diff --git "a/api_server_extended.py" "b/api_server_extended.py"
--- "a/api_server_extended.py"
+++ "b/api_server_extended.py"
@@ -1,3920 +1,4059 @@
-#!/usr/bin/env python3
-"""
-API Server Extended - HuggingFace Spaces Deployment Ready
-Complete Admin API with Real Data Only - NO MOCKS
-"""
-
-import os
-import threading
-import asyncio
-import sqlite3
-import httpx
-import json
-import subprocess
-import logging
-from pathlib import Path
-from typing import Optional, Dict, Any, List
-from datetime import datetime
-from contextlib import asynccontextmanager
-from collections import defaultdict
-
-logger = logging.getLogger(__name__)
-
-from fastapi import FastAPI, HTTPException, Response, Request
-from fastapi.middleware.cors import CORSMiddleware
-from fastapi.responses import JSONResponse, FileResponse, HTMLResponse
-from fastapi.staticfiles import StaticFiles
-from starlette.middleware.base import BaseHTTPMiddleware
-from pydantic import BaseModel
-
-# Environment variables
-USE_MOCK_DATA = os.getenv("USE_MOCK_DATA", "false").lower() == "true"
-PORT = int(os.getenv("PORT", "7860"))
-
-# Paths - In Docker container, use /app as base
-WORKSPACE_ROOT = Path("/app" if Path("/app").exists() else (Path("/workspace") if Path("/workspace").exists() else Path(".")))
-DB_PATH = WORKSPACE_ROOT / "data" / "database" / "crypto_monitor.db"
-LOG_DIR = WORKSPACE_ROOT / "logs"
-PROVIDERS_CONFIG_PATH = WORKSPACE_ROOT / "providers_config_extended.json"
-AUTO_DISCOVERY_REPORT_PATH = WORKSPACE_ROOT / "PROVIDER_AUTO_DISCOVERY_REPORT.json"
-API_REGISTRY_PATH = WORKSPACE_ROOT / "all_apis_merged_2025.json"
-
-# Ensure directories exist
-DB_PATH.parent.mkdir(parents=True, exist_ok=True)
-LOG_DIR.mkdir(parents=True, exist_ok=True)
-
-# Global state for providers
-_provider_state = {
- "providers": {},
- "pools": {},
- "logs": [],
- "last_check": None,
- "stats": {"total": 0, "online": 0, "offline": 0, "degraded": 0}
-}
-
-
-# ===== Database Setup =====
-def init_database():
- """Initialize SQLite database with required tables"""
- conn = sqlite3.connect(str(DB_PATH))
- cursor = conn.cursor()
-
- cursor.execute("""
- CREATE TABLE IF NOT EXISTS prices (
- id INTEGER PRIMARY KEY AUTOINCREMENT,
- symbol TEXT NOT NULL,
- name TEXT,
- price_usd REAL NOT NULL,
- volume_24h REAL,
- market_cap REAL,
- percent_change_24h REAL,
- rank INTEGER,
- timestamp DATETIME DEFAULT CURRENT_TIMESTAMP
- )
- """)
-
- cursor.execute("""
- CREATE TABLE IF NOT EXISTS sentiment_analysis (
- id INTEGER PRIMARY KEY AUTOINCREMENT,
- text TEXT NOT NULL,
- sentiment_label TEXT NOT NULL,
- confidence REAL NOT NULL,
- model_used TEXT,
- analysis_type TEXT,
- symbol TEXT,
- scores TEXT,
- timestamp DATETIME DEFAULT CURRENT_TIMESTAMP
- )
- """)
-
- cursor.execute("""
- CREATE TABLE IF NOT EXISTS news_articles (
- id INTEGER PRIMARY KEY AUTOINCREMENT,
- title TEXT NOT NULL,
- content TEXT,
- url TEXT,
- source TEXT,
- sentiment_label TEXT,
- sentiment_confidence REAL,
- related_symbols TEXT,
- published_date DATETIME,
- analyzed_at DATETIME DEFAULT CURRENT_TIMESTAMP
- )
- """)
-
- cursor.execute("CREATE INDEX IF NOT EXISTS idx_prices_symbol ON prices(symbol)")
- cursor.execute("CREATE INDEX IF NOT EXISTS idx_prices_timestamp ON prices(timestamp)")
- cursor.execute("CREATE INDEX IF NOT EXISTS idx_sentiment_timestamp ON sentiment_analysis(timestamp)")
- cursor.execute("CREATE INDEX IF NOT EXISTS idx_sentiment_symbol ON sentiment_analysis(symbol)")
- cursor.execute("CREATE INDEX IF NOT EXISTS idx_news_published ON news_articles(published_date)")
-
- conn.commit()
- conn.close()
- print(f"[OK] Database initialized at {DB_PATH}")
-
-
-def save_price_to_db(price_data: Dict[str, Any]):
- """Save price data to SQLite"""
- try:
- conn = sqlite3.connect(str(DB_PATH))
- cursor = conn.cursor()
- cursor.execute("""
- INSERT INTO prices (symbol, name, price_usd, volume_24h, market_cap, percent_change_24h, rank)
- VALUES (?, ?, ?, ?, ?, ?, ?)
- """, (
- price_data.get("symbol"),
- price_data.get("name"),
- price_data.get("price_usd", 0.0),
- price_data.get("volume_24h"),
- price_data.get("market_cap"),
- price_data.get("percent_change_24h"),
- price_data.get("rank")
- ))
- conn.commit()
- conn.close()
- except Exception as e:
- print(f"Error saving price to database: {e}")
-
-
-def get_price_history_from_db(symbol: str, limit: int = 10) -> List[Dict[str, Any]]:
- """Get price history from SQLite"""
- try:
- conn = sqlite3.connect(str(DB_PATH))
- conn.row_factory = sqlite3.Row
- cursor = conn.cursor()
- cursor.execute("""
- SELECT * FROM prices
- WHERE symbol = ?
- ORDER BY timestamp DESC
- LIMIT ?
- """, (symbol, limit))
- rows = cursor.fetchall()
- conn.close()
- return [dict(row) for row in rows]
- except Exception as e:
- print(f"Error fetching price history: {e}")
- return []
-
-
-def get_latest_prices_from_db() -> Dict[str, Dict[str, Any]]:
- """Get latest prices for BTC, ETH, BNB from database as fallback"""
- try:
- conn = sqlite3.connect(str(DB_PATH))
- conn.row_factory = sqlite3.Row
- cursor = conn.cursor()
-
- # Get latest price for each symbol
- symbols = ["BTC", "ETH", "BNB"]
- latest_prices = {}
-
- for symbol in symbols:
- cursor.execute("""
- SELECT * FROM prices
- WHERE symbol = ?
- ORDER BY timestamp DESC
- LIMIT 1
- """, (symbol,))
- row = cursor.fetchone()
- if row:
- latest_prices[symbol] = dict(row)
-
- conn.close()
- return latest_prices
- except Exception as e:
- logger.warning(f"Error fetching latest prices from database: {e}")
- return {}
-
-
-# ===== Provider Management =====
-def load_providers_config() -> Dict[str, Any]:
- """Load providers from providers_config_extended.json"""
- try:
- if PROVIDERS_CONFIG_PATH.exists():
- with open(PROVIDERS_CONFIG_PATH, 'r', encoding='utf-8') as f:
- config = json.load(f)
- # Validate structure
- if not isinstance(config, dict):
- logger.warning("Providers config is not a dict, returning empty")
- return {"providers": {}}
- if "providers" not in config:
- logger.warning("Providers config missing 'providers' key, adding it")
- config["providers"] = {}
- return config
- logger.warning(f"Providers config file not found at {PROVIDERS_CONFIG_PATH}")
- return {"providers": {}}
- except json.JSONDecodeError as e:
- logger.error(f"JSON decode error loading providers config: {e}")
- return {"providers": {}}
- except Exception as e:
- logger.error(f"Error loading providers config: {e}")
- return {"providers": {}}
-
-
-def load_apl_report() -> Dict[str, Any]:
- """Load APL validation report (alias for auto-discovery report)"""
- return load_auto_discovery_report()
-
-def load_auto_discovery_report() -> Dict[str, Any]:
- """Load PROVIDER_AUTO_DISCOVERY_REPORT.json"""
- try:
- if AUTO_DISCOVERY_REPORT_PATH.exists():
- with open(AUTO_DISCOVERY_REPORT_PATH, 'r', encoding='utf-8') as f:
- return json.load(f)
- return {}
- except Exception as e:
- logger.error(f"Error loading auto-discovery report: {e}")
- return {}
-
-def load_api_registry() -> Dict[str, Any]:
- """Load all_apis_merged_2025.json"""
- try:
- if API_REGISTRY_PATH.exists():
- with open(API_REGISTRY_PATH, 'r', encoding='utf-8') as f:
- return json.load(f)
- return {}
- except Exception as e:
- logger.error(f"Error loading API registry: {e}")
- return {}
-
-
-# ===== Deduplication Helpers =====
-def deduplicate_providers(providers_list: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
- """
- Deduplicate providers by id, or by name+base_url if no id.
- Merge tags/categories when duplicates are found.
- """
- seen = {}
- result = []
-
- for provider in providers_list:
- # Determine unique key
- provider_id = provider.get("id") or provider.get("provider_id")
- if provider_id:
- key = f"id:{provider_id}"
- else:
- name = provider.get("name", "unknown")
- base_url = provider.get("base_url", "")
- key = f"name_url:{name}:{base_url}"
-
- if key in seen:
- # Merge tags/categories
- existing = seen[key]
- existing_tags = set(existing.get("tags", []) if isinstance(existing.get("tags"), list) else [])
- new_tags = set(provider.get("tags", []) if isinstance(provider.get("tags"), list) else [])
- existing["tags"] = list(existing_tags | new_tags)
-
- # Merge categories if different
- existing_cat = existing.get("category", "")
- new_cat = provider.get("category", "")
- if new_cat and new_cat != existing_cat:
- if existing_cat:
- existing["categories"] = list(set([existing_cat, new_cat]))
- else:
- existing["category"] = new_cat
- else:
- # Ensure tags is a list
- if "tags" not in provider:
- provider["tags"] = []
- elif not isinstance(provider["tags"], list):
- provider["tags"] = [provider["tags"]]
-
- seen[key] = provider
- result.append(provider)
-
- return result
-
-
-def deduplicate_resources(resources_list: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
- """
- Deduplicate resources by id, or by name+url if no id.
- """
- seen = {}
- result = []
-
- for resource in resources_list:
- # Determine unique key
- resource_id = resource.get("id")
- if resource_id:
- key = f"id:{resource_id}"
- else:
- name = resource.get("name", "unknown")
- url = resource.get("url") or resource.get("base_url", "")
- path = resource.get("path", "")
- key = f"name_url:{name}:{url}{path}"
-
- if key not in seen:
- seen[key] = resource
- result.append(resource)
-
- return result
-
-
-def filter_resources_by_query(resources: List[Dict[str, Any]], query: str) -> List[Dict[str, Any]]:
- """
- Filter resources by search query (case-insensitive).
- Searches in name, description, category, and tags.
- """
- if not query:
- return resources
-
- query_lower = query.lower()
- filtered = []
-
- for resource in resources:
- # Search in name
- if query_lower in resource.get("name", "").lower():
- filtered.append(resource)
- continue
-
- # Search in description
- if query_lower in resource.get("description", "").lower():
- filtered.append(resource)
- continue
-
- # Search in category
- if query_lower in resource.get("category", "").lower():
- filtered.append(resource)
- continue
-
- # Search in tags
- tags = resource.get("tags", [])
- if isinstance(tags, list):
- if any(query_lower in str(tag).lower() for tag in tags):
- filtered.append(resource)
- continue
-
- return filtered
-
-
-# ===== Real Data Providers =====
-HEADERS = {
- "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
- "Accept": "application/json"
-}
-
-
-async def fetch_coingecko_simple_price() -> Dict[str, Any]:
- """Fetch real price data from CoinGecko API with proper error handling"""
- url = "https://api.coingecko.com/api/v3/simple/price"
- params = {
- "ids": "bitcoin,ethereum,binancecoin",
- "vs_currencies": "usd",
- "include_market_cap": "true",
- "include_24hr_vol": "true",
- "include_24hr_change": "true"
- }
-
- try:
- async with httpx.AsyncClient(timeout=15.0, headers=HEADERS) as client:
- response = await client.get(url, params=params)
- if response.status_code != 200:
- logger.warning(f"CoinGecko API returned HTTP {response.status_code}")
- raise Exception(f"CoinGecko API error: HTTP {response.status_code}")
- return response.json()
- except httpx.TimeoutException:
- logger.warning("CoinGecko API request timed out")
- raise Exception("CoinGecko API request timed out")
- except httpx.RequestError as e:
- logger.warning(f"CoinGecko API request error: {str(e)}")
- raise Exception(f"CoinGecko API request failed: {str(e)}")
- except Exception as e:
- logger.warning(f"CoinGecko API error: {str(e)}")
- raise
-
-
-async def fetch_fear_greed_index() -> Dict[str, Any]:
- """Fetch real Fear & Greed Index from Alternative.me"""
- url = "https://api.alternative.me/fng/"
- params = {"limit": "1", "format": "json"}
-
- async with httpx.AsyncClient(timeout=15.0, headers=HEADERS) as client:
- response = await client.get(url, params=params)
- if response.status_code != 200:
- raise HTTPException(status_code=503, detail=f"Alternative.me API error: HTTP {response.status_code}")
- return response.json()
-
-
-async def fetch_coingecko_trending() -> Dict[str, Any]:
- """Fetch real trending coins from CoinGecko"""
- url = "https://api.coingecko.com/api/v3/search/trending"
-
- async with httpx.AsyncClient(timeout=15.0, headers=HEADERS) as client:
- response = await client.get(url)
- if response.status_code != 200:
- raise HTTPException(status_code=503, detail=f"CoinGecko trending API error: HTTP {response.status_code}")
- return response.json()
-
-
-# ===== Self-Healing Health Registry =====
-from dataclasses import dataclass, field
-from typing import Callable
-import time as time_module
-
-@dataclass
-class ProviderHealthEntry:
- """Health tracking entry for a provider/resource"""
- id: str
- name: str
- status: str = "unknown" # "healthy", "degraded", "unavailable", "unknown"
- last_success: Optional[float] = None
- last_error: Optional[float] = None
- error_count: int = 0
- success_count: int = 0
- cooldown_until: Optional[float] = None
- last_error_message: Optional[str] = None
-
-class HealthRegistry:
- """
- Self-healing health registry for providers and external API endpoints.
- Tracks failures, implements cooldowns, and provides graceful degradation.
- """
- def __init__(self):
- self._providers: Dict[str, ProviderHealthEntry] = {}
- self._lock = threading.Lock()
- # Load config
- try:
- from config import get_settings
- self.settings = get_settings()
- except:
- # Fallback defaults if config not available
- class FallbackSettings:
- health_error_threshold = 3
- health_cooldown_seconds = 300
- health_success_recovery_count = 2
- self.settings = FallbackSettings()
-
- def _get_or_create_entry(self, provider_id: str, provider_name: str = None) -> ProviderHealthEntry:
- """Get or create health entry for a provider"""
- if provider_id not in self._providers:
- self._providers[provider_id] = ProviderHealthEntry(
- id=provider_id,
- name=provider_name or provider_id,
- status="unknown"
- )
- return self._providers[provider_id]
-
- def update_on_success(self, provider_id: str, provider_name: str = None):
- """Update health registry after successful provider call"""
- with self._lock:
- entry = self._get_or_create_entry(provider_id, provider_name)
- entry.last_success = time_module.time()
- entry.success_count += 1
-
- # Reset error count gradually
- if entry.error_count > 0:
- entry.error_count = max(0, entry.error_count - 1)
-
- # Recovery logic
- if entry.success_count >= self.settings.health_success_recovery_count:
- entry.status = "healthy"
- entry.cooldown_until = None
-
- def update_on_failure(self, provider_id: str, error_msg: str, provider_name: str = None):
- """Update health registry after failed provider call"""
- with self._lock:
- entry = self._get_or_create_entry(provider_id, provider_name)
- entry.last_error = time_module.time()
- entry.error_count += 1
- entry.last_error_message = error_msg[:500] # Limit error message length
- entry.success_count = 0
-
- # Determine status based on error count
- if entry.error_count >= self.settings.health_error_threshold:
- entry.status = "unavailable"
- entry.cooldown_until = time_module.time() + self.settings.health_cooldown_seconds
- elif entry.error_count >= (self.settings.health_error_threshold // 2):
- entry.status = "degraded"
- else:
- entry.status = "healthy"
-
- def is_in_cooldown(self, provider_id: str) -> bool:
- """Check if provider is in cooldown period"""
- if provider_id not in self._providers:
- return False
- entry = self._providers[provider_id]
- if entry.cooldown_until is None:
- return False
- return time_module.time() < entry.cooldown_until
-
- def get_status(self, provider_id: str) -> Optional[str]:
- """Get current status of a provider"""
- if provider_id not in self._providers:
- return "unknown"
- return self._providers[provider_id].status
-
- def get_all_entries(self) -> List[Dict[str, Any]]:
- """Get all health entries as list of dicts"""
- with self._lock:
- return [
- {
- "id": entry.id,
- "name": entry.name,
- "status": entry.status,
- "last_success": entry.last_success,
- "last_error": entry.last_error,
- "error_count": entry.error_count,
- "success_count": entry.success_count,
- "cooldown_until": entry.cooldown_until,
- "in_cooldown": self.is_in_cooldown(entry.id),
- "last_error_message": entry.last_error_message
- }
- for entry in self._providers.values()
- ]
-
- def get_summary(self) -> Dict[str, Any]:
- """Get summary statistics of health registry"""
- with self._lock:
- total = len(self._providers)
- healthy = sum(1 for e in self._providers.values() if e.status == "healthy")
- degraded = sum(1 for e in self._providers.values() if e.status == "degraded")
- unavailable = sum(1 for e in self._providers.values() if e.status == "unavailable")
- unknown = sum(1 for e in self._providers.values() if e.status == "unknown")
- in_cooldown = sum(1 for e in self._providers.values() if self.is_in_cooldown(e.id))
-
- return {
- "total": total,
- "healthy": healthy,
- "degraded": degraded,
- "unavailable": unavailable,
- "unknown": unknown,
- "in_cooldown": in_cooldown
- }
-
-# Global health registry instance
-_health_registry = HealthRegistry()
-
-
-async def call_provider_safe(
- provider_id: str,
- provider_name: str,
- call_func: Callable,
- *args,
- **kwargs
-) -> Dict[str, Any]:
- """
- Safely call a provider with health tracking.
-
- Args:
- provider_id: Unique identifier for the provider
- provider_name: Human-readable name
- call_func: Async function to call
- *args, **kwargs: Arguments to pass to call_func
-
- Returns:
- Dict with status and data or error
- """
- # Check if provider is in cooldown
- if _health_registry.is_in_cooldown(provider_id):
- entry = _health_registry._providers[provider_id]
- cooldown_remaining = int(entry.cooldown_until - time_module.time())
- return {
- "status": "cooldown",
- "error": f"Provider in cooldown for {cooldown_remaining}s",
- "provider_id": provider_id,
- "cooldown_remaining": cooldown_remaining
- }
-
- try:
- # Call the provider function
- result = await call_func(*args, **kwargs)
- # Update health on success
- _health_registry.update_on_success(provider_id, provider_name)
- return {
- "status": "success",
- "data": result,
- "provider_id": provider_id
- }
- except httpx.TimeoutException as e:
- error_msg = f"Timeout: {str(e)[:200]}"
- _health_registry.update_on_failure(provider_id, error_msg, provider_name)
- return {
- "status": "timeout",
- "error": error_msg,
- "provider_id": provider_id
- }
- except httpx.HTTPStatusError as e:
- error_msg = f"HTTP {e.response.status_code}: {str(e)[:200]}"
- _health_registry.update_on_failure(provider_id, error_msg, provider_name)
- return {
- "status": "http_error",
- "error": error_msg,
- "provider_id": provider_id,
- "status_code": e.response.status_code
- }
- except Exception as e:
- error_msg = f"{type(e).__name__}: {str(e)[:200]}"
- _health_registry.update_on_failure(provider_id, error_msg, provider_name)
- return {
- "status": "error",
- "error": error_msg,
- "provider_id": provider_id
- }
-
-
-# ===== Lifespan Management =====
-@asynccontextmanager
-async def lifespan(app: FastAPI):
- """Application lifespan manager"""
- print("=" * 80)
- print("Starting Crypto Monitor Admin API")
- print("=" * 80)
- init_database()
-
- # Load providers
- config = load_providers_config()
- _provider_state["providers"] = config.get("providers", {})
- print(f"[OK] Loaded {len(_provider_state['providers'])} providers from config")
-
- # Load auto-discovery report
- apl_report = load_auto_discovery_report()
- if apl_report:
- print(f"[OK] Loaded auto-discovery report with validation data")
-
- # Load API registry
- api_registry = load_api_registry()
- if api_registry:
- metadata = api_registry.get("metadata", {})
- print(f"[OK] Loaded API registry: {metadata.get('name', 'unknown')} v{metadata.get('version', 'unknown')}")
-
- # Initialize AI models
- try:
- from ai_models import initialize_models, registry_status, HF_MAX_STARTUP_MODELS
- model_init_result = initialize_models(max_models=HF_MAX_STARTUP_MODELS)
- registry_info = registry_status()
- print(f"[OK] AI Models initialized: {model_init_result}")
- print(f"[OK] HF Registry status: {registry_info}")
- except Exception as e:
- print(f"[WARN] AI Models initialization failed: {e}")
-
- # Validate unified resources
- try:
- from backend.services.resource_validator import validate_unified_resources
- validation_report = validate_unified_resources(str(WORKSPACE_ROOT / "api-resources" / "crypto_resources_unified_2025-11-11.json"))
- print(f"[OK] Resource validation: {validation_report['local_backend_routes']['routes_count']} local routes")
- if validation_report['local_backend_routes']['duplicate_signatures'] > 0:
- print(f"[WARN] Found {validation_report['local_backend_routes']['duplicate_signatures']} duplicate route signatures")
- except Exception as e:
- print(f"[WARN] Resource validation failed: {e}")
-
- print(f"[OK] Server ready on port {PORT}")
- print("=" * 80)
- yield
- print("Shutting down...")
-
-
-# ===== FastAPI Application =====
-app = FastAPI(
- title="Crypto Monitor Admin API",
- description="Real-time cryptocurrency data API with Admin Dashboard",
- version="5.0.0",
- lifespan=lifespan
-)
-
-# CORS Middleware
-app.add_middleware(
- CORSMiddleware,
- allow_origins=["*"],
- allow_credentials=True,
- allow_methods=["*"],
- allow_headers=["*"],
-)
-
-# Middleware to ensure HTML responses have correct Content-Type
-class HTMLContentTypeMiddleware(BaseHTTPMiddleware):
- async def dispatch(self, request: Request, call_next):
- response = await call_next(request)
- if isinstance(response, HTMLResponse):
- response.headers["Content-Type"] = "text/html; charset=utf-8"
- response.headers["X-Content-Type-Options"] = "nosniff"
- return response
-
-app.add_middleware(HTMLContentTypeMiddleware)
-
-# Mount static files
-try:
- static_path = WORKSPACE_ROOT / "static"
- if static_path.exists():
- app.mount("/static", StaticFiles(directory=str(static_path)), name="static")
- logger.info(f"Mounted static files from {static_path}")
- else:
- # Create static directories if they don't exist
- static_path.mkdir(parents=True, exist_ok=True)
- (static_path / "css").mkdir(exist_ok=True)
- (static_path / "js").mkdir(exist_ok=True)
- logger.info(f"Created static directories at {static_path}")
-except Exception as e:
- logger.warning(f"Could not mount static files: {e}")
-
-# Serve trading pairs file
-@app.get("/trading_pairs.txt")
-async def get_trading_pairs():
- """Serve trading pairs text file"""
- from fastapi.responses import PlainTextResponse
- trading_pairs_file = WORKSPACE_ROOT / "trading_pairs.txt"
- if trading_pairs_file.exists():
- return FileResponse(trading_pairs_file, media_type="text/plain")
- return PlainTextResponse("BTCUSDT\nETHUSDT\nBNBUSDT\nSOLUSDT", status_code=200)
-
-
-# ===== HTML UI Endpoints =====
-@app.get("/", response_class=HTMLResponse)
-async def root():
- """Serve main dashboard"""
- index_path = WORKSPACE_ROOT / "index.html"
- if index_path.exists():
- content = index_path.read_text(encoding="utf-8", errors="ignore")
- return HTMLResponse(
- content=content,
- media_type="text/html",
- headers={
- "Content-Type": "text/html; charset=utf-8",
- "X-Content-Type-Options": "nosniff"
- }
- )
- return HTMLResponse(
- "Cryptocurrency Data & Analysis API
See /docs for API documentation
",
- headers={"Content-Type": "text/html; charset=utf-8"}
- )
-
-@app.get("/index.html", response_class=HTMLResponse)
-async def index():
- """Serve index.html"""
- index_path = WORKSPACE_ROOT / "index.html"
- if index_path.exists():
- content = index_path.read_text(encoding="utf-8", errors="ignore")
- return HTMLResponse(
- content=content,
- media_type="text/html",
- headers={
- "Content-Type": "text/html; charset=utf-8",
- "X-Content-Type-Options": "nosniff"
- }
- )
- return HTMLResponse(
- "index.html not found
",
- headers={"Content-Type": "text/html; charset=utf-8"}
- )
-
-@app.get("/test.html", response_class=HTMLResponse)
-async def test_page():
- """Serve test.html for debugging"""
- test_path = WORKSPACE_ROOT / "test.html"
- if test_path.exists():
- content = test_path.read_text(encoding="utf-8", errors="ignore")
- return HTMLResponse(
- content=content,
- media_type="text/html",
- headers={
- "Content-Type": "text/html; charset=utf-8",
- "X-Content-Type-Options": "nosniff"
- }
- )
- return HTMLResponse(
- "✅ Server is Running
WORKSPACE_ROOT: " + str(WORKSPACE_ROOT) + "
",
- headers={"Content-Type": "text/html; charset=utf-8"}
- )
-
-@app.get("/ai-tools", response_class=HTMLResponse)
-async def ai_tools_page(request: Request):
- """
- Serve the standalone AI Tools page.
-
- This page provides:
- - Sentiment Playground: POST /api/sentiment/analyze
- - Text Summarizer: POST /api/ai/summarize
- - Model Status & Diagnostics: GET /api/models/status, /api/models/list
- """
- ai_tools_path = WORKSPACE_ROOT / "templates" / "ai_tools.html"
- if ai_tools_path.exists():
- content = ai_tools_path.read_text(encoding="utf-8", errors="ignore")
- return HTMLResponse(
- content=content,
- media_type="text/html",
- headers={
- "Content-Type": "text/html; charset=utf-8",
- "X-Content-Type-Options": "nosniff"
- }
- )
- return HTMLResponse(
- "AI Tools page not found
",
- headers={"Content-Type": "text/html; charset=utf-8"}
- )
-
-@app.get("/debug-info", response_class=HTMLResponse)
-async def debug_info():
- """Debug endpoint to show server configuration"""
- import os
- info = f"""
-
-
-
-
- Debug Info
-
-
-
- 🔍 Server Debug Information
- Paths:
-
-WORKSPACE_ROOT: {WORKSPACE_ROOT}
-Current Dir: {Path.cwd()}
-index.html exists: {"✅ YES" if (WORKSPACE_ROOT / "index.html").exists() else "❌ NO"}
-static dir exists: {"✅ YES" if (WORKSPACE_ROOT / "static").exists() else "❌ NO"}
-
- Files in WORKSPACE_ROOT:
-
-{chr(10).join([f"- {f.name}" for f in sorted(WORKSPACE_ROOT.glob("*.html"))[:20]])}
-
- Environment:
-
-Python: {os.sys.version}
-Port: 7860
-Host: 127.0.0.1
-
- Quick Links:
-
-
-
- """
- return HTMLResponse(content=info, headers={"Content-Type": "text/html; charset=utf-8"})
-
-
-# ===== Health & Status Endpoints =====
-@app.get("/health")
-async def health():
- """Health check endpoint (legacy)"""
- return {
- "status": "healthy",
- "timestamp": datetime.now().isoformat(),
- "database": str(DB_PATH),
- "use_mock_data": USE_MOCK_DATA,
- "providers_loaded": len(_provider_state["providers"])
- }
-
-
-@app.get("/api/health")
-async def api_health():
- """API health check endpoint - never crashes"""
- try:
- version = "1.0.0"
- try:
- # Try to get version from metadata
- api_registry = load_api_registry()
- metadata = api_registry.get("metadata", {})
- if metadata.get("version"):
- version = metadata.get("version")
- except Exception:
- pass
-
- return {
- "status": "ok",
- "timestamp": datetime.now().isoformat(),
- "version": version
- }
- except Exception as e:
- # Even if something goes wrong, return a clean response
- logger.error(f"Health check error: {e}")
- return JSONResponse(
- status_code=200,
- content={
- "status": "ok",
- "timestamp": datetime.now().isoformat(),
- "version": "unknown"
- }
- )
-
-
-@app.get("/api/status")
-async def get_status():
- """System status with real aggregated data"""
- try:
- # Load providers
- config = load_providers_config()
- providers = config.get("providers", {})
-
- # Count free vs paid providers
- free_count = sum(1 for p in providers.values()
- if not p.get("requires_auth", False) and p.get("rate_limit"))
- paid_count = sum(1 for p in providers.values()
- if p.get("requires_auth", False))
-
- # Load resources from unified file
- resources_json = WORKSPACE_ROOT / "api-resources" / "crypto_resources_unified_2025-11-11.json"
- resources_data = {"total": 0, "categories": {}}
-
- if resources_json.exists():
- try:
- with open(resources_json, 'r', encoding='utf-8') as f:
- unified_data = json.load(f)
- registry = unified_data.get('registry', {})
-
- for category, items in registry.items():
- if category == 'metadata':
- continue
- if isinstance(items, list):
- count = len(items)
- resources_data['total'] += count
-
- # Group similar categories
- cat_key = category.replace('_', '-')
- if cat_key not in resources_data['categories']:
- resources_data['categories'][cat_key] = 0
- resources_data['categories'][cat_key] += count
- except Exception as e:
- logger.error(f"Error loading resources: {e}")
-
- # Get model count
- model_count = 0
- try:
- from ai_models import MODEL_SPECS
- model_count = len(MODEL_SPECS) if MODEL_SPECS else 0
- except Exception:
- pass
-
- # Get system health metrics
- online_count = 0
- degraded_count = 0
- offline_count = 0
- response_times = []
-
- # Try to get health status from providers if available
- # This is a simplified version - in production you'd check actual provider health
- system_health = "ok" if len(providers) > 0 else "unknown"
-
- return {
- "status": "ok",
- "system_health": system_health,
- "timestamp": datetime.now().isoformat(),
- "last_update": datetime.now().isoformat(),
- "providers": {
- "total": len(providers),
- "free": free_count,
- "paid": paid_count
- },
- "online": online_count,
- "degraded": degraded_count,
- "offline": offline_count,
- "avg_response_time_ms": round(sum(response_times) / len(response_times), 2) if response_times else 0,
- "resources": resources_data,
- "models": {
- "total": model_count
- }
- }
- except Exception as e:
- logger.error(f"Status endpoint error: {e}")
- return {
- "status": "error",
- "timestamp": datetime.now().isoformat(),
- "error": str(e),
- "providers": {"total": 0, "free": 0, "paid": 0},
- "resources": {"total": 0, "categories": {}}
- }
-
-
-@app.get("/api/stats")
-async def get_stats():
- """System statistics"""
- config = load_providers_config()
- providers = config.get("providers", {})
-
- # Group by category
- categories = defaultdict(int)
- for p in providers.values():
- cat = p.get("category", "unknown")
- categories[cat] += 1
-
- return {
- "total_providers": len(providers),
- "categories": dict(categories),
- "total_categories": len(categories),
- "timestamp": datetime.now().isoformat()
- }
-
-
-# ===== Market Data Endpoint =====
-@app.get("/api/market")
-async def get_market_data():
- """Market data from CoinGecko with database fallback"""
- cryptocurrencies = []
- coin_mapping = {
- "bitcoin": {"name": "Bitcoin", "symbol": "BTC", "rank": 1, "image": "https://assets.coingecko.com/coins/images/1/small/bitcoin.png"},
- "ethereum": {"name": "Ethereum", "symbol": "ETH", "rank": 2, "image": "https://assets.coingecko.com/coins/images/279/small/ethereum.png"},
- "binancecoin": {"name": "BNB", "symbol": "BNB", "rank": 3, "image": "https://assets.coingecko.com/coins/images/825/small/bnb-icon2_2x.png"}
- }
-
- data_source = "CoinGecko API (Real Data)"
- use_fallback = False
-
- # Try to fetch from CoinGecko API first
- try:
- data = await fetch_coingecko_simple_price()
-
- for coin_id, coin_info in coin_mapping.items():
- if coin_id in data:
- coin_data = data[coin_id]
- crypto_entry = {
- "rank": coin_info["rank"],
- "name": coin_info["name"],
- "symbol": coin_info["symbol"],
- "price": coin_data.get("usd", 0),
- "change_24h": coin_data.get("usd_24h_change", 0),
- "market_cap": coin_data.get("usd_market_cap", 0),
- "volume_24h": coin_data.get("usd_24h_vol", 0),
- "image": coin_info["image"]
- }
- cryptocurrencies.append(crypto_entry)
-
- # Save to database
- try:
- save_price_to_db({
- "symbol": coin_info["symbol"],
- "name": coin_info["name"],
- "price_usd": crypto_entry["price"],
- "volume_24h": crypto_entry["volume_24h"],
- "market_cap": crypto_entry["market_cap"],
- "percent_change_24h": crypto_entry["change_24h"],
- "rank": coin_info["rank"]
- })
- except Exception as db_error:
- logger.warning(f"Failed to save price to database: {db_error}")
-
- except Exception as e:
- logger.warning(f"Failed to fetch from CoinGecko API: {str(e)}, trying database fallback...")
- use_fallback = True
- data_source = "Database (Cached Data)"
-
- # Fallback to database
- latest_prices = get_latest_prices_from_db()
-
- for coin_id, coin_info in coin_mapping.items():
- symbol = coin_info["symbol"]
- if symbol in latest_prices:
- db_data = latest_prices[symbol]
- crypto_entry = {
- "rank": coin_info["rank"],
- "name": coin_info["name"],
- "symbol": coin_info["symbol"],
- "price": db_data.get("price_usd", 0),
- "change_24h": db_data.get("percent_change_24h", 0),
- "market_cap": db_data.get("market_cap", 0),
- "volume_24h": db_data.get("volume_24h", 0),
- "image": coin_info["image"]
- }
- cryptocurrencies.append(crypto_entry)
- else:
- # If no database data, add placeholder with zero values
- logger.warning(f"No cached data found for {symbol}")
- cryptocurrencies.append({
- "rank": coin_info["rank"],
- "name": coin_info["name"],
- "symbol": coin_info["symbol"],
- "price": 0,
- "change_24h": 0,
- "market_cap": 0,
- "volume_24h": 0,
- "image": coin_info["image"]
- })
-
- # If still no data, return empty structure with message
- if not cryptocurrencies:
- logger.error("No market data available from API or database")
- return {
- "cryptocurrencies": [],
- "total_market_cap": 0,
- "btc_dominance": 0,
- "timestamp": datetime.now().isoformat(),
- "source": "No data available",
- "error": "Unable to fetch market data. Please try again later.",
- "message": "Market data temporarily unavailable"
- }
-
- # Calculate dominance
- total_market_cap = sum(c["market_cap"] for c in cryptocurrencies)
- btc_dominance = 0
- if total_market_cap > 0:
- btc_entry = next((c for c in cryptocurrencies if c["symbol"] == "BTC"), None)
- if btc_entry:
- btc_dominance = (btc_entry["market_cap"] / total_market_cap) * 100
-
- response = {
- "cryptocurrencies": cryptocurrencies,
- "total_market_cap": total_market_cap,
- "btc_dominance": btc_dominance,
- "timestamp": datetime.now().isoformat(),
- "source": data_source
- }
-
- if use_fallback:
- response["warning"] = "Using cached data from database. API unavailable."
-
- return response
-
-
-@app.get("/api/market/history")
-async def get_market_history(symbol: str = "BTC", limit: int = 10):
- """Get price history from database - REAL DATA ONLY"""
- history = get_price_history_from_db(symbol.upper(), limit)
-
- if not history:
- return {
- "symbol": symbol,
- "history": [],
- "count": 0,
- "message": "No history available"
- }
-
- return {
- "symbol": symbol,
- "history": history,
- "count": len(history),
- "source": "SQLite Database (Real Data)"
- }
-
-
-@app.get("/api/sentiment")
-async def get_sentiment():
- """Sentiment data from Alternative.me - REAL DATA ONLY"""
- try:
- data = await fetch_fear_greed_index()
-
- if "data" in data and len(data["data"]) > 0:
- fng_data = data["data"][0]
- return {
- "fear_greed_index": int(fng_data["value"]),
- "fear_greed_label": fng_data["value_classification"],
- "timestamp": datetime.now().isoformat(),
- "source": "Alternative.me API (Real Data)"
- }
-
- raise HTTPException(status_code=503, detail="Invalid response from Alternative.me")
-
- except Exception as e:
- raise HTTPException(status_code=503, detail=f"Failed to fetch sentiment: {str(e)}")
-
-
-@app.post("/api/sentiment")
-async def analyze_sentiment_simple(request: Dict[str, Any]):
- """Analyze sentiment with mode routing - simplified endpoint"""
- try:
- from ai_models import (
- analyze_crypto_sentiment,
- analyze_financial_sentiment,
- analyze_social_sentiment,
- _registry,
- MODEL_SPECS,
- ModelNotAvailable
- )
-
- text = request.get("text", "").strip()
- if not text:
- raise HTTPException(status_code=400, detail="Text is required")
-
- mode = request.get("mode", "auto").lower()
- model_key = request.get("model_key")
-
- # If model_key is provided, use that specific model
- if model_key:
- if model_key not in MODEL_SPECS:
- raise HTTPException(status_code=404, detail=f"Model key '{model_key}' not found")
-
- try:
- pipeline = _registry.get_pipeline(model_key)
- spec = MODEL_SPECS[model_key]
-
- # Handle trading signal models specially
- if spec.category == "trading_signal":
- raw_result = pipeline(text, max_length=200, num_return_sequences=1)
- if isinstance(raw_result, list) and raw_result:
- raw_result = raw_result[0]
- generated_text = raw_result.get("generated_text", str(raw_result))
-
- decision = "HOLD"
- if "buy" in generated_text.lower():
- decision = "BUY"
- elif "sell" in generated_text.lower():
- decision = "SELL"
-
- return {
- "sentiment": decision.lower(),
- "confidence": 0.7,
- "raw_label": decision,
- "mode": "trading",
- "model": model_key,
- "extra": {
- "decision": decision,
- "rationale": generated_text,
- "raw": raw_result
- }
- }
-
- # Regular sentiment analysis
- raw_result = pipeline(text[:512])
- if isinstance(raw_result, list) and raw_result:
- raw_result = raw_result[0]
-
- label = raw_result.get("label", "neutral").upper()
- score = raw_result.get("score", 0.5)
-
- # Map to standard format
- mapped = "Bullish" if "POSITIVE" in label or "BULLISH" in label or "LABEL_2" in label else (
- "Bearish" if "NEGATIVE" in label or "BEARISH" in label or "LABEL_0" in label else "Neutral"
- )
-
- return {
- "sentiment": mapped,
- "confidence": score,
- "raw_label": label,
- "mode": mode,
- "model": model_key,
- "extra": {"raw": raw_result}
- }
-
- except ModelNotAvailable as e:
- logger.warning(f"Model {model_key} not available: {e}")
- raise HTTPException(status_code=503, detail=f"Model not available: {str(e)}")
-
- # Mode-based routing (no explicit model key)
- result = None
- actual_model = None
-
- if mode == "crypto" or mode == "auto":
- result = analyze_crypto_sentiment(text)
- actual_model = "crypto_sent_kk08" # Default crypto model
- elif mode == "social":
- result = analyze_social_sentiment(text)
- actual_model = "crypto_sent_social" # ElKulako/cryptobert
- elif mode == "financial":
- result = analyze_financial_sentiment(text)
- actual_model = "crypto_sent_fin" # FinTwitBERT
- elif mode == "news":
- result = analyze_financial_sentiment(text) # Use financial for news
- actual_model = "crypto_sent_fin"
- elif mode == "trading":
- # Try to use trading model
- try:
- pipeline = _registry.get_pipeline("crypto_trading_lm")
- raw_result = pipeline(text, max_length=200, num_return_sequences=1)
- if isinstance(raw_result, list) and raw_result:
- raw_result = raw_result[0]
- generated_text = raw_result.get("generated_text", str(raw_result))
-
- decision = "HOLD"
- if "buy" in generated_text.lower():
- decision = "BUY"
- elif "sell" in generated_text.lower():
- decision = "SELL"
-
- return {
- "sentiment": decision,
- "confidence": 0.7,
- "raw_label": decision,
- "mode": "trading",
- "model": "crypto_trading_lm",
- "extra": {
- "decision": decision,
- "rationale": generated_text
- }
- }
- except ModelNotAvailable:
- # Fallback to crypto sentiment
- result = analyze_crypto_sentiment(text)
- actual_model = "crypto_sent_kk08"
- else:
- result = analyze_crypto_sentiment(text) # Default fallback
- actual_model = "crypto_sent_kk08"
-
- if not result:
- raise HTTPException(status_code=500, detail="Sentiment analysis failed")
-
- # Standardize result format
- sentiment = result.get("label", "Neutral")
- confidence = result.get("confidence", 0.5)
-
- # Capitalize first letter
- sentiment_formatted = sentiment.capitalize() if isinstance(sentiment, str) else "Neutral"
-
- return {
- "sentiment": sentiment_formatted,
- "confidence": confidence,
- "raw_label": sentiment,
- "mode": mode,
- "model": actual_model,
- "extra": result
- }
-
- except HTTPException:
- raise
- except Exception as e:
- logger.error(f"Sentiment analysis error: {e}")
- raise HTTPException(status_code=500, detail=f"Analysis failed: {str(e)}")
-
-
-@app.get("/api/resources")
-async def get_resources(q: Optional[str] = None):
- """Get all resources with optional search query and deduplication"""
- try:
- resources_list = []
-
- # Load from unified resources file
- resources_json = WORKSPACE_ROOT / "api-resources" / "crypto_resources_unified_2025-11-11.json"
- if resources_json.exists():
- try:
- with open(resources_json, 'r', encoding='utf-8') as f:
- unified_data = json.load(f)
- registry = unified_data.get('registry', {})
-
- for category, items in registry.items():
- if category == 'metadata':
- continue
- if isinstance(items, list):
- for item in items:
- # Normalize resource structure
- resource = {
- "id": item.get("id"),
- "name": item.get("name", item.get("title", "Unknown")),
- "category": category,
- "url": item.get("url") or item.get("base_url", ""),
- "free": item.get("free", True),
- "auth_required": item.get("auth_required", False) or (item.get("auth", {}).get("type") != "none" if "auth" in item else False),
- "tags": item.get("tags", []) if isinstance(item.get("tags"), list) else [],
- "description": item.get("description", "") or item.get("note", "")
- }
-
- # Additional fields if present
- if "method" in item:
- resource["method"] = item["method"]
- if "path" in item:
- resource["path"] = item["path"]
- if "endpoint" in item:
- resource["endpoint"] = item["endpoint"]
-
- resources_list.append(resource)
- except Exception as e:
- logger.error(f"Error loading unified resources: {e}")
-
- # Load from API registry (all_apis_merged_2025.json)
- api_registry = load_api_registry()
- if api_registry and "raw_files" in api_registry:
- # Parse raw files for additional resources (basic extraction)
- for raw_file in api_registry.get("raw_files", [])[:10]: # Limit to first 10
- content = raw_file.get("content", "")
- filename = raw_file.get("filename", "")
-
- # Simple extraction: look for URLs in content
- import re
- urls = re.findall(r'https?://[^\s<>"]+', content)
- for url in urls[:5]: # Limit URLs per file
- resources_list.append({
- "id": None,
- "name": f"Resource from {filename}",
- "category": "discovered",
- "url": url,
- "free": True,
- "auth_required": False,
- "tags": ["auto-discovered"],
- "description": f"Auto-discovered from {filename}"
- })
-
- # Apply deduplication
- deduplicated_resources = deduplicate_resources(resources_list)
-
- # Apply search filter if query provided
- if q:
- deduplicated_resources = filter_resources_by_query(deduplicated_resources, q)
-
- return deduplicated_resources
-
- except Exception as e:
- logger.error(f"Error in get_resources: {e}")
- raise HTTPException(status_code=500, detail=f"Failed to fetch resources: {str(e)}")
-
-
-@app.get("/api/resources/summary")
-async def get_resources_summary():
- """Get resources summary for HTML dashboard (includes API registry metadata and local routes)"""
- try:
- # Import MODEL_SPECS first as source of truth for models count
- try:
- from ai_models import MODEL_SPECS
- models_count = len(MODEL_SPECS) if MODEL_SPECS else 0
- except Exception as e:
- logger.warning(f"Failed to import MODEL_SPECS: {e}")
- models_count = 0
-
- # Load API registry for metadata
- api_registry = load_api_registry()
- metadata = api_registry.get("metadata", {}) if api_registry else {}
-
- # Try to load resources from JSON files
- resources_json = WORKSPACE_ROOT / "api-resources" / "crypto_resources_unified_2025-11-11.json"
-
- summary = {
- "total_resources": 0,
- "free_resources": 0,
- "models_available": models_count, # Use MODEL_SPECS as source of truth
- "local_routes_count": 0,
- "categories": {}
- }
-
- # Load from unified resources
- if resources_json.exists():
- try:
- with open(resources_json, 'r', encoding='utf-8') as f:
- data = json.load(f)
- registry = data.get('registry', {})
-
- # Process all categories
- for category, items in registry.items():
- if category == 'metadata':
- continue
- if isinstance(items, list):
- count = len(items)
- summary['total_resources'] += count
- summary['categories'][category] = {
- "count": count,
- "type": "local" if category == "local_backend_routes" else "external"
- }
-
- # Track local routes separately
- if category == 'local_backend_routes':
- summary['local_routes_count'] = count
-
- free_count = sum(1 for item in items if item.get('free', False) or item.get('auth', {}).get('type') == 'none')
- summary['free_resources'] += free_count
- except Exception as e:
- logger.warning(f"Failed to load resources JSON: {e}")
-
- # Ensure models_available is always non-zero if MODEL_SPECS is available
- if summary['models_available'] == 0 and models_count > 0:
- summary['models_available'] = models_count
-
- # If no resources found, provide fallback data but keep models count from MODEL_SPECS
- if summary['total_resources'] == 0:
- logger.warning("No resources found in JSON files, using fallback data")
- summary['total_resources'] = 15
- summary['free_resources'] = 12
- # Ensure models count is at least from MODEL_SPECS or fallback minimum
- summary['models_available'] = max(summary['models_available'], models_count, 7)
- summary['categories'] = {
- 'market_data': 5,
- 'news': 3,
- 'sentiment': 2,
- 'blockchain': 3,
- 'defi': 2
- }
-
- return {
- "success": True,
- "summary": summary,
- "api_registry_metadata": metadata,
- "timestamp": datetime.now().isoformat()
- }
- except Exception as e:
- logger.error(f"Error in get_resources_summary: {e}")
- # Return fallback data on error, but try to get models count from MODEL_SPECS
- try:
- from ai_models import MODEL_SPECS
- fallback_models = len(MODEL_SPECS) if MODEL_SPECS else 7
- except:
- fallback_models = 7
-
- return {
- "success": True,
- "summary": {
- "total_resources": 15,
- "free_resources": 12,
- "models_available": fallback_models,
- "local_routes_count": 0,
- "categories": {
- 'market_data': 5,
- 'news': 3,
- 'sentiment': 2,
- 'blockchain': 3,
- 'defi': 2
- }
- },
- "error": str(e),
- "timestamp": datetime.now().isoformat()
- }
-
-@app.get("/api/resources/apis")
-async def get_resources_apis():
- """Get API registry with local and external routes"""
- registry = load_api_registry()
-
- # Load unified resources for local routes
- resources_json = WORKSPACE_ROOT / "api-resources" / "crypto_resources_unified_2025-11-11.json"
- local_routes = []
- unified_metadata = {}
-
- if resources_json.exists():
- try:
- with open(resources_json, 'r', encoding='utf-8') as f:
- unified_data = json.load(f)
- unified_registry = unified_data.get('registry', {})
- unified_metadata = unified_registry.get('metadata', {})
- local_routes = unified_registry.get('local_backend_routes', [])
- except Exception as e:
- logger.error(f"Error loading unified resources: {e}")
-
- # Process legacy registry
- categories = set()
- metadata = {}
- raw_files = []
- trimmed_files = []
-
- if registry:
- metadata = registry.get("metadata", {})
- raw_files = registry.get("raw_files", [])
-
- # Extract categories from raw file content (basic parsing)
- for raw_file in raw_files[:5]: # Limit to first 5 files for performance
- content = raw_file.get("content", "")
- # Simple category detection from content
- if "market data" in content.lower() or "price" in content.lower():
- categories.add("market_data")
- if "explorer" in content.lower() or "blockchain" in content.lower():
- categories.add("block_explorer")
- if "rpc" in content.lower() or "node" in content.lower():
- categories.add("rpc_nodes")
- if "cors" in content.lower() or "proxy" in content.lower():
- categories.add("cors_proxy")
- if "news" in content.lower():
- categories.add("news")
- if "sentiment" in content.lower() or "fear" in content.lower():
- categories.add("sentiment")
- if "whale" in content.lower():
- categories.add("whale_tracking")
-
- # Provide trimmed raw files (first 500 chars each)
- for raw_file in raw_files[:10]: # Limit to 10 files
- content = raw_file.get("content", "")
- trimmed_files.append({
- "filename": raw_file.get("filename", ""),
- "preview": content[:500] + "..." if len(content) > 500 else content,
- "size": len(content)
- })
-
- # Add local category
- if local_routes:
- categories.add("local")
-
- return {
- "ok": True,
- "metadata": {
- "name": metadata.get("name", "") or unified_metadata.get("description", ""),
- "version": metadata.get("version", "") or unified_metadata.get("version", ""),
- "description": metadata.get("description", ""),
- "created_at": metadata.get("created_at", ""),
- "source_files": metadata.get("source_files", []),
- "updated": unified_metadata.get("updated", "")
- },
- "categories": list(categories),
- "local_routes": {
- "count": len(local_routes),
- "routes": local_routes[:20] # Return first 20 for preview
- },
- "raw_files_preview": trimmed_files,
- "total_raw_files": len(raw_files),
- "sources": ["all_apis_merged_2025.json", "crypto_resources_unified_2025-11-11.json"]
- }
-
-@app.get("/api/resources/apis/raw")
-async def get_resources_apis_raw():
- """Get raw files from API registry (trimmed to avoid huge payloads)"""
- registry = load_api_registry()
-
- if not registry:
- return {
- "ok": False,
- "error": "API registry file not found"
- }
-
- raw_files = registry.get("raw_files", [])
-
- # Return trimmed versions (first 1000 chars each, max 20 files)
- trimmed = []
- for raw_file in raw_files[:20]:
- content = raw_file.get("content", "")
- trimmed.append({
- "filename": raw_file.get("filename", ""),
- "preview": content[:1000] + "..." if len(content) > 1000 else content,
- "full_size": len(content)
- })
-
- return {
- "ok": True,
- "files": trimmed,
- "total_files": len(raw_files),
- "showing": min(20, len(raw_files)),
- "source": "all_apis_merged_2025.json"
- }
-
-
-@app.get("/api/trending")
-async def get_trending():
- """Trending coins from CoinGecko - REAL DATA ONLY"""
- try:
- data = await fetch_coingecko_trending()
-
- trending_coins = []
- if "coins" in data:
- for item in data["coins"][:10]:
- coin = item.get("item", {})
- trending_coins.append({
- "id": coin.get("id"),
- "name": coin.get("name"),
- "symbol": coin.get("symbol"),
- "market_cap_rank": coin.get("market_cap_rank"),
- "thumb": coin.get("thumb"),
- "score": coin.get("score", 0)
- })
-
- return {
- "trending": trending_coins,
- "count": len(trending_coins),
- "timestamp": datetime.now().isoformat(),
- "source": "CoinGecko API (Real Data)"
- }
-
- except Exception as e:
- raise HTTPException(status_code=503, detail=f"Failed to fetch trending: {str(e)}")
-
-
-# ===== Providers Management Endpoints =====
-@app.get("/api/providers")
-async def get_providers():
- """Get all providers with deduplication applied"""
- try:
- # Load primary config
- config = load_providers_config()
- providers_dict = config.get("providers", {})
-
- # Load auto-discovery report for validation status
- discovery_report = load_auto_discovery_report()
- discovery_results = {}
- if discovery_report and "http_providers" in discovery_report:
- for result in discovery_report["http_providers"].get("results", []):
- discovery_results[result.get("provider_id")] = result
-
- # Build provider list from primary config
- providers_list = []
- for provider_id, provider_data in providers_dict.items():
- # Merge with auto-discovery data if available
- discovery_data = discovery_results.get(provider_id, {})
-
- # Determine auth requirement
- auth_required = provider_data.get("requires_auth", False)
- free = not auth_required
-
- # Extract tags from provider data
- tags = []
- if "tags" in provider_data:
- tags = provider_data["tags"] if isinstance(provider_data["tags"], list) else [provider_data["tags"]]
-
- # Build description
- description = provider_data.get("description", "") or provider_data.get("note", "")
- if not description and provider_data.get("name"):
- description = f"{provider_data.get('name')} - {provider_data.get('category', 'unknown')} provider"
-
- provider_entry = {
- "id": provider_id,
- "name": provider_data.get("name", provider_id),
- "category": provider_data.get("category", "unknown"),
- "base_url": provider_data.get("base_url", ""),
- "auth_required": auth_required,
- "free": free,
- "tags": tags,
- "description": description,
- "type": provider_data.get("type", "http"),
- "priority": provider_data.get("priority", 0),
- "weight": provider_data.get("weight", 0),
- "rate_limit": provider_data.get("rate_limit", {}),
- "endpoints": provider_data.get("endpoints", {}),
- "status": discovery_data.get("status", "UNKNOWN") if discovery_data else "unvalidated",
- "validated_at": provider_data.get("validated_at"),
- "response_time_ms": discovery_data.get("response_time_ms") or provider_data.get("response_time_ms"),
- "added_by": provider_data.get("added_by", "manual")
- }
- providers_list.append(provider_entry)
-
- # Add HF Models as providers (with proper structure)
- try:
- from ai_models import MODEL_SPECS, _registry
- for model_key, spec in MODEL_SPECS.items():
- is_loaded = model_key in _registry._pipelines
- providers_list.append({
- "id": f"hf_model_{model_key}",
- "name": f"HF Model: {spec.model_id}",
- "category": spec.category,
- "base_url": f"/api/models/{model_key}/predict",
- "auth_required": spec.requires_auth,
- "free": not spec.requires_auth,
- "tags": ["huggingface", "ai-model", spec.task, spec.category],
- "description": f"Hugging Face {spec.task} model for {spec.category}",
- "type": "hf_model",
- "status": "available" if is_loaded else "not_loaded",
- "model_key": model_key,
- "model_id": spec.model_id,
- "task": spec.task,
- "added_by": "hf_models"
- })
- except Exception as e:
- logger.warning(f"Could not add HF models as providers: {e}")
-
- # Apply deduplication
- deduplicated_providers = deduplicate_providers(providers_list)
-
- return {
- "providers": deduplicated_providers,
- "total": len(deduplicated_providers),
- "source": "providers_config_extended.json + PROVIDER_AUTO_DISCOVERY_REPORT.json + HF Models (deduplicated)"
- }
- except Exception as e:
- logger.error(f"Error in get_providers: {e}")
- return {
- "providers": [],
- "total": 0,
- "error": str(e),
- "source": "error"
- }
-
-
-@app.get("/api/providers/{provider_id}")
-async def get_provider_detail(provider_id: str):
- """Get specific provider details"""
- # Check if it's an HF model provider
- if provider_id.startswith("hf_model_"):
- model_key = provider_id.replace("hf_model_", "")
- try:
- from ai_models import MODEL_SPECS, _registry
- if model_key not in MODEL_SPECS:
- raise HTTPException(status_code=404, detail=f"Model {model_key} not found")
-
- spec = MODEL_SPECS[model_key]
- is_loaded = model_key in _registry._pipelines
-
- return {
- "provider_id": provider_id,
- "name": f"HF Model: {spec.model_id}",
- "category": spec.category,
- "type": "hf_model",
- "status": "available" if is_loaded else "not_loaded",
- "model_key": model_key,
- "model_id": spec.model_id,
- "task": spec.task,
- "requires_auth": spec.requires_auth,
- "endpoint": f"/api/models/{model_key}/predict",
- "usage": {
- "method": "POST",
- "url": f"/api/models/{model_key}/predict",
- "body": {"text": "string", "options": {}}
- },
- "added_by": "hf_models"
- }
- except HTTPException:
- raise
- except Exception as e:
- raise HTTPException(status_code=500, detail=str(e))
-
- # Regular provider
- config = load_providers_config()
- providers = config.get("providers", {})
-
- if provider_id not in providers:
- raise HTTPException(status_code=404, detail=f"Provider {provider_id} not found")
-
- return {
- "provider_id": provider_id,
- **providers[provider_id]
- }
-
-
-@app.get("/api/providers/category/{category}")
-async def get_providers_by_category(category: str):
- """Get providers by category"""
- config = load_providers_config()
- providers = config.get("providers", {})
-
- filtered = {
- pid: data for pid, data in providers.items()
- if data.get("category") == category
- }
-
- return {
- "category": category,
- "providers": filtered,
- "count": len(filtered)
- }
-
-
-# ===== Pools Endpoints (Placeholder - to be implemented) =====
-@app.get("/api/pools")
-async def get_pools():
- """Get provider pools"""
- return {
- "pools": [],
- "message": "Pools feature not yet implemented in this version"
- }
-
-
-# ===== Logs Endpoints =====
-@app.get("/api/logs/recent")
-async def get_recent_logs():
- """Get recent logs"""
- return {
- "logs": _provider_state.get("logs", [])[-50:],
- "count": min(50, len(_provider_state.get("logs", [])))
- }
-
-
-@app.get("/api/logs/errors")
-async def get_error_logs():
- """Get error logs"""
- all_logs = _provider_state.get("logs", [])
- errors = [log for log in all_logs if log.get("level") == "ERROR"]
- return {
- "errors": errors[-50:],
- "count": len(errors)
- }
-
-
-# ===== Diagnostics Endpoints =====
-@app.post("/api/diagnostics/run")
-async def run_diagnostics(auto_fix: bool = False):
- """Run system diagnostics"""
- issues = []
- fixes_applied = []
-
- # Check database
- if not DB_PATH.exists():
- issues.append({"type": "database", "message": "Database file not found"})
- if auto_fix:
- init_database()
- fixes_applied.append("Initialized database")
-
- # Check providers config
- if not PROVIDERS_CONFIG_PATH.exists():
- issues.append({"type": "config", "message": "Providers config not found"})
-
- # Check auto-discovery report
- if not AUTO_DISCOVERY_REPORT_PATH.exists():
- issues.append({"type": "auto_discovery", "message": "Auto-discovery report not found"})
-
- return {
- "status": "completed",
- "issues_found": len(issues),
- "issues": issues,
- "fixes_applied": fixes_applied if auto_fix else [],
- "timestamp": datetime.now().isoformat()
- }
-
-
-@app.get("/api/diagnostics/last")
-async def get_last_diagnostics():
- """Get last diagnostics results"""
- # Would load from file in real implementation
- return {
- "status": "no_previous_run",
- "message": "No previous diagnostics run found"
- }
-
-
-@app.get("/api/diagnostics/health")
-async def get_diagnostics_health():
- """
- Get comprehensive health status of all providers and models.
- Returns health registry data for diagnostics and observability.
- """
- try:
- # Get provider health
- provider_health = _health_registry.get_all_entries()
- provider_summary = _health_registry.get_summary()
-
- # Get model health
- model_health = []
- model_summary = {
- "total": 0,
- "healthy": 0,
- "degraded": 0,
- "unavailable": 0,
- "unknown": 0,
- "in_cooldown": 0
- }
-
- try:
- from ai_models import get_model_health_registry
- model_health = get_model_health_registry()
- # Calculate model summary
- model_summary["total"] = len(model_health)
- for model in model_health:
- status = model.get("status", "unknown")
- model_summary[status] = model_summary.get(status, 0) + 1
- if model.get("in_cooldown", False):
- model_summary["in_cooldown"] += 1
- except Exception as e:
- logger.warning(f"Could not load model health: {e}")
-
- return {
- "status": "success",
- "timestamp": datetime.now().isoformat(),
- "providers": {
- "summary": provider_summary,
- "entries": provider_health
- },
- "models": {
- "summary": model_summary,
- "entries": model_health
- },
- "overall_health": {
- "providers_ok": provider_summary["healthy"] >= (provider_summary["total"] // 2) if provider_summary["total"] > 0 else True,
- "models_ok": model_summary["healthy"] >= (model_summary["total"] // 4) if model_summary["total"] > 0 else True
- }
- }
- except Exception as e:
- logger.error(f"Error getting health diagnostics: {e}")
- return {
- "status": "error",
- "error": str(e),
- "timestamp": datetime.now().isoformat()
- }
-
-
-@app.post("/api/diagnostics/run-test")
-async def run_diagnostic_test():
- """
- Run test_models_diagnostic.py and return results.
- Execute the Python script and capture stdout/stderr.
- """
- import subprocess
- import time
-
- start_time = time.time()
-
- try:
- # Find the diagnostic script - check multiple possible locations
- diagnostic_script = None
- possible_paths = [
- WORKSPACE_ROOT / "test_models_diagnostic.py",
- Path("test_models_diagnostic.py"),
- Path(__file__).parent / "test_models_diagnostic.py",
- ]
-
- for path in possible_paths:
- if path.exists():
- diagnostic_script = path
- break
-
- if not diagnostic_script:
- return {
- "status": "error",
- "output": "test_models_diagnostic.py not found. Searched in:\n" + "\n".join([str(p) for p in possible_paths]),
- "timestamp": datetime.now().isoformat(),
- "duration_seconds": 0,
- "summary": {
- "transformers_available": False,
- "hf_hub_connected": False,
- "models_loaded": 0,
- "critical_issues": ["Diagnostic script not found"]
- }
- }
-
- # Execute the diagnostic script
- result = subprocess.run(
- ["python3", str(diagnostic_script)],
- capture_output=True,
- text=True,
- timeout=60, # 60 second timeout
- cwd=str(diagnostic_script.parent)
- )
-
- duration = time.time() - start_time
-
- # Combine stdout and stderr
- full_output = result.stdout
- if result.stderr:
- full_output += "\n--- STDERR ---\n" + result.stderr
-
- # Parse output for summary information
- summary = {
- "transformers_available": "✅ transformers:" in full_output and "OK" in full_output,
- "hf_hub_connected": "✅ Hub connection:" in full_output and "OK" in full_output,
- "models_loaded": 0, # Would need more parsing to count actual loaded models
- "critical_issues": []
- }
-
- # Check for critical issues
- if "❌ transformers:" in full_output:
- summary["critical_issues"].append("Transformers library not available")
- if "❌ Authenticated access:" in full_output and "FAILED" in full_output:
- summary["critical_issues"].append("HuggingFace authentication failed")
- if "❌ Model not available" in full_output:
- summary["critical_issues"].append("AI models failed to load")
-
- return {
- "status": "success",
- "output": full_output,
- "timestamp": datetime.now().isoformat(),
- "duration_seconds": round(duration, 2),
- "summary": summary
- }
-
- except subprocess.TimeoutExpired:
- duration = time.time() - start_time
- return {
- "status": "timeout",
- "output": f"Test timed out after {duration:.1f} seconds",
- "timestamp": datetime.now().isoformat(),
- "duration_seconds": round(duration, 2),
- "summary": {
- "transformers_available": False,
- "hf_hub_connected": False,
- "models_loaded": 0,
- "critical_issues": ["Test execution timed out"]
- }
- }
-
- except Exception as e:
- duration = time.time() - start_time
- return {
- "status": "error",
- "output": f"Error running diagnostic test: {str(e)}",
- "timestamp": datetime.now().isoformat(),
- "duration_seconds": round(duration, 2),
- "summary": {
- "transformers_available": False,
- "hf_hub_connected": False,
- "models_loaded": 0,
- "critical_issues": [f"Execution error: {str(e)}"]
- }
- }
-
-
-@app.post("/api/diagnostics/self-heal")
-async def trigger_self_heal(model_key: Optional[str] = None):
- """
- Trigger self-healing actions for models.
- Safe, idempotent, and non-blocking.
-
- Query params:
- model_key: Specific model to reinitialize (optional)
- """
- try:
- from ai_models import attempt_model_reinit, get_model_health_registry
-
- results = []
-
- if model_key:
- # Reinit specific model
- result = attempt_model_reinit(model_key)
- results.append({
- "model_key": model_key,
- **result
- })
- else:
- # Reinit all failed models that are out of cooldown
- model_health = get_model_health_registry()
- failed_models = [
- m for m in model_health
- if m.get("status") in ["unavailable", "degraded"]
- and not m.get("in_cooldown", False)
- ]
-
- for model in failed_models[:5]: # Limit to 5 at a time to avoid blocking
- result = attempt_model_reinit(model["key"])
- results.append({
- "model_key": model["key"],
- **result
- })
-
- success_count = sum(1 for r in results if r.get("status") == "success")
-
- return {
- "status": "completed",
- "timestamp": datetime.now().isoformat(),
- "results": results,
- "summary": {
- "total_attempts": len(results),
- "successful": success_count,
- "failed": len(results) - success_count
- }
- }
- except Exception as e:
- logger.error(f"Error in self-heal: {e}")
- return {
- "status": "error",
- "error": str(e),
- "timestamp": datetime.now().isoformat()
- }
-
-
-# ===== APL (Auto Provider Loader) Endpoints =====
-@app.post("/api/apl/run")
-async def run_apl_scan():
- """Run APL provider scan"""
- try:
- # Run APL script
- result = subprocess.run(
- ["python3", str(WORKSPACE_ROOT / "auto_provider_loader.py")],
- capture_output=True,
- text=True,
- timeout=300,
- cwd=str(WORKSPACE_ROOT)
- )
-
- # Reload providers after APL run
- config = load_providers_config()
- _provider_state["providers"] = config.get("providers", {})
-
- return {
- "status": "completed",
- "stdout": result.stdout[-1000:], # Last 1000 chars
- "returncode": result.returncode,
- "providers_count": len(_provider_state["providers"]),
- "timestamp": datetime.now().isoformat()
- }
-
- except subprocess.TimeoutExpired:
- return {
- "status": "timeout",
- "message": "APL scan timed out after 5 minutes"
- }
- except Exception as e:
- raise HTTPException(status_code=500, detail=f"APL scan failed: {str(e)}")
-
-
-@app.get("/api/apl/report")
-async def get_apl_report():
- """Get APL validation report (alias for auto-discovery report)"""
- return await get_providers_auto_discovery_report()
-
-@app.get("/api/providers/auto-discovery-report")
-async def get_providers_auto_discovery_report():
- """Get PROVIDER_AUTO_DISCOVERY_REPORT.json"""
- report = load_auto_discovery_report()
-
- if not report:
- return {
- "ok": False,
- "error": "Auto-discovery report file not found",
- "message": f"Report file not found at {AUTO_DISCOVERY_REPORT_PATH}"
- }
-
- return {
- "ok": True,
- "report": report,
- "source": "PROVIDER_AUTO_DISCOVERY_REPORT.json"
- }
-
-@app.get("/api/providers/health-summary")
-async def get_providers_health_summary():
- """Get simplified health summary from auto-discovery report + local routes - always returns 200"""
- try:
- report = load_auto_discovery_report()
-
- # Load local routes for health checking
- resources_json = WORKSPACE_ROOT / "api-resources" / "crypto_resources_unified_2025-11-11.json"
- local_routes = []
- local_health = {"total": 0, "checked": 0, "up": 0, "down": 0}
-
- if resources_json.exists():
- try:
- with open(resources_json, 'r', encoding='utf-8') as f:
- unified_data = json.load(f)
- unified_registry = unified_data.get('registry', {})
- local_routes = unified_registry.get('local_backend_routes', [])
- local_health["total"] = len(local_routes)
-
- # Quick health check for up to 10 local routes
- async with httpx.AsyncClient(timeout=2.0) as client:
- routes_to_check = [r for r in local_routes if 'ws://' not in r.get('base_url', '')][:10]
- for route in routes_to_check:
- base_url = route.get('base_url', '').replace('{API_BASE}', f'http://localhost:{PORT}')
- if 'http' in base_url:
- try:
- response = await client.get(base_url, timeout=2.0)
- local_health["checked"] += 1
- if response.status_code < 500:
- local_health["up"] += 1
- else:
- local_health["down"] += 1
- except:
- local_health["checked"] += 1
- local_health["down"] += 1
- except Exception as e:
- logger.error(f"Error checking local routes health: {e}")
-
- if not report or "stats" not in report:
- return JSONResponse(
- status_code=200,
- content={
- "ok": False,
- "error": "Auto-discovery report not found or invalid",
- "message": f"Report file not found at {AUTO_DISCOVERY_REPORT_PATH}",
- "summary": {
- "total_active_providers": 0,
- "http_valid": 0,
- "http_invalid": 0,
- "http_conditional": 0,
- "hf_valid": 0,
- "hf_invalid": 0,
- "hf_conditional": 0,
- "status_breakdown": {"VALID": 0, "INVALID": 0, "CONDITIONALLY_AVAILABLE": 0},
- "execution_time_sec": 0,
- "timestamp": "",
- "local_routes": local_health
- }
- }
- )
-
- stats = report.get("stats", {})
- http_providers = report.get("http_providers", {})
- hf_providers = report.get("hf_providers", {})
-
- # Count by status
- status_counts = {"VALID": 0, "INVALID": 0, "CONDITIONALLY_AVAILABLE": 0}
- for result in http_providers.get("results", []):
- status = result.get("status", "UNKNOWN")
- if status in status_counts:
- status_counts[status] += 1
-
- return JSONResponse(
- status_code=200,
- content={
- "ok": True,
- "summary": {
- "total_active_providers": stats.get("total_active_providers", 0),
- "http_valid": stats.get("http_valid", 0),
- "http_invalid": stats.get("http_invalid", 0),
- "http_conditional": stats.get("http_conditional", 0),
- "hf_valid": stats.get("hf_valid", 0),
- "hf_invalid": stats.get("hf_invalid", 0),
- "hf_conditional": stats.get("hf_conditional", 0),
- "status_breakdown": status_counts,
- "execution_time_sec": stats.get("execution_time_sec", 0),
- "timestamp": stats.get("timestamp", ""),
- "local_routes": local_health
- },
- "source": "PROVIDER_AUTO_DISCOVERY_REPORT.json + local routes"
- }
- )
- except Exception as e:
- logger.error(f"Error loading health summary: {e}")
- return JSONResponse(
- status_code=200,
- content={
- "ok": False,
- "error": str(e),
- "summary": {
- "total_active_providers": 0,
- "http_valid": 0,
- "http_invalid": 0,
- "http_conditional": 0,
- "hf_valid": 0,
- "hf_invalid": 0,
- "hf_conditional": 0,
- "status_breakdown": {"VALID": 0, "INVALID": 0, "CONDITIONALLY_AVAILABLE": 0},
- "execution_time_sec": 0,
- "timestamp": "",
- "local_routes": {"total": 0, "checked": 0, "up": 0, "down": 0}
- }
- }
- )
-
-@app.get("/api/apl/summary")
-async def get_apl_summary():
- """Get APL summary statistics (alias for health-summary)"""
- return await get_providers_health_summary()
-
-
-# ===== HF Models Endpoints =====
-@app.get("/api/hf/models")
-async def get_hf_models():
- """Get HuggingFace models from APL report"""
- report = load_apl_report()
-
- if not report:
- return {"models": [], "count": 0}
-
- hf_models = report.get("hf_models", {}).get("results", [])
-
- return {
- "models": hf_models,
- "count": len(hf_models),
- "source": "APL Validation Report (Real Data)"
- }
-
-
-@app.get("/api/hf/health")
-async def get_hf_health():
- """Get HF services health"""
- try:
- from backend.services.hf_registry import REGISTRY
- health = REGISTRY.health()
- return health
- except Exception as e:
- return {
- "ok": False,
- "error": f"HF registry not available: {str(e)}"
- }
-
-
-# ===== DeFi Endpoint =====
-@app.get("/api/defi")
-async def get_defi():
- """DeFi endpoint"""
- return {
- "success": True,
- "message": "DeFi data endpoint",
- "data": [],
- "timestamp": datetime.now().isoformat()
- }
-
-
-# ===== News Endpoint (compatible with UI) =====
-@app.get("/api/news")
-async def get_news_api(limit: int = 20):
- """Get news (compatible with UI) - with external API fallback"""
- try:
- # Try to get news from database first
- conn = sqlite3.connect(str(DB_PATH))
- cursor = conn.cursor()
- cursor.execute("""
- SELECT * FROM news_articles
- ORDER BY analyzed_at DESC
- LIMIT ?
- """, (limit,))
- rows = cursor.fetchall()
- columns = [desc[0] for desc in cursor.description]
- conn.close()
-
- results = []
- for row in rows:
- record = dict(zip(columns, row))
- if record.get("related_symbols"):
- try:
- record["related_symbols"] = json.loads(record["related_symbols"])
- except:
- pass
- results.append(record)
-
- # If database is empty, fetch from external API
- if len(results) == 0:
- logger.info("No news in database, fetching from external API...")
- try:
- # Get API key from environment
- cryptocompare_api_key = os.getenv("CRYPTOCOMPARE_API_KEY", "968a5e25552b4cb5ba3280361d8444ab")
-
- async with httpx.AsyncClient(timeout=10.0) as client:
- # Try CryptoCompare News API with API key
- response = await client.get(
- "https://min-api.cryptocompare.com/data/v2/news/?lang=EN",
- headers={
- "User-Agent": "Mozilla/5.0",
- "authorization": f"Apikey {cryptocompare_api_key}"
- }
- )
- if response.status_code == 200:
- data = response.json()
- if data.get("Data"):
- for article in data["Data"][:limit]:
- results.append({
- "id": article.get("id"),
- "title": article.get("title", ""),
- "content": article.get("body", "")[:500],
- "url": article.get("url", ""),
- "source": article.get("source", "CryptoCompare"),
- "sentiment_label": None,
- "sentiment_confidence": None,
- "related_symbols": article.get("categories", "").split("|") if article.get("categories") else [],
- "published_date": datetime.fromtimestamp(article.get("published_on", 0)).isoformat() if article.get("published_on") else None,
- "analyzed_at": datetime.now().isoformat()
- })
- logger.info(f"Fetched {len(results)} news articles from CryptoCompare")
- except Exception as api_error:
- logger.warning(f"External news API failed: {api_error}")
-
- return {
- "success": True,
- "news": results,
- "count": len(results),
- "source": "database" if len(results) > 0 and rows else "external_api"
- }
- except Exception as e:
- logger.error(f"Error in get_news_api: {e}")
- return {
- "success": False,
- "news": [],
- "count": 0,
- "error": str(e)
- }
-
-
-# ===== Logs Endpoints =====
-@app.get("/api/logs/summary")
-async def get_logs_summary():
- """Get logs summary"""
- try:
- return {
- "success": True,
- "total": len(_provider_state.get("logs", [])),
- "recent": _provider_state.get("logs", [])[-10:],
- "timestamp": datetime.now().isoformat()
- }
- except Exception as e:
- return {
- "success": False,
- "error": str(e)
- }
-
-
-# ===== Diagnostics Endpoints =====
-@app.get("/api/diagnostics/errors")
-async def get_diagnostics_errors():
- """Get diagnostic errors"""
- try:
- return {
- "success": True,
- "errors": [],
- "timestamp": datetime.now().isoformat()
- }
- except Exception as e:
- return {
- "success": False,
- "errors": [],
- "error": str(e)
- }
-
-
-# ===== Resources Endpoints =====
-@app.get("/api/resources/search")
-async def search_resources(q: str = "", source: str = "all"):
- """Search resources"""
- try:
- return {
- "success": True,
- "query": q,
- "source": source,
- "results": [],
- "count": 0
- }
- except Exception as e:
- return {
- "success": False,
- "error": str(e)
- }
-
-
-# ===== V2 API Endpoints (compatibility) =====
-@app.post("/api/v2/export/{export_type}")
-async def export_v2(export_type: str, data: Dict[str, Any] = None):
- """V2 export endpoint"""
- return {
- "success": True,
- "type": export_type,
- "message": "Export functionality",
- "data": data or {}
- }
-
-
-@app.post("/api/v2/backup")
-async def backup_v2():
- """V2 backup endpoint"""
- return {
- "success": True,
- "message": "Backup functionality",
- "timestamp": datetime.now().isoformat()
- }
-
-
-@app.post("/api/v2/import/providers")
-async def import_providers_v2(data: Dict[str, Any]):
- """V2 import providers endpoint"""
- return {
- "success": True,
- "message": "Import providers functionality",
- "data": data
- }
-
-
-# ===== HuggingFace ML Sentiment Endpoints =====
-@app.post("/api/sentiment/analyze")
-async def analyze_sentiment(request: Dict[str, Any]):
- """Analyze sentiment using Hugging Face models"""
- try:
- from ai_models import (
- analyze_crypto_sentiment,
- analyze_financial_sentiment,
- analyze_social_sentiment,
- analyze_market_text,
- _registry,
- MODEL_SPECS,
- ModelNotAvailable
- )
-
- text = request.get("text", "").strip()
- if not text:
- raise HTTPException(status_code=400, detail="Text is required")
-
- mode = request.get("mode", "auto").lower()
- source = request.get("source", "user")
- model_key = request.get("model_key")
- symbol = request.get("symbol")
-
- try:
- # If model_key is provided, use that specific model
- if model_key and model_key in MODEL_SPECS:
- try:
- pipeline = _registry.get_pipeline(model_key)
- spec = MODEL_SPECS[model_key]
-
- # Handle different task types
- if spec.task == "text-generation":
- # For trading signal models or generation models
- raw_result = pipeline(text, max_length=200, num_return_sequences=1)
- if isinstance(raw_result, list) and raw_result:
- raw_result = raw_result[0]
-
- generated_text = raw_result.get("generated_text", str(raw_result))
-
- # Parse trading signals if applicable
- if spec.category == "trading_signal":
- # Extract signal from generated text
- decision = "HOLD"
- if "buy" in generated_text.lower():
- decision = "BUY"
- elif "sell" in generated_text.lower():
- decision = "SELL"
-
- return {
- "ok": True,
- "available": True,
- "sentiment": decision.lower(),
- "label": decision.lower(),
- "score": 0.7,
- "confidence": 0.7,
- "model": model_key,
- "engine": "huggingface",
- "mode": "trading",
- "extra": {
- "decision": decision,
- "rationale": generated_text,
- "raw": raw_result
- }
- }
- else:
- # Generation model - return generated text
- return {
- "ok": True,
- "available": True,
- "sentiment": "neutral",
- "label": "neutral",
- "score": 0.5,
- "confidence": 0.5,
- "model": model_key,
- "engine": "huggingface",
- "mode": "generation",
- "extra": {
- "generated_text": generated_text,
- "raw": raw_result
- }
- }
- else:
- # Text classification / sentiment
- raw_result = pipeline(text[:512])
- if isinstance(raw_result, list) and raw_result:
- raw_result = raw_result[0]
-
- label = raw_result.get("label", "neutral").upper()
- score = raw_result.get("score", 0.5)
-
- # Map labels to standard format
- mapped = "bullish" if "POSITIVE" in label or "BULLISH" in label or "LABEL_2" in label else (
- "bearish" if "NEGATIVE" in label or "BEARISH" in label or "LABEL_0" in label else "neutral"
- )
-
- return {
- "ok": True,
- "available": True,
- "sentiment": mapped,
- "label": mapped,
- "score": score,
- "confidence": score,
- "raw_label": label,
- "model": model_key,
- "engine": "huggingface",
- "mode": mode,
- "extra": {
- "vote": score if mapped == "bullish" else (-score if mapped == "bearish" else 0.0),
- "raw": raw_result
- }
- }
- except ModelNotAvailable as e:
- logger.warning(f"Model {model_key} not available: {e}")
- return {
- "ok": False,
- "available": False,
- "error": f"Model {model_key} not available: {str(e)}",
- "label": "neutral",
- "sentiment": "neutral",
- "score": 0.0,
- "confidence": 0.0
- }
-
- # Default mode-based analysis
- if mode == "crypto":
- result = analyze_crypto_sentiment(text)
- elif mode == "financial":
- result = analyze_financial_sentiment(text)
- elif mode == "social":
- result = analyze_social_sentiment(text)
- elif mode == "trading":
- # Try to use trading signal model
- result = analyze_crypto_sentiment(text)
- else:
- result = analyze_market_text(text)
-
- sentiment_label = result.get("label", "neutral")
- confidence = result.get("confidence", result.get("score", 0.5))
- model_used = result.get("model_count", result.get("model", result.get("engine", "unknown")))
-
- # Prepare response compatible with frontend format
- response_data = {
- "ok": True,
- "available": True,
- "sentiment": sentiment_label.lower(),
- "label": sentiment_label.lower(),
- "confidence": float(confidence),
- "score": float(confidence),
- "model": f"{model_used} models" if isinstance(model_used, int) else str(model_used),
- "engine": result.get("engine", "huggingface"),
- "mode": mode
- }
-
- # Add details if available for score bars
- if result.get("scores"):
- scores_dict = result.get("scores", {})
- if isinstance(scores_dict, dict):
- labels_list = []
- scores_list = []
- for lbl, scr in scores_dict.items():
- labels_list.append(lbl)
- scores_list.append(float(scr) if isinstance(scr, (int, float)) else float(scr.get("score", 0.5)) if isinstance(scr, dict) else 0.5)
- if labels_list:
- response_data["details"] = {
- "labels": labels_list,
- "scores": scores_list
- }
-
- # Save to database
- try:
- conn = sqlite3.connect(str(DB_PATH))
- cursor = conn.cursor()
- cursor.execute("""
- INSERT INTO sentiment_analysis
- (text, sentiment_label, confidence, model_used, analysis_type, symbol, scores)
- VALUES (?, ?, ?, ?, ?, ?, ?)
- """, (
- text[:500],
- sentiment_label,
- confidence,
- f"{model_used} models" if isinstance(model_used, int) else str(model_used),
- mode,
- symbol,
- json.dumps(result.get("scores", {}))
- ))
- conn.commit()
- conn.close()
- except Exception as db_error:
- logger.warning(f"Failed to save to database: {db_error}")
-
- return response_data
-
- except Exception as e:
- # Unexpected error - log and return error response
- logger.error(f"Sentiment analysis unexpected error: {str(e)}")
- return {
- "ok": False,
- "available": False,
- "error": f"Analysis failed: {str(e)}",
- "sentiment": "neutral",
- "label": "neutral",
- "confidence": 0.0,
- "score": 0.0
- }
-
- except HTTPException:
- raise
- except Exception as e:
- raise HTTPException(status_code=500, detail=f"Sentiment analysis failed: {str(e)}")
-
-
-@app.post("/api/ai/summarize")
-async def summarize_text(request: Dict[str, Any]):
- """
- Summarize text using Hugging Face models or simple text processing.
-
- Expects: { "text": "string", "max_sentences": 3 }
- Returns: { "ok": true, "summary": "...", "sentences": ["...", "..."] }
- """
- try:
- text = request.get("text", "").strip()
- max_sentences = request.get("max_sentences", 3)
-
- if not text:
- return {
- "ok": False,
- "error": "Text is required"
- }
-
- # Try to use Hugging Face summarization model if available
- try:
- from ai_models import MODEL_SPECS, _registry, ModelNotAvailable
-
- # Check if summarization model is available
- summarization_key = None
- for key, spec in MODEL_SPECS.items():
- if spec.task == "summarization":
- summarization_key = key
- break
-
- if summarization_key:
- try:
- pipeline = _registry.get_pipeline(summarization_key)
- # Use HF model for summarization
- # Try with parameters first, then fallback to simple call
- try:
- summary_result = pipeline(text, max_length=max_sentences * 50, min_length=max_sentences * 20, do_sample=False)
- except TypeError:
- # Some pipelines don't accept these parameters
- summary_result = pipeline(text)
-
- if isinstance(summary_result, list) and summary_result:
- summary_text = summary_result[0].get("summary_text", summary_result[0].get("generated_text", str(summary_result[0])))
- elif isinstance(summary_result, dict):
- summary_text = summary_result.get("summary_text", summary_result.get("generated_text", str(summary_result)))
- else:
- summary_text = str(summary_result)
-
- # Split into sentences
- sentences = [s.strip() + ("." if not s.strip().endswith((".", "!", "?")) else "") for s in summary_text.split(". ") if s.strip()]
- sentences = sentences[:max_sentences]
-
- return {
- "ok": True,
- "summary": summary_text,
- "sentences": sentences
- }
- except ModelNotAvailable:
- # Fall through to simple summarizer
- pass
- except Exception as e:
- logger.warning(f"HF summarization failed: {e}, using fallback")
- # Fall through to simple summarizer
- pass
- except Exception as e:
- logger.warning(f"HF summarization model not available: {e}")
- # Fall through to simple summarizer
-
- # Simple placeholder summarizer: split by sentences and take first N
- sentences = []
- current_sentence = ""
-
- for char in text:
- current_sentence += char
- if char in ".!?":
- sentence = current_sentence.strip()
- if sentence:
- sentences.append(sentence)
- current_sentence = ""
- if len(sentences) >= max_sentences:
- break
-
- # If we didn't get enough sentences, add the rest
- if len(sentences) < max_sentences and current_sentence.strip():
- sentences.append(current_sentence.strip())
-
- # If still no sentences, just truncate
- if not sentences:
- words = text.split()
- chunk_size = len(words) // max_sentences
- sentences = []
- for i in range(max_sentences):
- start_idx = i * chunk_size
- end_idx = start_idx + chunk_size if i < max_sentences - 1 else len(words)
- if start_idx < len(words):
- sentence = " ".join(words[start_idx:end_idx])
- if sentence:
- sentences.append(sentence)
-
- summary = " ".join(sentences)
-
- return {
- "ok": True,
- "summary": summary,
- "sentences": sentences[:max_sentences]
- }
-
- except Exception as e:
- logger.error(f"Summarization failed: {e}")
- return {
- "ok": False,
- "error": f"Summarization failed: {str(e)}"
- }
-
-
-@app.post("/api/news/analyze")
-async def analyze_news(request: Dict[str, Any]):
- """Analyze news article sentiment using HF models"""
- try:
- from ai_models import analyze_news_item
-
- title = request.get("title", "").strip()
- content = request.get("content", request.get("description", "")).strip()
- url = request.get("url", "")
- source = request.get("source", "unknown")
- published_date = request.get("published_date")
-
- if not title and not content:
- raise HTTPException(status_code=400, detail="Title or content is required")
-
- try:
- news_item = {
- "title": title,
- "description": content
- }
- result = analyze_news_item(news_item)
-
- sentiment_label = result.get("sentiment", "neutral")
- sentiment_confidence = result.get("sentiment_confidence", 0.5)
- sentiment_details = result.get("sentiment_details", {})
- related_symbols = request.get("related_symbols", [])
-
- # Check if HF models were used (for diagnostics)
- hf_available = sentiment_details.get("engine", "unknown") == "huggingface" if isinstance(sentiment_details, dict) else True
-
- # Save to database (always)
- saved_to_db = False
- try:
- conn = sqlite3.connect(str(DB_PATH))
- cursor = conn.cursor()
- cursor.execute("""
- INSERT INTO news_articles
- (title, content, url, source, sentiment_label, sentiment_confidence, related_symbols, published_date)
- VALUES (?, ?, ?, ?, ?, ?, ?, ?)
- """, (
- title[:500],
- content[:2000] if content else None,
- url,
- source,
- sentiment_label,
- sentiment_confidence,
- json.dumps(related_symbols) if related_symbols else None,
- published_date
- ))
- conn.commit()
- conn.close()
- saved_to_db = True
- except Exception as db_error:
- logger.warning(f"Failed to save to database: {db_error}")
-
- return {
- "success": True,
- "available": True,
- "hf_models_available": hf_available,
- "news": {
- "title": title,
- "sentiment": sentiment_label,
- "confidence": sentiment_confidence,
- "details": sentiment_details
- },
- "saved_to_db": saved_to_db
- }
-
- except Exception as e:
- logger.error(f"News analysis error: {str(e)}")
- return {
- "success": False,
- "available": False,
- "error": f"Analysis failed: {str(e)}",
- "news": {
- "title": title,
- "sentiment": "neutral",
- "confidence": 0.0
- }
- }
-
- except HTTPException:
- raise
- except Exception as e:
- raise HTTPException(status_code=500, detail=f"News analysis failed: {str(e)}")
-
-
-@app.get("/api/sentiment/history")
-async def get_sentiment_history(
- symbol: Optional[str] = None,
- limit: int = 50
-):
- """Get sentiment analysis history from database"""
- try:
- conn = sqlite3.connect(str(DB_PATH))
- cursor = conn.cursor()
-
- if symbol:
- cursor.execute("""
- SELECT * FROM sentiment_analysis
- WHERE symbol = ?
- ORDER BY timestamp DESC
- LIMIT ?
- """, (symbol.upper(), limit))
- else:
- cursor.execute("""
- SELECT * FROM sentiment_analysis
- ORDER BY timestamp DESC
- LIMIT ?
- """, (limit,))
-
- rows = cursor.fetchall()
- columns = [desc[0] for desc in cursor.description]
- conn.close()
-
- results = []
- for row in rows:
- record = dict(zip(columns, row))
- if record.get("scores"):
- try:
- record["scores"] = json.loads(record["scores"])
- except:
- pass
- results.append(record)
-
- return {
- "success": True,
- "count": len(results),
- "results": results
- }
-
- except Exception as e:
- raise HTTPException(status_code=500, detail=f"Failed to fetch sentiment history: {str(e)}")
-
-
-@app.post("/api/news/fetch")
-async def fetch_and_save_news(limit: int = 50):
- """Fetch news from CryptoCompare API and save to database"""
- try:
- cryptocompare_api_key = os.getenv("CRYPTOCOMPARE_API_KEY", "968a5e25552b4cb5ba3280361d8444ab")
-
- async with httpx.AsyncClient(timeout=15.0) as client:
- response = await client.get(
- "https://min-api.cryptocompare.com/data/v2/news/?lang=EN",
- headers={
- "User-Agent": "Mozilla/5.0",
- "authorization": f"Apikey {cryptocompare_api_key}"
- }
- )
-
- if response.status_code != 200:
- return {
- "success": False,
- "error": f"CryptoCompare API returned {response.status_code}",
- "saved": 0
- }
-
- data = response.json()
-
- if not data.get("Data"):
- return {
- "success": False,
- "error": "No news data returned from API",
- "saved": 0
- }
-
- # Save to database
- conn = sqlite3.connect(str(DB_PATH))
- cursor = conn.cursor()
- saved_count = 0
-
- for article in data["Data"][:limit]:
- try:
- # Check if article already exists
- cursor.execute("SELECT id FROM news_articles WHERE url = ?", (article.get("url", ""),))
- if cursor.fetchone():
- continue # Skip duplicates
-
- # Extract related symbols from categories
- categories = article.get("categories", "").split("|") if article.get("categories") else []
- related_symbols_json = json.dumps(categories)
-
- # Insert news article
- cursor.execute("""
- INSERT INTO news_articles (
- title, content, url, source,
- related_symbols, published_date, analyzed_at
- ) VALUES (?, ?, ?, ?, ?, ?, ?)
- """, (
- article.get("title", ""),
- article.get("body", "")[:1000], # Limit content length
- article.get("url", ""),
- article.get("source", "CryptoCompare"),
- related_symbols_json,
- datetime.fromtimestamp(article.get("published_on", 0)).isoformat() if article.get("published_on") else None,
- datetime.now().isoformat()
- ))
- saved_count += 1
- except Exception as e:
- logger.warning(f"Error saving article: {e}")
- continue
-
- conn.commit()
- conn.close()
-
- logger.info(f"[OK] Saved {saved_count} news articles to database")
-
- return {
- "success": True,
- "saved": saved_count,
- "total_fetched": len(data["Data"][:limit]),
- "message": f"Successfully saved {saved_count} news articles"
- }
-
- except Exception as e:
- logger.error(f"Error fetching news: {e}")
- return {
- "success": False,
- "error": str(e),
- "saved": 0
- }
-
-
-@app.get("/api/news/latest")
-async def get_latest_news(
- limit: int = 20,
- sentiment: Optional[str] = None
-):
- """Get latest analyzed news from database"""
- try:
- conn = sqlite3.connect(str(DB_PATH))
- cursor = conn.cursor()
-
- if sentiment:
- cursor.execute("""
- SELECT * FROM news_articles
- WHERE sentiment_label = ?
- ORDER BY analyzed_at DESC
- LIMIT ?
- """, (sentiment.lower(), limit))
- else:
- cursor.execute("""
- SELECT * FROM news_articles
- ORDER BY analyzed_at DESC
- LIMIT ?
- """, (limit,))
-
- rows = cursor.fetchall()
- columns = [desc[0] for desc in cursor.description]
- conn.close()
-
- results = []
- for row in rows:
- record = dict(zip(columns, row))
- if record.get("related_symbols"):
- try:
- record["related_symbols"] = json.loads(record["related_symbols"])
- except:
- pass
- results.append(record)
-
- return {
- "success": True,
- "count": len(results),
- "news": results
- }
-
- except Exception as e:
- raise HTTPException(status_code=500, detail=f"Failed to fetch news: {str(e)}")
-
-
-@app.post("/api/news/summarize")
-async def summarize_news(request: Dict[str, Any]):
- """
- Summarize crypto/financial news using Hugging Face Crypto-Financial-News-Summarizer model
-
- Expects: { "title": "News Title", "content": "Full article text" }
- Returns: { "summary": "Summarized news paragraph", "model": "Crypto-Financial-News-Summarizer" }
- """
- try:
- from ai_models import MODEL_SPECS, _registry, ModelNotAvailable
-
- title = request.get("title", "").strip()
- content = request.get("content", "").strip()
-
- if not title and not content:
- raise HTTPException(status_code=400, detail="Title or content is required")
-
- # Combine title and content for summarization
- text_to_summarize = f"{title}. {content}" if title and content else (title or content)
-
- try:
- # Try to use the Crypto-Financial-News-Summarizer model
- summarization_key = "summarization_0"
-
- if summarization_key in MODEL_SPECS:
- try:
- pipeline = _registry.get_pipeline(summarization_key)
- spec = MODEL_SPECS[summarization_key]
-
- # Use HF model for summarization
- # Limit input text to avoid token length issues
- max_input_length = 1024
- text_input = text_to_summarize[:max_input_length]
-
- try:
- # Try with parameters first
- summary_result = pipeline(
- text_input,
- max_length=150,
- min_length=50,
- do_sample=False,
- truncation=True
- )
- except TypeError:
- # Some pipelines don't accept these parameters
- summary_result = pipeline(text_input, truncation=True)
-
- # Extract summary text from result
- if isinstance(summary_result, list) and summary_result:
- summary_text = summary_result[0].get("summary_text", summary_result[0].get("generated_text", str(summary_result[0])))
- elif isinstance(summary_result, dict):
- summary_text = summary_result.get("summary_text", summary_result.get("generated_text", str(summary_result)))
- else:
- summary_text = str(summary_result)
-
- return {
- "success": True,
- "summary": summary_text,
- "model": spec.model_id,
- "available": True,
- "input_length": len(text_input),
- "title": title,
- "timestamp": datetime.now().isoformat()
- }
-
- except ModelNotAvailable as e:
- logger.warning(f"Crypto-Financial-News-Summarizer not available: {e}")
- # Fall through to fallback
- except Exception as e:
- logger.warning(f"HF summarization failed: {e}, using fallback")
- # Fall through to fallback
-
- # Fallback: Simple extractive summarization
- # Split into sentences and take the most important ones
- sentences = []
- current_sentence = ""
-
- for char in text_to_summarize:
- current_sentence += char
- if char in ".!?":
- sentence = current_sentence.strip()
- if sentence and len(sentence) > 10: # Filter out very short sentences
- sentences.append(sentence)
- current_sentence = ""
- if len(sentences) >= 5: # Take first 5 sentences max
- break
-
- # If we didn't get enough sentences, add the rest
- if len(sentences) < 3 and current_sentence.strip():
- sentences.append(current_sentence.strip())
-
- # Take first 3 sentences as summary
- summary = " ".join(sentences[:3]) if sentences else text_to_summarize[:500]
-
- return {
- "success": True,
- "summary": summary,
- "model": "fallback_extractive",
- "available": False,
- "note": "Using fallback extractive summarization (HF model not available)",
- "title": title,
- "timestamp": datetime.now().isoformat()
- }
-
- except Exception as e:
- logger.error(f"Summarization error: {str(e)}")
- return {
- "success": False,
- "error": f"Summarization failed: {str(e)}",
- "summary": "",
- "model": "error",
- "available": False
- }
-
- except HTTPException:
- raise
- except Exception as e:
- raise HTTPException(status_code=500, detail=f"News summarization failed: {str(e)}")
-
-
-@app.get("/api/models/status")
-async def get_models_status():
- """Get AI models status and registry info - honest status reporting"""
- try:
- from ai_models import (
- get_model_info, registry_status, HF_MODE, TRANSFORMERS_AVAILABLE,
- INFERENCE_API_MODE, _registry,
- )
-
- model_info = get_model_info()
- registry_info = registry_status()
- loaded_count = len(_registry._pipelines) + len(_registry._inference_ready)
-
- # Determine honest status
- if HF_MODE == "off":
- status = "disabled"
- status_message = "HF models are disabled (HF_MODE=off). To enable them, set HF_MODE=public or HF_MODE=auth in the environment."
- elif INFERENCE_API_MODE and loaded_count > 0:
- status = "ok" if len(_registry._failed_models) == 0 else "partial"
- status_message = f"{loaded_count} model(s) ready via HF Inference API"
- elif not TRANSFORMERS_AVAILABLE and not INFERENCE_API_MODE:
- status = "transformers_unavailable"
- status_message = "Transformers library is not installed. Models cannot be loaded."
- elif not _registry._initialized:
- status = "not_initialized"
- status_message = "Models have not been initialized yet."
- elif loaded_count == 0:
- status = "no_models_loaded"
- status_message = f"No models could be loaded. {len(_registry._failed_models)} models failed. Check model IDs or HF access."
- elif loaded_count > 0:
- status = "ok" if len(_registry._failed_models) == 0 else "partial"
- backend = "inference API" if INFERENCE_API_MODE else "local"
- status_message = f"{loaded_count} model(s) loaded successfully ({backend})"
- if len(_registry._failed_models) > 0:
- status_message += f", {len(_registry._failed_models)} failed"
- else:
- status = "unknown"
- status_message = "Unknown status"
-
- # Format failed models as list of [key, error] tuples for ai_tools.html
- failed_list = []
- for key, error in list(_registry._failed_models.items())[:10]:
- failed_list.append([key, str(error)])
-
- return {
- "success": True,
- "status": status,
- "status_message": status_message,
- "hf_mode": HF_MODE,
- "inference_api_mode": INFERENCE_API_MODE,
- "models_loaded": loaded_count,
- "models_failed": len(_registry._failed_models),
- "transformers_available": TRANSFORMERS_AVAILABLE,
- "initialized": _registry._initialized,
- "models": model_info,
- "registry": registry_info,
- "failed": failed_list, # Format: [[key, error], ...] for ai_tools.html
- "failed_models": list(_registry._failed_models.keys())[:10], # Keep for backward compatibility
- "loaded_models": list(_registry._pipelines.keys()) + list(_registry._inference_ready),
- "database": {
- "path": str(DB_PATH),
- "exists": DB_PATH.exists()
- }
- }
- except Exception as e:
- logger.error(f"Error getting models status: {e}")
- return {
- "success": False,
- "status": "error",
- "status_message": f"Error retrieving model status: {str(e)}",
- "error": str(e),
- "hf_mode": "unknown",
- "models_loaded": 0,
- "models_failed": 0
- }
-
-
-@app.post("/api/models/initialize")
-async def initialize_ai_models():
- """Initialize AI models (force reload)"""
- try:
- from ai_models import initialize_models, _registry, HF_MAX_STARTUP_MODELS
-
- result = initialize_models(max_models=HF_MAX_STARTUP_MODELS)
- registry_status = _registry.get_registry_status()
-
- return registry_status
- except Exception as e:
- logger.error(f"Failed to initialize models: {e}")
- return {
- "models_total": 0,
- "models_loaded": 0,
- "models_failed": 0,
- "items": [],
- "error": str(e)
- }
-
-
-# ===== Model-based Data Endpoints (Using HF Models as Data Sources) =====
-@app.get("/api/models/list")
-async def list_available_models():
- """List all available Hugging Face models as data sources"""
- try:
- from ai_models import get_model_info, MODEL_SPECS, _registry, CRYPTO_SENTIMENT_MODELS, SOCIAL_SENTIMENT_MODELS, FINANCIAL_SENTIMENT_MODELS, NEWS_SENTIMENT_MODELS, GENERATION_MODELS, TRADING_SIGNAL_MODELS
-
- model_info = get_model_info()
-
- # Model descriptions
- model_descriptions = {
- "kk08/CryptoBERT": "Crypto sentiment binary classification model trained on cryptocurrency-related text",
- "ElKulako/cryptobert": "Crypto social sentiment classifier (Bullish/Neutral/Bearish) for social media and news",
- "StephanAkkerman/FinTwitBERT-sentiment": "Financial tweet sentiment analysis model for market-related social media content",
- "OpenC/crypto-gpt-o3-mini": "Crypto and DeFi text generation model for analysis and content creation",
- "ElKulako/cryptobert": "Crypto sentiment model used for trading signal generation (buy/sell/hold based on sentiment)",
- "cardiffnlp/twitter-roberta-base-sentiment-latest": "General Twitter sentiment analysis (fallback model)",
- "ProsusAI/finbert": "Financial sentiment analysis model for news and financial documents",
- "FurkanGozukara/Crypto-Financial-News-Summarizer": "Specialized model for summarizing cryptocurrency and financial news articles"
- }
-
- models_list = []
- for key, spec in MODEL_SPECS.items():
- is_loaded = key in _registry._pipelines or key in getattr(_registry, "_inference_ready", set())
- error_msg = None
- if key in _registry._failed_models:
- error_msg = str(_registry._failed_models[key])
-
- models_list.append({
- "key": key,
- "id": key,
- "name": spec.model_id,
- "model_id": spec.model_id,
- "task": spec.task,
- "category": spec.category,
- "requires_auth": spec.requires_auth,
- "loaded": is_loaded,
- "error": error_msg,
- "description": model_descriptions.get(spec.model_id, f"{spec.category} model for {spec.task}"),
- "endpoint": f"/api/models/{key}/predict"
- })
-
- return {
- "success": True,
- "total_models": len(models_list),
- "models": models_list,
- "categories": {
- "crypto_sentiment": CRYPTO_SENTIMENT_MODELS,
- "social_sentiment": SOCIAL_SENTIMENT_MODELS,
- "financial_sentiment": FINANCIAL_SENTIMENT_MODELS,
- "news_sentiment": NEWS_SENTIMENT_MODELS,
- "generation": GENERATION_MODELS,
- "trading_signals": TRADING_SIGNAL_MODELS,
- "summarization": ["FurkanGozukara/Crypto-Financial-News-Summarizer"]
- },
- "model_info": model_info
- }
- except Exception as e:
- return {
- "success": False,
- "error": str(e),
- "models": []
- }
-
-
-@app.get("/api/models/{model_key}/info")
-async def get_model_info_endpoint(model_key: str):
- """Get information about a specific model"""
- try:
- from ai_models import MODEL_SPECS, ModelNotAvailable, _registry
-
- if model_key not in MODEL_SPECS:
- raise HTTPException(status_code=404, detail=f"Model {model_key} not found")
-
- spec = MODEL_SPECS[model_key]
- is_loaded = model_key in _registry._pipelines
-
- return {
- "success": True,
- "model_key": model_key,
- "model_id": spec.model_id,
- "task": spec.task,
- "category": spec.category,
- "requires_auth": spec.requires_auth,
- "is_loaded": is_loaded,
- "endpoint": f"/api/models/{model_key}/predict",
- "usage": {
- "method": "POST",
- "url": f"/api/models/{model_key}/predict",
- "body": {"text": "string", "options": {}}
- }
- }
- except HTTPException:
- raise
- except Exception as e:
- raise HTTPException(status_code=500, detail=str(e))
-
-
-@app.post("/api/models/{model_key}/predict")
-async def predict_with_model(model_key: str, request: Dict[str, Any]):
- """Use a specific model to generate predictions/data"""
- try:
- from ai_models import MODEL_SPECS, _registry, ModelNotAvailable
-
- if model_key not in MODEL_SPECS:
- raise HTTPException(status_code=404, detail=f"Model {model_key} not found")
-
- spec = MODEL_SPECS[model_key]
- text = request.get("text", "").strip()
-
- if not text:
- raise HTTPException(status_code=400, detail="Text is required")
-
- try:
- pipeline = _registry.get_pipeline(model_key)
- result = pipeline(text[:512])
-
- if isinstance(result, list) and result:
- result = result[0]
-
- return {
- "success": True,
- "available": True,
- "model_key": model_key,
- "model_id": spec.model_id,
- "task": spec.task,
- "input": text[:100],
- "output": result,
- "timestamp": datetime.now().isoformat()
- }
- except ModelNotAvailable as e:
- return {
- "success": False,
- "available": False,
- "model_key": model_key,
- "model_id": spec.model_id,
- "error": str(e),
- "reason": "model_unavailable"
- }
-
- except HTTPException:
- raise
- except Exception as e:
- raise HTTPException(status_code=500, detail=f"Prediction failed: {str(e)}")
-
-
-@app.post("/api/models/batch/predict")
-async def batch_predict(request: Dict[str, Any]):
- """Batch prediction using multiple models"""
- try:
- from ai_models import MODEL_SPECS, _registry, ModelNotAvailable
-
- texts = request.get("texts", [])
- model_keys = request.get("models", [])
-
- if not texts:
- raise HTTPException(status_code=400, detail="Texts array is required")
-
- if not model_keys:
- model_keys = list(MODEL_SPECS.keys())[:5]
-
- results = []
- for text in texts:
- if not text.strip():
- continue
-
- text_results = {}
- for model_key in model_keys:
- if model_key not in MODEL_SPECS:
- continue
-
- try:
- spec = MODEL_SPECS[model_key]
- pipeline = _registry.get_pipeline(model_key)
- result = pipeline(text[:512])
-
- if isinstance(result, list) and result:
- result = result[0]
-
- text_results[model_key] = {
- "model_id": spec.model_id,
- "result": result,
- "success": True
- }
- except ModelNotAvailable:
- text_results[model_key] = {
- "success": False,
- "error": "Model not available"
- }
- except Exception as e:
- text_results[model_key] = {
- "success": False,
- "error": str(e)
- }
-
- results.append({
- "text": text[:100],
- "predictions": text_results
- })
-
- return {
- "success": True,
- "total_texts": len(results),
- "models_used": model_keys,
- "results": results,
- "timestamp": datetime.now().isoformat()
- }
-
- except HTTPException:
- raise
- except Exception as e:
- raise HTTPException(status_code=500, detail=f"Batch prediction failed: {str(e)}")
-
-
-@app.post("/api/analyze/text")
-async def analyze_text(request: Dict[str, Any]):
- """
- Analyze or generate text using crypto-gpt-o3-mini generation model.
-
- Expects: { "prompt": "...", "mode": "analysis" | "generation" }
- Returns: { "text": "...", "model": "OpenC/crypto-gpt-o3-mini" }
- """
- try:
- from ai_models import MODEL_SPECS, _registry, ModelNotAvailable
-
- prompt = request.get("prompt", "").strip()
- mode = request.get("mode", "analysis").lower()
- max_length = request.get("max_length", 200)
-
- if not prompt:
- raise HTTPException(status_code=400, detail="Prompt is required")
-
- # Find generation model (crypto-gpt-o3-mini) - use specific key first
- generation_key = "crypto_ai_analyst" if "crypto_ai_analyst" in MODEL_SPECS else None
-
- # Fallback: search by category or model name
- if not generation_key:
- for key, spec in MODEL_SPECS.items():
- if spec.category == "analysis_generation" or "crypto-gpt" in spec.model_id.lower():
- generation_key = key
- break
-
- if not generation_key:
- return {
- "success": False,
- "available": False,
- "error": "Crypto text generation model not configured",
- "text": ""
- }
-
- try:
- spec = MODEL_SPECS[generation_key]
- pipeline = _registry.get_pipeline(generation_key)
-
- # Generate text
- result = pipeline(prompt, max_length=max_length, num_return_sequences=1, truncation=True)
-
- if isinstance(result, list) and result:
- result = result[0]
-
- generated_text = result.get("generated_text", str(result))
-
- return {
- "success": True,
- "available": True,
- "text": generated_text,
- "model": spec.model_id,
- "mode": mode,
- "prompt": prompt[:100],
- "timestamp": datetime.now().isoformat()
- }
-
- except ModelNotAvailable as e:
- logger.warning(f"Generation model not available: {e}")
- return {
- "success": False,
- "available": False,
- "error": f"Model not available: {str(e)}",
- "text": "",
- "note": "HF model unavailable - check model configuration"
- }
-
- except HTTPException:
- raise
- except Exception as e:
- logger.error(f"Text analysis failed: {e}")
- raise HTTPException(status_code=500, detail=f"Text analysis failed: {str(e)}")
-
-
-@app.post("/api/trading/decision")
-async def trading_decision(request: Dict[str, Any]):
- """
- Get trading decision based on sentiment analysis.
- Uses sentiment analysis to determine BUY/SELL/HOLD signals.
-
- Expects: { "symbol": "BTC", "context": "market context..." }
- Returns: {
- "decision": "BUY" | "SELL" | "HOLD",
- "confidence": float,
- "rationale": "explanation",
- "raw": {...}
- }
- """
- try:
- from ai_models import analyze_crypto_sentiment
-
- symbol = request.get("symbol", "").strip().upper()
- context = request.get("context", "").strip()
-
- if not symbol:
- raise HTTPException(status_code=400, detail="Symbol is required")
-
- # Build text for sentiment analysis
- if context:
- analysis_text = f"{symbol} {context}"
- else:
- analysis_text = f"{symbol} market analysis"
-
- # Default response in case of any failure
- default_response = {
- "success": True,
- "available": True,
- "decision": "HOLD",
- "confidence": 0.5,
- "rationale": "Sentiment analysis unavailable - defaulting to HOLD",
- "symbol": symbol,
- "model": "fallback",
- "context_provided": bool(context),
- "timestamp": datetime.now().isoformat()
- }
-
- try:
- # Analyze sentiment using crypto sentiment model
- sentiment_result = analyze_crypto_sentiment(analysis_text)
-
- # Extract sentiment label and confidence
- sentiment_label = sentiment_result.get("label", "neutral").lower()
- confidence = sentiment_result.get("confidence", 0.5)
-
- # Map sentiment to trading decision
- decision = "HOLD" # Default
- if sentiment_label == "bullish":
- decision = "BUY"
- elif sentiment_label == "bearish":
- decision = "SELL"
- else: # neutral or unknown
- decision = "HOLD"
-
- # Build rationale
- rationale = f"Sentiment analysis indicates {sentiment_label} sentiment (confidence: {confidence:.2f})"
- if context:
- rationale += f" based on: {context[:200]}"
-
- return {
- "success": True,
- "available": True,
- "decision": decision,
- "confidence": float(confidence),
- "rationale": rationale,
- "symbol": symbol,
- "model": sentiment_result.get("engine", "sentiment_analysis"),
- "sentiment": sentiment_label,
- "context_provided": bool(context),
- "raw": sentiment_result,
- "timestamp": datetime.now().isoformat()
- }
-
- except Exception as e:
- logger.warning(f"Sentiment analysis failed for trading decision: {e}")
- # Return default HOLD response instead of crashing
- default_response["error"] = f"Sentiment analysis failed: {str(e)[:100]}"
- default_response["note"] = "Using default HOLD signal due to analysis failure"
- return default_response
-
- except HTTPException:
- raise
- except Exception as e:
- logger.error(f"Trading decision failed: {e}")
- # Return safe default instead of raising exception
- return {
- "success": True,
- "available": False,
- "error": f"Trading decision processing failed: {str(e)[:100]}",
- "decision": "HOLD",
- "confidence": 0.5,
- "rationale": "Error occurred during analysis - defaulting to HOLD for safety",
- "symbol": request.get("symbol", "UNKNOWN"),
- "timestamp": datetime.now().isoformat()
- }
-
-
-@app.get("/api/models/data/generated")
-async def get_generated_data(
- limit: int = 50,
- model_key: Optional[str] = None,
- symbol: Optional[str] = None
-):
- """Get data generated by models from database"""
- try:
- conn = sqlite3.connect(str(DB_PATH))
- cursor = conn.cursor()
-
- if model_key and symbol:
- cursor.execute("""
- SELECT * FROM sentiment_analysis
- WHERE analysis_type = ? AND symbol = ?
- ORDER BY timestamp DESC
- LIMIT ?
- """, (model_key, symbol.upper(), limit))
- elif model_key:
- cursor.execute("""
- SELECT * FROM sentiment_analysis
- WHERE analysis_type = ?
- ORDER BY timestamp DESC
- LIMIT ?
- """, (model_key, limit))
- elif symbol:
- cursor.execute("""
- SELECT * FROM sentiment_analysis
- WHERE symbol = ?
- ORDER BY timestamp DESC
- LIMIT ?
- """, (symbol.upper(), limit))
- else:
- cursor.execute("""
- SELECT * FROM sentiment_analysis
- ORDER BY timestamp DESC
- LIMIT ?
- """, (limit,))
-
- rows = cursor.fetchall()
- columns = [desc[0] for desc in cursor.description]
- conn.close()
-
- results = []
- for row in rows:
- record = dict(zip(columns, row))
- if record.get("scores"):
- try:
- record["scores"] = json.loads(record["scores"])
- except:
- pass
- results.append(record)
-
- return {
- "success": True,
- "count": len(results),
- "data": results,
- "source": "models",
- "timestamp": datetime.now().isoformat()
- }
-
- except Exception as e:
- raise HTTPException(status_code=500, detail=f"Failed to fetch generated data: {str(e)}")
-
-
-@app.get("/api/models/data/stats")
-async def get_models_data_stats():
- """Get statistics about data generated by models"""
- try:
- conn = sqlite3.connect(str(DB_PATH))
- cursor = conn.cursor()
-
- cursor.execute("SELECT COUNT(*) FROM sentiment_analysis")
- total_analyses = cursor.fetchone()[0]
-
- cursor.execute("SELECT COUNT(DISTINCT symbol) FROM sentiment_analysis WHERE symbol IS NOT NULL")
- unique_symbols = cursor.fetchone()[0]
-
- cursor.execute("SELECT COUNT(DISTINCT analysis_type) FROM sentiment_analysis")
- unique_types = cursor.fetchone()[0]
-
- cursor.execute("""
- SELECT sentiment_label, COUNT(*) as count
- FROM sentiment_analysis
- GROUP BY sentiment_label
- """)
- sentiment_dist = {row[0]: row[1] for row in cursor.fetchall()}
-
- cursor.execute("""
- SELECT analysis_type, COUNT(*) as count
- FROM sentiment_analysis
- GROUP BY analysis_type
- """)
- type_dist = {row[0]: row[1] for row in cursor.fetchall()}
-
- conn.close()
-
- return {
- "success": True,
- "statistics": {
- "total_analyses": total_analyses,
- "unique_symbols": unique_symbols,
- "unique_model_types": unique_types,
- "sentiment_distribution": sentiment_dist,
- "model_type_distribution": type_dist
- },
- "timestamp": datetime.now().isoformat()
- }
-
- except Exception as e:
- raise HTTPException(status_code=500, detail=f"Failed to fetch statistics: {str(e)}")
-
-
-@app.post("/api/hf/run-sentiment")
-async def run_hf_sentiment(data: Dict[str, Any]):
- """Run sentiment analysis using HF models (compatible with UI)"""
- try:
- from ai_models import analyze_market_text, ModelNotAvailable
-
- texts = data.get("texts", [])
- if isinstance(texts, str):
- texts = [texts]
-
- if not texts or not any(t.strip() for t in texts):
- raise HTTPException(status_code=400, detail="At least one text is required")
-
- try:
- all_results = []
- total_vote = 0.0
- count = 0
- models_available = False
-
- for text in texts:
- if not text.strip():
- continue
-
- result = analyze_market_text(text.strip())
-
- # Check if models are available
- if result.get("available", True):
- models_available = True
-
- label = result.get("label", "neutral")
- confidence = result.get("confidence", 0.5)
-
- vote_score = 0.0
- if label == "bullish":
- vote_score = confidence
- elif label == "bearish":
- vote_score = -confidence
-
- total_vote += vote_score
- count += 1
-
- all_results.append({
- "text": text[:100],
- "label": label,
- "confidence": confidence,
- "vote": vote_score,
- "available": result.get("available", True)
- })
-
- avg_vote = total_vote / count if count > 0 else 0.0
-
- return {
- "available": models_available,
- "vote": avg_vote,
- "results": all_results,
- "count": count,
- "average_confidence": sum(r["confidence"] for r in all_results) / len(all_results) if all_results else 0.0
- }
-
- except ModelNotAvailable as e:
- return {
- "available": False,
- "vote": 0.0,
- "results": [],
- "count": 0,
- "average_confidence": 0.0,
- "error": str(e),
- "reason": "model_unavailable"
- }
-
- except HTTPException:
- raise
- except Exception as e:
- raise HTTPException(status_code=500, detail=f"Sentiment analysis failed: {str(e)}")
-
-
-
-
-# ===== Short Hunter / v2 compatibility routes (free Binance + CoinGecko) =====
-try:
- from api_compat_routes import register_compat_routes
-
- register_compat_routes(app)
-except Exception as compat_error:
- logger.warning(f"Compat routes not loaded: {compat_error}")
-
-# ===== Main Entry Point =====
-if __name__ == "__main__":
- import uvicorn
- print(f"Starting Crypto Monitor Admin Server on port {PORT}")
- uvicorn.run(app, host="0.0.0.0", port=PORT, log_level="info")
+#!/usr/bin/env python3
+"""
+API Server Extended - HuggingFace Spaces Deployment Ready
+Complete Admin API with Real Data Only - NO MOCKS
+"""
+
+import os
+import threading
+import asyncio
+import sqlite3
+import httpx
+import json
+import subprocess
+import logging
+from pathlib import Path
+from typing import Optional, Dict, Any, List
+from datetime import datetime
+from contextlib import asynccontextmanager
+from collections import defaultdict
+
+try:
+ from api_hub_registry import get_secret, provider_runtime_summary
+except Exception:
+ def get_secret(name):
+ return os.getenv(name)
+ def provider_runtime_summary():
+ return {"totalProviders": 0, "categories": {}}
+
+logger = logging.getLogger(__name__)
+
+from fastapi import FastAPI, HTTPException, Response, Request
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import JSONResponse, FileResponse, HTMLResponse
+from fastapi.staticfiles import StaticFiles
+from starlette.middleware.base import BaseHTTPMiddleware
+from pydantic import BaseModel
+
+# Environment variables
+USE_MOCK_DATA = os.getenv("USE_MOCK_DATA", "false").lower() == "true"
+PORT = int(os.getenv("PORT", "7860"))
+
+# Paths - In Docker container, use /app as base
+WORKSPACE_ROOT = Path("/app" if Path("/app").exists() else (Path("/workspace") if Path("/workspace").exists() else Path(".")))
+DB_PATH = WORKSPACE_ROOT / "data" / "database" / "crypto_monitor.db"
+LOG_DIR = WORKSPACE_ROOT / "logs"
+PROVIDERS_CONFIG_PATH = WORKSPACE_ROOT / "providers_config_extended.json"
+AUTO_DISCOVERY_REPORT_PATH = WORKSPACE_ROOT / "PROVIDER_AUTO_DISCOVERY_REPORT.json"
+API_REGISTRY_PATH = WORKSPACE_ROOT / "all_apis_merged_2025.json"
+
+# Ensure directories exist
+DB_PATH.parent.mkdir(parents=True, exist_ok=True)
+LOG_DIR.mkdir(parents=True, exist_ok=True)
+
+# Global state for providers
+_provider_state = {
+ "providers": {},
+ "pools": {},
+ "logs": [],
+ "last_check": None,
+ "stats": {"total": 0, "online": 0, "offline": 0, "degraded": 0}
+}
+
+
+# ===== Database Setup =====
+def init_database():
+ """Initialize SQLite database with required tables"""
+ conn = sqlite3.connect(str(DB_PATH))
+ cursor = conn.cursor()
+
+ cursor.execute("""
+ CREATE TABLE IF NOT EXISTS prices (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ symbol TEXT NOT NULL,
+ name TEXT,
+ price_usd REAL NOT NULL,
+ volume_24h REAL,
+ market_cap REAL,
+ percent_change_24h REAL,
+ rank INTEGER,
+ timestamp DATETIME DEFAULT CURRENT_TIMESTAMP
+ )
+ """)
+
+ cursor.execute("""
+ CREATE TABLE IF NOT EXISTS sentiment_analysis (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ text TEXT NOT NULL,
+ sentiment_label TEXT NOT NULL,
+ confidence REAL NOT NULL,
+ model_used TEXT,
+ analysis_type TEXT,
+ symbol TEXT,
+ scores TEXT,
+ timestamp DATETIME DEFAULT CURRENT_TIMESTAMP
+ )
+ """)
+
+ cursor.execute("""
+ CREATE TABLE IF NOT EXISTS news_articles (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ title TEXT NOT NULL,
+ content TEXT,
+ url TEXT,
+ source TEXT,
+ sentiment_label TEXT,
+ sentiment_confidence REAL,
+ related_symbols TEXT,
+ published_date DATETIME,
+ analyzed_at DATETIME DEFAULT CURRENT_TIMESTAMP
+ )
+ """)
+
+ cursor.execute("CREATE INDEX IF NOT EXISTS idx_prices_symbol ON prices(symbol)")
+ cursor.execute("CREATE INDEX IF NOT EXISTS idx_prices_timestamp ON prices(timestamp)")
+ cursor.execute("CREATE INDEX IF NOT EXISTS idx_sentiment_timestamp ON sentiment_analysis(timestamp)")
+ cursor.execute("CREATE INDEX IF NOT EXISTS idx_sentiment_symbol ON sentiment_analysis(symbol)")
+ cursor.execute("CREATE INDEX IF NOT EXISTS idx_news_published ON news_articles(published_date)")
+
+ conn.commit()
+ conn.close()
+ print(f"[OK] Database initialized at {DB_PATH}")
+
+
+def save_price_to_db(price_data: Dict[str, Any]):
+ """Save price data to SQLite"""
+ try:
+ conn = sqlite3.connect(str(DB_PATH))
+ cursor = conn.cursor()
+ cursor.execute("""
+ INSERT INTO prices (symbol, name, price_usd, volume_24h, market_cap, percent_change_24h, rank)
+ VALUES (?, ?, ?, ?, ?, ?, ?)
+ """, (
+ price_data.get("symbol"),
+ price_data.get("name"),
+ price_data.get("price_usd", 0.0),
+ price_data.get("volume_24h"),
+ price_data.get("market_cap"),
+ price_data.get("percent_change_24h"),
+ price_data.get("rank")
+ ))
+ conn.commit()
+ conn.close()
+ except Exception as e:
+ print(f"Error saving price to database: {e}")
+
+
+def get_price_history_from_db(symbol: str, limit: int = 10) -> List[Dict[str, Any]]:
+ """Get price history from SQLite"""
+ try:
+ conn = sqlite3.connect(str(DB_PATH))
+ conn.row_factory = sqlite3.Row
+ cursor = conn.cursor()
+ cursor.execute("""
+ SELECT * FROM prices
+ WHERE symbol = ?
+ ORDER BY timestamp DESC
+ LIMIT ?
+ """, (symbol, limit))
+ rows = cursor.fetchall()
+ conn.close()
+ return [dict(row) for row in rows]
+ except Exception as e:
+ print(f"Error fetching price history: {e}")
+ return []
+
+
+def get_latest_prices_from_db() -> Dict[str, Dict[str, Any]]:
+ """Get latest prices for BTC, ETH, BNB from database as fallback"""
+ try:
+ conn = sqlite3.connect(str(DB_PATH))
+ conn.row_factory = sqlite3.Row
+ cursor = conn.cursor()
+
+ # Get latest price for each symbol
+ symbols = ["BTC", "ETH", "BNB"]
+ latest_prices = {}
+
+ for symbol in symbols:
+ cursor.execute("""
+ SELECT * FROM prices
+ WHERE symbol = ?
+ ORDER BY timestamp DESC
+ LIMIT 1
+ """, (symbol,))
+ row = cursor.fetchone()
+ if row:
+ latest_prices[symbol] = dict(row)
+
+ conn.close()
+ return latest_prices
+ except Exception as e:
+ logger.warning(f"Error fetching latest prices from database: {e}")
+ return {}
+
+
+# ===== Provider Management =====
+def load_providers_config() -> Dict[str, Any]:
+ """Load providers from providers_config_extended.json"""
+ try:
+ if PROVIDERS_CONFIG_PATH.exists():
+ with open(PROVIDERS_CONFIG_PATH, 'r', encoding='utf-8') as f:
+ config = json.load(f)
+ # Validate structure
+ if not isinstance(config, dict):
+ logger.warning("Providers config is not a dict, returning empty")
+ return {"providers": {}}
+ if "providers" not in config:
+ logger.warning("Providers config missing 'providers' key, adding it")
+ config["providers"] = {}
+ return config
+ logger.warning(f"Providers config file not found at {PROVIDERS_CONFIG_PATH}")
+ return {"providers": {}}
+ except json.JSONDecodeError as e:
+ logger.error(f"JSON decode error loading providers config: {e}")
+ return {"providers": {}}
+ except Exception as e:
+ logger.error(f"Error loading providers config: {e}")
+ return {"providers": {}}
+
+
+def load_apl_report() -> Dict[str, Any]:
+ """Load APL validation report (alias for auto-discovery report)"""
+ return load_auto_discovery_report()
+
+def load_auto_discovery_report() -> Dict[str, Any]:
+ """Load PROVIDER_AUTO_DISCOVERY_REPORT.json"""
+ try:
+ if AUTO_DISCOVERY_REPORT_PATH.exists():
+ with open(AUTO_DISCOVERY_REPORT_PATH, 'r', encoding='utf-8') as f:
+ return json.load(f)
+ return {}
+ except Exception as e:
+ logger.error(f"Error loading auto-discovery report: {e}")
+ return {}
+
+def load_api_registry() -> Dict[str, Any]:
+ """Load all_apis_merged_2025.json"""
+ try:
+ if API_REGISTRY_PATH.exists():
+ with open(API_REGISTRY_PATH, 'r', encoding='utf-8') as f:
+ return json.load(f)
+ return {}
+ except Exception as e:
+ logger.error(f"Error loading API registry: {e}")
+ return {}
+
+
+# ===== Deduplication Helpers =====
+def deduplicate_providers(providers_list: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+ """
+ Deduplicate providers by id, or by name+base_url if no id.
+ Merge tags/categories when duplicates are found.
+ """
+ seen = {}
+ result = []
+
+ for provider in providers_list:
+ # Determine unique key
+ provider_id = provider.get("id") or provider.get("provider_id")
+ if provider_id:
+ key = f"id:{provider_id}"
+ else:
+ name = provider.get("name", "unknown")
+ base_url = provider.get("base_url", "")
+ key = f"name_url:{name}:{base_url}"
+
+ if key in seen:
+ # Merge tags/categories
+ existing = seen[key]
+ existing_tags = set(existing.get("tags", []) if isinstance(existing.get("tags"), list) else [])
+ new_tags = set(provider.get("tags", []) if isinstance(provider.get("tags"), list) else [])
+ existing["tags"] = list(existing_tags | new_tags)
+
+ # Merge categories if different
+ existing_cat = existing.get("category", "")
+ new_cat = provider.get("category", "")
+ if new_cat and new_cat != existing_cat:
+ if existing_cat:
+ existing["categories"] = list(set([existing_cat, new_cat]))
+ else:
+ existing["category"] = new_cat
+ else:
+ # Ensure tags is a list
+ if "tags" not in provider:
+ provider["tags"] = []
+ elif not isinstance(provider["tags"], list):
+ provider["tags"] = [provider["tags"]]
+
+ seen[key] = provider
+ result.append(provider)
+
+ return result
+
+
+def deduplicate_resources(resources_list: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+ """
+ Deduplicate resources by id, or by name+url if no id.
+ """
+ seen = {}
+ result = []
+
+ for resource in resources_list:
+ # Determine unique key
+ resource_id = resource.get("id")
+ if resource_id:
+ key = f"id:{resource_id}"
+ else:
+ name = resource.get("name", "unknown")
+ url = resource.get("url") or resource.get("base_url", "")
+ path = resource.get("path", "")
+ key = f"name_url:{name}:{url}{path}"
+
+ if key not in seen:
+ seen[key] = resource
+ result.append(resource)
+
+ return result
+
+
+def filter_resources_by_query(resources: List[Dict[str, Any]], query: str) -> List[Dict[str, Any]]:
+ """
+ Filter resources by search query (case-insensitive).
+ Searches in name, description, category, and tags.
+ """
+ if not query:
+ return resources
+
+ query_lower = query.lower()
+ filtered = []
+
+ for resource in resources:
+ # Search in name
+ if query_lower in resource.get("name", "").lower():
+ filtered.append(resource)
+ continue
+
+ # Search in description
+ if query_lower in resource.get("description", "").lower():
+ filtered.append(resource)
+ continue
+
+ # Search in category
+ if query_lower in resource.get("category", "").lower():
+ filtered.append(resource)
+ continue
+
+ # Search in tags
+ tags = resource.get("tags", [])
+ if isinstance(tags, list):
+ if any(query_lower in str(tag).lower() for tag in tags):
+ filtered.append(resource)
+ continue
+
+ return filtered
+
+
+# ===== Real Data Providers =====
+HEADERS = {
+ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
+ "Accept": "application/json"
+}
+
+
+async def fetch_coingecko_simple_price() -> Dict[str, Any]:
+ """Fetch real price data from CoinGecko API with proper error handling"""
+ url = "https://api.coingecko.com/api/v3/simple/price"
+ params = {
+ "ids": "bitcoin,ethereum,binancecoin",
+ "vs_currencies": "usd",
+ "include_market_cap": "true",
+ "include_24hr_vol": "true",
+ "include_24hr_change": "true"
+ }
+
+ try:
+ async with httpx.AsyncClient(timeout=15.0, headers=HEADERS) as client:
+ response = await client.get(url, params=params)
+ if response.status_code != 200:
+ logger.warning(f"CoinGecko API returned HTTP {response.status_code}")
+ raise Exception(f"CoinGecko API error: HTTP {response.status_code}")
+ return response.json()
+ except httpx.TimeoutException:
+ logger.warning("CoinGecko API request timed out")
+ raise Exception("CoinGecko API request timed out")
+ except httpx.RequestError as e:
+ logger.warning(f"CoinGecko API request error: {str(e)}")
+ raise Exception(f"CoinGecko API request failed: {str(e)}")
+ except Exception as e:
+ logger.warning(f"CoinGecko API error: {str(e)}")
+ raise
+
+
+async def fetch_fear_greed_index() -> Dict[str, Any]:
+ """Fetch real Fear & Greed Index from Alternative.me"""
+ url = "https://api.alternative.me/fng/"
+ params = {"limit": "1", "format": "json"}
+
+ async with httpx.AsyncClient(timeout=15.0, headers=HEADERS) as client:
+ response = await client.get(url, params=params)
+ if response.status_code != 200:
+ raise HTTPException(status_code=503, detail=f"Alternative.me API error: HTTP {response.status_code}")
+ return response.json()
+
+
+async def fetch_coingecko_trending() -> Dict[str, Any]:
+ """Fetch real trending coins from CoinGecko"""
+ url = "https://api.coingecko.com/api/v3/search/trending"
+
+ async with httpx.AsyncClient(timeout=15.0, headers=HEADERS) as client:
+ response = await client.get(url)
+ if response.status_code != 200:
+ raise HTTPException(status_code=503, detail=f"CoinGecko trending API error: HTTP {response.status_code}")
+ return response.json()
+
+
+# ===== Self-Healing Health Registry =====
+from dataclasses import dataclass, field
+from typing import Callable
+import time as time_module
+
+@dataclass
+class ProviderHealthEntry:
+ """Health tracking entry for a provider/resource"""
+ id: str
+ name: str
+ status: str = "unknown" # "healthy", "degraded", "unavailable", "unknown"
+ last_success: Optional[float] = None
+ last_error: Optional[float] = None
+ error_count: int = 0
+ success_count: int = 0
+ cooldown_until: Optional[float] = None
+ last_error_message: Optional[str] = None
+
+class HealthRegistry:
+ """
+ Self-healing health registry for providers and external API endpoints.
+ Tracks failures, implements cooldowns, and provides graceful degradation.
+ """
+ def __init__(self):
+ self._providers: Dict[str, ProviderHealthEntry] = {}
+ self._lock = threading.Lock()
+ # Load config
+ try:
+ from config import get_settings
+ self.settings = get_settings()
+ except:
+ # Fallback defaults if config not available
+ class FallbackSettings:
+ health_error_threshold = 3
+ health_cooldown_seconds = 300
+ health_success_recovery_count = 2
+ self.settings = FallbackSettings()
+
+ def _get_or_create_entry(self, provider_id: str, provider_name: str = None) -> ProviderHealthEntry:
+ """Get or create health entry for a provider"""
+ if provider_id not in self._providers:
+ self._providers[provider_id] = ProviderHealthEntry(
+ id=provider_id,
+ name=provider_name or provider_id,
+ status="unknown"
+ )
+ return self._providers[provider_id]
+
+ def update_on_success(self, provider_id: str, provider_name: str = None):
+ """Update health registry after successful provider call"""
+ with self._lock:
+ entry = self._get_or_create_entry(provider_id, provider_name)
+ entry.last_success = time_module.time()
+ entry.success_count += 1
+
+ # Reset error count gradually
+ if entry.error_count > 0:
+ entry.error_count = max(0, entry.error_count - 1)
+
+ # Recovery logic
+ if entry.success_count >= self.settings.health_success_recovery_count:
+ entry.status = "healthy"
+ entry.cooldown_until = None
+
+ def update_on_failure(self, provider_id: str, error_msg: str, provider_name: str = None):
+ """Update health registry after failed provider call"""
+ with self._lock:
+ entry = self._get_or_create_entry(provider_id, provider_name)
+ entry.last_error = time_module.time()
+ entry.error_count += 1
+ entry.last_error_message = error_msg[:500] # Limit error message length
+ entry.success_count = 0
+
+ # Determine status based on error count
+ if entry.error_count >= self.settings.health_error_threshold:
+ entry.status = "unavailable"
+ entry.cooldown_until = time_module.time() + self.settings.health_cooldown_seconds
+ elif entry.error_count >= (self.settings.health_error_threshold // 2):
+ entry.status = "degraded"
+ else:
+ entry.status = "healthy"
+
+ def is_in_cooldown(self, provider_id: str) -> bool:
+ """Check if provider is in cooldown period"""
+ if provider_id not in self._providers:
+ return False
+ entry = self._providers[provider_id]
+ if entry.cooldown_until is None:
+ return False
+ return time_module.time() < entry.cooldown_until
+
+ def get_status(self, provider_id: str) -> Optional[str]:
+ """Get current status of a provider"""
+ if provider_id not in self._providers:
+ return "unknown"
+ return self._providers[provider_id].status
+
+ def get_all_entries(self) -> List[Dict[str, Any]]:
+ """Get all health entries as list of dicts"""
+ with self._lock:
+ return [
+ {
+ "id": entry.id,
+ "name": entry.name,
+ "status": entry.status,
+ "last_success": entry.last_success,
+ "last_error": entry.last_error,
+ "error_count": entry.error_count,
+ "success_count": entry.success_count,
+ "cooldown_until": entry.cooldown_until,
+ "in_cooldown": self.is_in_cooldown(entry.id),
+ "last_error_message": entry.last_error_message
+ }
+ for entry in self._providers.values()
+ ]
+
+ def get_summary(self) -> Dict[str, Any]:
+ """Get summary statistics of health registry"""
+ with self._lock:
+ total = len(self._providers)
+ healthy = sum(1 for e in self._providers.values() if e.status == "healthy")
+ degraded = sum(1 for e in self._providers.values() if e.status == "degraded")
+ unavailable = sum(1 for e in self._providers.values() if e.status == "unavailable")
+ unknown = sum(1 for e in self._providers.values() if e.status == "unknown")
+ in_cooldown = sum(1 for e in self._providers.values() if self.is_in_cooldown(e.id))
+
+ return {
+ "total": total,
+ "healthy": healthy,
+ "degraded": degraded,
+ "unavailable": unavailable,
+ "unknown": unknown,
+ "in_cooldown": in_cooldown
+ }
+
+# Global health registry instance
+_health_registry = HealthRegistry()
+
+
+async def call_provider_safe(
+ provider_id: str,
+ provider_name: str,
+ call_func: Callable,
+ *args,
+ **kwargs
+) -> Dict[str, Any]:
+ """
+ Safely call a provider with health tracking.
+
+ Args:
+ provider_id: Unique identifier for the provider
+ provider_name: Human-readable name
+ call_func: Async function to call
+ *args, **kwargs: Arguments to pass to call_func
+
+ Returns:
+ Dict with status and data or error
+ """
+ # Check if provider is in cooldown
+ if _health_registry.is_in_cooldown(provider_id):
+ entry = _health_registry._providers[provider_id]
+ cooldown_remaining = int(entry.cooldown_until - time_module.time())
+ return {
+ "status": "cooldown",
+ "error": f"Provider in cooldown for {cooldown_remaining}s",
+ "provider_id": provider_id,
+ "cooldown_remaining": cooldown_remaining
+ }
+
+ try:
+ # Call the provider function
+ result = await call_func(*args, **kwargs)
+ # Update health on success
+ _health_registry.update_on_success(provider_id, provider_name)
+ return {
+ "status": "success",
+ "data": result,
+ "provider_id": provider_id
+ }
+ except httpx.TimeoutException as e:
+ error_msg = f"Timeout: {str(e)[:200]}"
+ _health_registry.update_on_failure(provider_id, error_msg, provider_name)
+ return {
+ "status": "timeout",
+ "error": error_msg,
+ "provider_id": provider_id
+ }
+ except httpx.HTTPStatusError as e:
+ error_msg = f"HTTP {e.response.status_code}: {str(e)[:200]}"
+ _health_registry.update_on_failure(provider_id, error_msg, provider_name)
+ return {
+ "status": "http_error",
+ "error": error_msg,
+ "provider_id": provider_id,
+ "status_code": e.response.status_code
+ }
+ except Exception as e:
+ error_msg = f"{type(e).__name__}: {str(e)[:200]}"
+ _health_registry.update_on_failure(provider_id, error_msg, provider_name)
+ return {
+ "status": "error",
+ "error": error_msg,
+ "provider_id": provider_id
+ }
+
+
+# ===== Lifespan Management =====
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+ """Application lifespan manager"""
+ print("=" * 80)
+ print("Starting Crypto Monitor Admin API")
+ print("=" * 80)
+ init_database()
+
+ # Load providers
+ config = load_providers_config()
+ _provider_state["providers"] = config.get("providers", {})
+ print(f"[OK] Loaded {len(_provider_state['providers'])} providers from config")
+
+ # Load auto-discovery report
+ apl_report = load_auto_discovery_report()
+ if apl_report:
+ print(f"[OK] Loaded auto-discovery report with validation data")
+
+ # Load API registry
+ api_registry = load_api_registry()
+ if api_registry:
+ metadata = api_registry.get("metadata", {})
+ print(f"[OK] Loaded API registry: {metadata.get('name', 'unknown')} v{metadata.get('version', 'unknown')}")
+
+ # Initialize AI models
+ try:
+ from ai_models import initialize_models, registry_status, HF_MAX_STARTUP_MODELS
+ model_init_result = initialize_models(max_models=HF_MAX_STARTUP_MODELS)
+ registry_info = registry_status()
+ print(f"[OK] AI Models initialized: {model_init_result}")
+ print(f"[OK] HF Registry status: {registry_info}")
+ except Exception as e:
+ print(f"[WARN] AI Models initialization failed: {e}")
+
+ # Validate unified resources
+ try:
+ from backend.services.resource_validator import validate_unified_resources
+ validation_report = validate_unified_resources(str(WORKSPACE_ROOT / "api-resources" / "crypto_resources_unified_2025-11-11.json"))
+ print(f"[OK] Resource validation: {validation_report['local_backend_routes']['routes_count']} local routes")
+ if validation_report['local_backend_routes']['duplicate_signatures'] > 0:
+ print(f"[WARN] Found {validation_report['local_backend_routes']['duplicate_signatures']} duplicate route signatures")
+ except Exception as e:
+ print(f"[WARN] Resource validation failed: {e}")
+
+ print(f"[OK] Server ready on port {PORT}")
+ print("=" * 80)
+ yield
+ print("Shutting down...")
+
+
+# ===== FastAPI Application =====
+app = FastAPI(
+ title="Crypto Monitor Admin API",
+ description="Real-time cryptocurrency data API with Admin Dashboard",
+ version="5.0.0",
+ lifespan=lifespan
+)
+
+# CORS Middleware
+app.add_middleware(
+ CORSMiddleware,
+ allow_origins=["*"],
+ allow_credentials=True,
+ allow_methods=["*"],
+ allow_headers=["*"],
+)
+
+# Middleware to ensure HTML responses have correct Content-Type
+class HTMLContentTypeMiddleware(BaseHTTPMiddleware):
+ async def dispatch(self, request: Request, call_next):
+ response = await call_next(request)
+ if isinstance(response, HTMLResponse):
+ response.headers["Content-Type"] = "text/html; charset=utf-8"
+ response.headers["X-Content-Type-Options"] = "nosniff"
+ return response
+
+app.add_middleware(HTMLContentTypeMiddleware)
+
+# Mount static files
+try:
+ static_path = WORKSPACE_ROOT / "static"
+ if static_path.exists():
+ app.mount("/static", StaticFiles(directory=str(static_path)), name="static")
+ logger.info(f"Mounted static files from {static_path}")
+ else:
+ # Create static directories if they don't exist
+ static_path.mkdir(parents=True, exist_ok=True)
+ (static_path / "css").mkdir(exist_ok=True)
+ (static_path / "js").mkdir(exist_ok=True)
+ logger.info(f"Created static directories at {static_path}")
+except Exception as e:
+ logger.warning(f"Could not mount static files: {e}")
+
+# Serve trading pairs file
+@app.get("/trading_pairs.txt")
+async def get_trading_pairs():
+ """Serve trading pairs text file"""
+ from fastapi.responses import PlainTextResponse
+ trading_pairs_file = WORKSPACE_ROOT / "trading_pairs.txt"
+ if trading_pairs_file.exists():
+ return FileResponse(trading_pairs_file, media_type="text/plain")
+ return PlainTextResponse("BTCUSDT\nETHUSDT\nBNBUSDT\nSOLUSDT", status_code=200)
+
+
+# ===== HTML UI Endpoints =====
+@app.get("/", response_class=HTMLResponse)
+async def root():
+ """Serve main dashboard"""
+ index_path = WORKSPACE_ROOT / "index.html"
+ if index_path.exists():
+ content = index_path.read_text(encoding="utf-8", errors="ignore")
+ return HTMLResponse(
+ content=content,
+ media_type="text/html",
+ headers={
+ "Content-Type": "text/html; charset=utf-8",
+ "X-Content-Type-Options": "nosniff"
+ }
+ )
+ return HTMLResponse(
+ "Cryptocurrency Data & Analysis API
See /docs for API documentation
",
+ headers={"Content-Type": "text/html; charset=utf-8"}
+ )
+
+@app.get("/index.html", response_class=HTMLResponse)
+async def index():
+ """Serve index.html"""
+ index_path = WORKSPACE_ROOT / "index.html"
+ if index_path.exists():
+ content = index_path.read_text(encoding="utf-8", errors="ignore")
+ return HTMLResponse(
+ content=content,
+ media_type="text/html",
+ headers={
+ "Content-Type": "text/html; charset=utf-8",
+ "X-Content-Type-Options": "nosniff"
+ }
+ )
+ return HTMLResponse(
+ "index.html not found
",
+ headers={"Content-Type": "text/html; charset=utf-8"}
+ )
+
+@app.get("/test.html", response_class=HTMLResponse)
+async def test_page():
+ """Serve test.html for debugging"""
+ test_path = WORKSPACE_ROOT / "test.html"
+ if test_path.exists():
+ content = test_path.read_text(encoding="utf-8", errors="ignore")
+ return HTMLResponse(
+ content=content,
+ media_type="text/html",
+ headers={
+ "Content-Type": "text/html; charset=utf-8",
+ "X-Content-Type-Options": "nosniff"
+ }
+ )
+ return HTMLResponse(
+ "✅ Server is Running
WORKSPACE_ROOT: " + str(WORKSPACE_ROOT) + "
",
+ headers={"Content-Type": "text/html; charset=utf-8"}
+ )
+
+@app.get("/ai-tools", response_class=HTMLResponse)
+async def ai_tools_page(request: Request):
+ """
+ Serve the standalone AI Tools page.
+
+ This page provides:
+ - Sentiment Playground: POST /api/sentiment/analyze
+ - Text Summarizer: POST /api/ai/summarize
+ - Model Status & Diagnostics: GET /api/models/status, /api/models/list
+ """
+ ai_tools_path = WORKSPACE_ROOT / "templates" / "ai_tools.html"
+ if ai_tools_path.exists():
+ content = ai_tools_path.read_text(encoding="utf-8", errors="ignore")
+ return HTMLResponse(
+ content=content,
+ media_type="text/html",
+ headers={
+ "Content-Type": "text/html; charset=utf-8",
+ "X-Content-Type-Options": "nosniff"
+ }
+ )
+ return HTMLResponse(
+ "AI Tools page not found
",
+ headers={"Content-Type": "text/html; charset=utf-8"}
+ )
+
+@app.get("/debug-info", response_class=HTMLResponse)
+async def debug_info():
+ """Debug endpoint to show server configuration"""
+ import os
+ info = f"""
+
+
+
+
+ Debug Info
+
+
+
+ 🔍 Server Debug Information
+ Paths:
+
+WORKSPACE_ROOT: {WORKSPACE_ROOT}
+Current Dir: {Path.cwd()}
+index.html exists: {"✅ YES" if (WORKSPACE_ROOT / "index.html").exists() else "❌ NO"}
+static dir exists: {"✅ YES" if (WORKSPACE_ROOT / "static").exists() else "❌ NO"}
+
+ Files in WORKSPACE_ROOT:
+
+{chr(10).join([f"- {f.name}" for f in sorted(WORKSPACE_ROOT.glob("*.html"))[:20]])}
+
+ Environment:
+
+Python: {os.sys.version}
+Port: 7860
+Host: 127.0.0.1
+
+ Quick Links:
+
+
+
+ """
+ return HTMLResponse(content=info, headers={"Content-Type": "text/html; charset=utf-8"})
+
+
+# ===== Health & Status Endpoints =====
+@app.get("/health")
+async def health():
+ """Health check endpoint (legacy)"""
+ return {
+ "status": "healthy",
+ "timestamp": datetime.now().isoformat(),
+ "database": str(DB_PATH),
+ "use_mock_data": USE_MOCK_DATA,
+ "providers_loaded": len(_provider_state["providers"])
+ }
+
+
+@app.get("/api/health")
+async def api_health():
+ """Stable API health check for HuggingFace Space consumers."""
+ try:
+ version = "4.1.0-short-hunter-compat"
+ try:
+ api_registry = load_api_registry()
+ metadata = api_registry.get("metadata", {}) if isinstance(api_registry, dict) else {}
+ version = metadata.get("version") or version
+ except Exception:
+ pass
+
+ return {
+ "ok": True,
+ "success": True,
+ "status": "ok",
+ "service": "Datasourceforcryptocurrency-4",
+ "version": version,
+ "timestamp": datetime.now().isoformat(),
+ "uptime": None,
+ "errors": [],
+ }
+ except Exception as e:
+ logger.error(f"Health check error: {e}")
+ return JSONResponse(
+ status_code=200,
+ content={
+ "ok": True,
+ "success": True,
+ "status": "ok",
+ "service": "Datasourceforcryptocurrency-4",
+ "version": "unknown",
+ "timestamp": datetime.now().isoformat(),
+ "errors": [],
+ },
+ )
+
+@app.get("/api/status")
+async def get_status():
+ """Capability-level status. Missing optional capabilities never mark the whole Space down."""
+ try:
+ config = load_providers_config()
+ providers_config = config.get("providers", {}) if isinstance(config, dict) else {}
+ resources_json = WORKSPACE_ROOT / "api-resources" / "crypto_resources_unified_2025-11-11.json"
+
+ resources_data = {"total": 0, "categories": {}}
+ errors = []
+ if resources_json.exists():
+ try:
+ with open(resources_json, "r", encoding="utf-8") as f:
+ unified_data = json.load(f)
+ registry = unified_data.get("registry", {}) if isinstance(unified_data, dict) else {}
+ for category, items in registry.items():
+ if category == "metadata":
+ continue
+ if isinstance(items, list):
+ count = len(items)
+ resources_data["total"] += count
+ resources_data["categories"][category.replace("_", "-")] = resources_data["categories"].get(category.replace("_", "-"), 0) + count
+ except Exception as resource_error:
+ errors.append(f"resources_load_failed: {resource_error}")
+
+ model_count = 0
+ try:
+ from ai_models import MODEL_SPECS
+ model_count = len(MODEL_SPECS) if MODEL_SPECS else 0
+ except Exception as model_error:
+ errors.append(f"model_registry_unavailable: {model_error}")
+
+ # Capability model: this endpoint reports route/provider capability, not a slow live external probe.
+ capabilities = {
+ "market": {"status": "available", "providers": ["CoinGecko public", "database cache"]},
+ "coinsTop": {"status": "available", "providers": ["CoinGecko public"]},
+ "trending": {"status": "available", "providers": ["CoinGecko public"]},
+ "ohlcv": {"status": "available", "providers": ["Binance public", "KuCoin public"]},
+ "klines": {"status": "available", "providers": ["Binance public", "KuCoin public"]},
+ "indicators": {"status": "available", "providers": ["local OHLCV computation"]},
+ "sentiment": {"status": "available" if model_count > 0 else "partial", "providers": ["Alternative.me", "HuggingFace models"]},
+ "news": {"status": "available_or_empty", "providers": ["database", "CryptoCompare public"]},
+ "orderbook": {"status": "available", "providers": ["KuCoin public", "Binance public"]},
+ }
+
+ missing_capabilities = [name for name, cap in capabilities.items() if cap.get("status") in {"unavailable", "missing"}]
+ degraded_capabilities = [name for name, cap in capabilities.items() if cap.get("status") in {"partial", "degraded", "available_or_empty"}]
+
+ if missing_capabilities and len(missing_capabilities) >= len(capabilities):
+ data_state = "UNAVAILABLE"
+ elif missing_capabilities:
+ data_state = "DEGRADED"
+ elif degraded_capabilities:
+ data_state = "PARTIAL"
+ else:
+ data_state = "COMPLETE"
+
+ provider_health = health_registry.get_summary() if "health_registry" in globals() else {}
+
+ return {
+ "ok": True,
+ "success": True,
+ "status": "ok",
+ "service": "Datasourceforcryptocurrency-4",
+ "dataState": data_state,
+ "timestamp": datetime.now().isoformat(),
+ "last_update": datetime.now().isoformat(),
+ "providers": {
+ "configuredTotal": len(providers_config),
+ "health": provider_health,
+ "market": capabilities["market"],
+ "ohlcv": capabilities["ohlcv"],
+ "indicators": capabilities["indicators"],
+ "sentiment": capabilities["sentiment"],
+ "news": capabilities["news"],
+ "orderbook": capabilities["orderbook"],
+ },
+ "capabilities": capabilities,
+ "missingCapabilities": missing_capabilities,
+ "degradedCapabilities": degraded_capabilities,
+ "errors": errors,
+ "resources": resources_data,
+ "models": {"total": model_count},
+ }
+ except Exception as e:
+ logger.error(f"Status endpoint error: {e}")
+ return {
+ "ok": False,
+ "success": False,
+ "status": "error",
+ "dataState": "DEGRADED",
+ "timestamp": datetime.now().isoformat(),
+ "providers": {},
+ "capabilities": {},
+ "missingCapabilities": [],
+ "errors": [str(e)],
+ "resources": {"total": 0, "categories": {}},
+ }
+
+@app.get("/api/stats")
+async def get_stats():
+ """System statistics"""
+ config = load_providers_config()
+ providers = config.get("providers", {})
+
+ # Group by category
+ categories = defaultdict(int)
+ for p in providers.values():
+ cat = p.get("category", "unknown")
+ categories[cat] += 1
+
+ return {
+ "total_providers": len(providers),
+ "categories": dict(categories),
+ "total_categories": len(categories),
+ "timestamp": datetime.now().isoformat()
+ }
+
+
+# ===== Market Data Endpoint =====
+@app.get("/api/market")
+async def get_market_data(limit: int = 100):
+ """Normalized market data with backward-compatible fields."""
+ cryptocurrencies = []
+ errors = []
+
+ # Optional primary: CoinMarketCap if configured in HuggingFace Space secrets.
+ cmc_key = get_secret("COINMARKETCAP_KEY")
+ if cmc_key:
+ try:
+ cmc_symbols = "BTC,ETH,BNB,SOL,XRP,DOGE,ADA,TRX,AVAX,LINK,DOT,MATIC,TON,LTC,BCH,UNI,ATOM,ETC,APT,ARB,OP,NEAR,FIL,INJ,SUI,SEI"
+ async with httpx.AsyncClient(timeout=httpx.Timeout(8.0, connect=3.0), headers={**HEADERS, "X-CMC_PRO_API_KEY": cmc_key}) as client:
+ response = await client.get(
+ "https://pro-api.coinmarketcap.com/v1/cryptocurrency/quotes/latest",
+ params={"symbol": cmc_symbols, "convert": "USD"},
+ )
+ if response.status_code == 200:
+ payload = response.json()
+ data = payload.get("data") if isinstance(payload, dict) else {}
+ if isinstance(data, dict):
+ for sym, item in data.items():
+ quote = ((item.get("quote") or {}).get("USD") or {}) if isinstance(item, dict) else {}
+ cryptocurrencies.append({
+ "rank": item.get("cmc_rank"),
+ "name": item.get("name"),
+ "symbol": f"{sym.upper()}USDT",
+ "baseSymbol": sym.upper(),
+ "price": quote.get("price"),
+ "change24h": quote.get("percent_change_24h"),
+ "change_24h": quote.get("percent_change_24h"),
+ "marketCap": quote.get("market_cap"),
+ "market_cap": quote.get("market_cap"),
+ "volume24h": quote.get("volume_24h"),
+ "volume_24h": quote.get("volume_24h"),
+ "source": "coinmarketcap_quotes",
+ })
+ else:
+ errors.append(f"coinmarketcap_http_{response.status_code}")
+ except Exception as cmc_error:
+ errors.append(f"coinmarketcap_failed: {cmc_error}")
+
+ # Primary public fallback: CoinGecko coins/markets gives richer rows for scanners.
+ try:
+ async with httpx.AsyncClient(timeout=httpx.Timeout(8.0, connect=3.0), headers=HEADERS) as client:
+ response = await client.get(
+ "https://api.coingecko.com/api/v3/coins/markets",
+ params={
+ "vs_currency": "usd",
+ "order": "market_cap_desc",
+ "per_page": min(max(int(limit or 100), 1), 250),
+ "page": 1,
+ "sparkline": "false",
+ "price_change_percentage": "24h",
+ },
+ )
+ if response.status_code == 200:
+ payload = response.json()
+ if isinstance(payload, list):
+ for item in payload:
+ base_symbol = str(item.get("symbol", "")).upper()
+ normalized_symbol = f"{base_symbol}USDT" if base_symbol and not base_symbol.endswith("USDT") else base_symbol
+ cryptocurrencies.append({
+ "rank": item.get("market_cap_rank"),
+ "name": item.get("name"),
+ "symbol": normalized_symbol,
+ "baseSymbol": base_symbol,
+ "price": item.get("current_price"),
+ "change24h": item.get("price_change_percentage_24h"),
+ "change_24h": item.get("price_change_percentage_24h"),
+ "marketCap": item.get("market_cap"),
+ "market_cap": item.get("market_cap"),
+ "volume24h": item.get("total_volume"),
+ "volume_24h": item.get("total_volume"),
+ "image": item.get("image"),
+ "source": "coingecko_markets",
+ })
+ else:
+ errors.append(f"coingecko_markets_http_{response.status_code}")
+ except Exception as market_error:
+ errors.append(f"coingecko_markets_failed: {market_error}")
+
+ # Fallback: existing simple-price helper for BTC/ETH/BNB.
+ if not cryptocurrencies:
+ coin_mapping = {
+ "bitcoin": {"name": "Bitcoin", "symbol": "BTCUSDT", "rank": 1, "image": "https://assets.coingecko.com/coins/images/1/small/bitcoin.png"},
+ "ethereum": {"name": "Ethereum", "symbol": "ETHUSDT", "rank": 2, "image": "https://assets.coingecko.com/coins/images/279/small/ethereum.png"},
+ "binancecoin": {"name": "BNB", "symbol": "BNBUSDT", "rank": 3, "image": "https://assets.coingecko.com/coins/images/825/small/bnb-icon2_2x.png"},
+ }
+ try:
+ data = await fetch_coingecko_simple_price()
+ for coin_id, coin_info in coin_mapping.items():
+ coin_data = data.get(coin_id, {}) if isinstance(data, dict) else {}
+ if coin_data:
+ cryptocurrencies.append({
+ "rank": coin_info["rank"],
+ "name": coin_info["name"],
+ "symbol": coin_info["symbol"],
+ "baseSymbol": coin_info["symbol"].replace("USDT", ""),
+ "price": coin_data.get("usd", 0),
+ "change24h": coin_data.get("usd_24h_change", 0),
+ "change_24h": coin_data.get("usd_24h_change", 0),
+ "marketCap": coin_data.get("usd_market_cap", 0),
+ "market_cap": coin_data.get("usd_market_cap", 0),
+ "volume24h": coin_data.get("usd_24h_vol", 0),
+ "volume_24h": coin_data.get("usd_24h_vol", 0),
+ "image": coin_info["image"],
+ "source": "coingecko_simple_price",
+ })
+ except Exception as simple_error:
+ errors.append(f"coingecko_simple_price_failed: {simple_error}")
+
+ # Secondary fallback: CryptoCompare prices, with optional key.
+ if not cryptocurrencies:
+ try:
+ cc_key = get_secret("CRYPTOCOMPARE_KEY")
+ params = {"fsyms": "BTC,ETH,BNB,SOL,XRP,DOGE,ADA,TRX,AVAX,LINK", "tsyms": "USD"}
+ if cc_key:
+ params["api_key"] = cc_key
+ async with httpx.AsyncClient(timeout=httpx.Timeout(8.0, connect=3.0), headers=HEADERS) as client:
+ response = await client.get("https://min-api.cryptocompare.com/data/pricemultifull", params=params)
+ if response.status_code == 200:
+ payload = response.json()
+ raw = (payload.get("RAW") or {}) if isinstance(payload, dict) else {}
+ for sym, row in raw.items():
+ usd = (row or {}).get("USD") or {}
+ cryptocurrencies.append({
+ "rank": None,
+ "name": sym.upper(),
+ "symbol": f"{sym.upper()}USDT",
+ "baseSymbol": sym.upper(),
+ "price": usd.get("PRICE"),
+ "change24h": usd.get("CHANGEPCT24HOUR"),
+ "change_24h": usd.get("CHANGEPCT24HOUR"),
+ "marketCap": usd.get("MKTCAP"),
+ "market_cap": usd.get("MKTCAP"),
+ "volume24h": usd.get("VOLUME24HOURTO"),
+ "volume_24h": usd.get("VOLUME24HOURTO"),
+ "source": "cryptocompare_pricemultifull",
+ })
+ else:
+ errors.append(f"cryptocompare_prices_http_{response.status_code}")
+ except Exception as cc_error:
+ errors.append(f"cryptocompare_prices_failed: {cc_error}")
+
+ # Last fallback: cached DB rows. Zero-placeholder rows are no longer advertised as valid data.
+ if not cryptocurrencies:
+ latest_prices = get_latest_prices_from_db()
+ for symbol, db_data in latest_prices.items():
+ cryptocurrencies.append({
+ "rank": db_data.get("rank"),
+ "name": db_data.get("name"),
+ "symbol": f"{symbol}USDT" if not str(symbol).upper().endswith("USDT") else str(symbol).upper(),
+ "baseSymbol": symbol,
+ "price": db_data.get("price_usd", 0),
+ "change24h": db_data.get("percent_change_24h", 0),
+ "change_24h": db_data.get("percent_change_24h", 0),
+ "marketCap": db_data.get("market_cap", 0),
+ "market_cap": db_data.get("market_cap", 0),
+ "volume24h": db_data.get("volume_24h", 0),
+ "volume_24h": db_data.get("volume_24h", 0),
+ "source": "sqlite_cache",
+ })
+
+ total_market_cap = sum((c.get("marketCap") or c.get("market_cap") or 0) for c in cryptocurrencies)
+ btc_entry = next((c for c in cryptocurrencies if str(c.get("symbol", "")).startswith("BTC")), None)
+ btc_dominance = ((btc_entry.get("marketCap") or btc_entry.get("market_cap") or 0) / total_market_cap * 100) if btc_entry and total_market_cap else 0
+
+ if not cryptocurrencies:
+ return {
+ "success": False,
+ "data": [],
+ "cryptocurrencies": [],
+ "count": 0,
+ "dataState": "UNAVAILABLE",
+ "missingCapabilities": ["market"],
+ "errors": errors or ["No market provider returned data"],
+ "source": "none",
+ "timestamp": datetime.now().isoformat(),
+ }
+
+ return {
+ "success": True,
+ "data": cryptocurrencies,
+ "cryptocurrencies": cryptocurrencies,
+ "count": len(cryptocurrencies),
+ "total_market_cap": total_market_cap,
+ "btc_dominance": btc_dominance,
+ "dataState": "COMPLETE" if not errors else "PARTIAL",
+ "missingCapabilities": [],
+ "errors": errors,
+ "source": cryptocurrencies[0].get("source", "mixed_public_sources"),
+ "timestamp": datetime.now().isoformat(),
+ }
+
+@app.get("/api/market/history")
+async def get_market_history(symbol: str = "BTC", limit: int = 10):
+ """Get price history from database - REAL DATA ONLY"""
+ history = get_price_history_from_db(symbol.upper(), limit)
+
+ if not history:
+ return {
+ "symbol": symbol,
+ "history": [],
+ "count": 0,
+ "message": "No history available"
+ }
+
+ return {
+ "symbol": symbol,
+ "history": history,
+ "count": len(history),
+ "source": "SQLite Database (Real Data)"
+ }
+
+
+@app.get("/api/sentiment")
+async def get_sentiment():
+ """Sentiment data from Alternative.me - REAL DATA ONLY"""
+ try:
+ data = await fetch_fear_greed_index()
+
+ if "data" in data and len(data["data"]) > 0:
+ fng_data = data["data"][0]
+ return {
+ "fear_greed_index": int(fng_data["value"]),
+ "fear_greed_label": fng_data["value_classification"],
+ "timestamp": datetime.now().isoformat(),
+ "source": "Alternative.me API (Real Data)"
+ }
+
+ raise HTTPException(status_code=503, detail="Invalid response from Alternative.me")
+
+ except Exception as e:
+ raise HTTPException(status_code=503, detail=f"Failed to fetch sentiment: {str(e)}")
+
+
+@app.post("/api/sentiment")
+async def analyze_sentiment_simple(request: Dict[str, Any]):
+ """Analyze sentiment with mode routing - simplified endpoint"""
+ try:
+ from ai_models import (
+ analyze_crypto_sentiment,
+ analyze_financial_sentiment,
+ analyze_social_sentiment,
+ _registry,
+ MODEL_SPECS,
+ ModelNotAvailable
+ )
+
+ text = request.get("text", "").strip()
+ if not text:
+ raise HTTPException(status_code=400, detail="Text is required")
+
+ mode = request.get("mode", "auto").lower()
+ model_key = request.get("model_key")
+
+ # If model_key is provided, use that specific model
+ if model_key:
+ if model_key not in MODEL_SPECS:
+ raise HTTPException(status_code=404, detail=f"Model key '{model_key}' not found")
+
+ try:
+ pipeline = _registry.get_pipeline(model_key)
+ spec = MODEL_SPECS[model_key]
+
+ # Handle trading signal models specially
+ if spec.category == "trading_signal":
+ raw_result = pipeline(text, max_length=200, num_return_sequences=1)
+ if isinstance(raw_result, list) and raw_result:
+ raw_result = raw_result[0]
+ generated_text = raw_result.get("generated_text", str(raw_result))
+
+ decision = "HOLD"
+ if "buy" in generated_text.lower():
+ decision = "BUY"
+ elif "sell" in generated_text.lower():
+ decision = "SELL"
+
+ return {
+ "sentiment": decision.lower(),
+ "confidence": 0.7,
+ "raw_label": decision,
+ "mode": "trading",
+ "model": model_key,
+ "extra": {
+ "decision": decision,
+ "rationale": generated_text,
+ "raw": raw_result
+ }
+ }
+
+ # Regular sentiment analysis
+ raw_result = pipeline(text[:512])
+ if isinstance(raw_result, list) and raw_result:
+ raw_result = raw_result[0]
+
+ label = raw_result.get("label", "neutral").upper()
+ score = raw_result.get("score", 0.5)
+
+ # Map to standard format
+ mapped = "Bullish" if "POSITIVE" in label or "BULLISH" in label or "LABEL_2" in label else (
+ "Bearish" if "NEGATIVE" in label or "BEARISH" in label or "LABEL_0" in label else "Neutral"
+ )
+
+ return {
+ "sentiment": mapped,
+ "confidence": score,
+ "raw_label": label,
+ "mode": mode,
+ "model": model_key,
+ "extra": {"raw": raw_result}
+ }
+
+ except ModelNotAvailable as e:
+ logger.warning(f"Model {model_key} not available: {e}")
+ raise HTTPException(status_code=503, detail=f"Model not available: {str(e)}")
+
+ # Mode-based routing (no explicit model key)
+ result = None
+ actual_model = None
+
+ if mode == "crypto" or mode == "auto":
+ result = analyze_crypto_sentiment(text)
+ actual_model = "crypto_sent_kk08" # Default crypto model
+ elif mode == "social":
+ result = analyze_social_sentiment(text)
+ actual_model = "crypto_sent_social" # ElKulako/cryptobert
+ elif mode == "financial":
+ result = analyze_financial_sentiment(text)
+ actual_model = "crypto_sent_fin" # FinTwitBERT
+ elif mode == "news":
+ result = analyze_financial_sentiment(text) # Use financial for news
+ actual_model = "crypto_sent_fin"
+ elif mode == "trading":
+ # Try to use trading model
+ try:
+ pipeline = _registry.get_pipeline("crypto_trading_lm")
+ raw_result = pipeline(text, max_length=200, num_return_sequences=1)
+ if isinstance(raw_result, list) and raw_result:
+ raw_result = raw_result[0]
+ generated_text = raw_result.get("generated_text", str(raw_result))
+
+ decision = "HOLD"
+ if "buy" in generated_text.lower():
+ decision = "BUY"
+ elif "sell" in generated_text.lower():
+ decision = "SELL"
+
+ return {
+ "sentiment": decision,
+ "confidence": 0.7,
+ "raw_label": decision,
+ "mode": "trading",
+ "model": "crypto_trading_lm",
+ "extra": {
+ "decision": decision,
+ "rationale": generated_text
+ }
+ }
+ except ModelNotAvailable:
+ # Fallback to crypto sentiment
+ result = analyze_crypto_sentiment(text)
+ actual_model = "crypto_sent_kk08"
+ else:
+ result = analyze_crypto_sentiment(text) # Default fallback
+ actual_model = "crypto_sent_kk08"
+
+ if not result:
+ raise HTTPException(status_code=500, detail="Sentiment analysis failed")
+
+ # Standardize result format
+ sentiment = result.get("label", "Neutral")
+ confidence = result.get("confidence", 0.5)
+
+ # Capitalize first letter
+ sentiment_formatted = sentiment.capitalize() if isinstance(sentiment, str) else "Neutral"
+
+ return {
+ "sentiment": sentiment_formatted,
+ "confidence": confidence,
+ "raw_label": sentiment,
+ "mode": mode,
+ "model": actual_model,
+ "extra": result
+ }
+
+ except HTTPException:
+ raise
+ except Exception as e:
+ logger.error(f"Sentiment analysis error: {e}")
+ raise HTTPException(status_code=500, detail=f"Analysis failed: {str(e)}")
+
+
+@app.get("/api/resources")
+async def get_resources(q: Optional[str] = None):
+ """Get all resources with optional search query and deduplication"""
+ try:
+ resources_list = []
+
+ # Load from unified resources file
+ resources_json = WORKSPACE_ROOT / "api-resources" / "crypto_resources_unified_2025-11-11.json"
+ if resources_json.exists():
+ try:
+ with open(resources_json, 'r', encoding='utf-8') as f:
+ unified_data = json.load(f)
+ registry = unified_data.get('registry', {})
+
+ for category, items in registry.items():
+ if category == 'metadata':
+ continue
+ if isinstance(items, list):
+ for item in items:
+ # Normalize resource structure
+ resource = {
+ "id": item.get("id"),
+ "name": item.get("name", item.get("title", "Unknown")),
+ "category": category,
+ "url": item.get("url") or item.get("base_url", ""),
+ "free": item.get("free", True),
+ "auth_required": item.get("auth_required", False) or (item.get("auth", {}).get("type") != "none" if "auth" in item else False),
+ "tags": item.get("tags", []) if isinstance(item.get("tags"), list) else [],
+ "description": item.get("description", "") or item.get("note", "")
+ }
+
+ # Additional fields if present
+ if "method" in item:
+ resource["method"] = item["method"]
+ if "path" in item:
+ resource["path"] = item["path"]
+ if "endpoint" in item:
+ resource["endpoint"] = item["endpoint"]
+
+ resources_list.append(resource)
+ except Exception as e:
+ logger.error(f"Error loading unified resources: {e}")
+
+ # Load from API registry (all_apis_merged_2025.json)
+ api_registry = load_api_registry()
+ if api_registry and "raw_files" in api_registry:
+ # Parse raw files for additional resources (basic extraction)
+ for raw_file in api_registry.get("raw_files", [])[:10]: # Limit to first 10
+ content = raw_file.get("content", "")
+ filename = raw_file.get("filename", "")
+
+ # Simple extraction: look for URLs in content
+ import re
+ urls = re.findall(r'https?://[^\s<>"]+', content)
+ for url in urls[:5]: # Limit URLs per file
+ resources_list.append({
+ "id": None,
+ "name": f"Resource from {filename}",
+ "category": "discovered",
+ "url": url,
+ "free": True,
+ "auth_required": False,
+ "tags": ["auto-discovered"],
+ "description": f"Auto-discovered from {filename}"
+ })
+
+ # Apply deduplication
+ deduplicated_resources = deduplicate_resources(resources_list)
+
+ # Apply search filter if query provided
+ if q:
+ deduplicated_resources = filter_resources_by_query(deduplicated_resources, q)
+
+ return deduplicated_resources
+
+ except Exception as e:
+ logger.error(f"Error in get_resources: {e}")
+ raise HTTPException(status_code=500, detail=f"Failed to fetch resources: {str(e)}")
+
+
+@app.get("/api/resources/summary")
+async def get_resources_summary():
+ """Get resources summary for HTML dashboard (includes API registry metadata and local routes)"""
+ try:
+ # Import MODEL_SPECS first as source of truth for models count
+ try:
+ from ai_models import MODEL_SPECS
+ models_count = len(MODEL_SPECS) if MODEL_SPECS else 0
+ except Exception as e:
+ logger.warning(f"Failed to import MODEL_SPECS: {e}")
+ models_count = 0
+
+ # Load API registry for metadata
+ api_registry = load_api_registry()
+ metadata = api_registry.get("metadata", {}) if api_registry else {}
+
+ # Try to load resources from JSON files
+ resources_json = WORKSPACE_ROOT / "api-resources" / "crypto_resources_unified_2025-11-11.json"
+
+ summary = {
+ "total_resources": 0,
+ "free_resources": 0,
+ "models_available": models_count, # Use MODEL_SPECS as source of truth
+ "local_routes_count": 0,
+ "categories": {}
+ }
+
+ # Load from unified resources
+ if resources_json.exists():
+ try:
+ with open(resources_json, 'r', encoding='utf-8') as f:
+ data = json.load(f)
+ registry = data.get('registry', {})
+
+ # Process all categories
+ for category, items in registry.items():
+ if category == 'metadata':
+ continue
+ if isinstance(items, list):
+ count = len(items)
+ summary['total_resources'] += count
+ summary['categories'][category] = {
+ "count": count,
+ "type": "local" if category == "local_backend_routes" else "external"
+ }
+
+ # Track local routes separately
+ if category == 'local_backend_routes':
+ summary['local_routes_count'] = count
+
+ free_count = sum(1 for item in items if item.get('free', False) or item.get('auth', {}).get('type') == 'none')
+ summary['free_resources'] += free_count
+ except Exception as e:
+ logger.warning(f"Failed to load resources JSON: {e}")
+
+ # Ensure models_available is always non-zero if MODEL_SPECS is available
+ if summary['models_available'] == 0 and models_count > 0:
+ summary['models_available'] = models_count
+
+ # If no resources found, provide fallback data but keep models count from MODEL_SPECS
+ if summary['total_resources'] == 0:
+ logger.warning("No resources found in JSON files, using fallback data")
+ summary['total_resources'] = 15
+ summary['free_resources'] = 12
+ # Ensure models count is at least from MODEL_SPECS or fallback minimum
+ summary['models_available'] = max(summary['models_available'], models_count, 7)
+ summary['categories'] = {
+ 'market_data': 5,
+ 'news': 3,
+ 'sentiment': 2,
+ 'blockchain': 3,
+ 'defi': 2
+ }
+
+ return {
+ "success": True,
+ "summary": summary,
+ "api_registry_metadata": metadata,
+ "timestamp": datetime.now().isoformat()
+ }
+ except Exception as e:
+ logger.error(f"Error in get_resources_summary: {e}")
+ # Return fallback data on error, but try to get models count from MODEL_SPECS
+ try:
+ from ai_models import MODEL_SPECS
+ fallback_models = len(MODEL_SPECS) if MODEL_SPECS else 7
+ except:
+ fallback_models = 7
+
+ return {
+ "success": True,
+ "summary": {
+ "total_resources": 15,
+ "free_resources": 12,
+ "models_available": fallback_models,
+ "local_routes_count": 0,
+ "categories": {
+ 'market_data': 5,
+ 'news': 3,
+ 'sentiment': 2,
+ 'blockchain': 3,
+ 'defi': 2
+ }
+ },
+ "error": str(e),
+ "timestamp": datetime.now().isoformat()
+ }
+
+@app.get("/api/resources/apis")
+async def get_resources_apis():
+ """Get API registry with local and external routes"""
+ registry = load_api_registry()
+
+ # Load unified resources for local routes
+ resources_json = WORKSPACE_ROOT / "api-resources" / "crypto_resources_unified_2025-11-11.json"
+ local_routes = []
+ unified_metadata = {}
+
+ if resources_json.exists():
+ try:
+ with open(resources_json, 'r', encoding='utf-8') as f:
+ unified_data = json.load(f)
+ unified_registry = unified_data.get('registry', {})
+ unified_metadata = unified_registry.get('metadata', {})
+ local_routes = unified_registry.get('local_backend_routes', [])
+ except Exception as e:
+ logger.error(f"Error loading unified resources: {e}")
+
+ # Process legacy registry
+ categories = set()
+ metadata = {}
+ raw_files = []
+ trimmed_files = []
+
+ if registry:
+ metadata = registry.get("metadata", {})
+ raw_files = registry.get("raw_files", [])
+
+ # Extract categories from raw file content (basic parsing)
+ for raw_file in raw_files[:5]: # Limit to first 5 files for performance
+ content = raw_file.get("content", "")
+ # Simple category detection from content
+ if "market data" in content.lower() or "price" in content.lower():
+ categories.add("market_data")
+ if "explorer" in content.lower() or "blockchain" in content.lower():
+ categories.add("block_explorer")
+ if "rpc" in content.lower() or "node" in content.lower():
+ categories.add("rpc_nodes")
+ if "cors" in content.lower() or "proxy" in content.lower():
+ categories.add("cors_proxy")
+ if "news" in content.lower():
+ categories.add("news")
+ if "sentiment" in content.lower() or "fear" in content.lower():
+ categories.add("sentiment")
+ if "whale" in content.lower():
+ categories.add("whale_tracking")
+
+ # Provide trimmed raw files (first 500 chars each)
+ for raw_file in raw_files[:10]: # Limit to 10 files
+ content = raw_file.get("content", "")
+ trimmed_files.append({
+ "filename": raw_file.get("filename", ""),
+ "preview": content[:500] + "..." if len(content) > 500 else content,
+ "size": len(content)
+ })
+
+ # Add local category
+ if local_routes:
+ categories.add("local")
+
+ return {
+ "ok": True,
+ "metadata": {
+ "name": metadata.get("name", "") or unified_metadata.get("description", ""),
+ "version": metadata.get("version", "") or unified_metadata.get("version", ""),
+ "description": metadata.get("description", ""),
+ "created_at": metadata.get("created_at", ""),
+ "source_files": metadata.get("source_files", []),
+ "updated": unified_metadata.get("updated", "")
+ },
+ "categories": list(categories),
+ "local_routes": {
+ "count": len(local_routes),
+ "routes": local_routes[:20] # Return first 20 for preview
+ },
+ "raw_files_preview": trimmed_files,
+ "total_raw_files": len(raw_files),
+ "sources": ["all_apis_merged_2025.json", "crypto_resources_unified_2025-11-11.json"]
+ }
+
+@app.get("/api/resources/apis/raw")
+async def get_resources_apis_raw():
+ """Get raw files from API registry (trimmed to avoid huge payloads)"""
+ registry = load_api_registry()
+
+ if not registry:
+ return {
+ "ok": False,
+ "error": "API registry file not found"
+ }
+
+ raw_files = registry.get("raw_files", [])
+
+ # Return trimmed versions (first 1000 chars each, max 20 files)
+ trimmed = []
+ for raw_file in raw_files[:20]:
+ content = raw_file.get("content", "")
+ trimmed.append({
+ "filename": raw_file.get("filename", ""),
+ "preview": content[:1000] + "..." if len(content) > 1000 else content,
+ "full_size": len(content)
+ })
+
+ return {
+ "ok": True,
+ "files": trimmed,
+ "total_files": len(raw_files),
+ "showing": min(20, len(raw_files)),
+ "source": "all_apis_merged_2025.json"
+ }
+
+
+@app.get("/api/trending")
+async def get_trending(limit: int = 10):
+ """Trending coins from CoinGecko, normalized for API clients."""
+ try:
+ data = await fetch_coingecko_trending()
+ trending_coins = []
+ if isinstance(data, dict) and "coins" in data:
+ for item in data["coins"][: min(max(int(limit or 10), 1), 50)]:
+ coin = item.get("item", {})
+ sym = str(coin.get("symbol", "")).upper()
+ trending_coins.append({
+ "id": coin.get("id"),
+ "name": coin.get("name"),
+ "symbol": f"{sym}USDT" if sym and not sym.endswith("USDT") else sym,
+ "baseSymbol": sym,
+ "marketCapRank": coin.get("market_cap_rank"),
+ "market_cap_rank": coin.get("market_cap_rank"),
+ "thumb": coin.get("thumb"),
+ "score": coin.get("score", 0),
+ "source": "coingecko_trending",
+ })
+
+ return {
+ "success": True,
+ "data": trending_coins,
+ "trending": trending_coins,
+ "count": len(trending_coins),
+ "timestamp": datetime.now().isoformat(),
+ "source": "CoinGecko API (Real Data)",
+ "errors": [],
+ }
+ except Exception as e:
+ logger.warning(f"Trending fetch failed: {e}")
+ return {
+ "success": False,
+ "data": [],
+ "trending": [],
+ "count": 0,
+ "timestamp": datetime.now().isoformat(),
+ "source": "coingecko_trending",
+ "errors": [str(e)],
+ "missingCapabilities": ["trending"],
+ }
+
+# ===== Providers Management Endpoints =====
+@app.get("/api/providers")
+async def get_providers():
+ """Get all providers with deduplication applied"""
+ try:
+ # Load primary config
+ config = load_providers_config()
+ providers_dict = config.get("providers", {})
+
+ # Load auto-discovery report for validation status
+ discovery_report = load_auto_discovery_report()
+ discovery_results = {}
+ if discovery_report and "http_providers" in discovery_report:
+ for result in discovery_report["http_providers"].get("results", []):
+ discovery_results[result.get("provider_id")] = result
+
+ # Build provider list from primary config
+ providers_list = []
+ for provider_id, provider_data in providers_dict.items():
+ # Merge with auto-discovery data if available
+ discovery_data = discovery_results.get(provider_id, {})
+
+ # Determine auth requirement
+ auth_required = provider_data.get("requires_auth", False)
+ free = not auth_required
+
+ # Extract tags from provider data
+ tags = []
+ if "tags" in provider_data:
+ tags = provider_data["tags"] if isinstance(provider_data["tags"], list) else [provider_data["tags"]]
+
+ # Build description
+ description = provider_data.get("description", "") or provider_data.get("note", "")
+ if not description and provider_data.get("name"):
+ description = f"{provider_data.get('name')} - {provider_data.get('category', 'unknown')} provider"
+
+ provider_entry = {
+ "id": provider_id,
+ "name": provider_data.get("name", provider_id),
+ "category": provider_data.get("category", "unknown"),
+ "base_url": provider_data.get("base_url", ""),
+ "auth_required": auth_required,
+ "free": free,
+ "tags": tags,
+ "description": description,
+ "type": provider_data.get("type", "http"),
+ "priority": provider_data.get("priority", 0),
+ "weight": provider_data.get("weight", 0),
+ "rate_limit": provider_data.get("rate_limit", {}),
+ "endpoints": provider_data.get("endpoints", {}),
+ "status": discovery_data.get("status", "UNKNOWN") if discovery_data else "unvalidated",
+ "validated_at": provider_data.get("validated_at"),
+ "response_time_ms": discovery_data.get("response_time_ms") or provider_data.get("response_time_ms"),
+ "added_by": provider_data.get("added_by", "manual")
+ }
+ providers_list.append(provider_entry)
+
+ # Add HF Models as providers (with proper structure)
+ try:
+ from ai_models import MODEL_SPECS, _registry
+ for model_key, spec in MODEL_SPECS.items():
+ is_loaded = model_key in _registry._pipelines
+ providers_list.append({
+ "id": f"hf_model_{model_key}",
+ "name": f"HF Model: {spec.model_id}",
+ "category": spec.category,
+ "base_url": f"/api/models/{model_key}/predict",
+ "auth_required": spec.requires_auth,
+ "free": not spec.requires_auth,
+ "tags": ["huggingface", "ai-model", spec.task, spec.category],
+ "description": f"Hugging Face {spec.task} model for {spec.category}",
+ "type": "hf_model",
+ "status": "available" if is_loaded else "not_loaded",
+ "model_key": model_key,
+ "model_id": spec.model_id,
+ "task": spec.task,
+ "added_by": "hf_models"
+ })
+ except Exception as e:
+ logger.warning(f"Could not add HF models as providers: {e}")
+
+ # Apply deduplication
+ deduplicated_providers = deduplicate_providers(providers_list)
+
+ return {
+ "providers": deduplicated_providers,
+ "total": len(deduplicated_providers),
+ "source": "providers_config_extended.json + PROVIDER_AUTO_DISCOVERY_REPORT.json + HF Models (deduplicated)"
+ }
+ except Exception as e:
+ logger.error(f"Error in get_providers: {e}")
+ return {
+ "providers": [],
+ "total": 0,
+ "error": str(e),
+ "source": "error"
+ }
+
+
+@app.get("/api/providers/{provider_id}")
+async def get_provider_detail(provider_id: str):
+ """Get specific provider details"""
+ # Check if it's an HF model provider
+ if provider_id.startswith("hf_model_"):
+ model_key = provider_id.replace("hf_model_", "")
+ try:
+ from ai_models import MODEL_SPECS, _registry
+ if model_key not in MODEL_SPECS:
+ raise HTTPException(status_code=404, detail=f"Model {model_key} not found")
+
+ spec = MODEL_SPECS[model_key]
+ is_loaded = model_key in _registry._pipelines
+
+ return {
+ "provider_id": provider_id,
+ "name": f"HF Model: {spec.model_id}",
+ "category": spec.category,
+ "type": "hf_model",
+ "status": "available" if is_loaded else "not_loaded",
+ "model_key": model_key,
+ "model_id": spec.model_id,
+ "task": spec.task,
+ "requires_auth": spec.requires_auth,
+ "endpoint": f"/api/models/{model_key}/predict",
+ "usage": {
+ "method": "POST",
+ "url": f"/api/models/{model_key}/predict",
+ "body": {"text": "string", "options": {}}
+ },
+ "added_by": "hf_models"
+ }
+ except HTTPException:
+ raise
+ except Exception as e:
+ raise HTTPException(status_code=500, detail=str(e))
+
+ # Regular provider
+ config = load_providers_config()
+ providers = config.get("providers", {})
+
+ if provider_id not in providers:
+ raise HTTPException(status_code=404, detail=f"Provider {provider_id} not found")
+
+ return {
+ "provider_id": provider_id,
+ **providers[provider_id]
+ }
+
+
+@app.get("/api/providers/category/{category}")
+async def get_providers_by_category(category: str):
+ """Get providers by category"""
+ config = load_providers_config()
+ providers = config.get("providers", {})
+
+ filtered = {
+ pid: data for pid, data in providers.items()
+ if data.get("category") == category
+ }
+
+ return {
+ "category": category,
+ "providers": filtered,
+ "count": len(filtered)
+ }
+
+
+# ===== Pools Endpoints (Placeholder - to be implemented) =====
+@app.get("/api/pools")
+async def get_pools():
+ """Get provider pools"""
+ return {
+ "pools": [],
+ "message": "Pools feature not yet implemented in this version"
+ }
+
+
+# ===== Logs Endpoints =====
+@app.get("/api/logs/recent")
+async def get_recent_logs():
+ """Get recent logs"""
+ return {
+ "logs": _provider_state.get("logs", [])[-50:],
+ "count": min(50, len(_provider_state.get("logs", [])))
+ }
+
+
+@app.get("/api/logs/errors")
+async def get_error_logs():
+ """Get error logs"""
+ all_logs = _provider_state.get("logs", [])
+ errors = [log for log in all_logs if log.get("level") == "ERROR"]
+ return {
+ "errors": errors[-50:],
+ "count": len(errors)
+ }
+
+
+# ===== Diagnostics Endpoints =====
+@app.post("/api/diagnostics/run")
+async def run_diagnostics(auto_fix: bool = False):
+ """Run system diagnostics"""
+ issues = []
+ fixes_applied = []
+
+ # Check database
+ if not DB_PATH.exists():
+ issues.append({"type": "database", "message": "Database file not found"})
+ if auto_fix:
+ init_database()
+ fixes_applied.append("Initialized database")
+
+ # Check providers config
+ if not PROVIDERS_CONFIG_PATH.exists():
+ issues.append({"type": "config", "message": "Providers config not found"})
+
+ # Check auto-discovery report
+ if not AUTO_DISCOVERY_REPORT_PATH.exists():
+ issues.append({"type": "auto_discovery", "message": "Auto-discovery report not found"})
+
+ return {
+ "status": "completed",
+ "issues_found": len(issues),
+ "issues": issues,
+ "fixes_applied": fixes_applied if auto_fix else [],
+ "timestamp": datetime.now().isoformat()
+ }
+
+
+@app.get("/api/diagnostics/last")
+async def get_last_diagnostics():
+ """Get last diagnostics results"""
+ # Would load from file in real implementation
+ return {
+ "status": "no_previous_run",
+ "message": "No previous diagnostics run found"
+ }
+
+
+@app.get("/api/diagnostics/health")
+async def get_diagnostics_health():
+ """
+ Get comprehensive health status of all providers and models.
+ Returns health registry data for diagnostics and observability.
+ """
+ try:
+ # Get provider health
+ provider_health = _health_registry.get_all_entries()
+ provider_summary = _health_registry.get_summary()
+
+ # Get model health
+ model_health = []
+ model_summary = {
+ "total": 0,
+ "healthy": 0,
+ "degraded": 0,
+ "unavailable": 0,
+ "unknown": 0,
+ "in_cooldown": 0
+ }
+
+ try:
+ from ai_models import get_model_health_registry
+ model_health = get_model_health_registry()
+ # Calculate model summary
+ model_summary["total"] = len(model_health)
+ for model in model_health:
+ status = model.get("status", "unknown")
+ model_summary[status] = model_summary.get(status, 0) + 1
+ if model.get("in_cooldown", False):
+ model_summary["in_cooldown"] += 1
+ except Exception as e:
+ logger.warning(f"Could not load model health: {e}")
+
+ return {
+ "status": "success",
+ "timestamp": datetime.now().isoformat(),
+ "providers": {
+ "summary": provider_summary,
+ "entries": provider_health
+ },
+ "models": {
+ "summary": model_summary,
+ "entries": model_health
+ },
+ "overall_health": {
+ "providers_ok": provider_summary["healthy"] >= (provider_summary["total"] // 2) if provider_summary["total"] > 0 else True,
+ "models_ok": model_summary["healthy"] >= (model_summary["total"] // 4) if model_summary["total"] > 0 else True
+ }
+ }
+ except Exception as e:
+ logger.error(f"Error getting health diagnostics: {e}")
+ return {
+ "status": "error",
+ "error": str(e),
+ "timestamp": datetime.now().isoformat()
+ }
+
+
+@app.post("/api/diagnostics/run-test")
+async def run_diagnostic_test():
+ """
+ Run test_models_diagnostic.py and return results.
+ Execute the Python script and capture stdout/stderr.
+ """
+ import subprocess
+ import time
+
+ start_time = time.time()
+
+ try:
+ # Find the diagnostic script - check multiple possible locations
+ diagnostic_script = None
+ possible_paths = [
+ WORKSPACE_ROOT / "test_models_diagnostic.py",
+ Path("test_models_diagnostic.py"),
+ Path(__file__).parent / "test_models_diagnostic.py",
+ ]
+
+ for path in possible_paths:
+ if path.exists():
+ diagnostic_script = path
+ break
+
+ if not diagnostic_script:
+ return {
+ "status": "error",
+ "output": "test_models_diagnostic.py not found. Searched in:\n" + "\n".join([str(p) for p in possible_paths]),
+ "timestamp": datetime.now().isoformat(),
+ "duration_seconds": 0,
+ "summary": {
+ "transformers_available": False,
+ "hf_hub": False,
+ "models_loaded": 0,
+ "critical_issues": ["Diagnostic script not found"]
+ }
+ }
+
+ # Execute the diagnostic script
+ result = subprocess.run(
+ ["python3", str(diagnostic_script)],
+ capture_output=True,
+ text=True,
+ timeout=60, # 60 second timeout
+ cwd=str(diagnostic_script.parent)
+ )
+
+ duration = time.time() - start_time
+
+ # Combine stdout and stderr
+ full_output = result.stdout
+ if result.stderr:
+ full_output += "\n--- STDERR ---\n" + result.stderr
+
+ # Parse output for summary information
+ summary = {
+ "transformers_available": "✅ transformers:" in full_output and "OK" in full_output,
+ "hf_hub": "✅ Hub connection:" in full_output and "OK" in full_output,
+ "models_loaded": 0, # Would need more parsing to count actual loaded models
+ "critical_issues": []
+ }
+
+ # Check for critical issues
+ if "❌ transformers:" in full_output:
+ summary["critical_issues"].append("Transformers library not available")
+ if "❌ Authenticated access:" in full_output and "FAILED" in full_output:
+ summary["critical_issues"].append("HuggingFace authentication failed")
+ if "❌ Model not available" in full_output:
+ summary["critical_issues"].append("AI models failed to load")
+
+ return {
+ "status": "success",
+ "output": full_output,
+ "timestamp": datetime.now().isoformat(),
+ "duration_seconds": round(duration, 2),
+ "summary": summary
+ }
+
+ except subprocess.TimeoutExpired:
+ duration = time.time() - start_time
+ return {
+ "status": "timeout",
+ "output": f"Test timed out after {duration:.1f} seconds",
+ "timestamp": datetime.now().isoformat(),
+ "duration_seconds": round(duration, 2),
+ "summary": {
+ "transformers_available": False,
+ "hf_hub": False,
+ "models_loaded": 0,
+ "critical_issues": ["Test execution timed out"]
+ }
+ }
+
+ except Exception as e:
+ duration = time.time() - start_time
+ return {
+ "status": "error",
+ "output": f"Error running diagnostic test: {str(e)}",
+ "timestamp": datetime.now().isoformat(),
+ "duration_seconds": round(duration, 2),
+ "summary": {
+ "transformers_available": False,
+ "hf_hub": False,
+ "models_loaded": 0,
+ "critical_issues": [f"Execution error: {str(e)}"]
+ }
+ }
+
+
+@app.post("/api/diagnostics/self-heal")
+async def trigger_self_heal(model_key: Optional[str] = None):
+ """
+ Trigger self-healing actions for models.
+ Safe, idempotent, and non-blocking.
+
+ Query params:
+ model_key: Specific model to reinitialize (optional)
+ """
+ try:
+ from ai_models import attempt_model_reinit, get_model_health_registry
+
+ results = []
+
+ if model_key:
+ # Reinit specific model
+ result = attempt_model_reinit(model_key)
+ results.append({
+ "model_key": model_key,
+ **result
+ })
+ else:
+ # Reinit all failed models that are out of cooldown
+ model_health = get_model_health_registry()
+ failed_models = [
+ m for m in model_health
+ if m.get("status") in ["unavailable", "degraded"]
+ and not m.get("in_cooldown", False)
+ ]
+
+ for model in failed_models[:5]: # Limit to 5 at a time to avoid blocking
+ result = attempt_model_reinit(model["key"])
+ results.append({
+ "model_key": model["key"],
+ **result
+ })
+
+ success_count = sum(1 for r in results if r.get("status") == "success")
+
+ return {
+ "status": "completed",
+ "timestamp": datetime.now().isoformat(),
+ "results": results,
+ "summary": {
+ "total_attempts": len(results),
+ "successful": success_count,
+ "failed": len(results) - success_count
+ }
+ }
+ except Exception as e:
+ logger.error(f"Error in self-heal: {e}")
+ return {
+ "status": "error",
+ "error": str(e),
+ "timestamp": datetime.now().isoformat()
+ }
+
+
+# ===== APL (Auto Provider Loader) Endpoints =====
+@app.post("/api/apl/run")
+async def run_apl_scan():
+ """Run APL provider scan"""
+ try:
+ # Run APL script
+ result = subprocess.run(
+ ["python3", str(WORKSPACE_ROOT / "auto_provider_loader.py")],
+ capture_output=True,
+ text=True,
+ timeout=300,
+ cwd=str(WORKSPACE_ROOT)
+ )
+
+ # Reload providers after APL run
+ config = load_providers_config()
+ _provider_state["providers"] = config.get("providers", {})
+
+ return {
+ "status": "completed",
+ "stdout": result.stdout[-1000:], # Last 1000 chars
+ "returncode": result.returncode,
+ "providers_count": len(_provider_state["providers"]),
+ "timestamp": datetime.now().isoformat()
+ }
+
+ except subprocess.TimeoutExpired:
+ return {
+ "status": "timeout",
+ "message": "APL scan timed out after 5 minutes"
+ }
+ except Exception as e:
+ raise HTTPException(status_code=500, detail=f"APL scan failed: {str(e)}")
+
+
+@app.get("/api/apl/report")
+async def get_apl_report():
+ """Get APL validation report (alias for auto-discovery report)"""
+ return await get_providers_auto_discovery_report()
+
+@app.get("/api/providers/auto-discovery-report")
+async def get_providers_auto_discovery_report():
+ """Get PROVIDER_AUTO_DISCOVERY_REPORT.json"""
+ report = load_auto_discovery_report()
+
+ if not report:
+ return {
+ "ok": False,
+ "error": "Auto-discovery report file not found",
+ "message": f"Report file not found at {AUTO_DISCOVERY_REPORT_PATH}"
+ }
+
+ return {
+ "ok": True,
+ "report": report,
+ "source": "PROVIDER_AUTO_DISCOVERY_REPORT.json"
+ }
+
+@app.get("/api/providers/health-summary")
+async def get_providers_health_summary():
+ """Get simplified health summary from auto-discovery report + local routes - always returns 200"""
+ try:
+ report = load_auto_discovery_report()
+
+ # Load local routes for health checking
+ resources_json = WORKSPACE_ROOT / "api-resources" / "crypto_resources_unified_2025-11-11.json"
+ local_routes = []
+ local_health = {"total": 0, "checked": 0, "up": 0, "down": 0}
+
+ if resources_json.exists():
+ try:
+ with open(resources_json, 'r', encoding='utf-8') as f:
+ unified_data = json.load(f)
+ unified_registry = unified_data.get('registry', {})
+ local_routes = unified_registry.get('local_backend_routes', [])
+ local_health["total"] = len(local_routes)
+
+ # Quick health check for up to 10 local routes
+ async with httpx.AsyncClient(timeout=2.0) as client:
+ routes_to_check = [r for r in local_routes if 'ws://' not in r.get('base_url', '')][:10]
+ for route in routes_to_check:
+ base_url = route.get('base_url', '').replace('{API_BASE}', f'http://localhost:{PORT}')
+ if 'http' in base_url:
+ try:
+ response = await client.get(base_url, timeout=2.0)
+ local_health["checked"] += 1
+ if response.status_code < 500:
+ local_health["up"] += 1
+ else:
+ local_health["down"] += 1
+ except:
+ local_health["checked"] += 1
+ local_health["down"] += 1
+ except Exception as e:
+ logger.error(f"Error checking local routes health: {e}")
+
+ if not report or "stats" not in report:
+ return JSONResponse(
+ status_code=200,
+ content={
+ "ok": False,
+ "error": "Auto-discovery report not found or invalid",
+ "message": f"Report file not found at {AUTO_DISCOVERY_REPORT_PATH}",
+ "summary": {
+ "total_active_providers": 0,
+ "http_valid": 0,
+ "http_invalid": 0,
+ "http_conditional": 0,
+ "hf_valid": 0,
+ "hf_invalid": 0,
+ "hf_conditional": 0,
+ "status_breakdown": {"VALID": 0, "INVALID": 0, "CONDITIONALLY_AVAILABLE": 0},
+ "execution_time_sec": 0,
+ "timestamp": "",
+ "local_routes": local_health
+ }
+ }
+ )
+
+ stats = report.get("stats", {})
+ http_providers = report.get("http_providers", {})
+ hf_providers = report.get("hf_providers", {})
+
+ # Count by status
+ status_counts = {"VALID": 0, "INVALID": 0, "CONDITIONALLY_AVAILABLE": 0}
+ for result in http_providers.get("results", []):
+ status = result.get("status", "UNKNOWN")
+ if status in status_counts:
+ status_counts[status] += 1
+
+ return JSONResponse(
+ status_code=200,
+ content={
+ "ok": True,
+ "summary": {
+ "total_active_providers": stats.get("total_active_providers", 0),
+ "http_valid": stats.get("http_valid", 0),
+ "http_invalid": stats.get("http_invalid", 0),
+ "http_conditional": stats.get("http_conditional", 0),
+ "hf_valid": stats.get("hf_valid", 0),
+ "hf_invalid": stats.get("hf_invalid", 0),
+ "hf_conditional": stats.get("hf_conditional", 0),
+ "status_breakdown": status_counts,
+ "execution_time_sec": stats.get("execution_time_sec", 0),
+ "timestamp": stats.get("timestamp", ""),
+ "local_routes": local_health
+ },
+ "source": "PROVIDER_AUTO_DISCOVERY_REPORT.json + local routes"
+ }
+ )
+ except Exception as e:
+ logger.error(f"Error loading health summary: {e}")
+ return JSONResponse(
+ status_code=200,
+ content={
+ "ok": False,
+ "error": str(e),
+ "summary": {
+ "total_active_providers": 0,
+ "http_valid": 0,
+ "http_invalid": 0,
+ "http_conditional": 0,
+ "hf_valid": 0,
+ "hf_invalid": 0,
+ "hf_conditional": 0,
+ "status_breakdown": {"VALID": 0, "INVALID": 0, "CONDITIONALLY_AVAILABLE": 0},
+ "execution_time_sec": 0,
+ "timestamp": "",
+ "local_routes": {"total": 0, "checked": 0, "up": 0, "down": 0}
+ }
+ }
+ )
+
+@app.get("/api/apl/summary")
+async def get_apl_summary():
+ """Get APL summary statistics (alias for health-summary)"""
+ return await get_providers_health_summary()
+
+
+# ===== HF Models Endpoints =====
+@app.get("/api/hf/models")
+async def get_hf_models():
+ """Get HuggingFace models from APL report"""
+ report = load_apl_report()
+
+ if not report:
+ return {"models": [], "count": 0}
+
+ hf_models = report.get("hf_models", {}).get("results", [])
+
+ return {
+ "models": hf_models,
+ "count": len(hf_models),
+ "source": "APL Validation Report (Real Data)"
+ }
+
+
+@app.get("/api/hf/health")
+async def get_hf_health():
+ """Get HF services health"""
+ try:
+ from backend.services.hf_registry import REGISTRY
+ health = REGISTRY.health()
+ return health
+ except Exception as e:
+ return {
+ "ok": False,
+ "error": f"HF registry not available: {str(e)}"
+ }
+
+
+# ===== DeFi Endpoint =====
+@app.get("/api/defi")
+async def get_defi():
+ """DeFi endpoint"""
+ return {
+ "success": True,
+ "message": "DeFi data endpoint",
+ "data": [],
+ "timestamp": datetime.now().isoformat()
+ }
+
+
+# ===== News Endpoint (compatible with UI) =====
+@app.get("/api/news")
+async def get_news_api(limit: int = 20):
+ """Get news as structured JSON. Empty news is not fatal."""
+ results = []
+ errors = []
+ db_rows_available = False
+ try:
+ conn = sqlite3.connect(str(DB_PATH))
+ cursor = conn.cursor()
+ cursor.execute("""
+ SELECT * FROM news_articles
+ ORDER BY analyzed_at DESC
+ LIMIT ?
+ """, (limit,))
+ rows = cursor.fetchall()
+ columns = [desc[0] for desc in cursor.description]
+ conn.close()
+ db_rows_available = bool(rows)
+ for row in rows:
+ record = dict(zip(columns, row))
+ if record.get("related_symbols"):
+ try:
+ record["related_symbols"] = json.loads(record["related_symbols"])
+ except Exception:
+ pass
+ results.append(record)
+ except Exception as db_error:
+ errors.append(f"database_news_failed: {db_error}")
+
+ if not results:
+ try:
+ newsapi_key = get_secret("NEWSAPI_KEY")
+ if newsapi_key:
+ async with httpx.AsyncClient(timeout=httpx.Timeout(8.0, connect=3.0), headers=HEADERS) as client:
+ response = await client.get(
+ "https://newsapi.org/v2/everything",
+ params={"q": "crypto OR bitcoin OR ethereum", "language": "en", "pageSize": min(max(int(limit or 20), 1), 100), "apiKey": newsapi_key, "sortBy": "publishedAt"},
+ )
+ if response.status_code == 200:
+ payload = response.json()
+ for article in (payload.get("articles") or [])[:limit]:
+ results.append({
+ "title": article.get("title", ""),
+ "content": article.get("description") or article.get("content") or "",
+ "url": article.get("url", ""),
+ "source": (article.get("source") or {}).get("name", "NewsAPI"),
+ "sentiment_label": None,
+ "sentiment_confidence": None,
+ "related_symbols": [],
+ "published_date": article.get("publishedAt"),
+ "analyzed_at": datetime.now().isoformat(),
+ })
+ else:
+ errors.append(f"newsapi_http_{response.status_code}")
+ except Exception as newsapi_error:
+ errors.append(f"newsapi_failed: {newsapi_error}")
+
+ if not results:
+ try:
+ cryptocompare_api_key = get_secret("CRYPTOCOMPARE_KEY")
+ headers = {"User-Agent": "Mozilla/5.0"}
+ if cryptocompare_api_key:
+ headers["authorization"] = f"Apikey {cryptocompare_api_key}"
+ async with httpx.AsyncClient(timeout=httpx.Timeout(8.0, connect=3.0)) as client:
+ response = await client.get("https://min-api.cryptocompare.com/data/v2/news/?lang=EN", headers=headers)
+ if response.status_code == 200:
+ data = response.json()
+ for article in (data.get("Data") or [])[:limit]:
+ results.append({
+ "id": article.get("id"),
+ "title": article.get("title", ""),
+ "content": article.get("body", "")[:500],
+ "url": article.get("url", ""),
+ "source": article.get("source", "CryptoCompare"),
+ "sentiment_label": None,
+ "sentiment_confidence": None,
+ "related_symbols": article.get("categories", "").split("|") if article.get("categories") else [],
+ "published_date": datetime.fromtimestamp(article.get("published_on", 0)).isoformat() if article.get("published_on") else None,
+ "analyzed_at": datetime.now().isoformat(),
+ })
+ else:
+ errors.append(f"cryptocompare_news_http_{response.status_code}")
+ except Exception as api_error:
+ errors.append(f"external_news_failed: {api_error}")
+
+ return {
+ "success": True,
+ "data": results,
+ "news": results,
+ "count": len(results),
+ "status": "available" if results else "empty",
+ "source": "database" if db_rows_available else "external_or_empty",
+ "errors": errors,
+ "timestamp": datetime.now().isoformat(),
+ }
+
+# ===== Logs Endpoints =====
+@app.get("/api/logs/summary")
+async def get_logs_summary():
+ """Get logs summary"""
+ try:
+ return {
+ "success": True,
+ "total": len(_provider_state.get("logs", [])),
+ "recent": _provider_state.get("logs", [])[-10:],
+ "timestamp": datetime.now().isoformat()
+ }
+ except Exception as e:
+ return {
+ "success": False,
+ "error": str(e)
+ }
+
+
+# ===== Diagnostics Endpoints =====
+@app.get("/api/diagnostics/errors")
+async def get_diagnostics_errors():
+ """Get diagnostic errors"""
+ try:
+ return {
+ "success": True,
+ "errors": [],
+ "timestamp": datetime.now().isoformat()
+ }
+ except Exception as e:
+ return {
+ "success": False,
+ "errors": [],
+ "error": str(e)
+ }
+
+
+# ===== Resources Endpoints =====
+@app.get("/api/resources/search")
+async def search_resources(q: str = "", source: str = "all"):
+ """Search resources"""
+ try:
+ return {
+ "success": True,
+ "query": q,
+ "source": source,
+ "results": [],
+ "count": 0
+ }
+ except Exception as e:
+ return {
+ "success": False,
+ "error": str(e)
+ }
+
+
+# ===== V2 API Endpoints (compatibility) =====
+@app.post("/api/v2/export/{export_type}")
+async def export_v2(export_type: str, data: Dict[str, Any] = None):
+ """V2 export endpoint"""
+ return {
+ "success": True,
+ "type": export_type,
+ "message": "Export functionality",
+ "data": data or {}
+ }
+
+
+@app.post("/api/v2/backup")
+async def backup_v2():
+ """V2 backup endpoint"""
+ return {
+ "success": True,
+ "message": "Backup functionality",
+ "timestamp": datetime.now().isoformat()
+ }
+
+
+@app.post("/api/v2/import/providers")
+async def import_providers_v2(data: Dict[str, Any]):
+ """V2 import providers endpoint"""
+ return {
+ "success": True,
+ "message": "Import providers functionality",
+ "data": data
+ }
+
+
+# ===== HuggingFace ML Sentiment Endpoints =====
+@app.post("/api/sentiment/analyze")
+async def analyze_sentiment(request: Dict[str, Any]):
+ """Analyze sentiment using Hugging Face models"""
+ try:
+ from ai_models import (
+ analyze_crypto_sentiment,
+ analyze_financial_sentiment,
+ analyze_social_sentiment,
+ analyze_market_text,
+ _registry,
+ MODEL_SPECS,
+ ModelNotAvailable
+ )
+
+ text = request.get("text", "").strip()
+ if not text:
+ raise HTTPException(status_code=400, detail="Text is required")
+
+ mode = request.get("mode", "auto").lower()
+ source = request.get("source", "user")
+ model_key = request.get("model_key")
+ symbol = request.get("symbol")
+
+ try:
+ # If model_key is provided, use that specific model
+ if model_key and model_key in MODEL_SPECS:
+ try:
+ pipeline = _registry.get_pipeline(model_key)
+ spec = MODEL_SPECS[model_key]
+
+ # Handle different task types
+ if spec.task == "text-generation":
+ # For trading signal models or generation models
+ raw_result = pipeline(text, max_length=200, num_return_sequences=1)
+ if isinstance(raw_result, list) and raw_result:
+ raw_result = raw_result[0]
+
+ generated_text = raw_result.get("generated_text", str(raw_result))
+
+ # Parse trading signals if applicable
+ if spec.category == "trading_signal":
+ # Extract signal from generated text
+ decision = "HOLD"
+ if "buy" in generated_text.lower():
+ decision = "BUY"
+ elif "sell" in generated_text.lower():
+ decision = "SELL"
+
+ return {
+ "ok": True,
+ "available": True,
+ "sentiment": decision.lower(),
+ "label": decision.lower(),
+ "score": 0.7,
+ "confidence": 0.7,
+ "model": model_key,
+ "engine": "huggingface",
+ "mode": "trading",
+ "extra": {
+ "decision": decision,
+ "rationale": generated_text,
+ "raw": raw_result
+ }
+ }
+ else:
+ # Generation model - return generated text
+ return {
+ "ok": True,
+ "available": True,
+ "sentiment": "neutral",
+ "label": "neutral",
+ "score": 0.5,
+ "confidence": 0.5,
+ "model": model_key,
+ "engine": "huggingface",
+ "mode": "generation",
+ "extra": {
+ "generated_text": generated_text,
+ "raw": raw_result
+ }
+ }
+ else:
+ # Text classification / sentiment
+ raw_result = pipeline(text[:512])
+ if isinstance(raw_result, list) and raw_result:
+ raw_result = raw_result[0]
+
+ label = raw_result.get("label", "neutral").upper()
+ score = raw_result.get("score", 0.5)
+
+ # Map labels to standard format
+ mapped = "bullish" if "POSITIVE" in label or "BULLISH" in label or "LABEL_2" in label else (
+ "bearish" if "NEGATIVE" in label or "BEARISH" in label or "LABEL_0" in label else "neutral"
+ )
+
+ return {
+ "ok": True,
+ "available": True,
+ "sentiment": mapped,
+ "label": mapped,
+ "score": score,
+ "confidence": score,
+ "raw_label": label,
+ "model": model_key,
+ "engine": "huggingface",
+ "mode": mode,
+ "extra": {
+ "vote": score if mapped == "bullish" else (-score if mapped == "bearish" else 0.0),
+ "raw": raw_result
+ }
+ }
+ except ModelNotAvailable as e:
+ logger.warning(f"Model {model_key} not available: {e}")
+ return {
+ "ok": False,
+ "available": False,
+ "error": f"Model {model_key} not available: {str(e)}",
+ "label": "neutral",
+ "sentiment": "neutral",
+ "score": 0.0,
+ "confidence": 0.0
+ }
+
+ # Default mode-based analysis
+ if mode == "crypto":
+ result = analyze_crypto_sentiment(text)
+ elif mode == "financial":
+ result = analyze_financial_sentiment(text)
+ elif mode == "social":
+ result = analyze_social_sentiment(text)
+ elif mode == "trading":
+ # Try to use trading signal model
+ result = analyze_crypto_sentiment(text)
+ else:
+ result = analyze_market_text(text)
+
+ sentiment_label = result.get("label", "neutral")
+ confidence = result.get("confidence", result.get("score", 0.5))
+ model_used = result.get("model_count", result.get("model", result.get("engine", "unknown")))
+
+ # Prepare response compatible with frontend format
+ response_data = {
+ "ok": True,
+ "available": True,
+ "sentiment": sentiment_label.lower(),
+ "label": sentiment_label.lower(),
+ "confidence": float(confidence),
+ "score": float(confidence),
+ "model": f"{model_used} models" if isinstance(model_used, int) else str(model_used),
+ "engine": result.get("engine", "huggingface"),
+ "mode": mode
+ }
+
+ # Add details if available for score bars
+ if result.get("scores"):
+ scores_dict = result.get("scores", {})
+ if isinstance(scores_dict, dict):
+ labels_list = []
+ scores_list = []
+ for lbl, scr in scores_dict.items():
+ labels_list.append(lbl)
+ scores_list.append(float(scr) if isinstance(scr, (int, float)) else float(scr.get("score", 0.5)) if isinstance(scr, dict) else 0.5)
+ if labels_list:
+ response_data["details"] = {
+ "labels": labels_list,
+ "scores": scores_list
+ }
+
+ # Save to database
+ try:
+ conn = sqlite3.connect(str(DB_PATH))
+ cursor = conn.cursor()
+ cursor.execute("""
+ INSERT INTO sentiment_analysis
+ (text, sentiment_label, confidence, model_used, analysis_type, symbol, scores)
+ VALUES (?, ?, ?, ?, ?, ?, ?)
+ """, (
+ text[:500],
+ sentiment_label,
+ confidence,
+ f"{model_used} models" if isinstance(model_used, int) else str(model_used),
+ mode,
+ symbol,
+ json.dumps(result.get("scores", {}))
+ ))
+ conn.commit()
+ conn.close()
+ except Exception as db_error:
+ logger.warning(f"Failed to save to database: {db_error}")
+
+ return response_data
+
+ except Exception as e:
+ # Unexpected error - log and return error response
+ logger.error(f"Sentiment analysis unexpected error: {str(e)}")
+ return {
+ "ok": False,
+ "available": False,
+ "error": f"Analysis failed: {str(e)}",
+ "sentiment": "neutral",
+ "label": "neutral",
+ "confidence": 0.0,
+ "score": 0.0
+ }
+
+ except HTTPException:
+ raise
+ except Exception as e:
+ raise HTTPException(status_code=500, detail=f"Sentiment analysis failed: {str(e)}")
+
+
+@app.post("/api/ai/summarize")
+async def summarize_text(request: Dict[str, Any]):
+ """
+ Summarize text using Hugging Face models or simple text processing.
+
+ Expects: { "text": "string", "max_sentences": 3 }
+ Returns: { "ok": true, "summary": "...", "sentences": ["...", "..."] }
+ """
+ try:
+ text = request.get("text", "").strip()
+ max_sentences = request.get("max_sentences", 3)
+
+ if not text:
+ return {
+ "ok": False,
+ "error": "Text is required"
+ }
+
+ # Try to use Hugging Face summarization model if available
+ try:
+ from ai_models import MODEL_SPECS, _registry, ModelNotAvailable
+
+ # Check if summarization model is available
+ summarization_key = None
+ for key, spec in MODEL_SPECS.items():
+ if spec.task == "summarization":
+ summarization_key = key
+ break
+
+ if summarization_key:
+ try:
+ pipeline = _registry.get_pipeline(summarization_key)
+ # Use HF model for summarization
+ # Try with parameters first, then fallback to simple call
+ try:
+ summary_result = pipeline(text, max_length=max_sentences * 50, min_length=max_sentences * 20, do_sample=False)
+ except TypeError:
+ # Some pipelines don't accept these parameters
+ summary_result = pipeline(text)
+
+ if isinstance(summary_result, list) and summary_result:
+ summary_text = summary_result[0].get("summary_text", summary_result[0].get("generated_text", str(summary_result[0])))
+ elif isinstance(summary_result, dict):
+ summary_text = summary_result.get("summary_text", summary_result.get("generated_text", str(summary_result)))
+ else:
+ summary_text = str(summary_result)
+
+ # Split into sentences
+ sentences = [s.strip() + ("." if not s.strip().endswith((".", "!", "?")) else "") for s in summary_text.split(". ") if s.strip()]
+ sentences = sentences[:max_sentences]
+
+ return {
+ "ok": True,
+ "summary": summary_text,
+ "sentences": sentences
+ }
+ except ModelNotAvailable:
+ # Fall through to simple summarizer
+ pass
+ except Exception as e:
+ logger.warning(f"HF summarization failed: {e}, using fallback")
+ # Fall through to simple summarizer
+ pass
+ except Exception as e:
+ logger.warning(f"HF summarization model not available: {e}")
+ # Fall through to simple summarizer
+
+ # Simple placeholder summarizer: split by sentences and take first N
+ sentences = []
+ current_sentence = ""
+
+ for char in text:
+ current_sentence += char
+ if char in ".!?":
+ sentence = current_sentence.strip()
+ if sentence:
+ sentences.append(sentence)
+ current_sentence = ""
+ if len(sentences) >= max_sentences:
+ break
+
+ # If we didn't get enough sentences, add the rest
+ if len(sentences) < max_sentences and current_sentence.strip():
+ sentences.append(current_sentence.strip())
+
+ # If still no sentences, just truncate
+ if not sentences:
+ words = text.split()
+ chunk_size = len(words) // max_sentences
+ sentences = []
+ for i in range(max_sentences):
+ start_idx = i * chunk_size
+ end_idx = start_idx + chunk_size if i < max_sentences - 1 else len(words)
+ if start_idx < len(words):
+ sentence = " ".join(words[start_idx:end_idx])
+ if sentence:
+ sentences.append(sentence)
+
+ summary = " ".join(sentences)
+
+ return {
+ "ok": True,
+ "summary": summary,
+ "sentences": sentences[:max_sentences]
+ }
+
+ except Exception as e:
+ logger.error(f"Summarization failed: {e}")
+ return {
+ "ok": False,
+ "error": f"Summarization failed: {str(e)}"
+ }
+
+
+@app.post("/api/news/analyze")
+async def analyze_news(request: Dict[str, Any]):
+ """Analyze news article sentiment using HF models"""
+ try:
+ from ai_models import analyze_news_item
+
+ title = request.get("title", "").strip()
+ content = request.get("content", request.get("description", "")).strip()
+ url = request.get("url", "")
+ source = request.get("source", "unknown")
+ published_date = request.get("published_date")
+
+ if not title and not content:
+ raise HTTPException(status_code=400, detail="Title or content is required")
+
+ try:
+ news_item = {
+ "title": title,
+ "description": content
+ }
+ result = analyze_news_item(news_item)
+
+ sentiment_label = result.get("sentiment", "neutral")
+ sentiment_confidence = result.get("sentiment_confidence", 0.5)
+ sentiment_details = result.get("sentiment_details", {})
+ related_symbols = request.get("related_symbols", [])
+
+ # Check if HF models were used (for diagnostics)
+ hf_available = sentiment_details.get("engine", "unknown") == "huggingface" if isinstance(sentiment_details, dict) else True
+
+ # Save to database (always)
+ saved_to_db = False
+ try:
+ conn = sqlite3.connect(str(DB_PATH))
+ cursor = conn.cursor()
+ cursor.execute("""
+ INSERT INTO news_articles
+ (title, content, url, source, sentiment_label, sentiment_confidence, related_symbols, published_date)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+ """, (
+ title[:500],
+ content[:2000] if content else None,
+ url,
+ source,
+ sentiment_label,
+ sentiment_confidence,
+ json.dumps(related_symbols) if related_symbols else None,
+ published_date
+ ))
+ conn.commit()
+ conn.close()
+ saved_to_db = True
+ except Exception as db_error:
+ logger.warning(f"Failed to save to database: {db_error}")
+
+ return {
+ "success": True,
+ "available": True,
+ "hf_available": hf_available,
+ "news": {
+ "title": title,
+ "sentiment": sentiment_label,
+ "confidence": sentiment_confidence,
+ "details": sentiment_details
+ },
+ "saved_to_db": saved_to_db
+ }
+
+ except Exception as e:
+ logger.error(f"News analysis error: {str(e)}")
+ return {
+ "success": False,
+ "available": False,
+ "error": f"Analysis failed: {str(e)}",
+ "news": {
+ "title": title,
+ "sentiment": "neutral",
+ "confidence": 0.0
+ }
+ }
+
+ except HTTPException:
+ raise
+ except Exception as e:
+ raise HTTPException(status_code=500, detail=f"News analysis failed: {str(e)}")
+
+
+@app.get("/api/sentiment/history")
+async def get_sentiment_history(
+ symbol: Optional[str] = None,
+ limit: int = 50
+):
+ """Get sentiment analysis history from database"""
+ try:
+ conn = sqlite3.connect(str(DB_PATH))
+ cursor = conn.cursor()
+
+ if symbol:
+ cursor.execute("""
+ SELECT * FROM sentiment_analysis
+ WHERE symbol = ?
+ ORDER BY timestamp DESC
+ LIMIT ?
+ """, (symbol.upper(), limit))
+ else:
+ cursor.execute("""
+ SELECT * FROM sentiment_analysis
+ ORDER BY timestamp DESC
+ LIMIT ?
+ """, (limit,))
+
+ rows = cursor.fetchall()
+ columns = [desc[0] for desc in cursor.description]
+ conn.close()
+
+ results = []
+ for row in rows:
+ record = dict(zip(columns, row))
+ if record.get("scores"):
+ try:
+ record["scores"] = json.loads(record["scores"])
+ except:
+ pass
+ results.append(record)
+
+ return {
+ "success": True,
+ "count": len(results),
+ "results": results
+ }
+
+ except Exception as e:
+ raise HTTPException(status_code=500, detail=f"Failed to fetch sentiment history: {str(e)}")
+
+
+@app.post("/api/news/fetch")
+async def fetch_and_save_news(limit: int = 50):
+ """Fetch news from CryptoCompare API and save to database"""
+ try:
+ cryptocompare_api_key = os.getenv("CRYPTOCOMPARE_API_KEY", "")
+
+ async with httpx.AsyncClient(timeout=15.0) as client:
+ response = await client.get(
+ "https://min-api.cryptocompare.com/data/v2/news/?lang=EN",
+ headers={
+ "User-Agent": "Mozilla/5.0",
+ "authorization": f"Apikey {cryptocompare_api_key}"
+ }
+ )
+
+ if response.status_code != 200:
+ return {
+ "success": False,
+ "error": f"CryptoCompare API returned {response.status_code}",
+ "saved": 0
+ }
+
+ data = response.json()
+
+ if not data.get("Data"):
+ return {
+ "success": False,
+ "error": "No news data returned from API",
+ "saved": 0
+ }
+
+ # Save to database
+ conn = sqlite3.connect(str(DB_PATH))
+ cursor = conn.cursor()
+ saved_count = 0
+
+ for article in data["Data"][:limit]:
+ try:
+ # Check if article already exists
+ cursor.execute("SELECT id FROM news_articles WHERE url = ?", (article.get("url", ""),))
+ if cursor.fetchone():
+ continue # Skip duplicates
+
+ # Extract related symbols from categories
+ categories = article.get("categories", "").split("|") if article.get("categories") else []
+ related_symbols_json = json.dumps(categories)
+
+ # Insert news article
+ cursor.execute("""
+ INSERT INTO news_articles (
+ title, content, url, source,
+ related_symbols, published_date, analyzed_at
+ ) VALUES (?, ?, ?, ?, ?, ?, ?)
+ """, (
+ article.get("title", ""),
+ article.get("body", "")[:1000], # Limit content length
+ article.get("url", ""),
+ article.get("source", "CryptoCompare"),
+ related_symbols_json,
+ datetime.fromtimestamp(article.get("published_on", 0)).isoformat() if article.get("published_on") else None,
+ datetime.now().isoformat()
+ ))
+ saved_count += 1
+ except Exception as e:
+ logger.warning(f"Error saving article: {e}")
+ continue
+
+ conn.commit()
+ conn.close()
+
+ logger.info(f"[OK] Saved {saved_count} news articles to database")
+
+ return {
+ "success": True,
+ "saved": saved_count,
+ "total_fetched": len(data["Data"][:limit]),
+ "message": f"Successfully saved {saved_count} news articles"
+ }
+
+ except Exception as e:
+ logger.error(f"Error fetching news: {e}")
+ return {
+ "success": False,
+ "error": str(e),
+ "saved": 0
+ }
+
+
+@app.get("/api/news/latest")
+async def get_latest_news(limit: int = 20, sentiment: Optional[str] = None):
+ """Get latest analyzed news; empty is a valid non-fatal state."""
+ try:
+ conn = sqlite3.connect(str(DB_PATH))
+ cursor = conn.cursor()
+ if sentiment:
+ cursor.execute("""
+ SELECT * FROM news_articles
+ WHERE sentiment_label = ?
+ ORDER BY analyzed_at DESC
+ LIMIT ?
+ """, (sentiment.lower(), limit))
+ else:
+ cursor.execute("""
+ SELECT * FROM news_articles
+ ORDER BY analyzed_at DESC
+ LIMIT ?
+ """, (limit,))
+ rows = cursor.fetchall()
+ columns = [desc[0] for desc in cursor.description]
+ conn.close()
+
+ results = []
+ for row in rows:
+ record = dict(zip(columns, row))
+ if record.get("related_symbols"):
+ try:
+ record["related_symbols"] = json.loads(record["related_symbols"])
+ except Exception:
+ pass
+ results.append(record)
+
+ return {
+ "success": True,
+ "data": results,
+ "news": results,
+ "count": len(results),
+ "status": "available" if results else "empty",
+ "errors": [],
+ "timestamp": datetime.now().isoformat(),
+ }
+ except Exception as e:
+ logger.warning(f"Latest news fetch failed: {e}")
+ return {
+ "success": True,
+ "data": [],
+ "news": [],
+ "count": 0,
+ "status": "empty",
+ "errors": [str(e)],
+ "timestamp": datetime.now().isoformat(),
+ }
+
+@app.post("/api/news/summarize")
+async def summarize_news(request: Dict[str, Any]):
+ """
+ Summarize crypto/financial news using Hugging Face Crypto-Financial-News-Summarizer model
+
+ Expects: { "title": "News Title", "content": "Full article text" }
+ Returns: { "summary": "Summarized news paragraph", "model": "Crypto-Financial-News-Summarizer" }
+ """
+ try:
+ from ai_models import MODEL_SPECS, _registry, ModelNotAvailable
+
+ title = request.get("title", "").strip()
+ content = request.get("content", "").strip()
+
+ if not title and not content:
+ raise HTTPException(status_code=400, detail="Title or content is required")
+
+ # Combine title and content for summarization
+ text_to_summarize = f"{title}. {content}" if title and content else (title or content)
+
+ try:
+ # Try to use the Crypto-Financial-News-Summarizer model
+ summarization_key = "summarization_0"
+
+ if summarization_key in MODEL_SPECS:
+ try:
+ pipeline = _registry.get_pipeline(summarization_key)
+ spec = MODEL_SPECS[summarization_key]
+
+ # Use HF model for summarization
+ # Limit input text to avoid token length issues
+ max_input_length = 1024
+ text_input = text_to_summarize[:max_input_length]
+
+ try:
+ # Try with parameters first
+ summary_result = pipeline(
+ text_input,
+ max_length=150,
+ min_length=50,
+ do_sample=False,
+ truncation=True
+ )
+ except TypeError:
+ # Some pipelines don't accept these parameters
+ summary_result = pipeline(text_input, truncation=True)
+
+ # Extract summary text from result
+ if isinstance(summary_result, list) and summary_result:
+ summary_text = summary_result[0].get("summary_text", summary_result[0].get("generated_text", str(summary_result[0])))
+ elif isinstance(summary_result, dict):
+ summary_text = summary_result.get("summary_text", summary_result.get("generated_text", str(summary_result)))
+ else:
+ summary_text = str(summary_result)
+
+ return {
+ "success": True,
+ "summary": summary_text,
+ "model": spec.model_id,
+ "available": True,
+ "input_length": len(text_input),
+ "title": title,
+ "timestamp": datetime.now().isoformat()
+ }
+
+ except ModelNotAvailable as e:
+ logger.warning(f"Crypto-Financial-News-Summarizer not available: {e}")
+ # Fall through to fallback
+ except Exception as e:
+ logger.warning(f"HF summarization failed: {e}, using fallback")
+ # Fall through to fallback
+
+ # Fallback: Simple extractive summarization
+ # Split into sentences and take the most important ones
+ sentences = []
+ current_sentence = ""
+
+ for char in text_to_summarize:
+ current_sentence += char
+ if char in ".!?":
+ sentence = current_sentence.strip()
+ if sentence and len(sentence) > 10: # Filter out very short sentences
+ sentences.append(sentence)
+ current_sentence = ""
+ if len(sentences) >= 5: # Take first 5 sentences max
+ break
+
+ # If we didn't get enough sentences, add the rest
+ if len(sentences) < 3 and current_sentence.strip():
+ sentences.append(current_sentence.strip())
+
+ # Take first 3 sentences as summary
+ summary = " ".join(sentences[:3]) if sentences else text_to_summarize[:500]
+
+ return {
+ "success": True,
+ "summary": summary,
+ "model": "fallback_extractive",
+ "available": False,
+ "note": "Using fallback extractive summarization (HF model not available)",
+ "title": title,
+ "timestamp": datetime.now().isoformat()
+ }
+
+ except Exception as e:
+ logger.error(f"Summarization error: {str(e)}")
+ return {
+ "success": False,
+ "error": f"Summarization failed: {str(e)}",
+ "summary": "",
+ "model": "error",
+ "available": False
+ }
+
+ except HTTPException:
+ raise
+ except Exception as e:
+ raise HTTPException(status_code=500, detail=f"News summarization failed: {str(e)}")
+
+
+@app.get("/api/models/status")
+async def get_models_status():
+ """Get AI models status and registry info - honest status reporting"""
+ try:
+ from ai_models import (
+ get_model_info, registry_status, HF_MODE, TRANSFORMERS_AVAILABLE,
+ INFERENCE_API_MODE, _registry,
+ )
+
+ model_info = get_model_info()
+ registry_info = registry_status()
+ loaded_count = len(_registry._pipelines) + len(_registry._inference_ready)
+
+ # Determine honest status
+ if HF_MODE == "off":
+ status = "disabled"
+ status_message = "HF models are disabled (HF_MODE=off). To enable them, set HF_MODE=public or HF_MODE=auth in the environment."
+ elif INFERENCE_API_MODE and loaded_count > 0:
+ status = "ok" if len(_registry._failed_models) == 0 else "partial"
+ status_message = f"{loaded_count} model(s) ready via HF Inference API"
+ elif not TRANSFORMERS_AVAILABLE and not INFERENCE_API_MODE:
+ status = "transformers_unavailable"
+ status_message = "Transformers library is not installed. Models cannot be loaded."
+ elif not _registry._initialized:
+ status = "not_initialized"
+ status_message = "Models have not been initialized yet."
+ elif loaded_count == 0:
+ status = "no_models_loaded"
+ status_message = f"No models could be loaded. {len(_registry._failed_models)} models failed. Check model IDs or HF access."
+ elif loaded_count > 0:
+ status = "ok" if len(_registry._failed_models) == 0 else "partial"
+ backend = "inference API" if INFERENCE_API_MODE else "local"
+ status_message = f"{loaded_count} model(s) loaded successfully ({backend})"
+ if len(_registry._failed_models) > 0:
+ status_message += f", {len(_registry._failed_models)} failed"
+ else:
+ status = "unknown"
+ status_message = "Unknown status"
+
+ # Format failed models as list of [key, error] tuples for ai_tools.html
+ failed_list = []
+ for key, error in list(_registry._failed_models.items())[:10]:
+ failed_list.append([key, str(error)])
+
+ return {
+ "success": True,
+ "status": status,
+ "status_message": status_message,
+ "hf_mode": HF_MODE,
+ "inference_api_mode": INFERENCE_API_MODE,
+ "models_loaded": loaded_count,
+ "models_failed": len(_registry._failed_models),
+ "transformers_available": TRANSFORMERS_AVAILABLE,
+ "initialized": _registry._initialized,
+ "models": model_info,
+ "registry": registry_info,
+ "failed": failed_list, # Format: [[key, error], ...] for ai_tools.html
+ "failed_models": list(_registry._failed_models.keys())[:10], # Keep for backward compatibility
+ "loaded_models": list(_registry._pipelines.keys()) + list(_registry._inference_ready),
+ "database": {
+ "path": str(DB_PATH),
+ "exists": DB_PATH.exists()
+ }
+ }
+ except Exception as e:
+ logger.error(f"Error getting models status: {e}")
+ return {
+ "success": False,
+ "status": "error",
+ "status_message": f"Error retrieving model status: {str(e)}",
+ "error": str(e),
+ "hf_mode": "unknown",
+ "models_loaded": 0,
+ "models_failed": 0
+ }
+
+
+@app.post("/api/models/initialize")
+async def initialize_ai_models():
+ """Initialize AI models (force reload)"""
+ try:
+ from ai_models import initialize_models, _registry, HF_MAX_STARTUP_MODELS
+
+ result = initialize_models(max_models=HF_MAX_STARTUP_MODELS)
+ registry_status = _registry.get_registry_status()
+
+ return registry_status
+ except Exception as e:
+ logger.error(f"Failed to initialize models: {e}")
+ return {
+ "models_total": 0,
+ "models_loaded": 0,
+ "models_failed": 0,
+ "items": [],
+ "error": str(e)
+ }
+
+
+# ===== Model-based Data Endpoints (Using HF Models as Data Sources) =====
+@app.get("/api/models/list")
+async def list_available_models():
+ """List all available Hugging Face models as data sources"""
+ try:
+ from ai_models import get_model_info, MODEL_SPECS, _registry, CRYPTO_SENTIMENT_MODELS, SOCIAL_SENTIMENT_MODELS, FINANCIAL_SENTIMENT_MODELS, NEWS_SENTIMENT_MODELS, GENERATION_MODELS, TRADING_SIGNAL_MODELS
+
+ model_info = get_model_info()
+
+ # Model descriptions
+ model_descriptions = {
+ "kk08/CryptoBERT": "Crypto sentiment binary classification model trained on cryptocurrency-related text",
+ "ElKulako/cryptobert": "Crypto social sentiment classifier (Bullish/Neutral/Bearish) for social media and news",
+ "StephanAkkerman/FinTwitBERT-sentiment": "Financial tweet sentiment analysis model for market-related social media content",
+ "OpenC/crypto-gpt-o3-mini": "Crypto and DeFi text generation model for analysis and content creation",
+ "ElKulako/cryptobert": "Crypto sentiment model used for trading signal generation (buy/sell/hold based on sentiment)",
+ "cardiffnlp/twitter-roberta-base-sentiment-latest": "General Twitter sentiment analysis (fallback model)",
+ "ProsusAI/finbert": "Financial sentiment analysis model for news and financial documents",
+ "FurkanGozukara/Crypto-Financial-News-Summarizer": "Specialized model for summarizing cryptocurrency and financial news articles"
+ }
+
+ models_list = []
+ for key, spec in MODEL_SPECS.items():
+ is_loaded = key in _registry._pipelines or key in getattr(_registry, "_inference_ready", set())
+ error_msg = None
+ if key in _registry._failed_models:
+ error_msg = str(_registry._failed_models[key])
+
+ models_list.append({
+ "key": key,
+ "id": key,
+ "name": spec.model_id,
+ "model_id": spec.model_id,
+ "task": spec.task,
+ "category": spec.category,
+ "requires_auth": spec.requires_auth,
+ "loaded": is_loaded,
+ "error": error_msg,
+ "description": model_descriptions.get(spec.model_id, f"{spec.category} model for {spec.task}"),
+ "endpoint": f"/api/models/{key}/predict"
+ })
+
+ return {
+ "success": True,
+ "total_models": len(models_list),
+ "models": models_list,
+ "categories": {
+ "crypto_sentiment": CRYPTO_SENTIMENT_MODELS,
+ "social_sentiment": SOCIAL_SENTIMENT_MODELS,
+ "financial_sentiment": FINANCIAL_SENTIMENT_MODELS,
+ "news_sentiment": NEWS_SENTIMENT_MODELS,
+ "generation": GENERATION_MODELS,
+ "trading_signals": TRADING_SIGNAL_MODELS,
+ "summarization": ["FurkanGozukara/Crypto-Financial-News-Summarizer"]
+ },
+ "model_info": model_info
+ }
+ except Exception as e:
+ return {
+ "success": False,
+ "error": str(e),
+ "models": []
+ }
+
+
+@app.get("/api/models/{model_key}/info")
+async def get_model_info_endpoint(model_key: str):
+ """Get information about a specific model"""
+ try:
+ from ai_models import MODEL_SPECS, ModelNotAvailable, _registry
+
+ if model_key not in MODEL_SPECS:
+ raise HTTPException(status_code=404, detail=f"Model {model_key} not found")
+
+ spec = MODEL_SPECS[model_key]
+ is_loaded = model_key in _registry._pipelines
+
+ return {
+ "success": True,
+ "model_key": model_key,
+ "model_id": spec.model_id,
+ "task": spec.task,
+ "category": spec.category,
+ "requires_auth": spec.requires_auth,
+ "is_loaded": is_loaded,
+ "endpoint": f"/api/models/{model_key}/predict",
+ "usage": {
+ "method": "POST",
+ "url": f"/api/models/{model_key}/predict",
+ "body": {"text": "string", "options": {}}
+ }
+ }
+ except HTTPException:
+ raise
+ except Exception as e:
+ raise HTTPException(status_code=500, detail=str(e))
+
+
+@app.post("/api/models/{model_key}/predict")
+async def predict_with_model(model_key: str, request: Dict[str, Any]):
+ """Use a specific model to generate predictions/data"""
+ try:
+ from ai_models import MODEL_SPECS, _registry, ModelNotAvailable
+
+ if model_key not in MODEL_SPECS:
+ raise HTTPException(status_code=404, detail=f"Model {model_key} not found")
+
+ spec = MODEL_SPECS[model_key]
+ text = request.get("text", "").strip()
+
+ if not text:
+ raise HTTPException(status_code=400, detail="Text is required")
+
+ try:
+ pipeline = _registry.get_pipeline(model_key)
+ result = pipeline(text[:512])
+
+ if isinstance(result, list) and result:
+ result = result[0]
+
+ return {
+ "success": True,
+ "available": True,
+ "model_key": model_key,
+ "model_id": spec.model_id,
+ "task": spec.task,
+ "input": text[:100],
+ "output": result,
+ "timestamp": datetime.now().isoformat()
+ }
+ except ModelNotAvailable as e:
+ return {
+ "success": False,
+ "available": False,
+ "model_key": model_key,
+ "model_id": spec.model_id,
+ "error": str(e),
+ "reason": "model_unavailable"
+ }
+
+ except HTTPException:
+ raise
+ except Exception as e:
+ raise HTTPException(status_code=500, detail=f"Prediction failed: {str(e)}")
+
+
+@app.post("/api/models/batch/predict")
+async def batch_predict(request: Dict[str, Any]):
+ """Batch prediction using multiple models"""
+ try:
+ from ai_models import MODEL_SPECS, _registry, ModelNotAvailable
+
+ texts = request.get("texts", [])
+ model_keys = request.get("models", [])
+
+ if not texts:
+ raise HTTPException(status_code=400, detail="Texts array is required")
+
+ if not model_keys:
+ model_keys = list(MODEL_SPECS.keys())[:5]
+
+ results = []
+ for text in texts:
+ if not text.strip():
+ continue
+
+ text_results = {}
+ for model_key in model_keys:
+ if model_key not in MODEL_SPECS:
+ continue
+
+ try:
+ spec = MODEL_SPECS[model_key]
+ pipeline = _registry.get_pipeline(model_key)
+ result = pipeline(text[:512])
+
+ if isinstance(result, list) and result:
+ result = result[0]
+
+ text_results[model_key] = {
+ "model_id": spec.model_id,
+ "result": result,
+ "success": True
+ }
+ except ModelNotAvailable:
+ text_results[model_key] = {
+ "success": False,
+ "error": "Model not available"
+ }
+ except Exception as e:
+ text_results[model_key] = {
+ "success": False,
+ "error": str(e)
+ }
+
+ results.append({
+ "text": text[:100],
+ "predictions": text_results
+ })
+
+ return {
+ "success": True,
+ "total_texts": len(results),
+ "models_used": model_keys,
+ "results": results,
+ "timestamp": datetime.now().isoformat()
+ }
+
+ except HTTPException:
+ raise
+ except Exception as e:
+ raise HTTPException(status_code=500, detail=f"Batch prediction failed: {str(e)}")
+
+
+@app.post("/api/analyze/text")
+async def analyze_text(request: Dict[str, Any]):
+ """
+ Analyze or generate text using crypto-gpt-o3-mini generation model.
+
+ Expects: { "prompt": "...", "mode": "analysis" | "generation" }
+ Returns: { "text": "...", "model": "OpenC/crypto-gpt-o3-mini" }
+ """
+ try:
+ from ai_models import MODEL_SPECS, _registry, ModelNotAvailable
+
+ prompt = request.get("prompt", "").strip()
+ mode = request.get("mode", "analysis").lower()
+ max_length = request.get("max_length", 200)
+
+ if not prompt:
+ raise HTTPException(status_code=400, detail="Prompt is required")
+
+ # Find generation model (crypto-gpt-o3-mini) - use specific key first
+ generation_key = "crypto_ai_analyst" if "crypto_ai_analyst" in MODEL_SPECS else None
+
+ # Fallback: search by category or model name
+ if not generation_key:
+ for key, spec in MODEL_SPECS.items():
+ if spec.category == "analysis_generation" or "crypto-gpt" in spec.model_id.lower():
+ generation_key = key
+ break
+
+ if not generation_key:
+ return {
+ "success": False,
+ "available": False,
+ "error": "Crypto text generation model not configured",
+ "text": ""
+ }
+
+ try:
+ spec = MODEL_SPECS[generation_key]
+ pipeline = _registry.get_pipeline(generation_key)
+
+ # Generate text
+ result = pipeline(prompt, max_length=max_length, num_return_sequences=1, truncation=True)
+
+ if isinstance(result, list) and result:
+ result = result[0]
+
+ generated_text = result.get("generated_text", str(result))
+
+ return {
+ "success": True,
+ "available": True,
+ "text": generated_text,
+ "model": spec.model_id,
+ "mode": mode,
+ "prompt": prompt[:100],
+ "timestamp": datetime.now().isoformat()
+ }
+
+ except ModelNotAvailable as e:
+ logger.warning(f"Generation model not available: {e}")
+ return {
+ "success": False,
+ "available": False,
+ "error": f"Model not available: {str(e)}",
+ "text": "",
+ "note": "HF model unavailable - check model configuration"
+ }
+
+ except HTTPException:
+ raise
+ except Exception as e:
+ logger.error(f"Text analysis failed: {e}")
+ raise HTTPException(status_code=500, detail=f"Text analysis failed: {str(e)}")
+
+
+@app.post("/api/trading/decision")
+async def trading_decision(request: Dict[str, Any]):
+ """
+ Get trading decision based on sentiment analysis.
+ Uses sentiment analysis to determine BUY/SELL/HOLD signals.
+
+ Expects: { "symbol": "BTC", "context": "market context..." }
+ Returns: {
+ "decision": "BUY" | "SELL" | "HOLD",
+ "confidence": float,
+ "rationale": "explanation",
+ "raw": {...}
+ }
+ """
+ try:
+ from ai_models import analyze_crypto_sentiment
+
+ symbol = request.get("symbol", "").strip().upper()
+ context = request.get("context", "").strip()
+
+ if not symbol:
+ raise HTTPException(status_code=400, detail="Symbol is required")
+
+ # Build text for sentiment analysis
+ if context:
+ analysis_text = f"{symbol} {context}"
+ else:
+ analysis_text = f"{symbol} market analysis"
+
+ # Default response in case of any failure
+ default_response = {
+ "success": True,
+ "available": True,
+ "decision": "HOLD",
+ "confidence": 0.5,
+ "rationale": "Sentiment analysis unavailable - defaulting to HOLD",
+ "symbol": symbol,
+ "model": "fallback",
+ "context_provided": bool(context),
+ "timestamp": datetime.now().isoformat()
+ }
+
+ try:
+ # Analyze sentiment using crypto sentiment model
+ sentiment_result = analyze_crypto_sentiment(analysis_text)
+
+ # Extract sentiment label and confidence
+ sentiment_label = sentiment_result.get("label", "neutral").lower()
+ confidence = sentiment_result.get("confidence", 0.5)
+
+ # Map sentiment to trading decision
+ decision = "HOLD" # Default
+ if sentiment_label == "bullish":
+ decision = "BUY"
+ elif sentiment_label == "bearish":
+ decision = "SELL"
+ else: # neutral or unknown
+ decision = "HOLD"
+
+ # Build rationale
+ rationale = f"Sentiment analysis indicates {sentiment_label} sentiment (confidence: {confidence:.2f})"
+ if context:
+ rationale += f" based on: {context[:200]}"
+
+ return {
+ "success": True,
+ "available": True,
+ "decision": decision,
+ "confidence": float(confidence),
+ "rationale": rationale,
+ "symbol": symbol,
+ "model": sentiment_result.get("engine", "sentiment_analysis"),
+ "sentiment": sentiment_label,
+ "context_provided": bool(context),
+ "raw": sentiment_result,
+ "timestamp": datetime.now().isoformat()
+ }
+
+ except Exception as e:
+ logger.warning(f"Sentiment analysis failed for trading decision: {e}")
+ # Return default HOLD response instead of crashing
+ default_response["error"] = f"Sentiment analysis failed: {str(e)[:100]}"
+ default_response["note"] = "Using default HOLD signal due to analysis failure"
+ return default_response
+
+ except HTTPException:
+ raise
+ except Exception as e:
+ logger.error(f"Trading decision failed: {e}")
+ # Return safe default instead of raising exception
+ return {
+ "success": True,
+ "available": False,
+ "error": f"Trading decision processing failed: {str(e)[:100]}",
+ "decision": "HOLD",
+ "confidence": 0.5,
+ "rationale": "Error occurred during analysis - defaulting to HOLD for safety",
+ "symbol": request.get("symbol", "UNKNOWN"),
+ "timestamp": datetime.now().isoformat()
+ }
+
+
+@app.get("/api/models/data/generated")
+async def get_generated_data(
+ limit: int = 50,
+ model_key: Optional[str] = None,
+ symbol: Optional[str] = None
+):
+ """Get data generated by models from database"""
+ try:
+ conn = sqlite3.connect(str(DB_PATH))
+ cursor = conn.cursor()
+
+ if model_key and symbol:
+ cursor.execute("""
+ SELECT * FROM sentiment_analysis
+ WHERE analysis_type = ? AND symbol = ?
+ ORDER BY timestamp DESC
+ LIMIT ?
+ """, (model_key, symbol.upper(), limit))
+ elif model_key:
+ cursor.execute("""
+ SELECT * FROM sentiment_analysis
+ WHERE analysis_type = ?
+ ORDER BY timestamp DESC
+ LIMIT ?
+ """, (model_key, limit))
+ elif symbol:
+ cursor.execute("""
+ SELECT * FROM sentiment_analysis
+ WHERE symbol = ?
+ ORDER BY timestamp DESC
+ LIMIT ?
+ """, (symbol.upper(), limit))
+ else:
+ cursor.execute("""
+ SELECT * FROM sentiment_analysis
+ ORDER BY timestamp DESC
+ LIMIT ?
+ """, (limit,))
+
+ rows = cursor.fetchall()
+ columns = [desc[0] for desc in cursor.description]
+ conn.close()
+
+ results = []
+ for row in rows:
+ record = dict(zip(columns, row))
+ if record.get("scores"):
+ try:
+ record["scores"] = json.loads(record["scores"])
+ except:
+ pass
+ results.append(record)
+
+ return {
+ "success": True,
+ "count": len(results),
+ "data": results,
+ "source": "models",
+ "timestamp": datetime.now().isoformat()
+ }
+
+ except Exception as e:
+ raise HTTPException(status_code=500, detail=f"Failed to fetch generated data: {str(e)}")
+
+
+@app.get("/api/models/data/stats")
+async def get_models_data_stats():
+ """Get statistics about data generated by models"""
+ try:
+ conn = sqlite3.connect(str(DB_PATH))
+ cursor = conn.cursor()
+
+ cursor.execute("SELECT COUNT(*) FROM sentiment_analysis")
+ total_analyses = cursor.fetchone()[0]
+
+ cursor.execute("SELECT COUNT(DISTINCT symbol) FROM sentiment_analysis WHERE symbol IS NOT NULL")
+ unique_symbols = cursor.fetchone()[0]
+
+ cursor.execute("SELECT COUNT(DISTINCT analysis_type) FROM sentiment_analysis")
+ unique_types = cursor.fetchone()[0]
+
+ cursor.execute("""
+ SELECT sentiment_label, COUNT(*) as count
+ FROM sentiment_analysis
+ GROUP BY sentiment_label
+ """)
+ sentiment_dist = {row[0]: row[1] for row in cursor.fetchall()}
+
+ cursor.execute("""
+ SELECT analysis_type, COUNT(*) as count
+ FROM sentiment_analysis
+ GROUP BY analysis_type
+ """)
+ type_dist = {row[0]: row[1] for row in cursor.fetchall()}
+
+ conn.close()
+
+ return {
+ "success": True,
+ "statistics": {
+ "total_analyses": total_analyses,
+ "unique_symbols": unique_symbols,
+ "unique_model_types": unique_types,
+ "sentiment_distribution": sentiment_dist,
+ "model_type_distribution": type_dist
+ },
+ "timestamp": datetime.now().isoformat()
+ }
+
+ except Exception as e:
+ raise HTTPException(status_code=500, detail=f"Failed to fetch statistics: {str(e)}")
+
+
+@app.post("/api/hf/run-sentiment")
+async def run_hf_sentiment(data: Dict[str, Any]):
+ """Run sentiment analysis using HF models (compatible with UI)"""
+ try:
+ from ai_models import analyze_market_text, ModelNotAvailable
+
+ texts = data.get("texts", [])
+ if isinstance(texts, str):
+ texts = [texts]
+
+ if not texts or not any(t.strip() for t in texts):
+ raise HTTPException(status_code=400, detail="At least one text is required")
+
+ try:
+ all_results = []
+ total_vote = 0.0
+ count = 0
+ models_available = False
+
+ for text in texts:
+ if not text.strip():
+ continue
+
+ result = analyze_market_text(text.strip())
+
+ # Check if models are available
+ if result.get("available", True):
+ models_available = True
+
+ label = result.get("label", "neutral")
+ confidence = result.get("confidence", 0.5)
+
+ vote_score = 0.0
+ if label == "bullish":
+ vote_score = confidence
+ elif label == "bearish":
+ vote_score = -confidence
+
+ total_vote += vote_score
+ count += 1
+
+ all_results.append({
+ "text": text[:100],
+ "label": label,
+ "confidence": confidence,
+ "vote": vote_score,
+ "available": result.get("available", True)
+ })
+
+ avg_vote = total_vote / count if count > 0 else 0.0
+
+ return {
+ "available": models_available,
+ "vote": avg_vote,
+ "results": all_results,
+ "count": count,
+ "average_confidence": sum(r["confidence"] for r in all_results) / len(all_results) if all_results else 0.0
+ }
+
+ except ModelNotAvailable as e:
+ return {
+ "available": False,
+ "vote": 0.0,
+ "results": [],
+ "count": 0,
+ "average_confidence": 0.0,
+ "error": str(e),
+ "reason": "model_unavailable"
+ }
+
+ except HTTPException:
+ raise
+ except Exception as e:
+ raise HTTPException(status_code=500, detail=f"Sentiment analysis failed: {str(e)}")
+
+
+
+
+# ===== Short Hunter / v2 compatibility routes (free Binance + CoinGecko) =====
+try:
+ from api_compat_routes import register_compat_routes
+
+ register_compat_routes(app)
+except Exception as compat_error:
+ logger.warning(f"Compat routes not loaded: {compat_error}")
+
+# ===== Main Entry Point =====
+if __name__ == "__main__":
+ import uvicorn
+ print(f"Starting Crypto Monitor Admin Server on port {PORT}")
+ uvicorn.run(app, host="0.0.0.0", port=PORT, log_level="info")