Fixed community dataset download

marionette/main.py

@@ -2,19 +2,20 @@ from __future__ import annotations
 
 import json
 import logging
+import platform
 import re
 import shutil
 import threading
 import time
 from dataclasses import dataclass
 from datetime import datetime
-import platform
 from pathlib import Path
 from tempfile import TemporaryDirectory
 from textwrap import dedent
 from typing import Any
 
 import numpy as np
+import requests
 from fastapi import HTTPException
 from pydantic import BaseModel, Field
 
@@ -49,6 +50,7 @@ DATASET_REGISTRY_FILENAME = "dataset_registry.json"
 DEFAULT_DATASET_LABEL = "Local dataset"
 COMMUNITY_DATASET_TAG = "reachy_mini_community_moves"
 MAX_COMMUNITY_DATASETS = 50
+HF_DATASETS_API_URL = "https://huggingface.co/api/datasets"
 
 logger = logging.getLogger(__name__)
 
@@ -900,37 +902,58 @@ class Marionette(ReachyMiniApp):
         return entry
 
     def _list_community_datasets(self) -> list[dict[str, Any]]:
-        self._ensure_hf_backend("browse community datasets.")
-        api = HfApi()
-        dataset_filter = None
-        if DatasetFilter is not None:
+        datasets: list[Any] = self._fetch_community_datasets_http()
+        if not datasets and HfApi is not None:
+            dataset_filter = None
+            if DatasetFilter is not None:
+                try:
+                    dataset_filter = DatasetFilter(tags=[COMMUNITY_DATASET_TAG])
+                except Exception:  # pragma: no cover - incompatible hub version
+                    logger.debug("DatasetFilter unavailable, falling back to text search.")
+                    dataset_filter = None
+            search_query = None if dataset_filter else COMMUNITY_DATASET_TAG
+            logger.info("Listing community datasets via HfApi for tag %s", COMMUNITY_DATASET_TAG)
             try:
-                dataset_filter = DatasetFilter(tags=[COMMUNITY_DATASET_TAG])
-            except Exception:  # pragma: no cover - incompatible hub version
-                logger.debug("DatasetFilter unavailable, falling back to search parameter.")
-                dataset_filter = None
+                api = HfApi()
+                datasets = api.list_datasets(
+                    filter=dataset_filter,
+                    search=search_query,
+                    limit=MAX_COMMUNITY_DATASETS,
+                    full=True,
+                )
+            except Exception as exc:  # pragma: no cover - network failure
+                logger.warning("HfApi dataset listing failed: %s", exc)
+                datasets = []
+        if not datasets:
+            raise HTTPException(status_code=502, detail="Unable to list community datasets from Hugging Face.")
 
-        logger.info("Listing community datasets for tag %s", COMMUNITY_DATASET_TAG)
-        try:
-            datasets = api.list_datasets(
-                filter=dataset_filter,
-                search=None if dataset_filter else f"tag:{COMMUNITY_DATASET_TAG}",
-                limit=MAX_COMMUNITY_DATASETS,
-                full=True,
-            )
-        except Exception as exc:  # pragma: no cover - network failure
-            logger.exception("Unable to list community datasets")
-            raise HTTPException(status_code=502, detail=f"Unable to list datasets: {exc}") from exc
+        filtered_items: list[Any] = []
+        for item in datasets:
+            tags = getattr(item, "tags", None) or item.get("tags") if isinstance(item, dict) else []
+            if tags and COMMUNITY_DATASET_TAG in tags:
+                filtered_items.append(item)
+        if not filtered_items:
+            filtered_items = datasets
 
         results = []
-        for item in datasets:
-            repo_id = getattr(item, "id", None) or getattr(item, "repo_id", None)
+        for item in filtered_items:
+            repo_id = (
+                getattr(item, "id", None)
+                or getattr(item, "repo_id", None)
+                or (item.get("id") if isinstance(item, dict) else None)
+                or (item.get("repo_id") if isinstance(item, dict) else None)
+            )
             if not repo_id:
                 continue
-            card = getattr(item, "cardData", None) or {}
+            card = (
+                getattr(item, "cardData", None)
+                or getattr(item, "card_data", None)
+                or (item.get("cardData") if isinstance(item, dict) else None)
+                or {}
+            )
             pretty = card.get("pretty_name") or repo_id
-            description = card.get("short_description") or card.get("description") or ""
-            updated = getattr(item, "lastModified", None)
+            description = card.get("short_description") or card.get("description") or item.get("description", "")
+            updated = getattr(item, "lastModified", None) or item.get("lastModified")
             if isinstance(updated, datetime):
                 updated_str = updated.isoformat()
             else:
@@ -941,15 +964,48 @@ class Marionette(ReachyMiniApp):
                     "repo_id": repo_id,
                     "pretty_name": pretty,
                     "description": description,
-                    "author": getattr(item, "author", None),
-                    "likes": getattr(item, "likes", None),
-                    "downloads": getattr(item, "downloads", None),
+                    "author": getattr(item, "author", None) or item.get("author"),
+                    "likes": getattr(item, "likes", None) or item.get("likes"),
+                    "downloads": getattr(item, "downloads", None) or item.get("downloads"),
                     "last_modified": updated_str,
-                    "tags": getattr(item, "tags", None),
+                    "tags": getattr(item, "tags", None) or item.get("tags"),
                 }
             )
         return results
 
+    def _fetch_community_datasets_http(self) -> list[dict[str, Any]]:
+        base_params = {
+            "limit": MAX_COMMUNITY_DATASETS,
+            "full": "true",
+            "sort": "downloads",
+            "direction": "-1",
+        }
+        attempts = [
+            ("tag_search", {"search": f"tag:{COMMUNITY_DATASET_TAG}"}),
+            ("keyword_search", {"search": COMMUNITY_DATASET_TAG}),
+            ("legacy_filter", {"filter": COMMUNITY_DATASET_TAG}),
+        ]
+        for label, extra_params in attempts:
+            params = base_params.copy()
+            params.update(extra_params)
+            try:
+                logger.debug("HTTP dataset listing (%s) with params %s", label, params)
+                resp = requests.get(
+                    HF_DATASETS_API_URL,
+                    params=params,
+                    timeout=30,
+                )
+                resp.raise_for_status()
+                data = resp.json()
+                if isinstance(data, list) and data:
+                    logger.info("Fetched %d community datasets via HTTP (%s)", len(data), label)
+                    return data
+                if isinstance(data, list):
+                    logger.debug("HTTP dataset listing (%s) returned zero results.", label)
+            except Exception as exc:
+                logger.warning("HTTP dataset listing (%s) failed: %s", label, exc)
+        return []
+
     def _build_hf_readme(
         self,
         dataset_entry: DatasetEntry,

marionette/static/main.js

@@ -670,7 +670,7 @@ function renderCommunityDatasets(datasets) {
   communityList.innerHTML = "";
   if (!datasets.length) {
     const li = document.createElement("li");
-    li.textContent = "No community datasets found yet.";
+    li.textContent = "No community datasets found for #reachy_mini_community_moves.";
     communityList.appendChild(li);
     return;
   }
@@ -756,8 +756,8 @@ async function fetchCommunityDatasets() {
     renderCommunityDatasets(communityDatasets);
     if (communityStatus) {
       communityStatus.textContent = communityDatasets.length
-        ? `Found ${communityDatasets.length} datasets.`
-        : "No datasets published yet.";
+        ? `Found ${communityDatasets.length} dataset${communityDatasets.length === 1 ? "" : "s"} tagged #reachy_mini_community_moves.`
+        : "No datasets tagged #reachy_mini_community_moves yet.";
     }
   } catch (error) {
     console.error("Community fetch error:", error);

pyproject.toml

@@ -12,7 +12,7 @@ requires-python = ">=3.10"
 dependencies = [
     "reachy-mini",
     "soundfile",
-    "huggingface_hub",
+    "huggingface-hub==0.34.4",
 ]
 keywords = ["reachy-mini-app"]