Fix Wikipedia and HF harvest buttons

Wikipedia fix:
- Add User-Agent header to all requests (Wikipedia API returns 403 without it)
- Extract shared _upload_jsonl() helper to reduce code duplication

HF harvest fix:
- Remove audio re-upload approach (500 HTTP requests -> Gradio timeout)
- Instead write dataset references to dataset_sources.jsonl on the Hub
- One small file upload (~instant) instead of gigabytes of audio
- Kaggle notebook Cell 4 now reads dataset_sources.jsonl dynamically:
any dataset registered in Space Self-Teaching tab is automatically
picked up at training time with no notebook edits needed

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

app.py

@@ -748,128 +748,99 @@ def _bulk_upload(lang_label: str, zip_file, csv_text: str) -> str:
 
 # ── Internet self-teaching handlers ───────────────────────────────────────────
 
-def _harvest_wikipedia(lang_label: str, max_articles: int = 100) -> str:
-    """Fetch Wikipedia text for this language and append to vocabulary.jsonl."""
-    if _hf_api is None:
-        return "⚠️ HF_TOKEN not set."
-    lang = SUPPORTED_LANGUAGES.get(lang_label, "bam")
-    if lang not in ("bam", "ful"):
-        return "⚠️ Wikipedia harvest only supported for Bambara and Fula."
-
-    from src.data.web_harvester import harvest_wikipedia_text
-    entries = harvest_wikipedia_text(lang, max_articles=max_articles)
-    if not entries:
-        return "⚠️ No text harvested — check network or try again."
-
-    # Append to vocabulary.jsonl
+def _upload_jsonl(repo_path: str, entries: list[dict]) -> tuple[int, str | None]:
+    """Append entries to a jsonl file on the Hub. Returns (total_lines, error_or_None)."""
     from huggingface_hub import hf_hub_download
     for attempt in range(2):
         try:
             local = hf_hub_download(
-                repo_id=FEEDBACK_REPO_ID, filename="vocabulary.jsonl",
+                repo_id=FEEDBACK_REPO_ID, filename=repo_path,
                 repo_type="dataset", token=HF_TOKEN,
             )
             with open(local, encoding="utf-8") as f:
                 existing = f.read()
         except Exception:
             existing = ""
-        new_lines = "".join(json.dumps(e, ensure_ascii=False) + "\n" for e in entries)
-        updated   = existing + new_lines
+        updated = existing + "".join(json.dumps(e, ensure_ascii=False) + "\n" for e in entries)
         try:
             _hf_api.upload_file(
                 path_or_fileobj=io.BytesIO(updated.encode("utf-8")),
-                path_in_repo="vocabulary.jsonl",
+                path_in_repo=repo_path,
                 repo_id=FEEDBACK_REPO_ID,
                 repo_type="dataset",
             )
-            break
+            return updated.count("\n"), None
         except Exception as e:
             if attempt == 1:
-                return f"❌ Upload failed: {e}"
+                return 0, str(e)
+    return 0, "unknown error"
+
+
+def _harvest_wikipedia(lang_label: str, max_articles: int = 100) -> str:
+    """Fetch Wikipedia text and append to vocabulary.jsonl."""
+    if _hf_api is None:
+        return "⚠️ HF_TOKEN not set."
+    lang = SUPPORTED_LANGUAGES.get(lang_label, "bam")
+    if lang not in ("bam", "ful"):
+        return "⚠️ Supported for Bambara and Fula only."
 
+    try:
+        from src.data.web_harvester import harvest_wikipedia_text
+        entries = harvest_wikipedia_text(lang, max_articles=max_articles)
+    except Exception as e:
+        return f"❌ Harvest error: {e}"
+
+    if not entries:
+        return "⚠️ No sentences extracted. Wikipedia may be temporarily unavailable."
+
+    total, err = _upload_jsonl("vocabulary.jsonl", entries)
+    if err:
+        return f"❌ Upload failed: {err}"
     return (
         f"✅ Wikipedia harvest complete!\n"
-        f"  Language : {lang_label}\n"
-        f"  Sentences added : {len(entries)}\n"
-        f"  Total vocabulary entries: {updated.count(chr(10))}"
+        f"  Language         : {lang_label}\n"
+        f"  Sentences added  : {len(entries)}\n"
+        f"  Vocabulary total : {total} entries"
     )
 
 
 def _harvest_hf_dataset(lang_label: str, max_samples: int = 500) -> str:
-    """Pull audio+transcription from public HF datasets into corrections.jsonl."""
+    """
+    Register an HF dataset as a training source by writing its config to
+    dataset_sources.jsonl on the Hub. The Kaggle notebook reads this file
+    at Cell 4 and loads the dataset directly — no audio re-upload needed.
+    """
     if _hf_api is None:
         return "⚠️ HF_TOKEN not set."
     lang = SUPPORTED_LANGUAGES.get(lang_label, "bam")
     if lang not in ("bam", "ful"):
-        return "⚠️ HF dataset harvest only supported for Bambara and Fula."
+        return "⚠️ Supported for Bambara and Fula only."
 
-    from src.data.web_harvester import harvest_hf_audio, HF_ASR_SOURCES
-    sources = HF_ASR_SOURCES.get(lang, [])
-    if not sources:
+    from src.data.web_harvester import get_hf_dataset_refs
+    refs = get_hf_dataset_refs(lang)
+    if not refs:
         return f"⚠️ No HF dataset configured for {lang}."
 
-    records = []
-    errors  = 0
-    for wav_bytes, text, repo_path in harvest_hf_audio(lang, HF_TOKEN):
-        try:
-            _hf_api.upload_file(
-                path_or_fileobj=io.BytesIO(wav_bytes),
-                path_in_repo=repo_path,
-                repo_id=FEEDBACK_REPO_ID,
-                repo_type="dataset",
-            )
-            ts = repo_path.split("_")[-1].replace(".wav", "")
-            records.append({
-                "id": ts, "timestamp": datetime.now(timezone.utc).isoformat(),
-                "language": lang,
-                "audio_file": repo_path,
-                "transcription": text, "corrected_text": text,
-                "source": f"hf_harvest:{sources[0]['repo']}",
-                "is_correction": False, "model": WHISPER_MODEL_ID,
-            })
-            if len(records) >= max_samples:
-                break
-        except Exception:
-            errors += 1
-            if errors > 20:
-                break
-
-    if not records:
-        return "⚠️ No samples harvested. Dataset may require accepting terms on HuggingFace first."
-
-    # Append to corrections.jsonl
-    from huggingface_hub import hf_hub_download
-    for attempt in range(2):
-        try:
-            local = hf_hub_download(
-                repo_id=FEEDBACK_REPO_ID, filename="corrections.jsonl",
-                repo_type="dataset", token=HF_TOKEN,
-            )
-            with open(local, encoding="utf-8") as f:
-                existing = f.read()
-        except Exception:
-            existing = ""
-        new_lines = "".join(json.dumps(r, ensure_ascii=False) + "\n" for r in records)
-        updated   = existing + new_lines
-        try:
-            _hf_api.upload_file(
-                path_or_fileobj=io.BytesIO(updated.encode("utf-8")),
-                path_in_repo="corrections.jsonl",
-                repo_id=FEEDBACK_REPO_ID,
-                repo_type="dataset",
-            )
-            break
-        except Exception as e:
-            if attempt == 1:
-                return f"❌ corrections.jsonl update failed: {e}"
-
-    total = updated.count("\n")
-    _maybe_auto_trigger()
+    # Write dataset references with user-requested max_samples cap
+    entries = []
+    for ref in refs:
+        entry = dict(ref)
+        entry["max"]       = max_samples
+        entry["enabled"]   = True
+        entry["added_at"]  = datetime.now(timezone.utc).isoformat()
+        entries.append(entry)
+
+    total, err = _upload_jsonl("dataset_sources.jsonl", entries)
+    if err:
+        return f"❌ Upload failed: {err}"
+
+    repos = ", ".join(r["repo"] for r in refs)
     return (
-        f"✅ HF dataset harvest complete!\n"
-        f"  Source   : {sources[0]['repo']}\n"
-        f"  Imported : {len(records)} samples ({errors} errors)\n"
-        f"  Dataset  : {total} total corrections\n"
+        f"✅ Dataset registered for training!\n"
+        f"  Source(s) : {repos}\n"
+        f"  Max samples : {max_samples}\n"
+        f"  The Kaggle notebook will load this dataset directly at training time.\n"
+        f"  Total registered sources: {total}"
     )
 
 

notebooks/kaggle_master_trainer.ipynb

@@ -71,7 +71,9 @@
    "id": "cell-ext-config",
    "metadata": {},
    "outputs": [],
-   "source": "# -- Cell 4: External dataset configuration -----------------------------------\n# Data source reality check (as of 2026):\n#\n#   Bambara ASR:\n#     - google/WaxalNLP  -> no 'bam' subset exists\n#     - Common Voice 'bm' -> moved to Mozilla Data Collective (not on HF)\n#     - PRIMARY SOURCE    -> user corrections in sahel-agri-feedback\n#\n#   Fula ASR:\n#     - google/WaxalNLP 'ful_asr' -> AVAILABLE (handled in Cell 8)\n#     - Common Voice 'ff' -> moved to Mozilla Data Collective (not on HF)\n#\n# To add a new dataset later, add an entry with enabled=True.\n# Any HF dataset with an 'audio' column and a text column works.\n\nEXTERNAL_DATASETS = [\n    # Example -- uncomment and set enabled=True when a Bambara HF dataset appears:\n    # {\n    #     'enabled'    : False,\n    #     'repo_id'    : 'MALIBA-AI/bambara-asr',   # check HF for availability\n    #     'config'     : None,\n    #     'split'      : 'train',\n    #     'text_col'   : 'transcription',\n    #     'lang'       : 'bam',\n    #     'max_samples': 5_000,\n    # },\n]\n\nactive = [d for d in EXTERNAL_DATASETS if d.get('enabled') and d.get('lang') == TRAIN_LANG]\nprint(f'External sources active for {TRAIN_LANG}: {len(active)}')\nif not active:\n    if TRAIN_LANG == 'bam':\n        print('Bambara: no external HF dataset available.')\n        print('  Training will use user corrections from sahel-agri-feedback.')\n        print('  Collect corrections via the Space to grow this dataset over time.')\n    elif TRAIN_LANG == 'ful':\n        print('Fula: WaxalNLP ful_asr loaded in Cell 8 -- no extra external source needed.')"
+   "source": [
+    "# -- Cell 4: External dataset configuration -----------------------------------\n# EXTERNAL_DATASETS is loaded dynamically from dataset_sources.jsonl in the\n# feedback repo. The Space's Self-Teaching tab writes dataset references there\n# when the user clicks \"Import from HuggingFace\". This cell reads that file\n# so any dataset registered in the Space is automatically used here.\n\nimport json as _json\nfrom huggingface_hub import hf_hub_download as _hf_dl\n\nEXTERNAL_DATASETS = []\n\n# -- Load dataset_sources.jsonl from Hub (written by Space Self-Teaching tab) --\ntry:\n    _src_path = _hf_dl(\n        repo_id=FEEDBACK_REPO_ID, filename='dataset_sources.jsonl',\n        repo_type='dataset', token=HF_TOKEN,\n    )\n    with open(_src_path, encoding='utf-8') as _f:\n        for _line in _f:\n            _line = _line.strip()\n            if not _line:\n                continue\n            _entry = _json.loads(_line)\n            if not _entry.get('enabled'):\n                continue\n            # Normalise keys to what Cell 9 expects\n            EXTERNAL_DATASETS.append({\n                'enabled'    : True,\n                'repo_id'    : _entry.get('repo',      _entry.get('repo_id', '')),\n                'config'     : _entry.get('config'),\n                'split'      : _entry.get('split',     'train'),\n                'text_col'   : _entry.get('text_col',  'transcription'),\n                'lang'       : _entry.get('lang',      _entry.get('language', TRAIN_LANG)),\n                'max_samples': _entry.get('max',       _entry.get('max_samples', 2_000)),\n            })\n    print(f'dataset_sources.jsonl: loaded {len(EXTERNAL_DATASETS)} source(s)')\nexcept Exception as _e:\n    print(f'dataset_sources.jsonl not found or empty ({_e}) -- using hardcoded list only')\n\nactive = [d for d in EXTERNAL_DATASETS if d.get('lang') == TRAIN_LANG]\nprint(f'External sources active for {TRAIN_LANG}: {len(active)}')\nfor _d in active:\n    print(f\"  - {_d['repo_id']} / {_d['config']}  (max {_d['max_samples']} samples)\")\nif not active:\n    if TRAIN_LANG == 'bam':\n        print('Bambara: no external source yet.')\n        print('  In the Space -> Self-Teaching tab -> Import from HuggingFace (Bambara).')\n    elif TRAIN_LANG == 'ful':\n        print('Fula: WaxalNLP ful_asr loaded in Cell 8 -- no extra source needed.')\n"
+   ]
   },
   {
    "cell_type": "code",

src/data/web_harvester.py

@@ -5,32 +5,49 @@ into the sahel-agri-feedback HF dataset repo.
 Sources:
   - RobotsMali/jeli-asr         (HF, 33k Bambara audio+text samples)
   - google/fleurs ff_sn          (HF, Fula audio+text)
-  - bm.wikipedia.org / ff.wikipedia.org  (Wikipedia API, text only → vocabulary.jsonl)
+  - bm.wikipedia.org / ff.wikipedia.org  (Wikipedia API, text only -> vocabulary.jsonl)
 
-All writes go through the same corrections.jsonl / vocabulary.jsonl files
-that the Kaggle training notebook reads — no special handling needed.
+HF audio datasets are registered by reference in dataset_sources.jsonl — the
+Kaggle notebook loads them directly at training time.  This avoids re-uploading
+gigabytes of audio through the Space (which would timeout every time).
 """
 from __future__ import annotations
 
 import io
 import json
 import time
-from datetime import datetime, timezone
-from typing import Generator
 
 WIKI_APIS = {
     "bam": "https://bm.wikipedia.org/w/api.php",
     "ful": "https://ff.wikipedia.org/w/api.php",
 }
 
-HF_ASR_SOURCES = {
+# Wikipedia requires a descriptive User-Agent or returns 403
+_UA = "SahelVoiceAI/1.0 (Bambara/Fula language research; huggingface.co/spaces/ous-sow/sahel-agri-voice)"
+
+# Datasets registered by reference — Kaggle notebook loads them directly
+HF_DATASET_REGISTRY = {
     "bam": [
-        {"repo": "RobotsMali/jeli-asr",    "config": "jeli-asr", "split": "train",
-         "audio_col": "audio", "text_col": "bam", "max": 5_000},
+        {
+            "repo":      "RobotsMali/jeli-asr",
+            "config":    "jeli-asr",
+            "split":     "train",
+            "audio_col": "audio",
+            "text_col":  "bam",
+            "max":       5_000,
+            "license":   "cc-by-4.0",
+        },
     ],
     "ful": [
-        {"repo": "google/fleurs",           "config": "ff_sn",    "split": "train",
-         "audio_col": "audio", "text_col": "transcription", "max": 2_000},
+        {
+            "repo":      "google/fleurs",
+            "config":    "ff_sn",
+            "split":     "train",
+            "audio_col": "audio",
+            "text_col":  "transcription",
+            "max":       2_000,
+            "license":   "cc-by-4.0",
+        },
     ],
 }
 
@@ -39,8 +56,8 @@ HF_ASR_SOURCES = {
 
 def harvest_wikipedia_text(lang: str, max_articles: int = 100) -> list[dict]:
     """
-    Fetch up to max_articles article extracts from the language's Wikipedia.
-    Returns list of {word, translation, language} dicts suitable for vocabulary.jsonl.
+    Fetch up to max_articles article extracts from the language Wikipedia.
+    Returns list of {word, translation, language, source} dicts for vocabulary.jsonl.
     """
     import urllib.request, urllib.parse
 
@@ -48,17 +65,15 @@ def harvest_wikipedia_text(lang: str, max_articles: int = 100) -> list[dict]:
     if not api_url:
         return []
 
-    # Step 1: get a list of article titles
-    params = urllib.parse.urlencode({
-        "action": "query",
-        "list": "allpages",
-        "aplimit": max_articles,
-        "apfilterredir": "nonredirects",
-        "format": "json",
-    })
-    with urllib.request.urlopen(f"{api_url}?{params}", timeout=15) as r:
-        data = json.loads(r.read())
+    def _get(params: dict) -> dict:
+        url = f"{api_url}?{urllib.parse.urlencode(params)}"
+        req = urllib.request.Request(url, headers={"User-Agent": _UA})
+        with urllib.request.urlopen(req, timeout=20) as r:
+            return json.loads(r.read())
 
+    # Step 1: get article titles
+    data   = _get({"action": "query", "list": "allpages", "aplimit": max_articles,
+                   "apfilterredir": "nonredirects", "format": "json"})
     titles = [p["title"] for p in data.get("query", {}).get("allpages", [])]
     if not titles:
         return []
@@ -67,105 +82,34 @@ def harvest_wikipedia_text(lang: str, max_articles: int = 100) -> list[dict]:
     entries = []
     for i in range(0, len(titles), 20):
         batch = titles[i:i + 20]
-        params2 = urllib.parse.urlencode({
-            "action": "query",
-            "titles": "|".join(batch),
-            "prop": "extracts",
-            "exsentences": 3,
-            "exlimit": len(batch),
-            "explaintext": True,
-            "format": "json",
-        })
         try:
-            with urllib.request.urlopen(f"{api_url}?{params2}", timeout=15) as r:
-                data2 = json.loads(r.read())
+            data2 = _get({"action": "query", "titles": "|".join(batch),
+                          "prop": "extracts", "exsentences": 3,
+                          "exlimit": len(batch), "explaintext": True, "format": "json"})
             for page in data2.get("query", {}).get("pages", {}).values():
                 extract = (page.get("extract") or "").strip()
                 title   = page.get("title", "").strip()
                 if not extract or not title:
                     continue
-                # Split into sentences, keep those 3–20 words
                 for sentence in extract.replace("\n", " ").split("."):
                     sentence = sentence.strip()
                     words = sentence.split()
                     if 3 <= len(words) <= 20:
                         entries.append({
-                            "word": sentence,
-                            "translation": title,   # use article title as loose context
-                            "language": lang,
-                            "source": "wikipedia",
+                            "word":        sentence,
+                            "translation": title,
+                            "language":    lang,
+                            "source":      "wikipedia",
                         })
         except Exception:
             pass
-        time.sleep(0.3)  # be polite to Wikipedia servers
+        time.sleep(0.3)
 
     return entries
 
 
-# ── HF dataset audio harvest ──────────────────────────────────────────────────
-
-def harvest_hf_audio(
-    lang: str,
-    hf_token: str | None,
-    progress_cb=None,
-) -> Generator[tuple[bytes, str, str], None, None]:
-    """
-    Yield (wav_bytes, transcription, audio_repo_path) for each sample
-    in the configured HF ASR sources for this language.
-
-    Caller is responsible for writing to corrections.jsonl + uploading audio.
-    progress_cb(current, total, message) called periodically if provided.
-    """
-    import numpy as np
-
-    sources = HF_ASR_SOURCES.get(lang, [])
-    for src in sources:
-        try:
-            from datasets import load_dataset, Audio as HFAudio
-        except ImportError:
-            continue
+# ── HF dataset registration (reference-based, no re-upload) ──────────────────
 
-        try:
-            ds = load_dataset(
-                src["repo"], src["config"],
-                split=src["split"],
-                streaming=True,
-                token=hf_token,
-                trust_remote_code=False,
-            )
-            ds = ds.cast_column(src["audio_col"], HFAudio(sampling_rate=16_000))
-            total = src["max"]
-            for i, sample in enumerate(ds.take(total)):
-                if progress_cb:
-                    progress_cb(i + 1, total, f"{src['repo']} ({lang})")
-                try:
-                    audio_arr = np.array(sample[src["audio_col"]]["array"], dtype=np.float32)
-                    text = (sample.get(src["text_col"]) or "").strip()
-                    if not text or len(audio_arr) < 3_200:  # skip < 0.2s
-                        continue
-                    # Convert to WAV bytes
-                    wav_bytes = _numpy_to_wav_bytes(audio_arr, 16_000)
-                    ts = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S_%f")
-                    repo_path = f"audio/{lang}_{ts}.wav"
-                    yield wav_bytes, text, repo_path
-                except Exception:
-                    continue
-        except Exception:
-            continue
-
-
-def _numpy_to_wav_bytes(audio: "np.ndarray", sr: int) -> bytes:
-    import struct, io as _io
-    audio_pcm = (audio * 32767).clip(-32768, 32767).astype("<i2")
-    data_bytes = audio_pcm.tobytes()
-    buf = _io.BytesIO()
-    # WAV header
-    buf.write(b"RIFF")
-    buf.write(struct.pack("<I", 36 + len(data_bytes)))
-    buf.write(b"WAVE")
-    buf.write(b"fmt ")
-    buf.write(struct.pack("<IHHIIHH", 16, 1, 1, sr, sr * 2, 2, 16))
-    buf.write(b"data")
-    buf.write(struct.pack("<I", len(data_bytes)))
-    buf.write(data_bytes)
-    return buf.getvalue()
+def get_hf_dataset_refs(lang: str) -> list[dict]:
+    """Return the dataset reference dicts for this language."""
+    return HF_DATASET_REGISTRY.get(lang, [])