feat(02-03): compact JMdict loader with ranked headword lookup

- load(): lru_cache singleton over data/jmdict/jmdict-compact.json.gz; frozen Entry
records with file order; by_id / by_kanji / by_kana indexes (453,630 headword keys)
- lookup(lemma, level_key, reading, level_of): rank (reading match, on JLPT list,
common, -order) so いる->居る, ある->有る, こんにちは via its kana headword
- warmup() and glosses_for(entry, max_senses=3); LFS-pointer guard on load
- measured: without the JLPT term 居る/射る/要る tie and file order picks 射る, so
the empty-level_of test asserts only that 煎る loses

src/japanese_avatar/nlp/jmdict.py

@@ -0,0 +1,190 @@
+"""The compact JMdict as a read-only, process-wide singleton with a ranked headword lookup.
+
+``data/jmdict/jmdict-compact.json.gz`` (plan 02-01) is a positional projection of jmdict-eng
+``3.6.2+20260831182826``: ``[id, kanji_texts, kana_texts, senses, common]`` per entry, in JMdict
+file order. :func:`load` gunzips and parses it once per process (``lru_cache``), builds one
+:class:`Entry` per record and two headword indexes (``by_kanji``, ``by_kana``; a headword may
+name several entries, kept in file order), and hands back a :class:`Lexicon` that nothing
+mutates afterwards. It is therefore safe to share across Gradio sessions: there is no
+module-level mutable state here - the only cache is the one ``lru_cache`` owns on ``load``.
+
+:func:`lookup` is the Sudachi-lemma -> JMdict-entry step. It is a headword match, so homographs
+need a rule: 02-RESEARCH.md § Common Pitfalls 2 measured the naive join picking 煎る for いる,
+或る for ある and 酔う for よう. The rank is ``(reading matches a kana form, id is on a JLPT list,
+JMdict common flag, earlier file order)`` - reading first, JLPT second, common third, and JMdict's
+own order as the LAST resort only (a lowest-id tie-break picked 煎る 1391500 over 居る 1577980).
+
+Importing this module reads nothing; call :func:`warmup` at app start so the first visitor does
+not pay the load (the ``voice.tts.warmup`` precedent).
+"""
+
+from __future__ import annotations
+
+import gzip
+import json
+import time
+from collections.abc import Mapping
+from dataclasses import dataclass
+from functools import lru_cache
+from pathlib import Path
+
+
+def _repo_root() -> Path:
+    """Nearest ancestor holding ``pyproject.toml`` (the checkout), else the CWD."""
+    for parent in Path(__file__).resolve().parents:
+        if (parent / "pyproject.toml").is_file():
+            return parent
+    return Path.cwd()
+
+
+REPO_ROOT = _repo_root()
+COMPACT_PATH = REPO_ROOT / "data" / "jmdict" / "jmdict-compact.json.gz"
+
+#: How many senses :func:`glosses_for` returns by default (D-07: "first two or three senses").
+DEFAULT_MAX_SENSES = 3
+
+
+@dataclass(frozen=True, slots=True)
+class Entry:
+    """One JMdict entry as the compact file records it.
+
+    ``glosses`` is up to 3 senses x up to 3 English glosses; ``order`` is the record's index in
+    the compact file (JMdict file order), used only as the final ranking tie-break.
+    """
+
+    id: int
+    kanji: tuple[str, ...]
+    kana: tuple[str, ...]
+    glosses: tuple[tuple[str, ...], ...]
+    common: bool
+    order: int
+
+
+class Lexicon:
+    """The loaded dictionary: entries by id and the two headword indexes.
+
+    Built once by :func:`load` and never mutated afterwards. ``by_kanji`` and ``by_kana`` map a
+    headword string to every entry that lists it, in file order.
+    """
+
+    __slots__ = ("by_id", "by_kana", "by_kanji", "meta")
+
+    def __init__(self, meta: Mapping[str, object], entries: list[Entry]) -> None:
+        self.meta: Mapping[str, object] = dict(meta)
+        by_id: dict[int, Entry] = {}
+        by_kanji: dict[str, list[Entry]] = {}
+        by_kana: dict[str, list[Entry]] = {}
+        for entry in entries:
+            by_id[entry.id] = entry
+            for text in entry.kanji:
+                by_kanji.setdefault(text, []).append(entry)
+            for text in entry.kana:
+                by_kana.setdefault(text, []).append(entry)
+        self.by_id: Mapping[int, Entry] = by_id
+        self.by_kanji: Mapping[str, list[Entry]] = by_kanji
+        self.by_kana: Mapping[str, list[Entry]] = by_kana
+
+    def __len__(self) -> int:
+        return len(self.by_id)
+
+    def __repr__(self) -> str:  # pragma: no cover - debugging aid
+        return (
+            f"Lexicon(entries={len(self.by_id)}, kanji_keys={len(self.by_kanji)}, "
+            f"kana_keys={len(self.by_kana)})"
+        )
+
+
+def _entry_from_record(order: int, record: list) -> Entry:
+    entry_id, kanji, kana, senses, common = record
+    return Entry(
+        id=int(entry_id),
+        kanji=tuple(kanji),
+        kana=tuple(kana),
+        glosses=tuple(tuple(sense) for sense in senses),
+        common=bool(common),
+        order=order,
+    )
+
+
+@lru_cache(maxsize=1)
+def load() -> Lexicon:
+    """Gunzip, parse and index the compact JMdict once per process.
+
+    Raises ``FileNotFoundError`` with a pointer at ``git lfs pull`` when the file is missing or is
+    still an LFS pointer, because a bare gzip error tells the Space log nothing useful.
+    """
+    if not COMPACT_PATH.is_file():
+        raise FileNotFoundError(
+            f"{COMPACT_PATH} is missing. It is committed through Git LFS; run `git lfs pull` "
+            "or regenerate it with scripts/build_jmdict.py."
+        )
+    with COMPACT_PATH.open("rb") as raw:
+        if raw.read(2) != b"\x1f\x8b":
+            raise FileNotFoundError(
+                f"{COMPACT_PATH} is not a gzip file - it is probably a Git LFS pointer; "
+                "run `git lfs pull`."
+            )
+    with gzip.open(COMPACT_PATH, "rt", encoding="utf-8") as fh:
+        data = json.load(fh)
+    entries = [_entry_from_record(i, record) for i, record in enumerate(data["entries"])]
+    return Lexicon(data.get("meta", {}), entries)
+
+
+def warmup() -> float:
+    """Load the dictionary ahead of the first request. Returns seconds elapsed.
+
+    Idempotent: :func:`load` is cached, so repeat calls cost nothing.
+    """
+    started = time.perf_counter()
+    load()
+    return time.perf_counter() - started
+
+
+def _rank(entry: Entry, reading_hira: str, level_of: Mapping[int, int]) -> tuple:
+    """Higher is better: reading match, on a JLPT list, JMdict common, earlier in the file."""
+    return (reading_hira in entry.kana, entry.id in level_of, entry.common, -entry.order)
+
+
+def lookup(
+    lemma: str,
+    level_key: str,
+    reading_hira: str,
+    level_of: Mapping[int, int],
+) -> Entry | None:
+    """Resolve a Sudachi unit to the JMdict entry the ranking rule picks, or ``None``.
+
+    ``level_key`` (the head's ``normalized_form``: 話せる -> 話す, いい -> 良い) is tried before
+    ``lemma`` (the ``dictionary_form``); both are looked up as kanji and as kana headwords.
+    Candidates are deduplicated by id keeping the first occurrence, then the best by :func:`_rank`
+    wins. ``level_of`` is the JLPT join (``levels.vocab_levels()``); an empty mapping is allowed.
+    """
+    lex = load()
+    seen: set[int] = set()
+    candidates: list[Entry] = []
+    for key in (level_key, lemma):
+        if not key:
+            continue
+        for entry in (*lex.by_kanji.get(key, ()), *lex.by_kana.get(key, ())):
+            if entry.id not in seen:
+                seen.add(entry.id)
+                candidates.append(entry)
+    if not candidates:
+        return None
+    return max(candidates, key=lambda e: _rank(e, reading_hira, level_of))
+
+
+def glosses_for(entry: Entry, max_senses: int = DEFAULT_MAX_SENSES) -> list[list[str]]:
+    """The entry's English glosses as plain lists, capped at ``max_senses`` senses (D-07)."""
+    return [list(sense) for sense in entry.glosses[:max_senses]]
+
+
+__all__ = [
+    "COMPACT_PATH",
+    "DEFAULT_MAX_SENSES",
+    "Entry",
+    "Lexicon",
+    "glosses_for",
+    "load",
+    "lookup",
+    "warmup",
+]

tests/test_jmdict.py

@@ -77,10 +77,12 @@ def test_entries_are_frozen_records(lexicon):
 def test_iru_is_oru_not_iru_roast(lexicon):
     hit = jmdict.lookup("いる", "居る", "いる", {IRU_ORU: 5})
     assert hit is not None and hit.id == IRU_ORU
-    # Without any JLPT knowledge the reading match plus `common` still beats 煎る.
-    bare = jmdict.lookup("いる", "いる", "いる", {})
+    # Without any JLPT knowledge the reading match plus `common` still beats 煎る. Measured:
+    # 居る / 射る / 要る then tie on (reading, common) and file order picks 射る 1322180 - the
+    # JLPT term is what makes 居る win, which is why lookup() takes level_of at all.
+    bare = jmdict.lookup("いる", "居る", "いる", {})
     assert bare is not None and bare.id != IRU_ROAST
-    assert bare.id == IRU_ORU
+    assert "いる" in bare.kana and bare.common
 
 
 def test_aru_naru_kore(lexicon):