feat(02-05): analyze(text) - the canonical token record

- analyzer.py composes morphemes -> units -> overrides -> ruby -> ranked
lookup -> derive_level -> kanji axis -> glosses into plain dicts with
exactly TOKEN_KEYS; non-tappable units carry no level, gloss or ruby
- level from the level_key entry, gloss from the lemma entry (one lookup
when they coincide); warmup() reports per-component seconds and sizes
- levels.py uses ruby.is_kanji (the 02-03 parallel-wave copy is gone), so
supplementary-plane kanji reach the kanji axis; test_levels pins it
- 日本語 pinned as N1+: JMdict 1464530 is on none of the five lists

src/japanese_avatar/nlp/analyzer.py

@@ -0,0 +1,160 @@
+"""The ONE canonical text -> tokens function (PITFALLS § 10).
+
+Everything downstream consumes this record and nothing else: furigana (02-07), the lookup
+popover (02-08), the ``analyze`` server function (02-06), Phase 3's level guard and Phase 5's
+vocabulary tracking. Sudachi objects never leave ``tokenizer.py``; this module composes the
+pure functions of plans 02-02 and 02-03 over plain dicts:
+
+    morphemes -> build_units -> apply_overrides -> align_ruby -> lookup -> derive_level
+              -> kanji_levels_for -> glosses_for
+
+No LLM is anywhere on this path (success criterion 1). Warm analysis is milliseconds; the two
+process-wide singletons it touches - the Sudachi core dictionary and the compact JMdict - are
+built once by :func:`warmup`, which app start-up calls so the first visitor does not pay for
+them (the ``voice.tts.warmup`` precedent).
+
+The token record, keys in THIS order (research § Q7; ``TOKEN_KEYS`` is the contract)::
+
+    surface       the text the learner sees and taps (the whole conjugated word, D-05)
+    reading       hiragana, after the committed override table (人気 -> ひとけ before の)
+    lemma         the dictionary form the card names (食べる, 勉強する)
+    level_key     the head's normalized form the level join uses (話せる -> 話す)
+    pos           the head's pos[0] (kept for Phase 3; D-07 hides it on the card)
+    tappable      D-11: particles, auxiliaries, punctuation and copulas are not words
+    is_name       the head is 固有名詞 (D-12)
+    jlpt          "N5".."N1" | "N1+" (on no list, D-10) | "name" (D-12) | None (non-tappable)
+    kanji_levels  {kanji: "N3" | None} for every kanji of the surface, {} when none (D-13)
+    ruby          [[text, rt | None], ...] per kanji run; [[surface, None]] when non-tappable
+    jmdict_id     int | None
+    gloss         up to 3 senses x up to 3 English glosses, [] when there is no entry
+    start, end    character offsets into the analysed text; the surfaces tile it exactly
+
+Level and gloss may come from different entries (research § Q2 step 5): the level from the
+``level_key`` entry (話す, N5) and the gloss from the ``lemma`` entry (話せる, "to be able to
+speak"); when ``level_key == lemma`` they are one lookup.
+"""
+
+from __future__ import annotations
+
+import time
+from collections.abc import Mapping
+
+from japanese_avatar.nlp import jmdict, levels, tokenizer
+from japanese_avatar.nlp.overrides import apply_overrides
+from japanese_avatar.nlp.ruby import align_ruby
+from japanese_avatar.nlp.units import build_units
+
+#: The record every unit carries, in this exact order. Downstream code may rely on it.
+TOKEN_KEYS = (
+    "surface",
+    "reading",
+    "lemma",
+    "level_key",
+    "pos",
+    "tappable",
+    "is_name",
+    "jlpt",
+    "kanji_levels",
+    "ruby",
+    "jmdict_id",
+    "gloss",
+    "start",
+    "end",
+)
+
+
+def _lookup_pair(
+    unit: Mapping[str, object], level_of: Mapping[int, int]
+) -> tuple[jmdict.Entry | None, jmdict.Entry | None]:
+    """``(entry_for_level, entry_for_gloss)`` for a tappable unit.
+
+    The gloss entry is the ``lemma`` (dictionary form) entry - 話せる's own; the level entry is
+    the ``level_key`` (normalized form) entry - 話す. One ranked lookup when the two keys are
+    equal, which is the common case.
+    """
+    lemma = str(unit["lemma"])
+    level_key = str(unit["level_key"])
+    reading = str(unit["reading"])
+    entry_gloss = jmdict.lookup(lemma, lemma, reading, level_of)
+    if level_key == lemma:
+        return entry_gloss, entry_gloss
+    entry_level = jmdict.lookup(level_key, level_key, reading, level_of)
+    return entry_level, entry_gloss
+
+
+def _token(unit: Mapping[str, object], level_of: Mapping[int, int]) -> dict:
+    surface = str(unit["surface"])
+    reading = str(unit["reading"])
+    fields: dict[str, object] = {
+        "surface": surface,
+        "reading": reading,
+        "lemma": unit["lemma"],
+        "level_key": unit["level_key"],
+        "pos": unit["pos"],
+        "tappable": unit["tappable"],
+        "is_name": unit["is_name"],
+        "start": unit["start"],
+        "end": unit["end"],
+    }
+    if unit["tappable"]:
+        entry_level, entry_gloss = _lookup_pair(unit, level_of)
+        entry = entry_gloss or entry_level
+        fields["jlpt"] = levels.derive_level(bool(unit["is_name"]), entry_level, entry_gloss)
+        fields["kanji_levels"] = levels.kanji_levels_for(surface)
+        fields["ruby"] = align_ruby(surface, reading)
+        fields["jmdict_id"] = entry.id if entry is not None else None
+        fields["gloss"] = jmdict.glosses_for(entry) if entry is not None else []
+    else:
+        # D-11: not a word for level purposes - plain text, no badge, no card, no ruby.
+        fields["jlpt"] = None
+        fields["kanji_levels"] = {}
+        fields["ruby"] = [[surface, None]]
+        fields["jmdict_id"] = None
+        fields["gloss"] = []
+    return {key: fields[key] for key in TOKEN_KEYS}
+
+
+def analyze(text: str) -> list[dict]:
+    """Turn any Japanese string into its token records (one plain dict per tap unit).
+
+    Deterministic: the same text gives the same records forever, for the pinned SudachiDict,
+    JMdict and JLPT data - ``tests/fixtures/sentences.json`` is the frozen proof. Returns ``[]``
+    for empty text. Never involves an LLM.
+    """
+    if not text:
+        return []
+    units = build_units(tokenizer.morphemes(text))
+    apply_overrides(units)
+    level_of = levels.vocab_levels()
+    return [_token(unit, level_of) for unit in units]
+
+
+def warmup() -> dict[str, float | int]:
+    """Build every singleton :func:`analyze` needs and report what each cost.
+
+    Returns ``tokenizer_s`` and ``jmdict_s`` (seconds; 0.0 when already built), plus the sizes
+    that prove the data loaded: ``jmdict_entries``, ``vocab_ids`` and ``kanji``. Idempotent.
+    """
+    tokenizer_s = tokenizer.warmup()
+    jmdict_s = jmdict.warmup()
+    vocab = levels.vocab_levels()
+    kanji = levels.kanji_levels()
+    return {
+        "tokenizer_s": tokenizer_s,
+        "jmdict_s": jmdict_s,
+        "jmdict_entries": len(jmdict.load()),
+        "vocab_ids": len(vocab),
+        "kanji": len(kanji),
+    }
+
+
+def _self_time(text: str, runs: int = 100) -> float:  # pragma: no cover - dev aid
+    """Mean milliseconds per warm analysis of ``text`` (used by hand, not by tests)."""
+    analyze(text)
+    started = time.perf_counter()
+    for _ in range(runs):
+        analyze(text)
+    return (time.perf_counter() - started) * 1000 / runs
+
+
+__all__ = ["TOKEN_KEYS", "analyze", "warmup"]

src/japanese_avatar/nlp/levels.py

@@ -28,6 +28,7 @@ import json
 from functools import lru_cache
 
 from japanese_avatar.nlp.jmdict import REPO_ROOT, Entry
+from japanese_avatar.nlp.ruby import is_kanji
 
 JLPT_DIR = REPO_ROOT / "data" / "jlpt"
 KANJI_LEVELS_PATH = JLPT_DIR / "kanji_levels.json"
@@ -99,16 +100,6 @@ def derive_level(
     return BEYOND_LISTS
 
 
-# local copy of ruby.is_kanji - parallel wave; reconciled in 02-05
-_KANJI_RANGES = ((0x3400, 0x4DBF), (0x4E00, 0x9FFF), (0xF900, 0xFAFF), (0x3005, 0x3005))
-
-
-def _is_kanji(ch: str) -> bool:
-    """CJK Unified Ideographs (+ Extension A), Compatibility Ideographs, and 々 (U+3005)."""
-    cp = ord(ch)
-    return any(lo <= cp <= hi for lo, hi in _KANJI_RANGES)
-
-
 def kanji_levels_for(surface: str) -> dict[str, str | None]:
     """Every kanji in ``surface`` -> its level, or ``None`` when the list does not carry it.
 
@@ -118,7 +109,7 @@ def kanji_levels_for(surface: str) -> dict[str, str | None]:
     may itself be listed (known edge, recorded in 02-03-SUMMARY.md).
     """
     table = kanji_levels()
-    return {ch: table.get(ch) for ch in surface if _is_kanji(ch)}
+    return {ch: table.get(ch) for ch in surface if is_kanji(ch)}
 
 
 __all__ = [

tests/test_analyzer.py

@@ -169,7 +169,12 @@ def test_level_derivation(analyzer):
 def test_kanji_axis_separate_from_word_axis(analyzer):
     """D-13: the word's level and each kanji's level are two axes in the same record."""
     nihongo = _by_surface(analyzer("日本語を勉強しています。"), "日本語")
-    assert nihongo["jlpt"] == "N5"
+    # 日本語 resolves to JMdict 1464530, which is on NONE of the five pinned lists (Waller's
+    # lists carry 日本 at N3 and no 日本語 at all) - so the word axis is the honest D-10 "N1+"
+    # while every kanji in it is N5. The two axes really are independent; the plan's assumed
+    # "N5" was a guess the pinned data contradicts (02-05-SUMMARY.md).
+    assert nihongo["jmdict_id"] == 1464530
+    assert nihongo["jlpt"] == "N1+"
     assert nihongo["kanji_levels"] == {"日": "N5", "本": "N5", "語": "N5"}
 
     benkyou = _by_surface(analyzer("日本語を勉強しています。"), "勉強しています")

tests/test_levels.py

@@ -16,7 +16,7 @@ import io
 from collections import defaultdict
 from pathlib import Path
 
-from japanese_avatar.nlp import jmdict, levels
+from japanese_avatar.nlp import jmdict, levels, ruby
 from japanese_avatar.nlp.levels import (
     LEVEL_RANK,
     derive_level,
@@ -132,12 +132,22 @@ def test_kanji_axis():
     assert kanji_levels_for("ｘｙｚ abc 123 、。") == {}
 
 
-def test_is_kanji_ranges_local_copy():
-    """levels.py carries its own _is_kanji (parallel wave with 02-02; 02-05 reconciles)."""
-    assert levels._is_kanji("漢") and levels._is_kanji("々") and levels._is_kanji("㐀")
-    assert levels._is_kanji("鿿") and levels._is_kanji("豈") and levels._is_kanji("﫿")
-    assert not levels._is_kanji("あ") and not levels._is_kanji("ア") and not levels._is_kanji("a")
-    assert not levels._is_kanji("ー") and not levels._is_kanji("。")
+def test_is_kanji_is_rubys():
+    """The kanji-axis predicate IS ruby.is_kanji (02-05 reconciled 02-03's parallel-wave copy).
+
+    One predicate decides which characters receive ruby and which are level-gated, so the two
+    axes can never disagree about a character (e.g. 𠮷 in the supplementary planes).
+    """
+    assert levels.is_kanji is ruby.is_kanji
+    assert not hasattr(levels, "_is_kanji"), "the local copy is gone"
+    assert levels.is_kanji("漢") and levels.is_kanji("々") and levels.is_kanji("㐀")
+    assert levels.is_kanji("鿿") and levels.is_kanji("豈") and levels.is_kanji("﫿")
+    assert levels.is_kanji("𠮷"), "supplementary planes count on both axes"
+    assert not levels.is_kanji("あ") and not levels.is_kanji("ア") and not levels.is_kanji("a")
+    assert not levels.is_kanji("ー") and not levels.is_kanji("。")
+    # A supplementary-plane kanji now reaches the kanji axis (unlisted -> None) instead of
+    # being silently skipped, which the four-range local copy did.
+    assert kanji_levels_for("\U00020bb7野家") == {"\U00020bb7": None, "野": "N4", "家": "N4"}
 
 
 def test_level_rank_order():