feat(02-02): mode-C morphemes to whole-word units (rules A1-A7)

- tokenizer.py: lazy lru_cache Sudachi core singleton, warmup(), plain-dict morphemes()
- units.py: build_units() implementing the A0 guard and A1-A7 over dicts, no Sudachi import
- nlp/__init__.py: extend the package docstring with the boundary discipline

src/japanese_avatar/nlp/__init__.py

@@ -1 +1,8 @@
-"""Japanese language core: tokenizer, dictionary, JLPT levels, translation (Phase 2)."""
+"""Japanese language core: tokenizer, dictionary, JLPT levels, translation (Phase 2).
+
+One canonical ``analyze(text)`` and the pure functions under it. Only ``tokenizer`` touches
+SudachiPy, and only inside its functions, so ``import japanese_avatar.nlp`` constructs nothing and
+stays importable where the wheel is absent. ``units``, ``ruby`` and ``overrides`` are pure
+functions over plain dicts and strings - the ``voice.visemes`` discipline: no engine object
+crosses a function boundary.
+"""

src/japanese_avatar/nlp/tokenizer.py

@@ -0,0 +1,62 @@
+"""SudachiPy tokenizer singleton and the plain-dict morpheme boundary.
+
+Same shape as ``voice.tts``: nothing is built at import time, ``sudachipy`` is imported inside
+the functions, and the dictionary (``sudachidict_core``, memory-mapped, ~0.1-0.2 s to open) is
+created once by :func:`get_tokenizer` and shared read-only across sessions. :func:`warmup` lets
+app startup pay that cost before the first visitor does.
+
+:func:`morphemes` is the only place a Sudachi object is touched. It returns plain dicts so that
+``units.build_units`` and everything downstream is testable without the engine and no Sudachi
+type ever crosses a function boundary.
+"""
+
+from __future__ import annotations
+
+import functools
+import time
+from typing import Any
+
+#: Sudachi's longest split. D-05's tap unit is a whole word, and mode C still emits 食べ|まし|た,
+#: which ``units.build_units`` joins back together.
+_SPLIT_MODE = "C"
+
+
+@functools.lru_cache(maxsize=1)
+def get_tokenizer() -> Any:
+    """The process-wide Sudachi tokenizer over the core dictionary. Read-only; safe to share."""
+    from sudachipy import Dictionary
+
+    return Dictionary(dict="core").create()
+
+
+def warmup() -> float:
+    """Build the tokenizer now and return the seconds it took (0.0 once it is already built)."""
+    t0 = time.perf_counter()
+    get_tokenizer()
+    return time.perf_counter() - t0
+
+
+def morphemes(text: str) -> list[dict]:
+    """Mode-C morphemes of ``text`` as plain dicts, one per Sudachi morpheme, in text order.
+
+    Keys: ``surface``, ``reading`` (``reading_form()``, KATAKANA; punctuation returns itself),
+    ``lemma`` (``dictionary_form()``), ``norm`` (``normalized_form()``: 話せる -> 話す,
+    いい -> 良い), ``pos`` (the 6-tuple ``part_of_speech()`` as a list of str), ``oov``
+    (``is_oov()``).
+    """
+    if not text:
+        return []
+    from sudachipy import SplitMode
+
+    mode = getattr(SplitMode, _SPLIT_MODE)
+    return [
+        {
+            "surface": m.surface(),
+            "reading": m.reading_form(),
+            "lemma": m.dictionary_form(),
+            "norm": m.normalized_form(),
+            "pos": list(m.part_of_speech()),
+            "oov": bool(m.is_oov()),
+        }
+        for m in get_tokenizer().tokenize(text, mode)
+    ]

src/japanese_avatar/nlp/units.py

@@ -0,0 +1,196 @@
+"""Mode-C morphemes -> whole-word tap units (D-05). Pure function over plain dicts, no Sudachi.
+
+Sudachi's longest split still emits 食べ|まし|た; the learner perceives 食べました. This module
+walks the morphemes ``tokenizer.morphemes`` returned and joins inflectional material onto its
+stem with a small, deterministic, POS-driven rule set. A unit has a HEAD (its content morpheme,
+which supplies lemma / level_key / pos / tappable) and a TAIL (its last morpheme, which the
+attach rules inspect).
+
+The joining rule (02-02-PLAN.md, the binding form of 02-RESEARCH.md § Q1). Walk left to right;
+a new unit starts at every morpheme UNLESS one of these attaches it to the previous unit. A0 is
+a guard evaluated BEFORE A1-A7:
+
+A0  The PREVIOUS UNIT is a breaker - head pos[0] in NON_TAPPABLE_POS0, head lemma だ/です, or
+    head pos[0:2] == 形状詞,助動詞語幹 - so NO attach rule fires: breakers accept nothing.
+    きれい|じゃ|なかった, 人気|の|ない, 話せる|よう|に, 行かなけれ|ば|ならない.
+A1  pos[0] == 助動詞, lemma not だ/です, tail pos[0] in {動詞, 形容詞, 助動詞}.
+    ます, た, たい, ない(助動詞), させる, られる, れる, ぬ, う/よう, まい.
+A2  pos[0:2] == 形容詞,非自立可能 and norm == 無い and (tail is 助動詞, or tail is
+    動詞/形容詞 in 未然形/連用形). 食べたく|なかっ|た, 美味しく|ない - the 形容詞 tail
+    is REQUIRED (research's prose omitted it).
+A3  pos[0:2] == 助詞,接続助詞 and surface in {て, で} and tail pos[0] in
+    {動詞, 形容詞, 助動詞}. 食べ|て, 読ん|で; NOT ば / ながら / から.
+A4  pos[0:2] == 動詞,非自立可能 and the tail is an A3-attached て/で.
+    し|て|いる, 読んで|ください, 見て|みる.
+A5  pos[0:2] == 動詞,非自立可能 and lemma == する and the HEAD is 名詞 with
+    pos[2] == サ変可能. 勉強|し|ています -> lemma 勉強する.
+A6  pos[0] == 接尾辞. 田中|さん, 三|人 (the head keeps lemma / level_key).
+A7  The previous morpheme is 接頭辞: the PREFIX joins the NEXT unit (attach forward).
+    お|願い, ご|飯.
+
+Contract: this table plus the expectations in ``tests/test_units.py`` ARE the joining contract.
+Only Sudachi's MORPHEME output (surfaces, readings, POS tuples) is "observed and pinned" -
+never a unit split. If the rule as tabulated ever yields a split that contradicts a test
+expectation, the rule or its implementation is wrong and is fixed; the expectation stands.
+
+Non-tappable units (D-11): head pos[0] in NON_TAPPABLE_POS0, or head 形状詞,助動詞語幹
+(よう / そう / みたい), or head lemma だ/です. Everything else whose head pos[0] is in
+TAPPABLE_POS is tappable, including an OOV numeral (三, norm "3"). Proper nouns are tappable
+with ``is_name`` set (D-12).
+"""
+
+from __future__ import annotations
+
+import jaconv
+
+#: Head POS that make a unit a tappable word (D-05 / D-11).
+TAPPABLE_POS = frozenset(
+    {"名詞", "動詞", "形容詞", "形状詞", "副詞", "代名詞", "連体詞", "接続詞", "感動詞"}
+)
+
+#: Head POS of a plain-text unit (D-11). Such a unit is also a chain breaker (A0).
+NON_TAPPABLE_POS0 = frozenset({"助詞", "助動詞", "補助記号", "空白", "記号"})
+
+#: Copula lemmas: never attach (A1), never tappable, and break the chain (A0).
+CHAIN_BREAKER_LEMMAS = frozenset({"だ", "です"})
+
+#: The 形状詞 auxiliary stems (よう / そう / みたい): plain text and a chain breaker.
+_AUX_STEM = ["形状詞", "助動詞語幹"]
+
+#: Tails that inflectional material (A1, A3) may follow.
+_INFLECTING = frozenset({"動詞", "形容詞", "助動詞"})
+
+_TE_DE = frozenset({"て", "で"})
+
+_A2_TAIL_FORMS = ("未然形", "連用形")
+
+
+def is_breaker(head: dict) -> bool:
+    """A0 / D-11: a unit with this head accepts no attachment and is not tappable."""
+    pos = head["pos"]
+    return (
+        pos[0] in NON_TAPPABLE_POS0 or head["lemma"] in CHAIN_BREAKER_LEMMAS or pos[:2] == _AUX_STEM
+    )
+
+
+class _Unit:
+    """Builder for one unit. Internal; ``build_units`` emits plain dicts."""
+
+    __slots__ = ("head_index", "morphs", "start", "suru", "tail_is_te")
+
+    def __init__(self, start: int) -> None:
+        self.morphs: list[dict] = []
+        self.head_index: int | None = None
+        self.start = start
+        self.tail_is_te = False  # the tail is an A3-attached て/で (enables A4)
+        self.suru = False  # A5 fired: lemma gains する
+
+    @property
+    def head(self) -> dict:
+        idx = 0 if self.head_index is None else self.head_index
+        return self.morphs[idx]
+
+    @property
+    def tail(self) -> dict:
+        return self.morphs[-1]
+
+    @property
+    def pending_prefix(self) -> bool:
+        return self.head_index is None
+
+    def add(self, m: dict, *, as_head: bool = False, rule: str | None = None) -> None:
+        if as_head:
+            self.head_index = len(self.morphs)
+        self.morphs.append(m)
+        self.tail_is_te = rule == "A3"
+        if rule == "A5":
+            self.suru = True
+
+    def to_dict(self) -> dict:
+        head = self.head
+        surface = "".join(m["surface"] for m in self.morphs)
+        reading = jaconv.kata2hira("".join(m["reading"] for m in self.morphs))
+        lemma = head["lemma"] + ("する" if self.suru else "")
+        return {
+            "surface": surface,
+            "reading": reading,
+            "lemma": lemma,
+            "level_key": head["norm"],
+            "pos": head["pos"][0],
+            "tappable": head["pos"][0] in TAPPABLE_POS and not is_breaker(head),
+            "is_name": head["pos"][1] == "固有名詞",
+            "start": self.start,
+            "end": self.start + len(surface),
+            "morpheme_count": len(self.morphs),
+        }
+
+
+def attach_rule(m: dict, unit: _Unit) -> str | None:
+    """The rule (``"A1"``..``"A6"``) that attaches morpheme ``m`` to ``unit``, or ``None``.
+
+    A7 (prefix attaches forward) is handled by the walk in :func:`build_units`, because it is the
+    only rule that looks at the NEXT morpheme rather than the previous unit.
+    """
+    head, tail = unit.head, unit.tail
+    if is_breaker(head):  # A0: breakers accept nothing
+        return None
+    pos = m["pos"]
+    tail_pos = tail["pos"]
+
+    if pos[0] == "助動詞" and m["lemma"] not in CHAIN_BREAKER_LEMMAS and tail_pos[0] in _INFLECTING:
+        return "A1"
+
+    if (
+        pos[:2] == ["形容詞", "非自立可能"]
+        and m["norm"] == "無い"
+        and (
+            tail_pos[0] == "助動詞"
+            or (tail_pos[0] in {"動詞", "形容詞"} and tail_pos[5].startswith(_A2_TAIL_FORMS))
+        )
+    ):
+        return "A2"
+
+    if pos[:2] == ["助詞", "接続助詞"] and m["surface"] in _TE_DE and tail_pos[0] in _INFLECTING:
+        return "A3"
+
+    if pos[:2] == ["動詞", "非自立可能"]:
+        if unit.tail_is_te:
+            return "A4"
+        if m["lemma"] == "する" and head["pos"][0] == "名詞" and head["pos"][2] == "サ変可能":
+            return "A5"
+
+    if pos[0] == "接尾辞":
+        return "A6"
+
+    return None
+
+
+def build_units(morphs: list[dict]) -> list[dict]:
+    """Join mode-C morphemes into whole-word units by rules A0-A7. Input is not mutated.
+
+    Returns one plain dict per unit with keys ``surface``, ``reading`` (hiragana), ``lemma``,
+    ``level_key``, ``pos``, ``tappable``, ``is_name``, ``start``, ``end``, ``morpheme_count``;
+    ``start``/``end`` are character offsets into the text the morphemes came from, and the
+    surfaces tile it exactly.
+    """
+    units: list[_Unit] = []
+    offset = 0
+    for m in morphs:
+        if units and units[-1].pending_prefix:
+            # A7: the previous morpheme(s) were 接頭辞 - this morpheme becomes the head of that
+            # unit (a second prefix in a row keeps the unit pending).
+            units[-1].add(m, as_head=m["pos"][0] != "接頭辞")
+        elif m["pos"][0] == "接頭辞":
+            unit = _Unit(offset)
+            unit.add(m)  # head pending until the next morpheme arrives
+            units.append(unit)
+        else:
+            rule = attach_rule(m, units[-1]) if units else None
+            if rule is None:
+                unit = _Unit(offset)
+                unit.add(m, as_head=True)
+                units.append(unit)
+            else:
+                units[-1].add(m, rule=rule)
+        offset += len(m["surface"])
+    return [u.to_dict() for u in units]