feat: add glossary placeholder restore

hachimimt-local.zip

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a3511d1f3a5a8e0e44de5cbd24daedc7e9226c3bd9bf0efb0d7688972b48a0d9
-size 136689
+oid sha256:db6a42ac152095b5d31d2a72e7f7ff849df0b7453f7cede80ccba4af9a869193
+size 141638

src/app.py

@@ -47,11 +47,15 @@ _patch_windows_platform()
 import gradio as gr
 
 from glossary import (
+    PLACEHOLDER_SCOPE_DOCUMENT,
+    PLACEHOLDER_SCOPE_LINE,
     GlossaryValidationError,
     apply_glossary_rows,
     compile_glossary,
     glossary_table_rows,
+    protect_glossary_text,
     read_glossary_file,
+    restore_glossary_rows,
     write_glossary_file,
 )
 import hardware
@@ -1516,7 +1520,7 @@ def _layout_lines(text: str) -> list[str]:
     return (text or "").replace("\r\n", "\n").replace("\r", "\n").split("\n")
 
 
-def rebuild_paragraph_layout(
+def rebuild_paragraph_layout(
     source_text: str, rows: list[tuple[int, str, str]]
 ) -> str:
     """Ghép bản dịch của các dòng (rows) trở lại ĐÚNG bố cục dòng của nguồn.
@@ -1535,10 +1539,64 @@ def rebuild_paragraph_layout(
     sentinel = object()
     if next(translated_iter, sentinel) is not sentinel:
         raise RuntimeError("Số row dịch lớn hơn số dòng nguồn không rỗng")
-    return "\n".join(output)
-
-
-def apply_postprocess_rows(
+    return "\n".join(output)
+
+
+def _result_text_from_rows(
+    source_text: str,
+    rows: list[tuple[int, str, str]],
+    backend: str,
+) -> str:
+    if backend == Backend.CT2.value:
+        return rebuild_paragraph_layout(source_text, rows)
+    return "\n".join(translated_vi for _, _, translated_vi in rows)
+
+
+def _retry_glossary_failed_rows(
+    rows: list[tuple[int, str, str]],
+    failed_indices: tuple[int, ...],
+    *,
+    beam_size: int,
+) -> list[tuple[int, str, str]]:
+    """Translate failed placeholder rows again from their original source."""
+    failed = set(failed_indices)
+    source_by_index = {
+        index: source_zh
+        for index, source_zh, _translated_vi in rows
+        if index in failed
+    }
+    missing = failed - set(source_by_index)
+    if missing:
+        raise RuntimeError(
+            "Không tìm thấy dòng nguồn để fallback glossary: "
+            + ", ".join(str(index) for index in sorted(missing))
+        )
+
+    fallback_by_index: dict[int, str] = {}
+    for index in failed_indices:
+        fallback_text: str | None = None
+        for _done, _total, _message, result_rows, result_text in translator.translate_text_iter(
+            source_by_index[index],
+            chunk_mode="paragraph",
+            beam_size=beam_size,
+        ):
+            if result_rows is not None and result_text is not None:
+                fallback_text = result_text
+        if fallback_text is None:
+            raise RuntimeError(f"Fallback glossary không trả kết quả cho dòng {index}.")
+        fallback_by_index[index] = fallback_text.strip()
+
+    return [
+        (
+            index,
+            source_zh,
+            fallback_by_index.get(index, translated_vi),
+        )
+        for index, source_zh, translated_vi in rows
+    ]
+
+
+def apply_postprocess_rows(
     rows: list[tuple[int, str, str]],
     *,
     honorific_kinship: bool,
@@ -1680,11 +1738,22 @@ def _translate_run(
             raise gr.Error(f"Glossary không hợp lệ: {_exception_message(exc)}") from exc
         honorific_kinship = bool(honorific_kinship)
         honorific_pronouns = bool(honorific_pronouns)
-        honorific_on = honorific_kinship or honorific_pronouns
-        pronoun_harmonizer_v9 = bool(pronoun_harmonizer_v9)
-
-        if is_model_downloaded(model_key, backend):
-            load_msg = "Đang nạp model..."
+        honorific_on = honorific_kinship or honorific_pronouns
+        pronoun_harmonizer_v9 = bool(pronoun_harmonizer_v9)
+        placeholder_scope = (
+            PLACEHOLDER_SCOPE_LINE
+            if backend == Backend.CT2.value
+            else PLACEHOLDER_SCOPE_DOCUMENT
+        )
+        glossary_protection = protect_glossary_text(
+            source,
+            glossary_entries,
+            scope=placeholder_scope,
+        )
+        model_source = glossary_protection.text
+
+        if is_model_downloaded(model_key, backend):
+            load_msg = "Đang nạp model..."
         else:
             label = MODELS[model_key].label if model_key in MODELS else model_key
             load_msg = f"Đang tải model {label} từ Hugging Face (lần đầu, vui lòng đợi)..."
@@ -1694,20 +1763,29 @@ def _translate_run(
         resolve_batch_size(auto_batch, manual_batch)
         status = ensure_model(model_key, backend, beam_size)
 
-        set_progress(2, f"{normalize_msg} Đang chia chunk...", session=session)
-        yield _progress_stream_update(session=session)
+        protection_note = (
+            f" Đã bảo vệ {glossary_protection.protected_occurrences} tên/thuật ngữ."
+            if glossary_protection.protected_occurrences
+            else ""
+        )
+        set_progress(
+            2,
+            f"{normalize_msg}{protection_note} Đang chia chunk...",
+            session=session,
+        )
+        yield _progress_stream_update(session=session)
 
         rows: list[tuple[int, str, str]] = []
         full_text = ""
         last_progress_update = 0.0
         translate_start = time.perf_counter()
-        run_id = uuid.uuid4().hex[:8]
-
-        for done, total, message, result_rows, result_text in translator.translate_text_iter(
-            source,
-            chunk_mode=chunk_mode,
-            beam_size=int(beam_size),
-        ):
+        run_id = uuid.uuid4().hex[:8]
+
+        for done, total, message, result_rows, result_text in translator.translate_text_iter(
+            model_source,
+            chunk_mode=chunk_mode,
+            beam_size=int(beam_size),
+        ):
             if result_rows is not None and result_text is not None:
                 rows = result_rows
                 full_text = result_text
@@ -1725,12 +1803,33 @@ def _translate_run(
             pct = round(done / max(total, 1) * 100, 1)
             detail = f"{message} ({pct}%)"
             set_progress(pct, detail, session=session)
-            last_progress_update = now
-            yield _progress_stream_update(session=session)
-
-        translate_seconds = time.perf_counter() - translate_start
-
-        rows, full_text, postprocess_report = apply_postprocess_rows(
+            last_progress_update = now
+            yield _progress_stream_update(session=session)
+
+        translate_profile = dict(translator.last_profile)
+        glossary_restore_report = None
+        glossary_retry_rows = 0
+        if glossary_protection.protected_occurrences:
+            rows, glossary_restore_report = restore_glossary_rows(
+                rows,
+                glossary_protection,
+            )
+            if glossary_restore_report.failed_indices:
+                glossary_retry_rows = glossary_restore_report.failed_rows
+                set_progress(
+                    99,
+                    f"Glossary: dịch lại {glossary_retry_rows} dòng có placeholder lỗi...",
+                    session=session,
+                )
+                yield _progress_stream_update(session=session)
+                rows = _retry_glossary_failed_rows(
+                    rows,
+                    glossary_restore_report.failed_indices,
+                    beam_size=int(beam_size),
+                )
+            full_text = _result_text_from_rows(source, rows, backend)
+
+        rows, full_text, postprocess_report = apply_postprocess_rows(
             rows,
             honorific_kinship=honorific_kinship,
             honorific_pronouns=honorific_pronouns,
@@ -1752,19 +1851,20 @@ def _translate_run(
             try:
                 rows, glossary_report = apply_glossary_rows(rows, glossary_entries)
                 if glossary_report.changed_rows:
-                    full_text = rebuild_paragraph_layout(source, rows)
+                    full_text = _result_text_from_rows(source, rows, backend)
             except Exception as exc:
                 rows = before_glossary_rows
                 full_text = before_glossary_text
                 glossary_warning = _exception_message(exc)
 
+        translate_seconds = time.perf_counter() - translate_start
+
         # KHÔNG auto-tạo file .txt mỗi lần dịch (tránh rác temp) — file chỉ sinh
         # khi user bấm nút "Xuất bản dịch .txt" (export_btn.click). Bản dịch đầy đủ
         # vẫn nằm trong ô + full_text_state để xuất khi cần.
-        download_path = None
-        duration = _format_duration(translate_seconds)
-        translate_profile = translator.last_profile
-        chunk_count = int(translate_profile.get("chunks") or len(rows))
+        download_path = None
+        duration = _format_duration(translate_seconds)
+        chunk_count = int(translate_profile.get("chunks") or len(rows))
         fallback_chunks = int(translate_profile.get("paragraph_fallback_chunks") or 0)
         fallback_lines = int(translate_profile.get("paragraph_fallback_lines") or 0)
         measured_chunks = max(chunk_count + fallback_chunks, 1)
@@ -1801,10 +1901,22 @@ def _translate_run(
             glossary_note = (
                 f" Glossary: {glossary_report.entries} mục, "
                 f"{glossary_report.source_hits} hit nguồn; "
-                f"{glossary_report.replacements} thay thế, "
+                f"{glossary_report.replacements} thay thế alias, "
                 f"{glossary_report.satisfied} đã đúng, "
                 f"{glossary_report.unresolved} chưa khớp alias."
             )
+            if glossary_restore_report is not None:
+                glossary_note += (
+                    f" Placeholder: {glossary_restore_report.protected_occurrences} bảo vệ, "
+                    f"{glossary_restore_report.restored_occurrences} restore"
+                )
+                if glossary_restore_report.skipped_occurrences:
+                    glossary_note += (
+                        f", {glossary_restore_report.skipped_occurrences} vượt pool"
+                    )
+                if glossary_retry_rows:
+                    glossary_note += f", {glossary_retry_rows} dòng retry"
+                glossary_note += "."
         elif glossary_warning:
             glossary_note = (
                 f" Glossary lỗi, đã giữ bản trước glossary: {glossary_warning}."
@@ -2148,9 +2260,11 @@ def build_ui() -> gr.Blocks:
         with gr.Accordion("📚 Tên riêng & thuật ngữ (glossary)", open=False,
                           elem_id="glossary-accordion"):
             gr.Markdown(
-                "Glossary chỉ chuẩn hóa **alias tiếng Việt** khi dòng nguồn chứa đúng "
-                "`source_zh`; không tự chèn tên bị model bỏ sót. "
-                "`Loại` chỉ để quản lý và **không bắt buộc**. Nhiều alias ngăn bằng `|`.",
+                "Glossary thay đúng `source_zh` bằng **placeholder đã kiểm định** trước khi "
+                "Marian dịch, rồi khôi phục thành `target_vi`; vì vậy tên hiếm không phụ "
+                "thuộc vào cách model tự phiên âm. Nếu placeholder bị mất/đổi/lặp, dòng đó "
+                "được dịch lại từ nguồn gốc và mới dùng alias làm fallback. "
+                "`Loại` chỉ để quản lý và **không bắt buộc**.",
                 elem_classes=["honorific-hint"],
             )
             glossary_table = gr.Dataframe(
@@ -2159,7 +2273,7 @@ def build_ui() -> gr.Blocks:
                     "Nguồn Trung *",
                     "Đích Việt *",
                     "Loại (tùy chọn)",
-                    "Alias Việt (ngăn bằng |)",
+                    "Alias fallback (tùy chọn, ngăn bằng |)",
                     "Bật",
                 ],
                 datatype=["str", "str", "str", "str", "bool"],

src/glossary.py

@@ -1,8 +1,9 @@
-"""User glossary parsing and safe post-translation canonicalization.
+"""User glossary parsing, placeholder protection, restore, and alias fallback.
 
-The MVP intentionally does not inject placeholders into model input.  It only
-canonicalizes an explicitly listed Vietnamese alias when the corresponding
-Chinese term is present in the same source row.
+Enabled glossary entries are protected before Marian decoding with a small
+model-tested placeholder pool.  Exact placeholders are restored to the
+canonical Vietnamese target before the existing post-processing pipeline.
+Alias canonicalization remains as a compatibility fallback.
 """
 
 from __future__ import annotations
@@ -19,6 +20,12 @@ from typing import Any
 GLOSSARY_HEADERS = ("source_zh", "target_vi", "type", "aliases_vi", "enabled")
 MAX_GLOSSARY_BYTES = 2_000_000
 MAX_GLOSSARY_ROWS = 5_000
+# Frozen from rare_name_eval_v2: every marker reached 127/127 survival with
+# zero overcopy on all eight CT2 models exposed by qt2 (2026-07-27).
+GLOSSARY_PLACEHOLDERS = ("QX7", "KX7", "RX7", "ZX9", "ZQ1", "VX1")
+PLACEHOLDER_SCOPE_LINE = "line"
+PLACEHOLDER_SCOPE_DOCUMENT = "document"
+PLACEHOLDER_SCOPES = {PLACEHOLDER_SCOPE_LINE, PLACEHOLDER_SCOPE_DOCUMENT}
 
 
 class GlossaryValidationError(ValueError):
@@ -44,6 +51,42 @@ class GlossaryReport:
     unresolved_terms: tuple[str, ...] = ()
 
 
+@dataclass(frozen=True, slots=True)
+class GlossaryPlaceholderBinding:
+    marker: str
+    entry: GlossaryEntry
+    occurrences: int
+
+
+@dataclass(frozen=True, slots=True)
+class GlossaryProtectedRow:
+    index: int
+    original_source: str
+    protected_source: str
+    bindings: tuple[GlossaryPlaceholderBinding, ...]
+
+
+@dataclass(frozen=True, slots=True)
+class GlossaryProtection:
+    text: str
+    rows: tuple[GlossaryProtectedRow, ...]
+    scope: str
+    markers: tuple[str, ...]
+    source_hits: int
+    protected_occurrences: int
+    skipped_occurrences: int
+
+
+@dataclass(frozen=True, slots=True)
+class GlossaryRestoreReport:
+    protected_occurrences: int
+    restored_occurrences: int
+    skipped_occurrences: int
+    failed_rows: int
+    overcopy_rows: int
+    failed_indices: tuple[int, ...] = ()
+
+
 def _clean_text(value: Any) -> str:
     if value is None:
         return ""
@@ -219,6 +262,260 @@ def _source_matches(
     return selected
 
 
+def _source_spans(
+    source_text: str,
+    entries: Sequence[GlossaryEntry],
+) -> list[tuple[int, int, GlossaryEntry]]:
+    """Return non-overlapping source spans, preferring the longest key."""
+    candidates: list[tuple[int, int, int, GlossaryEntry]] = []
+    for entry_index, entry in enumerate(entries):
+        start = source_text.find(entry.source_zh)
+        while start >= 0:
+            end = start + len(entry.source_zh)
+            candidates.append((start, -len(entry.source_zh), entry_index, entry))
+            start = source_text.find(entry.source_zh, start + 1)
+    candidates.sort(key=lambda item: item[:3])
+
+    selected: list[tuple[int, int, GlossaryEntry]] = []
+    occupied_until = -1
+    for start, negative_length, _entry_index, entry in candidates:
+        end = start - negative_length
+        if start < occupied_until:
+            continue
+        selected.append((start, end, entry))
+        occupied_until = end
+    return selected
+
+
+def _line_body_and_ending(raw_line: str) -> tuple[str, str]:
+    for ending in ("\r\n", "\n", "\r"):
+        if raw_line.endswith(ending):
+            return raw_line[: -len(ending)], ending
+    return raw_line, ""
+
+
+def _placeholder_pattern(marker: str) -> re.Pattern[str]:
+    # ASCII boundaries reject mutated forms such as QX7A while still accepting
+    # markers next to Chinese characters in source and normal punctuation in VI.
+    return re.compile(
+        rf"(?<![A-Za-z0-9]){re.escape(marker)}(?![A-Za-z0-9])",
+        re.IGNORECASE,
+    )
+
+
+def protect_glossary_text(
+    text: str,
+    entries: Sequence[GlossaryEntry],
+    *,
+    scope: str = PLACEHOLDER_SCOPE_LINE,
+    markers: Sequence[str] = GLOSSARY_PLACEHOLDERS,
+) -> GlossaryProtection:
+    """Replace matched source terms with model-tested placeholders.
+
+    ``line`` scope reuses the finite marker pool on every non-blank source line
+    and is used by CT2, whose output rows preserve source-line identity.
+    ``document`` scope assigns each marker to one entry for the entire input and
+    is the conservative fallback for backends that return arbitrary chunks.
+    """
+    if scope not in PLACEHOLDER_SCOPES:
+        raise ValueError(f"Placeholder scope không hợp lệ: {scope!r}.")
+
+    marker_pool = tuple(
+        dict.fromkeys(_clean_text(marker) for marker in markers if _clean_text(marker))
+    )
+    collision_values = [text]
+    for entry in entries:
+        collision_values.extend((entry.target_vi, *entry.aliases_vi))
+    marker_pool = tuple(
+        marker
+        for marker in marker_pool
+        if not any(marker.casefold() in value.casefold() for value in collision_values)
+    )
+
+    protected_parts: list[str] = []
+    protected_rows: list[GlossaryProtectedRow] = []
+    source_hits = 0
+    protected_occurrences = 0
+    row_index = 0
+    document_assignments: dict[GlossaryEntry, str] = {}
+
+    for raw_line in text.splitlines(keepends=True):
+        body, ending = _line_body_and_ending(raw_line)
+        if not body.strip():
+            protected_parts.append(raw_line)
+            continue
+
+        row_index += 1
+        spans = _source_spans(body, entries)
+        source_hits += len(spans)
+        if not spans or not marker_pool:
+            protected_parts.append(raw_line)
+            continue
+
+        ordered_entries: list[GlossaryEntry] = []
+        seen_entries: set[GlossaryEntry] = set()
+        for _start, _end, entry in spans:
+            if entry not in seen_entries:
+                ordered_entries.append(entry)
+                seen_entries.add(entry)
+
+        if scope == PLACEHOLDER_SCOPE_DOCUMENT:
+            for entry in ordered_entries:
+                if entry in document_assignments:
+                    continue
+                if len(document_assignments) >= len(marker_pool):
+                    break
+                document_assignments[entry] = marker_pool[len(document_assignments)]
+            assignments = document_assignments
+        else:
+            assignments = {
+                entry: marker
+                for entry, marker in zip(ordered_entries, marker_pool, strict=False)
+            }
+
+        counts: dict[GlossaryEntry, int] = {}
+        protected_body = body
+        for start, end, entry in reversed(spans):
+            marker = assignments.get(entry)
+            if marker is None:
+                continue
+            protected_body = protected_body[:start] + marker + protected_body[end:]
+            counts[entry] = counts.get(entry, 0) + 1
+
+        bindings = tuple(
+            GlossaryPlaceholderBinding(
+                marker=assignments[entry],
+                entry=entry,
+                occurrences=counts[entry],
+            )
+            for entry in ordered_entries
+            if entry in counts
+        )
+        protected_occurrences += sum(binding.occurrences for binding in bindings)
+        if bindings:
+            protected_rows.append(
+                GlossaryProtectedRow(
+                    index=row_index,
+                    original_source=body.strip(),
+                    protected_source=protected_body.strip(),
+                    bindings=bindings,
+                )
+            )
+        protected_parts.append(protected_body + ending)
+
+    return GlossaryProtection(
+        text="".join(protected_parts),
+        rows=tuple(protected_rows),
+        scope=scope,
+        markers=marker_pool,
+        source_hits=source_hits,
+        protected_occurrences=protected_occurrences,
+        skipped_occurrences=source_hits - protected_occurrences,
+    )
+
+
+def restore_glossary_rows(
+    rows: Iterable[tuple[int, str, str]],
+    protection: GlossaryProtection,
+) -> tuple[list[tuple[int, str, str]], GlossaryRestoreReport]:
+    """Restore exact placeholders and fail a whole row on any count mismatch."""
+    protected_by_index = {row.index: row for row in protection.rows}
+    document_entries: dict[str, GlossaryEntry] = {}
+    if protection.scope == PLACEHOLDER_SCOPE_DOCUMENT:
+        for protected_row in protection.rows:
+            for binding in protected_row.bindings:
+                existing = document_entries.get(binding.marker.casefold())
+                if existing is not None and existing != binding.entry:
+                    raise RuntimeError(
+                        f"Placeholder {binding.marker!r} ánh xạ tới nhiều glossary entry."
+                    )
+                document_entries[binding.marker.casefold()] = binding.entry
+
+    output_rows: list[tuple[int, str, str]] = []
+    restored_occurrences = 0
+    failed_indices: list[int] = []
+    overcopy_rows = 0
+    seen_line_indices: set[int] = set()
+
+    for index, source_zh, translated_vi in rows:
+        if protection.scope == PLACEHOLDER_SCOPE_LINE:
+            protected_row = protected_by_index.get(index)
+            if protected_row is None:
+                output_rows.append((index, source_zh, translated_vi))
+                continue
+            seen_line_indices.add(index)
+            bindings = protected_row.bindings
+            source_matches_row = source_zh.strip() == protected_row.protected_source
+        else:
+            bindings = tuple(
+                GlossaryPlaceholderBinding(
+                    marker=marker,
+                    entry=entry,
+                    occurrences=len(_placeholder_pattern(marker).findall(source_zh)),
+                )
+                for marker, entry in (
+                    (marker, document_entries[marker.casefold()])
+                    for marker in protection.markers
+                    if marker.casefold() in document_entries
+                )
+                if _placeholder_pattern(marker).search(source_zh)
+            )
+            if not bindings:
+                output_rows.append((index, source_zh, translated_vi))
+                continue
+            source_matches_row = True
+
+        restored_source = source_zh
+        expected_counts = {binding.marker.casefold(): binding.occurrences for binding in bindings}
+        actual_counts = {
+            marker.casefold(): len(_placeholder_pattern(marker).findall(translated_vi))
+            for marker in protection.markers
+        }
+        count_mismatch = any(
+            actual_counts.get(marker.casefold(), 0) != expected_counts.get(marker.casefold(), 0)
+            for marker in protection.markers
+        )
+        row_overcopy = any(
+            actual_counts.get(marker.casefold(), 0) > expected_counts.get(marker.casefold(), 0)
+            for marker in protection.markers
+        )
+
+        for binding in bindings:
+            restored_source = _placeholder_pattern(binding.marker).sub(
+                lambda _match, value=binding.entry.source_zh: value,
+                restored_source,
+            )
+
+        if not source_matches_row or count_mismatch:
+            failed_indices.append(index)
+            overcopy_rows += int(row_overcopy)
+            output_rows.append((index, restored_source, translated_vi))
+            continue
+
+        restored_vi = translated_vi
+        for binding in bindings:
+            restored_vi = _placeholder_pattern(binding.marker).sub(
+                lambda _match, value=binding.entry.target_vi: value,
+                restored_vi,
+            )
+            restored_occurrences += binding.occurrences
+        output_rows.append((index, restored_source, restored_vi))
+
+    if protection.scope == PLACEHOLDER_SCOPE_LINE:
+        failed_indices.extend(sorted(set(protected_by_index) - seen_line_indices))
+
+    unique_failed_indices = tuple(dict.fromkeys(failed_indices))
+    report = GlossaryRestoreReport(
+        protected_occurrences=protection.protected_occurrences,
+        restored_occurrences=restored_occurrences,
+        skipped_occurrences=protection.skipped_occurrences,
+        failed_rows=len(unique_failed_indices),
+        overcopy_rows=overcopy_rows,
+        failed_indices=unique_failed_indices,
+    )
+    return output_rows, report
+
+
 def _literal_pattern(value: str) -> re.Pattern[str]:
     left = r"(?<!\w)" if value[0].isalnum() or value[0] == "_" else ""
     right = r"(?!\w)" if value[-1].isalnum() or value[-1] == "_" else ""