scripts/vendor_modules.py

"""Vendor the avatar's 3D runtime into avatar/vendor/ so no CDN sits in the render path.

esm.sh was the right choice for iteration and the wrong one to ship: a flagship portfolio
Space that renders a blank canvas because a volunteer-run CDN blipped during a recruiter's
visit is an unacceptable failure mode (01-RESEARCH.md § Production hardening). This script
makes vendoring reproducible rather than a one-off download:

1. Fetch each pinned esm.sh ENTRY point and read where it re-exports from - both the
   ``X-Esm-Path`` header and the ``export * from`` line, which must agree. Nothing is
   hardcoded from a guess; the resolved inner path is printed.
2. Download the resolved inner build (the file the browser actually executes today).
3. Rewrite every reference to three.js - absolute ``/three@0.185.1/es2022/three.mjs`` and
   relative ``../../../three.mjs`` alike - to ``./three.mjs``, then assert that NO absolute
   (``https://``) or root-relative (``/``) specifier survives in any of the three files. A
   single missed one silently loads a second three.js instance, and the symptom of that is
   a T-posed statue with no error.
4. Fetch each package's LICENSE text (MIT permits redistribution with the notice kept).
5. Write avatar/vendor/README.md with every file's SHA256, byte size, source URLs, the
   resolution date and the exact command to regenerate.

Re-running produces byte-identical modules; the README keeps its recorded date unless a
hash changed. Run from the repo root::

    .venv/Scripts/python.exe scripts/vendor_modules.py

Stdlib only, on purpose: this must work in a bare interpreter with no project extras.
"""

from __future__ import annotations

import hashlib
import re
import sys
import urllib.request
from datetime import UTC, datetime
from pathlib import Path

REPO_ROOT = Path(__file__).resolve().parent.parent
VENDOR_DIR = REPO_ROOT / "avatar" / "vendor"
README = VENDOR_DIR / "README.md"

ESM = "https://esm.sh"
THREE_VERSION = "0.185.1"
THREE_VRM_VERSION = "3.5.5"

# The one path every vendored module must end up importing three.js from.
LOCAL_THREE = "./three.mjs"
# What esm.sh's builds reference instead. The GLTFLoader build sits three directories
# below the package root and reaches three.mjs relatively; three-vrm is a different
# package and reaches it absolutely.
ABSOLUTE_THREE = f"/three@{THREE_VERSION}/es2022/three.mjs"
RELATIVE_THREE = re.compile(r"(?:\.\./)+three\.mjs")

# name on disk -> the esm.sh entry point that resolves to it. Entry points, not inner
# paths: esm.sh may rebuild an inner path (the three-vrm one carries a build hash), and
# reading it from the entry each time is what keeps this script honest.
MODULES: dict[str, str] = {
    "three.mjs": f"{ESM}/three@{THREE_VERSION}",
    "GLTFLoader.mjs": f"{ESM}/three@{THREE_VERSION}/examples/jsm/loaders/GLTFLoader.js",
    "three-vrm.mjs": f"{ESM}/@pixiv/three-vrm@{THREE_VRM_VERSION}?deps=three@{THREE_VERSION}",
}

# The packages' own LICENSE files, from the npm tarballs as unpkg serves them.
LICENSES: dict[str, str] = {
    "LICENSE-three.txt": f"https://unpkg.com/three@{THREE_VERSION}/LICENSE",
    "LICENSE-three-vrm.txt": f"https://unpkg.com/@pixiv/three-vrm@{THREE_VRM_VERSION}/LICENSE",
}

# Every module specifier, matched STRUCTURALLY - the keyword, the optional binding
# clause, then `from` and the quoted path - rather than by the word `from` alone. esm.sh
# output is minified (no whitespace between tokens, so `\s*` everywhere), and its builds
# contain prose such as `"resized from ("` and `colors from "srgb-linear"` that a loose
# `from\s*["']` scan reports as imports.
STATIC_IMPORT = re.compile(
    r"""\b(?:import|export)\s*"""
    r"""(?:\*\s*as\s+[\w$]+|\{[^}]*\}|\*|[\w$]+(?:\s*,\s*\{[^}]*\})?)?"""
    r"""\s*from\s*["']([^"']+)["']"""
)
SIDE_EFFECT_IMPORT = re.compile(r"""\bimport\s*["']([^"']+)["']""")
DYNAMIC_IMPORT = re.compile(r"""\bimport\s*\(\s*["']([^"']+)["']""")
REEXPORT = re.compile(r"""export\s*\*\s*from\s*["']([^"']+)["']""")


def specifiers(source: str) -> list[str]:
    found: list[str] = []
    for pattern in (STATIC_IMPORT, SIDE_EFFECT_IMPORT, DYNAMIC_IMPORT):
        found.extend(pattern.findall(source))
    return found


def fetch(url: str) -> tuple[bytes, dict[str, str]]:
    req = urllib.request.Request(url, headers={"User-Agent": "japanese-learning-avatar vendoring"})
    with urllib.request.urlopen(req, timeout=60) as resp:  # noqa: S310 - pinned https URLs
        return resp.read(), {k.lower(): v for k, v in resp.headers.items()}


def resolve_inner_path(entry_url: str) -> str:
    """Read the entry point's re-export target and cross-check it with X-Esm-Path."""
    body, headers = fetch(entry_url)
    text = body.decode("utf-8")
    match = REEXPORT.search(text)
    if not match:
        raise SystemExit(f"{entry_url}: no `export * from` line in the entry point:\n{text}")
    from_line = match.group(1)
    from_header = headers.get("x-esm-path")
    if from_header and from_header != from_line:
        raise SystemExit(
            f"{entry_url}: X-Esm-Path {from_header!r} disagrees with the re-export {from_line!r}"
        )
    return from_line


def rewrite_three_imports(source: str) -> str:
    source = source.replace(ABSOLUTE_THREE, LOCAL_THREE)
    return RELATIVE_THREE.sub(LOCAL_THREE, source)


def remote_specifiers(source: str) -> list[str]:
    """Specifiers that would leave the vendor directory: anything not `./<name>`."""
    return [s for s in specifiers(source) if not s.startswith("./")]


def sha256(data: bytes) -> str:
    return hashlib.sha256(data).hexdigest()


def recorded_date(readme_text: str) -> str | None:
    m = re.search(r"^\*\*Resolved:\*\* (\d{4}-\d{2}-\d{2})", readme_text, re.M)
    return m.group(1) if m else None


def recorded_hashes(readme_text: str) -> set[str]:
    return set(re.findall(r"`([0-9a-f]{64})`", readme_text))


def write_readme(rows: list[dict], licenses: list[dict], date: str) -> None:
    total = sum(r["bytes"] for r in rows)
    lines = [
        "# avatar/vendor - the 3D runtime, served by the Space itself",
        "",
        "Generated by `scripts/vendor_modules.py`. **Do not hand-edit the `.mjs` files**:",
        "`tests/test_vendor.py::test_vendored_hashes_match_readme` compares them with the",
        "hashes below, so an edit fails the quick loop until this file is regenerated.",
        "",
        f"**Resolved:** {date}  ",
        f"**Pins:** three@{THREE_VERSION}, @pixiv/three-vrm@{THREE_VRM_VERSION} "
        f"(built against that three)  ",
        f"**Total vendored module bytes:** {total:,}",
        "",
        "Regenerate (from the repo root; re-running is idempotent - byte-identical modules):",
        "",
        "```",
        ".venv/Scripts/python.exe scripts/vendor_modules.py",
        "```",
        "",
        "## Modules",
        "",
        "| File | Bytes | SHA256 | Entry point | Resolved inner build |",
        "|---|---|---|---|---|",
    ]
    for r in rows:
        lines.append(
            f"| `{r['name']}` | {r['bytes']:,} | `{r['sha256']}` | {r['entry']} | "
            f"`{ESM}{r['inner']}` |"
        )
    lines += [
        "",
        "## What was rewritten, and why it matters",
        "",
        f"esm.sh's builds reference three.js as `{ABSOLUTE_THREE}` (three-vrm) and as",
        "`../../../three.mjs` (GLTFLoader). Both are rewritten to `./three.mjs`, so all three",
        "modules share ONE three.js instance served from this directory. The script then asserts",
        "that no `https://` and no root-relative `/` specifier survives in any of the three files:",
        "a single missed one loads a second three.js, `@pixiv/three-vrm`'s `instanceof` checks",
        "fail, `expressionManager` comes back undefined, and the avatar renders as a T-posed",
        "statue with no error in the console. The deployed suite asserts `threeInstanceCount == 1`",
        "for",
        "the same reason; `avatar/vrm-stage.js` counts resource entries whose name includes",
        "`three.mjs`, so the file name above is load-bearing.",
        "",
        "These are esm.sh's es2022 builds of the packages, not the npm tarballs' own module files:",
        "they are byte-for-byte what the browser executed from the CDN before vendoring, so this",
        "change alters where the bytes come from and nothing else.",
        "",
        "## Licences",
        "",
        "| File | Bytes | SHA256 | Source |",
        "|---|---|---|---|",
    ]
    for lic in licenses:
        lines.append(f"| `{lic['name']}` | {lic['bytes']:,} | `{lic['sha256']}` | {lic['url']} |")
    lines += [
        "",
        "three.js and @pixiv/three-vrm are MIT; GLTFLoader is part of three.js and under its",
        "licence. MIT permits redistribution provided the copyright and permission notice is kept,",
        "which is what the two files above are. `LICENSES.md` at the repo root is the project-wide",
        "record and points here.",
        "",
        "## Deliberately NOT vendored: `@huggingface/transformers` (avatar/asr.js)",
        "",
        "`avatar/asr.js` still loads `@huggingface/transformers@4.2.0` from esm.sh, and",
        "`tests/test_vendor.py::test_runtime_has_no_esm_sh` exempts that one file explicitly.",
        "Vendoring the JS shim alone would buy nothing: the library pulls its own multi-megabyte",
        "ONNX Runtime WASM assets and the 135.8 MB Whisper model from the Hugging Face CDN",
        "regardless, so the ASR path depends on a CDN either way - and it is the push-to-talk",
        "path, not the render path. The avatar renders, breathes and speaks typed turns with",
        "esm.sh unreachable; only the microphone would degrade. Making browser ASR CDN-free means",
        "vendoring the WASM runtime and the model, which is a hosting decision for a later phase.",
        "",
    ]
    README.write_text("\n".join(lines), encoding="utf-8", newline="\n")


def main() -> int:
    VENDOR_DIR.mkdir(parents=True, exist_ok=True)
    previous = README.read_text(encoding="utf-8") if README.exists() else ""

    rows: list[dict] = []
    sources: dict[str, str] = {}
    for name, entry in MODULES.items():
        inner = resolve_inner_path(entry)
        print(f"{name}: {entry}\n    -> {inner}")
        body, _ = fetch(f"{ESM}{inner}")
        source = rewrite_three_imports(body.decode("utf-8"))
        sources[name] = source
        rows.append({"name": name, "entry": entry, "inner": inner})

    # The whole point. Checked across every file BEFORE anything is written, so a bad
    # build never lands on disk half-vendored.
    for name, source in sources.items():
        leaks = remote_specifiers(source)
        if leaks:
            raise SystemExit(
                f"{name}: {len(leaks)} specifier(s) still leave the vendor directory: {leaks}"
            )
        if name != "three.mjs" and LOCAL_THREE not in source:
            raise SystemExit(f"{name}: never imports {LOCAL_THREE}; the rewrite missed it")

    for row in rows:
        data = sources[row["name"]].encode("utf-8")
        path = VENDOR_DIR / row["name"]
        path.write_bytes(data)
        row["bytes"] = len(data)
        row["sha256"] = sha256(data)
        print(f"    wrote {path.relative_to(REPO_ROOT)} ({row['bytes']:,} bytes) {row['sha256']}")

    licenses: list[dict] = []
    for name, url in LICENSES.items():
        data, _ = fetch(url)
        text = data.decode("utf-8")
        if "MIT License" not in text and "Permission is hereby granted" not in text:
            raise SystemExit(f"{url} does not look like an MIT licence text:\n{text[:200]}")
        (VENDOR_DIR / name).write_bytes(data)
        licenses.append({"name": name, "url": url, "bytes": len(data), "sha256": sha256(data)})
        print(f"    wrote avatar/vendor/{name} ({len(data):,} bytes)")

    hashes_now = {r["sha256"] for r in rows} | {lic["sha256"] for lic in licenses}
    date = recorded_date(previous)
    if date is None or not hashes_now <= recorded_hashes(previous):
        date = datetime.now(UTC).strftime("%Y-%m-%d")
    write_readme(rows, licenses, date)
    total = sum(r["bytes"] for r in rows)
    print(f"total vendored module bytes: {total:,}; README resolved date {date}")
    return 0


if __name__ == "__main__":
    sys.exit(main())