"""Vendor the avatar's 3D runtime into avatar/vendor/ so no CDN sits in the render path. esm.sh was the right choice for iteration and the wrong one to ship: a flagship portfolio Space that renders a blank canvas because a volunteer-run CDN blipped during a recruiter's visit is an unacceptable failure mode (01-RESEARCH.md ยง Production hardening). This script makes vendoring reproducible rather than a one-off download: 1. Fetch each pinned esm.sh ENTRY point and read where it re-exports from - both the ``X-Esm-Path`` header and the ``export * from`` line, which must agree. Nothing is hardcoded from a guess; the resolved inner path is printed. 2. Download the resolved inner build (the file the browser actually executes today). 3. Rewrite every reference to three.js - absolute ``/three@0.185.1/es2022/three.mjs`` and relative ``../../../three.mjs`` alike - to ``./three.mjs``, then assert that NO absolute (``https://``) or root-relative (``/``) specifier survives in any of the three files. A single missed one silently loads a second three.js instance, and the symptom of that is a T-posed statue with no error. 4. Fetch each package's LICENSE text (MIT permits redistribution with the notice kept). 5. Write avatar/vendor/README.md with every file's SHA256, byte size, source URLs, the resolution date and the exact command to regenerate. Re-running produces byte-identical modules; the README keeps its recorded date unless a hash changed. Run from the repo root:: .venv/Scripts/python.exe scripts/vendor_modules.py Stdlib only, on purpose: this must work in a bare interpreter with no project extras. """ from __future__ import annotations import hashlib import re import sys import urllib.request from datetime import UTC, datetime from pathlib import Path REPO_ROOT = Path(__file__).resolve().parent.parent VENDOR_DIR = REPO_ROOT / "avatar" / "vendor" README = VENDOR_DIR / "README.md" ESM = "https://esm.sh" THREE_VERSION = "0.185.1" THREE_VRM_VERSION = "3.5.5" # The one path every vendored module must end up importing three.js from. LOCAL_THREE = "./three.mjs" # What esm.sh's builds reference instead. The GLTFLoader build sits three directories # below the package root and reaches three.mjs relatively; three-vrm is a different # package and reaches it absolutely. ABSOLUTE_THREE = f"/three@{THREE_VERSION}/es2022/three.mjs" RELATIVE_THREE = re.compile(r"(?:\.\./)+three\.mjs") # name on disk -> the esm.sh entry point that resolves to it. Entry points, not inner # paths: esm.sh may rebuild an inner path (the three-vrm one carries a build hash), and # reading it from the entry each time is what keeps this script honest. MODULES: dict[str, str] = { "three.mjs": f"{ESM}/three@{THREE_VERSION}", "GLTFLoader.mjs": f"{ESM}/three@{THREE_VERSION}/examples/jsm/loaders/GLTFLoader.js", "three-vrm.mjs": f"{ESM}/@pixiv/three-vrm@{THREE_VRM_VERSION}?deps=three@{THREE_VERSION}", } # The packages' own LICENSE files, from the npm tarballs as unpkg serves them. LICENSES: dict[str, str] = { "LICENSE-three.txt": f"https://unpkg.com/three@{THREE_VERSION}/LICENSE", "LICENSE-three-vrm.txt": f"https://unpkg.com/@pixiv/three-vrm@{THREE_VRM_VERSION}/LICENSE", } # Every module specifier, matched STRUCTURALLY - the keyword, the optional binding # clause, then `from` and the quoted path - rather than by the word `from` alone. esm.sh # output is minified (no whitespace between tokens, so `\s*` everywhere), and its builds # contain prose such as `"resized from ("` and `colors from "srgb-linear"` that a loose # `from\s*["']` scan reports as imports. STATIC_IMPORT = re.compile( r"""\b(?:import|export)\s*""" r"""(?:\*\s*as\s+[\w$]+|\{[^}]*\}|\*|[\w$]+(?:\s*,\s*\{[^}]*\})?)?""" r"""\s*from\s*["']([^"']+)["']""" ) SIDE_EFFECT_IMPORT = re.compile(r"""\bimport\s*["']([^"']+)["']""") DYNAMIC_IMPORT = re.compile(r"""\bimport\s*\(\s*["']([^"']+)["']""") REEXPORT = re.compile(r"""export\s*\*\s*from\s*["']([^"']+)["']""") def specifiers(source: str) -> list[str]: found: list[str] = [] for pattern in (STATIC_IMPORT, SIDE_EFFECT_IMPORT, DYNAMIC_IMPORT): found.extend(pattern.findall(source)) return found def fetch(url: str) -> tuple[bytes, dict[str, str]]: req = urllib.request.Request(url, headers={"User-Agent": "japanese-learning-avatar vendoring"}) with urllib.request.urlopen(req, timeout=60) as resp: # noqa: S310 - pinned https URLs return resp.read(), {k.lower(): v for k, v in resp.headers.items()} def resolve_inner_path(entry_url: str) -> str: """Read the entry point's re-export target and cross-check it with X-Esm-Path.""" body, headers = fetch(entry_url) text = body.decode("utf-8") match = REEXPORT.search(text) if not match: raise SystemExit(f"{entry_url}: no `export * from` line in the entry point:\n{text}") from_line = match.group(1) from_header = headers.get("x-esm-path") if from_header and from_header != from_line: raise SystemExit( f"{entry_url}: X-Esm-Path {from_header!r} disagrees with the re-export {from_line!r}" ) return from_line def rewrite_three_imports(source: str) -> str: source = source.replace(ABSOLUTE_THREE, LOCAL_THREE) return RELATIVE_THREE.sub(LOCAL_THREE, source) def remote_specifiers(source: str) -> list[str]: """Specifiers that would leave the vendor directory: anything not `./`.""" return [s for s in specifiers(source) if not s.startswith("./")] def sha256(data: bytes) -> str: return hashlib.sha256(data).hexdigest() def recorded_date(readme_text: str) -> str | None: m = re.search(r"^\*\*Resolved:\*\* (\d{4}-\d{2}-\d{2})", readme_text, re.M) return m.group(1) if m else None def recorded_hashes(readme_text: str) -> set[str]: return set(re.findall(r"`([0-9a-f]{64})`", readme_text)) def write_readme(rows: list[dict], licenses: list[dict], date: str) -> None: total = sum(r["bytes"] for r in rows) lines = [ "# avatar/vendor - the 3D runtime, served by the Space itself", "", "Generated by `scripts/vendor_modules.py`. **Do not hand-edit the `.mjs` files**:", "`tests/test_vendor.py::test_vendored_hashes_match_readme` compares them with the", "hashes below, so an edit fails the quick loop until this file is regenerated.", "", f"**Resolved:** {date} ", f"**Pins:** three@{THREE_VERSION}, @pixiv/three-vrm@{THREE_VRM_VERSION} " f"(built against that three) ", f"**Total vendored module bytes:** {total:,}", "", "Regenerate (from the repo root; re-running is idempotent - byte-identical modules):", "", "```", ".venv/Scripts/python.exe scripts/vendor_modules.py", "```", "", "## Modules", "", "| File | Bytes | SHA256 | Entry point | Resolved inner build |", "|---|---|---|---|---|", ] for r in rows: lines.append( f"| `{r['name']}` | {r['bytes']:,} | `{r['sha256']}` | {r['entry']} | " f"`{ESM}{r['inner']}` |" ) lines += [ "", "## What was rewritten, and why it matters", "", f"esm.sh's builds reference three.js as `{ABSOLUTE_THREE}` (three-vrm) and as", "`../../../three.mjs` (GLTFLoader). Both are rewritten to `./three.mjs`, so all three", "modules share ONE three.js instance served from this directory. The script then asserts", "that no `https://` and no root-relative `/` specifier survives in any of the three files:", "a single missed one loads a second three.js, `@pixiv/three-vrm`'s `instanceof` checks", "fail, `expressionManager` comes back undefined, and the avatar renders as a T-posed", "statue with no error in the console. The deployed suite asserts `threeInstanceCount == 1`", "for", "the same reason; `avatar/vrm-stage.js` counts resource entries whose name includes", "`three.mjs`, so the file name above is load-bearing.", "", "These are esm.sh's es2022 builds of the packages, not the npm tarballs' own module files:", "they are byte-for-byte what the browser executed from the CDN before vendoring, so this", "change alters where the bytes come from and nothing else.", "", "## Licences", "", "| File | Bytes | SHA256 | Source |", "|---|---|---|---|", ] for lic in licenses: lines.append(f"| `{lic['name']}` | {lic['bytes']:,} | `{lic['sha256']}` | {lic['url']} |") lines += [ "", "three.js and @pixiv/three-vrm are MIT; GLTFLoader is part of three.js and under its", "licence. MIT permits redistribution provided the copyright and permission notice is kept,", "which is what the two files above are. `LICENSES.md` at the repo root is the project-wide", "record and points here.", "", "## Deliberately NOT vendored: `@huggingface/transformers` (avatar/asr.js)", "", "`avatar/asr.js` still loads `@huggingface/transformers@4.2.0` from esm.sh, and", "`tests/test_vendor.py::test_runtime_has_no_esm_sh` exempts that one file explicitly.", "Vendoring the JS shim alone would buy nothing: the library pulls its own multi-megabyte", "ONNX Runtime WASM assets and the 135.8 MB Whisper model from the Hugging Face CDN", "regardless, so the ASR path depends on a CDN either way - and it is the push-to-talk", "path, not the render path. The avatar renders, breathes and speaks typed turns with", "esm.sh unreachable; only the microphone would degrade. Making browser ASR CDN-free means", "vendoring the WASM runtime and the model, which is a hosting decision for a later phase.", "", ] README.write_text("\n".join(lines), encoding="utf-8", newline="\n") def main() -> int: VENDOR_DIR.mkdir(parents=True, exist_ok=True) previous = README.read_text(encoding="utf-8") if README.exists() else "" rows: list[dict] = [] sources: dict[str, str] = {} for name, entry in MODULES.items(): inner = resolve_inner_path(entry) print(f"{name}: {entry}\n -> {inner}") body, _ = fetch(f"{ESM}{inner}") source = rewrite_three_imports(body.decode("utf-8")) sources[name] = source rows.append({"name": name, "entry": entry, "inner": inner}) # The whole point. Checked across every file BEFORE anything is written, so a bad # build never lands on disk half-vendored. for name, source in sources.items(): leaks = remote_specifiers(source) if leaks: raise SystemExit( f"{name}: {len(leaks)} specifier(s) still leave the vendor directory: {leaks}" ) if name != "three.mjs" and LOCAL_THREE not in source: raise SystemExit(f"{name}: never imports {LOCAL_THREE}; the rewrite missed it") for row in rows: data = sources[row["name"]].encode("utf-8") path = VENDOR_DIR / row["name"] path.write_bytes(data) row["bytes"] = len(data) row["sha256"] = sha256(data) print(f" wrote {path.relative_to(REPO_ROOT)} ({row['bytes']:,} bytes) {row['sha256']}") licenses: list[dict] = [] for name, url in LICENSES.items(): data, _ = fetch(url) text = data.decode("utf-8") if "MIT License" not in text and "Permission is hereby granted" not in text: raise SystemExit(f"{url} does not look like an MIT licence text:\n{text[:200]}") (VENDOR_DIR / name).write_bytes(data) licenses.append({"name": name, "url": url, "bytes": len(data), "sha256": sha256(data)}) print(f" wrote avatar/vendor/{name} ({len(data):,} bytes)") hashes_now = {r["sha256"] for r in rows} | {lic["sha256"] for lic in licenses} date = recorded_date(previous) if date is None or not hashes_now <= recorded_hashes(previous): date = datetime.now(UTC).strftime("%Y-%m-%d") write_readme(rows, licenses, date) total = sum(r["bytes"] for r in rows) print(f"total vendored module bytes: {total:,}; README resolved date {date}") return 0 if __name__ == "__main__": sys.exit(main())