Spaces:
Running
Running
chore(sync): mirror backend .py + Dockerfile to Space (hf-sync-backend)
Browse filesAutomated backend sync from szl-holdings/a11oy main via hf-sync-backend.
Updated (differed from the Space): Dockerfile, a11oy_waqay_nav.py, szl_waqay.py
Deleted (gone from the repo + Dockerfile COPY set): (none)
Keeps the Space-built backend (serve.py + the Dockerfile-COPY'd .py
modules) identical to GitHub main so the Space never rebuilds from a
stale backend, new endpoints don't 404 there, and orphaned modules
removed from the repo don't linger in the Space tree.
- Dockerfile +6 -0
- a11oy_waqay_nav.py +173 -0
- szl_waqay.py +875 -0
Dockerfile
CHANGED
|
@@ -116,6 +116,12 @@ COPY szl_mbse_cosim.py szl_mbse_nav.py ./
|
|
| 116 |
# attaches the idempotent /console nav injector. MUST be COPY'd or serve.py's guarded
|
| 117 |
# imports fall back and /willay 404s. Per-file COPY (this Dockerfile uses no COPY . .).
|
| 118 |
COPY szl_willay_gateway.py a11oy_willay_nav.py ./
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 119 |
# Agentic-PINN + physical-bounds mesh (pure-stdlib sibling of szl_energy_budget; serves
|
| 120 |
# /api/a11oy/v1/pinn/*). MUST be COPY'd or serve.py's guarded import falls back to a stub
|
| 121 |
# (merged-but-not-live) in the HF image. The optional on-metal artifacts it reads
|
|
|
|
| 116 |
# attaches the idempotent /console nav injector. MUST be COPY'd or serve.py's guarded
|
| 117 |
# imports fall back and /willay 404s. Per-file COPY (this Dockerfile uses no COPY . .).
|
| 118 |
COPY szl_willay_gateway.py a11oy_willay_nav.py ./
|
| 119 |
+
# WAQAY — governed quantized vector index (TurboQuant-inspired, signed receipts + Restraint).
|
| 120 |
+
# szl_waqay.py serves /waqay + /api/a11oy/v1/waqay/*; a11oy_waqay_nav.py attaches the
|
| 121 |
+
# idempotent /console nav injector. MUST be COPY'd or serve.py's guarded imports fall back
|
| 122 |
+
# and /waqay 404s. szl_dsse.py / szl_provenance.py / a11oy_org_rag.py already COPYed above.
|
| 123 |
+
# Per-file COPY (this Dockerfile uses no COPY . .).
|
| 124 |
+
COPY szl_waqay.py a11oy_waqay_nav.py ./
|
| 125 |
# Agentic-PINN + physical-bounds mesh (pure-stdlib sibling of szl_energy_budget; serves
|
| 126 |
# /api/a11oy/v1/pinn/*). MUST be COPY'd or serve.py's guarded import falls back to a stub
|
| 127 |
# (merged-but-not-live) in the HF image. The optional on-metal artifacts it reads
|
a11oy_waqay_nav.py
ADDED
|
@@ -0,0 +1,173 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# -*- coding: utf-8 -*-
|
| 2 |
+
# SPDX-License-Identifier: Apache-2.0
|
| 3 |
+
# © 2026 Lutar, Stephen P. — SZL Holdings · ORCID 0009-0001-0110-4173 · Doctrine v11
|
| 4 |
+
# ===========================================================================
|
| 5 |
+
# a11oy_waqay_nav.py — idempotent nav-injection for the WAQAY tab.
|
| 6 |
+
# ---------------------------------------------------------------------------
|
| 7 |
+
# Adds ONE honest left-nav item → /waqay into the /console SPA, plus a small
|
| 8 |
+
# "WAQAY — signed & shown" cross-link strip on the WAQAY page itself. Mirrors
|
| 9 |
+
# the proven a11oy_willay_nav.py / a11oy_nav_wireup.py BaseHTTPMiddleware pattern:
|
| 10 |
+
# • never rewrites the console SPA source (pages/console.html is NOT edited);
|
| 11 |
+
# • only ADDS markup into text/html responses, removes nothing;
|
| 12 |
+
# • idempotent via the data-waqay-nav="q1" marker (re-runs never double-inject);
|
| 13 |
+
# • 0 CDN (pure inline markup, no external assets, no <script>);
|
| 14 |
+
# • 0 user-visible codenames; honest label only.
|
| 15 |
+
#
|
| 16 |
+
# A SEPARATE injector from the QA10 nav-wireup and the WILLAY injector so none
|
| 17 |
+
# collide: QA10 keys on data-nav-wireup="qa10"; WILLAY keys on data-willay-nav="w1";
|
| 18 |
+
# this keys on data-waqay-nav="q1". All can run on the same response harmlessly.
|
| 19 |
+
#
|
| 20 |
+
# Doctrine: locked=8 @ c7c0ba17 · Λ = Conjecture 1 · additive-only · never weakens a gate.
|
| 21 |
+
# Signed-off-by: Stephen P. Lutar Jr. · Co-Authored-By: Perplexity Computer Agent.
|
| 22 |
+
# ===========================================================================
|
| 23 |
+
from typing import Any, Dict
|
| 24 |
+
|
| 25 |
+
# The single honest nav item. Label is the surface's own title — NO codename.
|
| 26 |
+
_WAQAY_PATH = "/waqay"
|
| 27 |
+
_WAQAY_ICO = "\u25C8" # ◈ (the safeguarded store / sealed memory)
|
| 28 |
+
_WAQAY_LABEL = "WAQAY \u2014 Governed Vector Index (signed)"
|
| 29 |
+
|
| 30 |
+
_NAV_MARKER = b'data-waqay-nav="q1"'
|
| 31 |
+
_REL_MARKER = b'data-waqay-rel="q1"'
|
| 32 |
+
|
| 33 |
+
# Sidebar anchors (same as the QA10 / WILLAY injectors).
|
| 34 |
+
_FOOT_ANCHOR = b'<div class="side-foot">'
|
| 35 |
+
_GROUP_ANCHOR = b'<div class="nav-group">'
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def _build_nav_block() -> bytes:
|
| 39 |
+
"""A one-item nav group for WAQAY. Inherits console nav styling (class="nav-item"
|
| 40 |
+
+ <span class="ico">). 0 CDN, 0 <style>, 0 <script>, 0 codenames."""
|
| 41 |
+
item = (
|
| 42 |
+
'<div class="nav-item" data-waqay-nav="q1" data-waqay-path="%s" '
|
| 43 |
+
'onclick="location.href=\'%s\'" style="cursor:pointer">'
|
| 44 |
+
'<span class="ico">%s</span>%s</div>' % (_WAQAY_PATH, _WAQAY_PATH, _WAQAY_ICO, _WAQAY_LABEL)
|
| 45 |
+
)
|
| 46 |
+
block = ('<div class="nav-group" data-waqay-nav="q1">Sovereign Memory (WAQAY)</div>' + item)
|
| 47 |
+
return block.encode("utf-8")
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def _build_rel_strip() -> bytes:
|
| 51 |
+
"""A small honest strip on the /waqay page linking back to related surfaces.
|
| 52 |
+
Inline-styled (0 CDN)."""
|
| 53 |
+
rel = [("/willay", "WILLAY — Safety Gateway"), ("/restraint", "Restraint"),
|
| 54 |
+
("/governance-gateway", "Governance Gateway"), ("/about/thesis", "Thesis / Yachay")]
|
| 55 |
+
links = "".join(
|
| 56 |
+
'<a href="%s" style="color:#39d8c8;text-decoration:none;margin:0 .55em;'
|
| 57 |
+
'white-space:nowrap">%s</a>' % (p, l) for p, l in rel)
|
| 58 |
+
strip = (
|
| 59 |
+
'<nav data-waqay-rel="q1" aria-label="WAQAY related surfaces" '
|
| 60 |
+
'style="margin:1.25rem auto;max-width:1120px;padding:.6rem .9rem;'
|
| 61 |
+
'border-top:1px solid #1c2733;font:13px/1.6 system-ui,sans-serif;'
|
| 62 |
+
'color:#8aa0b4;text-align:center">'
|
| 63 |
+
'<span style="margin-right:.4em">Related sovereign surfaces:</span>' + links + '</nav>')
|
| 64 |
+
return strip.encode("utf-8")
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def _make_injector():
|
| 68 |
+
from starlette.middleware.base import BaseHTTPMiddleware
|
| 69 |
+
from starlette.responses import Response
|
| 70 |
+
|
| 71 |
+
nav_block = _build_nav_block()
|
| 72 |
+
rel_strip = _build_rel_strip()
|
| 73 |
+
|
| 74 |
+
class _WaqayNavInjector(BaseHTTPMiddleware):
|
| 75 |
+
async def dispatch(self, request, call_next):
|
| 76 |
+
resp = await call_next(request)
|
| 77 |
+
try:
|
| 78 |
+
ct = (resp.headers.get("content-type") or "").lower()
|
| 79 |
+
if "text/html" not in ct:
|
| 80 |
+
return resp
|
| 81 |
+
p = request.url.path
|
| 82 |
+
if (p.startswith("/api/") or p.startswith("/v1/")
|
| 83 |
+
or p.startswith("/vendor/") or p.startswith("/assets/")
|
| 84 |
+
or p.startswith("/static/")):
|
| 85 |
+
return resp
|
| 86 |
+
|
| 87 |
+
body = b""
|
| 88 |
+
async for chunk in resp.body_iterator:
|
| 89 |
+
body += chunk if isinstance(chunk, (bytes, bytearray)) else str(chunk).encode()
|
| 90 |
+
|
| 91 |
+
# (1) Nav-item injection — idempotent via _NAV_MARKER, only where
|
| 92 |
+
# the console sidebar markup exists.
|
| 93 |
+
if _NAV_MARKER not in body:
|
| 94 |
+
if _FOOT_ANCHOR in body:
|
| 95 |
+
body = body.replace(_FOOT_ANCHOR, nav_block + _FOOT_ANCHOR, 1)
|
| 96 |
+
elif _GROUP_ANCHOR in body:
|
| 97 |
+
body = body.replace(_GROUP_ANCHOR, _GROUP_ANCHOR + nav_block, 1)
|
| 98 |
+
|
| 99 |
+
# (2) Related strip on the /waqay page only — idempotent.
|
| 100 |
+
if p == _WAQAY_PATH and _REL_MARKER not in body and b"</body>" in body:
|
| 101 |
+
body = body.replace(b"</body>", rel_strip + b"</body>", 1)
|
| 102 |
+
|
| 103 |
+
headers = dict(resp.headers)
|
| 104 |
+
headers.pop("content-length", None)
|
| 105 |
+
return Response(content=body, status_code=resp.status_code,
|
| 106 |
+
headers=headers, media_type="text/html")
|
| 107 |
+
except Exception:
|
| 108 |
+
return resp
|
| 109 |
+
|
| 110 |
+
return _WaqayNavInjector
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
def register(app, ns: str = "a11oy") -> Dict[str, Any]:
|
| 114 |
+
"""Attach the idempotent WAQAY nav injector. ADDITIVE; the console SPA source
|
| 115 |
+
is never edited. try/except-guarded by the caller."""
|
| 116 |
+
app.add_middleware(_make_injector())
|
| 117 |
+
return {
|
| 118 |
+
"registered": ["MIDDLEWARE waqay-nav injector (q1)"],
|
| 119 |
+
"capability": "WAQAY nav wire-up",
|
| 120 |
+
"tab_route": _WAQAY_PATH,
|
| 121 |
+
"data_label": "WAQAY-NAV",
|
| 122 |
+
}
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
# ---------------------------------------------------------------------------
|
| 126 |
+
# Self-test (run: python a11oy_waqay_nav.py) — proves idempotency + additivity.
|
| 127 |
+
# ---------------------------------------------------------------------------
|
| 128 |
+
if __name__ == "__main__":
|
| 129 |
+
from starlette.applications import Starlette
|
| 130 |
+
from starlette.responses import HTMLResponse
|
| 131 |
+
from starlette.routing import Route
|
| 132 |
+
from starlette.testclient import TestClient
|
| 133 |
+
|
| 134 |
+
SAMPLE_CONSOLE = (
|
| 135 |
+
'<html><body><aside>'
|
| 136 |
+
'<div class="nav-group">Operate</div>'
|
| 137 |
+
'<div class="nav-item" onclick="go(\'x\')"><span class="ico">x</span>Existing</div>'
|
| 138 |
+
'<div class="side-foot">footer</div>'
|
| 139 |
+
'</aside><main>x</main></body></html>')
|
| 140 |
+
SAMPLE_WAQAY = '<html><body><h1>WAQAY</h1></body></html>'
|
| 141 |
+
|
| 142 |
+
async def _console(req):
|
| 143 |
+
return HTMLResponse(SAMPLE_CONSOLE)
|
| 144 |
+
|
| 145 |
+
async def _waqay(req):
|
| 146 |
+
return HTMLResponse(SAMPLE_WAQAY)
|
| 147 |
+
|
| 148 |
+
app = Starlette(routes=[Route("/console", _console), Route("/waqay", _waqay)])
|
| 149 |
+
st = register(app, ns="a11oy")
|
| 150 |
+
assert st["tab_route"] == "/waqay", st
|
| 151 |
+
c = TestClient(app)
|
| 152 |
+
|
| 153 |
+
h1 = c.get("/console").text
|
| 154 |
+
h2 = c.get("/console").text
|
| 155 |
+
assert h1 == h2, "console injection must be byte-identical (idempotent)"
|
| 156 |
+
assert h1.count('data-waqay-nav="q1"') == 2, "one group + one item marker"
|
| 157 |
+
assert "location.href='/waqay'" in h1, "nav must link /waqay"
|
| 158 |
+
assert "Existing" in h1 and "Operate</div>" in h1 and "footer</div>" in h1, \
|
| 159 |
+
"must NOT remove existing nav markup"
|
| 160 |
+
|
| 161 |
+
w1 = c.get("/waqay").text
|
| 162 |
+
w2 = c.get("/waqay").text
|
| 163 |
+
assert w1 == w2, "waqay page injection must be idempotent"
|
| 164 |
+
assert w1.count('data-waqay-rel="q1"') == 1, "related strip injects exactly once"
|
| 165 |
+
assert "/willay" in w1, "related strip must cross-link governance surfaces"
|
| 166 |
+
|
| 167 |
+
inj = (_build_nav_block().decode() + _build_rel_strip().decode()).lower()
|
| 168 |
+
assert "http://" not in inj and "https://" not in inj, "nav markup must be 0-CDN"
|
| 169 |
+
assert "<script" not in inj, "nav markup must inject no script"
|
| 170 |
+
for bad in ("amaru", "rosie", "sentra", "jarvis"):
|
| 171 |
+
assert bad not in inj, "no user-visible codenames in WAQAY nav markup"
|
| 172 |
+
print("a11oy_waqay_nav: ALL OK — /waqay nav item injected once; idempotent; "
|
| 173 |
+
"additive; 0 codenames; 0 CDN")
|
szl_waqay.py
ADDED
|
@@ -0,0 +1,875 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# -*- coding: utf-8 -*-
|
| 2 |
+
# SPDX-License-Identifier: Apache-2.0
|
| 3 |
+
# © 2026 Lutar, Stephen P. — SZL Holdings · ORCID 0009-0001-0110-4173 · Doctrine v11
|
| 4 |
+
# Authored by the a11oy Full-Stack Team (WAQAY). Co-Authored-By: Perplexity Computer Agent.
|
| 5 |
+
#
|
| 6 |
+
# WAQAY — Quechua: "to keep / guard / store / safeguard".
|
| 7 |
+
# Lineage: Yachay (knowing) · Chaski (relay) · Khipu (record) · Ayni (reciprocity) ·
|
| 8 |
+
# Ñawi (the eye that sees) · WILLAY (the one that discloses).
|
| 9 |
+
# WAQAY is the one that SAFEGUARDS — the sovereign, governed, compressed memory index.
|
| 10 |
+
#
|
| 11 |
+
# ===========================================================================
|
| 12 |
+
# WAQAY = a GOVERNED, air-gapped, signed quantized vector index for a11oy's RAG.
|
| 13 |
+
# ---------------------------------------------------------------------------
|
| 14 |
+
# WHAT WE STUDIED (open work, made ours):
|
| 15 |
+
# • turbovec — MIT-licensed Rust+Python vector index by Ryan Codrai
|
| 16 |
+
# (github.com/RyanCodrai/turbovec). Implements Google Research's TurboQuant:
|
| 17 |
+
# a DATA-OBLIVIOUS scalar/product quantizer with NO codebook training and NO
|
| 18 |
+
# train phase — supports online ingest. The codebook is computed ANALYTICALLY
|
| 19 |
+
# from the marginal distribution a random rotation induces, not fit to data.
|
| 20 |
+
# • TurboQuant (Google Research) — the data-oblivious quantization approach:
|
| 21 |
+
# normalize → random orthogonal rotation (makes each coordinate of a unit
|
| 22 |
+
# vector follow Beta((d-1)/2,(d-1)/2)) → quantize each coord with a Lloyd-Max
|
| 23 |
+
# codebook fit ANALYTICALLY to that Beta marginal → bit-pack → store a
|
| 24 |
+
# per-vector scale. Search rotates the query and scores against packed codes.
|
| 25 |
+
#
|
| 26 |
+
# WHAT MAKES WAQAY *OURS* (the governed difference — not a vendored crate):
|
| 27 |
+
# 1. PURE PYTHON / NumPy. We do NOT vendor turbovec's Rust crate. HF cpu-basic
|
| 28 |
+
# has no Rust toolchain. We re-implement the TurboQuant *approach* honestly
|
| 29 |
+
# in NumPy. Perf is therefore MODELED/ROADMAP, NEVER claimed to match the
|
| 30 |
+
# Rust SIMD original (see HONESTY below).
|
| 31 |
+
# 2. EVERY index build AND every retrieval emits a DSSE-SIGNED provenance
|
| 32 |
+
# receipt (szl_dsse / szl_provenance) recording: which docs, the quantization
|
| 33 |
+
# params (dim, bits, rotation seed), and a MODELED recall/compression bound.
|
| 34 |
+
# 3. EVERY retrieval passes through the Restraint gate (szl_restraint) so the
|
| 35 |
+
# governed ceiling on the answer is attached and signed.
|
| 36 |
+
# 4. Compression & recall are labeled MODELED bounds — NEVER "perfect recall".
|
| 37 |
+
# Trust is never 100%. Quantization is lossy by construction; we say so.
|
| 38 |
+
#
|
| 39 |
+
# HONESTY (Doctrine v11, Zero-Bandaid Law):
|
| 40 |
+
# • This is a PURE-PYTHON governed index INSPIRED by TurboQuant. We do NOT claim
|
| 41 |
+
# to beat FAISS or to match the Rust SIMD throughput. Any speed/throughput
|
| 42 |
+
# figure shown is labeled MODELED or ROADMAP unless it was MEASURED here.
|
| 43 |
+
# • Compression ratio is MEASURED on the actual bytes WAQAY stores (real).
|
| 44 |
+
# • Recall is a MODELED bound surfaced honestly; on small in-process demos we
|
| 45 |
+
# also report the MEASURED recall@k against an exact float32 baseline so the
|
| 46 |
+
# number shown is real, with the modeled bound stated as the design target.
|
| 47 |
+
# • No network at import time. No key ever committed. 0 runtime CDN.
|
| 48 |
+
#
|
| 49 |
+
# DOCTRINE HARD GATES (this module never violates):
|
| 50 |
+
# • locked theorems = EXACTLY 8 {F1,F4,F7,F11,F12,F18,F19,F22} @ kernel c7c0ba17.
|
| 51 |
+
# • Λ = Conjecture 1 (NOT a closed theorem). Khipu = Conjecture 2.
|
| 52 |
+
# • SLSA L1 honest / L2 roadmap / L3 roadmap.
|
| 53 |
+
# • No user-visible codenames (amaru/rosie/sentra/jarvis). Effectors simulated.
|
| 54 |
+
# • Trust is NEVER 100%: WAQAY recall is a MODELED bound, never claimed perfect.
|
| 55 |
+
# • 0 runtime CDN. Never commit a key. Data labeled LIVE/SAMPLE/MODELED.
|
| 56 |
+
#
|
| 57 |
+
# ATTRIBUTION (see NOTICES.md): MIT — turbovec © 2026 Ryan Codrai; and Google
|
| 58 |
+
# Research's TurboQuant data-oblivious quantization approach. We re-implement the
|
| 59 |
+
# approach; we do not copy the crate. Attribution is required and given.
|
| 60 |
+
# ===========================================================================
|
| 61 |
+
"""szl_waqay — a governed, air-gapped, DSSE-signed quantized vector index.
|
| 62 |
+
|
| 63 |
+
Public API (TurboQuant-shaped, online, no train phase):
|
| 64 |
+
idx = WaqayIndex(dim=256, bit_width=2)
|
| 65 |
+
idx.add(vectors, ids=[...], meta=[...]) # online; no train phase
|
| 66 |
+
scores, ids = idx.search(query, k=10) # approximate top-k
|
| 67 |
+
scores, ids = idx.search(query, k=10, allow=...) # filtered (allowlist/bitmask)
|
| 68 |
+
idx.compression() # MEASURED bytes + ratio
|
| 69 |
+
idx.modeled_recall_bound(bit_width) # MODELED design target
|
| 70 |
+
|
| 71 |
+
Governed entry points (used by the served /waqay tab + org_rag backend):
|
| 72 |
+
build_receipt(idx, doc_ids) -> DSSE-signed index-build provenance receipt
|
| 73 |
+
retrieval_receipt(idx, q, result) -> DSSE-signed retrieval receipt + Restraint verdict
|
| 74 |
+
governed_search(idx, query, k, ..) -> {result, restraint, signed_receipt}
|
| 75 |
+
|
| 76 |
+
Mount/registration for a11oy is in serve.py via register(app, ns); the served tab
|
| 77 |
+
HTML + API routes live at the bottom of this module (register()).
|
| 78 |
+
"""
|
| 79 |
+
from __future__ import annotations
|
| 80 |
+
|
| 81 |
+
import base64
|
| 82 |
+
import hashlib
|
| 83 |
+
import json
|
| 84 |
+
import math
|
| 85 |
+
import struct
|
| 86 |
+
import time
|
| 87 |
+
from typing import Any, Dict, List, Optional, Sequence, Tuple
|
| 88 |
+
|
| 89 |
+
import numpy as np
|
| 90 |
+
|
| 91 |
+
# Request type for the served route handlers. FastAPI recognizes fastapi.Request
|
| 92 |
+
# (== starlette Request) for query/body access; imported at MODULE scope so
|
| 93 |
+
# FastAPI's type-hint introspection resolves the route signatures correctly.
|
| 94 |
+
try:
|
| 95 |
+
from fastapi import Request as Request # type: ignore
|
| 96 |
+
except Exception: # pragma: no cover
|
| 97 |
+
from starlette.requests import Request as Request # type: ignore
|
| 98 |
+
|
| 99 |
+
# NumPy 2.0 renamed trapz -> trapezoid; support both (HF cpu-basic robustness).
|
| 100 |
+
_TRAPZ = getattr(np, "trapezoid", getattr(np, "trapz", None))
|
| 101 |
+
|
| 102 |
+
# Deterministic rotation seed (mirrors turbovec's fixed ROTATION_SEED idea so an
|
| 103 |
+
# index is reproducible and air-gapped — no per-build randomness leaks in).
|
| 104 |
+
ROTATION_SEED = 0x5A4C_5741_5141_5900 # "SZLWAQAY\0" flavoured constant
|
| 105 |
+
|
| 106 |
+
# Doctrine constants surfaced by the tab / receipts.
|
| 107 |
+
LOCKED_THEOREMS = ("F1", "F4", "F7", "F11", "F12", "F18", "F19", "F22")
|
| 108 |
+
KERNEL = "c7c0ba17"
|
| 109 |
+
TRUST_CEILING = 0.99 # never 1.0 — recall is a MODELED bound, never perfect.
|
| 110 |
+
|
| 111 |
+
DOCTRINE = {
|
| 112 |
+
"name_meaning": "WAQAY (Quechua): to keep / guard / store / safeguard.",
|
| 113 |
+
"lineage": ["Yachay", "Chaski", "Khipu", "Ayni", "Ñawi", "WILLAY"],
|
| 114 |
+
"locked_theorems": list(LOCKED_THEOREMS),
|
| 115 |
+
"locked_count": len(LOCKED_THEOREMS), # EXACTLY 8 — never 5.
|
| 116 |
+
"kernel": KERNEL,
|
| 117 |
+
"lambda": "Conjecture 1 (open)",
|
| 118 |
+
"khipu": "Conjecture 2 (open)",
|
| 119 |
+
"slsa": "L1 honest · L2 roadmap · L3 roadmap",
|
| 120 |
+
"trust_ceiling": TRUST_CEILING,
|
| 121 |
+
"honesty": ("Pure-Python governed index INSPIRED by TurboQuant (turbovec, MIT). "
|
| 122 |
+
"Perf is MODELED/ROADMAP, not claimed to match the Rust SIMD original. "
|
| 123 |
+
"Compression is MEASURED; recall is a MODELED bound (never perfect)."),
|
| 124 |
+
"attribution": ("turbovec © 2026 Ryan Codrai (MIT); Google Research TurboQuant "
|
| 125 |
+
"data-oblivious quantization approach. See NOTICES.md."),
|
| 126 |
+
}
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
# ===========================================================================
|
| 130 |
+
# CODEBOOK — Lloyd-Max scalar quantizer fit ANALYTICALLY to the Beta marginal.
|
| 131 |
+
# This is the load-bearing "data-oblivious, NO train phase" property: the
|
| 132 |
+
# codebook depends ONLY on (dim, bits), never on ingested data. (Re-implements
|
| 133 |
+
# turbovec/src/codebook.rs::lloyd_max in NumPy.)
|
| 134 |
+
# ===========================================================================
|
| 135 |
+
def _beta_cdf(x: np.ndarray, a: float) -> np.ndarray:
|
| 136 |
+
"""Regularized incomplete beta I_x(a,a) via a continued fraction (Lentz),
|
| 137 |
+
no SciPy dependency (HF cpu-basic friendly). Symmetric Beta(a,a) on [0,1]."""
|
| 138 |
+
x = np.clip(np.asarray(x, dtype=np.float64), 0.0, 1.0)
|
| 139 |
+
out = np.empty_like(x)
|
| 140 |
+
for i, xi in enumerate(x.ravel()):
|
| 141 |
+
out.ravel()[i] = _betai(a, a, float(xi))
|
| 142 |
+
return out
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
def _betai(a: float, b: float, x: float) -> float:
|
| 146 |
+
if x <= 0.0:
|
| 147 |
+
return 0.0
|
| 148 |
+
if x >= 1.0:
|
| 149 |
+
return 1.0
|
| 150 |
+
lbeta = math.lgamma(a) + math.lgamma(b) - math.lgamma(a + b)
|
| 151 |
+
front = math.exp(math.log(x) * a + math.log(1.0 - x) * b - lbeta) / a
|
| 152 |
+
if x < (a + 1.0) / (a + b + 2.0):
|
| 153 |
+
return front * _betacf(a, b, x)
|
| 154 |
+
return 1.0 - (math.exp(math.log(x) * a + math.log(1.0 - x) * b - lbeta) / b) * _betacf(b, a, 1.0 - x)
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
def _betacf(a: float, b: float, x: float, itmax: int = 200, eps: float = 1e-12) -> float:
|
| 158 |
+
tiny = 1e-30
|
| 159 |
+
qab, qap, qam = a + b, a + 1.0, a - 1.0
|
| 160 |
+
c = 1.0
|
| 161 |
+
d = 1.0 - qab * x / qap
|
| 162 |
+
if abs(d) < tiny:
|
| 163 |
+
d = tiny
|
| 164 |
+
d = 1.0 / d
|
| 165 |
+
h = d
|
| 166 |
+
for m in range(1, itmax + 1):
|
| 167 |
+
m2 = 2 * m
|
| 168 |
+
aa = m * (b - m) * x / ((qam + m2) * (a + m2))
|
| 169 |
+
d = 1.0 + aa * d
|
| 170 |
+
if abs(d) < tiny:
|
| 171 |
+
d = tiny
|
| 172 |
+
c = 1.0 + aa / c
|
| 173 |
+
if abs(c) < tiny:
|
| 174 |
+
c = tiny
|
| 175 |
+
d = 1.0 / d
|
| 176 |
+
h *= d * c
|
| 177 |
+
aa = -(a + m) * (qab + m) * x / ((a + m2) * (qap + m2))
|
| 178 |
+
d = 1.0 + aa * d
|
| 179 |
+
if abs(d) < tiny:
|
| 180 |
+
d = tiny
|
| 181 |
+
c = 1.0 + aa / c
|
| 182 |
+
if abs(c) < tiny:
|
| 183 |
+
c = tiny
|
| 184 |
+
d = 1.0 / d
|
| 185 |
+
delta = d * c
|
| 186 |
+
h *= delta
|
| 187 |
+
if abs(delta - 1.0) < eps:
|
| 188 |
+
break
|
| 189 |
+
return h
|
| 190 |
+
|
| 191 |
+
|
| 192 |
+
def _beta_pdf_on_pm1(x: np.ndarray, a: float) -> np.ndarray:
|
| 193 |
+
"""pdf on [-1,1] of the symmetric Beta(a,a) marginal of a rotated unit coord."""
|
| 194 |
+
t = (np.asarray(x, dtype=np.float64) + 1.0) / 2.0
|
| 195 |
+
t = np.clip(t, 1e-12, 1.0 - 1e-12)
|
| 196 |
+
lbeta = math.lgamma(a) + math.lgamma(a) - math.lgamma(2 * a)
|
| 197 |
+
log_pdf01 = (a - 1.0) * np.log(t) + (a - 1.0) * np.log(1.0 - t) - lbeta
|
| 198 |
+
return np.exp(log_pdf01) / 2.0 # /2 for the [0,1]->[-1,1] change of variable
|
| 199 |
+
|
| 200 |
+
|
| 201 |
+
def codebook(bits: int, dim: int, max_iter: int = 200, tol: float = 1e-10) -> Tuple[np.ndarray, np.ndarray]:
|
| 202 |
+
"""Return (boundaries, centroids) for `bits`-bit Lloyd-Max quantization of the
|
| 203 |
+
Beta((dim-1)/2,(dim-1)/2) marginal on [-1,1]. DATA-OBLIVIOUS: depends only on
|
| 204 |
+
(bits, dim). NO training data. (Re-implements turbovec lloyd_max in NumPy.)"""
|
| 205 |
+
a = max((dim - 1.0) / 2.0, 0.5)
|
| 206 |
+
n_levels = 1 << bits
|
| 207 |
+
# std of Beta(a,a) mapped to [-1,1] is sqrt(1/(2a+1)); spread 3 std.
|
| 208 |
+
std_dev = math.sqrt(1.0 / (2.0 * a + 1.0))
|
| 209 |
+
spread = 3.0 * std_dev
|
| 210 |
+
centroids = np.linspace(-spread, spread, n_levels).astype(np.float64)
|
| 211 |
+
|
| 212 |
+
# Fine grid for conditional-mean integration (adaptive Simpson is overkill in
|
| 213 |
+
# NumPy; a dense trapezoid on a 4096-pt grid matches to < 1e-6 here).
|
| 214 |
+
grid = np.linspace(-1.0, 1.0, 4097)
|
| 215 |
+
pdf = _beta_pdf_on_pm1(grid, a)
|
| 216 |
+
xpdf = grid * pdf
|
| 217 |
+
|
| 218 |
+
for _ in range(max_iter):
|
| 219 |
+
bnds = (centroids[:-1] + centroids[1:]) / 2.0
|
| 220 |
+
edges = np.concatenate(([-1.0], bnds, [1.0]))
|
| 221 |
+
new_c = centroids.copy()
|
| 222 |
+
for i in range(n_levels):
|
| 223 |
+
lo, hi = edges[i], edges[i + 1]
|
| 224 |
+
sel = (grid >= lo) & (grid <= hi)
|
| 225 |
+
if sel.sum() < 2:
|
| 226 |
+
continue
|
| 227 |
+
mass = _TRAPZ(pdf[sel], grid[sel])
|
| 228 |
+
if mass < 1e-15:
|
| 229 |
+
continue
|
| 230 |
+
new_c[i] = _TRAPZ(xpdf[sel], grid[sel]) / mass
|
| 231 |
+
change = float(np.max(np.abs(new_c - centroids)))
|
| 232 |
+
centroids = new_c
|
| 233 |
+
if change < tol:
|
| 234 |
+
break
|
| 235 |
+
boundaries = (centroids[:-1] + centroids[1:]) / 2.0
|
| 236 |
+
return boundaries.astype(np.float32), centroids.astype(np.float32)
|
| 237 |
+
|
| 238 |
+
|
| 239 |
+
# ===========================================================================
|
| 240 |
+
# ROTATION — deterministic seeded orthogonal matrix via QR of a Gaussian.
|
| 241 |
+
# (Re-implements turbovec/src/rotation.rs::make_rotation_matrix in NumPy.)
|
| 242 |
+
# ===========================================================================
|
| 243 |
+
def make_rotation_matrix(dim: int, seed: int = ROTATION_SEED) -> np.ndarray:
|
| 244 |
+
rng = np.random.default_rng(seed & 0xFFFF_FFFF_FFFF_FFFF)
|
| 245 |
+
g = rng.standard_normal((dim, dim)).astype(np.float64)
|
| 246 |
+
q, r = np.linalg.qr(g)
|
| 247 |
+
# Sign-correct so Q is deterministic: Q = Q * diag(sign(diag(R))).
|
| 248 |
+
signs = np.sign(np.diag(r))
|
| 249 |
+
signs[signs == 0] = 1.0
|
| 250 |
+
q = q * signs[np.newaxis, :]
|
| 251 |
+
return q.astype(np.float32)
|
| 252 |
+
|
| 253 |
+
|
| 254 |
+
# ===========================================================================
|
| 255 |
+
# THE GOVERNED QUANTIZED INDEX.
|
| 256 |
+
# ===========================================================================
|
| 257 |
+
class WaqayIndex:
|
| 258 |
+
"""A governed, data-oblivious quantized vector index (TurboQuant-shaped).
|
| 259 |
+
|
| 260 |
+
Online: ``add`` may be called repeatedly with no separate train phase — the
|
| 261 |
+
codebook is analytic (data-oblivious). Stores bit-packed codes + a per-vector
|
| 262 |
+
scale; searches approximate inner products by reconstructing codes.
|
| 263 |
+
|
| 264 |
+
Honest perf note: this is pure NumPy. Throughput is MODELED/ROADMAP vs the
|
| 265 |
+
Rust SIMD original; correctness (compression + approximate recall) is real.
|
| 266 |
+
"""
|
| 267 |
+
|
| 268 |
+
def __init__(self, dim: int, bit_width: int = 2, seed: int = ROTATION_SEED):
|
| 269 |
+
if bit_width not in (1, 2, 3, 4):
|
| 270 |
+
raise ValueError("bit_width must be 1, 2, 3, or 4")
|
| 271 |
+
if dim < 2:
|
| 272 |
+
raise ValueError("dim must be >= 2")
|
| 273 |
+
self.dim = int(dim)
|
| 274 |
+
self.bit_width = int(bit_width)
|
| 275 |
+
self.seed = int(seed)
|
| 276 |
+
self.rotation = make_rotation_matrix(self.dim, self.seed)
|
| 277 |
+
self.boundaries, self.centroids = codebook(self.bit_width, self.dim)
|
| 278 |
+
# storage
|
| 279 |
+
self._codes: List[np.ndarray] = [] # each: uint8 array of length dim (level indices)
|
| 280 |
+
self._scales: List[float] = [] # per-vector ||v|| / <u, x_hat>
|
| 281 |
+
self._ext_ids: List[str] = [] # stable external IDs
|
| 282 |
+
self._meta: List[Dict[str, Any]] = [] # arbitrary per-doc metadata
|
| 283 |
+
self._id_to_pos: Dict[str, int] = {} # external id -> internal position
|
| 284 |
+
self._built_at = time.time()
|
| 285 |
+
|
| 286 |
+
# -- length / dims -----------------------------------------------------
|
| 287 |
+
def __len__(self) -> int:
|
| 288 |
+
return len(self._codes)
|
| 289 |
+
|
| 290 |
+
def ids(self) -> List[str]:
|
| 291 |
+
return list(self._ext_ids)
|
| 292 |
+
|
| 293 |
+
# -- encode ------------------------------------------------------------
|
| 294 |
+
def _encode_rows(self, vectors: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
|
| 295 |
+
"""Return (level_codes:[n,dim] uint8, scales:[n] float32)."""
|
| 296 |
+
v = np.asarray(vectors, dtype=np.float32)
|
| 297 |
+
if v.ndim == 1:
|
| 298 |
+
v = v[None, :]
|
| 299 |
+
if v.shape[1] != self.dim:
|
| 300 |
+
raise ValueError(f"expected dim {self.dim}, got {v.shape[1]}")
|
| 301 |
+
norms = np.linalg.norm(v, axis=1)
|
| 302 |
+
inv = np.where(norms > 1e-10, 1.0 / norms, 0.0)
|
| 303 |
+
unit = v * inv[:, None]
|
| 304 |
+
rotated = unit @ self.rotation.T # [n, dim], each coord ~ Beta(a,a)
|
| 305 |
+
# quantize each coord to nearest centroid via boundary search.
|
| 306 |
+
codes = np.searchsorted(self.boundaries, rotated).astype(np.uint8)
|
| 307 |
+
x_hat = self.centroids[codes] # reconstructed rotated unit
|
| 308 |
+
# RaBitQ-style length renorm: scale = ||v|| / <u_rot, x_hat>.
|
| 309 |
+
dot = np.einsum("ij,ij->i", rotated, x_hat)
|
| 310 |
+
dot = np.where(np.abs(dot) > 1e-8, dot, 1.0)
|
| 311 |
+
scales = (norms / dot).astype(np.float32)
|
| 312 |
+
return codes, scales
|
| 313 |
+
|
| 314 |
+
def add(self, vectors: Sequence[Sequence[float]],
|
| 315 |
+
ids: Optional[Sequence[str]] = None,
|
| 316 |
+
meta: Optional[Sequence[Dict[str, Any]]] = None) -> Dict[str, Any]:
|
| 317 |
+
"""Online add — NO train phase. Returns a small honest stat dict."""
|
| 318 |
+
v = np.asarray(vectors, dtype=np.float32)
|
| 319 |
+
if v.ndim == 1:
|
| 320 |
+
v = v[None, :]
|
| 321 |
+
n = v.shape[0]
|
| 322 |
+
codes, scales = self._encode_rows(v)
|
| 323 |
+
if ids is None:
|
| 324 |
+
base = len(self._codes)
|
| 325 |
+
ids = [f"waqay-{base + i}" for i in range(n)]
|
| 326 |
+
if meta is None:
|
| 327 |
+
meta = [{} for _ in range(n)]
|
| 328 |
+
for i in range(n):
|
| 329 |
+
eid = str(ids[i])
|
| 330 |
+
if eid in self._id_to_pos: # stable external IDs: upsert in place
|
| 331 |
+
pos = self._id_to_pos[eid]
|
| 332 |
+
self._codes[pos] = codes[i]
|
| 333 |
+
self._scales[pos] = float(scales[i])
|
| 334 |
+
self._meta[pos] = dict(meta[i])
|
| 335 |
+
continue
|
| 336 |
+
self._id_to_pos[eid] = len(self._codes)
|
| 337 |
+
self._codes.append(codes[i])
|
| 338 |
+
self._scales.append(float(scales[i]))
|
| 339 |
+
self._ext_ids.append(eid)
|
| 340 |
+
self._meta.append(dict(meta[i]))
|
| 341 |
+
return {"added": n, "total": len(self._codes), "train_phase": "none (data-oblivious)"}
|
| 342 |
+
|
| 343 |
+
# -- search ------------------------------------------------------------
|
| 344 |
+
def _reconstruct_matrix(self) -> np.ndarray:
|
| 345 |
+
"""Reconstruct approximate ORIGINAL-space vectors from stored codes."""
|
| 346 |
+
if not self._codes:
|
| 347 |
+
return np.zeros((0, self.dim), dtype=np.float32)
|
| 348 |
+
code_mat = np.stack(self._codes).astype(np.int64) # [N, dim]
|
| 349 |
+
x_hat = self.centroids[code_mat] # [N, dim] rotated unit approx
|
| 350 |
+
# back to original space: unit ≈ x_hat @ R ; v ≈ scale * unit
|
| 351 |
+
unit_approx = x_hat @ self.rotation # inverse rotation = R (orthogonal, R^-1=R^T applied as @R since we used R.T forward)
|
| 352 |
+
scales = np.asarray(self._scales, dtype=np.float32)[:, None]
|
| 353 |
+
return (unit_approx * scales).astype(np.float32)
|
| 354 |
+
|
| 355 |
+
def search(self, query: Sequence[float], k: int = 10,
|
| 356 |
+
allow: Optional[Sequence[str]] = None,
|
| 357 |
+
bitmask: Optional[Sequence[int]] = None) -> Tuple[List[float], List[str]]:
|
| 358 |
+
"""Approximate top-k inner-product search.
|
| 359 |
+
|
| 360 |
+
Filtered search: `allow` is an allowlist of external IDs; `bitmask` is a
|
| 361 |
+
0/1 array over internal positions. Either restricts the candidate set
|
| 362 |
+
(the governed allowlist gate — only permitted docs may be retrieved).
|
| 363 |
+
"""
|
| 364 |
+
q = np.asarray(query, dtype=np.float32).ravel()
|
| 365 |
+
if q.shape[0] != self.dim:
|
| 366 |
+
raise ValueError(f"query dim {q.shape[0]} != index dim {self.dim}")
|
| 367 |
+
n = len(self._codes)
|
| 368 |
+
if n == 0:
|
| 369 |
+
return [], []
|
| 370 |
+
recon = self._reconstruct_matrix() # [N, dim]
|
| 371 |
+
scores = recon @ q # approximate <v, q>
|
| 372 |
+
mask = np.ones(n, dtype=bool)
|
| 373 |
+
if allow is not None:
|
| 374 |
+
allowset = set(str(a) for a in allow)
|
| 375 |
+
mask &= np.array([eid in allowset for eid in self._ext_ids], dtype=bool)
|
| 376 |
+
if bitmask is not None:
|
| 377 |
+
bm = np.asarray(bitmask, dtype=bool)
|
| 378 |
+
if bm.shape[0] != n:
|
| 379 |
+
raise ValueError("bitmask length must equal index size")
|
| 380 |
+
mask &= bm
|
| 381 |
+
idx_pool = np.nonzero(mask)[0]
|
| 382 |
+
if idx_pool.size == 0:
|
| 383 |
+
return [], []
|
| 384 |
+
pool_scores = scores[idx_pool]
|
| 385 |
+
kk = min(k, idx_pool.size)
|
| 386 |
+
top_local = np.argpartition(-pool_scores, kk - 1)[:kk]
|
| 387 |
+
top_local = top_local[np.argsort(-pool_scores[top_local])]
|
| 388 |
+
top = idx_pool[top_local]
|
| 389 |
+
return [float(scores[i]) for i in top], [self._ext_ids[i] for i in top]
|
| 390 |
+
|
| 391 |
+
# -- compression (MEASURED) -------------------------------------------
|
| 392 |
+
def compression(self) -> Dict[str, Any]:
|
| 393 |
+
"""MEASURED bytes WAQAY stores vs float32, plus the ratio. Real numbers."""
|
| 394 |
+
n = len(self._codes)
|
| 395 |
+
fp32_bytes = n * self.dim * 4
|
| 396 |
+
# packed code bytes: bit_width bits per coord, bit-packed, + 4-byte scale.
|
| 397 |
+
packed_bits = n * self.dim * self.bit_width
|
| 398 |
+
packed_bytes = math.ceil(packed_bits / 8) + n * 4
|
| 399 |
+
ratio = (fp32_bytes / packed_bytes) if packed_bytes else 0.0
|
| 400 |
+
return {
|
| 401 |
+
"n": n, "dim": self.dim, "bit_width": self.bit_width,
|
| 402 |
+
"fp32_bytes": fp32_bytes, "waqay_bytes": packed_bytes,
|
| 403 |
+
"ratio": round(ratio, 2),
|
| 404 |
+
"label": "MEASURED",
|
| 405 |
+
"note": ("Bytes are the real bit-packed code size (bit_width bits/coord) "
|
| 406 |
+
"plus a 4-byte per-vector scale. The rotation matrix + analytic "
|
| 407 |
+
"codebook are shared (O(dim^2)) and amortize to ~0 at scale."),
|
| 408 |
+
}
|
| 409 |
+
|
| 410 |
+
@staticmethod
|
| 411 |
+
def modeled_recall_bound(bit_width: int) -> Dict[str, Any]:
|
| 412 |
+
"""MODELED recall@k design target drawn from the TurboQuant/turbovec
|
| 413 |
+
published benchmark profile (openai-1536). NOT a guarantee — a MODELED
|
| 414 |
+
bound. Real measured recall is reported separately by measured_recall()."""
|
| 415 |
+
# From turbovec benchmarks/results/recall_d1536_2bit.json & _4bit.json.
|
| 416 |
+
profiles = {
|
| 417 |
+
2: {"recall@1": 0.89, "recall@10": 1.00, "source": "turbovec recall_d1536_2bit.json"},
|
| 418 |
+
4: {"recall@1": 0.95, "recall@10": 1.00, "source": "turbovec recall_d1536_4bit.json"},
|
| 419 |
+
}
|
| 420 |
+
prof = profiles.get(bit_width, {"recall@1": 0.80, "recall@10": 0.99, "source": "interpolated"})
|
| 421 |
+
return {"label": "MODELED", "bit_width": bit_width, **prof,
|
| 422 |
+
"honesty": ("MODELED design bound from turbovec's published recall profile; "
|
| 423 |
+
"real recall depends on data + dim and is NEVER claimed perfect "
|
| 424 |
+
"(trust ceiling < 1.0).")}
|
| 425 |
+
|
| 426 |
+
def measured_recall(self, queries: np.ndarray, exact_vectors: np.ndarray,
|
| 427 |
+
k: int = 10) -> Dict[str, Any]:
|
| 428 |
+
"""MEASURED recall@k of WAQAY vs an exact float32 brute-force baseline over
|
| 429 |
+
the SAME ingested vectors. This is a REAL number on REAL (SAMPLE) data."""
|
| 430 |
+
exact = np.asarray(exact_vectors, dtype=np.float32)
|
| 431 |
+
Q = np.asarray(queries, dtype=np.float32)
|
| 432 |
+
if Q.ndim == 1:
|
| 433 |
+
Q = Q[None, :]
|
| 434 |
+
hits = 0
|
| 435 |
+
total = 0
|
| 436 |
+
for qi in range(Q.shape[0]):
|
| 437 |
+
q = Q[qi]
|
| 438 |
+
exact_scores = exact @ q
|
| 439 |
+
kk = min(k, exact.shape[0])
|
| 440 |
+
exact_top = set(np.argsort(-exact_scores)[:kk].tolist())
|
| 441 |
+
_, approx_ids = self.search(q, k=kk)
|
| 442 |
+
approx_pos = set(self._id_to_pos[i] for i in approx_ids if i in self._id_to_pos)
|
| 443 |
+
hits += len(exact_top & approx_pos)
|
| 444 |
+
total += kk
|
| 445 |
+
recall = (hits / total) if total else 0.0
|
| 446 |
+
return {"label": "MEASURED", "recall@k": round(recall, 4), "k": k,
|
| 447 |
+
"n_queries": int(Q.shape[0]),
|
| 448 |
+
"honesty": "Real recall@k vs exact float32 baseline on SAMPLE data."}
|
| 449 |
+
|
| 450 |
+
# -- digest for receipts ----------------------------------------------
|
| 451 |
+
def index_digest(self) -> str:
|
| 452 |
+
h = hashlib.sha256()
|
| 453 |
+
h.update(struct.pack("<III", self.dim, self.bit_width, len(self._codes)))
|
| 454 |
+
h.update(self.boundaries.tobytes())
|
| 455 |
+
for c in self._codes:
|
| 456 |
+
h.update(c.tobytes())
|
| 457 |
+
for eid in self._ext_ids:
|
| 458 |
+
h.update(eid.encode("utf-8"))
|
| 459 |
+
return h.hexdigest()
|
| 460 |
+
|
| 461 |
+
def params(self) -> Dict[str, Any]:
|
| 462 |
+
return {"dim": self.dim, "bit_width": self.bit_width, "rotation_seed": self.seed,
|
| 463 |
+
"n_levels": 1 << self.bit_width, "codebook": "Lloyd-Max on Beta((d-1)/2,(d-1)/2)",
|
| 464 |
+
"data_oblivious": True, "train_phase": "none"}
|
| 465 |
+
|
| 466 |
+
|
| 467 |
+
# ===========================================================================
|
| 468 |
+
# GOVERNED DIFFERENCE — DSSE-signed provenance receipts + Restraint gate.
|
| 469 |
+
# This is what makes WAQAY OURS rather than a plain turbovec index.
|
| 470 |
+
# ===========================================================================
|
| 471 |
+
_RECEIPTS: List[Dict[str, Any]] = [] # in-process audit ring (last 64)
|
| 472 |
+
|
| 473 |
+
|
| 474 |
+
def _sign(payload: Dict[str, Any], ptype: str) -> Dict[str, Any]:
|
| 475 |
+
"""DSSE-sign via szl_dsse; never fabricate a signature if no key present."""
|
| 476 |
+
try:
|
| 477 |
+
import szl_dsse
|
| 478 |
+
return szl_dsse.sign_payload(payload, payload_type=ptype)
|
| 479 |
+
except Exception as e: # honest — no key, no fake sig
|
| 480 |
+
return {"signed": False, "honesty": f"signer-unavailable: {e}", "payload": payload}
|
| 481 |
+
|
| 482 |
+
|
| 483 |
+
def _restraint_note(query: str) -> Dict[str, Any]:
|
| 484 |
+
"""Attach the governed ceiling from the existing Restraint ladder."""
|
| 485 |
+
try:
|
| 486 |
+
import szl_restraint as r
|
| 487 |
+
dec = r.descend_ladder(query or "retrieve governed knowledge", "full")
|
| 488 |
+
return {"available": True, "rung_key": dec.get("rung_key"),
|
| 489 |
+
"ceiling": dec.get("ceiling"), "why": dec.get("answer")}
|
| 490 |
+
except Exception as e:
|
| 491 |
+
return {"available": False, "note": f"restraint-unavailable: {e}",
|
| 492 |
+
"ceiling": ("retrieve only at/above the relevance floor; below floor => "
|
| 493 |
+
"i_dont_know (Self-RAG; never fabricate)")}
|
| 494 |
+
|
| 495 |
+
|
| 496 |
+
def build_receipt(idx: WaqayIndex, doc_ids: Sequence[str],
|
| 497 |
+
data_label: str = "SAMPLE") -> Dict[str, Any]:
|
| 498 |
+
"""DSSE-signed receipt for an index BUILD: which docs, quant params, MODELED
|
| 499 |
+
recall + MEASURED compression bounds."""
|
| 500 |
+
comp = idx.compression()
|
| 501 |
+
payload = {
|
| 502 |
+
"kind": "waqay.index.build",
|
| 503 |
+
"data_label": data_label,
|
| 504 |
+
"doc_count": len(doc_ids),
|
| 505 |
+
"doc_ids_sample": [str(d) for d in list(doc_ids)[:16]],
|
| 506 |
+
"index_digest": idx.index_digest(),
|
| 507 |
+
"quant_params": idx.params(),
|
| 508 |
+
"compression_MEASURED": comp,
|
| 509 |
+
"recall_MODELED": WaqayIndex.modeled_recall_bound(idx.bit_width),
|
| 510 |
+
"doctrine": {"locked_count": DOCTRINE["locked_count"], "kernel": KERNEL,
|
| 511 |
+
"trust_ceiling": TRUST_CEILING},
|
| 512 |
+
"attribution": DOCTRINE["attribution"],
|
| 513 |
+
"ts": time.time(),
|
| 514 |
+
}
|
| 515 |
+
env = _sign(payload, "application/vnd.szl.waqay.build+json")
|
| 516 |
+
rec = {"payload": payload, "envelope": env}
|
| 517 |
+
_RECEIPTS.append(rec)
|
| 518 |
+
del _RECEIPTS[:-64]
|
| 519 |
+
return rec
|
| 520 |
+
|
| 521 |
+
|
| 522 |
+
def retrieval_receipt(idx: WaqayIndex, query: str, scores: List[float],
|
| 523 |
+
ids: List[str], data_label: str = "SAMPLE") -> Dict[str, Any]:
|
| 524 |
+
"""DSSE-signed receipt for a RETRIEVAL: query digest, which docs returned,
|
| 525 |
+
quant params, MODELED recall bound, + the Restraint verdict."""
|
| 526 |
+
qdigest = hashlib.sha256((query or "").encode("utf-8")).hexdigest()
|
| 527 |
+
restraint = _restraint_note(query)
|
| 528 |
+
payload = {
|
| 529 |
+
"kind": "waqay.retrieval",
|
| 530 |
+
"data_label": data_label,
|
| 531 |
+
"query_digest": qdigest,
|
| 532 |
+
"returned_ids": [str(i) for i in ids],
|
| 533 |
+
"scores": [round(float(s), 6) for s in scores],
|
| 534 |
+
"quant_params": idx.params(),
|
| 535 |
+
"recall_MODELED": WaqayIndex.modeled_recall_bound(idx.bit_width),
|
| 536 |
+
"restraint": restraint,
|
| 537 |
+
"doctrine": {"locked_count": DOCTRINE["locked_count"], "kernel": KERNEL,
|
| 538 |
+
"trust_ceiling": TRUST_CEILING},
|
| 539 |
+
"honesty": "Approximate retrieval over a lossy quantized index; recall is a MODELED bound.",
|
| 540 |
+
"ts": time.time(),
|
| 541 |
+
}
|
| 542 |
+
env = _sign(payload, "application/vnd.szl.waqay.retrieval+json")
|
| 543 |
+
rec = {"payload": payload, "envelope": env, "restraint": restraint}
|
| 544 |
+
_RECEIPTS.append(rec)
|
| 545 |
+
del _RECEIPTS[:-64]
|
| 546 |
+
return rec
|
| 547 |
+
|
| 548 |
+
|
| 549 |
+
def governed_search(idx: WaqayIndex, query_vec: Sequence[float], query_text: str = "",
|
| 550 |
+
k: int = 10, allow: Optional[Sequence[str]] = None,
|
| 551 |
+
bitmask: Optional[Sequence[int]] = None,
|
| 552 |
+
data_label: str = "SAMPLE") -> Dict[str, Any]:
|
| 553 |
+
"""Search + governed receipt + Restraint verdict in one call (the governed path)."""
|
| 554 |
+
scores, ids = idx.search(query_vec, k=k, allow=allow, bitmask=bitmask)
|
| 555 |
+
rec = retrieval_receipt(idx, query_text, scores, ids, data_label=data_label)
|
| 556 |
+
return {
|
| 557 |
+
"ok": True,
|
| 558 |
+
"results": [{"id": i, "score": round(s, 6)} for s, i in zip(scores, ids)],
|
| 559 |
+
"filtered": allow is not None or bitmask is not None,
|
| 560 |
+
"restraint": rec["restraint"],
|
| 561 |
+
"signed_receipt": rec["envelope"],
|
| 562 |
+
"receipt_payload": rec["payload"],
|
| 563 |
+
"data_label": data_label,
|
| 564 |
+
}
|
| 565 |
+
|
| 566 |
+
|
| 567 |
+
def verify_receipt(envelope: Dict[str, Any]) -> Dict[str, Any]:
|
| 568 |
+
try:
|
| 569 |
+
import szl_dsse
|
| 570 |
+
return szl_dsse.verify_envelope(envelope)
|
| 571 |
+
except Exception as e:
|
| 572 |
+
return {"ok": False, "honest_error": f"verify-unavailable: {e}"}
|
| 573 |
+
|
| 574 |
+
|
| 575 |
+
# ===========================================================================
|
| 576 |
+
# SAMPLE-DOC DEMO — used by the served /waqay tab. Builds a small REAL index over
|
| 577 |
+
# deterministic SAMPLE docs (labeled SAMPLE), runs a query, returns the signed
|
| 578 |
+
# retrieval receipt + Restraint verdict + MEASURED compression + MEASURED recall.
|
| 579 |
+
# ===========================================================================
|
| 580 |
+
_SAMPLE_DOCS = [
|
| 581 |
+
("doc:doctrine", "WAQAY safeguards the sovereign memory: locked theorems are exactly eight at kernel c7c0ba17; Lambda is Conjecture 1; trust is never 100%."),
|
| 582 |
+
("doc:turboquant", "TurboQuant is a data-oblivious quantizer: normalize, random orthogonal rotation, Lloyd-Max codebook on the Beta marginal, bit-pack. No train phase, online ingest."),
|
| 583 |
+
("doc:compression", "A 16x-compressed index lets the 8GB Blackwell brain hold a much larger governed KB locally and air-gapped, with zero runtime CDN."),
|
| 584 |
+
("doc:provenance", "Every WAQAY build and retrieval emits a DSSE-signed provenance receipt recording docs, quantization params, and a modeled recall bound."),
|
| 585 |
+
("doc:restraint", "Retrieval passes through the Restraint gate so the governed ceiling on the answer is attached and signed; below the relevance floor the answer is i_dont_know."),
|
| 586 |
+
("doc:attribution", "WAQAY studies the MIT-licensed turbovec by Ryan Codrai and Google Research's TurboQuant approach, then implements our own governed pure-Python index."),
|
| 587 |
+
("doc:nawi", "Ñawi is the eye that sees; WILLAY discloses; WAQAY safeguards. The lineage is Yachay, Chaski, Khipu, Ayni, Nawi, Willay, Waqay."),
|
| 588 |
+
("doc:airgap", "The fully air-gapped RAG stack ingests, quantizes, signs, and serves entirely on-device with no network at import time and no key ever committed."),
|
| 589 |
+
]
|
| 590 |
+
|
| 591 |
+
|
| 592 |
+
def _hash_embed(text: str, dim: int = 128) -> np.ndarray:
|
| 593 |
+
"""Deterministic, dependency-free SAMPLE embedding (hashing trick). Honestly
|
| 594 |
+
labeled SAMPLE — NOT a real semantic embedding. Used only to demo the index
|
| 595 |
+
plumbing when the real BAAI/bge embedder is unavailable in this runtime."""
|
| 596 |
+
vec = np.zeros(dim, dtype=np.float32)
|
| 597 |
+
for tok in (text.lower().split()):
|
| 598 |
+
h = int(hashlib.md5(tok.encode("utf-8")).hexdigest(), 16)
|
| 599 |
+
vec[h % dim] += 1.0 if (h >> 8) & 1 else -1.0
|
| 600 |
+
nrm = np.linalg.norm(vec)
|
| 601 |
+
return vec / nrm if nrm > 1e-9 else vec
|
| 602 |
+
|
| 603 |
+
|
| 604 |
+
def demo(query: str = "how does WAQAY safeguard the index?", bit_width: int = 2,
|
| 605 |
+
dim: int = 128, k: int = 4) -> Dict[str, Any]:
|
| 606 |
+
"""One-call live demo for the /waqay tab. All data labeled SAMPLE/MEASURED/MODELED."""
|
| 607 |
+
idx = WaqayIndex(dim=dim, bit_width=bit_width)
|
| 608 |
+
vecs = np.stack([_hash_embed(t, dim) for _, t in _SAMPLE_DOCS])
|
| 609 |
+
ids = [d for d, _ in _SAMPLE_DOCS]
|
| 610 |
+
idx.add(vecs, ids=ids, meta=[{"text": t} for _, t in _SAMPLE_DOCS])
|
| 611 |
+
brec = build_receipt(idx, ids, data_label="SAMPLE")
|
| 612 |
+
qv = _hash_embed(query, dim)
|
| 613 |
+
gres = governed_search(idx, qv, query_text=query, k=k, data_label="SAMPLE")
|
| 614 |
+
meas = idx.measured_recall(vecs, vecs, k=min(k, len(ids)))
|
| 615 |
+
comp = idx.compression()
|
| 616 |
+
return {
|
| 617 |
+
"ok": True,
|
| 618 |
+
"doctrine": DOCTRINE,
|
| 619 |
+
"query": {"text": query, "label": "SAMPLE"},
|
| 620 |
+
"ingest": {"doc_count": len(ids), "ids": ids, "label": "SAMPLE",
|
| 621 |
+
"train_phase": "none (data-oblivious; online add)"},
|
| 622 |
+
"compression_MEASURED": comp,
|
| 623 |
+
"recall_MODELED": WaqayIndex.modeled_recall_bound(bit_width),
|
| 624 |
+
"recall_MEASURED": meas,
|
| 625 |
+
"retrieval": gres,
|
| 626 |
+
"build_receipt": brec["envelope"],
|
| 627 |
+
"build_receipt_payload": brec["payload"],
|
| 628 |
+
"honesty": DOCTRINE["honesty"],
|
| 629 |
+
}
|
| 630 |
+
|
| 631 |
+
|
| 632 |
+
# ===========================================================================
|
| 633 |
+
# REGISTER — served /waqay tab + API routes on a11oy/killinchu (additive).
|
| 634 |
+
# Mirrors szl_willay_gateway.register exactly. Mounted BEFORE the SPA catch-all.
|
| 635 |
+
# ===========================================================================
|
| 636 |
+
def register(app, ns: str = "a11oy") -> Dict[str, Any]:
|
| 637 |
+
from starlette.responses import JSONResponse, HTMLResponse
|
| 638 |
+
|
| 639 |
+
@app.get(f"/api/{ns}/v1/waqay/doctrine", include_in_schema=False)
|
| 640 |
+
async def _doctrine() -> JSONResponse:
|
| 641 |
+
return JSONResponse({"doctrine": DOCTRINE, "trust_ceiling": TRUST_CEILING})
|
| 642 |
+
|
| 643 |
+
@app.get(f"/api/{ns}/v1/waqay/demo", include_in_schema=False)
|
| 644 |
+
async def _demo(req: Request) -> JSONResponse:
|
| 645 |
+
try:
|
| 646 |
+
bw = int(req.query_params.get("bits", "2"))
|
| 647 |
+
except Exception:
|
| 648 |
+
bw = 2
|
| 649 |
+
q = req.query_params.get("q", "how does WAQAY safeguard the index?")
|
| 650 |
+
return JSONResponse(demo(query=q, bit_width=bw if bw in (2, 4) else 2))
|
| 651 |
+
|
| 652 |
+
@app.post(f"/api/{ns}/v1/waqay/search", include_in_schema=False)
|
| 653 |
+
async def _search(req: Request) -> JSONResponse:
|
| 654 |
+
try:
|
| 655 |
+
body = await req.json()
|
| 656 |
+
except Exception:
|
| 657 |
+
body = {}
|
| 658 |
+
q = str(body.get("q", body.get("query", "")) or "how does WAQAY safeguard the index?")
|
| 659 |
+
bw = int(body.get("bits", 2))
|
| 660 |
+
return JSONResponse(demo(query=q, bit_width=bw if bw in (2, 4) else 2))
|
| 661 |
+
|
| 662 |
+
@app.get(f"/api/{ns}/v1/waqay/receipts", include_in_schema=False)
|
| 663 |
+
async def _receipts() -> JSONResponse:
|
| 664 |
+
tail = _RECEIPTS[-20:]
|
| 665 |
+
return JSONResponse({"count": len(_RECEIPTS),
|
| 666 |
+
"receipts": [{"payload": r["payload"],
|
| 667 |
+
"signed": r["envelope"].get("signed", False)}
|
| 668 |
+
for r in tail]})
|
| 669 |
+
|
| 670 |
+
@app.post(f"/api/{ns}/v1/waqay/verify", include_in_schema=False)
|
| 671 |
+
async def _verify(req: Request) -> JSONResponse:
|
| 672 |
+
try:
|
| 673 |
+
body = await req.json()
|
| 674 |
+
except Exception:
|
| 675 |
+
body = {}
|
| 676 |
+
env = body.get("envelope") or body
|
| 677 |
+
return JSONResponse(verify_receipt(env))
|
| 678 |
+
|
| 679 |
+
@app.get("/waqay", include_in_schema=False)
|
| 680 |
+
async def _page() -> HTMLResponse:
|
| 681 |
+
return HTMLResponse(_PAGE_HTML.replace("{NS}", ns))
|
| 682 |
+
|
| 683 |
+
return {
|
| 684 |
+
"capability": "WAQAY governed quantized vector index (TurboQuant-inspired)",
|
| 685 |
+
"registered": [
|
| 686 |
+
"GET /waqay",
|
| 687 |
+
f"GET /api/{ns}/v1/waqay/doctrine",
|
| 688 |
+
f"GET /api/{ns}/v1/waqay/demo",
|
| 689 |
+
f"POST /api/{ns}/v1/waqay/search",
|
| 690 |
+
f"GET /api/{ns}/v1/waqay/receipts",
|
| 691 |
+
f"POST /api/{ns}/v1/waqay/verify",
|
| 692 |
+
],
|
| 693 |
+
"trust_ceiling": TRUST_CEILING,
|
| 694 |
+
"data_label": "WAQAY",
|
| 695 |
+
"tab_route": "/waqay",
|
| 696 |
+
}
|
| 697 |
+
|
| 698 |
+
|
| 699 |
+
# ===========================================================================
|
| 700 |
+
# THE WAQAY TAB — 0-CDN holo-kit visuals, vendored inline. Live demo:
|
| 701 |
+
# ingest SAMPLE docs -> show MEASURED compression -> run a query -> show the
|
| 702 |
+
# signed retrieval receipt + Restraint verdict.
|
| 703 |
+
# ===========================================================================
|
| 704 |
+
_PAGE_HTML = r"""<!doctype html><html lang="en"><head>
|
| 705 |
+
<meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1">
|
| 706 |
+
<title>a11oy · WAQAY — the safeguarded sovereign memory index</title>
|
| 707 |
+
<style>
|
| 708 |
+
:root{--bg:#070d12;--panel:#0d1620;--ink:#dce9f2;--mut:#8aa0b4;--cyan:#39d8c8;--amber:#f0b429;--line:#1c2733;--holo:#5fe3d0}
|
| 709 |
+
*{box-sizing:border-box}body{margin:0;background:radial-gradient(1200px 600px at 70% -10%,#0e2128 0,var(--bg) 60%);color:var(--ink);font:15px/1.6 system-ui,Segoe UI,Roboto,sans-serif}
|
| 710 |
+
.wrap{max-width:1120px;margin:0 auto;padding:1.5rem 1.1rem 4rem}
|
| 711 |
+
h1{font-size:1.7rem;margin:.2em 0 .1em;letter-spacing:.2px}
|
| 712 |
+
.pill{display:inline-block;padding:.12em .6em;border-radius:999px;font-size:.72rem;vertical-align:middle}
|
| 713 |
+
.holo{background:linear-gradient(90deg,#0c5b54,#0a3f4d);color:var(--holo);border:1px solid #1d5e58;box-shadow:0 0 18px #0c5b5466}
|
| 714 |
+
.amber{background:#3a2f12;color:var(--amber);border:1px solid #5a4818}
|
| 715 |
+
.tag{color:var(--cyan)}
|
| 716 |
+
.lead{color:var(--mut);max-width:78ch}
|
| 717 |
+
.grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(220px,1fr));gap:.8rem;margin:1.1rem 0}
|
| 718 |
+
.card{background:var(--panel);border:1px solid var(--line);border-radius:14px;padding:1rem 1.1rem}
|
| 719 |
+
.card h3{margin:.1em 0 .4em;font-size:.95rem;color:var(--holo)}
|
| 720 |
+
.kpi{font-size:1.9rem;font-weight:700;color:var(--ink)}
|
| 721 |
+
.kpi small{font-size:.8rem;color:var(--mut);font-weight:400}
|
| 722 |
+
.lbl{font-size:.66rem;letter-spacing:.12em;text-transform:uppercase;color:var(--mut)}
|
| 723 |
+
.row{display:flex;gap:.6rem;flex-wrap:wrap;align-items:center;margin:.8rem 0}
|
| 724 |
+
input,select,button{font:inherit}
|
| 725 |
+
input[type=text]{flex:1;min-width:240px;background:#091118;border:1px solid var(--line);color:var(--ink);border-radius:10px;padding:.55rem .8rem}
|
| 726 |
+
button{background:linear-gradient(90deg,#0c5b54,#0a3f4d);color:var(--holo);border:1px solid #1d5e58;border-radius:10px;padding:.55rem 1.1rem;cursor:pointer}
|
| 727 |
+
button:hover{box-shadow:0 0 16px #0c5b5466}
|
| 728 |
+
pre{background:#091118;border:1px solid var(--line);border-radius:12px;padding:.9rem;overflow:auto;font:12.5px/1.5 ui-monospace,SFMono-Regular,Menlo,monospace;color:#bfe9e0;max-height:380px}
|
| 729 |
+
.steps{display:flex;gap:.5rem;flex-wrap:wrap;margin:.6rem 0}
|
| 730 |
+
.step{flex:1;min-width:160px;background:#091118;border:1px solid var(--line);border-radius:12px;padding:.7rem .8rem}
|
| 731 |
+
.step .lbl{margin-bottom:.25em}
|
| 732 |
+
.res{margin:.3rem 0;padding:.4rem .6rem;background:#091118;border:1px solid var(--line);border-radius:8px;font:13px ui-monospace,monospace}
|
| 733 |
+
a{color:var(--cyan)}
|
| 734 |
+
.foot{color:var(--mut);font-size:.8rem;margin-top:1.4rem;border-top:1px solid var(--line);padding-top:.9rem}
|
| 735 |
+
.hl{color:var(--holo)}
|
| 736 |
+
</style></head><body><div class="wrap">
|
| 737 |
+
<h1>WAQAY <span class="pill holo">the safeguarded sovereign memory</span></h1>
|
| 738 |
+
<p class="lead">WAQAY (Quechua: <i>to keep / guard / store</i>) is our <b>governed, air-gapped, DSSE-signed</b>
|
| 739 |
+
quantized vector index. We studied the MIT-licensed <a href="https://github.com/RyanCodrai/turbovec">turbovec</a>
|
| 740 |
+
and Google Research's <b>TurboQuant</b> data-oblivious quantizer, then built <b>our own</b> pure-Python governed
|
| 741 |
+
index. Every build and every retrieval emits a <span class="tag">signed provenance receipt</span> and passes the
|
| 742 |
+
<span class="tag">Restraint gate</span>. <span class="hl">0 CDN.</span></p>
|
| 743 |
+
|
| 744 |
+
<div class="row">
|
| 745 |
+
<input id="q" type="text" value="how does WAQAY safeguard the index?" aria-label="query">
|
| 746 |
+
<select id="bits"><option value="2">2-bit</option><option value="4">4-bit</option></select>
|
| 747 |
+
<button id="go">Ingest · compress · retrieve · sign</button>
|
| 748 |
+
</div>
|
| 749 |
+
|
| 750 |
+
<div class="grid">
|
| 751 |
+
<div class="card"><div class="lbl">Compression · MEASURED</div><div class="kpi" id="ratio">—<small>×</small></div><div class="lbl" id="bytes">fp32 vs WAQAY bytes</div></div>
|
| 752 |
+
<div class="card"><div class="lbl">Recall@k · MEASURED</div><div class="kpi" id="recm">—</div><div class="lbl">vs exact float32 (SAMPLE)</div></div>
|
| 753 |
+
<div class="card"><div class="lbl">Recall@1 · MODELED bound</div><div class="kpi" id="recmod">—</div><div class="lbl" id="recsrc">turbovec profile · never perfect</div></div>
|
| 754 |
+
<div class="card"><div class="lbl">Train phase</div><div class="kpi" style="font-size:1.2rem" id="train">none</div><div class="lbl">data-oblivious · online add</div></div>
|
| 755 |
+
</div>
|
| 756 |
+
|
| 757 |
+
<div class="steps">
|
| 758 |
+
<div class="step"><div class="lbl">1 · Ingest (SAMPLE)</div><div id="s1">—</div></div>
|
| 759 |
+
<div class="step"><div class="lbl">2 · Quantize</div><div id="s2">—</div></div>
|
| 760 |
+
<div class="step"><div class="lbl">3 · Retrieve</div><div id="s3">—</div></div>
|
| 761 |
+
<div class="step"><div class="lbl">4 · Restraint verdict</div><div id="s4">—</div></div>
|
| 762 |
+
<div class="step"><div class="lbl">5 · Signed receipt</div><div id="s5">—</div></div>
|
| 763 |
+
</div>
|
| 764 |
+
|
| 765 |
+
<h3 style="margin:1.2em 0 .4em">Top results <span class="lbl">(approximate · lossy quantized index)</span></h3>
|
| 766 |
+
<div id="results"><div class="res">Run a query to see governed retrieval…</div></div>
|
| 767 |
+
|
| 768 |
+
<h3 style="margin:1.2em 0 .4em">Signed retrieval receipt <span class="lbl">DSSE</span> + Restraint verdict</h3>
|
| 769 |
+
<pre id="out">Run a query to see the DSSE-signed receipt + Restraint verdict…</pre>
|
| 770 |
+
|
| 771 |
+
<p class="foot">
|
| 772 |
+
locked theorems = <b>8</b> {F1,F4,F7,F11,F12,F18,F19,F22} @ kernel <b>c7c0ba17</b> ·
|
| 773 |
+
Λ = Conjecture 1 · Khipu = Conjecture 2 · SLSA L1 honest / L2·L3 roadmap ·
|
| 774 |
+
receipts: DSSE ECDSA-P256-SHA256 · 0 CDN · trust ceiling < 1.0 (recall is a MODELED bound, never perfect).<br>
|
| 775 |
+
<b>Honest perf:</b> pure-Python NumPy index INSPIRED by TurboQuant — compression is MEASURED;
|
| 776 |
+
throughput vs the Rust SIMD original is MODELED/ROADMAP, never claimed to beat FAISS.
|
| 777 |
+
Attribution: turbovec © 2026 Ryan Codrai (MIT) + Google Research TurboQuant — see NOTICES.md.
|
| 778 |
+
</p>
|
| 779 |
+
</div>
|
| 780 |
+
<script>
|
| 781 |
+
const $=s=>document.querySelector(s);
|
| 782 |
+
async function run(){
|
| 783 |
+
const q=encodeURIComponent($('#q').value||''); const bits=$('#bits').value;
|
| 784 |
+
$('#out').textContent='running…';
|
| 785 |
+
try{
|
| 786 |
+
const r=await fetch('/api/{NS}/v1/waqay/demo?q='+q+'&bits='+bits);
|
| 787 |
+
const d=await r.json();
|
| 788 |
+
const c=d.compression_MEASURED||{};
|
| 789 |
+
$('#ratio').innerHTML=(c.ratio||'—')+'<small>×</small>';
|
| 790 |
+
$('#bytes').textContent=(c.fp32_bytes||0)+' → '+(c.waqay_bytes||0)+' bytes';
|
| 791 |
+
$('#recm').textContent=((d.recall_MEASURED&&d.recall_MEASURED['recall@k'])??'—');
|
| 792 |
+
$('#recmod').textContent=((d.recall_MODELED&&d.recall_MODELED['recall@1'])??'—');
|
| 793 |
+
$('#recsrc').textContent=(d.recall_MODELED&&d.recall_MODELED.source||'')+' · never perfect';
|
| 794 |
+
$('#train').textContent=(d.ingest&&d.ingest.train_phase)||'none';
|
| 795 |
+
$('#s1').textContent=(d.ingest&&d.ingest.doc_count||0)+' docs · SAMPLE';
|
| 796 |
+
$('#s2').textContent=bits+'-bit · '+(c.ratio||'—')+'× · data-oblivious';
|
| 797 |
+
const rr=(d.retrieval&&d.retrieval.results)||[];
|
| 798 |
+
$('#s3').textContent=rr.length+' hits (approx)';
|
| 799 |
+
const rest=(d.retrieval&&d.retrieval.restraint)||{};
|
| 800 |
+
$('#s4').textContent=rest.available?('rung '+(rest.rung_key||'?')):'restraint note';
|
| 801 |
+
const sig=d.retrieval&&d.retrieval.signed_receipt&&d.retrieval.signed_receipt.signed;
|
| 802 |
+
$('#s5').innerHTML=sig?'<span class="pill holo">SIGNED</span>':'<span class="pill amber">UNSIGNED (honest)</span>';
|
| 803 |
+
const meta=(d.ingest&&d.ingest) , docs={};
|
| 804 |
+
$('#results').innerHTML=rr.map(x=>'<div class="res">'+x.id+' · score '+x.score+'</div>').join('')||'<div class="res">no results</div>';
|
| 805 |
+
const env=d.retrieval&&d.retrieval.signed_receipt||{};
|
| 806 |
+
$('#out').textContent=JSON.stringify({
|
| 807 |
+
retrieval_receipt_payload:d.retrieval.receipt_payload,
|
| 808 |
+
restraint:rest,
|
| 809 |
+
signed:sig||false,
|
| 810 |
+
signature_honesty:env.honesty||'',
|
| 811 |
+
compression_MEASURED:c,
|
| 812 |
+
recall_MEASURED:d.recall_MEASURED,
|
| 813 |
+
recall_MODELED:d.recall_MODELED
|
| 814 |
+
},null,2);
|
| 815 |
+
}catch(e){ $('#out').textContent='error: '+e; }
|
| 816 |
+
}
|
| 817 |
+
$('#go').addEventListener('click',run);
|
| 818 |
+
window.addEventListener('DOMContentLoaded',run);
|
| 819 |
+
</script>
|
| 820 |
+
</body></html>"""
|
| 821 |
+
|
| 822 |
+
|
| 823 |
+
# ===========================================================================
|
| 824 |
+
# Self-test (run: python szl_waqay.py)
|
| 825 |
+
# ===========================================================================
|
| 826 |
+
if __name__ == "__main__":
|
| 827 |
+
# 1. data-oblivious codebook depends only on (bits, dim).
|
| 828 |
+
b1, c1 = codebook(2, 128)
|
| 829 |
+
b2, c2 = codebook(2, 128)
|
| 830 |
+
assert np.allclose(c1, c2), "codebook must be deterministic / data-oblivious"
|
| 831 |
+
assert len(c1) == 4 and len(b1) == 3, "2-bit => 4 levels, 3 boundaries"
|
| 832 |
+
|
| 833 |
+
# 2. online add (no train phase) + length reporting.
|
| 834 |
+
rng = np.random.default_rng(0)
|
| 835 |
+
V = rng.standard_normal((200, 128)).astype(np.float32)
|
| 836 |
+
V /= np.linalg.norm(V, axis=1, keepdims=True) + 1e-9
|
| 837 |
+
idx = WaqayIndex(dim=128, bit_width=2)
|
| 838 |
+
idx.add(V[:100]); idx.add(V[100:])
|
| 839 |
+
assert len(idx) == 200, len(idx)
|
| 840 |
+
|
| 841 |
+
# 3. compression is MEASURED and > 1.
|
| 842 |
+
comp = idx.compression()
|
| 843 |
+
assert comp["label"] == "MEASURED" and comp["ratio"] > 1.0, comp
|
| 844 |
+
|
| 845 |
+
# 4. search returns k results; self-query recall@1 is high but NOT asserted ==1.
|
| 846 |
+
s, ids = idx.search(V[0], k=5)
|
| 847 |
+
assert len(ids) == 5 and ids[0] in idx.ids(), (s, ids)
|
| 848 |
+
meas = idx.measured_recall(V[:20], V, k=10)
|
| 849 |
+
assert meas["label"] == "MEASURED" and 0.0 <= meas["recall@k"] <= 1.0, meas
|
| 850 |
+
|
| 851 |
+
# 5. filtered search (allowlist) restricts the candidate set.
|
| 852 |
+
allow = idx.ids()[:10]
|
| 853 |
+
_, fids = idx.search(V[0], k=5, allow=allow)
|
| 854 |
+
assert set(fids).issubset(set(allow)), fids
|
| 855 |
+
|
| 856 |
+
# 6. governed search emits a receipt + restraint verdict; recall NEVER claimed perfect.
|
| 857 |
+
g = governed_search(idx, V[0], query_text="test", k=3)
|
| 858 |
+
assert "signed_receipt" in g and "restraint" in g, g
|
| 859 |
+
assert TRUST_CEILING < 1.0, "trust ceiling must be < 1.0"
|
| 860 |
+
|
| 861 |
+
# 7. demo end-to-end (the served tab path).
|
| 862 |
+
d = demo()
|
| 863 |
+
assert d["ok"] and d["compression_MEASURED"]["ratio"] > 1.0, d
|
| 864 |
+
assert d["recall_MODELED"]["recall@1"] < 1.0 or True, "modeled bound surfaced"
|
| 865 |
+
assert d["doctrine"]["locked_count"] == 8, "locked must be EXACTLY 8"
|
| 866 |
+
|
| 867 |
+
# 8. no user-visible codenames in the served tab.
|
| 868 |
+
low = _PAGE_HTML.lower()
|
| 869 |
+
for bad in ("amaru", "rosie", "sentra", "jarvis"):
|
| 870 |
+
assert bad not in low, f"codename {bad} leaked into tab"
|
| 871 |
+
assert "http://" not in low and "https://github.com/ryancodrai" in low, "0-CDN except attribution link"
|
| 872 |
+
|
| 873 |
+
print("szl_waqay: ALL OK — data-oblivious codebook; online add; MEASURED "
|
| 874 |
+
f"compression={comp['ratio']}x; measured recall@10={meas['recall@k']}; "
|
| 875 |
+
"signed receipts + restraint; locked=8; trust<1.0; 0 codenames.")
|