| <!doctype html> |
| <html lang="en"> |
| <head> |
| <meta charset="utf-8"> |
| <meta name="viewport" content="width=device-width, initial-scale=1"> |
| <title>Tessera - span-level language ID</title> |
| <style> |
| :root{ |
| --bg:#fbfbfa; --fg:#1a1a19; --muted:#6b6b68; --line:#e3e3df; |
| --card:#ffffff; --accent:#2f6f4f; --warn:#b3261e; --warnbg:#fdecea; |
| --code:#f4f4f1; |
| } |
| @media (prefers-color-scheme: dark){ |
| :root:not([data-theme="light"]){ |
| --bg:#17171a; --fg:#e9e9e6; --muted:#9a9a96; --line:#303036; |
| --card:#1e1e22; --accent:#7fc9a0; --warn:#f2b8b5; --warnbg:#3a1f1d; |
| --code:#232329; |
| } |
| } |
| *{box-sizing:border-box} |
| body{margin:0;background:var(--bg);color:var(--fg); |
| font:15px/1.55 ui-sans-serif,system-ui,-apple-system,"Segoe UI",Roboto,sans-serif} |
| .wrap{max-width:1080px;margin:0 auto;padding:28px 20px 80px} |
| h1{font-size:26px;margin:0 0 4px;letter-spacing:-.01em} |
| .sub{color:var(--muted);margin:0 0 20px;font-size:15px} |
| .lede{border-left:3px solid var(--accent);padding:2px 0 2px 14px;margin:0 0 22px} |
| .grid{display:grid;grid-template-columns:1fr 370px;gap:18px} |
| @media(max-width:860px){.grid{grid-template-columns:1fr}} |
| textarea{width:100%;min-height:190px;padding:12px;border:1px solid var(--line); |
| border-radius:8px;background:var(--card);color:var(--fg);font:14px/1.5 ui-monospace, |
| "SF Mono",Menlo,monospace;resize:vertical} |
| button{background:var(--accent);color:#fff;border:0;border-radius:7px; |
| padding:9px 18px;font-size:14px;font-weight:600;cursor:pointer} |
| button:disabled{opacity:.5;cursor:default} |
| .row{display:flex;gap:10px;align-items:center;margin:10px 0;flex-wrap:wrap} |
| .card{background:var(--card);border:1px solid var(--line);border-radius:9px;padding:14px} |
| .warncard{background:var(--warnbg);border:1px solid var(--warn);border-radius:9px; |
| padding:12px 14px;color:var(--fg)} |
| .muted{color:var(--muted);font-size:13px} |
| table{border-collapse:collapse;width:100%;font-size:13px} |
| th,td{text-align:left;padding:5px 8px;border-bottom:1px solid var(--line); |
| vertical-align:top} |
| th{color:var(--muted);font-weight:600;font-size:12px;text-transform:uppercase; |
| letter-spacing:.04em} |
| td.txt{font-family:ui-monospace,Menlo,monospace;word-break:break-word;max-width:340px} |
| .chip{display:inline-block;padding:1px 7px;border-radius:20px;font-size:12px; |
| font-family:ui-monospace,Menlo,monospace;white-space:nowrap} |
| .spanview{line-height:2.3;word-break:break-word} |
| .spanview mark{padding:2px 5px;border-radius:5px;color:#111} |
| .drop{opacity:.45;text-decoration:line-through} |
| .bar{height:8px;background:var(--line);border-radius:5px;overflow:hidden;margin:8px 0} |
| .bar>i{display:block;height:100%;background:var(--accent);width:0;transition:width .2s} |
| .ex{background:none;color:var(--accent);border:1px solid var(--line); |
| padding:5px 11px;font-weight:500} |
| a{color:var(--accent)} |
| code{background:var(--code);padding:1px 5px;border-radius:4px; |
| font-family:ui-monospace,Menlo,monospace;font-size:.92em} |
| .scroll{overflow-x:auto} |
| #verdict table td{padding:4px 6px;font-size:12.5px} |
| #verdict table td:first-child{white-space:nowrap;font-size:12px} |
| #verdict table td code{font-size:11.5px;padding:1px 4px} |
| .grid{align-items:start} |
| footer{margin-top:34px;padding-top:18px;border-top:1px solid var(--line); |
| color:var(--muted);font-size:13px} |
| </style> |
| </head> |
| <body> |
| <div class="wrap"> |
| <h1>Tessera</h1> |
| <p class="sub">Span-level, multi-label language identification.</p> |
|
|
| <div class="lede"> |
| <p style="margin:0 0 8px">Document-level language ID gives <strong>one language per |
| page</strong>. Real pages are not one language: the navigation is English, the body |
| might be Ojibwe, the comments might be three other things. Filtering a corpus on the |
| document label throws that text away, and low-resource languages are the ones that |
| cannot spare it.</p> |
| <p style="margin:0" class="muted">On the benchmark corpus, document-level LID |
| recovers <strong>0.0%</strong> of the low-resource spans it would otherwise discard |
| or misfile. Tessera recovers <strong>86%</strong> (90% on the rarest tier). |
| Tessera runs <em>behind</em> GlotLID / OpenLID, not instead of them.</p> |
| </div> |
|
|
| <div id="status" class="card" style="margin-bottom:16px"> |
| <div id="statusText">Loading the model (17 MB, runs entirely in your browser)…</div> |
| <div class="bar"><i id="prog"></i></div> |
| </div> |
|
|
| <div class="grid"> |
| <div> |
| <textarea id="input" spellcheck="false" placeholder="Paste a web page, a forum thread, a comment section…"></textarea> |
| <div class="row"> |
| <button id="go" disabled>Analyse</button> |
| <label class="muted">threshold |
| <input id="thr" type="range" min="0.05" max="0.95" step="0.05" value="0.5"> |
| <span id="thrv">0.50</span> |
| </label> |
| </div> |
| <div class="row" id="examples"></div> |
| </div> |
| <div> |
| <div id="verdict" class="card"> |
| <div class="muted">Analyse some text to see what a single-label filter would |
| cost.</div> |
| </div> |
| </div> |
| </div> |
|
|
| <h3 style="margin:26px 0 8px">Spans</h3> |
| <div id="spans" class="card spanview muted">—</div> |
|
|
| <h3 style="margin:26px 0 8px">Detail</h3> |
| <div class="card scroll"><table id="table"> |
| <thead><tr><th>range</th><th>len</th><th>role</th><th>languages</th><th>tier</th> |
| <th>score</th><th>text</th></tr></thead><tbody></tbody></table></div> |
|
|
| <footer> |
| <p><strong>What this demo does and does not show.</strong> The model runs in your |
| browser from a 17 MB float16 export, verified against the Python reference to |
| 3e-04. For the three built-in examples the document-level verdicts shown are |
| <em>measured</em>, by running GlotLID v3, OpenLID and fastText lid.176 on those |
| exact strings offline; fastText models cannot run in a static page, so they are |
| embedded rather than computed live. For text you paste, the panel falls back to a |
| stand-in built from Tessera's own majority span language, which is <em>more |
| favourable to document-level filtering</em> than the real thing and therefore |
| understates the problem. Every published number in the repository uses GlotLID v3.</p> |
| <p>Trained on synthetic composed multilingual pages built from UDHR-LID (CC0, 430 |
| languages) and FLORES-200 (CC-BY-SA-4.0, 204 languages). Synthetic construction is |
| what makes exact span boundaries possible; it is also a limitation, and the |
| repository reports its real-data checks and known weaknesses separately.</p> |
| <p><a href="https://github.com/NagaYu/tessera">Code</a> · |
| <a href="https://huggingface.co/NagaYu/tessera-span-lid">Model</a> · |
| <a href="https://huggingface.co/datasets/NagaYu/tessera-mixed-pages">Dataset</a> |
| · Apache-2.0</p> |
| <p id="selfcheck" class="muted"></p> |
| </footer> |
| </div> |
|
|
| <script type="module"> |
| import { load, selfCheck, segment, stripHtml } from "./tessera.js"; |
| |
| |
| |
| |
| |
| const BASELINES = {"Buried body": {"glotlid": {"label": "anp_Deva", "p": 0.974, "kept": true}, "openlid": {"label": "ast_Latn", "p": 0.9973, "kept": true}, "fasttext176": {"label": "eng_Latn", "p": 0.1659, "kept": false}}, "Minority comment": {"glotlid": {"label": "ind_Latn", "p": 0.3477, "kept": false}, "openlid": {"label": "ind_Latn", "p": 0.847, "kept": true}, "fasttext176": {"label": "eng_Latn", "p": 0.3731, "kept": false}}, "HTML page": {"glotlid": {"label": "chr_Cher", "p": 0.9728, "kept": true}, "openlid": {"label": "bak_Cyrl", "p": 0.9995, "kept": true}, "fasttext176": {"label": "rus_Cyrl", "p": 0.493, "kept": false}}}; |
| |
| const EXAMPLES = { |
| "Buried body": `Home | About | Contact | Subscribe |
| ᐊᓂᔑᓈᐯᒧᐎᓐ ᑭᑫᓐᑕᒧᐎᓐ |
| ᑲᑭᓇᐌᓀᓐ ᐅᑕᔭᓐ ᑌᐸᑫᑕᑯᓯᐎᓐ ᒋᑭᑕᑶᐸᓐ ᑲᔦ ᑕᐱᑕ ᐃᔑᑲᓇᐗᐸᑎᐎᓐ ᐅᐌᓂ ᐅᑐᑕᑲᓀᓯᐎᓐ. |
| ᑲᐎᓐ ᐊᐎᔭ ᑕᐅᒋᑎᐸᑯᓇᓯ ᐅᐌᓂ ᐅᑐᑕᑲᓀᓯᐎᓐ ᑲᐃᔑᓇᑲᑫᒋᑫᐎᓇᓐ ᑭᔥᐱᓐ ᑌᐺ. |
| Reply | Share | Report |
| © 2024 Example Media Inc.`, |
| "Minority comment": `Welcome to the community forum |
| This article covers the history of the region in some detail. |
| ሕድሕድ ሰብ ኩሉ መሰላትን ናጽነታትን ክረክብ መሰል ኦለዎ። |
| Sangat bagus sekali artikel ini, terima kasih banyak |
| 2024 |
| Terms of Service | Privacy Policy`, |
| "HTML page": `<html><body><nav>Accueil | Boutique | Contact</nav> |
| <h1>ᏣᎳᎩ ᎦᏬᏂᎯᏍᏗ</h1> |
| <p>ᏂᎦᏗᏳ ᏴᏫ ᏗᏂᏲᎯ ᎠᎴ ᎤᏂᏂᎩᏛ ᎨᏒ ᏚᏳᎪᏛ ᎠᎴ ᏚᏳᎪᏛ ᏗᎨᏥᎳᏫᏎᏗ.</p> |
| <div class="comments"><p>Привет всем, отличная статья</p></div> |
| <footer>© 2024</footer></body></html>`, |
| }; |
| |
| const PALETTE = ["#bfdcff","#ffd9a8","#c3ebc8","#f8c4dc","#fff2a8", |
| "#b6e8ef","#ffc9b4","#dbcdf0","#d9edb6","#f0dcc0"]; |
| const colours = new Map(); |
| const colourFor = (l) => { |
| if (!colours.has(l)) colours.set(l, PALETTE[colours.size % PALETTE.length]); |
| return colours.get(l); |
| }; |
| |
| const $ = (id) => document.getElementById(id); |
| let tagger = null; |
| |
| for (const [name, text] of Object.entries(EXAMPLES)) { |
| const b = document.createElement("button"); |
| b.className = "ex"; b.textContent = name; |
| b.onclick = () => { $("input").value = text; run(); }; |
| $("examples").appendChild(b); |
| } |
| $("thr").oninput = (e) => { $("thrv").textContent = (+e.target.value).toFixed(2); |
| if (tagger && $("input").value.trim()) run(); }; |
| |
| (async () => { |
| try { |
| $("prog").style.width = "35%"; |
| tagger = await load("."); |
| $("prog").style.width = "80%"; |
| const chk = selfCheck(tagger); |
| $("prog").style.width = "100%"; |
| const n = tagger.labels.length; |
| $("statusText").innerHTML = chk.ok |
| ? `Model ready — ${n} labels, ${tagger.meta.total_floats.toLocaleString()} weights. |
| <span class="muted">Self-check against the Python reference passed |
| (max deviation ${chk.worstDiff.toExponential(1)}, float16 rounding).</span>` |
| : `<strong style="color:var(--warn)">Self-check FAILED</strong> — this port |
| disagrees with the Python reference by ${chk.worstDiff.toExponential(2)}. |
| Results below should not be trusted.`; |
| $("selfcheck").textContent = |
| `Self-check: ${chk.rows.length} reference probes, worst deviation ` + |
| `${chk.worstDiff.toExponential(2)} (threshold 5e-3).`; |
| $("go").disabled = false; |
| $("go").onclick = run; |
| $("input").value = EXAMPLES["Buried body"]; |
| run(); |
| } catch (e) { |
| $("statusText").innerHTML = |
| `<strong style="color:var(--warn)">Could not load the model.</strong> ${e.message}`; |
| } |
| })(); |
| |
| function run() { |
| const raw = $("input").value; |
| if (!raw.trim()) return; |
| const text = stripHtml(raw); |
| const spans = segment(text); |
| if (!spans.length) return; |
| const thr = +$("thr").value; |
| const t0 = performance.now(); |
| const probs = tagger.predict(spans); |
| const ms = performance.now() - t0; |
| |
| const rows = spans.map((s, i) => { |
| const top = tagger.topK(probs[i], 4, thr); |
| const best = tagger.topK(probs[i], 1, 0)[0]; |
| return { ...s, labels: top.map((t) => t.label), best, |
| keep: top.length > 0, score: best ? best.score : 0 }; |
| }); |
| |
| |
| const view = $("spans"); |
| view.className = "card spanview"; |
| view.innerHTML = ""; |
| let cur = 0; |
| for (const r of rows) { |
| if (r.start > cur) view.append(document.createTextNode(text.slice(cur, r.start))); |
| const m = document.createElement("mark"); |
| m.textContent = r.text; |
| const lab = r.labels[0] || "und"; |
| m.style.background = r.keep ? colourFor(lab) : "transparent"; |
| if (!r.keep) m.className = "drop"; |
| m.title = `${r.labels.join(", ") || "below threshold"} — ${r.score.toFixed(3)}`; |
| view.append(m); |
| cur = r.end; |
| } |
| if (cur < text.length) view.append(document.createTextNode(text.slice(cur))); |
| |
| |
| const tb = $("table").querySelector("tbody"); |
| tb.innerHTML = ""; |
| for (const r of rows) { |
| const tr = document.createElement("tr"); |
| const tier = r.labels[0] ? (tagger.meta.tiers[r.labels[0]] || "-") : "-"; |
| tr.innerHTML = |
| `<td>${r.start}:${r.end}</td><td>${r.end - r.start}</td><td>${r.role}</td>` + |
| `<td>${r.labels.map((l) => `<span class="chip" style="background:${colourFor(l)};color:#111">${l}</span>`).join(" ") || '<span class="muted">below threshold</span>'}</td>` + |
| `<td>${tier}</td><td>${r.score.toFixed(3)}</td>` + |
| `<td class="txt">${escapeHtml(r.text.slice(0, 90))}</td>`; |
| tb.appendChild(tr); |
| } |
| |
| |
| const chars = {}; |
| let total = 0; |
| for (const r of rows) { |
| total += r.end - r.start; |
| if (!r.keep || !r.labels.length) continue; |
| const l = r.labels[0]; |
| if (l === "und") continue; |
| chars[l] = (chars[l] || 0) + (r.end - r.start); |
| } |
| const ranked = Object.entries(chars).sort((a, b) => b[1] - a[1]); |
| const major = ranked[0] ? ranked[0][0] : null; |
| const lost = ranked.slice(1).reduce((a, b) => a + b[1], 0); |
| const pct = total ? (100 * lost / total) : 0; |
| |
| |
| const exName = Object.keys(EXAMPLES).find((k) => EXAMPLES[k] === raw); |
| let head; |
| if (exName && BASELINES[exName]) { |
| const b = BASELINES[exName]; |
| const line = (sys, d) => { |
| const verdict = d.kept |
| ? `kept, filed as <code>${d.label}</code>` |
| : `<strong style="color:var(--warn)">discarded</strong> (<code>${d.label}</code> below threshold)`; |
| return `<tr><td><code>${sys}</code></td><td>${verdict} |
| <span class="muted">p=${d.p}</span></td></tr>`; |
| }; |
| head = `<div class="muted" style="margin-bottom:6px">What real document-level LID |
| does with this exact page</div> |
| <table style="margin-bottom:10px"> |
| ${line("GlotLID v3", b.glotlid)} |
| ${line("OpenLID", b.openlid)} |
| ${line("fastText lid.176", b.fasttext176)} |
| </table> |
| <div class="warncard">Every one of them assigns <strong>a single label to |
| the whole page</strong>. Tessera finds |
| ${ranked.length} language${ranked.length === 1 ? "" : "s"} in it.</div>`; |
| } else if (major) { |
| head = `<div class="muted" style="margin-bottom:6px">If this page were filtered on a |
| single language label</div> |
| <div style="font-size:15px;margin-bottom:10px">it would be filed as |
| <span class="chip" style="background:${colourFor(major)};color:#111">${major}</span></div> |
| <div class="warncard"><strong>${lost.toLocaleString()} of |
| ${total.toLocaleString()} characters (${pct.toFixed(1)}%)</strong> are not in |
| that language and would be discarded or misfiled.</div> |
| <div class="muted" style="margin-top:8px">This stand-in uses Tessera's own |
| majority span language, which is <em>more generous to document-level |
| filtering</em> than a real document-level LID: on the built-in examples |
| GlotLID returns things like <code>anp_Deva</code> at p=0.97 for a page with |
| no Devanagari on it. Load an example to see the measured verdicts.</div>`; |
| } else { |
| head = `<div class="muted">No language asserted above the threshold.</div>`; |
| } |
| $("verdict").innerHTML = head + |
| `<div class="muted" style="margin-top:10px">Languages Tessera found: |
| ${ranked.map(([l]) => `<span class="chip" style="background:${colourFor(l)};color:#111">${l}</span>`).join(" ") || "none"}</div> |
| <div class="muted" style="margin-top:8px">${spans.length} spans in |
| ${ms.toFixed(0)} ms, in your browser.</div>`; |
| } |
| |
| function escapeHtml(s) { |
| return s.replace(/[&<>"']/g, (c) => ({ "&": "&", "<": "<", ">": ">", |
| '"': """, "'": "'" }[c])); |
| } |
| </script> |
| </body> |
| </html> |
|
|
