| <!doctype html> |
| <html lang="en"> |
| <head> |
| <meta charset="utf-8"> |
| <meta name="viewport" content="width=device-width, initial-scale=1"> |
| <meta name="description" content="A compact, interactive autopsy of a seed lottery in a 12.7M-parameter language model."> |
| <meta name="robots" content="index,follow"> |
| <meta name="theme-color" content="#f5f0e7"> |
| <link rel="canonical" href="https://porkicoder.com/research/the-sniff-test.html"> |
| <meta property="og:type" content="article"> |
| <meta property="og:title" content="The Sniff Test: A Seed Lottery in a 12.7M-Parameter Language Model"> |
| <meta property="og:description" content="A literary initialization appeared to stop repeated words, but its favorable direction did not recur consistently under matched random starts."> |
| <meta property="og:url" content="https://porkicoder.com/research/the-sniff-test.html"> |
| <meta name="twitter:card" content="summary"> |
| <meta name="twitter:title" content="The Sniff Test: A Seed Lottery in a Tiny Language Model"> |
| <meta name="twitter:description" content="A literary initialization appeared to stop repeated words, but its favorable direction did not recur consistently under matched random starts."> |
| <title>The Sniff Test: A Seed Lottery in a 12.7M-Parameter Language Model</title> |
| <script type="application/ld+json"> |
| { |
| "@context": "https://schema.org", |
| "@type": "ScholarlyArticle", |
| "headline": "The Sniff Test: A Seed Lottery in a 12.7M-Parameter Language Model", |
| "description": "A literary initialization appeared to stop repeated words, but its favorable direction did not recur consistently under matched random starts.", |
| "author": {"@type": "Person", "name": "MD Ishtiaque Hossain"}, |
| "publisher": {"@type": "Organization", "name": "PorkiCoder Research"}, |
| "datePublished": "2026-08-12", |
| "isAccessibleForFree": true, |
| "url": "https://porkicoder.com/research/the-sniff-test.html" |
| } |
| </script> |
| <style> |
| :root { |
| --paper: #f5f0e7; |
| --paper-deep: #ece3d4; |
| --ink: #17211f; |
| --muted: #58625f; |
| --faint: #d6cec1; |
| --sniff: #176b87; |
| --sniff-soft: #d9edf2; |
| --base: #b4513f; |
| --base-soft: #f3dfd9; |
| --tie: #7b8480; |
| --good: #32735f; |
| --warn: #9b6b24; |
| --shadow: 0 18px 50px rgba(44, 35, 24, 0.08); |
| --sans: Inter, ui-sans-serif, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; |
| --serif: Charter, "Iowan Old Style", "Bitstream Charter", Georgia, serif; |
| --mono: "SFMono-Regular", Consolas, "Liberation Mono", monospace; |
| } |
| |
| * { box-sizing: border-box; } |
| |
| html { |
| scroll-behavior: smooth; |
| background: var(--paper); |
| } |
| |
| body { |
| margin: 0; |
| overflow-x: hidden; |
| color: var(--ink); |
| background: |
| radial-gradient(circle at 12% 0%, rgba(23, 107, 135, 0.09), transparent 27rem), |
| radial-gradient(circle at 90% 16%, rgba(180, 81, 63, 0.07), transparent 24rem), |
| var(--paper); |
| font-family: var(--serif); |
| font-size: 18px; |
| line-height: 1.58; |
| text-rendering: optimizeLegibility; |
| } |
| |
| a { color: var(--sniff); text-underline-offset: 0.14em; } |
| a:hover { color: #0b5169; } |
| |
| .reading-progress { |
| position: fixed; |
| z-index: 20; |
| inset: 0 auto auto 0; |
| width: 0; |
| height: 3px; |
| background: linear-gradient(90deg, var(--base), var(--sniff)); |
| } |
| |
| .paper { |
| width: min(calc(100% - 2rem), 1120px); |
| margin: 0 auto; |
| padding: 4.5rem 0 5rem; |
| } |
| |
| .measure { width: min(100%, 760px); margin-inline: auto; } |
| |
| .front { |
| width: min(100%, 920px); |
| margin: 0 auto 3rem; |
| text-align: center; |
| } |
| |
| .eyebrow, |
| .meta, |
| .figure-label, |
| .kicker, |
| .tiny, |
| button, |
| select, |
| .stat-label, |
| .legend, |
| figcaption, |
| table, |
| .claim-tag, |
| .formula-label { |
| font-family: var(--sans); |
| } |
| |
| .eyebrow { |
| margin: 0 0 1rem; |
| color: var(--sniff); |
| font-size: 0.76rem; |
| font-weight: 750; |
| letter-spacing: 0.13em; |
| text-transform: uppercase; |
| } |
| |
| h1 { |
| margin: 0; |
| font-size: clamp(3.2rem, 10vw, 7.6rem); |
| line-height: 0.88; |
| letter-spacing: -0.065em; |
| font-weight: 650; |
| } |
| |
| .subtitle { |
| width: min(100%, 760px); |
| margin: 1.45rem auto 1.7rem; |
| color: #34403d; |
| font-size: clamp(1.1rem, 2.2vw, 1.45rem); |
| line-height: 1.35; |
| } |
| |
| .meta { |
| display: flex; |
| flex-wrap: wrap; |
| justify-content: center; |
| gap: 0.35rem 1rem; |
| margin: 0; |
| color: var(--muted); |
| font-size: 0.83rem; |
| } |
| |
| .abstract { |
| position: relative; |
| margin: 2.5rem auto 0; |
| padding: 1.5rem 1.6rem; |
| border: 1px solid var(--faint); |
| border-radius: 18px; |
| background: rgba(255, 255, 255, 0.54); |
| box-shadow: var(--shadow); |
| text-align: left; |
| } |
| |
| .abstract h2 { |
| margin: 0 0 0.45rem; |
| font: 750 0.76rem/1 var(--sans); |
| letter-spacing: 0.1em; |
| text-transform: uppercase; |
| } |
| |
| .abstract p { margin: 0; font-size: 1rem; } |
| |
| nav.contents { |
| display: flex; |
| flex-wrap: wrap; |
| justify-content: center; |
| gap: 0.5rem 1.1rem; |
| width: min(100%, 860px); |
| margin: 0 auto 4rem; |
| padding: 0.85rem 1rem; |
| border-block: 1px solid var(--faint); |
| font: 650 0.79rem/1.4 var(--sans); |
| } |
| |
| nav.contents a { color: var(--muted); text-decoration: none; } |
| nav.contents a:hover { color: var(--sniff); } |
| |
| section { scroll-margin-top: 1rem; } |
| |
| h2 { |
| margin: 4rem 0 0.9rem; |
| font-size: clamp(1.8rem, 4vw, 2.65rem); |
| line-height: 1.08; |
| letter-spacing: -0.025em; |
| } |
| |
| h3 { |
| margin: 2rem 0 0.45rem; |
| font-size: 1.2rem; |
| line-height: 1.25; |
| } |
| |
| p { margin: 0.72rem 0; } |
| strong { font-weight: 720; } |
| |
| .lead { |
| margin-top: 0; |
| color: #35413e; |
| font-size: 1.14rem; |
| line-height: 1.48; |
| } |
| |
| .kicker { |
| display: block; |
| margin-bottom: 0.55rem; |
| color: var(--sniff); |
| font-size: 0.75rem; |
| font-weight: 800; |
| letter-spacing: 0.12em; |
| text-transform: uppercase; |
| } |
| |
| .claim-ladder { |
| display: grid; |
| grid-template-columns: repeat(3, 1fr); |
| gap: 0.75rem; |
| width: min(100%, 1000px); |
| margin: 2rem auto 3.5rem; |
| } |
| |
| .claim { |
| min-height: 12rem; |
| padding: 1.1rem; |
| border: 1px solid var(--faint); |
| border-top: 5px solid var(--tie); |
| border-radius: 14px; |
| background: rgba(255, 255, 255, 0.48); |
| } |
| |
| .claim.robust { border-top-color: var(--good); } |
| .claim.descriptive { border-top-color: var(--warn); } |
| .claim.unproven { border-top-color: var(--base); } |
| |
| .claim-tag { |
| display: block; |
| margin-bottom: 0.45rem; |
| color: var(--muted); |
| font-size: 0.69rem; |
| font-weight: 800; |
| letter-spacing: 0.08em; |
| text-transform: uppercase; |
| } |
| |
| .claim strong { display: block; font-size: 1.08rem; line-height: 1.25; } |
| .claim p { margin: 0.55rem 0 0; color: var(--muted); font-size: 0.88rem; line-height: 1.45; } |
| |
| .hero-numbers { |
| display: grid; |
| grid-template-columns: repeat(2, minmax(0, 1fr)); |
| gap: 0.8rem; |
| width: min(100%, 900px); |
| margin: 1.8rem auto; |
| } |
| |
| .hero-number { |
| padding: 1.35rem; |
| border: 1px solid var(--faint); |
| border-radius: 16px; |
| background: rgba(255, 255, 255, 0.5); |
| } |
| |
| .hero-number .value { |
| display: block; |
| font: 750 clamp(2rem, 5vw, 3.8rem)/1 var(--sans); |
| letter-spacing: -0.055em; |
| } |
| |
| .hero-number .value.neutral { color: var(--tie); } |
| .hero-number .value.sniff { color: var(--sniff); } |
| .hero-number p { margin: 0.5rem 0 0; color: var(--muted); font-size: 0.9rem; } |
| |
| figure.interactive { |
| width: min(100%, 1060px); |
| margin: 2.25rem auto 3rem; |
| padding: clamp(1rem, 3vw, 1.7rem); |
| border: 1px solid var(--faint); |
| border-radius: 20px; |
| background: rgba(255, 255, 255, 0.65); |
| box-shadow: var(--shadow); |
| } |
| |
| .figure-head { |
| display: flex; |
| align-items: flex-start; |
| justify-content: space-between; |
| gap: 1rem; |
| margin-bottom: 1rem; |
| } |
| |
| .figure-label { |
| display: block; |
| margin-bottom: 0.2rem; |
| color: var(--muted); |
| font-size: 0.69rem; |
| font-weight: 800; |
| letter-spacing: 0.1em; |
| text-transform: uppercase; |
| } |
| |
| figure h3 { margin: 0; font-size: 1.25rem; font-style: normal; } |
| |
| .controls { |
| display: flex; |
| flex-wrap: wrap; |
| gap: 0.42rem; |
| margin: 0.65rem 0 1rem; |
| } |
| |
| button, |
| select { |
| min-height: 2.75rem; |
| padding: 0.42rem 0.72rem; |
| border: 1px solid #b9b2a8; |
| border-radius: 999px; |
| color: var(--ink); |
| background: #fffdf8; |
| font-size: 0.76rem; |
| font-weight: 700; |
| cursor: pointer; |
| } |
| |
| button:hover, |
| button:focus-visible, |
| select:focus-visible { border-color: var(--sniff); outline: 3px solid rgba(23, 107, 135, 0.16); } |
| button[aria-pressed="true"] { color: #fff; border-color: var(--sniff); background: var(--sniff); } |
| |
| .figure-stats { |
| display: grid; |
| grid-template-columns: repeat(4, 1fr); |
| gap: 0.55rem; |
| margin: 0.65rem 0 0.9rem; |
| } |
| |
| .mini-stat { |
| padding: 0.65rem 0.72rem; |
| border-radius: 10px; |
| background: #f7f3eb; |
| } |
| |
| .stat-label { |
| display: block; |
| color: var(--muted); |
| font-size: 0.65rem; |
| font-weight: 750; |
| letter-spacing: 0.05em; |
| text-transform: uppercase; |
| } |
| |
| .mini-stat strong { |
| display: block; |
| margin-top: 0.12rem; |
| font: 750 1.05rem/1.2 var(--sans); |
| font-variant-numeric: tabular-nums; |
| } |
| |
| .chart-shell { |
| position: relative; |
| min-height: 300px; |
| border: 1px solid #ded7ca; |
| border-radius: 13px; |
| background: #fffdf8; |
| overflow: clip; |
| } |
| |
| svg.chart { display: block; width: 100%; height: auto; min-height: 300px; } |
| svg text { font-family: var(--sans); fill: var(--muted); } |
| .chart-scrubber { cursor: crosshair; touch-action: pan-y; } |
| .chart-scrubber:focus { outline: none; } |
| .chart-scrubber:focus-visible { |
| stroke: var(--sniff); |
| stroke-width: 4; |
| stroke-dasharray: 8 4; |
| vector-effect: non-scaling-stroke; |
| } |
| |
| .data-error { |
| display: grid; |
| min-height: 7rem; |
| margin: 0; |
| padding: 1rem; |
| place-items: center; |
| color: var(--ink); |
| background: var(--base-soft); |
| font: 700 0.82rem/1.45 var(--sans); |
| text-align: center; |
| } |
| |
| .learner-question, |
| .live-takeaway, |
| .plain-key, |
| .chart-note, |
| .transition-detail, |
| .progressive-facts, |
| .control-guide, |
| .matrix-wrap, |
| .control-stage { |
| font-family: var(--sans); |
| } |
| |
| .learner-question { |
| margin: 0 0 0.35rem; |
| font-size: clamp(1rem, 2vw, 1.16rem); |
| font-weight: 780; |
| line-height: 1.32; |
| } |
| |
| .figure-instruction { |
| margin: 0 0 0.9rem; |
| color: var(--muted); |
| font-size: 0.9rem; |
| } |
| |
| .plain-key { |
| margin: 0 0 0.9rem; |
| padding: 0.62rem 0.75rem; |
| border-left: 4px solid var(--tie); |
| background: #f7f3eb; |
| color: var(--muted); |
| font-size: 0.78rem; |
| line-height: 1.45; |
| } |
| |
| .plain-key strong { color: var(--ink); } |
| |
| .live-takeaway { |
| margin: 0.65rem 0 0.9rem; |
| padding: 0.7rem 0.8rem; |
| border: 1px solid #b9d4dc; |
| border-radius: 10px; |
| background: var(--sniff-soft); |
| color: #123f4c; |
| font-size: 0.86rem; |
| font-weight: 720; |
| line-height: 1.4; |
| } |
| |
| .live-takeaway::before { |
| content: "What to notice: "; |
| font-weight: 820; |
| } |
| |
| .chart-note { |
| margin: 0.55rem 0 0; |
| color: var(--muted); |
| font-size: 0.74rem; |
| line-height: 1.4; |
| } |
| |
| .chart-note strong { color: var(--ink); } |
| |
| .annotated-chart text.direct-label { |
| fill: var(--ink); |
| font-size: 12px; |
| font-weight: 760; |
| paint-order: stroke; |
| stroke: #fffdf8; |
| stroke-width: 4px; |
| stroke-linejoin: round; |
| } |
| |
| .ci-band { fill: rgba(23, 107, 135, 0.16); } |
| .ci-whisker { stroke: var(--sniff); stroke-width: 6; stroke-linecap: round; } |
| .judge-dot { stroke: #fffdf8; stroke-width: 2; } |
| |
| .transition-matrix { |
| display: grid; |
| grid-template-columns: minmax(7.5rem, 0.7fr) repeat(2, minmax(8.5rem, 1fr)); |
| gap: 0.45rem; |
| margin: 0.75rem 0; |
| } |
| |
| .matrix-heading, |
| .matrix-row-label, |
| .transition-cell { |
| min-width: 0; |
| padding: 0.72rem; |
| border-radius: 11px; |
| font-family: var(--sans); |
| } |
| |
| .matrix-heading, |
| .matrix-row-label { |
| display: grid; |
| align-content: center; |
| color: var(--muted); |
| background: #f1ece3; |
| font-size: 0.72rem; |
| font-weight: 780; |
| line-height: 1.25; |
| } |
| |
| .matrix-heading { text-align: center; } |
| |
| .transition-cell { |
| min-height: 7.2rem; |
| border: 1px solid #c9c2b7; |
| border-radius: 11px; |
| color: var(--ink); |
| background: #fffdf8; |
| text-align: left; |
| } |
| |
| .transition-cell strong, |
| .transition-cell span { display: block; } |
| .transition-cell strong { margin: 0.15rem 0; font-size: 1.4rem; } |
| .transition-cell span { margin-top: 0.2rem; font-size: 0.72rem; line-height: 1.35; } |
| .transition-cell[aria-pressed="true"] { border-color: var(--sniff); color: #fff; background: var(--sniff); } |
| .transition-cell[aria-pressed="true"] .stat-label { color: #fff; } |
| |
| .transition-detail { |
| min-height: 4.5rem; |
| margin: 0.55rem 0; |
| padding: 0.75rem 0.85rem; |
| border: 1px solid #ded7ca; |
| border-radius: 10px; |
| background: #fffdf8; |
| font-size: 0.82rem; |
| line-height: 1.45; |
| } |
| |
| .transition-detail strong { display: block; margin-bottom: 0.2rem; } |
| |
| details.progressive-facts { |
| margin-top: 0.65rem; |
| border: 1px solid #ded7ca; |
| border-radius: 10px; |
| background: #f7f3eb; |
| } |
| |
| details.progressive-facts summary { |
| min-height: 2.75rem; |
| padding: 0.7rem 0.8rem; |
| cursor: pointer; |
| font-size: 0.82rem; |
| font-weight: 780; |
| } |
| |
| .progressive-facts ul { margin: 0; padding: 0 1rem 0.85rem 2rem; } |
| .progressive-facts li { margin: 0.35rem 0; font-size: 0.8rem; line-height: 1.4; } |
| |
| .control-guide { |
| display: grid; |
| grid-template-columns: repeat(4, minmax(0, 1fr)); |
| gap: 0.45rem; |
| margin: 0.75rem 0 1rem; |
| } |
| |
| .guide-button { |
| min-height: 4.8rem; |
| padding: 0.7rem; |
| border-radius: 11px; |
| text-align: left; |
| line-height: 1.3; |
| } |
| |
| .guide-button b, |
| .guide-button span { display: block; } |
| .guide-button b { margin-bottom: 0.2rem; font-size: 0.69rem; letter-spacing: 0.05em; text-transform: uppercase; } |
| .guide-button span { font-size: 0.76rem; } |
| |
| .control-stage { |
| min-height: 23rem; |
| padding: 0.8rem; |
| border: 1px solid #ded7ca; |
| border-radius: 13px; |
| background: #fffdf8; |
| } |
| |
| .control-stage-head { |
| display: flex; |
| align-items: flex-start; |
| justify-content: space-between; |
| gap: 0.8rem; |
| margin-bottom: 0.7rem; |
| } |
| |
| .control-stage-head h4 { margin: 0; font: 780 1rem/1.3 var(--sans); } |
| .control-stage-head p { margin: 0.2rem 0 0; color: var(--muted); font-size: 0.78rem; } |
| |
| .slope-grid { |
| display: grid; |
| gap: 0.7rem; |
| } |
| |
| .slope-grid.three { grid-template-columns: repeat(3, minmax(0, 1fr)); } |
| |
| .slope-card { |
| padding: 0.7rem; |
| border: 1px solid #ded7ca; |
| border-radius: 10px; |
| background: #f9f6f0; |
| } |
| |
| .slope-card h5 { margin: 0 0 0.35rem; font: 780 0.78rem/1.3 var(--sans); } |
| .slope-card svg { display: block; width: 100%; height: auto; } |
| .slope-reading { margin: 0.35rem 0 0; color: var(--muted); font-size: 0.73rem; line-height: 1.35; } |
| |
| .matrix-tools { |
| display: flex; |
| flex-wrap: wrap; |
| gap: 0.45rem; |
| margin-bottom: 0.75rem; |
| } |
| |
| .matrix-wrap { |
| width: 100%; |
| overflow-x: auto; |
| overscroll-behavior-inline: contain; |
| } |
| |
| .control-matrix { |
| min-width: 720px; |
| border-collapse: separate; |
| border-spacing: 0.35rem; |
| } |
| |
| .control-matrix th, |
| .control-matrix td { border: 0; padding: 0.45rem; } |
| .control-matrix thead th { border: 0; text-align: center; } |
| .control-matrix tbody tr:last-child td { border: 0; } |
| .control-matrix .stub { width: 8rem; text-align: left; } |
| |
| .matrix-cell { |
| min-width: 8rem; |
| border: 1px solid #cfc7bb; |
| border-radius: 9px; |
| background: #f7f3eb; |
| text-align: center; |
| } |
| |
| .matrix-cell strong, |
| .matrix-cell span { display: block; } |
| .matrix-cell strong { font-size: 0.9rem; } |
| .matrix-cell span { margin-top: 0.15rem; color: var(--muted); font-size: 0.65rem; } |
| .matrix-cell.pending { |
| background: repeating-linear-gradient(135deg, #f7f3eb, #f7f3eb 8px, #e6dfd4 8px, #e6dfd4 16px); |
| color: #4d5754; |
| } |
| |
| .matrix-cell.pending strong { font-size: 0.75rem; } |
| .matrix-note { margin: 0.6rem 0 0; color: var(--muted); font-size: 0.72rem; } |
| |
| .sr-only { |
| position: absolute; |
| width: 1px; |
| height: 1px; |
| padding: 0; |
| margin: -1px; |
| overflow: hidden; |
| clip: rect(0, 0, 0, 0); |
| white-space: nowrap; |
| border: 0; |
| } |
| |
| .no-js-fallback { |
| margin: 0.65rem 0; |
| padding: 0.75rem; |
| border: 1px solid #ded7ca; |
| border-radius: 10px; |
| background: #f7f3eb; |
| font: 0.8rem/1.45 var(--sans); |
| } |
| |
| .has-js .no-js-fallback { display: none; } |
| |
| html:not(.has-js) figure.interactive .controls, |
| html:not(.has-js) figure.interactive .live-takeaway, |
| html:not(.has-js) figure.interactive .figure-stats, |
| html:not(.has-js) figure.interactive .chart-shell, |
| html:not(.has-js) figure.interactive .row-inspector, |
| html:not(.has-js) figure.interactive .transition-matrix, |
| html:not(.has-js) figure.interactive .transition-detail, |
| html:not(.has-js) figure.interactive .control-guide, |
| html:not(.has-js) figure.interactive .control-stage { display: none; } |
| |
| .row-inspector { |
| display: grid; |
| grid-template-columns: repeat(3, 1fr) auto; |
| gap: 0.65rem; |
| align-items: stretch; |
| margin-top: 0.7rem; |
| } |
| |
| .title-card { |
| min-width: 0; |
| padding: 0.62rem 0.7rem; |
| border-radius: 10px; |
| background: #f7f3eb; |
| } |
| |
| .title-card.base { border-left: 4px solid var(--base); } |
| .title-card.sniff { border-left: 4px solid var(--sniff); } |
| .title-card.teacher { border-left: 4px solid var(--tie); } |
| .title-card strong { display: block; overflow-wrap: anywhere; font: 720 0.88rem/1.3 var(--sans); } |
| .title-card .stat-label { margin-bottom: 0.2rem; } |
| |
| .delta-chip { |
| display: grid; |
| place-items: center; |
| min-width: 5.5rem; |
| padding: 0.65rem; |
| border-radius: 10px; |
| color: #fff; |
| background: #4d5754; |
| font: 750 0.9rem/1 var(--sans); |
| } |
| |
| figcaption { |
| display: grid; |
| grid-template-columns: 1fr 1fr; |
| gap: 1rem; |
| margin-top: 1rem; |
| color: var(--muted); |
| font-size: 0.78rem; |
| line-height: 1.45; |
| } |
| |
| figcaption strong { display: block; margin-bottom: 0.18rem; color: var(--ink); } |
| |
| .result-grid { |
| display: grid; |
| grid-template-columns: 1.15fr 0.85fr; |
| gap: 1rem; |
| width: min(100%, 960px); |
| margin: 1.6rem auto; |
| } |
| |
| .result-card { |
| padding: 1.15rem; |
| border: 1px solid var(--faint); |
| border-radius: 14px; |
| background: rgba(255, 255, 255, 0.46); |
| } |
| |
| .result-card h3 { margin-top: 0; } |
| .result-card p:last-child { margin-bottom: 0; } |
| |
| .transition-summary { |
| display: grid; |
| grid-template-columns: 1fr auto 1fr; |
| gap: 1rem; |
| align-items: center; |
| max-width: 680px; |
| margin: 0.5rem auto 1.2rem; |
| text-align: center; |
| } |
| |
| .transition-summary strong { |
| display: block; |
| font: 780 clamp(2rem, 5vw, 3.6rem)/1 var(--sans); |
| letter-spacing: -0.045em; |
| } |
| |
| .transition-summary .base strong { color: var(--base); } |
| .transition-summary .sniff strong { color: var(--sniff); } |
| .transition-summary .arrow { color: var(--muted); font: 400 2rem/1 var(--sans); } |
| .transition-summary span { font: 650 0.74rem/1.3 var(--sans); color: var(--muted); } |
| |
| .formula { |
| margin: 1rem 0; |
| padding: 0.9rem 1rem; |
| border-left: 4px solid var(--sniff); |
| background: rgba(255, 255, 255, 0.52); |
| font-size: 1.03rem; |
| overflow-x: auto; |
| } |
| |
| .formula-label { |
| display: block; |
| margin-bottom: 0.3rem; |
| color: var(--muted); |
| font-size: 0.67rem; |
| font-weight: 780; |
| letter-spacing: 0.08em; |
| text-transform: uppercase; |
| } |
| |
| .hypotheses { |
| display: grid; |
| grid-template-columns: repeat(2, 1fr); |
| gap: 0.7rem; |
| margin: 1.2rem 0; |
| } |
| |
| .hypothesis { |
| padding: 0.85rem; |
| border: 1px solid var(--faint); |
| border-radius: 11px; |
| background: rgba(255, 255, 255, 0.43); |
| } |
| |
| .hypothesis strong { display: block; font-family: var(--sans); font-size: 0.88rem; } |
| .hypothesis span { color: var(--muted); font-size: 0.85rem; } |
| |
| .table-wrap { margin: 1.2rem 0; overflow-x: auto; } |
| table { width: 100%; border-collapse: collapse; font-size: 0.78rem; line-height: 1.4; } |
| th, td { padding: 0.5rem; border-bottom: 1px solid #ddd5c8; text-align: left; vertical-align: top; } |
| thead th { border-block: 2px solid var(--ink); font-weight: 760; } |
| td.num, th.num { text-align: right; font-variant-numeric: tabular-nums; } |
| tbody tr:last-child td { border-bottom: 2px solid var(--ink); } |
| |
| .method-strip { |
| display: grid; |
| grid-template-columns: repeat(4, 1fr); |
| gap: 0.6rem; |
| margin: 1.4rem 0; |
| } |
| |
| .method-step { |
| position: relative; |
| padding: 0.8rem; |
| border: 1px solid var(--faint); |
| border-radius: 11px; |
| background: rgba(255, 255, 255, 0.46); |
| font: 0.78rem/1.4 var(--sans); |
| } |
| |
| .method-step b { display: block; margin-bottom: 0.25rem; color: var(--sniff); font-size: 0.7rem; letter-spacing: 0.07em; text-transform: uppercase; } |
| |
| .boundary { |
| margin: 1.3rem 0; |
| padding: 1rem 1.1rem; |
| border: 1px solid #d7b5a9; |
| border-radius: 12px; |
| background: var(--base-soft); |
| } |
| |
| .boundary strong { font-family: var(--sans); } |
| |
| .refs { |
| margin: 0.8rem 0 0; |
| padding: 0; |
| list-style: none; |
| counter-reset: refs; |
| } |
| |
| .refs li { |
| position: relative; |
| margin: 0.55rem 0; |
| padding-left: 2rem; |
| counter-increment: refs; |
| font-size: 0.88rem; |
| } |
| |
| .refs li::before { |
| content: counter(refs) "."; |
| position: absolute; |
| left: 0; |
| color: var(--muted); |
| font-family: var(--sans); |
| font-size: 0.78rem; |
| } |
| |
| footer { |
| width: min(100%, 760px); |
| margin: 4rem auto 0; |
| padding-top: 1rem; |
| border-top: 1px solid var(--faint); |
| color: var(--muted); |
| font: 0.75rem/1.45 var(--sans); |
| } |
| |
| code, .mono { font-family: var(--mono); font-size: 0.86em; } |
| .sniff-text { color: var(--sniff); } |
| .base-text { color: var(--base); } |
| .muted { color: var(--muted); } |
| .nowrap { white-space: nowrap; } |
| .js-only { display: none; } |
| .has-js .js-only { display: initial; } |
| |
| @media (max-width: 800px) { |
| .paper { width: min(calc(100% - 1.1rem), 1120px); padding-top: 2.5rem; } |
| .claim-ladder, .result-grid { grid-template-columns: 1fr; } |
| .claim { min-height: 0; } |
| .figure-stats { grid-template-columns: repeat(2, 1fr); } |
| .row-inspector { grid-template-columns: 1fr; } |
| .delta-chip { min-height: 2.5rem; } |
| .method-strip { grid-template-columns: repeat(2, 1fr); } |
| figcaption { grid-template-columns: 1fr; } |
| .control-guide { grid-template-columns: repeat(2, minmax(0, 1fr)); } |
| .slope-grid.three { grid-template-columns: 1fr; } |
| } |
| |
| @media (max-width: 520px) { |
| body { font-size: 16px; } |
| h1 { font-size: clamp(2.75rem, 14vw, 3.35rem); } |
| .hero-numbers, .hypotheses { grid-template-columns: 1fr; } |
| .abstract { padding: 1.1rem; } |
| .figure-head { display: block; } |
| .method-strip { grid-template-columns: 1fr; } |
| .transition-matrix { grid-template-columns: 5.6rem repeat(2, minmax(0, 1fr)); gap: 0.3rem; } |
| .matrix-heading, .matrix-row-label, .transition-cell { padding: 0.5rem; } |
| .transition-cell { min-height: 7.8rem; } |
| .control-guide { grid-template-columns: 1fr; } |
| .control-stage { padding: 0.55rem; } |
| .chart-shell { min-height: 250px; } |
| svg.chart { min-height: 250px; } |
| .control-stage-head { display: block; } |
| .control-stage-head > .tiny { display: block; margin-top: 0.35rem; } |
| .matrix-wrap { overflow: visible; } |
| .control-matrix { min-width: 0; width: 100%; border-spacing: 0; } |
| .control-matrix thead { position: absolute; width: 1px; height: 1px; margin: -1px; overflow: hidden; clip: rect(0, 0, 0, 0); } |
| .control-matrix tbody, |
| .control-matrix tr, |
| .control-matrix th, |
| .control-matrix td { display: block; width: 100%; } |
| .control-matrix tbody { display: grid; gap: 0.65rem; } |
| .control-matrix tbody tr { padding: 0.55rem; border: 1px solid #ded7ca; border-radius: 10px; } |
| .control-matrix .stub { width: 100%; padding: 0.2rem 0.3rem 0.55rem; } |
| .control-matrix td { margin-top: 0.35rem; padding: 0.65rem; } |
| .control-matrix td::before { content: attr(data-label); display: block; margin-bottom: 0.2rem; color: var(--muted); font-size: 0.65rem; font-weight: 780; text-transform: uppercase; } |
| } |
| |
| @media (prefers-reduced-motion: reduce) { |
| html { scroll-behavior: auto; } |
| * { transition-duration: 0.01ms !important; animation-duration: 0.01ms !important; } |
| } |
| |
| @media print { |
| :root { --paper: #fff; } |
| body { background: #fff; font-size: 10.5pt; } |
| .paper { width: 100%; padding: 0; } |
| .reading-progress, nav.contents, .controls, .row-inspector, .control-guide, .matrix-tools { display: none !important; } |
| figure.interactive { box-shadow: none; break-inside: auto; } |
| .control-stage { min-height: 0; } |
| .chart-shell { min-height: 0; overflow: visible; } |
| svg.chart { min-height: 0; } |
| .transition-matrix { break-inside: avoid; } |
| details.progressive-facts[open] { break-inside: avoid; } |
| details.progressive-facts > :not(summary) { display: block !important; } |
| .no-js-fallback { display: block !important; } |
| .claim-ladder, .hero-numbers { break-inside: avoid; } |
| a { color: inherit; text-decoration: none; } |
| h2, h3 { break-after: avoid; } |
| @page { margin: 18mm; } |
| } |
| </style> |
| </head> |
| <body> |
| <div class="reading-progress" id="reading-progress" aria-hidden="true"></div> |
| <main class="paper" id="paper"> |
| <header class="front"> |
| <p class="eyebrow">Exploratory model analysis</p> |
| <h1>The Sniff Test</h1> |
| <p class="subtitle">A seed lottery in a 12.7M-parameter language model</p> |
| <p class="meta"> |
| <span>MD Ishtiaque Hossain</span> |
| <span>PorkiCoder Research</span> |
| <span>12 August 2026</span> |
| <span>12.7M parameter T5 student</span> |
| </p> |
|
|
| <section class="abstract" id="abstract"> |
| <h2>Abstract</h2> |
| <p> |
| <strong>In one sentence: a tiny model appeared to stop repeating words after its embeddings read Verne, but the favorable direction did not recur consistently with matched random starts, so one run is not evidence of a mechanism.</strong> In the original pair, repeated-token titles fell by 37%, from 16.0% to 10.1%, while teacher similarity fell by less than 0.2%; the visible error changed without a fidelity gain. Among titles of at least three tokens, repeats fell by 42% while the number of titles that long fell by only 0.9%, arguing against an explanation based solely on fewer longer titles. The comparison had also changed the random transformer body. In a deterministic 31-run control, ordered Verne produced 6.9% more repeated titles than frequency-matched shuffled Verne, 14.75% versus 13.79%. It was more repetitive in six of nine matches, tied once, less repetitive twice, and 1.0% lower in teacher similarity. The data do not support a stable literary sequence-order effect. |
| </p> |
| </section> |
| </header> |
|
|
| <nav class="contents" aria-label="Paper sections"> |
| <a href="#paradox">The paradox</a> |
| <a href="#stutter">The signal</a> |
| <a href="#mechanism">Mechanism</a> |
| <a href="#controls">Control test</a> |
| <a href="#method">Method</a> |
| <a href="#limits">Boundaries</a> |
| <a href="#future">Future work</a> |
| </nav> |
|
|
| <section class="measure" id="paradox"> |
| <span class="kicker">1. The paradox</span> |
| <h2>The average stood still. The model did not.</h2> |
| <p class="lead"> |
| The sniff checkpoint scored 0.3512 against the teacher; the baseline scored 0.3519. Their paired mean difference was -0.00066, but they emitted different titles on 661 of 907 tasks. The apparent tie hides a redistribution of errors. |
| </p> |
| </section> |
|
|
| <div class="hero-numbers" aria-label="Headline score comparison"> |
| <div class="hero-number"> |
| <span class="value neutral">-0.00066</span> |
| <p>Mean change in automatic teacher similarity. The thread-cluster bootstrap 95% interval crosses zero: about -0.0114 to +0.0100.</p> |
| </div> |
| <div class="hero-number"> |
| <span class="value sniff">+0.152</span> |
| <p>Mean change from the later 0 to 10 judge. This number is descriptive because evaluator calls were unblinded and dependent.</p> |
| </div> |
| </div> |
|
|
| <figure class="interactive" id="paradox-figure"> |
| <div class="figure-head"> |
| <div> |
| <span class="figure-label">Interactive 1</span> |
| <h3>The cancellation engine</h3> |
| </div> |
| <span class="tiny muted">Each point is an aggregate block</span> |
| </div> |
| <p class="learner-question">How can the average stay near zero when hundreds of titles change?</p> |
| <p class="figure-instruction">Choose a score and subset, then inspect aggregate blocks with a pointer or arrow keys.</p> |
| <p class="plain-key"><strong>Key.</strong> Change means sniff minus baseline. Above zero favors sniff; below zero favors baseline.</p> |
| <div class="controls" role="group" aria-label="Choose cancellation chart data"> |
| <button type="button" data-paradox-metric="dS" aria-pressed="true">Similarity to teacher</button> |
| <button type="button" data-paradox-metric="dJ" aria-pressed="false">Later judge score</button> |
| <select id="paradox-filter" aria-label="Task subset"> |
| <option value="all">All 907 tasks</option> |
| <option value="changed">661 changed titles</option> |
| <option value="exact_unseen">855 exact-input-unseen</option> |
| <option value="thread_disjoint">288 known thread-disjoint</option> |
| </select> |
| </div> |
| <p class="live-takeaway" id="paradox-takeaway" aria-live="polite">Opposing gains and losses nearly cancel even though the outputs differ.</p> |
| <div class="figure-stats" id="paradox-stats"></div> |
| <div class="chart-shell"> |
| <svg class="chart annotated-chart" id="paradox-chart" viewBox="0 0 920 340" role="group" aria-label="Cumulative paired score difference with an interactive aggregate-block inspector"></svg> |
| </div> |
| <p class="chart-note"><strong>Reading the line.</strong> The endpoint is the net after block gains and losses cancel. Mixed order is not training time.</p> |
| <div class="row-inspector" id="row-inspector"> |
| <div class="title-card base"><span class="stat-label">Aggregate block</span><strong>Use the chart slider</strong></div> |
| <div class="title-card sniff"><span class="stat-label">Sniff / tie / baseline</span><strong>to inspect grouped counts</strong></div> |
| <div class="title-card teacher"><span class="stat-label">Privacy</span><strong>No row-level outcomes published</strong></div> |
| <div class="delta-chip">Net</div> |
| </div> |
| <p class="no-js-fallback"><strong>Static reading.</strong> Gains of +40.819 and losses of -41.415 nearly cancel; mean change is -0.00066.</p> |
| <figcaption> |
| <span><strong>Takeaway</strong>The endpoint is nearly zero for S because +40.819 similarity points and -41.415 points almost exactly cancel. A mean tie is not behavioral identity.</span> |
| <span><strong>What this does not show</strong>The sorted views are explanatory orderings, not time. Task-level movement does not estimate retraining variance.</span> |
| </figcaption> |
| </figure> |
|
|
| <section class="measure"> |
| <p> |
| The sniff checkpoint won on 285 tasks by S, tied on 306, and lost on 316. Mean absolute movement was 0.0907, about 138 times larger than the net mean. The judge advantage was concentrated where S also rose: those rows summed to +244 judge points, versus -108 where S fell and +2 on S ties. This accounting does not identify why judge scores differed or rule out dimensions orthogonal to S. |
| </p> |
| <p> |
| Nor did the sniff rise on a lexical task-overlap proxy or distributional diversity. Unique titles fell from 313 to 306, title entropy fell from 5.149 to 5.101 nats, and mean task-token overlap fell from 0.2962 to 0.2924. <em>Review Auth</em> expanded from 16 to 51 outputs; on those 51 sniff rows, paired judge differences summed to -44. One narrower output pattern differed. |
| </p> |
| </section> |
|
|
| <figure class="interactive" id="judge-figure"> |
| <div class="figure-head"> |
| <div> |
| <span class="figure-label">Interactive 2</span> |
| <h3>One mean, two units of uncertainty</h3> |
| </div> |
| <span class="tiny muted">Two checks, one estimate</span> |
| </div> |
| <p class="learner-question">How much confidence should we place in the later judge advantage?</p> |
| <p class="figure-instruction">Switch between uncertainty across evaluation threads and across nine complete scoring calls.</p> |
| <p class="plain-key"><strong>Key.</strong> The dot is the mean; the band is its 95% interval. Crossing zero means no average difference remains compatible with that check.</p> |
| <div class="controls" role="group" aria-label="Choose judge uncertainty check"> |
| <button type="button" data-judge-unit="task" aria-pressed="true">Main check: resample threads</button> |
| <button type="button" data-judge-unit="batch" aria-pressed="false">Sensitivity: compare scoring calls</button> |
| </div> |
| <p class="live-takeaway" id="judge-takeaway" aria-live="polite">The thread-resampling interval stays above zero, but this does not remove evaluator bias.</p> |
| <div class="figure-stats" id="judge-stats"></div> |
| <div class="chart-shell"> |
| <svg class="chart annotated-chart" id="judge-chart" viewBox="0 0 920 330" role="img" aria-label="Judge score changes with a 95 percent uncertainty interval"></svg> |
| </div> |
| <p class="chart-note"><strong>Dots are scoring calls, not a time series.</strong> Nine cover 100 tasks; the last covers seven.</p> |
| <p class="no-js-fallback"><strong>Static reading.</strong> The thread interval is +0.039 to +0.266; the nine-call interval, -0.017 to +0.337, crosses zero.</p> |
| <figcaption> |
| <span><strong>Takeaway</strong>The thread-cluster interval excludes zero, but an unweighted t sensitivity across nine complete 100-task blocks includes zero.</span> |
| <span><strong>What this does not show</strong>The blocks are sequential batches from one evaluator, not independent replications. The rubric named systems, exposed earlier means, fixed candidate order, and capped doubled titles at four.</span> |
| </figcaption> |
| </figure> |
|
|
| <section class="measure" id="stutter"> |
| <span class="kicker">2. The signal inside the tie</span> |
| <h2>The sniff checkpoint stuttered less.</h2> |
| <p class="lead"> |
| Without using judge scores in its definition, we post hoc flag a title that repeats any case-insensitive ASCII-alphanumeric token, as in <em>Test Test</em> or <em>Bot Bot Status</em>. The exploratory flag is a deterministic property of the generated title. |
| </p> |
| </section> |
|
|
| <figure class="interactive" id="repeat-figure"> |
| <div class="figure-head"> |
| <div> |
| <span class="figure-label">Interactive 3</span> |
| <h3>The repetition machine</h3> |
| </div> |
| <span class="tiny muted">Paired 2 × 2 table</span> |
| </div> |
| <p class="learner-question">Which tasks account for the drop in repeated-title outputs?</p> |
| <p class="figure-instruction">Select a matrix cell to read its count and judge-score contribution.</p> |
| <p class="plain-key"><strong>Key.</strong> Each task occupies one cell. Rows describe baseline repetition; columns describe sniff repetition.</p> |
| <div class="transition-summary" aria-label="Repeated token title rate"> |
| <div class="base"><strong>145</strong><span>baseline repeats<br>16.0%</span></div> |
| <div class="arrow" aria-hidden="true">→</div> |
| <div class="sniff"><strong>92</strong><span>sniff repeats<br>10.1%</span></div> |
| </div> |
| <p class="live-takeaway" id="repeat-takeaway" aria-live="polite">Repairs outnumber new stutters 87 to 34, producing 53 fewer repeated-title outputs.</p> |
| <div class="transition-matrix" id="transition-matrix" role="group" aria-label="Paired repeated-title transition matrix"></div> |
| <div class="transition-detail" id="transition-detail">Select a matrix cell for its aggregate interpretation.</div> |
| <details class="progressive-facts"> |
| <summary>Is the drop just because sniff titles became shorter?</summary> |
| <ul> |
| <li>Titles with at least three tokens barely changed: 430 baseline versus 426 sniff.</li> |
| <li>Repeated titles with at least three tokens fell from 121 to 70, a 42% reduction.</li> |
| <li>Exact two-token doubles barely changed: 24 baseline versus 22 sniff.</li> |
| </ul> |
| </details> |
| <p class="no-js-fallback"><strong>Static reading.</strong> Counts are 728 stayed clean, 87 repaired, 34 new stutters, and 58 stayed repeated.</p> |
| <figcaption> |
| <span><strong>Takeaway</strong>The checkpoint removed repetition on 87 tasks and introduced it on 34. The difference is -5.84 percentage points, with 95% thread-cluster bootstrap interval about -8.16 to -3.58. The post hoc, unadjusted McNemar p is 1.57 × 10<sup>-6</sup>.</span> |
| <span><strong>What this does not show</strong>The rubric's “garbled doubles” category only partly matches this exact-token flag, and it explicitly penalized doubled titles. The repetition count itself does not use judge scores.</span> |
| </figcaption> |
| </figure> |
|
|
| <section class="measure"> |
| <p> |
| The same direction appears in one nested subset and one older ID-disjoint panel. On 288 tasks with known conversation threads disjoint from training, repeats fell from 46 to 30 (nominal exact paired p = 0.0166). On the repeatedly inspected 62-task panel they fell from 10 to 4 (p = 0.070). In the full set, repeated-token titles of at least three tokens fell from 121 to 70, a 42% reduction. The number of titles that long barely moved, from 430 to 426, arguing against an explanation based solely on fewer titles reaching three tokens. Exact two-token doubles changed only from 24 to 22 (p = 0.824). Removing repetition aligned with +65 judge points and introducing it with -29. Their +36 arithmetic difference is 26% of the total +138, not a causal mediation estimate. |
| </p> |
| <div class="result-grid"> |
| <div class="result-card"> |
| <h3>Consistent fixed-checkpoint pattern</h3> |
| <p>The realized sniff checkpoint emits fewer repeated-token titles across the development evaluation and both reported subsets, mostly among longer titles.</p> |
| </div> |
| <div class="result-card"> |
| <h3>Not a general quality win</h3> |
| <p>Teacher similarity, exact match, two-word compliance, lexical task overlap, title diversity, and grammar validity do not show a reliable lift. On 62 tasks, S fell from 0.2109 to 0.1916.</p> |
| </div> |
| </div> |
| </section> |
|
|
| <section class="measure" id="mechanism"> |
| <span class="kicker">3. Why the mirage was plausible</span> |
| <h2>A local decoding change can look like learned restraint.</h2> |
| <p class="lead"> |
| Repetition is a known failure mode of autoregressive generation. One token wins, becomes the next decoder input, and helps itself or a related frequent token win again. |
| </p> |
| <p> |
| T5 ties its input embedding to its output projection, so the intervention touched both sides of this loop. A copied row could change how a token enters the decoder and how decoder state becomes its next-token score. It need not teach software semantics to alter local margins among common words. |
| </p> |
| <div class="formula" role="note"> |
| <span class="formula-label">A mechanistic sketch, not an identified causal path</span> |
| sniffed tied rows → different early token margins → different greedy path → fewer self-reinforcing repeats |
| </div> |
| <p> |
| This sketch explains how one checkpoint could change, not why the grid changed. KD pulls students toward one teacher, but training can preserve different boundaries near greedy ties. A small displacement can reroute decoding without moving the average. Prior work establishes the ingredients; we did not measure token margins inside these models. |
| </p> |
| <p> |
| This is underspecification: random training choices return models with similar aggregate scores but different stress-test behavior. The original fixed pair reduced repeated titles by 53, but it was confounded. One matched ordered arm reduced them by 44 against its body-matched baseline, yet direction varied across the grid. Preserving Verne's order improved neither repetition nor fidelity on average. |
| </p> |
| </section> |
|
|
| <section class="measure" id="controls"> |
| <span class="kicker">4. The matched test</span> |
| <h2>The favorable direction does not recur consistently.</h2> |
| <p class="lead"> |
| Audit found that the original baseline and sniff checkpoint did not start from the same transformer body. The transplant script created a fresh, unseeded body in a separate process. The original comparison therefore changed both the embedding rows and the body initialization. |
| </p> |
| <p> |
| We recommissioned an RTX 6000 Ada and built a crossed follow-up. Three body seeds were held bit-identical across arms. Three donor seeds created each embedding treatment. Every arm received the same 3,000-step KD schedule. The designated baseline and no-op pair produced identical histories and outputs; its run log recorded identical checkpoint hashes before those weights were stripped. |
| </p> |
| <p> |
| The random donor is not a pure pretraining control: it was initialized and saved in fp32, whereas ordered and shuffled donors trained in bf16 before fp32 export, so random-versus-trained does not isolate pretraining from precision. Ordered versus shuffled shares initialization, precision path, token multiset, and schedule, making it the narrower sequence-order test. |
| </p> |
| <div class="table-wrap" aria-labelledby="control-result-caption"> |
| <table> |
| <caption id="control-result-caption"><strong>Complete 3 by 3 sequence-order contrast.</strong> Negative values favor ordered Verne; positive values mean ordered emitted more repeated titles than frequency-matched shuffled Verne. Ordered improved 2 matches, tied 1, and worsened 6.</caption> |
| <thead> |
| <tr><th>Ordered minus shuffled repeated titles</th><th class="num">Donor 501</th><th class="num">Donor 502</th><th class="num">Donor 503</th></tr> |
| </thead> |
| <tbody> |
| <tr><td>Body 101</td><td class="num">-6</td><td class="num">+5</td><td class="num">-3</td></tr> |
| <tr><td>Body 202</td><td class="num">+31</td><td class="num">+13</td><td class="num">+17</td></tr> |
| <tr><td>Body 303</td><td class="num">+3</td><td class="num">+18</td><td class="num">0</td></tr> |
| </tbody> |
| </table> |
| </div> |
| <p class="note"> |
| <strong>Complete result, 31 of 31 runs.</strong> Ordered Verne averaged 0.96 percentage points, or 8.7 titles per 907 tasks, more repetition than shuffled Verne. Its teacher similarity averaged 0.00358 lower and improved in only four of nine matches. After subtracting each body's baseline and averaging the nine fixed cells, all three families had more repetition and lower S overall: random +1.67 points and -0.00773 S, shuffled +1.00 and -0.00291, ordered +1.96 and -0.00649. The fixed grid is too small for a population estimate and provides no support for the proposed stable sequence-order advantage. |
| </p> |
| </section> |
|
|
| <figure class="interactive" id="control-figure"> |
| <div class="figure-head"> |
| <div> |
| <span class="figure-label">Interactive 4</span> |
| <h3>The matched-seed walkthrough</h3> |
| </div> |
| <span class="tiny muted" id="control-status">Checking control data</span> |
| </div> |
| <p class="learner-question">Do matched starts preserve the apparent benefit, and does ordered Verne beat shuffled Verne consistently?</p> |
| <p class="figure-instruction">First compare each transplant with its body-matched baseline, then inspect the cleaner ordered-versus-shuffled contrast.</p> |
| <p class="plain-key"><strong>Key.</strong> Body draw sets the student; donor draw sets transplanted rows. Percentage points, or pp, compare repetition rates.</p> |
| <p class="live-takeaway" id="control-takeaway" aria-live="polite">Control status and matched outcomes are read from the validated payload.</p> |
| <div class="control-guide" role="group" aria-label="Control walkthrough steps"> |
| <button type="button" class="guide-button" data-control-step="1" aria-pressed="true" aria-controls="control-stage"><b>Step 1</b><span>One matched comparison</span></button> |
| <button type="button" class="guide-button" data-control-step="2" aria-pressed="false" aria-controls="control-stage"><b>Step 2</b><span>Hold donor draw fixed</span></button> |
| <button type="button" class="guide-button" data-control-step="3" aria-pressed="false" aria-controls="control-stage"><b>Step 3</b><span>Hold body draw fixed</span></button> |
| <button type="button" class="guide-button" data-control-step="4" aria-pressed="false" aria-controls="control-stage"><b>Step 4</b><span>See the full matrix</span></button> |
| </div> |
| <div class="figure-stats" id="control-stats"></div> |
| <div class="control-stage" id="control-stage"></div> |
| <p class="no-js-fallback"><strong>Static reading.</strong> All 31 runs completed. Ordered Verne produced fewer repeats than shuffled Verne in two matched cells, tied once, and produced more in six; its mean difference was +0.96 percentage points.</p> |
| <figcaption> |
| <span><strong>Takeaway</strong><span id="control-caption-takeaway">Control interpretation is generated from the validated run matrix.</span></span> |
| <span><strong>What this does not show</strong>Three body seeds and three donor seeds support a mechanism screen, not a universal estimate across corpora, architectures, or training recipes.</span> |
| </figcaption> |
| </figure> |
|
|
| <section class="measure" id="method"> |
| <span class="kicker">5. Method in brief</span> |
| <h2>A small experiment with a narrow claim.</h2> |
| <div class="method-strip" aria-label="Experimental pipeline"> |
| <div class="method-step"><b>1. Sniff</b>One epoch, 335 steps, of T5 span corruption on 5,488,228 Verne SentencePiece tokens.</div> |
| <div class="method-step"><b>2. Transplant</b>Keep 6,985 tied input and output rows; initialize a 12.7M parameter student.</div> |
| <div class="method-step"><b>3. Distill</b>Train 3,000 steps on 3,847 grammar-gated teacher titles; select on validation-100 S.</div> |
| <div class="method-step"><b>4. Compare</b>Pair predictions by task; inspect similarity, exactness, validity, repetition, and judge score.</div> |
| </div> |
|
|
| <h3>Model and objective</h3> |
| <p> |
| The teacher is a 77M parameter FLAN-T5-small model fine-tuned for terminal-tab titles. The comparator, <span class="mono">kd_vocab</span>, is a randomly initialized T5 student with width 384, four encoder layers, two decoder layers, and a 6,985-id vocabulary. The sniff arm replaces its tied input and output rows with rows from a 32,100-id Verne donor. The original script did not match the remaining random body. Targets are valid teacher replies, not human-gold titles. Training minimizes: |
| </p> |
| <div class="formula"> |
| <span class="formula-label">Knowledge-distillation loss</span> |
| L = 0.7 T<sup>2</sup> KL(p<sub>teacher</sub><sup>T</sup> || p<sub>student</sub><sup>T</sup>) + 0.3 CE, with T = 2. |
| </div> |
| <p> |
| The KL direction above matches the implementation. The sniff uses sequence length 512, 15% corruption, mean span length 3, batch 32, AdamW learning rate 2 × 10<sup>-3</sup>, and 50 warmup steps. Its Python seed 42 controlled masking and order, but the original donor initialization was not torch-seeded. KD uses batch 32, learning rate 3 × 10<sup>-4</sup>, 100 warmup steps, seed 42, and 3,000 steps. Decoding is greedy with a 12-token cap and no-repeat-bigram constraint. That constraint blocks a repeated bigram, not a unigram appearing twice. It does not enforce the desired one-to-three-word product rule. Exactly two words occurred on 460 of 907 sniff titles and 461 baseline titles. |
| </p> |
|
|
| <h3>Evaluation and uncertainty</h3> |
| <p> |
| The automatic score is the mean of normalized character similarity and token F1 against the teacher title: |
| </p> |
| <div class="formula"> |
| <span class="formula-label">Paired teacher similarity</span> |
| S<sub>i</sub> = 0.5 [lev_sim(y<sub>i</sub>, t<sub>i</sub>) + token_F1(y<sub>i</sub>, t<sub>i</sub>)]. |
| </div> |
| <p> |
| Reported 95% intervals resample 20,000 evaluation-thread clusters; rows without a known thread are singleton clusters. Exact McNemar tests use discordant row pairs and are labeled nominal because they do not cluster threads or adjust for post hoc metric selection. Intervals describe this task collection, not retraining seeds or users. The judge figure also reports an unweighted t sensitivity across nine complete 100-task scoring blocks. Those sequential blocks came from one evaluator and are not independent replications. All 907 analyses are exploratory because this set was repeatedly inspected during the campaign. |
| </p> |
|
|
| <h3>Data boundary</h3> |
| <p> |
| The 907 rows are ID-disjoint from the title-training split, but 52 serialized inputs occur in training under other IDs. Of 758 rows with known conversation IDs, 470 share a thread with training; 288 are demonstrably thread-disjoint. On the full set, exact teacher-title counts are 62 for baseline and 63 for sniff. After removing exact-input overlaps, they reverse to 56 for baseline and 55 for sniff. The S tie (-0.00036) and repetition gap (130 to 82) remain. We therefore call this a reused development evaluation, not an untouched holdout. |
| </p> |
|
|
| <div class="table-wrap"> |
| <table> |
| <caption><strong>Fixed-checkpoint outcomes on 907 tasks.</strong> Paired change is sniff minus baseline.</caption> |
| <thead> |
| <tr><th>Measure, 907 tasks</th><th class="num">Baseline</th><th class="num">Sniff checkpoint</th><th class="num">Paired change</th></tr> |
| </thead> |
| <tbody> |
| <tr><td>Teacher similarity S</td><td class="num">0.3519</td><td class="num">0.3512</td><td class="num">-0.00066</td></tr> |
| <tr><td>Exact teacher title</td><td class="num">62</td><td class="num">63</td><td class="num">+1</td></tr> |
| <tr><td>Valid one-to-three-word title</td><td class="num">864</td><td class="num">870</td><td class="num">+6</td></tr> |
| <tr><td>Exactly two words</td><td class="num">461</td><td class="num">460</td><td class="num">-1</td></tr> |
| <tr><td>Repeated-token title</td><td class="num">145</td><td class="num">92</td><td class="num">-53</td></tr> |
| <tr><td>Unblinded judge, 0 to 10</td><td class="num">2.991</td><td class="num">3.143</td><td class="num">+0.152</td></tr> |
| </tbody> |
| </table> |
| </div> |
| </section> |
|
|
| <section class="measure" id="limits"> |
| <span class="kicker">6. Claim boundary</span> |
| <h2>What survived the sniff test.</h2> |
| <div class="claim-ladder"> |
| <div class="claim robust"> |
| <span class="claim-tag">Observed in the original pair</span> |
| <strong>The realized sniff checkpoint emitted fewer repeated-token titles.</strong> |
| <p>The 37% reduction is genuine for those two fixed checkpoints and remains after the reported data checks.</p> |
| </div> |
| <div class="claim descriptive"> |
| <span class="claim-tag">Supported by matched controls</span> |
| <strong>Preserving Verne sequence order had no stable advantage.</strong> |
| <p>Ordered was more repetitive in six of nine ordered-versus-shuffled matches and slightly worse on both balanced means.</p> |
| </div> |
| <div class="claim unproven"> |
| <span class="claim-tag">Not established</span> |
| <strong>Small models are universally dominated by seed.</strong> |
| <p>We tested one 12.7M architecture and nine fixed body-donor matches, not model size as an experimental factor.</p> |
| </div> |
| </div> |
| <div class="boundary"> |
| <strong>Final claim.</strong> The original fixed-pair reduction of 53 repeated titles is genuine behavior of those two checkpoints, not evidence that Verne taught restraint. Its favorable direction did not recur consistently in the matched grid. A single tiny-model run can show a conspicuous behavioral improvement with no aggregate fidelity gain, then fail to generalize across starts. This is a concrete seed-lottery autopsy, not a new universal law of small models. |
| </div> |
| </section> |
|
|
| <section class="measure" id="future"> |
| <span class="kicker">Future research</span> |
| <h2>Three decisive tests, in order.</h2> |
| <ol> |
| <li><strong>Test the decoding mechanism.</strong> Compare tied and untied embeddings while tracking token logits and top-two margins. If the proposed feedback loop is real, margin changes should precede repeated decoding.</li> |
| <li><strong>Predict robust starts instead of cherry-picking them.</strong> Screen many starts with inexpensive early diagnostics and a multi-objective validation rule, then evaluate the chosen checkpoint once on untouched data. Compare this seed-screening policy with random restarts under equal compute.</li> |
| <li><strong>Test the size claim.</strong> Use identical precision paths, more body and donor draws, several model sizes and corpora, a locked thread-disjoint set, and blinded randomized-order human ratings.</li> |
| </ol> |
| </section> |
|
|
| <section class="measure" id="references"> |
| <span class="kicker">References</span> |
| <ol class="refs"> |
| <li>Raffel, C. et al. 2020. <a href="https://jmlr.org/papers/v21/20-074.html">Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer</a>. JMLR 21(140).</li> |
| <li>Hinton, G., Vinyals, O., and Dean, J. 2015. <a href="https://arxiv.org/abs/1503.02531">Distilling the Knowledge in a Neural Network</a>.</li> |
| <li>Gururangan, S. et al. 2020. <a href="https://aclanthology.org/2020.acl-main.740/">Don't Stop Pretraining: Adapt Language Models to Domains and Tasks</a>. ACL.</li> |
| <li>Press, O. and Wolf, L. 2017. <a href="https://aclanthology.org/E17-2025/">Using the Output Embedding to Improve Language Models</a>. EACL.</li> |
| <li>Holtzman, A. et al. 2020. <a href="https://openreview.net/forum?id=rygGQyrFvH">The Curious Case of Neural Text Degeneration</a>. ICLR.</li> |
| <li>Welleck, S. et al. 2020. <a href="https://openreview.net/forum?id=SJeYe0NtvH">Neural Text Generation with Unlikelihood Training</a>. ICLR.</li> |
| <li>D'Amour, A. et al. 2022. <a href="https://jmlr.org/papers/v23/20-1335.html">Underspecification Presents Challenges for Credibility in Modern Machine Learning</a>. JMLR 23(226).</li> |
| <li>Mosbach, M., Andriushchenko, M., and Klakow, D. 2021. <a href="https://openreview.net/forum?id=nzpLWnVAyah">On the Stability of Fine-tuning BERT</a>. ICLR.</li> |
| <li>Reimers, N. and Gurevych, I. 2017. <a href="https://aclanthology.org/D17-1035/">Reporting Score Distributions Makes a Difference</a>. EMNLP.</li> |
| <li>Dodge, J. et al. 2020. <a href="https://arxiv.org/abs/2002.06305">Fine-Tuning Pretrained Language Models: Weight Initializations, Data Orders, and Early Stopping</a>.</li> |
| </ol> |
| </section> |
|
|
| <footer> |
| <p><strong>Reproducibility.</strong> This two-file web artifact loads <code>assets/sniff_paper_data.js</code>, generated from aligned prediction and judge records by <code>scripts/build_sniff_paper_data.py</code>. The public payload contains aggregate outcome blocks, not production task text, source IDs, or row-level outcomes. The designated full 3,000-step baseline/no-op pair has identical released histories and outputs; its run log records identical checkpoint hashes, but stripped weights prevent recomputing that byte comparison now.</p> |
| <p>The Sniff Test · MD Ishtiaque Hossain · PorkiCoder Research · 12 August 2026</p> |
| </footer> |
| </main> |
|
|
| <script src="assets/sniff_paper_data.js"></script> |
| <script> |
| document.documentElement.classList.add("has-js"); |
| |
| (() => { |
| "use strict"; |
| |
| const isRecord = (value) => value !== null && typeof value === "object" && !Array.isArray(value); |
| const isFiniteNumber = (value) => typeof value === "number" && Number.isFinite(value); |
| const transitionKeys = ["removed", "introduced", "both", "neither"]; |
| const bodySeeds = [101, 202, 303]; |
| const donorSeeds = [501, 502, 503]; |
| const treatmentFamilies = ["random", "shuffled", "ordered"]; |
| const payloadFailureMessage = "Interactive data are unavailable or incompatible. The article text remains readable; reload after the paper and data asset are updated together."; |
| |
| const showPayloadFailure = (error) => { |
| if (document.documentElement.dataset.sniffPayloadFailure === "true") return; |
| document.documentElement.dataset.sniffPayloadFailure = "true"; |
| console.error("Sniff Test interactive payload rejected", error); |
| document.querySelectorAll("figure.interactive").forEach((figure, index) => { |
| figure.querySelectorAll("button, select").forEach((control) => { |
| control.disabled = true; |
| }); |
| figure.querySelectorAll(".figure-stats, .row-inspector, .transition-matrix, .transition-detail, .control-stage").forEach((output) => { |
| output.replaceChildren(); |
| }); |
| const notice = document.createElement("p"); |
| notice.className = "data-error"; |
| notice.setAttribute("role", index === 0 ? "alert" : "status"); |
| notice.textContent = payloadFailureMessage; |
| const chartShell = figure.querySelector(".chart-shell"); |
| if (chartShell) chartShell.replaceChildren(notice); |
| else figure.insertBefore(notice, figure.querySelector("figcaption")); |
| const liveTakeaway = figure.querySelector(".live-takeaway"); |
| if (liveTakeaway) liveTakeaway.textContent = "The interactive payload failed validation, so no dynamic result is shown."; |
| }); |
| const status = document.getElementById("control-status"); |
| const takeaway = document.getElementById("control-takeaway"); |
| if (status) status.textContent = "Control data unavailable"; |
| if (takeaway) takeaway.textContent = "The control payload could not be validated, so no control result is displayed."; |
| }; |
| |
| const requirePayload = (condition, message) => { |
| if (!condition) throw new TypeError(message); |
| }; |
| |
| const sameMembers = (left, right) => Array.isArray(left) |
| && left.length === right.length |
| && new Set(left).size === left.length |
| && right.every((value) => left.includes(value)); |
| |
| const expectedControlCells = (() => { |
| const cells = []; |
| bodySeeds.forEach((body) => { |
| cells.push(`${body}:baseline`); |
| if (body === 101) cells.push(`${body}:noop`); |
| donorSeeds.forEach((donor) => treatmentFamilies.forEach((family) => { |
| cells.push(`${body}:${family}_d${donor}`); |
| })); |
| }); |
| return cells; |
| })(); |
| |
| const isValidControlRun = (run) => { |
| if (!isRecord(run) || !bodySeeds.includes(run.body_seed) || typeof run.arm !== "string" || typeof run.family !== "string") return false; |
| if (!isFiniteNumber(run.repeat907) || run.repeat907 < 0 || run.repeat907 > 1 || !isFiniteNumber(run.S907) || run.S907 < 0 || run.S907 > 1) return false; |
| if (["baseline", "noop"].includes(run.family)) return run.arm === run.family && run.donor_seed === null && (run.family !== "noop" || run.body_seed === 101); |
| const match = /^(random|shuffled|ordered)_d(501|502|503)$/.exec(run.arm); |
| return Boolean(match && match[1] === run.family && Number(match[2]) === run.donor_seed); |
| }; |
| |
| const validDisplayMetric = (controls, metric) => { |
| const summary = controls.display_metrics?.[metric]; |
| if (!isRecord(summary) || !isRecord(summary.families) || !isRecord(summary.deltas_vs_matched_baseline)) return false; |
| const scale = metric === "repeat907" ? 100 : 1; |
| if (summary.mean_unit !== (metric === "repeat907" ? "percent" : "score_points")) return false; |
| if (summary.delta_unit !== (metric === "repeat907" ? "percentage_points" : "score_points")) return false; |
| const near = (left, right) => left === null && right === null || isFiniteNumber(left) && isFiniteNumber(right) && Math.abs(left - right) < 1e-9; |
| const baselines = new Map(controls.runs.filter((run) => run.family === "baseline").map((run) => [run.body_seed, run[metric]])); |
| return ["baseline", ...treatmentFamilies].every((family) => { |
| const values = controls.runs.filter((run) => run.family === family).map((run) => run[metric]); |
| const cell = summary.families[family]; |
| const expectedMean = values.length ? values.reduce((total, value) => total + value, 0) / values.length * scale : null; |
| return isRecord(cell) && cell.n === values.length && near(cell.mean, expectedMean); |
| }) && treatmentFamilies.every((family) => { |
| const values = controls.runs |
| .filter((run) => run.family === family && baselines.has(run.body_seed)) |
| .map((run) => (run[metric] - baselines.get(run.body_seed)) * scale); |
| const expectedDelta = values.length ? values.reduce((total, value) => total + value, 0) / values.length : null; |
| return near(summary.deltas_vs_matched_baseline[family], expectedDelta); |
| }); |
| }; |
| |
| const validatePayload = (candidate) => { |
| requirePayload(isRecord(candidate), "missing payload object"); |
| requirePayload(candidate.schema === 2, "schema 2 is required"); |
| const shape = candidate.shape; |
| requirePayload(isRecord(shape) && shape.views === "aggregate_blocks" && Number.isInteger(shape.minimum_block_n) && shape.minimum_block_n >= 8 && shape.row_level_outcomes === false, "incompatible aggregate shape"); |
| requirePayload(isRecord(shape.examples) && shape.examples.selection === "omitted_no_outcome_independent_public_sample" && shape.examples.count === 0 && Array.isArray(shape.examples.fields) && shape.examples.fields.length === 0, "incompatible example boundary"); |
| requirePayload(isRecord(shape.controls) && shape.controls.run_metrics === "one_record_per_validated_control_cell" && shape.controls.repeated_title_run_unit === "proportion" && shape.controls.repeated_title_summary_unit === "percentage_points", "incompatible control units"); |
| requirePayload(isRecord(candidate.summary), "missing summary"); |
| requirePayload(candidate.summary.n === 907, "unexpected evaluation size"); |
| requirePayload(isRecord(candidate.summary.judge), "missing judge summary"); |
| requirePayload(Array.isArray(candidate.summary.judge.batch_means) && candidate.summary.judge.batch_means.length === 10 && candidate.summary.judge.batch_means.every(isFiniteNumber), "invalid judge blocks"); |
| requirePayload(isFiniteNumber(candidate.summary.judge.delta) && isFiniteNumber(candidate.summary.judge.complete_batch_mean), "invalid judge means"); |
| ["ci_thread_cluster_bootstrap", "complete_batch_ci_t"].forEach((key) => { |
| const interval = candidate.summary.judge[key]; |
| requirePayload(Array.isArray(interval) && interval.length === 2 && interval.every(isFiniteNumber), `invalid judge interval ${key}`); |
| }); |
| requirePayload(isRecord(candidate.summary.repetition), "missing repetition summary"); |
| ["transition_counts", "transition_judge_points"].forEach((field) => { |
| const values = candidate.summary.repetition[field]; |
| requirePayload(isRecord(values) && transitionKeys.every((key) => isFiniteNumber(values[key])), `invalid repetition field ${field}`); |
| }); |
| requirePayload(transitionKeys.reduce((total, key) => total + candidate.summary.repetition.transition_counts[key], 0) === 907, "transition counts do not sum to 907"); |
| requirePayload(transitionKeys.reduce((total, key) => total + candidate.summary.repetition.transition_judge_points[key], 0) === 138, "transition judge points do not sum to 138"); |
| requirePayload(isRecord(candidate.views), "missing aggregate views"); |
| ["all", "changed", "exact_unseen", "thread_disjoint"].forEach((subset) => { |
| requirePayload(isRecord(candidate.views[subset]), `missing view ${subset}`); |
| ["dS", "dJ"].forEach((metric) => { |
| requirePayload(isRecord(candidate.views[subset][metric]), `missing view ${subset}.${metric}`); |
| ["mixed", "cancel", "reverse"].forEach((order) => { |
| const view = candidate.views[subset][metric][order]; |
| requirePayload(isRecord(view) && Number.isInteger(view.n) && view.n > 0, `invalid view ${subset}.${metric}.${order}`); |
| requirePayload(["mean", "positive_mass", "negative_mass", "wins", "ties", "losses"].every((key) => isFiniteNumber(view[key])), `invalid view totals ${subset}.${metric}.${order}`); |
| requirePayload(Array.isArray(view.blocks) && view.blocks.length > 0, `missing blocks ${subset}.${metric}.${order}`); |
| requirePayload(view.blocks.every((block) => isRecord(block) && Number.isInteger(block.n) && block.n >= shape.minimum_block_n && ["sum", "wins", "ties", "losses"].every((key) => isFiniteNumber(block[key]))), `invalid blocks ${subset}.${metric}.${order}`); |
| requirePayload(view.blocks.reduce((total, block) => total + block.n, 0) === view.n, `block count mismatch ${subset}.${metric}.${order}`); |
| }); |
| }); |
| }); |
| requirePayload(Array.isArray(candidate.examples) && candidate.examples.length === 0, "public examples must be omitted"); |
| requirePayload(isRecord(candidate.controls) && candidate.controls.schema === 2 && Array.isArray(candidate.controls.runs), "invalid controls"); |
| requirePayload(["not_available", "partial", "invalid", "complete"].includes(candidate.controls.status), "invalid control status"); |
| const design = candidate.controls.design; |
| requirePayload(isRecord(design) && design.expected_runs === 31 && sameMembers(design.body_seeds, bodySeeds) && sameMembers(design.donor_seeds, donorSeeds) && sameMembers(design.treatment_families, treatmentFamilies), "incompatible control design"); |
| requirePayload(Array.isArray(design.expected_cells) && sameMembers(design.expected_cells.map((cell) => `${cell.body_seed}:${cell.arm}`), expectedControlCells), "unexpected control cells"); |
| requirePayload(candidate.controls.runs.every(isValidControlRun), "invalid control run"); |
| const observedCells = candidate.controls.runs.map((run) => `${run.body_seed}:${run.arm}`); |
| requirePayload(new Set(observedCells).size === observedCells.length && observedCells.every((cell) => expectedControlCells.includes(cell)), "duplicate or unexpected control run"); |
| requirePayload(design.observed_runs === observedCells.length && Number.isInteger(design.started_runs) && design.started_runs >= design.observed_runs && design.started_runs <= 31, "control progress mismatch"); |
| requirePayload(Array.isArray(design.missing_cells) && sameMembers(design.missing_cells.map((cell) => `${cell.body_seed}:${cell.arm}`), expectedControlCells.filter((cell) => !observedCells.includes(cell))), "control missing-cell mismatch"); |
| requirePayload(isRecord(candidate.controls.validation) && Array.isArray(candidate.controls.validation.errors), "invalid control validation report"); |
| if (["partial", "complete"].includes(candidate.controls.status)) { |
| requirePayload(candidate.controls.validation.errors.length === 0 && candidate.controls.validation.run_logs_present === true, "unvalidated control outcomes"); |
| requirePayload(validDisplayMetric(candidate.controls, "repeat907") && validDisplayMetric(candidate.controls, "S907"), "control display metrics disagree with runs"); |
| } |
| if (candidate.controls.status === "partial") requirePayload(observedCells.length > 0 && observedCells.length < 31 && candidate.controls.validation.matrix_exact === false, "invalid partial status"); |
| if (candidate.controls.status === "complete") requirePayload(observedCells.length === 31 && candidate.controls.validation.matrix_exact === true && candidate.controls.validation.workflow_done_logged === true, "invalid complete status"); |
| if (candidate.controls.status === "not_available") requirePayload(observedCells.length === 0, "not-available controls contain runs"); |
| return candidate; |
| }; |
| |
| let data; |
| try { |
| data = validatePayload(window.SNIFF_PAPER_DATA); |
| } catch (error) { |
| showPayloadFailure(error); |
| return; |
| } |
| const safeInteraction = (handler) => (...args) => { |
| try { |
| return handler(...args); |
| } catch (error) { |
| showPayloadFailure(error); |
| return undefined; |
| } |
| }; |
| |
| const COLORS = { |
| sniff: "#176b87", |
| base: "#b4513f", |
| tie: "#7b8480", |
| grid: "#ddd5c8", |
| paper: "#fffdf8", |
| ink: "#17211f" |
| }; |
| const escapeHtml = (value) => String(value).replace(/[&<>"']/g, (char) => ({ |
| "&": "&", "<": "<", ">": ">", "\"": """, "'": "'" |
| })[char]); |
| const signed = (value, digits = 3) => isFiniteNumber(value) ? `${value > 0 ? "+" : ""}${value.toFixed(digits)}` : "Pending"; |
| const svgEl = (name, attrs = {}) => { |
| const node = document.createElementNS("http://www.w3.org/2000/svg", name); |
| Object.entries(attrs).forEach(([key, value]) => node.setAttribute(key, value)); |
| return node; |
| }; |
| const addText = (svg, text, x, y, attrs = {}) => { |
| const node = svgEl("text", {x, y, ...attrs}); |
| node.textContent = text; |
| svg.appendChild(node); |
| return node; |
| }; |
| const addSvgMeta = (svg, idPrefix, title, description) => { |
| const titleNode = svgEl("title", {id: `${idPrefix}-title`}); |
| titleNode.textContent = title; |
| const descriptionNode = svgEl("desc", {id: `${idPrefix}-description`}); |
| descriptionNode.textContent = description; |
| svg.append(titleNode, descriptionNode); |
| svg.setAttribute("aria-labelledby", `${idPrefix}-title ${idPrefix}-description`); |
| }; |
| const statCard = (label, value) => `<div class="mini-stat"><span class="stat-label">${escapeHtml(label)}</span><strong>${escapeHtml(value)}</strong></div>`; |
| |
| const progress = document.getElementById("reading-progress"); |
| const updateProgress = () => { |
| const max = document.documentElement.scrollHeight - innerHeight; |
| progress.style.width = `${max > 0 ? (scrollY / max) * 100 : 0}%`; |
| }; |
| addEventListener("scroll", updateProgress, {passive: true}); |
| updateProgress(); |
| |
| let paradoxMetric = "dS"; |
| const paradoxChart = document.getElementById("paradox-chart"); |
| const paradoxStats = document.getElementById("paradox-stats"); |
| const paradoxTakeaway = document.getElementById("paradox-takeaway"); |
| const inspector = document.getElementById("row-inspector"); |
| const filterSelect = document.getElementById("paradox-filter"); |
| |
| const inspectBlock = (block, index, total) => { |
| const value = block.sum; |
| const digits = paradoxMetric === "dS" ? 3 : 0; |
| const color = value > 0 ? COLORS.sniff : value < 0 ? COLORS.base : "#4d5754"; |
| inspector.innerHTML = ` |
| <div class="title-card base"><span class="stat-label">Aggregate block</span><strong>${index + 1} of ${total}, grouping ${block.n} tasks</strong></div> |
| <div class="title-card sniff"><span class="stat-label">Sniff / tie / baseline</span><strong>${block.wins} / ${block.ties} / ${block.losses}</strong></div> |
| <div class="title-card teacher"><span class="stat-label">Privacy</span><strong>No row-level outcomes published</strong></div> |
| <div class="delta-chip" style="background:${color}">${signed(value, digits)} block net</div>`; |
| }; |
| |
| const renderParadox = () => { |
| const view = data.views[filterSelect.value][paradoxMetric].mixed; |
| const blocks = view.blocks; |
| const metricName = paradoxMetric === "dS" ? "similarity to teacher" : "later judge score"; |
| const digits = paradoxMetric === "dS" ? 3 : 0; |
| paradoxStats.innerHTML = [ |
| statCard("Average change", signed(view.mean, paradoxMetric === "dS" ? 4 : 3)), |
| statCard("Sniff / tie / baseline", `${view.wins} / ${view.ties} / ${view.losses}`), |
| statCard("Total gains", `+${view.positive_mass.toFixed(paradoxMetric === "dS" ? 2 : 0)}`), |
| statCard("Total losses", view.negative_mass.toFixed(paradoxMetric === "dS" ? 2 : 0)) |
| ].join(""); |
| |
| paradoxChart.replaceChildren(); |
| const compact = matchMedia("(max-width: 520px)").matches; |
| const W = compact ? 420 : 920; |
| const H = 340; |
| const left = compact ? 48 : 66; |
| const right = compact ? 12 : 22; |
| const top = compact ? 36 : 24; |
| const bottom = 48; |
| paradoxChart.setAttribute("viewBox", `0 0 ${W} ${H}`); |
| const cumulative = [{n: 0, sum: 0}]; |
| blocks.forEach((block) => cumulative.push({n: cumulative.at(-1).n + block.n, sum: cumulative.at(-1).sum + block.sum})); |
| const endpoint = cumulative.at(-1).sum; |
| addSvgMeta( |
| paradoxChart, |
| "paradox-svg", |
| `Cancellation in ${metricName}`, |
| `${blocks.length} aggregate blocks in a fixed mixed order. Total gains are ${signed(view.positive_mass, digits)}, total losses are ${signed(view.negative_mass, digits)}, and the endpoint is ${signed(endpoint, digits)}. This ordering is not time.` |
| ); |
| const yMin = Math.min(0, ...cumulative.map((point) => point.sum)); |
| const yMax = Math.max(0, ...cumulative.map((point) => point.sum)); |
| const pad = Math.max((yMax - yMin) * 0.08, paradoxMetric === "dS" ? 0.3 : 3); |
| const lo = yMin - pad, hi = yMax + pad; |
| const x = (count) => left + count / Math.max(view.n, 1) * (W - left - right); |
| const y = (value) => top + (hi - value) / (hi - lo) * (H - top - bottom); |
| |
| [0, 0.25, 0.5, 0.75, 1].forEach((fraction) => { |
| const value = lo + (hi - lo) * fraction; |
| const yy = y(value); |
| paradoxChart.appendChild(svgEl("line", {x1: left, x2: W - right, y1: yy, y2: yy, stroke: COLORS.grid, "stroke-width": 1})); |
| addText(paradoxChart, value.toFixed(paradoxMetric === "dS" ? 1 : 0), left - 10, yy + 4, {"text-anchor": "end", "font-size": 11}); |
| }); |
| paradoxChart.appendChild(svgEl("line", {x1: left, x2: W - right, y1: y(0), y2: y(0), stroke: COLORS.ink, "stroke-width": 1.6})); |
| const path = cumulative.map((point, index) => `${index ? "L" : "M"}${x(point.n).toFixed(2)},${y(point.sum).toFixed(2)}`).join(" "); |
| paradoxChart.appendChild(svgEl("path", {d: path, fill: "none", stroke: COLORS.sniff, "stroke-width": 3, "stroke-linejoin": "round"})); |
| paradoxChart.appendChild(svgEl("circle", {cx: x(0), cy: y(0), r: 4.5, fill: COLORS.ink, stroke: COLORS.paper, "stroke-width": 2})); |
| paradoxChart.appendChild(svgEl("circle", {cx: x(view.n), cy: y(endpoint), r: 5.5, fill: endpoint >= 0 ? COLORS.sniff : COLORS.base, stroke: COLORS.paper, "stroke-width": 2})); |
| addText(paradoxChart, "aggregate tasks included, fixed mixed order", (left + W - right) / 2, H - 12, {"text-anchor": "middle", "font-size": 12}); |
| addText(paradoxChart, paradoxMetric === "dS" ? "cumulative similarity change" : "cumulative judge points", 18, H / 2, {transform: `rotate(-90 18 ${H / 2})`, "text-anchor": "middle", "font-size": 12}); |
| addText(paradoxChart, "start: 0", left + 8, Math.max(top + 15, y(0) - 10), {class: "direct-label"}); |
| addText(paradoxChart, "zero: no net difference", left + 105, y(0) - 8, {class: "direct-label"}); |
| addText(paradoxChart, `end: ${signed(endpoint, digits)}`, W - right - 4, Math.max(top + 15, Math.min(H - bottom - 9, y(endpoint) - 10)), {class: "direct-label", "text-anchor": "end"}); |
| addText(paradoxChart, `${compact ? "cancel" : "cancellation"}: +${view.positive_mass.toFixed(digits)} plus ${view.negative_mass.toFixed(digits)}`, W - right - 4, top + 14, {class: "direct-label", "text-anchor": "end"}); |
| |
| paradoxTakeaway.textContent = `${metricName[0].toUpperCase()}${metricName.slice(1)} gains of +${view.positive_mass.toFixed(digits)} and losses of ${view.negative_mass.toFixed(digits)} end at ${signed(endpoint, digits)} across ${view.n} tasks.`; |
| |
| const cursor = svgEl("line", {x1: left, x2: left, y1: top, y2: H - bottom, stroke: COLORS.tie, "stroke-width": 1, "stroke-dasharray": "4 4"}); |
| paradoxChart.appendChild(cursor); |
| const overlay = svgEl("rect", { |
| x: left, |
| y: top, |
| width: W - left - right, |
| height: H - top - bottom, |
| fill: "transparent", |
| class: "chart-scrubber", |
| tabindex: 0, |
| focusable: "true", |
| role: "slider", |
| "aria-label": `Aggregate block inspector for ${metricName}`, |
| "aria-orientation": "horizontal", |
| "aria-valuemin": 1, |
| "aria-valuemax": blocks.length, |
| "aria-valuenow": 1, |
| "aria-describedby": "paradox-takeaway row-inspector" |
| }); |
| const selectBlock = (index) => { |
| const selected = Math.max(0, Math.min(blocks.length - 1, index)); |
| const block = blocks[selected]; |
| overlay.dataset.index = String(selected); |
| overlay.setAttribute("aria-valuenow", String(selected + 1)); |
| overlay.setAttribute("aria-valuetext", `Aggregate block ${selected + 1} of ${blocks.length}, grouping ${block.n} tasks, ${signed(block.sum, digits)} ${metricName} block net`); |
| cursor.setAttribute("x1", x(cumulative[selected + 1].n)); |
| cursor.setAttribute("x2", x(cumulative[selected + 1].n)); |
| inspectBlock(block, selected, blocks.length); |
| }; |
| const pointAt = (clientX) => { |
| const rect = paradoxChart.getBoundingClientRect(); |
| const local = (clientX - rect.left) / rect.width * W; |
| const target = Math.max(0, Math.min(view.n, (local - left) / (W - left - right) * view.n)); |
| let index = cumulative.slice(1).findIndex((point) => point.n >= target); |
| if (index < 0) index = blocks.length - 1; |
| selectBlock(index); |
| }; |
| overlay.addEventListener("pointermove", safeInteraction((event) => pointAt(event.clientX))); |
| overlay.addEventListener("pointerdown", safeInteraction((event) => { |
| pointAt(event.clientX); |
| overlay.focus({preventScroll: true}); |
| })); |
| overlay.addEventListener("focus", safeInteraction(() => selectBlock(Number(overlay.dataset.index || 0)))); |
| overlay.addEventListener("keydown", safeInteraction((event) => { |
| if (!["ArrowLeft", "ArrowRight", "Home", "End"].includes(event.key)) return; |
| event.preventDefault(); |
| const current = Number(overlay.dataset.index || 0); |
| const next = event.key === "Home" |
| ? 0 |
| : event.key === "End" |
| ? blocks.length - 1 |
| : current + (event.key === "ArrowRight" ? 1 : -1); |
| selectBlock(next); |
| })); |
| paradoxChart.appendChild(overlay); |
| overlay.dataset.index = "0"; |
| selectBlock(0); |
| }; |
| |
| document.querySelectorAll("[data-paradox-metric]").forEach((button) => button.addEventListener("click", safeInteraction(() => { |
| paradoxMetric = button.dataset.paradoxMetric; |
| document.querySelectorAll("[data-paradox-metric]").forEach((candidate) => candidate.setAttribute("aria-pressed", String(candidate === button))); |
| renderParadox(); |
| }))); |
| filterSelect.addEventListener("change", safeInteraction(renderParadox)); |
| |
| const judgeChart = document.getElementById("judge-chart"); |
| const judgeStats = document.getElementById("judge-stats"); |
| const judgeTakeaway = document.getElementById("judge-takeaway"); |
| let judgeUnit = "task"; |
| const renderJudge = () => { |
| const allValues = data.summary.judge.batch_means; |
| const values = judgeUnit === "task" ? allValues : allValues.slice(0, 9); |
| const ci = judgeUnit === "task" ? data.summary.judge.ci_thread_cluster_bootstrap : data.summary.judge.complete_batch_ci_t; |
| const point = judgeUnit === "task" ? data.summary.judge.delta : data.summary.judge.complete_batch_mean; |
| const method = judgeUnit === "task" ? "resampling evaluation threads" : "comparing nine complete scoring calls"; |
| const crossesZero = ci[0] <= 0 && ci[1] >= 0; |
| const compact = matchMedia("(max-width: 520px)").matches; |
| const W = compact ? 460 : 920; |
| const H = compact ? 350 : 330; |
| const left = compact ? 96 : 90; |
| const right = compact ? 18 : 36; |
| const axisY = compact ? 315 : 292; |
| const lo = -1, hi = 0.65; |
| const x = (value) => left + (value - lo) / (hi - lo) * (W - left - right); |
| const callRows = values.map((value, index) => ({ |
| value, |
| call: index + 1, |
| tasks: index === 9 ? 7 : 100 |
| })); |
| const markerY = (call) => (compact ? 184 : 170) + (call - 1) * 11.5; |
| judgeChart.replaceChildren(); |
| judgeChart.setAttribute("viewBox", `0 0 ${W} ${H}`); |
| addSvgMeta( |
| judgeChart, |
| "judge-svg", |
| `Judge uncertainty from ${method}`, |
| `The average change is ${signed(point, 3)} on a 0 to 10 scale. The 95 percent interval is ${signed(ci[0], 3)} to ${signed(ci[1], 3)} and ${crossesZero ? "crosses" : "does not cross"} zero. Separate scoring-call means shown are ${values.map((value) => signed(value, 2)).join(", ")}. The dots are not a time series.` |
| ); |
| [-1, -0.5, 0, 0.5].forEach((value) => { |
| judgeChart.appendChild(svgEl("line", {x1: x(value), x2: x(value), y1: 28, y2: axisY, stroke: value === 0 ? COLORS.ink : COLORS.grid, "stroke-width": value === 0 ? 2 : 1})); |
| addText(judgeChart, signed(value, 1), x(value), axisY + 22, {"text-anchor": "middle", "font-size": 11}); |
| }); |
| const bandStart = x(ci[0]); |
| const bandEnd = x(ci[1]); |
| judgeChart.appendChild(svgEl("rect", {x: bandStart, y: 49, width: Math.max(2, bandEnd - bandStart), height: 58, rx: 10, class: "ci-band"})); |
| judgeChart.appendChild(svgEl("line", {x1: bandStart, x2: bandEnd, y1: 78, y2: 78, class: "ci-whisker"})); |
| judgeChart.appendChild(svgEl("line", {x1: bandStart, x2: bandStart, y1: 65, y2: 91, stroke: COLORS.sniff, "stroke-width": 3})); |
| judgeChart.appendChild(svgEl("line", {x1: bandEnd, x2: bandEnd, y1: 65, y2: 91, stroke: COLORS.sniff, "stroke-width": 3})); |
| judgeChart.appendChild(svgEl("circle", {cx: x(point), cy: 78, r: 8, fill: COLORS.ink, stroke: COLORS.paper, "stroke-width": 2})); |
| addText(judgeChart, `95% interval ${signed(ci[0], 3)} to ${signed(ci[1], 3)}`, (bandStart + bandEnd) / 2, 37, {class: "direct-label", "text-anchor": "middle"}); |
| addText(judgeChart, `mean ${signed(point, 3)}`, x(point), 125, {class: "direct-label", "text-anchor": "middle"}); |
| addText(judgeChart, "zero: no average change", x(0) + 7, 22, {class: "direct-label"}); |
| addText(judgeChart, "separate scoring-call means", left, 163, {class: "direct-label"}); |
| callRows.forEach(({value, call, tasks}) => { |
| const yy = markerY(call); |
| const dot = svgEl("circle", {cx: x(value), cy: yy, r: tasks === 7 ? 7 : 6, fill: value >= 0 ? COLORS.sniff : COLORS.base, class: "judge-dot"}); |
| judgeChart.appendChild(dot); |
| addText(judgeChart, tasks === 7 ? "call 10, 7 tasks" : `call ${call}`, left - 10, yy + 4, {"text-anchor": "end", "font-size": 10, "font-weight": 700}); |
| }); |
| judgeChart.appendChild(svgEl("line", {x1: left, x2: W - right, y1: axisY, y2: axisY, stroke: COLORS.ink, "stroke-width": 1.5})); |
| addText(judgeChart, "paired judge-score change on a 0 to 10 scale", (left + W - right) / 2, H - 5, {"text-anchor": "middle", "font-size": 12}); |
| if (judgeUnit === "task") { |
| judgeStats.innerHTML = [statCard("Average change", `${signed(point, 3)} / 10`), statCard("95% interval", `${signed(ci[0], 3)} to ${signed(ci[1], 3)}`), statCard("What was resampled", "Evaluation threads"), statCard("Zero crossing", "No")].join(""); |
| judgeTakeaway.textContent = "The main thread-resampling interval stays above zero, but the evaluator was unblinded and the calls were dependent."; |
| } else { |
| judgeStats.innerHTML = [statCard("Average across calls", `${signed(point, 3)} / 10`), statCard("95% interval", `${signed(ci[0], 3)} to ${signed(ci[1], 3)}`), statCard("What was compared", "9 complete calls"), statCard("Zero crossing", "Yes")].join(""); |
| judgeTakeaway.textContent = "The sensitivity interval across nine complete scoring calls crosses zero, so this check is compatible with no average advantage."; |
| } |
| judgeChart.setAttribute("aria-label", `${method}: mean ${signed(point, 3)}, 95 percent interval ${signed(ci[0], 3)} to ${signed(ci[1], 3)}, ${crossesZero ? "crosses zero" : "above zero"}`); |
| }; |
| document.querySelectorAll("[data-judge-unit]").forEach((button) => button.addEventListener("click", safeInteraction(() => { |
| judgeUnit = button.dataset.judgeUnit; |
| document.querySelectorAll("[data-judge-unit]").forEach((candidate) => candidate.setAttribute("aria-pressed", String(candidate === button))); |
| renderJudge(); |
| }))); |
| |
| const transitionLabels = { |
| neither: {name: "Stayed clean", path: "baseline clean, sniff clean", explanation: "Neither checkpoint repeated a title token."}, |
| introduced: {name: "New stutter", path: "baseline clean, sniff repeated", explanation: "Only the sniff checkpoint repeated a title token."}, |
| removed: {name: "Repair", path: "baseline repeated, sniff clean", explanation: "Only the baseline repeated a title token."}, |
| both: {name: "Stayed repeated", path: "baseline repeated, sniff repeated", explanation: "Both checkpoints repeated a title token."} |
| }; |
| let selectedTransition = "removed"; |
| const transitionMatrix = document.getElementById("transition-matrix"); |
| const transitionDetail = document.getElementById("transition-detail"); |
| const repeatTakeaway = document.getElementById("repeat-takeaway"); |
| const updateTransition = (key) => { |
| selectedTransition = key; |
| const counts = data.summary.repetition.transition_counts; |
| const points = data.summary.repetition.transition_judge_points; |
| transitionMatrix.querySelectorAll("[data-transition]").forEach((button) => { |
| button.setAttribute("aria-pressed", String(button.dataset.transition === key)); |
| }); |
| const label = transitionLabels[key]; |
| transitionDetail.innerHTML = `<strong>${escapeHtml(label.name)}: ${counts[key]} tasks</strong>${escapeHtml(label.explanation)} Together these tasks contributed ${signed(points[key], 0)} points to the later judge difference.`; |
| repeatTakeaway.textContent = `${label.name} contains ${counts[key]} tasks and contributes ${signed(points[key], 0)} later judge points. Across the full matrix, repairs outnumber new stutters 87 to 34.`; |
| }; |
| const transitionButton = (key) => { |
| const count = data.summary.repetition.transition_counts[key]; |
| const points = data.summary.repetition.transition_judge_points[key]; |
| const label = transitionLabels[key]; |
| return `<button type="button" class="transition-cell" data-transition="${key}" aria-pressed="${key === selectedTransition}" aria-controls="transition-detail" aria-label="${escapeHtml(label.name)}, ${escapeHtml(label.path)}, ${count} tasks, ${signed(points, 0)} judge points"> |
| <span class="stat-label">${escapeHtml(label.name)}</span> |
| <strong>${count}</strong> |
| <span>${escapeHtml(label.path)}<br>Judge ${signed(points, 0)} points</span> |
| </button>`; |
| }; |
| const renderTransitions = () => { |
| transitionMatrix.innerHTML = ` |
| <div class="matrix-heading" aria-hidden="true">Baseline down<br>Sniff across</div> |
| <div class="matrix-heading">Sniff clean</div> |
| <div class="matrix-heading">Sniff repeated</div> |
| <div class="matrix-row-label">Baseline clean</div> |
| ${transitionButton("neither")} |
| ${transitionButton("introduced")} |
| <div class="matrix-row-label">Baseline repeated</div> |
| ${transitionButton("removed")} |
| ${transitionButton("both")}`; |
| transitionMatrix.querySelectorAll("[data-transition]").forEach((button) => button.addEventListener("click", safeInteraction(() => { |
| if (button.dataset.transition !== selectedTransition) updateTransition(button.dataset.transition); |
| }))); |
| updateTransition(selectedTransition); |
| }; |
| |
| const controlRuns = data.controls.runs; |
| const controlRunMap = new Map(controlRuns.map((run) => [`${run.body_seed}:${run.arm}`, run])); |
| const controlStats = document.getElementById("control-stats"); |
| const controlStatus = document.getElementById("control-status"); |
| const controlTakeaway = document.getElementById("control-takeaway"); |
| const controlCaptionTakeaway = document.getElementById("control-caption-takeaway"); |
| const controlStage = document.getElementById("control-stage"); |
| const guideButtons = [...document.querySelectorAll("[data-control-step]")]; |
| const familyLabels = { |
| random: "Random rows", |
| shuffled: "Shuffled Verne", |
| ordered: "Ordered Verne" |
| }; |
| const outcomeLabels = { |
| repeat907: "Repeated titles", |
| S907: "Similarity to teacher" |
| }; |
| const noopChecks = Array.isArray(data.controls.noop_checks) ? data.controls.noop_checks : []; |
| const noopEvidence = noopChecks.find((check) => isRecord(check) && check.all_exact === true); |
| const noopFailed = noopChecks.some((check) => isRecord(check) && check.all_exact === false); |
| const noopExact = Boolean(noopEvidence); |
| const exactMatrix = controlRuns.length === 31 && expectedControlCells.every((cell) => controlRunMap.has(cell)); |
| let effectiveControlStatus = data.controls.status; |
| if (data.controls.validation.errors.length || noopFailed) effectiveControlStatus = "invalid"; |
| if (["partial", "complete"].includes(effectiveControlStatus) && !noopExact) effectiveControlStatus = "invalid"; |
| if (effectiveControlStatus === "complete" && (!exactMatrix || data.controls.validation.matrix_exact !== true)) effectiveControlStatus = "invalid"; |
| if (effectiveControlStatus === "partial" && exactMatrix) effectiveControlStatus = "invalid"; |
| |
| let controlStep = 1; |
| let matrixFamily = "ordered"; |
| let matrixMetric = "repeat907"; |
| |
| const controlValue = (metric, value) => { |
| if (!isFiniteNumber(value)) return "Not run yet"; |
| return metric === "repeat907" ? `${(value * 100).toFixed(2)}%` : value.toFixed(4); |
| }; |
| const controlDelta = (metric, value) => { |
| if (!isFiniteNumber(value)) return "Not run yet"; |
| return metric === "repeat907" ? `${signed(value * 100, 2)} pp` : signed(value, 4); |
| }; |
| const matchedRuns = (body, donor, family = "ordered") => ({ |
| baseline: controlRunMap.get(`${body}:baseline`), |
| treatment: controlRunMap.get(`${body}:${family}_d${donor}`) |
| }); |
| const matchedDelta = (body, donor, family, metric) => { |
| const {baseline, treatment} = matchedRuns(body, donor, family); |
| return baseline && treatment ? treatment[metric] - baseline[metric] : null; |
| }; |
| const controlRange = (metric) => { |
| const scale = metric === "repeat907" ? 100 : 1; |
| const values = controlRuns.filter((run) => run.family !== "noop").map((run) => run[metric] * scale); |
| const min = Math.min(...values); |
| const max = Math.max(...values); |
| const pad = Math.max((max - min) * 0.16, metric === "repeat907" ? 0.8 : 0.003); |
| return {lo: min - pad, hi: max + pad, scale}; |
| }; |
| const slopeMarkup = (body, donor, family = "ordered", metric = "repeat907") => { |
| const {baseline, treatment} = matchedRuns(body, donor, family); |
| const title = `Body starting draw ${body}, donor draw ${donor}`; |
| if (!baseline || !treatment) return `<article class="slope-card"> |
| <h5>${escapeHtml(title)}</h5> |
| <div class="matrix-cell pending"><strong>Not run yet</strong><span>Missing, not zero</span></div> |
| <p class="slope-reading">This matched comparison is pending.</p> |
| </article>`; |
| const {lo, hi, scale} = controlRange(metric); |
| const leftValue = baseline[metric] * scale; |
| const rightValue = treatment[metric] * scale; |
| const delta = treatment[metric] - baseline[metric]; |
| const y = (value) => 18 + (hi - value) / (hi - lo) * 92; |
| const leftY = y(leftValue); |
| const rightY = y(rightValue); |
| const favorable = metric === "repeat907" ? delta < 0 : delta > 0; |
| const color = Math.abs(delta) < 1e-12 ? COLORS.tie : favorable ? "#32735f" : COLORS.base; |
| const id = `slope-${body}-${donor}-${family}-${metric}`; |
| const leftLabel = controlValue(metric, baseline[metric]); |
| const rightLabel = controlValue(metric, treatment[metric]); |
| const deltaLabel = controlDelta(metric, delta); |
| return `<article class="slope-card"> |
| <h5>${escapeHtml(title)}</h5> |
| <svg viewBox="0 0 280 155" role="img" aria-labelledby="${id}-title ${id}-desc"> |
| <title id="${id}-title">${escapeHtml(title)}, no transplant compared with ${escapeHtml(familyLabels[family])}</title> |
| <desc id="${id}-desc">No transplant ${leftLabel}; ${escapeHtml(familyLabels[family])} ${rightLabel}; matched change ${deltaLabel}.</desc> |
| <line x1="62" x2="218" y1="${leftY.toFixed(2)}" y2="${rightY.toFixed(2)}" stroke="${color}" stroke-width="4" stroke-linecap="round"></line> |
| <circle cx="62" cy="${leftY.toFixed(2)}" r="7" fill="${COLORS.tie}" stroke="${COLORS.paper}" stroke-width="2"></circle> |
| <circle cx="218" cy="${rightY.toFixed(2)}" r="7" fill="${color}" stroke="${COLORS.paper}" stroke-width="2"></circle> |
| <text x="62" y="${Math.max(12, leftY - 11).toFixed(2)}" text-anchor="middle" class="direct-label">${leftLabel}</text> |
| <text x="218" y="${Math.max(12, rightY - 11).toFixed(2)}" text-anchor="middle" class="direct-label">${rightLabel}</text> |
| <text x="62" y="136" text-anchor="middle" font-size="11">No transplant</text> |
| <text x="218" y="136" text-anchor="middle" font-size="11">${escapeHtml(familyLabels[family])}</text> |
| </svg> |
| <p class="slope-reading">${leftLabel} to ${rightLabel}, a matched change of <strong>${deltaLabel}</strong>.</p> |
| </article>`; |
| }; |
| |
| const controlStageHead = (title, instruction) => `<div class="control-stage-head"><div><h4>${escapeHtml(title)}</h4><p>${escapeHtml(instruction)}</p></div><span class="tiny muted">Step ${controlStep} of 4</span></div>`; |
| |
| const availableDeltas = (pairs, family = "ordered", metric = "repeat907") => pairs |
| .map(([body, donor]) => matchedDelta(body, donor, family, metric)) |
| .filter(isFiniteNumber); |
| |
| const mixedSigns = (values) => values.some((value) => value < -1e-12) && values.some((value) => value > 1e-12); |
| |
| const renderControlStepOne = () => { |
| const delta = matchedDelta(101, 501, "ordered", "repeat907"); |
| controlStage.innerHTML = `${controlStageHead("Start with one matched pair", "Hold body starting draw 101 and donor draw 501 fixed. Change only no transplant versus ordered Verne rows.")} |
| <div class="slope-grid">${slopeMarkup(101, 501)}</div>`; |
| controlTakeaway.textContent = isFiniteNumber(delta) |
| ? `For body starting draw 101 and donor draw 501, ordered Verne changes repeated titles by ${controlDelta("repeat907", delta)} relative to no transplant. One pair cannot identify a general effect.` |
| : "The first matched pair has not run yet; no value is imputed."; |
| }; |
| |
| const renderControlStepTwo = () => { |
| const pairs = bodySeeds.map((body) => [body, 501]); |
| const deltas = availableDeltas(pairs); |
| controlStage.innerHTML = `${controlStageHead("Hold donor draw 501 fixed", "Now repeat the same ordered-Verne comparison across three body starting draws.")} |
| <div class="slope-grid three">${pairs.map(([body, donor]) => slopeMarkup(body, donor)).join("")}</div>`; |
| controlTakeaway.textContent = mixedSigns(deltas) |
| ? "With donor draw 501 fixed, ordered Verne lowers repetition in one body starting draw and raises it in another. The direction reverses across bodies." |
| : deltas.length === 3 |
| ? "All three body comparisons are available; compare their directions before making a mechanism claim." |
| : `${deltas.length} of 3 body comparisons are available. Missing comparisons are pending, not zero.`; |
| }; |
| |
| const renderControlStepThree = () => { |
| const pairs = donorSeeds.map((donor) => [101, donor]); |
| const deltas = availableDeltas(pairs); |
| controlStage.innerHTML = `${controlStageHead("Hold body starting draw 101 fixed", "Now vary the donor draw while keeping the student body unchanged.")} |
| <div class="slope-grid three">${pairs.map(([body, donor]) => slopeMarkup(body, donor)).join("")}</div>`; |
| controlTakeaway.textContent = mixedSigns(deltas) |
| ? "Within body starting draw 101, changing the donor draw reverses the ordered-Verne repetition effect. Donor initialization matters in this body." |
| : deltas.length === 3 |
| ? "All three donor comparisons are available; their spread shows how donor initialization changes the matched result." |
| : `${deltas.length} of 3 donor comparisons are available. Missing comparisons are pending, not zero.`; |
| }; |
| |
| const updateMatrixTakeaway = () => { |
| const missing = 31 - controlRuns.length; |
| const shownMissing = bodySeeds.reduce((total, body) => total + donorSeeds.filter((donor) => { |
| const treatmentMissing = !controlRunMap.has(`${body}:${matrixFamily}_d${donor}`); |
| const comparatorMissing = matrixFamily === "ordered" |
| ? !controlRunMap.has(`${body}:shuffled_d${donor}`) |
| : !controlRunMap.has(`${body}:baseline`); |
| return treatmentMissing || comparatorMissing; |
| }).length, 0); |
| if (effectiveControlStatus !== "complete") { |
| controlTakeaway.textContent = `Showing ${familyLabels[matrixFamily].toLowerCase()} for ${outcomeLabels[matrixMetric].toLowerCase()} with ${controlRuns.length} of 31 validated runs. ${missing} planned runs remain; ${shownMissing} cells in this selected view are hatched. This unbalanced progress view is not a family estimate.`; |
| return; |
| } |
| if (matrixFamily === "ordered") { |
| const contrasts = bodySeeds.flatMap((body) => donorSeeds.map((donor) => { |
| const ordered = controlRunMap.get(`${body}:ordered_d${donor}`); |
| const shuffled = controlRunMap.get(`${body}:shuffled_d${donor}`); |
| return ordered && shuffled ? ordered[matrixMetric] - shuffled[matrixMetric] : null; |
| })).filter(isFiniteNumber); |
| const mean = contrasts.reduce((total, value) => total + value, 0) / contrasts.length; |
| const favorable = contrasts.filter((value) => matrixMetric === "repeat907" ? value < -1e-12 : value > 1e-12).length; |
| const tied = contrasts.filter((value) => Math.abs(value) <= 1e-12).length; |
| const adverse = contrasts.length - favorable - tied; |
| const directions = matrixMetric === "repeat907" ? "lower / tied / higher repetition" : "higher / tied / lower similarity"; |
| controlTakeaway.textContent = `All 31 runs are validated. Ordered versus shuffled Verne gives ${favorable} / ${tied} / ${adverse} cells with ${directions}; the mean change is ${controlDelta(matrixMetric, mean)}. This does not show a consistent ordered-Verne advantage.`; |
| return; |
| } |
| const matchedMean = data.controls.display_metrics[matrixMetric].deltas_vs_matched_baseline[matrixFamily]; |
| const precisionWarning = matrixFamily === "random" ? " Random-versus-trained is also confounded by the precision path." : ""; |
| controlTakeaway.textContent = `All 31 runs are validated. ${familyLabels[matrixFamily]} has a matched mean change of ${matrixMetric === "repeat907" ? `${signed(matchedMean, 2)} pp` : signed(matchedMean, 4)} versus no transplant across nine fixed cells.${precisionWarning}`; |
| }; |
| |
| const renderMatrixTable = () => { |
| const target = document.getElementById("control-matrix-target"); |
| if (!target) return; |
| const rows = bodySeeds.map((body) => { |
| const baseline = controlRunMap.get(`${body}:baseline`); |
| const cells = donorSeeds.map((donor) => { |
| const treatment = controlRunMap.get(`${body}:${matrixFamily}_d${donor}`); |
| const comparator = matrixFamily === "ordered" |
| ? controlRunMap.get(`${body}:shuffled_d${donor}`) |
| : baseline; |
| const comparatorName = matrixFamily === "ordered" ? "Shuffled Verne" : "No transplant"; |
| if (!treatment || !comparator) { |
| const reason = !comparator ? `${comparatorName} not run yet` : `${familyLabels[matrixFamily]} not run yet`; |
| return `<td class="matrix-cell pending" data-label="Donor draw ${donor}"><strong>Matched pair incomplete</strong><span>${escapeHtml(reason)}; missing, not zero</span></td>`; |
| } |
| const delta = treatment[matrixMetric] - comparator[matrixMetric]; |
| return matrixFamily === "ordered" |
| ? `<td class="matrix-cell" data-label="Donor draw ${donor}"><strong>${controlDelta(matrixMetric, delta)}</strong><span>Ordered ${controlValue(matrixMetric, treatment[matrixMetric])}<br>Shuffled ${controlValue(matrixMetric, comparator[matrixMetric])}</span></td>` |
| : `<td class="matrix-cell" data-label="Donor draw ${donor}"><strong>${controlValue(matrixMetric, treatment[matrixMetric])}</strong><span>${controlDelta(matrixMetric, delta)} vs no transplant</span></td>`; |
| }).join(""); |
| const referenceText = matrixFamily === "ordered" |
| ? "Ordered minus shuffled" |
| : `No transplant ${baseline ? controlValue(matrixMetric, baseline[matrixMetric]) : "not run yet"}`; |
| return `<tr><th scope="row" class="stub">Body starting draw ${body}<br><span class="tiny muted">${escapeHtml(referenceText)}</span></th>${cells}</tr>`; |
| }).join(""); |
| const comparisonText = matrixFamily === "ordered" |
| ? "Ordered minus shuffled Verne" |
| : `${familyLabels[matrixFamily]} minus no transplant`; |
| target.innerHTML = `<div class="matrix-wrap"><table class="control-matrix" id="control-matrix-table"> |
| <caption class="sr-only">${escapeHtml(comparisonText)}, ${escapeHtml(outcomeLabels[matrixMetric])}, by body starting draw and donor draw. Missing cells are not run yet, not zero.</caption> |
| <thead><tr><th class="stub">Matched comparison</th>${donorSeeds.map((donor) => `<th scope="col">Donor draw ${donor}</th>`).join("")}</tr></thead> |
| <tbody>${rows}</tbody> |
| </table></div><p class="matrix-note">Hatched cells are missing and are never treated as zero. ${matrixFamily === "ordered" ? "Each cell is ordered minus frequency-matched shuffled Verne; lower repetition and higher similarity favor ordered." : "Each value is paired with the no-transplant baseline from the same body starting draw."}</p>`; |
| updateMatrixTakeaway(); |
| }; |
| |
| const renderControlStepFour = () => { |
| controlStage.innerHTML = `${controlStageHead("Inspect the full crossed matrix", "Choose one row family and one outcome. Each cell keeps body and donor identity visible.")} |
| <div class="matrix-tools" role="group" aria-label="Choose transplanted row family"> |
| ${treatmentFamilies.map((family) => `<button type="button" data-matrix-family="${family}" aria-pressed="${family === matrixFamily}" aria-controls="control-matrix-table">${escapeHtml(family === "ordered" ? "Ordered vs shuffled" : familyLabels[family])}</button>`).join("")} |
| </div> |
| <div class="matrix-tools" role="group" aria-label="Choose control outcome"> |
| <button type="button" data-matrix-metric="repeat907" aria-pressed="${matrixMetric === "repeat907"}" aria-controls="control-matrix-table">Repeated titles</button> |
| <button type="button" data-matrix-metric="S907" aria-pressed="${matrixMetric === "S907"}" aria-controls="control-matrix-table">Similarity to teacher</button> |
| </div> |
| <div id="control-matrix-target"></div>`; |
| controlStage.querySelectorAll("[data-matrix-family]").forEach((button) => button.addEventListener("click", safeInteraction(() => { |
| matrixFamily = button.dataset.matrixFamily; |
| controlStage.querySelectorAll("[data-matrix-family]").forEach((candidate) => candidate.setAttribute("aria-pressed", String(candidate === button))); |
| renderMatrixTable(); |
| }))); |
| controlStage.querySelectorAll("[data-matrix-metric]").forEach((button) => button.addEventListener("click", safeInteraction(() => { |
| matrixMetric = button.dataset.matrixMetric; |
| controlStage.querySelectorAll("[data-matrix-metric]").forEach((candidate) => candidate.setAttribute("aria-pressed", String(candidate === button))); |
| renderMatrixTable(); |
| }))); |
| renderMatrixTable(); |
| }; |
| |
| const renderControlStep = () => { |
| if (controlStep === 1) renderControlStepOne(); |
| else if (controlStep === 2) renderControlStepTwo(); |
| else if (controlStep === 3) renderControlStepThree(); |
| else renderControlStepFour(); |
| }; |
| |
| const renderControls = () => { |
| const treatmentCoverage = bodySeeds.map((body) => controlRuns.filter((run) => run.body_seed === body && treatmentFamilies.includes(run.family)).length); |
| const missing = 31 - controlRuns.length; |
| const noopText = noopExact |
| ? noopEvidence.historical_model_exact_log_only === true ? "Exact, log evidence" : "Exact" |
| : noopFailed ? "Failed" : "Not verified"; |
| if (effectiveControlStatus === "not_available" || effectiveControlStatus === "invalid") { |
| guideButtons.forEach((button) => { button.disabled = true; }); |
| controlStatus.textContent = effectiveControlStatus === "invalid" ? "Control data failed validation" : "Control data not available"; |
| controlStats.innerHTML = [statCard("Planned", "31 runs"), statCard("Validated", String(controlRuns.length)), statCard("No-op check", noopText), statCard("Status", effectiveControlStatus === "invalid" ? "Invalid" : "Not available")].join(""); |
| controlTakeaway.textContent = effectiveControlStatus === "invalid" |
| ? "The control payload failed validation, so control outcomes are suppressed." |
| : "No validated control outcomes are available in this payload."; |
| controlCaptionTakeaway.textContent = "No control interpretation is made because validated outcomes are unavailable."; |
| controlStage.innerHTML = `<p class="data-error" role="status">${effectiveControlStatus === "invalid" ? "Control records failed validation; outcomes are suppressed." : "No validated control outcomes are available."}</p>`; |
| return; |
| } |
| const complete = effectiveControlStatus === "complete"; |
| controlStatus.textContent = complete ? "31 planned runs complete" : `${controlRuns.length} of 31 validated runs available`; |
| controlStats.innerHTML = [ |
| statCard("Validated", `${controlRuns.length} / 31`), |
| statCard("Not run yet", String(missing)), |
| statCard("No-op check", noopText), |
| statCard("Treatments by body", treatmentCoverage.join(" / ")) |
| ].join(""); |
| controlCaptionTakeaway.textContent = complete |
| ? "The complete matrix contains every body-by-donor treatment cell. Interpretation should follow consistency across matched cells, not an isolated best run." |
| : `This partial matrix is unbalanced: validated treatment counts by body starting draw are ${treatmentCoverage.join(", ")}. Family averages are descriptive progress, not completed estimates.`; |
| guideButtons.forEach((button) => button.addEventListener("click", safeInteraction(() => { |
| const next = Number(button.dataset.controlStep); |
| if (next === controlStep) return; |
| controlStep = next; |
| guideButtons.forEach((candidate) => candidate.setAttribute("aria-pressed", String(candidate === button))); |
| renderControlStep(); |
| }))); |
| renderControlStep(); |
| }; |
| try { |
| renderParadox(); |
| renderJudge(); |
| renderTransitions(); |
| renderControls(); |
| const compactMedia = matchMedia("(max-width: 520px)"); |
| compactMedia.addEventListener?.("change", safeInteraction(() => { |
| renderParadox(); |
| renderJudge(); |
| })); |
| const printableDetails = [...document.querySelectorAll("details.progressive-facts")]; |
| let prePrintOpenStates = []; |
| addEventListener("beforeprint", () => { |
| prePrintOpenStates = printableDetails.map((detail) => detail.open); |
| printableDetails.forEach((detail) => { detail.open = true; }); |
| }); |
| addEventListener("afterprint", () => { |
| printableDetails.forEach((detail, index) => { detail.open = prePrintOpenStates[index] ?? detail.open; }); |
| }); |
| } catch (error) { |
| showPayloadFailure(error); |
| } |
| })(); |
| </script> |
| </body> |
| </html> |
|
|
