File size: 4,033 Bytes
9d1ec74 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 | #!/usr/bin/env python3
"""RB3 decontam reference-index: 13-gram word-shingles z benchmarkow eval (Glint-parity).
Cel: zbudowac indeks n-gramow z DOKLADNIE tych zbiorow ktore ewaluujemy (WikiText-2 test,
BLiMP wszystkie configi good+bad, ARC-Easy+Challenge test), zeby filtr UFW mogl wyrzucic
zanieczyszczone dokumenty. Prawda-z-bajtow: zrodla identyczne z glint_parity_eval.py.
Reguly (RB3-spec):
- WikiText-2 + ARC (E+C): drop-UFW-doc na ANY-13gram-match (male zbiory, czystosc krytyczna).
- BLiMP: match-fraction > 0.5% (krotkie zdania, unikamy ciecia generycznych).
Indeks: blake2b-8B hash kazdego 13-gramu (word-level, znormalizowany). Zapis: decontam_index.json
{version, n13, hard_hashes:[...], blimp_hashes:[...], counts:{...}}.
Uzycie: python build_decontam_index.py [--out decontam_index.json]
"""
import argparse
import hashlib
import json
import re
import sys
from pathlib import Path
# reuse dokladnych loaderow + configow z eval (te same zrodla = wlasciwa dekontaminacja)
sys.path.insert(0, str(Path(__file__).parent))
from glint_parity_eval import _rows, BLIMP_CONFIGS # noqa: E402
N = 13 # 13-gram word-level (konwencja GPT-3/FineWeb)
def normalize(text):
text = text.lower()
text = re.sub(r"[^\w\s]", " ", text) # strip punctuation
text = re.sub(r"\s+", " ", text).strip()
return text
def shingle_hashes(text, n=N):
words = normalize(text).split()
out = set()
for i in range(len(words) - n + 1):
sh = " ".join(words[i:i + n])
out.add(hashlib.blake2b(sh.encode("utf-8"), digest_size=8).hexdigest())
return out
def collect(texts, label, n=N):
idx = set()
docs = 0
for t in texts:
if not t or not t.strip():
continue
docs += 1
idx |= shingle_hashes(t, n)
print(f" [{label}] docs={docs:,} shingles({n}gram)={len(idx):,}", flush=True)
return idx
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--out", default="decontam_index.json")
a = ap.parse_args()
print("=== RB3 decontam-index build (13-gram shingles) ===", flush=True)
# --- HARD sources (drop-on-any-match): WikiText-2 test + ARC-E/C test ---
hard_texts = []
print("WikiText-2 test...", flush=True)
hard_texts.extend(r["text"] for r in _rows("Salesforce/wikitext", "wikitext-2-raw-v1", "test"))
for cfg in ("ARC-Easy", "ARC-Challenge"):
print(f"ARC {cfg} test...", flush=True)
for ex in _rows("allenai/ai2_arc", cfg, "test"):
hard_texts.append(ex["question"])
hard_texts.extend(ex["choices"]["text"])
hard = collect(hard_texts, "hard-13gram", n=13)
synth_hard = collect(hard_texts, "hard-8gram-synth-paraphrase-gate", n=8)
# --- BLiMP (fraction-threshold source): all configs, good+bad ---
print(f"BLiMP ({len(BLIMP_CONFIGS)} configs) good+bad...", flush=True)
blimp_texts = []
for c in BLIMP_CONFIGS:
for e in _rows("nyu-mll/blimp", c, "train"):
blimp_texts.append(e["sentence_good"])
blimp_texts.append(e["sentence_bad"])
blimp = collect(blimp_texts, "blimp-all")
out = {
"version": 2, "n": N, "n_synth": 8,
"hard_hashes": sorted(hard), # WikiText-2 + ARC, 13-gram (drop-on-any, all sources)
"blimp_hashes": sorted(blimp), # BLiMP, 13-gram (fraction>0.5%)
"synth_hard_hashes": sorted(synth_hard), # WikiText-2 + ARC, 8-gram (drop-on-any, SYNTHETIC only)
"counts": {"hard": len(hard), "blimp": len(blimp), "synth_hard": len(synth_hard),
"hard_sources": "wikitext2-test + arc-easy/challenge-test",
"blimp_sources": f"blimp-{len(BLIMP_CONFIGS)}cfg-good+bad",
"synth_hard_note": "8-gram paraphrase-proxy for synthetic sources (Cosmopedia/Nemotron)"},
}
Path(a.out).write_text(json.dumps(out), encoding="utf-8")
print(f"=== WROTE {a.out}: hard={len(hard):,} blimp={len(blimp):,} 13-gram-hashes ===", flush=True)
if __name__ == "__main__":
main()
|