RB4 runda 3: blend_rb4/extract/launch_r3 + build_v2_blend fix (eot z tokenizera, brak klucza synth = stop); skan egress GREEN

blend_rb4.py

@@ -0,0 +1,139 @@
+#!/usr/bin/env python3
+"""RB4 blend builder (runda 3+), zastepuje blend_edu.py z rundy 1.
+
+Poprawki wobec rundy 1 (confoundy z 2026-09-25):
+- separator dokumentu brany z tokenizera (<|endoftext|>), nigdy vocab-1;
+- zrodlo probkowane z CALEGO pliku na granicach dokumentow (seedowany wybor), nie z glowy;
+- manifest z tokenami per zrodlo i per region (dokumenty czatu/QA = zawieraja <|im_start|>).
+
+Tryby:
+  index   --src arcmix.bin --out arcmix_index.npz
+          granice dokumentow + flaga QA (dokument zawiera <|im_start|>)
+  sample  --src arcmix.bin --index arcmix_index.npz --tokens N [--qa-weight W] --out X.bin
+          seedowany wybor dokumentow (dokument QA wystepuje W razy w puli) az do N tokenow,
+          zapis w przetasowanej kolejnosci
+  mix     --a A.bin --b B.bin --a-tokens N --b-tokens M --out X.bin
+          dwa pliki z tym samym separatorem -> przetasowanie na poziomie dokumentow
+
+Kazdy tryb zapisuje <out>.json (manifest) obok wyjscia.
+"""
+import argparse
+import hashlib
+import json
+from pathlib import Path
+
+import numpy as np
+
+CHUNK = 250_000_000
+
+
+def special_ids(tokenizer):
+    added = {t["content"]: t["id"] for t in json.load(open(tokenizer, encoding="utf-8"))["added_tokens"]}
+    return added["<|endoftext|>"], added["<|im_start|>"]
+
+
+def doc_bounds(arr, eos):
+    """(starts, lens) dokumentow konczacych sie eos; ogon bez eos pomijany."""
+    ends = []
+    for lo in range(0, len(arr), CHUNK):
+        ends.append(np.flatnonzero(np.asarray(arr[lo:lo + CHUNK]) == eos) + lo)
+    ends = np.concatenate(ends).astype(np.int64)
+    starts = np.concatenate([[0], ends[:-1] + 1])
+    return starts, ends - starts + 1
+
+
+def write_docs(out_path, parts):
+    """parts: iterowalne (array, start, len); zapis + sha256."""
+    sha, total, buf, buflen = hashlib.sha256(), 0, [], 0
+    with open(out_path, "wb") as out:
+        for arr, s, n in parts:
+            buf.append(np.asarray(arr[s:s + n]))
+            buflen += n
+            if buflen >= 50_000_000:
+                x = np.concatenate(buf)
+                x.tofile(out)
+                sha.update(x.tobytes())
+                total += len(x)
+                buf, buflen = [], 0
+        if buf:
+            x = np.concatenate(buf)
+            x.tofile(out)
+            sha.update(x.tobytes())
+            total += len(x)
+    return total, sha.hexdigest()
+
+
+def cmd_index(a):
+    eos, im_start = special_ids(a.tokenizer)
+    arr = np.memmap(a.src, dtype=np.uint16, mode="r")
+    starts, lens = doc_bounds(arr, eos)
+    qa_pos = np.concatenate([np.flatnonzero(np.asarray(arr[lo:lo + CHUNK]) == im_start) + lo
+                             for lo in range(0, len(arr), CHUNK)])
+    qa = np.zeros(len(starts), dtype=bool)
+    qa[np.searchsorted(starts, qa_pos, side="right") - 1] = True
+    np.savez(a.out, starts=starts, lens=lens, qa=qa)
+    meta = {"src": a.src, "src_tokens": int(len(arr)), "eos": eos, "docs": int(len(starts)),
+            "doc_tokens": int(lens.sum()), "qa_docs": int(qa.sum()), "qa_tokens": int(lens[qa].sum())}
+    Path(a.out + ".json").write_text(json.dumps(meta, indent=1))
+    print(json.dumps(meta), flush=True)
+
+
+def cmd_sample(a):
+    eos, _ = special_ids(a.tokenizer)
+    arr = np.memmap(a.src, dtype=np.uint16, mode="r")
+    idx = np.load(a.index)
+    starts, lens, qa = idx["starts"], idx["lens"], idx["qa"]
+    pool = np.concatenate([np.arange(len(starts))] + [np.flatnonzero(qa)] * (a.qa_weight - 1))
+    rng = np.random.default_rng(a.seed)
+    rng.shuffle(pool)
+    take = pool[:np.searchsorted(np.cumsum(lens[pool]), a.tokens) + 1]
+    total, sha = write_docs(a.out, ((arr, int(starts[i]), int(lens[i])) for i in take))
+    qa_tok = int(lens[take][qa[take]].sum())
+    meta = {"mode": "sample", "src": a.src, "seed": a.seed, "eos": eos, "qa_weight": a.qa_weight,
+            "docs": int(len(take)), "unique_docs": int(len(np.unique(take))), "tokens": total,
+            "qa_tokens": qa_tok, "qa_share": qa_tok / total,
+            "src_qa_share": float(lens[qa].sum() / lens.sum()), "sha256": sha}
+    Path(a.out + ".json").write_text(json.dumps(meta, indent=1))
+    print(json.dumps(meta), flush=True)
+
+
+def cmd_mix(a):
+    eos, _ = special_ids(a.tokenizer)
+    A = np.memmap(a.a, dtype=np.uint16, mode="r")
+    B = np.memmap(a.b, dtype=np.uint16, mode="r")
+    sa, la = doc_bounds(A, eos)
+    sb, lb = doc_bounds(B, eos)
+    rng = np.random.default_rng(a.seed)
+    oa, ob = rng.permutation(len(sa)), rng.permutation(len(sb))
+    oa = oa[:np.searchsorted(np.cumsum(la[oa]), a.a_tokens) + 1]
+    ob = ob[:np.searchsorted(np.cumsum(lb[ob]), a.b_tokens) + 1]
+    tagged = np.concatenate([np.stack([np.zeros_like(oa), oa], 1), np.stack([np.ones_like(ob), ob], 1)])
+    rng.shuffle(tagged)
+    src = ((A, int(sa[i]), int(la[i])) if t == 0 else (B, int(sb[i]), int(lb[i])) for t, i in tagged)
+    total, sha = write_docs(a.out, src)
+    at, bt = int(la[oa].sum()), int(lb[ob].sum())
+    meta = {"mode": "mix", "a": a.a, "b": a.b, "seed": a.seed, "eos": eos,
+            "a_docs": int(len(oa)), "a_tokens": at, "b_docs": int(len(ob)), "b_tokens": bt,
+            "tokens": total, "a_share": at / total, "sha256": sha}
+    Path(a.out + ".json").write_text(json.dumps(meta, indent=1))
+    print(json.dumps(meta), flush=True)
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--tokenizer", required=True)
+    ap.add_argument("--seed", type=int, default=1337)
+    sub = ap.add_subparsers(dest="cmd", required=True)
+    p = sub.add_parser("index"); p.add_argument("--src", required=True); p.add_argument("--out", required=True)
+    p = sub.add_parser("sample"); p.add_argument("--src", required=True); p.add_argument("--index", required=True)
+    p.add_argument("--tokens", type=int, required=True); p.add_argument("--qa-weight", type=int, default=1)
+    p.add_argument("--out", required=True)
+    p = sub.add_parser("mix"); p.add_argument("--a", required=True); p.add_argument("--b", required=True)
+    p.add_argument("--a-tokens", type=int, required=True); p.add_argument("--b-tokens", type=int, required=True)
+    p.add_argument("--out", required=True)
+    a = ap.parse_args()
+    {"index": cmd_index, "sample": cmd_sample, "mix": cmd_mix}[a.cmd](a)
+
+
+if __name__ == "__main__":
+    main()

build_v2_blend.py

@@ -92,12 +92,19 @@ def main():
 
     tok = Tokenizer.from_file(a.tokenizer)
     vocab = tok.get_vocab_size()
-    eot = a.eot_id if a.eot_id is not None else vocab - 1
+    eos_id = tok.token_to_id("<|endoftext|>")
+    if eos_id is None:
+        raise SystemExit("tokenizer nie ma <|endoftext|>; podaj --eot-id jawnie")
+    eot = a.eot_id if a.eot_id is not None else eos_id
+    if eot != eos_id:
+        print(f"WARN: --eot-id={eot} rozni sie od <|endoftext|>={eos_id}", flush=True)
     idx = json.loads(Path(a.decontam_index).read_text())
+    missing = [k for k in ("hard_hashes", "blimp_hashes", "synth_hard_hashes") if k not in idx]
+    if missing:
+        # Bramka Harta/Wartownika 2026-09-25: brak klucza = stop, nigdy cichy fallback na 13-gram.
+        raise SystemExit(f"decontam index {a.decontam_index} nie ma kluczy: {missing}")
     hard = set(idx["hard_hashes"]); blimp = set(idx["blimp_hashes"])
-    # synthetic paraphrase-gate reuses hard_hashes at 8-gram granularity: rebuild an 8-gram set from
-    # the same eval sources if provided, else fall back to the 13-gram hard set (still drop-on-any).
-    synth_hard = set(idx.get("synth_hard_hashes", idx["hard_hashes"]))
+    synth_hard = set(idx["synth_hard_hashes"])
     print(f"tokenizer vocab={vocab} eot={eot} | decontam hard={len(hard):,} blimp={len(blimp):,} "
           f"synth_hard={len(synth_hard):,}", flush=True)
 
@@ -183,7 +190,9 @@ def main():
     train_fh.close(); val_fh.close()
     meta = {"blend": report, "total_tok": total_written, "synthetic_tok": synth_tok,
             "synthetic_frac": round(synth_tok / max(total_written, 1), 4),
-            "vocab": vocab, "train_tok": state["train"], "val_tok": state["val"], "seed": a.seed}
+            "vocab": vocab, "eot": eot, "train_tok": state["train"], "val_tok": state["val"], "seed": a.seed,
+            "decontam": {"index": str(a.decontam_index), "hard": len(hard), "blimp": len(blimp),
+                         "synth_hard": len(synth_hard)}}
     (outd / "mix_meta.json").write_text(json.dumps(meta, indent=2))
     print(f"WROTE {outd}/train.bin ({state['train']:,} tok) + val.bin ({state['val']:,}) + mix_meta.json", flush=True)
 

extract_edu_from_blend.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+"""Odzysk tokenow edu z blendu fork-B (RB4 runda 1) + naprawa separatora.
+
+Blend fork-B = przetasowane bloki: arcmix[0:~1.15B] (separator dokumentu 12285, zero 12287)
+oraz dokumenty edu zakonczone 12287 (<|im_end|>, blad buildu: eot=vocab-1).
+
+Dzielimy strumien na kawalki konczace sie 12287. Kawalek bez 12285 i bez 12286 = czysty
+dokument edu -> zapis z separatorem podmienionym na 12285. Kawalek zawierajacy 12285/12286
+niesie blok arcmixu -> odrzucony w calosci (traci sie co najwyzej jeden dokument edu na blok).
+
+Wyjscie: edu.bin (uint16) + edu.json (liczby do weryfikacji wobec logu buildu).
+"""
+import argparse
+import hashlib
+import json
+
+import numpy as np
+
+EOS, IM_START, IM_END = 12285, 12286, 12287
+CHUNK = 200_000_000
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--blend", required=True)
+    ap.add_argument("--out", required=True)
+    a = ap.parse_args()
+
+    b = np.memmap(a.blend, dtype=np.uint16, mode="r")
+    n = len(b)
+    out = open(a.out, "wb")
+    sha = hashlib.sha256()
+    kept_docs = kept_tok = drop_pieces = drop_tok = 0
+    carry = np.empty(0, dtype=np.uint16)
+    for lo in range(0, n, CHUNK):
+        x = np.concatenate([carry, np.asarray(b[lo:lo + CHUNK])])
+        ends = np.flatnonzero(x == IM_END)
+        if lo + CHUNK >= n and (len(ends) == 0 or ends[-1] != len(x) - 1):
+            tail = len(x) - (ends[-1] + 1 if len(ends) else 0)
+            drop_pieces += 1
+            drop_tok += tail
+        if len(ends) == 0:
+            carry = x
+            continue
+        starts = np.concatenate([[0], ends[:-1] + 1])
+        bad = np.zeros(len(ends), dtype=bool)
+        for code in (EOS, IM_START):
+            pos = np.flatnonzero(x[:ends[-1] + 1] == code)
+            bad[np.searchsorted(ends, pos)] = True
+        seg = x[:ends[-1] + 1].copy()
+        seg[ends] = EOS
+        keep_mask = np.repeat(~bad, ends - starts + 1)
+        kept = seg[keep_mask]
+        kept.tofile(out)
+        sha.update(kept.tobytes())
+        kept_docs += int((~bad).sum())
+        kept_tok += int(len(kept))
+        drop_pieces += int(bad.sum())
+        drop_tok += int((ends - starts + 1)[bad].sum())
+        carry = x[ends[-1] + 1:]
+    out.close()
+    stats = {"blend_tokens": int(n), "edu_docs": kept_docs, "edu_tokens": kept_tok,
+             "dropped_pieces": drop_pieces, "dropped_tokens": drop_tok,
+             "separator": EOS, "sha256": sha.hexdigest()}
+    json.dump(stats, open(a.out.rsplit(".", 1)[0] + ".json", "w"), indent=1)
+    print(json.dumps(stats), flush=True)
+
+
+if __name__ == "__main__":
+    main()

launch_r3.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+# launch_r3.sh — RB4 runda 3: fork z ckpt 320k, config 1:1 jak control, rozni sie TYLKO danymi.
+# Czeka az obecny trening na podzie sie skonczy (WAITPID) i az dane beda kompletne (manifest),
+# weryfikuje krok bazy (ck["step"] == 320000), dopiero wtedy startuje trener + autopush + pusher.
+#
+# Uzycie: bash launch_r3.sh WORK DATA OUT BASE HFSUB NAME WAITPID
+set -euo pipefail
+WORK="$1"; DATA="$2"; OUT="$3"; BASE="$4"; HFSUB="$5"; NAME="$6"; WAITPID="$7"
+cd "$WORK"
+echo "[$(date -u +%H:%M:%SZ)] launch_r3: czekam na pid $WAITPID i $DATA/train.bin.json"
+while kill -0 "$WAITPID" 2>/dev/null; do sleep 20; done
+while [ ! -f "$DATA/train.bin.json" ]; do sleep 20; done
+[ -e "$DATA/val.bin" ] || { echo "!! brak $DATA/val.bin"; exit 1; }
+
+STEP=$(python3 -c "import torch,sys; print(int(torch.load(sys.argv[1], map_location='cpu')['step']))" "$BASE")
+[ "$STEP" = "320000" ] || { echo "!! baza $BASE ma step $STEP, oczekiwano 320000"; exit 1; }
+mkdir -p "$OUT"
+[ -e "$OUT/ckpt.pt" ] && { echo "!! $OUT/ckpt.pt juz istnieje, nie nadpisuje"; exit 1; }
+cp "$BASE" "$OUT/ckpt.pt"
+echo "[$(date -u +%H:%M:%SZ)] baza OK (step $STEP), start treningu $NAME"
+
+nohup python -u train_gpt_ref.py --data-dir "$DATA" --out-dir "$OUT" \
+  --n-layer 14 --n-embd 576 --n-head 9 --block 1024 --vocab 12288 --norm rmsnorm --pos rope \
+  --rope-theta 100000.0 --ffn swiglu --ffn-mult 2.667 --value-residual --qk-norm \
+  --batch 32 --steps 400000 --lr 6e-4 --min-lr 6e-5 --warmup 2000 --wd 0.1 \
+  --optimizer muon --muon-lr 0.02 --dtype uint16 --seed 1337 \
+  --ckpt-every 20000 --eval-every 999999 --log-every 200 \
+  --events-jsonl "$OUT/events.jsonl" --compile --resume > "$OUT/train.log" 2>&1 &
+echo "TRAIN-PID=$!"
+sleep 5
+nohup python -u ckpt_autopush.py "$OUT/train.log" "$OUT/ckpt.pt" "$HFSUB" > "$OUT/autopush.log" 2>&1 &
+echo "AUTOPUSH-PID=$!"
+set -a; . /root/.fabryka.env; set +a
+nohup python -u fabryka_push.py "$OUT/events.jsonl" "$NAME" gollem-v5 13107200000 > "$OUT/fabryka_push.log" 2>&1 &
+echo "PUSHER-PID=$!"
+echo "[$(date -u +%H:%M:%SZ)] launch_r3: wszystko uruchomione"