Upload folder using huggingface_hub

src/aigc_detector.egg-info/PKG-INFO

@@ -0,0 +1,20 @@
+Metadata-Version: 2.4
+Name: aigc-detector
+Version: 0.1.0
+Requires-Python: >=3.11
+Requires-Dist: torch>=2.2
+Requires-Dist: torchvision>=0.17
+Requires-Dist: numpy>=1.26
+Requires-Dist: pandas>=2.2
+Requires-Dist: pyarrow>=15
+Requires-Dist: pillow>=10
+Requires-Dist: scikit-learn>=1.4
+Requires-Dist: wandb>=0.16
+Requires-Dist: psutil>=5.9
+Requires-Dist: opencv-python-headless>=4.9
+Provides-Extra: dev
+Requires-Dist: pytest>=8; extra == "dev"
+Provides-Extra: data
+Requires-Dist: datasets>=2.18; extra == "data"
+Requires-Dist: huggingface_hub>=0.23; extra == "data"
+Requires-Dist: modelscope>=1.13; extra == "data"

src/aigc_detector.egg-info/SOURCES.txt

@@ -0,0 +1,80 @@
+README.md
+pyproject.toml
+src/__init__.py
+src/config.py
+src/aigc_detector.egg-info/PKG-INFO
+src/aigc_detector.egg-info/SOURCES.txt
+src/aigc_detector.egg-info/dependency_links.txt
+src/aigc_detector.egg-info/requires.txt
+src/aigc_detector.egg-info/top_level.txt
+src/analysis/__init__.py
+src/analysis/format_probe.py
+src/analysis/subset_weakness.py
+src/data/__init__.py
+src/data/dataset.py
+src/data/preprocess.py
+src/data/shard_reader.py
+src/data/aug/__init__.py
+src/data/aug/augment.py
+src/data/aug/augment_impls.py
+src/data/aug/deshortcut.py
+src/data/aug/random_resolution.py
+src/data/aug/symmetry_audit.py
+src/data/aug/warp.py
+src/data/provenance/__init__.py
+src/data/provenance/audit.py
+src/data/provenance/cleanup.py
+src/data/provenance/dda_audit.py
+src/data/provenance/dedup_audit.py
+src/data/provenance/diskcheck.py
+src/data/provenance/families.py
+src/data/provenance/feature_cache.py
+src/data/provenance/finalize.py
+src/data/provenance/journal.py
+src/data/provenance/manifest_build.py
+src/data/provenance/orchestrate.py
+src/data/provenance/push.py
+src/data/provenance/rangezip.py
+src/data/provenance/resolver.py
+src/data/provenance/run_curator.py
+src/data/provenance/selection.py
+src/data/provenance/sources.py
+src/data/provenance/splitter.py
+src/data/provenance/subsample.py
+src/data/provenance/val_holdout.py
+src/data/provenance/wildfake.py
+src/data/provenance/writer.py
+src/eval/__init__.py
+src/eval/error_gallery.py
+src/eval/explain.py
+src/eval/harness.py
+src/eval/metrics.py
+src/eval/predictions.py
+src/eval/stream.py
+src/inference/__init__.py
+src/inference/predict.py
+src/inference/server.py
+src/model/__init__.py
+src/model/_dinov2.py
+src/model/backbone.py
+src/model/detector.py
+src/model/freq.py
+src/model/fusion.py
+src/model/head.py
+src/train/__init__.py
+src/train/checkpoint.py
+src/train/compile_backbone.py
+src/train/driver.py
+src/train/logging_utils.py
+src/train/loop.py
+src/train/loss.py
+src/train/preflight.py
+src/train/watchdog.py
+tests/test_aug.py
+tests/test_evaluation.py
+tests/test_explainability.py
+tests/test_inference.py
+tests/test_model.py
+tests/test_preprocessing.py
+tests/test_provenance.py
+tests/test_training.py

src/aigc_detector.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

src/aigc_detector.egg-info/requires.txt

@@ -0,0 +1,18 @@
+torch>=2.2
+torchvision>=0.17
+numpy>=1.26
+pandas>=2.2
+pyarrow>=15
+pillow>=10
+scikit-learn>=1.4
+wandb>=0.16
+psutil>=5.9
+opencv-python-headless>=4.9
+
+[data]
+datasets>=2.18
+huggingface_hub>=0.23
+modelscope>=1.13
+
+[dev]
+pytest>=8

src/aigc_detector.egg-info/top_level.txt

@@ -0,0 +1,8 @@
+__init__
+analysis
+config
+data
+eval
+inference
+model
+train

src/analysis/format_probe.py

@@ -0,0 +1,115 @@
+# src/analysis/format_probe.py
+"""Format/codec-shortcut probe — validates that the holdout AUC is content, not codec.
+
+A format-only rule ("PNG = fake, JPEG = real") scores ~0.845 on our
+holdout — essentially identical to the model's 0.84. DINOv2 never sees filenames, but it DOES
+see JPEG quantization/blocking/chroma-subsampling residuals, and published detectors have been
+shown to treat JPEG compression as evidence of authenticity ("Fake or JPEG?", arXiv 2403.17608).
+Our resolution defenses do nothing for this axis.
+
+The decisive test: restrict to a SINGLE codec (JPEG) present in BOTH classes, size-match, and
+stratify by generator. If the AUC collapses toward 0.5 inside that controlled slice, most of the
+headline is codec/provenance, not detection. If it stays strong, the shortcut is not what's
+driving the score. This module computes exactly that controlled AUC alongside the raw AUC.
+
+`format` is derived from the image path extension (verbatim from the sniffed source format in
+save_source_image), so no manifest schema change is needed — .jpg/.jpeg -> JPEG, .png -> PNG.
+"""
+import os
+
+import numpy as np
+import pandas as pd
+
+from eval.metrics import roc_auc
+from analysis.subset_weakness import resolution_band
+
+
+def format_of(path_or_ext) -> str:
+    """Map a path or extension to a coarse codec label: JPEG / PNG / WEBP / OTHER."""
+    ext = os.path.splitext(str(path_or_ext))[1].lower().lstrip(".") or str(path_or_ext).lower().lstrip(".")
+    if ext in ("jpg", "jpeg"):
+        return "JPEG"
+    if ext == "png":
+        return "PNG"
+    if ext == "webp":
+        return "WEBP"
+    return "OTHER"
+
+
+def _format_col(df) -> pd.Series:
+    """Return a 'format' Series for df, using an existing column or deriving from image_path."""
+    if "format" in df.columns and df["format"].notna().any():
+        return df["format"].astype(str)
+    src = "image_path" if "image_path" in df.columns else "path"
+    return df[src].map(format_of)
+
+
+def format_leakage_auc(pred_df, condition="clean") -> dict:
+    """How well does codec ALONE separate the classes? Scores format as an ordinal signal
+    (JPEG=0, PNG=1) against the label — the 'format-only classifier' baseline.
+    A value near the model's AUC means codec could account for most of the headline."""
+    df = pred_df[pred_df["condition"] == condition] if "condition" in pred_df else pred_df
+    fmt = _format_col(df)
+    # PNG=1 leans fake, JPEG=0 leans real (our corpus polarity); WEBP/OTHER map to 0.5.
+    score = fmt.map({"PNG": 1.0, "JPEG": 0.0}).fillna(0.5).to_numpy()
+    y = df["label"].to_numpy()
+    auc = roc_auc(y, score) if len(np.unique(y)) == 2 else float("nan")
+    counts = {f"{lab}:{f}": int(n) for (lab, f), n in df.assign(_f=fmt).groupby(["label", "_f"]).size().items()}
+    return {"format_only_auc": float(auc), "counts": counts}
+
+
+def format_controlled_auc(pred_df, keep_format="JPEG", condition="clean", size_match=True,
+                          min_n=15) -> dict:
+    """Model AUC restricted to a single codec present in BOTH classes (default JPEG), optionally
+    size-band-matched, so codec cannot be the discriminator. Returns the controlled AUC, the raw
+    AUC on the same condition, the drop, and per-generator controlled AUCs.
+
+    A large drop (controlled << raw, controlled -> ~0.5) is the smoking gun: the headline was
+    codec/provenance, not detection. A small drop means the score survives codec control."""
+    df = pred_df[pred_df["condition"] == condition].copy() if "condition" in pred_df else pred_df.copy()
+    df["_fmt"] = _format_col(df).values
+    y_all, p_all = df["label"].to_numpy(), df["prob"].to_numpy()
+    raw_auc = roc_auc(y_all, p_all) if len(np.unique(y_all)) == 2 else float("nan")
+
+    sub = df[df["_fmt"] == keep_format]
+    # size-match: keep only resolution bands where BOTH classes are present, so within the kept
+    # codec the classes also share the resolution support (removes the residual size confound).
+    matched_bands = None
+    if size_match and "native_w" in sub.columns and sub["native_w"].notna().any():
+        sub = sub.assign(_band=sub["native_w"].map(resolution_band))
+        bands_real = set(sub[sub["label"] == 0]["_band"])
+        bands_fake = set(sub[sub["label"] == 1]["_band"])
+        matched_bands = sorted(bands_real & bands_fake)
+        sub = sub[sub["_band"].isin(matched_bands)]
+
+    y, p = sub["label"].to_numpy(), sub["prob"].to_numpy()
+    n_real = int((sub["label"] == 0).sum())
+    n_fake = int((sub["label"] == 1).sum())
+    ctrl_auc = (roc_auc(y, p) if len(np.unique(y)) == 2 and min(n_real, n_fake) >= min_n
+                else float("nan"))
+
+    per_gen = []
+    if "generator_id" in sub.columns:
+        reals = sub[sub["label"] == 0]["prob"].to_numpy()
+        for gen, g in sub[sub["label"] == 1].groupby("generator_id"):
+            fp = g["prob"].to_numpy()
+            yy = np.r_[np.zeros(len(reals)), np.ones(len(fp))]
+            pp = np.r_[reals, fp]
+            a = roc_auc(yy, pp) if len(fp) and len(reals) else float("nan")
+            per_gen.append({"generator_id": str(gen), "auc": a, "n_fake": int(len(fp)),
+                            "reliable": len(fp) >= min_n})
+    per_gen_df = pd.DataFrame(per_gen, columns=["generator_id", "auc", "n_fake", "reliable"])
+    if not per_gen_df.empty:
+        per_gen_df = per_gen_df.sort_values("auc", na_position="last").reset_index(drop=True)
+
+    return {
+        "keep_format": keep_format,
+        "raw_auc": float(raw_auc),
+        "controlled_auc": float(ctrl_auc),
+        "drop": float(raw_auc - ctrl_auc) if not (np.isnan(raw_auc) or np.isnan(ctrl_auc)) else float("nan"),
+        "n_real": n_real, "n_fake": n_fake,
+        "matched_bands": matched_bands,
+        "size_matched": bool(size_match and matched_bands is not None),
+        "per_generator": per_gen_df,
+        "reliable": min(n_real, n_fake) >= min_n,
+    }

src/analysis/subset_weakness.py

@@ -0,0 +1,95 @@
+# src/analysis/subset_weakness.py
+"""Subset-weakness analysis — WHICH slices does the detector fail on?
+
+The aggregate AUC hides generalization failures. A slice with AUC < 0.5 (below chance) means
+the model ranks that slice's fakes as MORE real than reals — a genuine failure mode, not just
+noise. This module scores each subset (generator, real_source, resolution band, format) against
+the shared real pool and surfaces the weak ones, always with the sample count N so a real
+failure is distinguished from a small-sample artifact.
+
+Motivated by the observed styleGAN ≈ 0.5 slice: GANs are a systematically weaker subset for a
+DINOv2-semantic-feature detector, which averaging into a 0.9+ headline would hide.
+"""
+import numpy as np
+import pandas as pd
+
+from eval.metrics import roc_auc
+
+
+def resolution_band(w):
+    """Bucket a width into a coarse resolution band (for slicing by resolution)."""
+    w = float(w)
+    if w < 256:
+        return "<256"
+    if w < 512:
+        return "256-512"
+    if w < 1024:
+        return "512-1024"
+    return ">=1024"
+
+
+def subset_weakness(pred_df, by="generator_id", condition="clean", min_n=15):
+    """Per-subset AUC worst-first, with N and a reliability flag. Auto-detects which class the
+    `by` column actually varies on (`by='real_source'` previously grouped FAKE rows silently,
+    which all carry real_source='none', so it never measured per-real-source weakness):
+
+    - FAKE-side attribute (e.g. generator_id): each fake subset vs the SHARED real pool.
+    - REAL-side attribute (e.g. real_source): each real subset vs the SHARED fake pool. A LOW AUC
+      here means the detector ranks that real source as more fake than average (a real-content
+      weakness) — the diagnostic the report intends.
+
+    Returns a DataFrame [subset, auc, n_fake, n_real, reliable]."""
+    df = pred_df[pred_df["condition"] == condition] if "condition" in pred_df else pred_df
+    if by not in df.columns:
+        return pd.DataFrame(columns=["subset", "auc", "n_fake", "n_real", "reliable"])
+    reals = df[df["label"] == 0]
+    fakes = df[df["label"] == 1]
+    # which class does `by` actually distinguish? Use the side with >1 distinct non-null value.
+    real_varies = reals[by].nunique(dropna=True) > 1
+    fake_varies = fakes[by].nunique(dropna=True) > 1
+    real_side = real_varies and not fake_varies      # real_source-type attribute
+    rows = []
+    if real_side:
+        # each REAL subset vs the shared FAKE pool
+        fake_probs = fakes["prob"].to_numpy()
+        for subset, g in reals.groupby(by):
+            rp = g["prob"].to_numpy()
+            y = np.r_[np.zeros(len(rp)), np.ones(len(fake_probs))]
+            p = np.r_[rp, fake_probs]
+            auc = roc_auc(y, p) if len(rp) and len(fake_probs) else float("nan")
+            rows.append({"subset": str(subset), "auc": auc, "n_fake": int(len(fake_probs)),
+                         "n_real": int(len(rp)), "reliable": len(rp) >= min_n})
+    else:
+        # each FAKE subset vs the shared REAL pool
+        real_probs = reals["prob"].to_numpy()
+        for subset, g in fakes.groupby(by):
+            fp = g["prob"].to_numpy()
+            y = np.r_[np.zeros(len(real_probs)), np.ones(len(fp))]
+            p = np.r_[real_probs, fp]
+            auc = roc_auc(y, p) if len(fp) and len(real_probs) else float("nan")
+            rows.append({"subset": str(subset), "auc": auc, "n_fake": int(len(fp)),
+                         "n_real": int(len(real_probs)), "reliable": len(fp) >= min_n})
+    out = pd.DataFrame(rows, columns=["subset", "auc", "n_fake", "n_real", "reliable"])
+    return out.sort_values("auc", na_position="last").reset_index(drop=True)
+
+
+def find_weak_subsets(pred_df, dims=("generator_id", "real_source"), auc_threshold=0.7,
+                      min_n=15, condition="clean"):
+    """Scan multiple slice dimensions and return every subset below `auc_threshold` that has
+    enough samples to be reliable. Adds a resolution-band dimension if native_w is present.
+    Returns a list of dicts {dim, subset, auc, n_fake} worst-first — the failure-mode report."""
+    df = pred_df.copy()
+    dims = list(dims)
+    if "native_w" in df.columns and "res_band" not in df.columns:
+        df["res_band"] = df["native_w"].map(resolution_band)
+        dims.append("res_band")
+    weak = []
+    for dim in dims:
+        if dim not in df.columns:
+            continue
+        rep = subset_weakness(df, by=dim, condition=condition, min_n=min_n)
+        for _, r in rep.iterrows():
+            if r["reliable"] and r["auc"] < auc_threshold:
+                weak.append({"dim": dim, "subset": r["subset"], "auc": float(r["auc"]),
+                             "n_fake": int(r["n_fake"])})
+    return sorted(weak, key=lambda x: x["auc"])

src/config.py

@@ -0,0 +1,81 @@
+# src/config.py
+from dataclasses import dataclass
+
+@dataclass
+class Config:
+    resolution: int = 518
+    patch_grid: int = 37
+    n_patch: int = 1369
+    embed_dim: int = 768
+    backbone: str = "stub"          # "stub" (local CI) | "dinov2_vitb14_reg" (GPU host)
+    # REJECTED (kept for reference): RINE-style last-N fusion never beat single-layer readout.
+    # n_intermediate_layers: int = 1
+    extract_layer: int = 8          # SHIP DEFAULT (train.py trains 8 and 10 for the ensemble) — >0 = read patch tokens after DINOv2 block L (1-indexed) and TRUNCATE the stack to L blocks (real FLOP saving); tests whether localized signal survives earlier. 0 = full depth. Mutually exclusive with n_intermediate_layers>1.
+    # REJECTED: MIL top-k image logits (k=16/64 and DC-centered) gave no transfer gain — the
+    # head's self-attention broadcasts the verdict into every patch, so top-k of a broadcast
+    # is the broadcast. Left commented for the record.
+    # mil_top_k: int = 0
+    # mil_centered: bool = False
+    label_smooth: float = 0.05       # BCE target smoothing eps: targets eps/1−eps instead of 0/1 (bounds optimal logits, calibrates probs). 0 = hard labels.
+    # REJECTED: warp minimal-pair augmentation taught the warp artifact, not anatomy
+    # (unseen-GAN AUC −0.027). Implementation retained in data/aug/warp.py for reference.
+    # warp_strength: float = 1.0
+    # p_global_warp: float = 0.0
+    corruption_scale: float = 1.0   # train-only: >1 stretches corruption severity ranges toward their harsh end (clamped). Ship default 1.0 — scaling alone measured negative; keep at 1.0 unless re-validated together with lambda_feat.
+    lambda_feat: float = 1.5        # feature-consistency weight: cosine drift of the pooled trainable representation across the clean/corrupted pair (label-blind, both classes) — forces the decision path onto corruption-stable semantic/geometric directions.
+    fusion_project: bool = True     # fusion projection: True=dense DxD (~590K); False=scalar layer-mix (K params, preferred at low data)
+    train_impls: dict | None = None # {family:[impl,...]} operator subset for TRAINING — set BEFORE training so held-out impls stay genuinely unseen (the impl-holdout gap is only valid when training is restricted to this subset)
+    data_parallel: bool = False     # DataParallel across all visible GPUs (e.g. dual T4): splits the heavy frozen-backbone forward for ~2x + halves per-GPU batch memory. No idle GPU regardless of arm count.
+    head_hidden: int = 0            # 0 = linear-probe head; >0 = MLP head (Linear-GELU-Linear)
+    head_pool: str = "attn"         # "attn" (baseline MIL) | "dist" (+mean/std/top-k patch-logit stats) | "nbr" (dist + zero-init 3x3 depthwise conv over the patch grid = local adjacency; density-gate redirect, orthogonal to global-realism signal) | "enc" (dist + zero-init self-attention encoder w/ 2D pos-embed = configural "sequence-not-bag" test)
+    # NULLED (never shipped): opt-in spectral branch — JPEG/blur corruption destroys the
+    # frequency signal it reads; only viable as a fusion member, which we did not ship.
+    # use_freq_branch: bool = False
+    enc_layers: int = 2             # EncoderPoolHead: transformer-encoder depth (head_pool="enc")
+    enc_heads: int = 8              # EncoderPoolHead: self-attention heads (head_pool="enc")
+    use_pos_embed: bool = True      # EncoderPoolHead: add fixed 2D sin/cos positional embedding before the encoder
+    # SUPERSEDED by deshortcut=True (the full pipeline); kept for ablation only.
+    # defeat_resolution: bool = False
+    deshortcut: bool = False        # train: thumbnail-degrade+contrast+sharpness+codec aug (stronger shortcut defeat)
+    p_codec: float = 0.5            # deshortcut: P(random JPEG/PNG re-encode) — kills the codec-history shortcut
+    random_resolution: bool = False # train: per-batch random resolution (varied token grid, no res anchor)
+    res_lo: int = 392               # random_resolution range lo (mult of 14)
+    res_hi: int = 644               # random_resolution range hi (mult of 14)
+    fair_eval: bool = False         # eval: normalize real+fake to common size+format (no res/format tell)
+    feature_mode: str = "jit"       # Phase-2: "jit"|"cache" — cache path not yet implemented (no consumer in Phase 1)
+    compile: bool = False           # torch.compile toggle (GPU-host only)
+    amp: bool = False               # fp16 autocast on the FROZEN backbone only (T4 tensor cores ~2x); head stays fp32
+    gpu_preprocess: bool = False    # workers emit uint8 (resize only); float/normalize/res-resize happen ON GPU (halves worker CPU, 4x smaller host transfers). Bilinear resize commutes with per-channel normalize, so numerics match the CPU path.
+    lambda_aug: float = 0.8
+    lambda_cons_max: float = 0.3
+    # NULLED (never shipped): teacher distillation from a pretrained forensics model was cut
+    # early on licensing + complexity grounds; the frozen-backbone head needs no teacher.
+    # lambda_kd: float = 0.0
+    pos_weight: float = 0.0         # 0 = unweighted BCE; >0 scales the fake(1) term (calibration)
+    entropy_gate: float = 0.5
+    stack_prob: float = 0.1
+    seed: int = 0
+    epochs: int = 10
+    epoch_frac: float = 1.0         # train on this fraction of the loader per epoch (0.5 = half an epoch); shuffle=True makes the half-pass a random subsample
+    batch_size: int = 32
+    eval_batch_size: int = 32       # eval forward-pass chunk (predict_fn); bounds VRAM on large val sets
+    eval_pool_max: int = 64         # max decoded val images resident at once (bounds HOST RAM in streaming eval)
+    eval_workers: int = 0           # >0 = thread-pool the eval corruption prep (PIL releases the GIL); 0 = serial (legacy)
+    holdout_eval_cap: int = 0       # 0=eval whole holdout; >0=stratified subsample cap (speed)
+    val_eval_cap: int = 0           # 0=eval whole val EVERY epoch; >0=stratified per-epoch subsample (a 7.5k val x 19 conds = hours of CPU-bound corruption PER EPOCH on the 47k corpus)
+    lr: float = 1e-3
+    lr_schedule: str = "none"      # "none" | "onecycle" (per-step mid-epoch warmup+cosine)
+    num_workers: int = 2
+    pin_memory: bool = True
+    max_images: int = 0             # 0 = all
+    data_root: str = "data"
+    cache_dir: str = "cache"
+    # Weights & Biases logging (OFF by default; training hosts flip use_wandb=True)
+    use_wandb: bool = False
+    wandb_project: str = "track5-aigc"
+    wandb_run_name: str = ""        # empty -> wandb autogenerates
+    log_every: int = 10             # log train metrics every N steps
+
+    @classmethod
+    def smoke(cls) -> "Config":
+        return cls(max_images=200, epochs=1, compile=False, batch_size=8)

src/data/aug/augment.py

@@ -0,0 +1,77 @@
+# src/data/augment.py
+import io
+import numpy as np
+from PIL import Image, ImageFilter, ImageEnhance
+
+CORRUPTIONS = ["jpeg", "blur", "resize", "noise", "jitter", "crop"]
+
+EVAL_GRID = {
+    "jpeg":   [90, 70, 50, 30],
+    "blur":   [0.5, 1.0, 2.0],
+    "resize": [0.5, 0.25],
+    "noise":  [0.02, 0.05, 0.10],
+    "jitter": [0.2],          # ±20%
+    "crop":   [0.8],          # keep 80%
+}
+
+# Continuous training ranges (min,max) spanning the eval grid.
+_TRAIN_RANGE = {
+    "jpeg":   (30.0, 95.0),
+    "blur":   (0.3, 2.2),
+    "resize": (0.25, 0.9),
+    "noise":  (0.01, 0.12),
+    "jitter": (0.05, 0.25),
+    "crop":   (0.7, 0.95),
+}
+
+def apply_corruption(img: Image.Image, name: str, severity: float) -> Image.Image:
+    img = img.convert("RGB")
+    if name == "jpeg":
+        buf = io.BytesIO()
+        img.save(buf, format="JPEG", quality=int(round(severity)))
+        buf.seek(0)
+        return Image.open(buf).convert("RGB")
+    if name == "blur":
+        return img.filter(ImageFilter.GaussianBlur(radius=float(severity)))
+    if name == "resize":
+        w, h = img.size
+        small = img.resize((max(1, int(w*severity)), max(1, int(h*severity))), Image.BILINEAR)
+        return small.resize((w, h), Image.BILINEAR)
+    if name == "noise":
+        arr = np.asarray(img).astype(np.float32) / 255.0
+        rng = np.random.default_rng(int(severity * 1e6))  # deterministic per severity
+        arr = arr + rng.normal(0, float(severity), arr.shape)
+        return Image.fromarray((np.clip(arr, 0, 1) * 255).astype("uint8"))
+    if name == "jitter":
+        out = ImageEnhance.Brightness(img).enhance(1 + severity)
+        out = ImageEnhance.Contrast(out).enhance(1 + severity)
+        out = ImageEnhance.Color(out).enhance(1 + severity)
+        return out
+    if name == "crop":
+        w, h = img.size
+        cw, ch = int(w*severity), int(h*severity)
+        left, top = (w - cw)//2, (h - ch)//2
+        return img.crop((left, top, left+cw, top+ch))
+    raise ValueError(f"unknown corruption {name}")
+
+# Direction of "harsher" per corruption: -1 = lower value is harsher (jpeg quality,
+# resize/crop keep-fraction), +1 = higher value is harsher (blur radius, noise sigma, jitter).
+_HARSH_DIR = {"jpeg": -1, "blur": +1, "resize": -1, "noise": +1, "jitter": +1, "crop": -1}
+# Hard floors/ceilings for scaled ranges (beyond these, reals become unrecognizable and
+# clean-AUC pays for it).
+_HARSH_LIMIT = {"jpeg": 12.0, "blur": 4.5, "resize": 0.12, "noise": 0.25, "jitter": 0.5,
+                "crop": 0.5}
+
+
+def sample_train_corruption(rng: np.random.Generator, scale: float = 1.0) -> tuple[str, float]:
+    """scale>1 stretches each range toward its HARSH end by (scale-1) x range-width, clamped
+    at _HARSH_LIMIT — 'drown the residual texture signal' training. scale=1 = original ranges."""
+    name = CORRUPTIONS[rng.integers(len(CORRUPTIONS))]
+    lo, hi = _TRAIN_RANGE[name]
+    if scale > 1.0:
+        stretch = (scale - 1.0) * (hi - lo)
+        if _HARSH_DIR[name] < 0:
+            lo = max(_HARSH_LIMIT[name], lo - stretch)
+        else:
+            hi = min(_HARSH_LIMIT[name], hi + stretch)
+    return name, float(rng.uniform(lo, hi))

src/data/aug/augment_impls.py

@@ -0,0 +1,146 @@
+# src/data/augment_impls.py
+"""Multi-IMPLEMENTATION corruption bank for the operator-realization holdout.
+
+We are scored on 6 corruptions but do NOT control the grader's exact operator (their
+JPEG quantization/subsampling, their resize kernel). Training a detector to be robust to
+ONE implementation risks overfitting that operator's artifact. This bank exposes several
+GENUINELY DIVERGENT implementations per family so we can train on one subset and evaluate
+on a held-out subset — measuring (and defending against) operator overfit.
+
+The divergence axes are empirically chosen: naive "PIL-jpeg vs cv2-jpeg" is a FAKE
+holdout (both wrap libjpeg → byte-identical). What actually changes the artifact is the
+PARAMETERIZATION — JPEG chroma subsampling (4:4:4 vs 4:2:0), resize KERNEL (bilinear /
+bicubic / lanczos / area), blur border handling. Those are the implementations here.
+
+cv2 is an OPTIONAL backend (may be absent on minimal images); every family has at
+least two PIL-only implementations so the holdout works with the standard library alone.
+"""
+import io
+import numpy as np
+from PIL import Image, ImageFilter, ImageEnhance
+
+try:
+    import cv2 as _cv2
+except Exception:                       # cv2 optional — PIL-only impls still give a holdout
+    _cv2 = None
+
+
+# ---- JPEG: the artifact axis is chroma subsampling + codec, not the wrapper library ----
+def _jpeg_pil(img, q, subsampling):
+    buf = io.BytesIO()
+    img.save(buf, format="JPEG", quality=int(round(q)), subsampling=subsampling)
+    return Image.open(io.BytesIO(buf.getvalue())).convert("RGB")
+
+def _jpeg_444(img, q): return _jpeg_pil(img, q, 0)     # no chroma subsampling
+def _jpeg_420(img, q): return _jpeg_pil(img, q, 2)     # 2x2 chroma subsampling (very common)
+
+def _jpeg_cv2(img, q):
+    a = _cv2.cvtColor(np.asarray(img), _cv2.COLOR_RGB2BGR)
+    ok, enc = _cv2.imencode(".jpg", a, [_cv2.IMWRITE_JPEG_QUALITY, int(round(q))])
+    dec = _cv2.imdecode(enc, _cv2.IMREAD_COLOR)
+    return Image.fromarray(_cv2.cvtColor(dec, _cv2.COLOR_BGR2RGB))
+
+
+# ---- RESIZE: down-then-up; the artifact axis is the KERNEL ----
+def _resize_pil(img, s, flt):
+    w, h = img.size
+    small = img.resize((max(1, int(w * s)), max(1, int(h * s))), flt)
+    return small.resize((w, h), flt).convert("RGB")
+
+def _resize_bilinear(img, s): return _resize_pil(img, s, Image.BILINEAR)
+def _resize_bicubic(img, s):  return _resize_pil(img, s, Image.BICUBIC)
+def _resize_lanczos(img, s):  return _resize_pil(img, s, Image.LANCZOS)
+
+def _resize_cv2_area(img, s):
+    a = np.asarray(img); h, w = a.shape[:2]
+    small = _cv2.resize(a, (max(1, int(w * s)), max(1, int(h * s))), interpolation=_cv2.INTER_AREA)
+    up = _cv2.resize(small, (w, h), interpolation=_cv2.INTER_LINEAR)
+    return Image.fromarray(up)
+
+
+# ---- BLUR: PIL Gaussian vs a box-blur approximation vs cv2 (border handling differs) ----
+def _blur_gauss(img, sigma): return img.filter(ImageFilter.GaussianBlur(float(sigma))).convert("RGB")
+def _blur_box(img, sigma):
+    # box radius chosen so variance ≈ Gaussian sigma² (uniform kernel, different tail)
+    r = max(1, int(round(float(sigma) * 1.7)))
+    return img.filter(ImageFilter.BoxBlur(r)).convert("RGB")
+def _blur_cv2(img, sigma):
+    a = _cv2.GaussianBlur(np.asarray(img), (0, 0), float(sigma))
+    return Image.fromarray(a)
+
+
+# ---- NOISE: additive gaussian in float vs uint8 domain (clipping order differs) ----
+def _noise_float(img, sigma):
+    arr = np.asarray(img).astype(np.float32) / 255.0
+    rng = np.random.default_rng(int(sigma * 1e6))
+    arr = np.clip(arr + rng.normal(0, float(sigma), arr.shape), 0, 1)
+    return Image.fromarray((arr * 255).astype("uint8"))
+def _noise_uint8(img, sigma):
+    arr = np.asarray(img).astype(np.int16)
+    rng = np.random.default_rng(int(sigma * 1e6) + 1)
+    arr = arr + rng.normal(0, float(sigma) * 255.0, arr.shape).astype(np.int16)
+    return Image.fromarray(np.clip(arr, 0, 255).astype("uint8"))
+
+
+# ---- JITTER: order of brightness/contrast/color ops differs the result ----
+def _jitter_bcs(img, s):
+    out = ImageEnhance.Brightness(img).enhance(1 + s)
+    out = ImageEnhance.Contrast(out).enhance(1 + s)
+    return ImageEnhance.Color(out).enhance(1 + s).convert("RGB")
+def _jitter_csb(img, s):
+    out = ImageEnhance.Color(img).enhance(1 + s)
+    out = ImageEnhance.Contrast(out).enhance(1 + s)
+    return ImageEnhance.Brightness(out).enhance(1 + s).convert("RGB")
+
+
+# ---- CROP: center vs corner (both keep `s` fraction; different pixels retained) ----
+def _crop_center(img, s):
+    w, h = img.size; cw, ch = int(w * s), int(h * s)
+    l, t = (w - cw) // 2, (h - ch) // 2
+    return img.crop((l, t, l + cw, t + ch)).convert("RGB")
+def _crop_corner(img, s):
+    w, h = img.size; cw, ch = int(w * s), int(h * s)
+    return img.crop((0, 0, cw, ch)).convert("RGB")
+
+
+# Registry: family -> {impl_name: fn}. cv2 impls added only when cv2 is importable.
+IMPL_BANK = {
+    "jpeg":   {"pil444": _jpeg_444, "pil420": _jpeg_420},
+    "resize": {"bilinear": _resize_bilinear, "bicubic": _resize_bicubic, "lanczos": _resize_lanczos},
+    "blur":   {"gauss": _blur_gauss, "box": _blur_box},
+    "noise":  {"float": _noise_float, "uint8": _noise_uint8},
+    "jitter": {"bcs": _jitter_bcs, "csb": _jitter_csb},
+    "crop":   {"center": _crop_center, "corner": _crop_corner},
+}
+if _cv2 is not None:
+    IMPL_BANK["jpeg"]["cv2"] = _jpeg_cv2
+    IMPL_BANK["resize"]["cv2area"] = _resize_cv2_area
+    IMPL_BANK["blur"]["cv2"] = _blur_cv2
+
+
+def list_impls(family):
+    return list(IMPL_BANK[family])
+
+
+def apply_impl(img, family, impl, severity):
+    return IMPL_BANK[family][impl](img.convert("RGB"), severity)
+
+
+def split_impls_train_test(seed=0):
+    """Partition each family's implementations into disjoint train/test sets (the
+    implementation holdout). Deterministic; every family with >=2 impls contributes at
+    least one impl to each split. The TEST impls are never seen during training, so the
+    train→test AUC gap measures operator-overfit."""
+    import random
+    rng = random.Random(seed)
+    train, test = {}, {}
+    for fam, impls in IMPL_BANK.items():
+        names = sorted(impls)
+        rng.shuffle(names)
+        if len(names) == 1:                       # can't hold out; use in both (documented)
+            train[fam] = test[fam] = list(names)
+            continue
+        n_test = max(1, len(names) // 2)
+        test[fam] = sorted(names[:n_test])
+        train[fam] = sorted(names[n_test:])
+    return train, test

src/data/aug/deshortcut.py

@@ -0,0 +1,85 @@
+# src/data/deshortcut.py
+"""Shortcut-defeating augmentations.
+
+The eval exposed low-level tells: the COCO reals are 200px JPEG thumbnails while fakes are
+large/sharp, so absolute resolution/format/sharpness separate the classes perfectly (AUC 1.0)
+without any AI-detection. These augmentations push BOTH classes, label-blindly, through the
+SAME low-level degradations so those statistics carry no label signal and the model must use
+content:
+
+- thumbnail_degrade: downsize to a small size -> JPEG re-encode -> upsize back to original.
+  This REPRODUCES exactly what a 200px COCO thumbnail underwent, applied to every image, so a
+  large sharp fake sometimes looks thumbnail-degraded too. The single most targeted fix.
+- randomize_contrast / randomize_sharpness: jitter the contrast and sharpness so those global
+  statistics (which also differ between thumbnail-reals and full-res-fakes) are not a tell.
+
+All are seeded by a caller-supplied numpy Generator (the dataset's label-blind rng) so the
+degradation is class-independent and reproducible.
+"""
+import io
+import numpy as np
+from PIL import Image, ImageEnhance
+
+
+def thumbnail_degrade(img, rng, min_side=96, max_side=384, jpeg_q=(40, 95)):
+    """Downsize -> JPEG -> upsize back to the original size, reproducing the thumbnail pipeline
+    the small COCO reals went through. Random intermediate size + JPEG quality per call."""
+    img = img.convert("RGB")
+    w, h = img.size
+    side = int(rng.integers(min_side, max_side + 1))
+    # preserve aspect: scale the SHORT side to `side`
+    if w <= h:
+        nw, nh = side, max(1, int(round(h * side / w)))
+    else:
+        nw, nh = max(1, int(round(w * side / h))), side
+    small = img.resize((nw, nh), Image.BILINEAR)
+    q = int(rng.integers(jpeg_q[0], jpeg_q[1] + 1))
+    buf = io.BytesIO(); small.save(buf, format="JPEG", quality=q)
+    small = Image.open(io.BytesIO(buf.getvalue())).convert("RGB")
+    return small.resize((w, h), Image.BILINEAR)
+
+
+def randomize_contrast(img, rng, lo=0.7, hi=1.3):
+    """Jitter contrast so global contrast is not a real/fake tell."""
+    return ImageEnhance.Contrast(img.convert("RGB")).enhance(float(rng.uniform(lo, hi)))
+
+
+def randomize_sharpness(img, rng, lo=0.4, hi=1.8):
+    """Jitter sharpness (blur<1, sharpen>1) so the sharp-fake / soft-thumbnail-real tell is
+    randomized out."""
+    return ImageEnhance.Sharpness(img.convert("RGB")).enhance(float(rng.uniform(lo, hi)))
+
+
+def randomize_codec(img, rng, jpeg_q=(45, 95)):
+    """Re-encode through a RANDOM codec so compression history carries no label signal.
+
+    The #1 measured shortcut: our reals are all JPEG, GAN/Imagen fakes all PNG, so 'JPEG⟺real' holds
+    on the real side no matter what — and DINOv2 reads JPEG quantization/blocking residuals. This
+    picks JPEG (random quality) OR lossless PNG per call, label-blindly, so a real is sometimes
+    seen as clean-PNG and a fake sometimes as JPEG-compressed. Deliberately: we do
+    NOT blanket-JPEG (that double-compresses the already-JPEG reals while single-compressing PNG
+    fakes — a NEW tell); randomizing the codec per image within BOTH classes avoids it."""
+    img = img.convert("RGB")
+    if rng.random() < 0.5:
+        q = int(rng.integers(jpeg_q[0], jpeg_q[1] + 1))
+        buf = io.BytesIO(); img.save(buf, format="JPEG", quality=q)
+        return Image.open(io.BytesIO(buf.getvalue())).convert("RGB")
+    buf = io.BytesIO(); img.save(buf, format="PNG")           # lossless re-encode
+    return Image.open(io.BytesIO(buf.getvalue())).convert("RGB")
+
+
+def apply_deshortcut(img, rng, p_thumbnail=0.5, p_contrast=0.5, p_sharpness=0.5, p_codec=0.5):
+    """Compose the shortcut-defeating augs, each applied with its probability. Label-blind:
+    the caller passes the same rng distribution for real and fake, so none of these low-level
+    statistics (resolution, sharpness, contrast, AND codec history) can encode the label.
+    Returns an RGB image of the original size."""
+    out = img.convert("RGB")
+    if rng.random() < p_thumbnail:
+        out = thumbnail_degrade(out, rng)
+    if rng.random() < p_contrast:
+        out = randomize_contrast(out, rng)
+    if rng.random() < p_sharpness:
+        out = randomize_sharpness(out, rng)
+    if rng.random() < p_codec:
+        out = randomize_codec(out, rng)
+    return out

src/data/aug/random_resolution.py

@@ -0,0 +1,61 @@
+# src/data/random_resolution.py
+"""Per-batch random resolution (user-directed).
+
+Forcing every image to a fixed 518×518 bakes the resolution difference in (thumbnails always
+upscaled, large images always downscaled) and lets the model memorize a fixed 37×37 token
+layout. Instead we resize each BATCH to a random valid resolution (a multiple of the patch
+size), so:
+  - the same content is seen at varied token-grid densities (a general robustness aug), and
+  - absolute source resolution is randomized across batches, removing an anchor for the
+    resolution shortcut.
+
+DINOv2 interpolates its position embeddings for arbitrary resolution natively, and the MIL
+head is patch-count-agnostic, so no model change is needed. Per-BATCH (not per-image) because
+a batch is one tensor and must share a size anyway — which is also the cheap, worker-safe
+implementation the user suggested ("the entire batch can resize").
+"""
+import numpy as np
+import torch
+import torch.nn.functional as F
+from torch.utils.data.dataloader import default_collate
+
+
+def valid_resolutions(patch=14, lo=392, hi=644):
+    """Resolutions that are multiples of `patch` in [lo, hi] — the sizes DINOv2 accepts."""
+    k0 = (lo + patch - 1) // patch
+    k1 = hi // patch
+    return [k * patch for k in range(k0, k1 + 1)]
+
+
+class _RandomResolutionCollate:
+    """Picklable collate callable (a plain closure isn't picklable, so DataLoader workers with
+    num_workers>0 can't use it). Stacks a batch (default_collate) then resizes the image tensors
+    to ONE random valid resolution for that batch. Non-image fields pass through unchanged.
+    lo==hi disables randomization (deterministic base size). Each worker seeds independently."""
+    def __init__(self, patch=14, lo=392, hi=644, seed=0, image_keys=("clean_img", "aug_img")):
+        self.sizes = valid_resolutions(patch, lo, hi)
+        self.seed = seed
+        self.image_keys = image_keys
+        self._rng = None                      # lazily created per-process (post-fork/pickle)
+
+    def __call__(self, samples):
+        if self._rng is None:
+            import os
+            self._rng = np.random.default_rng((self.seed, os.getpid()))
+        batch = default_collate(samples)
+        target = int(self.sizes[self._rng.integers(len(self.sizes))]) if len(self.sizes) > 1 else self.sizes[0]
+        for k in self.image_keys:
+            if k in batch and torch.is_tensor(batch[k]) and batch[k].shape[-1] != target:
+                if batch[k].dtype == torch.uint8:
+                    # gpu_preprocess path: DEFER the resize to the GPU (gpu_finish) —
+                    # interpolating a big fp32 batch here was pure worker-CPU burn.
+                    batch["target_res"] = target
+                else:
+                    batch[k] = F.interpolate(batch[k], size=(target, target),
+                                             mode="bilinear", align_corners=False)
+        return batch
+
+
+def random_resolution_collate(patch=14, lo=392, hi=644, seed=0, image_keys=("clean_img", "aug_img")):
+    """Build a picklable per-batch random-resolution collate (see _RandomResolutionCollate)."""
+    return _RandomResolutionCollate(patch, lo, hi, seed, image_keys)

src/data/aug/symmetry_audit.py

@@ -0,0 +1,53 @@
+# src/data/symmetry_audit.py
+"""Empirical class-symmetry audit.
+
+Class-INDEPENDENT augmentation sampling is necessary but not sufficient for "no shortcut":
+real and fake images can still receive different REALIZED corruptions if the sources differ
+in resolution, format, or transform failure rate. The decisive test is empirical — fit a
+cheap classifier on the augmentation METADATA alone (which corruption, severity, realized
+dims) and check its AUC at predicting the label. If aug metadata predicts the label, the
+detector could learn "was-corrupted-this-way" instead of "is-synthetic" — the shortcut that
+"Fake or JPEG?" exposed. A passing audit shows AUC ≈ 0.5.
+"""
+import numpy as np
+import pandas as pd
+
+
+def aug_metadata_leakage_auc(df, seed=0):
+    """AUC of a logistic model predicting `label` from augmentation metadata only
+    (aug_id one-hot + severity + any realized-dim columns present). ~0.5 = no leakage."""
+    from sklearn.linear_model import LogisticRegression
+    from sklearn.model_selection import cross_val_predict
+    from sklearn.metrics import roc_auc_score
+
+    y = df["label"].to_numpy().astype(int)
+    if len(np.unique(y)) < 2:
+        return 0.5
+    feats = []
+    if "aug_id" in df:
+        feats.append(pd.get_dummies(df["aug_id"], prefix="aug"))
+    for col in ("severity", "out_w", "out_h", "out_bytes"):
+        if col in df:
+            feats.append(df[[col]].astype(float))
+    if not feats:
+        return 0.5
+    X = pd.concat(feats, axis=1).to_numpy(dtype=float)
+    # cross-validated out-of-fold probabilities -> honest AUC (no train-on-test optimism)
+    clf = LogisticRegression(max_iter=500)
+    try:
+        proba = cross_val_predict(clf, X, y, cv=5, method="predict_proba")[:, 1]
+    except Exception:
+        clf.fit(X, y); proba = clf.predict_proba(X)[:, 1]
+    return float(roc_auc_score(y, proba))
+
+
+def audit_class_symmetry(df, threshold=0.6, seed=0):
+    """Report whether augmentation metadata leaks the label. passed = AUC <= threshold."""
+    auc = aug_metadata_leakage_auc(df, seed=seed)
+    return {
+        "leakage_auc": auc,
+        "passed": auc <= threshold,
+        "threshold": threshold,
+        "note": ("aug metadata should NOT predict the label; AUC>threshold means real/fake "
+                 "receive systematically different corruptions -> a 'was-corrupted' shortcut."),
+    }

src/data/aug/warp.py

@@ -0,0 +1,120 @@
+# src/data/warp.py
+"""Anatomical-plausibility augmentation: local discontinuous warps.
+
+Motivation (run-4..8 + XAI): every reader of DINOv2-B features hits the same GigaGAN
+ceiling — texture separates train classes so cheaply that anatomy/arrangement is never
+learned. A real photo and its LOCALLY-warped twin form a minimal pair: identical camera,
+JPEG history, and texture statistics, differing ONLY in local geometric plausibility. With
+the warped twin labeled FAKE, texture is uninformative on the pair by construction and the
+gradient's only descent direction is the geometry axis (the six-fingers class of wrongness).
+
+Two warp families with OPPOSITE roles:
+  local_deform  -> label-flipping (real -> warped twin labeled fake): 1-3 localized mesh
+                   displacements — a region's geometry disagrees with its surroundings.
+  global_warp   -> class-SYMMETRIC aug (both classes, label-blind): mild affine/perspective,
+                   breaks no anatomy. Its role is to put warp-resampling smear on BOTH
+                   classes so "was resampled" cannot become the shortcut for "fake".
+
+Both re-quantize through JPEG so interpolation smoothness is not the tell either.
+"""
+import io
+
+import numpy as np
+from PIL import Image
+
+
+def _jpeg(img: Image.Image, quality: int) -> Image.Image:
+    buf = io.BytesIO()
+    img.save(buf, "JPEG", quality=int(quality))
+    buf.seek(0)
+    return Image.open(buf).convert("RGB")
+
+
+def _remap_mesh(arr: np.ndarray, dx: np.ndarray, dy: np.ndarray) -> np.ndarray:
+    """Bilinear remap of HxWx3 uint8 by per-pixel displacement fields (float32 HxW)."""
+    h, w = arr.shape[:2]
+    yy, xx = np.meshgrid(np.arange(h, dtype=np.float32), np.arange(w, dtype=np.float32),
+                         indexing="ij")
+    sx = np.clip(xx + dx, 0, w - 1)
+    sy = np.clip(yy + dy, 0, h - 1)
+    x0 = np.floor(sx).astype(np.int32); y0 = np.floor(sy).astype(np.int32)
+    x1 = np.minimum(x0 + 1, w - 1);     y1 = np.minimum(y0 + 1, h - 1)
+    fx = (sx - x0)[..., None]; fy = (sy - y0)[..., None]
+    a = arr[y0, x0].astype(np.float32); b = arr[y0, x1].astype(np.float32)
+    c = arr[y1, x0].astype(np.float32); d = arr[y1, x1].astype(np.float32)
+    out = (a * (1 - fx) + b * fx) * (1 - fy) + (c * (1 - fx) + d * fx) * fy
+    return np.clip(out, 0, 255).astype(np.uint8)
+
+
+def local_deform(img: Image.Image, rng: np.random.Generator,
+                 strength: float = 1.0, jpeg_q: (int, int) = (70, 92),
+                 max_side: int = 640) -> Image.Image:
+    """1-3 localized smooth-bump displacements — geometry locally inconsistent with the
+    surround, texture untouched. strength scales displacement amplitude.
+
+    CPU note: the numpy remap is O(H*W); native-res warping of a 1500px image is ~6x the
+    work of 640px for zero semantic gain (deshortcut crushes to ~200px downstream anyway).
+    Warp at <=max_side and restore the ORIGINAL size afterward, so the twin's resolution
+    statistics stay identical to its real original (no size leak)."""
+    orig_size = img.size
+    img = img.convert("RGB")
+    if max(img.size) > max_side:
+        s = max_side / max(img.size)
+        img = img.resize((max(1, int(img.size[0] * s)), max(1, int(img.size[1] * s))),
+                         Image.BILINEAR)
+    arr = np.asarray(img)
+    h, w = arr.shape[:2]
+    dx = np.zeros((h, w), np.float32); dy = np.zeros((h, w), np.float32)
+    n = int(rng.integers(1, 4))
+    for _ in range(n):
+        # bump center anywhere; radius 8-25% of min side; amplitude 4-12% of radius x strength
+        cy, cx = rng.uniform(0.15, 0.85) * h, rng.uniform(0.15, 0.85) * w
+        rad = rng.uniform(0.08, 0.25) * min(h, w)
+        amp = rng.uniform(0.15, 0.45) * rad * strength
+        ang = rng.uniform(0, 2 * np.pi)
+        yy, xx = np.meshgrid(np.arange(h, dtype=np.float32), np.arange(w, dtype=np.float32),
+                             indexing="ij")
+        d2 = ((yy - cy) ** 2 + (xx - cx) ** 2) / (rad ** 2)
+        bump = np.exp(-d2 * 2.0).astype(np.float32)          # smooth, compact support
+        dx += np.cos(ang) * amp * bump
+        dy += np.sin(ang) * amp * bump
+    out_img = Image.fromarray(_remap_mesh(arr, dx, dy))
+    if out_img.size != orig_size:
+        out_img = out_img.resize(orig_size, Image.BILINEAR)   # restore native size (no leak)
+    q = int(rng.integers(jpeg_q[0], jpeg_q[1] + 1))
+    return _jpeg(out_img, q)
+
+
+def global_warp(img: Image.Image, rng: np.random.Generator,
+                jpeg_q: (int, int) = (70, 92)) -> Image.Image:
+    """Mild global perspective/affine jitter (anatomy-preserving), then JPEG. Class-symmetric:
+    exists so BOTH classes carry warp-resampling residue."""
+    w, h = img.size
+    j = 0.04                                                  # corner jitter, fraction of side
+    src = [(0, 0), (w, 0), (w, h), (0, h)]
+    dst = [(x + rng.uniform(-j, j) * w, y + rng.uniform(-j, j) * h) for x, y in src]
+    # solve the 8-dof perspective transform mapping dst -> src (PIL wants inverse coeffs)
+    A = []
+    for (x, y), (u, v) in zip(dst, src):
+        A.append([x, y, 1, 0, 0, 0, -u * x, -u * y])
+        A.append([0, 0, 0, x, y, 1, -v * x, -v * y])
+    B = np.array([c for (u, v) in src for c in (u, v)], dtype=np.float64)
+    coeffs = np.linalg.solve(np.array(A, dtype=np.float64), B)
+    out = img.transform((w, h), Image.PERSPECTIVE, tuple(coeffs), resample=Image.BILINEAR)
+    q = int(rng.integers(jpeg_q[0], jpeg_q[1] + 1))
+    return _jpeg(out, q)
+
+
+def add_warp_fakes(train_df, frac: float, seed: int = 0):
+    """Duplicate a `frac` sample of REAL train rows as warped fakes: label=1, warp_fake=1,
+    generator_id='warp_real'. The original real rows stay — the minimal pair is explicit
+    (same content under both labels; only the warp separates them)."""
+    reals = train_df[train_df.label == 0]
+    dup = reals.sample(int(len(reals) * frac), random_state=seed).copy()
+    dup["label"] = 1
+    dup["warp_fake"] = 1
+    dup["generator_id"] = "warp_real"
+    out = train_df.copy()
+    out["warp_fake"] = 0
+    import pandas as pd
+    return pd.concat([out, dup], ignore_index=True)

src/data/dataset.py

@@ -0,0 +1,108 @@
+# src/data/dataset.py
+import numpy as np
+from pandas import notna as pd_notna
+from PIL import Image
+from torch.utils.data import Dataset
+from data.aug.augment import apply_corruption, sample_train_corruption
+from data.preprocess import preprocess_image
+
+class AigcDataset(Dataset):
+    def __init__(self, df, cfg, train=True, resolver=None):
+        self.df = df.reset_index(drop=True)
+        self.cfg = cfg
+        self.train = train
+        self.res = cfg.resolution
+        self.resolver = resolver
+        # Epoch folded into the aug seed so each image gets a DIFFERENT corruption realization
+        # every epoch (a seed of only (cfg.seed, img_id) froze the aug across epochs,
+        # killing augmentation diversity). Epoch is class-INDEPENDENT, so label-blindness holds.
+        # The driver calls set_epoch(e) before each epoch's loop.
+        self.epoch = 0
+
+    def set_epoch(self, epoch):
+        self.epoch = int(epoch)
+
+    def __len__(self):
+        return len(self.df)
+
+    def _prep(self, img):
+        if getattr(self.cfg, "gpu_preprocess", False):
+            from data.preprocess import preprocess_image_uint8
+            return preprocess_image_uint8(img, self.res)   # float+normalize done on GPU
+        return preprocess_image(img, self.res)
+
+    def __getitem__(self, i):
+        row = self.df.iloc[i]
+        if self.resolver is not None and "ref_type" in row:
+            img = self.resolver.resolve(row)
+        else:
+            with Image.open(row["path"]) as im:
+                img = im.convert("RGB")
+        # LABEL-BLIND seed: derive the aug RNG from the IMMUTABLE img_id, not the row index
+        # i (which correlates with class when the manifest is label-sorted). This guarantees
+        # the realized corruption cannot carry label signal via seeding.
+        _idkey = row.get("img_id") if "img_id" in row and pd_notna(row.get("img_id")) else \
+            row.get("rel_path", row.get("path", i))
+        # epoch is part of the seed (train only) so the realized corruption VARIES across epochs
+        # while staying label-blind (epoch does not depend on the label).
+        _epoch = self.epoch if self.train else 0
+        seed_key = abs(hash((self.cfg.seed, _epoch, str(_idkey)))) % (2**32)
+        rng = np.random.default_rng(seed_key)
+        # DEFEAT THE RESOLUTION/FORMAT/SHARPNESS SHORTCUT (label-blind, both classes): the eval
+        # showed reals (200px JPEG thumbnails) vs fakes (large/sharp) separate by low-level
+        # statistics ALONE (resolution-only AUC 1.0). apply_deshortcut pushes every image through
+        # the SAME degradations — thumbnail(downsize->JPEG->upsize, reproducing what the COCO
+        # reals underwent) + contrast + sharpness jitter — so those stats carry no label signal.
+        # Applied BEFORE clean/aug so both views inherit it. (defeat_resolution kept as the
+        # lighter resolution-only variant for ablation.)
+        # WARP-FAKE rows (label-flipped duplicates of reals, add_warp_fakes): apply the local
+        # deform FIRST so the twin then rides the identical deshortcut+corruption pipeline as
+        # its real original — the minimal pair differs ONLY in local geometry. Seed offset by a
+        # constant so the twin's downstream aug draw still matches nothing label-correlated.
+        if row.get("warp_fake", 0) == 1:
+            from data.aug.warp import local_deform
+            img = local_deform(img, np.random.default_rng((seed_key + 777) % (2**32)),
+                               strength=getattr(self.cfg, "warp_strength", 1.0))
+        # class-SYMMETRIC global warp (both classes, label-blind): puts warp-resampling residue
+        # on reals and fakes alike so "was resampled" cannot proxy for "warp_fake".
+        if self.train and rng.random() < getattr(self.cfg, "p_global_warp", 0.0):
+            from data.aug.warp import global_warp
+            img = global_warp(img, rng)
+        if self.train and getattr(self.cfg, "deshortcut", False):
+            from data.aug.deshortcut import apply_deshortcut
+            img = apply_deshortcut(img, rng, p_codec=getattr(self.cfg, "p_codec", 0.5))
+        elif self.train and getattr(self.cfg, "defeat_resolution", False):
+            from data.preprocess import defeat_resolution_shortcut
+            img = defeat_resolution_shortcut(img, self.res, rng)
+        name, sev = sample_train_corruption(
+            rng, scale=getattr(self.cfg, "corruption_scale", 1.0) if self.train else 1.0)
+        # CLASS-SYMMETRIC by construction: corruption is sampled from a class-INDEPENDENT
+        # distribution and applied to every image, so "was-corrupted" carries no label
+        # signal (prevents the JPEG/resize dataset-shortcut that "Fake or JPEG?" exposed).
+        # When train_impls is set, draw from the TRAINING implementation subset only, so the
+        # held-out implementations stay genuinely unseen for the invariance measurement.
+        train_impls = getattr(self.cfg, "train_impls", None)
+        if train_impls:
+            from data.aug.augment_impls import apply_impl
+            allowed = train_impls.get(name)
+            impl = allowed[int(rng.integers(len(allowed)))] if allowed else None
+            aug = apply_impl(img, name, impl, sev) if impl else apply_corruption(img, name, sev)
+        else:
+            aug = apply_corruption(img, name, sev)
+        # img_id: the curator manifest identifies images by rel_path, not img_id — fall back
+        # to rel_path (then path, then row index) so the dataset reads any manifest schema.
+        img_id = row.get("img_id") if "img_id" in row and pd_notna(row.get("img_id")) else \
+            row.get("rel_path", row.get("path", i))
+        out = {
+            "clean_img": self._prep(img),
+            "aug_img": self._prep(aug),
+            "aug_id": name,
+            "severity": float(sev),
+            "label": float(row["label"]),
+            "source": row["source_dataset"],
+            "generator_id": row["generator_id"],
+            "img_id": str(img_id),
+        }
+        if "teacher_prob" in row and row["teacher_prob"] == row["teacher_prob"]:   # not NaN
+            out["teacher_prob"] = float(row["teacher_prob"])
+        return out

src/data/preprocess.py

@@ -0,0 +1,76 @@
+# src/data/preprocess.py
+import io
+from PIL import Image
+from torchvision import transforms
+
+_MEAN = [0.485, 0.456, 0.406]
+_STD = [0.229, 0.224, 0.225]
+_NORM = transforms.Normalize(mean=_MEAN, std=_STD)
+_TO = transforms.ToTensor()
+
+
+def preprocess_image_uint8(img, resolution):
+    """CPU-cheap half of preprocessing: resize + HWC->CHW uint8 tensor. The float
+    conversion + normalization (the expensive per-pixel CPU work) moves to the GPU
+    via gpu_finish — workers then ship 4x fewer bytes and burn ~half the CPU."""
+    import numpy as np
+    import torch
+    img = img.convert("RGB").resize((resolution, resolution), Image.BILINEAR)
+    arr = np.asarray(img, dtype=np.uint8)             # HWC
+    return torch.from_numpy(arr).permute(2, 0, 1).contiguous()
+
+
+def gpu_finish(x, target_res=None):
+    """Finish preprocessing ON DEVICE for a uint8 [B,C,H,W] batch: float/255,
+    ImageNet-normalize, optional resize to the batch's random resolution. Exactly
+    equivalent to ToTensor+Normalize (ToTensor is uint8/255) + collate interpolate,
+    minus the worker CPU. No-op passthrough for float inputs (already normalized)."""
+    import torch
+    import torch.nn.functional as F
+    if x.dtype == torch.uint8:
+        x = x.float().div_(255.0)
+        mean = torch.tensor(_MEAN, device=x.device).view(1, 3, 1, 1)
+        std = torch.tensor(_STD, device=x.device).view(1, 3, 1, 1)
+        x = x.sub_(mean).div_(std)
+    if target_res is not None and x.shape[-1] != int(target_res):
+        x = F.interpolate(x, size=(int(target_res), int(target_res)),
+                          mode="bilinear", align_corners=False)
+    return x
+
+def preprocess_image(img, resolution):
+    img = img.convert("RGB").resize((resolution, resolution), Image.BILINEAR)
+    return _NORM(_TO(img))
+
+
+def defeat_resolution_shortcut(img, resolution, rng):
+    """Randomize the resolution PATH so absolute source resolution carries no label signal.
+
+    The eval exposed a catastrophic shortcut: reals were all 200x200 thumbnails and fakes were
+    large, so a plain resize-to-518 leaves reals upscale-blurred and fakes downscale-sharp — a
+    'was-this-upscaled' tell (resolution-only AUC = 1.0). Here every image is first resized to a
+    RANDOM intermediate size (thumbnailing large images, and sometimes down-then-up), then to the
+    model resolution, with a random interpolation. Applied to BOTH classes, this randomizes out
+    the upscale/downscale signature so the model must use content, not resolution."""
+    img = img.convert("RGB")
+    # random intermediate short-side in [96, 1.4x resolution]; random interp both ways
+    inter = int(rng.integers(96, int(resolution * 1.4)))
+    filt_down = rng.choice([Image.BILINEAR, Image.BICUBIC, Image.LANCZOS, Image.NEAREST])
+    filt_up = rng.choice([Image.BILINEAR, Image.BICUBIC, Image.NEAREST])
+    small = img.resize((inter, inter), filt_down)
+    return small.resize((resolution, resolution), filt_up)
+
+
+def fair_eval_normalize(img, resolution, jpeg_quality=90, common_small=200):
+    """Fair-eval normalizer: equalize real and fake so absolute resolution/format is not the
+    discriminator. Push every image DOWN to a small common size (`common_small`, matching the
+    holdout reals' 200px thumbnails) + JPEG re-encode, THEN up to the model resolution.
+
+    Downsizing to the common-small size is load-bearing: an earlier version that resized straight
+    to `resolution` (518) UPSCALED the small reals and AMPLIFIED the sharpness gap (a small real
+    upscaled gets even smoother; a large fake downscaled stays sharp) — sharpness-oracle AUC rose
+    0.63→0.77. Matching everything DOWN to the real's information level destroys the fake's detail
+    advantage instead (oracle AUC → ~0.4), so any residual AUC is content, not resolution."""
+    img = img.convert("RGB").resize((common_small, common_small), Image.BILINEAR)
+    buf = io.BytesIO(); img.save(buf, format="JPEG", quality=jpeg_quality)   # common format
+    img = Image.open(io.BytesIO(buf.getvalue())).convert("RGB")
+    return img.resize((resolution, resolution), Image.BILINEAR)              # up to model res

src/data/provenance/audit.py

@@ -0,0 +1,17 @@
+# src/curator/audit.py
+from data.provenance.dda_audit import audit_class_separability
+
+def audit_corpus(df) -> dict:
+    dda = audit_class_separability(df)
+    per_family = df.groupby("family").size().to_dict()
+    n_real = int((df["label"] == 0).sum())
+    n_fake = int((df["label"] == 1).sum())
+    if dda["flag"]:
+        report = (f"SHORTCUT FLAGGED: resolution_auc={dda['resolution_auc']:.3f} "
+                  f"compression_auc={dda['compression_auc']:.3f} (>0.65). "
+                  f"Mitigate: resolution/JPEG-normalize BOTH classes or rebalance sources, then re-audit.")
+    else:
+        report = (f"clean: resolution_auc={dda['resolution_auc']:.3f} "
+                  f"compression_auc={dda['compression_auc']:.3f} (both <=0.65).")
+    return {**dda, "per_family_counts": per_family,
+            "class_balance": {"n_real": n_real, "n_fake": n_fake}, "report": report}

src/data/provenance/cleanup.py

@@ -0,0 +1,60 @@
+# src/shards/cleanup.py
+"""Marker-guarded cleanup. Every recursive delete must target a directory that the
+pipeline itself created (contains .track5_run_root) — filesystem roots, homes, the
+workspace, and anything unmarked are refused. All operations report item count and
+bytes reclaimed so GC is observable, not best-effort."""
+import os
+import shutil
+import tempfile
+
+MARKER = ".track5_run_root"
+
+
+def make_run_root(parent, prefix="run_"):
+    root = tempfile.mkdtemp(prefix=prefix, dir=parent)
+    open(os.path.join(root, MARKER), "w").close()
+    return root
+
+
+def _tree_stats(target):
+    n, b = 0, 0
+    for dp, _dirs, files in os.walk(target):
+        for f in files:
+            n += 1
+            try:
+                b += os.path.getsize(os.path.join(dp, f))
+            except OSError:
+                pass
+    return n, b
+
+
+def safe_rmtree(target):
+    resolved = os.path.realpath(target)
+    home = os.path.realpath(os.path.expanduser("~"))
+    cwd = os.path.realpath(os.getcwd())
+    if (not resolved or resolved == os.path.sep or os.path.dirname(resolved) == resolved
+            or resolved in (home, cwd)):
+        raise ValueError(f"refusing to delete unsafe target: {resolved}")
+    if not os.path.isfile(os.path.join(resolved, MARKER)):
+        raise ValueError(f"refusing: {resolved} has no {MARKER} marker")
+    n, b = _tree_stats(resolved)
+    shutil.rmtree(resolved)
+    return n, b
+
+
+def reclaim_unreferenced(raw_root, referenced_relpaths):
+    """Delete every file under raw_root whose posix relpath is NOT in
+    referenced_relpaths; prune empty dirs. Returns (n_deleted, bytes_reclaimed)."""
+    referenced = set(referenced_relpaths)
+    n, b = 0, 0
+    for dp, _dirs, files in os.walk(raw_root, topdown=False):
+        for f in files:
+            p = os.path.join(dp, f)
+            rel = os.path.relpath(p, raw_root).replace(os.sep, "/")
+            if rel not in referenced:
+                b += os.path.getsize(p)
+                os.remove(p)
+                n += 1
+        if dp != raw_root and not os.listdir(dp):
+            os.rmdir(dp)
+    return n, b

src/data/provenance/dda_audit.py

@@ -0,0 +1,16 @@
+# src/manifest/dda_audit.py
+import numpy as np
+from eval.metrics import roc_auc
+
+def audit_class_separability(df) -> dict:
+    labels = df["label"].to_numpy()
+    res = (df["native_w"] * df["native_h"]).to_numpy().astype(float)
+    comp = df["jpeg_bytes_per_pixel"].to_numpy().astype(float)
+    # AUC of a single scalar predicting label; flip so AUC>=0.5
+    def _one_sided(x):
+        a = roc_auc(labels, x)
+        return max(a, 1 - a) if not np.isnan(a) else float("nan")
+    res_auc = _one_sided(res)
+    comp_auc = _one_sided(comp)
+    flag = bool((res_auc > 0.65) or (comp_auc > 0.65))
+    return {"resolution_auc": res_auc, "compression_auc": comp_auc, "flag": flag}

src/data/provenance/dedup_audit.py

@@ -0,0 +1,184 @@
+# src/curator/dedup_audit.py
+"""Near-duplicate audit + dedup for the corpus manifest. OmniFake (the aggregator)
+is known to partially CONTAIN WildFake+GenImage images, so sources overlap; this
+module finds near-identical images across sources via a DCT pHash and collapses
+each duplicate cluster down to one row, per a source-priority policy."""
+import numpy as np
+import pandas as pd
+from PIL import Image
+
+from data.provenance.manifest_build import MANIFEST_COLUMNS
+
+DEFAULT_PRIORITY = ["wildfake_valset", "wildfake", "sid_set", "commforensics",
+                     "deepface", "omnifake", "cifake"]
+
+def _dct_matrix(n: int) -> np.ndarray:
+    """(n,n) DCT-II basis: D[k,n] = cos(pi*(2n+1)*k/(2N)). dct2(img) = D @ img @ D.T."""
+    k = np.arange(n).reshape(-1, 1)
+    x = np.arange(n).reshape(1, -1)
+    return np.cos(np.pi * (2 * x + 1) * k / (2 * n))
+
+def phash(image_path, hash_size=8) -> int:
+    """DCT-based pHash, numpy-only (no imagehash/scipy dependency). Deterministic:
+    same file -> same int every call. Robust to JPEG re-encode (low-freq DCT
+    coefficients barely move under quantization noise)."""
+    n = hash_size * 4
+    with Image.open(image_path) as im:
+        img = np.asarray(im.convert("L").resize((n, n), Image.Resampling.LANCZOS), dtype=np.float64)
+    d = _dct_matrix(n)
+    dct2 = d @ img @ d.T
+    block = dct2[:hash_size, :hash_size]
+    # Exclude the [0,0] DC term from the median (it's just average brightness,
+    # not structure) so the threshold reflects the actual frequency content.
+    flat = block.flatten()
+    median = np.median(np.delete(flat, 0))
+    bits = (flat > median)
+    h = 0
+    for b in bits:
+        h = (h << 1) | int(b)
+    return h
+
+def hamming(a: int, b: int) -> int:
+    return (a ^ b).bit_count()
+
+_POP_C1 = np.uint64(0x5555555555555555)
+_POP_C2 = np.uint64(0x3333333333333333)
+_POP_C4 = np.uint64(0x0f0f0f0f0f0f0f0f)
+_POP_H01 = np.uint64(0x0101010101010101)
+
+def _popcount64(x):
+    """Vectorized 64-bit popcount over a uint64 numpy array (SWAR / bit-twiddle). ~50x faster
+    than a Python-per-element loop, so the Hamming all-pairs scan is dominated by SIMD, not
+    interpreter overhead."""
+    x = x - ((x >> np.uint64(1)) & _POP_C1)
+    x = (x & _POP_C2) + ((x >> np.uint64(2)) & _POP_C2)
+    x = (x + (x >> np.uint64(4))) & _POP_C4
+    return (x * _POP_H01) >> np.uint64(56)
+
+def _within_threshold_neighbors(hashes_u64, threshold):
+    """Yield (i, j>i) index pairs with Hamming(phash) <= threshold, computed row-at-a-time
+    with a VECTORIZED popcount (bounded memory O(n), one XOR-broadcast per row). This is the
+    O(n^2)-comparisons scan but with a tiny constant factor — measured faster than LSH banding
+    on mostly-unique real-image hashes (banding degrades to millions of chance-collision
+    candidates when bands are narrow). Exact: no false negatives, no tuning."""
+    n = len(hashes_u64)
+    for i in range(n):
+        if i + 1 >= n:
+            break
+        d = _popcount64(hashes_u64[i] ^ hashes_u64[i + 1:])
+        for off in np.nonzero(d <= threshold)[0]:
+            yield i, i + 1 + int(off)
+
+
+def _phash_many(paths, workers=8):
+    """Compute phash for a list of paths, in PARALLEL over `workers` threads, preserving
+    input order. pHash is decode(disk-I/O) + DCT(numpy, releases the GIL), so threads overlap
+    the I/O and the released-GIL compute — measured ~20-27x over sequential even on 2 cores.
+    workers<=1 runs sequentially."""
+    if workers is None or workers <= 1 or len(paths) < 32:
+        return [phash(p) for p in paths]
+    from concurrent.futures import ThreadPoolExecutor
+    with ThreadPoolExecutor(max_workers=workers) as ex:
+        return list(ex.map(phash, paths))            # ex.map preserves order
+
+
+def add_phash(df, workers=8):
+    """Copy of df with a uint64 `phash` column (64-bit DCT pHash of each row's file)."""
+    out = df.copy()
+    hashes = _phash_many(out["path"].tolist(), workers=workers)
+    out["phash"] = np.asarray([int(h) & ((1 << 64) - 1) for h in hashes], dtype=np.uint64)
+    return out
+
+
+def cross_leak_mask(cand_hashes, hold_hashes, threshold=6):
+    """Boolean mask over cand_hashes: True where Hamming distance to ANY holdout
+    hash is <= threshold. Vectorized row-at-a-time over the (smaller) holdout side."""
+    cand = np.asarray(cand_hashes, dtype=np.uint64)
+    hold = np.asarray(hold_hashes, dtype=np.uint64)
+    mask = np.zeros(len(cand), dtype=bool)
+    if len(cand) == 0 or len(hold) == 0:
+        return mask
+    for h in hold:
+        d = _popcount64(cand ^ h)
+        mask |= d <= threshold
+    return mask
+
+
+def dedup_manifest(df, threshold=6, priority=None, phash_workers=8):
+    """Collapse near-duplicate rows (hamming(phash) <= threshold) to one survivor
+    per cluster, keeping the row whose source ranks best in `priority`.
+
+    Complexity: O(n^2) comparisons but VECTORIZED (numpy SWAR popcount, row-at-a-time), so the
+    constant factor is ~50x smaller than a Python-per-pair loop — 30k rows in ~10s vs minutes,
+    100k in ~2min. (LSH banding was tried and REJECTED: on mostly-unique real-image hashes,
+    narrow bands collide by chance and generate millions of spurious candidates, making it
+    SLOWER than the vectorized scan while adding tuning + false-negative risk.) Output is
+    identical to an exhaustive scan.
+    """
+    priority = list(priority) if priority is not None else DEFAULT_PRIORITY
+    rank = {src: i for i, src in enumerate(priority)}
+    n_sources = len(priority)
+
+    work = df.copy()
+    work["_phash"] = _phash_many(work["path"].tolist(), workers=phash_workers)
+    work["_rank"] = work["source_dataset"].map(lambda s: rank.get(s, n_sources))
+
+    n = len(work)
+    hashes = work["_phash"].to_numpy()
+    hashes_u64 = np.asarray([int(h) & ((1 << 64) - 1) for h in hashes], dtype=np.uint64)
+    ranks = work["_rank"].to_numpy()
+    dropped = np.zeros(n, dtype=bool)
+
+    # Gather within-threshold neighbours per row (vectorized), then the SAME union-find-free
+    # clustering: for each surviving row i, its within-threshold matches form a cluster; keep
+    # the best-priority survivor, drop the rest. Ties broken by stable first-seen (ascending i).
+    from collections import defaultdict
+    neighbors = defaultdict(list)
+    for i, j in _within_threshold_neighbors(hashes_u64, threshold):
+        neighbors[i].append(j)
+    for i in range(n):
+        if dropped[i]:
+            continue
+        cluster = [i]
+        for j in neighbors.get(i, ()):
+            if not dropped[j]:
+                cluster.append(j)
+        if len(cluster) == 1:
+            continue
+        best = min(cluster, key=lambda idx: (ranks[idx], idx))
+        for idx in cluster:
+            if idx != best:
+                dropped[idx] = True
+
+    kept_mask = ~dropped
+    cols = [c for c in MANIFEST_COLUMNS if c in df.columns]
+    deduped_df = df.loc[kept_mask, cols].reset_index(drop=True)
+    # Persist the survivor's OWN pHash so downstream cross-artifact leak subtraction
+    # (candidate vs official holdout) is manifest-only — no re-reading 45k images.
+    deduped_df["phash"] = hashes_u64[kept_mask]
+
+    # --- provenance table ---------------------------------------------------
+    sources_present = list(dict.fromkeys(df["source_dataset"]))  # first-seen order
+    sampled = df["source_dataset"].value_counts()
+    after = df.loc[kept_mask, "source_dataset"].value_counts()
+
+    rows = []
+    for src in sources_present:
+        s = int(sampled.get(src, 0))
+        a = int(after.get(src, 0))
+        rows.append({"source_dataset": src, "total_available": pd.NA,
+                     "sampled": s, "after_dedup": a, "dropped_as_dupe": s - a})
+
+    # Sort by priority order (unlisted sources after all listed, in first-seen order).
+    rows.sort(key=lambda r: rank.get(r["source_dataset"], n_sources))
+
+    total = {"source_dataset": "TOTAL", "total_available": pd.NA,
+             "sampled": sum(r["sampled"] for r in rows),
+             "after_dedup": sum(r["after_dedup"] for r in rows),
+             "dropped_as_dupe": sum(r["dropped_as_dupe"] for r in rows)}
+    rows.append(total)
+
+    provenance_df = pd.DataFrame(
+        rows, columns=["source_dataset", "total_available", "sampled", "after_dedup", "dropped_as_dupe"])
+
+    return deduped_df, provenance_df

src/data/provenance/diskcheck.py

@@ -0,0 +1,38 @@
+# src/shards/diskcheck.py
+"""Multi-filesystem disk accounting. The failed corpus run's telemetry watched ONE
+filesystem while the client archived on another; fs_stats observes every filesystem
+that holds run data (run root, system temp) and assert_headroom
+stops BEFORE an operation that cannot fit its peak + emergency reserve."""
+import os
+import shutil
+import tempfile
+
+RESERVE_BYTES = 5 * 2**30
+
+
+def fs_stats(paths):
+    out = {}
+    for p in paths:
+        p = os.path.realpath(p)
+        if not os.path.exists(p):
+            continue
+        dev = os.stat(p).st_dev
+        key = str(dev)
+        if key not in out:
+            du = shutil.disk_usage(p)
+            out[key] = {"paths": [], "used": du.used, "free": du.free, "total": du.total}
+        out[key]["paths"].append(p)
+    return out
+
+
+def default_watch_paths(run_root):
+    paths = [run_root, tempfile.gettempdir()]
+    return paths
+
+
+def assert_headroom(path, needed_bytes, reserve_bytes=RESERVE_BYTES):
+    free = shutil.disk_usage(os.path.realpath(path)).free
+    if free < needed_bytes + reserve_bytes:
+        raise RuntimeError(
+            f"disk headroom: need {needed_bytes + reserve_bytes} bytes on {path}, "
+            f"free {free} — stopping before the operation (state preserved)")

src/data/provenance/families.py

@@ -0,0 +1,32 @@
+# src/curator/families.py
+import re
+
+FAMILIES = [
+    "sd1x", "sd2x", "sdxl", "sd3", "flux", "dalle", "midjourney",
+    "kandinsky", "gan", "other_diffusion", "commercial_other", "unknown",
+]
+
+# Ordered rules: first match wins. (name_lower) -> family.
+# Order matters: sdxl/sd3 before the generic sd1x/sd2x check; specific before generic.
+_RULES = [
+    (r"stable-diffusion-xl|sdxl|sd-xl", "sdxl"),
+    (r"stable-diffusion-3|sd-?3|sd3", "sd3"),
+    (r"stable-diffusion-2|sd-?2|sd2", "sd2x"),
+    (r"stable-diffusion-v?1|sd-?1|sd1|stable-diffusion-1", "sd1x"),
+    (r"flux", "flux"),
+    (r"dall-?e|dalle", "dalle"),
+    (r"midjourney|mj[_-]?v?\d", "midjourney"),
+    (r"kandinsky", "kandinsky"),
+    (r"stylegan|prousgan|progan|biggan|cyclegan|stargan|gaugan|gansformer|projectedgan|\bgan\b", "gan"),
+    (r"\badm\b|glide|vqdm|wukong|\bldm\b|ddpm|ddim|denoising.?diffusion", "other_diffusion"),
+    (r"firefly|ideogram|\bimagen\b", "commercial_other"),   # imagen = Google's commercial model
+]
+
+def classify_family(model_name: str) -> str:
+    if not model_name:
+        return "unknown"
+    n = model_name.lower()
+    for pat, fam in _RULES:
+        if re.search(pat, n):
+            return fam
+    return "unknown"

src/data/provenance/feature_cache.py

@@ -0,0 +1,36 @@
+# src/curator/feature_cache.py
+import hashlib, os
+
+def cache_path(img_id, backbone_id, resolution, cache_dir) -> str:
+    key = f"{img_id}|{backbone_id}|{resolution}"
+    h = hashlib.sha1(key.encode()).hexdigest()[:16]
+    return os.path.join(cache_dir, f"{h}.pt")
+
+def shard_indices(df, shard_idx, n_shards):
+    return list(range(shard_idx, len(df), n_shards))
+
+def extract_shard(manifest_df, shard_idx, n_shards, cfg, cache_dir):
+    """GPU-host real path: load backbone, forward each shard image, save features.
+    Local: exercised only via the pure helpers above (this body needs a backbone)."""
+    import torch
+    from model.backbone import build_backbone
+    from data.preprocess import preprocess_image
+    from PIL import Image
+    os.makedirs(cache_dir, exist_ok=True)
+    bb = build_backbone(cfg)
+    if torch.cuda.is_available():
+        bb = bb.cuda()
+    idxs = shard_indices(manifest_df, shard_idx, n_shards)
+    for i in idxs:
+        row = manifest_df.iloc[i]
+        cp = cache_path(row["img_id"] if "img_id" in row else row["path"],
+                        cfg.backbone, cfg.resolution, cache_dir)
+        if os.path.exists(cp):
+            continue
+        with Image.open(row["path"]) as im:
+            x = preprocess_image(im.convert("RGB"), cfg.resolution).unsqueeze(0)
+        if torch.cuda.is_available():
+            x = x.cuda()
+        with torch.no_grad():
+            feat = bb(x).squeeze(0).cpu().half()
+        torch.save(feat, cp)

src/data/provenance/finalize.py

@@ -0,0 +1,38 @@
+# src/curator/finalize.py
+"""Cross-artifact finalization (manifest-only; no image I/O, no shard rewrites).
+The official holdout always wins: finalization removes candidate TRAINING rows that
+collide with holdout rows (exact identity first, then pHash<=threshold), never the
+reverse. Splits are assigned only AFTER subtraction, and the finalized manifest pins
+the immutable candidate dataset version — 'latest' resolution is refused."""
+from data.provenance.dedup_audit import cross_leak_mask
+from data.provenance.splitter import assign_mixed_split
+
+
+def _ident(df):
+    col = "img_id" if "img_id" in df.columns else "rel_path"
+    return df[col].astype(str)
+
+
+def finalize_training_manifest(cand_df, holdout_df, candidate_ref, threshold=6,
+                               val_frac=0.15, seed=0):
+    if "@" not in candidate_ref:
+        raise ValueError(
+            f"candidate_ref must pin a version (slug@version), got {candidate_ref!r}")
+    cand = cand_df.copy()
+    n0 = len(cand)
+    hold_ids = set(_ident(holdout_df)) if len(holdout_df) else set()
+    cand = cand[~_ident(cand).isin(hold_ids)]
+    n_exact = n0 - len(cand)
+    if len(holdout_df) and len(cand):
+        mask = cross_leak_mask(cand["phash"].to_numpy(),
+                               holdout_df["phash"].to_numpy(), threshold=threshold)
+        cand = cand[~mask]
+    n_phash = n0 - n_exact - len(cand)
+    final = assign_mixed_split(cand.reset_index(drop=True), val_frac=val_frac, seed=seed)
+    for part in ("train", "val"):
+        labs = set(final[final["split"] == part]["label"])
+        assert labs == {0, 1}, f"finalize: split {part} lacks both classes ({labs})"
+    final["candidate_dataset"] = candidate_ref
+    report = {"n_exact_dropped": int(n_exact), "n_phash_dropped": int(n_phash),
+              "n_final": int(len(final))}
+    return final, report

src/data/provenance/journal.py

@@ -0,0 +1,48 @@
+# src/shards/journal.py
+"""Append-only shard progress journal. States are monotonic; deletion decisions
+downstream depend on the DURABLY persisted state (fsync per append), never on
+in-memory state. A compact parquet snapshot can be regenerated at any time."""
+import json
+import os
+import time
+
+STATES = ("planned", "writing", "verified", "indexed", "raw_cleaned", "remote_confirmed")
+_ORD = {s: i for i, s in enumerate(STATES)}
+
+
+class Journal:
+    def __init__(self, run_root):
+        self.path = os.path.join(run_root, "journal.jsonl")
+        self._latest = {}
+        if os.path.exists(self.path):
+            with open(self.path, encoding="utf-8") as f:
+                for line in f:
+                    if line.strip():
+                        rec = json.loads(line)
+                        self._latest[rec["shard_name"]] = rec
+
+    def state(self, shard_name):
+        rec = self._latest.get(shard_name)
+        return rec["state"] if rec else None
+
+    def load(self):
+        return dict(self._latest)
+
+    def advance(self, shard_name, state, **info):
+        if state not in _ORD:
+            raise ValueError(f"unknown state {state!r}")
+        cur = self.state(shard_name)
+        if cur is not None and _ORD[state] <= _ORD[cur]:
+            raise ValueError(f"non-monotonic {shard_name}: {cur} -> {state}")
+        rec = {"shard_name": shard_name, "state": state, "ts": time.time(), **info}
+        prev = self._latest.get(shard_name, {})
+        rec = {**{k: v for k, v in prev.items() if k not in ("state", "ts")}, **rec}
+        with open(self.path, "a", encoding="utf-8") as f:
+            f.write(json.dumps(rec) + "\n")
+            f.flush()
+            os.fsync(f.fileno())
+        self._latest[shard_name] = rec
+
+    def snapshot_parquet(self, path):
+        import pandas as pd
+        pd.DataFrame(list(self._latest.values())).to_parquet(path, index=False)

src/data/provenance/manifest_build.py

@@ -0,0 +1,54 @@
+# src/curator/manifest_build.py
+import os
+import pandas as pd
+from PIL import Image
+from data.provenance.families import classify_family
+
+MANIFEST_COLUMNS = ["path","label","source_dataset","generator_id","family",
+                    "real_source","native_w","native_h","jpeg_bytes_per_pixel",
+                    "ref_type","mount_slug","rel_path","split"]
+
+def jpeg_bytes_per_pixel(path) -> float:
+    try:
+        size = os.path.getsize(path)
+        with Image.open(path) as im:
+            w, h = im.size
+        return float(size) / float(w * h) if w and h else float("nan")
+    except Exception:
+        return float("nan")
+
+
+_EXT = {"JPEG": ".jpg", "PNG": ".png", "WEBP": ".webp", "BMP": ".bmp", "GIF": ".gif", "TIFF": ".tiff"}
+
+def save_source_image(raw: bytes, path_noext: str):
+    """Persist a source image blob VERBATIM (no re-encode) and return (path, w, h).
+
+    Re-encoding is the wrong move for this corpus: JPEG bakes a scored corruption into
+    the 'clean' ground truth; PNG re-encode of already-JPEG sources ~4x's disk (blows the
+    typical working-disk limits) for ZERO fidelity gain — the JPEG artifacts are already in
+    the source pixels. Writing the source bytes unchanged adds nothing and preserves the
+    original size. We sniff format+dims via PIL (header only, no full decode) to pick the
+    extension and fill native_w/h; a corrupt/undecodable blob raises (caller skips it)."""
+    import io
+    with Image.open(io.BytesIO(raw)) as im:      # header sniff; verify() would consume it
+        fmt, (w, h) = im.format, im.size
+        im.verify()                              # raises on a truncated/corrupt blob
+    ext = _EXT.get(fmt, ".img")
+    path = path_noext + ext
+    os.makedirs(os.path.dirname(path), exist_ok=True)
+    with open(path, "wb") as f:
+        f.write(raw)
+    return path, w, h
+
+def assemble_manifest(records: pd.DataFrame) -> pd.DataFrame:
+    df = records.copy()
+    df["family"] = df["generator_id"].fillna("none").map(classify_family)
+    # Resolution columns come from the sources (Tasks 3/4); default if a record lacks them.
+    if "ref_type" not in df.columns:
+        df["ref_type"] = "materialized"
+    if "mount_slug" not in df.columns:
+        df["mount_slug"] = ""
+    if "rel_path" not in df.columns:
+        df["rel_path"] = df["path"].map(lambda p: os.path.basename(str(p)))
+    df["split"] = ""
+    return df[MANIFEST_COLUMNS]

src/data/provenance/orchestrate.py

@@ -0,0 +1,122 @@
+# src/shards/orchestrate.py
+"""Per-shard transactions: plan -> write+verify -> journal -> delete raws. Raw files
+for a shard are deleted ONLY after the journal durably records the verified shard,
+so conversion never holds a second corpus-sized copy and a crash at any point leaves
+either the raws or a verified shard (never neither)."""
+import hashlib
+import os
+import shutil
+import zipfile
+
+import pandas as pd
+
+from data.provenance import diskcheck
+from data.provenance.writer import plan_shards, write_shard
+
+
+def _sha256_file(path):
+    h = hashlib.sha256()
+    with open(path, "rb") as f:
+        for chunk in iter(lambda: f.read(1 << 20), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def resume_pending(journal, shard_dir, raw_root):
+    """Startup/retry recovery: drop stale .partial files, and finish raw deletion for
+    shards journaled verified/indexed but not raw_cleaned. A digest mismatch on a
+    completed shard is a hard failure."""
+    for f in os.listdir(shard_dir):
+        if f.endswith(".partial"):
+            os.remove(os.path.join(shard_dir, f))
+    for name, rec in journal.load().items():
+        if rec["state"] in ("verified", "indexed"):
+            path = os.path.join(shard_dir, name)
+            if _sha256_file(path) != rec["sha256"]:
+                raise AssertionError(f"digest mismatch on completed shard {name}")
+            _delete_members(raw_root, rec.get("members", []))
+            if rec["state"] == "verified":
+                journal.advance(name, "indexed")
+            journal.advance(name, "raw_cleaned")
+
+
+def _delete_members(raw_root, member_names):
+    freed = 0
+    for rel in member_names:
+        p = os.path.join(raw_root, rel.replace("/", os.sep))
+        if os.path.exists(p):
+            freed += os.path.getsize(p)
+            os.remove(p)
+    # prune empty dirs bottom-up
+    for dp, _dirs, _files in os.walk(raw_root, topdown=False):
+        if dp != raw_root and not os.listdir(dp):
+            os.rmdir(dp)
+    return freed
+
+
+def shard_corpus(manifest_df, raw_root, shard_dir, journal, target_bytes=1 << 30,
+                 prefix="corpus", log=print):
+    """Shard every manifest row (transactionally, resumable via the journal) and
+    return (candidates_df, shards_df). candidates_df drops `path` (raws are deleted)
+    and carries shard_name/member_name/byte_size/crc32 with ref_type='shard'."""
+    df = manifest_df.copy()
+    if "byte_size" not in df.columns:
+        # Resume path: raws of completed shards are deleted — recover their sizes from
+        # the verified shards' central directories so packing stays deterministic.
+        sizes = {}
+        if os.path.isdir(shard_dir):
+            for f in os.listdir(shard_dir):
+                if f.endswith(".shard"):
+                    with zipfile.ZipFile(os.path.join(shard_dir, f)) as zf:
+                        for zi in zf.infolist():
+                            sizes[zi.filename] = zi.file_size
+        def _size(row):
+            if os.path.exists(row["path"]):
+                return os.path.getsize(row["path"])
+            if row["rel_path"] in sizes:
+                return sizes[row["rel_path"]]
+            raise FileNotFoundError(
+                f"no raw file and no shard member for {row['rel_path']}")
+        df["byte_size"] = df.apply(_size, axis=1)
+    plan = plan_shards(df, target_bytes=target_bytes, prefix=prefix)
+    shard_records = []
+    for name, group in plan.groupby("shard_name", sort=True):
+        state = journal.state(name)
+        if state == "raw_cleaned":
+            # resumed past this shard: recover per-member CRCs + ledger row from the
+            # verified archive itself (raws are gone; the shard is the source of truth)
+            with zipfile.ZipFile(os.path.join(shard_dir, name)) as zf:
+                crc = {zi.filename: zi.CRC for zi in zf.infolist()}
+            plan.loc[group.index, "crc32"] = group["member_name"].map(crc)
+            rec0 = journal.load()[name]
+            shard_records.append({"shard_name": name, "n_members": len(crc),
+                                  "payload_bytes": int(group["byte_size"].sum()),
+                                  "zip_bytes": os.path.getsize(os.path.join(shard_dir, name)),
+                                  "sha256": rec0["sha256"]})
+            continue
+        members = list(zip(group["member_name"], group["path"]))
+        diskcheck.assert_headroom(shard_dir, needed_bytes=int(group["byte_size"].sum()))
+        if state is None:
+            journal.advance(name, "planned", n_members=len(members))
+        journal.advance(name, "writing")
+        rec = write_shard(os.path.join(shard_dir, name), members)
+        journal.advance(name, "verified", sha256=rec["sha256"],
+                        members=[m["member_name"] for m in rec["members"]])
+        journal.advance(name, "indexed")
+        freed = _delete_members(raw_root, [m for m, _ in members])
+        free_after = shutil.disk_usage(shard_dir).free
+        journal.advance(name, "raw_cleaned", bytes_reclaimed=freed,
+                        free_after=free_after)
+        log(f"[shard] {name}: {rec['n_members']} members, "
+            f"{rec['zip_bytes']} zip bytes, freed {freed}, free {free_after}")
+        crc = {m["member_name"]: m["crc32"] for m in rec["members"]}
+        plan.loc[group.index, "crc32"] = group["member_name"].map(crc)
+        shard_records.append({"shard_name": name, "n_members": rec["n_members"],
+                              "payload_bytes": rec["payload_bytes"],
+                              "zip_bytes": rec["zip_bytes"], "sha256": rec["sha256"]})
+    cand = plan.drop(columns=["path"]).copy()
+    cand["ref_type"] = "shard"
+    cand["crc32"] = cand["crc32"].astype("int64")
+    return cand.reset_index(drop=True), pd.DataFrame(
+        shard_records, columns=["shard_name", "n_members", "payload_bytes",
+                                "zip_bytes", "sha256"])

src/data/provenance/push.py

@@ -0,0 +1,11 @@
+# src/data/provenance/push.py
+"""Manifest write-out. The corpus is built and consumed locally; there is no
+remote publishing step."""
+import os
+
+
+def write_manifest(df, out_dir) -> str:
+    os.makedirs(out_dir, exist_ok=True)
+    path = os.path.join(out_dir, "manifest.parquet")
+    df.to_parquet(path, index=False)
+    return path

src/data/provenance/rangezip.py

@@ -0,0 +1,266 @@
+# src/curator/rangezip.py
+"""Random-access reader for a REMOTE zip over HTTP byte ranges, using ONLY explicit
+ranges (bytes=a-b) — never suffix ranges (bytes=-N), which ModelScope's file endpoint
+rejects with 400 (remotezip uses suffix ranges, hence its failure here).
+
+Why this works at all: a ZIP has a Central Directory at the END mapping every member
+to a byte offset, and each member is compressed INDEPENDENTLY. So to pull one image out
+of a 25 GB zip we: (1) range-read the tail to find + parse the central directory, then
+(2) range-read just that member's local-header+data and inflate it. Transfer is ~the
+member size, not the whole archive.
+
+Only the subset of ZIP needed for WildFake is implemented: STORED (0) + DEFLATE (8),
+32-bit offsets, and ZIP64 end-of-central-directory when present. No encryption.
+"""
+import struct
+import zlib
+import urllib.request
+
+_EOCD_SIG = b"PK\x05\x06"
+_EOCD64_LOC_SIG = b"PK\x06\x07"
+_EOCD64_SIG = b"PK\x06\x06"
+_CEN_SIG = b"PK\x01\x02"
+
+
+class _ShortBuffer(Exception):
+    """Raised by _decode_at when the fetched buffer doesn't hold the full member; carries
+    the exact byte span now known so the caller re-fetches precisely (no truncation)."""
+    def __init__(self, needed):
+        self.needed = needed
+        super().__init__(f"buffer short, need {needed} bytes")
+
+
+class RangeZip:
+    def __init__(self, url, headers=None, session=None):
+        self.url = url
+        self.headers = headers or {}
+        # CONNECTION REUSE: one persistent keep-alive session for ALL range reads to this
+        # zip's host, so the TLS handshake to ModelScope (China, ~2s) is paid ONCE instead
+        # of per request. This is the dominant cost — reusing the SSL/TCP connection is what
+        # makes many range reads cheap. Falls back to urllib (still per-request) if requests
+        # is unavailable.
+        self._session = session if session is not None else self._make_session()
+        self.size = self._content_length()
+        self._entries = self._read_central_directory()   # name -> (offset, comp_size, method)
+
+    @staticmethod
+    def _make_session():
+        try:
+            import requests
+            from requests.adapters import HTTPAdapter
+            s = requests.Session()
+            # a small pool of persistent connections, reused across range GETs (keep-alive).
+            ad = HTTPAdapter(pool_connections=8, pool_maxsize=32, max_retries=3)
+            s.mount("https://", ad); s.mount("http://", ad)
+            return s
+        except Exception:
+            return None                                  # urllib fallback (no pooling)
+
+    # --- HTTP helpers (explicit ranges only) --------------------------------
+    def _content_length(self):
+        if self._session is not None:
+            r = self._session.head(self.url, headers=self.headers, allow_redirects=True, timeout=60)
+            r.raise_for_status()
+            return int(r.headers["Content-Length"])
+        req = urllib.request.Request(self.url, method="HEAD", headers=self.headers)
+        with urllib.request.urlopen(req) as r:
+            return int(r.headers["Content-Length"])
+
+    def _get_range(self, lo, hi):
+        """Fetch bytes [lo, hi] inclusive via an explicit Range header, over the persistent
+        keep-alive session (reused SSL/TCP connection) when available."""
+        h = dict(self.headers); h["Range"] = f"bytes={lo}-{hi}"
+        if self._session is not None:
+            r = self._session.get(self.url, headers=h, timeout=300)
+            r.raise_for_status()
+            return r.content
+        req = urllib.request.Request(self.url, headers=h)
+        with urllib.request.urlopen(req) as r:
+            return r.read()
+
+    # Tail window read to bootstrap the directory. Sized so the CENTRAL DIRECTORY of most
+    # zips fits inside it (WildFake: reals 3-17MB, typical fakes 5-16MB), letting us slice
+    # the directory straight out of this one read — no second GET. Only the giant
+    # 200k+-member zips (CD 20-52MB) need a follow-up, and then only for the missing prefix.
+    _TAIL = 16 * 1024 * 1024
+
+    # --- central directory --------------------------------------------------
+    def _read_central_directory(self):
+        tail_len = min(self._TAIL, self.size)
+        tail_start = self.size - tail_len
+        tail = self._get_range(tail_start, self.size - 1)         # ONE read, big window
+        i = tail.rfind(_EOCD_SIG)
+        if i < 0:
+            raise ValueError("EOCD not found (not a zip or comment too large)")
+        eocd = tail[i:i + 22]
+        cd_size = struct.unpack("<I", eocd[12:16])[0]
+        cd_off = struct.unpack("<I", eocd[16:20])[0]
+        total = struct.unpack("<H", eocd[10:12])[0]
+        # ZIP64 path: offsets == 0xFFFFFFFF -> find the ZIP64 EOCD via its locator.
+        if cd_off == 0xFFFFFFFF or cd_size == 0xFFFFFFFF or total == 0xFFFF:
+            j = tail.rfind(_EOCD64_LOC_SIG, 0, i)
+            if j < 0:
+                raise ValueError("ZIP64 locator not found")
+            eocd64_off = struct.unpack("<Q", tail[j + 8:j + 16])[0]
+            # the ZIP64 EOCD is itself usually in our tail window; slice if so, else fetch.
+            if eocd64_off >= tail_start:
+                head = tail[eocd64_off - tail_start: eocd64_off - tail_start + 56]
+            else:
+                head = self._get_range(eocd64_off, eocd64_off + 56 - 1)
+            if head[:4] != _EOCD64_SIG:
+                raise ValueError("bad ZIP64 EOCD signature")
+            cd_size = struct.unpack("<Q", head[40:48])[0]
+            cd_off = struct.unpack("<Q", head[48:56])[0]
+        # If the whole central directory is already inside the tail we read, SLICE it —
+        # no second request. Otherwise fetch only the prefix we're missing and prepend.
+        if cd_off >= tail_start:
+            cd = tail[cd_off - tail_start: cd_off - tail_start + cd_size]
+        else:
+            missing = self._get_range(cd_off, tail_start - 1)     # only the part below the tail
+            cd = missing + tail[:max(0, (cd_off + cd_size) - tail_start)]
+        return self._parse_central(cd)
+
+    def _parse_central(self, cd):
+        entries, p = {}, 0
+        while p + 4 <= len(cd) and cd[p:p + 4] == _CEN_SIG:
+            method = struct.unpack("<H", cd[p + 10:p + 12])[0]
+            comp_size = struct.unpack("<I", cd[p + 20:p + 24])[0]
+            n = struct.unpack("<H", cd[p + 28:p + 30])[0]
+            m = struct.unpack("<H", cd[p + 30:p + 32])[0]
+            k = struct.unpack("<H", cd[p + 32:p + 34])[0]
+            lho = struct.unpack("<I", cd[p + 42:p + 46])[0]
+            name = cd[p + 46:p + 46 + n].decode("utf-8", "replace")
+            extra = cd[p + 46 + n:p + 46 + n + m]
+            # ZIP64 extra: patch 0xFFFFFFFF comp_size / offset from the extra field.
+            if comp_size == 0xFFFFFFFF or lho == 0xFFFFFFFF:
+                comp_size, lho = _zip64_patch(extra, comp_size, lho)
+            entries[name] = (lho, comp_size, method)
+            p += 46 + n + m + k
+        return entries
+
+    # --- public API ---------------------------------------------------------
+    def namelist(self):
+        return list(self._entries)
+
+    def image_members(self, exts=(".png", ".jpg", ".jpeg", ".webp", ".bmp")):
+        return [n for n in self._entries if n.lower().endswith(exts)]
+
+    # Generous slack past comp_size to cover the LOCAL header (30 fixed + name + extra
+    # fields). The local extra field can carry ZIP64/alignment/other records and is NOT
+    # bounded by 256B — a too-small slack truncates the compressed stream (zlib error -5).
+    # 4KB comfortably covers any realistic local header; _decode_at still VERIFIES the
+    # buffer actually holds the full member and raises if not (never silently truncates).
+    _LH_SLACK = 30 + 4096
+
+    def _decode_at(self, buf, base, lho, comp_size, method):
+        """Decode ONE member from an in-memory buffer `buf` that starts at absolute byte
+        `base` and contains the member's local header (at `lho`) + its compressed data.
+        Raises _ShortBuffer if the buffer doesn't hold the full compressed payload — the
+        caller then re-fetches the exact span (never returns a truncated stream)."""
+        off = lho - base
+        if off + 30 > len(buf):
+            raise _ShortBuffer(30)                       # not even the fixed header present
+        n = struct.unpack("<H", buf[off + 26:off + 28])[0]
+        m = struct.unpack("<H", buf[off + 28:off + 30])[0]
+        start = off + 30 + n + m
+        end = start + comp_size
+        if end > len(buf):
+            raise _ShortBuffer(30 + n + m + comp_size)   # exact span now known
+        raw = buf[start:end]
+        if method == 0:
+            return raw
+        if method == 8:
+            return zlib.decompress(raw, -15)
+        raise ValueError(f"unsupported compression method {method}")
+
+    def read(self, name):
+        """DECOMPRESSED bytes of one member via a range fetch of its local header + data.
+        Retries once with the exact span if the first (slack-estimated) window was short."""
+        lho, comp_size, method = self._entries[name]
+        span = self._LH_SLACK + comp_size
+        window = self._get_range(lho, min(lho + span, self.size) - 1)
+        try:
+            return self._decode_at(window, lho, lho, comp_size, method)
+        except _ShortBuffer as e:
+            window = self._get_range(lho, min(lho + e.needed, self.size) - 1)
+            return self._decode_at(window, lho, lho, comp_size, method)
+
+    def read_many(self, names, max_gap=1 << 20, workers=16):
+        """Fetch MANY members by COALESCING nearby members into contiguous segments, then
+        fetching those segments IN PARALLEL over the shared keep-alive session.
+
+        The parallelism is at the SEGMENT level (not the zip level): a single giant WildFake
+        zip has hundreds of segments, and with only ~12 zips, per-zip concurrency alone
+        barely distributes — one big zip would serialize. Running its segment GETs across
+        `workers` threads (all reusing one pooled SSL connection) is what actually saturates
+        the link. Returns {name: decompressed_bytes}.
+        """
+        from concurrent.futures import ThreadPoolExecutor
+        # each member spans [lho, lho + LH_SLACK + comp_size); sort by start offset
+        spans = []
+        for nm in names:
+            lho, comp_size, method = self._entries[nm]
+            end = lho + self._LH_SLACK + comp_size
+            spans.append((lho, end, nm, comp_size, method))
+        spans.sort()
+
+        # 1) build the list of coalesced segments (each = one contiguous range + its members)
+        segments = []
+        i = 0
+        while i < len(spans):
+            seg_lo, seg_hi = spans[i][0], spans[i][1]
+            members = [spans[i]]
+            j = i + 1
+            while j < len(spans) and spans[j][0] <= seg_hi + max_gap:
+                seg_hi = max(seg_hi, spans[j][1])
+                members.append(spans[j])
+                j += 1
+            segments.append((seg_lo, min(seg_hi, self.size), members))
+            i = j
+
+        # 2) fetch + decode each segment; segments run in PARALLEL over the pooled session
+        out, lock = {}, __import__("threading").Lock()
+        def _do(seg):
+            seg_lo, seg_hi, members = seg
+            buf = self._get_range(seg_lo, seg_hi - 1)          # ONE GET per segment
+            local = {}
+            for lho, _end, nm, comp_size, method in members:
+                try:
+                    local[nm] = self._decode_at(buf, seg_lo, lho, comp_size, method)
+                except _ShortBuffer:
+                    local[nm] = self.read(nm)
+            with lock:
+                out.update(local)
+        if workers > 1 and len(segments) > 1:
+            with ThreadPoolExecutor(max_workers=workers) as ex:
+                list(ex.map(_do, segments))
+        else:
+            for seg in segments:
+                _do(seg)
+        return out
+
+
+def _zip64_patch(extra, comp_size, lho):
+    p = 0
+    while p + 4 <= len(extra):
+        hid, sz = struct.unpack("<HH", extra[p:p + 4])
+        body = extra[p + 4:p + 4 + sz]
+        if hid == 0x0001:                             # ZIP64 extended info
+            q = 0
+            # order: uncompressed, compressed, offset — only present when the
+            # corresponding central field was 0xFFFFFFFF. We only need comp+offset.
+            # Uncompressed (skip if present): can't know without the orig flags, so we
+            # conservatively read fields in order and assign the ones we need.
+            vals = []
+            while q + 8 <= len(body):
+                vals.append(struct.unpack("<Q", body[q:q + 8])[0]); q += 8
+            # Heuristic matching WildFake zips: [uncomp, comp, offset] present.
+            if len(vals) >= 3:
+                comp_size, lho = vals[1], vals[2]
+            elif len(vals) == 2:
+                comp_size, lho = vals[0], vals[1]
+            elif len(vals) == 1:
+                lho = vals[0]
+            break
+        p += 4 + sz
+    return comp_size, lho

src/data/provenance/resolver.py

@@ -0,0 +1,53 @@
+# src/curator/resolver.py
+import os
+from PIL import Image
+
+# Optional per-slug marker sub-path used to DISCOVER a mounted dataset's root when
+# Mount layouts are non-uniform across hosts. The corpus is ALL-MATERIALIZED from
+# official sources (commercial-safe), so no default mount markers are configured;
+# the mechanism is retained for optional reference-mounting of any future
+# commercial-safe corpus directory (pass `markers={slug: marker}` to SourceResolver).
+MOUNT_MARKERS: dict[str, str] = {}
+
+class SourceResolver:
+    def __init__(self, input_root="./data/raw", fakes_slug="track5-corpus-fakes", markers=None):
+        self.input_root = input_root
+        self.fakes_slug = fakes_slug
+        self.markers = markers if markers is not None else MOUNT_MARKERS
+        self._root_cache = {}
+
+    def discover_root(self, slug) -> str:
+        if slug in self._root_cache:
+            return self._root_cache[slug]
+        marker = self.markers[slug]
+        # find a dir whose path ends with the marker; its PARENT-of-marker is the root
+        marker_norm = marker.replace("\\", "/")
+        for dp, dirs, _ in os.walk(self.input_root):
+            rel = os.path.relpath(dp, self.input_root)
+            rel_norm = rel.replace("\\", "/")
+            # Anchor to a path-component boundary so marker "dataset" does NOT match
+            # ".../coco-2017-dataset" (no "/dataset" segment) while ".../lhq-1024/dataset" does.
+            if rel_norm == marker_norm or rel_norm.endswith("/" + marker_norm):
+                # root = dp with the marker suffix stripped, so root/rel_path (which
+                # includes the marker prefix) resolves
+                depth = len(marker.replace("\\", "/").split("/"))
+                root = dp
+                for _ in range(depth):
+                    root = os.path.dirname(root)
+                self._root_cache[slug] = root
+                return root
+        raise FileNotFoundError(f"resolver: could not discover mount root for {slug} (marker {marker}) under {self.input_root}")
+
+    def _path(self, row) -> str:
+        if row["ref_type"] == "mount":
+            return os.path.join(self.discover_root(row["mount_slug"]), row["rel_path"])
+        if row["ref_type"] == "materialized":
+            return os.path.join(self.input_root, self.fakes_slug, row["rel_path"])
+        raise ValueError(f"unknown ref_type {row['ref_type']}")
+
+    def resolve(self, row) -> Image.Image:
+        p = self._path(row)
+        if not os.path.exists(p):
+            raise FileNotFoundError(f"resolver: {p} (ref_type={row['ref_type']}, slug={row.get('mount_slug')})")
+        with Image.open(p) as im:
+            return im.convert("RGB").copy()

src/data/provenance/run_curator.py

@@ -0,0 +1,110 @@
+# src/curator/run_curator.py
+import argparse, os, tempfile
+import pandas as pd
+from data.provenance import sources, subsample, manifest_build, audit, push, dedup_audit
+from data.provenance.splitter import assign_mixed_split
+
+
+def _write_stage_manifest(frame, out_dir, stage):
+    path = os.path.join(out_dir, f"manifest_{stage}.parquet")
+    frame.to_parquet(path, index=False)
+    counts = (frame.groupby(["label", "source_dataset", "family"], dropna=False)
+              .size().rename("count").reset_index())
+    counts.insert(0, "stage", stage)
+    return counts
+
+def _write_build_metadata(tables_dir, **config):
+    import json, subprocess, time, uuid
+    try:
+        commit = subprocess.run(["git", "rev-parse", "HEAD"], capture_output=True,
+                                text=True).stdout.strip()
+    except Exception:  # noqa: BLE001
+        commit = "unknown"
+    meta = {"code_commit": commit or "unknown", "build_id": uuid.uuid4().hex,
+            "built_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+            "schema_version": 1, "license_policy": "commercial-only-2026-08-30",
+            "config": config}
+    with open(os.path.join(tables_dir, "build_metadata.json"), "w") as f:
+        json.dump(meta, f, indent=1)
+
+
+def run_curator(per_family_cap=30000, seed=0, out_dir=None, input_root="./data/raw",
+                do_push=False, allow_shortcut=False, corpus_sources=None, max_examine=None,
+                progress_cb=None, sharded=False, run_root=None):
+    out_dir = out_dir or tempfile.mkdtemp(prefix="corpus_")
+    # Corpus is ALL-MATERIALIZED from official commercial-safe sources (JPEG q95) into
+    # a versioned corpus directory. Sources emit BOTH real (label 0) and fake (label 1)
+    # rows — e.g. SID_Set/WildFake/OmniFake carry reals; CommForensics is fake-only.
+    fakes_out_dir = os.path.join(out_dir, "track5-corpus-fakes")
+    if corpus_sources is None:
+        corpus_sources = sources.default_corpus_sources(per_family_cap)
+    if sharded:
+        # Sharded tail (spec: sharded-corpus-pipeline): candidate corpus only — NO split
+        # (the finalizer assigns splits after official-holdout subtraction), deterministic
+        # ZIP shards + audit tables in a flat publish dir, raws deleted per verified shard.
+        from data.provenance import cleanup as _cleanup, diskcheck, journal as _journal
+        from data.provenance.orchestrate import shard_corpus
+        root = _cleanup.make_run_root(run_root or tempfile.gettempdir())
+        tables_dir = os.path.join(root, "publish")
+        os.makedirs(tables_dir, exist_ok=True)
+        print("[disk]", diskcheck.fs_stats(diskcheck.default_watch_paths(root)), flush=True)
+    else:
+        tables_dir = out_dir
+    raw = sources.pull_all(corpus_sources, fakes_out_dir, per_family_cap, seed,
+                           max_examine=max_examine, progress_cb=progress_cb)
+    # --- pure logic ---
+    # assemble_manifest FIRST so `family` (and split/ref columns) exist before
+    # fake-family capping, which must never touch explicitly licensed real quotas.
+    mani = manifest_build.assemble_manifest(raw)
+    stage_counts = [_write_stage_manifest(mani, tables_dir, "raw")]
+    mani = subsample.cap_fake_families(mani, per_family_cap, seed)
+    stage_counts.append(_write_stage_manifest(mani, tables_dir, "capped"))
+    # Dedup AFTER cap (bounds the O(n^2) pHash scan to the capped set) but BEFORE
+    # splitting. OmniFake overlaps
+    # WildFake/GenImage; near-identical images collapse to one, priority-ordered.
+    # provenance = the {sampled, after_dedup, dropped_as_dupe} attribution deliverable.
+    mani, provenance = dedup_audit.dedup_manifest(mani)
+    stage_counts.append(_write_stage_manifest(mani, tables_dir, "deduped"))
+    pd.concat(stage_counts, ignore_index=True).to_parquet(
+        os.path.join(tables_dir, "stage_counts.parquet"), index=False)
+    prov_path = os.path.join(tables_dir, "provenance.parquet")
+    provenance.to_parquet(prov_path, index=False)
+    if sharded:
+        # GC checkpoint: cap/dedup rejects are unreferenced now — reclaim before sharding
+        from data.provenance.cleanup import reclaim_unreferenced
+        n_gc, b_gc = reclaim_unreferenced(fakes_out_dir, set(mani["rel_path"]))
+        print(f"[gc] post-dedup reclaim: {n_gc} files, {b_gc} bytes", flush=True)
+        j = _journal.Journal(root)
+        mani_ns = mani.drop(columns=["split"], errors="ignore")
+        cand, shards_df = shard_corpus(mani_ns, fakes_out_dir, tables_dir, j)
+        cand.to_parquet(os.path.join(tables_dir, "manifest_candidates.parquet"), index=False)
+        shards_df.to_parquet(os.path.join(tables_dir, "shards.parquet"), index=False)
+        _write_build_metadata(tables_dir, per_family_cap=per_family_cap, seed=seed)
+        print("[disk]", diskcheck.fs_stats(diskcheck.default_watch_paths(root)), flush=True)
+        return tables_dir
+    mani = assign_mixed_split(mani, val_frac=0.15, seed=seed)
+    # --- audit gate ---
+    rep = audit.audit_corpus(mani)
+    print(rep["report"]); print("balance:", rep["class_balance"]); print("families:", rep["per_family_counts"])
+    if rep["flag"] and not allow_shortcut:
+        raise SystemExit("DDA shortcut flagged — mitigate and re-run, or pass --allow-shortcut.")
+    path = push.write_manifest(mani, out_dir)
+    print("manifest:", path)
+    print("provenance:", prov_path); print(provenance.to_string(index=False))
+    return out_dir
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--per-family-cap", type=int, default=30000)
+    ap.add_argument("--seed", type=int, default=0)
+    ap.add_argument("--out-dir", default=None)
+    ap.add_argument("--push", action="store_true")
+    ap.add_argument("--allow-shortcut", action="store_true")
+    ap.add_argument("--max-examine", type=int, default=None,
+                    help="cap examples streamed per source (bandwidth guard)")
+    a = ap.parse_args()
+    run_curator(per_family_cap=a.per_family_cap, seed=a.seed, out_dir=a.out_dir,
+                do_push=a.push, allow_shortcut=a.allow_shortcut, max_examine=a.max_examine)
+
+if __name__ == "__main__":
+    main()

src/data/provenance/selection.py

@@ -0,0 +1,52 @@
+# src/curator/selection.py
+"""The DECISION between the two passes: given a scan inventory (what we HAVE), compute
+a selection plan (what we'll PULL). Pure logic on the scan DataFrame — no I/O — so the
+sampling policy is reviewable before any image is touched.
+
+inventory columns: [source_dataset, group_key, label, count]  (from DatasetSource.scan)
+plan: {(source_dataset, group_key, label): quota}  (consumed by DatasetSource.pull)
+"""
+
+
+def plan_selection(inventory, per_group=None, balance=False):
+    """Cap each (source, group, label) at min(available, per_group); optionally balance
+    classes by trimming the majority class to the minority total.
+
+    per_group=None -> take everything available (quota = count). balance=True -> after
+    capping, scale the larger class's quotas down proportionally so n_fake == n_real
+    (as evenly as possible), preserving per-group proportions."""
+    plan = {}
+    for r in inventory.itertuples():
+        avail = int(r.count)
+        quota = avail if per_group is None else min(avail, per_group)
+        if quota > 0:
+            plan[(r.source_dataset, r.group_key, int(r.label))] = quota
+
+    if not balance:
+        return plan
+
+    n = {0: 0, 1: 0}
+    for (_s, _g, lbl), q in plan.items():
+        n[lbl] += q
+    if n[0] == 0 or n[1] == 0:
+        return plan                     # can't balance with a missing class; leave as-is
+    keep, drop = (0, 1) if n[0] < n[1] else (1, 0)
+    target = n[keep]                    # trim the majority class down to the minority total
+    # Distribute `target` across the majority groups proportionally, largest-remainder.
+    maj = {k: v for k, v in plan.items() if k[2] == drop}
+    total = sum(maj.values())
+    scaled, rema = {}, []
+    for k, v in maj.items():
+        exact = v * target / total
+        base = int(exact)
+        scaled[k] = base
+        rema.append((exact - base, k))
+    short = target - sum(scaled.values())
+    for _frac, k in sorted(rema, reverse=True)[:short]:
+        scaled[k] += 1
+    for k, v in scaled.items():
+        if v > 0:
+            plan[k] = v
+        else:
+            plan.pop(k, None)           # a group scaled to 0 drops out
+    return plan

src/data/provenance/sources.py

@@ -0,0 +1,502 @@
+"""Per-dataset source abstraction. Each dataset = a DatasetSource subclass; the
+base class owns the shared record-building + per-family cap loop (DRY). Heavy
+imports (datasets, huggingface_hub) are deferred inside _iter_examples so importing
+this module stays cheap and pytest-collection-safe (no downloads at import)."""
+import abc, os
+import pandas as pd
+from data.provenance.families import classify_family
+from data.provenance.manifest_build import jpeg_bytes_per_pixel, save_source_image
+
+RECORD_COLUMNS = ["path","label","source_dataset","generator_id",
+                  "real_source","native_w","native_h","jpeg_bytes_per_pixel",
+                  "ref_type","mount_slug","rel_path"]
+
+# --- Defensive field readers (shared) ---------------------------------------
+# The commercial-safe sources have heterogeneous / under-documented schemas
+# (ModelScope cards are bare, OmniFake's viewer is broken). These readers coerce
+# a raw example into (label, generator, PIL image) across the likely field names,
+# so subclasses' _to_fields stay small and testable with hand-built dicts.
+
+def _as_pil(obj):
+    """Coerce a PIL image / bytes / {'bytes'|'path'} dict / path-string -> PIL.Image."""
+    import io
+    from PIL import Image
+    if isinstance(obj, Image.Image):
+        return obj
+    if isinstance(obj, (bytes, bytearray)):
+        return Image.open(io.BytesIO(bytes(obj)))
+    if isinstance(obj, dict):
+        if obj.get("bytes"):
+            return Image.open(io.BytesIO(obj["bytes"]))
+        if obj.get("path"):
+            return Image.open(obj["path"])
+    if isinstance(obj, str):
+        return Image.open(obj)
+    raise TypeError(f"_as_pil: cannot coerce {type(obj)!r} to a PIL image")
+
+def _read_label(ex, default=None):
+    """Best-effort 0(real)/1(fake) from an example. Returns None (sentinel) when NO
+    label field is resolvable — callers MUST skip such rows, never default them to
+    fake (defaulting a real image into the fake class poisons the real/fake boundary)."""
+    for k in ("label", "labels", "target", "is_fake", "fake"):
+        if ex.get(k) is not None:
+            v = ex[k]
+            if isinstance(v, str):
+                s = v.strip().lower()
+                if s in ("real", "0", "authentic", "genuine"): return 0
+                if s in ("fake", "1", "synthetic", "ai", "generated"): return 1
+            try:
+                return 1 if int(v) != 0 else 0
+            except (TypeError, ValueError):
+                pass
+    for k in ("category", "class", "folder", "subset", "type"):
+        v = ex.get(k)
+        if isinstance(v, str):
+            return 0 if v.strip().lower() in ("real", "0_real", "authentic") else 1
+    return default
+
+def _read_generator(ex):
+    """Best-effort generator/model name string (feeds classify_family)."""
+    for k in ("generator", "model", "model_name", "gen_model", "method",
+              "source_model", "generator_name", "label_name", "category",
+              "class", "folder"):
+        v = ex.get(k)
+        if isinstance(v, str) and v:
+            return v
+    return ""
+
+def _read_image(ex):
+    """Return the raw image object from an example (caller passes to _as_pil)."""
+    for k in ("image", "img", "image_data", "image_bytes", "picture"):
+        if ex.get(k) is not None:
+            return ex[k]
+    raise KeyError("_read_image: no image field in example")
+
+class DatasetSource(abc.ABC):
+    name: str = "source"
+
+    @abc.abstractmethod
+    def _iter_examples(self):
+        """Yield raw examples (network I/O lives here, in subclasses)."""
+    @abc.abstractmethod
+    def _to_fields(self, example) -> dict:
+        """Map one example -> {label, generator_id, real_source, image: PIL.Image}."""
+
+    def _group_key(self, f) -> str:
+        """The split/selection group for a materialized-field dict: family for fakes
+        (label 1), real_source for reals (label 0). This is the unit scan counts and
+        plan_selection quotas over."""
+        return classify_family(f["generator_id"]) if f["label"] == 1 else f["real_source"]
+
+    def scan(self, max_examine=None) -> pd.DataFrame:
+        """PASS 1 — inventory only, NO images saved. Returns a DataFrame with columns
+        [source_dataset, group_key, label, count]: how many examples this source has per
+        (group, label). Default implementation walks _iter_examples()/_to_fields() but
+        decodes NO pixels beyond what _to_fields needs; subclasses with cheap metadata
+        (e.g. WildFake's label CSVs) should OVERRIDE scan() to avoid touching images.
+        max_examine bounds examined examples for streaming sources."""
+        counts = {}
+        for n_seen, ex in enumerate(self._iter_examples()):
+            if max_examine is not None and n_seen >= max_examine:
+                break
+            f = self._to_fields(ex)
+            if f is None:
+                continue
+            key = (self._group_key(f), int(f["label"]))
+            counts[key] = counts.get(key, 0) + 1
+        rows = [{"source_dataset": self.name, "group_key": g, "label": l, "count": c}
+                for (g, l), c in counts.items()]
+        return pd.DataFrame(rows, columns=["source_dataset", "group_key", "label", "count"])
+
+    def pull(self, out_dir, seed=0, per_family_cap=None, max_examine=None,
+             selection=None, progress_cb=None, progress_every=100) -> pd.DataFrame:
+        # PASS 2 — materialize. Two quota modes:
+        #   selection (two-pass): {(source, group_key, label): quota} from plan_selection.
+        #   per_family_cap (legacy one-pass): flat cap per family.
+        # max_examine bounds examples EXAMINED (streamed) — the bandwidth guard for a
+        # streaming source that would otherwise transfer bytes of every example.
+        # progress_cb({done, ...}) fires every progress_every materialized rows so a long
+        # network stream (e.g. CF PublicEval) is monitorable, same contract WildFake uses.
+        os.makedirs(out_dir, exist_ok=True)
+        counts, rows, n_skipped = {}, [], 0
+        import time as _t
+        _t0 = _t.time()
+        for n_seen, ex in enumerate(self._iter_examples()):
+            if max_examine is not None and n_seen >= max_examine:
+                break
+            f = self._to_fields(ex)
+            if f is None:                       # unknown-label row: SKIP, do not
+                n_skipped += 1                  # materialize and do not count vs cap
+                continue                        # (defaulting to fake would poison reals)
+            # Counter keyed by GROUP_KEY (family for fakes, real_source for reals) so
+            # per-group quotas don't collide — all reals share fam=="unknown", so a
+            # fam-keyed counter would merge distinct real sources under one quota.
+            gkey = self._group_key(f)
+            if selection is not None:
+                quota = selection.get((self.name, gkey, int(f["label"])), 0)
+                if counts.get(gkey, 0) >= quota:
+                    continue
+            elif (f["label"] == 1 and per_family_cap is not None
+                  and counts.get(gkey, 0) >= per_family_cap):
+                continue
+            elif (f["label"] == 0 and getattr(self, "real_cap", None) is not None
+                  and counts.get(gkey, 0) >= self.real_cap):
+                # per-real-source cap (label 0): bounds a materialized real source (e.g. CF
+                # PublicEval reals) without a two-pass selection. Keyed by real_source group,
+                # so distinct real sources each get their own quota. None => uncapped (SID
+                # controls its own real count via its n_real quota, so it leaves this unset).
+                continue
+            # Namespace rel_path by source name so it round-trips through the pushed fakes
+            # dataset root: file lands at <out_dir>/<name>/fakes/x.<ext> and the resolver
+            # opens input_root/<fakes_slug>/<name>/fakes/x.<ext>.
+            # PREFER VERBATIM source bytes (f["raw"], set by _to_fields when it has the
+            # original blob) — no re-encode, so no baked-in corruption and no PNG bloat. Only
+            # fall back to a lossless-PNG re-encode when a source hands us a PIL with no bytes
+            # (e.g. SID); JPEG re-encode is never used (it would corrupt the clean ground truth).
+            base = f"{self.name}/fakes/{self.name}_{gkey}_{counts.get(gkey,0)}"
+            p_noext = os.path.join(out_dir, base)
+            raw = f.get("raw")
+            if raw is not None:
+                p, w, h = save_source_image(raw, p_noext)
+            else:
+                from PIL import Image
+                p = p_noext + ".png"
+                os.makedirs(os.path.dirname(p), exist_ok=True)
+                f["image"].convert("RGB").save(p, format="PNG", optimize=False)
+                with Image.open(p) as im:
+                    w, h = im.size
+            rel = os.path.relpath(p, out_dir).replace(os.sep, "/")
+            rows.append({"path": p, "label": f["label"], "source_dataset": self.name,
+                         "generator_id": f["generator_id"], "real_source": f["real_source"],
+                         "native_w": w, "native_h": h, "jpeg_bytes_per_pixel": jpeg_bytes_per_pixel(p),
+                         "ref_type": "materialized", "mount_slug": "", "rel_path": rel})
+            counts[gkey] = counts.get(gkey, 0) + 1
+            if progress_cb is not None and len(rows) % progress_every == 0:
+                dt = max(_t.time() - _t0, 1e-9)
+                progress_cb({"done": len(rows), "imgs_per_s": len(rows) / dt})
+        print(f"[{self.name}] skipped {n_skipped} unlabeled examples")
+        return pd.DataFrame(rows, columns=RECORD_COLUMNS)
+
+def _iter_parquet_rows(fileobj, columns, batch_size=256):
+    """Yield rows (dicts of `columns`) from a parquet fileobj ONE BATCH AT A TIME.
+
+    Peak RAM is a single `batch_size`-row batch, NEVER a whole row-group. This is the
+    load-bearing memory bound: CommForensics PublicEval shards are packed as a SINGLE
+    row-group of up to 6.5GB (8640 rows of raw image blobs), so HF `datasets`
+    streaming (which materializes a whole row-group's Arrow table before yielding row 1)
+    climbs to ~30GB and OOM-kills a modest host. pyarrow.iter_batches decodes only the
+    requested columns and only one batch of rows at a time -> bounded, provably."""
+    import pyarrow.parquet as pq
+    pf = pq.ParquetFile(fileobj)
+    for batch in pf.iter_batches(batch_size=batch_size, columns=columns):
+        d = batch.to_pydict()
+        for i in range(len(batch)):
+            yield {c: d[c][i] for c in columns}
+
+
+class CommunityForensicsSource(DatasetSource):
+    name = "commforensics"
+    # LICENSE-MANDATORY (docs/specs/DATASET_LICENSES.md): materialize ONLY the
+    # redistribution-safe PublicEval split (schema: image_data / model_name / nsfw_flag /
+    # real_source / label "1:Fake 0:Real"). The Systematic / Manual / Commercial splits are
+    # FORBIDDEN (fakes paired with reals of unknown redistributability). We read PublicEval's
+    # parquet shards DIRECTLY via pyarrow (batch-bounded RAM, see _iter_parquet_rows) rather
+    # than HF `datasets` streaming, which OOMs on the single-6.5GB-row-group shard layout.
+    # The separate -Small repo is DROPPED: 760GB of 4GB single-row-group shards, same hazard
+    # for marginal gain — PublicEval alone is 21 generators (Dalle2/3, FLUX, MidjourneyV5/6,
+    # Firefly, Imagen3, Ideogram, kandinsky, stable_cascade, GALIP, DFGAN, ...) + reals.
+    HF_REPO = "OwensLab/CommunityForensics"
+    HF_SPLIT_DIR = "data/publicEval"                 # PublicEval shards only
+    N_SHARDS = 6
+    # HELD-OUT GENERATORS: CF PublicEval contains Dalle3 + Imagen3, the exact families the
+    # OOS holdout quarantines (WildFake DALLE3/GigaGAN/Imagen). Training on them would make the
+    # "unseen generator" holdout no longer unseen -> excluded here to keep the eval honest.
+    # (Dalle2 is kept — a genuinely different, non-held-out model.)
+    #
+    # CANONICAL match: exact case-sensitive strings were brittle — "Imagen" vs
+    # "Imagen3", "DALLE3" vs "Dalle3" vs "dall-e 3" would slip through and leak the holdout. We
+    # normalize (lowercase, strip non-alphanumerics) and match the held-out FAMILY prefixes, so
+    # every spelling/version of a held-out family is caught. GigaGAN added even though CF likely
+    # lacks it — a no-op if absent, a leak-stopper if present.
+    _HELD_OUT_CANON = ("dalle3", "imagen", "gigagan")   # family prefixes, canonicalized
+    _COLUMNS = ["image_data", "model_name", "nsfw_flag", "real_source", "label"]
+
+    def __init__(self, families_wanted=None, include_reals=True, real_cap=None,
+                 include_fakes=True):
+        self.families_wanted = families_wanted
+        self.include_reals = bool(include_reals)
+        self.include_fakes = bool(include_fakes)     # False = reals-only (additive CF-reals set)
+        # real_cap: per-real-source quota (label 0), enforced in the base pull(). Bounds a
+        # materialized CF PublicEval real pull without a two-pass selection. None = uncapped.
+        self.real_cap = int(real_cap) if real_cap is not None else None
+
+    @staticmethod
+    def _canon(name):
+        """Lowercase + strip non-alphanumerics so 'DALL-E 3'/'Dalle3'/'dalle_3' all -> 'dalle3'."""
+        import re
+        return re.sub(r"[^a-z0-9]", "", str(name or "").lower())
+
+    def _is_held_out(self, model_name):
+        """True if model_name canonicalizes to (or starts with) any held-out family. 'Imagen3'
+        and 'Imagen' both match family 'imagen'; 'Dalle3' matches 'dalle3' but 'Dalle2' does NOT
+        (canon 'dalle2' has no held-out prefix)."""
+        c = self._canon(model_name)
+        return any(c.startswith(h) for h in self._HELD_OUT_CANON)
+
+    def _shards(self):
+        """Yield (path, cleanup) per PublicEval parquet shard, smallest-first (a capped
+        pull fills its quota from the fewest shards). Each shard is DOWNLOADED TO LOCAL
+        DISK first (hf_hub_download: sequential streamed GET with built-in retry/resume),
+        then read locally — NOT read over an fsspec https handle. Parquet's columnar
+        layout makes iter_batches seek around the file; over HTTP that is hundreds of
+        random range reads that time out (FSTimeoutError observed). One sequential
+        download to disk + local batched read is fast, resilient, and still RAM-bounded.
+        cleanup() deletes the shard so peak disk is ONE shard (~3-6GB), not all 27GB.
+        Overridden in tests."""
+        from huggingface_hub import hf_hub_download
+        from concurrent.futures import ThreadPoolExecutor
+        # smallest-first by known size ordering (shard 0 = 3.3GB ... shard 2 = 6.5GB)
+        order = (0, 4, 5, 1, 3, 2)[:self.N_SHARDS]
+
+        def _dl(i):
+            return hf_hub_download(self.HF_REPO,
+                                   f"{self.HF_SPLIT_DIR}/HFCF_PublicEval_{i}.parquet",
+                                   repo_type="dataset")
+
+        # PREFETCH: download shard N+1 in the background while shard N is being read/decoded,
+        # overlapping the multi-minute download with processing. Peak disk = 2 shards (~12GB,
+        # safe) since cleanup deletes shard N before N+2 starts. One prefetch worker (serial
+        # downloads still saturate the HF pipe; more would just contend).
+        with ThreadPoolExecutor(max_workers=1) as pool:
+            fut = pool.submit(_dl, order[0])
+            for k, i in enumerate(order):
+                path = fut.result()                        # current shard (already downloading)
+                if k + 1 < len(order):
+                    fut = pool.submit(_dl, order[k + 1])   # kick off the NEXT while we process
+                def _cleanup(p=path):
+                    try: os.remove(p)
+                    except OSError: pass
+                yield path, _cleanup
+
+    def _iter_examples(self):
+        for path, cleanup in self._shards():
+            try:
+                with open(path, "rb") as f:
+                    for ex in _iter_parquet_rows(f, self._COLUMNS):
+                        if ex.get("nsfw_flag"):
+                            continue                 # NSFW: never materialize
+                        if not self.include_reals and _read_label(ex) == 0:
+                            continue                 # aggregate real license chain unverified
+                        if not self.include_fakes and _read_label(ex) == 1:
+                            continue                 # reals-only (additive CF-reals dataset)
+                        if self._is_held_out(ex.get("model_name")):
+                            continue                 # keep the OOS holdout genuinely unseen
+                        yield ex
+            finally:
+                cleanup()                            # bounded disk: one shard at a time
+    def _to_fields(self, ex):
+        # Record per-image model_name (OpenRAIL-M pass-through). Respect the label field
+        # so any real rows in a safe subset land as label 0.
+        lbl = _read_label(ex)
+        if lbl is None:                         # unresolvable label -> skip (never default fake)
+            return None
+        img = ex.get("image_data") if ex.get("image_data") is not None else ex.get("image")
+        # image_data is the ORIGINAL encoded blob (bytes or {"bytes":...}); pass it through as
+        # `raw` so pull() writes it VERBATIM (no re-encode, no baked corruption, no PNG bloat).
+        raw = img if isinstance(img, (bytes, bytearray)) else (img.get("bytes") if isinstance(img, dict) else None)
+        return {"label": lbl,
+                "generator_id": "none" if lbl == 0 else (ex.get("model_name", "") or ""),
+                "real_source": (ex.get("real_source", "none") or "none") if lbl == 0 else "none",
+                "image": _as_pil(img), "raw": raw}
+
+class SIDSetSource(DatasetSource):
+    name = "sid_set"
+    def __init__(self, n_real, n_fake):
+        self.quotas = (int(n_real), int(n_fake))
+
+    def _iter_examples(self):
+        from datasets import Image as HFImage, load_dataset
+        dataset = load_dataset("saberzl/SID_Set", split="train", streaming=True)
+        dataset = dataset.cast_column("image", HFImage(decode=False))
+        seen = [0, 0]
+        for ex in dataset:
+            lbl = int(ex["label"])
+            if lbl not in (0, 1) or seen[lbl] >= self.quotas[lbl]:
+                continue
+            seen[lbl] += 1
+            yield ex
+            if tuple(seen) == self.quotas:
+                break
+
+    def _to_fields(self, ex):
+        lbl = int(ex["label"])
+        image = ex["image"]
+        raw = image.get("bytes") if isinstance(image, dict) else None
+        return {"label": lbl, "generator_id": "none" if lbl == 0 else "sid_synth",
+                "real_source": ex.get("real_source", "sid"), "image": _as_pil(image),
+                "raw": raw}
+
+# WildFakeSource lives in data.curator.wildfake — it fetches ONLY selected members via
+# RangeZip (range requests over the ModelScope zips), never whole 25-54GB archives, and
+# handles the mapped per-generator taxonomy + §5.4 held-out exclusion. Re-exported here
+# so default_corpus_sources / callers keep one import site.
+from data.provenance.wildfake import WildFakeSource   # noqa: E402,F401
+
+class OmniFakeSource(DatasetSource):
+    """OmniFake (MIT) via the OFFICIAL HF datasets loader: MoeNew/OmniFake (streaming).
+    45 generators + real. It is imagefolder-style: the class/label encodes real-vs-fake
+    and the folder name IS the generator (e.g. FLUX_Dev, DALLE_3, StyleGAN_3, real)."""
+    name = "omnifake"
+    hf_id = "MoeNew/OmniFake"
+    def __init__(self, split="train"): self.split = split
+    def _iter_examples(self):
+        from datasets import load_dataset
+        ds = load_dataset(self.hf_id, split=self.split, streaming=True)
+        # imagefolder datasets expose an int `label` + a `label`->foldername mapping via
+        # features; surface the folder name so classify_family sees the generator string.
+        names = None
+        try:
+            feat = ds.features.get("label")
+            names = getattr(feat, "names", None)
+        except Exception:
+            names = None
+        for ex in ds:
+            if names is not None and isinstance(ex.get("label"), int):
+                ex = dict(ex); ex["label_name"] = names[ex["label"]]
+            yield ex
+    def _to_fields(self, ex):
+        # The folder name is the ground truth for BOTH real/fake and generator.
+        name = ex.get("label_name") or _read_generator(ex)
+        is_real = isinstance(name, str) and name.strip().lower() in ("real", "0_real", "authentic")
+        return {"label": 0 if is_real else 1,
+                "generator_id": "none" if is_real else (name or ""),
+                "real_source": "omnifake" if is_real else "none",
+                "image": _as_pil(_read_image(ex))}
+
+# NOTE: DeepFace_part_v4 was RETIRED (probe finding): it is 15x HiFiMask_*.zip = face
+# ANTI-SPOOFING / presentation-attack data (real faces behind masks), NOT AIGC
+# fully-synthetic-vs-genuine. WildFake's Images/Real/ffhq.zip already covers real faces
+# Apache-clean, so DeepFace added no fitting signal. Do not re-add without a real->fake
+# face-generation source.
+
+class CifakeSource(DatasetSource):
+    """CIFAKE (MIT), aux 32x32. Reads a local CIFAKE directory
+    (birdy654/cifake-real-and-ai-generated-synthetic-images). Label from the
+    FAKE/REAL folder name. No network in _iter_examples — walks the mounted dir."""
+    name = "cifake"
+    def __init__(self, root, split="train"):
+        self.root = root; self.split = split
+    def _iter_examples(self):
+        import glob
+        base = os.path.join(self.root, self.split)
+        # CIFAKE layout: <root>/<split>/{REAL,FAKE}/*.jpg
+        for cls in ("FAKE", "REAL"):
+            for fp in glob.glob(os.path.join(base, cls, "*")):
+                yield {"path": fp, "cls": cls}
+    def _to_fields(self, ex):
+        is_real = ex["cls"].strip().upper() == "REAL"
+        return {"label": 0 if is_real else 1,
+                "generator_id": "none" if is_real else "cifake_synth",
+                "real_source": "cifake" if is_real else "none",
+                "image": _as_pil(ex["path"])}
+
+# NOTE: real images now come from the SAME materialized commercial-safe sources that
+# provide fakes (SID_Set / WildFake / OmniFake carry real rows at label 0; WildFake's
+# Images/Real/{coco,ffhq,celebahq,afhq,church}.zip are the primary diverse-real supply).
+# There is no separate real-pool recipe source — each source's _to_fields sets label +
+# real_source.
+
+# Generators held OUT of training (the OOS holdout axes). Their fakes never enter the
+# training corpus; the holdout build (WildFakeSource(holdout_only=True)) selects them +
+# COCO-val2017 reals. DALLE3 is the spec §5.4 set; GigaGAN (GAN) + Imagen (research
+# diffusion) add two more OOS axes (user decision, 2026-08-29).
+WILDFAKE_HELD_OUT = frozenset({"DALLE3", "GigaGAN", "Imagen"})
+
+def default_corpus_sources(per_family_cap):
+    """Commercial-safe training defaults: aggregate sources contribute fakes only;
+    SID_Set supplies independently capped real and fully-synthetic rows."""
+    return [
+        WildFakeSource(per_gen_cap=per_family_cap, held_out=WILDFAKE_HELD_OUT,
+                       exclude_val2017=True, include_reals=False),
+        CommunityForensicsSource(include_reals=False),
+        SIDSetSource(n_real=per_family_cap, n_fake=per_family_cap),
+    ]
+
+def wildfake_holdout_source(per_gen_cap=3000, spec_exact=False):
+    """The §5.4 OOS holdout: held-out generators' fakes (DALLE3/GigaGAN/Imagen) +
+    COCO-val2017 reals. Materialized separately, NEVER trained on.
+
+    per_gen_cap (default 3000) bounds each generator — without it this pulls ALL held-out
+    images (DALLE3 8843 + GigaGAN 27610 + Imagen 47435 = ~84k), slow and unnecessary for a
+    stable AUC.
+
+    spec_exact=True materializes the MANDATED §5.4 set: DALLE3 = all 8843, COCO-val2017 = all
+    4998 (via per-group overrides = None → take-all), while GigaGAN/Imagen stay bounded at
+    per_gen_cap (they are our bonus OOS axes, not part of §5.4). Use this for the graded number."""
+    overrides = None
+    if spec_exact:
+        # None => take-all for the §5.4-mandated groups; GigaGAN/Imagen fall back to per_gen_cap.
+        overrides = {"DALLE3": None, "coco": None}
+    return WildFakeSource(per_gen_cap=per_gen_cap, held_out=WILDFAKE_HELD_OUT,
+                          exclude_val2017=True, holdout_only=True, per_gen_overrides=overrides)
+
+def official_holdout_source():
+    """The OFFICIAL holdout artifact (spec: sharded-corpus-pipeline): ALL WildFake
+    COCO-val2017 reals + ALL DALLE3 fakes. No quota or cap code path for these two
+    groups (None == take-all); every other held-out group is excluded (0)."""
+    return WildFakeSource(per_gen_cap=None, held_out=WILDFAKE_HELD_OUT,
+                          exclude_val2017=True, holdout_only=True,
+                          per_gen_overrides={"DALLE3": None, "coco": None,
+                                             "GigaGAN": 0, "Imagen": 0})
+
+
+def cf_reals_source(real_cap=None):
+    """ADDITIVE dataset source: Community Forensics PublicEval REALS only (fake rows
+    excluded), bounded per real_source by real_cap. These are reals from the SAME
+    acquisition pipeline as the CF fakes already in the corpus — the fix for the source
+    confound (val real-side AUC 0.618: the detector partly reads SID-pipeline = real).
+    PublicEval is the redistribution-safe split (user-approved 2026-08-31); held-out
+    generators are irrelevant to reals but the filter is harmless. Published as its own
+    dataset (track5-cf-reals) and mounted ALONGSIDE the existing corpus — no rebuild,
+    no overwrite; the training box's RoutingResolver discovers it by shard prefix."""
+    return CommunityForensicsSource(include_reals=True, include_fakes=False,
+                                    real_cap=real_cap, families_wanted=None)
+
+
+def sid_fakes_source(n_fake=2000):
+    """ADDITIVE dataset source: SID_Set FAKES only (n_real=0), tagged generator_id
+    'sid_synth'. The other half of the source-confound fix: putting fakes into the
+    real-dominant SID pipeline breaks 'SID-pipeline = real' (symmetric to cf_reals_source
+    breaking 'CF-pipeline = fake'). This is also the SID-fake scale fix (the live corpus
+    has only 199 SID fakes — the family-cap bug; the per-generator cap fix lands only on a
+    rebuild, but this additive dataset delivers the full 2000 WITHOUT one). Published as
+    track5-sid-fakes, mounted alongside the corpus. SIDSetSource's own quota yields exactly
+    n_fake fakes (no cap_fake_families in the standalone build path)."""
+    return SIDSetSource(n_real=0, n_fake=int(n_fake))
+
+
+def oos_validation_source(n_per_gen=3000):
+    """Separate OOS validation artifact: fixed samples of GigaGAN + Imagen. These are
+    validation data, not official holdout, and never enter training."""
+    return WildFakeSource(per_gen_cap=None, held_out=WILDFAKE_HELD_OUT,
+                          exclude_val2017=True, holdout_only=True,
+                          per_gen_overrides={"GigaGAN": int(n_per_gen),
+                                             "Imagen": int(n_per_gen),
+                                             "DALLE3": 0, "coco": 0})
+
+
+def pull_all(sources, out_dir, per_family_cap=None, seed=0, max_examine=None,
+             selection=None, progress_cb=None) -> pd.DataFrame:
+    # Pass the SAME dataset root to every source; pull() namespaces by source name via
+    # rel_path (<name>/fakes/...), so files land at out_dir/<name>/fakes/... and round-trip.
+    # progress_cb is forwarded only to sources whose pull() accepts it (WildFake); others
+    # ignore it. Kept per-source so long network stages stream live progress to wandb.
+    import inspect
+    parts = []
+    for s in sources:
+        kw = dict(seed=seed, per_family_cap=per_family_cap,
+                  max_examine=max_examine, selection=selection)
+        if progress_cb is not None and "progress_cb" in inspect.signature(s.pull).parameters:
+            kw["progress_cb"] = progress_cb
+        parts.append(s.pull(out_dir, **kw))
+    return pd.concat(parts, ignore_index=True) if parts else pd.DataFrame(columns=RECORD_COLUMNS)

src/data/provenance/splitter.py

@@ -0,0 +1,118 @@
+# src/manifest/splitter.py
+import numpy as np
+import pandas as pd
+
+def assign_split(df, mode, val_frac, group_col, holdout_values, seed):
+    df = df.copy()
+    if mode == "holdout":
+        df["split"] = np.where(df["generator_id"].isin(holdout_values), "val", "train")
+        return df
+    if mode == "grouped":
+        rng = np.random.default_rng(seed)
+        # each group's stratum label = most common label within the group
+        grp_label = df.groupby(group_col)["label"].agg(lambda s: s.mode().iloc[0])
+        val_groups = set()
+        for lbl, groups_in_stratum in grp_label.groupby(grp_label):
+            gvals = np.asarray(groups_in_stratum.index)      # ndarray -> safe to shuffle
+            rng.shuffle(gvals)
+            n = len(gvals)
+            n_val = max(1, int(round(n * val_frac)))
+            if n >= 2:
+                n_val = min(n_val, n - 1)                      # keep >=1 in train per label
+            val_groups.update(gvals[:n_val])
+        df["split"] = np.where(df[group_col].isin(val_groups), "val", "train")
+        return df
+    raise ValueError(f"unknown split mode {mode}")
+
+
+def assign_mixed_split(df, val_frac=0.15, seed=0):
+    """Group-hold out fake families while randomly splitting a single-source real pool.
+
+    SID reals all have ``real_source='sid'``. Treating that one source as a grouped unit
+    would put every real in one split. Fakes still need family-level isolation, while
+    genuine SID rows can be deterministically sampled because there is no second real
+    source whose acquisition signature could leak across the boundary.
+    """
+    if not 0 < val_frac < 1:
+        raise ValueError("val_frac must be between 0 and 1")
+
+    out = df.copy()
+    out["split"] = "train"
+    rng = np.random.default_rng(seed)
+
+    real_indices = out.index[out["label"] == 0].to_numpy(copy=True)
+    rng.shuffle(real_indices)
+    if len(real_indices) >= 2:
+        n_real_val = min(max(1, int(round(len(real_indices) * val_frac))),
+                         len(real_indices) - 1)
+        out.loc[real_indices[:n_real_val], "split"] = "val"
+
+    fake = out[out["label"] == 1].copy()
+    if not fake.empty:
+        fake["_family"] = fake["family"].astype(str).str.strip().str.lower()
+        if fake["_family"].nunique() >= 2:
+            fake = assign_split(fake, mode="grouped", val_frac=val_frac,
+                                group_col="_family", holdout_values=None, seed=seed)
+            out.loc[fake.index, "split"] = fake["split"]
+        elif len(fake) >= 2:
+            fake_indices = fake.index.to_numpy(copy=True)
+            rng.shuffle(fake_indices)
+            n_fake_val = min(max(1, int(round(len(fake_indices) * val_frac))),
+                             len(fake_indices) - 1)
+            out.loc[fake_indices[:n_fake_val], "split"] = "val"
+
+    for label in (0, 1):
+        label_rows = out[out["label"] == label]
+        if len(label_rows) >= 2 and set(label_rows["split"]) != {"train", "val"}:
+            raise RuntimeError(f"mixed split failed to retain label {label} in both splits")
+    return out
+
+
+def assign_hybrid_split(df, grouped_frac=0.1, random_frac=0.1, seed=0):
+    """Per-epoch val = TWO slices measuring different things (user decision):
+      - val_grouped: whole generators/real-sources held out of train (cross-generator
+        generalization signal). group = generator_id for fakes, real_source for reals.
+      - val_random:  a stratified-by-label random slice of the REMAINING rows
+        (in-distribution recall signal).
+    The rest is train. Returns df with split in {train, val_grouped, val_random}.
+    Both val slices carry both classes (stratified); grouped holds whole groups out."""
+    df = df.copy().reset_index(drop=True)
+    rng = np.random.default_rng(seed)
+    # CASE-CANONICAL group key: the same generator under different casing (e.g. WildFake's
+    # "DALLE2" vs CommForensics' "Dalle2") must be ONE group, or the whole-generator holdout
+    # leaks — training on one casing while "holding out" the other is not a genuine unseen
+    # generator. Lowercase the grouping key so such variants collapse to one held-out group.
+    group = df.apply(lambda r: (r["generator_id"] if r["label"] == 1
+                                else r["real_source"]).strip().lower(), axis=1)
+    df["_grp"] = group
+
+    # 1) val_grouped: hold out whole groups, ~grouped_frac of rows, stratified by the
+    #    group's modal label so BOTH classes are represented in the grouped-val slice.
+    grp_label = df.groupby("_grp")["label"].agg(lambda s: s.mode().iloc[0])
+    val_groups = set()
+    for lbl, groups_in_stratum in grp_label.groupby(grp_label):
+        gvals = np.asarray(groups_in_stratum.index)
+        rng.shuffle(gvals)
+        n = len(gvals)
+        n_val = max(1, int(round(n * grouped_frac)))
+        if n >= 2:
+            n_val = min(n_val, n - 1)          # keep >=1 group in train per label
+        val_groups.update(gvals[:n_val])
+    split = np.where(df["_grp"].isin(val_groups), "val_grouped", "train")
+
+    # 2) val_random: from the remaining TRAIN rows, take ~random_frac (of the whole df)
+    #    at random, stratified by label so both classes appear.
+    n_random_target = int(round(len(df) * random_frac))
+    train_idx = np.where(split == "train")[0]
+    if n_random_target > 0 and len(train_idx) > 0:
+        labels = df["label"].to_numpy()
+        picked = []
+        for lbl in (0, 1):
+            pool = train_idx[labels[train_idx] == lbl]
+            k = min(len(pool), max(1, n_random_target // 2))
+            if len(pool):
+                picked.extend(rng.choice(pool, size=k, replace=False))
+        split[np.array(picked, dtype=int)] = "val_random"
+
+    df["split"] = split
+    return df.drop(columns=["_grp"])

src/data/provenance/subsample.py

@@ -0,0 +1,34 @@
+import pandas as pd
+
+
+def cap_fake_families(df, cap, seed):
+    """Cap synthetic rows per GENERATOR while retaining every real row.
+
+    The cap exists to stop any single prolific generator from dominating the fake
+    class. It is keyed on ``generator_id`` — one quota per distinct generator — NOT on
+    the coarse ``family`` label.
+
+    Keying on ``family`` silently conflated every generator that classifies to
+    ``family="unknown"`` (SID's ``sid_synth`` plus the ~20k WildFake/CommForensics rows
+    whose model name matches no family rule) into ONE bucket, so a cap of N kept only N
+    rows *across all of them*. That is exactly how the 2,000 requested SID fakes
+    collapsed to 199 (2000 * 2000 / 22000 ≈ 182). Per-generator keying gives each
+    generator its own quota, preserving generator diversity (the unseen-generator axis
+    the hidden test grades) instead of pooling distinct models.
+
+    Real supply (label 0) is controlled by each licensed source's explicit quota and is
+    never capped here — reals all carry ``generator_id="none"``, so capping them would
+    collapse every genuine image into one bucket and recreate the scarcity this curator
+    is intended to avoid.
+    """
+    if cap is None:
+        return df.copy().reset_index(drop=True)
+    if cap <= 0:
+        raise ValueError("cap must be positive or None")
+
+    parts = [df[df["label"] == 0]]
+    for _gen, group in df[df["label"] == 1].groupby("generator_id", sort=False):
+        if len(group) > cap:
+            group = group.sample(n=cap, random_state=seed)
+        parts.append(group)
+    return pd.concat(parts).sort_index().reset_index(drop=True)

src/data/provenance/val_holdout.py

@@ -0,0 +1,192 @@
+"""Track5 §5.4 Validation/Demo Holdout — a QUARANTINED benchmark.
+
+Spec (techjam spec, §5.4): a fixed subset of WildFake for participants to
+*demonstrate* performance — "Do not use the following data during training."
+Non-AIGC COCO val2017 = 4998 reals, AIGC DALL-E Advanced = 8843 fakes.
+
+Design:
+- The holdout is MATERIALIZED into its own dataset (`wildfake_valset`), separate
+  from the training corpus, and every row is stamped `split="val_holdout"` — a
+  distinct quarantine tag, NOT train/val.
+- The load-bearing deliverable is `assert_no_holdout_in_training`: a hard guard
+  that raises if any holdout row (by split OR by source_dataset) reaches a
+  training manifest, so the demo data can never leak into training.
+
+Same manifest schema as the training corpus (MANIFEST_COLUMNS) so the resolver
+and eval harness operate on it unchanged. Import-safe: the ModelScope SDK import
+is deferred inside WildFakeValSource._iter_examples.
+"""
+import pandas as pd
+from data.provenance.families import classify_family
+from data.provenance.manifest_build import MANIFEST_COLUMNS
+from data.provenance.sources import DatasetSource, RECORD_COLUMNS
+
+# --- Quarantine constants (the whole point: this data never trains) ---------
+VAL_HOLDOUT_SPLIT = "val_holdout"
+WILDFAKE_VALSET_SOURCE = "wildfake_valset"
+DALLE_ADVANCED_GENERATOR = "dalle_advanced"   # classify_family(...) -> "dalle"
+COCO_VAL2017_REAL_SOURCE = "coco_val2017"
+
+# WildFake ModelScope dataset + the two partitions this holdout draws from.
+# RUNTIME-VERIFY: the exact internal partition/subset identifiers below are the
+# spec's human labels, NOT confirmed against WildFake's on-disk parquet schema
+# (needs network). Confirm before the real pull; see WildFakeValSource.
+WILDFAKE_MODELSCOPE_ID = "hy2628982280/WildFake"
+COCO_VAL2017_PARTITION = "COCO_val2017"        # RUNTIME-VERIFY exact name
+DALLE_ADVANCED_PARTITION = "DALLE_Advanced"    # RUNTIME-VERIFY exact name
+
+
+def build_val_holdout_manifest(records: pd.DataFrame) -> pd.DataFrame:
+    """Pure logic: raw holdout records -> a MANIFEST_COLUMNS manifest.
+
+    Normalizes identity columns purely from `label` (0=COCO real, 1=DALL-E fake)
+    so the manifest is correct regardless of what the raw records carried, and
+    stamps every row with the quarantine split.
+    """
+    df = records.copy()
+    is_fake = df["label"].astype(int) == 1
+    df["source_dataset"] = WILDFAKE_VALSET_SOURCE
+    df["generator_id"] = is_fake.map(
+        lambda f: DALLE_ADVANCED_GENERATOR if f else "none")
+    df["real_source"] = is_fake.map(
+        lambda f: "none" if f else COCO_VAL2017_REAL_SOURCE)
+    df["family"] = df["generator_id"].map(classify_family)
+    df["split"] = VAL_HOLDOUT_SPLIT
+    # Backfill any resolution columns a raw record might lack, mirroring
+    # assemble_manifest's defensiveness.
+    if "ref_type" not in df.columns:
+        df["ref_type"] = "materialized"
+    if "mount_slug" not in df.columns:
+        df["mount_slug"] = ""
+    if "rel_path" not in df.columns:
+        df["rel_path"] = df["path"]
+    return df[MANIFEST_COLUMNS]
+
+
+def _holdout_mask(df: pd.DataFrame) -> pd.Series:
+    """Row-wise True where a row is holdout — by split OR by source_dataset.
+    Both signals are checked so a row that lost one tag is still caught."""
+    mask = pd.Series(False, index=df.index)
+    if "split" in df.columns:
+        mask = mask | (df["split"] == VAL_HOLDOUT_SPLIT)
+    if "source_dataset" in df.columns:
+        mask = mask | (df["source_dataset"] == WILDFAKE_VALSET_SOURCE)
+    return mask
+
+
+def assert_no_holdout_in_training(train_manifest_df: pd.DataFrame) -> None:
+    """HARD GUARD — raise ValueError if any quarantined holdout row is present.
+
+    Call this on any manifest destined for training. Detects the holdout by
+    `split == "val_holdout"` OR `source_dataset == "wildfake_valset"`."""
+    mask = _holdout_mask(train_manifest_df)
+    n = int(mask.sum())
+    if n:
+        raise ValueError(
+            f"Train-exclusion guard tripped: {n} quarantined val_holdout row(s) "
+            f"found in a training manifest (split=='{VAL_HOLDOUT_SPLIT}' or "
+            f"source_dataset=='{WILDFAKE_VALSET_SOURCE}'). §5.4 forbids training "
+            f"on the demo holdout.")
+
+
+def filter_out_holdout(df: pd.DataFrame) -> pd.DataFrame:
+    """Defensive: drop any holdout rows, returning a guard-clean training df."""
+    return df[~_holdout_mask(df)].reset_index(drop=True)
+
+
+def drop_holdout_image_leaks(train_df: pd.DataFrame, holdout_df: pd.DataFrame,
+                             threshold: int = 6) -> pd.DataFrame:
+    """IMAGE-level leakage guard (complements the TAG-level assert_no_holdout_in_training).
+
+    TWO passes, cheap-and-exact first:
+    1. EXACT identity: drop any train row whose stable id (img_id, else rel_path) matches a
+       holdout row's. This catches the case both sets draw the SAME source image (e.g. both pull
+       COCO reals from WildFake) — no file I/O, 100% reliable. This is the leak the pHash pass
+       under-caught in practice (95 shared COCO reals survived a threshold-6 pHash sweep because
+       phash requires reading every file, which is slow/fragile at 12k holdout images).
+    2. RE-ENCODE identity: drop any remaining train row whose DCT pHash is within `threshold`
+       (Hamming) of ANY holdout image — catches a re-encoded copy that arrives with clean tags
+       and a different id. Reuses dedup_audit's pHash.
+    """
+    if len(train_df) == 0 or len(holdout_df) == 0:
+        return train_df.reset_index(drop=True)
+
+    def _idcol(df):
+        for c in ("img_id", "rel_path"):
+            if c in df.columns and df[c].notna().any():
+                return df[c].astype(str)
+        return df["path"].astype(str)
+
+    # --- pass 1: exact id match (no file reads) ---
+    hold_ids = set(_idcol(holdout_df))
+    tr_ids = _idcol(train_df)
+    keep_exact = ~tr_ids.isin(hold_ids)
+    df = train_df[keep_exact].reset_index(drop=True)
+    if len(df) == 0:
+        return df
+
+    # --- pass 2: pHash re-encode match on what remains ---
+    from data.provenance.dedup_audit import phash, hamming
+    holdout_hashes = [phash(p) for p in holdout_df["path"]]
+    def _leaks(p):
+        h = phash(p)
+        return any(hamming(h, hh) <= threshold for hh in holdout_hashes)
+    keep = ~df["path"].map(_leaks)
+    return df[keep].reset_index(drop=True)
+
+
+class WildFakeValSource(DatasetSource):
+    """Materializes the §5.4 holdout from WildFake (ModelScope parquet) into its
+    own `wildfake_valset` dataset: COCO val2017 reals (label 0) + DALL-E Advanced
+    fakes (label 1). Heavy ModelScope import is DEFERRED inside _iter_examples
+    for import-safety (this module imports with no network / no SDK present)."""
+    name = WILDFAKE_VALSET_SOURCE
+
+    def __init__(self, coco_partition=COCO_VAL2017_PARTITION,
+                 dalle_partition=DALLE_ADVANCED_PARTITION,
+                 modelscope_id=WILDFAKE_MODELSCOPE_ID):
+        self.coco_partition = coco_partition
+        self.dalle_partition = dalle_partition
+        self.modelscope_id = modelscope_id
+
+    def _iter_examples(self):
+        # DEFERRED heavy import — keeps `import data.provenance.val_holdout` cheap
+        # and network-free (pytest-collection-safe).
+        from modelscope.msdatasets import MsDataset
+        # RUNTIME-VERIFY: WildFake's exact partition/subset names are not
+        # confirmed offline. We tag each example with which partition it came
+        # from so _to_fields can assign the label deterministically. If the
+        # subset_name identifiers below are wrong, this is the single place to
+        # fix — do NOT silently guess elsewhere.
+        for partition, tag in ((self.coco_partition, "real"),
+                               (self.dalle_partition, "fake")):
+            ds = MsDataset.load(self.modelscope_id, subset_name=partition,
+                                split="train")
+            for ex in ds:
+                ex = dict(ex)
+                ex["_holdout_tag"] = tag
+                yield ex
+
+    def _to_fields(self, ex) -> dict:
+        is_fake = ex.get("_holdout_tag") == "fake"
+        return {
+            "label": 1 if is_fake else 0,
+            "generator_id": DALLE_ADVANCED_GENERATOR if is_fake else "none",
+            "real_source": "none" if is_fake else COCO_VAL2017_REAL_SOURCE,
+            "image": ex["image"],
+        }
+
+
+def evaluate_on_holdout(predict_fn, holdout_df, cfg, logger=None) -> dict:
+    """Thin wrapper: run the existing eval harness on the quarantined holdout.
+    No harness logic is duplicated — this only exists to name the intent."""
+    from eval.harness import evaluate
+    return evaluate(predict_fn, holdout_df, cfg, logger=logger)
+
+
+__all__ = [
+    "VAL_HOLDOUT_SPLIT", "WILDFAKE_VALSET_SOURCE", "DALLE_ADVANCED_GENERATOR",
+    "COCO_VAL2017_REAL_SOURCE", "build_val_holdout_manifest",
+    "assert_no_holdout_in_training", "filter_out_holdout", "drop_holdout_image_leaks",
+    "WildFakeValSource", "evaluate_on_holdout", "RECORD_COLUMNS",
+]

src/data/provenance/wildfake.py

@@ -0,0 +1,393 @@
+# src/curator/wildfake.py
+"""WildFake member classification, derived from the mapped taxonomy
+(docs/specs/wildfake_membermap.log). Given a zip's source-relative path and an image
+member's in-zip path, return (label, generator, timing, real_source).
+
+Two member-path shapes exist in WildFake:
+  STRUCTURED: <Family>/<Typical|Advanced>/<Generator>/...  (DALLE, GAN_based, Other_based,
+              personalizedSD, SDwithAdaptor) -> generator = segment after the timing dir.
+  FLAT:       <hash>.png or <ZipStem>/imgs/<hash>.png  (ADM, VQDM, Imagen, Midjourney/SD
+              part zips) -> no per-generator subfolder, so generator = the ZIP's stem and
+              timing comes from the zip PATH (…/Typical|Advanced/… when present).
+Reals live under Images/Real/<src>.zip -> label 0, real_source = zip stem.
+"""
+import os
+
+_TIMINGS = ("Typical", "Advanced")
+
+
+def _zip_stem(zip_rel):
+    """'Images/Diffusion_based/DALLE.zip' -> 'DALLE'; multi-part
+    'Images/Diffusion_based/Midjourney/Advanced/part_1.zip' -> 'Midjourney'."""
+    parts = zip_rel.split("/")
+    # For part-zips the generator is the dir two-or-more levels up (…/Midjourney/Advanced/part_1.zip
+    # -> Midjourney); for single zips it's the filename stem.
+    stem = os.path.splitext(parts[-1])[0]
+    if stem.startswith("part_") or stem.startswith("part"):
+        # walk up past the timing dir to the generator dir
+        for i in range(len(parts) - 2, 0, -1):
+            if parts[i] not in _TIMINGS:
+                return parts[i]
+    return stem
+
+
+def _zip_timing(zip_rel):
+    """Timing from the zip PATH for flat part-zips (…/Advanced/part_1.zip -> 'Advanced')."""
+    for seg in zip_rel.split("/"):
+        if seg in _TIMINGS:
+            return seg
+    return None
+
+
+def classify_member(zip_rel, member):
+    """-> {label, generator, timing, real_source}. Pure string logic (no I/O)."""
+    is_real = "/Real/" in zip_rel or zip_rel.startswith("Images/Real")
+    if is_real:
+        return {"label": 0, "generator": "none", "timing": None,
+                "real_source": _zip_stem(zip_rel)}
+    segs = member.split("/")
+    # STRUCTURED: find a timing segment; generator is the NEXT segment.
+    for i, s in enumerate(segs[:-1]):
+        if s in _TIMINGS:
+            gen = segs[i + 1] if i + 1 < len(segs) else _zip_stem(zip_rel)
+            return {"label": 1, "generator": gen, "timing": s, "real_source": "none"}
+    # FLAT: generator = zip stem; timing from the zip path (part-zips) else None.
+    return {"label": 1, "generator": _zip_stem(zip_rel),
+            "timing": _zip_timing(zip_rel), "real_source": "none"}
+
+
+# --- §5.4 holdout member predicates (for quarantine + dalle3 exclusion) -----------
+def is_val2017_member(member):
+    """COCO val2017 reals (the §5.4 demo reals, 4998)."""
+    m = member.replace("\\", "/")
+    return "/val2017/" in m or m.startswith("coco/coco2017/val2017")
+
+
+def is_dalle3_member(member):
+    """DALLE3 = §5.4 DALL-E-Advanced fakes (8843), the held-out OOS generator."""
+    return "DALLE3" in member or "/Advanced/DALLE3" in member
+
+
+def plan_wildfake_members(zip_to_members, per_gen_cap, held_out=frozenset(),
+                          exclude_val2017=True, seed=0, holdout_only=False, stratify=False,
+                          oversample=0, per_gen_overrides=None, include_reals=True):
+    """PASS-1.5 selection for WildFake: from {zip_rel: [member,...]} choose which members
+    to fetch, PER GENERATOR, honoring held-out exclusion. Pure/offline (no network).
+
+    held_out: generator names kept OUT of training (their members are dropped unless
+              holdout_only, which selects ONLY them + the val2017 reals = the OOS set).
+    exclude_val2017: drop COCO val2017 reals from a training pull (they're §5.4 holdout).
+    stratify: sample the per-generator cap PROPORTIONALLY across the Typical/Advanced
+              timing strata (WildFake distinguishes older vs newer generations), so a
+              generator's quota isn't accidentally drawn all from one era. False = flat
+              uniform sample per generator (original behavior).
+    Returns a list of (zip_rel, member, fields), capped at per_gen_cap per generator.
+    """
+    import random
+    rng = random.Random(seed)
+    # bucket members by generator (fakes)/real_source (reals) AND timing stratum
+    buckets = {}
+    for zip_rel, members in zip_to_members.items():
+        for m in members:
+            c = classify_member(zip_rel, m)
+            if c["label"] == 0 and not include_reals:
+                continue
+            gen = c["generator"] if c["label"] == 1 else c["real_source"]
+            is_ho = (gen in held_out) or is_dalle3_member(m)
+            is_v2017 = is_val2017_member(m)
+            if holdout_only:
+                if not (is_ho or is_v2017):
+                    continue
+            else:
+                if is_ho:
+                    continue
+                if is_v2017 and exclude_val2017:
+                    continue
+                if c["label"] == 0 and not is_v2017 and "val2017" in m:
+                    continue
+            key = (zip_rel, gen, c["label"])
+            strat = c.get("timing") or "none"
+            buckets.setdefault(key, {}).setdefault(strat, []).append((m, c))
+
+    def _emit(zip_rel, gen, label, chosen, plan):
+        for m, c in chosen:
+            plan.append((zip_rel, m, {"label": c["label"],
+                                      "generator_id": "none" if c["label"] == 0 else gen,
+                                      "real_source": gen if c["label"] == 0 else "none"}))
+
+    # OVERSAMPLE: fetch a few EXTRA candidates per generator beyond per_gen_cap so the
+    # save loop can replace undecodable images and still land EXACTLY at the cap (count
+    # stability). oversample=0 preserves original behavior.
+    # per_gen_overrides: {gen_or_real_source: cap} overrides per_gen_cap for named groups —
+    # used for the spec-exact §5.4 holdout (DALLE3 all 8843, coco all 4998, GigaGAN/Imagen
+    # bounded). A None override means "take all" for that group.
+    overrides = per_gen_overrides or {}
+    plan = []
+    for (zip_rel, gen, label), strata in buckets.items():
+        all_items = [it for lst in strata.values() for it in lst]
+        cap = overrides[gen] if gen in overrides else per_gen_cap
+        if cap == 0:
+            continue          # explicit 0 = group EXCLUDED (oversample must not resurrect it)
+        eff_cap = None if cap is None else cap + max(0, oversample)
+        if eff_cap is None or len(all_items) <= eff_cap:
+            _emit(zip_rel, gen, label, all_items, plan)
+            continue
+        if not stratify or len(strata) <= 1:
+            _emit(zip_rel, gen, label, rng.sample(all_items, eff_cap), plan)
+            continue
+        # STRATIFIED: split the cap across timing strata proportionally to their size,
+        # largest-remainder for the leftover, so both eras are represented.
+        total = len(all_items)
+        alloc, rema = {}, []
+        for s, lst in strata.items():
+            exact = eff_cap * len(lst) / total
+            alloc[s] = min(len(lst), int(exact))
+            rema.append((exact - int(exact), s))
+        short = eff_cap - sum(alloc.values())
+        for _frac, s in sorted(rema, reverse=True):
+            if short <= 0:
+                break
+            if alloc[s] < len(strata[s]):
+                alloc[s] += 1; short -= 1
+        for s, k in alloc.items():
+            _emit(zip_rel, gen, label, rng.sample(strata[s], k), plan)
+    return plan
+
+
+class WildFakeSource:
+    """WildFake source over the ModelScope raw zips. BULK-downloads each generator zip
+    whole (one sustained connection — far faster than per-member range requests, which
+    pay a full TLS handshake to ModelScope/China per image), extracts only the SAMPLED
+    members (per-generator cap + held-out exclusion), then DELETES the zip. `download_workers`
+    whole zips run concurrently. scan()/pull() build on plan_wildfake_members; zip_opener/
+    zip_list are injectable for unit tests."""
+    name = "wildfake"
+    modelscope_id = "hy2628982280/WildFake"
+
+    def __init__(self, per_gen_cap=2000, held_out=frozenset(), exclude_val2017=True,
+                 seed=0, holdout_only=False, zip_opener=None, zip_list=None,
+                 download_workers=2, mode="segmented", stratify=True,
+                 seg_workers=16, max_gap=256 * 1024, per_gen_overrides=None,
+                 include_reals=True):
+        self.per_gen_cap = per_gen_cap
+        self.per_gen_overrides = per_gen_overrides or {}
+        self.held_out = frozenset(held_out)
+        self.exclude_val2017 = exclude_val2017
+        self.seed = seed
+        self.holdout_only = holdout_only
+        self.include_reals = bool(include_reals)
+        self._zip_opener = zip_opener
+        self._zip_list = zip_list
+        # download_workers: number of ZIPS processed concurrently.
+        self.download_workers = max(int(download_workers), 1)
+        # seg_workers/max_gap: WITHIN a zip, read_many fetches coalesced segments across
+        # seg_workers threads. Bandwidth probe measured the ModelScope CDN throttles
+        # ~8MB/s PER CONNECTION, scaling to ~97MB/s at 16 conns -> seg_workers=16. max_gap
+        # 256KB (not 1MB) keeps segments smaller/more-numerous so there's enough of them to
+        # actually use 16 workers (over-coalescing into 3 huge segments would cap it at 3).
+        self.seg_workers = max(int(seg_workers), 1)
+        self.max_gap = int(max_gap)
+        # mode: "segmented" (default, FAST) = RangeZip + coalesced parallel range reads.
+        #   "bulk" = download whole zip then extract. stratify: proportional Typical/Advanced.
+        self.mode = mode
+        self.stratify = stratify
+
+    # --- ModelScope wiring (deferred; overridden in tests) ------------------
+    def _list_zips(self):
+        if self._zip_list is not None:
+            return self._zip_list
+        from modelscope.hub.api import HubApi
+        api = HubApi()
+        all_zips = [getattr(f, "path", None) or str(f)
+                    for f in api.list_repo_files(self.modelscope_id, repo_type="dataset")
+                    if (str(getattr(f, "path", f)).startswith("Images/")
+                        and str(getattr(f, "path", f)).lower().endswith(".zip"))]
+        return self._dedupe_parts(all_zips)
+
+    @staticmethod
+    def _dedupe_parts(zips):
+        """Multi-part generator zips (…/Midjourney/Advanced/part_1.zip … part_7.zip) are
+        REDUNDANT splits of ONE generator — part_1 alone holds 30k+ images, far more than
+        any per-gen cap needs. Opening all 14 SD/Midjourney 52GB parts just to parse their
+        central directories is THE build bottleneck (dir-parse is 30s/zip). Keep only the
+        FIRST part per (generator, timing) group; non-part zips pass through untouched."""
+        import os
+        seen_groups, kept = set(), []
+        for z in zips:
+            base = os.path.basename(z)
+            if base.startswith("part_") or base.startswith("part"):
+                group = os.path.dirname(z)          # …/Midjourney/Advanced  (gen+timing)
+                if group in seen_groups:
+                    continue                         # already have a part for this generator
+                seen_groups.add(group)
+            kept.append(z)
+        return kept
+
+    def _open_zip(self, zip_rel):
+        """Return a zip handle exposing .image_members(), .read(name), optionally
+        .read_many(names), .close(). Test opener injected verbatim. Otherwise:
+          segmented -> RangeZip (reads central dir only; read_many coalesces sampled
+                       members into a few contiguous range GETs — no over-download, no
+                       per-image handshake). THE fast path.
+          bulk      -> download the whole zip, extract from disk, delete on close."""
+        if self._zip_opener is not None:
+            return self._zip_opener(zip_rel)
+        from modelscope.hub.api import HubApi
+        api = HubApi()
+        if self.mode == "segmented":
+            from data.provenance.rangezip import RangeZip
+            url = api.get_dataset_file_url(zip_rel, "WildFake", "hy2628982280", "master")
+            rz = RangeZip(url)
+            seg_workers, max_gap = self.seg_workers, self.max_gap
+
+            class _Seg:
+                def image_members(self): return rz.image_members()
+                def read(self, name): return rz.read(name)
+                def read_many(self, names):
+                    # MEASURED (bandwidth probe): ModelScope CDN throttles ~8MB/s PER
+                    # CONNECTION but scales near-linearly to ~16 conns (97MB/s). So we want
+                    # many concurrent range GETs. Small max_gap => more, smaller segments
+                    # => higher effective concurrency (up to seg_workers).
+                    return rz.read_many(names, max_gap=max_gap, workers=seg_workers)
+                def close(self): pass                       # nothing downloaded to delete
+            return _Seg()
+        # bulk fallback
+        import os, zipfile
+        from modelscope import dataset_file_download
+        local = dataset_file_download(dataset_id=self.modelscope_id, file_path=zip_rel)
+        z = zipfile.ZipFile(local)
+
+        class _Handle:
+            def image_members(self):
+                return [n for n in z.namelist()
+                        if n.lower().endswith((".jpg", ".jpeg", ".png", ".webp", ".bmp"))]
+            def read(self, name): return z.read(name)
+            def close(self):
+                z.close()
+                try: os.remove(local)
+                except OSError: pass
+        return _Handle()
+
+    def _members_of(self, handle):
+        return handle.image_members()
+
+    def scan(self, max_examine=None):
+        """Inventory per generator. Downloads each zip to read its member list, then
+        releases it (bounded disk). For a pure-metadata scan without downloads, use the
+        label CSVs (kernel-side); this scan is the source-agnostic fallback."""
+        import pandas as pd
+        counts = {}
+        for zr in self._list_zips():
+            h = self._open_zip(zr)
+            try:
+                for m in h.image_members():
+                    c = classify_member(zr, m)
+                    if c["label"] == 0 and not self.include_reals:
+                        continue
+                    gkey = c["generator"] if c["label"] == 1 else c["real_source"]
+                    counts[(gkey, c["label"])] = counts.get((gkey, c["label"]), 0) + 1
+            finally:
+                h.close()
+        rows = [{"source_dataset": self.name, "group_key": g, "label": l, "count": c}
+                for (g, l), c in counts.items()]
+        return pd.DataFrame(rows, columns=["source_dataset", "group_key", "label", "count"])
+
+    def pull(self, out_dir, seed=0, per_family_cap=None, max_examine=None, selection=None,
+             progress_cb=None, progress_every=200):
+        """BULK per-zip materialization: for each zip, download it whole (one fast
+        connection), plan+extract only the members we sampled (held-out excluded), write
+        JPEGs, then DELETE the zip before the next — peak disk = one zip. INTERLEAVED:
+        the next zip downloads (in a background thread) while the current one extracts,
+        so download and extract overlap and fetching starts within seconds (no upfront
+        all-directories barrier).
+
+        progress_cb: fn(dict) called after each zip with {done,total,imgs_per_s,elapsed_s,
+        zips_done,n_zips,eta_s} so a long build streams live to wandb."""
+        import os, time, pandas as pd
+        from concurrent.futures import ThreadPoolExecutor, as_completed
+        from PIL import Image
+        from data.provenance.sources import RECORD_COLUMNS, _as_pil
+        from data.provenance.manifest_build import jpeg_bytes_per_pixel, save_source_image
+        os.makedirs(out_dir, exist_ok=True)
+
+        zips = self._list_zips()
+        n_zips = len(zips)
+        t0 = time.perf_counter()
+
+        def _process_zip(zr):
+            """Download one whole zip, extract its planned members, delete it. Fully
+            independent -> safe to run many in parallel. Each (generator lives in exactly
+            ONE zip after part-dedup) so per-zip filename counters never collide."""
+            handle = self._open_zip(zr)
+            zip_rows, local_counts = [], {}
+            try:
+                plan = plan_wildfake_members({zr: handle.image_members()},
+                                             per_gen_cap=self.per_gen_cap,
+                                             held_out=self.held_out,
+                                             exclude_val2017=self.exclude_val2017,
+                                             seed=self.seed, holdout_only=self.holdout_only,
+                                             stratify=self.stratify,
+                                             per_gen_overrides=self.per_gen_overrides,
+                                             include_reals=self.include_reals,
+                                             # +5% reserves (min 8) to replace undecodable blobs
+                                             oversample=max(8, (self.per_gen_cap or 0) // 20))
+                # Batch-fetch all sampled members in a FEW coalesced range GETs when the
+                # handle supports it (segmented mode); else read one at a time (bulk/test).
+                want = [m for _zr, m, _f in plan]
+                if hasattr(handle, "read_many") and want:
+                    blobs = handle.read_many(want)
+                else:
+                    blobs = {m: handle.read(m) for m in want}
+                for _zr, m, f in plan:
+                    gkey = f["generator_id"] if f["label"] == 1 else f["real_source"]
+                    # COUNT-STABLE: the plan is OVERSAMPLED (cap + buffer). Stop saving a
+                    # generator once it reaches the true cap — extra candidates are reserves
+                    # that only get used to replace undecodable images below.
+                    if self.per_gen_cap is not None and local_counts.get(gkey, 0) >= self.per_gen_cap:
+                        continue
+                    zt = _zr.replace("/", "_").replace(".zip", "")
+                    # VERBATIM source bytes (no re-encode): JPEG re-encode bakes a scored
+                    # corruption into the 'clean' ground truth; PNG re-encode ~4x's disk (blows
+                    # working-disk limits) for zero fidelity gain. save_source_image writes the
+                    # source blob unchanged with the sniffed extension.
+                    base = f"{self.name}/fakes/{self.name}_{gkey}_{zt}_{local_counts.get(gkey,0)}"
+                    p_noext = os.path.join(out_dir, base)
+                    try:
+                        # A corrupt/undecodable blob must NOT abort the whole run — skip it; a
+                        # reserve candidate takes its slot so the per-generator count stays exact.
+                        p, w, h = save_source_image(blobs[m], p_noext)
+                        rel = os.path.relpath(p, out_dir).replace(os.sep, "/")
+                    except Exception as e:
+                        print(f"[wildfake] skip undecodable {_zr}:{m} ({type(e).__name__}: {e})", flush=True)
+                        import glob as _g
+                        for _f in _g.glob(p_noext + ".*"):   # remove any partial write
+                            try: os.remove(_f)
+                            except OSError: pass
+                        continue
+                    zip_rows.append({"path": p, "label": f["label"], "source_dataset": self.name,
+                                     "generator_id": f["generator_id"], "real_source": f["real_source"],
+                                     "native_w": w, "native_h": h,
+                                     "jpeg_bytes_per_pixel": jpeg_bytes_per_pixel(p),
+                                     "ref_type": "materialized", "mount_slug": "", "rel_path": rel})
+                    local_counts[gkey] = local_counts.get(gkey, 0) + 1
+            finally:
+                handle.close()
+            return zip_rows
+
+        # N whole zips downloaded+extracted CONCURRENTLY (each a sustained connection;
+        # parallel streams saturate total bandwidth). Peak disk ~= download_workers zips.
+        rows, zips_done = [], 0
+        with ThreadPoolExecutor(max_workers=self.download_workers) as ex:
+            futs = {ex.submit(_process_zip, zr): zr for zr in zips}
+            for fut in as_completed(futs):
+                rows.extend(fut.result())
+                zips_done += 1
+                if progress_cb:
+                    dt = time.perf_counter() - t0
+                    progress_cb({"phase": "fetch", "done": len(rows), "zips_done": zips_done,
+                                 "n_zips": n_zips, "imgs_per_s": len(rows) / dt if dt > 0 else 0.0,
+                                 "elapsed_s": dt,
+                                 "eta_s": (n_zips - zips_done) * (dt / zips_done) if zips_done else None})
+                print(f"[wildfake] zip {zips_done}/{n_zips} {futs[fut]} -> {len(rows)} total "
+                      f"imgs ({time.perf_counter()-t0:.0f}s)", flush=True)
+        return pd.DataFrame(rows, columns=RECORD_COLUMNS)

src/data/provenance/writer.py

@@ -0,0 +1,98 @@
+# src/shards/writer.py
+"""Deterministic ZIP64 shard planning + writing (spec: sharded-corpus-pipeline).
+
+Members are ZIP_STORED (images are already compressed), ordered by a stable sort,
+with fixed timestamps/attrs so identical inputs reproduce identical SHA-256 digests.
+write_shard is transactional up to the atomic rename: it writes `<path>.partial`,
+re-opens and verifies (names, order, count, sizes, per-member CRC by reading every
+member), computes the digest, then renames. On any failure only the .partial is
+removed; raw inputs are never touched here (deletion is the orchestrator's job,
+and only AFTER the journal records the verified shard)."""
+import hashlib
+import os
+import zipfile
+
+_FIXED_DT = (1980, 1, 1, 0, 0, 0)
+
+
+def validate_member_name(name: str) -> str:
+    if not name or "\\" in name or name.startswith("/"):
+        raise ValueError(f"unsafe member name: {name!r}")
+    parts = name.split("/")
+    if any(p in ("", ".", "..") for p in parts):
+        raise ValueError(f"unsafe member name: {name!r}")
+    return name
+
+
+def plan_shards(df, target_bytes=1 << 30, prefix="corpus"):
+    """Assign every manifest row to a deterministic shard. Returns a sorted copy with
+    shard_name, member_name, byte_size columns. Close-before-exceed packing; a single
+    oversized member occupies its own shard."""
+    out = df.copy()
+    out["member_name"] = [validate_member_name(r) for r in out["rel_path"]]
+    if out["member_name"].duplicated().any():
+        dupes = out.loc[out["member_name"].duplicated(), "member_name"].head(3).tolist()
+        raise ValueError(f"duplicate member names: {dupes}")
+    ident = out["img_id"] if "img_id" in out.columns else out["rel_path"]
+    out["_grp"] = out["family"].where(out["label"] == 1, out["real_source"])
+    out["_ident"] = ident.astype(str)
+    out = out.sort_values(["source_dataset", "label", "_grp", "_ident", "rel_path"],
+                          kind="mergesort").reset_index(drop=True)
+    # honor a pre-supplied byte_size column (resume path: raws of completed shards are
+    # deleted, their sizes recovered from the shard central directories by the caller)
+    if "byte_size" not in out.columns:
+        out["byte_size"] = [os.path.getsize(p) for p in out["path"]]
+    names, cur_bytes, cur_idx = [], 0, 0
+    for size in out["byte_size"]:
+        if cur_bytes > 0 and cur_bytes + size > target_bytes:
+            cur_idx += 1
+            cur_bytes = 0
+        names.append(f"{prefix}-{cur_idx:05d}.shard")   # .shard NOT .zip: keeps generic file-sync tools from auto-extracting the archives
+        cur_bytes += size
+    out["shard_name"] = names
+    return out.drop(columns=["_grp", "_ident"])
+
+
+def write_shard(zip_path: str, members) -> dict:
+    """Write members [(member_name, src_path)...] to zip_path via .partial + verify +
+    digest + atomic rename. Returns shard record with per-member sizes and CRCs."""
+    partial = zip_path + ".partial"
+    try:
+        with open(partial, "wb") as fh:
+            with zipfile.ZipFile(fh, "w", compression=zipfile.ZIP_STORED,
+                                 allowZip64=True) as zf:
+                for name, src in members:
+                    validate_member_name(name)
+                    zi = zipfile.ZipInfo(name, date_time=_FIXED_DT)
+                    zi.compress_type = zipfile.ZIP_STORED
+                    zi.create_system = 3
+                    zi.external_attr = 0o644 << 16
+                    with open(src, "rb") as f:
+                        zf.writestr(zi, f.read())
+            fh.flush()
+            os.fsync(fh.fileno())
+        # --- verify by re-reading the closed archive, never trusting write state ---
+        rec_members, payload = [], 0
+        with zipfile.ZipFile(partial) as zf:
+            if zf.namelist() != [n for n, _ in members]:
+                raise AssertionError(f"shard member order mismatch in {zip_path}")
+            for name, src in members:
+                info = zf.getinfo(name)
+                data = zf.read(name)              # zipfile CRC-checks on read
+                if len(data) != os.path.getsize(src):
+                    raise AssertionError(f"size mismatch for {name}")
+                rec_members.append({"member_name": name, "byte_size": len(data),
+                                    "crc32": info.CRC})
+                payload += len(data)
+        h = hashlib.sha256()
+        with open(partial, "rb") as f:
+            for chunk in iter(lambda: f.read(1 << 20), b""):
+                h.update(chunk)
+        os.replace(partial, zip_path)             # atomic on same filesystem
+        return {"sha256": h.hexdigest(), "zip_bytes": os.path.getsize(zip_path),
+                "payload_bytes": payload, "n_members": len(rec_members),
+                "members": rec_members}
+    except BaseException:
+        if os.path.exists(partial):
+            os.remove(partial)
+        raise

src/data/shard_reader.py

@@ -0,0 +1,49 @@
+# src/data/shard_reader.py
+"""Read training images directly from mounted ZIP shards — no extraction.
+
+Handles are opened lazily per worker process and kept in a small LRU (default 8).
+The pid check recreates handles after DataLoader fork/spawn: a zipfile handle
+shared across processes corrupts reads. Rows resolve via shard_name + member_name
+from the candidate manifest."""
+import io
+import os
+import zipfile
+from collections import OrderedDict
+
+from PIL import Image
+
+
+class ShardResolver:
+    def __init__(self, shard_dir, max_handles=8):
+        self.shard_dir = shard_dir
+        self.max_handles = int(max_handles)
+        self._handles = OrderedDict()
+        self._pid = os.getpid()
+
+    def _get_zip(self, shard_name):
+        if os.getpid() != self._pid:          # forked/spawned worker: stale handles
+            self._handles = OrderedDict()
+            self._pid = os.getpid()
+        zf = self._handles.get(shard_name)
+        if zf is None:
+            zf = zipfile.ZipFile(os.path.join(self.shard_dir, shard_name))
+            self._handles[shard_name] = zf
+            while len(self._handles) > self.max_handles:
+                _, old = self._handles.popitem(last=False)
+                old.close()
+        else:
+            self._handles.move_to_end(shard_name)
+        return zf
+
+    def resolve(self, row):
+        zf = self._get_zip(row["shard_name"])
+        data = zf.read(row["member_name"])
+        img = Image.open(io.BytesIO(data))
+        img.load()
+        return img.convert("RGB")
+
+    def __getstate__(self):                    # picklable into DataLoader workers
+        return {"shard_dir": self.shard_dir, "max_handles": self.max_handles}
+
+    def __setstate__(self, state):
+        self.__init__(state["shard_dir"], state["max_handles"])

src/eval/error_gallery.py

@@ -0,0 +1,69 @@
+# src/eval/error_gallery.py
+"""Offline error-gallery construction from the predictions parquet.
+
+The predictions table (one row per image x condition, with provenance) is the ONLY
+input the model produces — galleries are built here, offline, AFTER a run, and can be
+rebuilt at any bands without re-running training. Images are pulled on demand from the
+mounted source via SourceResolver (rel_path is the stable address), so nothing is
+saved during the run.
+
+Banding is by PROBABILITY, not a 0.5 threshold (ROC AUC is rank-only, so 0.5 is
+meaningless). Two bands the user asked for, stratified per dataset:
+  - confident_wrong : LABEL-RELATIVE — a fake (label 1) with prob < conf, or a real
+                      (label 0) with prob > 1-conf. The embarrassing, high-confidence misses.
+  - borderline      : prob in [border_lo, border_hi]. The model is genuinely unsure.
+"""
+import pandas as pd
+
+
+def select_gallery_rows(pred_df, n_per=30, conf=0.15, border=(0.4, 0.6),
+                        condition="clean", seed=0):
+    """Return a DataFrame of selected rows with an added `band` column.
+
+    n_per caps rows per (band x source_dataset). Selection is on `condition` (default
+    "clean"). confident_wrong sorted by how wrong (most confident-wrong first);
+    borderline sampled. Deterministic given seed.
+    """
+    df = pred_df[pred_df["condition"] == condition].copy()
+    lo, hi = border
+    is_fake = df["label"] == 1
+    # LABEL-RELATIVE confident-wrong: fake&low OR real&high.
+    cw = df[(is_fake & (df["prob"] < conf)) | (~is_fake & (df["prob"] > 1 - conf))].copy()
+    cw["band"] = "confident_wrong"
+    # "wrongness" = distance of prob from the correct end (1 for fakes, 0 for reals).
+    cw["_wrong"] = (cw["label"] - cw["prob"]).abs()
+    bl = df[(df["prob"] >= lo) & (df["prob"] <= hi)].copy()
+    bl["band"] = "borderline"
+    bl["_wrong"] = (bl["prob"] - 0.5).abs()   # closeness to 0.5 (most-unsure first)
+
+    out = []
+    for band, g, ascending in (("confident_wrong", cw, False), ("borderline", bl, True)):
+        for _src, grp in g.groupby("source_dataset"):
+            # confident_wrong: most-wrong first (desc). borderline: most-unsure first (asc dist).
+            grp = grp.sort_values("_wrong", ascending=ascending).head(n_per)
+            out.append(grp)
+    if not out:
+        return pd.DataFrame(columns=list(pred_df.columns) + ["band"])
+    res = pd.concat(out, ignore_index=True).drop(columns=["_wrong"])
+    return res
+
+
+def render_gallery(selected_df, resolver, out_dir, thumb=256):
+    """Pull each selected image via `resolver` (offline, from the mounted source) and
+    write a 256px thumbnail into out_dir/<band>/<source>/. Filename encodes prob+label so
+    the miss is legible at a glance. Returns {(band,source): count}. Never crashes on a
+    single bad image (logs + skips) — the selection table is the source of truth."""
+    import os
+    counts = {}
+    for _, r in selected_df.iterrows():
+        try:
+            img = resolver.resolve(r)                      # rel_path/source_dataset -> PIL
+            img = img.convert("RGB"); img.thumbnail((thumb, thumb))
+            d = os.path.join(out_dir, r["band"], str(r["source_dataset"]))
+            os.makedirs(d, exist_ok=True)
+            tag = f"L{int(r['label'])}_p{float(r['prob']):.2f}_{os.path.basename(str(r['img_id']))}.jpg"
+            img.save(os.path.join(d, tag), quality=88)
+            k = (r["band"], r["source_dataset"]); counts[k] = counts.get(k, 0) + 1
+        except Exception as e:  # noqa: BLE001 — one bad image must not kill the gallery
+            print(f"[gallery] skip {r.get('img_id')}: {e}")
+    return counts

src/eval/explain.py

@@ -0,0 +1,99 @@
+# src/eval/explain.py
+"""Explainability for the detector — the business-facing "why was this flagged" layer.
+
+The student's AttentionPoolPatchHead already emits a per-patch attention map (the MIL
+pooling weights). We turn it into (a) a spatial heatmap overlay a human moderator can read,
+(b) a compact per-image rationale (verdict + confidence + where the model looked), and
+(c) an aggregate reliance summary (how real/fake separation holds up per corruption). No
+extra forward pass — the attention is a free byproduct of the pooling head.
+
+Why this matters for the business case: a content-moderation / provenance system cannot act
+on a black-box score. A localized "this region looks synthetic" plus a robustness profile is
+what makes the flag actionable and auditable.
+"""
+import numpy as np
+
+
+def attention_heatmap(attn, img_size):
+    """Upsample a per-patch attention vector (length = n_side², a square grid) to a
+    (H, W) heatmap normalized to [0,1]. Nearest-neighbour upsample keeps patch structure."""
+    n = len(attn)
+    side = int(round(np.sqrt(n)))
+    if side * side != n:
+        raise ValueError(f"attention length {n} is not a square patch grid")
+    grid = np.asarray(attn, dtype=float).reshape(side, side)
+    rng = grid.max() - grid.min()
+    grid = (grid - grid.min()) / rng if rng > 0 else np.zeros_like(grid)
+    W, H = img_size
+    ys = (np.arange(H) * side // H).clip(0, side - 1)
+    xs = (np.arange(W) * side // W).clip(0, side - 1)
+    return grid[ys][:, xs]
+
+
+def explain_prediction(prob, attn, img_size, threshold=0.5):
+    """Per-image explanation: verdict, calibrated-ish confidence, attention concentration
+    (focus_fraction = share of attention mass in the top 10% of patches — high means the
+    decision hinges on a localized region), and a short human rationale."""
+    attn = np.asarray(attn, dtype=float)
+    k = max(1, int(round(0.1 * len(attn))))
+    focus_fraction = float(np.sort(attn)[::-1][:k].sum() / (attn.sum() + 1e-9))
+    verdict = "ai-generated" if prob >= threshold else "authentic"
+    confidence = float(abs(prob - 0.5) * 2)      # 0 at the boundary, 1 at the extremes
+    where = ("concentrated in a localized region" if focus_fraction > 0.4
+             else "spread broadly across the image")
+    rationale = (f"Flagged as {verdict} (p={prob:.2f}); the detector's evidence is "
+                 f"{where} (focus={focus_fraction:.0%}).")
+    return {"verdict": verdict, "confidence": confidence,
+            "focus_fraction": focus_fraction, "heatmap": attention_heatmap(attn, img_size),
+            "rationale": rationale}
+
+
+def aggregate_reliance(pred_df):
+    """Per-condition real/fake separation = mean prob(fake rows) − mean prob(real rows).
+    A large separation on clean that DEGRADES under a corruption tells the business exactly
+    which real-world transformation weakens the detector — the actionable robustness story."""
+    summ = {}
+    for cond, g in pred_df.groupby("condition"):
+        fake = g[g["label"] == 1]["prob"]
+        real = g[g["label"] == 0]["prob"]
+        sep = float(fake.mean() - real.mean()) if len(fake) and len(real) else float("nan")
+        summ[cond] = {"separation": sep, "n": int(len(g)),
+                      "fake_mean": float(fake.mean()) if len(fake) else float("nan"),
+                      "real_mean": float(real.mean()) if len(real) else float("nan")}
+    return summ
+
+
+def deletion_faithfulness(predict_fn, img, attn, n_regions=8, seed=0):
+    """Deletion-test faithfulness of an attention explanation.
+
+    An attention map is only a real explanation if the regions it highlights are the ones
+    the model actually USES. We measure this by DELETING (graying out) the top-attention
+    patches and seeing how much the prediction drops, versus deleting random patches. Score
+    = (prob_full − prob_delete_top) − (prob_full − prob_delete_random) = the extra prediction
+    change attributable to removing the ATTENDED region. Higher = more faithful attention.
+    Returns a scalar; positive means attention aligns with the model's causal evidence."""
+    import numpy as np
+    from PIL import Image
+    rng = np.random.default_rng(seed)
+    a = np.asarray(img.convert("RGB")).copy()
+    H, W = a.shape[:2]
+    n = len(attn)
+    side = int(round(np.sqrt(n)))
+    attn = np.asarray(attn, dtype=float)
+
+    def _gray_patches(idxs):
+        out = a.copy()
+        for idx in idxs:
+            r, c = idx // side, idx % side
+            y0, y1 = r * H // side, (r + 1) * H // side
+            x0, x1 = c * W // side, (c + 1) * W // side
+            out[y0:y1, x0:x1] = 128            # neutral gray = "delete"
+        return Image.fromarray(out)
+
+    k = max(1, min(n_regions, n))
+    top_idx = np.argsort(attn)[::-1][:k]
+    rand_idx = rng.choice(n, size=k, replace=False)
+    p_full = float(predict_fn([img])[0])
+    p_top = float(predict_fn([_gray_patches(top_idx)])[0])
+    p_rand = float(predict_fn([_gray_patches(rand_idx)])[0])
+    return (p_full - p_top) - (p_full - p_rand)      # = p_rand - p_top

src/eval/harness.py

@@ -0,0 +1,210 @@
+# src/eval/harness.py
+import numpy as np
+from PIL import Image
+from data.aug.augment import CORRUPTIONS, EVAL_GRID, apply_corruption
+from eval.metrics import roc_auc, aggregate_scores
+from eval.predictions import build_predictions_df
+from eval.stream import plan_eval_batches
+from analysis.format_probe import format_of as _format_of
+
+def evaluate(predict_fn, val_df, cfg, logger=None, corruption_fn=None, resolver=None,
+             watchdog=None) -> dict:
+    """Streaming bounded-memory evaluation over clean + the 6-corruption grid.
+
+    corruption_fn(img, name, severity)->PIL lets the caller SWAP the corruption
+    implementation (default: the canonical single-impl `apply_corruption`). Passing an
+    implementation-holdout function (see evaluate_impl_holdout) measures AUC under
+    operators NOT seen during training — the corruption-implementation-invariance result."""
+    corrupt = corruption_fn or apply_corruption
+    labels = val_df["label"].to_numpy()
+    # sharded rows (ref_type='shard') carry no usable path: resolve via the resolver
+    shard_rows = None
+    if resolver is not None and "shard_name" in val_df.columns:
+        shard_rows = val_df.reset_index(drop=True).to_dict("records")
+    paths = val_df["path"].tolist() if "path" in val_df.columns else [""] * len(val_df)
+    n = len(paths)
+    # img_id column if present, else fall back to path (aligned to `paths` order).
+    if "img_id" in val_df:
+        img_ids = val_df["img_id"].tolist()
+    else:
+        img_ids = list(paths)
+    # Provenance for offline error-gallery stratification + reconstruction. rel_path is
+    # the STABLE source address (source_dataset + rel_path -> SourceResolver re-fetches
+    # the image); image_path is an ephemeral /tmp path. Absent columns default to "".
+    def _col(name):
+        return val_df[name].tolist() if name in val_df else [""] * n
+    source_datasets = _col("source_dataset")
+    generator_ids = _col("generator_id")
+    rel_paths = _col("rel_path")
+    # native_w + format carry the codec/size axes so the format-shortcut probe (the decisive
+    # holdout test: JPEG-only, size-matched AUC) is derivable OFFLINE from the predictions table.
+    # native_w defaults to NaN; format is derived from the path extension when the column is absent.
+    native_ws = val_df["native_w"].tolist() if "native_w" in val_df else [float("nan")] * n
+
+    # The 19 eval conditions, index 0 = clean. condition_idx -> (name, severity, key).
+    conditions = [("clean", None, "clean")]
+    for name in CORRUPTIONS:
+        for sev in EVAL_GRID[name]:
+            conditions.append((name, sev, f"{name}@{sev}"))
+    n_cond = len(conditions)
+
+    # BOUNDED-MEMORY STREAMING (fixes host-RAM OOM): the old harness decoded EVERY val
+    # image into a list held across all conditions (+a full copy per condition) — tens
+    # of GB on a large val set. Instead we stream (image x condition) work through
+    # plan_eval_batches under a hard cap of `pool_max` decoded images resident at once;
+    # each image is decoded on first use and evicted the moment its last condition is
+    # scored. Memory is O(pool_max), independent of val-set size.
+    pool_max = max(getattr(cfg, "eval_pool_max", 64), 1)
+    batch = max(getattr(cfg, "eval_batch_size", 32), 1)
+    prob_mat = np.full((n_cond, n), np.nan, dtype=float)   # [condition][image]
+    cache = {}          # item_idx -> decoded PIL.Image (at most pool_max live)
+    done = np.zeros(n, dtype=int)   # conditions scored per item -> evict at n_cond
+
+    # FAIR EVAL: normalize real+fake through an identical resize+recompress so absolute
+    # resolution/format can't be the discriminator (the holdout's reals were 200px JPEG, fakes
+    # large PNG -> resolution-only AUC 1.0). Opt-in; any AUC after this is real detection signal.
+    fair = getattr(cfg, "fair_eval", False)
+    fair_res = getattr(cfg, "resolution", 518)
+    # PARALLEL CPU PREP (cfg.eval_workers>0): decode + corruption are PIL C calls that
+    # release the GIL, so a THREAD pool parallelizes them for real — the serial loop left
+    # 29 of 30 cores idle through every eval phase (GPU util 0%, the v12 4h stall's true
+    # anatomy). Decode stays serialized per item (cache coherence); corruption fans out.
+    eval_workers = int(getattr(cfg, "eval_workers", 0))
+    pool = None
+    if eval_workers > 0:
+        from concurrent.futures import ThreadPoolExecutor
+        pool = ThreadPoolExecutor(max_workers=eval_workers)
+
+    def _decode(item):
+        if item not in cache:
+            if resolver is not None and shard_rows is not None:
+                im = resolver.resolve(shard_rows[item]).convert("RGB")
+            else:
+                with Image.open(paths[item]) as im2:
+                    im = im2.convert("RGB")
+            if fair:
+                from data.preprocess import fair_eval_normalize
+                im = fair_eval_normalize(im, fair_res)
+            cache[item] = im
+        return cache[item]
+
+    def _make(item_cond):
+        item, cond = item_cond
+        name, sev, _ = conditions[cond]
+        base_im = cache[item]
+        return base_im if name == "clean" else corrupt(base_im, name, sev)
+
+    # MID-EVAL PROGRESS: eval phases used to be silent for up to hours (GPU idle, no
+    # output — indistinguishable from a hang). Log every ~30s: cells done/total, rate,
+    # ETA — to stdout (run.log) and, when a logger is given, to wandb as eval/progress.
+    import time as _time
+    total_cells = n * n_cond
+    cells_done = 0
+    _t0 = _time.time()
+    _t_last = _t0
+    for bat in plan_eval_batches(n, n_cond, pool_max, batch):
+        for item, _cond in bat:
+            _decode(item)                     # serial: fills cache once per item
+        coords = list(bat)
+        if pool is not None:
+            imgs = list(pool.map(_make, coords))
+        else:
+            imgs = [_make(ic) for ic in coords]
+        probs = predict_fn(imgs)
+        cells_done += len(coords)
+        if watchdog is not None:
+            watchdog.update(cells_done=cells_done, cells_total=total_cells)
+        if _time.time() - _t_last >= 30:
+            _t_last = _time.time()
+            rate = cells_done / max(_time.time() - _t0, 1e-9)
+            eta = (total_cells - cells_done) / max(rate, 1e-9)
+            print(f"[eval] {cells_done}/{total_cells} cells "
+                  f"({100*cells_done/total_cells:.0f}%) | {rate:.0f} cells/s | "
+                  f"eta {eta/60:.1f} min", flush=True)
+            if logger is not None:
+                try:
+                    logger.log({"eval/progress_frac": cells_done / total_cells,
+                                "eval/cells_per_s": rate})
+                except Exception:  # noqa: BLE001 - progress logging never breaks eval
+                    pass
+        for (item, cond), p in zip(coords, probs):
+            prob_mat[cond, item] = float(p)
+            done[item] += 1
+            if done[item] >= n_cond:
+                cache.pop(item, None)      # last condition scored -> free the image
+    if pool is not None:
+        pool.shutdown(wait=False)
+
+    # Assemble per-condition AUCs + the long-format predictions table from prob_mat.
+    pred_records = []
+    per = {}
+    for cond, (name, sev, key) in enumerate(conditions):
+        col = prob_mat[cond]
+        per[key] = roc_auc(labels, col)
+        for i in range(n):
+            pred_records.append({
+                "img_id": img_ids[i], "image_path": paths[i],
+                "source_dataset": source_datasets[i], "generator_id": generator_ids[i],
+                "rel_path": rel_paths[i], "native_w": native_ws[i],
+                "format": _format_of(paths[i]), "label": int(labels[i]),
+                "condition": name, "severity": sev, "prob": float(col[i]),
+            })
+
+    agg = aggregate_scores(per)
+    predictions = build_predictions_df(pred_records)
+    result = {"per_condition": per, **agg, "predictions": predictions}
+
+    # Attention-map diagnostics are intentionally skipped: predict_fn only returns
+    # sigmoid probs (attn is discarded), so measuring attention peakedness would
+    # require a second forward pass / harness refactor. Not worth it — see report.
+    if logger is not None:
+        metrics = {f"eval/{k}": v for k, v in per.items()}   # e.g. eval/jpeg@30
+        metrics.update({
+            "eval/auc_clean": agg["auc_clean"],
+            "eval/auc_robust_mean": agg["auc_robust_mean"],
+            "eval/auc_robust_worst": agg["auc_robust_worst"],
+            "eval/final": agg["final"],
+        })
+        logger.log(metrics)
+
+    return result
+
+
+def _impl_corruption_fn(impl_choice, seed=0):
+    """Build a corruption_fn(img,name,sev) that applies a SPECIFIC implementation per
+    family. impl_choice maps family -> list of allowed impl names; one is chosen
+    deterministically per (family, severity) so the eval is reproducible."""
+    import random
+    from data.aug.augment_impls import apply_impl, list_impls
+    def fn(img, name, sev):
+        allowed = impl_choice.get(name) or list_impls(name)
+        # deterministic pick within the allowed set, varied by severity
+        r = random.Random(f"{name}:{sev}:{seed}")
+        impl = allowed[r.randrange(len(allowed))]
+        return apply_impl(img, name, impl, sev)
+    return fn
+
+
+def evaluate_impl_holdout(predict_fn, val_df, cfg, train_impls, test_impls,
+                          logger=None, seed=0, resolver=None) -> dict:
+    """Corruption-IMPLEMENTATION-invariance measurement: evaluate the SAME model under
+    (a) the training implementations and (b) the held-out implementations, and report the
+    AUC gap. A small gap = the detector generalizes across operators (did NOT overfit the
+    grader's exact JPEG/resize/blur); a large gap = operator overfit.
+
+    Returns {train_impl: <evaluate result>, test_impl: <evaluate result>, gap: {...}}.
+    train_impls / test_impls are {family: [impl_name,...]} from split_impls_train_test."""
+    seen = evaluate(predict_fn, val_df, cfg, logger=None,
+                    corruption_fn=_impl_corruption_fn(train_impls, seed), resolver=resolver)
+    held = evaluate(predict_fn, val_df, cfg, logger=None,
+                    corruption_fn=_impl_corruption_fn(test_impls, seed), resolver=resolver)
+    gap = {
+        "auc_robust_mean_seen": seen["auc_robust_mean"],
+        "auc_robust_mean_held": held["auc_robust_mean"],
+        "robust_mean_gap": seen["auc_robust_mean"] - held["auc_robust_mean"],
+        "final_seen": seen["final"], "final_held": held["final"],
+        "final_gap": seen["final"] - held["final"],
+    }
+    if logger is not None:
+        logger.log({f"implholdout/{k}": v for k, v in gap.items()})
+    return {"train_impl": seen, "test_impl": held, "gap": gap}

src/eval/metrics.py

@@ -0,0 +1,58 @@
+# src/eval/metrics.py
+import numpy as np
+from sklearn.metrics import roc_auc_score
+
+def roc_auc(labels: np.ndarray, probs: np.ndarray) -> float:
+    labels = np.asarray(labels); probs = np.asarray(probs)
+    if len(np.unique(labels)) < 2:
+        return float("nan")
+    return float(roc_auc_score(labels, probs))
+
+def stratified_auc(df, by="generator_id", prob_col="prob", label_col="label"):
+    """Per-group AUC: each FAKE group (e.g. one generator) scored vs the SHARED real pool.
+    Reveals which generators the model separates well — logged live to wandb per epoch.
+
+    df must carry label/prob/`by`. Reals (label 0) form one pool reused for every group.
+    Groups with <2 classes (no reals, or the group has no fakes) are skipped/NaN."""
+    import numpy as np
+    reals = df[df[label_col] == 0]
+    if len(reals) == 0:
+        return {}
+    real_labels = reals[label_col].to_numpy()
+    real_probs = reals[prob_col].to_numpy()
+    out = {}
+    for gval, grp in df[df[label_col] == 1].groupby(by):
+        labels = np.concatenate([real_labels, grp[label_col].to_numpy()])
+        probs = np.concatenate([real_probs, grp[prob_col].to_numpy()])
+        out[str(gval)] = roc_auc(labels, probs)
+    return out
+
+
+def aggregate_scores(per_condition: dict[str, float]) -> dict:
+    """Aggregate per-condition AUCs into the competition score.
+
+    The organizer's metric is mean over the SIX corruption FAMILIES (jpeg/blur/resize/noise/
+    jitter/crop), each family weighted EQUALLY. Our eval grid probes several severities per
+    family (jpeg@30/50/70/90, ...), so a naive mean over all severity cells over-weights
+    families with more severities (jpeg had 4 votes, jitter/crop 1). We therefore average the
+    severities WITHIN each family first (family AUC), then take the mean over families — so the
+    number matches the score that will actually be computed. `robust_worst` is the worst FAMILY.
+    Condition keys are 'family@severity' (or a bare 'family' with no '@')."""
+    clean = per_condition["clean"]
+    families: dict[str, list] = {}
+    for k, v in per_condition.items():
+        if k == "clean":
+            continue
+        fam = k.split("@", 1)[0]                 # 'jpeg@30' -> 'jpeg'; 'blur' -> 'blur'
+        families.setdefault(fam, []).append(v)
+    family_auc = {fam: float(np.nanmean(vs)) for fam, vs in families.items()}
+    fam_vals = list(family_auc.values())
+    robust_mean = float(np.mean(fam_vals)) if fam_vals else float("nan")
+    robust_worst = float(np.min(fam_vals)) if fam_vals else float("nan")
+    return {
+        "auc_clean": clean,
+        "auc_robust_mean": robust_mean,
+        "auc_robust_worst": robust_worst,
+        "per_family_auc": family_auc,
+        "final": 0.5 * clean + 0.5 * robust_mean,
+    }

src/eval/predictions.py

@@ -0,0 +1,51 @@
+# src/eval/predictions.py
+"""Per-image prediction table for offline error analysis.
+
+The eval harness already computes per-image sigmoid probabilities under every
+(clean + corruption) condition and, historically, discarded them after folding
+into roc_auc(). This module captures those probs into a tidy long-format table
+so precision/recall and TP/FP/TN/FN sampling can be derived OFFLINE at ANY
+threshold — nothing here recomputes a forward pass.
+
+Schema (one row per image x condition), columns in this EXACT order:
+    img_id, image_path, source_dataset, generator_id, rel_path,
+    label, condition, severity, prob
+  - source_dataset/generator_id: provenance, so galleries stratify by dataset/generator
+  - rel_path : the STABLE source-relative address (source_dataset + rel_path re-addresses
+               the image via SourceResolver offline; image_path is an ephemeral /tmp path)
+  - condition: "clean" or a corruption name (jpeg/blur/resize/noise/jitter/crop)
+  - severity : None for clean, else the eval-grid severity value
+  - prob     : sigmoid prob in [0,1] for that image under that condition
+"""
+import pandas as pd
+
+COLUMNS = ["img_id", "image_path", "source_dataset", "generator_id", "rel_path",
+           "native_w", "format", "label", "condition", "severity", "prob"]
+
+
+def build_predictions_df(records) -> pd.DataFrame:
+    """Build the predictions DataFrame from an iterable of row dicts.
+
+    Each record must carry the COLUMNS keys. Column order is enforced so the
+    downstream (label, prob)-at-any-threshold contract is stable.
+    """
+    df = pd.DataFrame(list(records), columns=COLUMNS)
+    return df
+
+
+def save_predictions(df: pd.DataFrame, out_dir, prefix="") -> dict[str, str]:
+    """Write {prefix}predictions.parquet AND {prefix}predictions.csv into out_dir.
+
+    Returns {"parquet": <path>, "csv": <path>}. Both formats are written: parquet
+    for typed/compact offline analysis, csv for eyeballing / spreadsheet use.
+    `prefix` (e.g. 'base_' / 'fuse4scalar_') keeps per-arm artifacts SEPARATE — the
+    v10 sequential A/B overwrote the base arm's files, destroying the paired oracle.
+    """
+    import os
+
+    os.makedirs(out_dir, exist_ok=True)
+    parquet_path = os.path.join(out_dir, f"{prefix}predictions.parquet")
+    csv_path = os.path.join(out_dir, f"{prefix}predictions.csv")
+    df.to_parquet(parquet_path, index=False)
+    df.to_csv(csv_path, index=False)
+    return {"parquet": parquet_path, "csv": csv_path}