"""Build the 'unique activity' test set: one held-out-template row per action, sampled from the model test split. Prefers clean (no typo) variants and alternates Hinglish / English so the report reads naturally.""" import json, random, sys, os, collections HERE = os.path.dirname(os.path.abspath(__file__)); ROOT = os.path.dirname(HERE) sys.path.insert(0, os.path.join(ROOT, "model")) from catalog import ACTIONS # noqa random.seed(2026) src = sys.argv[1] if len(sys.argv) > 1 else os.path.join(ROOT, "..", "union_model", "data", "test.jsonl") rows = [json.loads(l) for l in open(src)] by = collections.defaultdict(list) for r in rows: by[r["action"]].append(r) out = [] for a in ACTIONS: cand = by.get(a, []) if not cand: continue clean = [r for r in cand if "typo" not in r.get("variant", "") and len(r["text"]) < 90] or cand hi = [r for r in clean if any(w in r["text"].lower().split() for w in ("karo", "kar", "do", "batao", "dikhao", "kholo", "ka", "ki", "ke", "pe", "se", "hai", "kya"))] en = [r for r in clean if r not in hi] pick = random.choice(hi if (len(out) % 2 == 0 and hi) else (en or hi or clean)) dev = "pi" if ACTIONS[a]["platforms"] == ["pi"] else "windows" if ACTIONS[a]["platforms"] == ["windows"] else "both" out.append({"text": pick["text"], "action": a, "target": pick["target"], "target_raw": pick.get("target_raw", pick["target"]), "device": dev, "risk": ACTIONS[a]["risk"], "group": pick["group"]}) with open(os.path.join(HERE, "curated_prompts.jsonl"), "w", encoding="utf-8") as f: for r in out: f.write(json.dumps(r, ensure_ascii=False) + "\n") print("curated prompts:", len(out), "actions covered", len({r['action'] for r in out}), "of", len(ACTIONS))