| """Keep only webgym episodes that reached a verified task_done (same filter as run_v17s.sh applies to gym_raw2): |
| finetune/out/gym_raw3/g_<kind>_<seed0>.jsonl -> finetune/out/gym/clean3_<kind>_<seed0>.jsonl |
| Prints per-file and per-kind episode / case counts.""" |
| import json, glob, os, collections |
| O = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "out") |
| tot = collections.defaultdict(lambda: [0, 0, 0]) |
| for f in sorted(glob.glob(f"{O}/gym_raw3/g_*.jsonl")): |
| rows = [json.loads(l) for l in open(f)] |
| good = {(r["gym_kind"], r["seed"]) for r in rows if r["skill"] == "task_done"} |
| keep = [r for r in rows if (r["gym_kind"], r["seed"]) in good] |
| out = f.replace("gym_raw3/g_", "gym/clean3_") |
| open(out, "w").write("".join(json.dumps(r, ensure_ascii=False) + "\n" for r in keep)) |
| kind = os.path.basename(f).split("_")[1] |
| n_eps = len({r["seed"] for r in rows}) |
| print(f"{os.path.basename(out):32s} episodes {len(good):4d}/{n_eps:<4d} cases {len(keep):6d}") |
| t = tot[kind]; t[0] += len(good); t[1] += n_eps; t[2] += len(keep) |
| for k, (g, n, c) in tot.items(): |
| print(f"== {k:11s} clean episodes {g}/{n} cases {c}") |
|
|
