"""Which Online-Mind2Web start sites load for our headless browser (vs. an anti-bot wall / access denied)?
    python finetune/probe_sites.py [out.json]"""
import json, os, re, sys, time
sys.path.insert(0, "/home/ckl/projects/S/jev-ultrafast"); sys.path.insert(0, "/home/ckl/projects/S/laya/apps")
import browser_suite  # noqa: F401
from browser_suite_om2w import held_out_tasks
from jev_ultrafast.browser import Browser
WALL = re.compile(r"access denied|just a moment|verify you are human|are you a robot|captcha|security verification|"
                  r"request blocked|forbidden|unusual traffic|pardon our interruption|press & hold|not available in your (country|region)", re.I)
sites = sorted({t["website"] for t in held_out_tasks()})
res = {}
for u in sites:
    try:
        b = Browser(u); time.sleep(5)
        p = b.observe(screenshot=False); b.close()
        blocked = bool(WALL.search(p["title"] + " " + p["text"][:600])) or len(p["actions"]) < 5
        res[u] = {"ok": not blocked, "title": p["title"][:60], "n_actions": len(p["actions"])}
    except Exception as e:
        res[u] = {"ok": False, "title": f"error {type(e).__name__}", "n_actions": 0}
    print(("OK   " if res[u]["ok"] else "WALL ") + u[:50], "|", res[u]["title"], flush=True)
json.dump(res, open(sys.argv[1] if len(sys.argv) > 1 else "finetune/out/om2w_sites.json", "w"), indent=1)
print(f"== {sum(r['ok'] for r in res.values())}/{len(res)} sites usable")