| """Which Online-Mind2Web start sites load for our headless browser (vs. an anti-bot wall / access denied)? |
| python finetune/probe_sites.py [out.json]""" |
| import json, os, re, sys, time |
| sys.path.insert(0, "/home/ckl/projects/S/jev-ultrafast"); sys.path.insert(0, "/home/ckl/projects/S/laya/apps") |
| import browser_suite |
| from browser_suite_om2w import held_out_tasks |
| from jev_ultrafast.browser import Browser |
| WALL = re.compile(r"access denied|just a moment|verify you are human|are you a robot|captcha|security verification|" |
| r"request blocked|forbidden|unusual traffic|pardon our interruption|press & hold|not available in your (country|region)", re.I) |
| sites = sorted({t["website"] for t in held_out_tasks()}) |
| res = {} |
| for u in sites: |
| try: |
| b = Browser(u); time.sleep(5) |
| p = b.observe(screenshot=False); b.close() |
| blocked = bool(WALL.search(p["title"] + " " + p["text"][:600])) or len(p["actions"]) < 5 |
| res[u] = {"ok": not blocked, "title": p["title"][:60], "n_actions": len(p["actions"])} |
| except Exception as e: |
| res[u] = {"ok": False, "title": f"error {type(e).__name__}", "n_actions": 0} |
| print(("OK " if res[u]["ok"] else "WALL ") + u[:50], "|", res[u]["title"], flush=True) |
| json.dump(res, open(sys.argv[1] if len(sys.argv) > 1 else "finetune/out/om2w_sites.json", "w"), indent=1) |
| print(f"== {sum(r['ok'] for r in res.values())}/{len(res)} sites usable") |
|
|
