File size: 1,441 Bytes
454b3e6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 | """Which Online-Mind2Web start sites load for our headless browser (vs. an anti-bot wall / access denied)?
python finetune/probe_sites.py [out.json]"""
import json, os, re, sys, time
sys.path.insert(0, "/home/ckl/projects/S/jev-ultrafast"); sys.path.insert(0, "/home/ckl/projects/S/laya/apps")
import browser_suite # noqa: F401
from browser_suite_om2w import held_out_tasks
from jev_ultrafast.browser import Browser
WALL = re.compile(r"access denied|just a moment|verify you are human|are you a robot|captcha|security verification|"
r"request blocked|forbidden|unusual traffic|pardon our interruption|press & hold|not available in your (country|region)", re.I)
sites = sorted({t["website"] for t in held_out_tasks()})
res = {}
for u in sites:
try:
b = Browser(u); time.sleep(5)
p = b.observe(screenshot=False); b.close()
blocked = bool(WALL.search(p["title"] + " " + p["text"][:600])) or len(p["actions"]) < 5
res[u] = {"ok": not blocked, "title": p["title"][:60], "n_actions": len(p["actions"])}
except Exception as e:
res[u] = {"ok": False, "title": f"error {type(e).__name__}", "n_actions": 0}
print(("OK " if res[u]["ok"] else "WALL ") + u[:50], "|", res[u]["title"], flush=True)
json.dump(res, open(sys.argv[1] if len(sys.argv) > 1 else "finetune/out/om2w_sites.json", "w"), indent=1)
print(f"== {sum(r['ok'] for r in res.values())}/{len(res)} sites usable")
|