code/apps/browser_diag.py

"""Direct diagnostic: can laya pick the right element on a real Wikipedia element table, under different prompt formats?"""
import json, os, random, sys, time
import httpx
sys.path.insert(0, "/home/ckl/projects/S/jev-ultrafast")
os.environ["BU_CDP_URL"] = "http://127.0.0.1:9222"
from jev_ultrafast.browser import Browser
from jev_ultrafast.model import action_space
from jev_ultrafast.questions import NEXT_ACTION, TARGET

S1 = "http://127.0.0.1:8791/v1/systemone"
page = None
def get_page():
    global page
    if page is None:
        b = Browser("https://en.wikipedia.org/wiki/Main_Page"); page = b.observe(screenshot=False); b.close()
    return page

GOALS = [("Click the 'Random article' link in the navigation.", "Random article"),
         ("Open the site's search box so a query can be typed.", "Search"),
         ("Log in to Wikipedia.", "Log in"),
         ("Open the Talk page for the main page.", "Talk"),
         ("Read the community portal.", "Community portal")]

def ask(state, questions):
    r = httpx.post(S1, json={"model": "x", "state": state, "questions": questions}, timeout=120).json()
    return r["answers"]

def run(fmt, K=1):
    p = get_page(); elements, targets, controls = action_space(p["actions"])
    cands = targets["CLICK"]
    hits, ranks = 0, []
    for goal, want in GOALS:
        gold = [i for i, a in cands.items() if a["label"].split(" → ")[0].strip().lower() == want.lower()]
        if not gold: print("  no gold for", want); continue
        if fmt == "jev":
            crit = {i: {"element": f"[{i}] {a['label']}", "current_value": a.get("current_value", a.get("value", "")), **{k: a[k] for k in ("role", "checked", "selected", "expanded") if k in a}} for i, a in cands.items()}
            state = {"page": {k: p[k] for k in ("url", "title", "text")}, "elements": elements, "recent_actions": []}
            ins = {"goal": goal, "operation": "CLICK", "rules": [NEXT_ACTION, TARGET]}
        else:
            crit = {i: a["label"].split(" → ")[0][:60] for i, a in cands.items()}
            state = {"goal": goal, "page_title": p["title"], "url": p["url"]}
            ins = f"Goal: {goal}  Which element should be clicked next?"
        probs = {i: 0.0 for i in crit}
        for k in range(K):
            keys = list(crit); random.Random(k).shuffle(keys) if K > 1 else None
            a = ask(state, {"t": {"type": "choice", "instructions": ins, "criteria": {i: crit[i] for i in keys}}})["t"]
            for i, v in a["probabilities"].items(): probs[i] += v / K
        order = sorted(probs, key=probs.get, reverse=True)
        rank = min(order.index(g) for g in gold) + 1
        ranks.append(rank); hits += rank == 1
        print(f"  {want:18s} gold={gold} top={order[:3]} p_gold={max(probs[g] for g in gold):.3f} rank={rank}")
    print(f"  => top1 {hits}/{len(ranks)}  mean rank {sum(ranks)/len(ranks):.1f} of {len(cands)}")

fmt = sys.argv[1]; K = int(sys.argv[2]) if len(sys.argv) > 2 else 1
print(f"== format={fmt} permutations={K}  variant={httpx.get('http://127.0.0.1:8791/').json()['variant']}")
run(fmt, K)