"""DAgger in webgym: the current laya model drives, the scripted expert labels every state the MODEL visits. python finetune/webgym/dagger_gym.py [seed0=500001] [kinds=flight,hotel,shop] [beta=0.3] Needs: headless Chromium (BU_CDP_URL), the webgym server (:8811) and the laya systemone server on TYPESAFE_BASE_URL (default :8791) serving the policy. No text model is needed: when the policy types, the text comes from the task spec. Per step: the expert's gold action for the current state is recorded (full goal + current sub-goal, and sub-goal DONE at transitions), then with prob. 1-beta the MODEL's action is executed (else the expert's). A wrong DONE / BLOCKED from the model is recorded and replaced by the expert's action. Cases carry source "webgym_dagger" and `agree` (model == expert). """ import json, os, random, sys, time sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, "/home/ckl/projects/S/jev-ultrafast") os.environ.setdefault("TYPESAFE_BASE_URL", "http://127.0.0.1:8791"); os.environ.setdefault("TYPESAFE_API_KEY", "local") import expert as E import spec as S from jev_ultrafast.browser import Browser from jev_ultrafast.model import choose def expert_gold(ex, b, sp, page, st, meta): """(gold_op, action, text, sub_index) for the current state, or ("DONE", None, None, None) when the task is done.""" if ex.wrong_results(meta): a = next((x for x in page["actions"] if x["label"] == "Modify search"), None) if a is None: raise E.Stop("wrong results and no way back") return "CLICK", a, None, None cur = ex.current(st, meta) if cur is None: return "DONE", None, None, None key = sp["subgoals"][cur][0] op, sel, val = ex.step_for(key, st, page) a = E.to_action(b, page, op, sel, val) if a is None and op != "PRESS_ENTER": if E.js(b, "!!document.getElementById('cookie')"): op, sel, val = "CLICK", "[data-g=cookie-accept]", None; a = E.to_action(b, page, op, sel, val) elif op != "CLICK_TEXT" and E.where(b, sel) in (1, -1): sid = "scroll_down" if E.where(b, sel) == 1 else "scroll_up" a = next((x for x in page["actions"] if x["id"] == sid), None); op = sid.upper() elif op == "CLICK_TEXT": a = next((x for x in page["actions"] if x["id"] == "scroll_down"), None); op = "SCROLL_DOWN" if a is None: raise E.Stop(f"{op} {sel} not actionable") gold_op = op if op in ("SCROLL_DOWN", "SCROLL_UP", "PRESS_ENTER") else {"CLICK_TEXT": "CLICK"}.get(op, op) return gold_op, a, (val if op == "TYPE_TEXT" else None), cur def text_for(b, sp, node, gold_a, gold_text): """Text the policy types into `node`: the expert's text if it is the same field, else the spec value of that field.""" if gold_a is not None and gold_a.get("node") == node and gold_text: return gold_text for f in sp.get("fields", []): if E.node_of(b, f"[data-g={f['key']}-input]") == node: t = sp["target"].get(f["key"]) if f["w"] == "combo" and t: return E.S_name(t) if f["w"] == "date" and t: y, m, d = t.split("-"); return f"{m}/{d}/{y}" return str(t) if t is not None else "test" if sp["kind"] == "shop" and sp["post"].get("q"): return sp["post"]["q"] return gold_text or "test" def run(kind, seed, out, rng, beta, max_steps=50): sp = S.make(kind, seed) b = Browser(f"{E.BASE}/t/{kind}/{seed}") ex = E.Expert(sp, b, rng) hist, n, agree, steps, prev_sub, seen = [], 0, 0, 0, None, {} try: page = E.observe(b) for step in range(max_steps): info = E.status(b); st = info.get("st") or {}; meta = info.get("meta") if st and "open" not in st: st["open"] = [] gold_op, ga, gtext, cur = expert_gold(ex, b, sp, page, st, meta) if prev_sub is not None and cur != prev_sub: out.write(json.dumps(E.case(page, sp["subgoals"][prev_sub][1], "DONE", "DONE", None, hist, sp, "subgoal_done") | {"source": "webgym_dagger"}, ensure_ascii=False) + "\n"); n += 1 # the model's own decision on this state d = choose(page, sp["goal"], hist) m_choice = d["choice"] m_ok = (gold_op == "DONE" and m_choice == "DONE") or (ga is not None and m_choice == ga["id"]) agree += m_ok; steps += 1 gid = "DONE" if gold_op == "DONE" else ga["id"] # a policy stuck in a loop revisits the same state: keep at most 2 copies of (state, gold) per episode sig = (page["url"], page["fingerprint"], gid, hist[-1]["action"] if hist else "") seen[sig] = seen.get(sig, 0) + 1 write = seen[sig] <= 2 if write: out.write(json.dumps(E.case(page, sp["goal"], gold_op, gid, ga, hist, sp, "dagger") | {"source": "webgym_dagger", "agree": m_ok}, ensure_ascii=False) + "\n"); n += 1 if cur is not None and write: out.write(json.dumps(E.case(page, sp["subgoals"][cur][1], gold_op, gid, ga, hist, sp, "dagger@sub") | {"source": "webgym_dagger", "agree": m_ok}, ensure_ascii=False) + "\n"); n += 1 if gold_op == "DONE" and (m_choice == "DONE" or rng.random() < beta): return n, agree, steps, True # which action to execute use_model = rng.random() >= beta and m_choice not in ("DONE", "BLOCKED") if use_model: a = next((x for x in page["actions"] if x["id"] == m_choice), None) if a is None: use_model = False if not use_model: if gold_op == "DONE": return n, agree, steps, True a = ga text = None if a["kind"] == "fill": text = gtext if a is ga else text_for(b, sp, a.get("node"), ga, gtext) before = page["fingerprint"] try: b.act(a, page, text=text) except E.StalePage: page = E.observe(b); continue time.sleep(0.2) page = E.observe(b) hist.append(E.hist_entry(a, text, page["fingerprint"] != before)) prev_sub = cur return n, agree, steps, False finally: b.close() def main(): out_f, n_ep = sys.argv[1], int(sys.argv[2]) seed0 = int(sys.argv[3]) if len(sys.argv) > 3 else 500001 kinds = (sys.argv[4] if len(sys.argv) > 4 else "flight,hotel,shop").split(",") beta = float(sys.argv[5]) if len(sys.argv) > 5 else 0.3 rng = random.Random(seed0) tot = {"ep": 0, "done": 0, "cases": 0, "agree": 0, "steps": 0, "err": 0} t0 = time.time(); seed = seed0 with open(out_f, "a") as out: while tot["ep"] + tot["err"] < n_ep: if seed % 10 == 0: seed += 1; continue kind = kinds[(tot["ep"] + tot["err"]) % len(kinds)] try: c, ag, s, done = run(kind, seed, out, rng, beta) tot["ep"] += 1; tot["done"] += done; tot["cases"] += c; tot["agree"] += ag; tot["steps"] += s except Exception as e: tot["err"] += 1 seed += 1 if (tot["ep"] + tot["err"]) % 20 == 0: print(f" {tot} agree={tot['agree']/max(1,tot['steps']):.2f} {time.time()-t0:.0f}s", flush=True) print("done", tot, f"agree={tot['agree']/max(1,tot['steps']):.2f}") if __name__ == "__main__": main()