code: rollouts, teacher eval, corrected suite checks

code/apps/browser_suite.py

@@ -57,13 +57,17 @@ def run(name, url, goal, check, max_steps=20):
             for state in agent.run():
                 steps = len(state["history"]); status = state["status"]; page = state["page"]
                 if steps >= max_steps: break
+            time.sleep(1.5)                                  # let a slow navigation (GitHub's Turbo, etc.) land before judging
+            try:
+                page = agent.browser.observe(screenshot=False)
+            except Exception:
+                pass
     except Exception as e:
         status = f"error:{type(e).__name__}"
     wall = time.time() - t0
     ok = bool(page and check(page["url"], page["title"], page.get("text", "")))
     if page and name == "internet-dropdown":
-        ok = any(a.get("kind") == "select" and a.get("current_value") == "2" for a in page["actions"]) or \
-             any(a.get("kind") == "select" and a.get("value") == "2" and a.get("selected") for a in page["actions"])
+        ok = any(a.get("kind") == "select" and str(a.get("current_value", "")).strip() in ("2", "Option 2") for a in page["actions"])
     if page and name == "internet-checkbox":
         cbs = [a for a in page["actions"] if a.get("role") == "checkbox"]
         ok = bool(cbs) and bool(cbs[0].get("checked"))

code/finetune/README.md

@@ -34,6 +34,17 @@ DONE / TYPE_TEXT / SELECT 操作样本 3 倍过采样，16020 条训练样本，
 Wikipedia 搜索任务首次正确选 TYPE_TEXT 并由本地 Qwen 填入 "Python programming language"，但之后重复填同一字段而没有提交——
 训练数据里缺"字段已填好 → 下一步提交"的样本。GitHub Issues 点成 Releases、HN new 提前 DONE。
 
+## 最终对比 v2（2026-09-21，修正检查脚本后：下拉按 current_value 文本判定、导航后等 1.5 s 再判）
+
+| 模型 | 真实任务 16×3 | 留出 top-1 | 每步 |
+|---|---|---|---|
+| v10（421M） | 50% | 0.656 | 41–50 ms |
+| **v10s（322M）** | **62%** | 0.631 | 17–23 ms |
+| v11s（322M + 682 条滚动/搜索/下拉脚本轨迹） | 56% | 0.623 | 19 ms |
+
+10 个任务 3/3 稳过、6 个 3/3 稳挂；v10s/v11s 差异小于站点随机波动。老师对比：27B 思考预算 300 最好（0.861/0.603，4.7 s/步），
+仍不超过 laya v11s（0.890/0.623，0.021 s）；8B/27B 都不适合做 DAgger 老师或 System-2 兜底。
+
 ## 最终对比（2026-09-21，16 个真实任务 × 3 次，apps/browser_suite.py）
 
 | 模型 | 底座 | 一步延迟 | 留出目标 top-1 | 真实任务通过率 | +门控 τ=0.7（Qwen3-8B 兜底） |

code/finetune/run_suite_fixed.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+# after the teacher eval: re-run the 16-task suite x3 for v11s, v10s and v10 with the corrected checks
+S=/tmp/claude-1000/-home-ckl-projects-S/8a7ce50a-5640-4606-837a-93df5ff48c83/scratchpad
+cd /home/ckl/projects/S/laya && source env.sh; O=finetune/out
+until grep -q "TEACHER_EVAL_DONE" $O/teacher_eval.log; do sleep 60; done
+bash $S/stop_all.sh >/dev/null 2>&1; bash $S/bonsai.sh stop >/dev/null 2>&1
+curl -s -m 3 http://127.0.0.1:9222/json/version >/dev/null || (nohup chromium --headless=new --remote-debugging-port=9222 --user-data-dir=$S/chrome-profile --window-size=1120,780 --no-first-run --lang=en-US about:blank >/dev/null 2>&1 &); sleep 3
+for ck in laya-browser-v11s laya-browser-v10s laya-browser-v10; do
+  echo "== suite $ck x3 (fixed checks)"
+  ESCALATE_TAU=0 bash $S/restart_s1.sh $PWD/$O/$ck 999 >/dev/null
+  (cd ../jev-ultrafast && REPEATS=3 SUITE_OUT=$PWD/../laya/$O/suite_fixed_$ck.json timeout 3000 .venv/bin/python ../laya/apps/browser_suite.py 2>&1 | grep -E "^==|per task")
+done
+bash $S/stop_all.sh >/dev/null 2>&1
+echo SUITE_FIXED_DONE

code/finetune/run_teacher_eval.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+# after v11: compare teachers on 80 held-out cases (40 Mind2Web human-labelled + 40 live), then laya v10s / v11s on the same cases.
+S=/tmp/claude-1000/-home-ckl-projects-S/8a7ce50a-5640-4606-837a-93df5ff48c83/scratchpad
+cd /home/ckl/projects/S/laya && source env.sh
+O=finetune/out; P=.venv/bin/python
+until grep -q "V11_DONE" $O/v11.log; do sleep 60; done
+bash $S/stop_all.sh >/dev/null 2>&1
+echo "== 27B (bonsai), thinking off"
+bash $S/bonsai.sh start
+$P finetune/teacher_eval.py $O/pages.jsonl $O/eval_cases.jsonl http://127.0.0.1:30001/v1 "bonsai-27b think=off" 80
+echo "== 27B, thinking full"
+$P finetune/teacher_eval.py $O/pages.jsonl $O/eval_cases.jsonl http://127.0.0.1:30001/v1 "bonsai-27b think=full" 40 --think
+bash $S/bonsai.sh stop
+echo "== 27B, thinking budget 300 tokens (--reasoning-budget 300)"
+bash $S/bonsai.sh start --reasoning-budget 300
+$P finetune/teacher_eval.py $O/pages.jsonl $O/eval_cases.jsonl http://127.0.0.1:30001/v1 "bonsai-27b think=budget300" 80 --think
+bash $S/bonsai.sh stop
+echo "== 27B, thinking with --reasoning-effort low (server flag)"
+bash $S/bonsai.sh start --reasoning-effort low
+$P finetune/teacher_eval.py $O/pages.jsonl $O/eval_cases.jsonl http://127.0.0.1:30001/v1 "bonsai-27b think=low" 40 --think
+bash $S/bonsai.sh stop
+echo "== Qwen3-8B-AWQ, thinking off"
+bash $S/start_infra.sh >/dev/null 2>&1
+$P finetune/teacher_eval.py $O/pages.jsonl $O/eval_cases.jsonl http://127.0.0.1:30000/v1 "qwen3-8b think=off" 80
+bash $S/stop_all.sh >/dev/null 2>&1
+echo "== laya v10s / v11s on the full eval set (for reference)"
+LAYA_FMT=v3 $P finetune/eval.py $O/pages.jsonl $O/eval_cases.jsonl $PWD/$O/laya-browser-v10s 2>&1 | grep -vE "TileLang|Warn|warn|Fetch" | head -1
+LAYA_FMT=v3 $P finetune/eval.py $O/pages.jsonl $O/eval_cases.jsonl $PWD/$O/laya-browser-v11s 2>&1 | grep -vE "TileLang|Warn|warn|Fetch" | head -1
+echo TEACHER_EVAL_DONE

code/finetune/run_v11.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+# v11s: v10s recipe + scripted scroll / search-submit / select trajectories (rollout_cases, x3), then 16 tasks x3.
+S=/tmp/claude-1000/-home-ckl-projects-S/8a7ce50a-5640-4606-837a-93df5ff48c83/scratchpad
+cd /home/ckl/projects/S/laya && source env.sh
+O=finetune/out; P=.venv/bin/python
+until ! pgrep -f "finetune/rollouts.py" >/dev/null; do sleep 60; done
+echo "== rollout cases: $(wc -l < $O/rollout_cases.jsonl)"
+export LAYA_BASE=$(ls -d ~/.cache/huggingface/hub/models--convaiinnovations--laya/snapshots/*/multilingual | head -1)
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True CKPT=0 COMPILE=0 LAYA_FMT=v3 LAYA_HEAD=768
+echo "== build v11s"
+$P finetune/build_items.py $O/pages.jsonl $O/cases.jsonl $O/ $O/done_cases.jsonl $O/step2_cases.jsonl $O/m2w_cases.jsonl $O/dagger_cases.jsonl $O/rollout_cases.jsonl
+echo "== train v11s (4 epochs)"; $P finetune/train.py $O/train_items.pt $O/laya-browser-v11s 4 2>&1 | grep --line-buffered -E "=== epoch|saved|Error|Traceback"
+echo "== calibrate"; $P finetune/calibrate.py $O/pages.jsonl $O/eval_cases.jsonl $PWD/$O/laya-browser-v11s 2>&1 | grep -vE "TileLang|Warn|warn|Fetch"
+echo "== eval v11s"; $P finetune/eval.py $O/pages.jsonl $O/eval_cases.jsonl $PWD/$O/laya-browser-v11s 2>&1 | grep -vE "TileLang|Warn|warn|Fetch"
+echo "== suite v11s x3"
+bash $S/start_infra.sh >/dev/null 2>&1
+export ESCALATE_TAU=0
+bash $S/restart_s1.sh $PWD/$O/laya-browser-v11s 999 >/dev/null
+(cd ../jev-ultrafast && REPEATS=3 SUITE_OUT=$PWD/../laya/$O/suite_final_v11s.json timeout 3000 .venv/bin/python ../laya/apps/browser_suite.py 2>&1 | grep -E "^==|per task")
+bash $S/stop_all.sh >/dev/null 2>&1
+echo V11_DONE

code/finetune/teacher_eval.py

@@ -0,0 +1,51 @@
+"""Teacher quality on held-out cases: operation accuracy and target top-1 for an LLM teacher (OpenAI-compatible endpoint).
+
+    python finetune/teacher_eval.py out/pages.jsonl out/eval_cases.jsonl <base_url> <label> [n=80] [--think] [--budget N]
+
+Same element table / goal / history the laya model sees (format v3), same gold. Prints accuracy and seconds per decision.
+"""
+import json, os, random, sys, time
+import httpx
+os.environ.setdefault("LAYA_FMT", "v3")
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from common_ft import build_request, gold_for
+
+SYS = """You are the decision head of a browser agent (like TypeSafe Jev). Given the goal, the actions so far, the page and a numbered table of
+controls with the operations each supports, output the single best NEXT step as JSON only:
+{"operation": "CLICK"|"TYPE_TEXT"|"SELECT"|"DONE"|"BLOCKED"|"WAIT"|"SCROLL_DOWN"|"SCROLL_UP", "target": "<control key or null>"}
+Rules: use TYPE_TEXT (not CLICK) for a text field the goal needs filled; SELECT for a dropdown option; CLICK for links/buttons/checkboxes.
+DONE only if every requirement of the goal is already visibly satisfied. A field already showing the requested value is done.
+If the needed control is not in the table and the page is taller than the viewport, SCROLL_DOWN."""
+
+def main():
+    pages = [json.loads(l) for l in open(sys.argv[1])]; cases = [json.loads(l) for l in open(sys.argv[2])]
+    base, label = sys.argv[3].rstrip("/"), sys.argv[4]
+    n = int(next((a for a in sys.argv[5:] if a.isdigit()), "80")); think = "--think" in sys.argv
+    budget = int(sys.argv[sys.argv.index("--budget") + 1]) if "--budget" in sys.argv else None
+    rng = random.Random(0); rng.shuffle(cases)
+    sel = [c for c in cases if c.get("source") == "mind2web"][: n // 2] + [c for c in cases if c.get("source") != "mind2web"][: n // 2]
+    op_ok = tgt_ok = tgt_n = 0; secs = []; fails = 0
+    for c in sel:
+        state, questions, targets, controls = build_request(c.get("page_obj") or pages[c["page"]], c["goal"], c.get("history", []))
+        gop, gidx = gold_for(c, targets, controls)
+        if gop is None: continue
+        ctr = [{"op": qid[:-7].upper(), "target": k, "control": v} for qid, q in questions.items() if qid.endswith("_target") for k, v in q["criteria"].items()]
+        user = {"goal": c["goal"], "actions_so_far": state.get("recent_actions", []), "page": state["page"], "operations": list(questions["operation"]["criteria"]), "controls": ctr[:150]}
+        body = {"model": "x", "temperature": 0.0, "max_tokens": 1500 if think else 60, "response_format": {"type": "json_object"},
+                "chat_template_kwargs": {"enable_thinking": think},
+                "messages": [{"role": "system", "content": SYS}, {"role": "user", "content": json.dumps(user, ensure_ascii=False)}]}
+        if budget is not None: body["reasoning_budget"] = budget
+        t = time.time()
+        try:
+            r = httpx.post(base + "/chat/completions", json=body, timeout=600).json(); v = json.loads(r["choices"][0]["message"]["content"] or "{}")
+        except Exception as e:
+            fails += 1; continue
+        secs.append(time.time() - t)
+        op = str(v.get("operation", "")).upper(); op_ok += op == gop
+        if gidx is not None:
+            tgt_n += 1; tgt_ok += (op == gop and str(v.get("target")) == str(gidx))
+    m = len(secs)
+    print(f"{label:28s} n={m:3d} op acc {op_ok/max(1,m):.3f}  target top-1 {tgt_ok/max(1,tgt_n):.3f}  {sum(secs)/max(1,m):.2f} s/decision  fails={fails}")
+
+if __name__ == "__main__":
+    main()