code/finetune/noul_curve.py

"""Threshold curve of the goal_done (noul) head on held-out eval cases: done recall vs. not-done-judged-done.
    python finetune/noul_curve.py <pages> <eval_cases> <ckpt> [max_cases=4000]"""
import json, os, random, sys
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))); sys.path.insert(0, "/home/ckl/projects/S/laya-upstream")
from common_ft import build_request, goal_done_question
import laya
pages = [json.loads(l) for l in open(sys.argv[1])]; cases = [json.loads(l) for l in open(sys.argv[2])]
pos = [c for c in cases if c["gold_op"] == "DONE"]; neg = [c for c in cases if c["gold_op"] != "DONE"]
random.Random(0).shuffle(neg); cases = pos + neg[:int(sys.argv[4]) if len(sys.argv) > 4 else 4000]
agent = laya.load(sys.argv[3]); agent.cfg["max_len"], agent.cfg["head_max_len"] = int(os.environ.get("LAYA_MAXLEN", "1024")), 768; agent.accelerate()
ps = []
for c in cases:
    state, _, _, _ = build_request(c.get("page_obj") or pages[c["page"]], c["goal"], c.get("history", []))
    ps.append((agent.predict(state, {"g": goal_done_question(c["goal"])})["answers"]["g"]["noul"], c["gold_op"] == "DONE"))
P = [p for p, y in ps if y]; N = [p for p, y in ps if not y]
auc = sum((p > q) + 0.5 * (p == q) for p in P for q in N) / (len(P) * len(N))
print(f"AUC {auc:.3f}  (done n={len(P)}, not-done n={len(N)})")
for t in (0.05, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5):
    print(f"  reject DONE if p < {t:.2f}: keeps {sum(p >= t for p in P) / len(P):.2f} of true DONEs, lets through {sum(p >= t for p in N) / len(N):.3f} of not-done states")