"""Generate testresult.html: every unique activity prompt -> expected intent, model output, executor command (Windows + Pi),
device, model status, executor status (real execution for safe read-only actions on this machine, dry-run otherwise).
  python eval/make_report.py <run_dir_with_eval/>   (uses model/ numpy runtime for the curated rows)"""
import json, os, sys, html, time, collections, platform as _pf
HERE = os.path.dirname(os.path.abspath(__file__)); ROOT = os.path.dirname(HERE)
sys.path.insert(0, os.path.join(ROOT, "model")); sys.path.insert(0, os.path.join(ROOT, "executor"))
from runtime import CommandModel  # noqa
from catalog import ACTIONS  # noqa
import executor as EX  # noqa
run_dir = sys.argv[1] if len(sys.argv) > 1 else None
_sp = os.path.join(run_dir, "eval", "summary.json") if run_dir else os.path.join(HERE, "model_eval_summary.json")
summary = json.load(open(_sp)) if os.path.exists(_sp) else {}
curated = [json.loads(l) for l in open(os.path.join(HERE, "curated_prompts.jsonl"), encoding="utf-8")]
model = CommandModel(os.path.join(ROOT, "model"))
esc = lambda s: html.escape(str(s if s is not None else ""))
EXEC_HERE = {"sh", "py"}   # kinds that can really run on this Linux box
rows = []; t_all = time.time(); exec_counts = collections.Counter(); model_ok = 0
for r in curated:
    out = model.predict(r["text"]); cmd = {k: v for k, v in out.items() if k != "_meta"}; canon = out["_meta"]["canonical"]
    ok = canon == r["target"]; model_ok += ok
    plans = {pl: EX.plan(cmd, pl, r["text"]) for pl in ("pi", "win")}
    ppi = plans["pi"]; st, outp = "", ""
    if not ppi.supported: st, outp = "n/a on Pi/Linux", ppi.note
    elif ppi.kind not in EXEC_HERE: st, outp = "dry-run", f"{ppi.kind} step (needs desktop/keyboard); command rendered only"
    elif ppi.risk != "safe": st, outp = f"dry-run ({ppi.risk}, gated)", "rendered only; UI asks for confirmation before running"
    elif not ok: st, outp = "skipped (model mismatch)", "not executed because the parse differed from the expected intent"
    else:
        res = EX.run(ppi, cmd, dry_run=False); outp = res["output"][:300]
        st = "executed: tool not on this machine" if ("command not found" in outp or ("No such file" in outp and "/sys/" in outp)) else f"executed: {res['status']}"
    exec_counts[st.split(":")[0].split(" (")[0]] += 1
    rows.append({"text": r["text"], "action": r["action"], "device": r["device"], "risk": r["risk"], "expected": r["target"], "pred": canon, "json": cmd, "ok": ok, "conf": out["_meta"]["confidence"],
                 "cmd_win": plans["win"].command if plans["win"].supported else "— " + plans["win"].note, "cmd_pi": ppi.command if ppi.supported else "— " + ppi.note, "kind_pi": ppi.kind, "exec_status": st, "exec_out": outp})
acc = model_ok / len(rows)
def cell_json(j): return "<br>".join(f"<span class=k>{esc(k)}</span>={esc('••••' if k == 'password' else v)}" for k, v in j.items() if k != "action")
summ_rows = "".join(f"<tr><td>{esc(n)}</td><td>{s['n']}</td><td class='{ 'g' if s['exact']>=0.99 else 'y' if s['exact']>=0.95 else 'r'}'><b>{s['exact']*100:.2f}%</b></td><td>{s['action']*100:.2f}%</td><td class=small>{esc(', '.join(f'{a}→{b} ({n})' for (a,b),n in s['top_confusions'][:5]))}</td></tr>" for n, s in summary.items())
by_risk = collections.Counter(r["risk"] for r in rows); by_dev = collections.Counter(r["device"] for r in rows)
table = "".join(f"""<tr class="{'ok' if r['ok'] else 'fail'}"><td>{i+1}</td><td>{esc(r['text'])}</td><td><b>{esc(r['action'])}</b><div class=small>{esc(ACTIONS[r['action']]['desc'])}</div></td>
<td><code>{esc(r['pred'])}</code><div class=small>{cell_json(r['json'])}</div><div class=small>conf {r['conf']*100:.1f}%</div></td>
<td class="{'g' if r['ok'] else 'r'}"><b>{'PASS' if r['ok'] else 'FAIL'}</b>{'' if r['ok'] else '<div class=small>expected: <code>'+esc(r['expected'])+'</code></div>'}</td>
<td><span class="pill {esc(r['risk'])}">{esc(r['risk'])}</span><div class=small>{esc(r['device'])}</div></td>
<td><div class=small><b>Windows</b></div><pre>{esc(r['cmd_win'][:400])}</pre><div class=small><b>Raspberry Pi</b> [{esc(r['kind_pi'])}]</div><pre>{esc(r['cmd_pi'][:400])}</pre></td>
<td><b>{esc(r['exec_status'])}</b><pre>{esc(r['exec_out'][:300])}</pre></td></tr>""" for i, r in enumerate(rows))
doc = f"""<!doctype html><html lang=en><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1"><title>Union Command Agent — test results</title>
<style>:root{{--bg:#0f1115;--panel:#171a21;--panel2:#1e222b;--border:#2a2f3a;--text:#e6e8ee;--muted:#9aa3b2;--acc:#c51a4a;--ok:#6cc04a;--warn:#f2c94c;--bad:#ff6b6b}}
body{{margin:0;background:var(--bg);color:var(--text);font:13.5px/1.45 system-ui,Segoe UI,Roboto,sans-serif}}header{{background:var(--panel);border-bottom:1px solid var(--border);padding:14px 20px}}h1{{margin:0 0 4px;font-size:18px}}main{{padding:16px 20px 60px;max-width:1700px;margin:0 auto}}
.grid{{display:grid;grid-template-columns:repeat(auto-fit,minmax(180px,1fr));gap:10px;margin:12px 0}}.stat{{background:var(--panel);border:1px solid var(--border);border-radius:8px;padding:10px 12px}}.stat b{{font-size:22px;display:block}}.stat span{{color:var(--muted);font-size:12px}}
table{{width:100%;border-collapse:collapse;background:var(--panel);border:1px solid var(--border);border-radius:8px;overflow:hidden;margin:10px 0 22px}}th{{text-align:left;padding:8px 10px;font-size:11px;letter-spacing:.04em;text-transform:uppercase;color:var(--muted);background:var(--panel2);border-bottom:1px solid var(--border)}}
td{{padding:8px 10px;border-bottom:1px solid var(--border);vertical-align:top}}tr.fail td{{background:#1f1517}}pre{{margin:2px 0;white-space:pre-wrap;word-break:break-all;background:#0b0d12;padding:5px 7px;border-radius:5px;font-size:11.5px;max-height:120px;overflow:auto}}code{{font-family:ui-monospace,Consolas,monospace;font-size:12px}}
.small{{color:var(--muted);font-size:11.5px}}.k{{color:var(--muted)}}.g{{color:var(--ok)}}.y{{color:var(--warn)}}.r{{color:var(--bad)}}.pill{{display:inline-block;font-size:11px;padding:1px 7px;border-radius:10px;border:1px solid var(--border)}}.safe{{color:var(--ok)}}.caution{{color:var(--warn)}}.critical{{color:var(--bad)}}
h2{{font-size:15px;margin:22px 0 6px}}p.note{{color:var(--muted);max-width:1100px}}input{{width:100%;padding:8px 10px;background:#0b0d12;color:var(--text);border:1px solid var(--border);border-radius:6px;font:inherit;margin:6px 0}}</style></head><body>
<header><h1>Union Command Agent — test results</h1><div class=small>generated {time.strftime('%Y-%m-%d %H:%M')} on {esc(_pf.node())} ({esc(_pf.system())}) · model: char-level transformer seq2seq, {sum(v.size for v in model.W.values())/1e6:.2f}M params, grammar-constrained decoding · {len(ACTIONS)} actions</div></header><main>
<div class=grid><div class=stat><b class="{'g' if acc>=0.99 else 'y'}">{acc*100:.2f}%</b><span>unique-activity prompts exact match ({model_ok}/{len(rows)}) — one held-out-template prompt per action</span></div>
{''.join(f"<div class=stat><b class='{'g' if s['exact']>=0.99 else 'y' if s['exact']>=0.95 else 'r'}'>{s['exact']*100:.2f}%</b><span>{esc(n)} exact (n={s['n']})</span></div>" for n, s in summary.items() if n in ('val','test_strict','curated_strict','B_golden_final'))}
<div class=stat><b>{len(rows)}</b><span>activities: safe {by_risk['safe']} · caution {by_risk['caution']} · critical {by_risk['critical']}</span></div><div class=stat><b>{by_dev['both']}/{by_dev['pi']}/{by_dev['windows']}</b><span>both / Pi-only / Windows-only</span></div></div>
<h2>Model accuracy per set</h2><p class=note><b>exact</b> = full canonical target (action + every slot value) matches. <b>action</b> = action label only. <i>val</i> shares templates with training. <i>test</i> / <i>curated</i> are held-out <b>templates</b>; since v6 the dataset also contains machine paraphrases of templates, so a held-out paraphrase can have a sibling paraphrase in train — <i>test_strict</i> / <i>curated_strict</i> keep only base-template rows (no sibling anywhere in train) and are the honest generalisation numbers. A_* / B_* are the two source projects' own evaluation sets relabelled to the union schema (their label conventions differ from the union in places, so they under-count).</p>
<table><thead><tr><th>set</th><th>n</th><th>exact</th><th>action</th><th>top confusions</th></tr></thead><tbody>{summ_rows}</tbody></table>
<h2>Executor status on this machine</h2><p class=note>Safe, read-only actions whose Pi/Linux command is runnable here were <b>actually executed</b> and their output captured. Caution/critical actions are rendered but gated (the UI asks for confirmation; critical ones require typing the action name). Key/desktop steps need a display. {esc(', '.join(f'{k}: {v}' for k, v in exec_counts.most_common()))}.</p>
<h2>Every unique activity ({len(rows)})</h2><input id=q placeholder="filter… (e.g. gpio, wifi, FAIL, critical)" oninput="f()">
<table id=t><thead><tr><th>#</th><th style="width:15%">prompt</th><th style="width:11%">intent (expected)</th><th style="width:15%">model output</th><th>model</th><th>risk / device</th><th style="width:30%">executor command</th><th style="width:14%">executor status</th></tr></thead><tbody>{table}</tbody></table>
<script>function f(){{const q=document.getElementById('q').value.toLowerCase();document.querySelectorAll('#t tbody tr').forEach(r=>r.style.display=r.textContent.toLowerCase().includes(q)?'':'none')}}</script></main></body></html>"""
open(os.path.join(HERE, "testresult.html"), "w", encoding="utf-8").write(doc)
print(f"curated exact {acc*100:.2f}% ({model_ok}/{len(rows)}); executor: {dict(exec_counts)}; report in {time.time()-t_all:.0f}s -> eval/testresult.html")