eval/smoke_test.py

"""Smoke test: model + executor, no UI, nothing is executed (plans only).
  python eval/smoke_test.py          # fixed checks (~5 s)
  python eval/smoke_test.py --full   # + all 371 curated activity prompts and the wifi held-out set (~1-2 min)
Exit code 0 = every MUST check passed. KNOWN checks are phrasings the CPU model (run5) gets wrong; they are
expected to pass after the Colab model is installed and are reported, not failed on.
"""
import json, os, sys, time
HERE = os.path.dirname(os.path.abspath(__file__)); ROOT = os.path.dirname(HERE)
sys.path[:0] = [os.path.join(ROOT, "model"), os.path.join(ROOT, "executor")]
from runtime import CommandModel  # noqa
import executor as EX  # noqa

# (prompt, expected canonical target, level)   level: MUST | KNOWN
MODEL_CASES = [
    ("awaaz ko pachaas pe rakh do", 'set_volume value=50', "MUST"),
    ("volume badhao", 'volume_up', "MUST"),
    ("brightness 70 kar do", 'set_brightness value=70', "MUST"),
    ("board pin 11 pe led jalao", 'gpio_on pin=11 numbering=board', "MUST"),
    ("gpio 17 ko off karo", 'gpio_off pin=17', "MUST"),
    ("wifi Airtel_5G se connect karo password Wifi@2024", 'connect_wifi ssid="airtel_5g" password="wifi@2024"', "MUST"),
    ("connect to wifi JioFiber-Home with password jio#12345", 'connect_wifi ssid="jiofiber-home" password="jio#12345"', "MUST"),
    ("connect wifi Guest_Network pwd guest@123", 'connect_wifi ssid="guest_network" password="guest@123"', "MUST"),
    ("konse wifi networks available hai", 'list_wifi', "MUST"),
    ("wifi band karo", 'wifi_off', "MUST"),
    ("chrome kholo", 'open_app app=chrome', "MUST"),
    # run5 gets these wrong (see HANDOVER.md); the executor guard turns most of them into a clarification instead of a bad command
    ("mere ghar ka wifi TP-Link_A1B2 hai, password hai Pass@word99, connect kar do", 'connect_wifi ssid="tp-link_a1b2" password="pass@word99"', "KNOWN"),
    ("ssid: Office WiFi 5G password: Off!ce2024 se jud jao", 'connect_wifi ssid="office wifi 5g" password="off!ce2024"', "KNOWN"),
    ("wifi se connect karo naam MyNet pass abc12345", 'connect_wifi ssid="mynet" password="abc12345"', "KNOWN"),
    ("wifi Airtel_5G bhool jao", 'forget_wifi ssid="airtel_5g"', "KNOWN"),
]

# executor-only checks: (parsed command, platform, predicate on plan, description)
def _has(s): return lambda p: s in (p.command or "")
EXEC_CASES = [
    ({"action": "set_volume", "value": 50}, "pi", lambda p: p.supported and p.risk == "safe" and "50%" in p.command, "volume -> wpctl 50%"),
    ({"action": "gpio_on", "pin": 11, "numbering": "board"}, "pi", _has("pinctrl set 17 op dh"), "board 11 -> BCM 17"),
    ({"action": "gpio_on", "pin": 2, "numbering": "board"}, "pi", lambda p: not p.supported, "board pin 2 (5V) refused"),
    ({"action": "gpio_on", "pin": 40}, "pi", lambda p: not p.supported, "BCM 40 refused"),
    ({"action": "connect_wifi", "ssid": "Office WiFi", "password": "a'b&c"}, "pi", _has("'Office WiFi' password 'a'\"'\"'b&c'"), "Pi: ssid/password shell-quoted"),
    ({"action": "connect_wifi", "ssid": "Airtel_5G"}, "pi", lambda p: "password" not in p.command, "Pi: no empty 'password' arg"),
    ({"action": "connect_wifi", "ssid": "Airtel_5G", "password": "Wifi@2024"}, "win", lambda p: "netsh wlan add profile" in p.command and "keyMaterial" in p.command, "Windows: profile with key, then connect"),
    ({"action": "connect_wifi", "ssid": "TP-Link", "password": "hai"}, "pi", lambda p: p.action == "clarify", "filler password -> clarify"),
    ({"action": "connect_wifi", "ssid": "Office WiFi", "password": "Office WiFi"}, "pi", lambda p: p.action == "clarify", "password == ssid -> clarify"),
    ({"action": "connect_wifi", "ssid": "MyNet pass abc12345"}, "pi", lambda p: p.action == "clarify", "password inside ssid -> clarify"),
    ({"action": "connect_wifi", "ssid": "baad mein", "password": "rudra@11"}, "pi", lambda p: p.action == "clarify", "'baad mein' copied as ssid -> clarify"),
    ({"action": "connect_wifi", "ssid": "Redmi Note 13", "password": "pehle:"}, "pi", lambda p: p.action == "clarify", "'pehle:' copied as password -> clarify"),
    ({"action": "shutdown"}, "pi", lambda p: p.risk == "critical" and not EX.gate(p)[0] and EX.gate(p, confirm_text="shutdown")[0], "critical needs typed confirmation"),
    ({"action": "connect_wifi", "ssid": "a", "password": "bbbbbbbb"}, "pi", lambda p: not EX.gate(p)[0] and EX.gate(p, allow_caution=True)[0], "caution needs allow flag"),
    ({"action": "run_command", "command": "ls -la"}, "pi", lambda p: p.risk == "critical", "run_command is critical"),
    ({"action": "set_volume", "value": 40}, "win", lambda p: "= None;" not in p.command and "$None" not in p.command and "[Audio]::Volume=40/100" in p.command, "Windows audio helper uses C# null, not Python None"),
    # optional 5th field = the user's sentence: a pin-driving action from a bare 'pin N' must ask board vs BCM
    ({"action": "gpio_on", "pin": 11}, "pi", lambda p: p.action == "clarify" and "board pin 11" in p.note and "gpio 11" in p.note, "bare 'pin no 11' write -> clarify", "on windows 11 set pin no 11 high"),
    ({"action": "gpio_toggle", "pin": 17}, "pi", lambda p: p.action == "clarify", "bare 'pin 17' toggle (caution) -> clarify", "pin 17 toggle karo"),
    ({"action": "gpio_on", "pin": 17}, "pi", _has("pinctrl set 17 op dh"), "'gpio 17' write runs", "gpio 17 on karo"),
    ({"action": "gpio_on", "pin": 11, "numbering": "board"}, "pi", _has("pinctrl set 17 op dh"), "'board pin 11' write runs", "board pin 11 pe led jalao"),
    ({"action": "gpio_off", "pin": 21, "numbering": "board"}, "pi", lambda p: p.action == "gpio_off", "'40-pin header pin 21' write runs", "40-pin header pin 21 pe heater off karo"),
    ({"action": "gpio_info", "pin": 11}, "pi", lambda p: p.action == "gpio_info", "bare pin on a safe read -> no clarify", "pin 11 input hai ya output"),
]

def main(full):
    t0 = time.time(); m = CommandModel(os.path.join(ROOT, "model"))
    print(f"model: {sum(v.size for v in m.W.values())/1e6:.2f}M params, loaded in {time.time()-t0:.1f}s\n")
    must_fail = 0; known_pass = 0; known = 0
    print("== model parse ==")
    for text, exp, level in MODEL_CASES:
        out = m.predict(text); got = out["_meta"]["canonical"]; ok = got == exp
        if level == "KNOWN":
            known += 1; known_pass += ok; tag = "KNOWN-OK " if ok else "KNOWN-BAD"
        else:
            must_fail += not ok; tag = "PASS     " if ok else "FAIL     "
        cmd = {k: v for k, v in out.items() if k != "_meta"}; pl = EX.plan(cmd, "pi", text)
        extra = "" if ok else f"   expected: {exp}"
        print(f"{tag} {text}\n          -> {got}   [executor: {pl.action}{' - ' + pl.note if pl.action == 'clarify' else ''}]{extra}")
    print("\n== executor ==")
    for cmd, plat, pred, desc, *text in EXEC_CASES:
        p = EX.plan(cmd, plat, text[0] if text else None)
        try: ok = bool(pred(p))
        except Exception as e: ok = False; desc += f" ({e})"
        must_fail += not ok
        print(f"{'PASS' if ok else 'FAIL'}  [{plat}] {desc}" + ("" if ok else f"\n      got: {p.action} {p.supported} {p.risk} {(p.command or p.note)[:160]}"))
    if full:
        for name, path in (("curated (one prompt per action)", os.path.join(HERE, "curated_prompts.jsonl")), ("wifi_holdout", os.path.join(HERE, "wifi_holdout.jsonl"))):
            if not os.path.exists(path): continue
            rows = [json.loads(l) for l in open(path, encoding="utf-8")]; t1 = time.time()
            bad = [(r, m.predict(r["text"])["_meta"]["canonical"]) for r in rows]
            bad = [(r, g) for r, g in bad if g != r["target"]]
            acc = 1 - len(bad) / len(rows)
            print(f"\n== {name}: exact {acc*100:.2f}% ({len(rows)-len(bad)}/{len(rows)}) in {time.time()-t1:.0f}s ==")
            for r, g in bad[:10]: print(f"  MISS {r['text']}\n       got {g}\n       exp {r['target']}")
    print(f"\nsummary: MUST failures = {must_fail}; KNOWN phrasings fixed = {known_pass}/{known}")
    sys.exit(1 if must_fail else 0)

if __name__ == "__main__": main("--full" in sys.argv)