eval_push.py

#!/usr/bin/env python3
"""Push offline board-evals (Hart's glint_parity numbers) to an EXISTING fabryka run as
clean board/* metrics (not eval_-prefixed). Attaches by FABRYKA_RUN_ID.

Usage: FABRYKA_API_KEY=... python eval_push.py RUN_ID RUN_NAME STEP 'JSON_METRICS'
  e.g. eval_push.py 538058fc-394b-4cb4-9057-b190685dbf2a GoLLeM-v5-64M-forkB-arcmix-edu \
       340000 '{"board/arc_easy":46.42,"board/blimp":77.53}'
"""
import json
import os
import sys

run_id = sys.argv[1]
name = sys.argv[2]
step = int(sys.argv[3])
metrics = json.loads(sys.argv[4])
os.environ["FABRYKA_RUN_ID"] = run_id  # must be set BEFORE RunClient reads it


def _patch():
    """Make installed SDK honor FABRYKA_RUN_ID (idempotent; pusher may have done it already)."""
    try:
        import fabryka
        p = os.path.join(os.path.dirname(fabryka.__file__), "client.py")
        s = open(p, encoding="utf-8").read()
        if "FABRYKA_RUN_ID" not in s:
            s = s.replace("self.run_id = str(uuid.uuid4())",
                          'self.run_id = os.getenv("FABRYKA_RUN_ID") or str(uuid.uuid4())')
            open(p, "w", encoding="utf-8").write(s)
            for mod in [m for m in sys.modules if m == "fabryka" or m.startswith("fabryka.")]:
                del sys.modules[mod]
    except Exception as e:
        print(f"[eval_push] patch skipped: {e}", flush=True)


_patch()
from fabryka import RunClient

client = RunClient(api_url=os.environ.get("FABRYKA_API_URL", "https://track.fabryka.ai"),
                   api_key=os.environ["FABRYKA_API_KEY"])
client.init(project="gollem-v5", name=name)
client.log({k: float(v) for k, v in metrics.items()}, step=step)
# SDK _atexit calls finish(state="failed") on exit; null run_id so this short-lived push does NOT
# override the live pusher's run-state (we only append a metric, never own/finish the run).
client.run_id = None
print(f"[eval_push] pushed {metrics} @ step {step} -> run {run_id}", flush=True)