fix: make receipt evaluation portable
Browse filesCopy the complete repository into the isolated re-forge directory so packaged kernels and doctrine inputs resolve on fresh checkouts. Avoid privileged symlinks and non-ASCII status output on Windows.
- scripts/eval.py +6 -11
scripts/eval.py
CHANGED
|
@@ -14,20 +14,15 @@ print(f"model.joblib sha256 {'MATCHES receipt' if got == want else 'MISMATCH —
|
|
| 14 |
if got != want:
|
| 15 |
sys.exit(1)
|
| 16 |
with tempfile.TemporaryDirectory() as td:
|
| 17 |
-
|
| 18 |
-
shutil.
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
# from a fresh clone that ships it.
|
| 22 |
-
if os.path.isfile(f"{root}/Modelfile"):
|
| 23 |
-
shutil.copy(f"{root}/Modelfile", f"{td}/Modelfile")
|
| 24 |
-
out = subprocess.run([sys.executable, f"{td}/scripts/forge.py"],
|
| 25 |
-
capture_output=True, text=True, cwd=td)
|
| 26 |
print(out.stdout[-500:] if out.returncode == 0 else out.stderr[-800:])
|
| 27 |
if out.returncode:
|
| 28 |
sys.exit(1)
|
| 29 |
-
re_receipt = json.load(open(f"{
|
| 30 |
d = abs(re_receipt["metrics_MEASURED"]["fidelity_vs_rule_checker"]
|
| 31 |
- receipt["metrics_MEASURED"]["fidelity_vs_rule_checker"])
|
| 32 |
-
print(f"re-measured fidelity delta vs receipt: {d:.4f} ({'OK
|
| 33 |
sys.exit(0 if d <= 0.02 else 1)
|
|
|
|
| 14 |
if got != want:
|
| 15 |
sys.exit(1)
|
| 16 |
with tempfile.TemporaryDirectory() as td:
|
| 17 |
+
copy = f"{td}/repo"
|
| 18 |
+
shutil.copytree(root, copy, ignore=shutil.ignore_patterns(".cache"))
|
| 19 |
+
out = subprocess.run([sys.executable, f"{copy}/scripts/forge.py"],
|
| 20 |
+
capture_output=True, text=True, cwd=copy)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 21 |
print(out.stdout[-500:] if out.returncode == 0 else out.stderr[-800:])
|
| 22 |
if out.returncode:
|
| 23 |
sys.exit(1)
|
| 24 |
+
re_receipt = json.load(open(f"{copy}/TRAINING_RECEIPT.json"))
|
| 25 |
d = abs(re_receipt["metrics_MEASURED"]["fidelity_vs_rule_checker"]
|
| 26 |
- receipt["metrics_MEASURED"]["fidelity_vs_rule_checker"])
|
| 27 |
+
print(f"re-measured fidelity delta vs receipt: {d:.4f} ({'OK <=0.02' if d <= 0.02 else 'FAIL'})")
|
| 28 |
sys.exit(0 if d <= 0.02 else 1)
|