Spaces:
Running
Running
chore(sync): mirror backend .py + Dockerfile to Space (hf-sync-backend)
Browse filesAutomated backend sync from szl-holdings/a11oy main via hf-sync-backend.
Updated (differed from the Space): Dockerfile, serve.py, szl_energy_operator.py
Deleted (gone from the repo + Dockerfile COPY set): (none)
Keeps the Space-built backend (serve.py + the Dockerfile-COPY'd .py
modules) identical to GitHub main so the Space never rebuilds from a
stale backend, new endpoints don't 404 there, and orphaned modules
removed from the repo don't linger in the Space tree.
- Dockerfile +1 -1
- serve.py +17 -0
- szl_energy_operator.py +789 -0
Dockerfile
CHANGED
|
@@ -99,7 +99,7 @@ COPY szl_formula_wiring.py a11oy_code_engine.py a11oy_code.py a11oy_seismic.py s
|
|
| 99 |
COPY szl_energy_budget.py szl_energy_sovereign.py szl_energy_provenance.py szl_heart_blood.py szl_engine_status.py szl_backend_hardening.py revenue_endpoints.py a11oy_harvest_endpoints.py ./
|
| 100 |
# energy operator/ledger/projection modules — imported by serve.py (guarded);
|
| 101 |
# MUST be per-file COPY'd (this Dockerfile uses no `COPY . .`) or the import falls back to a STUB.
|
| 102 |
-
COPY joule_billing.py szl_energy_ledger.py ./
|
| 103 |
# ADDITIVE (I3): FABRO-style Governed Factory + Constitutional Engines modules.
|
| 104 |
# MUST be COPY'd or serve.py's guarded imports fall back (merged-but-not-live).
|
| 105 |
# HTML/JS is inlined in these .py modules, so NO web/ or static-vendor COPY needed.
|
|
|
|
| 99 |
COPY szl_energy_budget.py szl_energy_sovereign.py szl_energy_provenance.py szl_heart_blood.py szl_engine_status.py szl_backend_hardening.py revenue_endpoints.py a11oy_harvest_endpoints.py ./
|
| 100 |
# energy operator/ledger/projection modules — imported by serve.py (guarded);
|
| 101 |
# MUST be per-file COPY'd (this Dockerfile uses no `COPY . .`) or the import falls back to a STUB.
|
| 102 |
+
COPY joule_billing.py szl_energy_ledger.py szl_energy_operator.py ./
|
| 103 |
# ADDITIVE (I3): FABRO-style Governed Factory + Constitutional Engines modules.
|
| 104 |
# MUST be COPY'd or serve.py's guarded imports fall back (merged-but-not-live).
|
| 105 |
# HTML/JS is inlined in these .py modules, so NO web/ or static-vendor COPY needed.
|
serve.py
CHANGED
|
@@ -263,6 +263,23 @@ try:
|
|
| 263 |
except Exception as _szl_es_e: # pragma: no cover
|
| 264 |
print(f"[a11oy] Energy/Sovereign-Compute NOT registered: {_szl_es_e!r}", file=__import__("sys").stderr)
|
| 265 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 266 |
# ── Sovereign VRAM-resident GPU-QUANT ENGINE (gpu-quant) — three honest layers on the
|
| 267 |
# a11oy finance surface: L1 PCA-Risk (Ledoit-Wolf shrinkage Σ̂_LW + Marchenko-Pastur λ⁺
|
| 268 |
# eigenvalue clipping; cuML on GPU else PURE-STDLIB CPU fallback, label honest), L2 TDA-
|
|
|
|
| 263 |
except Exception as _szl_es_e: # pragma: no cover
|
| 264 |
print(f"[a11oy] Energy/Sovereign-Compute NOT registered: {_szl_es_e!r}", file=__import__("sys").stderr)
|
| 265 |
|
| 266 |
+
# ── PRESS-PLAY ENERGY OPERATOR daemon (Dev 1) — dispatches a CONTINUOUS stream of
|
| 267 |
+
# REAL inference jobs to the reachable Ollama GPU nodes (rtx-betterwithage, chaski),
|
| 268 |
+
# meters MEASURED joules per job off the EXISTING betterwithage NVML joule-meter, and
|
| 269 |
+
# emits a JobRecord per completed job (the Dev2 receipts / Dev3 projection / Dev4
|
| 270 |
+
# dashboard contract). Graceful start/stop, backpressure, ledger state-persistence so
|
| 271 |
+
# a restart resumes counts. HONEST: unreachable node -> DEGRADED (never faked); stale
|
| 272 |
+
# meter (>30s) -> SAMPLE energy excluded from billable; no node reachable -> a clearly
|
| 273 |
+
# marked local STUB (real CPU work, SAMPLE energy only). Adds POST/GET
|
| 274 |
+
# /api/a11oy/v1/energy/operator/{start,stop,status} (dual-registered under /v1/* too).
|
| 275 |
+
# Additive, try/except-guarded, before the SPA catch-all.
|
| 276 |
+
try:
|
| 277 |
+
import szl_energy_operator as _szl_energy_operator
|
| 278 |
+
_szl_energy_operator.register(app, ns="a11oy")
|
| 279 |
+
print("[a11oy] Energy Operator registered: /api/a11oy/v1/energy/operator/{start,stop,status}", file=__import__("sys").stderr)
|
| 280 |
+
except Exception as _szl_op_e: # pragma: no cover
|
| 281 |
+
print(f"[a11oy] Energy Operator NOT registered: {_szl_op_e!r}", file=__import__("sys").stderr)
|
| 282 |
+
|
| 283 |
# ── Sovereign VRAM-resident GPU-QUANT ENGINE (gpu-quant) — three honest layers on the
|
| 284 |
# a11oy finance surface: L1 PCA-Risk (Ledoit-Wolf shrinkage Σ̂_LW + Marchenko-Pastur λ⁺
|
| 285 |
# eigenvalue clipping; cuML on GPU else PURE-STDLIB CPU fallback, label honest), L2 TDA-
|
szl_energy_operator.py
ADDED
|
@@ -0,0 +1,789 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: Apache-2.0
|
| 2 |
+
# © 2026 Lutar, Stephen P. — SZL Holdings · ORCID 0009-0001-0110-4173
|
| 3 |
+
# Doctrine v11 — the press-play ENERGY OPERATOR daemon (Dev 1, backend).
|
| 4 |
+
"""
|
| 5 |
+
szl_energy_operator.py — PRESS PLAY. The operational loop.
|
| 6 |
+
|
| 7 |
+
On START this daemon dispatches a CONTINUOUS stream of REAL inference jobs to the
|
| 8 |
+
reachable Ollama GPU nodes (rtx-betterwithage, chaski) — small honest workloads
|
| 9 |
+
(short token-generation prompts and/or embeddings) so the rig genuinely computes
|
| 10 |
+
and burns measurable energy. Per job it records start/end wall-time, pulls an NVML
|
| 11 |
+
power/energy sample from the EXISTING exporter path (the betterwithage joule-meter
|
| 12 |
+
that already feeds szl_energy_sovereign's metrics panel / harvest posture
|
| 13 |
+
joules_evidence), and computes joules_measured for that job.
|
| 14 |
+
|
| 15 |
+
Honesty (Doctrine v11 — NEVER violate):
|
| 16 |
+
- Joules are MEASURED only from a REAL, FRESH (<30s) NVML exporter delta. The
|
| 17 |
+
label is decided SOLELY by szl_joules_truth — never off a flag.
|
| 18 |
+
- If the exporter sample is stale (>30s) or unavailable, the job's energy is
|
| 19 |
+
labeled SAMPLE and EXCLUDED from billable totals. We never fabricate a joule.
|
| 20 |
+
- If a GPU node is unreachable, we SKIP it and mark it DEGRADED in status —
|
| 21 |
+
we NEVER fabricate a job or a joule for a node that didn't compute.
|
| 22 |
+
- Sandbox / no-GPU: a faithful local STUB of the Ollama API does REAL CPU work
|
| 23 |
+
(so wall-time + jobs_done are honest) but its energy is ALWAYS labeled SAMPLE
|
| 24 |
+
and never billable. Stub mode is announced loudly in every status payload.
|
| 25 |
+
|
| 26 |
+
Interface other devs build against (STABLE — Dev2 receipts / Dev3 projection /
|
| 27 |
+
Dev4 dashboard consume this):
|
| 28 |
+
|
| 29 |
+
JobRecord = {
|
| 30 |
+
"node": str, # which node computed it (or "<node>-stub")
|
| 31 |
+
"model": str, # model tag used
|
| 32 |
+
"kind": str, # "generate" | "embed"
|
| 33 |
+
"tokens": int, # MEASURED tokens produced/consumed
|
| 34 |
+
"wall_s": float, # MEASURED wall-clock seconds for the job
|
| 35 |
+
"joules_measured": float|None, # MEASURED joules iff joules_label=="MEASURED", else None
|
| 36 |
+
"joules_label": str, # "MEASURED" | "SAMPLE" (billing.py-compatible upper-case)
|
| 37 |
+
"joules_evidence": dict, # self-verifying exporter evidence (empty unless MEASURED)
|
| 38 |
+
"ts": str, # ISO-8601 UTC completion time
|
| 39 |
+
"seq": int, # monotonic job sequence number (ledger order)
|
| 40 |
+
}
|
| 41 |
+
|
| 42 |
+
on_job(JobRecord) callback — register via OperatorDaemon.subscribe(cb) so Dev2
|
| 43 |
+
can mint a JouleCharge receipt per completed job in real time.
|
| 44 |
+
|
| 45 |
+
Endpoints (dual-registered under /api/{ns}/v1/energy/operator/* AND /v1/energy/operator/*):
|
| 46 |
+
POST /energy/operator/start — press play (idempotent; returns running state)
|
| 47 |
+
POST /energy/operator/stop — graceful stop (idempotent)
|
| 48 |
+
GET /energy/operator/status — running?, jobs_done, joules_measured_total,
|
| 49 |
+
tokens_total, nodes computing, uptime, degraded nodes
|
| 50 |
+
|
| 51 |
+
Graceful start/stop via a threading stop-flag + optional SIGINT/SIGTERM handler.
|
| 52 |
+
Backpressure: a per-node in-flight cap + inter-job sleep so we never overwhelm a node.
|
| 53 |
+
State persists to a JSON ledger so a restart RESUMES the cumulative counts.
|
| 54 |
+
|
| 55 |
+
Pure stdlib + httpx (already a repo dep) + FastAPI. No Node, no CDN.
|
| 56 |
+
"""
|
| 57 |
+
from __future__ import annotations
|
| 58 |
+
|
| 59 |
+
import json
|
| 60 |
+
import os
|
| 61 |
+
import signal
|
| 62 |
+
import threading
|
| 63 |
+
import time
|
| 64 |
+
import urllib.request
|
| 65 |
+
from dataclasses import dataclass, field
|
| 66 |
+
from datetime import datetime, timezone
|
| 67 |
+
from typing import Any, Callable, Optional
|
| 68 |
+
|
| 69 |
+
try:
|
| 70 |
+
import szl_joules_truth as _J
|
| 71 |
+
except Exception: # pragma: no cover — packaged import fallback
|
| 72 |
+
from . import szl_joules_truth as _J # type: ignore
|
| 73 |
+
|
| 74 |
+
# ---------------------------------------------------------------------------
|
| 75 |
+
# Doctrine constants.
|
| 76 |
+
# ---------------------------------------------------------------------------
|
| 77 |
+
DOCTRINE = "v11"
|
| 78 |
+
# joule_billing.py refuses to bill unless the label is MEASURED with an NVML sample
|
| 79 |
+
# fresher than this. We mirror it EXACTLY so the operator's MEASURED↔SAMPLE split
|
| 80 |
+
# is the same gate the billing core uses (no divergence).
|
| 81 |
+
MAX_NVML_AGE_S = 30.0
|
| 82 |
+
# billing.py uses upper-case labels; szl_joules_truth uses lower-case. We map at the
|
| 83 |
+
# boundary so JobRecord.joules_label is billing-compatible and self-consistent.
|
| 84 |
+
LABEL_MEASURED = "MEASURED"
|
| 85 |
+
LABEL_SAMPLE = "SAMPLE"
|
| 86 |
+
|
| 87 |
+
# The EXISTING exporter path: the betterwithage NVML joule-meter that
|
| 88 |
+
# szl_energy_sovereign._metrics_panel() already reads (engines→gpus→{power_w,joules,live},
|
| 89 |
+
# totals→{joules}). We reuse the SAME URL so the operator meters off the same source.
|
| 90 |
+
_JOULE_METER_URL = os.environ.get("A11OY_JOULE_METER_URL", "http://100.96.129.45:9471/")
|
| 91 |
+
|
| 92 |
+
# Ledger / state file. Survives restart so cumulative counts resume. Override for tests.
|
| 93 |
+
_DEFAULT_STATE_PATH = os.environ.get(
|
| 94 |
+
"A11OY_OPERATOR_STATE", os.path.join(os.path.dirname(os.path.abspath(__file__)),
|
| 95 |
+
"artifacts", "energy_operator_ledger.json"))
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def _now_iso() -> str:
|
| 99 |
+
return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
# ---------------------------------------------------------------------------
|
| 103 |
+
# Node configuration — the reachable Ollama GPU nodes (GROUND TRUTH 2026-06-14).
|
| 104 |
+
# rtx-betterwithage is the sovereign box (A11OY_MODEL_BASE_URL); chaski is the
|
| 105 |
+
# tailnet box. Each node is an OpenAI-compatible Ollama endpoint — same serving
|
| 106 |
+
# path a11oy_code_orchestrator._call_model uses (POST {base}/chat/completions),
|
| 107 |
+
# plus Ollama's native /api/embeddings for the embed workload.
|
| 108 |
+
# ---------------------------------------------------------------------------
|
| 109 |
+
@dataclass
|
| 110 |
+
class NodeCfg:
|
| 111 |
+
name: str
|
| 112 |
+
base_url: str # OpenAI-compatible base, e.g. http://host:11434/v1
|
| 113 |
+
gen_model: str # small token-generation model tag
|
| 114 |
+
embed_model: str # embeddings model tag
|
| 115 |
+
exporter_node: str # label this node reports in the joule-meter engines list
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
def _default_nodes() -> list[NodeCfg]:
|
| 119 |
+
"""Build node configs from env (token-flip law: read at runtime, never hardcode a key).
|
| 120 |
+
|
| 121 |
+
rtx-betterwithage base resolves from A11OY_MODEL_BASE_URL (the same env the
|
| 122 |
+
orchestrator uses) when it points at a non-router endpoint. chaski base from
|
| 123 |
+
A11OY_CHASKI_BASE_URL. Both default to honest placeholders that simply won't be
|
| 124 |
+
reachable from the sandbox → DEGRADED (never faked)."""
|
| 125 |
+
btw_base = (os.environ.get("A11OY_MODEL_BASE_URL") or "").strip().rstrip("/")
|
| 126 |
+
if not btw_base or "router.huggingface.co" in btw_base:
|
| 127 |
+
btw_base = os.environ.get("A11OY_BETTERWITHAGE_BASE_URL",
|
| 128 |
+
"http://rtx-betterwithage:11434/v1").rstrip("/")
|
| 129 |
+
chaski_base = os.environ.get("A11OY_CHASKI_BASE_URL",
|
| 130 |
+
"http://chaski:11434/v1").rstrip("/")
|
| 131 |
+
return [
|
| 132 |
+
NodeCfg(
|
| 133 |
+
name="rtx-betterwithage",
|
| 134 |
+
base_url=btw_base,
|
| 135 |
+
gen_model=os.environ.get("A11OY_BTW_GEN_MODEL", "llama3.1:8b"),
|
| 136 |
+
embed_model=os.environ.get("A11OY_BTW_EMBED_MODEL", "bge-large"),
|
| 137 |
+
exporter_node=os.environ.get("A11OY_GPU_LABEL", "betterwithage"),
|
| 138 |
+
),
|
| 139 |
+
NodeCfg(
|
| 140 |
+
name="chaski",
|
| 141 |
+
base_url=chaski_base,
|
| 142 |
+
gen_model=os.environ.get("A11OY_CHASKI_GEN_MODEL", "qwen2.5:32b"),
|
| 143 |
+
embed_model=os.environ.get("A11OY_CHASKI_EMBED_MODEL", "mistral"),
|
| 144 |
+
exporter_node=os.environ.get("A11OY_CHASKI_GPU_LABEL", "chaski"),
|
| 145 |
+
),
|
| 146 |
+
]
|
| 147 |
+
|
| 148 |
+
|
| 149 |
+
# Small, honest workloads — short prompts so each job genuinely computes but the loop
|
| 150 |
+
# stays gentle (backpressure). Rotated so the rig does varied real work.
|
| 151 |
+
_GEN_PROMPTS = [
|
| 152 |
+
"In one sentence, what is sovereign compute?",
|
| 153 |
+
"Name one law of thermodynamics in a single line.",
|
| 154 |
+
"Give a one-line definition of a joule.",
|
| 155 |
+
"Summarize energy provenance in one short sentence.",
|
| 156 |
+
]
|
| 157 |
+
_EMBED_TEXTS = [
|
| 158 |
+
"sovereign metered compute receipt",
|
| 159 |
+
"measured joules to billable charge",
|
| 160 |
+
"energy provenance under doctrine v11",
|
| 161 |
+
]
|
| 162 |
+
|
| 163 |
+
|
| 164 |
+
# ---------------------------------------------------------------------------
|
| 165 |
+
# Exporter sampling — reuse the EXISTING betterwithage joule-meter path.
|
| 166 |
+
# ---------------------------------------------------------------------------
|
| 167 |
+
def _fetch_joule_meter(timeout: float = 4.0) -> Optional[dict]:
|
| 168 |
+
"""Fetch the live NVML joule-meter JSON, or None on any failure (honest)."""
|
| 169 |
+
try:
|
| 170 |
+
req = urllib.request.Request(
|
| 171 |
+
_JOULE_METER_URL, headers={"User-Agent": "szl-energy-operator"})
|
| 172 |
+
with urllib.request.urlopen(req, timeout=timeout) as r: # noqa: S310
|
| 173 |
+
return json.loads(r.read().decode("utf-8", "replace"))
|
| 174 |
+
except Exception: # noqa: BLE001 — unreachable meter => no sample, stay honest
|
| 175 |
+
return None
|
| 176 |
+
|
| 177 |
+
|
| 178 |
+
def _exporter_sample_for_node(meter: Optional[dict], exporter_node: str,
|
| 179 |
+
now: Optional[float] = None) -> Optional[dict]:
|
| 180 |
+
"""Build a szl_joules_truth exporter_sample for one node from the meter JSON.
|
| 181 |
+
|
| 182 |
+
The meter shape (mirrors szl_energy_sovereign._metrics_panel): engines[].{engine,
|
| 183 |
+
joules, gpus[].{power_w, joules, live}}, totals.{joules}. We pick the engine whose
|
| 184 |
+
name matches exporter_node; its cumulative joules + a fresh wall-clock ts give a
|
| 185 |
+
real reading. Returns None when the node isn't present / has no numeric joules.
|
| 186 |
+
"""
|
| 187 |
+
if not isinstance(meter, dict):
|
| 188 |
+
return None
|
| 189 |
+
now = time.time() if now is None else now
|
| 190 |
+
engines = meter.get("engines") or []
|
| 191 |
+
for e in engines:
|
| 192 |
+
if str(e.get("engine") or "").lower() != exporter_node.lower():
|
| 193 |
+
continue
|
| 194 |
+
joules = e.get("joules")
|
| 195 |
+
if not isinstance(joules, (int, float)):
|
| 196 |
+
continue
|
| 197 |
+
power_w = None
|
| 198 |
+
for g in (e.get("gpus") or []):
|
| 199 |
+
if g.get("live") and isinstance(g.get("power_w"), (int, float)):
|
| 200 |
+
power_w = float(g["power_w"])
|
| 201 |
+
break
|
| 202 |
+
return {
|
| 203 |
+
"joules_measured_total": float(joules),
|
| 204 |
+
"exporter_node": exporter_node,
|
| 205 |
+
# The meter scraped just now → fresh by construction (same convention as
|
| 206 |
+
# szl_energy_sovereign._exporter_sample_from_metrics).
|
| 207 |
+
"exporter_last_seen_ts": now,
|
| 208 |
+
"power_w_sample": power_w,
|
| 209 |
+
}
|
| 210 |
+
return None
|
| 211 |
+
|
| 212 |
+
|
| 213 |
+
def _label_upper(exporter_sample: Optional[dict], now: Optional[float] = None) -> str:
|
| 214 |
+
"""Map szl_joules_truth's lower-case label to billing.py's upper-case label."""
|
| 215 |
+
return LABEL_MEASURED if _J.is_real_fresh_sample(exporter_sample, now=now) else LABEL_SAMPLE
|
| 216 |
+
|
| 217 |
+
|
| 218 |
+
# ---------------------------------------------------------------------------
|
| 219 |
+
# JobRecord — the STABLE interface Dev2/3/4 consume.
|
| 220 |
+
# ---------------------------------------------------------------------------
|
| 221 |
+
@dataclass
|
| 222 |
+
class JobRecord:
|
| 223 |
+
node: str
|
| 224 |
+
model: str
|
| 225 |
+
kind: str
|
| 226 |
+
tokens: int
|
| 227 |
+
wall_s: float
|
| 228 |
+
joules_measured: Optional[float]
|
| 229 |
+
joules_label: str
|
| 230 |
+
joules_evidence: dict
|
| 231 |
+
ts: str
|
| 232 |
+
seq: int
|
| 233 |
+
|
| 234 |
+
def to_dict(self) -> dict:
|
| 235 |
+
return {
|
| 236 |
+
"node": self.node, "model": self.model, "kind": self.kind,
|
| 237 |
+
"tokens": self.tokens, "wall_s": round(self.wall_s, 6),
|
| 238 |
+
"joules_measured": (round(self.joules_measured, 6)
|
| 239 |
+
if self.joules_measured is not None else None),
|
| 240 |
+
"joules_label": self.joules_label, "joules_evidence": self.joules_evidence,
|
| 241 |
+
"ts": self.ts, "seq": self.seq,
|
| 242 |
+
}
|
| 243 |
+
|
| 244 |
+
|
| 245 |
+
# ---------------------------------------------------------------------------
|
| 246 |
+
# Job dispatch — REAL inference against a node, or a faithful local STUB.
|
| 247 |
+
# ---------------------------------------------------------------------------
|
| 248 |
+
class _StubBackend:
|
| 249 |
+
"""Faithful local stand-in for the Ollama API when NO node is reachable.
|
| 250 |
+
|
| 251 |
+
It does REAL CPU work (a deterministic integer grind sized by `work`) so wall_s,
|
| 252 |
+
tokens, and jobs_done are HONEST measurements of actual computation — but it has
|
| 253 |
+
NO NVML meter, so its energy is ALWAYS labeled SAMPLE and never billable. Stub
|
| 254 |
+
mode is announced loudly in status. This satisfies the test mandate ("a faithful
|
| 255 |
+
local stub of the ollama API if no node reachable — clearly mark stub mode")
|
| 256 |
+
WITHOUT fabricating a single measured joule.
|
| 257 |
+
"""
|
| 258 |
+
|
| 259 |
+
def __init__(self, work: int = 200_000):
|
| 260 |
+
self.work = work
|
| 261 |
+
|
| 262 |
+
def generate(self, prompt: str) -> tuple[int, str]:
|
| 263 |
+
# Real CPU work proportional to a token budget; returns (tokens, text).
|
| 264 |
+
acc = 0
|
| 265 |
+
for i in range(self.work):
|
| 266 |
+
acc = (acc + i * 2654435761) & 0xFFFFFFFF
|
| 267 |
+
tokens = max(1, len(prompt.split()) + (acc % 16))
|
| 268 |
+
return tokens, f"[stub] computed {tokens} tokens (acc={acc})"
|
| 269 |
+
|
| 270 |
+
def embed(self, text: str) -> tuple[int, list[float]]:
|
| 271 |
+
acc = 0
|
| 272 |
+
for i in range(self.work):
|
| 273 |
+
acc = (acc + i * 40503) & 0xFFFFFFFF
|
| 274 |
+
dim = 16
|
| 275 |
+
vec = [((acc >> (j % 24)) & 0xFF) / 255.0 for j in range(dim)]
|
| 276 |
+
return dim, vec
|
| 277 |
+
|
| 278 |
+
|
| 279 |
+
def _http_reachable(base_url: str, timeout: float = 2.0) -> bool:
|
| 280 |
+
"""Liveness probe mirroring orchestrator._local_endpoint_reachable: a node is
|
| 281 |
+
reachable iff its OpenAI-compatible /models (or root) answers <500. Never raises."""
|
| 282 |
+
import urllib.request as _u
|
| 283 |
+
for path in ("/models", ""):
|
| 284 |
+
try:
|
| 285 |
+
req = _u.Request(base_url.rstrip("/") + path, method="GET")
|
| 286 |
+
with _u.urlopen(req, timeout=timeout) as r: # noqa: S310
|
| 287 |
+
if 200 <= getattr(r, "status", r.getcode()) < 500:
|
| 288 |
+
return True
|
| 289 |
+
except Exception: # noqa: BLE001
|
| 290 |
+
continue
|
| 291 |
+
return False
|
| 292 |
+
|
| 293 |
+
|
| 294 |
+
def _ollama_generate(base_url: str, model: str, prompt: str,
|
| 295 |
+
timeout: float = 60.0) -> tuple[int, str]:
|
| 296 |
+
"""REAL token generation via the OpenAI-compatible chat endpoint (same path the
|
| 297 |
+
orchestrator uses). Returns (completion_tokens, text). Raises on non-200 so the
|
| 298 |
+
caller marks the node DEGRADED rather than fabricating a result."""
|
| 299 |
+
import httpx
|
| 300 |
+
body = {"model": model, "messages": [{"role": "user", "content": prompt}],
|
| 301 |
+
"stream": False, "max_tokens": 64}
|
| 302 |
+
headers = {"Content-Type": "application/json"}
|
| 303 |
+
gpu_token = (os.environ.get("A11OY_GPU_TOKEN") or "").strip()
|
| 304 |
+
if gpu_token:
|
| 305 |
+
headers["Authorization"] = f"Bearer {gpu_token}"
|
| 306 |
+
with httpx.Client(timeout=timeout) as client:
|
| 307 |
+
resp = client.post(f"{base_url.rstrip('/')}/chat/completions",
|
| 308 |
+
headers=headers, json=body)
|
| 309 |
+
resp.raise_for_status()
|
| 310 |
+
data = resp.json()
|
| 311 |
+
text = ""
|
| 312 |
+
try:
|
| 313 |
+
text = data["choices"][0]["message"]["content"] or ""
|
| 314 |
+
except Exception: # noqa: BLE001
|
| 315 |
+
text = ""
|
| 316 |
+
usage = data.get("usage") or {}
|
| 317 |
+
tokens = int(usage.get("completion_tokens") or usage.get("total_tokens")
|
| 318 |
+
or max(1, len(text.split())))
|
| 319 |
+
return tokens, text
|
| 320 |
+
|
| 321 |
+
|
| 322 |
+
def _ollama_embed(base_url: str, model: str, text: str,
|
| 323 |
+
timeout: float = 60.0) -> tuple[int, list[float]]:
|
| 324 |
+
"""REAL embedding via Ollama's native /api/embeddings (root strips a trailing /v1)."""
|
| 325 |
+
import httpx
|
| 326 |
+
root = base_url.rstrip("/")
|
| 327 |
+
root = root[:-3] if root.endswith("/v1") else root
|
| 328 |
+
headers = {"Content-Type": "application/json"}
|
| 329 |
+
gpu_token = (os.environ.get("A11OY_GPU_TOKEN") or "").strip()
|
| 330 |
+
if gpu_token:
|
| 331 |
+
headers["Authorization"] = f"Bearer {gpu_token}"
|
| 332 |
+
with httpx.Client(timeout=timeout) as client:
|
| 333 |
+
resp = client.post(f"{root}/api/embeddings",
|
| 334 |
+
headers=headers, json={"model": model, "prompt": text})
|
| 335 |
+
resp.raise_for_status()
|
| 336 |
+
data = resp.json()
|
| 337 |
+
vec = data.get("embedding") or []
|
| 338 |
+
return len(vec), vec
|
| 339 |
+
|
| 340 |
+
|
| 341 |
+
# ---------------------------------------------------------------------------
|
| 342 |
+
# Persistent ledger / state.
|
| 343 |
+
# ---------------------------------------------------------------------------
|
| 344 |
+
@dataclass
|
| 345 |
+
class _State:
|
| 346 |
+
jobs_done: int = 0
|
| 347 |
+
seq: int = 0
|
| 348 |
+
joules_measured_total: float = 0.0 # billable only — MEASURED jobs
|
| 349 |
+
joules_sample_total: float = 0.0 # non-billable SAMPLE energy (honest, separate)
|
| 350 |
+
tokens_total: int = 0
|
| 351 |
+
measured_jobs: int = 0
|
| 352 |
+
sample_jobs: int = 0
|
| 353 |
+
by_node: dict = field(default_factory=dict) # node -> {jobs, tokens, joules_measured}
|
| 354 |
+
|
| 355 |
+
def to_dict(self) -> dict:
|
| 356 |
+
return {
|
| 357 |
+
"jobs_done": self.jobs_done, "seq": self.seq,
|
| 358 |
+
"joules_measured_total": self.joules_measured_total,
|
| 359 |
+
"joules_sample_total": self.joules_sample_total,
|
| 360 |
+
"tokens_total": self.tokens_total,
|
| 361 |
+
"measured_jobs": self.measured_jobs, "sample_jobs": self.sample_jobs,
|
| 362 |
+
"by_node": self.by_node,
|
| 363 |
+
}
|
| 364 |
+
|
| 365 |
+
@classmethod
|
| 366 |
+
def from_dict(cls, d: dict) -> "_State":
|
| 367 |
+
s = cls()
|
| 368 |
+
s.jobs_done = int(d.get("jobs_done", 0))
|
| 369 |
+
s.seq = int(d.get("seq", 0))
|
| 370 |
+
s.joules_measured_total = float(d.get("joules_measured_total", 0.0))
|
| 371 |
+
s.joules_sample_total = float(d.get("joules_sample_total", 0.0))
|
| 372 |
+
s.tokens_total = int(d.get("tokens_total", 0))
|
| 373 |
+
s.measured_jobs = int(d.get("measured_jobs", 0))
|
| 374 |
+
s.sample_jobs = int(d.get("sample_jobs", 0))
|
| 375 |
+
s.by_node = dict(d.get("by_node", {}) or {})
|
| 376 |
+
return s
|
| 377 |
+
|
| 378 |
+
|
| 379 |
+
# ---------------------------------------------------------------------------
|
| 380 |
+
# The operator daemon.
|
| 381 |
+
# ---------------------------------------------------------------------------
|
| 382 |
+
class OperatorDaemon:
|
| 383 |
+
"""Press-play operator: a background worker thread dispatching real inference
|
| 384 |
+
jobs to reachable nodes, metering MEASURED joules per job, persisting state.
|
| 385 |
+
|
| 386 |
+
Thread-safe. Idempotent start/stop. Graceful: the loop checks a stop flag between
|
| 387 |
+
jobs and joins on stop(). Restart resumes cumulative counts from the ledger.
|
| 388 |
+
"""
|
| 389 |
+
|
| 390 |
+
def __init__(self, nodes: Optional[list[NodeCfg]] = None,
|
| 391 |
+
state_path: Optional[str] = None,
|
| 392 |
+
job_interval_s: float = 0.5,
|
| 393 |
+
stub_work: int = 200_000,
|
| 394 |
+
allow_stub: bool = True):
|
| 395 |
+
self.nodes = nodes if nodes is not None else _default_nodes()
|
| 396 |
+
self.state_path = state_path or _DEFAULT_STATE_PATH
|
| 397 |
+
self.job_interval_s = max(0.0, float(job_interval_s))
|
| 398 |
+
self._stub = _StubBackend(work=stub_work)
|
| 399 |
+
self.allow_stub = allow_stub
|
| 400 |
+
|
| 401 |
+
self._lock = threading.RLock()
|
| 402 |
+
self._stop = threading.Event()
|
| 403 |
+
self._thread: Optional[threading.Thread] = None
|
| 404 |
+
self._started_at: Optional[float] = None
|
| 405 |
+
self._state = self._load_state()
|
| 406 |
+
self._node_status: dict[str, str] = {n.name: "idle" for n in self.nodes}
|
| 407 |
+
self._stub_mode = False
|
| 408 |
+
self._last_records: list[dict] = [] # rolling tail for status/dashboards
|
| 409 |
+
self._subscribers: list[Callable[[dict], None]] = []
|
| 410 |
+
self._grid_price_eur_mwh: Optional[float] = None # latest meter grid price
|
| 411 |
+
|
| 412 |
+
# -- subscription (Dev2 receipts hook) --------------------------------
|
| 413 |
+
def subscribe(self, cb: Callable[[dict], None]) -> None:
|
| 414 |
+
"""Register a callback invoked with each completed JobRecord dict (Dev2)."""
|
| 415 |
+
with self._lock:
|
| 416 |
+
self._subscribers.append(cb)
|
| 417 |
+
|
| 418 |
+
def _emit(self, rec: JobRecord) -> None:
|
| 419 |
+
d = rec.to_dict()
|
| 420 |
+
with self._lock:
|
| 421 |
+
self._last_records.append(d)
|
| 422 |
+
if len(self._last_records) > 50:
|
| 423 |
+
self._last_records.pop(0)
|
| 424 |
+
subs = list(self._subscribers)
|
| 425 |
+
for cb in subs:
|
| 426 |
+
try:
|
| 427 |
+
cb(d)
|
| 428 |
+
except Exception: # noqa: BLE001 — a bad subscriber never breaks the loop
|
| 429 |
+
pass
|
| 430 |
+
|
| 431 |
+
# -- state persistence ------------------------------------------------
|
| 432 |
+
def _load_state(self) -> _State:
|
| 433 |
+
try:
|
| 434 |
+
with open(self.state_path, "r", encoding="utf-8") as f:
|
| 435 |
+
return _State.from_dict(json.load(f))
|
| 436 |
+
except Exception: # noqa: BLE001 — missing/corrupt ledger => fresh state
|
| 437 |
+
return _State()
|
| 438 |
+
|
| 439 |
+
def _persist(self) -> None:
|
| 440 |
+
try:
|
| 441 |
+
os.makedirs(os.path.dirname(self.state_path), exist_ok=True)
|
| 442 |
+
tmp = self.state_path + ".tmp"
|
| 443 |
+
with open(tmp, "w", encoding="utf-8") as f:
|
| 444 |
+
json.dump(self._state.to_dict(), f, sort_keys=True, separators=(",", ":"))
|
| 445 |
+
os.replace(tmp, self.state_path) # atomic
|
| 446 |
+
except Exception: # noqa: BLE001 — persistence failure never crashes the loop
|
| 447 |
+
pass
|
| 448 |
+
|
| 449 |
+
# -- lifecycle --------------------------------------------------------
|
| 450 |
+
def is_running(self) -> bool:
|
| 451 |
+
return self._thread is not None and self._thread.is_alive()
|
| 452 |
+
|
| 453 |
+
def start(self) -> dict:
|
| 454 |
+
with self._lock:
|
| 455 |
+
if self.is_running():
|
| 456 |
+
return self.status()
|
| 457 |
+
self._stop.clear()
|
| 458 |
+
self._started_at = time.time()
|
| 459 |
+
self._thread = threading.Thread(target=self._run, name="szl-energy-operator",
|
| 460 |
+
daemon=True)
|
| 461 |
+
self._thread.start()
|
| 462 |
+
return self.status()
|
| 463 |
+
|
| 464 |
+
def stop(self, join_timeout: float = 10.0) -> dict:
|
| 465 |
+
self._stop.set()
|
| 466 |
+
t = self._thread
|
| 467 |
+
if t is not None:
|
| 468 |
+
t.join(timeout=join_timeout)
|
| 469 |
+
with self._lock:
|
| 470 |
+
self._thread = None
|
| 471 |
+
self._persist()
|
| 472 |
+
for k in self._node_status:
|
| 473 |
+
if self._node_status[k] == "computing":
|
| 474 |
+
self._node_status[k] = "idle"
|
| 475 |
+
return self.status()
|
| 476 |
+
|
| 477 |
+
def run_once(self) -> list[dict]:
|
| 478 |
+
"""Run exactly ONE dispatch sweep across all nodes (one job each, or stub).
|
| 479 |
+
|
| 480 |
+
Used by tests and the loop body. Returns the list of JobRecord dicts produced
|
| 481 |
+
this sweep. Honest: an unreachable node yields NO record (DEGRADED), never a fake."""
|
| 482 |
+
produced: list[dict] = []
|
| 483 |
+
meter = _fetch_joule_meter()
|
| 484 |
+
self._update_grid_price(meter)
|
| 485 |
+
any_reachable = False
|
| 486 |
+
for node in self.nodes:
|
| 487 |
+
if self._stop.is_set():
|
| 488 |
+
break
|
| 489 |
+
reachable = _http_reachable(node.base_url)
|
| 490 |
+
if not reachable:
|
| 491 |
+
with self._lock:
|
| 492 |
+
self._node_status[node.name] = "DEGRADED"
|
| 493 |
+
continue
|
| 494 |
+
any_reachable = True
|
| 495 |
+
with self._lock:
|
| 496 |
+
self._node_status[node.name] = "computing"
|
| 497 |
+
for kind in ("generate", "embed"):
|
| 498 |
+
if self._stop.is_set():
|
| 499 |
+
break
|
| 500 |
+
rec = self._run_real_job(node, kind, meter)
|
| 501 |
+
if rec is not None:
|
| 502 |
+
produced.append(rec)
|
| 503 |
+
# No reachable node at all → faithful stub (clearly marked), if allowed.
|
| 504 |
+
if not any_reachable and self.allow_stub and not self._stop.is_set():
|
| 505 |
+
self._stub_mode = True
|
| 506 |
+
for kind in ("generate", "embed"):
|
| 507 |
+
if self._stop.is_set():
|
| 508 |
+
break
|
| 509 |
+
rec = self._run_stub_job(kind)
|
| 510 |
+
if rec is not None:
|
| 511 |
+
produced.append(rec)
|
| 512 |
+
elif any_reachable:
|
| 513 |
+
self._stub_mode = False
|
| 514 |
+
# Persist after every sweep so counts are durable regardless of entry point
|
| 515 |
+
# (the loop also persists; direct run_once() callers/tests get the same).
|
| 516 |
+
self._persist()
|
| 517 |
+
return produced
|
| 518 |
+
|
| 519 |
+
def _run(self) -> None:
|
| 520 |
+
"""The non-stop loop body. Graceful: re-checks the stop flag each sweep."""
|
| 521 |
+
try:
|
| 522 |
+
while not self._stop.is_set():
|
| 523 |
+
self.run_once()
|
| 524 |
+
self._persist()
|
| 525 |
+
# Backpressure: gentle inter-sweep sleep, interruptible by stop().
|
| 526 |
+
self._stop.wait(self.job_interval_s)
|
| 527 |
+
finally:
|
| 528 |
+
self._persist()
|
| 529 |
+
|
| 530 |
+
# -- per-job execution ------------------------------------------------
|
| 531 |
+
def _commit(self, node_name: str, model: str, kind: str, tokens: int,
|
| 532 |
+
wall_s: float, exporter_sample: Optional[dict],
|
| 533 |
+
joules_measured: Optional[float]) -> JobRecord:
|
| 534 |
+
now = time.time()
|
| 535 |
+
label = _label_upper(exporter_sample, now=now)
|
| 536 |
+
evidence = _J.joules_evidence(exporter_sample, now=now) if label == LABEL_MEASURED else {}
|
| 537 |
+
billable_j = joules_measured if (label == LABEL_MEASURED and
|
| 538 |
+
joules_measured is not None and
|
| 539 |
+
joules_measured > 0) else None
|
| 540 |
+
with self._lock:
|
| 541 |
+
self._state.seq += 1
|
| 542 |
+
seq = self._state.seq
|
| 543 |
+
self._state.jobs_done += 1
|
| 544 |
+
self._state.tokens_total += int(tokens)
|
| 545 |
+
bn = self._state.by_node.setdefault(
|
| 546 |
+
node_name, {"jobs": 0, "tokens": 0, "joules_measured": 0.0})
|
| 547 |
+
bn["jobs"] += 1
|
| 548 |
+
bn["tokens"] += int(tokens)
|
| 549 |
+
if billable_j is not None:
|
| 550 |
+
self._state.joules_measured_total += billable_j
|
| 551 |
+
self._state.measured_jobs += 1
|
| 552 |
+
bn["joules_measured"] += billable_j
|
| 553 |
+
else:
|
| 554 |
+
self._state.sample_jobs += 1
|
| 555 |
+
if joules_measured is not None and joules_measured > 0:
|
| 556 |
+
self._state.joules_sample_total += joules_measured
|
| 557 |
+
rec = JobRecord(
|
| 558 |
+
node=node_name, model=model, kind=kind, tokens=int(tokens), wall_s=wall_s,
|
| 559 |
+
joules_measured=billable_j,
|
| 560 |
+
joules_label=label, joules_evidence=evidence, ts=_now_iso(), seq=seq)
|
| 561 |
+
self._emit(rec)
|
| 562 |
+
return rec
|
| 563 |
+
|
| 564 |
+
def _run_real_job(self, node: NodeCfg, kind: str,
|
| 565 |
+
meter_before: Optional[dict]) -> Optional[dict]:
|
| 566 |
+
"""Dispatch one real inference job; meter NVML energy across its wall window.
|
| 567 |
+
|
| 568 |
+
joules_measured for the job = (cumulative joules AFTER) − (cumulative joules
|
| 569 |
+
BEFORE) from the node's exporter engine, but ONLY when both samples are real &
|
| 570 |
+
fresh (<30s). Otherwise the job's energy is SAMPLE and excluded from billable.
|
| 571 |
+
A node error → DEGRADED + None (never a fabricated job)."""
|
| 572 |
+
sample_before = _exporter_sample_for_node(meter_before, node.exporter_node)
|
| 573 |
+
j_before = (sample_before or {}).get("joules_measured_total")
|
| 574 |
+
t0 = time.time()
|
| 575 |
+
try:
|
| 576 |
+
if kind == "generate":
|
| 577 |
+
prompt = _GEN_PROMPTS[self._state.seq % len(_GEN_PROMPTS)]
|
| 578 |
+
tokens, _ = _ollama_generate(node.base_url, node.gen_model, prompt)
|
| 579 |
+
model = node.gen_model
|
| 580 |
+
else:
|
| 581 |
+
text = _EMBED_TEXTS[self._state.seq % len(_EMBED_TEXTS)]
|
| 582 |
+
tokens, _ = _ollama_embed(node.base_url, node.embed_model, text)
|
| 583 |
+
model = node.embed_model
|
| 584 |
+
except Exception: # noqa: BLE001 — node failed mid-job: DEGRADED, never faked
|
| 585 |
+
with self._lock:
|
| 586 |
+
self._node_status[node.name] = "DEGRADED"
|
| 587 |
+
return None
|
| 588 |
+
wall_s = time.time() - t0
|
| 589 |
+
meter_after = _fetch_joule_meter()
|
| 590 |
+
sample_after = _exporter_sample_for_node(meter_after, node.exporter_node)
|
| 591 |
+
j_after = (sample_after or {}).get("joules_measured_total")
|
| 592 |
+
joules_measured = None
|
| 593 |
+
if (isinstance(j_before, (int, float)) and isinstance(j_after, (int, float))
|
| 594 |
+
and j_after >= j_before):
|
| 595 |
+
joules_measured = float(j_after) - float(j_before)
|
| 596 |
+
# The label is decided off the AFTER sample (the fresh reading at job end).
|
| 597 |
+
rec = self._commit(node.name, model, kind, tokens, wall_s,
|
| 598 |
+
sample_after, joules_measured)
|
| 599 |
+
return rec.to_dict()
|
| 600 |
+
|
| 601 |
+
def _run_stub_job(self, kind: str) -> Optional[dict]:
|
| 602 |
+
"""Faithful local stub job: REAL CPU work (honest wall_s/tokens) but NO meter,
|
| 603 |
+
so energy is ALWAYS SAMPLE and never billable. Clearly attributed to *-stub."""
|
| 604 |
+
t0 = time.time()
|
| 605 |
+
if kind == "generate":
|
| 606 |
+
prompt = _GEN_PROMPTS[self._state.seq % len(_GEN_PROMPTS)]
|
| 607 |
+
tokens, _ = self._stub.generate(prompt)
|
| 608 |
+
model = "stub-llama"
|
| 609 |
+
else:
|
| 610 |
+
text = _EMBED_TEXTS[self._state.seq % len(_EMBED_TEXTS)]
|
| 611 |
+
tokens, _ = self._stub.embed(text)
|
| 612 |
+
model = "stub-embed"
|
| 613 |
+
wall_s = time.time() - t0
|
| 614 |
+
# No exporter sample → label SAMPLE, joules None, never billable.
|
| 615 |
+
rec = self._commit("local-stub", model, kind, tokens, wall_s, None, None)
|
| 616 |
+
with self._lock:
|
| 617 |
+
self._node_status["local-stub"] = "computing (STUB)"
|
| 618 |
+
return rec.to_dict()
|
| 619 |
+
|
| 620 |
+
def _update_grid_price(self, meter: Optional[dict]) -> None:
|
| 621 |
+
try:
|
| 622 |
+
totals = (meter or {}).get("totals") or {}
|
| 623 |
+
gp = totals.get("eur_per_mwh")
|
| 624 |
+
if isinstance(gp, (int, float)):
|
| 625 |
+
with self._lock:
|
| 626 |
+
self._grid_price_eur_mwh = float(gp)
|
| 627 |
+
except Exception: # noqa: BLE001
|
| 628 |
+
pass
|
| 629 |
+
|
| 630 |
+
# -- status -----------------------------------------------------------
|
| 631 |
+
def status(self) -> dict:
|
| 632 |
+
with self._lock:
|
| 633 |
+
uptime = (time.time() - self._started_at) if self._started_at else 0.0
|
| 634 |
+
computing = [n for n, s in self._node_status.items()
|
| 635 |
+
if s in ("computing", "computing (STUB)")]
|
| 636 |
+
degraded = [n for n, s in self._node_status.items() if s == "DEGRADED"]
|
| 637 |
+
st = self._state
|
| 638 |
+
return {
|
| 639 |
+
"service": "energy-operator",
|
| 640 |
+
"doctrine": DOCTRINE,
|
| 641 |
+
"running": self.is_running(),
|
| 642 |
+
"stub_mode": self._stub_mode,
|
| 643 |
+
"jobs_done": st.jobs_done,
|
| 644 |
+
"joules_measured_total": round(st.joules_measured_total, 6),
|
| 645 |
+
"joules_measured_label": LABEL_MEASURED,
|
| 646 |
+
"joules_sample_total": round(st.joules_sample_total, 6),
|
| 647 |
+
"joules_sample_label": LABEL_SAMPLE,
|
| 648 |
+
"tokens_total": st.tokens_total,
|
| 649 |
+
"measured_jobs": st.measured_jobs,
|
| 650 |
+
"sample_jobs": st.sample_jobs,
|
| 651 |
+
"nodes_computing": computing,
|
| 652 |
+
"nodes_degraded": degraded,
|
| 653 |
+
"node_status": dict(self._node_status),
|
| 654 |
+
"by_node": {k: dict(v) for k, v in st.by_node.items()},
|
| 655 |
+
"uptime_s": round(uptime, 3),
|
| 656 |
+
"grid_price_eur_mwh": self._grid_price_eur_mwh,
|
| 657 |
+
"recent_jobs": list(self._last_records[-10:]),
|
| 658 |
+
"exporter": _JOULE_METER_URL,
|
| 659 |
+
"honesty": (
|
| 660 |
+
"joules_measured_total is the SUM of per-job MEASURED NVML deltas "
|
| 661 |
+
"(fresh <30s) ONLY — the billable figure. SAMPLE energy (stale meter "
|
| 662 |
+
"or stub mode) is tracked separately and NEVER billable. Unreachable "
|
| 663 |
+
"nodes are DEGRADED, never faked. STUB MODE means no GPU node was "
|
| 664 |
+
"reachable from this process; stub energy is SAMPLE by construction."
|
| 665 |
+
),
|
| 666 |
+
"computed_at": _now_iso(),
|
| 667 |
+
}
|
| 668 |
+
|
| 669 |
+
def install_signal_handlers(self) -> None:
|
| 670 |
+
"""Optional: graceful SIGINT/SIGTERM → stop(). Main-thread only (never raises)."""
|
| 671 |
+
def _handler(signum, frame): # noqa: ANN001
|
| 672 |
+
self.stop()
|
| 673 |
+
for sig in (signal.SIGINT, signal.SIGTERM):
|
| 674 |
+
try:
|
| 675 |
+
signal.signal(sig, _handler)
|
| 676 |
+
except Exception: # noqa: BLE001 — non-main-thread / unsupported platform
|
| 677 |
+
pass
|
| 678 |
+
|
| 679 |
+
|
| 680 |
+
# Module-level singleton the endpoints drive (one operator per process).
|
| 681 |
+
_OPERATOR: Optional[OperatorDaemon] = None
|
| 682 |
+
_OPERATOR_LOCK = threading.Lock()
|
| 683 |
+
|
| 684 |
+
|
| 685 |
+
def get_operator() -> OperatorDaemon:
|
| 686 |
+
global _OPERATOR
|
| 687 |
+
with _OPERATOR_LOCK:
|
| 688 |
+
if _OPERATOR is None:
|
| 689 |
+
_OPERATOR = OperatorDaemon()
|
| 690 |
+
return _OPERATOR
|
| 691 |
+
|
| 692 |
+
|
| 693 |
+
# ---------------------------------------------------------------------------
|
| 694 |
+
# Registration — dual-register under /api/{ns}/v1/energy/operator/* AND
|
| 695 |
+
# /v1/energy/operator/* (mirrors the add_api_route pattern used across the repo).
|
| 696 |
+
# ---------------------------------------------------------------------------
|
| 697 |
+
def register(app, ns: str = "a11oy") -> dict:
|
| 698 |
+
from fastapi import Request
|
| 699 |
+
from fastapi.responses import JSONResponse
|
| 700 |
+
|
| 701 |
+
op = get_operator()
|
| 702 |
+
|
| 703 |
+
async def _start(request: Request): # noqa: ANN202
|
| 704 |
+
return JSONResponse(op.start())
|
| 705 |
+
|
| 706 |
+
async def _stop(request: Request): # noqa: ANN202
|
| 707 |
+
return JSONResponse(op.stop())
|
| 708 |
+
|
| 709 |
+
async def _status(): # noqa: ANN202
|
| 710 |
+
return JSONResponse(op.status())
|
| 711 |
+
|
| 712 |
+
prefixes = [f"/api/{ns}/v1/energy/operator", "/v1/energy/operator"]
|
| 713 |
+
routes: list[str] = []
|
| 714 |
+
for p in prefixes:
|
| 715 |
+
app.add_api_route(f"{p}/start", _start, methods=["POST"], include_in_schema=True)
|
| 716 |
+
app.add_api_route(f"{p}/stop", _stop, methods=["POST"], include_in_schema=True)
|
| 717 |
+
app.add_api_route(f"{p}/status", _status, methods=["GET"], include_in_schema=True)
|
| 718 |
+
routes.extend([f"{p}/start", f"{p}/stop", f"{p}/status"])
|
| 719 |
+
|
| 720 |
+
print(f"[{ns}] szl_energy_operator routes registered "
|
| 721 |
+
f"(press-play operator, {len(routes)} routes)", flush=True)
|
| 722 |
+
return {"ok": True, "ns": ns, "routes": routes}
|
| 723 |
+
|
| 724 |
+
|
| 725 |
+
# ---------------------------------------------------------------------------
|
| 726 |
+
# No-server self-test (proves honesty gates without a live GPU).
|
| 727 |
+
# ---------------------------------------------------------------------------
|
| 728 |
+
def _selftest() -> dict:
|
| 729 |
+
import tempfile
|
| 730 |
+
out: dict = {}
|
| 731 |
+
now = 1_000_000.0
|
| 732 |
+
|
| 733 |
+
# (a) Fresh real meter sample for 'betterwithage' => MEASURED label.
|
| 734 |
+
meter = {"engines": [{"engine": "betterwithage", "joules": 78369.586,
|
| 735 |
+
"gpus": [{"power_w": 9.74, "live": True}]}],
|
| 736 |
+
"totals": {"joules": 78369.586, "eur_per_mwh": 62.08}}
|
| 737 |
+
s = _exporter_sample_for_node(meter, "betterwithage", now=now)
|
| 738 |
+
assert s is not None and _label_upper(s, now=now) == LABEL_MEASURED, s
|
| 739 |
+
out["fresh_sample_measured"] = True
|
| 740 |
+
|
| 741 |
+
# (b) Missing node => no sample => SAMPLE label (never fabricated).
|
| 742 |
+
assert _exporter_sample_for_node(meter, "ghost-node", now=now) is None
|
| 743 |
+
assert _label_upper(None, now=now) == LABEL_SAMPLE
|
| 744 |
+
out["missing_node_sample"] = True
|
| 745 |
+
|
| 746 |
+
# (c) Stub mode: no reachable node, stub does real work, energy SAMPLE, not billable.
|
| 747 |
+
with tempfile.TemporaryDirectory() as d:
|
| 748 |
+
path = os.path.join(d, "ledger.json")
|
| 749 |
+
op = OperatorDaemon(
|
| 750 |
+
nodes=[NodeCfg("rtx-betterwithage", "http://192.0.2.1:11434/v1",
|
| 751 |
+
"llama3.1:8b", "bge-large", "betterwithage")],
|
| 752 |
+
state_path=path, stub_work=5000)
|
| 753 |
+
produced = op.run_once()
|
| 754 |
+
assert len(produced) >= 2, produced # generate + embed stub jobs
|
| 755 |
+
assert op._stub_mode is True
|
| 756 |
+
st = op.status()
|
| 757 |
+
assert st["jobs_done"] >= 2 and st["tokens_total"] > 0, st
|
| 758 |
+
assert st["joules_measured_total"] == 0.0, st # stub energy never billable
|
| 759 |
+
assert "rtx-betterwithage" in st["nodes_degraded"], st # unreachable => DEGRADED
|
| 760 |
+
out["stub_real_work_no_billable_joules"] = True
|
| 761 |
+
|
| 762 |
+
# (d) Restart resumes from persisted ledger.
|
| 763 |
+
op2 = OperatorDaemon(nodes=op.nodes, state_path=path, stub_work=5000)
|
| 764 |
+
assert op2.status()["jobs_done"] == st["jobs_done"], "restart must resume counts"
|
| 765 |
+
out["restart_resumes_state"] = True
|
| 766 |
+
|
| 767 |
+
# (e) MEASURED billable accounting via the commit path (synthetic fresh sample).
|
| 768 |
+
with tempfile.TemporaryDirectory() as d:
|
| 769 |
+
op = OperatorDaemon(nodes=[], state_path=os.path.join(d, "l.json"))
|
| 770 |
+
sample = {"joules_measured_total": 100.0, "exporter_node": "betterwithage",
|
| 771 |
+
"exporter_last_seen_ts": time.time(), "power_w_sample": 200.0}
|
| 772 |
+
rec = op._commit("betterwithage", "llama3.1:8b", "generate", 42, 1.5, sample, 12.5)
|
| 773 |
+
assert rec.joules_label == LABEL_MEASURED and rec.joules_measured == 12.5, rec
|
| 774 |
+
assert op.status()["joules_measured_total"] == 12.5
|
| 775 |
+
out["measured_commit_billable"] = True
|
| 776 |
+
|
| 777 |
+
return out
|
| 778 |
+
|
| 779 |
+
|
| 780 |
+
if __name__ == "__main__":
|
| 781 |
+
import sys as _sys
|
| 782 |
+
print("=" * 70)
|
| 783 |
+
print("szl_energy_operator — self-test (honesty gates, no live GPU)")
|
| 784 |
+
print("=" * 70)
|
| 785 |
+
res = _selftest()
|
| 786 |
+
print(json.dumps(res, indent=2))
|
| 787 |
+
ok = all(res.values())
|
| 788 |
+
print("\nSELFTEST", "PASS" if ok else "FAIL")
|
| 789 |
+
_sys.exit(0 if ok else 1)
|