betterwithage commited on
Commit
b0c66c5
·
verified ·
1 Parent(s): b069295

chore(sync): mirror backend .py + Dockerfile to Space (hf-sync-backend)

Browse files

Automated backend sync from szl-holdings/a11oy main via hf-sync-backend.
Updated (differed from the Space): Dockerfile, serve.py, szl_energy_operator.py
Deleted (gone from the repo + Dockerfile COPY set): (none)

Keeps the Space-built backend (serve.py + the Dockerfile-COPY'd .py
modules) identical to GitHub main so the Space never rebuilds from a
stale backend, new endpoints don't 404 there, and orphaned modules
removed from the repo don't linger in the Space tree.

Files changed (3) hide show
  1. Dockerfile +1 -1
  2. serve.py +17 -0
  3. szl_energy_operator.py +789 -0
Dockerfile CHANGED
@@ -99,7 +99,7 @@ COPY szl_formula_wiring.py a11oy_code_engine.py a11oy_code.py a11oy_seismic.py s
99
  COPY szl_energy_budget.py szl_energy_sovereign.py szl_energy_provenance.py szl_heart_blood.py szl_engine_status.py szl_backend_hardening.py revenue_endpoints.py a11oy_harvest_endpoints.py ./
100
  # energy operator/ledger/projection modules — imported by serve.py (guarded);
101
  # MUST be per-file COPY'd (this Dockerfile uses no `COPY . .`) or the import falls back to a STUB.
102
- COPY joule_billing.py szl_energy_ledger.py ./
103
  # ADDITIVE (I3): FABRO-style Governed Factory + Constitutional Engines modules.
104
  # MUST be COPY'd or serve.py's guarded imports fall back (merged-but-not-live).
105
  # HTML/JS is inlined in these .py modules, so NO web/ or static-vendor COPY needed.
 
99
  COPY szl_energy_budget.py szl_energy_sovereign.py szl_energy_provenance.py szl_heart_blood.py szl_engine_status.py szl_backend_hardening.py revenue_endpoints.py a11oy_harvest_endpoints.py ./
100
  # energy operator/ledger/projection modules — imported by serve.py (guarded);
101
  # MUST be per-file COPY'd (this Dockerfile uses no `COPY . .`) or the import falls back to a STUB.
102
+ COPY joule_billing.py szl_energy_ledger.py szl_energy_operator.py ./
103
  # ADDITIVE (I3): FABRO-style Governed Factory + Constitutional Engines modules.
104
  # MUST be COPY'd or serve.py's guarded imports fall back (merged-but-not-live).
105
  # HTML/JS is inlined in these .py modules, so NO web/ or static-vendor COPY needed.
serve.py CHANGED
@@ -263,6 +263,23 @@ try:
263
  except Exception as _szl_es_e: # pragma: no cover
264
  print(f"[a11oy] Energy/Sovereign-Compute NOT registered: {_szl_es_e!r}", file=__import__("sys").stderr)
265
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
266
  # ── Sovereign VRAM-resident GPU-QUANT ENGINE (gpu-quant) — three honest layers on the
267
  # a11oy finance surface: L1 PCA-Risk (Ledoit-Wolf shrinkage Σ̂_LW + Marchenko-Pastur λ⁺
268
  # eigenvalue clipping; cuML on GPU else PURE-STDLIB CPU fallback, label honest), L2 TDA-
 
263
  except Exception as _szl_es_e: # pragma: no cover
264
  print(f"[a11oy] Energy/Sovereign-Compute NOT registered: {_szl_es_e!r}", file=__import__("sys").stderr)
265
 
266
+ # ── PRESS-PLAY ENERGY OPERATOR daemon (Dev 1) — dispatches a CONTINUOUS stream of
267
+ # REAL inference jobs to the reachable Ollama GPU nodes (rtx-betterwithage, chaski),
268
+ # meters MEASURED joules per job off the EXISTING betterwithage NVML joule-meter, and
269
+ # emits a JobRecord per completed job (the Dev2 receipts / Dev3 projection / Dev4
270
+ # dashboard contract). Graceful start/stop, backpressure, ledger state-persistence so
271
+ # a restart resumes counts. HONEST: unreachable node -> DEGRADED (never faked); stale
272
+ # meter (>30s) -> SAMPLE energy excluded from billable; no node reachable -> a clearly
273
+ # marked local STUB (real CPU work, SAMPLE energy only). Adds POST/GET
274
+ # /api/a11oy/v1/energy/operator/{start,stop,status} (dual-registered under /v1/* too).
275
+ # Additive, try/except-guarded, before the SPA catch-all.
276
+ try:
277
+ import szl_energy_operator as _szl_energy_operator
278
+ _szl_energy_operator.register(app, ns="a11oy")
279
+ print("[a11oy] Energy Operator registered: /api/a11oy/v1/energy/operator/{start,stop,status}", file=__import__("sys").stderr)
280
+ except Exception as _szl_op_e: # pragma: no cover
281
+ print(f"[a11oy] Energy Operator NOT registered: {_szl_op_e!r}", file=__import__("sys").stderr)
282
+
283
  # ── Sovereign VRAM-resident GPU-QUANT ENGINE (gpu-quant) — three honest layers on the
284
  # a11oy finance surface: L1 PCA-Risk (Ledoit-Wolf shrinkage Σ̂_LW + Marchenko-Pastur λ⁺
285
  # eigenvalue clipping; cuML on GPU else PURE-STDLIB CPU fallback, label honest), L2 TDA-
szl_energy_operator.py ADDED
@@ -0,0 +1,789 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ # © 2026 Lutar, Stephen P. — SZL Holdings · ORCID 0009-0001-0110-4173
3
+ # Doctrine v11 — the press-play ENERGY OPERATOR daemon (Dev 1, backend).
4
+ """
5
+ szl_energy_operator.py — PRESS PLAY. The operational loop.
6
+
7
+ On START this daemon dispatches a CONTINUOUS stream of REAL inference jobs to the
8
+ reachable Ollama GPU nodes (rtx-betterwithage, chaski) — small honest workloads
9
+ (short token-generation prompts and/or embeddings) so the rig genuinely computes
10
+ and burns measurable energy. Per job it records start/end wall-time, pulls an NVML
11
+ power/energy sample from the EXISTING exporter path (the betterwithage joule-meter
12
+ that already feeds szl_energy_sovereign's metrics panel / harvest posture
13
+ joules_evidence), and computes joules_measured for that job.
14
+
15
+ Honesty (Doctrine v11 — NEVER violate):
16
+ - Joules are MEASURED only from a REAL, FRESH (<30s) NVML exporter delta. The
17
+ label is decided SOLELY by szl_joules_truth — never off a flag.
18
+ - If the exporter sample is stale (>30s) or unavailable, the job's energy is
19
+ labeled SAMPLE and EXCLUDED from billable totals. We never fabricate a joule.
20
+ - If a GPU node is unreachable, we SKIP it and mark it DEGRADED in status —
21
+ we NEVER fabricate a job or a joule for a node that didn't compute.
22
+ - Sandbox / no-GPU: a faithful local STUB of the Ollama API does REAL CPU work
23
+ (so wall-time + jobs_done are honest) but its energy is ALWAYS labeled SAMPLE
24
+ and never billable. Stub mode is announced loudly in every status payload.
25
+
26
+ Interface other devs build against (STABLE — Dev2 receipts / Dev3 projection /
27
+ Dev4 dashboard consume this):
28
+
29
+ JobRecord = {
30
+ "node": str, # which node computed it (or "<node>-stub")
31
+ "model": str, # model tag used
32
+ "kind": str, # "generate" | "embed"
33
+ "tokens": int, # MEASURED tokens produced/consumed
34
+ "wall_s": float, # MEASURED wall-clock seconds for the job
35
+ "joules_measured": float|None, # MEASURED joules iff joules_label=="MEASURED", else None
36
+ "joules_label": str, # "MEASURED" | "SAMPLE" (billing.py-compatible upper-case)
37
+ "joules_evidence": dict, # self-verifying exporter evidence (empty unless MEASURED)
38
+ "ts": str, # ISO-8601 UTC completion time
39
+ "seq": int, # monotonic job sequence number (ledger order)
40
+ }
41
+
42
+ on_job(JobRecord) callback — register via OperatorDaemon.subscribe(cb) so Dev2
43
+ can mint a JouleCharge receipt per completed job in real time.
44
+
45
+ Endpoints (dual-registered under /api/{ns}/v1/energy/operator/* AND /v1/energy/operator/*):
46
+ POST /energy/operator/start — press play (idempotent; returns running state)
47
+ POST /energy/operator/stop — graceful stop (idempotent)
48
+ GET /energy/operator/status — running?, jobs_done, joules_measured_total,
49
+ tokens_total, nodes computing, uptime, degraded nodes
50
+
51
+ Graceful start/stop via a threading stop-flag + optional SIGINT/SIGTERM handler.
52
+ Backpressure: a per-node in-flight cap + inter-job sleep so we never overwhelm a node.
53
+ State persists to a JSON ledger so a restart RESUMES the cumulative counts.
54
+
55
+ Pure stdlib + httpx (already a repo dep) + FastAPI. No Node, no CDN.
56
+ """
57
+ from __future__ import annotations
58
+
59
+ import json
60
+ import os
61
+ import signal
62
+ import threading
63
+ import time
64
+ import urllib.request
65
+ from dataclasses import dataclass, field
66
+ from datetime import datetime, timezone
67
+ from typing import Any, Callable, Optional
68
+
69
+ try:
70
+ import szl_joules_truth as _J
71
+ except Exception: # pragma: no cover — packaged import fallback
72
+ from . import szl_joules_truth as _J # type: ignore
73
+
74
+ # ---------------------------------------------------------------------------
75
+ # Doctrine constants.
76
+ # ---------------------------------------------------------------------------
77
+ DOCTRINE = "v11"
78
+ # joule_billing.py refuses to bill unless the label is MEASURED with an NVML sample
79
+ # fresher than this. We mirror it EXACTLY so the operator's MEASURED↔SAMPLE split
80
+ # is the same gate the billing core uses (no divergence).
81
+ MAX_NVML_AGE_S = 30.0
82
+ # billing.py uses upper-case labels; szl_joules_truth uses lower-case. We map at the
83
+ # boundary so JobRecord.joules_label is billing-compatible and self-consistent.
84
+ LABEL_MEASURED = "MEASURED"
85
+ LABEL_SAMPLE = "SAMPLE"
86
+
87
+ # The EXISTING exporter path: the betterwithage NVML joule-meter that
88
+ # szl_energy_sovereign._metrics_panel() already reads (engines→gpus→{power_w,joules,live},
89
+ # totals→{joules}). We reuse the SAME URL so the operator meters off the same source.
90
+ _JOULE_METER_URL = os.environ.get("A11OY_JOULE_METER_URL", "http://100.96.129.45:9471/")
91
+
92
+ # Ledger / state file. Survives restart so cumulative counts resume. Override for tests.
93
+ _DEFAULT_STATE_PATH = os.environ.get(
94
+ "A11OY_OPERATOR_STATE", os.path.join(os.path.dirname(os.path.abspath(__file__)),
95
+ "artifacts", "energy_operator_ledger.json"))
96
+
97
+
98
+ def _now_iso() -> str:
99
+ return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
100
+
101
+
102
+ # ---------------------------------------------------------------------------
103
+ # Node configuration — the reachable Ollama GPU nodes (GROUND TRUTH 2026-06-14).
104
+ # rtx-betterwithage is the sovereign box (A11OY_MODEL_BASE_URL); chaski is the
105
+ # tailnet box. Each node is an OpenAI-compatible Ollama endpoint — same serving
106
+ # path a11oy_code_orchestrator._call_model uses (POST {base}/chat/completions),
107
+ # plus Ollama's native /api/embeddings for the embed workload.
108
+ # ---------------------------------------------------------------------------
109
+ @dataclass
110
+ class NodeCfg:
111
+ name: str
112
+ base_url: str # OpenAI-compatible base, e.g. http://host:11434/v1
113
+ gen_model: str # small token-generation model tag
114
+ embed_model: str # embeddings model tag
115
+ exporter_node: str # label this node reports in the joule-meter engines list
116
+
117
+
118
+ def _default_nodes() -> list[NodeCfg]:
119
+ """Build node configs from env (token-flip law: read at runtime, never hardcode a key).
120
+
121
+ rtx-betterwithage base resolves from A11OY_MODEL_BASE_URL (the same env the
122
+ orchestrator uses) when it points at a non-router endpoint. chaski base from
123
+ A11OY_CHASKI_BASE_URL. Both default to honest placeholders that simply won't be
124
+ reachable from the sandbox → DEGRADED (never faked)."""
125
+ btw_base = (os.environ.get("A11OY_MODEL_BASE_URL") or "").strip().rstrip("/")
126
+ if not btw_base or "router.huggingface.co" in btw_base:
127
+ btw_base = os.environ.get("A11OY_BETTERWITHAGE_BASE_URL",
128
+ "http://rtx-betterwithage:11434/v1").rstrip("/")
129
+ chaski_base = os.environ.get("A11OY_CHASKI_BASE_URL",
130
+ "http://chaski:11434/v1").rstrip("/")
131
+ return [
132
+ NodeCfg(
133
+ name="rtx-betterwithage",
134
+ base_url=btw_base,
135
+ gen_model=os.environ.get("A11OY_BTW_GEN_MODEL", "llama3.1:8b"),
136
+ embed_model=os.environ.get("A11OY_BTW_EMBED_MODEL", "bge-large"),
137
+ exporter_node=os.environ.get("A11OY_GPU_LABEL", "betterwithage"),
138
+ ),
139
+ NodeCfg(
140
+ name="chaski",
141
+ base_url=chaski_base,
142
+ gen_model=os.environ.get("A11OY_CHASKI_GEN_MODEL", "qwen2.5:32b"),
143
+ embed_model=os.environ.get("A11OY_CHASKI_EMBED_MODEL", "mistral"),
144
+ exporter_node=os.environ.get("A11OY_CHASKI_GPU_LABEL", "chaski"),
145
+ ),
146
+ ]
147
+
148
+
149
+ # Small, honest workloads — short prompts so each job genuinely computes but the loop
150
+ # stays gentle (backpressure). Rotated so the rig does varied real work.
151
+ _GEN_PROMPTS = [
152
+ "In one sentence, what is sovereign compute?",
153
+ "Name one law of thermodynamics in a single line.",
154
+ "Give a one-line definition of a joule.",
155
+ "Summarize energy provenance in one short sentence.",
156
+ ]
157
+ _EMBED_TEXTS = [
158
+ "sovereign metered compute receipt",
159
+ "measured joules to billable charge",
160
+ "energy provenance under doctrine v11",
161
+ ]
162
+
163
+
164
+ # ---------------------------------------------------------------------------
165
+ # Exporter sampling — reuse the EXISTING betterwithage joule-meter path.
166
+ # ---------------------------------------------------------------------------
167
+ def _fetch_joule_meter(timeout: float = 4.0) -> Optional[dict]:
168
+ """Fetch the live NVML joule-meter JSON, or None on any failure (honest)."""
169
+ try:
170
+ req = urllib.request.Request(
171
+ _JOULE_METER_URL, headers={"User-Agent": "szl-energy-operator"})
172
+ with urllib.request.urlopen(req, timeout=timeout) as r: # noqa: S310
173
+ return json.loads(r.read().decode("utf-8", "replace"))
174
+ except Exception: # noqa: BLE001 — unreachable meter => no sample, stay honest
175
+ return None
176
+
177
+
178
+ def _exporter_sample_for_node(meter: Optional[dict], exporter_node: str,
179
+ now: Optional[float] = None) -> Optional[dict]:
180
+ """Build a szl_joules_truth exporter_sample for one node from the meter JSON.
181
+
182
+ The meter shape (mirrors szl_energy_sovereign._metrics_panel): engines[].{engine,
183
+ joules, gpus[].{power_w, joules, live}}, totals.{joules}. We pick the engine whose
184
+ name matches exporter_node; its cumulative joules + a fresh wall-clock ts give a
185
+ real reading. Returns None when the node isn't present / has no numeric joules.
186
+ """
187
+ if not isinstance(meter, dict):
188
+ return None
189
+ now = time.time() if now is None else now
190
+ engines = meter.get("engines") or []
191
+ for e in engines:
192
+ if str(e.get("engine") or "").lower() != exporter_node.lower():
193
+ continue
194
+ joules = e.get("joules")
195
+ if not isinstance(joules, (int, float)):
196
+ continue
197
+ power_w = None
198
+ for g in (e.get("gpus") or []):
199
+ if g.get("live") and isinstance(g.get("power_w"), (int, float)):
200
+ power_w = float(g["power_w"])
201
+ break
202
+ return {
203
+ "joules_measured_total": float(joules),
204
+ "exporter_node": exporter_node,
205
+ # The meter scraped just now → fresh by construction (same convention as
206
+ # szl_energy_sovereign._exporter_sample_from_metrics).
207
+ "exporter_last_seen_ts": now,
208
+ "power_w_sample": power_w,
209
+ }
210
+ return None
211
+
212
+
213
+ def _label_upper(exporter_sample: Optional[dict], now: Optional[float] = None) -> str:
214
+ """Map szl_joules_truth's lower-case label to billing.py's upper-case label."""
215
+ return LABEL_MEASURED if _J.is_real_fresh_sample(exporter_sample, now=now) else LABEL_SAMPLE
216
+
217
+
218
+ # ---------------------------------------------------------------------------
219
+ # JobRecord — the STABLE interface Dev2/3/4 consume.
220
+ # ---------------------------------------------------------------------------
221
+ @dataclass
222
+ class JobRecord:
223
+ node: str
224
+ model: str
225
+ kind: str
226
+ tokens: int
227
+ wall_s: float
228
+ joules_measured: Optional[float]
229
+ joules_label: str
230
+ joules_evidence: dict
231
+ ts: str
232
+ seq: int
233
+
234
+ def to_dict(self) -> dict:
235
+ return {
236
+ "node": self.node, "model": self.model, "kind": self.kind,
237
+ "tokens": self.tokens, "wall_s": round(self.wall_s, 6),
238
+ "joules_measured": (round(self.joules_measured, 6)
239
+ if self.joules_measured is not None else None),
240
+ "joules_label": self.joules_label, "joules_evidence": self.joules_evidence,
241
+ "ts": self.ts, "seq": self.seq,
242
+ }
243
+
244
+
245
+ # ---------------------------------------------------------------------------
246
+ # Job dispatch — REAL inference against a node, or a faithful local STUB.
247
+ # ---------------------------------------------------------------------------
248
+ class _StubBackend:
249
+ """Faithful local stand-in for the Ollama API when NO node is reachable.
250
+
251
+ It does REAL CPU work (a deterministic integer grind sized by `work`) so wall_s,
252
+ tokens, and jobs_done are HONEST measurements of actual computation — but it has
253
+ NO NVML meter, so its energy is ALWAYS labeled SAMPLE and never billable. Stub
254
+ mode is announced loudly in status. This satisfies the test mandate ("a faithful
255
+ local stub of the ollama API if no node reachable — clearly mark stub mode")
256
+ WITHOUT fabricating a single measured joule.
257
+ """
258
+
259
+ def __init__(self, work: int = 200_000):
260
+ self.work = work
261
+
262
+ def generate(self, prompt: str) -> tuple[int, str]:
263
+ # Real CPU work proportional to a token budget; returns (tokens, text).
264
+ acc = 0
265
+ for i in range(self.work):
266
+ acc = (acc + i * 2654435761) & 0xFFFFFFFF
267
+ tokens = max(1, len(prompt.split()) + (acc % 16))
268
+ return tokens, f"[stub] computed {tokens} tokens (acc={acc})"
269
+
270
+ def embed(self, text: str) -> tuple[int, list[float]]:
271
+ acc = 0
272
+ for i in range(self.work):
273
+ acc = (acc + i * 40503) & 0xFFFFFFFF
274
+ dim = 16
275
+ vec = [((acc >> (j % 24)) & 0xFF) / 255.0 for j in range(dim)]
276
+ return dim, vec
277
+
278
+
279
+ def _http_reachable(base_url: str, timeout: float = 2.0) -> bool:
280
+ """Liveness probe mirroring orchestrator._local_endpoint_reachable: a node is
281
+ reachable iff its OpenAI-compatible /models (or root) answers <500. Never raises."""
282
+ import urllib.request as _u
283
+ for path in ("/models", ""):
284
+ try:
285
+ req = _u.Request(base_url.rstrip("/") + path, method="GET")
286
+ with _u.urlopen(req, timeout=timeout) as r: # noqa: S310
287
+ if 200 <= getattr(r, "status", r.getcode()) < 500:
288
+ return True
289
+ except Exception: # noqa: BLE001
290
+ continue
291
+ return False
292
+
293
+
294
+ def _ollama_generate(base_url: str, model: str, prompt: str,
295
+ timeout: float = 60.0) -> tuple[int, str]:
296
+ """REAL token generation via the OpenAI-compatible chat endpoint (same path the
297
+ orchestrator uses). Returns (completion_tokens, text). Raises on non-200 so the
298
+ caller marks the node DEGRADED rather than fabricating a result."""
299
+ import httpx
300
+ body = {"model": model, "messages": [{"role": "user", "content": prompt}],
301
+ "stream": False, "max_tokens": 64}
302
+ headers = {"Content-Type": "application/json"}
303
+ gpu_token = (os.environ.get("A11OY_GPU_TOKEN") or "").strip()
304
+ if gpu_token:
305
+ headers["Authorization"] = f"Bearer {gpu_token}"
306
+ with httpx.Client(timeout=timeout) as client:
307
+ resp = client.post(f"{base_url.rstrip('/')}/chat/completions",
308
+ headers=headers, json=body)
309
+ resp.raise_for_status()
310
+ data = resp.json()
311
+ text = ""
312
+ try:
313
+ text = data["choices"][0]["message"]["content"] or ""
314
+ except Exception: # noqa: BLE001
315
+ text = ""
316
+ usage = data.get("usage") or {}
317
+ tokens = int(usage.get("completion_tokens") or usage.get("total_tokens")
318
+ or max(1, len(text.split())))
319
+ return tokens, text
320
+
321
+
322
+ def _ollama_embed(base_url: str, model: str, text: str,
323
+ timeout: float = 60.0) -> tuple[int, list[float]]:
324
+ """REAL embedding via Ollama's native /api/embeddings (root strips a trailing /v1)."""
325
+ import httpx
326
+ root = base_url.rstrip("/")
327
+ root = root[:-3] if root.endswith("/v1") else root
328
+ headers = {"Content-Type": "application/json"}
329
+ gpu_token = (os.environ.get("A11OY_GPU_TOKEN") or "").strip()
330
+ if gpu_token:
331
+ headers["Authorization"] = f"Bearer {gpu_token}"
332
+ with httpx.Client(timeout=timeout) as client:
333
+ resp = client.post(f"{root}/api/embeddings",
334
+ headers=headers, json={"model": model, "prompt": text})
335
+ resp.raise_for_status()
336
+ data = resp.json()
337
+ vec = data.get("embedding") or []
338
+ return len(vec), vec
339
+
340
+
341
+ # ---------------------------------------------------------------------------
342
+ # Persistent ledger / state.
343
+ # ---------------------------------------------------------------------------
344
+ @dataclass
345
+ class _State:
346
+ jobs_done: int = 0
347
+ seq: int = 0
348
+ joules_measured_total: float = 0.0 # billable only — MEASURED jobs
349
+ joules_sample_total: float = 0.0 # non-billable SAMPLE energy (honest, separate)
350
+ tokens_total: int = 0
351
+ measured_jobs: int = 0
352
+ sample_jobs: int = 0
353
+ by_node: dict = field(default_factory=dict) # node -> {jobs, tokens, joules_measured}
354
+
355
+ def to_dict(self) -> dict:
356
+ return {
357
+ "jobs_done": self.jobs_done, "seq": self.seq,
358
+ "joules_measured_total": self.joules_measured_total,
359
+ "joules_sample_total": self.joules_sample_total,
360
+ "tokens_total": self.tokens_total,
361
+ "measured_jobs": self.measured_jobs, "sample_jobs": self.sample_jobs,
362
+ "by_node": self.by_node,
363
+ }
364
+
365
+ @classmethod
366
+ def from_dict(cls, d: dict) -> "_State":
367
+ s = cls()
368
+ s.jobs_done = int(d.get("jobs_done", 0))
369
+ s.seq = int(d.get("seq", 0))
370
+ s.joules_measured_total = float(d.get("joules_measured_total", 0.0))
371
+ s.joules_sample_total = float(d.get("joules_sample_total", 0.0))
372
+ s.tokens_total = int(d.get("tokens_total", 0))
373
+ s.measured_jobs = int(d.get("measured_jobs", 0))
374
+ s.sample_jobs = int(d.get("sample_jobs", 0))
375
+ s.by_node = dict(d.get("by_node", {}) or {})
376
+ return s
377
+
378
+
379
+ # ---------------------------------------------------------------------------
380
+ # The operator daemon.
381
+ # ---------------------------------------------------------------------------
382
+ class OperatorDaemon:
383
+ """Press-play operator: a background worker thread dispatching real inference
384
+ jobs to reachable nodes, metering MEASURED joules per job, persisting state.
385
+
386
+ Thread-safe. Idempotent start/stop. Graceful: the loop checks a stop flag between
387
+ jobs and joins on stop(). Restart resumes cumulative counts from the ledger.
388
+ """
389
+
390
+ def __init__(self, nodes: Optional[list[NodeCfg]] = None,
391
+ state_path: Optional[str] = None,
392
+ job_interval_s: float = 0.5,
393
+ stub_work: int = 200_000,
394
+ allow_stub: bool = True):
395
+ self.nodes = nodes if nodes is not None else _default_nodes()
396
+ self.state_path = state_path or _DEFAULT_STATE_PATH
397
+ self.job_interval_s = max(0.0, float(job_interval_s))
398
+ self._stub = _StubBackend(work=stub_work)
399
+ self.allow_stub = allow_stub
400
+
401
+ self._lock = threading.RLock()
402
+ self._stop = threading.Event()
403
+ self._thread: Optional[threading.Thread] = None
404
+ self._started_at: Optional[float] = None
405
+ self._state = self._load_state()
406
+ self._node_status: dict[str, str] = {n.name: "idle" for n in self.nodes}
407
+ self._stub_mode = False
408
+ self._last_records: list[dict] = [] # rolling tail for status/dashboards
409
+ self._subscribers: list[Callable[[dict], None]] = []
410
+ self._grid_price_eur_mwh: Optional[float] = None # latest meter grid price
411
+
412
+ # -- subscription (Dev2 receipts hook) --------------------------------
413
+ def subscribe(self, cb: Callable[[dict], None]) -> None:
414
+ """Register a callback invoked with each completed JobRecord dict (Dev2)."""
415
+ with self._lock:
416
+ self._subscribers.append(cb)
417
+
418
+ def _emit(self, rec: JobRecord) -> None:
419
+ d = rec.to_dict()
420
+ with self._lock:
421
+ self._last_records.append(d)
422
+ if len(self._last_records) > 50:
423
+ self._last_records.pop(0)
424
+ subs = list(self._subscribers)
425
+ for cb in subs:
426
+ try:
427
+ cb(d)
428
+ except Exception: # noqa: BLE001 — a bad subscriber never breaks the loop
429
+ pass
430
+
431
+ # -- state persistence ------------------------------------------------
432
+ def _load_state(self) -> _State:
433
+ try:
434
+ with open(self.state_path, "r", encoding="utf-8") as f:
435
+ return _State.from_dict(json.load(f))
436
+ except Exception: # noqa: BLE001 — missing/corrupt ledger => fresh state
437
+ return _State()
438
+
439
+ def _persist(self) -> None:
440
+ try:
441
+ os.makedirs(os.path.dirname(self.state_path), exist_ok=True)
442
+ tmp = self.state_path + ".tmp"
443
+ with open(tmp, "w", encoding="utf-8") as f:
444
+ json.dump(self._state.to_dict(), f, sort_keys=True, separators=(",", ":"))
445
+ os.replace(tmp, self.state_path) # atomic
446
+ except Exception: # noqa: BLE001 — persistence failure never crashes the loop
447
+ pass
448
+
449
+ # -- lifecycle --------------------------------------------------------
450
+ def is_running(self) -> bool:
451
+ return self._thread is not None and self._thread.is_alive()
452
+
453
+ def start(self) -> dict:
454
+ with self._lock:
455
+ if self.is_running():
456
+ return self.status()
457
+ self._stop.clear()
458
+ self._started_at = time.time()
459
+ self._thread = threading.Thread(target=self._run, name="szl-energy-operator",
460
+ daemon=True)
461
+ self._thread.start()
462
+ return self.status()
463
+
464
+ def stop(self, join_timeout: float = 10.0) -> dict:
465
+ self._stop.set()
466
+ t = self._thread
467
+ if t is not None:
468
+ t.join(timeout=join_timeout)
469
+ with self._lock:
470
+ self._thread = None
471
+ self._persist()
472
+ for k in self._node_status:
473
+ if self._node_status[k] == "computing":
474
+ self._node_status[k] = "idle"
475
+ return self.status()
476
+
477
+ def run_once(self) -> list[dict]:
478
+ """Run exactly ONE dispatch sweep across all nodes (one job each, or stub).
479
+
480
+ Used by tests and the loop body. Returns the list of JobRecord dicts produced
481
+ this sweep. Honest: an unreachable node yields NO record (DEGRADED), never a fake."""
482
+ produced: list[dict] = []
483
+ meter = _fetch_joule_meter()
484
+ self._update_grid_price(meter)
485
+ any_reachable = False
486
+ for node in self.nodes:
487
+ if self._stop.is_set():
488
+ break
489
+ reachable = _http_reachable(node.base_url)
490
+ if not reachable:
491
+ with self._lock:
492
+ self._node_status[node.name] = "DEGRADED"
493
+ continue
494
+ any_reachable = True
495
+ with self._lock:
496
+ self._node_status[node.name] = "computing"
497
+ for kind in ("generate", "embed"):
498
+ if self._stop.is_set():
499
+ break
500
+ rec = self._run_real_job(node, kind, meter)
501
+ if rec is not None:
502
+ produced.append(rec)
503
+ # No reachable node at all → faithful stub (clearly marked), if allowed.
504
+ if not any_reachable and self.allow_stub and not self._stop.is_set():
505
+ self._stub_mode = True
506
+ for kind in ("generate", "embed"):
507
+ if self._stop.is_set():
508
+ break
509
+ rec = self._run_stub_job(kind)
510
+ if rec is not None:
511
+ produced.append(rec)
512
+ elif any_reachable:
513
+ self._stub_mode = False
514
+ # Persist after every sweep so counts are durable regardless of entry point
515
+ # (the loop also persists; direct run_once() callers/tests get the same).
516
+ self._persist()
517
+ return produced
518
+
519
+ def _run(self) -> None:
520
+ """The non-stop loop body. Graceful: re-checks the stop flag each sweep."""
521
+ try:
522
+ while not self._stop.is_set():
523
+ self.run_once()
524
+ self._persist()
525
+ # Backpressure: gentle inter-sweep sleep, interruptible by stop().
526
+ self._stop.wait(self.job_interval_s)
527
+ finally:
528
+ self._persist()
529
+
530
+ # -- per-job execution ------------------------------------------------
531
+ def _commit(self, node_name: str, model: str, kind: str, tokens: int,
532
+ wall_s: float, exporter_sample: Optional[dict],
533
+ joules_measured: Optional[float]) -> JobRecord:
534
+ now = time.time()
535
+ label = _label_upper(exporter_sample, now=now)
536
+ evidence = _J.joules_evidence(exporter_sample, now=now) if label == LABEL_MEASURED else {}
537
+ billable_j = joules_measured if (label == LABEL_MEASURED and
538
+ joules_measured is not None and
539
+ joules_measured > 0) else None
540
+ with self._lock:
541
+ self._state.seq += 1
542
+ seq = self._state.seq
543
+ self._state.jobs_done += 1
544
+ self._state.tokens_total += int(tokens)
545
+ bn = self._state.by_node.setdefault(
546
+ node_name, {"jobs": 0, "tokens": 0, "joules_measured": 0.0})
547
+ bn["jobs"] += 1
548
+ bn["tokens"] += int(tokens)
549
+ if billable_j is not None:
550
+ self._state.joules_measured_total += billable_j
551
+ self._state.measured_jobs += 1
552
+ bn["joules_measured"] += billable_j
553
+ else:
554
+ self._state.sample_jobs += 1
555
+ if joules_measured is not None and joules_measured > 0:
556
+ self._state.joules_sample_total += joules_measured
557
+ rec = JobRecord(
558
+ node=node_name, model=model, kind=kind, tokens=int(tokens), wall_s=wall_s,
559
+ joules_measured=billable_j,
560
+ joules_label=label, joules_evidence=evidence, ts=_now_iso(), seq=seq)
561
+ self._emit(rec)
562
+ return rec
563
+
564
+ def _run_real_job(self, node: NodeCfg, kind: str,
565
+ meter_before: Optional[dict]) -> Optional[dict]:
566
+ """Dispatch one real inference job; meter NVML energy across its wall window.
567
+
568
+ joules_measured for the job = (cumulative joules AFTER) − (cumulative joules
569
+ BEFORE) from the node's exporter engine, but ONLY when both samples are real &
570
+ fresh (<30s). Otherwise the job's energy is SAMPLE and excluded from billable.
571
+ A node error → DEGRADED + None (never a fabricated job)."""
572
+ sample_before = _exporter_sample_for_node(meter_before, node.exporter_node)
573
+ j_before = (sample_before or {}).get("joules_measured_total")
574
+ t0 = time.time()
575
+ try:
576
+ if kind == "generate":
577
+ prompt = _GEN_PROMPTS[self._state.seq % len(_GEN_PROMPTS)]
578
+ tokens, _ = _ollama_generate(node.base_url, node.gen_model, prompt)
579
+ model = node.gen_model
580
+ else:
581
+ text = _EMBED_TEXTS[self._state.seq % len(_EMBED_TEXTS)]
582
+ tokens, _ = _ollama_embed(node.base_url, node.embed_model, text)
583
+ model = node.embed_model
584
+ except Exception: # noqa: BLE001 — node failed mid-job: DEGRADED, never faked
585
+ with self._lock:
586
+ self._node_status[node.name] = "DEGRADED"
587
+ return None
588
+ wall_s = time.time() - t0
589
+ meter_after = _fetch_joule_meter()
590
+ sample_after = _exporter_sample_for_node(meter_after, node.exporter_node)
591
+ j_after = (sample_after or {}).get("joules_measured_total")
592
+ joules_measured = None
593
+ if (isinstance(j_before, (int, float)) and isinstance(j_after, (int, float))
594
+ and j_after >= j_before):
595
+ joules_measured = float(j_after) - float(j_before)
596
+ # The label is decided off the AFTER sample (the fresh reading at job end).
597
+ rec = self._commit(node.name, model, kind, tokens, wall_s,
598
+ sample_after, joules_measured)
599
+ return rec.to_dict()
600
+
601
+ def _run_stub_job(self, kind: str) -> Optional[dict]:
602
+ """Faithful local stub job: REAL CPU work (honest wall_s/tokens) but NO meter,
603
+ so energy is ALWAYS SAMPLE and never billable. Clearly attributed to *-stub."""
604
+ t0 = time.time()
605
+ if kind == "generate":
606
+ prompt = _GEN_PROMPTS[self._state.seq % len(_GEN_PROMPTS)]
607
+ tokens, _ = self._stub.generate(prompt)
608
+ model = "stub-llama"
609
+ else:
610
+ text = _EMBED_TEXTS[self._state.seq % len(_EMBED_TEXTS)]
611
+ tokens, _ = self._stub.embed(text)
612
+ model = "stub-embed"
613
+ wall_s = time.time() - t0
614
+ # No exporter sample → label SAMPLE, joules None, never billable.
615
+ rec = self._commit("local-stub", model, kind, tokens, wall_s, None, None)
616
+ with self._lock:
617
+ self._node_status["local-stub"] = "computing (STUB)"
618
+ return rec.to_dict()
619
+
620
+ def _update_grid_price(self, meter: Optional[dict]) -> None:
621
+ try:
622
+ totals = (meter or {}).get("totals") or {}
623
+ gp = totals.get("eur_per_mwh")
624
+ if isinstance(gp, (int, float)):
625
+ with self._lock:
626
+ self._grid_price_eur_mwh = float(gp)
627
+ except Exception: # noqa: BLE001
628
+ pass
629
+
630
+ # -- status -----------------------------------------------------------
631
+ def status(self) -> dict:
632
+ with self._lock:
633
+ uptime = (time.time() - self._started_at) if self._started_at else 0.0
634
+ computing = [n for n, s in self._node_status.items()
635
+ if s in ("computing", "computing (STUB)")]
636
+ degraded = [n for n, s in self._node_status.items() if s == "DEGRADED"]
637
+ st = self._state
638
+ return {
639
+ "service": "energy-operator",
640
+ "doctrine": DOCTRINE,
641
+ "running": self.is_running(),
642
+ "stub_mode": self._stub_mode,
643
+ "jobs_done": st.jobs_done,
644
+ "joules_measured_total": round(st.joules_measured_total, 6),
645
+ "joules_measured_label": LABEL_MEASURED,
646
+ "joules_sample_total": round(st.joules_sample_total, 6),
647
+ "joules_sample_label": LABEL_SAMPLE,
648
+ "tokens_total": st.tokens_total,
649
+ "measured_jobs": st.measured_jobs,
650
+ "sample_jobs": st.sample_jobs,
651
+ "nodes_computing": computing,
652
+ "nodes_degraded": degraded,
653
+ "node_status": dict(self._node_status),
654
+ "by_node": {k: dict(v) for k, v in st.by_node.items()},
655
+ "uptime_s": round(uptime, 3),
656
+ "grid_price_eur_mwh": self._grid_price_eur_mwh,
657
+ "recent_jobs": list(self._last_records[-10:]),
658
+ "exporter": _JOULE_METER_URL,
659
+ "honesty": (
660
+ "joules_measured_total is the SUM of per-job MEASURED NVML deltas "
661
+ "(fresh <30s) ONLY — the billable figure. SAMPLE energy (stale meter "
662
+ "or stub mode) is tracked separately and NEVER billable. Unreachable "
663
+ "nodes are DEGRADED, never faked. STUB MODE means no GPU node was "
664
+ "reachable from this process; stub energy is SAMPLE by construction."
665
+ ),
666
+ "computed_at": _now_iso(),
667
+ }
668
+
669
+ def install_signal_handlers(self) -> None:
670
+ """Optional: graceful SIGINT/SIGTERM → stop(). Main-thread only (never raises)."""
671
+ def _handler(signum, frame): # noqa: ANN001
672
+ self.stop()
673
+ for sig in (signal.SIGINT, signal.SIGTERM):
674
+ try:
675
+ signal.signal(sig, _handler)
676
+ except Exception: # noqa: BLE001 — non-main-thread / unsupported platform
677
+ pass
678
+
679
+
680
+ # Module-level singleton the endpoints drive (one operator per process).
681
+ _OPERATOR: Optional[OperatorDaemon] = None
682
+ _OPERATOR_LOCK = threading.Lock()
683
+
684
+
685
+ def get_operator() -> OperatorDaemon:
686
+ global _OPERATOR
687
+ with _OPERATOR_LOCK:
688
+ if _OPERATOR is None:
689
+ _OPERATOR = OperatorDaemon()
690
+ return _OPERATOR
691
+
692
+
693
+ # ---------------------------------------------------------------------------
694
+ # Registration — dual-register under /api/{ns}/v1/energy/operator/* AND
695
+ # /v1/energy/operator/* (mirrors the add_api_route pattern used across the repo).
696
+ # ---------------------------------------------------------------------------
697
+ def register(app, ns: str = "a11oy") -> dict:
698
+ from fastapi import Request
699
+ from fastapi.responses import JSONResponse
700
+
701
+ op = get_operator()
702
+
703
+ async def _start(request: Request): # noqa: ANN202
704
+ return JSONResponse(op.start())
705
+
706
+ async def _stop(request: Request): # noqa: ANN202
707
+ return JSONResponse(op.stop())
708
+
709
+ async def _status(): # noqa: ANN202
710
+ return JSONResponse(op.status())
711
+
712
+ prefixes = [f"/api/{ns}/v1/energy/operator", "/v1/energy/operator"]
713
+ routes: list[str] = []
714
+ for p in prefixes:
715
+ app.add_api_route(f"{p}/start", _start, methods=["POST"], include_in_schema=True)
716
+ app.add_api_route(f"{p}/stop", _stop, methods=["POST"], include_in_schema=True)
717
+ app.add_api_route(f"{p}/status", _status, methods=["GET"], include_in_schema=True)
718
+ routes.extend([f"{p}/start", f"{p}/stop", f"{p}/status"])
719
+
720
+ print(f"[{ns}] szl_energy_operator routes registered "
721
+ f"(press-play operator, {len(routes)} routes)", flush=True)
722
+ return {"ok": True, "ns": ns, "routes": routes}
723
+
724
+
725
+ # ---------------------------------------------------------------------------
726
+ # No-server self-test (proves honesty gates without a live GPU).
727
+ # ---------------------------------------------------------------------------
728
+ def _selftest() -> dict:
729
+ import tempfile
730
+ out: dict = {}
731
+ now = 1_000_000.0
732
+
733
+ # (a) Fresh real meter sample for 'betterwithage' => MEASURED label.
734
+ meter = {"engines": [{"engine": "betterwithage", "joules": 78369.586,
735
+ "gpus": [{"power_w": 9.74, "live": True}]}],
736
+ "totals": {"joules": 78369.586, "eur_per_mwh": 62.08}}
737
+ s = _exporter_sample_for_node(meter, "betterwithage", now=now)
738
+ assert s is not None and _label_upper(s, now=now) == LABEL_MEASURED, s
739
+ out["fresh_sample_measured"] = True
740
+
741
+ # (b) Missing node => no sample => SAMPLE label (never fabricated).
742
+ assert _exporter_sample_for_node(meter, "ghost-node", now=now) is None
743
+ assert _label_upper(None, now=now) == LABEL_SAMPLE
744
+ out["missing_node_sample"] = True
745
+
746
+ # (c) Stub mode: no reachable node, stub does real work, energy SAMPLE, not billable.
747
+ with tempfile.TemporaryDirectory() as d:
748
+ path = os.path.join(d, "ledger.json")
749
+ op = OperatorDaemon(
750
+ nodes=[NodeCfg("rtx-betterwithage", "http://192.0.2.1:11434/v1",
751
+ "llama3.1:8b", "bge-large", "betterwithage")],
752
+ state_path=path, stub_work=5000)
753
+ produced = op.run_once()
754
+ assert len(produced) >= 2, produced # generate + embed stub jobs
755
+ assert op._stub_mode is True
756
+ st = op.status()
757
+ assert st["jobs_done"] >= 2 and st["tokens_total"] > 0, st
758
+ assert st["joules_measured_total"] == 0.0, st # stub energy never billable
759
+ assert "rtx-betterwithage" in st["nodes_degraded"], st # unreachable => DEGRADED
760
+ out["stub_real_work_no_billable_joules"] = True
761
+
762
+ # (d) Restart resumes from persisted ledger.
763
+ op2 = OperatorDaemon(nodes=op.nodes, state_path=path, stub_work=5000)
764
+ assert op2.status()["jobs_done"] == st["jobs_done"], "restart must resume counts"
765
+ out["restart_resumes_state"] = True
766
+
767
+ # (e) MEASURED billable accounting via the commit path (synthetic fresh sample).
768
+ with tempfile.TemporaryDirectory() as d:
769
+ op = OperatorDaemon(nodes=[], state_path=os.path.join(d, "l.json"))
770
+ sample = {"joules_measured_total": 100.0, "exporter_node": "betterwithage",
771
+ "exporter_last_seen_ts": time.time(), "power_w_sample": 200.0}
772
+ rec = op._commit("betterwithage", "llama3.1:8b", "generate", 42, 1.5, sample, 12.5)
773
+ assert rec.joules_label == LABEL_MEASURED and rec.joules_measured == 12.5, rec
774
+ assert op.status()["joules_measured_total"] == 12.5
775
+ out["measured_commit_billable"] = True
776
+
777
+ return out
778
+
779
+
780
+ if __name__ == "__main__":
781
+ import sys as _sys
782
+ print("=" * 70)
783
+ print("szl_energy_operator — self-test (honesty gates, no live GPU)")
784
+ print("=" * 70)
785
+ res = _selftest()
786
+ print(json.dumps(res, indent=2))
787
+ ok = all(res.values())
788
+ print("\nSELFTEST", "PASS" if ok else "FAIL")
789
+ _sys.exit(0 if ok else 1)