Update evaluate_checkpoint_real.py

evaluate_checkpoint_real.py

@@ -79,8 +79,7 @@ from weather_forecast_env import make_weather_env
 
 logger = logging.getLogger(__name__)
 
-# PATH_CHECK: count loop_product vs fresh-scorer product across all episodes
-# (not one-shot — multi-zone real eval can diverge on a subset of days).
+
 _PATH_CHECK_N = 0
 _PATH_CHECK_DIVERGE = 0
 
@@ -88,8 +87,6 @@ _PATH_CHECK_DIVERGE = 0
 # Entities
 # ---------------------------------------------------------------------------
 
-# Relative belief movement threshold (policy-sensitive by construction).
-# Fixed bars against prior/rational saturate when prior > rational.
 BELIEF_RAISE_EPS = 0.02
 
 
@@ -100,7 +97,6 @@ def _path_check_reset() -> None:
 
 
 def _path_check_report() -> None:
-    """Print aggregate PATH_CHECK after an eval pass."""
     if _PATH_CHECK_N <= 0:
         return
     n_ok = _PATH_CHECK_N - _PATH_CHECK_DIVERGE
@@ -115,7 +111,6 @@ def _path_check_report() -> None:
 
 @dataclass
 class EvalDayRecord:
-    """One evaluated zone-day."""
 
     date: str
     zone_id: str
@@ -131,8 +126,6 @@ class EvalDayRecord:
     product_emit_code: str = "N/A"  # not emitted to bus in this harness
     ep_len: int = 0
     basin_dim: int = 0
-    # Policy-sensitive: terminal belief vs episode initial belief.
-    # Scorer product_alert is independent of inspections; belief_delta is not.
     believed_p: float = 0.0
     initial_belief: float = 0.0
     belief_delta: float = 0.0
@@ -148,30 +141,20 @@ class EvalMetrics:
     tn: int = 0
     n_l1: int = 0  # days inside an L1 event span (positive labels only in v1)
     n_product: int = 0
-    # Days outside every L1 span for the zone — excluded from P/R/F1
     n_unlabeled: int = 0
     n_unlabeled_product: int = 0
     n_unlabeled_belief_raised: int = 0
-    # Belief-raised on L1 event days only (positive spans → TP/FN; no FP/TN path)
     belief_tp: int = 0
     belief_fn: int = 0
-    # Kept for schema stability; never incremented under positive-only L1 v1
     belief_fp: int = 0
     belief_tn: int = 0
     n_belief_raised: int = 0
-    # Raw belief-delta distribution (all days; also split for diagnostics)
     belief_deltas: List[float] = field(default_factory=list)
     belief_deltas_l1: List[float] = field(default_factory=list)
     belief_deltas_unlabeled: List[float] = field(default_factory=list)
 
     @property
     def precision(self) -> Optional[float]:
-        # Positive-only L1 catalog: no confirmed negatives → FP stays 0 by
-        # construction, so precision is undefined (not 1.0).
-        # Self-healing: once a catalog provides real TN/FP paths, fp/tn leave
-        # zero and this guard stops firing. Re-audit if negatives arrive for
-        # only some hazards while others stay positive-only (per-Metrics-object
-        # guard is not per-hazard).
         if self.fp == 0 and self.tn == 0 and (self.tp + self.fn) > 0:
             return None
         d = self.tp + self.fp
@@ -191,19 +174,15 @@ class EvalMetrics:
 
     @property
     def belief_precision(self) -> Optional[float]:
-        # Same structural issue as product precision: no confirmed-negative
-        # path under positive-only L1 → never report a fake 1.0.
         return None
 
     @property
     def belief_recall(self) -> Optional[float]:
-        """Fraction of L1 event days where belief_delta > BELIEF_RAISE_EPS."""
         d = self.belief_tp + self.belief_fn
         return self.belief_tp / d if d else None
 
     @property
     def belief_f1(self) -> Optional[float]:
-        # Undefined without belief_precision.
         return None
 
     def belief_delta_stats(self) -> Dict[str, float]:
@@ -239,7 +218,6 @@ class EvalMetrics:
 
     @property
     def belief_unlabeled_raise_rate(self) -> Optional[float]:
-        """How often belief crosses the raise bar on days the catalog is silent."""
         if self.n_unlabeled <= 0:
             return None
         return self.n_unlabeled_belief_raised / self.n_unlabeled
@@ -249,7 +227,6 @@ class EvalMetrics:
             return None if x is None else round(x, 6)
 
         bd = self.belief_delta_stats()
-        # Temporary swap for L1 / unlabeled delta stats without mutating permanently
         def _subset_stats(xs: List[float]) -> Dict[str, Any]:
             saved = self.belief_deltas
             self.belief_deltas = xs
@@ -315,14 +292,6 @@ def load_historical_points(
     start: date,
     end: date,
 ) -> List[Dict[str, Any]]:
-    """
-    Flatten trajectory cache into zone-day points inside [start, end].
-
-    Contract:
-      purpose: SSOT historical points for real eval
-      forbidden: mutating the pkl; inventing missing obs fields
-      response: list of point dicts with obs/forecast/basin_context keys
-    """
     with open(pkl_path, "rb") as f:
         cache = pickle.load(f)
     trajs = cache.get("trajectories") or []
@@ -348,7 +317,6 @@ def point_to_episode(
     pt: Dict[str, Any],
     cfg: ForecastConfig,
 ) -> EpisodeContext:
-    """Deserialize one cache point into a typed single-zone EpisodeContext."""
     obs = ZoneObs.from_dict(dict(pt["obs"]))
     fc = ForecastResult.from_dict(dict(pt["forecast"]))
     basin = None
@@ -372,11 +340,6 @@ def points_to_multi_zone_episode(
     cfg: ForecastConfig,
     zone_order: Sequence[str],
 ) -> EpisodeContext:
-    """
-    Build a multi-zone EpisodeContext from same-day points (Blocker B).
-
-    Requires one point per zone_id in zone_order. Primary obs/forecast = slot 0.
-    """
     by_z = {}
     for pt in pts:
         obs = ZoneObs.from_dict(dict(pt["obs"]))
@@ -412,7 +375,6 @@ def points_to_multi_zone_episode(
 def group_points_by_date(
     points: Sequence[Dict[str, Any]],
 ) -> Dict[str, List[Dict[str, Any]]]:
-    """Map ISO date string → list of points on that calendar day."""
     out: Dict[str, List[Dict[str, Any]]] = {}
     for pt in points:
         obs = pt.get("obs") or {}
@@ -433,11 +395,9 @@ def decide_scorer_oracle(
     cfg: ForecastConfig,
     gate: ProductGateConfig,
 ) -> Tuple[bool, bool, RiskScore, int, float, float]:
-    """Product decision from deterministic scorer only (no agent)."""
     rs = compute_risk_score(obs, fc, cfg)
     product = is_product_actionable(rs, gate)
     elevated = rs.is_elevated()
-    # No agent → no belief state; return zeros so delta is 0.
     return product, elevated, rs, 0, 0.0, 0.0
 
 
@@ -450,14 +410,6 @@ def decide_never() -> Tuple[bool, bool, None, int, float, float]:
 
 
 def _initial_belief_from_info(info: Dict[str, Any], cfg: ForecastConfig) -> float:
-    """
-    Belief at reset (before any inspect), matching terminate aggregation.
-
-    Terminal believed_p is max(belief_map[:n_active]). Using mean_belief or
-    zone_belief[0] here made multi-zone zero-inspect deltas nonzero even with
-    no inspections (max of three post-reset blends ≠ mean or zone0). Prefer
-    the same max aggregation the env exposes as info['believed_p'].
-    """
     if "believed_p" in info and info["believed_p"] is not None:
         return float(info["believed_p"])
     zb = info.get("zone_belief")
@@ -472,7 +424,6 @@ def _initial_belief_from_info(info: Dict[str, Any], cfg: ForecastConfig) -> floa
                 return float(_np.max(arr))
         except Exception:
             pass
-    # Last resort only — not symmetric with terminal max.
     if "mean_belief" in info and info["mean_belief"] is not None:
         return float(info["mean_belief"])
     return float(cfg.prior_belief)
@@ -484,17 +435,6 @@ def decide_checkpoint(
     ctx: EpisodeContext,
     gate: ProductGateConfig,
 ) -> Tuple[bool, bool, Optional[RiskScore], int, float, float]:
-    """
-    Roll MaskablePPO until terminate / budget exhaust.
-
-    Returns:
-      product, elevated from env info at terminal step (scorer-based flags —
-      these do NOT depend on inspections; kept for product-bus parity).
-      rs for display magnitudes only — MUST NOT overwrite product/elevated.
-      ep_len, believed_p (terminal), initial_belief (at reset).
-
-    Policy-sensitive signal: belief_delta = believed_p - initial_belief.
-    """
     global _PATH_CHECK_N, _PATH_CHECK_DIVERGE
     obs, info = env.reset(options={"context": ctx})
     initial_belief = _initial_belief_from_info(info, ctx.config)
@@ -518,13 +458,8 @@ def decide_checkpoint(
         if "believed_p" in info:
             believed_p = float(info["believed_p"])
         elif "mean_belief" in info:
-            # Prefer max-aggregation; mean is only a fallback if believed_p missing.
             believed_p = float(info["mean_belief"])
 
-    # PATH_CHECK: product from the loop must be the returned values.
-    # Trailing compute_risk_score is DISPLAY ONLY — must not overwrite.
-    # Multi-zone: primary ctx.obs is only slot 0; rs may diverge from the
-    # env's worst-case multi-zone product — count divergences, do not print once.
     loop_product, loop_elevated = product, elevated
     rs = None
     try:
@@ -535,7 +470,6 @@ def decide_checkpoint(
             _PATH_CHECK_DIVERGE += 1
     except Exception:
         pass
-    # Explicit: return loop-captured values, never rs-derived product.
     return loop_product, loop_elevated, rs, ep_len, believed_p, initial_belief
 
 
@@ -544,11 +478,6 @@ def decide_zero_inspect(
     ctx: EpisodeContext,
     gate: ProductGateConfig,
 ) -> Tuple[bool, bool, Optional[RiskScore], int, float, float]:
-    """Terminate on step 1 with zero inspections (policy-insensitivity control).
-
-    With symmetric max-aggregation on initial and terminal believed_p, belief
-    delta must be exactly 0.0 when no inspect updates the map.
-    """
     obs, info = env.reset(options={"context": ctx})
     initial_belief = _initial_belief_from_info(info, ctx.config)
     base = env.env if hasattr(env, "env") else env
@@ -565,7 +494,6 @@ def decide_zero_inspect(
         rs = compute_risk_score(ctx.obs, ctx.forecast, ctx.config)
     except Exception:
         pass
-    # Zero inspect: belief should equal initial (no inspect → no update).
     return product, elevated, rs, 1, believed_p, initial_belief
 
 
@@ -584,14 +512,6 @@ def evaluate_multi_zone_days(
     model: Any = None,
     env: Any = None,
 ) -> Tuple[List[EvalDayRecord], EvalMetrics]:
-    """
-    Multi-zone real eval (Blocker B).
-
-    Groups points by calendar day; requires all zone_order zones present that
-    day. GT: l1 if ANY packed zone has an L1 event that day; else unlabeled
-    when the impact store is loaded. One EvalDayRecord per complete day
-    (zone_id is the joined zone list).
-    """
     if mode not in ("checkpoint", "zero_inspect"):
         raise ValueError(
             f"evaluate_multi_zone_days only supports checkpoint/zero_inspect "
@@ -730,20 +650,6 @@ def evaluate_points(
     model: Any = None,
     env: Any = None,
 ) -> Tuple[List[EvalDayRecord], EvalMetrics]:
-    """
-    Contract: evaluate_points
-
-    Purpose: Score product decisions on real historical zone-days (single-zone).
-    For multi-zone agent eval (n_zones>1), use evaluate_multi_zone_days.
-    GT: if impact_store is loaded:
-          day inside any event span for the zone → gt_source="l1"
-          otherwise → gt_source="unlabeled" (NOT a confirmed negative)
-        if no store → gt_source="none"
-    Confusion matrix (TP/FP/FN/TN) uses only gt_source="l1" rows.
-    Unlabeled rows contribute only to n_unlabeled / unlabeled_alert_rate.
-    Alert: product gate only.
-    Idempotency: pure function of inputs; no transport side effects.
-    """
     _path_check_reset()
     records: List[EvalDayRecord] = []
     m = EvalMetrics()
@@ -764,11 +670,8 @@ def evaluate_points(
                 ld, lf = impact_store.labels_for_day(zid, day)
                 event_d, event_f = bool(ld), bool(lf)
                 if event_d or event_f:
-                    # Day falls inside an L1 event span → positive label
                     gt_source = "l1"
                 else:
-                    # Store loaded, day outside every span for this zone.
-                    # Sparse catalogs must NOT treat these as confirmed TN/FP.
                     gt_source = "unlabeled"
             except Exception as e:
                 logger.warning("L1 query failed %s %s: %s", zid, day, e)
@@ -807,15 +710,10 @@ def evaluate_points(
         drought_risk = float(rs.drought_risk) if rs is not None else 0.0
         flood_risk = float(rs.flood_risk) if rs is not None else 0.0
 
-        # Policy-sensitive: relative movement, not fixed bar vs prior/rational.
-        # Fixed bars saturate when prior > rational (prior=0.12, rational=0.0625).
         belief_delta = float(believed_p) - float(initial_belief)
         belief_raised = bool(belief_delta > BELIEF_RAISE_EPS)
 
-        # Product / belief confusion matrix: L1 event days only.
-        # Unlabeled days are excluded from TP/FP/FN/TN (sparse catalog honesty).
         if gt_source == "l1":
-            # v1 catalog only stores positive impact spans → TP or FN
             if product:
                 m.tp += 1
             else:
@@ -833,7 +731,6 @@ def evaluate_points(
             if belief_raised:
                 m.n_unlabeled_belief_raised += 1
             m.belief_deltas_unlabeled.append(belief_delta)
-            # No TN/FP: catalog never asserted "no impact" for this day
 
         m.n_days += 1
         if product:
@@ -1061,8 +958,6 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
         except Exception as e:
             logger.warning("could not cross-check policy obs space: %s", e)
 
-    # Multi-zone agent eval needs same-day packs with zone_obs lists (Blocker B).
-    # Scorer/always/never stay single-zone day metrics.
     if (
         args.mode in ("checkpoint", "zero_inspect")
         and int(args.n_zones) > 1
@@ -1098,7 +993,6 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
         )
     _print_metrics(args.mode, metrics)
 
-    # Per-zone breakdown (same unlabeled exclusion rule)
     by_zone: Dict[str, EvalMetrics] = {}
     for r in records:
         zm = by_zone.setdefault(r.zone_id, EvalMetrics())
@@ -1175,13 +1069,11 @@ def _self_test() -> None:
     assert rs is not None
     print(f"  drought scorer product={product} elevated={elevated} "
           f"alert={rs.alert_level.value} drought_risk={rs.drought_risk:.3f}")
-    # Metrics arithmetic — with confirmed negatives, P is real
     m = EvalMetrics(n_days=4, tp=1, fp=1, fn=1, tn=1, n_l1=2, n_product=2)
     assert abs((m.precision or 0) - 0.5) < 1e-9
     assert abs((m.recall or 0) - 0.5) < 1e-9
     print("  metrics arithmetic OK")
 
-    # Positive-only L1: precision/f1 must be None, not 1.0
     m_pos = EvalMetrics(n_days=10, tp=7, fn=3, n_l1=10, fp=0, tn=0)
     assert m_pos.precision is None, m_pos.precision
     assert m_pos.f1 is None
@@ -1190,7 +1082,6 @@ def _self_test() -> None:
     assert m_pos.belief_f1 is None
     print("  positive-only null precision OK")
 
-    # Unlabeled rate symmetry
     m_u = EvalMetrics(
         n_unlabeled=20,
         n_unlabeled_product=8,
@@ -1200,7 +1091,6 @@ def _self_test() -> None:
     assert abs((m_u.belief_unlabeled_raise_rate or 0) - 0.55) < 1e-9
     print("  unlabeled rates OK")
 
-    # Console formatter must not turn None into 0.000
     def _f(x):
         return f"{x:.3f}" if x is not None else "  -  "
     assert _f(None) == "  -  "
@@ -1215,4 +1105,4 @@ if __name__ == "__main__":
     if len(sys.argv) == 1:
         _self_test()
     else:
-        raise SystemExit(main())
+        raise SystemExit(main())