"""Nova Outcome Backfill (REQ-317, SPEC §5.8, P3 Wave 2). The `fact_decision.outcome` column in the metrics cold store is written `pending` by the collector (it ingests `nova.ai.decision.made` events, which are emitted *before* the run executes the apply). Once the run completes (`nova.run.completed`, exit 0) or fails (`nova.run.failed`, exit non-zero), the outcome must be transitioned `pending -> succeeded`/`failed` so the Post-Pilot AI Decision Accuracy denominator is grounded (an outcome that is stuck `pending` cannot be scored). Architecture (grounded in what the ledger + collector actually do): * The Decision Ledger (`core/metrics/decision_ledger.py`) is an **append-only hash-chain** of CloudEvents envelopes — there is no `fact_decision` table *inside* the ledger DB; facts live in the separate collector cold store (`core/metrics/collector.py`, `nova_metrics.db`). The ledger is never UPDATEd in place (that would break the SHA-256 chain — see `verify_chain()`). * Therefore the backfill does TWO things: 1. Appends a new audit event `nova.outcome.backfilled` to the ledger (preserves the hash chain; auditable via `replay_run`). 2. UPDATEs the `fact_decision` row in the cold store (the row is keyed by `decision_id`; `outcome` + `backfilled_at` are mutable — they are facts, not chain events). Idempotent + terminal: * If `outcome` is already `succeeded`/`failed` (i.e. not `pending`), the call is a no-op and returns `{"status": "already_backfilled", "existing_outcome": }`. A terminal outcome is NEVER overwritten (defense against double-backfill and against flipping a `succeeded` run to `failed` retroactively or vice versa). * The same `outcome` value is re-asserted harmlessly (still a no-op). REQ-317: `outcome` ∈ {"succeeded", "failed"} only — `pending` is the initial state and may not be written by the backfill (it would undo the transition). An invalid value raises `ValueError`. Future milestones may add `'policy'` to `escalation_reason` (REQ-318); this module is scoped to outcome only. """ import datetime import json import os import sqlite3 import sys from pathlib import Path from typing import Optional, Dict, Any sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from core.metrics.event_envelope import make_event, append_event from core.metrics.decision_ledger import append as ledger_append, _LEDGER_PATH # The collector cold store path is mirrored here so the backfill can be # invoked without importing the collector (avoids a circular import: # the collector calls into backfill at run.completed/run.failed time). _METRICS_DIR = os.path.join( os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), "metrics", ) _STORE_PATH = os.path.join(_METRICS_DIR, "nova_metrics.db") _VALID_OUTCOMES = {"succeeded", "failed"} _PENDING = "pending" def _iso8601_now(): return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") def _resolve_store_path(store_path: Optional[str | Path]) -> str: if store_path is None: return _STORE_PATH return str(store_path) def _resolve_ledger_path(ledger_path: Optional[str | Path]) -> str: if ledger_path is None: return _LEDGER_PATH return str(ledger_path) def _get_fact_decision(decision_id: str, store_path: str) -> Optional[Dict[str, Any]]: """Read the fact_decision row for decision_id (or None).""" if not os.path.isfile(store_path): return None conn = sqlite3.connect(store_path) conn.row_factory = sqlite3.Row row = conn.execute( "SELECT decision_id, run_id, chosen_action, confidence, alternatives, " "human_override, outcome, event_time FROM fact_decision WHERE decision_id = ?", (decision_id,), ).fetchone() conn.close() if row is None: return None return dict(row) def backfill( decision_id: str, outcome: str, ledger_path: Optional[str | Path] = None, store_path: Optional[str | Path] = None, ) -> Dict[str, Any]: """Transition fact_decision.outcome from `pending` to `outcome`. Args: decision_id: the decision id (== run_id for v1.26). outcome: the terminal outcome; must be in {"succeeded", "failed"}. ledger_path: optional override for the Decision Ledger SQLite DB. store_path: optional override for the collector cold store SQLite DB. Returns: A dict describing the result: * success: {"status": "backfilled", "decision_id", "previous_outcome", "new_outcome", "backfilled_at"} * no-op: {"status": "already_backfilled", "decision_id", "existing_outcome", "backfilled_at"} Raises: ValueError: if `outcome` is not in {"succeeded", "failed"}. KeyError: if `decision_id` is not present in fact_decision. """ if outcome not in _VALID_OUTCOMES: raise ValueError( f"outcome must be one of {sorted(_VALID_OUTCOMES)}, got: {outcome!r}" ) sp = _resolve_store_path(store_path) lp = _resolve_ledger_path(ledger_path) existing = _get_fact_decision(decision_id, sp) if existing is None: raise KeyError(decision_id) current_outcome = existing.get("outcome") or _PENDING backfilled_at = _iso8601_now() if current_outcome != _PENDING: # Idempotent + terminal: do NOT overwrite a non-pending outcome. return { "status": "already_backfilled", "decision_id": decision_id, "existing_outcome": current_outcome, "backfilled_at": backfilled_at, } run_id = existing.get("run_id") or decision_id # 1. UPDATE the fact_decision row in the cold store (mutable fact). conn = sqlite3.connect(sp) # Add backfilled_at column idempotently (schema was added in v1.26 P3 W2; # older cold stores created by P2 lack it — ALTER TABLE is a no-op if # the column already exists). try: conn.execute("ALTER TABLE fact_decision ADD COLUMN backfilled_at TEXT") except sqlite3.OperationalError: pass # column already exists conn.execute( "UPDATE fact_decision SET outcome = ?, backfilled_at = ? WHERE decision_id = ?", (outcome, backfilled_at, decision_id), ) conn.commit() conn.close() # 2. Append an audit event to the append-only Decision Ledger (preserves # the hash chain — the ledger is never UPDATEd in place). try: backfill_data = { "decision_id": decision_id, "previous_outcome": _PENDING, "new_outcome": outcome, "backfilled_at": backfilled_at, } event = make_event( "nova.outcome.backfilled", run_id, existing.get("environment", ""), backfill_data, contract_id=existing.get("contract_id", ""), actor_type="outcome-backfill", actor_id="outcome_backfill", ) append_event(event) ledger_append(event, db_path=lp) except Exception: # Metrics emission must never break the backfill — the cold store # UPDATE is the source of truth for the denominator; the ledger # event is audit chrome. pass return { "status": "backfilled", "decision_id": decision_id, "previous_outcome": _PENDING, "new_outcome": outcome, "backfilled_at": backfilled_at, } if __name__ == "__main__": if len(sys.argv) < 3: print("usage: outcome_backfill.py ", file=sys.stderr) sys.exit(2) _did = sys.argv[1] _out = sys.argv[2] try: _r = backfill(_did, _out) print(json.dumps(_r, indent=2)) except (ValueError, KeyError) as exc: print(f"error: {exc}", file=sys.stderr) sys.exit(1)