acdl/core/metrics/outcome_backfill.py

"""Nova Outcome Backfill (REQ-317, SPEC §5.8, P3 Wave 2).

The `fact_decision.outcome` column in the metrics cold store is written
`pending` by the collector (it ingests `nova.ai.decision.made` events,
which are emitted *before* the run executes the apply). Once the run
completes (`nova.run.completed`, exit 0) or fails (`nova.run.failed`,
exit non-zero), the outcome must be transitioned `pending ->
succeeded`/`failed` so the Post-Pilot AI Decision Accuracy denominator is
grounded (an outcome that is stuck `pending` cannot be scored).

Architecture (grounded in what the ledger + collector actually do):

  * The Decision Ledger (`core/metrics/decision_ledger.py`) is an
    **append-only hash-chain** of CloudEvents envelopes — there is no
    `fact_decision` table *inside* the ledger DB; facts live in the
    separate collector cold store (`core/metrics/collector.py`,
    `nova_metrics.db`). The ledger is never UPDATEd in place (that would
    break the SHA-256 chain — see `verify_chain()`).
  * Therefore the backfill does TWO things:
      1. Appends a new audit event `nova.outcome.backfilled` to the
         ledger (preserves the hash chain; auditable via `replay_run`).
      2. UPDATEs the `fact_decision` row in the cold store (the row is
         keyed by `decision_id`; `outcome` + `backfilled_at` are
         mutable — they are facts, not chain events).

Idempotent + terminal:
  * If `outcome` is already `succeeded`/`failed` (i.e. not `pending`),
    the call is a no-op and returns `{"status": "already_backfilled",
    "existing_outcome": <current>}`. A terminal outcome is NEVER
    overwritten (defense against double-backfill and against flipping a
    `succeeded` run to `failed` retroactively or vice versa).
  * The same `outcome` value is re-asserted harmlessly (still a no-op).

REQ-317: `outcome` ∈ {"succeeded", "failed"} only — `pending` is the
initial state and may not be written by the backfill (it would undo the
transition). An invalid value raises `ValueError`.

Future milestones may add `'policy'` to `escalation_reason` (REQ-318);
this module is scoped to outcome only.
"""

import datetime
import json
import os
import sqlite3
import sys
from pathlib import Path
from typing import Optional, Dict, Any

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from core.metrics.event_envelope import make_event, append_event
from core.metrics.decision_ledger import append as ledger_append, _LEDGER_PATH

# The collector cold store path is mirrored here so the backfill can be
# invoked without importing the collector (avoids a circular import:
# the collector calls into backfill at run.completed/run.failed time).
_METRICS_DIR = os.path.join(
    os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
    "metrics",
)
_STORE_PATH = os.path.join(_METRICS_DIR, "nova_metrics.db")

_VALID_OUTCOMES = {"succeeded", "failed"}
_PENDING = "pending"


def _iso8601_now():
    return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")


def _resolve_store_path(store_path: Optional[str | Path]) -> str:
    if store_path is None:
        return _STORE_PATH
    return str(store_path)


def _resolve_ledger_path(ledger_path: Optional[str | Path]) -> str:
    if ledger_path is None:
        return _LEDGER_PATH
    return str(ledger_path)


def _get_fact_decision(decision_id: str, store_path: str) -> Optional[Dict[str, Any]]:
    """Read the fact_decision row for decision_id (or None)."""
    if not os.path.isfile(store_path):
        return None
    conn = sqlite3.connect(store_path)
    conn.row_factory = sqlite3.Row
    row = conn.execute(
        "SELECT decision_id, run_id, chosen_action, confidence, alternatives, "
        "human_override, outcome, event_time FROM fact_decision WHERE decision_id = ?",
        (decision_id,),
    ).fetchone()
    conn.close()
    if row is None:
        return None
    return dict(row)


def backfill(
    decision_id: str,
    outcome: str,
    ledger_path: Optional[str | Path] = None,
    store_path: Optional[str | Path] = None,
) -> Dict[str, Any]:
    """Transition fact_decision.outcome from `pending` to `outcome`.

    Args:
        decision_id: the decision id (== run_id for v1.26).
        outcome: the terminal outcome; must be in {"succeeded", "failed"}.
        ledger_path: optional override for the Decision Ledger SQLite DB.
        store_path: optional override for the collector cold store SQLite DB.

    Returns:
        A dict describing the result:
          * success: {"status": "backfilled", "decision_id", "previous_outcome",
                      "new_outcome", "backfilled_at"}
          * no-op:   {"status": "already_backfilled", "decision_id",
                      "existing_outcome", "backfilled_at"}

    Raises:
        ValueError: if `outcome` is not in {"succeeded", "failed"}.
        KeyError:   if `decision_id` is not present in fact_decision.
    """
    if outcome not in _VALID_OUTCOMES:
        raise ValueError(
            f"outcome must be one of {sorted(_VALID_OUTCOMES)}, got: {outcome!r}"
        )

    sp = _resolve_store_path(store_path)
    lp = _resolve_ledger_path(ledger_path)

    existing = _get_fact_decision(decision_id, sp)
    if existing is None:
        raise KeyError(decision_id)

    current_outcome = existing.get("outcome") or _PENDING
    backfilled_at = _iso8601_now()

    if current_outcome != _PENDING:
        # Idempotent + terminal: do NOT overwrite a non-pending outcome.
        return {
            "status": "already_backfilled",
            "decision_id": decision_id,
            "existing_outcome": current_outcome,
            "backfilled_at": backfilled_at,
        }

    run_id = existing.get("run_id") or decision_id

    # 1. UPDATE the fact_decision row in the cold store (mutable fact).
    conn = sqlite3.connect(sp)
    # Add backfilled_at column idempotently (schema was added in v1.26 P3 W2;
    # older cold stores created by P2 lack it — ALTER TABLE is a no-op if
    # the column already exists).
    try:
        conn.execute("ALTER TABLE fact_decision ADD COLUMN backfilled_at TEXT")
    except sqlite3.OperationalError:
        pass  # column already exists
    conn.execute(
        "UPDATE fact_decision SET outcome = ?, backfilled_at = ? WHERE decision_id = ?",
        (outcome, backfilled_at, decision_id),
    )
    conn.commit()
    conn.close()

    # 2. Append an audit event to the append-only Decision Ledger (preserves
    #    the hash chain — the ledger is never UPDATEd in place).
    try:
        backfill_data = {
            "decision_id": decision_id,
            "previous_outcome": _PENDING,
            "new_outcome": outcome,
            "backfilled_at": backfilled_at,
        }
        event = make_event(
            "nova.outcome.backfilled",
            run_id,
            existing.get("environment", ""),
            backfill_data,
            contract_id=existing.get("contract_id", ""),
            actor_type="outcome-backfill",
            actor_id="outcome_backfill",
        )
        append_event(event)
        ledger_append(event, db_path=lp)
    except Exception:
        # Metrics emission must never break the backfill — the cold store
        # UPDATE is the source of truth for the denominator; the ledger
        # event is audit chrome.
        pass

    return {
        "status": "backfilled",
        "decision_id": decision_id,
        "previous_outcome": _PENDING,
        "new_outcome": outcome,
        "backfilled_at": backfilled_at,
    }


if __name__ == "__main__":
    if len(sys.argv) < 3:
        print("usage: outcome_backfill.py <decision_id> <succeeded|failed>", file=sys.stderr)
        sys.exit(2)
    _did = sys.argv[1]
    _out = sys.argv[2]
    try:
        _r = backfill(_did, _out)
        print(json.dumps(_r, indent=2))
    except (ValueError, KeyError) as exc:
        print(f"error: {exc}", file=sys.stderr)
        sys.exit(1)