feat(P03 W2): outcome backfill (REQ-317) + escalation_reason (REQ-318)
REQ-317: core/metrics/outcome_backfill.py backfills fact_decision.outcome pending -> succeeded/failed after run.completed/run.failed; idempotent + terminal (does not overwrite a non-pending outcome); wired into the collector. The Post-Pilot AI Decision Accuracy denominator is now grounded (fact_decision.outcome is not stuck pending). REQ-318: ai.decision.made on a block band carries escalation_reason: 'confidence' (the only value in v1.26 — a block is always confidence- driven; future milestones may add 'policy'). Persisted into fact_run by the collector. The Post-Pilot Human Escalation Frequency denominator is now grounded. ---ci--- project: acdl phase: 3 milestone: v1.26 status: execute wave: W2 ---
This commit is contained in:
Jon Chery
@@ -0,0 +1,213 @@
|
||||
"""Nova Outcome Backfill (REQ-317, SPEC §5.8, P3 Wave 2).
|
||||
|
||||
The `fact_decision.outcome` column in the metrics cold store is written
|
||||
`pending` by the collector (it ingests `nova.ai.decision.made` events,
|
||||
which are emitted *before* the run executes the apply). Once the run
|
||||
completes (`nova.run.completed`, exit 0) or fails (`nova.run.failed`,
|
||||
exit non-zero), the outcome must be transitioned `pending ->
|
||||
succeeded`/`failed` so the Post-Pilot AI Decision Accuracy denominator is
|
||||
grounded (an outcome that is stuck `pending` cannot be scored).
|
||||
|
||||
Architecture (grounded in what the ledger + collector actually do):
|
||||
|
||||
* The Decision Ledger (`core/metrics/decision_ledger.py`) is an
|
||||
**append-only hash-chain** of CloudEvents envelopes — there is no
|
||||
`fact_decision` table *inside* the ledger DB; facts live in the
|
||||
separate collector cold store (`core/metrics/collector.py`,
|
||||
`nova_metrics.db`). The ledger is never UPDATEd in place (that would
|
||||
break the SHA-256 chain — see `verify_chain()`).
|
||||
* Therefore the backfill does TWO things:
|
||||
1. Appends a new audit event `nova.outcome.backfilled` to the
|
||||
ledger (preserves the hash chain; auditable via `replay_run`).
|
||||
2. UPDATEs the `fact_decision` row in the cold store (the row is
|
||||
keyed by `decision_id`; `outcome` + `backfilled_at` are
|
||||
mutable — they are facts, not chain events).
|
||||
|
||||
Idempotent + terminal:
|
||||
* If `outcome` is already `succeeded`/`failed` (i.e. not `pending`),
|
||||
the call is a no-op and returns `{"status": "already_backfilled",
|
||||
"existing_outcome": <current>}`. A terminal outcome is NEVER
|
||||
overwritten (defense against double-backfill and against flipping a
|
||||
`succeeded` run to `failed` retroactively or vice versa).
|
||||
* The same `outcome` value is re-asserted harmlessly (still a no-op).
|
||||
|
||||
REQ-317: `outcome` ∈ {"succeeded", "failed"} only — `pending` is the
|
||||
initial state and may not be written by the backfill (it would undo the
|
||||
transition). An invalid value raises `ValueError`.
|
||||
|
||||
Future milestones may add `'policy'` to `escalation_reason` (REQ-318);
|
||||
this module is scoped to outcome only.
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Optional, Dict, Any
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
from core.metrics.event_envelope import make_event, append_event
|
||||
from core.metrics.decision_ledger import append as ledger_append, _LEDGER_PATH
|
||||
|
||||
# The collector cold store path is mirrored here so the backfill can be
|
||||
# invoked without importing the collector (avoids a circular import:
|
||||
# the collector calls into backfill at run.completed/run.failed time).
|
||||
_METRICS_DIR = os.path.join(
|
||||
os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
|
||||
"metrics",
|
||||
)
|
||||
_STORE_PATH = os.path.join(_METRICS_DIR, "nova_metrics.db")
|
||||
|
||||
_VALID_OUTCOMES = {"succeeded", "failed"}
|
||||
_PENDING = "pending"
|
||||
|
||||
|
||||
def _iso8601_now():
|
||||
return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
|
||||
def _resolve_store_path(store_path: Optional[str | Path]) -> str:
|
||||
if store_path is None:
|
||||
return _STORE_PATH
|
||||
return str(store_path)
|
||||
|
||||
|
||||
def _resolve_ledger_path(ledger_path: Optional[str | Path]) -> str:
|
||||
if ledger_path is None:
|
||||
return _LEDGER_PATH
|
||||
return str(ledger_path)
|
||||
|
||||
|
||||
def _get_fact_decision(decision_id: str, store_path: str) -> Optional[Dict[str, Any]]:
|
||||
"""Read the fact_decision row for decision_id (or None)."""
|
||||
if not os.path.isfile(store_path):
|
||||
return None
|
||||
conn = sqlite3.connect(store_path)
|
||||
conn.row_factory = sqlite3.Row
|
||||
row = conn.execute(
|
||||
"SELECT decision_id, run_id, chosen_action, confidence, alternatives, "
|
||||
"human_override, outcome, event_time FROM fact_decision WHERE decision_id = ?",
|
||||
(decision_id,),
|
||||
).fetchone()
|
||||
conn.close()
|
||||
if row is None:
|
||||
return None
|
||||
return dict(row)
|
||||
|
||||
|
||||
def backfill(
|
||||
decision_id: str,
|
||||
outcome: str,
|
||||
ledger_path: Optional[str | Path] = None,
|
||||
store_path: Optional[str | Path] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Transition fact_decision.outcome from `pending` to `outcome`.
|
||||
|
||||
Args:
|
||||
decision_id: the decision id (== run_id for v1.26).
|
||||
outcome: the terminal outcome; must be in {"succeeded", "failed"}.
|
||||
ledger_path: optional override for the Decision Ledger SQLite DB.
|
||||
store_path: optional override for the collector cold store SQLite DB.
|
||||
|
||||
Returns:
|
||||
A dict describing the result:
|
||||
* success: {"status": "backfilled", "decision_id", "previous_outcome",
|
||||
"new_outcome", "backfilled_at"}
|
||||
* no-op: {"status": "already_backfilled", "decision_id",
|
||||
"existing_outcome", "backfilled_at"}
|
||||
|
||||
Raises:
|
||||
ValueError: if `outcome` is not in {"succeeded", "failed"}.
|
||||
KeyError: if `decision_id` is not present in fact_decision.
|
||||
"""
|
||||
if outcome not in _VALID_OUTCOMES:
|
||||
raise ValueError(
|
||||
f"outcome must be one of {sorted(_VALID_OUTCOMES)}, got: {outcome!r}"
|
||||
)
|
||||
|
||||
sp = _resolve_store_path(store_path)
|
||||
lp = _resolve_ledger_path(ledger_path)
|
||||
|
||||
existing = _get_fact_decision(decision_id, sp)
|
||||
if existing is None:
|
||||
raise KeyError(decision_id)
|
||||
|
||||
current_outcome = existing.get("outcome") or _PENDING
|
||||
backfilled_at = _iso8601_now()
|
||||
|
||||
if current_outcome != _PENDING:
|
||||
# Idempotent + terminal: do NOT overwrite a non-pending outcome.
|
||||
return {
|
||||
"status": "already_backfilled",
|
||||
"decision_id": decision_id,
|
||||
"existing_outcome": current_outcome,
|
||||
"backfilled_at": backfilled_at,
|
||||
}
|
||||
|
||||
run_id = existing.get("run_id") or decision_id
|
||||
|
||||
# 1. UPDATE the fact_decision row in the cold store (mutable fact).
|
||||
conn = sqlite3.connect(sp)
|
||||
# Add backfilled_at column idempotently (schema was added in v1.26 P3 W2;
|
||||
# older cold stores created by P2 lack it — ALTER TABLE is a no-op if
|
||||
# the column already exists).
|
||||
try:
|
||||
conn.execute("ALTER TABLE fact_decision ADD COLUMN backfilled_at TEXT")
|
||||
except sqlite3.OperationalError:
|
||||
pass # column already exists
|
||||
conn.execute(
|
||||
"UPDATE fact_decision SET outcome = ?, backfilled_at = ? WHERE decision_id = ?",
|
||||
(outcome, backfilled_at, decision_id),
|
||||
)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
# 2. Append an audit event to the append-only Decision Ledger (preserves
|
||||
# the hash chain — the ledger is never UPDATEd in place).
|
||||
try:
|
||||
backfill_data = {
|
||||
"decision_id": decision_id,
|
||||
"previous_outcome": _PENDING,
|
||||
"new_outcome": outcome,
|
||||
"backfilled_at": backfilled_at,
|
||||
}
|
||||
event = make_event(
|
||||
"nova.outcome.backfilled",
|
||||
run_id,
|
||||
existing.get("environment", ""),
|
||||
backfill_data,
|
||||
contract_id=existing.get("contract_id", ""),
|
||||
actor_type="outcome-backfill",
|
||||
actor_id="outcome_backfill",
|
||||
)
|
||||
append_event(event)
|
||||
ledger_append(event, db_path=lp)
|
||||
except Exception:
|
||||
# Metrics emission must never break the backfill — the cold store
|
||||
# UPDATE is the source of truth for the denominator; the ledger
|
||||
# event is audit chrome.
|
||||
pass
|
||||
|
||||
return {
|
||||
"status": "backfilled",
|
||||
"decision_id": decision_id,
|
||||
"previous_outcome": _PENDING,
|
||||
"new_outcome": outcome,
|
||||
"backfilled_at": backfilled_at,
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) < 3:
|
||||
print("usage: outcome_backfill.py <decision_id> <succeeded|failed>", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
_did = sys.argv[1]
|
||||
_out = sys.argv[2]
|
||||
try:
|
||||
_r = backfill(_did, _out)
|
||||
print(json.dumps(_r, indent=2))
|
||||
except (ValueError, KeyError) as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
Reference in New Issue
Block a user