feat(P03 W2): outcome backfill (REQ-317) + escalation_reason (REQ-318)

REQ-317: core/metrics/outcome_backfill.py backfills fact_decision.outcome pending -> succeeded/failed after run.completed/run.failed; idempotent + terminal (does not overwrite a non-pending outcome); wired into the collector. The Post-Pilot AI Decision Accuracy denominator is now grounded (fact_decision.outcome is not stuck pending). REQ-318: ai.decision.made on a block band carries escalation_reason: 'confidence' (the only value in v1.26 — a block is always confidence- driven; future milestones may add 'policy'). Persisted into fact_run by the collector. The Post-Pilot Human Escalation Frequency denominator is now grounded. ---ci--- project: acdl phase: 3 milestone: v1.26 status: execute wave: W2 ---
2026-08-18 22:14:03 +00:00
parent 804c52aa90
commit 51b886f3f6
7 changed files with 682 additions and 19 deletions
@@ -0,0 +1,211 @@
+"""Tests for escalation_reason on ai.decision.made (REQ-318, SPEC §5.8, P3 W2).
+
+Covers:
+  * block band carries escalation_reason == "confidence" + human_override True
+  * pass band has escalation_reason ABSENT + human_override False
+  * fact_run persists escalation_reason (collector wiring)
+
+Follows the fixture pattern in tests/test_metrics_emitters.py.
+"""
+
+import json
+import os
+import sqlite3
+import sys
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(ROOT))
+
+
+@pytest.fixture
+def tmp_metrics(tmp_path, monkeypatch):
+    """Redirect metrics/ to a tmp dir for isolated testing."""
+    metrics_dir = tmp_path / "metrics"
+    metrics_dir.mkdir()
+    runs_dir = metrics_dir / "runs"
+    runs_dir.mkdir()
+    events_log = metrics_dir / "events.jsonl"
+    ledger_db = metrics_dir / "decision_ledger.db"
+    store_db = metrics_dir / "nova_metrics.db"
+
+    monkeypatch.setattr("core.metrics.event_envelope.METRICS_DIR", str(metrics_dir))
+    monkeypatch.setattr("core.metrics.event_envelope.EVENTS_LOG", str(events_log))
+    monkeypatch.setattr("core.metrics.run_manifest._METRICS_DIR", str(metrics_dir))
+    monkeypatch.setattr("core.metrics.run_manifest._RUNS_DIR", str(runs_dir))
+    monkeypatch.setattr("core.metrics.decision_ledger._LEDGER_PATH", str(ledger_db))
+    monkeypatch.setattr("core.metrics.collector._METRICS_DIR", str(metrics_dir))
+    monkeypatch.setattr("core.metrics.collector._STORE_PATH", str(store_db))
+    monkeypatch.setattr("core.metrics.collector._RUNS_DIR", str(runs_dir))
+    monkeypatch.setattr("core.metrics.collector._LEDGER_DB", str(ledger_db))
+    monkeypatch.setattr("core.metrics.outcome_backfill._METRICS_DIR", str(metrics_dir))
+    monkeypatch.setattr("core.metrics.outcome_backfill._STORE_PATH", str(store_db))
+    monkeypatch.setattr("core.metrics.outcome_backfill._LEDGER_PATH", str(ledger_db))
+    return {
+        "metrics_dir": metrics_dir,
+        "events_log": events_log,
+        "ledger_db": ledger_db,
+        "store_db": store_db,
+        "runs_dir": runs_dir,
+    }
+
+
+def _base_inputs():
+    return {
+        "policy": [{"result": "pass", "severity": "info"}],
+        "validation": {"schema": True, "stack_resolved": True,
+                       "tf_validated": True, "tf_planned": True},
+        "freshness": {"age_days": 0, "max_age_days": 7},
+        "source": {"submitter": "dev", "commit_sha": "abc"},
+        "history": {"prior_rollbacks": 0, "prior_policy_fails": 0},
+        "nfrs": {"conformance": 1.0},
+    }
+
+
+def _block_inputs():
+    """A critical PCR triggers a hard override (score=0, band=block)."""
+    inputs = _base_inputs()
+    inputs["policy"] = [{"result": "fail", "severity": "critical", "ruleId": "CKV_X"}]
+    return inputs
+
+
+def _read_decision_event(events_log):
+    lines = events_log.read_text().strip().split("\n")
+    for line in lines:
+        ev = json.loads(line)
+        if ev["type"] == "nova.ai.decision.made":
+            return ev
+    return None
+
+
+def test_block_band_has_escalation_reason(tmp_metrics):
+    """A block (critical PCR hard override) carries escalation_reason='confidence'."""
+    from core.confidence_signal import compute
+    sig = compute("cid-block-1", "dev", _block_inputs())
+    assert sig.band == "block"
+    ev = _read_decision_event(tmp_metrics["events_log"])
+    assert ev is not None
+    data = ev["data"]
+    assert data["chosen_action"] == "block"
+    assert data["human_override"] is True
+    assert data.get("escalation_reason") == "confidence"
+
+
+def test_pass_band_no_escalation_reason(tmp_metrics):
+    """A clean dev apply (pass band) has NO escalation_reason + human_override False."""
+    from core.confidence_signal import compute
+    sig = compute("cid-pass-1", "dev", _base_inputs())
+    assert sig.band == "pass"
+    ev = _read_decision_event(tmp_metrics["events_log"])
+    assert ev is not None
+    data = ev["data"]
+    assert data["chosen_action"] == "pass"
+    assert data["human_override"] is False
+    # escalation_reason must be ABSENT on a non-block band.
+    assert "escalation_reason" not in data
+
+
+def test_low_confidence_block_has_escalation_reason(tmp_metrics):
+    """A score below (threshold - 0.10) blocks on confidence grounds."""
+    from core.confidence_signal import compute
+    # Freshness maximally stale + a high-severity policy fail drags the
+    # score well below the dev threshold of 0.50 - 0.10 = 0.40.
+    inputs = _base_inputs()
+    inputs["freshness"] = {"age_days": 7, "max_age_days": 7}
+    inputs["policy"] = [{"result": "fail", "severity": "high", "ruleId": "CKV_Y"}]
+    sig = compute("cid-block-2", "dev", inputs)
+    assert sig.band == "block"
+    ev = _read_decision_event(tmp_metrics["events_log"])
+    data = ev["data"]
+    assert data.get("escalation_reason") == "confidence"
+
+
+def test_fact_run_persists_escalation_reason(tmp_metrics):
+    """The collector persists escalation_reason into fact_run + fact_decision.
+
+    End-to-end: confidence_signal emits ai.decision.made (block) →
+    run_manifest.complete_run writes the manifest with escalation_reason →
+    collector.collect_run_manifests + collect_decision_ledger populate
+    fact_run.escalation_reason + fact_decision.escalation_reason.
+    """
+    from core.confidence_signal import compute
+    from core.metrics.run_manifest import complete_run
+    from core.metrics.collector import collect_run_manifests, collect_decision_ledger
+
+    # Emit a block decision.
+    os.environ["NOVA_RUN_ID"] = "run-esc-1"
+    try:
+        sig = compute("cid-esc-1", "dev", _block_inputs())
+        assert sig.band == "block"
+    finally:
+        os.environ.pop("NOVA_RUN_ID", None)
+
+    # Complete the run with escalation_reason carried into the manifest.
+    manifest = complete_run(
+        "run-esc-1", "cid-esc-1", "dev",
+        stages=[{"name": "apply", "duration_ms": 100, "exit_code": 0}],
+        exit_code=0,
+        confidence={"score": sig.score, "band": sig.band, "perInput": sig.perInput},
+        decision_id="run-esc-1",
+        escalation_reason="confidence",
+    )
+    assert manifest["escalation_reason"] == "confidence"
+
+    # Collector reads the manifest → fact_run.
+    collect_run_manifests()
+    # Collector reads the ledger → fact_decision.
+    collect_decision_ledger()
+
+    conn = sqlite3.connect(str(tmp_metrics["store_db"]))
+    conn.row_factory = sqlite3.Row
+    run_row = conn.execute(
+        "SELECT run_id, escalation_reason, decision_id FROM fact_run WHERE run_id = ?",
+        ("run-esc-1",),
+    ).fetchone()
+    dec_row = conn.execute(
+        "SELECT decision_id, escalation_reason, human_override FROM fact_decision WHERE decision_id = ?",
+        ("run-esc-1",),
+    ).fetchone()
+    conn.close()
+
+    assert run_row is not None
+    assert run_row["escalation_reason"] == "confidence"
+    assert run_row["decision_id"] == "run-esc-1"
+    assert dec_row is not None
+    assert dec_row["escalation_reason"] == "confidence"
+    assert dec_row["human_override"] == 1
+
+
+def test_fact_run_no_escalation_reason_on_pass(tmp_metrics):
+    """A pass-band run has escalation_reason NULL in fact_run."""
+    from core.confidence_signal import compute
+    from core.metrics.run_manifest import complete_run
+    from core.metrics.collector import collect_run_manifests
+
+    os.environ["NOVA_RUN_ID"] = "run-esc-pass-1"
+    try:
+        sig = compute("cid-esc-pass-1", "dev", _base_inputs())
+        assert sig.band == "pass"
+    finally:
+        os.environ.pop("NOVA_RUN_ID", None)
+
+    complete_run(
+        "run-esc-pass-1", "cid-esc-pass-1", "dev",
+        stages=[{"name": "apply", "duration_ms": 100, "exit_code": 0}],
+        exit_code=0,
+        confidence={"score": sig.score, "band": sig.band, "perInput": sig.perInput},
+        decision_id="run-esc-pass-1",
+        # escalation_reason intentionally omitted (pass band).
+    )
+    collect_run_manifests()
+    conn = sqlite3.connect(str(tmp_metrics["store_db"]))
+    conn.row_factory = sqlite3.Row
+    row = conn.execute(
+        "SELECT run_id, escalation_reason FROM fact_run WHERE run_id = ?",
+        ("run-esc-pass-1",),
+    ).fetchone()
+    conn.close()
+    assert row is not None
+    assert row["escalation_reason"] is None
@@ -0,0 +1,152 @@
+"""Tests for Nova Outcome Backfill (REQ-317, SPEC §5.8, P3 W2).
+
+Covers the fact_decision.outcome transition pending -> succeeded/failed:
+  * happy path (succeeded, failed)
+  * idempotency (already_backfilled is a no-op)
+  * terminal defense (does NOT flip succeeded -> failed)
+  * invalid outcome raises ValueError
+  * unknown decision_id raises KeyError
+
+Uses a tmp SQLite cold store + Decision Ledger (does NOT touch the real
+metrics/decision_ledger.db). Follows the fixture pattern in
+tests/test_metrics_collector.py + tests/test_metrics_emitters.py.
+"""
+
+import json
+import os
+import sqlite3
+import sys
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(ROOT))
+
+
+@pytest.fixture
+def tmp_backfill_env(tmp_path, monkeypatch):
+    """Redirect metrics/ to a tmp dir + seed a fact_decision row (pending)."""
+    metrics_dir = tmp_path / "metrics"
+    metrics_dir.mkdir()
+    store_db = metrics_dir / "nova_metrics.db"
+    ledger_db = metrics_dir / "decision_ledger.db"
+    events_log = metrics_dir / "events.jsonl"
+
+    monkeypatch.setattr("core.metrics.event_envelope.METRICS_DIR", str(metrics_dir))
+    monkeypatch.setattr("core.metrics.event_envelope.EVENTS_LOG", str(events_log))
+    monkeypatch.setattr("core.metrics.decision_ledger._LEDGER_PATH", str(ledger_db))
+    monkeypatch.setattr("core.metrics.outcome_backfill._METRICS_DIR", str(metrics_dir))
+    monkeypatch.setattr("core.metrics.outcome_backfill._STORE_PATH", str(store_db))
+    monkeypatch.setattr("core.metrics.outcome_backfill._LEDGER_PATH", str(ledger_db))
+
+    # Initialize the cold store schema + a pending fact_decision row.
+    from core.metrics.collector import _init_store
+    _init_store(str(store_db))
+
+    conn = sqlite3.connect(str(store_db))
+    conn.execute(
+        "INSERT INTO fact_decision "
+        "(decision_id, run_id, chosen_action, confidence, alternatives, "
+        "human_override, escalation_reason, outcome, backfilled_at, event_time) "
+        "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+        ("dec-1", "run-1", "block", 0.42, "{}", 1, "confidence",
+         "pending", None, "2026-08-18T00:00:00Z"),
+    )
+    conn.commit()
+    conn.close()
+
+    return {
+        "metrics_dir": metrics_dir,
+        "store_db": store_db,
+        "ledger_db": ledger_db,
+        "events_log": events_log,
+        "decision_id": "dec-1",
+    }
+
+
+def _get_fact_decision(store_db, decision_id):
+    conn = sqlite3.connect(str(store_db))
+    conn.row_factory = sqlite3.Row
+    row = conn.execute(
+        "SELECT decision_id, outcome, backfilled_at FROM fact_decision WHERE decision_id = ?",
+        (decision_id,),
+    ).fetchone()
+    conn.close()
+    return dict(row) if row else None
+
+
+def test_backfill_succeeded(tmp_backfill_env):
+    from core.metrics.outcome_backfill import backfill
+    result = backfill(tmp_backfill_env["decision_id"], "succeeded")
+    assert result["status"] == "backfilled"
+    assert result["previous_outcome"] == "pending"
+    assert result["new_outcome"] == "succeeded"
+    assert result["backfilled_at"]
+    fact = _get_fact_decision(tmp_backfill_env["store_db"], "dec-1")
+    assert fact["outcome"] == "succeeded"
+    assert fact["backfilled_at"] == result["backfilled_at"]
+
+
+def test_backfill_failed(tmp_backfill_env):
+    from core.metrics.outcome_backfill import backfill
+    result = backfill(tmp_backfill_env["decision_id"], "failed")
+    assert result["status"] == "backfilled"
+    assert result["new_outcome"] == "failed"
+    fact = _get_fact_decision(tmp_backfill_env["store_db"], "dec-1")
+    assert fact["outcome"] == "failed"
+
+
+def test_backfill_idempotent(tmp_backfill_env):
+    """Already succeeded → backfill('succeeded') again → no-op."""
+    from core.metrics.outcome_backfill import backfill
+    backfill(tmp_backfill_env["decision_id"], "succeeded")
+    result = backfill(tmp_backfill_env["decision_id"], "succeeded")
+    assert result["status"] == "already_backfilled"
+    assert result["existing_outcome"] == "succeeded"
+    fact = _get_fact_decision(tmp_backfill_env["store_db"], "dec-1")
+    assert fact["outcome"] == "succeeded"
+
+
+def test_backfill_does_not_overwrite(tmp_backfill_env):
+    """Already succeeded → backfill('failed') → must NOT flip to failed.
+
+    A terminal outcome is never overwritten (defense against double-backfill
+    and against retroactively flipping succeeded -> failed).
+    """
+    from core.metrics.outcome_backfill import backfill
+    backfill(tmp_backfill_env["decision_id"], "succeeded")
+    result = backfill(tmp_backfill_env["decision_id"], "failed")
+    assert result["status"] == "already_backfilled"
+    assert result["existing_outcome"] == "succeeded"
+    fact = _get_fact_decision(tmp_backfill_env["store_db"], "dec-1")
+    assert fact["outcome"] == "succeeded"
+
+
+def test_backfill_invalid_outcome(tmp_backfill_env):
+    from core.metrics.outcome_backfill import backfill
+    with pytest.raises(ValueError):
+        backfill(tmp_backfill_env["decision_id"], "pending")
+    with pytest.raises(ValueError):
+        backfill(tmp_backfill_env["decision_id"], "garbage")
+
+
+def test_backfill_unknown_decision(tmp_backfill_env):
+    from core.metrics.outcome_backfill import backfill
+    with pytest.raises(KeyError):
+        backfill("nonexistent-decision-id", "succeeded")
+
+
+def test_backfill_appends_ledger_event(tmp_backfill_env):
+    """The backfill appends nova.outcome.backfilled to the Decision Ledger
+    (preserves the hash chain — the ledger is never UPDATEd in place)."""
+    from core.metrics.outcome_backfill import backfill
+    from core.metrics.decision_ledger import query_by_run, verify_chain
+    backfill(tmp_backfill_env["decision_id"], "succeeded")
+    entries = query_by_run("run-1", db_path=str(tmp_backfill_env["ledger_db"]))
+    types = [e["event_type"] for e in entries]
+    assert "nova.outcome.backfilled" in types
+    # Hash chain still intact.
+    ok, broken, _ = verify_chain(db_path=str(tmp_backfill_env["ledger_db"]))
+    assert ok, f"chain broken after backfill: {broken}"
+    assert broken == 0