"""Tests for escalation_reason on ai.decision.made (REQ-318, SPEC §5.8, P3 W2). Covers: * block band carries escalation_reason == "confidence" + human_override True * pass band has escalation_reason ABSENT + human_override False * fact_run persists escalation_reason (collector wiring) Follows the fixture pattern in tests/test_metrics_emitters.py. """ import json import os import sqlite3 import sys from pathlib import Path import pytest ROOT = Path(__file__).resolve().parent.parent sys.path.insert(0, str(ROOT)) @pytest.fixture def tmp_metrics(tmp_path, monkeypatch): """Redirect metrics/ to a tmp dir for isolated testing.""" metrics_dir = tmp_path / "metrics" metrics_dir.mkdir() runs_dir = metrics_dir / "runs" runs_dir.mkdir() events_log = metrics_dir / "events.jsonl" ledger_db = metrics_dir / "decision_ledger.db" store_db = metrics_dir / "nova_metrics.db" monkeypatch.setattr("core.metrics.event_envelope.METRICS_DIR", str(metrics_dir)) monkeypatch.setattr("core.metrics.event_envelope.EVENTS_LOG", str(events_log)) monkeypatch.setattr("core.metrics.run_manifest._METRICS_DIR", str(metrics_dir)) monkeypatch.setattr("core.metrics.run_manifest._RUNS_DIR", str(runs_dir)) monkeypatch.setattr("core.metrics.decision_ledger._LEDGER_PATH", str(ledger_db)) monkeypatch.setattr("core.metrics.collector._METRICS_DIR", str(metrics_dir)) monkeypatch.setattr("core.metrics.collector._STORE_PATH", str(store_db)) monkeypatch.setattr("core.metrics.collector._RUNS_DIR", str(runs_dir)) monkeypatch.setattr("core.metrics.collector._LEDGER_DB", str(ledger_db)) monkeypatch.setattr("core.metrics.outcome_backfill._METRICS_DIR", str(metrics_dir)) monkeypatch.setattr("core.metrics.outcome_backfill._STORE_PATH", str(store_db)) monkeypatch.setattr("core.metrics.outcome_backfill._LEDGER_PATH", str(ledger_db)) return { "metrics_dir": metrics_dir, "events_log": events_log, "ledger_db": ledger_db, "store_db": store_db, "runs_dir": runs_dir, } def _base_inputs(): return { "policy": [{"result": "pass", "severity": "info"}], "validation": {"schema": True, "stack_resolved": True, "tf_validated": True, "tf_planned": True}, "freshness": {"age_days": 0, "max_age_days": 7}, "source": {"submitter": "dev", "commit_sha": "abc"}, "history": {"prior_rollbacks": 0, "prior_policy_fails": 0}, "nfrs": {"conformance": 1.0}, } def _block_inputs(): """A critical PCR triggers a hard override (score=0, band=block).""" inputs = _base_inputs() inputs["policy"] = [{"result": "fail", "severity": "critical", "ruleId": "CKV_X"}] return inputs def _read_decision_event(events_log): lines = events_log.read_text().strip().split("\n") for line in lines: ev = json.loads(line) if ev["type"] == "nova.ai.decision.made": return ev return None def test_block_band_has_escalation_reason(tmp_metrics): """A block (critical PCR hard override) carries escalation_reason='confidence'.""" from core.confidence_signal import compute sig = compute("cid-block-1", "dev", _block_inputs()) assert sig.band == "block" ev = _read_decision_event(tmp_metrics["events_log"]) assert ev is not None data = ev["data"] assert data["chosen_action"] == "block" assert data["human_override"] is True assert data.get("escalation_reason") == "confidence" def test_pass_band_no_escalation_reason(tmp_metrics): """A clean dev apply (pass band) has NO escalation_reason + human_override False.""" from core.confidence_signal import compute sig = compute("cid-pass-1", "dev", _base_inputs()) assert sig.band == "pass" ev = _read_decision_event(tmp_metrics["events_log"]) assert ev is not None data = ev["data"] assert data["chosen_action"] == "pass" assert data["human_override"] is False # escalation_reason must be ABSENT on a non-block band. assert "escalation_reason" not in data def test_low_confidence_block_has_escalation_reason(tmp_metrics): """A score below (threshold - 0.10) blocks on confidence grounds.""" from core.confidence_signal import compute # Freshness maximally stale + a high-severity policy fail drags the # score well below the dev threshold of 0.50 - 0.10 = 0.40. inputs = _base_inputs() inputs["freshness"] = {"age_days": 7, "max_age_days": 7} inputs["policy"] = [{"result": "fail", "severity": "high", "ruleId": "CKV_Y"}] sig = compute("cid-block-2", "dev", inputs) assert sig.band == "block" ev = _read_decision_event(tmp_metrics["events_log"]) data = ev["data"] assert data.get("escalation_reason") == "confidence" def test_fact_run_persists_escalation_reason(tmp_metrics): """The collector persists escalation_reason into fact_run + fact_decision. End-to-end: confidence_signal emits ai.decision.made (block) → run_manifest.complete_run writes the manifest with escalation_reason → collector.collect_run_manifests + collect_decision_ledger populate fact_run.escalation_reason + fact_decision.escalation_reason. """ from core.confidence_signal import compute from core.metrics.run_manifest import complete_run from core.metrics.collector import collect_run_manifests, collect_decision_ledger # Emit a block decision. os.environ["NOVA_RUN_ID"] = "run-esc-1" try: sig = compute("cid-esc-1", "dev", _block_inputs()) assert sig.band == "block" finally: os.environ.pop("NOVA_RUN_ID", None) # Complete the run with escalation_reason carried into the manifest. manifest = complete_run( "run-esc-1", "cid-esc-1", "dev", stages=[{"name": "apply", "duration_ms": 100, "exit_code": 0}], exit_code=0, confidence={"score": sig.score, "band": sig.band, "perInput": sig.perInput}, decision_id="run-esc-1", escalation_reason="confidence", ) assert manifest["escalation_reason"] == "confidence" # Collector reads the manifest → fact_run. collect_run_manifests() # Collector reads the ledger → fact_decision. collect_decision_ledger() conn = sqlite3.connect(str(tmp_metrics["store_db"])) conn.row_factory = sqlite3.Row run_row = conn.execute( "SELECT run_id, escalation_reason, decision_id FROM fact_run WHERE run_id = ?", ("run-esc-1",), ).fetchone() dec_row = conn.execute( "SELECT decision_id, escalation_reason, human_override FROM fact_decision WHERE decision_id = ?", ("run-esc-1",), ).fetchone() conn.close() assert run_row is not None assert run_row["escalation_reason"] == "confidence" assert run_row["decision_id"] == "run-esc-1" assert dec_row is not None assert dec_row["escalation_reason"] == "confidence" assert dec_row["human_override"] == 1 def test_fact_run_no_escalation_reason_on_pass(tmp_metrics): """A pass-band run has escalation_reason NULL in fact_run.""" from core.confidence_signal import compute from core.metrics.run_manifest import complete_run from core.metrics.collector import collect_run_manifests os.environ["NOVA_RUN_ID"] = "run-esc-pass-1" try: sig = compute("cid-esc-pass-1", "dev", _base_inputs()) assert sig.band == "pass" finally: os.environ.pop("NOVA_RUN_ID", None) complete_run( "run-esc-pass-1", "cid-esc-pass-1", "dev", stages=[{"name": "apply", "duration_ms": 100, "exit_code": 0}], exit_code=0, confidence={"score": sig.score, "band": sig.band, "perInput": sig.perInput}, decision_id="run-esc-pass-1", # escalation_reason intentionally omitted (pass band). ) collect_run_manifests() conn = sqlite3.connect(str(tmp_metrics["store_db"])) conn.row_factory = sqlite3.Row row = conn.execute( "SELECT run_id, escalation_reason FROM fact_run WHERE run_id = ?", ("run-esc-pass-1",), ).fetchone() conn.close() assert row is not None assert row["escalation_reason"] is None