acdl/tests/test_confidence_escalation_reason.py

"""Tests for escalation_reason on ai.decision.made (REQ-318, SPEC §5.8, P3 W2).

Covers:
  * block band carries escalation_reason == "confidence" + human_override True
  * pass band has escalation_reason ABSENT + human_override False
  * fact_run persists escalation_reason (collector wiring)

Follows the fixture pattern in tests/test_metrics_emitters.py.
"""

import json
import os
import sqlite3
import sys
from pathlib import Path

import pytest

ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT))


@pytest.fixture
def tmp_metrics(tmp_path, monkeypatch):
    """Redirect metrics/ to a tmp dir for isolated testing."""
    metrics_dir = tmp_path / "metrics"
    metrics_dir.mkdir()
    runs_dir = metrics_dir / "runs"
    runs_dir.mkdir()
    events_log = metrics_dir / "events.jsonl"
    ledger_db = metrics_dir / "decision_ledger.db"
    store_db = metrics_dir / "nova_metrics.db"

    monkeypatch.setattr("core.metrics.event_envelope.METRICS_DIR", str(metrics_dir))
    monkeypatch.setattr("core.metrics.event_envelope.EVENTS_LOG", str(events_log))
    monkeypatch.setattr("core.metrics.run_manifest._METRICS_DIR", str(metrics_dir))
    monkeypatch.setattr("core.metrics.run_manifest._RUNS_DIR", str(runs_dir))
    monkeypatch.setattr("core.metrics.decision_ledger._LEDGER_PATH", str(ledger_db))
    monkeypatch.setattr("core.metrics.collector._METRICS_DIR", str(metrics_dir))
    monkeypatch.setattr("core.metrics.collector._STORE_PATH", str(store_db))
    monkeypatch.setattr("core.metrics.collector._RUNS_DIR", str(runs_dir))
    monkeypatch.setattr("core.metrics.collector._LEDGER_DB", str(ledger_db))
    monkeypatch.setattr("core.metrics.outcome_backfill._METRICS_DIR", str(metrics_dir))
    monkeypatch.setattr("core.metrics.outcome_backfill._STORE_PATH", str(store_db))
    monkeypatch.setattr("core.metrics.outcome_backfill._LEDGER_PATH", str(ledger_db))
    return {
        "metrics_dir": metrics_dir,
        "events_log": events_log,
        "ledger_db": ledger_db,
        "store_db": store_db,
        "runs_dir": runs_dir,
    }


def _base_inputs():
    return {
        "policy": [{"result": "pass", "severity": "info"}],
        "validation": {"schema": True, "stack_resolved": True,
                       "tf_validated": True, "tf_planned": True},
        "freshness": {"age_days": 0, "max_age_days": 7},
        "source": {"submitter": "dev", "commit_sha": "abc"},
        "history": {"prior_rollbacks": 0, "prior_policy_fails": 0},
        "nfrs": {"conformance": 1.0},
    }


def _block_inputs():
    """A critical PCR triggers a hard override (score=0, band=block)."""
    inputs = _base_inputs()
    inputs["policy"] = [{"result": "fail", "severity": "critical", "ruleId": "CKV_X"}]
    return inputs


def _read_decision_event(events_log):
    lines = events_log.read_text().strip().split("\n")
    for line in lines:
        ev = json.loads(line)
        if ev["type"] == "nova.ai.decision.made":
            return ev
    return None


def test_block_band_has_escalation_reason(tmp_metrics):
    """A block (critical PCR hard override) carries escalation_reason='confidence'."""
    from core.confidence_signal import compute
    sig = compute("cid-block-1", "dev", _block_inputs())
    assert sig.band == "block"
    ev = _read_decision_event(tmp_metrics["events_log"])
    assert ev is not None
    data = ev["data"]
    assert data["chosen_action"] == "block"
    assert data["human_override"] is True
    assert data.get("escalation_reason") == "confidence"


def test_pass_band_no_escalation_reason(tmp_metrics):
    """A clean dev apply (pass band) has NO escalation_reason + human_override False."""
    from core.confidence_signal import compute
    sig = compute("cid-pass-1", "dev", _base_inputs())
    assert sig.band == "pass"
    ev = _read_decision_event(tmp_metrics["events_log"])
    assert ev is not None
    data = ev["data"]
    assert data["chosen_action"] == "pass"
    assert data["human_override"] is False
    # escalation_reason must be ABSENT on a non-block band.
    assert "escalation_reason" not in data


def test_low_confidence_block_has_escalation_reason(tmp_metrics):
    """A score below (threshold - 0.10) blocks on confidence grounds."""
    from core.confidence_signal import compute
    # Freshness maximally stale + a high-severity policy fail drags the
    # score well below the dev threshold of 0.50 - 0.10 = 0.40.
    inputs = _base_inputs()
    inputs["freshness"] = {"age_days": 7, "max_age_days": 7}
    inputs["policy"] = [{"result": "fail", "severity": "high", "ruleId": "CKV_Y"}]
    sig = compute("cid-block-2", "dev", inputs)
    assert sig.band == "block"
    ev = _read_decision_event(tmp_metrics["events_log"])
    data = ev["data"]
    assert data.get("escalation_reason") == "confidence"


def test_fact_run_persists_escalation_reason(tmp_metrics):
    """The collector persists escalation_reason into fact_run + fact_decision.

    End-to-end: confidence_signal emits ai.decision.made (block) →
    run_manifest.complete_run writes the manifest with escalation_reason →
    collector.collect_run_manifests + collect_decision_ledger populate
    fact_run.escalation_reason + fact_decision.escalation_reason.
    """
    from core.confidence_signal import compute
    from core.metrics.run_manifest import complete_run
    from core.metrics.collector import collect_run_manifests, collect_decision_ledger

    # Emit a block decision.
    os.environ["NOVA_RUN_ID"] = "run-esc-1"
    try:
        sig = compute("cid-esc-1", "dev", _block_inputs())
        assert sig.band == "block"
    finally:
        os.environ.pop("NOVA_RUN_ID", None)

    # Complete the run with escalation_reason carried into the manifest.
    manifest = complete_run(
        "run-esc-1", "cid-esc-1", "dev",
        stages=[{"name": "apply", "duration_ms": 100, "exit_code": 0}],
        exit_code=0,
        confidence={"score": sig.score, "band": sig.band, "perInput": sig.perInput},
        decision_id="run-esc-1",
        escalation_reason="confidence",
    )
    assert manifest["escalation_reason"] == "confidence"

    # Collector reads the manifest → fact_run.
    collect_run_manifests()
    # Collector reads the ledger → fact_decision.
    collect_decision_ledger()

    conn = sqlite3.connect(str(tmp_metrics["store_db"]))
    conn.row_factory = sqlite3.Row
    run_row = conn.execute(
        "SELECT run_id, escalation_reason, decision_id FROM fact_run WHERE run_id = ?",
        ("run-esc-1",),
    ).fetchone()
    dec_row = conn.execute(
        "SELECT decision_id, escalation_reason, human_override FROM fact_decision WHERE decision_id = ?",
        ("run-esc-1",),
    ).fetchone()
    conn.close()

    assert run_row is not None
    assert run_row["escalation_reason"] == "confidence"
    assert run_row["decision_id"] == "run-esc-1"
    assert dec_row is not None
    assert dec_row["escalation_reason"] == "confidence"
    assert dec_row["human_override"] == 1


def test_fact_run_no_escalation_reason_on_pass(tmp_metrics):
    """A pass-band run has escalation_reason NULL in fact_run."""
    from core.confidence_signal import compute
    from core.metrics.run_manifest import complete_run
    from core.metrics.collector import collect_run_manifests

    os.environ["NOVA_RUN_ID"] = "run-esc-pass-1"
    try:
        sig = compute("cid-esc-pass-1", "dev", _base_inputs())
        assert sig.band == "pass"
    finally:
        os.environ.pop("NOVA_RUN_ID", None)

    complete_run(
        "run-esc-pass-1", "cid-esc-pass-1", "dev",
        stages=[{"name": "apply", "duration_ms": 100, "exit_code": 0}],
        exit_code=0,
        confidence={"score": sig.score, "band": sig.band, "perInput": sig.perInput},
        decision_id="run-esc-pass-1",
        # escalation_reason intentionally omitted (pass band).
    )
    collect_run_manifests()
    conn = sqlite3.connect(str(tmp_metrics["store_db"]))
    conn.row_factory = sqlite3.Row
    row = conn.execute(
        "SELECT run_id, escalation_reason FROM fact_run WHERE run_id = ?",
        ("run-esc-pass-1",),
    ).fetchone()
    conn.close()
    assert row is not None
    assert row["escalation_reason"] is None