acdl/tests/test_outcome_backfill.py

"""Tests for Nova Outcome Backfill (REQ-317, SPEC §5.8, P3 W2).

Covers the fact_decision.outcome transition pending -> succeeded/failed:
  * happy path (succeeded, failed)
  * idempotency (already_backfilled is a no-op)
  * terminal defense (does NOT flip succeeded -> failed)
  * invalid outcome raises ValueError
  * unknown decision_id raises KeyError

Uses a tmp SQLite cold store + Decision Ledger (does NOT touch the real
metrics/decision_ledger.db). Follows the fixture pattern in
tests/test_metrics_collector.py + tests/test_metrics_emitters.py.
"""

import json
import os
import sqlite3
import sys
from pathlib import Path

import pytest

ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT))


@pytest.fixture
def tmp_backfill_env(tmp_path, monkeypatch):
    """Redirect metrics/ to a tmp dir + seed a fact_decision row (pending)."""
    metrics_dir = tmp_path / "metrics"
    metrics_dir.mkdir()
    store_db = metrics_dir / "nova_metrics.db"
    ledger_db = metrics_dir / "decision_ledger.db"
    events_log = metrics_dir / "events.jsonl"

    monkeypatch.setattr("core.metrics.event_envelope.METRICS_DIR", str(metrics_dir))
    monkeypatch.setattr("core.metrics.event_envelope.EVENTS_LOG", str(events_log))
    monkeypatch.setattr("core.metrics.decision_ledger._LEDGER_PATH", str(ledger_db))
    monkeypatch.setattr("core.metrics.outcome_backfill._METRICS_DIR", str(metrics_dir))
    monkeypatch.setattr("core.metrics.outcome_backfill._STORE_PATH", str(store_db))
    monkeypatch.setattr("core.metrics.outcome_backfill._LEDGER_PATH", str(ledger_db))

    # Initialize the cold store schema + a pending fact_decision row.
    from core.metrics.collector import _init_store
    _init_store(str(store_db))

    conn = sqlite3.connect(str(store_db))
    conn.execute(
        "INSERT INTO fact_decision "
        "(decision_id, run_id, chosen_action, confidence, alternatives, "
        "human_override, escalation_reason, outcome, backfilled_at, event_time) "
        "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
        ("dec-1", "run-1", "block", 0.42, "{}", 1, "confidence",
         "pending", None, "2026-08-18T00:00:00Z"),
    )
    conn.commit()
    conn.close()

    return {
        "metrics_dir": metrics_dir,
        "store_db": store_db,
        "ledger_db": ledger_db,
        "events_log": events_log,
        "decision_id": "dec-1",
    }


def _get_fact_decision(store_db, decision_id):
    conn = sqlite3.connect(str(store_db))
    conn.row_factory = sqlite3.Row
    row = conn.execute(
        "SELECT decision_id, outcome, backfilled_at FROM fact_decision WHERE decision_id = ?",
        (decision_id,),
    ).fetchone()
    conn.close()
    return dict(row) if row else None


def test_backfill_succeeded(tmp_backfill_env):
    from core.metrics.outcome_backfill import backfill
    result = backfill(tmp_backfill_env["decision_id"], "succeeded")
    assert result["status"] == "backfilled"
    assert result["previous_outcome"] == "pending"
    assert result["new_outcome"] == "succeeded"
    assert result["backfilled_at"]
    fact = _get_fact_decision(tmp_backfill_env["store_db"], "dec-1")
    assert fact["outcome"] == "succeeded"
    assert fact["backfilled_at"] == result["backfilled_at"]


def test_backfill_failed(tmp_backfill_env):
    from core.metrics.outcome_backfill import backfill
    result = backfill(tmp_backfill_env["decision_id"], "failed")
    assert result["status"] == "backfilled"
    assert result["new_outcome"] == "failed"
    fact = _get_fact_decision(tmp_backfill_env["store_db"], "dec-1")
    assert fact["outcome"] == "failed"


def test_backfill_idempotent(tmp_backfill_env):
    """Already succeeded → backfill('succeeded') again → no-op."""
    from core.metrics.outcome_backfill import backfill
    backfill(tmp_backfill_env["decision_id"], "succeeded")
    result = backfill(tmp_backfill_env["decision_id"], "succeeded")
    assert result["status"] == "already_backfilled"
    assert result["existing_outcome"] == "succeeded"
    fact = _get_fact_decision(tmp_backfill_env["store_db"], "dec-1")
    assert fact["outcome"] == "succeeded"


def test_backfill_does_not_overwrite(tmp_backfill_env):
    """Already succeeded → backfill('failed') → must NOT flip to failed.

    A terminal outcome is never overwritten (defense against double-backfill
    and against retroactively flipping succeeded -> failed).
    """
    from core.metrics.outcome_backfill import backfill
    backfill(tmp_backfill_env["decision_id"], "succeeded")
    result = backfill(tmp_backfill_env["decision_id"], "failed")
    assert result["status"] == "already_backfilled"
    assert result["existing_outcome"] == "succeeded"
    fact = _get_fact_decision(tmp_backfill_env["store_db"], "dec-1")
    assert fact["outcome"] == "succeeded"


def test_backfill_invalid_outcome(tmp_backfill_env):
    from core.metrics.outcome_backfill import backfill
    with pytest.raises(ValueError):
        backfill(tmp_backfill_env["decision_id"], "pending")
    with pytest.raises(ValueError):
        backfill(tmp_backfill_env["decision_id"], "garbage")


def test_backfill_unknown_decision(tmp_backfill_env):
    from core.metrics.outcome_backfill import backfill
    with pytest.raises(KeyError):
        backfill("nonexistent-decision-id", "succeeded")


def test_backfill_appends_ledger_event(tmp_backfill_env):
    """The backfill appends nova.outcome.backfilled to the Decision Ledger
    (preserves the hash chain — the ledger is never UPDATEd in place)."""
    from core.metrics.outcome_backfill import backfill
    from core.metrics.decision_ledger import query_by_run, verify_chain
    backfill(tmp_backfill_env["decision_id"], "succeeded")
    entries = query_by_run("run-1", db_path=str(tmp_backfill_env["ledger_db"]))
    types = [e["event_type"] for e in entries]
    assert "nova.outcome.backfilled" in types
    # Hash chain still intact.
    ok, broken, _ = verify_chain(db_path=str(tmp_backfill_env["ledger_db"]))
    assert ok, f"chain broken after backfill: {broken}"
    assert broken == 0