"""Tests for Nova Outcome Backfill (REQ-317, SPEC §5.8, P3 W2). Covers the fact_decision.outcome transition pending -> succeeded/failed: * happy path (succeeded, failed) * idempotency (already_backfilled is a no-op) * terminal defense (does NOT flip succeeded -> failed) * invalid outcome raises ValueError * unknown decision_id raises KeyError Uses a tmp SQLite cold store + Decision Ledger (does NOT touch the real metrics/decision_ledger.db). Follows the fixture pattern in tests/test_metrics_collector.py + tests/test_metrics_emitters.py. """ import json import os import sqlite3 import sys from pathlib import Path import pytest ROOT = Path(__file__).resolve().parent.parent sys.path.insert(0, str(ROOT)) @pytest.fixture def tmp_backfill_env(tmp_path, monkeypatch): """Redirect metrics/ to a tmp dir + seed a fact_decision row (pending).""" metrics_dir = tmp_path / "metrics" metrics_dir.mkdir() store_db = metrics_dir / "nova_metrics.db" ledger_db = metrics_dir / "decision_ledger.db" events_log = metrics_dir / "events.jsonl" monkeypatch.setattr("core.metrics.event_envelope.METRICS_DIR", str(metrics_dir)) monkeypatch.setattr("core.metrics.event_envelope.EVENTS_LOG", str(events_log)) monkeypatch.setattr("core.metrics.decision_ledger._LEDGER_PATH", str(ledger_db)) monkeypatch.setattr("core.metrics.outcome_backfill._METRICS_DIR", str(metrics_dir)) monkeypatch.setattr("core.metrics.outcome_backfill._STORE_PATH", str(store_db)) monkeypatch.setattr("core.metrics.outcome_backfill._LEDGER_PATH", str(ledger_db)) # Initialize the cold store schema + a pending fact_decision row. from core.metrics.collector import _init_store _init_store(str(store_db)) conn = sqlite3.connect(str(store_db)) conn.execute( "INSERT INTO fact_decision " "(decision_id, run_id, chosen_action, confidence, alternatives, " "human_override, escalation_reason, outcome, backfilled_at, event_time) " "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", ("dec-1", "run-1", "block", 0.42, "{}", 1, "confidence", "pending", None, "2026-08-18T00:00:00Z"), ) conn.commit() conn.close() return { "metrics_dir": metrics_dir, "store_db": store_db, "ledger_db": ledger_db, "events_log": events_log, "decision_id": "dec-1", } def _get_fact_decision(store_db, decision_id): conn = sqlite3.connect(str(store_db)) conn.row_factory = sqlite3.Row row = conn.execute( "SELECT decision_id, outcome, backfilled_at FROM fact_decision WHERE decision_id = ?", (decision_id,), ).fetchone() conn.close() return dict(row) if row else None def test_backfill_succeeded(tmp_backfill_env): from core.metrics.outcome_backfill import backfill result = backfill(tmp_backfill_env["decision_id"], "succeeded") assert result["status"] == "backfilled" assert result["previous_outcome"] == "pending" assert result["new_outcome"] == "succeeded" assert result["backfilled_at"] fact = _get_fact_decision(tmp_backfill_env["store_db"], "dec-1") assert fact["outcome"] == "succeeded" assert fact["backfilled_at"] == result["backfilled_at"] def test_backfill_failed(tmp_backfill_env): from core.metrics.outcome_backfill import backfill result = backfill(tmp_backfill_env["decision_id"], "failed") assert result["status"] == "backfilled" assert result["new_outcome"] == "failed" fact = _get_fact_decision(tmp_backfill_env["store_db"], "dec-1") assert fact["outcome"] == "failed" def test_backfill_idempotent(tmp_backfill_env): """Already succeeded → backfill('succeeded') again → no-op.""" from core.metrics.outcome_backfill import backfill backfill(tmp_backfill_env["decision_id"], "succeeded") result = backfill(tmp_backfill_env["decision_id"], "succeeded") assert result["status"] == "already_backfilled" assert result["existing_outcome"] == "succeeded" fact = _get_fact_decision(tmp_backfill_env["store_db"], "dec-1") assert fact["outcome"] == "succeeded" def test_backfill_does_not_overwrite(tmp_backfill_env): """Already succeeded → backfill('failed') → must NOT flip to failed. A terminal outcome is never overwritten (defense against double-backfill and against retroactively flipping succeeded -> failed). """ from core.metrics.outcome_backfill import backfill backfill(tmp_backfill_env["decision_id"], "succeeded") result = backfill(tmp_backfill_env["decision_id"], "failed") assert result["status"] == "already_backfilled" assert result["existing_outcome"] == "succeeded" fact = _get_fact_decision(tmp_backfill_env["store_db"], "dec-1") assert fact["outcome"] == "succeeded" def test_backfill_invalid_outcome(tmp_backfill_env): from core.metrics.outcome_backfill import backfill with pytest.raises(ValueError): backfill(tmp_backfill_env["decision_id"], "pending") with pytest.raises(ValueError): backfill(tmp_backfill_env["decision_id"], "garbage") def test_backfill_unknown_decision(tmp_backfill_env): from core.metrics.outcome_backfill import backfill with pytest.raises(KeyError): backfill("nonexistent-decision-id", "succeeded") def test_backfill_appends_ledger_event(tmp_backfill_env): """The backfill appends nova.outcome.backfilled to the Decision Ledger (preserves the hash chain — the ledger is never UPDATEd in place).""" from core.metrics.outcome_backfill import backfill from core.metrics.decision_ledger import query_by_run, verify_chain backfill(tmp_backfill_env["decision_id"], "succeeded") entries = query_by_run("run-1", db_path=str(tmp_backfill_env["ledger_db"])) types = [e["event_type"] for e in entries] assert "nova.outcome.backfilled" in types # Hash chain still intact. ok, broken, _ = verify_chain(db_path=str(tmp_backfill_env["ledger_db"])) assert ok, f"chain broken after backfill: {broken}" assert broken == 0