acdl/tests/test_metrics_collector.py

"""Tests for Nova metrics collector (P2, REQ-189/200).

Tests the collector's idempotent re-run property (REQ-200) and the
SQLite cold store schema.
"""

import json
import os
import sqlite3
import sys
from pathlib import Path

import pytest

ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT))


@pytest.fixture
def tmp_store(tmp_path, monkeypatch):
    """Redirect metrics/ to a tmp dir for isolated testing."""
    metrics_dir = tmp_path / "metrics"
    metrics_dir.mkdir()
    runs_dir = metrics_dir / "runs"
    runs_dir.mkdir()
    lifecycle_dir = metrics_dir / "lifecycle"
    lifecycle_dir.mkdir()
    store_db = metrics_dir / "nova_metrics.db"
    ledger_db = metrics_dir / "decision_ledger.db"

    monkeypatch.setattr("core.metrics.collector._METRICS_DIR", str(metrics_dir))
    monkeypatch.setattr("core.metrics.collector._STORE_PATH", str(store_db))
    monkeypatch.setattr("core.metrics.collector._RUNS_DIR", str(runs_dir))
    monkeypatch.setattr("core.metrics.collector._LEDGER_DB", str(ledger_db))
    monkeypatch.setattr("core.metrics.collector._REPO_ROOT", str(tmp_path))
    monkeypatch.setattr("core.metrics.collector._REGRESSION_REPORT", str(tmp_path / "REGRESSION_REPORT.json"))
    monkeypatch.setattr("core.metrics.collector._COVERAGE_JSON", str(metrics_dir / "coverage.json"))
    monkeypatch.setattr("core.metrics.collector._TEST_RESULTS_XML", str(metrics_dir / "test-results.xml"))
    monkeypatch.setattr("core.metrics.decision_ledger._LEDGER_PATH", str(ledger_db))
    return {"metrics_dir": metrics_dir, "store_db": store_db, "ledger_db": ledger_db, "runs_dir": runs_dir}


def _write_regression_report(path, run_id="regr-test-1"):
    report = {
        "run_id": run_id,
        "run_at_utc": "2026-08-04T12:00:00Z",
        "milestone": "v1.17",
        "phase": 0,
        "summary": {"Verified": 18, "Decayed": 0, "Broken": 0, "Skipped": 4},
        "passed": True,
        "results": [
            {"capability_id": "CAP-001", "name": "test cap", "status": "Verified", "tier": "local", "duration_ms": 100, "detail": "ok"},
            {"capability_id": "CAP-002", "name": "test cap 2", "status": "Skipped", "tier": "live-aws", "duration_ms": 50, "detail": "D-096"},
        ],
    }
    with open(path, "w") as f:
        json.dump(report, f)


def _write_run_manifest(runs_dir, run_id="run-test-1"):
    manifest = {
        "run_id": run_id,
        "contract_id": "cid-1",
        "environment": "dev",
        "started_at": "2026-08-04T12:00:00Z",
        "completed_at": "2026-08-04T12:01:00Z",
        "exit_code": 0,
        "stages": [{"name": "resolve", "duration_ms": 100, "exit_code": 0}],
        "outcome": "succeeded",
        "confidence": {"score": 0.9, "band": "pass", "perInput": {"policy": 1.0}},
        "hitl": {"gate": "dev", "result": "autonomous", "block": False},
        "cost_estimate_usd": -12.5,
        "decision_id": run_id,
    }
    with open(runs_dir / f"{run_id}.json", "w") as f:
        json.dump(manifest, f)


def _write_junit(path):
    xml = """<?xml version="1.0" encoding="utf-8"?>
<testsuites>
  <testsuite name="test_metrics" tests="10" failures="0" errors="0" skipped="0" time="1.5">
    <testcase name="test_one" time="0.1"/>
  </testsuite>
</testsuites>"""
    path.write_text(xml)


def _write_coverage(path):
    with open(path, "w") as f:
        json.dump({"totals": {"percent_covered": 85.5}}, f)


def test_collector_init(tmp_store):
    from core.metrics.collector import _init_store
    _init_store()
    assert tmp_store["store_db"].exists()
    conn = sqlite3.connect(str(tmp_store["store_db"]))
    tables = conn.execute("SELECT name FROM sqlite_master WHERE type='table'").fetchall()
    conn.close()
    table_names = [t[0] for t in tables]
    assert "fact_run" in table_names
    assert "fact_capability" in table_names
    assert "fact_decision" in table_names
    assert "dim_capability" in table_names
    assert "dim_milestone" in table_names


def test_collector_regression_report(tmp_store):
    from core.metrics.collector import collect_regression_report
    _write_regression_report(tmp_store["metrics_dir"].parent / "REGRESSION_REPORT.json")
    count = collect_regression_report()
    assert count == 2
    conn = sqlite3.connect(str(tmp_store["store_db"]))
    rows = conn.execute("SELECT capability_id, status FROM fact_capability").fetchall()
    conn.close()
    assert len(rows) == 2
    assert rows[0][0] == "CAP-001"


def test_collector_run_manifests(tmp_store):
    from core.metrics.collector import collect_run_manifests
    _write_run_manifest(tmp_store["runs_dir"])
    count = collect_run_manifests()
    assert count == 1
    conn = sqlite3.connect(str(tmp_store["store_db"]))
    row = conn.execute("SELECT run_id, confidence_score, cost_estimate_usd FROM fact_run").fetchone()
    conn.close()
    assert row[0] == "run-test-1"
    assert row[1] == 0.9
    assert row[2] == -12.5


def test_collector_idempotent(tmp_store):
    """REQ-200: re-running the collector produces identical row counts."""
    from core.metrics.collector import collect_all
    _write_regression_report(tmp_store["metrics_dir"].parent / "REGRESSION_REPORT.json")
    _write_run_manifest(tmp_store["runs_dir"])
    _write_junit(tmp_store["metrics_dir"] / "test-results.xml")
    _write_coverage(tmp_store["metrics_dir"] / "coverage.json")

    result1 = collect_all()
    conn = sqlite3.connect(str(tmp_store["store_db"]))
    cap_count_1 = conn.execute("SELECT COUNT(*) FROM fact_capability").fetchone()[0]
    run_count_1 = conn.execute("SELECT COUNT(*) FROM fact_run").fetchone()[0]
    conn.close()

    result2 = collect_all()
    conn = sqlite3.connect(str(tmp_store["store_db"]))
    cap_count_2 = conn.execute("SELECT COUNT(*) FROM fact_capability").fetchone()[0]
    run_count_2 = conn.execute("SELECT COUNT(*) FROM fact_run").fetchone()[0]
    conn.close()

    assert cap_count_1 == cap_count_2
    assert run_count_1 == run_count_2


def test_collector_decision_ledger(tmp_store):
    from core.metrics.event_envelope import make_event
    from core.metrics.decision_ledger import append
    from core.metrics.collector import collect_decision_ledger
    ev = make_event("nova.ai.decision.made", "run-dl-collect-1", "dev",
                     {"decision_id": "run-dl-collect-1", "chosen_action": "pass",
                      "confidence": 0.94, "alternatives": {"policy": 1.0},
                      "human_override": False, "outcome": "succeeded"})
    append(ev)
    count = collect_decision_ledger()
    assert count == 1
    conn = sqlite3.connect(str(tmp_store["store_db"]))
    row = conn.execute("SELECT decision_id, confidence, chosen_action FROM fact_decision").fetchone()
    conn.close()
    assert row[0] == "run-dl-collect-1"
    assert row[1] == 0.94
    assert row[2] == "pass"


def test_collector_test_results(tmp_store):
    from core.metrics.collector import collect_test_results
    _write_junit(tmp_store["metrics_dir"] / "test-results.xml")
    _write_coverage(tmp_store["metrics_dir"] / "coverage.json")
    count = collect_test_results()
    assert count == 1
    conn = sqlite3.connect(str(tmp_store["store_db"]))
    row = conn.execute("SELECT total_tests, passed, coverage_pct FROM fact_test").fetchone()
    conn.close()
    assert row[0] == 10
    assert row[1] == 10
    assert row[2] == 85.5


def test_collector_all(tmp_store):
    from core.metrics.collector import collect_all
    _write_regression_report(tmp_store["metrics_dir"].parent / "REGRESSION_REPORT.json")
    _write_run_manifest(tmp_store["runs_dir"])
    _write_junit(tmp_store["metrics_dir"] / "test-results.xml")
    _write_coverage(tmp_store["metrics_dir"] / "coverage.json")
    result = collect_all()
    assert result["capabilities"] == 2
    assert result["runs"] == 1
    assert result["tests"] == 1