"""Tests for Nova metrics collector (P2, REQ-189/200). Tests the collector's idempotent re-run property (REQ-200) and the SQLite cold store schema. """ import json import os import sqlite3 import sys from pathlib import Path import pytest ROOT = Path(__file__).resolve().parent.parent sys.path.insert(0, str(ROOT)) @pytest.fixture def tmp_store(tmp_path, monkeypatch): """Redirect metrics/ to a tmp dir for isolated testing.""" metrics_dir = tmp_path / "metrics" metrics_dir.mkdir() runs_dir = metrics_dir / "runs" runs_dir.mkdir() lifecycle_dir = metrics_dir / "lifecycle" lifecycle_dir.mkdir() store_db = metrics_dir / "nova_metrics.db" ledger_db = metrics_dir / "decision_ledger.db" monkeypatch.setattr("core.metrics.collector._METRICS_DIR", str(metrics_dir)) monkeypatch.setattr("core.metrics.collector._STORE_PATH", str(store_db)) monkeypatch.setattr("core.metrics.collector._RUNS_DIR", str(runs_dir)) monkeypatch.setattr("core.metrics.collector._LEDGER_DB", str(ledger_db)) monkeypatch.setattr("core.metrics.collector._REPO_ROOT", str(tmp_path)) monkeypatch.setattr("core.metrics.collector._REGRESSION_REPORT", str(tmp_path / "REGRESSION_REPORT.json")) monkeypatch.setattr("core.metrics.collector._COVERAGE_JSON", str(metrics_dir / "coverage.json")) monkeypatch.setattr("core.metrics.collector._TEST_RESULTS_XML", str(metrics_dir / "test-results.xml")) monkeypatch.setattr("core.metrics.decision_ledger._LEDGER_PATH", str(ledger_db)) return {"metrics_dir": metrics_dir, "store_db": store_db, "ledger_db": ledger_db, "runs_dir": runs_dir} def _write_regression_report(path, run_id="regr-test-1"): report = { "run_id": run_id, "run_at_utc": "2026-08-04T12:00:00Z", "milestone": "v1.17", "phase": 0, "summary": {"Verified": 18, "Decayed": 0, "Broken": 0, "Skipped": 4}, "passed": True, "results": [ {"capability_id": "CAP-001", "name": "test cap", "status": "Verified", "tier": "local", "duration_ms": 100, "detail": "ok"}, {"capability_id": "CAP-002", "name": "test cap 2", "status": "Skipped", "tier": "live-aws", "duration_ms": 50, "detail": "D-096"}, ], } with open(path, "w") as f: json.dump(report, f) def _write_run_manifest(runs_dir, run_id="run-test-1"): manifest = { "run_id": run_id, "contract_id": "cid-1", "environment": "dev", "started_at": "2026-08-04T12:00:00Z", "completed_at": "2026-08-04T12:01:00Z", "exit_code": 0, "stages": [{"name": "resolve", "duration_ms": 100, "exit_code": 0}], "outcome": "succeeded", "confidence": {"score": 0.9, "band": "pass", "perInput": {"policy": 1.0}}, "hitl": {"gate": "dev", "result": "autonomous", "block": False}, "cost_estimate_usd": -12.5, "decision_id": run_id, } with open(runs_dir / f"{run_id}.json", "w") as f: json.dump(manifest, f) def _write_junit(path): xml = """ """ path.write_text(xml) def _write_coverage(path): with open(path, "w") as f: json.dump({"totals": {"percent_covered": 85.5}}, f) def test_collector_init(tmp_store): from core.metrics.collector import _init_store _init_store() assert tmp_store["store_db"].exists() conn = sqlite3.connect(str(tmp_store["store_db"])) tables = conn.execute("SELECT name FROM sqlite_master WHERE type='table'").fetchall() conn.close() table_names = [t[0] for t in tables] assert "fact_run" in table_names assert "fact_capability" in table_names assert "fact_decision" in table_names assert "dim_capability" in table_names assert "dim_milestone" in table_names def test_collector_regression_report(tmp_store): from core.metrics.collector import collect_regression_report _write_regression_report(tmp_store["metrics_dir"].parent / "REGRESSION_REPORT.json") count = collect_regression_report() assert count == 2 conn = sqlite3.connect(str(tmp_store["store_db"])) rows = conn.execute("SELECT capability_id, status FROM fact_capability").fetchall() conn.close() assert len(rows) == 2 assert rows[0][0] == "CAP-001" def test_collector_run_manifests(tmp_store): from core.metrics.collector import collect_run_manifests _write_run_manifest(tmp_store["runs_dir"]) count = collect_run_manifests() assert count == 1 conn = sqlite3.connect(str(tmp_store["store_db"])) row = conn.execute("SELECT run_id, confidence_score, cost_estimate_usd FROM fact_run").fetchone() conn.close() assert row[0] == "run-test-1" assert row[1] == 0.9 assert row[2] == -12.5 def test_collector_idempotent(tmp_store): """REQ-200: re-running the collector produces identical row counts.""" from core.metrics.collector import collect_all _write_regression_report(tmp_store["metrics_dir"].parent / "REGRESSION_REPORT.json") _write_run_manifest(tmp_store["runs_dir"]) _write_junit(tmp_store["metrics_dir"] / "test-results.xml") _write_coverage(tmp_store["metrics_dir"] / "coverage.json") result1 = collect_all() conn = sqlite3.connect(str(tmp_store["store_db"])) cap_count_1 = conn.execute("SELECT COUNT(*) FROM fact_capability").fetchone()[0] run_count_1 = conn.execute("SELECT COUNT(*) FROM fact_run").fetchone()[0] conn.close() result2 = collect_all() conn = sqlite3.connect(str(tmp_store["store_db"])) cap_count_2 = conn.execute("SELECT COUNT(*) FROM fact_capability").fetchone()[0] run_count_2 = conn.execute("SELECT COUNT(*) FROM fact_run").fetchone()[0] conn.close() assert cap_count_1 == cap_count_2 assert run_count_1 == run_count_2 def test_collector_decision_ledger(tmp_store): from core.metrics.event_envelope import make_event from core.metrics.decision_ledger import append from core.metrics.collector import collect_decision_ledger ev = make_event("nova.ai.decision.made", "run-dl-collect-1", "dev", {"decision_id": "run-dl-collect-1", "chosen_action": "pass", "confidence": 0.94, "alternatives": {"policy": 1.0}, "human_override": False, "outcome": "succeeded"}) append(ev) count = collect_decision_ledger() assert count == 1 conn = sqlite3.connect(str(tmp_store["store_db"])) row = conn.execute("SELECT decision_id, confidence, chosen_action FROM fact_decision").fetchone() conn.close() assert row[0] == "run-dl-collect-1" assert row[1] == 0.94 assert row[2] == "pass" def test_collector_test_results(tmp_store): from core.metrics.collector import collect_test_results _write_junit(tmp_store["metrics_dir"] / "test-results.xml") _write_coverage(tmp_store["metrics_dir"] / "coverage.json") count = collect_test_results() assert count == 1 conn = sqlite3.connect(str(tmp_store["store_db"])) row = conn.execute("SELECT total_tests, passed, coverage_pct FROM fact_test").fetchone() conn.close() assert row[0] == 10 assert row[1] == 10 assert row[2] == 85.5 def test_collector_all(tmp_store): from core.metrics.collector import collect_all _write_regression_report(tmp_store["metrics_dir"].parent / "REGRESSION_REPORT.json") _write_run_manifest(tmp_store["runs_dir"]) _write_junit(tmp_store["metrics_dir"] / "test-results.xml") _write_coverage(tmp_store["metrics_dir"] / "coverage.json") result = collect_all() assert result["capabilities"] == 2 assert result["runs"] == 1 assert result["tests"] == 1