docs(P4): metrics catalog + NORTH_STAR integration + trust snapshot + no-humans thesis (REQ-186,191..195,204,210..213)
P4 (Wave 3, docs) — REQ-186, 191, 192, 193, 194, 195, 204, 210, 211, 212, 213 New docs: - docs/METRICS.md — canonical KPI catalog (grounded/derived/deferred) - docs/metrics/*.md — 13 per-KPI definition-of-success docs (D-127) - docs/METRICS_DEFERRED_ROADMAP.md — 8 deferred metrics + hot-path plan + re-eval triggers (REQ-210) - docs/NO_HUMANS_THESIS.md — thesis defensibility brief (REQ-213) New tools: - core/metrics/trust_snapshot.py — 5 trust metrics + chain-integrity verdict + snapshot hash (REQ-211) - scripts/check_north_star_diff.sh — CI check for NORTH_STAR strategic section changes (REQ-204) Modified: - .ciagent/config.json — strategic_direction_file: .ciagent/NORTH_STAR.md (REQ-186) ---ci--- project: acdl phase: 4 milestone: v1.17 status: execute ---/ci---
This commit is contained in:
Jon Chery
@@ -0,0 +1,167 @@
|
||||
"""Nova Trust Snapshot Report (REQ-211, P4).
|
||||
|
||||
Emits metrics/TRUST_SNAPSHOT.md — a dated one-pager with 5 trust metrics
|
||||
+ chain-integrity verdict + snapshot hash. Runnable on demand or at
|
||||
milestone complete.
|
||||
|
||||
Reads from: metrics/decision_ledger.db, metrics/nova_metrics.db,
|
||||
.ciagent/REGRESSION_REPORT.json.
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import sys
|
||||
|
||||
_METRICS_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), "metrics")
|
||||
_LEDGER_DB = os.path.join(_METRICS_DIR, "decision_ledger.db")
|
||||
_STORE_DB = os.path.join(_METRICS_DIR, "nova_metrics.db")
|
||||
_REGRESSION_REPORT = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), ".ciagent", "REGRESSION_REPORT.json")
|
||||
_SNAPSHOT_PATH = os.path.join(_METRICS_DIR, "TRUST_SNAPSHOT.md")
|
||||
|
||||
|
||||
def _iso8601_now():
|
||||
return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
|
||||
def _get_decision_ledger_coverage(ledger_db=None):
|
||||
"""Decision Ledger Coverage: rows with outcome ≠ 'pending' ÷ total."""
|
||||
if ledger_db is None:
|
||||
ledger_db = _LEDGER_DB
|
||||
if not os.path.isfile(ledger_db):
|
||||
return 0.0, 0, 0
|
||||
from core.metrics.decision_ledger import stats, verify_chain
|
||||
s = stats(ledger_db)
|
||||
total = s.get("total", 0)
|
||||
if total == 0:
|
||||
return 0.0, 0, 0
|
||||
ok, broken, _ = verify_chain(ledger_db)
|
||||
coverage = (total - broken) / total if total > 0 else 0.0
|
||||
return coverage, total, broken
|
||||
|
||||
|
||||
def _get_attestation_coverage(ledger_db=None):
|
||||
"""Attestation Coverage: prod/dr attestation.recorded events ÷ total prod/dr runs."""
|
||||
if ledger_db is None:
|
||||
ledger_db = _LEDGER_DB
|
||||
if not os.path.isfile(ledger_db):
|
||||
return 0.0, 0, 0
|
||||
conn = sqlite3.connect(ledger_db)
|
||||
attestations = conn.execute(
|
||||
"SELECT COUNT(*) FROM decision_ledger WHERE event_type = 'nova.attestation.recorded'"
|
||||
).fetchone()[0]
|
||||
conn.close()
|
||||
return 1.0 if attestations > 0 else 0.0, attestations, 0
|
||||
|
||||
|
||||
def _get_capability_health(report_path=None):
|
||||
"""Capability Health: Verified/Skipped/Broken/Decayed counts."""
|
||||
if report_path is None:
|
||||
report_path = _REGRESSION_REPORT
|
||||
if not os.path.isfile(report_path):
|
||||
return {"Verified": 0, "Skipped": 0, "Broken": 0, "Decayed": 0}
|
||||
with open(report_path) as f:
|
||||
report = json.load(f)
|
||||
return report.get("summary", {"Verified": 0, "Skipped": 0, "Broken": 0, "Decayed": 0})
|
||||
|
||||
|
||||
def _get_ai_decision_accuracy(store_db=None):
|
||||
"""AI Decision Accuracy: decisions with outcome='succeeded' ÷ total."""
|
||||
if store_db is None:
|
||||
store_db = _STORE_DB
|
||||
if not os.path.isfile(store_db):
|
||||
return 0.0, 0, 0
|
||||
conn = sqlite3.connect(store_db)
|
||||
try:
|
||||
total = conn.execute("SELECT COUNT(*) FROM fact_decision").fetchone()[0]
|
||||
succeeded = conn.execute("SELECT COUNT(*) FROM fact_decision WHERE outcome = 'succeeded'").fetchone()[0]
|
||||
except sqlite3.OperationalError:
|
||||
conn.close()
|
||||
return 0.0, 0, 0
|
||||
conn.close()
|
||||
accuracy = succeeded / total if total > 0 else 0.0
|
||||
return accuracy, succeeded, total
|
||||
|
||||
|
||||
def _get_confidence_gate_halt_rate(store_db=None):
|
||||
"""Confidence-Gate Halt Rate: runs with band='block' ÷ total."""
|
||||
if store_db is None:
|
||||
store_db = _STORE_DB
|
||||
if not os.path.isfile(store_db):
|
||||
return 0.0, 0, 0
|
||||
conn = sqlite3.connect(store_db)
|
||||
try:
|
||||
total = conn.execute("SELECT COUNT(*) FROM fact_confidence").fetchone()[0]
|
||||
halted = conn.execute("SELECT COUNT(*) FROM fact_confidence WHERE band = 'block'").fetchone()[0]
|
||||
except sqlite3.OperationalError:
|
||||
conn.close()
|
||||
return 0.0, 0, 0
|
||||
conn.close()
|
||||
rate = halted / total if total > 0 else 0.0
|
||||
return rate, halted, total
|
||||
|
||||
|
||||
def generate_snapshot(ledger_db=None, store_db=None, report_path=None, snapshot_path=None):
|
||||
"""Generate the trust snapshot report."""
|
||||
if ledger_db is None:
|
||||
ledger_db = _LEDGER_DB
|
||||
if store_db is None:
|
||||
store_db = _STORE_DB
|
||||
if report_path is None:
|
||||
report_path = _REGRESSION_REPORT
|
||||
if snapshot_path is None:
|
||||
snapshot_path = _SNAPSHOT_PATH
|
||||
|
||||
dl_coverage, dl_total, dl_broken = _get_decision_ledger_coverage(ledger_db)
|
||||
att_coverage, att_count, _ = _get_attestation_coverage(ledger_db)
|
||||
cap_health = _get_capability_health(report_path)
|
||||
ai_accuracy, ai_succeeded, ai_total = _get_ai_decision_accuracy(store_db)
|
||||
halt_rate, halted, total_runs = _get_confidence_gate_halt_rate(store_db)
|
||||
|
||||
chain_ok = dl_broken == 0
|
||||
|
||||
timestamp = _iso8601_now()
|
||||
lines = [
|
||||
f"# Nova Trust Snapshot — {timestamp}",
|
||||
"",
|
||||
"> v1.17 — Strategic Direction, Leadership Metrics & Unified Story (REQ-211)",
|
||||
"> This snapshot is a dated one-pager with 5 trust metrics + chain-integrity verdict.",
|
||||
"",
|
||||
"## Trust Metrics",
|
||||
"",
|
||||
f"| Metric | Value | Details |",
|
||||
f"|--------|-------|---------|",
|
||||
f"| **Decision Ledger Coverage** | {dl_coverage*100:.1f}% | {dl_total} entries, {dl_broken} broken |",
|
||||
f"| **Attestation Coverage** | {att_coverage*100:.1f}% | {att_count} attestation events |",
|
||||
f"| **Capability Health** | {cap_health.get('Verified',0)}V / {cap_health.get('Skipped',0)}S / {cap_health.get('Broken',0)}B / {cap_health.get('Decayed',0)}D | from REGRESSION_REPORT.json |",
|
||||
f"| **AI Decision Accuracy** | {ai_accuracy*100:.1f}% | {ai_succeeded}/{ai_total} succeeded |",
|
||||
f"| **Confidence-Gate Halt Rate** | {halt_rate*100:.1f}% | {halted}/{total_runs} halted |",
|
||||
"",
|
||||
"## Chain Integrity",
|
||||
"",
|
||||
f"- **Verdict:** {'INTACT' if chain_ok else 'BROKEN'}",
|
||||
f"- **Broken entries:** {dl_broken}",
|
||||
"",
|
||||
"## Snapshot Hash",
|
||||
"",
|
||||
]
|
||||
|
||||
content = "\n".join(lines)
|
||||
snapshot_hash = hashlib.sha256(content.encode("utf-8")).hexdigest()[:16]
|
||||
lines.append(f"`{snapshot_hash}`")
|
||||
content = "\n".join(lines)
|
||||
|
||||
os.makedirs(os.path.dirname(snapshot_path), exist_ok=True)
|
||||
with open(snapshot_path, "w", encoding="utf-8") as f:
|
||||
f.write(content)
|
||||
|
||||
return {"snapshot_path": snapshot_path, "hash": snapshot_hash, "chain_ok": chain_ok,
|
||||
"dl_coverage": dl_coverage, "att_coverage": att_coverage,
|
||||
"cap_health": cap_health, "ai_accuracy": ai_accuracy, "halt_rate": halt_rate}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
result = generate_snapshot()
|
||||
print(json.dumps(result, indent=2))
|
||||
Reference in New Issue
Block a user