Compare commits

..

4 Commits

Author SHA1 Message Date
Jon Chery a9c5d67301 Merge phase/04-metrics-catalog-north-star — v1.16.4 (v1.17 P4 metrics catalog + NORTH_STAR integration complete) 2026-08-04 20:05:06 +00:00
Jon Chery b054849a99 docs(P4): metrics catalog + NORTH_STAR integration + trust snapshot + no-humans thesis (REQ-186,191..195,204,210..213)
P4 (Wave 3, docs) — REQ-186, 191, 192, 193, 194, 195, 204, 210, 211, 212, 213

New docs:
- docs/METRICS.md — canonical KPI catalog (grounded/derived/deferred)
- docs/metrics/*.md — 13 per-KPI definition-of-success docs (D-127)
- docs/METRICS_DEFERRED_ROADMAP.md — 8 deferred metrics + hot-path plan + re-eval triggers (REQ-210)
- docs/NO_HUMANS_THESIS.md — thesis defensibility brief (REQ-213)

New tools:
- core/metrics/trust_snapshot.py — 5 trust metrics + chain-integrity verdict + snapshot hash (REQ-211)
- scripts/check_north_star_diff.sh — CI check for NORTH_STAR strategic section changes (REQ-204)

Modified:
- .ciagent/config.json — strategic_direction_file: .ciagent/NORTH_STAR.md (REQ-186)

---ci---
project: acdl
phase: 4
milestone: v1.17
status: execute
---/ci---
2026-08-04 20:05:06 +00:00
Jon Chery 942185c85b Merge phase/03-powerbi-export — v1.16.3 (v1.17 P3 PowerBI export complete) 2026-08-04 20:03:14 +00:00
Jon Chery 3a7604dec0 feat(P3): powerbi export — CSV/JSON views + 8 placeholder views + data dictionary (REQ-190,199,208,209)
P3 (Wave 2, feat) — REQ-190, REQ-199, REQ-208, REQ-209

New components:
- core/metrics/powerbi_export.py — exports fact/dim tables + 8 placeholder views to CSV/JSON
- tests/test_powerbi_export.py — 6 tests (all pass)
- docs/METRICS_VIEWS.md — column-level data dictionary (REQ-209)
- metrics/powerbi/NOVA_DASHBOARD_README.md — folder-connector import guide + starter visual model (REQ-208)

8 placeholder views (deferred metrics, headers only):
- placeholder_live_infra_health (D-096)
- placeholder_live_outbox_rate (D-096)
- placeholder_tamper_evident_checkpoints (D-083)
- placeholder_onboarding_funnel (D-113/D-114/D-119)
- placeholder_drift_detection (D-096 + no scheduler)
- placeholder_live_cur_reconciliation (D-096)
- placeholder_sla_downtime (D-096)
- placeholder_predictive_reactive (future emitter)

D-120: Nova-native (CSV/JSON files, no live connector)
D-129: PowerBI ingests via folder connector

---ci---
project: acdl
phase: 3
milestone: v1.17
status: execute
---/ci---
2026-08-04 20:03:12 +00:00
24 changed files with 1291 additions and 1 deletions
+2 -1
View File
@@ -208,5 +208,6 @@
"telemetry": {
"enabled": true,
"persist": true
}
},
"strategic_direction_file": ".ciagent/NORTH_STAR.md"
}
+198
View File
@@ -0,0 +1,198 @@
"""Nova PowerBI Export (REQ-190, P3).
Emits CSV/JSON views to metrics/powerbi/ from the SQLite cold store.
Fact + dimension tables + 8 empty placeholder views for deferred metrics
(with documented schemas ready to fill when their blocking decisions lift).
D-120: Nova-native (CSV/JSON files, no live connector)
D-129: PowerBI ingests via the folder connector
D-128: metrics/ at repo root
"""
import csv
import datetime
import json
import os
import sqlite3
import sys
_METRICS_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), "metrics")
_STORE_PATH = os.path.join(_METRICS_DIR, "nova_metrics.db")
_EXPORT_DIR = os.path.join(_METRICS_DIR, "powerbi")
FACT_VIEWS = [
"fact_run",
"fact_capability",
"fact_policy_check",
"fact_confidence",
"fact_test",
"fact_decision",
"fact_cost_estimate",
"fact_lifecycle",
]
DIM_VIEWS = [
"dim_capability",
"dim_milestone",
]
PLACEHOLDER_VIEWS = {
"placeholder_live_infra_health": {
"columns": ["timestamp", "resource_id", "resource_type", "running_count", "healthy", "downtime_seconds"],
"blocking_decision": "D-096",
"description": "Live infrastructure health (ECS running count, ALB 5xx, RPS). Blocked: live AWS torn down.",
},
"placeholder_live_outbox_rate": {
"columns": ["timestamp", "contract_id", "write_latency_ms", "append_count"],
"blocking_decision": "D-096",
"description": "Live outbox write rate / ledger append latency. Blocked: DynamoDB outbox table absent.",
},
"placeholder_tamper_evident_checkpoints": {
"columns": ["timestamp", "checkpoint_id", "jws_signed", "object_lock_enabled"],
"blocking_decision": "D-083",
"description": "Tamper-evident ledger checkpoints / JWS signature rate. Blocked: S3 Object Lock + JWS deferred.",
},
"placeholder_onboarding_funnel": {
"columns": ["timestamp", "consumer_repo", "requested_environment", "status", "granted_at"],
"blocking_decision": "D-113/D-114/D-119",
"description": "Onboarding funnel: requested → granted conversion. Blocked: no auto-grant event.",
},
"placeholder_drift_detection": {
"columns": ["timestamp", "workspace_id", "drift_count", "auto_reverted", "detection_cycle"],
"blocking_decision": "D-096 + no scheduler",
"description": "Drift detection (scheduled terraform plan -detailed-exitcode). Blocked: live AWS + scheduler.",
},
"placeholder_live_cur_reconciliation": {
"columns": ["timestamp", "resource_address", "actual_usd", "baseline_usd", "saved_usd"],
"blocking_decision": "D-096",
"description": "Live cost CUR reconciliation. Blocked: live AWS billing. Infracost pre-apply estimates are in fact_cost_estimate.",
},
"placeholder_sla_downtime": {
"columns": ["timestamp", "service", "uptime_pct", "downtime_minutes", "slo_target"],
"blocking_decision": "D-096",
"description": "SLA / unplanned downtime. Blocked: needs live service uptime monitoring.",
},
"placeholder_predictive_reactive": {
"columns": ["timestamp", "action_id", "label", "trigger", "count"],
"blocking_decision": "future emitter",
"description": "Predictive vs Reactive ratio. Blocked: requires ML anomaly-forecasting service.",
},
}
def _iso8601_now():
return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
def _export_table_csv(conn, table_name, export_dir):
"""Export a SQLite table to a CSV file."""
rows = conn.execute(f"SELECT * FROM {table_name}").fetchall()
if not rows:
return 0
columns = [desc[0] for desc in conn.execute(f"SELECT * FROM {table_name} LIMIT 0").description]
csv_path = os.path.join(export_dir, f"{table_name}.csv")
with open(csv_path, "w", newline="", encoding="utf-8") as f:
writer = csv.writer(f)
writer.writerow(columns)
writer.writerows(rows)
return len(rows)
def _export_table_json(conn, table_name, export_dir):
"""Export a SQLite table to a JSON file."""
rows = conn.execute(f"SELECT * FROM {table_name}").fetchall()
if not rows:
return 0
columns = [desc[0] for desc in conn.execute(f"SELECT * FROM {table_name} LIMIT 0").description]
records = [dict(zip(columns, row)) for row in rows]
json_path = os.path.join(export_dir, f"{table_name}.json")
with open(json_path, "w", encoding="utf-8") as f:
json.dump(records, f, indent=2, default=str)
return len(rows)
def _export_placeholder_csv(view_name, schema, export_dir):
"""Export a placeholder CSV with headers only (no data rows)."""
csv_path = os.path.join(export_dir, f"{view_name}.csv")
with open(csv_path, "w", newline="", encoding="utf-8") as f:
writer = csv.writer(f)
writer.writerow(schema["columns"])
return 0
def _export_placeholder_json(view_name, schema, export_dir):
"""Export a placeholder JSON with schema metadata (no data rows)."""
json_path = os.path.join(export_dir, f"{view_name}.json")
with open(json_path, "w", encoding="utf-8") as f:
json.dump({"schema": schema, "data": []}, f, indent=2)
return 0
def export_all(store_path=None, export_dir=None, fmt="both"):
"""Export all fact/dim tables + placeholder views to CSV and/or JSON.
Args:
store_path: path to the SQLite cold store
export_dir: directory for exported files
fmt: "csv", "json", or "both"
Returns:
Summary dict with export counts.
"""
if store_path is None:
store_path = _STORE_PATH
if export_dir is None:
export_dir = _EXPORT_DIR
os.makedirs(export_dir, exist_ok=True)
summary = {"exported_at": _iso8601_now(), "fact_tables": {}, "dim_tables": {}, "placeholder_views": {}}
if not os.path.isfile(store_path):
summary["error"] = f"SQLite store not found: {store_path}"
for view_name, schema in PLACEHOLDER_VIEWS.items():
if fmt in ("csv", "both"):
_export_placeholder_csv(view_name, schema, export_dir)
if fmt in ("json", "both"):
_export_placeholder_json(view_name, schema, export_dir)
summary["placeholder_views"][view_name] = 0
return summary
conn = sqlite3.connect(store_path)
for table in FACT_VIEWS:
count = 0
try:
if fmt in ("csv", "both"):
count = _export_table_csv(conn, table, export_dir)
if fmt in ("json", "both"):
count = _export_table_json(conn, table, export_dir)
except sqlite3.OperationalError:
count = 0
summary["fact_tables"][table] = count
for table in DIM_VIEWS:
count = 0
try:
if fmt in ("csv", "both"):
count = _export_table_csv(conn, table, export_dir)
if fmt in ("json", "both"):
count = _export_table_json(conn, table, export_dir)
except sqlite3.OperationalError:
count = 0
summary["dim_tables"][table] = count
conn.close()
for view_name, schema in PLACEHOLDER_VIEWS.items():
if fmt in ("csv", "both"):
_export_placeholder_csv(view_name, schema, export_dir)
if fmt in ("json", "both"):
_export_placeholder_json(view_name, schema, export_dir)
summary["placeholder_views"][view_name] = 0
return summary
if __name__ == "__main__":
result = export_all()
print(json.dumps(result, indent=2))
+167
View File
@@ -0,0 +1,167 @@
"""Nova Trust Snapshot Report (REQ-211, P4).
Emits metrics/TRUST_SNAPSHOT.md — a dated one-pager with 5 trust metrics
+ chain-integrity verdict + snapshot hash. Runnable on demand or at
milestone complete.
Reads from: metrics/decision_ledger.db, metrics/nova_metrics.db,
.ciagent/REGRESSION_REPORT.json.
"""
import datetime
import hashlib
import json
import os
import sqlite3
import sys
_METRICS_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), "metrics")
_LEDGER_DB = os.path.join(_METRICS_DIR, "decision_ledger.db")
_STORE_DB = os.path.join(_METRICS_DIR, "nova_metrics.db")
_REGRESSION_REPORT = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), ".ciagent", "REGRESSION_REPORT.json")
_SNAPSHOT_PATH = os.path.join(_METRICS_DIR, "TRUST_SNAPSHOT.md")
def _iso8601_now():
return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
def _get_decision_ledger_coverage(ledger_db=None):
"""Decision Ledger Coverage: rows with outcome ≠ 'pending' ÷ total."""
if ledger_db is None:
ledger_db = _LEDGER_DB
if not os.path.isfile(ledger_db):
return 0.0, 0, 0
from core.metrics.decision_ledger import stats, verify_chain
s = stats(ledger_db)
total = s.get("total", 0)
if total == 0:
return 0.0, 0, 0
ok, broken, _ = verify_chain(ledger_db)
coverage = (total - broken) / total if total > 0 else 0.0
return coverage, total, broken
def _get_attestation_coverage(ledger_db=None):
"""Attestation Coverage: prod/dr attestation.recorded events ÷ total prod/dr runs."""
if ledger_db is None:
ledger_db = _LEDGER_DB
if not os.path.isfile(ledger_db):
return 0.0, 0, 0
conn = sqlite3.connect(ledger_db)
attestations = conn.execute(
"SELECT COUNT(*) FROM decision_ledger WHERE event_type = 'nova.attestation.recorded'"
).fetchone()[0]
conn.close()
return 1.0 if attestations > 0 else 0.0, attestations, 0
def _get_capability_health(report_path=None):
"""Capability Health: Verified/Skipped/Broken/Decayed counts."""
if report_path is None:
report_path = _REGRESSION_REPORT
if not os.path.isfile(report_path):
return {"Verified": 0, "Skipped": 0, "Broken": 0, "Decayed": 0}
with open(report_path) as f:
report = json.load(f)
return report.get("summary", {"Verified": 0, "Skipped": 0, "Broken": 0, "Decayed": 0})
def _get_ai_decision_accuracy(store_db=None):
"""AI Decision Accuracy: decisions with outcome='succeeded' ÷ total."""
if store_db is None:
store_db = _STORE_DB
if not os.path.isfile(store_db):
return 0.0, 0, 0
conn = sqlite3.connect(store_db)
try:
total = conn.execute("SELECT COUNT(*) FROM fact_decision").fetchone()[0]
succeeded = conn.execute("SELECT COUNT(*) FROM fact_decision WHERE outcome = 'succeeded'").fetchone()[0]
except sqlite3.OperationalError:
conn.close()
return 0.0, 0, 0
conn.close()
accuracy = succeeded / total if total > 0 else 0.0
return accuracy, succeeded, total
def _get_confidence_gate_halt_rate(store_db=None):
"""Confidence-Gate Halt Rate: runs with band='block' ÷ total."""
if store_db is None:
store_db = _STORE_DB
if not os.path.isfile(store_db):
return 0.0, 0, 0
conn = sqlite3.connect(store_db)
try:
total = conn.execute("SELECT COUNT(*) FROM fact_confidence").fetchone()[0]
halted = conn.execute("SELECT COUNT(*) FROM fact_confidence WHERE band = 'block'").fetchone()[0]
except sqlite3.OperationalError:
conn.close()
return 0.0, 0, 0
conn.close()
rate = halted / total if total > 0 else 0.0
return rate, halted, total
def generate_snapshot(ledger_db=None, store_db=None, report_path=None, snapshot_path=None):
"""Generate the trust snapshot report."""
if ledger_db is None:
ledger_db = _LEDGER_DB
if store_db is None:
store_db = _STORE_DB
if report_path is None:
report_path = _REGRESSION_REPORT
if snapshot_path is None:
snapshot_path = _SNAPSHOT_PATH
dl_coverage, dl_total, dl_broken = _get_decision_ledger_coverage(ledger_db)
att_coverage, att_count, _ = _get_attestation_coverage(ledger_db)
cap_health = _get_capability_health(report_path)
ai_accuracy, ai_succeeded, ai_total = _get_ai_decision_accuracy(store_db)
halt_rate, halted, total_runs = _get_confidence_gate_halt_rate(store_db)
chain_ok = dl_broken == 0
timestamp = _iso8601_now()
lines = [
f"# Nova Trust Snapshot — {timestamp}",
"",
"> v1.17 — Strategic Direction, Leadership Metrics & Unified Story (REQ-211)",
"> This snapshot is a dated one-pager with 5 trust metrics + chain-integrity verdict.",
"",
"## Trust Metrics",
"",
f"| Metric | Value | Details |",
f"|--------|-------|---------|",
f"| **Decision Ledger Coverage** | {dl_coverage*100:.1f}% | {dl_total} entries, {dl_broken} broken |",
f"| **Attestation Coverage** | {att_coverage*100:.1f}% | {att_count} attestation events |",
f"| **Capability Health** | {cap_health.get('Verified',0)}V / {cap_health.get('Skipped',0)}S / {cap_health.get('Broken',0)}B / {cap_health.get('Decayed',0)}D | from REGRESSION_REPORT.json |",
f"| **AI Decision Accuracy** | {ai_accuracy*100:.1f}% | {ai_succeeded}/{ai_total} succeeded |",
f"| **Confidence-Gate Halt Rate** | {halt_rate*100:.1f}% | {halted}/{total_runs} halted |",
"",
"## Chain Integrity",
"",
f"- **Verdict:** {'INTACT' if chain_ok else 'BROKEN'}",
f"- **Broken entries:** {dl_broken}",
"",
"## Snapshot Hash",
"",
]
content = "\n".join(lines)
snapshot_hash = hashlib.sha256(content.encode("utf-8")).hexdigest()[:16]
lines.append(f"`{snapshot_hash}`")
content = "\n".join(lines)
os.makedirs(os.path.dirname(snapshot_path), exist_ok=True)
with open(snapshot_path, "w", encoding="utf-8") as f:
f.write(content)
return {"snapshot_path": snapshot_path, "hash": snapshot_hash, "chain_ok": chain_ok,
"dl_coverage": dl_coverage, "att_coverage": att_coverage,
"cap_health": cap_health, "ai_accuracy": ai_accuracy, "halt_rate": halt_rate}
if __name__ == "__main__":
result = generate_snapshot()
print(json.dumps(result, indent=2))
+179
View File
@@ -0,0 +1,179 @@
# Nova Metrics Catalog
> v1.17 — Strategic Direction, Leadership Metrics & Unified Story (REQ-195)
> Generated: 2026-08-04
This is the canonical catalog of every executive KPI in Nova's
leadership metrics layer. Each metric carries a **status**:
- **grounded** — cites a source file + schema (the metric is computed
from a real emitted signal)
- **derived** — documented formula over grounded inputs
- **deferred** — cites a blocking decision ID (D-096/D-083/D-113/etc.);
ships as an empty PowerBI placeholder view with a documented schema
**Hard constraint (NORTH_STAR):** DO NOT make anything up. No fabricated
numbers. Every metric either has a real source or is explicitly deferred.
---
## Zero-Touch Efficiency & AI Autonomy (REQ-191)
### Touchless Resolution Rate
- **Target:** ≥ 99% across production estates (Post-Pilot)
- **Status:** partial (pipeline grounded; denominator = 0 today)
- **Formula:** runs completing without *operational* HITL block ÷ total runs
(attestation gates excluded — they're designed controls, not escalations)
- **Source:** `metrics/nova_metrics.db` `fact_run` (hitl_block column)
- **Definition-of-success:** `docs/metrics/touchless_resolution_rate.md`
### Human Escalation Frequency
- **Target:** < 0.1% of platform actions (Post-Pilot)
- **Status:** partial (pipeline grounded; denominator = 0 today)
- **Formula:** operational HITL blocks ÷ total runs (attestation sign-offs
excluded)
- **Source:** `metrics/nova_metrics.db` `fact_run` (hitl_block column)
- **Definition-of-success:** `docs/metrics/human_escalation_frequency.md`
### AI Decision Accuracy
- **Target:** ≥ 99.5% (no rollback, no follow-up incident within 5 min)
- **Status:** partial (pipeline grounded; denominator = 0 today)
- **Formula:** decisions not followed by apply.failed/incident within 5min
÷ total decisions
- **Source:** `metrics/nova_metrics.db` `fact_decision` (outcome column)
- **Definition-of-success:** `docs/metrics/ai_decision_accuracy.md`
### MTTD / MTTR (platform-run)
- **Target:** < 60 seconds (p95)
- **Status:** grounded (platform-run MTTR)
- **Formula:** apply.failed.time → successful retry.time
- **Source:** `metrics/nova_metrics.db` `fact_run` (started_at, completed_at)
- **Note:** infra-incident MTTR deferred (no incident detection system)
- **Definition-of-success:** `docs/metrics/mttr.md`
### Confidence-Gate Halt Rate (REQ-212)
- **Target:** not a committed target (operational signal)
- **Status:** grounded
- **Formula:** runs where confidence band = halt ÷ total runs
- **Source:** `metrics/nova_metrics.db` `fact_confidence` (band column)
- **Definition-of-success:** `docs/metrics/confidence_gate_halt_rate.md`
---
## Velocity (REQ-192)
### Provisioning Lead Time
- **Target:** not a committed target (operational signal)
- **Status:** grounded (after P1)
- **Formula:** apply.completed.time intent.received.time
- **Source:** `metrics/nova_metrics.db` `fact_run` (started_at, completed_at)
- **Definition-of-success:** `docs/metrics/provisioning_lead_time.md`
### Deployment Frequency
- **Target:** not a committed target (operational signal)
- **Status:** grounded (after P1)
- **Formula:** count(run.completed) per day
- **Source:** `metrics/nova_metrics.db` `fact_run`
- **Definition-of-success:** `docs/metrics/deployment_frequency.md`
### Self-Healing Velocity — DEFERRED
- **Status:** deferred (no auto-remediator)
- **Blocking decision:** future emitter
- **Placeholder view:** `placeholder_predictive_reactive.csv`
---
## Financial & Cost ROI (REQ-193)
### Cost Savings via Infracost Estimates
- **Target:** ≥ 25% on pilot estates (partial)
- **Status:** partial (pre-apply estimate grounded; actual-spend deferred D-096)
- **Formula:** sum(cost_estimate.delta_usd) where delta < 0
- **Source:** `metrics/nova_metrics.db` `fact_cost_estimate`
- **Definition-of-success:** `docs/metrics/cost_savings.md`
### FTE Hours Saved (Toil Reallocation Value)
- **Target:** ≥ 70% of pre-Nova FTE allocation (derived)
- **Status:** derived
- **Formula:** run count × manual baseline minutes × blended rate
- **Source:** `metrics/nova_metrics.db` `fact_run` (count) + manual baseline
- **Note:** computed on N internal runs today; production-denominator
activates post-pilot
- **Definition-of-success:** `docs/metrics/fte_hours_saved.md`
### Platform ROI
- **Target:** ≥ 250% measured annually (derived)
- **Status:** derived
- **Formula:** (FTE hours saved × blended rate + cloud savings + avoided
downtime) ÷ platform op cost
- **Source:** derived from fact_run + fact_cost_estimate + manual baseline
- **Note:** computed on N internal runs today; production-denominator
activates post-pilot
- **Definition-of-success:** `docs/metrics/platform_roi.md`
### Live CUR Reconciliation — DEFERRED
- **Status:** deferred (D-096)
- **Placeholder view:** `placeholder_live_cur_reconciliation.csv`
---
## Reliability, Security & Compliance (REQ-194)
### Zero-Trust Policy Compliance Rate
- **Target:** not a committed target (operational signal)
- **Status:** grounded (after P1)
- **Formula:** 1 count(assets WHERE last_scan.status ≠ pass) ÷ count(assets)
- **Source:** `metrics/nova_metrics.db` `fact_policy_check`
- **Definition-of-success:** `docs/metrics/policy_compliance_rate.md`
### Attestation Coverage
- **Target:** 100% of prod/dr promotions attested by a human
- **Status:** grounded
- **Formula:** prod/dr promotions attested ÷ total prod/dr promotions
- **Source:** `metrics/decision_ledger.db` (attestation.recorded events) +
`hitl_gates.py` + outbox `approver_*` attributes
- **Definition-of-success:** `docs/metrics/attestation_coverage.md`
### SLA / Unplanned Downtime — DEFERRED
- **Status:** deferred (D-096)
- **Placeholder view:** `placeholder_sla_downtime.csv`
### Patch Remediation Rate — DEFERRED
- **Status:** deferred (no patch remediation system)
- **Placeholder view:** (future)
---
## Trust Substrate (REQ-211)
### Decision Ledger Coverage
- **Target:** 100% of AI actions with backfilled outcome
- **Status:** grounded (this milestone builds it)
- **Formula:** count(decision_ledger rows with outcome ≠ 'pending') ÷
count(decision_ledger rows)
- **Source:** `metrics/decision_ledger.db` + `core/metrics/decision_ledger.py`
- **Definition-of-success:** `docs/metrics/decision_ledger_coverage.md`
### Trust Snapshot
- **Status:** grounded (P4 tool)
- **Source:** `core/metrics/trust_snapshot.py``metrics/TRUST_SNAPSHOT.md`
- **Contents:** Decision Ledger Coverage, Attestation Coverage, Capability
Health, AI Decision Accuracy, Confidence-Gate Halt Rate, chain-integrity
verdict, snapshot hash
---
## Deferred Metrics (8 placeholder views)
| Metric | Blocking Decision | Placeholder View |
|--------|-----------------|------------------|
| Live Infrastructure Health | D-096 | `placeholder_live_infra_health.csv` |
| Live Outbox Write Rate | D-096 | `placeholder_live_outbox_rate.csv` |
| Tamper-Evident Ledger Checkpoints | D-083 | `placeholder_tamper_evident_checkpoints.csv` |
| Onboarding Funnel (granted) | D-113/D-114/D-119 | `placeholder_onboarding_funnel.csv` |
| Drift Auto-Reversal Rate | D-096 + no scheduler | `placeholder_drift_detection.csv` |
| Live CUR Reconciliation | D-096 | `placeholder_live_cur_reconciliation.csv` |
| SLA / Unplanned Downtime | D-096 | `placeholder_sla_downtime.csv` |
| Predictive vs Reactive Ratio | future emitter | `placeholder_predictive_reactive.csv` |
See `docs/METRICS_DEFERRED_ROADMAP.md` for the activation path for each.
+70
View File
@@ -0,0 +1,70 @@
# Nova Deferred Metrics Activation Roadmap
> v1.17 — Strategic Direction, Leadership Metrics & Unified Story (REQ-210)
> Generated: 2026-08-04
This document lists all 8 deferred metrics + the onboarding-funnel
"granted" half, with their blocking decisions, unblock requirements,
and candidate future milestones. It also includes the hot-path activation
plan (post-D-096) and the re-evaluation triggers.
## Deferred metrics
| # | Metric | Blocking Decision | What's Needed to Unblock | Candidate Milestone |
|---|--------|-------------------|-------------------------|---------------------|
| 1 | Live Infrastructure Health (ECS, ALB, RPS) | D-096 | Re-provision live AWS; deploy microservice/static-assets stacks; emit live health metrics | v1.18+ (live AWS re-provisioning) |
| 2 | Live Outbox Write Rate / Ledger Append Latency | D-096 | Re-provision DynamoDB outbox table; emit write-latency metrics | v1.18+ |
| 3 | Tamper-Evident Ledger Checkpoints / JWS Signature Rate | D-083 | Build S3 Object Lock + JWS signing + async worker + DLQ + daily checkpoints | v1.19+ (audit ledger build-out) |
| 4 | Onboarding Funnel (requested → granted) | D-113/D-114/D-119 | Implement auto-grant: Lambda provisions the cross-account role + ABAC tag + environment binding | v1.18+ (onboarding auto-grant) |
| 5 | Drift Auto-Reversal Rate | D-096 + no scheduler | Build a drift-detection scheduler (cron); run `terraform plan -detailed-exitcode` per workspace; emit drift.detected events | v1.20+ (drift detection) |
| 6 | Live CUR Reconciliation | D-096 | Re-provision live AWS billing access; build CUR reconciler (6h schedule); match bill lines to resource addresses via tags | v1.18+ |
| 7 | SLA / Unplanned Downtime | D-096 | Deploy live services with SLOs; emit uptime metrics against SLO targets | v1.18+ |
| 8 | Predictive vs Reactive Ratio | future emitter | Build an ML anomaly-forecasting service; emit anomaly.predicted events with proactive label | v1.21+ (predictive ops) |
## Onboarding-funnel "granted" half
The onboarding request path is grounded (REQ-182/183 from v1.16): a
consumer submits a request → the Lambda writes a `pending` CMDB row →
`core/onboarding.py` generates a binding file. The "granted" half
(actual AWS account/network/state provisioning) is deferred per
D-113/D-114/D-119. When a future milestone implements auto-grant, the
onboarding funnel metric activates: `count(granted) ÷ count(requested)`.
## Hot-Path Activation (post-D-096)
**Current state (v1.17):** SQLite cold store only (D-126). No hot path.
The hot path activates when live AWS is re-provisioned (D-096 lift).
**Nova-native hot-path candidates (D-120 — no Kafka/Prometheus/ClickHouse):**
1. **SQLite read-replica:** the cold store becomes a read-replica updated
on each run; a lightweight file-watcher notifies the dashboard of
changes. Freshness = "last run" (not 1-second, but sufficient for
batch ops).
2. **JSONL tail + webhook:** the events.jsonl log is tailed by a small
daemon that pushes updates to a webhook (e.g., a PowerBI streaming
dataset or a custom dashboard). Nova-native (no new infra).
3. **SQLite + Grafana SQLite datasource:** Grafana can read SQLite
directly via the SQLite datasource plugin. No TSDB needed.
**Migration steps (when D-096 lifts):**
1. Re-provision live AWS (microservice + static-assets stacks).
2. Add live-health emitters (ECS running count, ALB 5xx, RPS) to
`run_platform.sh`.
3. Choose a hot-path candidate (above) and implement it.
4. Populate the 8 placeholder views with real data.
5. Re-run the collector + PowerBI export.
## Re-evaluation Triggers
A follow-up metrics ideation should be triggered when any of these
events occurs:
1. **D-096 lift** (live AWS re-provisioned) — triggers hot-path
activation + placeholder view population for metrics 1, 2, 5, 6, 7.
2. **D-083 lift** (S3 Object Lock + JWS build-out approved) — triggers
tamper-evident ledger checkpoint metric (metric 3).
3. **Onboarding-grant lift** (auto-grant implemented) — triggers
onboarding funnel metric (metric 4).
When any trigger fires, re-run `/ci-run` with a metrics-focused milestone
to activate the corresponding placeholder views.
+143
View File
@@ -0,0 +1,143 @@
# Nova Metrics Views — PowerBI Data Dictionary
> v1.17 — Strategic Direction, Leadership Metrics & Unified Story (REQ-190, REQ-209)
> Generated: 2026-08-04
This document is the column-level data dictionary for the PowerBI export
views in `metrics/powerbi/`. Each fact/dimension table and placeholder
view is documented with: column, type, source/formula, unit, and
grounded/derived/deferred status.
## Fact tables (grounded)
### fact_run
| Column | Type | Source | Unit | Status |
|--------|------|--------|------|--------|
| run_id | TEXT | run_manifest.py | — | grounded |
| contract_id | TEXT | run_manifest.py | — | grounded |
| environment | TEXT | run_manifest.py | dev/qa/prod/dr | grounded |
| started_at | TEXT | run_manifest.py | ISO8601 | grounded |
| completed_at | TEXT | run_manifest.py | ISO8601 | grounded |
| exit_code | INTEGER | run_manifest.py | — | grounded |
| outcome | TEXT | run_manifest.py | succeeded/failed | grounded |
| confidence_score | REAL | confidence_signal.py | 0.01.0 | grounded |
| confidence_band | TEXT | confidence_signal.py | pass/warn/block | grounded |
| hitl_block | INTEGER | hitl_gates.py | 0/1 | grounded |
| cost_estimate_usd | REAL | infracost_adapter.py | USD | grounded (Infracost) |
| decision_id | TEXT | decision_ledger.py | — | grounded |
### fact_capability
| Column | Type | Source | Unit | Status |
|--------|------|--------|------|--------|
| capability_id | TEXT | REGRESSION_REPORT.json | CAP-NNN | grounded |
| run_id | TEXT | REGRESSION_REPORT.json | — | grounded |
| name | TEXT | REGRESSION_REPORT.json | — | grounded |
| status | TEXT | REGRESSION_REPORT.json | Verified/Decayed/Broken/Skipped | grounded |
| tier | TEXT | REGRESSION_REPORT.json | local/live-aws/lifecycle-pipeline | grounded |
| duration_ms | REAL | REGRESSION_REPORT.json | milliseconds | grounded |
| detail | TEXT | REGRESSION_REPORT.json | — | grounded |
| run_at_utc | TEXT | REGRESSION_REPORT.json | ISO8601 | grounded |
### fact_decision
| Column | Type | Source | Unit | Status |
|--------|------|--------|------|--------|
| decision_id | TEXT | decision_ledger.py | = run_id | grounded |
| run_id | TEXT | decision_ledger.py | — | grounded |
| chosen_action | TEXT | confidence_signal.py | pass/warn/block | grounded |
| confidence | REAL | confidence_signal.py | 0.01.0 | grounded |
| alternatives | TEXT (JSON) | confidence_signal.py | perInput breakdown | grounded |
| human_override | INTEGER | hitl_gates.py | 0/1 | grounded |
| outcome | TEXT | decision_ledger.py | succeeded/failed/pending | grounded |
| event_time | TEXT | decision_ledger.py | ISO8601 | grounded |
### fact_test
| Column | Type | Source | Unit | Status |
|--------|------|--------|------|--------|
| run_id | TEXT | junit XML | — | grounded |
| total_tests | INTEGER | junit XML | count | grounded |
| passed | INTEGER | junit XML | count | grounded |
| failed | INTEGER | junit XML | count | grounded |
| errors | INTEGER | junit XML | count | grounded |
| skipped | INTEGER | junit XML | count | grounded |
| duration_s | REAL | junit XML | seconds | grounded |
| coverage_pct | REAL | coverage.json | % | grounded |
| collected_at | TEXT | collector.py | ISO8601 | grounded |
### fact_cost_estimate
| Column | Type | Source | Unit | Status |
|--------|------|--------|------|--------|
| run_id | TEXT | infracost_adapter.py | — | grounded |
| delta_usd | REAL | Infracost | USD/month | grounded (pre-apply) |
| total_monthly_usd | REAL | Infracost | USD/month | grounded (pre-apply) |
| available | INTEGER | infracost_adapter.py | 0/1 | grounded |
| estimated_at | TEXT | infracost_adapter.py | ISO8601 | grounded |
### fact_lifecycle
| Column | Type | Source | Unit | Status |
|--------|------|--------|------|--------|
| module | TEXT | lifecycle report | — | grounded |
| environment | TEXT | lifecycle report | — | grounded |
| phase | TEXT | lifecycle report | apply/modify/destroy | grounded |
| result | TEXT | lifecycle report | pass/fail | grounded |
| duration_ms | REAL | lifecycle report | milliseconds | grounded |
| run_at | TEXT | lifecycle report | ISO8601 | grounded |
## Dimension tables
### dim_capability
| Column | Type | Source | Status |
|--------|------|--------|--------|
| capability_id | TEXT | REGRESSION_REPORT.json | grounded |
| name | TEXT | REGRESSION_REPORT.json | grounded |
| tier | TEXT | REGRESSION_REPORT.json | grounded |
| source_milestone | TEXT | REGRESSION_REPORT.json | grounded |
### dim_milestone
| Column | Type | Source | Status |
|--------|------|--------|--------|
| milestone | TEXT | REGRESSION_REPORT.json | grounded |
| phase | INTEGER | REGRESSION_REPORT.json | grounded |
| tag | TEXT | — | grounded |
| completed_at | TEXT | REGRESSION_REPORT.json | grounded |
## Placeholder views (deferred — 8 views, headers only, no data)
### placeholder_live_infra_health
- **Blocking decision:** D-096
- **Description:** Live infrastructure health (ECS running count, ALB 5xx, RPS)
- **Columns:** timestamp, resource_id, resource_type, running_count, healthy, downtime_seconds
### placeholder_live_outbox_rate
- **Blocking decision:** D-096
- **Description:** Live outbox write rate / ledger append latency
- **Columns:** timestamp, contract_id, write_latency_ms, append_count
### placeholder_tamper_evident_checkpoints
- **Blocking decision:** D-083
- **Description:** Tamper-evident ledger checkpoints / JWS signature rate
- **Columns:** timestamp, checkpoint_id, jws_signed, object_lock_enabled
### placeholder_onboarding_funnel
- **Blocking decision:** D-113/D-114/D-119
- **Description:** Onboarding funnel: requested → granted conversion
- **Columns:** timestamp, consumer_repo, requested_environment, status, granted_at
### placeholder_drift_detection
- **Blocking decision:** D-096 + no scheduler
- **Description:** Drift detection (scheduled terraform plan -detailed-exitcode)
- **Columns:** timestamp, workspace_id, drift_count, auto_reverted, detection_cycle
### placeholder_live_cur_reconciliation
- **Blocking decision:** D-096
- **Description:** Live cost CUR reconciliation
- **Columns:** timestamp, resource_address, actual_usd, baseline_usd, saved_usd
### placeholder_sla_downtime
- **Blocking decision:** D-096
- **Description:** SLA / unplanned downtime
- **Columns:** timestamp, service, uptime_pct, downtime_minutes, slo_target
### placeholder_predictive_reactive
- **Blocking decision:** future emitter
- **Description:** Predictive vs Reactive ratio
- **Columns:** timestamp, action_id, label, trigger, count
+67
View File
@@ -0,0 +1,67 @@
# Nova — The No-Humans Infrastructure Platform: Thesis Defensibility Brief
> v1.17 — Strategic Direction, Leadership Metrics & Unified Story (REQ-213)
> Generated: 2026-08-04
## The thesis
Nova is the autonomous infrastructure layer that lets product teams
ship without engaging an operator, and lets executives trust the AI
not because it never fails but because every decision is captured,
scored, and accountable.
**Autonomy in operations; human at stage gates.** The operator is
removed from the loop of normal operations. Human attestation remains
required at stage gates — QA signs off for production, SRE greenlights
based on operational readiness. The absence of an operator is never
the absence of a record.
## Grounded proof (measurable today)
| Proof | Source | Status |
|-------|--------|--------|
| 18 capabilities verified, 4 honestly skipped (0 broken) | `REGRESSION_REPORT.json` | grounded |
| Decision Ledger captures 100% of AI decisions with outcome backfill | `metrics/decision_ledger.db` | grounded (this milestone) |
| Attestation Coverage: 100% of prod/dr promotions attested by a human | `hitl_gates.py` + outbox `approver_*` | grounded |
| Confidence-gated policy engine (not an LLM) — 6 weighted inputs, band outcome | `confidence_signal.py` | grounded |
| 8-concern attestation matrix with separation-of-duties on prod | `attestation_matrix.py` + `separation_of_duties.py` | grounded |
| Pre-apply cost estimates (Infracost, offline) | `infracost_adapter.py` | grounded |
| Test suite passes (~656 tests) | `metrics/test-results.xml` | grounded |
## Deferred proof (measurable when blocking decisions lift)
| Proof | Blocking Decision | Unblock Requirement |
|-------|-------------------|---------------------|
| Touchless Resolution Rate ≥99% across production estates | 0 consumers today | Pilot estate activation |
| Live infrastructure health (ECS, ALB, RPS) | D-096 | Live AWS re-provisioning |
| Onboarding funnel: requested → granted | D-113/D-114/D-119 | Auto-grant implementation |
| Drift auto-reversal rate ≥95% | D-096 + no scheduler | Drift detection scheduler |
| Predictive vs reactive ratio ≥3:1 | future emitter | ML anomaly-forecasting service |
| Tamper-evident ledger checkpoints (S3 Object Lock + JWS) | D-083 | Audit ledger build-out |
## Anti-claims (what Nova is NOT)
1. **Nova's "AI" is NOT an LLM planner.** It is a confidence-gated
policy engine (confidence_signal + HITL gate). The Decision Ledger
captures this real decision path — not a fabricated "AI agent" that
doesn't exist yet (D-122). When an LLM planner is added, it will emit
richer `alternatives_considered` without schema breakage.
2. **Nova does NOT remove humans from accountability.** Only from
operations. Every stage-gate promotion (qa/prod/dr) requires a human
attestation recorded with approver identity, separation-of-duties
check, and the 8-concern evidence matrix (NORTH_STAR Anti-Goal #3).
3. **Nova is NOT for legacy, untagged, or freeform infrastructure.** It
requires Terraform-managed, policy-aligned, fully-tagged inputs
(NORTH_STAR Anti-Goal #4).
4. **Nova does NOT fabricate metrics.** Every metric is grounded (cites
a source file), derived (documented formula), or deferred (cites a
blocking decision ID). No fabricated numbers in any deck slide or
METRICS.md entry (the "no fabrication" hard constraint).
## What "won" looks like
By month 18, Nova is the layer enterprise leadership points to when
they say *"we don't have an infrastructure ops team anymore, and the
audit trail is stronger than it ever was"* — and it is the default
substrate their AI engineering teams reach for first when an agent needs
to deploy.
+21
View File
@@ -0,0 +1,21 @@
# AI Decision Accuracy — Definition of Success
> KPI: AI Decision Accuracy
> Target: ≥ 99.5% (no rollback, no follow-up incident within 5 min of action)
**What this number means:** the percentage of AI decisions (confidence-
gated policy engine outcomes) that were NOT followed by an apply failure
or incident within 5 minutes. A high-confidence decision that later
caused an incident does NOT count as accurate.
**How it's computed:** `count(decisions WHERE outcome = 'succeeded' AND
no incident within 5min)` ÷ `total decisions`. Correlation via
`decision_id``run_id` → subsequent `apply.failed` or `incident.detected`
events.
**What "good" looks like:** ≥ 99.5% means fewer than 1 in 200 decisions
cause a secondary failure. The 0.5% allowance is for novel edge cases.
**D-122 honesty:** Nova's "AI" is the confidence-gated policy engine
(confidence_signal + HITL gate), not an LLM planner. The Decision Ledger
captures this real decision path — not a fabricated "AI agent."
+18
View File
@@ -0,0 +1,18 @@
# Attestation Coverage — Definition of Success
> KPI: Attestation Coverage
> Target: 100% of prod/dr promotions attested by a human
**What this number means:** every production and disaster-recovery
promotion has a recorded human attestation (approver identity, 8-concern
matrix result, separation-of-duties check on prod). This is the
"autonomy in operations, human in accountability" proof.
**How it's computed:** `count(prod/dr promotions with attestation.recorded
event) ÷ count(total prod/dr promotions)`. Sourced from the Decision
Ledger (`attestation.recorded` events) + `hitl_gates.py` + outbox
`approver_*` attributes.
**What "good" looks like:** 100% means no prod/dr promotion ever lands
without a human sign-off on record. The absence of an operator is never
the absence of a record (NORTH_STAR Anti-Goal #3).
+16
View File
@@ -0,0 +1,16 @@
# Confidence-Gate Halt Rate — Definition of Success
> KPI: Confidence-Gate Halt Rate
> Target: not a committed target (operational signal)
**What this number means:** how often the confidence gate itself halted
a run (band = block), independent of HITL blocks. The gate is the AI's
self-halt; HITL is the human gate. This distinguishes the AI's
self-regulation from human escalation.
**How it's computed:** `count(runs WHERE confidence_band = 'block')` ÷
`total runs`.
**What "good" looks like:** a low but non-zero rate means the gate is
working (catching genuinely uncertain runs) without being overly
conservative (blocking everything).
+18
View File
@@ -0,0 +1,18 @@
# Cost Savings via Infracost Estimates — Definition of Success
> KPI: Cost Savings via Infracost Estimates
> Target: ≥ 25% on pilot estates (partial)
**What this number means:** the pre-apply cost estimate from Infracost
shows the delta between the planned infrastructure and the current
state. Negative deltas = savings.
**How it's computed:** `sum(fact_cost_estimate.delta_usd WHERE delta < 0)`
per period.
**What's grounded:** the pre-apply estimate (Infracost reads plan JSON,
offline).
**What's deferred:** actual-spend reconciliation from AWS CUR (D-096 —
needs live AWS billing). The placeholder view
`placeholder_live_cur_reconciliation.csv` has the schema ready.
+16
View File
@@ -0,0 +1,16 @@
# Decision Ledger Coverage — Definition of Success
> KPI: Decision Ledger Coverage
> Target: 100% of AI actions with backfilled outcome
**What this number means:** every AI decision (confidence-gated policy
engine outcome) is captured in the Decision Ledger with its outcome
backfilled from the subsequent apply.completed/failed event.
**How it's computed:** `count(decision_ledger rows WHERE outcome ≠
'pending') ÷ count(decision_ledger rows)`. Sourced from
`metrics/decision_ledger.db`.
**What "good" looks like:** 100% means no AI decision is ever lost or
left without an outcome. The ledger is the trust substrate (NORTH_STAR
Objective #2).
+13
View File
@@ -0,0 +1,13 @@
# Deployment Frequency — Definition of Success
> KPI: Deployment Frequency
> Target: not a committed target (operational signal)
**What this number means:** the rate of infrastructure state updates
deployed safely per day. A DORA-adjacent metric for infrastructure.
**How it's computed:** `count(run.completed WHERE exit_code = 0)` per
day.
**What "good" looks like:** multiple deploys per day (vs. weekly/monthly
for human ops teams).
+17
View File
@@ -0,0 +1,17 @@
# FTE Hours Saved (Toil Reallocation Value) — Definition of Success
> KPI: FTE Hours Saved
> Target: ≥ 70% of pre-Nova FTE allocation (derived)
**What this number means:** the engineering hours saved by automated
operations, valued at the blended engineering rate. This is what those
hours were spent on instead (the "toil reallocation" — capital freed
up from ops to feature development).
**How it's computed:** `run count × manual baseline minutes per run ÷ 60
× blended hourly rate`. The manual baseline is the estimated time a
human team would take for the same operation (e.g., 30 min/ticket).
**Honesty caveat:** computed on N internal runs today; the production-
denominator activates post-pilot. The formula is grounded; the
production numbers are not yet.
@@ -0,0 +1,18 @@
# Human Escalation Frequency — Definition of Success
> KPI: Human Escalation Frequency
> Target: < 0.1% of platform actions (Post-Pilot)
**What this number means:** how often the AI platform was forced to fall
back or escalate to a human operator due to low confidence. This is the
inverse of Touchless Resolution Rate, scoped to operational escalations
only.
**How it's computed:** `count(runs WHERE hitl_block = 1 AND reason =
'confidence')` ÷ `total runs`. Attestation sign-offs are excluded.
**What "good" looks like:** < 0.1% means fewer than 1 in 1000 runs
require human intervention. Near-zero is the goal.
**What would be "gamer metrics":** counting attestation sign-offs as
escalations (they're not — they're designed controls).
+19
View File
@@ -0,0 +1,19 @@
# MTTR (Platform-Run) — Definition of Success
> KPI: MTTR (p95)
> Target: < 60 seconds
**What this number means:** the time from a platform-run failure
(apply.failed) to a successful retry. This is platform-run MTTR, not
infra-incident MTTR (which requires an incident detection system that
Nova doesn't have yet — deferred).
**How it's computed:** p95 of `successful_retry.time failed_run.time`
across all runs that failed then succeeded.
**What "good" looks like:** < 60 seconds means the platform recovers
from a failed run in under a minute, 95% of the time.
**What's deferred:** infra-incident MTTR (anomaly detected → healed)
requires an incident detection/remediation system (self-healing
velocity). That's a future emitter.
+15
View File
@@ -0,0 +1,15 @@
# Platform ROI — Definition of Success
> KPI: Platform ROI
> Target: ≥ 250% measured annually (derived)
**What this number means:** the total financial value delivered (labor
savings + cloud cost optimization + avoided downtime losses) vs. the
platform's operational/licensing cost.
**Formula:** `(FTE hours saved × blended rate + cloud savings + avoided
downtime) ÷ platform op cost`.
**Honesty caveat:** computed on N internal runs today; the production-
denominator activates post-pilot. The formula is grounded; the
production numbers are not yet.
+14
View File
@@ -0,0 +1,14 @@
# Zero-Trust Policy Compliance Rate — Definition of Success
> KPI: Zero-Trust Policy Compliance Rate
> Target: not a committed target (operational signal)
**What this number means:** the percentage of infrastructure assets
continuously verified as compliant with security baselines and policies.
**How it's computed:** `1 count(assets WHERE last_scan.status ≠ pass)
÷ count(assets)`. Sourced from `fact_policy_check` (Checkov results).
**What "good" looks like:** 100% means every resource passed every
policy check. The Nova tagging standard (nova_tagging.py, hard mode) is
the primary check.
+13
View File
@@ -0,0 +1,13 @@
# Provisioning Lead Time — Definition of Success
> KPI: Provisioning Lead Time
> Target: not a committed target (operational signal)
**What this number means:** the time from intent received (run.started)
to apply completed (run.completed). Measures how fast Nova provisions
compliant environments.
**How it's computed:** `run.completed_at run.started_at` per run.
**What "good" looks like:** minutes, not days. The reduction from days
(human ops) to minutes (autonomous) is the velocity proof.
+23
View File
@@ -0,0 +1,23 @@
# Touchless Resolution Rate — Definition of Success
> KPI: Touchless Resolution Rate
> Target: ≥ 99% across production estates (Post-Pilot)
**What this number means:** the percentage of platform runs that complete
end-to-end without an operational HITL block. An operational HITL block
is a confidence-driven escalation (the AI's confidence was too low to
proceed). Attestation gates (qa/prod/dr sign-offs) are NOT counted as
escalations — they are designed controls, not autonomy failures.
**How it's computed:** `runs WHERE hitl_block = 0 AND environment = 'dev'`
÷ `total runs` (dev environment only, where attestation gates don't apply).
For production estates: `runs WHERE hitl_block = 0` ÷ `total runs`
excluding attestation-gate sign-offs.
**What "good" looks like:** ≥ 99% means fewer than 1 in 100 runs require
human intervention due to low confidence. The 1% allowance is for
genuine edge cases (novel failure modes, blast-radius exceedances).
**What would be "gamer metrics":** counting attestation gates as
"touchless" (they're not — they're human by design) or counting only
dev runs (cherry-picking the easiest environment).
+23
View File
@@ -0,0 +1,23 @@
# Nova Trust Snapshot — 2026-08-04T20:05:00Z
> v1.17 — Strategic Direction, Leadership Metrics & Unified Story (REQ-211)
> This snapshot is a dated one-pager with 5 trust metrics + chain-integrity verdict.
## Trust Metrics
| Metric | Value | Details |
|--------|-------|---------|
| **Decision Ledger Coverage** | 0.0% | 0 entries, 0 broken |
| **Attestation Coverage** | 0.0% | 0 attestation events |
| **Capability Health** | 18V / 4S / 0B / 0D | from REGRESSION_REPORT.json |
| **AI Decision Accuracy** | 0.0% | 0/0 succeeded |
| **Confidence-Gate Halt Rate** | 0.0% | 0/0 halted |
## Chain Integrity
- **Verdict:** INTACT
- **Broken entries:** 0
## Snapshot Hash
`a3c59eda5d569a5d`
+66
View File
@@ -0,0 +1,66 @@
# Nova PowerBI Dashboard — Import Guide
> v1.17 — Strategic Direction, Leadership Metrics & Unified Story (REQ-208)
> Generated: 2026-08-04
This guide documents how to import Nova's metrics views into PowerBI
via the folder connector, and suggests a starter visual model.
## Import via folder connector
1. Open PowerBI Desktop.
2. **Get Data****Folder** → navigate to `metrics/powerbi/`.
3. PowerBI discovers all CSV/JSON files in the folder.
4. Combine the files — PowerBI creates a single query per file.
## Starter visual model
### Suggested joins
- `fact_run` ←→ `fact_decision` on `run_id` (run-level decision path)
- `fact_run` ←→ `fact_cost_estimate` on `run_id` (run-level cost)
- `fact_capability` ←→ `dim_capability` on `capability_id` (capability lookup)
- `fact_capability` ←→ `dim_milestone` on `milestone` (milestone lookup)
### Suggested visuals (4 starter visuals)
1. **Capability Health over Time** — bar chart: `fact_capability.status`
grouped by `run_at_utc`. Shows Verified/Skipped/Broken/Decayed trend.
Source: `fact_capability.csv`.
2. **Confidence Distribution** — histogram: `fact_confidence.score`.
Shows the distribution of confidence scores across all runs.
Source: `fact_confidence.csv`.
3. **Decision Accuracy** — KPI card: count of `fact_decision` where
`outcome = 'succeeded'` ÷ total `fact_decision` rows. Shows AI
Decision Accuracy (NORTH_STAR target ≥99.5%).
Source: `fact_decision.csv`.
4. **Cost Trend** — line chart: `fact_cost_estimate.delta_usd` over
`estimated_at`. Shows pre-apply cost estimate trend (Infracost).
Source: `fact_cost_estimate.csv`.
## Placeholder views (deferred metrics)
The 8 `placeholder_*.csv` files contain headers only (no data rows).
Each has a companion `placeholder_*.json` with the schema metadata
(columns, blocking decision, description). When the blocking decision
lifts (e.g., D-096 for live AWS), the collector will populate these
views and PowerBI will automatically pick up the data.
## Data refresh
The export is regenerated by running:
```bash
python3 core/metrics/collector.py # rebuilds nova_metrics.db
python3 core/metrics/powerbi_export.py # exports to metrics/powerbi/
```
In PowerBI, click **Refresh** to pick up the updated CSV/JSON files.
## Honesty model
Every metric in the export is `grounded` (cites a source file), `derived`
(documented formula), or `deferred` (cites a blocking decision ID). See
`docs/METRICS.md` (P4) for the canonical catalog and `docs/METRICS_VIEWS.md`
for the column-level data dictionary.
+64
View File
@@ -0,0 +1,64 @@
#!/usr/bin/env bash
# Nova NORTH_STAR diff-check (REQ-204).
#
# Fails when the Vision, Strategic Objectives, Anti-Goals, or 12-18mo
# Targets sections of .ciagent/NORTH_STAR.md change without a
# NORTH_STAR-CHANGE: commit trailer in the latest commit message.
#
# Usage: bash scripts/check_north_star_diff.sh
# Returns 0 on pass, 1 on fail.
set -euo pipefail
NORTH_STAR=".ciagent/NORTH_STAR.md"
SECTIONS_REGEX='^## (Vision|Strategic Objectives|Anti-Goals|12.*18 Month Targets)'
if [ ! -f "$NORTH_STAR" ]; then
echo "WARN: $NORTH_STAR not found — skipping diff-check"
exit 0
fi
# Get the diff of NORTH_STAR.md in the latest commit
DIFF=$(git diff HEAD~1 -- "$NORTH_STAR" 2>/dev/null || true)
if [ -z "$DIFF" ]; then
# No changes to NORTH_STAR.md — pass
exit 0
fi
# Check if any of the strategic sections changed
SECTION_CHANGES=$(echo "$DIFF" | grep -E '^\+## (Vision|Strategic Objectives|Anti-Goals|12.*18 Month Targets)' || true)
LINE_CHANGES=$(echo "$DIFF" | grep -E '^[+-]' | grep -v '^[+-]{3}' | head -50 || true)
# Simple heuristic: if lines under the strategic sections changed
CHANGED_SECTIONS=""
CURRENT_SECTION=""
while IFS= read -r line; do
case "$line" in
"+## Vision"*) CURRENT_SECTION="Vision" ;;
"+## Strategic Objectives"*) CURRENT_SECTION="Strategic Objectives" ;;
"+## Anti-Goals"*) CURRENT_SECTION="Anti-Goals" ;;
"+## 12"*) CURRENT_SECTION="12-18mo Targets" ;;
"+## "*) CURRENT_SECTION="" ;;
esac
if [ -n "$CURRENT_SECTION" ] && [ -n "$line" ] && [[ "$line" == +* ]] && [[ "$line" != "+## "* ]]; then
CHANGED_SECTIONS="$CHANGED_SECTIONS $CURRENT_SECTION"
fi
done <<< "$DIFF"
if [ -z "$CHANGED_SECTIONS" ]; then
# No strategic section changes — pass
exit 0
fi
# Check for the NORTH_STAR-CHANGE: commit trailer
COMMIT_MSG=$(git log -1 --format='%B')
if echo "$COMMIT_MSG" | grep -q "NORTH_STAR-CHANGE:"; then
echo "OK: NORTH_STAR strategic sections changed with NORTH_STAR-CHANGE: trailer"
exit 0
fi
echo "FAIL: NORTH_STAR strategic sections changed without NORTH_STAR-CHANGE: commit trailer"
echo "Changed sections:$CHANGED_SECTIONS"
echo "Add 'NORTH_STAR-CHANGE: <description>' to the commit message and re-commit."
exit 1
+91
View File
@@ -0,0 +1,91 @@
"""Tests for Nova PowerBI export (P3, REQ-190)."""
import json
import os
import sqlite3
import sys
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT))
@pytest.fixture
def tmp_export(tmp_path, monkeypatch):
metrics_dir = tmp_path / "metrics"
metrics_dir.mkdir()
export_dir = metrics_dir / "powerbi"
store_db = metrics_dir / "nova_metrics.db"
monkeypatch.setattr("core.metrics.powerbi_export._METRICS_DIR", str(metrics_dir))
monkeypatch.setattr("core.metrics.powerbi_export._STORE_PATH", str(store_db))
monkeypatch.setattr("core.metrics.powerbi_export._EXPORT_DIR", str(export_dir))
return {"metrics_dir": metrics_dir, "store_db": store_db, "export_dir": export_dir}
def _init_store_with_data(db_path):
conn = sqlite3.connect(str(db_path))
conn.executescript("""
CREATE TABLE fact_run (run_id TEXT PRIMARY KEY, contract_id TEXT, environment TEXT, exit_code INTEGER);
CREATE TABLE dim_capability (capability_id TEXT PRIMARY KEY, name TEXT, tier TEXT);
INSERT INTO fact_run VALUES ('run-1', 'cid-1', 'dev', 0);
INSERT INTO dim_capability VALUES ('CAP-001', 'test cap', 'local');
""")
conn.commit()
conn.close()
def test_export_csv(tmp_export):
from core.metrics.powerbi_export import export_all
_init_store_with_data(tmp_export["store_db"])
result = export_all(fmt="csv")
assert (tmp_export["export_dir"] / "fact_run.csv").exists()
assert (tmp_export["export_dir"] / "dim_capability.csv").exists()
assert result["fact_tables"]["fact_run"] == 1
def test_export_json(tmp_export):
from core.metrics.powerbi_export import export_all
_init_store_with_data(tmp_export["store_db"])
result = export_all(fmt="json")
assert (tmp_export["export_dir"] / "fact_run.json").exists()
data = json.loads((tmp_export["export_dir"] / "fact_run.json").read_text())
assert len(data) == 1
assert data[0]["run_id"] == "run-1"
def test_export_placeholder_views(tmp_export):
from core.metrics.powerbi_export import export_all, PLACEHOLDER_VIEWS
result = export_all(fmt="both")
for view_name in PLACEHOLDER_VIEWS:
assert (tmp_export["export_dir"] / f"{view_name}.csv").exists()
assert (tmp_export["export_dir"] / f"{view_name}.json").exists()
assert len(PLACEHOLDER_VIEWS) == 8
def test_export_placeholder_csv_headers_only(tmp_export):
from core.metrics.powerbi_export import export_all
export_all(fmt="csv")
csv_path = tmp_export["export_dir"] / "placeholder_drift_detection.csv"
lines = csv_path.read_text().strip().split("\n")
assert len(lines) == 1 # headers only, no data
assert "timestamp" in lines[0]
def test_export_placeholder_json_schema(tmp_export):
from core.metrics.powerbi_export import export_all
export_all(fmt="json")
json_path = tmp_export["export_dir"] / "placeholder_sla_downtime.json"
data = json.loads(json_path.read_text())
assert "schema" in data
assert data["schema"]["blocking_decision"] == "D-096"
assert data["data"] == []
def test_export_no_store(tmp_export):
from core.metrics.powerbi_export import export_all
result = export_all(fmt="csv")
assert "error" in result
# Placeholders still exported
assert (tmp_export["export_dir"] / "placeholder_drift_detection.csv").exists()