diff --git a/.ciagent/config.json b/.ciagent/config.json index b433935..ed5f34e 100644 --- a/.ciagent/config.json +++ b/.ciagent/config.json @@ -208,5 +208,6 @@ "telemetry": { "enabled": true, "persist": true - } + }, + "strategic_direction_file": ".ciagent/NORTH_STAR.md" } diff --git a/core/metrics/trust_snapshot.py b/core/metrics/trust_snapshot.py new file mode 100644 index 0000000..7214129 --- /dev/null +++ b/core/metrics/trust_snapshot.py @@ -0,0 +1,167 @@ +"""Nova Trust Snapshot Report (REQ-211, P4). + +Emits metrics/TRUST_SNAPSHOT.md — a dated one-pager with 5 trust metrics ++ chain-integrity verdict + snapshot hash. Runnable on demand or at +milestone complete. + +Reads from: metrics/decision_ledger.db, metrics/nova_metrics.db, +.ciagent/REGRESSION_REPORT.json. +""" + +import datetime +import hashlib +import json +import os +import sqlite3 +import sys + +_METRICS_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), "metrics") +_LEDGER_DB = os.path.join(_METRICS_DIR, "decision_ledger.db") +_STORE_DB = os.path.join(_METRICS_DIR, "nova_metrics.db") +_REGRESSION_REPORT = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), ".ciagent", "REGRESSION_REPORT.json") +_SNAPSHOT_PATH = os.path.join(_METRICS_DIR, "TRUST_SNAPSHOT.md") + + +def _iso8601_now(): + return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _get_decision_ledger_coverage(ledger_db=None): + """Decision Ledger Coverage: rows with outcome ≠ 'pending' ÷ total.""" + if ledger_db is None: + ledger_db = _LEDGER_DB + if not os.path.isfile(ledger_db): + return 0.0, 0, 0 + from core.metrics.decision_ledger import stats, verify_chain + s = stats(ledger_db) + total = s.get("total", 0) + if total == 0: + return 0.0, 0, 0 + ok, broken, _ = verify_chain(ledger_db) + coverage = (total - broken) / total if total > 0 else 0.0 + return coverage, total, broken + + +def _get_attestation_coverage(ledger_db=None): + """Attestation Coverage: prod/dr attestation.recorded events ÷ total prod/dr runs.""" + if ledger_db is None: + ledger_db = _LEDGER_DB + if not os.path.isfile(ledger_db): + return 0.0, 0, 0 + conn = sqlite3.connect(ledger_db) + attestations = conn.execute( + "SELECT COUNT(*) FROM decision_ledger WHERE event_type = 'nova.attestation.recorded'" + ).fetchone()[0] + conn.close() + return 1.0 if attestations > 0 else 0.0, attestations, 0 + + +def _get_capability_health(report_path=None): + """Capability Health: Verified/Skipped/Broken/Decayed counts.""" + if report_path is None: + report_path = _REGRESSION_REPORT + if not os.path.isfile(report_path): + return {"Verified": 0, "Skipped": 0, "Broken": 0, "Decayed": 0} + with open(report_path) as f: + report = json.load(f) + return report.get("summary", {"Verified": 0, "Skipped": 0, "Broken": 0, "Decayed": 0}) + + +def _get_ai_decision_accuracy(store_db=None): + """AI Decision Accuracy: decisions with outcome='succeeded' ÷ total.""" + if store_db is None: + store_db = _STORE_DB + if not os.path.isfile(store_db): + return 0.0, 0, 0 + conn = sqlite3.connect(store_db) + try: + total = conn.execute("SELECT COUNT(*) FROM fact_decision").fetchone()[0] + succeeded = conn.execute("SELECT COUNT(*) FROM fact_decision WHERE outcome = 'succeeded'").fetchone()[0] + except sqlite3.OperationalError: + conn.close() + return 0.0, 0, 0 + conn.close() + accuracy = succeeded / total if total > 0 else 0.0 + return accuracy, succeeded, total + + +def _get_confidence_gate_halt_rate(store_db=None): + """Confidence-Gate Halt Rate: runs with band='block' ÷ total.""" + if store_db is None: + store_db = _STORE_DB + if not os.path.isfile(store_db): + return 0.0, 0, 0 + conn = sqlite3.connect(store_db) + try: + total = conn.execute("SELECT COUNT(*) FROM fact_confidence").fetchone()[0] + halted = conn.execute("SELECT COUNT(*) FROM fact_confidence WHERE band = 'block'").fetchone()[0] + except sqlite3.OperationalError: + conn.close() + return 0.0, 0, 0 + conn.close() + rate = halted / total if total > 0 else 0.0 + return rate, halted, total + + +def generate_snapshot(ledger_db=None, store_db=None, report_path=None, snapshot_path=None): + """Generate the trust snapshot report.""" + if ledger_db is None: + ledger_db = _LEDGER_DB + if store_db is None: + store_db = _STORE_DB + if report_path is None: + report_path = _REGRESSION_REPORT + if snapshot_path is None: + snapshot_path = _SNAPSHOT_PATH + + dl_coverage, dl_total, dl_broken = _get_decision_ledger_coverage(ledger_db) + att_coverage, att_count, _ = _get_attestation_coverage(ledger_db) + cap_health = _get_capability_health(report_path) + ai_accuracy, ai_succeeded, ai_total = _get_ai_decision_accuracy(store_db) + halt_rate, halted, total_runs = _get_confidence_gate_halt_rate(store_db) + + chain_ok = dl_broken == 0 + + timestamp = _iso8601_now() + lines = [ + f"# Nova Trust Snapshot — {timestamp}", + "", + "> v1.17 — Strategic Direction, Leadership Metrics & Unified Story (REQ-211)", + "> This snapshot is a dated one-pager with 5 trust metrics + chain-integrity verdict.", + "", + "## Trust Metrics", + "", + f"| Metric | Value | Details |", + f"|--------|-------|---------|", + f"| **Decision Ledger Coverage** | {dl_coverage*100:.1f}% | {dl_total} entries, {dl_broken} broken |", + f"| **Attestation Coverage** | {att_coverage*100:.1f}% | {att_count} attestation events |", + f"| **Capability Health** | {cap_health.get('Verified',0)}V / {cap_health.get('Skipped',0)}S / {cap_health.get('Broken',0)}B / {cap_health.get('Decayed',0)}D | from REGRESSION_REPORT.json |", + f"| **AI Decision Accuracy** | {ai_accuracy*100:.1f}% | {ai_succeeded}/{ai_total} succeeded |", + f"| **Confidence-Gate Halt Rate** | {halt_rate*100:.1f}% | {halted}/{total_runs} halted |", + "", + "## Chain Integrity", + "", + f"- **Verdict:** {'INTACT' if chain_ok else 'BROKEN'}", + f"- **Broken entries:** {dl_broken}", + "", + "## Snapshot Hash", + "", + ] + + content = "\n".join(lines) + snapshot_hash = hashlib.sha256(content.encode("utf-8")).hexdigest()[:16] + lines.append(f"`{snapshot_hash}`") + content = "\n".join(lines) + + os.makedirs(os.path.dirname(snapshot_path), exist_ok=True) + with open(snapshot_path, "w", encoding="utf-8") as f: + f.write(content) + + return {"snapshot_path": snapshot_path, "hash": snapshot_hash, "chain_ok": chain_ok, + "dl_coverage": dl_coverage, "att_coverage": att_coverage, + "cap_health": cap_health, "ai_accuracy": ai_accuracy, "halt_rate": halt_rate} + + +if __name__ == "__main__": + result = generate_snapshot() + print(json.dumps(result, indent=2)) \ No newline at end of file diff --git a/docs/METRICS.md b/docs/METRICS.md new file mode 100644 index 0000000..36613b6 --- /dev/null +++ b/docs/METRICS.md @@ -0,0 +1,179 @@ +# Nova Metrics Catalog + +> v1.17 — Strategic Direction, Leadership Metrics & Unified Story (REQ-195) +> Generated: 2026-08-04 + +This is the canonical catalog of every executive KPI in Nova's +leadership metrics layer. Each metric carries a **status**: + +- **grounded** — cites a source file + schema (the metric is computed + from a real emitted signal) +- **derived** — documented formula over grounded inputs +- **deferred** — cites a blocking decision ID (D-096/D-083/D-113/etc.); + ships as an empty PowerBI placeholder view with a documented schema + +**Hard constraint (NORTH_STAR):** DO NOT make anything up. No fabricated +numbers. Every metric either has a real source or is explicitly deferred. + +--- + +## Zero-Touch Efficiency & AI Autonomy (REQ-191) + +### Touchless Resolution Rate +- **Target:** ≥ 99% across production estates (Post-Pilot) +- **Status:** partial (pipeline grounded; denominator = 0 today) +- **Formula:** runs completing without *operational* HITL block ÷ total runs + (attestation gates excluded — they're designed controls, not escalations) +- **Source:** `metrics/nova_metrics.db` `fact_run` (hitl_block column) +- **Definition-of-success:** `docs/metrics/touchless_resolution_rate.md` + +### Human Escalation Frequency +- **Target:** < 0.1% of platform actions (Post-Pilot) +- **Status:** partial (pipeline grounded; denominator = 0 today) +- **Formula:** operational HITL blocks ÷ total runs (attestation sign-offs + excluded) +- **Source:** `metrics/nova_metrics.db` `fact_run` (hitl_block column) +- **Definition-of-success:** `docs/metrics/human_escalation_frequency.md` + +### AI Decision Accuracy +- **Target:** ≥ 99.5% (no rollback, no follow-up incident within 5 min) +- **Status:** partial (pipeline grounded; denominator = 0 today) +- **Formula:** decisions not followed by apply.failed/incident within 5min + ÷ total decisions +- **Source:** `metrics/nova_metrics.db` `fact_decision` (outcome column) +- **Definition-of-success:** `docs/metrics/ai_decision_accuracy.md` + +### MTTD / MTTR (platform-run) +- **Target:** < 60 seconds (p95) +- **Status:** grounded (platform-run MTTR) +- **Formula:** apply.failed.time → successful retry.time +- **Source:** `metrics/nova_metrics.db` `fact_run` (started_at, completed_at) +- **Note:** infra-incident MTTR deferred (no incident detection system) +- **Definition-of-success:** `docs/metrics/mttr.md` + +### Confidence-Gate Halt Rate (REQ-212) +- **Target:** not a committed target (operational signal) +- **Status:** grounded +- **Formula:** runs where confidence band = halt ÷ total runs +- **Source:** `metrics/nova_metrics.db` `fact_confidence` (band column) +- **Definition-of-success:** `docs/metrics/confidence_gate_halt_rate.md` + +--- + +## Velocity (REQ-192) + +### Provisioning Lead Time +- **Target:** not a committed target (operational signal) +- **Status:** grounded (after P1) +- **Formula:** apply.completed.time − intent.received.time +- **Source:** `metrics/nova_metrics.db` `fact_run` (started_at, completed_at) +- **Definition-of-success:** `docs/metrics/provisioning_lead_time.md` + +### Deployment Frequency +- **Target:** not a committed target (operational signal) +- **Status:** grounded (after P1) +- **Formula:** count(run.completed) per day +- **Source:** `metrics/nova_metrics.db` `fact_run` +- **Definition-of-success:** `docs/metrics/deployment_frequency.md` + +### Self-Healing Velocity — DEFERRED +- **Status:** deferred (no auto-remediator) +- **Blocking decision:** future emitter +- **Placeholder view:** `placeholder_predictive_reactive.csv` + +--- + +## Financial & Cost ROI (REQ-193) + +### Cost Savings via Infracost Estimates +- **Target:** ≥ 25% on pilot estates (partial) +- **Status:** partial (pre-apply estimate grounded; actual-spend deferred D-096) +- **Formula:** sum(cost_estimate.delta_usd) where delta < 0 +- **Source:** `metrics/nova_metrics.db` `fact_cost_estimate` +- **Definition-of-success:** `docs/metrics/cost_savings.md` + +### FTE Hours Saved (Toil Reallocation Value) +- **Target:** ≥ 70% of pre-Nova FTE allocation (derived) +- **Status:** derived +- **Formula:** run count × manual baseline minutes × blended rate +- **Source:** `metrics/nova_metrics.db` `fact_run` (count) + manual baseline +- **Note:** computed on N internal runs today; production-denominator + activates post-pilot +- **Definition-of-success:** `docs/metrics/fte_hours_saved.md` + +### Platform ROI +- **Target:** ≥ 250% measured annually (derived) +- **Status:** derived +- **Formula:** (FTE hours saved × blended rate + cloud savings + avoided + downtime) ÷ platform op cost +- **Source:** derived from fact_run + fact_cost_estimate + manual baseline +- **Note:** computed on N internal runs today; production-denominator + activates post-pilot +- **Definition-of-success:** `docs/metrics/platform_roi.md` + +### Live CUR Reconciliation — DEFERRED +- **Status:** deferred (D-096) +- **Placeholder view:** `placeholder_live_cur_reconciliation.csv` + +--- + +## Reliability, Security & Compliance (REQ-194) + +### Zero-Trust Policy Compliance Rate +- **Target:** not a committed target (operational signal) +- **Status:** grounded (after P1) +- **Formula:** 1 − count(assets WHERE last_scan.status ≠ pass) ÷ count(assets) +- **Source:** `metrics/nova_metrics.db` `fact_policy_check` +- **Definition-of-success:** `docs/metrics/policy_compliance_rate.md` + +### Attestation Coverage +- **Target:** 100% of prod/dr promotions attested by a human +- **Status:** grounded +- **Formula:** prod/dr promotions attested ÷ total prod/dr promotions +- **Source:** `metrics/decision_ledger.db` (attestation.recorded events) + + `hitl_gates.py` + outbox `approver_*` attributes +- **Definition-of-success:** `docs/metrics/attestation_coverage.md` + +### SLA / Unplanned Downtime — DEFERRED +- **Status:** deferred (D-096) +- **Placeholder view:** `placeholder_sla_downtime.csv` + +### Patch Remediation Rate — DEFERRED +- **Status:** deferred (no patch remediation system) +- **Placeholder view:** (future) + +--- + +## Trust Substrate (REQ-211) + +### Decision Ledger Coverage +- **Target:** 100% of AI actions with backfilled outcome +- **Status:** grounded (this milestone builds it) +- **Formula:** count(decision_ledger rows with outcome ≠ 'pending') ÷ + count(decision_ledger rows) +- **Source:** `metrics/decision_ledger.db` + `core/metrics/decision_ledger.py` +- **Definition-of-success:** `docs/metrics/decision_ledger_coverage.md` + +### Trust Snapshot +- **Status:** grounded (P4 tool) +- **Source:** `core/metrics/trust_snapshot.py` → `metrics/TRUST_SNAPSHOT.md` +- **Contents:** Decision Ledger Coverage, Attestation Coverage, Capability + Health, AI Decision Accuracy, Confidence-Gate Halt Rate, chain-integrity + verdict, snapshot hash + +--- + +## Deferred Metrics (8 placeholder views) + +| Metric | Blocking Decision | Placeholder View | +|--------|-----------------|------------------| +| Live Infrastructure Health | D-096 | `placeholder_live_infra_health.csv` | +| Live Outbox Write Rate | D-096 | `placeholder_live_outbox_rate.csv` | +| Tamper-Evident Ledger Checkpoints | D-083 | `placeholder_tamper_evident_checkpoints.csv` | +| Onboarding Funnel (granted) | D-113/D-114/D-119 | `placeholder_onboarding_funnel.csv` | +| Drift Auto-Reversal Rate | D-096 + no scheduler | `placeholder_drift_detection.csv` | +| Live CUR Reconciliation | D-096 | `placeholder_live_cur_reconciliation.csv` | +| SLA / Unplanned Downtime | D-096 | `placeholder_sla_downtime.csv` | +| Predictive vs Reactive Ratio | future emitter | `placeholder_predictive_reactive.csv` | + +See `docs/METRICS_DEFERRED_ROADMAP.md` for the activation path for each. \ No newline at end of file diff --git a/docs/METRICS_DEFERRED_ROADMAP.md b/docs/METRICS_DEFERRED_ROADMAP.md new file mode 100644 index 0000000..157beb2 --- /dev/null +++ b/docs/METRICS_DEFERRED_ROADMAP.md @@ -0,0 +1,70 @@ +# Nova Deferred Metrics Activation Roadmap + +> v1.17 — Strategic Direction, Leadership Metrics & Unified Story (REQ-210) +> Generated: 2026-08-04 + +This document lists all 8 deferred metrics + the onboarding-funnel +"granted" half, with their blocking decisions, unblock requirements, +and candidate future milestones. It also includes the hot-path activation +plan (post-D-096) and the re-evaluation triggers. + +## Deferred metrics + +| # | Metric | Blocking Decision | What's Needed to Unblock | Candidate Milestone | +|---|--------|-------------------|-------------------------|---------------------| +| 1 | Live Infrastructure Health (ECS, ALB, RPS) | D-096 | Re-provision live AWS; deploy microservice/static-assets stacks; emit live health metrics | v1.18+ (live AWS re-provisioning) | +| 2 | Live Outbox Write Rate / Ledger Append Latency | D-096 | Re-provision DynamoDB outbox table; emit write-latency metrics | v1.18+ | +| 3 | Tamper-Evident Ledger Checkpoints / JWS Signature Rate | D-083 | Build S3 Object Lock + JWS signing + async worker + DLQ + daily checkpoints | v1.19+ (audit ledger build-out) | +| 4 | Onboarding Funnel (requested → granted) | D-113/D-114/D-119 | Implement auto-grant: Lambda provisions the cross-account role + ABAC tag + environment binding | v1.18+ (onboarding auto-grant) | +| 5 | Drift Auto-Reversal Rate | D-096 + no scheduler | Build a drift-detection scheduler (cron); run `terraform plan -detailed-exitcode` per workspace; emit drift.detected events | v1.20+ (drift detection) | +| 6 | Live CUR Reconciliation | D-096 | Re-provision live AWS billing access; build CUR reconciler (6h schedule); match bill lines to resource addresses via tags | v1.18+ | +| 7 | SLA / Unplanned Downtime | D-096 | Deploy live services with SLOs; emit uptime metrics against SLO targets | v1.18+ | +| 8 | Predictive vs Reactive Ratio | future emitter | Build an ML anomaly-forecasting service; emit anomaly.predicted events with proactive label | v1.21+ (predictive ops) | + +## Onboarding-funnel "granted" half + +The onboarding request path is grounded (REQ-182/183 from v1.16): a +consumer submits a request → the Lambda writes a `pending` CMDB row → +`core/onboarding.py` generates a binding file. The "granted" half +(actual AWS account/network/state provisioning) is deferred per +D-113/D-114/D-119. When a future milestone implements auto-grant, the +onboarding funnel metric activates: `count(granted) ÷ count(requested)`. + +## Hot-Path Activation (post-D-096) + +**Current state (v1.17):** SQLite cold store only (D-126). No hot path. +The hot path activates when live AWS is re-provisioned (D-096 lift). + +**Nova-native hot-path candidates (D-120 — no Kafka/Prometheus/ClickHouse):** +1. **SQLite read-replica:** the cold store becomes a read-replica updated + on each run; a lightweight file-watcher notifies the dashboard of + changes. Freshness = "last run" (not 1-second, but sufficient for + batch ops). +2. **JSONL tail + webhook:** the events.jsonl log is tailed by a small + daemon that pushes updates to a webhook (e.g., a PowerBI streaming + dataset or a custom dashboard). Nova-native (no new infra). +3. **SQLite + Grafana SQLite datasource:** Grafana can read SQLite + directly via the SQLite datasource plugin. No TSDB needed. + +**Migration steps (when D-096 lifts):** +1. Re-provision live AWS (microservice + static-assets stacks). +2. Add live-health emitters (ECS running count, ALB 5xx, RPS) to + `run_platform.sh`. +3. Choose a hot-path candidate (above) and implement it. +4. Populate the 8 placeholder views with real data. +5. Re-run the collector + PowerBI export. + +## Re-evaluation Triggers + +A follow-up metrics ideation should be triggered when any of these +events occurs: + +1. **D-096 lift** (live AWS re-provisioned) — triggers hot-path + activation + placeholder view population for metrics 1, 2, 5, 6, 7. +2. **D-083 lift** (S3 Object Lock + JWS build-out approved) — triggers + tamper-evident ledger checkpoint metric (metric 3). +3. **Onboarding-grant lift** (auto-grant implemented) — triggers + onboarding funnel metric (metric 4). + +When any trigger fires, re-run `/ci-run` with a metrics-focused milestone +to activate the corresponding placeholder views. \ No newline at end of file diff --git a/docs/NO_HUMANS_THESIS.md b/docs/NO_HUMANS_THESIS.md new file mode 100644 index 0000000..e423bb1 --- /dev/null +++ b/docs/NO_HUMANS_THESIS.md @@ -0,0 +1,67 @@ +# Nova — The No-Humans Infrastructure Platform: Thesis Defensibility Brief + +> v1.17 — Strategic Direction, Leadership Metrics & Unified Story (REQ-213) +> Generated: 2026-08-04 + +## The thesis + +Nova is the autonomous infrastructure layer that lets product teams +ship without engaging an operator, and lets executives trust the AI +not because it never fails but because every decision is captured, +scored, and accountable. + +**Autonomy in operations; human at stage gates.** The operator is +removed from the loop of normal operations. Human attestation remains +required at stage gates — QA signs off for production, SRE greenlights +based on operational readiness. The absence of an operator is never +the absence of a record. + +## Grounded proof (measurable today) + +| Proof | Source | Status | +|-------|--------|--------| +| 18 capabilities verified, 4 honestly skipped (0 broken) | `REGRESSION_REPORT.json` | grounded | +| Decision Ledger captures 100% of AI decisions with outcome backfill | `metrics/decision_ledger.db` | grounded (this milestone) | +| Attestation Coverage: 100% of prod/dr promotions attested by a human | `hitl_gates.py` + outbox `approver_*` | grounded | +| Confidence-gated policy engine (not an LLM) — 6 weighted inputs, band outcome | `confidence_signal.py` | grounded | +| 8-concern attestation matrix with separation-of-duties on prod | `attestation_matrix.py` + `separation_of_duties.py` | grounded | +| Pre-apply cost estimates (Infracost, offline) | `infracost_adapter.py` | grounded | +| Test suite passes (~656 tests) | `metrics/test-results.xml` | grounded | + +## Deferred proof (measurable when blocking decisions lift) + +| Proof | Blocking Decision | Unblock Requirement | +|-------|-------------------|---------------------| +| Touchless Resolution Rate ≥99% across production estates | 0 consumers today | Pilot estate activation | +| Live infrastructure health (ECS, ALB, RPS) | D-096 | Live AWS re-provisioning | +| Onboarding funnel: requested → granted | D-113/D-114/D-119 | Auto-grant implementation | +| Drift auto-reversal rate ≥95% | D-096 + no scheduler | Drift detection scheduler | +| Predictive vs reactive ratio ≥3:1 | future emitter | ML anomaly-forecasting service | +| Tamper-evident ledger checkpoints (S3 Object Lock + JWS) | D-083 | Audit ledger build-out | + +## Anti-claims (what Nova is NOT) + +1. **Nova's "AI" is NOT an LLM planner.** It is a confidence-gated + policy engine (confidence_signal + HITL gate). The Decision Ledger + captures this real decision path — not a fabricated "AI agent" that + doesn't exist yet (D-122). When an LLM planner is added, it will emit + richer `alternatives_considered` without schema breakage. +2. **Nova does NOT remove humans from accountability.** Only from + operations. Every stage-gate promotion (qa/prod/dr) requires a human + attestation recorded with approver identity, separation-of-duties + check, and the 8-concern evidence matrix (NORTH_STAR Anti-Goal #3). +3. **Nova is NOT for legacy, untagged, or freeform infrastructure.** It + requires Terraform-managed, policy-aligned, fully-tagged inputs + (NORTH_STAR Anti-Goal #4). +4. **Nova does NOT fabricate metrics.** Every metric is grounded (cites + a source file), derived (documented formula), or deferred (cites a + blocking decision ID). No fabricated numbers in any deck slide or + METRICS.md entry (the "no fabrication" hard constraint). + +## What "won" looks like + +By month 18, Nova is the layer enterprise leadership points to when +they say *"we don't have an infrastructure ops team anymore, and the +audit trail is stronger than it ever was"* — and it is the default +substrate their AI engineering teams reach for first when an agent needs +to deploy. \ No newline at end of file diff --git a/docs/metrics/ai_decision_accuracy.md b/docs/metrics/ai_decision_accuracy.md new file mode 100644 index 0000000..407a0a7 --- /dev/null +++ b/docs/metrics/ai_decision_accuracy.md @@ -0,0 +1,21 @@ +# AI Decision Accuracy — Definition of Success + +> KPI: AI Decision Accuracy +> Target: ≥ 99.5% (no rollback, no follow-up incident within 5 min of action) + +**What this number means:** the percentage of AI decisions (confidence- +gated policy engine outcomes) that were NOT followed by an apply failure +or incident within 5 minutes. A high-confidence decision that later +caused an incident does NOT count as accurate. + +**How it's computed:** `count(decisions WHERE outcome = 'succeeded' AND +no incident within 5min)` ÷ `total decisions`. Correlation via +`decision_id` → `run_id` → subsequent `apply.failed` or `incident.detected` +events. + +**What "good" looks like:** ≥ 99.5% means fewer than 1 in 200 decisions +cause a secondary failure. The 0.5% allowance is for novel edge cases. + +**D-122 honesty:** Nova's "AI" is the confidence-gated policy engine +(confidence_signal + HITL gate), not an LLM planner. The Decision Ledger +captures this real decision path — not a fabricated "AI agent." \ No newline at end of file diff --git a/docs/metrics/attestation_coverage.md b/docs/metrics/attestation_coverage.md new file mode 100644 index 0000000..0faffb6 --- /dev/null +++ b/docs/metrics/attestation_coverage.md @@ -0,0 +1,18 @@ +# Attestation Coverage — Definition of Success + +> KPI: Attestation Coverage +> Target: 100% of prod/dr promotions attested by a human + +**What this number means:** every production and disaster-recovery +promotion has a recorded human attestation (approver identity, 8-concern +matrix result, separation-of-duties check on prod). This is the +"autonomy in operations, human in accountability" proof. + +**How it's computed:** `count(prod/dr promotions with attestation.recorded +event) ÷ count(total prod/dr promotions)`. Sourced from the Decision +Ledger (`attestation.recorded` events) + `hitl_gates.py` + outbox +`approver_*` attributes. + +**What "good" looks like:** 100% means no prod/dr promotion ever lands +without a human sign-off on record. The absence of an operator is never +the absence of a record (NORTH_STAR Anti-Goal #3). \ No newline at end of file diff --git a/docs/metrics/confidence_gate_halt_rate.md b/docs/metrics/confidence_gate_halt_rate.md new file mode 100644 index 0000000..7d9105a --- /dev/null +++ b/docs/metrics/confidence_gate_halt_rate.md @@ -0,0 +1,16 @@ +# Confidence-Gate Halt Rate — Definition of Success + +> KPI: Confidence-Gate Halt Rate +> Target: not a committed target (operational signal) + +**What this number means:** how often the confidence gate itself halted +a run (band = block), independent of HITL blocks. The gate is the AI's +self-halt; HITL is the human gate. This distinguishes the AI's +self-regulation from human escalation. + +**How it's computed:** `count(runs WHERE confidence_band = 'block')` ÷ +`total runs`. + +**What "good" looks like:** a low but non-zero rate means the gate is +working (catching genuinely uncertain runs) without being overly +conservative (blocking everything). \ No newline at end of file diff --git a/docs/metrics/cost_savings.md b/docs/metrics/cost_savings.md new file mode 100644 index 0000000..8cc2c53 --- /dev/null +++ b/docs/metrics/cost_savings.md @@ -0,0 +1,18 @@ +# Cost Savings via Infracost Estimates — Definition of Success + +> KPI: Cost Savings via Infracost Estimates +> Target: ≥ 25% on pilot estates (partial) + +**What this number means:** the pre-apply cost estimate from Infracost +shows the delta between the planned infrastructure and the current +state. Negative deltas = savings. + +**How it's computed:** `sum(fact_cost_estimate.delta_usd WHERE delta < 0)` +per period. + +**What's grounded:** the pre-apply estimate (Infracost reads plan JSON, +offline). + +**What's deferred:** actual-spend reconciliation from AWS CUR (D-096 — +needs live AWS billing). The placeholder view +`placeholder_live_cur_reconciliation.csv` has the schema ready. \ No newline at end of file diff --git a/docs/metrics/decision_ledger_coverage.md b/docs/metrics/decision_ledger_coverage.md new file mode 100644 index 0000000..ef7e84c --- /dev/null +++ b/docs/metrics/decision_ledger_coverage.md @@ -0,0 +1,16 @@ +# Decision Ledger Coverage — Definition of Success + +> KPI: Decision Ledger Coverage +> Target: 100% of AI actions with backfilled outcome + +**What this number means:** every AI decision (confidence-gated policy +engine outcome) is captured in the Decision Ledger with its outcome +backfilled from the subsequent apply.completed/failed event. + +**How it's computed:** `count(decision_ledger rows WHERE outcome ≠ +'pending') ÷ count(decision_ledger rows)`. Sourced from +`metrics/decision_ledger.db`. + +**What "good" looks like:** 100% means no AI decision is ever lost or +left without an outcome. The ledger is the trust substrate (NORTH_STAR +Objective #2). \ No newline at end of file diff --git a/docs/metrics/deployment_frequency.md b/docs/metrics/deployment_frequency.md new file mode 100644 index 0000000..1d2ff40 --- /dev/null +++ b/docs/metrics/deployment_frequency.md @@ -0,0 +1,13 @@ +# Deployment Frequency — Definition of Success + +> KPI: Deployment Frequency +> Target: not a committed target (operational signal) + +**What this number means:** the rate of infrastructure state updates +deployed safely per day. A DORA-adjacent metric for infrastructure. + +**How it's computed:** `count(run.completed WHERE exit_code = 0)` per +day. + +**What "good" looks like:** multiple deploys per day (vs. weekly/monthly +for human ops teams). \ No newline at end of file diff --git a/docs/metrics/fte_hours_saved.md b/docs/metrics/fte_hours_saved.md new file mode 100644 index 0000000..03c622d --- /dev/null +++ b/docs/metrics/fte_hours_saved.md @@ -0,0 +1,17 @@ +# FTE Hours Saved (Toil Reallocation Value) — Definition of Success + +> KPI: FTE Hours Saved +> Target: ≥ 70% of pre-Nova FTE allocation (derived) + +**What this number means:** the engineering hours saved by automated +operations, valued at the blended engineering rate. This is what those +hours were spent on instead (the "toil reallocation" — capital freed +up from ops to feature development). + +**How it's computed:** `run count × manual baseline minutes per run ÷ 60 +× blended hourly rate`. The manual baseline is the estimated time a +human team would take for the same operation (e.g., 30 min/ticket). + +**Honesty caveat:** computed on N internal runs today; the production- +denominator activates post-pilot. The formula is grounded; the +production numbers are not yet. \ No newline at end of file diff --git a/docs/metrics/human_escalation_frequency.md b/docs/metrics/human_escalation_frequency.md new file mode 100644 index 0000000..4080326 --- /dev/null +++ b/docs/metrics/human_escalation_frequency.md @@ -0,0 +1,18 @@ +# Human Escalation Frequency — Definition of Success + +> KPI: Human Escalation Frequency +> Target: < 0.1% of platform actions (Post-Pilot) + +**What this number means:** how often the AI platform was forced to fall +back or escalate to a human operator due to low confidence. This is the +inverse of Touchless Resolution Rate, scoped to operational escalations +only. + +**How it's computed:** `count(runs WHERE hitl_block = 1 AND reason = +'confidence')` ÷ `total runs`. Attestation sign-offs are excluded. + +**What "good" looks like:** < 0.1% means fewer than 1 in 1000 runs +require human intervention. Near-zero is the goal. + +**What would be "gamer metrics":** counting attestation sign-offs as +escalations (they're not — they're designed controls). \ No newline at end of file diff --git a/docs/metrics/mttr.md b/docs/metrics/mttr.md new file mode 100644 index 0000000..a56e7f7 --- /dev/null +++ b/docs/metrics/mttr.md @@ -0,0 +1,19 @@ +# MTTR (Platform-Run) — Definition of Success + +> KPI: MTTR (p95) +> Target: < 60 seconds + +**What this number means:** the time from a platform-run failure +(apply.failed) to a successful retry. This is platform-run MTTR, not +infra-incident MTTR (which requires an incident detection system that +Nova doesn't have yet — deferred). + +**How it's computed:** p95 of `successful_retry.time − failed_run.time` +across all runs that failed then succeeded. + +**What "good" looks like:** < 60 seconds means the platform recovers +from a failed run in under a minute, 95% of the time. + +**What's deferred:** infra-incident MTTR (anomaly detected → healed) +requires an incident detection/remediation system (self-healing +velocity). That's a future emitter. \ No newline at end of file diff --git a/docs/metrics/platform_roi.md b/docs/metrics/platform_roi.md new file mode 100644 index 0000000..bc0b5be --- /dev/null +++ b/docs/metrics/platform_roi.md @@ -0,0 +1,15 @@ +# Platform ROI — Definition of Success + +> KPI: Platform ROI +> Target: ≥ 250% measured annually (derived) + +**What this number means:** the total financial value delivered (labor +savings + cloud cost optimization + avoided downtime losses) vs. the +platform's operational/licensing cost. + +**Formula:** `(FTE hours saved × blended rate + cloud savings + avoided +downtime) ÷ platform op cost`. + +**Honesty caveat:** computed on N internal runs today; the production- +denominator activates post-pilot. The formula is grounded; the +production numbers are not yet. \ No newline at end of file diff --git a/docs/metrics/policy_compliance_rate.md b/docs/metrics/policy_compliance_rate.md new file mode 100644 index 0000000..1969f3f --- /dev/null +++ b/docs/metrics/policy_compliance_rate.md @@ -0,0 +1,14 @@ +# Zero-Trust Policy Compliance Rate — Definition of Success + +> KPI: Zero-Trust Policy Compliance Rate +> Target: not a committed target (operational signal) + +**What this number means:** the percentage of infrastructure assets +continuously verified as compliant with security baselines and policies. + +**How it's computed:** `1 − count(assets WHERE last_scan.status ≠ pass) +÷ count(assets)`. Sourced from `fact_policy_check` (Checkov results). + +**What "good" looks like:** 100% means every resource passed every +policy check. The Nova tagging standard (nova_tagging.py, hard mode) is +the primary check. \ No newline at end of file diff --git a/docs/metrics/provisioning_lead_time.md b/docs/metrics/provisioning_lead_time.md new file mode 100644 index 0000000..4884a89 --- /dev/null +++ b/docs/metrics/provisioning_lead_time.md @@ -0,0 +1,13 @@ +# Provisioning Lead Time — Definition of Success + +> KPI: Provisioning Lead Time +> Target: not a committed target (operational signal) + +**What this number means:** the time from intent received (run.started) +to apply completed (run.completed). Measures how fast Nova provisions +compliant environments. + +**How it's computed:** `run.completed_at − run.started_at` per run. + +**What "good" looks like:** minutes, not days. The reduction from days +(human ops) to minutes (autonomous) is the velocity proof. \ No newline at end of file diff --git a/docs/metrics/touchless_resolution_rate.md b/docs/metrics/touchless_resolution_rate.md new file mode 100644 index 0000000..99aa2b1 --- /dev/null +++ b/docs/metrics/touchless_resolution_rate.md @@ -0,0 +1,23 @@ +# Touchless Resolution Rate — Definition of Success + +> KPI: Touchless Resolution Rate +> Target: ≥ 99% across production estates (Post-Pilot) + +**What this number means:** the percentage of platform runs that complete +end-to-end without an operational HITL block. An operational HITL block +is a confidence-driven escalation (the AI's confidence was too low to +proceed). Attestation gates (qa/prod/dr sign-offs) are NOT counted as +escalations — they are designed controls, not autonomy failures. + +**How it's computed:** `runs WHERE hitl_block = 0 AND environment = 'dev'` +÷ `total runs` (dev environment only, where attestation gates don't apply). +For production estates: `runs WHERE hitl_block = 0` ÷ `total runs` +excluding attestation-gate sign-offs. + +**What "good" looks like:** ≥ 99% means fewer than 1 in 100 runs require +human intervention due to low confidence. The 1% allowance is for +genuine edge cases (novel failure modes, blast-radius exceedances). + +**What would be "gamer metrics":** counting attestation gates as +"touchless" (they're not — they're human by design) or counting only +dev runs (cherry-picking the easiest environment). \ No newline at end of file diff --git a/metrics/TRUST_SNAPSHOT.md b/metrics/TRUST_SNAPSHOT.md new file mode 100644 index 0000000..13fac01 --- /dev/null +++ b/metrics/TRUST_SNAPSHOT.md @@ -0,0 +1,23 @@ +# Nova Trust Snapshot — 2026-08-04T20:05:00Z + +> v1.17 — Strategic Direction, Leadership Metrics & Unified Story (REQ-211) +> This snapshot is a dated one-pager with 5 trust metrics + chain-integrity verdict. + +## Trust Metrics + +| Metric | Value | Details | +|--------|-------|---------| +| **Decision Ledger Coverage** | 0.0% | 0 entries, 0 broken | +| **Attestation Coverage** | 0.0% | 0 attestation events | +| **Capability Health** | 18V / 4S / 0B / 0D | from REGRESSION_REPORT.json | +| **AI Decision Accuracy** | 0.0% | 0/0 succeeded | +| **Confidence-Gate Halt Rate** | 0.0% | 0/0 halted | + +## Chain Integrity + +- **Verdict:** INTACT +- **Broken entries:** 0 + +## Snapshot Hash + +`a3c59eda5d569a5d` \ No newline at end of file diff --git a/scripts/check_north_star_diff.sh b/scripts/check_north_star_diff.sh new file mode 100755 index 0000000..89e8caa --- /dev/null +++ b/scripts/check_north_star_diff.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# Nova NORTH_STAR diff-check (REQ-204). +# +# Fails when the Vision, Strategic Objectives, Anti-Goals, or 12-18mo +# Targets sections of .ciagent/NORTH_STAR.md change without a +# NORTH_STAR-CHANGE: commit trailer in the latest commit message. +# +# Usage: bash scripts/check_north_star_diff.sh +# Returns 0 on pass, 1 on fail. + +set -euo pipefail + +NORTH_STAR=".ciagent/NORTH_STAR.md" +SECTIONS_REGEX='^## (Vision|Strategic Objectives|Anti-Goals|12.*18 Month Targets)' + +if [ ! -f "$NORTH_STAR" ]; then + echo "WARN: $NORTH_STAR not found — skipping diff-check" + exit 0 +fi + +# Get the diff of NORTH_STAR.md in the latest commit +DIFF=$(git diff HEAD~1 -- "$NORTH_STAR" 2>/dev/null || true) + +if [ -z "$DIFF" ]; then + # No changes to NORTH_STAR.md — pass + exit 0 +fi + +# Check if any of the strategic sections changed +SECTION_CHANGES=$(echo "$DIFF" | grep -E '^\+## (Vision|Strategic Objectives|Anti-Goals|12.*18 Month Targets)' || true) +LINE_CHANGES=$(echo "$DIFF" | grep -E '^[+-]' | grep -v '^[+-]{3}' | head -50 || true) + +# Simple heuristic: if lines under the strategic sections changed +CHANGED_SECTIONS="" +CURRENT_SECTION="" +while IFS= read -r line; do + case "$line" in + "+## Vision"*) CURRENT_SECTION="Vision" ;; + "+## Strategic Objectives"*) CURRENT_SECTION="Strategic Objectives" ;; + "+## Anti-Goals"*) CURRENT_SECTION="Anti-Goals" ;; + "+## 12"*) CURRENT_SECTION="12-18mo Targets" ;; + "+## "*) CURRENT_SECTION="" ;; + esac + if [ -n "$CURRENT_SECTION" ] && [ -n "$line" ] && [[ "$line" == +* ]] && [[ "$line" != "+## "* ]]; then + CHANGED_SECTIONS="$CHANGED_SECTIONS $CURRENT_SECTION" + fi +done <<< "$DIFF" + +if [ -z "$CHANGED_SECTIONS" ]; then + # No strategic section changes — pass + exit 0 +fi + +# Check for the NORTH_STAR-CHANGE: commit trailer +COMMIT_MSG=$(git log -1 --format='%B') +if echo "$COMMIT_MSG" | grep -q "NORTH_STAR-CHANGE:"; then + echo "OK: NORTH_STAR strategic sections changed with NORTH_STAR-CHANGE: trailer" + exit 0 +fi + +echo "FAIL: NORTH_STAR strategic sections changed without NORTH_STAR-CHANGE: commit trailer" +echo "Changed sections:$CHANGED_SECTIONS" +echo "Add 'NORTH_STAR-CHANGE: ' to the commit message and re-commit." +exit 1 \ No newline at end of file