acdl/core/metrics/powerbi_export.py

"""Nova PowerBI Export (REQ-190, P3).

Emits CSV/JSON views to metrics/powerbi/ from the SQLite cold store.
Fact + dimension tables + 8 empty placeholder views for deferred metrics
(with documented schemas ready to fill when their blocking decisions lift).

D-120: Nova-native (CSV/JSON files, no live connector)
D-129: PowerBI ingests via the folder connector
D-128: metrics/ at repo root
"""

import csv
import datetime
import json
import os
import sqlite3
import sys

_METRICS_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), "metrics")
_STORE_PATH = os.path.join(_METRICS_DIR, "nova_metrics.db")
_EXPORT_DIR = os.path.join(_METRICS_DIR, "powerbi")

FACT_VIEWS = [
    "fact_run",
    "fact_capability",
    "fact_policy_check",
    "fact_confidence",
    "fact_test",
    "fact_decision",
    "fact_cost_estimate",
    "fact_lifecycle",
]

DIM_VIEWS = [
    "dim_capability",
    "dim_milestone",
]

PLACEHOLDER_VIEWS = {
    "placeholder_live_infra_health": {
        "columns": ["timestamp", "resource_id", "resource_type", "running_count", "healthy", "downtime_seconds"],
        "blocking_decision": "D-096",
        "description": "Live infrastructure health (ECS running count, ALB 5xx, RPS). Blocked: live AWS torn down.",
    },
    "placeholder_live_outbox_rate": {
        "columns": ["timestamp", "contract_id", "write_latency_ms", "append_count"],
        "blocking_decision": "D-096",
        "description": "Live outbox write rate / ledger append latency. Blocked: DynamoDB outbox table absent.",
    },
    "placeholder_tamper_evident_checkpoints": {
        "columns": ["timestamp", "checkpoint_id", "jws_signed", "object_lock_enabled"],
        "blocking_decision": "D-083",
        "description": "Tamper-evident ledger checkpoints / JWS signature rate. Blocked: S3 Object Lock + JWS deferred.",
    },
    "placeholder_onboarding_funnel": {
        "columns": ["timestamp", "consumer_repo", "requested_environment", "status", "granted_at"],
        "blocking_decision": "D-113/D-114/D-119",
        "description": "Onboarding funnel: requested → granted conversion. Blocked: no auto-grant event.",
    },
    "placeholder_drift_detection": {
        "columns": ["timestamp", "workspace_id", "drift_count", "auto_reverted", "detection_cycle"],
        "blocking_decision": "D-096 + no scheduler",
        "description": "Drift detection (scheduled terraform plan -detailed-exitcode). Blocked: live AWS + scheduler.",
    },
    "placeholder_live_cur_reconciliation": {
        "columns": ["timestamp", "resource_address", "actual_usd", "baseline_usd", "saved_usd"],
        "blocking_decision": "D-096",
        "description": "Live cost CUR reconciliation. Blocked: live AWS billing. Infracost pre-apply estimates are in fact_cost_estimate.",
    },
    "placeholder_sla_downtime": {
        "columns": ["timestamp", "service", "uptime_pct", "downtime_minutes", "slo_target"],
        "blocking_decision": "D-096",
        "description": "SLA / unplanned downtime. Blocked: needs live service uptime monitoring.",
    },
    "placeholder_predictive_reactive": {
        "columns": ["timestamp", "action_id", "label", "trigger", "count"],
        "blocking_decision": "future emitter",
        "description": "Predictive vs Reactive ratio. Blocked: requires ML anomaly-forecasting service.",
    },
}


def _iso8601_now():
    return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")


def _export_table_csv(conn, table_name, export_dir):
    """Export a SQLite table to a CSV file."""
    rows = conn.execute(f"SELECT * FROM {table_name}").fetchall()
    if not rows:
        return 0
    columns = [desc[0] for desc in conn.execute(f"SELECT * FROM {table_name} LIMIT 0").description]
    csv_path = os.path.join(export_dir, f"{table_name}.csv")
    with open(csv_path, "w", newline="", encoding="utf-8") as f:
        writer = csv.writer(f)
        writer.writerow(columns)
        writer.writerows(rows)
    return len(rows)


def _export_table_json(conn, table_name, export_dir):
    """Export a SQLite table to a JSON file."""
    rows = conn.execute(f"SELECT * FROM {table_name}").fetchall()
    if not rows:
        return 0
    columns = [desc[0] for desc in conn.execute(f"SELECT * FROM {table_name} LIMIT 0").description]
    records = [dict(zip(columns, row)) for row in rows]
    json_path = os.path.join(export_dir, f"{table_name}.json")
    with open(json_path, "w", encoding="utf-8") as f:
        json.dump(records, f, indent=2, default=str)
    return len(rows)


def _export_placeholder_csv(view_name, schema, export_dir):
    """Export a placeholder CSV with headers only (no data rows)."""
    csv_path = os.path.join(export_dir, f"{view_name}.csv")
    with open(csv_path, "w", newline="", encoding="utf-8") as f:
        writer = csv.writer(f)
        writer.writerow(schema["columns"])
    return 0


def _export_placeholder_json(view_name, schema, export_dir):
    """Export a placeholder JSON with schema metadata (no data rows)."""
    json_path = os.path.join(export_dir, f"{view_name}.json")
    with open(json_path, "w", encoding="utf-8") as f:
        json.dump({"schema": schema, "data": []}, f, indent=2)
    return 0


def export_all(store_path=None, export_dir=None, fmt="both"):
    """Export all fact/dim tables + placeholder views to CSV and/or JSON.

    Args:
        store_path: path to the SQLite cold store
        export_dir: directory for exported files
        fmt: "csv", "json", or "both"

    Returns:
        Summary dict with export counts.
    """
    if store_path is None:
        store_path = _STORE_PATH
    if export_dir is None:
        export_dir = _EXPORT_DIR
    os.makedirs(export_dir, exist_ok=True)

    summary = {"exported_at": _iso8601_now(), "fact_tables": {}, "dim_tables": {}, "placeholder_views": {}}

    if not os.path.isfile(store_path):
        summary["error"] = f"SQLite store not found: {store_path}"
        for view_name, schema in PLACEHOLDER_VIEWS.items():
            if fmt in ("csv", "both"):
                _export_placeholder_csv(view_name, schema, export_dir)
            if fmt in ("json", "both"):
                _export_placeholder_json(view_name, schema, export_dir)
            summary["placeholder_views"][view_name] = 0
        return summary

    conn = sqlite3.connect(store_path)

    for table in FACT_VIEWS:
        count = 0
        try:
            if fmt in ("csv", "both"):
                count = _export_table_csv(conn, table, export_dir)
            if fmt in ("json", "both"):
                count = _export_table_json(conn, table, export_dir)
        except sqlite3.OperationalError:
            count = 0
        summary["fact_tables"][table] = count

    for table in DIM_VIEWS:
        count = 0
        try:
            if fmt in ("csv", "both"):
                count = _export_table_csv(conn, table, export_dir)
            if fmt in ("json", "both"):
                count = _export_table_json(conn, table, export_dir)
        except sqlite3.OperationalError:
            count = 0
        summary["dim_tables"][table] = count

    conn.close()

    for view_name, schema in PLACEHOLDER_VIEWS.items():
        if fmt in ("csv", "both"):
            _export_placeholder_csv(view_name, schema, export_dir)
        if fmt in ("json", "both"):
            _export_placeholder_json(view_name, schema, export_dir)
        summary["placeholder_views"][view_name] = 0

    return summary


if __name__ == "__main__":
    result = export_all()
    print(json.dumps(result, indent=2))