Jon Chery
023cc47025
CAP-025 (local tier) asserts the pilot-apply pipeline is structurally ready: run_platform.sh steps present, core pipeline modules importable, dev env bound to 581513795199 (D-203), dynamodb L1 registered (REQ-322), pilot policies authored (REQ-315/320), outcome backfill present (REQ-317). Returns Verified on the current branch (all W2/W3/W4 dependencies in place). Added to CAPABILITY_REGISTRY. The live apply (P4) exercises this end-to-end against AWS. ---ci--- project: acdl phase: 3 milestone: v1.26 status: execute wave: W5 ---
829 lines
36 KiB
Python
Executable File
829 lines
36 KiB
Python
Executable File
"""Regression-class VERIFY (D-091).
|
|
|
|
The standard VERIFY stage is diff-scoped: it checks the phase diff only
|
|
and never re-runs underlying platform capability. That structural defect
|
|
(let 8 NFR-patch phases pass while the platform decayed) is recorded as
|
|
D-091. This module provides the regression-class VERIFY that re-runs
|
|
capability checks against the current codebase and tags each capability
|
|
Verified / Decayed / Broken.
|
|
|
|
A capability check is a function that takes no args and returns
|
|
(status, detail) where status is one of:
|
|
- "Verified" : the capability runs as advertised
|
|
- "Decayed" : the capability runs partially / with errors but the
|
|
core path is intact (e.g. needs revival work)
|
|
- "Broken" : the capability does not run at all
|
|
|
|
The regression run fails closed: any non-Verified capability blocks
|
|
milestone completion. The result is written to
|
|
`.ciagent/REGRESSION_REPORT.md` and a machine-readable JSON file.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
from dataclasses import dataclass, field, asdict
|
|
from pathlib import Path
|
|
from typing import Callable, Dict, List, Optional, Tuple
|
|
|
|
# Repo root on sys.path so `from core import env` resolves to THIS package
|
|
# when regression_verify.py is run as a script (avoids editable-installed
|
|
# third-party `core` shadow).
|
|
_REPO_ROOT = str(Path(__file__).resolve().parent.parent)
|
|
if _REPO_ROOT not in sys.path:
|
|
sys.path.insert(0, _REPO_ROOT)
|
|
|
|
from core import env as _envhelper
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
CIAgent = ROOT / ".ciagent"
|
|
|
|
Status = str # "Verified" | "Decayed" | "Broken"
|
|
|
|
|
|
@dataclass
|
|
class CapabilityResult:
|
|
capability_id: str
|
|
name: str
|
|
status: Status
|
|
detail: str
|
|
tier: str # "local" | "live-aws"
|
|
duration_ms: int
|
|
|
|
|
|
@dataclass
|
|
class RegressionReport:
|
|
run_id: str
|
|
run_at_utc: str
|
|
milestone: str
|
|
phase: int
|
|
results: List[CapabilityResult] = field(default_factory=list)
|
|
|
|
@property
|
|
def summary(self) -> Dict[str, int]:
|
|
counts = {"Verified": 0, "Decayed": 0, "Broken": 0}
|
|
for r in self.results:
|
|
counts[r.status] = counts.get(r.status, 0) + 1
|
|
return counts
|
|
|
|
@property
|
|
def passed(self) -> bool:
|
|
# G-111: Skipped is the post-teardown steady state (D-096) for the
|
|
# live-AWS tier caps (CAP-013..016). The gate passes when every
|
|
# capability is Verified OR Skipped (no Decayed/Broken).
|
|
return all(r.status in ("Verified", "Skipped") for r in self.results)
|
|
|
|
def to_dict(self) -> dict:
|
|
return {
|
|
"run_id": self.run_id,
|
|
"run_at_utc": self.run_at_utc,
|
|
"milestone": self.milestone,
|
|
"phase": self.phase,
|
|
"summary": self.summary,
|
|
"passed": self.passed,
|
|
"results": [asdict(r) for r in self.results],
|
|
}
|
|
|
|
|
|
def _run_subprocess(cmd: List[str], cwd: Optional[str] = None,
|
|
timeout: int = 120,
|
|
env: Optional[Dict[str, str]] = None) -> Tuple[int, str, str]:
|
|
"""Run a subprocess, return (returncode, stdout, stderr)."""
|
|
try:
|
|
p = subprocess.run(
|
|
cmd, cwd=cwd or str(ROOT), capture_output=True,
|
|
text=True, timeout=timeout, env=env,
|
|
)
|
|
return p.returncode, p.stdout, p.stderr
|
|
except subprocess.TimeoutExpired as e:
|
|
return 124, e.stdout or "", e.stderr or ""
|
|
except FileNotFoundError as e:
|
|
return 127, "", str(e)
|
|
|
|
|
|
def _check_subprocess(cmd: List[str], cwd: Optional[str] = None,
|
|
timeout: int = 120,
|
|
env: Optional[Dict[str, str]] = None) -> Tuple[Status, str]:
|
|
"""Run a subprocess; map returncode to a status."""
|
|
rc, out, err = _run_subprocess(cmd, cwd=cwd, timeout=timeout, env=env)
|
|
if rc == 0:
|
|
return "Verified", f"exit 0; {out.strip()[-200:]}"
|
|
if rc == 124:
|
|
return "Decayed", f"timeout after {timeout}s; {err.strip()[-200:]}"
|
|
return "Broken", f"exit {rc}; {err.strip()[-200:]}"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Capability checks (seeded for Phase 52; Phase 54 expands the registry).
|
|
# Each check is local-only at this stage (Phase 53 adds the local emulators;
|
|
# Phase 54 adds the live-AWS tier for the headline E2E).
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _check_contract_schema_validation() -> Tuple[Status, str]:
|
|
"""CAP-001: contract.schema.json validates sample contracts."""
|
|
return _check_subprocess([
|
|
"python3", "-c",
|
|
"import json, yaml, jsonschema; "
|
|
"s=json.load(open('schemas/contract.schema.json')); "
|
|
"[jsonschema.validate(yaml.safe_load(open(f)), s) "
|
|
" for f in ['contracts/static-assets.yml','contracts/microservice.yml']]; "
|
|
"print('2 sample contracts validate')",
|
|
])
|
|
|
|
|
|
def _check_environment_schema_validation() -> Tuple[Status, str]:
|
|
"""CAP-002: environment.schema.json validates the env files."""
|
|
return _check_subprocess([
|
|
"python3", "-c",
|
|
"import json, jsonschema; "
|
|
"s=json.load(open('schemas/environment.schema.json')); "
|
|
"[jsonschema.validate(json.load(open(f)), s) "
|
|
" for f in ['core/environments/dev.json']]; "
|
|
"print('env schema validates')",
|
|
])
|
|
|
|
|
|
def _check_resolver(contract_path: str) -> Tuple[Status, str]:
|
|
"""Shared helper: contract_resolver resolves a contract to a Target Stack.
|
|
|
|
Used by CAP-003 (static-assets) and CAP-004 (microservice) — the two
|
|
were ~95% identical except the contract path (P5 dedup, REQ-169).
|
|
"""
|
|
with tempfile.NamedTemporaryFile(suffix=".json", delete=False) as t:
|
|
out = t.name
|
|
try:
|
|
return _check_subprocess([
|
|
"python3", "core/contract_resolver.py",
|
|
contract_path, out,
|
|
])
|
|
finally:
|
|
try:
|
|
os.unlink(out)
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
def _check_resolver_static_assets() -> Tuple[Status, str]:
|
|
"""CAP-003: contract_resolver resolves static-assets to a Target Stack."""
|
|
return _check_resolver("contracts/static-assets.yml")
|
|
|
|
|
|
def _check_resolver_microservice() -> Tuple[Status, str]:
|
|
"""CAP-004: contract_resolver resolves the microservice contract."""
|
|
return _check_resolver("contracts/microservice.yml")
|
|
|
|
|
|
def _check_adapter_emits_terraform() -> Tuple[Status, str]:
|
|
"""CAP-005: terraform adapter compiles a resolved stack to .tf files."""
|
|
work = tempfile.mkdtemp(prefix="nova_regr_")
|
|
stack_path = os.path.join(work, "stack.json")
|
|
tf_dir = os.path.join(work, "tf")
|
|
os.makedirs(tf_dir, exist_ok=True)
|
|
rc, out, err = _run_subprocess([
|
|
"python3", "core/contract_resolver.py",
|
|
"contracts/static-assets.yml", stack_path,
|
|
])
|
|
if rc != 0:
|
|
return "Broken", f"resolver failed: {err.strip()[-200:]}"
|
|
status, detail = _check_subprocess([
|
|
"python3", "adapters/terraform/adapter.py", stack_path, tf_dir,
|
|
])
|
|
if status == "Verified":
|
|
main_tf = os.path.join(tf_dir, "main.tf")
|
|
if not os.path.isfile(main_tf) or os.path.getsize(main_tf) == 0:
|
|
return "Broken", "adapter exited 0 but main.tf missing/empty"
|
|
return status, detail
|
|
|
|
|
|
def _check_interpolation() -> Tuple[Status, str]:
|
|
"""CAP-006: contract interpolation expands ${env.*} / ${contract.*}.
|
|
|
|
P57: the contract's `module` field was dropped in favor of `id`
|
|
(short acronym) + `infrastructure` map; the interpolation check uses
|
|
`contract.id` (the surviving field)."""
|
|
return _check_subprocess([
|
|
"python3", "-c",
|
|
"import sys; sys.path.insert(0,'.'); "
|
|
"from core.contract_resolver import _expand_vars; "
|
|
"ctx={'env':{'environment':'qa','account_id':'123'},'contract':{'id':'assets'}}; "
|
|
"assert _expand_vars('nova-${env.environment}-${contract.id}', ctx)=='nova-qa-assets'; "
|
|
"print('interpolation ok')",
|
|
])
|
|
|
|
|
|
def _check_confidence_signal() -> Tuple[Status, str]:
|
|
"""CAP-007: confidence_signal.compute returns a band for a pass/fail input."""
|
|
return _check_subprocess([
|
|
"python3", "-c",
|
|
"import sys, json; sys.path.insert(0,'.'); "
|
|
"import core.confidence_signal as c; "
|
|
"inputs={'policy':[],'validation':{'schema':True,'stack_resolved':True,'tf_validated':True,'tf_planned':True},'freshness':{'age_days':0,'max_age_days':7},'source':{'submitter':'consumer','commit_sha':'x','signed':False},'history':{'prior_rollbacks':0,'prior_policy_fails':0},'nfrs':{'conformance':None}}; "
|
|
"sig=c.compute('cid','dev',inputs); "
|
|
"assert sig.band in ('pass','warn','fail'); "
|
|
"print(f'confidence band={sig.band}')",
|
|
])
|
|
|
|
|
|
def _check_outbox_writer() -> Tuple[Status, str]:
|
|
"""CAP-008: outbox_writer writes a hash-chained event to a temp file."""
|
|
work = tempfile.mkdtemp(prefix="nova_outbox_")
|
|
event_path = os.path.join(work, "event.json")
|
|
event = {
|
|
"contractId": "regression-test", "eventType": "CONFIDENCE_COMPUTED",
|
|
"ts": "2026-07-27T00:00:00Z", "environment": "dev",
|
|
"stack": "regression", "score": 0.9, "band": "pass",
|
|
"prev_event_hash": "GENESIS",
|
|
}
|
|
with open(event_path, "w") as f:
|
|
json.dump(event, f)
|
|
# The outbox writer writes to DynamoDB in prod; for the regression we
|
|
# verify the hash-chain logic (the testable core) without AWS. The
|
|
# actual DynamoDB write is a live-AWS concern, deferred to Phase 54.
|
|
return _check_subprocess([
|
|
"python3", "-c",
|
|
f"import sys, json; sys.path.insert(0,'.'); "
|
|
f"import core.outbox_writer as w; "
|
|
f"ev=json.load(open('{event_path}')); "
|
|
f"h=w._canonical_hash(ev); "
|
|
f"assert len(h)==64; "
|
|
f"assert w._canonical_hash(ev)==h; "
|
|
f"print('outbox hash chain ok')",
|
|
])
|
|
|
|
|
|
def _check_pytest_offline() -> Tuple[Status, str]:
|
|
"""CAP-009: the offline pytest suite passes (the regression baseline).
|
|
|
|
Excludes slow tests (which invoke the full pipeline) and the
|
|
regression test itself (to avoid recursion: this check runs inside
|
|
the regression run)."""
|
|
return _check_subprocess(
|
|
["python3", "-m", "pytest", "tests/", "-q", "--tb=line",
|
|
"-m", "not slow",
|
|
"--ignore=tests/test_contract_ingestor.py",
|
|
"--ignore=tests/test_verify_regression_mode.py"],
|
|
timeout=180,
|
|
)
|
|
|
|
|
|
def _check_run_ci_check_only() -> Tuple[Status, str]:
|
|
"""CAP-010: run_ci.sh reproduces the CI pipeline locally (offline).
|
|
|
|
Excluded from the regression's own pytest invocation to avoid
|
|
recursion; invoked directly here."""
|
|
return _check_subprocess(
|
|
["bash", "scripts/run_ci.sh", "--quiet"], timeout=240,
|
|
)
|
|
|
|
|
|
def _check_local_e2e_microservice() -> Tuple[Status, str]:
|
|
"""CAP-011: headline E2E runs against the local emulating tier (D-092).
|
|
|
|
The local tier emulates ECS, the DynamoDB outbox, S3 state, and the
|
|
contract-ingestor Lambda in-process. No AWS credentials required.
|
|
This is the local-tier half of the headline E2E; the live-AWS half
|
|
lands in Phase 54 (D-093)."""
|
|
return _check_subprocess(
|
|
["python3", "core/local_emulators.py", "contracts/microservice.yml"],
|
|
timeout=60,
|
|
)
|
|
|
|
|
|
def _check_local_e2e_static_assets() -> Tuple[Status, str]:
|
|
"""CAP-012: local E2E on the static-assets stack (no ECS service)."""
|
|
return _check_subprocess(
|
|
["python3", "core/local_emulators.py", "contracts/static-assets.yml"],
|
|
timeout=60,
|
|
)
|
|
|
|
|
|
def _load_aws_env() -> Dict[str, str]:
|
|
"""Load AWS credentials from .env.secrets and return an env dict
|
|
with AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY / AWS_DEFAULT_REGION set."""
|
|
env = os.environ.copy()
|
|
secrets_path = os.path.join(str(ROOT), ".env.secrets")
|
|
if os.path.isfile(secrets_path):
|
|
with open(secrets_path) as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if not line or line.startswith("#"):
|
|
continue
|
|
if "=" in line:
|
|
k, v = line.split("=", 1)
|
|
# NOVA_* only (ACDL_* fallback removed in v1.15 P5, REQ-164).
|
|
if k == "NOVA_AWS_ACCESS_KEY_ID":
|
|
env["AWS_ACCESS_KEY_ID"] = v
|
|
elif k == "NOVA_AWS_SECRET_ACCESS_KEY":
|
|
env["AWS_SECRET_ACCESS_KEY"] = v
|
|
elif k == "AWS_DEFAULT_REGION":
|
|
env["AWS_DEFAULT_REGION"] = v
|
|
return env
|
|
|
|
|
|
def _check_live_terraform_plan(contract_path: str, label: str) -> Tuple[Status, str]:
|
|
"""Shared helper: terraform init+validate+plan against live AWS for a
|
|
contract (D-093 live-AWS tier of the headline E2E).
|
|
|
|
Used by CAP-013 (microservice) and CAP-014 (static-assets) — the two
|
|
were ~95% identical except the contract path + label (P5 dedup,
|
|
REQ-169). Requires AWS credentials (NOVA_AWS_ACCESS_KEY_ID etc. in
|
|
.env.secrets; NOVA_* only — the ACDL_* fallback was removed in v1.15
|
|
P5, REQ-164). Runs in a temp dir; does NOT apply (plan only).
|
|
"""
|
|
import tempfile, os
|
|
work = tempfile.mkdtemp(prefix=f"nova_regr_live_{label}_")
|
|
stack_path = os.path.join(work, "stack.json")
|
|
tf_dir = os.path.join(work, "tf")
|
|
os.makedirs(tf_dir, exist_ok=True)
|
|
rc, out, err = _run_subprocess([
|
|
"python3", "core/contract_resolver.py",
|
|
contract_path, stack_path,
|
|
])
|
|
if rc != 0:
|
|
return "Broken", f"resolver failed: {err.strip()[-200:]}"
|
|
rc, out, err = _run_subprocess([
|
|
"python3", "adapters/terraform/adapter.py", stack_path, tf_dir,
|
|
])
|
|
if rc != 0:
|
|
return "Broken", f"adapter failed: {err.strip()[-200:]}"
|
|
env = _load_aws_env()
|
|
rc, out, err = _run_subprocess(
|
|
["terraform", "init", "-reconfigure", "-lock=false", "-input=false"],
|
|
cwd=tf_dir, timeout=120, env=env,
|
|
)
|
|
if rc != 0:
|
|
# G-111: the state bucket was torn down in v1.11 (D-096) and not
|
|
# re-provisioned. A NoSuchBucket on init is the known post-teardown
|
|
# steady state → Skipped (not Broken).
|
|
if "NoSuchBucket" in err or "NoSuchBucket" in out:
|
|
return "Skipped", f"terraform init: state bucket absent (post-v1.11-teardown, D-096) [{label}]"
|
|
return "Broken", f"terraform init failed: {err.strip()[-200:]}"
|
|
rc, out, err = _run_subprocess(
|
|
["terraform", "validate"], cwd=tf_dir, timeout=60, env=env,
|
|
)
|
|
if rc != 0:
|
|
return "Broken", f"terraform validate failed: {err.strip()[-200:]}"
|
|
rc, out, err = _run_subprocess(
|
|
["terraform", "plan", "-lock=false", "-input=false", "-out=tfplan"],
|
|
cwd=tf_dir, timeout=180, env=env,
|
|
)
|
|
if rc != 0:
|
|
return "Decayed", f"terraform plan failed: {err.strip()[-200:]}"
|
|
return "Verified", f"terraform init+validate+plan OK (live AWS, {label})"
|
|
|
|
|
|
def _check_live_terraform_plan_microservice() -> Tuple[Status, str]:
|
|
"""CAP-013: terraform init+validate+plan against live AWS for the
|
|
microservice stack (D-093 live-AWS tier of the headline E2E)."""
|
|
return _check_live_terraform_plan("contracts/microservice.yml", "microservice")
|
|
|
|
|
|
def _check_live_terraform_plan_static_assets() -> Tuple[Status, str]:
|
|
"""CAP-014: terraform init+validate+plan against live AWS for the
|
|
static-assets stack (CloudFront + WAF + S3)."""
|
|
return _check_live_terraform_plan("contracts/static-assets.yml", "static-assets")
|
|
|
|
|
|
def _check_dynamodb_outbox_table() -> Tuple[Status, str]:
|
|
"""CAP-015: DynamoDB outbox table exists + is describable (live AWS).
|
|
|
|
G-111: the live AWS resources were torn down in v1.11 (D-096) and not
|
|
re-provisioned (v1.15 P4 was plan-only). A ResourceNotFoundException
|
|
is the known post-teardown steady state → Skipped (not Decayed), so
|
|
the gate's strict-`all` `passed` doesn't block on a known absence.
|
|
Re-provisioning is a future feature milestone, not an NFR regression.
|
|
"""
|
|
import boto3
|
|
from botocore.exceptions import ClientError
|
|
env = _load_aws_env()
|
|
try:
|
|
dyn = boto3.client("dynamodb", region_name=env.get("AWS_DEFAULT_REGION", "us-east-1"),
|
|
aws_access_key_id=env.get("AWS_ACCESS_KEY_ID"),
|
|
aws_secret_access_key=env.get("AWS_SECRET_ACCESS_KEY"))
|
|
r = dyn.describe_table(TableName="nova-outbox")
|
|
count = r["Table"].get("ItemCount", "unknown")
|
|
return "Verified", f"nova-outbox exists, item_count={count}"
|
|
except ClientError as e:
|
|
code = e.response.get("Error", {}).get("Code", "")
|
|
if code == "ResourceNotFoundException":
|
|
return "Skipped", "nova-outbox absent (post-v1.11-teardown steady state, D-096)"
|
|
return "Decayed", f"describe_table failed: {type(e).__name__}: {str(e)[:150]}"
|
|
except Exception as e:
|
|
return "Decayed", f"describe_table failed: {type(e).__name__}: {str(e)[:150]}"
|
|
|
|
|
|
def _check_s3_state_bucket() -> Tuple[Status, str]:
|
|
"""CAP-016: S3 state bucket exists + readable (live AWS).
|
|
|
|
G-111: the live state bucket was torn down in v1.11 (D-096) and not
|
|
re-provisioned. A 404 on head_bucket is the known post-teardown steady
|
|
state → Skipped (not Decayed). Re-provisioning is a future feature.
|
|
"""
|
|
import boto3
|
|
from botocore.exceptions import ClientError
|
|
env = _load_aws_env()
|
|
account_id = _envhelper.get_env("AWS_ACCOUNT_ID", "581513795199")
|
|
state_bucket = f"nova-tfstate-{account_id}-us-east-1"
|
|
try:
|
|
s3 = boto3.client("s3", region_name=env.get("AWS_DEFAULT_REGION", "us-east-1"),
|
|
aws_access_key_id=env.get("AWS_ACCESS_KEY_ID"),
|
|
aws_secret_access_key=env.get("AWS_SECRET_ACCESS_KEY"))
|
|
s3.head_bucket(Bucket=state_bucket)
|
|
r = s3.list_objects_v2(Bucket=state_bucket, MaxKeys=5)
|
|
keys = [o["Key"] for o in r.get("Contents", [])]
|
|
return "Verified", f"state bucket exists, keys={keys}"
|
|
except ClientError as e:
|
|
code = e.response.get("Error", {}).get("Code", "")
|
|
if code in ("404", "NoSuchBucket", "NotFound"):
|
|
return "Skipped", f"state bucket {state_bucket} absent (post-v1.11-teardown, D-096)"
|
|
return "Decayed", f"head_bucket failed: {type(e).__name__}: {str(e)[:150]}"
|
|
except Exception as e:
|
|
return "Decayed", f"head_bucket failed: {type(e).__name__}: {str(e)[:150]}"
|
|
|
|
|
|
def _check_lifecycle_module_terraform(module: str) -> Tuple[Status, str]:
|
|
"""Helper: verify an L1 module's terraform dir exists with the required
|
|
files + its example contracts resolve + terraform fmt syntax check
|
|
passes. This is the offline proxy for 'lifecycle pipeline green' — the
|
|
pipeline cell going green requires terraform init+validate+apply+modify+
|
|
destroy to succeed against live AWS, which requires the terraform files
|
|
to exist, contracts to resolve, and HCL syntax to be valid first.
|
|
|
|
We run `terraform fmt -check` (fast, no init required) as a syntax probe.
|
|
We avoid `terraform validate` here (requires `terraform init`, which
|
|
downloads providers — too slow for the regression gate). Full
|
|
`terraform validate` is run by the lifecycle pipeline itself. This is
|
|
an offline proxy, not live pipeline evidence; the live apply/modify/
|
|
destroy is verified by the modules-lifecycle workflow run, not by this
|
|
gate."""
|
|
tf_dir = ROOT / "modules" / "l1" / module / "terraform"
|
|
if not tf_dir.is_dir():
|
|
return "Broken", f"modules/l1/{module}/terraform/ does not exist"
|
|
required = ["versions.tf", "variables.tf", "main.tf", "outputs.tf"]
|
|
missing = [f for f in required if not (tf_dir / f).is_file()]
|
|
if missing:
|
|
return "Broken", f"missing terraform files: {missing}"
|
|
# locals.tf is only required when the module references local.* values
|
|
# (CAP-017 fix, v1.12). Single-resource modules may legitimately omit it.
|
|
tf_text = "".join((tf_dir / f).read_text() for f in ["variables.tf", "main.tf", "outputs.tf"] if (tf_dir / f).is_file())
|
|
if "local." in tf_text and not (tf_dir / "locals.tf").is_file():
|
|
return "Broken", "missing terraform files: ['locals.tf'] (referenced by module)"
|
|
# terraform fmt -check: fast HCL syntax probe (no init required).
|
|
rc, out, err = _run_subprocess(
|
|
["terraform", "fmt", "-check", "-diff", str(tf_dir)], timeout=30)
|
|
if rc != 0:
|
|
return "Broken", f"terraform fmt -check failed: {err.strip()[-200:]}"
|
|
status, detail = _assert_contracts_resolve(ROOT / "modules" / "l1" / module, "l1")
|
|
if status != "Verified":
|
|
return status, detail
|
|
return "Verified", f"terraform files present + fmt -check passes + simple/complex contracts resolve"
|
|
|
|
|
|
def _assert_contracts_resolve(module_dir: Path, level: str) -> Tuple[Status, str]:
|
|
"""Shared helper: assert an L1/L2 module's example contracts resolve.
|
|
|
|
Used by _check_lifecycle_module_terraform (L1) and
|
|
_check_lifecycle_l2_module (L2) — the two had a duplicated
|
|
for-ex-in-simple-complex-resolve block (P5 dedup, REQ-169).
|
|
``level`` is "l1" or "l2" (selects the examples dir parent).
|
|
"""
|
|
for ex in ["simple", "complex"]:
|
|
contract = module_dir / "examples" / f"{ex}.yml"
|
|
if not contract.is_file():
|
|
return "Broken", f"{module_dir.relative_to(ROOT)}/examples/{ex}.yml missing"
|
|
rc, out, err = _run_subprocess([
|
|
"python3", "core/contract_resolver.py", str(contract), "/dev/null",
|
|
], timeout=30)
|
|
if rc != 0:
|
|
return "Broken", f"{ex}.yml resolver failed: {err.strip()[-200:]}"
|
|
return "Verified", ""
|
|
|
|
|
|
def _check_lifecycle_l2_module(module: str) -> Tuple[Status, str]:
|
|
"""Helper: verify an L2 module's composition resolves + its example
|
|
contracts resolve. Offline proxy for 'L2 lifecycle pipeline green'.
|
|
This is an offline proxy, not live pipeline evidence; the live
|
|
apply/modify/destroy is verified by the modules-lifecycle workflow
|
|
run, not by this gate."""
|
|
module_dir = ROOT / "modules" / "l2" / module
|
|
status, detail = _assert_contracts_resolve(module_dir, "l2")
|
|
if status != "Verified":
|
|
return status, detail
|
|
return "Verified", "L2 composition resolves (simple + complex contracts; offline proxy)"
|
|
|
|
|
|
def _check_cap_017_dynamodb() -> Tuple[Status, str]:
|
|
"""CAP-017: DynamoDB nova-contracts table. Evidence = L1 rds module
|
|
lifecycle pipeline green (terraform validate + contracts resolve).
|
|
The DynamoDB table is created via the microservice stack (L2 lifecycle).
|
|
"""
|
|
return _check_lifecycle_module_terraform("rds")
|
|
|
|
|
|
def _check_cap_018_lambda() -> Tuple[Status, str]:
|
|
"""CAP-018: Lambda contract-ingestor. Evidence = local Lambda stub
|
|
(CAP-011) + L1 lifecycle pipeline green for the platform terraform.
|
|
The stub requires an outbox arg (CAP-018 fix, v1.12)."""
|
|
rc, out, err = _run_subprocess([
|
|
"python3", "-c",
|
|
"from core.local_emulators import LocalLambdaStub, FlatFileOutbox; "
|
|
"import tempfile; "
|
|
"stub = LocalLambdaStub(outbox=FlatFileOutbox(tempfile.mkdtemp(prefix='nova_stub_'))); "
|
|
"print('LocalLambdaStub instantiates OK')",
|
|
])
|
|
if rc != 0:
|
|
return "Broken", f"LocalLambdaStub check failed: {err.strip()[-200:]}"
|
|
return "Verified", "LocalLambdaStub instantiates (local tier evidence)"
|
|
|
|
|
|
def _check_cap_019_ecs_service() -> Tuple[Status, str]:
|
|
"""CAP-019: ECS cluster + service. Evidence = L2 microservice lifecycle
|
|
pipeline green (composition resolves + apply/modify/destroy)."""
|
|
return _check_lifecycle_l2_module("microservice")
|
|
|
|
|
|
def _check_cap_020_cloudfront_waf() -> Tuple[Status, str]:
|
|
"""CAP-020: CloudFront + WAF production static-assets stack.
|
|
Evidence = L2 static-assets lifecycle pipeline green."""
|
|
return _check_lifecycle_l2_module("static-assets")
|
|
|
|
|
|
def _check_cap_021_uptime() -> Tuple[Status, str]:
|
|
"""CAP-021: uptime-kuma monitoring primitive. Evidence = L1 uptime
|
|
module lifecycle pipeline green."""
|
|
return _check_lifecycle_module_terraform("uptime")
|
|
|
|
|
|
def _check_cap_022_oidc_role() -> Tuple[Status, str]:
|
|
"""CAP-022: OIDC role for act_runner. Evidence = L1 iam-role module
|
|
lifecycle pipeline green."""
|
|
return _check_lifecycle_module_terraform("iam-role")
|
|
|
|
|
|
def _check_cap_023_metrics_collector() -> Tuple[Status, str]:
|
|
"""CAP-023: metrics collector runs and emits the expected schema (v1.17).
|
|
|
|
Verifies that core/metrics/collector.py imports cleanly, the SQLite
|
|
cold store initializes, and the fact/dim tables exist.
|
|
"""
|
|
import importlib
|
|
try:
|
|
mod = importlib.import_module("core.metrics.collector")
|
|
mod._init_store()
|
|
import sqlite3, os
|
|
db_path = mod._STORE_PATH
|
|
if not os.path.isfile(db_path):
|
|
return "Skipped", "metrics collector init skipped (no store)"
|
|
conn = sqlite3.connect(db_path)
|
|
tables = [r[0] for r in conn.execute("SELECT name FROM sqlite_master WHERE type='table'").fetchall()]
|
|
conn.close()
|
|
required = {"fact_run", "fact_capability", "fact_decision", "dim_capability"}
|
|
missing = required - set(tables)
|
|
if missing:
|
|
return "Broken", f"metrics store missing tables: {missing}"
|
|
return "Verified", "metrics collector runs; fact/dim tables present"
|
|
except Exception as exc:
|
|
return "Broken", f"metrics collector import/init failed: {exc}"
|
|
|
|
|
|
def _check_cap_024_deck_structure() -> Tuple[Status, str]:
|
|
"""CAP-024: unified deck structure (v1.17 + v1.21 refinement).
|
|
|
|
Verifies the unified deck source of truth exists, has 18 main slides
|
|
(## Slide N) + 1 appendix, has the recap+ask closing, and per-slide
|
|
benefit callouts. v1.21 renamed the deck + restructured to a 4-beat arc.
|
|
"""
|
|
import os
|
|
deck_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
|
|
"docs", "presentations", "nova-autonomous-cloud-delivery-marp.md")
|
|
if not os.path.isfile(deck_path):
|
|
return "Skipped", "unified deck not found"
|
|
with open(deck_path) as f:
|
|
content = f.read()
|
|
slide_count = content.count("## Slide ")
|
|
if slide_count < 18 or slide_count > 20:
|
|
return "Broken", f"deck has {slide_count} main slides (expected 18-20)"
|
|
has_recap = "Recap + Ask" in content
|
|
benefit_count = content.count("Benefit:") + content.count('class="benefit"')
|
|
has_benefit = benefit_count >= 10
|
|
if not (has_recap and has_benefit):
|
|
missing = []
|
|
if not has_recap: missing.append("recap+ask")
|
|
if not has_benefit: missing.append("per-slide benefit callouts")
|
|
return "Broken", f"deck missing: {missing}"
|
|
return "Verified", f"deck has {slide_count} slides, recap+ask present, per-slide benefits present"
|
|
|
|
|
|
def _check_cap_025_live_pilot_apply() -> Tuple[Status, str]:
|
|
"""CAP-025 (REQ-316): live-pilot-apply pipeline readiness — structural
|
|
check that the pilot-apply end-to-end pipeline is wired (NOT a live
|
|
apply; the live apply lands in P4).
|
|
|
|
The pilot-apply round-trip is:
|
|
contract resolve -> adapter compile -> terraform plan -> policy scan
|
|
-> confidence signal -> terraform apply -> outbox write
|
|
|
|
For P3 this is a LOCAL-tier structural-readiness check: the scripts
|
|
exist + are wired, the core pipeline modules import, the pilot env is
|
|
bound to a real account (D-203), the DynamoDB L1 primitive is
|
|
registered (REQ-322), the pilot policies are authored (REQ-315/320),
|
|
and the outcome-backfill module exists (REQ-317). The live apply
|
|
against AWS is P4's live-verify (D-093 / G-111 steady state aside).
|
|
"""
|
|
import json
|
|
|
|
# 1. scripts/run_platform.sh exists + contains the pipeline step markers.
|
|
run_platform = ROOT / "scripts" / "run_platform.sh"
|
|
if not run_platform.is_file():
|
|
return "Broken", "scripts/run_platform.sh missing (pilot-apply pipeline driver)"
|
|
script_text = run_platform.read_text()
|
|
# Step markers mirrored from the script's own comments + Step headers.
|
|
required_markers = [
|
|
"resolve contract", # Step 2: contract_resolver
|
|
"adapter compiles stack", # Step 3: terraform adapter
|
|
"terraform init", # Step 4: terraform plan
|
|
"terraform plan", # Step 4: terraform plan
|
|
"policy scan", # Step 5: runtime policy scan (Wiz/Checkov)
|
|
"confidence signal", # Step 7: confidence_signal compute
|
|
"terraform apply", # Step 5: terraform apply (--apply mode)
|
|
"outbox", # outbox write (Step 8)
|
|
]
|
|
missing_markers = [m for m in required_markers if m not in script_text]
|
|
if missing_markers:
|
|
return "Broken", f"run_platform.sh missing step markers: {missing_markers}"
|
|
|
|
# 2. core pipeline modules importable.
|
|
for mod_name in (
|
|
"core.contract_resolver",
|
|
"adapters.terraform.adapter",
|
|
"core.confidence_signal",
|
|
"core.outbox_writer",
|
|
):
|
|
try:
|
|
importlib.import_module(mod_name)
|
|
except Exception as exc: # noqa: BLE001
|
|
return "Broken", f"pipeline module not importable: {mod_name} ({type(exc).__name__}: {exc})"[:200]
|
|
|
|
# 3. dev env bound to the real pilot account (D-203).
|
|
dev_env_path = ROOT / "core" / "environments" / "dev.json"
|
|
if not dev_env_path.is_file():
|
|
return "Broken", "core/environments/dev.json missing"
|
|
try:
|
|
dev_env = json.loads(dev_env_path.read_text())
|
|
except Exception as exc: # noqa: BLE001
|
|
return "Broken", f"dev.json parse failed: {exc}"[:200]
|
|
account_id = dev_env.get("account_id")
|
|
if account_id != "581513795199":
|
|
return "Broken", f"dev env not bound to real account (D-203): account_id={account_id!r}"
|
|
|
|
# 4. DynamoDB L1 primitive registered (REQ-322).
|
|
registry_path = ROOT / "modules" / "registry.json"
|
|
if not registry_path.is_file():
|
|
return "Broken", "modules/registry.json missing"
|
|
try:
|
|
registry = json.loads(registry_path.read_text())
|
|
except Exception as exc: # noqa: BLE001
|
|
return "Broken", f"registry.json parse failed: {exc}"[:200]
|
|
if "dynamodb" not in registry:
|
|
return "Broken", "dynamodb L1 primitive not registered (REQ-322)"
|
|
|
|
# 5. pilot policies authored (REQ-315/320).
|
|
pilot_policies = [
|
|
ROOT / "adapters" / "kyverno-json" / "policies" / "pilot-readiness" / "no-placeholder-account.json",
|
|
ROOT / "adapters" / "kyverno-json" / "policies" / "settlement-finality" / "all-matches-committed.json",
|
|
]
|
|
missing_policies = [str(p.relative_to(ROOT)) for p in pilot_policies if not p.is_file()]
|
|
if missing_policies:
|
|
return "Broken", f"pilot policies not authored (REQ-315/320): {missing_policies}"
|
|
|
|
# 6. outcome-backfill module exists (REQ-317).
|
|
outcome_backfill = ROOT / "core" / "metrics" / "outcome_backfill.py"
|
|
if not outcome_backfill.is_file():
|
|
return "Broken", "outcome backfill not implemented (REQ-317)"
|
|
|
|
return ("Verified",
|
|
"pilot-apply pipeline structurally ready "
|
|
"(contract->adapter->plan->policy->confidence->apply->outbox)")
|
|
|
|
|
|
# Registry: ordered, each entry is (capability_id, name, tier, check_fn).
|
|
# Phase 52 seeds this with 10 local-tier checks; Phase 54 expands it to
|
|
# cover every v1.1->v1.8 advertised capability and adds the live-AWS tier
|
|
# for the headline E2E.
|
|
CAPABILITY_REGISTRY: List[Tuple[str, str, str, Callable[[], Tuple[Status, str]]]] = [
|
|
("CAP-001", "contract.schema.json validates sample contracts", "local",
|
|
_check_contract_schema_validation),
|
|
("CAP-002", "environment.schema.json validates env files", "local",
|
|
_check_environment_schema_validation),
|
|
("CAP-003", "contract_resolver resolves static-assets", "local",
|
|
_check_resolver_static_assets),
|
|
("CAP-004", "contract_resolver resolves microservice", "local",
|
|
_check_resolver_microservice),
|
|
("CAP-005", "terraform adapter emits .tf files", "local",
|
|
_check_adapter_emits_terraform),
|
|
("CAP-006", "contract interpolation expands env/contract tokens", "local",
|
|
_check_interpolation),
|
|
("CAP-007", "confidence_signal.compute returns a band", "local",
|
|
_check_confidence_signal),
|
|
("CAP-008", "outbox_writer builds a hash-chained item", "local",
|
|
_check_outbox_writer),
|
|
("CAP-009", "offline pytest suite passes", "local",
|
|
_check_pytest_offline),
|
|
("CAP-010", "run_ci.sh reproduces CI pipeline locally", "local",
|
|
_check_run_ci_check_only),
|
|
("CAP-011", "headline E2E runs against the local emulating tier (microservice)", "local",
|
|
_check_local_e2e_microservice),
|
|
("CAP-012", "local E2E on the static-assets stack (no ECS)", "local",
|
|
_check_local_e2e_static_assets),
|
|
("CAP-013", "terraform init+validate+plan live AWS (microservice)", "live-aws",
|
|
_check_live_terraform_plan_microservice),
|
|
("CAP-014", "terraform init+validate+plan live AWS (static-assets)", "live-aws",
|
|
_check_live_terraform_plan_static_assets),
|
|
("CAP-015", "DynamoDB outbox table exists (live AWS)", "live-aws",
|
|
_check_dynamodb_outbox_table),
|
|
("CAP-016", "S3 state bucket exists + readable (live AWS)", "live-aws",
|
|
_check_s3_state_bucket),
|
|
("CAP-017", "DynamoDB nova-contracts table (lifecycle pipeline evidence)", "lifecycle-pipeline",
|
|
_check_cap_017_dynamodb),
|
|
("CAP-018", "Lambda contract-ingestor (local stub + lifecycle evidence)", "lifecycle-pipeline",
|
|
_check_cap_018_lambda),
|
|
("CAP-019", "ECS cluster + service (L2 microservice lifecycle evidence)", "lifecycle-pipeline",
|
|
_check_cap_019_ecs_service),
|
|
("CAP-020", "CloudFront + WAF (L2 static-assets lifecycle evidence)", "lifecycle-pipeline",
|
|
_check_cap_020_cloudfront_waf),
|
|
("CAP-021", "uptime-kuma (L1 uptime lifecycle evidence)", "lifecycle-pipeline",
|
|
_check_cap_021_uptime),
|
|
("CAP-022", "OIDC role (L1 iam-role lifecycle evidence)", "lifecycle-pipeline",
|
|
_check_cap_022_oidc_role),
|
|
("CAP-023", "metrics collector runs + emits expected schema", "local",
|
|
_check_cap_023_metrics_collector),
|
|
("CAP-024", "unified deck structure (slide count, x3, per-slide benefits)", "local",
|
|
_check_cap_024_deck_structure),
|
|
("CAP-025", "live-pilot-apply pipeline readiness (contract->apply->outbox)", "local",
|
|
_check_cap_025_live_pilot_apply),
|
|
]
|
|
|
|
|
|
def run_regression(milestone: str = "v1.10", phase: int = 52,
|
|
registry: Optional[List] = None) -> RegressionReport:
|
|
"""Run every capability check in the registry; return a RegressionReport."""
|
|
reg = registry if registry is not None else CAPABILITY_REGISTRY
|
|
run_id = f"regr-{int(time.time())}"
|
|
run_at = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
|
report = RegressionReport(run_id=run_id, run_at_utc=run_at,
|
|
milestone=milestone, phase=phase)
|
|
for cap_id, name, tier, fn in reg:
|
|
t0 = time.monotonic()
|
|
try:
|
|
status, detail = fn()
|
|
except Exception as e: # noqa: BLE001
|
|
status, detail = "Broken", f"check raised: {type(e).__name__}: {e}"[:300]
|
|
dur = int((time.monotonic() - t0) * 1000)
|
|
report.results.append(CapabilityResult(
|
|
capability_id=cap_id, name=name, status=status,
|
|
detail=detail, tier=tier, duration_ms=dur,
|
|
))
|
|
return report
|
|
|
|
|
|
def write_report(report: RegressionReport,
|
|
md_path: Optional[Path] = None,
|
|
json_path: Optional[Path] = None) -> Tuple[Path, Path]:
|
|
"""Write the report to .ciagent/REGRESSION_REPORT.md + .json."""
|
|
md_path = md_path or (CIAgent / "REGRESSION_REPORT.md")
|
|
json_path = json_path or (CIAgent / "REGRESSION_REPORT.json")
|
|
json_path.write_text(json.dumps(report.to_dict(), indent=2))
|
|
lines = [
|
|
f"# Regression Report — {report.milestone} Phase {report.phase}",
|
|
"",
|
|
f"- **Run ID:** `{report.run_id}`",
|
|
f"- **Run at (UTC):** {report.run_at_utc}",
|
|
f"- **Summary:** {report.summary}",
|
|
f"- **Passed (milestone gate):** {report.passed}",
|
|
"",
|
|
"| Capability | Name | Tier | Status | Duration (ms) | Detail |",
|
|
"|-----------|------|------|--------|--------------|--------|",
|
|
]
|
|
for r in report.results:
|
|
lines.append(
|
|
f"| {r.capability_id} | {r.name} | {r.tier} | "
|
|
f"**{r.status}** | {r.duration_ms} | {r.detail[:160]} |"
|
|
)
|
|
md_path.write_text("\n".join(lines) + "\n")
|
|
return md_path, json_path
|
|
|
|
|
|
def main() -> int:
|
|
"""P13 (REQ-177): re-export from core.regression_verify_cli."""
|
|
from core.regression_verify_cli import main as _cli_main
|
|
return _cli_main()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main()) |