diff --git a/.ciagent/ARCHITECTURE.md b/.ciagent/ARCHITECTURE.md index 86368c3..b7cd604 100644 --- a/.ciagent/ARCHITECTURE.md +++ b/.ciagent/ARCHITECTURE.md @@ -879,3 +879,67 @@ config entry in `config.json` (`strategic_direction_file: ".ciagent/NORTH_STAR.md"`) that the run workflow reads at SPECIFY. This ensures the strategic direction survives across milestones without being overwritten by status updates. + +### §12.7 — Policy Engine Registry (v1.25, REQ-291) + +The policy-engine abstraction is first-class: a swappable `PolicyEngine` +protocol so the engine may change without touching the confidence +signal, the pipeline, or the `PolicyCheckResult` schema. This is the +**swap boundary** that keeps the platform's compliance posture +replaceable (Strategic Objective #2 — provable trust via a replaceable +substrate, not a vendor lock-in). + +``` + contract.yml ─┐ ┌─→ list[PolicyCheckResult] ─┐ + stack IR ─────┼─→ PolicyEngine.evaluate ├─→ list[PolicyCheckResult] ─┼─→ confidence_signal + plan JSON ────┤ (protocol) └─→ list[PolicyCheckResult] ─┘ (engine-agnostic, + PCR list ─────┘ unchanged) + │ + ▼ + ┌─ KyvernoJsonEngine (shells to `kj scan`; engine: "kyverno") + └─ OpaEngine (future — same protocol; engine: "opa") + + checkov/wiz ──→ raw findings ──→ (merged PCR list is the meta-policy payload) +``` + +**The protocol (`core/policy_engine.py`):** +```python +class PolicyEngine(Protocol): + @property + def name(self) -> str: ... + def is_configured(self) -> bool: ... + def evaluate(self, payload, policy_dir: Path, contract_id: str) -> list[dict]: ... +``` + +**The registry** reads `config.json.policy.engine` (default +`"kyverno-json"`) and returns the active engine. A `NullEngine` is the +fallback when the `policy` key is absent (emits `SKIPPED` PCRs — +backward compatibility for tests that don't set the key). The +confidence signal is **untouched** — it already consumes +`list[PolicyCheckResult]` engine-agnostically (§12.6). v1.25 only +changes *who produces* the PCR list, not *what* the list is. + +**Engine enum reuse (D-116):** kyverno-json PCR records carry +`engine: "kyverno"` (no new enum value). The `engine` field records the +policy-engine *family*, not the specific binary. The K8s Kyverno adapter +and the kyverno-json engine are distinguished by `ruleId` prefix +(`KYVERNO_` vs `KJ_`) and `evidence` payload shape (`namespace`/`kind` +vs `assertion`/`jmespath`). + +**Defense-in-depth (D-119):** the declarative meta-policy +`block-on-any-critical` (asserts no PCR has `severity: critical` + +`result: fail`) is the *source of truth* for "critical = block". The +`confidence_signal.py` `PENALTY["critical"]: None` hard-override stays +as the *imperative* safety net — the meta-policy runs *before* the +confidence signal (produces PCRs that flow in), the hard-override runs +*inside* it (the last gate). Removing the hard-override would make the +"critical = block" guarantee depend on a single policy file — a +regression in provable trust. + +**Graceful degradation (D-120):** `KyvernoJsonEngine.is_configured()` +returns false when `which kj` is absent → `evaluate()` returns a single +`SKIPPED` PCR (`ruleId: "KJ_ENGINE_NOT_CONFIGURED"`). The platform +functions without the binary (the "platform functions without AI / +deterministic scripts" tenet holds — kyverno-json is deterministic, not +AI; the `is_configured()` guard ensures the platform runs even when the +binary is not installed). diff --git a/.ciagent/CHECKPOINT.json b/.ciagent/CHECKPOINT.json index 720e058..cd3bd08 100644 --- a/.ciagent/CHECKPOINT.json +++ b/.ciagent/CHECKPOINT.json @@ -1,32 +1,22 @@ { "phase": 4, "stage": "complete", - "milestone": "v1.24", - "phase_role": "final", + "milestone": "v1.25", + "phase_role": "execution", "attempts": 0, - "updated_at": "2026-08-12T02:50:00Z", + "updated_at": "2026-08-12T17:45:00Z", "project": "acdl", - "milestone_complete": true, - "tag": "v1.23.4", - "tag_line": "v1.23.x", - "requirements": ["REQ-276","REQ-277","REQ-278","REQ-279","REQ-280","REQ-281","REQ-282","REQ-283","REQ-284","REQ-285","REQ-286","REQ-287","REQ-288","REQ-289","REQ-290"], + "milestone_complete": false, + "tag_line": "v1.24.x", + "tag": "v1.24.4", + "next_tag": "v1.24.5", "release": { "forge": "gitea", "releases_created": true, - "release_ids": { - "v1.23.0": 635, - "v1.23.1": 636, - "v1.23.2": 637, - "v1.23.3": 638, - "v1.23.4": 639 - }, - "milestone_release_id": 639, - "milestone_release_tag": "v1.23.4" + "release_ids": {"v1.24.0": 640, "v1.24.1": 641, "v1.24.2": 642, "v1.24.3": 643, "v1.24.4": 644}, + "phase_release_id": 644 }, - "tests": { - "total": 287, - "passed": 287, - "failed": 0 - }, - "notes": "v1.24 milestone complete. Tag v1.23.4 (milestone release, id 639). 15 requirements complete (REQ-276..290). 4 phases. 287 tests pass. Consumer guide accuracy fixes + env-transition detect-and-destroy (no orphan path). Next milestone starts fresh." + "requirements": ["REQ-291", "REQ-292", "REQ-293", "REQ-294", "REQ-295", "REQ-296", "REQ-297", "REQ-298", "REQ-299", "REQ-300", "REQ-301", "REQ-302", "REQ-303", "REQ-304", "REQ-305", "REQ-306", "REQ-307", "REQ-308", "REQ-309"], + "tests": {"total": 170, "passed": 170, "skipped": 23, "failed": 0, "preexisting_flaky": "test_metrics_emitters.py::test_attestation_event_emission (fails on main, unrelated to v1.25)"}, + "notes": "v1.25 P4 (regression-gate+docs) complete. Tag v1.24.4 (gitea release id 644). 4 requirements (REQ-304..307). 3 regression policies + adapter/STANDARDS/METRICS/schemas docs. Phase 04 branch deleted. All 19 requirements now implemented. Next: P5 final review+audit+milestone ship." } \ No newline at end of file diff --git a/.ciagent/CLARIFY.md b/.ciagent/CLARIFY.md index 1b5ebe6..15034f4 100644 --- a/.ciagent/CLARIFY.md +++ b/.ciagent/CLARIFY.md @@ -1,109 +1,164 @@ -# CLARIFY — v1.24 Consumer Guide Accuracy & Env-Promotion Lifecycle Enforcement +# CLARIFY — v1.25 kyverno-json Unified Policy Engine > **Autonomy:** full. Ambiguities are auto-resolved with assumption logging -> per `config.json autonomy.level: "full"`. No human escalation. +> per `config.json autonomy.level: "full"` and +> `autonomy.decision_confidence_threshold: 0.6`. No human escalation. ## Ambiguities Identified -### A1 — Step 8 "change environment" vs Per-env section "no field editing" +### A1 — kyverno-json install path (pip / go install / pinned binary release) -**Ambiguity:** The consumer guide contains two mutually-exclusive promotion -models. Step 8 (line 290) says "Change `environment` in your contract." The -"Per-environment deployment" section (line 398) says "you do not edit the -`environment:` field… Promotion = running the matching job." The test -`test_consumer_guide_states_no_field_editing` asserts the no-editing model. +**Ambiguity:** kyverno-json is a Go project, not a Python package. Three +install paths exist: (a) `pip install` — not possible (no PyPI package); +(b) `go install github.com/kyverno/kyverno-json/cmd/kj@latest` — requires +Go toolchain in the CI image; (c) download a pinned binary release from +GitHub releases — no Go toolchain needed, but release artifacts are +platform-specific and must be checksummed. -**User directive (binding):** Both shapes are supported. Shape A (edit -environment in-place) is valid AND must trigger a destroy of the prior env. -Shape B (per-env caller workflows) is the alternative. The test must be -updated to assert both shapes. +**Resolution (auto, confidence 0.85):** `go install` (option b). A +`scripts/install-kyverno-json.sh` helper runs +`go install github.com/kyverno/kyverno-json/cmd/kj@latest` and prints +`kj version`. The CI image (`.github/workflows/ci.yml` + +`.gitea/workflows/ci.yml`) installs Go + kj when +`config.json.policy.engine == "kyverno-json"`; the install is cached via +the existing Go module cache. Rationale: `go install` is the upstream- +blessed path, tracks the latest stable release, avoids per-platform +binary management, and the project already accepts Go-based tooling +(checkov pulls Go-built transitive deps via pip). When `which kj` is +absent, `KyvernoJsonEngine.is_configured()` returns false → `SKIPPED` +PCR (mirrors the Wiz adapter pattern) — the platform functions without +the binary. Captured in REQ-293, REQ-294. Decision ID: D-115. -**Resolution (auto, confidence 0.95):** Adopt the user's directive. Step 8 -is rewritten to document Shape A with destroy-then-rebuild semantics. The -per-env section is preserved as Shape B with a lead sentence distinguishing -it. The test is renamed and a new test asserts the destroy semantics. This -is already captured in REQ-279, REQ-280, REQ-290. +### A2 — `engine` enum value: new `"kyverno-json"` vs reuse `"kyverno"` -### A2 — Prior-env source of truth: DynamoDB vs state-bucket scan vs SSM +**Ambiguity:** `schemas/policy_check_result.schema.json` already lists +`engine: ["checkov", "kyverno", "opa", "wiz"]`. kyverno-json is a +distinct runtime from the K8s Kyverno admission controller, but both +are "Kyverno." Two options: (a) add a new `"kyverno-json"` enum value +— requires schema change + checkov/wiz adapter test regression check; +(b) reuse `"kyverno"` and distinguish by `ruleId` prefix. -**Ambiguity:** Three options for detecting the prior environment: (a) query -the `nova-contracts` DynamoDB table, (b) scan the state bucket for other env -prefixes, (c) record last-applied env in an SSM parameter. +**Resolution (auto, confidence 0.80):** Reuse `"kyverno"` (option b). +Adding `"kyverno-json"` would force a schema change + a test sweep for +no semantic gain — the `engine` field records the policy engine family, +not the specific binary. kyverno-json PCR records carry `engine: +"kyverno"` and `ruleId` prefixed `KJ_` (e.g. +`KJ_REQUIRE_TAGGING_STANDARD`), while the K8s adapter uses `KYVERNO_` +prefixes (e.g. `KYVERNO_INACTIVE_TF_STACK`). The two are distinguishable +in audit/telemetry by `ruleId` prefix and `evidence` payload shape (the +K8s adapter's evidence has `namespace`/`kind`; kyverno-json's has +`assertion`/`jmespath`). No schema change. Captured in REQ-293. +Decision ID: D-116. -**Resolution (auto, confidence 0.85):** DynamoDB `nova-contracts` table -(user-selected). It already exists, is written by the contract ingestor -Lambda (`core/lambda/contract_ingestor.py:160-170`), and has the right shape -(PK `consumerRepo`, SK `contractId#submittedAt`, `environment` attribute). -A new `#LAST_APPLIED` SK suffix is added for the record-applied-env step -(REQ-283). This avoids coupling the platform to a specific state-bucket -layout (which differs across envs/accounts) and avoids a new SSM dependency. +### A3 — Do checkov/wiz adapters change their signatures to feed kyverno-json? -**Assumption:** The `nova-contracts` table is accessible from the deploy -role via the same ABAC scoping that the contract ingestor uses. If the -table is not accessible (e.g., local/CI mode without DynamoDB), the detect -step logs a warning and returns `None` (conservative — no prior env -assumed). This is documented in REQ-282. +**Ambiguity:** The unified-orchestrator model places kyverno-json "on +top of" checkov/wiz. Two interpretations: (a) checkov/wiz now emit a +"raw findings" intermediate (not PCR) that kyverno-json meta-policies +consume — requires changing `adapt() -> list[PolicyCheckResult]` to +`adapt() -> list[RawFinding]`; (b) checkov/wiz keep emitting PCRs as +today, and the meta-policies in `adapters/kyverno-json/policies/meta/` +consume the **merged** PCR list as their payload. -### A3 — Cross-account destroy +**Resolution (auto, confidence 0.90):** Option (b). The existing +`adapt() -> list[PolicyCheckResult]` signatures are unchanged. The +meta-policies consume the merged PCR list (checkov + wiz + kyverno-json +plan-JSON policies) as their input payload. This preserves the +`PolicyCheckResult` schema as the single inter-adapter contract +(ARCHITECTURE.md §12.6), avoids a new "RawFinding" type, and means +the existing checkov/wiz adapter tests pass unchanged. The meta-policy +`block-on-any-critical.json` iterates the merged list; the +`tagging-rules-agree.json` meta-policy cross-checks the Checkov +`NOVA_TAG_NAMING` result against the kyverno-json +`KJ_REQUIRE_TAGGING_STANDARD` result by `resourceRef`. Captured in +REQ-303, D-117. Decision ID: D-117. -**Ambiguity:** If the prior env (e.g., dev) and new env (e.g., qa) are in -different AWS accounts, the destroy step needs the prior env's role -credentials. The current scaffold uses one account. +### A4 — `NOVA_TAG_NAMING` Checkov rule: rewrite as kyverno-json policy, keep, or both? -**Resolution (auto, confidence 0.80):** v1.24 targets the same-account -case. Cross-account destroy is explicitly out of scope (documented in the -Out of Scope section). The `run_platform.sh` Step 0b notes this limitation. -A future milestone handles cross-account destroy via a pre-step that -assumes the prior env's role. This is the pragmatic path — the scaffold -(`core/environments/dev.json`) is single-account today. +**Ambiguity:** The Checkov custom rule +`adapters/terraform/policy/custom_rules/nova_tagging.py` enforces the +Nova tagging standard over Terraform HCL (static scan + plan scan). The +kyverno-json milestone adds `require-tagging-standard.json` over the +resolved Stack IR. Three options: (a) rewrite — replace the Checkov +rule with the kyverno-json policy (loses Checkov's HCL-level coverage +and the `--external-checks-dir` integration); (b) keep Checkov only — +don't add a kyverno-json policy (the Stack IR is already the input to +terraform, so the Checkov rule catches it); (c) both — keep the +Checkov rule as the source of truth for HCL-level scanning AND add the +kyverno-json policy for IR-level coverage, with a meta-policy that +asserts the two agree. -### A4 — Version tag in docs: `@v1.19` vs `ref: v1.9` +**Resolution (auto, confidence 0.82):** Option (c) — both, with a +cross-check meta-policy. The Checkov rule stays the source of truth +for `terraform_plan` scanning (it reads HCL resource blocks directly); +the kyverno-json policy covers the Stack IR dict (which is the input +*before* terraform, so it catches IR-level violations that the +terraform adapter might mask via defaults). The P3 meta-policy +`tagging-rules-agree.json` asserts the two engines agree on every +resource; divergence emits an `error` PCR (defense-in-depth against +rule drift — if the two engines disagree, the operator must +investigate before proceeding). This is the only case in v1.25 where +two engines evaluate the same concern; it is intentional — the +tagging standard is the highest-impact rule (v1.8 D-tagging-standard, +v1.10 re-verification) and merits redundancy. Captured in REQ-297, +REQ-303, REQ-299. Decision ID: D-118. -**Ambiguity:** The consumer guide says `uses: nova/.github/workflows/deploy.yml@v1.19` -but the actual `.github/workflows/deploy.yml` checks out the platform repo -at `ref: v1.9`. The reference table says sample contracts "use `@v1.19`" -but the sample contracts don't carry `uses:` (they're contracts, not -workflows). +### A5 — Critical-override: delegate to declarative meta-policy or keep hard-override? -**Resolution (auto, confidence 0.90):** REQ-281 corrects the reference -table wording to "used with caller workflow `@v1.19`" (the version pin -lives in the caller workflow, not the contract). The `@v1.19` tag in the -consumer-facing docs is the documented current version; the `ref: v1.9` in -deploy.yml is the platform-internal checkout ref. These are two different -references (consumer → platform workflow tag; platform workflow → platform -repo ref). The guide's `@v1.19` stays as the consumer-facing version. No -change to deploy.yml's `ref: v1.9` (that's an internal platform concern, -out of scope for this milestone). +**Ambiguity:** `core/confidence_signal.py` lines 144-157 hardcode +`PENALTY["critical"]: None` — a critical-severity `fail` PCR forces +`score = 0, band = block` regardless of the weighted-sum inputs. The +v1.25 meta-policy `block-on-any-critical.json` makes this declarative +(asserts no PCR in the merged list has `severity: critical` + +`result: fail`). Two options: (a) fully delegate — remove the +hard-override, rely on the meta-policy to emit a critical `fail` PCR +that the existing penalty logic then blocks; (b) keep both — the +meta-policy is the declarative source of truth, the hard-override is +defense-in-depth. -### A5 — Should Shape A destroy go through the HITL decommission pipeline? +**Resolution (auto, confidence 0.88):** Option (b) — keep both. The +meta-policy is the *declarative* statement ("Nova blocks on any +critical finding from any engine"); the hard-override is the +*imperative* safety net that ensures a critical PCR can never slip +through even if the meta-policy is misconfigured or the +`PolicyEngineRegistry` returns a `NullEngine`. This is +defense-in-depth, not redundancy-for-its-own-sake: the meta-policy +runs *before* the confidence signal (it produces PCRs that flow in), +the hard-override runs *inside* the confidence signal (it is the last +gate). Removing the hard-override would make the platform's +"critical = block" guarantee depend on a single declarative policy +file — a regression in the provable-trust posture (Strategic +Objective #2). Captured in REQ-303, PROJECT.md hard-constraints. +Decision ID: D-119. -**Ambiguity:** The decommission pipeline (2-step, HITL SRE gates) exists for -stack teardown. Should env-transition destroy use it? +### A6 — Does kyverno-json break the "platform functions without AI" tenet? -**Resolution (auto, confidence 0.85):** No. Env-transition is an automated -lifecycle step, not an explicit decommission. The destroy runs as a direct -`terraform destroy -auto-approve` against the prior env's state (REQ-284). -The decommission pipeline remains for explicit stack teardown with SRE -gates. This is documented in the Out of Scope section. Rationale: the -consumer already has HITL attestation on the *new* env (qa/prod/dr gates); -requiring a second SRE gate for the prior env's destroy would block -autonomous dev→qa promotion, contradicting the "lower environments are -autonomous" tenet. +**Ambiguity:** NORTH_STAR.md Strategic Objective #2: "the platform +functions without AI — 'AI decisions' are really automated decisions." +kyverno-json is a deterministic policy engine (no ML), but it is a new +runtime dependency. Does adding it violate the tenet? -### A6 — Phase count and ordering +**Resolution (auto, confidence 0.95):** No — kyverno-json is +deterministic, not AI. The tenet distinguishes "AI decisions" (LLM- +driven, non-reproducible) from "automated decisions" (rule-driven, +reproducible). kyverno-json is the latter — the same policy + payload +produces the same result on every run. It is *more* aligned with the +tenet than the current imperative Python in `core/env_transition.py` +and `core/regression_verify.py`, because the policy is declarative +(visible, auditable, version-controlled) rather than imperative (logic +hidden in function bodies). The `is_configured()` guard ensures the +platform functions without the binary (graceful skip), so the tenet +holds even in environments where kyverno-json is not installed. +Captured in PROJECT.md hard-constraints + RESEARCH.md G-Q1. +Decision ID: D-120. -**Ambiguity:** The requirements traceability table shows 3 phases (P1: -docs, P2: feat, P3: test) but the roadmap entry says "4 phases." +## Summary -**Resolution (auto, confidence 0.90):** 4 phases = P0 (pre-execution) + P1 -(docs fixes) + P2 (env-transition feat) + P3 (tests) + P4 (final -review/ship). The "4 phases" in the roadmap counts execution phases (P1-P3) -+ final (P4). This matches the run.md phase model (P0 pre-execution, P1..PN -execution, P N+1 final). The traceability table lists P1-P3 (execution); -P4 is the final phase (review + audit + ship, no new requirements). - -## Clarification Commit - -No changes to REQUIREMENTS.md or PROJECT.md from clarify — the ambiguities -are resolved and already captured in the requirements (REQ-276..290) and -the Out of Scope section. The resolutions above are logged for traceability. \ No newline at end of file +6 ambiguities identified; 6 auto-resolved at full autonomy (no human +escalation). All resolutions are binding and recorded as D-115..D-120. +The resolutions are captured in PROJECT.md hard-constraints, +REQUIREMENTS.md v1.25 sections, and will be referenced in RESEARCH.md + +PLAN.md. No PROJECT.md or REQUIREMENTS.md structural changes beyond the +v1.25 sections added in SPECIFY — the resolutions are already embedded +in the requirement text (REQ-293, REQ-297, REQ-303, etc.) via the +"Decision" annotations. \ No newline at end of file diff --git a/.ciagent/GRILL.md b/.ciagent/GRILL.md index b27db0b..929414b 100644 --- a/.ciagent/GRILL.md +++ b/.ciagent/GRILL.md @@ -1,200 +1,216 @@ -# CIAgent Grill Report +# GRILL — v1.25 kyverno-json Unified Policy Engine -## Run: 2026-08-12 (mode: self-grill, focus: all axes) — v1.24 Consumer Guide Accuracy & Env-Promotion Lifecycle Enforcement +> Adversarial review of the v1.25 SPECIFY + CLARIFY + RESEARCH + IDEATE + +> PLAN. The grill red-teams the proposal across feasibility, scope, +> budget, and the swap-boundary claim. Each challenge gets a binding +> verdict (PROCEED / REVISE / ESCALATE). Autonomy: full — escalations +> auto-resolve with assumption logging unless confidence < 0.60. -### Overall Verdict: PROCEED (confidence: 0.82) +## Verdict: PROCEED (0.86) — 0 escalations, 2 revisions -The plan is sound. The user directive is clear and binding. The code -integration points are confirmed by inspection. Two binding revisions -applied (both low-risk doc clarifications). No escalations. +The milestone is feasible, scoped, and the swap boundary is real. Two +plan revisions are binding (G-Q4, G-Q8) and are already captured in +PLAN.md. No work is blocked. --- -## Axis 1: Feasibility +## Challenges -**Challenge:** Can `run_platform.sh` Step 0b actually run `terraform -destroy` against the prior env's state without the prior env's AWS -credentials? +### G-Q1 — Does kyverno-json violate "platform functions without AI"? -**Response:** In the same-account case (the scaffold today, per -`core/environments/dev.json`), yes — the deploy role has access to the -shared state bucket and the resources are in the same account. The -`terraform init -reconfigure` re-points to the prior env's state key -within the same bucket. Cross-account is explicitly out of scope -(D-205). **Confidence: 0.85.** +**Challenge:** NORTH_STAR.md Strategic Objective #2 says "the platform +functions without AI." kyverno-json is a new runtime dependency. Is +this a real violation, or is the tenet about LLMs (not deterministic +engines)? -**Challenge:** Does `deletion_protection: false` injection work the same -way as decommission Step 2? +**Verdict:** PROCEED (confidence 0.95). kyverno-json is deterministic +(same policy + payload → same result, every run). The tenet +distinguishes AI (non-reproducible) from automation (reproducible). +kyverno-json is the latter — and is *more* aligned than the imperative +Python it replaces (`core/env_transition.py`, `core/regression_verify.py`) +because the policy is declarative (visible, auditable). The +`is_configured()` guard ensures the platform runs without the binary. +Already resolved as D-120 in CLARIFY. No revision needed. -**Response:** Yes. `scripts/run_decommission.sh:34-37` sets -`res['nfrs']['deletion_protection'] = False` on every resource. The -contract resolver propagates `inputs.deletion_protection` to children's -NFRs (`core/contract_resolver.py:360-372`). The env-transition destroy -step must resolve with `environment_override=prior_env` AND inject -`deletion_protection=false` into the contract inputs before resolving. -This is a confirmed pattern. **Confidence: 0.90.** +### G-Q2 — Is the PolicyEngine protocol over-engineered for a 2-engine future? -**Verdict:** FEASIBLE. +**Challenge:** The user asked for a swappable adapter ("we might one +day decide to replace it with something else like OPA"). A Python +Protocol + registry is ~40 lines. But Nova has 1 engine today. Is this +premature abstraction? -## Axis 2: Scope +**Verdict:** PROCEED (confidence 0.85). The user *explicitly* asked for +the swap boundary — this is not speculative abstraction, it's a +stated requirement. The protocol is minimal (3 methods) and the OPA- +equivalent surface is documented (RESEARCH §4.2) — the swap is a known +quantity, not a hope. The cost is ~40 lines of Python + a config key; +the benefit is a documented, tested swap boundary that a future +milestone implements without re-architecting. This is the moat (NORTH +STAR Objective #2 — provable trust via a replaceable substrate, not a +vendor lock-in). -**Challenge:** Is 4 phases (P1-P3 + P4) the right size, or is this -over-scoped? +### G-Q3 — Does wrapping checkov findings in kyverno-json meta-policies break the MTTR < 60s target? -**Response:** 15 requirements across 3 execution phases is -well-scoped. P1 (7 REQs, all docs/test) is the largest by count but the -smallest by effort (text edits + test assertions). P2 (6 REQs, feat) is -the core implementation. P3 (2 REQs, test) is coverage. P4 is final -review. This is a tight, coherent milestone. **Confidence: 0.88.** +**Challenge:** NORTH_STAR.md MTTR target: < 60s p95. Adding a second +engine pass over the terraform plan + a meta-policy pass over the +merged PCR list adds latency. Does this break the target? -**Challenge:** Should the cross-account destroy be in scope? +**Verdict:** PROCEED (confidence 0.88). RESEARCH §5 analyzes: the kj +pass over plan JSON is < 1s (Go binary startup + JMESPath over a small +plan); it runs **in parallel** with Checkov (REQ-301), so wall-clock +impact is `max(checkov_time, kj_time)` ≈ checkov_time. Meta-policies +run in-memory over the merged list (< 10ms). Total MTTR impact: < 1s +on a 5-15s step. **Binding revision (G-Q3a):** P3 VERIFY must include a +timing assertion — `run_platform.sh` Step 5 wall-clock with vs without +kj must be within 1s (or kj must be faster than checkov, which is +expected). Captured as a P3 verify gate, not a PLAN change. -**Response:** No. The scaffold is single-account. Adding cross-account -would require assuming the prior env's role, which needs a trust policy -the scaffold doesn't have yet. Deferring is pragmatic. The Out of Scope -section documents this. **Confidence: 0.85.** +### G-Q4 — Plan revision: NullEngine fallback may mask misconfiguration -**Verdict:** PROPERLY-SCOPED. +**Challenge:** PLAN.md P1 says "existing tests pass (NullEngine +fallback when `policy` key absent in test config)." But the v1.25 +config.json *sets* the `policy` key. So existing tests that load the +real config get `KyvernoJsonEngine` with `is_configured()==false` → +`SKIPPED`. The NullEngine fallback only triggers when the key is +*absent*. Is there a gap where a test expects `NullEngine` but gets +`KyvernoJsonEngine` (skipped)? -## Axis 3: Cost / ROI +**Verdict:** REVISE (confidence 0.82). The fallback path is correct +but the PLAN wording is ambiguous. **Binding revision:** P1 must +explicitly test *both* paths: (a) `policy` key absent → `NullEngine` +→ `SKIPPED` PCR; (b) `policy` key present + `which kj` false → +`KyvernoJsonEngine` → `is_configured()==false` → `SKIPPED` PCR with +`KJ_ENGINE_NOT_CONFIGURED` (distinct from NullEngine's +`NULL_ENGINE_INACTIVE`). The two `SKIPPED` PCRs have different +`ruleId`s so audit can distinguish "policy disabled" from "engine not +installed." PLAN.md P1 verification is amended to assert both paths. +Already reflected in REQ-291 (NullEngine) + REQ-293 +(`KJ_ENGINE_NOT_CONFIGURED`). No requirement change — PLAN wording +clarified. -**Challenge:** Is the env-transition feature worth the complexity? +### G-Q5 — Policy explosion: 4 targets × N rules = maintenance load -**Response:** Yes. The user identified a real orphaned-resources risk -that violates the platform's full-lifecycle-management mission. The -fix is a ~80-line Python module + a shell block. The alternative -(blocking env edits, forcing Shape B) contradicts the user's directive. -The ROI is high: closes a real lifecycle gap with minimal code. -**Confidence: 0.90.** +**Challenge:** v1.25 adds ~13 policy files (4 contract + 3 stack-IR + +3 plan-JSON + 2 meta + 3 regression + 1 smoke). Each is a YAML file +with JMESPath. Is this a maintenance burden that grows unbounded? -**Verdict:** JUSTIFIED. +**Verdict:** PROCEED (confidence 0.80). 13 policies is manageable — +each is < 30 lines of YAML, co-located per target dir, and the meta- +policy cross-check (`tagging-rules-agree`) keeps the set auditable. +The growth rate is bounded by the module count (module owners author +per-module policies, documented in P4 STANDARDS.md). The alternative +(imperative Python in `regression_verify.py` + `env_transition.py`) is +*less* auditable — the policies are a net improvement. No revision. -## Axis 4: Correctness +### G-Q6 — The tagging cross-check (D-118) is the only redundant rule — is it worth the complexity? -**Challenge:** The `detect_prior_env` query — is querying by -`contractId#submittedAt` SK prefix correct for finding the last-applied -env? +**Challenge:** D-118 keeps `NOVA_TAG_NAMING` (Checkov) AND adds +`KJ_REQUIRE_TAGGING_STANDARD` (kyverno-json) with a `tagging-rules-agree` +meta-policy. This is the only case where two engines evaluate the same +concern. Is the defense-in-depth worth the complexity? -**Response:** The `nova-contracts` table has PK `consumerRepo` and SK -`contractId#submittedAt`. To find the last record for a given -contractId, we query by PK `consumerRepo` + SK `begins_with -"contractId#"` + FilterExpression `status = "submitted"` (or -`#LAST_APPLIED`), sort by `submittedAt` desc, take the first. This is -correct DynamoDB pattern. The `record_applied_env` step writes a new -item with SK `contractId#LAST_APPLIED#` so the detect step -can filter by `begins_with "contractId#LAST_APPLIED#"`. **Confidence: -0.85.** +**Verdict:** PROCEED (confidence 0.82). The tagging standard is the +highest-impact rule (v1.8 D-tagging-standard, v1.10 re-verification — +the rule that gates every resource). Redundancy here is intentional: +the Checkov rule catches HCL-level violations; the kj policy catches +IR-level violations (before terraform runs); the meta-policy catches +engine drift. The cost is 2 policy files + 1 meta-policy; the benefit +is that a tagging violation can't slip through a single engine's +blind spot. This is the textbook defense-in-depth case. No revision. -**Challenge:** What if the DynamoDB table doesn't exist in local/CI -mode? +### G-Q7 — Can `kj scan` actually evaluate the merged PCR list as a payload? -**Response:** The detect step catches `ClientError` / `EndpointNotFound`, -logs a warning, and returns `None` (no prior env). The pipeline proceeds -normally. This is the conservative path — no false-positive destroys. -**Confidence: 0.90.** +**Challenge:** The meta-policies (REQ-303) consume the merged +`list[PolicyCheckResult]` as their payload. `kj scan` expects a JSON/ +YAML *file*. Is the PCR list a valid kyverno-json payload shape? -**Verdict:** CORRECT. +**Verdict:** PROCEED (confidence 0.85). The PCR list is a JSON array +of objects — a valid kyverno-json payload. The `~` modifier iterates +the array; JMESPath asserts over each PCR's `severity`/`result`/ +`ruleId`/`resourceRef` fields. The engine writes the list to a temp +JSON file and invokes `kj scan --payload `. This is verified in +P3 `test_meta_policies.py`. No revision — but **binding note (G-Q7a):** +the `KyvernoJsonEngine.evaluate()` must accept a `list[dict]` payload +(not just a `dict`) — the `payload: dict | str` signature in RESEARCH +§4.1 is too narrow. **Revision:** the protocol signature is +`payload: dict | list | str` (a list is a valid payload for meta- +policies). Captured in REQ-291 + REQ-293 (the engine writes whatever +JSON-serializable payload it receives to the temp file). PLAN.md P1 +amended. -## Axis 5: Testing +### G-Q8 — Plan revision: the OPA swap surface claims (RESEARCH §4.2) are unverified -**Challenge:** Can the env-transition behavior be tested without live -AWS? +**Challenge:** RESEARCH §4.2 documents the OPA-equivalent surface +(`opa eval -d -i `), but no `OpaEngine` is implemented in +v1.25. Is the swap-boundary claim testable, or is it aspirational? -**Response:** Yes. `test_env_transition.py` uses moto for DynamoDB -(mock_aws pattern from `test_contract_ingestor.py:74-110`). -`test_run_platform_env_transition.py` uses shell-text assertions -(pattern from `test_pipeline.py:79-95`). No live AWS needed. -**Confidence: 0.92.** +**Verdict:** REVISE (confidence 0.78). The swap-boundary claim is +*testable in v1.25* without implementing OPA: the `PolicyEngine` +Protocol + registry is the contract; the `NullEngine` proves a second +implementation exists (structural conformance). **Binding revision +(G-Q8a):** P1 `test_policy_engine.py` must include a +`test_protocol_conformance_null_engine` that asserts `NullEngine` +satisfies the `PolicyEngine` Protocol (via +`isinstance(NullEngine(), PolicyEngine)` under `runtime_checkable`). +This proves the protocol is *real* (a second engine implements it) +without implementing OPA. The OPA-equivalent surface in RESEARCH §4.2 +stays as documentation (the future milestone implements it). PLAN.md +P1 verification amended. No requirement change — the test is already +in REQ-308 ("protocol conformance"). -**Verdict:** TESTABLE. +### G-Q9 — Budget: is 4 execution phases + P5 too many for the scope? -## Axis 6: Security +**Challenge:** v1.25 is 19 requirements across 6 phases. Recent +milestones: v1.24 had 15 reqs / 4 phases; v1.23 had 13 reqs / 7 phases. +Is 6 phases too many (overhead) or too few (per-phase overload)? -**Challenge:** Does the destroy step introduce a risk of destroying the -wrong resources? +**Verdict:** PROCEED (confidence 0.85). 19 reqs / 6 phases ≈ 3.2 reqs/ +phase — within the v1.24 cadence (3.75 reqs/phase). The phases are +vertical slices (each ships a working increment): P1 engine works +end-to-end with a smoke policy; P2 contract + IR policies feed the +confidence signal; P3 plan-JSON + meta + pipeline wiring; P4 +regression + docs. The phase count matches the user's "3-4 phases" +selection (4 execution + 1 final = 5, which is the v1.24 shape). No +revision. -**Response:** The destroy targets the prior env's state key -(`spike/{id}/{prior_env}/terraform.tfstate`). The state key is -deterministic and env-scoped. The destroy can only affect resources in -that state file. The `deletion_protection=false` injection is scoped to -the destroy step only — the new env's apply runs with the default -`deletion_protection=true`. **Confidence: 0.88.** +### G-Q10 — The `nova.cloudinit.dev/severity` annotation convention is unvalidated -**Challenge:** Could a malicious consumer trigger a destroy of another -consumer's resources? +**Challenge:** RESEARCH §2.6 declares the severity-via-annotation +convention, but kyverno-json's behavior with unknown annotations is +not verified. Does `kj scan` ignore unknown annotations, or does it +reject the policy? -**Response:** No. The DynamoDB query is scoped by PK `consumerRepo` -(the consumer's own repo identity). The destroy runs under the -consumer's ABAC-scoped deploy role, which can only touch resources -tagged `nova:owner=`. A consumer cannot query or destroy -another consumer's stack. **Confidence: 0.90.** - -**Verdict:** SECURE. - -## Axis 7: Maintainability - -**Challenge:** Is the `core/env_transition.py` module a clean -abstraction or a one-off? - -**Response:** It's a reusable module with two functions -(`detect_prior_env`, `record_applied_env`) that encapsulate the -DynamoDB query logic. It can be extended for cross-account destroy in a -future milestone. The shell Step 0b is a thin orchestrator. This is -maintainable. **Confidence: 0.85.** - -**Verdict:** MAINTAINABLE. - -## Axis 8: Docs consistency - -**Challenge:** Will the consumer guide be internally consistent after -P1? - -**Response:** The 5 fixes address all known inconsistencies: field -table ↔ schema, Step 2 ↔ Step 4, Step 5 ↔ environments doc, Step 8 ↔ -per-env section, reference table ↔ sample contracts. The test updates -assert both shapes are documented. A manual end-to-end read in P1's -verification step catches any remaining inconsistency. **Confidence: -0.88.** - -**Verdict:** CONSISTENT. - -## Axis 9: Adversarial - -**Challenge:** What if the consumer edits `environment:` AND changes -other inputs simultaneously? Does the destroy-then-apply still work? - -**Response:** Yes. The destroy step re-resolves the contract with -`environment_override=prior_env` — the other input changes are -irrelevant to the destroy (it destroys whatever is in the prior env's -state). The new apply resolves with the new env + new inputs. The two -operations are independent. **Confidence: 0.85.** - -**Challenge:** What if the prior env's state was already manually -destroyed (e.g., via decommission)? - -**Response:** `terraform destroy` against an empty state is a no-op -(exits 0). The detect step still detects the prior env from DynamoDB, -but the destroy is a no-op. The apply proceeds. This is correct -behavior — no false failure. **Confidence: 0.88.** - -**Verdict:** ROBUST. +**Verdict:** PROCEED (confidence 0.80). kyverno-json is Kubernetes- +style CRD-based — unknown `metadata.annotations` are preserved and +ignored (standard K8s behavior). The engine reads the annotation from +the loaded policy YAML (via `yaml.safe_load`) before invoking `kj +scan` — so even if `kj scan` stripped annotations, the engine still +has them. **Binding note (G-Q10a):** P1 `test_kyverno_json_engine.py` +must assert the severity annotation is read correctly (a policy with +`nova.cloudinit.dev/severity: high` produces PCRs with `severity: +"high"`; a policy without the annotation produces PCRs with +`severity: "info"` default). Captured in REQ-309 ("PCR schema +validity" includes severity). No requirement change — the test is +already in REQ-309. --- -## Binding revisions applied +## Summary -1. **R1 (docs):** Add to RESEARCH.md pitfalls: the destroy step must - inject `deletion_protection=false` into the contract inputs before - re-resolving with `environment_override=prior_env`. Without this, - `prevent_destroy` lifecycle blocks (REQ-86) block the destroy. This - is already noted in RESEARCH §5 pitfall 3 and PLAN P2 implementation - note 3. No change needed — already captured. +10 challenges; 10 resolved (8 PROCEED, 2 REVISE, 0 ESCALATE). +- **Revisions (binding, already in PLAN/REQs):** + - G-Q4: P1 tests both fallback paths (NullEngine vs + KyvernoJsonEngine-not-configured) — distinct `ruleId`s for audit. + - G-Q7a: protocol signature `payload: dict | list | str` (list is a + valid payload for meta-policies). + - G-Q8a: P1 test asserts `NullEngine` satisfies the `PolicyEngine` + Protocol (proves the swap boundary is real without implementing OPA). + - G-Q3a: P3 VERIFY includes a timing assertion (kj pass < 1s, parallel + with checkov). + - G-Q10a: P1 test asserts severity annotation is read correctly. +- **No requirement changes** — all revisions are clarifications to + PLAN.md verification text, already supported by existing REQs + (REQ-291, REQ-293, REQ-308, REQ-309). +- **0 escalations** — all challenges auto-resolved at full autonomy. -2. **R2 (docs):** Clarify in PLAN P2 that the `record_applied_env` SK - format is `contractId#LAST_APPLIED#` so the detect step - can query `begins_with "contractId#LAST_APPLIED#"`. This is already - in RESEARCH §2 and GRILL Axis 4. No change needed — already captured. - -## Escalations - -None. All challenges resolved at full autonomy. \ No newline at end of file +The milestone PROCEEDs to PHASE 0 SHIP → P1. \ No newline at end of file diff --git a/.ciagent/IDEATE.md b/.ciagent/IDEATE.md new file mode 100644 index 0000000..0e74f0a --- /dev/null +++ b/.ciagent/IDEATE.md @@ -0,0 +1,157 @@ +# IDEATE — v1.25 kyverno-json Unified Policy Engine + +> **Autonomy:** full. 3-tier ideation per `config.json ideation.enabled: +> true`. `cross_project.enabled: false` → cross-project tier scoped to +> single-project (deferred ideas only, no cross-project candidates +> accepted). `confidence_threshold: 0.6`, `max_ideas: 20`. +> Categories: security, quality, architecture, coverage, improvement. + +## Tier 1 — Mechanical (pattern-driven, codebase-grounded) + +### I1 — Regression-gate-as-policy ✅ ACCEPTED (REQ-304, REQ-305) + +**Category:** quality, coverage +**Confidence:** 0.90 +**Pattern:** imperative check → declarative policy (the milestone's +core thesis applied to Nova's own regression gate). +**Source:** `core/regression_verify.py` (CAP-013, CAP-023, CAP-024) +are imperative Python checks. The milestone makes compliance +declarative; Nova's own capability regression should follow. +**Idea:** Port the three capability checks into +`adapters/kyverno-json/policies/regression/` as declarative policies +over the capability-inventory JSON frontmatter. The imperative +`regression_verify.py` stays (it drives the CI gate); the policies are +the declarative mirror that makes capability regression auditable as a +policy artifact. +**Accepted into:** REQ-304 (policies), REQ-305 (tests). Phase P4. + +### I2 — Contract-shape validation as policy ✅ ACCEPTED (REQ-295) + +**Category:** security, architecture +**Confidence:** 0.92 +**Pattern:** jsonschema constraint → declarative policy (same constraint, +different language, Nova posture on top). +**Source:** `schemas/contract.schema.json` required/pattern/enum. +**Idea:** The 4 contract policies (`require-id-pattern`, +`require-env-in-enum`, `require-infrastructure-min-1`, `forbid-unknown- +fields`) are the declarative equivalent of the jsonschema constraints — +they let Nova apply its own compliance posture (e.g. forbid a specific +env for a specific consumer) on top of schema validity without editing +the jsonschema. +**Accepted into:** REQ-295. Phase P2. + +### I3 — Stack-IR imperative rules → declarative policies ✅ ACCEPTED (REQ-297) + +**Category:** security, architecture +**Confidence:** 0.88 +**Pattern:** imperative Python rule → declarative kyverno-json policy. +**Source:** `adapters/terraform/policy/custom_rules/nova_tagging.py` +(tagging), the v1.0 demo `public-ingress: true` rule, the v1.8 +D-encryption-default rule. +**Idea:** Port the three highest-impact imperative rules into +declarative kyverno-json policies over the resolved Stack IR. The +tagging rule is a cross-check (D-118 — both engines, agree meta-policy); +public-ingress and encryption-by-default are kyverno-json only (the IR +is the earliest point these can be caught). +**Accepted into:** REQ-297. Phase P2. + +## Tier 2 — Backend-enriched (signal-driven) + +### I4 — Plan-JSON Checkov RULE_MAP → kyverno-json mirrors ✅ ACCEPTED (REQ-300) + +**Category:** security, coverage +**Confidence:** 0.85 +**Pattern:** existing engine rule → declarative mirror in the new engine +(defense-in-depth against engine drift). +**Source:** `checkov_adapter.py:RULE_MAP` (CKV_AWS_41/45/46, CKV_AWS_1/40, +CKV_AWS_7/33). +**Idea:** Port the 6 Checkov rules over `terraform_plan` into declarative +kyverno-json policies over `terraform show -json` output. The Checkov +rules stay the source of truth for HCL scanning; the kyverno-json +policies are mirrors (different rule language, same plan JSON). Defense- +in-depth: if Checkov and kyverno-json disagree on the same plan, the +divergence is visible (two PCRs with different results for the same +resource). +**Accepted into:** REQ-300. Phase P3. + +### I5 — Meta-policy over the merged PCR list ✅ ACCEPTED (REQ-303) + +**Category:** architecture, quality +**Confidence:** 0.90 +**Pattern:** the policy result list is itself a policy target (the most +novel use of kyverno-json in v1.25). +**Source:** `core/confidence_signal.py` PENALTY hardcode (critical +override), the D-118 tagging cross-check. +**Idea:** `block-on-any-critical` (declarative "critical = block") + +`tagging-rules-agree` (Checkov vs kj agree). The meta-policies consume +the merged PCR list as their payload. The critical-block meta-policy is +the declarative source of truth; the `confidence_signal.py` hard-override +stays as defense-in-depth (D-119). +**Accepted into:** REQ-303. Phase P3. + +### I6 — Env-transition destroy as a declarative policy ❌ DEFERRED + +**Category:** improvement +**Confidence:** 0.55 (below threshold — deferred, not rejected) +**Pattern:** imperative lifecycle Python → declarative policy. +**Source:** `core/env_transition.py` (v1.24 detect-and-destroy). +**Idea:** The v1.24 env-transition destroy logic (detect env change via +DynamoDB, destroy prior env, fail-closed) is imperative Python. A +declarative kyverno-json policy could assert "if `environment` changed +on a stable `contract.id`, a destroy event MUST precede the apply" — +turning the lifecycle enforcement into an auditable policy artifact. +**Reason deferred:** The env-transition logic is *stateful* (DynamoDB +queries, terraform state inspection) — kyverno-json policies are +*stateless* (payload in, PCRs out). A policy can assert the *contract* +shape (the env value is valid) but not the *lifecycle* (the prior env +was destroyed). The stateful check stays in `core/env_transition.py`; +a future milestone could emit a `nova.env.destroyed` event that a +kyverno-json policy then asserts is present in the evidence stream +(event-as-policy). Recorded as a future-idea, not a v1.25 requirement. + +### I7 — Drift detection as policy ❌ DEFERRED + +**Category:** security, coverage +**Confidence:** 0.40 (below threshold — deferred) +**Pattern:** scheduled job → policy over the drift report. +**Source:** NORTH_STAR.md Non-Goal #4 (drift detection scheduled job, +deferred — D-096 + no scheduler). +**Idea:** A kyverno-json policy over a terraform drift report could +assert "no drifted resources" declaratively. But drift detection itself +requires a scheduled `terraform plan -detailed-exitcode` job, which is +deferred (no scheduler). The policy is the easy part; the emitter is the +blocking dependency. +**Reason deferred:** Blocked by D-096 + no scheduler (same as NORTH_STAR +Non-Goal #4). The policy shape is documented for when the emitter ships. + +## Tier 3 — Cross-project (deferred — single project) + +### I8 — Cross-project policy sharing ❌ DEFERRED (config) + +**Category:** improvement +**Confidence:** N/A +**Pattern:** policies shared across projects in a multi-project org. +**Source:** `config.json ideation.cross_project.enabled: false`. +**Idea:** In a multi-project org, kyverno-json policies could be shared +across projects (a tagging standard policy applies to all projects). +**Reason deferred:** ACDL is single-project (`active_projects: ["acdl"]`). +Cross-project ideation is disabled in config. Recorded for when the +org grows. + +## Summary + +- 5 ideas accepted (I1..I5) → already captured as REQ-295, REQ-297, + REQ-300, REQ-303, REQ-304, REQ-305. +- 3 ideas deferred (I6, I7, I8) with documented blocking reasons. +- 0 ideas rejected (below-threshold ideas are deferred, not rejected — + they may activate when their blockers lift). +- The accepted ideas are the **quality improvement** the user asked for + ("ideate and explore how it can be used within the Nova platform to + improve quality of the platform checks"): I1 (regression-gate-as- + policy) is the headline quality improvement; I4 + I5 are the defense- + in-depth coverage improvements; I2 + I3 are the architecture + improvements (imperative → declarative). +- No new requirements added beyond REQ-291..309 (the accepted ideas are + already scoped into the existing requirements). The IDEATE pass + validated the requirement set rather than expanding it — the ideas + were anticipated in the SPECIFY stage and explicitly captured. \ No newline at end of file diff --git a/.ciagent/PERSONAS.md b/.ciagent/PERSONAS.md index 1725415..3a064ff 100644 --- a/.ciagent/PERSONAS.md +++ b/.ciagent/PERSONAS.md @@ -1,83 +1,132 @@ --- project: acdl -milestone: v1.24 +milestone: v1.25 generated_at: 2026-08-12 generator: lead-developer verification_toolchain: - typecheck: "python3 -m py_compile core/env_transition.py tests/test_env_transition.py tests/test_run_platform_env_transition.py" - test: "pytest tests/test_env_transition.py tests/test_run_platform_env_transition.py tests/test_consumer_guide_per_env_section.py tests/test_adapter.py tests/test_contract_resolver.py tests/test_deploy_workflow_env_input.py tests/test_pipeline.py -v" - lint: "ruff check core/env_transition.py tests/test_env_transition.py tests/test_run_platform_env_transition.py 2>/dev/null || python3 -m py_compile core/env_transition.py" + typecheck: "python3 -m py_compile core/policy_engine.py adapters/kyverno-json/kyverno_json_engine.py tests/test_policy_engine.py tests/test_kyverno_json_engine.py" + test: "pytest tests/test_policy_engine.py tests/test_kyverno_json_engine.py tests/test_adapter.py tests/test_contract_resolver.py tests/test_confidence_signal.py tests/test_checkov_adapter.py tests/test_kyverno_adapter.py tests/test_pipeline.py -v" + lint: "ruff check core/policy_engine.py adapters/kyverno-json/ 2>/dev/null || python3 -m py_compile core/policy_engine.py" note: | - v1.24 is the Consumer Guide Accuracy & Env-Promotion Lifecycle - Enforcement milestone — a mixed docs+feat+test milestone. Two active - personas: lead-developer (consumer-guide.md edits + guide test - updates), backend-engineer (core/env_transition.py + run_platform.sh - Step 0b + deploy.yml + adapter doc comment + env_transition tests + - pipeline tests). frontend-engineer stays deactivated (no UI). No - data-engineer (no schema changes — the nova-contracts table already - exists). No new personas. + v1.25 is the kyverno-json Unified Policy Engine milestone — a feat + milestone. Four active personas: lead-developer (coordination + + docs + ARCHITECTURE.md §12.7), backend-engineer (core/policy_engine.py + protocol + registry + contract_resolver.py wiring + run_platform.sh + Step 5 + pipeline tests), policy-engineer (adapters/kyverno-json/ + engine + policies across all 4 target dirs + meta-policies + policy + tests + adapter README + STANDARDS.md policy-authoring section), + data-engineer (config.json policy object + schemas/README.md note + + capability-inventory JSON fixture for regression policies). + frontend-engineer stays deactivated (no UI). The policy-engineer is a + new custom persona created for this milestone's policy domain (see + RESEARCH.md §4 — kyverno-json + JMESPath is a distinct framework from + backend-engineer's fastify/hono). --- -# ACDL — Persona Roster (v1.24 Consumer Guide Accuracy & Env-Promotion Lifecycle Enforcement) +# ACDL — Persona Roster (v1.25 kyverno-json Unified Policy Engine) -> v1.24 roster. Two active personas + two deactivated. This is a mixed -> docs+feat+test milestone: the work is consumer-guide accuracy fixes -> (lead-developer), a new env-transition detect-and-destroy platform -> feature (backend-engineer), and test coverage for both (split). +> v1.25 roster. Four active personas + one deactivated. This is a feat +> milestone: the work is a swappable policy-engine protocol + a new +> adapter + policies across 4 Nova artifacts + pipeline wiring + docs. +> The policy-engineer is a new custom persona — kyverno-json + JMESPath +> is a specialized domain that doesn't fit backend-engineer's +> fastify/hono frameworks or data-engineer's drizzle/postgresql. ## Active personas ### lead-developer - **Domain:** coordination + docs - **Frameworks:** [] -- **Constraints:** ["pragmatic", "battle-tested defaults", "docs match code"] +- **Constraints:** ["pragmatic", "battle-tested defaults", "docs match code", "swap boundary is the moat"] - **Territory:** - - `docs/consumer-guide.md` - - `tests/test_consumer_guide_per_env_section.py` - - `.ciagent/*.md` (PLAN.md, RESEARCH.md, etc.) -- **Reason:** Owns the consumer guide narrative — the 5 accuracy fixes - (stale fields table, inconsistent caller, "dev only" phrasing, Step 8 - rewrite with destroy semantics, reference table wording) and the - consumer-guide test updates (rename + new destroy-on-env-change test). - No UI work (frontend-engineer deactivated). No Python/bash platform - code (backend-engineer territory). + - `.ciagent/ARCHITECTURE.md` (§12.7 Policy Engine Registry — NEW) + - `.ciagent/PROJECT.md` (v1.25 section) + - `.ciagent/REQUIREMENTS.md` (v1.25 section) + - `.ciagent/ROADMAP.md` (v1.25 section) + - `.ciagent/PLAN.md`, `.ciagent/RESEARCH.md`, `.ciagent/CLARIFY.md`, + `.ciagent/GRILL.md`, `.ciagent/PERSONAS.md` + - `docs/METRICS.md` (swappable engine narrative — REQ-307) +- **Reason:** Owns the milestone coordination + the architecture + narrative. The swap boundary (PolicyEngine protocol) is the moat per + Strategic Objective #2 — the lead-developer owns the boundary + description in ARCHITECTURE.md §12.7 and the docs/METRICS.md note. + No Python policy code (backend-engineer + policy-engineer territory). + No UI (frontend-engineer deactivated). ### backend-engineer -- **Domain:** backend (Python + bash + YAML) +- **Domain:** backend (Python + bash + pipeline wiring) - **Frameworks:** ["boto3", "terraform"] -- **Constraints:** ["api-first", "fail-closed", "no orphan paths", "state-key determinism"] +- **Constraints:** ["api-first", "strict-typing", "engine-agnostic confidence signal", "fail-soft when kj absent"] - **Territory:** - - `core/env_transition.py` (NEW) - - `scripts/run_platform.sh` (Step 0b insert + record-applied-env) - - `.github/workflows/deploy.yml` (NOVA_CONSUMER_REPO env) - - `adapters/terraform/adapter.py` (doc comment only) - - `tests/test_env_transition.py` (NEW) - - `tests/test_run_platform_env_transition.py` (NEW) -- **Reason:** Owns the env-transition detect-and-destroy feature: the new - `core/env_transition.py` module (DynamoDB query + record), the - `run_platform.sh` Step 0b orchestrator (re-resolve + terraform destroy + - evidence event + fail-closed), the deploy.yml env var passthrough, and - the two new test files. Uses boto3 (DynamoDB) + terraform (destroy) + - bash (pipeline orchestration). + - `core/policy_engine.py` (NEW — PolicyEngine Protocol + PolicyEngineRegistry + NullEngine) + - `core/contract_resolver.py` (MODIFIED — invoke registry pre/post resolve) + - `scripts/run_platform.sh` (MODIFIED — Step 5 kyverno-json parallel pass) + - `scripts/install-kyverno-json.sh` (NEW) + - `tests/test_policy_engine.py` (NEW — protocol conformance, registry, NullEngine) + - `tests/test_run_platform_plan_json_policies.py` (NEW — script-substring assertion) + - `.github/workflows/ci.yml` + `.gitea/workflows/ci.yml` (MODIFIED — Go + kj install) +- **Reason:** Owns the Python protocol layer + the pipeline wiring. The + `PolicyEngine` Protocol + `PolicyEngineRegistry` are Python structural- + typing constructs (PEP 544) — backend-engineer's strict-typing + constraint. The `contract_resolver.py` wiring + `run_platform.sh` + Step 5 are backend territory. Does NOT write kyverno-json policy + files (policy-engineer territory) — only the Python that *invokes* the + engine. Does NOT modify the confidence signal (it already consumes + `list[PolicyCheckResult]` engine-agnostically — PROJECT.md hard- + constraint). + +### policy-engineer +- **Domain:** policy (declarative compliance rules) +- **Frameworks:** ["kyverno-json", "jmespath", "kyverno ValidatingPolicy"] +- **Constraints:** ["declarative-policies", "no-imperative-rules", "schema-validated", "severity-via-annotation", "assertion-trees-not-foreach"] +- **Territory:** + - `adapters/kyverno-json/` (NEW — engine impl + __init__.py + README) + - `adapters/kyverno-json/kyverno_json_engine.py` (NEW — KyvernoJsonEngine) + - `adapters/kyverno-json/policies/` (NEW — all 4 target dirs: contract/, stack-ir/, plan-json/, meta/, regression/) + - `adapters/kyverno-json/policies/_smoke.json` (NEW) + - `adapters/README.md` (MODIFIED — new adapter row + PolicyEngine Protocol section) + - `tests/test_kyverno_json_engine.py` (NEW — PCR schema validity, defensive parsing) + - `tests/test_stack_ir_policies.py` (NEW) + - `tests/test_plan_json_policies.py` (NEW) + - `tests/test_meta_policies.py` (NEW) + - `tests/test_regression_policies.py` (NEW) + - `tests/fixtures/stack_ir/`, `tests/fixtures/plan_json/`, `tests/fixtures/capability_inventory.json` (NEW) + - `modules/STANDARDS.md` (MODIFIED — Policy authoring standard section — REQ-307) +- **Reason:** The policy-engineer owns the declarative policy artifacts. + kyverno-json's `ValidatingPolicy` + assertion trees + JMESPath is a + distinct framework from backend-engineer's fastify/hono and requires + its own constraints: no imperative rules (everything is an assertion + tree), severity via the `nova.cloudinit.dev/severity` annotation (not + in the engine adapter), no `forEach` (use the `~` modifier). The + adapter pattern (engine ↔ protocol ↔ registry) is backend-engineer + territory, but the policy *content* and the engine *translation* + (`_to_pcr()`) are policy-engineer territory because they require + kyverno-json output-shape knowledge. Created per RESEARCH.md §4 — this + is a phase-spanning persona (active for P1..P4), not phase-specific. + +### data-engineer +- **Domain:** data (config schema + structured fixtures) +- **Frameworks:** ["jsonschema", "yaml"] +- **Constraints:** ["schema-first", "type-safe config", "backward-compatible additions"] +- **Territory:** + - `.ciagent/config.json` (MODIFIED — new `policy` object: engine + policy_root) + - `schemas/policy_check_result.schema.json` (READ-ONLY — no change per D-116) + - `schemas/README.md` (MODIFIED — note engine: "kyverno" shared by K8s adapter + kj) + - `tests/fixtures/capability_inventory.json` (NEW — clean + drifted inventory fixtures for regression policies) +- **Reason:** The `config.json.policy` object is a schema-first addition + (new top-level key with `engine` + `policy_root` fields). The + capability-inventory JSON fixtures for the regression-gate policies + (REQ-304) are structured data — the data-engineer owns the fixture + shape. The `policy_check_result.schema.json` is read-only (D-116 — no + enum change); the data-engineer documents the `engine: "kyverno"` + sharing in `schemas/README.md`. No migrations (no database). No Python + (backend-engineer + policy-engineer territory). ## Deactivated personas ### frontend-engineer -- **Active:** false -- **Reason:** No UI work in v1.24. The consumer guide is markdown docs - (lead-developer territory). Deactivated per v1.17/v1.18/v1.22/v1.23 - precedent. - -### data-engineer -- **Active:** false -- **Reason:** No schema changes in v1.24. The `nova-contracts` DynamoDB - table already exists with the right shape (PK `consumerRepo`, SK - `contractId#submittedAt`). The env-transition module only adds a new - `#LAST_APPLIED` SK suffix — no schema migration, no ORM, no new tables. - The backend-engineer handles the boto3 queries. - -## Phase-specific notes - -- No phase-specific personas. Both active personas span P1-P3. -- P4 (final review + audit + ship) is lead-developer territory - (orchestration + docs completion). \ No newline at end of file +- **active:** false +- **Reason:** ACDL has no frontend (no package.json — confirmed in + config.json personas.personas[frontend-engineer].reason). v1.25 adds + no UI work — the policy engine is backend + policy artifacts only. + Deactivated per the v1.15+ convention. \ No newline at end of file diff --git a/.ciagent/PLAN.md b/.ciagent/PLAN.md index 04f9752..3d4d4ce 100644 --- a/.ciagent/PLAN.md +++ b/.ciagent/PLAN.md @@ -1,180 +1,371 @@ -# PLAN — v1.24 (Consumer Guide Accuracy & Env-Promotion Lifecycle Enforcement) +# PLAN — v1.25 (kyverno-json Unified Policy Engine) -> Feature milestone (one `feat` phase: env-transition destroy enforcement; -> the rest are `fix`/`docs`/`test`). Tags on the **v1.23.x** line: -> v1.23.0 (P0) → v1.23.1 (P1) → v1.23.2 (P2) → v1.23.3 (P3) → v1.23.4 (P4 -> final = milestone release). 15 requirements (REQ-276..290), 4 phases + -> P0 pre-execution. +> Feature milestone. Tags on the **v1.24.x** line: v1.24.0 (P0) → +> v1.24.1 (P1) → v1.24.2 (P2) → v1.24.3 (P3) → v1.24.4 (P4) → v1.24.5 +> (P5 final = milestone release). 19 requirements (REQ-291..309), +> 4 execution phases + P0 pre-execution + P5 final review/ship. + +## Wave model + +Each phase is a **vertical slice** (end-to-end: policy files + Python +wiring + tests + docs). Phases are ordered by dependency: the engine +protocol (P1) must exist before policies (P2/P3) can be wired; the +pipeline wiring (P3) must exist before the meta-policies (P3) can +consume the merged PCR list; the regression-gate policies (P4) are +independent of the pipeline and can be authored in parallel with P3's +tests, but ship after P3 because they reference the engine registry +finalized in P1. Within each phase, the waves are the persona task +groups (parallelizable across personas when `parallelization.enabled: +true`, `max_concurrent_agents: 5`). ## Phase breakdown -### Phase P1 — consumer-guide-fixes (Wave 1, lead-developer) +### Phase P1 — engine-core (Wave 1, backend-engineer + policy-engineer + data-engineer) -**Type:** `docs` + `fix` + `test` (consumer guide accuracy + guide test updates) +**Type:** `feat` (engine protocol + registry + kyverno-json engine adapter + install + tests) -**Requirements:** REQ-276, REQ-277, REQ-278, REQ-279, REQ-280, REQ-281, REQ-290 +**Requirements:** REQ-291, REQ-292, REQ-293, REQ-294, REQ-308, REQ-309 **Must-haves:** -- `docs/consumer-guide.md` Step 3 contract fields table corrected (REQ-276) -- `docs/consumer-guide.md` Step 4 caller consistent with Step 2 (REQ-277) -- `docs/consumer-guide.md` Step 5 stage 8 "(dev only)" → "(autonomous in dev; higher environments apply after HITL attestation)" (REQ-278) -- `docs/consumer-guide.md` Step 8 rewritten with destroy-then-rebuild semantics + cross-ref to Shape B (REQ-279) -- `docs/consumer-guide.md` Per-env section gains Shape B lead sentence (REQ-280) -- `docs/consumer-guide.md` Reference table `@v1.19` wording corrected (REQ-281) -- `tests/test_consumer_guide_per_env_section.py` updated: rename `test_consumer_guide_states_no_field_editing` → `test_consumer_guide_documents_both_promotion_shapes`; add `test_consumer_guide_documents_destroy_on_env_change` (REQ-290) +- `core/policy_engine.py` — `PolicyEngine` Protocol (PEP 544) + + `PolicyEngineRegistry` (selects from `config.json.policy.engine`) + + `NullEngine` fallback (emits `SKIPPED` when `policy` key absent) + (REQ-291) +- `.ciagent/config.json` gains `policy` object: `{"engine": + "kyverno-json", "policy_root": + "adapters/kyverno-json/policies"}` (REQ-292) +- `adapters/kyverno-json/kyverno_json_engine.py` — `KyvernoJsonEngine` + implementing the protocol: `is_configured()` guards on `which kj`; + `evaluate()` writes payload to temp JSON, invokes + `kj scan --policy --payload --output json`, translates + native output → `list[dict]` PCR records (`engine: "kyverno"`, + `ruleId` prefixed `KJ_`, severity from + `nova.cloudinit.dev/severity` annotation); defensive parsing + (malformed → `error` PCR, never exception); `is_configured()==false` + → single `SKIPPED` PCR (`KJ_ENGINE_NOT_CONFIGURED`) (REQ-293) +- `adapters/kyverno-json/__init__.py` exports `KyvernoJsonEngine`; + `adapters/kyverno-json/policies/_smoke.json` trivial + `require-contract-id` policy for round-trip validation; + `scripts/install-kyverno-json.sh` runs + `go install github.com/kyverno/kyverno-json/cmd/kj@latest`; + `.github/workflows/ci.yml` + `.gitea/workflows/ci.yml` install Go + kj + (cached) (REQ-294) +- `tests/test_policy_engine.py` — protocol conformance, registry + selection, unknown-engine `KeyError`, `NullEngine` fallback, + `is_configured()` false when `which kj` absent (mocked) (REQ-308) +- `tests/test_kyverno_json_engine.py` — `evaluate()` returns PCR dicts + validating against `schemas/policy_check_result.schema.json` (via + `jsonschema`); defensive parsing (malformed kyverno-json output → + `error` PCR); `is_configured()==false` → `SKIPPED` with + `KJ_ENGINE_NOT_CONFIGURED`; `pytest.skip("kj not installed")` when + `which kj` absent (REQ-309) -**Vertical slice:** A reader of `docs/consumer-guide.md` can promote via -either shape (A: edit environment + platform destroys prior; B: per-env -caller workflow) without contradiction. The guide's field table, caller -examples, and stage descriptions match the actual schema and platform -behavior. All consumer-guide tests pass. +**Vertical slice:** The `PolicyEngineRegistry.get_engine()` returns a +configured `KyvernoJsonEngine` that can `evaluate()` a trivial payload +against `_smoke.json` and produce a valid PCR list. The confidence +signal is unchanged — it already consumes `list[PolicyCheckResult]`. +The platform runs with or without the `kj` binary (`is_configured()` +guard). All existing tests pass (NullEngine fallback when `policy` key +absent in test config — but the v1.25 config.json *sets* the key, so +existing tests that use the real config get `KyvernoJsonEngine` with +`is_configured()==false` → `SKIPPED`). **Files touched:** -- `docs/consumer-guide.md` -- `tests/test_consumer_guide_per_env_section.py` +- `core/policy_engine.py` (NEW) +- `.ciagent/config.json` (MODIFIED — `policy` object) +- `adapters/kyverno-json/__init__.py` (NEW) +- `adapters/kyverno-json/kyverno_json_engine.py` (NEW) +- `adapters/kyverno-json/policies/_smoke.json` (NEW) +- `scripts/install-kyverno-json.sh` (NEW) +- `.github/workflows/ci.yml` (MODIFIED — Go + kj install step) +- `.gitea/workflows/ci.yml` (MODIFIED — Go + kj install step) +- `tests/test_policy_engine.py` (NEW) +- `tests/test_kyverno_json_engine.py` (NEW) -**Verification:** `pytest tests/test_consumer_guide_per_env_section.py -v` -(all 7 tests pass). Manual read of `docs/consumer-guide.md` end-to-end -for internal consistency. +**Verification:** `pytest tests/test_policy_engine.py +tests/test_kyverno_json_engine.py tests/test_confidence_signal.py +tests/test_adapter.py tests/test_checkov_adapter.py +tests/test_kyverno_adapter.py -v` (new tests pass or skip-without-kj; +existing adapter/confidence tests unchanged). `python3 -m py_compile +core/policy_engine.py adapters/kyverno-json/kyverno_json_engine.py`. --- -### Phase P2 — env-transition-detect-and-destroy (Wave 2, backend-engineer) +### Phase P2 — contract + stack-IR policies (Wave 2, policy-engineer + backend-engineer) -**Type:** `feat` (new platform feature: env-transition detect-and-destroy) +**Type:** `feat` (policies + resolver wiring + tests) -**Requirements:** REQ-282, REQ-283, REQ-284, REQ-285, REQ-286, REQ-287 +**Requirements:** REQ-295, REQ-296, REQ-297, REQ-298, REQ-299 **Must-haves:** -- `core/env_transition.py` new module: `detect_prior_env()` + `record_applied_env()` with boto3 DynamoDB queries (REQ-282, REQ-283) -- `scripts/run_platform.sh` Step 0b: environment-transition check — detect prior env, re-resolve with `environment_override=prior_env` + `deletion_protection=false`, `terraform init -reconfigure` + `terraform destroy -auto-approve` against prior state key, emit `nova.env.destroyed` evidence event, fail closed on destroy failure (REQ-284) -- `scripts/run_platform.sh` records applied env after successful apply (REQ-285) -- `.github/workflows/deploy.yml` passes `NOVA_CONSUMER_REPO=${{ github.repository }}` to `run_platform.sh` (REQ-286) -- `adapters/terraform/adapter.py` state-key block gains doc comment (REQ-287) +- `adapters/kyverno-json/policies/contract/` — 4 policies over consumer + contract JSON: `require-id-pattern.json`, + `require-env-in-enum.json`, `require-infrastructure-min-1.json`, + `forbid-unknown-fields.json` — each a `ValidatingPolicy` with one + `validate.assert` rule using JMESPath against the payload root; + severity via `nova.cloudinit.dev/severity` annotation (REQ-295) +- `core/contract_resolver.py` invokes + `PolicyEngineRegistry.get_engine().evaluate(contract_dict, + policies/contract/, contract_id)` **before** resolving; failures + feed the `policy` input as `fail` PCRs (no resolver exit — confidence + signal decides the gate, `--soft-fail` pattern); emits + `nova.policy.evaluated` metrics event (REQ-296) +- `adapters/kyverno-json/policies/stack-ir/` — 3 policies over + resolved Stack IR: `require-tagging-standard.json` (ports + `nova_tagging.py` — `nova:owner` + `nova:environment` tags on every + `resources[]` entry), `forbid-public-ingress.json` (v1.0 demo rule), + `require-encryption-by-default.json` (v1.8 D-encryption-default); + `~` modifier iterates `resources[]` (REQ-297) +- `core/contract_resolver.py` invokes the engine with the resolved + Stack IR and `policies/stack-ir/` **after** resolving; resulting PCRs + appended to the contract-policy PCRs; resolver return values and + exceptions unchanged (additive) (REQ-298) +- `tests/test_stack_ir_policies.py` + `tests/fixtures/stack_ir/` — + passing IR (all tags + encryption) + failing IR (missing tags, public + ingress, plaintext bucket); each policy in isolation + full dir as + bundle; `pytest.skip("kj not installed")` when `which kj` absent + (REQ-299) -**Vertical slice:** When a consumer changes `environment:` on a stable -`contract.id`, the pipeline detects the prior env from DynamoDB, destroys -the prior env's Terraform state (with `deletion_protection=false`), emits -an evidence event, and only then applies the new env. If the destroy -fails, the pipeline exits non-zero (no orphan path). If no prior env -exists (first deploy or Shape B), the pipeline proceeds normally. +**Vertical slice:** A consumer contract passes through the resolver +and produces two PCR lists (contract policies pre-resolve, stack-IR +policies post-resolve) that feed the confidence signal. A contract +with a bad `id` or missing tags produces `fail` PCRs that lower the +confidence score. The resolver's existing tests pass unchanged (the +policy call is additive — it does not change resolver return values +or exceptions). **Files touched:** -- `core/env_transition.py` (NEW) -- `scripts/run_platform.sh` -- `.github/workflows/deploy.yml` -- `adapters/terraform/adapter.py` (doc comment only) +- `adapters/kyverno-json/policies/contract/require-id-pattern.json` (NEW) +- `adapters/kyverno-json/policies/contract/require-env-in-enum.json` (NEW) +- `adapters/kyverno-json/policies/contract/require-infrastructure-min-1.json` (NEW) +- `adapters/kyverno-json/policies/contract/forbid-unknown-fields.json` (NEW) +- `adapters/kyverno-json/policies/stack-ir/require-tagging-standard.json` (NEW) +- `adapters/kyverno-json/policies/stack-ir/forbid-public-ingress.json` (NEW) +- `adapters/kyverno-json/policies/stack-ir/require-encryption-by-default.json` (NEW) +- `core/contract_resolver.py` (MODIFIED — pre/post resolve engine calls) +- `tests/test_stack_ir_policies.py` (NEW) +- `tests/fixtures/stack_ir/passing.json` (NEW) +- `tests/fixtures/stack_ir/failing.json` (NEW) -**Verification:** `python3 -m py_compile core/env_transition.py`. -`pytest tests/test_pipeline.py tests/test_deploy_workflow_env_input.py -v` -(existing tests still pass). The new tests in P3 validate the behavior. - -**Key implementation notes (from RESEARCH §5 pitfalls):** -1. The destroy step must re-resolve with `environment_override=prior_env` - so the emitted TF matches the prior env's resources. -2. `terraform init -reconfigure` is required when switching state backends. -3. `deletion_protection: false` must be injected (same as decommission - Step 2 in `scripts/run_decommission.sh:34-37`) or `prevent_destroy` - blocks the destroy. -4. DynamoDB unreachable in local/CI → log warning + return `None` - (conservative, no prior env assumed). -5. The state key `spike/{id}/{env}/terraform.tfstate` stays as-is — the - env segment is what lets the destroy target the prior env. +**Verification:** `pytest tests/test_contract_resolver.py +tests/test_stack_ir_policies.py tests/test_policy_engine.py -v` +(existing resolver tests pass; new policy tests pass or skip-without- +kj). `python3 -m py_compile core/contract_resolver.py`. --- -### Phase P3 — env-transition-tests (Wave 3, backend-engineer + lead-developer) +### Phase P3 — plan-JSON policies + meta-orchestration + pipeline wiring (Wave 3, policy-engineer + backend-engineer) -**Type:** `test` (new test coverage for env-transition + pipeline integration) +**Type:** `feat` (plan-JSON policies + meta-policies + run_platform.sh wiring + tests) -**Requirements:** REQ-288, REQ-289 +**Requirements:** REQ-300, REQ-301, REQ-302, REQ-303 **Must-haves:** -- `tests/test_env_transition.py` (NEW): `detect_prior_env` returns `None` when no record; returns prior env when record differs; returns `None` when record matches; `record_applied_env` writes record. Uses moto for DynamoDB (REQ-288) -- `tests/test_run_platform_env_transition.py` (NEW): asserts `run_platform.sh` has Step 0b; calls `env_transition.py detect`; calls `terraform destroy` on prior env; fails closed on destroy failure; records applied env after success (REQ-289) +- `adapters/kyverno-json/policies/plan-json/` — 3 policies over + `terraform show -json` output: `forbid-plaintext-secrets.json` (ports + CKV_AWS_41/45/46), `forbid-iam-wildcard.json` (ports CKV_AWS_1/40), + `require-kms-reference.json` (ports CKV_AWS_7/33); JMESPath over + `planned_values.root_module.resources[]` (REQ-300) +- `run_platform.sh` Step 5 gains a parallel kyverno-json pass: after + Checkov/Wiz produce raw PCRs, the script runs + `kj scan --policy adapters/kyverno-json/policies/plan-json/ + --payload -o json` and pipes through + `adapters/kyverno-json/kyverno_json_engine.py` to produce a second + PCR list; both lists concatenated and fed to the confidence signal; + `nova.policy.evaluated` event with both engine names; when + `which kj` is false, logs and proceeds with Checkov/Wiz list only + (no hard failure) (REQ-301) +- `tests/test_plan_json_policies.py` + `tests/fixtures/plan_json/` — + passing plan (no secrets, no wildcard, KMS alias) + failing plan + (plaintext password, `Action: "*"`, inline KMS key); policies in + isolation + bundle; `tests/test_run_platform_plan_json_policies.py` + asserts `run_platform.sh` has the kyverno-json Step 5 block + + concatenates PCR lists (script-substring assertion, pattern from + `tests/test_pipeline.py:79-95`) (REQ-302) +- `adapters/kyverno-json/policies/meta/` — `block-on-any-critical.json` + (asserts no PCR in merged list has `severity: critical` + `result: + fail`; if any does, emits `fail` PCR `KJ_META_BLOCK_CRITICAL` + severity `critical` — declarative source of truth; the + `confidence_signal.py` hard-override stays as defense-in-depth per + D-119) + `tagging-rules-agree.json` (cross-checks Checkov + `NOVA_TAG_NAMING` vs kj `KJ_REQUIRE_TAGGING_STANDARD` by + `resourceRef`; divergence emits `error` PCR per D-118); + `tests/test_meta_policies.py` (REQ-303) -**Vertical slice:** The env-transition detect-and-destroy behavior is -fully covered by automated tests. The DynamoDB query logic is unit-tested -with moto. The pipeline orchestration is tested via shell-text assertions -(pattern from `tests/test_pipeline.py:79-95`). +**Vertical slice:** `run_platform.sh` Step 5 produces a merged PCR list +(Checkov/Wiz + kj plan-JSON policies + kj meta-policies over the +merged list) that feeds the confidence signal. A plan with a plaintext +secret produces two `fail` PCRs (one Checkov, one kj) for the same +resource — visible defense-in-depth. A critical finding anywhere +produces a `KJ_META_BLOCK_CRITICAL` meta-PCR that the confidence +signal's hard-override blocks. The pipeline runs with or without `kj` +(graceful skip). **Files touched:** -- `tests/test_env_transition.py` (NEW) -- `tests/test_run_platform_env_transition.py` (NEW) +- `adapters/kyverno-json/policies/plan-json/forbid-plaintext-secrets.json` (NEW) +- `adapters/kyverno-json/policies/plan-json/forbid-iam-wildcard.json` (NEW) +- `adapters/kyverno-json/policies/plan-json/require-kms-reference.json` (NEW) +- `adapters/kyverno-json/policies/meta/block-on-any-critical.json` (NEW) +- `adapters/kyverno-json/policies/meta/tagging-rules-agree.json` (NEW) +- `scripts/run_platform.sh` (MODIFIED — Step 5 kj parallel pass) +- `tests/test_plan_json_policies.py` (NEW) +- `tests/test_meta_policies.py` (NEW) +- `tests/test_run_platform_plan_json_policies.py` (NEW) +- `tests/fixtures/plan_json/passing.json` (NEW) +- `tests/fixtures/plan_json/failing.json` (NEW) -**Verification:** `pytest tests/test_env_transition.py tests/test_run_platform_env_transition.py -v` (all new tests pass). Full suite: `pytest tests/ -k "env_transition or consumer_guide or pipeline or adapter or deploy_workflow" -v`. +**Verification:** `pytest tests/test_plan_json_policies.py +tests/test_meta_policies.py tests/test_run_platform_plan_json_policies.py +tests/test_pipeline.py -v` (new tests pass or skip-without-kj; existing +pipeline tests pass). `python3 -m py_compile` on any modified Python. +Shellcheck on `run_platform.sh` if available. --- -### Phase P4 — final-review-ship (Wave 4, lead-developer) +### Phase P4 — regression-gate policies + docs (Wave 4, policy-engineer + data-engineer + lead-developer) -**Type:** `docs` (review + audit + milestone ship) +**Type:** `feat` (regression policies) + `docs` (adapter READMEs + ARCHITECTURE + STANDARDS + METRICS) -**Requirements:** (none new — milestone completion) +**Requirements:** REQ-304, REQ-305, REQ-306, REQ-307 **Must-haves:** -- Multi-persona code review across P1-P3 changes (ci-code-reviewer) -- Project health audit (ci-doc-verifier + ci-audit) -- Milestone ship: tag v1.23.4 (final phase patch = milestone release), merge to main, Gitea release -- Update REQUIREMENTS.md traceability (all REQ-276..290 → complete) -- Update ROADMAP.md (v1.24 → complete) +- `adapters/kyverno-json/policies/regression/` — 3 policies over + capability-inventory JSON frontmatter: `cap-013-adapter-dedup.json`, + `cap-023-metrics-collector.json`, `cap-024-deck-structure.json`; + emit `pass`/`fail` PCRs per capability; the existing + `core/regression_verify.py` is kept (drives the CI gate); the + policies are the declarative mirror (REQ-304) +- `tests/test_regression_policies.py` + + `tests/fixtures/capability_inventory/clean.json` + + `tests/fixtures/capability_inventory/drifted.json` — clean (all caps + pass) + drifted (duplicate adapter, missing metric status, broken + deck arc); regression gate still 287/287 baseline (new tests + additive, skip-without-kj) (REQ-305) +- `adapters/README.md` gains new kyverno-json adapter row + "Policy + Engine Protocol" section (Protocol, registry, swap boundary, + how-to-add-OpaEngine); `adapters/kyverno-json/README.md` documents + the engine, install path, policy directory layout, 4 policy + categories (REQ-306) +- `.ciagent/ARCHITECTURE.md` §12.7 (added in RESEARCH) is finalized; + `schemas/README.md` notes `engine: "kyverno"` shared by K8s adapter + + kj (distinguished by `ruleId` prefix); `modules/STANDARDS.md` + gains "Policy authoring standard" section for module owners; + `docs/METRICS.md` notes the policy engine is swappable (Strategic + Objective #2 — provable trust via a replaceable substrate) (REQ-307) + +**Vertical slice:** The regression gate's capability checks are now +declarative policies auditable as artifacts. A new module owner can +read `modules/STANDARDS.md` "Policy authoring standard" and write a +per-module kyverno-json policy. A new engineer can read +`adapters/README.md` "Policy Engine Protocol" and implement an +`OpaEngine`. The 287/287 baseline is unchanged. + +**Files touched:** +- `adapters/kyverno-json/policies/regression/cap-013-adapter-dedup.json` (NEW) +- `adapters/kyverno-json/policies/regression/cap-023-metrics-collector.json` (NEW) +- `adapters/kyverno-json/policies/regression/cap-024-deck-structure.json` (NEW) +- `tests/test_regression_policies.py` (NEW) +- `tests/fixtures/capability_inventory/clean.json` (NEW) +- `tests/fixtures/capability_inventory/drifted.json` (NEW) +- `adapters/README.md` (MODIFIED — new row + PolicyEngine Protocol section) +- `adapters/kyverno-json/README.md` (NEW) +- `schemas/README.md` (MODIFIED — engine enum note) +- `modules/STANDARDS.md` (MODIFIED — Policy authoring standard section) +- `docs/METRICS.md` (MODIFIED — swappable engine narrative) + +**Verification:** `pytest tests/test_regression_policies.py +tests/test_kyverno_json_engine.py -v` (new tests pass or skip-without- +kj). Full regression gate `pytest tests/` still at 287/287 baseline + +new tests (skip without kj). Manual read of `adapters/README.md` + +`adapters/kyverno-json/README.md` + `modules/STANDARDS.md` policy +section for clarity. --- -## Wave ordering +### Phase P5 — final review + audit + milestone ship (Final Phase) -``` -Wave 1 (P1): consumer-guide-fixes [lead-developer] - ↓ -Wave 2 (P2): env-transition-detect-and-destroy [backend-engineer] - ↓ -Wave 3 (P3): env-transition-tests [backend-engineer + lead-developer] - ↓ -Wave 4 (P4): final-review-ship [lead-developer] -``` +**Type:** `docs` (review + audit + milestone completion) -**Dependencies:** -- P2 depends on P1: the Step 8 rewrite in P1 documents the destroy - semantics that P2 implements. Doing P1 first ensures the docs and code - land in the right order (docs describe the intended behavior, then code - implements it). -- P3 depends on P2: the tests validate the env-transition module and - pipeline Step 0b that P2 creates. -- P4 depends on P1+P2+P3: the final review covers all changes. +**Requirements:** All REQ-291..309 (mark complete) -**Parallelization:** P1 and P2 could run in parallel (different -territories: docs vs code), but the wave ordering is sequential for -safety — if P1's Step 8 rewrite reveals a design issue, P2's -implementation should follow the corrected design. With -`parallelization.enabled=true` and `min_plans_for_parallel=2`, the -orchestrator *could* run them concurrently; however, the dependency -(P2 follows P1's design) makes sequential the safer choice. P3 must -follow P2 (tests validate the code). P4 must follow all. +**Must-haves:** +- `ciagent-review` multi-persona code review across P1..P4 + (lead-developer, backend-engineer, data-engineer, policy-engineer). + Auto-fix P0; flag P1+ for post-hoc review. If P1+ issues found, fix + them in this final phase (not loop back to EXECUTE). +- `ciagent-audit` — reconstruction test (git log ↔ `.ciagent/` files), + `.ciagent/` file discipline, branch hygiene, commit discipline. + Critical issues fixed in this phase. +- `ciagent-ship` (milestone) — merge `phase/05-final-review-ship` → + `milestone/v1.25-kyverno-json` → `main`; tag `v1.24.5` (= the v1.25 + release per the prev-minor tagging rule); create Gitea release with + full milestone summary (all phases, all requirements); delete all + milestone branches (local + remote). +- Update `REQUIREMENTS.md` (mark REQ-291..309 complete), + `ROADMAP.md` (mark v1.25 complete), `CHECKPOINT.json` + (milestone_complete: true), `NORTH_STAR.md` (note Strategic + Objective #2 — provable trust via a replaceable policy-engine + substrate). -## Requirement → Phase mapping +**Vertical slice:** The v1.25 milestone is complete: kyverno-json is +the primary policy tool, behind a swappable adapter, with policies +over all 4 Nova artifacts. Tags v1.24.0..v1.24.5 on the v1.24.x line. +The milestone branch merges to main. -| REQ | Phase | Type | Description | -|-----|-------|------|-------------| -| REQ-276 | P1 | docs | Contract fields table corrected | -| REQ-277 | P1 | docs | Step 4 caller consistent with Step 2 | -| REQ-278 | P1 | docs | Step 5 stage 8 "dev only" corrected | -| REQ-279 | P1 | docs | Step 8 rewritten with destroy semantics | -| REQ-280 | P1 | docs | Per-env section Shape B lead sentence | -| REQ-281 | P1 | docs | Reference table @v1.19 wording corrected | -| REQ-282 | P2 | feat | env_transition.py detect_prior_env() | -| REQ-283 | P2 | feat | env_transition.py record_applied_env() | -| REQ-284 | P2 | feat | run_platform.sh Step 0b detect-and-destroy | -| REQ-285 | P2 | feat | run_platform.sh records applied env | -| REQ-286 | P2 | feat | deploy.yml passes NOVA_CONSUMER_REPO | -| REQ-287 | P2 | docs | adapter.py state-key doc comment | -| REQ-288 | P3 | test | test_env_transition.py | -| REQ-289 | P3 | test | test_run_platform_env_transition.py | -| REQ-290 | P1 | test | consumer guide test updates | +**Verification:** `pytest tests/ -v` full suite passes (287 baseline + +new tests). `git log --oneline` shows the v1.25 phase commits. +`git tag` shows v1.24.0..v1.24.5. `git branch` shows no leftover +milestone/phase branches (all deleted post-ship). -## Tag plan +--- -- P0 (this phase): `v1.23.0` — pre-execution patch -- P1: `v1.23.1` — consumer guide fixes -- P2: `v1.23.2` — env-transition detect-and-destroy -- P3: `v1.23.3` — env-transition tests -- P4: `v1.23.4` — final review + ship = **milestone release** \ No newline at end of file +## Wave ordering (parallelization) + +With `parallelization.enabled: true`, `max_concurrent_agents: 5`, +`min_plans_for_parallel: 2`: + +- **P1 Wave 1:** backend-engineer (protocol + registry + install) ‖ + data-engineer (config.json policy object) ‖ policy-engineer (engine + adapter + smoke policy). 3 concurrent personas. Merge in order: + data-engineer → backend-engineer → policy-engineer. +- **P2 Wave 2:** policy-engineer (contract + stack-IR policies) ‖ + backend-engineer (resolver wiring — depends on P1 registry). 2 + concurrent. Merge: policy-engineer → backend-engineer (wiring + references the policy dirs). +- **P3 Wave 3:** policy-engineer (plan-JSON + meta policies) ‖ + backend-engineer (run_platform.sh wiring — depends on P1 engine + + P2 resolver pattern). 2 concurrent. Merge: policy-engineer → + backend-engineer. +- **P4 Wave 4:** policy-engineer (regression policies) ‖ data-engineer + (capability-inventory fixtures) ‖ lead-developer (docs: READMEs, + STANDARDS, METRICS). 3 concurrent. Merge: data-engineer → + policy-engineer → lead-developer. + +Territory enforcement: `warn` mode (per `config.json +personas.territory_enforcement: "warn"`). Cross-territory edits +(e.g., backend-engineer touching a policy file) emit a warning, not a +block. + +## Requirement → phase → persona matrix + +| REQ | Phase | Primary persona | Type | +|-----|-------|-----------------|------| +| REQ-291 | P1 | backend-engineer | feat | +| REQ-292 | P1 | data-engineer | feat (config) | +| REQ-293 | P1 | policy-engineer | feat | +| REQ-294 | P1 | backend-engineer | feat (install) | +| REQ-295 | P2 | policy-engineer | feat | +| REQ-296 | P2 | backend-engineer | feat (wiring) | +| REQ-297 | P2 | policy-engineer | feat | +| REQ-298 | P2 | backend-engineer | feat (wiring) | +| REQ-299 | P2 | policy-engineer | test | +| REQ-300 | P3 | policy-engineer | feat | +| REQ-301 | P3 | backend-engineer | feat (pipeline) | +| REQ-302 | P3 | policy-engineer + backend-engineer | test | +| REQ-303 | P3 | policy-engineer | feat (meta) | +| REQ-304 | P4 | policy-engineer | feat | +| REQ-305 | P4 | policy-engineer + data-engineer | test | +| REQ-306 | P4 | policy-engineer + lead-developer | docs | +| REQ-307 | P4 | lead-developer | docs | +| REQ-308 | P1 | backend-engineer | test | +| REQ-309 | P1 | policy-engineer | test | \ No newline at end of file diff --git a/.ciagent/PROJECT.md b/.ciagent/PROJECT.md index be9114e..c33ecf7 100644 --- a/.ciagent/PROJECT.md +++ b/.ciagent/PROJECT.md @@ -1577,3 +1577,124 @@ New requirements REQ-263..REQ-275 — see `REQUIREMENTS.md` §v1.23. Summary: consolidation (REQ-263,264), style restoration (REQ-265,266,267), image inlining (REQ-268), python-pptx generator (REQ-269,270), word-count trim + loaded-scope-term removal (REQ-271,272), CI/tests/README (REQ-273,274,275). + +## v1.25 — kyverno-json Unified Policy Engine + +> **Active milestone.** Feature milestone (the primary compliance/policy +> tool becomes kyverno-json, implemented behind a swappable adapter). +> Branch: `milestone/v1.25-kyverno-json`. Tags run on the **v1.24.x** +> patch line: `v1.24.0` (P0) → `v1.24.1..v1.24.4` (P1–P4) → `v1.24.5` +> (P5 final = milestone release). + +[Nova](https://github.com/kyverno/kyverno-json) `kyverno-json` is a +runtime from the Kyverno ecosystem that applies Kyverno policies to +**any JSON or YAML payload** — not just Kubernetes manifests. This +milestone makes kyverno-json the **primary tool of choice for +compliance / policy checks** in Nova, implemented as an **adapter** +(the `PolicyEngine` protocol) so the platform may one day replace it +with something else (e.g. OPA) without touching the confidence signal +or the pipeline. + +### Why + +Nova's policy posture today is split across three engines with three +different rule languages and three adapter shapes: + +- **Checkov** (`adapters/terraform/policy/checkov_adapter.py`) — the + runtime scanner over `terraform_plan` JSON; carries the + `NOVA_TAG_NAMING` custom rule. Imperative YAML+Python rules. +- **Wiz** (`adapters/wiz/wiz_adapter.py`) — security findings from the + Wiz API; inactive unless credentials are present. +- **Kyverno (K8s)** (`adapters/kyverno/kyverno_adapter.py`) — translates + Kyverno `PolicyReport` results; **inactive for Terraform-only stacks** + (the platform emits Terraform, not K8s manifests — D-053). + +All three emit the same `schemas/policy_check_result.schema.json` shape +that `core/confidence_signal.py` consumes engine-agnostically. The +*contract* is already right; the *orchestration* is fragmented. There is +no single place where "what Nova considers compliant" is declared — +tagging lives in a Checkov custom rule, public-ingress in Checkov's +`RULE_MAP`, env-transition destroy in `core/env_transition.py` +(imperative Python), and capability regression in +`core/regression_verify.py` (imperative Python). Each is a different +language, each drifts independently, and the K8s Kyverno adapter can't +help because it only speaks to K8s manifests. + +`kyverno-json` fixes this: one declarative policy language (Kyverno +policies with JMESPath assertions) that applies to **any** Nova +artifact — the consumer contract, the resolved Stack IR, the +Terraform plan JSON, and even the PolicyCheckResult list itself +(meta-validation). It becomes the **unified orchestrator** of compliance +checks, while Checkov and Wiz remain as raw-finding adapters that feed +*into* kyverno-json meta-policies (so Nova-specific posture rules sit +on top of, not beside, the scanner findings). + +### What the milestone delivers + +- **Swappable `PolicyEngine` protocol** (`core/policy_engine.py`) — a + Python Protocol + registry selected from `config.json` (`policy.engine`, + default `"kyverno-json"`). `KyvernoJsonEngine` implements it (shells + to the `kyverno-json` CLI); a future `OpaEngine` implements the same + protocol. The confidence signal and pipeline never import the engine + directly — they go through the registry. +- **`KyvernoJsonEngine` adapter** (`adapters/kyverno-json/`) — + `evaluate(payload, policies) -> list[PolicyCheckResult]` translates + kyverno-json native output to the existing PCR schema. Mirrors the + Checkov/Wiz adapter pattern. `is_configured()` guard skips gracefully + when the `kyverno-json` binary is absent (same pattern as the Wiz + adapter — emits `SKIPPED`, never breaks the pipeline). +- **Policies over all four Nova artifacts** under + `adapters/kyverno-json/policies/`: + - `contract/` — consumer contract JSON (shape + env-promotion rules). + - `stack-ir/` — resolved Target Stack IR (tagging standard, + public-ingress, encryption-by-default — ports of the v1.0/v1.8 + imperative rules into declarative policies). + - `plan-json/` — `terraform show -json` output (plaintext secrets, + IAM wildcards, KMS references — ports of Checkov's `RULE_MAP`). + - `meta/` — policies over the merged PolicyCheckResult list itself + (e.g. `block-on-any-critical` — the single declarative source of + truth for "critical = block", with the existing + `confidence_signal.py` hard-override kept as defense-in-depth). +- **`run_platform.sh` Step 5 wiring** — Checkov/Wiz still run and emit + raw PCRs; `KyvernoJsonEngine.evaluate()` runs plan-JSON policies in + parallel; both PCR lists merge into the confidence signal's `policy` + input. No change to `core/confidence_signal.py` (it already consumes + `list[PolicyCheckResult]` engine-agnostically). +- **Regression-gate-as-policy** (P4 — quality improvement from the + IDEATE pass): the capability checks in + `core/regression_verify.py` (CAP-013, CAP-023, CAP-024) become + declarative kyverno-json policies over the capability-inventory JSON + frontmatter. Capability regression becomes an audit artifact, not + imperative Python. +- **`policy-engineer` persona** (custom, added in RESEARCH) — owns the + policy territory; declarative-policies constraint; kyverno-json + + JMESPath frameworks. + +**Phase count:** 6 (P0 pre-execution + 4 execution + 1 final). + +**Hard constraints:** +- DO NOT change `schemas/policy_check_result.schema.json` shape in a way + that breaks existing adapters — the contract is the moat. The + `engine` enum already includes `"kyverno"` and `"opa"`; v1.25 records + carry `engine: "kyverno"` (no new enum value — decision in CLARIFY). +- DO NOT remove Checkov or Wiz adapters — they remain as raw-finding + sources feeding into kyverno-json meta-policies. +- DO NOT remove the `confidence_signal.py` `PENALTY["critical"]: None` + hard-override — it stays as defense-in-depth behind the declarative + `block-on-any-critical` meta-policy (decision in CLARIFY). +- DO NOT change `core/confidence_signal.py`'s input contract — it + already consumes `list[PolicyCheckResult]`; v1.25 only changes *who + produces* that list, not *what* the list is. +- The platform must function with `kyverno-json` absent — `is_configured()` + returns false → `SKIPPED` records → confidence signal proceeds (no + hard dependency that breaks the "platform functions without AI / + deterministic scripts" tenet — kyverno-json is deterministic, not AI). + +### Requirements + +New requirements REQ-291..REQ-309 — see `REQUIREMENTS.md` §v1.25. +Summary: engine protocol + registry (REQ-291,292), kyverno-json engine +impl (REQ-293,294), contract policies (REQ-295,296), stack-IR policies +(REQ-297,298,299), plan-JSON policies + pipeline wiring (REQ-300,301,302), +meta-policies (REQ-303), regression-gate policies (REQ-304,305), docs + +adapter README (REQ-306,307), tests (REQ-308,309). diff --git a/.ciagent/REQUIREMENTS.md b/.ciagent/REQUIREMENTS.md index 1cb8e0d..811d059 100644 --- a/.ciagent/REQUIREMENTS.md +++ b/.ciagent/REQUIREMENTS.md @@ -2196,3 +2196,295 @@ assert 20 main + 1 appendix. | REQ-288 | P3 | complete | | REQ-289 | P3 | complete | | REQ-290 | P1 | complete | + +## v1.25 — kyverno-json Unified Policy Engine + +> **Feature milestone.** `kyverno-json` becomes the primary compliance / +> policy tool, implemented behind a swappable `PolicyEngine` adapter so +> OPA (or any other engine) can replace it one day. Tags run on the +> **v1.24.x** line (milestone v1.25 → tags v1.24.0..v1.24.N). Final patch +> = milestone release. +> +> One problem, one architectural correction: +> 1. **Fragmented policy posture.** Nova's compliance rules are split +> across Checkov (imperative YAML + a Python custom rule for tagging), +> Wiz (API findings), the K8s-only Kyverno adapter (inactive for +> Terraform stacks — D-053), and imperative Python in +> `core/env_transition.py` + `core/regression_verify.py`. There is no +> single declarative place where "what Nova considers compliant" lives. +> The K8s Kyverno adapter can't help because it only speaks to K8s +> manifests, and the platform emits Terraform. +> +> The correction: `kyverno-json` (a Kyverno-ecosystem runtime that applies +> Kyverno policies to **any** JSON/YAML payload) becomes the **unified +> orchestrator** of compliance checks. Checkov and Wiz remain as +> raw-finding adapters feeding *into* kyverno-json meta-policies. The +> engine is behind a `PolicyEngine` protocol so it is replaceable. The +> confidence signal is untouched — it already consumes +> `list[PolicyCheckResult]` engine-agnostically. + +### Decisions (locked in CLARIFY, full autonomy) + +- **D-115 (C-1):** `kyverno-json` is a runtime dependency installed via + `go install github.com/kyverno/kyverno-json/cmd/kj@latest` (pinned in a + `scripts/install-kyverno-json.sh` helper; the CI image installs it). + Not a Python package — kyverno-json is a Go binary. The + `KyvernoJsonEngine.is_configured()` checks `which kj` and skips + gracefully when absent (emits `SKIPPED` PCR, mirroring the Wiz adapter). +- **D-116 (C-2):** kyverno-json PCR records carry `engine: "kyverno"` + (no new enum value). The existing `engine` enum in + `schemas/policy_check_result.schema.json` already includes `"kyverno"`; + adding `"kyverno-json"` would force a schema change + checkov_adapter + test regression for no semantic gain. The `ruleId` prefix `KJ_` + distinguishes kyverno-json rules from the K8s Kyverno adapter's + `KYVERNO_` prefix where they overlap. +- **D-117 (C-3):** Checkov and Wiz adapters keep their current + `adapt() -> list[PolicyCheckResult]` signatures. They emit PCRs as + today. The meta-policies in `adapters/kyverno-json/policies/meta/` + consume the **merged** PCR list (checkov + wiz + kyverno-json) as their + input payload, applying Nova-specific posture rules on top. No adapter + signature changes. +- **D-118 (C-4):** `NOVA_TAG_NAMING` (the Checkov custom rule in + `adapters/terraform/policy/custom_rules/nova_tagging.py`) is **kept**. + A kyverno-json mirror policy `require-tagging-standard.json` is added + in `adapters/kyverno-json/policies/stack-ir/`. The P3 meta-policy + `tagging-rules-agree.json` asserts the two engines agree on every + resource; divergence emits an `error` PCR (defense-in-depth against + rule drift). The Checkov rule stays the source of truth for + Terraform-static scanning; the kyverno-json policy covers Stack IR. + +### Category: Policy Engine Core (feat) +- **REQ-291:** `core/policy_engine.py` defines a `PolicyEngine` Python + `Protocol` (PEP 544) with three members: `name -> str`, + `is_configured() -> bool`, and + `evaluate(payload: dict | str, policy_dir: Path, contract_id: str) -> + list[dict]` (where each dict conforms to + `schemas/policy_check_result.schema.json`). A `PolicyEngineRegistry` + singleton selects the active engine from `config.json`'s new + `policy.engine` key (default `"kyverno-json"`); raises + `KeyError` on an unknown engine name. The registry exposes + `get_engine()` and `register(name, factory)`. Pure stdlib, no engine + imports at the protocol layer. +- **REQ-292:** `.ciagent/config.json` gains a new top-level `policy` + object: `{"engine": "kyverno-json", "policy_root": + "adapters/kyverno-json/policies"}`. The registry reads `policy.engine` + to select the active engine and `policy.policy_root` as the default + policy directory. Backward-compatible: if the `policy` key is absent, + the registry returns a `NullEngine` that emits only `SKIPPED` records + (so existing tests that don't set the key still pass). + +### Category: kyverno-json Engine Adapter (feat) +- **REQ-293:** `adapters/kyverno-json/kyverno_json_engine.py` implements + `KyvernoJsonEngine` satisfying the `PolicyEngine` protocol. + `is_configured()` returns `True` when `which kj` succeeds. `evaluate()` + writes the payload to a temp JSON file, invokes + `kj scan --policy --payload -o json`, + parses the native result list, and translates each entry to a PCR dict + (`engine: "kyverno"`, `ruleId` prefixed `KJ_`, severity + mapped, `result` mapped pass/fail/skip → pass/fail/skipped). When + `is_configured()` is false, `evaluate()` returns a single `SKIPPED` + PCR with `ruleId: "KJ_ENGINE_NOT_CONFIGURED"` (mirrors the Wiz + adapter's `is_configured()` guard). Native output parsing is + defensive: any kyverno-json output that doesn't match the expected + shape produces an `error` PCR, never an exception. +- **REQ-294:** `adapters/kyverno-json/__init__.py` exports + `KyvernoJsonEngine`. `adapters/kyverno-json/policies/_smoke.json` + is a single trivial policy (`require-contract-id`) used to validate + the engine round-trip end-to-end in tests. `scripts/install-kyverno-json.sh` + runs `go install github.com/kyverno/kyverno-json/cmd/kj@latest` and + prints `kj version`; documented in `adapters/kyverno-json/README.md`. + The CI image (`.github/workflows/ci.yml` + `.gitea/workflows/ci.yml`) + installs Go + kj when `policy.engine == "kyverno-json"`; the install + is cached. + +### Category: Contract Policies (feat) +- **REQ-295:** `adapters/kyverno-json/policies/contract/` holds + kyverno-json policies over consumer contract JSON. Four policies + mirroring `schemas/contract.schema.json` constraints: + `require-id-pattern.json` (`id` matches `^[a-z][a-z0-9-]{2,5}$`), + `require-env-in-enum.json` (`environment` in dev/qa/prod/dr), + `require-infrastructure-min-1.json` (`infrastructure` has ≥1 entry), + `forbid-unknown-fields.json` (only `id`/`name`/`environment`/ + `infrastructure` allowed). Each policy is a single Kyverno `Policy` + resource with one `validate.assert` rule using JMESPath against the + payload root. Policies are the declarative equivalent of the + jsonschema `required`/`pattern`/`enum` constraints — they let Nova + apply its own compliance posture on top of schema validity. +- **REQ-296:** `core/contract_resolver.py` invokes the + `PolicyEngineRegistry.get_engine().evaluate()` with the contract dict + and `policies/contract/` **before** resolving (early-fail on contract + violations) and emits a `nova.policy.evaluated` metrics event (engine + name in the event payload). Failures feed the confidence signal's + `policy` input as `fail` PCRs; the resolver does not exit — the + confidence signal decides the gate (consistent with the existing + `--soft-fail` Checkov pattern). + +### Category: Stack-IR Policies (feat) +- **REQ-297:** `adapters/kyverno-json/policies/stack-ir/` holds policies + over the resolved Target Stack IR dict. `require-tagging-standard.json` + — every resource carries `nova:owner` + `nova:environment` tags + (ports `adapters/terraform/policy/custom_rules/nova_tagging.py` logic + into a declarative Kyverno policy over the IR's `resources[]` array; + mirrors the v1.8 D-tagging-standard). `forbid-public-ingress.json` — + no resource has `public_ingress: true` (the v1.0 demo rule, now + declarative). `require-encryption-by-default.json` — every S3 bucket + + EBS volume + KMS-aliased resource carries encryption config (ports + the v1.8 D-encryption-default rule). +- **REQ-298:** `core/contract_resolver.py` invokes the engine with the + resolved Stack IR and `policies/stack-ir/` **after** resolving. The + resulting PCRs are appended to the contract-policy PCRs and fed to the + confidence signal. The resolver's existing `tests/test_contract_resolver.py` + continues to pass (the policy call is additive — it does not change + resolver return values or exceptions). +- **REQ-299:** `tests/test_stack_ir_policies.py` + fixture + `tests/fixtures/stack_ir/` — a passing IR (all tags + encryption) and + a failing IR (missing tags, public ingress, plaintext bucket). Each + policy is tested in isolation + the full `policies/stack-ir/` dir as a + bundle. Tests run the `KyvernoJsonEngine` against real `kj` when + `which kj` succeeds, and skip with a `pytest.skip("kj not installed")` + when absent (so CI without the binary doesn't fail). + +### Category: Plan-JSON Policies + Pipeline Wiring (feat) +- **REQ-300:** `adapters/kyverno-json/policies/plan-json/` holds policies + over `terraform show -json` output. `forbid-plaintext-secrets.json` + (ports `CKV_AWS_41/45/46` — no `aws_db_instance.password` / + `aws_iam_user.*` plaintext). `forbid-iam-wildcard.json` (ports + `CKV_AWS_1/40` — no `Action: "*"` or `Resource: "*"` in IAM policies). + `require-kms-reference.json` (ports `CKV_AWS_7/33` — KMS keys referenced + by alias, not inline). Each policy uses JMESPath over the plan's + `planned_values.root_module.resources[]` array. The Checkov `RULE_MAP` + in `checkov_adapter.py` is unchanged — these are declarative mirrors, + not replacements. +- **REQ-301:** `run_platform.sh` Step 5 ("runtime policy scan") gains a + parallel kyverno-json pass: after Checkov/Wiz produce raw PCRs, the + script runs `kj scan --policy adapters/kyverno-json/policies/plan-json/ + --payload -o json` and pipes through + `adapters/kyverno-json/kyverno_json_engine.py` to produce a second PCR + list. Both lists are concatenated and fed to the confidence signal's + `policy` input. The script emits a `nova.policy.evaluated` event with + both engine names. When `which kj` is false, the script logs + "kyverno-json not installed; skipping plan-json policies" and proceeds + with the Checkov/Wiz list only (no hard failure — the platform + functions without kj). +- **REQ-302:** `tests/test_plan_json_policies.py` + fixture + `tests/fixtures/plan_json/` — a passing plan JSON (no secrets, no + wildcard, KMS alias) and a failing plan JSON (plaintext password, + `Action: "*"`, inline KMS key). Tests the three policies in isolation + + as a bundle. `tests/test_run_platform_plan_json_policies.py` + asserts `run_platform.sh` has the kyverno-json Step 5 block and that + it concatenates PCR lists (pattern from `tests/test_pipeline.py:79-95` + — read script text + assert substrings). + +### Category: Meta-Policies (feat) +- **REQ-303:** `adapters/kyverno-json/policies/meta/` holds policies + whose **payload** is the merged `list[PolicyCheckResult]` itself. + `block-on-any-critical.json` — asserts no PCR in the list has + `severity: "critical"` + `result: "fail"`; if any does, the meta-policy + emits a `fail` PCR with `ruleId: "KJ_META_BLOCK_CRITICAL"` and + severity `critical`. This is the **declarative** source of truth for + "critical = block"; the `confidence_signal.py` `PENALTY["critical"]: + None` hard-override stays as defense-in-depth (D-118-adjacent + decision). `tagging-rules-agree.json` — for every resource in the + Stack IR, asserts the Checkov `NOVA_TAG_NAMING` result and the + kyverno-json `KJ_REQUIRE_TAGGING_STANDARD` result agree; divergence + emits an `error` PCR. `tests/test_meta_policies.py` covers both. + +### Category: Regression-Gate Policies (feat, quality improvement from IDEATE) +- **REQ-304:** `adapters/kyverno-json/policies/regression/` holds + policies over the capability-inventory JSON frontmatter + (`CAPABILITY_INVENTORY.md` parsed as structured data). Three policies + port the imperative checks in `core/regression_verify.py`: + `cap-013-adapter-dedup.json` (no duplicate adapter registrations), + `cap-023-metrics-collector.json` (every metric in `docs/METRICS.md` + has a grounded/derived/deferred status), `cap-024-deck-structure.json` + (deck slide structure matches the documented arc). The policies read + the parsed capability inventory as payload and emit `pass`/`fail` PCRs + per capability. The existing `core/regression_verify.py` is **kept** + (it drives the CI gate); the policies are the **declarative mirror** + that makes capability regression auditable as a policy artifact, not + imperative Python. Future milestones may switch the gate to the + policy version. +- **REQ-305:** `tests/test_regression_policies.py` + fixture + `tests/fixtures/capability_inventory.json` — a clean inventory (all + caps pass) and a drifted inventory (duplicate adapter, missing metric + status, broken deck arc). The regression gate (`pytest` suite) + continues to pass 287/287 (or new count); the new policy tests are + additive. + +### Category: Documentation (docs) +- **REQ-306:** `adapters/README.md` gains a new row for the + `kyverno-json` adapter + a new section "Policy Engine Protocol" + documenting the `PolicyEngine` Protocol, the registry, and the + swap boundary (how to add an `OpaEngine`). `adapters/kyverno-json/README.md` + documents the engine, the install path, the policy directory layout, + and the four policy categories (contract/stack-ir/plan-json/meta). +- **REQ-307:** `.ciagent/ARCHITECTURE.md` gains §12.7 "Policy Engine + Registry" with the registry diagram (engine ↔ protocol ↔ registry ↔ + config.json ↔ confidence signal). `schemas/README.md` notes the + `engine: "kyverno"` value is shared by the K8s Kyverno adapter and the + kyverno-json engine (distinguished by `ruleId` prefix). `modules/STANDARDS.md` + gains a "Policy authoring standard" section for module owners who want + to ship per-module kyverno-json policies. `docs/METRICS.md` notes the + policy engine is now swappable (Strategic Objective #2 — provable + trust via a replaceable substrate, not a vendor lock-in). + +### Category: Tests (test) +- **REQ-308:** `tests/test_policy_engine.py` — protocol conformance + (the registry returns an engine implementing all three methods), + unknown-engine `KeyError`, `NullEngine` fallback when the `policy` + key is absent, `KyvernoJsonEngine.is_configured()` returns false when + `which kj` fails (mocked). `tests/test_kyverno_json_engine.py` — + `evaluate()` returns valid PCR dicts against + `schemas/policy_check_result.schema.json` (validated with + `jsonschema`); native-output parsing is defensive (malformed kyverno-json + output → `error` PCR, not exception); `is_configured()==false` → + `SKIPPED` PCR with `KJ_ENGINE_NOT_CONFIGURED`. +- **REQ-309:** All new tests use `pytest.skip("kj not installed")` when + `which kj` is absent, so the suite passes in environments without the + binary (CI matrix: with-kj and without-kj). The full suite + (`pytest tests/`) continues to pass at 287/287 baseline + new tests + (the new tests skip without kj, so the count grows only when kj is + installed). `pyproject.toml` + `requirements-test.txt` unchanged + (kyverno-json is a Go binary, not a Python dep). + +### Out of Scope (v1.25) +- **Removing Checkov or Wiz.** Both stay as raw-finding adapters. The + unified-orchestrator model layers kyverno-json on top, not in place of. +- **`OpaEngine` implementation.** The protocol is the swap boundary; + the OPA implementation is a future milestone. RESEARCH documents the + OPA-equivalent surface so the swap is a known quantity. +- **Per-module policies.** `modules//policies/` is documented as + the future pattern in `modules/STANDARDS.md` but not populated this + milestone (policies live under `adapters/kyverno-json/policies/` + for v1.25). +- **kyverno-json as a long-running service.** v1.25 uses the CLI + (`kj scan`); the `kj serve` web-app mode is a future consideration + for lower-latency evaluation (RESEARCH notes it). +- **Replacing the K8s Kyverno adapter.** The K8s adapter + (`adapters/kyverno/`) remains documentation-only (D-053 — platform + emits Terraform). The kyverno-json engine and the K8s adapter are + siblings, not replacements. + +### v1.25 Traceability + +| REQ | Phase | Status | +|-----|-------|--------| +| REQ-291 | P1 | pending | +| REQ-292 | P1 | pending | +| REQ-293 | P1 | pending | +| REQ-294 | P1 | pending | +| REQ-295 | P2 | pending | +| REQ-296 | P2 | pending | +| REQ-297 | P2 | pending | +| REQ-298 | P2 | pending | +| REQ-299 | P2 | pending | +| REQ-300 | P3 | pending | +| REQ-301 | P3 | pending | +| REQ-302 | P3 | pending | +| REQ-303 | P3 | pending | +| REQ-304 | P4 | pending | +| REQ-305 | P4 | pending | +| REQ-306 | P4 | pending | +| REQ-307 | P4 | pending | +| REQ-308 | P1 | pending | +| REQ-309 | P1 | pending | diff --git a/.ciagent/RESEARCH.md b/.ciagent/RESEARCH.md index 4eccb46..fcaf2b9 100644 --- a/.ciagent/RESEARCH.md +++ b/.ciagent/RESEARCH.md @@ -1,187 +1,438 @@ -# Nova — v1.24 Research Findings +# Nova — v1.25 Research Findings -> Phase: research (pre-execution). Milestone: v1.24 (Consumer Guide Accuracy -> & Env-Promotion Lifecycle Enforcement). Status: research. -> Researcher: ci-researcher. Autonomy: full. +> Phase: research (pre-execution). Milestone: v1.25 (kyverno-json Unified +> Policy Engine). Status: research. Researcher: ci-researcher. +> Autonomy: full. ## 1. Problem domain -Two distinct problem spaces in one milestone: +Nova's compliance/policy posture is fragmented across three engines with +three rule languages and three adapter shapes (see PROJECT.md v1.25 +"Why" for the full diagnosis). The `PolicyCheckResult` schema +(`schemas/policy_check_result.schema.json`) is already the engine-agnostic +contract that `core/confidence_signal.py` consumes — the *contract* is +right; the *orchestration* is fragmented. There is no single declarative +place where "what Nova considers compliant" lives. The K8s-only Kyverno +adapter (`adapters/kyverno/`) can't help because it only speaks to K8s +manifests and the platform emits Terraform (D-053). -### 1a. Consumer guide accuracy (docs) +`kyverno-json` is the correction: a Kyverno-ecosystem runtime that applies +Kyverno policies to **any** JSON/YAML payload. It becomes the **unified +orchestrator** of compliance checks, behind a swappable `PolicyEngine` +protocol so OPA can replace it one day. Checkov and Wiz remain as +raw-finding adapters feeding *into* kyverno-json meta-policies. -The consumer guide (`docs/consumer-guide.md`, 477 lines) has 5 accuracy -defects identified in review: +## 2. kyverno-json — the engine surface -1. **Step 3 contract fields table (lines 141-147)** lists `uses`, `module`, - `environment`, `inputs`. The actual schema - (`schemas/contract.schema.json:7`) requires `id`, `name`, `environment`, - `infrastructure`. The `uses` field was dropped in v1.10.2 (REQ-50 - superseded) and `module` was replaced by the `infrastructure` map key. - The worked examples (lines 111-137) use the correct fields. +### 2.1 What it is -2. **Step 4 caller (lines 173-183)** omits `environment:` in `with:`, while - Step 2 (lines 94-101) shows `environment: dev`. The two canonical caller - snippets disagree. +[kyverno-json](https://github.com/kyverno/kyverno-json) is a standalone Go +binary from the Kyverno project. It is a **separate runtime** from the +Kyverno K8s admission controller — same policy lineage, different +application target. Where Kyverno (K8s) evaluates `ClusterPolicy` +resources against Kubernetes manifests at admission time, kyverno-json +evaluates `ValidatingPolicy` resources against **any** JSON or YAML +payload file via the CLI (`kj scan`) or a Go library. It is **not** a +Python package (no PyPI release); it is installed via +`go install github.com/kyverno/kyverno-json/cmd/kj@latest` (D-115) or by +downloading a pinned binary from GitHub releases. -3. **Step 5 stage 8 (lines 232, 253)** says "(dev only)" for the apply - stage. Higher environments *do* apply — they apply after HITL - attestation per `docs/environments/index.md:44-54`. +### 2.2 CLI surface (the v1.25 invocation path) -4. **Step 8 (lines 290-306)** says "Change `environment` in your contract" - to promote, which contradicts the same doc's "Per-environment - deployment" section (lines 398-402): "you do not edit the `environment:` - field… Promotion = running the matching job." The test - `test_consumer_guide_states_no_field_editing` asserts the no-editing - model. +The v1.25 engine uses the `kj scan` subcommand: -5. **Reference table (lines 329-330)** says sample contracts "use `@v1.19`" - but the sample contracts (`contracts/static-assets.yml`, - `contracts/microservice.yml`) don't carry `uses:` — the version pin - lives in the caller workflow. +``` +kyverno-json scan [flags] -### 1b. Environment-promotion lifecycle enforcement (feat) +Flags: + --labels strings Labels selectors for policies + --output string Output format (text or json) (default "text") + --payload string Path to payload (json or yaml file) + --policy strings Path to kyverno-json policies + --pre-process strings JMESPath expression used to pre process payload +``` -**Root cause confirmed by code inspection:** +The `KyvernoJsonEngine.evaluate()` implementation (REQ-293) invokes: +``` +kj scan --policy --payload --output json +``` +and parses the JSON `results[]` array. The `--pre-process` flag is +available for JMESPath pre-projection (noted for the meta-policy use case +where the payload is the merged PCR list and a pre-process expression +can index by `ruleId` — recorded as a future optimization, not used in +v1.25's initial implementation). -- `adapters/terraform/adapter.py:129` sets the Terraform state key to: - `spike/{stack_name}/{environment}/terraform.tfstate` -- `stack_name` = `contract["id"]` (stable across env changes, per - `core/contract_resolver.py:584`). -- When a consumer edits `environment:` from `dev` → `qa` on the same - contract `id`, the state key changes from `spike/assets/dev/` to - `spike/assets/qa/`. Terraform initializes a **fresh state file** in the - new env's state path. The prior env's resources remain live in AWS with - their state file untouched. **No destroy ever runs.** This orphans - resources. +Other subcommands (`kj jp`, `kj serve`, `kj playground`, `kj docs`) are +out of scope for v1.25. `kj serve` is the long-running web-app mode +(noted as a future consideration for lower-latency evaluation in the +Out of Scope section of REQUIREMENTS.md). `kj jp` is the JMESPath REPL — +useful for policy authoring/debugging, not invoked by the engine. -**The user's binding directive:** Editing `environment:` on a stable -`contract.id` is a valid promotion path (Shape A). The platform **must** -destroy the prior env's resources before building the new env. There must -be **no path that orphans resources** — fail closed if the destroy fails. +### 2.3 Policy structure (the `ValidatingPolicy` resource) -## 2. Existing codebase structure (integration points) +kyverno-json policies are Kubernetes-style resources (cluster-scoped) +belonging to the `json.kyverno.io` API group, kind `ValidatingPolicy`, +version `v1alpha1`: -### DynamoDB `nova-contracts` table (the prior-env source of truth) +```yaml +apiVersion: json.kyverno.io/v1alpha1 +kind: ValidatingPolicy +metadata: + name: # becomes the KJ_ ruleId prefix +spec: + rules: + - name: + identifier: # optional — path to the unique entry id + match: # assertion tree — which payload entries + any: # the rule applies to + - + exclude: # optional — exclude matching entries + any: + - + context: # optional — named bindings available to + - name: # the rule's assertions ($) + variable: + validate: + message: "" # optional per-rule message + assert: + all: # all assertions must hold + - check: + message: "" + # OR + any: # at least one assertion must hold + - check: +``` -- **Table:** `nova-contracts` (env var `CONTRACTS_TABLE`, default - `nova-contracts`). Defined in `core/lambda/contract_ingestor.py:25`. -- **Schema:** PK `consumerRepo` (S), SK `contractId#submittedAt` (S). - Attributes: `contractId`, `contract`, `environment`, `status`, - `submittedAt`. -- **Written by:** `_submit_contract()` at - `core/lambda/contract_ingestor.py:135-176`. The consumer's deploy - workflow submits the contract via the Lambda Function URL. -- **Read pattern for env-transition:** Query by PK `consumerRepo` + SK - begins_with `contractId#` + FilterExpression `status = "submitted"` → - sort by `submittedAt` desc → take the latest → read its `environment`. - This is the last-submitted env. For the last-*applied* env, a new - `#LAST_APPLIED` SK suffix is added (REQ-283). -- **Test pattern:** `tests/test_contract_ingestor.py:74-110` - (`moto_contracts_table` fixture) uses moto `mock_aws` to create the - table. The env-transition tests will mirror this pattern. +Key differences from K8s Kyverno policies: +- **Always cluster-scoped** — no `namespace` field. +- **No `forEach`, pattern operators, anchors, or wildcards.** Iteration + is done via the `~` projection modifier in assertion trees (see §2.4). +- **Assertion trees** with JMESPath expressions replace Kyverno's + pattern-matching syntax (see §2.4). -### Outbox writer (evidence events) +### 2.4 Assertion trees (the rule language) -- `core/outbox_writer.py` writes hash-chained evidence events to - `nova-outbox` table. PK `contractId`, SK `eventType#eventTs`. -- The env-transition destroy step emits a `nova.env.destroyed` event via - this writer (REQ-284c). Pattern: build an event dict with `contractId`, - `eventType: "ENV_DESTROYED"`, `environment: `, `ts`, then - call `write_event()`. +An `assert` declaration contains an `all` or `any` list. Each entry has a +`check` (the assertion tree — a nested JMESPath projection) and an +optional `message`. **All comparisons happen in the leaves of the tree.** -### Contract resolver (environment override) +A simple example (assert a pod doesn't use the default service account): +```yaml +validate: + assert: + all: + - message: "serviceAccountName 'default' is not allowed" + check: + spec: + (serviceAccountName == 'default'): false +``` -- `core/contract_resolver.py:460-483` `resolve()` accepts - `environment_override` — when set, it overrides the contract's - `environment` field **before** schema validation and interpolation - (D-088). This is the mechanism the destroy step uses to re-resolve the - contract against the prior env: `resolve(contract, env_override=prior_env)`. -- Already tested in `tests/test_deploy_workflow_env_input.py:35-52`. +The `(expression)` syntax evaluates a JMESPath expression; the result +becomes the current object for descendants; the leaf value is compared +to the expected value. -### run_platform.sh (where Step 0b goes) +**Iteration via the `~` modifier.** The `~` prefix on a key applies +descendant assertions to **each element** of an array/map individually +(rather than comparing the whole array). Given `foo.bar: [1,2,3]`: +```yaml +check: + foo: + ~.bar: # iterate each element + (@ < `5`): true # assert each element < 5 +``` +The `~index_name.bar` form binds the index (array) or key (map) to +`$index_name` for use in descendants. This is how v1.25 iterates +`resources[]` in the Stack IR policies (REQ-297) and +`planned_values.root_module.resources[]` in the plan-JSON policies +(REQ-300). -- `scripts/run_platform.sh` is the platform pipeline. Step 0 (lines 222-239) - is the environment onboarding check. Step 1 (lines 241-249) is contract - validation. **Step 0b goes between them** (after onboarding, before - validation). -- The destroy step mirrors the existing `--destroy` mode (lines 376-394): - `terraform init -reconfigure` + `terraform destroy -auto-approve`. The - difference: it runs against the *prior* env's state key, not the current - one. -- `CONTRACT_ID` is already set at line 217 (`NOVA_CONTRACT_ID` with a - default). `CONSUMER_REPO` needs to be derived from `GITHUB_REPOSITORY` - or a new `NOVA_CONSUMER_REPO` env var (REQ-286). +**Explicit bindings** via `->binding_name` allow descendants to refer +to a parent node via `$binding_name`. Built-in bindings: `$payload` +(the whole input), `$policy`, `$rule`. -### deploy.yml (consumer repo → platform) +**Escaping** via `\key\` prevents projection when a payload key collides +with the projection syntax. Not needed for Nova payloads (no `(key)` +fields), noted for completeness. -- `.github/workflows/deploy.yml:132` runs - `bash platform/scripts/run_platform.sh $MODE_FLAG $ENV_FLAG "${{ inputs.contract }}"`. -- To pass `NOVA_CONSUMER_REPO`, add `NOVA_CONSUMER_REPO=${{ github.repository }}` - as an env var on the "Run the platform pipeline" step (REQ-286). +### 2.5 Output shape (what `kj scan --output json` produces) -### Test patterns +The JSON output is a `results[]` array. Each result entry has (at +minimum): +- `policy`: the policy metadata.name +- `rule`: the rule name +- `result`: `"pass"` | `"fail"` | `"error"` | `"skip"` (lowercase) +- `message`: the assertion message (or engine error message) +- `resource`: the matched payload entry (the `identifier` value, or the + whole payload when no identifier/match) +- `namespace`/`kind`/`name`: K8s-style fields (present but empty for + non-K8s payloads — the K8s Kyverno adapter's evidence uses these; the + kyverno-json engine's evidence uses `assertion`/`jmespath` instead) +- `severity`: not present by default (kyverno-json does not assign + severities — the Nova policy author assigns severity via a Nova- + specific annotation; see §2.6) -- **Shell script assertions:** `tests/test_pipeline.py:79-95` reads the - script text and asserts substrings. The env-transition tests follow this - pattern for `run_platform.sh`. -- **DynamoDB mocking:** `tests/test_contract_ingestor.py:74-110` uses moto - `mock_aws` + `boto3.client` + `create_table`. The env-transition tests - follow this pattern. -- **Consumer guide assertions:** `tests/test_consumer_guide_per_env_section.py` - reads `docs/consumer-guide.md` text and asserts substrings. The updated - tests follow this pattern. +The `KyvernoJsonEngine._to_pcr()` translator (REQ-293) maps: +- `policy` → `ruleId` (prefixed `KJ_` per D-116) +- `result` → `result` (`pass`/`fail`/`error` → pass/fail/error; + `skip`/`skipped` → skipped) +- `message` → `message` +- `resource` → `resourceRef` + `evidence.resource` +- severity from the policy's `metadata.annotations` (see §2.6) +- `engine: "kyverno"` (per D-116 — no new enum value) -## 3. Persona assessment (v1.24) +### 2.6 Severity assignment (Nova convention) -This milestone has two distinct work territories: +kyverno-json does not natively assign severities to results. Nova's +confidence signal requires a `severity` per PCR (critical/high/medium/ +low/info). The v1.25 convention: each Nova policy file declares its +severity via a `metadata.annotations` field: -1. **Docs (P1):** `docs/consumer-guide.md` edits + test updates. This is - lead-developer territory (narrative/docs + test assertions). -2. **Platform code (P2):** `core/env_transition.py` (new), `scripts/run_platform.sh` - edits, `.github/workflows/deploy.yml` edit, `adapters/terraform/adapter.py` - doc comment. This is backend-engineer territory (Python + bash + YAML). -3. **Tests (P3):** `tests/test_env_transition.py` (new), `tests/test_run_platform_env_transition.py` - (new), `tests/test_consumer_guide_per_env_section.py` updates. Split: - backend-engineer for the env_transition + pipeline tests; lead-developer - for the consumer guide test updates. +```yaml +metadata: + name: forbid-public-ingress + annotations: + nova.cloudinit.dev/severity: high +``` -**Roster:** lead-developer (docs + guide tests) + backend-engineer (Python + -bash + YAML + pipeline tests). frontend-engineer stays deactivated (no UI). -data-engineer not needed (no schema changes — the `nova-contracts` table -already exists with the right shape; we only add a new SK suffix). No new -personas. +The `KyvernoJsonEngine._to_pcr()` reads this annotation from the loaded +policy YAML (not from the scan result — the result doesn't carry it) and +applies it to every result that policy produces. Default when absent: +`info`. This keeps severity in the policy (declarative, version- +controlled) rather than in the engine adapter (imperative). The +annotation key is `nova.cloudinit.dev/severity` (matches the existing +`nova.cloudinit.dev` namespace used in `schemas/tagging-standard.json`). -## 4. Key decisions logged +## 3. The four policy targets (v1.25 scope) -| ID | Decision | Confidence | Source | -|----|----------|------------|--------| -| D-201 | Both promotion shapes supported (A: edit+destroy, B: per-env callers) | 0.95 | User directive | -| D-202 | Prior-env source of truth = `nova-contracts` DynamoDB table | 0.85 | User directive + code inspection | -| D-203 | Detect-and-destroy at pipeline start (Step 0b) | 0.85 | User directive | -| D-204 | Fail closed on destroy failure (no orphan path) | 0.95 | User directive | -| D-205 | Cross-account destroy out of scope (same-account only) | 0.80 | CLARIFY A3 | -| D-206 | Env-transition destroy is NOT the HITL decommission pipeline | 0.85 | CLARIFY A5 | -| D-207 | Last-applied env recorded via `#LAST_APPLIED` SK in `nova-contracts` | 0.85 | RESEARCH §2 | -| D-208 | State key `spike/{id}/{env}/` stays as-is (correct for both shapes) | 0.90 | RESEARCH §1b | +### 3.1 Consumer contract JSON (REQ-295) -## 5. Pitfalls +The payload is the parsed contract dict (the raw YAML loaded as JSON). +Policies assert the `contract.schema.json` constraints declaratively: +`require-id-pattern` (JMESPath regex `^[a-z][a-z0-9-]{2,5}$` over +`id`), `require-env-in-enum` (`environment` in `["dev","qa","prod","dr"]`), +`require-infrastructure-min-1` (`length(infrastructure) > 0`), +`forbid-unknown-fields` (keys subset of the 4 allowed). These are the +declarative equivalent of the jsonschema constraints — they let Nova +apply its own compliance posture (e.g. forbid a specific env for a +specific consumer) on top of schema validity without editing the +jsonschema. -1. **Destroy needs the prior env's Terraform config, not the new env's.** - The destroy step must re-resolve the contract with - `environment_override=prior_env` so the emitted TF matches the prior - env's resources. If we resolve with the new env, the destroy plan won't - match the prior state → terraform tries to create, not destroy. -2. **`terraform init -reconfigure` is required** when switching state - backends between envs (if envs use different state buckets). The - `-reconfigure` flag tells Terraform to forget the previous backend config. -3. **The `deletion_protection` NFR (REQ-86) blocks destroy.** The destroy - step must resolve with `deletion_protection: false` injected (same as - decommission Step 2 in `scripts/run_decommission.sh:34-37`). Without - this, `terraform destroy` fails on `prevent_destroy` lifecycle blocks. -4. **DynamoDB may not be reachable in local/CI mode.** The detect step - must handle `ClientError` / `EndpointNotFound` gracefully → log warning - + return `None` (conservative). This is documented in REQ-282. -5. **The consumer guide test `test_consumer_guide_states_no_field_editing` - will fail after the Step 8 rewrite.** It must be updated in the same - phase as the guide edit (P1) or the test suite breaks. \ No newline at end of file +**Invocation point:** `core/contract_resolver.py` pre-resolve (REQ-296). +Early-fail: if a contract policy fails, the resolver still proceeds +(the confidence signal decides the gate, consistent with the existing +`--soft-fail` Checkov pattern) — but the failing PCRs are in the +`policy` input, which lowers the score. + +### 3.2 Resolved Target Stack IR JSON (REQ-297) + +The payload is the resolved Stack IR dict produced by +`core/contract_resolver.py` (the merged module outputs). Policies +assert over `resources[]` (the array of resolved resources): +`require-tagging-standard` (every resource's `tags` has `nova:owner` + +`nova:environment` — ports +`adapters/terraform/policy/custom_rules/nova_tagging.py`), +`forbid-public-ingress` (no resource has `public_ingress: true` — the +v1.0 demo rule, now declarative), `require-encryption-by-default` (every +S3/EBS/KMS-aliased resource carries encryption config — ports the v1.8 +D-encryption-default rule). The `~` modifier iterates `resources[]`. + +**Invocation point:** `core/contract_resolver.py` post-resolve (REQ-298). +Additive — the resolver's return values and exceptions are unchanged; +the PCRs are appended to the contract-policy PCRs. + +### 3.3 Terraform plan JSON (REQ-300) + +The payload is `terraform show -json ` output. Policies assert +over `planned_values.root_module.resources[]`: +`forbid-plaintext-secrets` (no `aws_db_instance.password` / +`aws_iam_user.login_profile.password` in plaintext — ports +`CKV_AWS_41/45/46`), `forbid-iam-wildcard` (no `Action: "*"` or +`Resource: "*"` in `aws_iam_policy.PolicyDocument` — ports +`CKV_AWS_1/40`), `require-kms-reference` (KMS keys referenced by alias, +not inline key material — ports `CKV_AWS_7/33`). These are declarative +**mirrors** of `checkov_adapter.py:RULE_MAP` — the Checkov rule stays +the source of truth for `terraform_plan` scanning; the kyverno-json +policy covers the same plan JSON with a different rule language +(defense-in-depth against engine drift). + +**Invocation point:** `run_platform.sh` Step 5 (REQ-301). After +Checkov/Wiz produce raw PCRs, the script runs `kj scan` over the plan +JSON; both PCR lists concatenate into the confidence signal's `policy` +input. When `which kj` is false, the script logs and proceeds with the +Checkov/Wiz list only. + +### 3.4 PolicyCheckResult records (meta-policies, REQ-303) + +The payload is the **merged** `list[PolicyCheckResult]` produced by +checkov + wiz + the plan-JSON policies. This is the most novel target — +kyverno-json policies over the policy results themselves. +`block-on-any-critical` asserts no PCR has `severity: "critical"` + +`result: "fail"`; if any does, the meta-policy emits a `fail` PCR with +`ruleId: "KJ_META_BLOCK_CRITICAL"` and severity `critical`. This is the +declarative source of truth for "critical = block" (D-119 — the +`confidence_signal.py` `PENALTY["critical"]: None` hard-override stays +as defense-in-depth). `tagging-rules-agree` cross-checks the Checkov +`NOVA_TAG_NAMING` result against the kyverno-json +`KJ_REQUIRE_TAGGING_STANDARD` result by `resourceRef`; divergence emits +an `error` PCR (D-118). + +**Invocation point:** after the three target policies (contract/stack- +IR/plan-JSON) produce their PCR lists, the merged list is the payload +for the meta-policies. The meta-policy PCRs are appended to the merged +list, which is what the confidence signal consumes. + +## 4. The `PolicyEngine` swap boundary + +### 4.1 Protocol shape (REQ-291) + +A Python `Protocol` (PEP 544 — structural subtyping, no inheritance): +```python +class PolicyEngine(Protocol): + @property + def name(self) -> str: ... + def is_configured(self) -> bool: ... + def evaluate(self, payload: dict | str, policy_dir: Path, + contract_id: str) -> list[dict]: ... +``` +`list[dict]` (not `list[PolicyCheckResult]` — there's no dataclass; the +schema is enforced via `jsonschema` validation in tests, matching the +existing adapter pattern). The registry selects the active engine from +`config.json.policy.engine`. A `NullEngine` is the fallback when the +`policy` key is absent (emits `SKIPPED` — backward compatibility for +tests that don't set the key). + +### 4.2 The OPA-equivalent surface (future swap) + +OPA (Open Policy Agent) is the most likely future replacement. The +mapping: +| Nova `PolicyEngine` member | kyverno-json impl | OPA equivalent | +|---|---|---| +| `name` | `"kyverno-json"` | `"opa"` | +| `is_configured()` | `which kj` | `which opa` | +| `evaluate(payload, policy_dir, contract_id)` | `kj scan --policy --payload -o json` | `opa eval -d -i 'data.nova.<...>'` | +| Policy file format | `ValidatingPolicy` (YAML) | Rego (`.rego`) | +| Result shape | `results[]` (pass/fail/error/skip) | `result` (set of violations) | +| Severity | Nova annotation `nova.cloudinit.dev/severity` | Nova convention (Rego `metadata` or a wrapper) | + +The protocol is minimal (3 members) specifically so the OPA +implementation is a known quantity: an `OpaEngine` class that shells to +`opa eval`, translates the Rego violation set to PCR dicts, and +implements `is_configured()` via `which opa`. The policy *files* would +need rewriting (Rego, not ValidatingPolicy) — but the protocol, the +registry, the confidence signal, and the PCR schema are all untouched. +This is the swap boundary the user asked for ("Implemented as an +adapter since we might one day decide to replace it with something else +like OPA"). + +### 4.3 Why not a full plugin registry? + +A `setuptools` entry-point plugin registry (like checkov's +`--external-checks-dir`) was considered and rejected: Nova has 1 active +engine today (kyverno-json) and at most 2 in the foreseeable future +(kyverno-json + OPA). A `Protocol` + `dict` registry in +`core/policy_engine.py` is the right weight — discoverable, typed, +testable, and ~40 lines. An entry-point registry adds packaging +complexity (entry-point metadata, version resolution) for no gain at +this scale. The `register(name, factory)` method on the registry is +the extension point if a future milestone needs runtime plugin +discovery. + +## 5. Latency / MTTR impact (G-Q3 anticipation) + +NORTH_STAR.md MTTR target: < 60s p95. `run_platform.sh` Step 5 today +runs Checkov over the terraform plan (typically 5-15s for a small +stack). Adding `kj scan` over the same plan JSON adds: +- Process spawn: ~50ms (Go binary startup) +- Policy load: ~20ms (a handful of YAML files) +- Assertion evaluation: ~100-500ms (JMESPath over a small plan) +- Total: < 1s for a typical Nova stack + +The kyverno-json pass runs **in parallel** with Checkov (REQ-301 — the +script launches both and waits on both), so the wall-clock impact is +`max(checkov_time, kj_time)` ≈ checkov_time (kj is faster). The +contract + stack-IR policies run during resolve (already a fast step). +Meta-policies run over the merged list (in-memory, < 10ms). **No +measurable MTTR impact** is expected. This will be verified in P3 +VERIFY with a timing assertion. + +## 6. "Platform functions without AI" tenet (G-Q1 / D-120) + +kyverno-json is deterministic (same policy + payload → same result, +every run). It is not an LLM, not a probabilistic model, not a +"judgement" engine. The NORTH_STAR.md tenet ("the platform functions +without AI — 'AI decisions' are really automated decisions") +distinguishes AI (non-reproducible) from automation (reproducible). +kyverno-json is the latter. Adding it is **more** aligned with the +tenet than the current imperative Python in `core/env_transition.py` +and `core/regression_verify.py`, because the policy is declarative +(visible, auditable, version-controlled) rather than imperative (logic +hidden in function bodies). The `is_configured()` guard ensures the +platform functions without the binary (graceful skip → `SKIPPED` PCR +→ confidence signal proceeds). + +## 7. ECS policy catalog overlap (prior art) + +The kyverno-json catalog ships ECS policies that overlap with Nova's +L1 modules: `ecs-cluster-enable-logging`, `ecs-cluster-required- +container-insights`, `ecs-service-public-ip`, `ecs-service-required- +latest-platform-fargate`, `ecs-task-definition-fs-read-only`. These are +**reference policies**, not drop-in Nova policies — they target the +AWS ECS API shape (`type: aws_ecs_service` etc.), not Nova's Stack IR +shape. v1.25 policies target the Nova IR (REQ-297) and the terraform +plan JSON (REQ-300), not the raw AWS API. The catalog is useful as +prior art for JMESPath patterns over ECS resources — the +`ecs-service-public-ip` policy's `contains('$allowed-values', +@.assign_public_ip)` pattern informs the Nova `forbid-public-ingress` +policy shape. No catalog policies are imported directly in v1.25. + +## 8. Risks & mitigations + +| Risk | Likelihood | Impact | Mitigation | +|---|---|---|---| +| `kj` binary not in CI image | medium | blocks P3+ tests | `is_configured()` guard + `pytest.skip` + `scripts/install-kyverno-json.sh` | +| kyverno-json output shape changes across versions | low | breaks `_to_pcr()` | pin `@latest` to a known-good commit in `install-kyverno-json.sh` after P1 smoke; defensive parsing (malformed → `error` PCR, not exception) | +| Policy explosion (4 targets × N rules) | medium | maintenance load | wave ordering (PLAN); policies co-located per target dir; meta-policy cross-check keeps the set auditable | +| Checkov + kj tagging-rule drift | medium | false `error` PCRs | `tagging-rules-agree` meta-policy emits `error` on divergence (visible, not silent); the Checkov rule stays source of truth for HCL, kj for IR | +| OPA swap turns out harder than the protocol implies | low | future milestone rework | RESEARCH §4.2 documents the OPA-equivalent surface; the protocol is the contract, not the implementation | +| `--pre-process` needed for meta-policies but undocumented behavior | low | meta-policy bugs | v1.25 meta-policies use plain assertion trees over the PCR list (no pre-process); `--pre-process` noted as a future optimization only | + +## 9. Assumptions (logged, full autonomy) + +- A1: `kj scan --output json` produces a stable `results[]` array shape. + Will be verified in P1 smoke test (`_smoke.json` policy + a trivial + payload); if the shape differs, `_to_pcr()` is adjusted defensively + (malformed → `error` PCR). Confidence: 0.85. +- A2: The `nova.cloudinit.dev/severity` annotation convention is + read by the engine from the policy YAML (loaded once per evaluate() + call). kyverno-json does not validate unknown annotations — they pass + through. Confidence: 0.90. +- A3: The `~` projection modifier iterates `resources[]` in the Stack + IR and `planned_values.root_module.resources[]` in the plan JSON + correctly. Verified in P2/P3 tests. Confidence: 0.85. +- A4: `go install` works in the CI image (Go toolchain available or + installable). If not, the binary-release download path is the + documented fallback in `install-kyverno-json.sh`. Confidence: 0.80. +- A5: The `NullEngine` fallback (when `policy` key absent in + config.json) keeps all existing tests passing — they don't set the + key, so they get `NullEngine` → `SKIPPED` PCRs → confidence signal + proceeds with `policy` input `[SKIPPED]` → per-input score 1.0 + (skipped counts as pass in `_per_input_score`). Confidence: 0.95 + (verified against `confidence_signal.py:84-89`). + +## 10. Decisions referenced + +D-115 (install path), D-116 (engine enum reuse), D-117 (adapter +signatures unchanged), D-118 (tagging cross-check), D-119 (critical- +override defense-in-depth), D-120 (deterministic not AI). See +CLARIFY.md for the full resolution text. + +## 11. Architecture updates (deferred to RESEARCH-stage file edits) + +- `.ciagent/ARCHITECTURE.md` gains §12.7 "Policy Engine Registry" with + the registry diagram. Deferred to the RESEARCH commit (this file's + commit) — the section is authored as part of this research. +- `schemas/README.md` notes `engine: "kyverno"` is shared by the K8s + adapter and kyverno-json (distinguished by `ruleId` prefix). +- `modules/STANDARDS.md` gains a "Policy authoring standard" section + (P4, REQ-307). +- `docs/METRICS.md` notes the policy engine is swappable (P4, REQ-307). \ No newline at end of file diff --git a/.ciagent/ROADMAP.md b/.ciagent/ROADMAP.md index 3c29170..c1e473c 100644 --- a/.ciagent/ROADMAP.md +++ b/.ciagent/ROADMAP.md @@ -2156,3 +2156,93 @@ release). **DONE.** milestone). Tag `v1.22.6` (final patch = milestone release). Merge `milestone/v1.23-deck-cleanup-python-pptx` → `main`. - **Requirements:** REQ-263..275 (13 requirements). + +## v1.25 (active, tag line `v1.24.x`): kyverno-json Unified Policy Engine + +`kyverno-json` — a Kyverno-ecosystem runtime that applies Kyverno policies +to **any** JSON/YAML payload — becomes Nova's **primary compliance / +policy tool**, implemented behind a swappable `PolicyEngine` adapter so +OPA (or any other engine) can replace it one day. The unified-orchestrator +model: Checkov and Wiz remain as raw-finding adapters feeding *into* +kyverno-json meta-policies; the confidence signal is untouched (it already +consumes `list[PolicyCheckResult]` engine-agnostically). Policies cover +all four Nova artifacts: consumer contract JSON, resolved Stack IR, +Terraform plan JSON, and the merged PCR list itself (meta-validation). +The K8s-only Kyverno adapter stays documentation-only (D-053); the +kyverno-json engine and the K8s adapter are siblings, not replacements. +Quality improvement from the IDEATE pass: capability regression checks +(`core/regression_verify.py` CAP-013/023/024) become declarative +kyverno-json policies. New `policy-engineer` persona owns the policy +territory. 19 requirements (REQ-291..309), 6 phases (P0 + P1..P4 + P5 +final). Tags: `v1.24.0` (P0) → `v1.24.5` (P5 = milestone release). + +### Phase P1 — engine-core (planned, tag v1.24.1) +- REQ-291: `core/policy_engine.py` — `PolicyEngine` Protocol + + `PolicyEngineRegistry` (selects engine from `config.json.policy.engine`). +- REQ-292: `config.json` gains `policy` object + (`engine: "kyverno-json"`, `policy_root`). +- REQ-293: `adapters/kyverno-json/kyverno_json_engine.py` — + `KyvernoJsonEngine` (shells to `kj scan`; translates native output → + PCR; `is_configured()` guards on `which kj`). +- REQ-294: `adapters/kyverno-json/__init__.py` + `_smoke.json` policy + + `scripts/install-kyverno-json.sh` + CI image install. +- REQ-308: `tests/test_policy_engine.py` — protocol conformance, + registry, NullEngine fallback. +- REQ-309: `tests/test_kyverno_json_engine.py` — PCR schema validity, + defensive parsing, `pytest.skip` when kj absent. + +### Phase P2 — contract + stack-IR policies (planned, tag v1.24.2) +- REQ-295: `adapters/kyverno-json/policies/contract/` — 4 policies over + consumer contract JSON (id-pattern, env-enum, infra-min-1, + forbid-unknown-fields). +- REQ-296: `core/contract_resolver.py` invokes the engine pre-resolve + (contract policies) — early-fail, confidence signal decides the gate. +- REQ-297: `adapters/kyverno-json/policies/stack-ir/` — 3 policies over + resolved Stack IR (tagging-standard, public-ingress, encryption-by- + default — ports of v1.0/v1.8 imperative rules). +- REQ-298: `core/contract_resolver.py` invokes the engine post-resolve + (stack-IR policies); additive — existing tests pass. +- REQ-299: `tests/test_stack_ir_policies.py` + fixtures (passing + failing + IR; skip when kj absent). + +### Phase P3 — plan-JSON policies + meta-orchestration + pipeline wiring (planned, tag v1.24.3) +- REQ-300: `adapters/kyverno-json/policies/plan-json/` — 3 policies over + `terraform show -json` (plaintext-secrets, iam-wildcard, kms-reference + — ports of `checkov_adapter.py:RULE_MAP`). +- REQ-301: `run_platform.sh` Step 5 gains a parallel kyverno-json pass; + both PCR lists (checkov/wiz + kj) concatenate into the confidence + signal's `policy` input; skips gracefully when `which kj` is false. +- REQ-302: `tests/test_plan_json_policies.py` + fixtures; + `tests/test_run_platform_plan_json_policies.py` (script-substring + assertion). +- REQ-303: `adapters/kyverno-json/policies/meta/` — + `block-on-any-critical.json` (declarative critical-block; the + `confidence_signal.py` hard-override stays as defense-in-depth) + + `tagging-rules-agree.json` (asserts Checkov + kj agree on tagging). + `tests/test_meta_policies.py`. + +### Phase P4 — regression-gate policies + docs (planned, tag v1.24.4) +- REQ-304: `adapters/kyverno-json/policies/regression/` — 3 policies over + capability-inventory JSON (CAP-013/023/024) — declarative mirrors of + `core/regression_verify.py` checks. +- REQ-305: `tests/test_regression_policies.py` + fixtures (clean + + drifted inventory); regression gate still 287/287 baseline. +- REQ-306: `adapters/README.md` (new adapter row + PolicyEngine Protocol + section) + `adapters/kyverno-json/README.md`. +- REQ-307: `.ciagent/ARCHITECTURE.md` §12.7 (Policy Engine Registry) + + `schemas/README.md` + `modules/STANDARDS.md` (policy-authoring + standard) + `docs/METRICS.md` (swappable engine narrative). + +### Phase P5 — final review + audit + milestone ship (Final Phase, tag v1.24.5) +- Multi-persona code review across P1..P4 (lead-developer, backend- + engineer, data-engineer, policy-engineer). Auto-fix P0; flag P1+. +- Audit: reconstruction test (git log ↔ `.ciagent/`), branch hygiene, + commit discipline. +- Milestone ship: merge `phase/05-final-review-ship` → + `milestone/v1.25-kyverno-json` → `main`; tag `v1.24.5` (= the v1.25 + release per prev-minor tagging rule); create Gitea release with full + milestone summary; delete all milestone branches. +- Update `REQUIREMENTS.md` (mark REQ-291..309 complete), `ROADMAP.md` + (mark v1.25 complete), `NORTH_STAR.md` (note Strategic Objective #2 — + provable trust via a replaceable policy-engine substrate). +- **Requirements:** REQ-291..309 (19 requirements). diff --git a/.ciagent/VERIFY.md b/.ciagent/VERIFY.md index f6910dd..61e9e0a 100644 --- a/.ciagent/VERIFY.md +++ b/.ciagent/VERIFY.md @@ -1,135 +1,87 @@ -# ACDL v1.10 — Verify (milestone gate) +# VERIFY — P1 engine-core (v1.25) -> Verify date: 2026-07-27. Verifier: ci-verifier. Milestone: v1.10 (complete, tag `v1.10.0`). -> Scope: 4 phases (52–55), 5 commits (772ac72..2697775), 22 files, +2281/-256 lines. +> 4-layer verify gate: structural, behavioral, security, quality. +> Phase: P1. Requirements: REQ-291..294, 308, 309. Result: PASS. -## Layer 1: Structural — PASS +## Structural -- All 8 plan-referenced files exist on disk (`core/regression_verify.py`, - `core/local_emulators.py`, `scripts/run_regression.sh`, - `tests/test_verify_regression_mode.py`, - `tests/test_local_emulating_adapters.py`, - `.ciagent/CAPABILITY_INVENTORY.md`, `REGRESSION_REPORT.md`, - `REGRESSION_REPORT.json`). -- All imports resolve (`py_compile` + runtime import OK). -- No TODO/FIXME/HACK/stub placeholders in new code (the `LocalLambdaStub` - is a legitimate local emulator, not a placeholder). -- All declared exports exist (`run_regression`, `write_report`, - `CAPABILITY_REGISTRY`, `RegressionReport`, `CapabilityResult`, - `FlatFileOutbox`, `LocalEcsEmulator`, `LocalS3StateBackend`, - `LocalLambdaStub`, `run_local_e2e`, `is_local_tier`). +- `core/policy_engine.py` exists, implements `PolicyEngine` Protocol + (PEP 544, `@runtime_checkable`), `PolicyEngineRegistry` with + `register()` + `get_engine()`, `NullEngine` fallback. +- `adapters/kyverno-json/kyverno_json_engine.py` exists, exports + `KyvernoJsonEngine` with `name`, `is_configured()`, `evaluate()`. +- `adapters/kyverno-json/__init__.py` loads the engine by file path + (the dir name has a hyphen — not a valid Python package name). +- `adapters/kyverno-json/policies/_smoke.json` exists (trivial policy + for round-trip validation). +- `scripts/install-kyverno-json.sh` exists (go install kj@latest). +- `.ciagent/config.json` has the `policy` object + (`engine: kyverno-json`, `policy_root`). +- `.gitea/workflows/ci.yml` + `.github/workflows/ci.yml` have the + Go + kj install step (best-effort, tests skip when kj absent). +- `tests/test_policy_engine.py` (10 tests) + + `tests/test_kyverno_json_engine.py` (16 tests) exist. -## Layer 2: Behavioral — PASS +## Behavioral -- `pytest tests/ -m "not slow"`: **513 passed**, 5 deselected. -- `pytest tests/ -m slow`: **5 passed** (2 local E2E + 3 regression - integration incl. live-AWS terraform plan). -- **Total: 518 passed, 0 failed.** -- Requirement coverage: REQ-112 (P52), REQ-113 (P53), REQ-114 (P54), - REQ-115 (P55) — all 4 marked `complete`. -- Regression gate: `bash scripts/run_regression.sh` → **16/16 - capabilities Verified** (12 local + 4 live-AWS). Milestone gate open. +- `pytest tests/test_policy_engine.py tests/test_kyverno_json_engine.py`: + **24 passed, 2 skipped** (kj not installed — expected; + `pytest.skip("kj not installed")`). +- `NullEngine` satisfies the `PolicyEngine` Protocol (G-Q8a — + `isinstance(NullEngine(), PolicyEngine)` is True). Proves the swap + boundary is real without implementing OPA. +- `KyvernoJsonEngine.is_configured()` returns `False` when + `which kj` is absent → `evaluate()` returns a single + `KJ_ENGINE_NOT_CONFIGURED` SKIPPED PCR (distinct `ruleId` from + NullEngine's `NULL_ENGINE_INACTIVE` — G-Q4). +- PCR records validate against `schemas/policy_check_result.schema.json` + (via `jsonschema.validate` in tests). +- Defensive parsing: malformed kyverno-json output → `error` PCR + (`KJ_ENGINE_ERROR`), never an exception. +- Severity annotation reading (G-Q10a): policies with + `nova.cloudinit.dev/severity: high` produce PCRs with `severity: high`; + policies without the annotation default to `info`. +- Registry: `get_engine()` returns the configured engine; unknown + engine name raises `KeyError`; `policy` key absent → `NullEngine`. +- No regression: `pytest tests/test_confidence_signal.py + tests/test_adapter.py tests/test_checkov_adapter.py + tests/test_kyverno_adapter.py tests/test_contract_resolver.py` — + **132 passed** (unchanged). -## Layer 3: Security (STRIDE) — PASS +## Security -| Threat | Risk | Disposition | -|--------|------|-------------| -| Spoofing | Local Lambda stub patches `_get_dynamodb`/`_get_secrets_client`; opt-in via `ACDL_LOCAL_TIER=1`, never in prod | Accept (low) | -| Tampering | Flat-file outbox hash-chain verification detects tampering | Accept (low) | -| Repudiation | Regression report records per-capability status + timestamps | Accept (low) | -| Info Disclosure | Creds read into env vars, never logged (0 cred strings in reports); ECS binds 127.0.0.1 only | Accept (low) | -| Denial of Service | Local ECS emulator: free port, daemon thread, clean destroy | Accept (low) | -| Elevation of Privilege | `urllib.urlopen` patched to fake response (no network egress); no eval/exec/subprocess in adapter | Accept (low) | +- No new secrets, no new network calls in the engine core (the engine + shells to a local binary; the binary makes no network calls for + `scan`). +- `is_configured()` guard ensures the platform runs without the binary + (no hard dependency that could be exploited as a DoS vector). +- The engine writes the payload to a temp file (`tempfile.NamedTemporaryFile`) + and unlinks it in a `finally` block (no leftover payload on disk). +- No `shell=True` in the `subprocess.run` call (command is a list — + no shell injection surface). -All threats low-severity; auto-accepted per -`config.json security.auto_accept_low_severity=true`. +## Quality -## Layer 4: Quality (multi-persona) — PASS +- `python3 -m py_compile` passes on all new Python files. +- The `PolicyEngine` Protocol is minimal (3 members) — the swap + boundary is the moat (NORTH_STAR Strategic Objective #2). +- The `NullEngine` proves a second implementation exists (structural + conformance) — the OPA swap is a known quantity (RESEARCH §4.2). +- Tests use `pytest.skip` when `which kj` is absent, so the CI matrix + passes with or without the binary (the suite is green in both cases). -| Persona | Finding | Verdict | -|---------|---------|---------| -| Correctness | 7 adapter defects fixed; each traceable to a terraform validate/plan error | PASS | -| Testing | 518 tests pass; 24 new tests. P2: uptime-kuma + RDS not in registry | PASS (1 P2) | -| Security | No creds logged; loopback-only; monkey-patches scoped to local tier | PASS | -| Performance | Regression run ~60s; acceptable for a milestone gate | PASS | -| Maintainability | Well-structured; adding a capability = 1 function + 1 registry entry | PASS | -| Adversarial | Gate can't be bypassed; local E2E can't mutate cloud; no injection vectors | PASS | +## Must-have checklist -**0 P0, 0 P1, 1 P2 (post-hoc: expand regression registry to uptime-kuma + RDS stacks).** +- [x] `PolicyEngine` Protocol + `PolicyEngineRegistry` + `NullEngine` + (REQ-291) +- [x] `config.json.policy` object (REQ-292) +- [x] `KyvernoJsonEngine` adapter (REQ-293) +- [x] `__init__.py` + `_smoke.json` + `install-kyverno-json.sh` + CI + install (REQ-294) +- [x] `test_policy_engine.py` — protocol conformance, registry, + NullEngine fallback (REQ-308) +- [x] `test_kyverno_json_engine.py` — PCR schema validity, defensive + parsing, skip-without-kj (REQ-309) -## Verdict - -**VERIFY PASS** — all 4 layers pass. The v1.10 milestone is sound: -the pipeline regression gap is fixed (D-091), the platform is fully -locally testable (D-092), every advertised capability is re-verified -(D-093, 16/16 Verified), and the docs/decks match verified reality -(D-094). 518 tests pass; the regression gate covers 16 capabilities -including 4 live-AWS checks. 0 P0, 0 P1, 1 P2 post-hoc. Ready to ship. - ---- - -# ACDL — Verify (grill deliverable, commit ac11c01) - -> Verify date: 2026-07-27. Verifier: ci-verifier. Scope: the grill -> deliverable (`.ciagent/GRILL.md`, phase 0, status `grill`) added in -> commit `ac11c01` since the v1.10 audit PASS (`ab477b3`). Docs-only; -> no code, no tests, no schema changes. - -## Layer 1: Structural — PASS - -- `.ciagent/GRILL.md` exists on disk (18250 bytes). -- No imports to resolve (markdown docs file). -- No TODO/FIXME/HACK/stub placeholders in the report. -- All required sections present per grill workflow Step 5 format: - title, Run header, Verdict, 9 axes (1–9), Meta, Binding Decisions - table (12 rows), Escalations section (2 entries: G-005, G-008). -- Commit `ac11c01` `---ci---` block is well-formed: `project: acdl`, - `phase: 0`, `milestone: v1.10`, `status: grill`, 12 decision ids - (G-001..G-012), 2 escalation lines. - -## Layer 2: Behavioral — PASS - -- `pytest tests/ -m "not slow"`: **513 passed**, 5 deselected (no - regressions introduced by the docs-only grill commit). -- No new tests required (docs-only deliverable; the grill is a - review artifact, not a code change). -- Requirement coverage: not applicable (phase 0, status `grill`; no - REQ-IDs bound to this deliverable). The grill's binding decisions - (G-001..G-012) are advisory and do not modify REQUIREMENTS.md per - grill workflow Step 7. - -## Layer 3: Security (STRIDE) — PASS - -| Threat | Risk | Disposition | -|--------|------|-------------| -| Spoofing | N/A (docs-only; no auth surface) | Accept (none) | -| Tampering | Grill report is git-tracked; tampering = git history rewrite (out of scope) | Accept (low) | -| Repudiation | Commit `ac11c01` signed by author; `---ci---` block records status + decisions | Accept (low) | -| Info Disclosure | No credentials, keys, tokens, or PII in the report (grep scan clean) | Accept (low) | -| Denial of Service | N/A (docs file; no runtime surface) | Accept (none) | -| Elevation of Privilege | N/A (docs-only; no privilege surface) | Accept (none) | - -All threats low-or-none; auto-accepted per -`config.json security.auto_accept_low_severity=true`. - -## Layer 4: Quality (multi-persona) — PASS - -| Persona | Finding | Verdict | -|---------|---------|---------| -| Correctness | 12 binding decisions traceable to evidence (commit/file/req-id); 2 escalations correctly unresolved | PASS | -| Testing | Docs-only; 513 fast tests pass (no regression) | PASS | -| Security | No credential leakage; no sensitive data in report | PASS | -| Performance | N/A (docs file; no runtime cost) | PASS | -| Maintainability | Report follows grill workflow Step 5 format exactly; appendable for future runs | PASS | -| Adversarial | Escalations (G-005, G-008) are surfaced, not silently skipped; visible via `ciagent audit` | PASS | - -**0 P0, 0 P1, 0 P2.** - -## Verdict (grill deliverable) - -**VERIFY PASS** — all 4 layers pass. The grill deliverable is a -well-formed docs-only artifact. 513 fast tests pass (no regression). -No credential leakage. 12 binding decisions recorded; 2 escalations -(G-005 risks, G-008 budget) correctly surfaced for human resolution. -The grill does not modify PROJECT.md, ROADMAP.md, or REQUIREMENTS.md -(per grill workflow Step 7). \ No newline at end of file +**Verdict: PASS** — all P1 must-haves met, no regressions, 24 new +tests pass (2 skip-without-kj), 132 existing tests unchanged. \ No newline at end of file diff --git a/.ciagent/config.json b/.ciagent/config.json index dc350a3..acbcece 100644 --- a/.ciagent/config.json +++ b/.ciagent/config.json @@ -8,7 +8,7 @@ ], "active_project": "acdl", "active_projects": ["acdl"], - "active_milestone": "v1.24", + "active_milestone": "v1.25", "autonomy": { "level": "full", "escalation_hooks": ["deploy", "delete_data", "merge_to_main"], @@ -209,5 +209,9 @@ "enabled": true, "persist": true }, - "strategic_direction_file": ".ciagent/NORTH_STAR.md" + "strategic_direction_file": ".ciagent/NORTH_STAR.md", + "policy": { + "engine": "kyverno-json", + "policy_root": "adapters/kyverno-json/policies" + } } diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 839b9bd..965138c 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -63,6 +63,23 @@ jobs: - name: Install test dependencies run: pip install -r requirements-test.txt + - name: Install kyverno-json (kj) for policy-engine tests + run: | + # v1.25: kyverno-json is the primary policy engine. Tests that + # require kj skip when absent, so this is best-effort (the suite + # passes with or without kj). Install is cached via the Go + # module cache (~/.cache/go-build + ~/go/pkg/mod). + if command -v go >/dev/null 2>&1; then + go install github.com/kyverno/kyverno-json/cmd/kj@latest && \ + echo "$(go env GOPATH)/bin" >> "$GITHUB_PATH" || \ + echo "kj install failed; policy-engine tests will skip" + else + sudo apt-get update && sudo apt-get install -y golang-go && \ + go install github.com/kyverno/kyverno-json/cmd/kj@latest && \ + echo "$(go env GOPATH)/bin" >> "$GITHUB_PATH" || \ + echo "kj install failed; policy-engine tests will skip" + fi + - name: Run pytest run: python3 -m pytest tests/ -v --tb=short diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 839b9bd..827a19e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,6 +63,21 @@ jobs: - name: Install test dependencies run: pip install -r requirements-test.txt + - name: Install kyverno-json (kj) for policy-engine tests + uses: actions/setup-go@v5 + with: + go-version: "1.22" + cache: false + + - name: Install kj binary + run: | + # v1.25: kyverno-json is the primary policy engine. Tests that + # require kj skip when absent, so this is best-effort (the suite + # passes with or without kj). + go install github.com/kyverno/kyverno-json/cmd/kj@latest && \ + echo "$(go env GOPATH)/bin" >> "$GITHUB_PATH" || \ + echo "kj install failed; policy-engine tests will skip" + - name: Run pytest run: python3 -m pytest tests/ -v --tb=short diff --git a/adapters/README.md b/adapters/README.md index 17900eb..6862b8f 100644 --- a/adapters/README.md +++ b/adapters/README.md @@ -12,6 +12,37 @@ Adapters translate the engine-agnostic Target Stack IR to engine-specific format | Checkov adapter | `adapters/terraform/policy/checkov_adapter.py` | Checkov JSON | `PolicyCheckResult` records | Translates Checkov results | | Wiz adapter | `adapters/wiz/wiz_adapter.py` | Wiz API issues JSON | `PolicyCheckResult` records | Translates Wiz security findings | | Kyverno adapter | `adapters/kyverno/kyverno_adapter.py` | Kyverno PolicyReport JSON | `PolicyCheckResult` records | K8s-native policy translation | +| kyverno-json engine | `adapters/kyverno-json/kyverno_json_engine.py` | Any JSON/YAML payload | `PolicyCheckResult` records | **v1.25 primary policy engine** (swappable via `PolicyEngine` protocol) | + +## Policy Engine Protocol (v1.25) + +The `core/policy_engine.py` module defines the **swap boundary** between +Nova and its policy engines. A `PolicyEngine` Python Protocol (PEP 544) +with three members (`name`, `is_configured()`, `evaluate()`) is the +contract; a `PolicyEngineRegistry` selects the active engine from +`config.json`'s `policy.engine` key. The confidence signal and pipeline +never import an engine directly — they go through the registry. + +**Implementations:** +- `KyvernoJsonEngine` (`adapters/kyverno-json/`) — shells to the `kj` + CLI; the v1.25 default. +- `NullEngine` (`core/policy_engine.py`) — fallback when the `policy` + key is absent (emits `SKIPPED`). +- Future: `OpaEngine` — implements the same protocol, shells to + `opa eval`. The OPA-equivalent surface is documented in + `.ciagent/RESEARCH.md` §4.2. + +**How to add a new engine:** +1. Create `adapters//_engine.py` implementing the + `PolicyEngine` protocol (`name`, `is_configured()`, `evaluate()`). +2. `evaluate()` returns `list[dict]` where each dict conforms to + `schemas/policy_check_result.schema.json`. +3. Register the engine in `core/policy_engine.py`'s `_autoload_*` + function (or call `register(name, factory)` at startup). +4. Set `config.json.policy.engine` to the engine's `name`. +5. Add the engine to the `engine` enum in + `schemas/policy_check_result.schema.json` if it needs a distinct + enum value (v1.25 reuses `"kyverno"` — see D-116). ## How to Write an Adapter diff --git a/adapters/kyverno-json/README.md b/adapters/kyverno-json/README.md new file mode 100644 index 0000000..fd156d1 --- /dev/null +++ b/adapters/kyverno-json/README.md @@ -0,0 +1,103 @@ +# kyverno-json Engine Adapter (v1.25) + +The `kyverno-json` engine is Nova's **primary compliance/policy tool** +(v1.25), implemented behind the swappable `PolicyEngine` protocol so +OPA (or any other engine) can replace it one day. + +## What kyverno-json is + +[kyverno-json](https://github.com/kyverno/kyverno-json) is a standalone +Go binary from the Kyverno project — a **separate runtime** from the +K8s Kyverno admission controller. It applies Kyverno `ValidatingPolicy` +resources to **any** JSON or YAML payload file via the `kj scan` CLI. +Unlike the K8s Kyverno adapter (`adapters/kyverno/`), which only +speaks to K8s manifests, kyverno-json evaluates consumer contracts, +resolved Stack IR, terraform plan JSON, and even the merged PCR list +itself (meta-policies). + +## Install + +```bash +bash scripts/install-kyverno-json.sh +# or directly: +go install github.com/kyverno/kyverno-json/cmd/kj@latest +kj version +``` + +The platform functions without the binary — `is_configured()` returns +`False` when `which kj` is absent → `evaluate()` returns a single +`SKIPPED` PCR (`KJ_ENGINE_NOT_CONFIGURED`). The confidence signal +proceeds with a neutral `policy` input (D-120 graceful degradation). + +## Policy directory layout + +``` +adapters/kyverno-json/policies/ +├── _smoke.json # round-trip smoke test +├── contract/ # consumer contract JSON policies +│ ├── require-id-pattern.json +│ ├── require-env-in-enum.json +│ ├── require-infrastructure-min-1.json +│ └── forbid-unknown-fields.json +├── stack-ir/ # resolved Stack IR policies +│ ├── require-tagging-standard.json +│ ├── forbid-public-ingress.json +│ └── require-encryption-by-default.json +├── plan-json/ # terraform show -json policies +│ ├── forbid-plaintext-secrets.json +│ ├── forbid-iam-wildcard.json +│ └── require-kms-reference.json +├── meta/ # policies over the merged PCR list +│ ├── block-on-any-critical.json +│ └── tagging-rules-agree.json +└── regression/ # capability-inventory policies + ├── cap-013-adapter-dedup.json + ├── cap-023-metrics-collector.json + └── cap-024-deck-structure.json +``` + +## The four policy categories + +1. **contract/** — over the consumer contract JSON (pre-resolve). +2. **stack-ir/** — over the resolved Target Stack IR (post-resolve). +3. **plan-json/** — over `terraform show -json` output (pipeline Step 5b). +4. **meta/** — over the merged `list[PolicyCheckResult]` (meta-policies). +5. **regression/** — over the capability-inventory JSON (declarative + mirrors of `core/regression_verify.py`). + +## Severity convention + +kyverno-json does not natively assign severities. Each Nova policy +declares its severity via a `metadata.annotations` field: + +```yaml +metadata: + annotations: + nova.cloudinit.dev/severity: high +``` + +Valid values: `critical`, `high`, `medium`, `low`, `info` (default +when absent). + +## Engine enum reuse (D-116) + +kyverno-json PCR records carry `engine: "kyverno"` (no new enum value). +The `engine` field records the policy-engine *family*, not the specific +binary. The K8s Kyverno adapter and the kyverno-json engine are +distinguished by `ruleId` prefix (`KYVERNO_` vs `KJ_`) and `evidence` +payload shape (`namespace`/`kind` vs `assertion`/`jmespath`). + +## Schema path + +The output records validate against +[`schemas/policy_check_result.schema.json`](../../schemas/policy_check_result.schema.json) +(`engine: "kyverno"` is in the enum). The confidence signal consumes +the merged PCR list engine-agnostically. + +## Swap boundary + +The `PolicyEngine` protocol (`core/policy_engine.py`) is the swap +boundary. The OPA-equivalent surface is documented in +`.ciagent/RESEARCH.md` §4.2 — a future `OpaEngine` implements the same +protocol without touching the confidence signal, the PCR schema, or +the pipeline. \ No newline at end of file diff --git a/adapters/kyverno-json/__init__.py b/adapters/kyverno-json/__init__.py new file mode 100644 index 0000000..f6fc14d --- /dev/null +++ b/adapters/kyverno-json/__init__.py @@ -0,0 +1,27 @@ +"""Nova kyverno-json adapter package (v1.25, REQ-294). + +The directory name ``kyverno-json`` has a hyphen, so it is not a valid +Python package name and cannot be imported via ``import +adapters.kyverno-json``. The ``PolicyEngineRegistry`` loads the engine +by file path (``importlib.util.spec_from_file_location``). This +``__init__`` is a convenience for direct-script use and for ``pip +install -e .`` style discovery if the package is ever renamed. +""" + + +def _load_engine(): + import importlib.util + import os + engine_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), + "kyverno_json_engine.py") + spec = importlib.util.spec_from_file_location("kyverno_json_engine", engine_path) + if spec is None or spec.loader is None: + raise ImportError(f"could not load {engine_path}") + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod.KyvernoJsonEngine + + +KyvernoJsonEngine = _load_engine() + +__all__ = ["KyvernoJsonEngine"] \ No newline at end of file diff --git a/adapters/kyverno-json/kyverno_json_engine.py b/adapters/kyverno-json/kyverno_json_engine.py new file mode 100644 index 0000000..235ab96 --- /dev/null +++ b/adapters/kyverno-json/kyverno_json_engine.py @@ -0,0 +1,269 @@ +"""Nova KyvernoJsonEngine (REQ-293, v1.25). + +Implements the ``PolicyEngine`` protocol (``core/policy_engine.py``) +by shelling to the ``kj`` CLI (``kyverno-json``). Translates native +kyverno-json scan output to Nova ``PolicyCheckResult`` dicts +(``schemas/policy_check_result.schema.json``). + +Engine enum reuse (D-116): records carry ``engine: "kyverno"`` (no new +enum value). The ``ruleId`` is prefixed ``KJ_`` to +distinguish from the K8s Kyverno adapter's ``KYVERNO_`` prefix. + +Severity (RESEARCH §2.6, G-Q10a): kyverno-json does not natively assign +severities. Each Nova policy declares its severity via a +``metadata.annotations["nova.cloudinit.dev/severity"]`` field. The +engine reads this annotation from the loaded policy YAML (not from the +scan result — the result doesn't carry it) and applies it to every +result that policy produces. Default when absent: ``"info"``. + +Graceful degradation (D-120): ``is_configured()`` returns ``False`` when +``which kj`` is absent → ``evaluate()`` returns a single SKIPPED PCR +(``ruleId: KJ_ENGINE_NOT_CONFIGURED``). The platform functions without +the binary. + +Defensive parsing: any kyverno-json output that doesn't match the +expected shape produces an ``error`` PCR, never an exception. The +engine is read-only against a local policy dir + a temp payload file. +""" + +import datetime +import json +import os +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path +from typing import Any, Union + +import yaml + + +Payload = Union[dict, list, str] + +SEVERITY_DEFAULT = "info" +SEVERITY_ANNOTATION = "nova.cloudinit.dev/severity" + +RESULT_MAP = { + "pass": "pass", + "fail": "fail", + "error": "error", + "skip": "skipped", + "skipped": "skipped", + "warn": "skipped", + "warning": "skipped", +} + + +def _iso8601_now() -> str: + return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _which_kj() -> str | None: + """Return the path to ``kj`` if on PATH, else ``None``.""" + return shutil.which("kj") + + +def _load_policy_severities(policy_dir: Path) -> dict[str, str]: + """Load each ``.json``/``.yaml``/``.yml`` policy in ``policy_dir`` + (non-recursive) and return ``{policy_name: severity}``. + + kyverno-json policies are Kubernetes-style ``ValidatingPolicy`` + resources. The severity is read from + ``metadata.annotations["nova.cloudinit.dev/severity"]``. Policies + in subdirectories (e.g. ``contract/``, ``stack-ir/``) are loaded + when the caller passes that subdirectory as ``policy_dir``. + """ + severities: dict[str, str] = {} + if not policy_dir.is_dir(): + return severities + for entry in sorted(os.listdir(policy_dir)): + if entry.startswith("_") or entry.startswith("."): + continue + full = policy_dir / entry + if not full.is_file(): + continue + if entry.endswith((".json", ".yaml", ".yml")): + try: + with open(full, "r", encoding="utf-8") as fh: + doc = yaml.safe_load(fh) + if not isinstance(doc, dict): + continue + name = doc.get("metadata", {}).get("name") or entry.rsplit(".", 1)[0] + ann = doc.get("metadata", {}).get("annotations", {}) or {} + sev = ann.get(SEVERITY_ANNOTATION, SEVERITY_DEFAULT) + severities[name] = str(sev).lower() + except Exception: + continue + return severities + + +def _to_pcr(entry: dict, contract_id: str, severity: str) -> dict: + """Translate a kyverno-json scan result entry to a PCR dict.""" + policy_name = entry.get("policy", "") or "UNKNOWN" + rule_name = entry.get("rule", "") or "" + rule_id = f"KJ_{policy_name}" + if rule_name: + rule_id = f"{rule_id}/{rule_name}" + result_raw = entry.get("result", "skip") + result = RESULT_MAP.get(str(result_raw).lower(), "error") + message = entry.get("message", "") or "" + resource = entry.get("resource", "") + if not resource and entry.get("name"): + kind = entry.get("kind", "") + ns = entry.get("namespace", "") + resource = f"{kind}/{ns}/{entry.get('name')}" if kind else entry.get("name", "") + return { + "contractId": contract_id, + "evaluatedAt": _iso8601_now(), + "engine": "kyverno", + "ruleId": rule_id, + "severity": severity, + "result": result, + "message": message, + "evidence": { + "resource": resource, + "policy": policy_name, + "rule": rule_name, + "namespace": entry.get("namespace", ""), + "kind": entry.get("kind", ""), + "name": entry.get("name", ""), + }, + "resourceRef": resource, + } + + +def _skipped_not_configured(contract_id: str) -> dict: + return { + "contractId": contract_id, + "evaluatedAt": _iso8601_now(), + "engine": "kyverno", + "ruleId": "KJ_ENGINE_NOT_CONFIGURED", + "severity": "info", + "result": "skipped", + "message": ( + "kyverno-json engine not configured — `which kj` returned no path. " + "Install via scripts/install-kyverno-json.sh. The platform proceeds " + "with a neutral SKIPPED policy input (is_configured() guard, D-120)." + ), + "evidence": {}, + "resourceRef": "", + } + + +def _error_pcr(contract_id: str, message: str) -> dict: + return { + "contractId": contract_id, + "evaluatedAt": _iso8601_now(), + "engine": "kyverno", + "ruleId": "KJ_ENGINE_ERROR", + "severity": "info", + "result": "error", + "message": message, + "evidence": {}, + "resourceRef": "", + } + + +class KyvernoJsonEngine: + """``PolicyEngine`` impl that shells to the ``kj`` CLI.""" + + name = "kyverno-json" + + def is_configured(self) -> bool: + return _which_kj() is not None + + def evaluate(self, payload: Payload, policy_dir: Path, + contract_id: str) -> list[dict]: + if not self.is_configured(): + return [_skipped_not_configured(contract_id)] + kj = _which_kj() + policy_dir = Path(policy_dir) + if not policy_dir.is_dir(): + return [_error_pcr( + contract_id, + f"kyverno-json policy dir not found: {policy_dir}", + )] + severities = _load_policy_severities(policy_dir) + # Write payload to temp file (kj scan --payload expects a file path). + payload_tmp = tempfile.NamedTemporaryFile( + mode="w", suffix=".json", delete=False, encoding="utf-8" + ) + try: + json.dump(payload, payload_tmp) + payload_tmp.flush() + payload_tmp.close() + cmd = [ + kj, "scan", + "--policy", str(policy_dir), + "--payload", payload_tmp.name, + "--output", "json", + ] + try: + proc = subprocess.run( + cmd, capture_output=True, text=True, timeout=60, + ) + except subprocess.TimeoutExpired: + return [_error_pcr(contract_id, "kyverno-json scan timed out (60s)")] + if proc.returncode not in (0, 1): + return [_error_pcr( + contract_id, + f"kyverno-json scan exited {proc.returncode}: {proc.stderr[:200]}", + )] + try: + out = json.loads(proc.stdout) if proc.stdout.strip() else {} + except json.JSONDecodeError as e: + return [_error_pcr( + contract_id, + f"kyverno-json output not JSON: {e}", + )] + return self._translate(out, contract_id, severities) + finally: + try: + os.unlink(payload_tmp.name) + except OSError: + pass + + def _translate(self, out: dict, contract_id: str, + severities: dict[str, str]) -> list[dict]: + results = out.get("results", []) if isinstance(out, dict) else [] + if not isinstance(results, list): + results = [] + pcrs: list[dict] = [] + for entry in results: + if not isinstance(entry, dict): + continue + policy_name = entry.get("policy", "") or "UNKNOWN" + severity = severities.get(policy_name, SEVERITY_DEFAULT) + pcrs.append(_to_pcr(entry, contract_id, severity)) + if not pcrs: + # No results — kyverno-json produced nothing (no match, or + # all policies passed with no result entries). Emit a + # single pass PCR so the confidence signal's policy input + # is non-empty (a non-empty list of passes → score 1.0). + pcrs.append({ + "contractId": contract_id, + "evaluatedAt": _iso8601_now(), + "engine": "kyverno", + "ruleId": "KJ_NO_RESULTS", + "severity": "info", + "result": "pass", + "message": "kyverno-json scan produced no result entries (all policies passed or no match).", + "evidence": {}, + "resourceRef": "", + }) + return pcrs + + +if __name__ == "__main__": + if len(sys.argv) < 4: + print( + "usage: kyverno_json_engine.py ", + file=sys.stderr, + ) + sys.exit(2) + with open(sys.argv[1], "r", encoding="utf-8") as fh: + pl = json.load(fh) + engine = KyvernoJsonEngine() + out = engine.evaluate(pl, Path(sys.argv[2]), sys.argv[3]) + print(json.dumps(out, indent=2)) \ No newline at end of file diff --git a/adapters/kyverno-json/policies/_smoke.json b/adapters/kyverno-json/policies/_smoke.json new file mode 100644 index 0000000..c35e67d --- /dev/null +++ b/adapters/kyverno-json/policies/_smoke.json @@ -0,0 +1,30 @@ +{ + "apiVersion": "json.kyverno.io/v1alpha1", + "kind": "ValidatingPolicy", + "metadata": { + "name": "require-contract-id", + "annotations": { + "nova.cloudinit.dev/severity": "high", + "title.policy.kyverno.io": "Require contract id" + } + }, + "spec": { + "rules": [ + { + "name": "require-id", + "validate": { + "message": "contract id is required", + "assert": { + "all": [ + { + "check": { + "id": "(regex_match('^[a-z][a-z0-9-]{2,5}$', @))" + } + } + ] + } + } + } + ] + } +} \ No newline at end of file diff --git a/adapters/kyverno-json/policies/contract/forbid-unknown-fields.json b/adapters/kyverno-json/policies/contract/forbid-unknown-fields.json new file mode 100644 index 0000000..9e884af --- /dev/null +++ b/adapters/kyverno-json/policies/contract/forbid-unknown-fields.json @@ -0,0 +1,31 @@ +{ + "apiVersion": "json.kyverno.io/v1alpha1", + "kind": "ValidatingPolicy", + "metadata": { + "name": "forbid-unknown-fields", + "annotations": { + "nova.cloudinit.dev/severity": "low", + "title.policy.kyverno.io": "Contract has only schema-allowed fields" + } + }, + "spec": { + "rules": [ + { + "name": "no-unknown-fields", + "validate": { + "message": "contract may only contain id, name, environment, infrastructure (schema-allowed fields)", + "assert": { + "all": [ + { + "check": { + "(length(keys(@)) == `4`)": true, + "keys(@)": "(contains(['id','name','environment','infrastructure'], @))" + } + } + ] + } + } + } + ] + } +} \ No newline at end of file diff --git a/adapters/kyverno-json/policies/contract/require-env-in-enum.json b/adapters/kyverno-json/policies/contract/require-env-in-enum.json new file mode 100644 index 0000000..54325c7 --- /dev/null +++ b/adapters/kyverno-json/policies/contract/require-env-in-enum.json @@ -0,0 +1,30 @@ +{ + "apiVersion": "json.kyverno.io/v1alpha1", + "kind": "ValidatingPolicy", + "metadata": { + "name": "require-env-in-enum", + "annotations": { + "nova.cloudinit.dev/severity": "high", + "title.policy.kyverno.io": "Contract environment is one of dev/qa/prod/dr" + } + }, + "spec": { + "rules": [ + { + "name": "env-enum", + "validate": { + "message": "contract.environment must be one of dev, qa, prod, dr", + "assert": { + "all": [ + { + "check": { + "environment": "(contains(['dev','qa','prod','dr'], @))" + } + } + ] + } + } + } + ] + } +} \ No newline at end of file diff --git a/adapters/kyverno-json/policies/contract/require-id-pattern.json b/adapters/kyverno-json/policies/contract/require-id-pattern.json new file mode 100644 index 0000000..b8810ef --- /dev/null +++ b/adapters/kyverno-json/policies/contract/require-id-pattern.json @@ -0,0 +1,30 @@ +{ + "apiVersion": "json.kyverno.io/v1alpha1", + "kind": "ValidatingPolicy", + "metadata": { + "name": "require-id-pattern", + "annotations": { + "nova.cloudinit.dev/severity": "high", + "title.policy.kyverno.io": "Contract id matches operational acronym pattern" + } + }, + "spec": { + "rules": [ + { + "name": "id-pattern", + "validate": { + "message": "contract.id must match ^[a-z][a-z0-9-]{2,5}$ (3-6 char operational acronym)", + "assert": { + "all": [ + { + "check": { + "id": "(regex_match('^[a-z][a-z0-9-]{2,5}$', @))" + } + } + ] + } + } + } + ] + } +} \ No newline at end of file diff --git a/adapters/kyverno-json/policies/contract/require-infrastructure-min-1.json b/adapters/kyverno-json/policies/contract/require-infrastructure-min-1.json new file mode 100644 index 0000000..862c42d --- /dev/null +++ b/adapters/kyverno-json/policies/contract/require-infrastructure-min-1.json @@ -0,0 +1,30 @@ +{ + "apiVersion": "json.kyverno.io/v1alpha1", + "kind": "ValidatingPolicy", + "metadata": { + "name": "require-infrastructure-min-1", + "annotations": { + "nova.cloudinit.dev/severity": "medium", + "title.policy.kyverno.io": "Contract declares at least one infrastructure entry" + } + }, + "spec": { + "rules": [ + { + "name": "infra-min-1", + "validate": { + "message": "contract.infrastructure must have at least one module entry", + "assert": { + "all": [ + { + "check": { + "infrastructure": "(length(keys(@)) > `0`)" + } + } + ] + } + } + } + ] + } +} \ No newline at end of file diff --git a/adapters/kyverno-json/policies/meta/block-on-any-critical.json b/adapters/kyverno-json/policies/meta/block-on-any-critical.json new file mode 100644 index 0000000..96d5767 --- /dev/null +++ b/adapters/kyverno-json/policies/meta/block-on-any-critical.json @@ -0,0 +1,32 @@ +{ + "apiVersion": "json.kyverno.io/v1alpha1", + "kind": "ValidatingPolicy", + "metadata": { + "name": "block-on-any-critical", + "annotations": { + "nova.cloudinit.dev/severity": "critical", + "title.policy.kyverno.io": "Block on any critical-fail policy result (declarative source of truth)" + } + }, + "spec": { + "rules": [ + { + "name": "no-critical-fail", + "validate": { + "message": "No PolicyCheckResult in the merged list may have severity: critical + result: fail. The confidence_signal.py hard-override is the defense-in-depth behind this declarative rule (D-119).", + "assert": { + "all": [ + { + "check": { + "~.[]": { + "(severity == 'critical' && result == 'fail')": false + } + } + } + ] + } + } + } + ] + } +} \ No newline at end of file diff --git a/adapters/kyverno-json/policies/meta/tagging-rules-agree.json b/adapters/kyverno-json/policies/meta/tagging-rules-agree.json new file mode 100644 index 0000000..71f5154 --- /dev/null +++ b/adapters/kyverno-json/policies/meta/tagging-rules-agree.json @@ -0,0 +1,41 @@ +{ + "apiVersion": "json.kyverno.io/v1alpha1", + "kind": "ValidatingPolicy", + "metadata": { + "name": "tagging-rules-agree", + "annotations": { + "nova.cloudinit.dev/severity": "medium", + "title.policy.kyverno.io": "Checkov NOVA_TAG_NAMING and kj KJ_REQUIRE_TAGGING_STANDARD agree per resource" + } + }, + "spec": { + "rules": [ + { + "name": "no-tagging-divergence", + "validate": { + "message": "For every resource, the Checkov NOVA_TAG_NAMING result and the kyverno-json KJ_REQUIRE_TAGGING_STANDARD result must agree. Divergence emits an error PCR (D-118, defense-in-depth against rule drift).", + "assert": { + "all": [ + { + "check": { + "~.[?(ruleId == 'NOVA_TAG_NAMING')]": { + "result->ckv_result": {}, + "($ckv_result == 'fail')": false + } + } + }, + { + "check": { + "~.[?(ruleId == 'KJ_REQUIRE_TAGGING_STANDARD')]": { + "result->kj_result": {}, + "($kj_result == 'fail')": false + } + } + } + ] + } + } + } + ] + } +} \ No newline at end of file diff --git a/adapters/kyverno-json/policies/plan-json/forbid-iam-wildcard.json b/adapters/kyverno-json/policies/plan-json/forbid-iam-wildcard.json new file mode 100644 index 0000000..ec67fba --- /dev/null +++ b/adapters/kyverno-json/policies/plan-json/forbid-iam-wildcard.json @@ -0,0 +1,49 @@ +{ + "apiVersion": "json.kyverno.io/v1alpha1", + "kind": "ValidatingPolicy", + "metadata": { + "name": "forbid-iam-wildcard", + "annotations": { + "nova.cloudinit.dev/severity": "high", + "title.policy.kyverno.io": "No IAM wildcard Actions or Resources" + } + }, + "spec": { + "rules": [ + { + "name": "no-wildcard-action", + "validate": { + "message": "IAM policy Action must not be '*' (ports CKV_AWS_1/40)", + "assert": { + "all": [ + { + "check": { + "planned_values.root_module.~.resources": { + "(type == 'aws_iam_policy' && contains(values.policy_document.Statement[].Action, '*'))": false + } + } + } + ] + } + } + }, + { + "name": "no-wildcard-resource", + "validate": { + "message": "IAM policy Resource must not be '*' (ports CKV_AWS_1/40)", + "assert": { + "all": [ + { + "check": { + "planned_values.root_module.~.resources": { + "(type == 'aws_iam_policy' && contains(values.policy_document.Statement[].Resource, '*'))": false + } + } + } + ] + } + } + } + ] + } +} \ No newline at end of file diff --git a/adapters/kyverno-json/policies/plan-json/forbid-plaintext-secrets.json b/adapters/kyverno-json/policies/plan-json/forbid-plaintext-secrets.json new file mode 100644 index 0000000..9df9b0b --- /dev/null +++ b/adapters/kyverno-json/policies/plan-json/forbid-plaintext-secrets.json @@ -0,0 +1,32 @@ +{ + "apiVersion": "json.kyverno.io/v1alpha1", + "kind": "ValidatingPolicy", + "metadata": { + "name": "forbid-plaintext-secrets", + "annotations": { + "nova.cloudinit.dev/severity": "high", + "title.policy.kyverno.io": "No plaintext secrets in the terraform plan" + } + }, + "spec": { + "rules": [ + { + "name": "no-plaintext-db-password", + "validate": { + "message": "aws_db_instance.password must not be a plaintext string (ports CKV_AWS_41/45/46)", + "assert": { + "all": [ + { + "check": { + "planned_values.root_module.~.resources": { + "(type == 'aws_db_instance' && contains(keys(values), 'password') && !contains(['${...}', ''], values.password))": false + } + } + } + ] + } + } + } + ] + } +} \ No newline at end of file diff --git a/adapters/kyverno-json/policies/plan-json/require-kms-reference.json b/adapters/kyverno-json/policies/plan-json/require-kms-reference.json new file mode 100644 index 0000000..b376807 --- /dev/null +++ b/adapters/kyverno-json/policies/plan-json/require-kms-reference.json @@ -0,0 +1,32 @@ +{ + "apiVersion": "json.kyverno.io/v1alpha1", + "kind": "ValidatingPolicy", + "metadata": { + "name": "require-kms-reference", + "annotations": { + "nova.cloudinit.dev/severity": "medium", + "title.policy.kyverno.io": "KMS keys referenced by alias, not inline key material" + } + }, + "spec": { + "rules": [ + { + "name": "kms-by-alias", + "validate": { + "message": "aws_kms_key resources should reference a customer-managed key alias, not inline key material (ports CKV_AWS_7/33)", + "assert": { + "all": [ + { + "check": { + "planned_values.root_module.~.resources": { + "(type == 'aws_kms_key' && !contains(keys(values), 'key_id') && !contains(keys(values), 'kms_key_id'))": false + } + } + } + ] + } + } + } + ] + } +} \ No newline at end of file diff --git a/adapters/kyverno-json/policies/regression/cap-013-adapter-dedup.json b/adapters/kyverno-json/policies/regression/cap-013-adapter-dedup.json new file mode 100644 index 0000000..1701640 --- /dev/null +++ b/adapters/kyverno-json/policies/regression/cap-013-adapter-dedup.json @@ -0,0 +1,30 @@ +{ + "apiVersion": "json.kyverno.io/v1alpha1", + "kind": "ValidatingPolicy", + "metadata": { + "name": "cap-013-adapter-dedup", + "annotations": { + "nova.cloudinit.dev/severity": "medium", + "title.policy.kyverno.io": "No duplicate adapter registrations (CAP-013 declarative mirror)" + } + }, + "spec": { + "rules": [ + { + "name": "no-duplicate-adapters", + "validate": { + "message": "Each adapter must be registered exactly once (no duplicate adapter names in the capability inventory). Declarative mirror of core/regression_verify.py CAP-013.", + "assert": { + "all": [ + { + "check": { + "adapters": "(length(duplicates(@)) == `0`)" + } + } + ] + } + } + } + ] + } +} \ No newline at end of file diff --git a/adapters/kyverno-json/policies/regression/cap-023-metrics-collector.json b/adapters/kyverno-json/policies/regression/cap-023-metrics-collector.json new file mode 100644 index 0000000..59d3e58 --- /dev/null +++ b/adapters/kyverno-json/policies/regression/cap-023-metrics-collector.json @@ -0,0 +1,32 @@ +{ + "apiVersion": "json.kyverno.io/v1alpha1", + "kind": "ValidatingPolicy", + "metadata": { + "name": "cap-023-metrics-collector", + "annotations": { + "nova.cloudinit.dev/severity": "medium", + "title.policy.kyverno.io": "Every metric has a grounded/derived/deferred status (CAP-023 declarative mirror)" + } + }, + "spec": { + "rules": [ + { + "name": "every-metric-has-status", + "validate": { + "message": "Every metric in docs/METRICS.md must declare a status (grounded, derived, or deferred). Declarative mirror of core/regression_verify.py CAP-023.", + "assert": { + "all": [ + { + "check": { + "~.metrics": { + "(contains(['grounded','derived','deferred'], status))": true + } + } + } + ] + } + } + } + ] + } +} \ No newline at end of file diff --git a/adapters/kyverno-json/policies/regression/cap-024-deck-structure.json b/adapters/kyverno-json/policies/regression/cap-024-deck-structure.json new file mode 100644 index 0000000..be39624 --- /dev/null +++ b/adapters/kyverno-json/policies/regression/cap-024-deck-structure.json @@ -0,0 +1,35 @@ +{ + "apiVersion": "json.kyverno.io/v1alpha1", + "kind": "ValidatingPolicy", + "metadata": { + "name": "cap-024-deck-structure", + "annotations": { + "nova.cloudinit.dev/severity": "low", + "title.policy.kyverno.io": "Deck structure matches the documented 4-beat arc (CAP-024 declarative mirror)" + } + }, + "spec": { + "rules": [ + { + "name": "deck-has-4-beats", + "validate": { + "message": "The deck must have the 4-beat arc: Problem, Solution, Proof, Roadmap+Ask. Declarative mirror of core/regression_verify.py CAP-024.", + "assert": { + "all": [ + { + "check": { + "deck.beats": "(length(@) >= `4`)" + } + }, + { + "check": { + "deck.beats": "(contains(@, 'Problem') && contains(@, 'Solution') && contains(@, 'Proof') && contains(@, 'Roadmap+Ask'))" + } + } + ] + } + } + } + ] + } +} \ No newline at end of file diff --git a/adapters/kyverno-json/policies/stack-ir/forbid-public-ingress.json b/adapters/kyverno-json/policies/stack-ir/forbid-public-ingress.json new file mode 100644 index 0000000..94641fd --- /dev/null +++ b/adapters/kyverno-json/policies/stack-ir/forbid-public-ingress.json @@ -0,0 +1,33 @@ +{ + "apiVersion": "json.kyverno.io/v1alpha1", + "kind": "ValidatingPolicy", + "metadata": { + "name": "forbid-public-ingress", + "annotations": { + "nova.cloudinit.dev/severity": "high", + "title.policy.kyverno.io": "No resource has public ingress enabled" + } + }, + "spec": { + "rules": [ + { + "name": "no-public-ingress", + "identifier": "id", + "validate": { + "message": "public_ingress: true is not allowed on any resource (v1.0 demo rule, now declarative)", + "assert": { + "all": [ + { + "check": { + "~.resources": { + "(inputs.public_ingress || `false`)": false + } + } + } + ] + } + } + } + ] + } +} \ No newline at end of file diff --git a/adapters/kyverno-json/policies/stack-ir/require-encryption-by-default.json b/adapters/kyverno-json/policies/stack-ir/require-encryption-by-default.json new file mode 100644 index 0000000..b646097 --- /dev/null +++ b/adapters/kyverno-json/policies/stack-ir/require-encryption-by-default.json @@ -0,0 +1,57 @@ +{ + "apiVersion": "json.kyverno.io/v1alpha1", + "kind": "ValidatingPolicy", + "metadata": { + "name": "require-encryption-by-default", + "annotations": { + "nova.cloudinit.dev/severity": "high", + "title.policy.kyverno.io": "S3 buckets and EBS volumes carry encryption config" + } + }, + "spec": { + "rules": [ + { + "name": "s3-encryption", + "identifier": "id", + "match": { + "any": [ + {"type": "aws:s3:bucket"} + ] + }, + "validate": { + "message": "S3 buckets must declare encryption config (inputs.bucket_encryption or inputs.kms_key_id)", + "assert": { + "all": [ + { + "check": { + "(contains(keys(inputs), 'bucket_encryption') || contains(keys(inputs), 'kms_key_id'))": true + } + } + ] + } + } + }, + { + "name": "ebs-encryption", + "identifier": "id", + "match": { + "any": [ + {"type": "aws:ebs:volume"} + ] + }, + "validate": { + "message": "EBS volumes must declare encryption (inputs.encrypted or inputs.kms_key_id)", + "assert": { + "all": [ + { + "check": { + "(contains(keys(inputs), 'encrypted') || contains(keys(inputs), 'kms_key_id'))": true + } + } + ] + } + } + } + ] + } +} \ No newline at end of file diff --git a/adapters/kyverno-json/policies/stack-ir/require-tagging-standard.json b/adapters/kyverno-json/policies/stack-ir/require-tagging-standard.json new file mode 100644 index 0000000..427b38f --- /dev/null +++ b/adapters/kyverno-json/policies/stack-ir/require-tagging-standard.json @@ -0,0 +1,36 @@ +{ + "apiVersion": "json.kyverno.io/v1alpha1", + "kind": "ValidatingPolicy", + "metadata": { + "name": "require-tagging-standard", + "annotations": { + "nova.cloudinit.dev/severity": "medium", + "title.policy.kyverno.io": "All resources carry required Nova tags" + } + }, + "spec": { + "rules": [ + { + "name": "require-nova-tags", + "identifier": "id", + "validate": { + "message": "Every taggable resource must carry nova:owner, nova:contract, nova:environment, nova:cost-center tags", + "assert": { + "all": [ + { + "check": { + "~.resources": { + "(contains(keys(tags || `[]`), 'nova:owner'))": true, + "(contains(keys(tags || `[]`), 'nova:contract'))": true, + "(contains(keys(tags || `[]`), 'nova:environment'))": true, + "(contains(keys(tags || `[]`), 'nova:cost-center'))": true + } + } + } + ] + } + } + } + ] + } +} \ No newline at end of file diff --git a/core/contract_resolver.py b/core/contract_resolver.py index ff7d7dd..ecc9a5d 100644 --- a/core/contract_resolver.py +++ b/core/contract_resolver.py @@ -488,6 +488,25 @@ def resolve(contract_path, repo_root=None, environment_override=None): # Validate contract against schema jsonschema.validate(contract, contract_schema) + # v1.25 (REQ-296): pre-resolve policy evaluation — run the active + # PolicyEngine over the contract dict with the contract/ policy + # dir BEFORE resolving. Failures feed the `policyResults` on the + # stack instance (the confidence signal's `policy` input). The + # resolver does NOT exit on policy failure — the confidence signal + # decides the gate (consistent with the existing --soft-fail + # Checkov pattern). + contract_pcrs: list = [] + try: + from core.policy_engine import get_engine, get_policy_root + _engine = get_engine() + _policy_root = get_policy_root() + contract_pcrs = _engine.evaluate( + contract, _policy_root / "contract", contract.get("id", "unknown") + ) + except Exception: + # Policy evaluation must never break the resolver. + contract_pcrs = [] + # Interpolation (D-081): expand ${env.} + ${contract.} # tokens AFTER schema validation (the schema sees raw tokens, which are # valid strings) and BEFORE IR resolution (the resolver sees concrete @@ -590,6 +609,12 @@ def resolve(contract_path, repo_root=None, environment_override=None): "data_sources": all_data_sources, } + # v1.25 (REQ-296): attach the pre-resolve contract-policy PCRs to + # the stack instance. The post-resolve stack-IR PCRs are appended + # after stack-schema validation (below). + if contract_pcrs: + stack_instance["policyResults"] = list(contract_pcrs) + # Add the human-readable title if contract.get("name"): stack_instance["stack"]["title"] = contract["name"] @@ -606,6 +631,28 @@ def resolve(contract_path, repo_root=None, environment_override=None): stack_schema = _load_schema(os.path.join(repo_root, "schemas", "stack.schema.json")) jsonschema.validate(stack_instance, stack_schema) + # v1.25 (REQ-298): post-resolve policy evaluation — run the active + # PolicyEngine over the resolved Stack IR with the stack-ir/ policy + # dir. The resulting PCRs are appended to the contract-policy PCRs + # on the stack instance (additive — the resolver's return value + # shape and exceptions are unchanged). The confidence signal + # consumes the merged list as its `policy` input. + try: + from core.policy_engine import get_engine, get_policy_root + engine = get_engine() + policy_root = get_policy_root() + stack_ir_pcrs = engine.evaluate( + stack_instance, policy_root / "stack-ir", contract.get("id", "unknown") + ) + stack_instance.setdefault("policyResults", []).extend(stack_ir_pcrs) + except Exception: + # Policy evaluation must never break the resolver — the + # confidence signal decides the gate. A failure here means the + # engine is misconfigured; the contract PCRs (if any) are still + # present, and the confidence signal proceeds with whatever + # `policy` input it receives (possibly empty → 0.5 neutral). + pass + return stack_instance diff --git a/core/policy_engine.py b/core/policy_engine.py new file mode 100644 index 0000000..e19928a --- /dev/null +++ b/core/policy_engine.py @@ -0,0 +1,212 @@ +"""Nova Policy Engine Registry (REQ-291, v1.25). + +The swappable policy-engine abstraction. A Python Protocol (PEP 544) +defines the engine contract; a registry selects the active engine from +``config.json``'s ``policy.engine`` key. This is the **swap boundary** +(ARCHITECTURE.md §12.7) — the confidence signal and pipeline never +import an engine directly; they go through the registry. A future +``OpaEngine`` implements the same protocol without touching the +confidence signal, the PCR schema, or the pipeline. + +The protocol is minimal (3 members) by design: + +- ``name`` — the engine's registry key (matches ``config.json.policy.engine``). +- ``is_configured()`` — returns False when the engine's binary is absent + (the registry's caller must skip gracefully, emitting SKIPPED PCRs). +- ``evaluate(payload, policy_dir, contract_id)`` — runs the engine's + policies over ``payload`` and returns a ``list[dict]`` where each dict + conforms to ``schemas/policy_check_result.schema.json``. + +A ``NullEngine`` is the fallback when the ``policy`` key is absent from +``config.json`` (backward compatibility for tests that don't set the +key — it emits a single SKIPPED PCR so the confidence signal proceeds +with a neutral ``policy`` input). + +Engine enum reuse (D-116): kyverno-json PCR records carry +``engine: "kyverno"`` (no new enum value). The ``engine`` field records +the policy-engine *family*, not the specific binary. The K8s Kyverno +adapter and the kyverno-json engine are distinguished by ``ruleId`` +prefix (``KYVERNO_`` vs ``KJ_``). +""" + +import json +import os +from pathlib import Path +from typing import Any, Callable, Protocol, Union, runtime_checkable + +import datetime + + +def _iso8601_now() -> str: + return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +Payload = Union[dict, list, str] + + +@runtime_checkable +class PolicyEngine(Protocol): + """The swap boundary for policy engines. + + Implementations: ``KyvernoJsonEngine`` (adapters/kyverno-json/), + ``NullEngine`` (this module), future ``OpaEngine``. + """ + + @property + def name(self) -> str: ... + + def is_configured(self) -> bool: ... + + def evaluate(self, payload: Payload, policy_dir: Path, + contract_id: str) -> list[dict]: ... + + +def _skipped_pcr(rule_id: str, message: str, contract_id: str) -> dict: + return { + "contractId": contract_id, + "evaluatedAt": _iso8601_now(), + "engine": "kyverno", + "ruleId": rule_id, + "severity": "info", + "result": "skipped", + "message": message, + "evidence": {}, + "resourceRef": "", + } + + +class NullEngine: + """Fallback when ``config.json.policy`` is absent. + + Emits a single SKIPPED PCR with ``ruleId: NULL_ENGINE_INACTIVE`` so + the confidence signal's ``policy`` input is non-null (the per-input + score for a single SKIPPED PCR is 1.0 — skipped counts as pass per + ``core/confidence_signal.py:84-89``). This keeps existing tests + passing when the ``policy`` key is not set. + """ + + name = "null" + + def is_configured(self) -> bool: + return False + + def evaluate(self, payload: Payload, policy_dir: Path, + contract_id: str) -> list[dict]: + return [_skipped_pcr( + "NULL_ENGINE_INACTIVE", + "NullEngine active — the `policy` key is absent from config.json. " + "No policy engine is configured; the confidence signal proceeds with " + "a neutral SKIPPED policy input.", + contract_id, + )] + + +_REGISTRY: dict[str, Callable[[], PolicyEngine]] = {} + + +def register(name: str, factory: Callable[[], PolicyEngine]) -> None: + """Register an engine factory under ``name``. + + The factory is called lazily by ``get_engine()`` so an engine's + binary dependency (e.g. ``kj``) is not required at import time. + """ + _REGISTRY[name] = factory + + +def _load_config_policy() -> dict | None: + """Read the ``policy`` object from ``.ciagent/config.json``. + + Returns ``None`` when the file is absent or the ``policy`` key is + missing (the caller falls back to ``NullEngine``). + """ + repo_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + cfg = os.path.join(repo_root, ".ciagent", "config.json") + if not os.path.isfile(cfg): + return None + try: + with open(cfg, "r", encoding="utf-8") as fh: + data = json.load(fh) + except (json.JSONDecodeError, OSError): + return None + return data.get("policy") + + +def get_engine() -> PolicyEngine: + """Return the active ``PolicyEngine`` from ``config.json``. + + Reads ``config.json.policy.engine`` (default ``"kyverno-json"``). + Falls back to ``NullEngine`` when the ``policy`` key is absent + (backward compatibility). Raises ``KeyError`` for an unknown engine + name (a typo in config — fail loud, not silent). + """ + policy_cfg = _load_config_policy() + if policy_cfg is None: + return NullEngine() + engine_name = policy_cfg.get("engine", "kyverno-json") + factory = _REGISTRY.get(engine_name) + if factory is None: + raise KeyError( + f"Unknown policy engine '{engine_name}' in config.json. " + f"Registered engines: {sorted(_REGISTRY.keys()) or ['(none)']}. " + f"Set policy.engine to a registered name or install the engine adapter." + ) + return factory() + + +def get_policy_root() -> Path: + """Return the configured policy root directory (or a default).""" + policy_cfg = _load_config_policy() + if policy_cfg is None: + return Path("adapters/kyverno-json/policies") + root = policy_cfg.get("policy_root", "adapters/kyverno-json/policies") + repo_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + if os.path.isabs(root): + return Path(root) + return Path(repo_root) / root + + +def _register_builtin(name: str, factory: Callable[[], PolicyEngine]) -> None: + register(name, factory) + + +def _autoload_kyverno_json() -> None: + """Register the kyverno-json engine if its adapter is importable. + + The adapter directory uses a hyphen (``adapters/kyverno-json/``), + so a plain ``import`` is not possible. Load the module by file path + via ``importlib.util``. Lazy import so ``core/policy_engine.py`` + does not require ``adapters/kyverno-json/`` at import time (the + adapter imports ``yaml``, which may be unavailable in minimal test + envs). + """ + try: + import importlib.util + repo_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + adapter_path = os.path.join( + repo_root, "adapters", "kyverno-json", "kyverno_json_engine.py" + ) + if not os.path.isfile(adapter_path): + return + spec = importlib.util.spec_from_file_location( + "kyverno_json_engine", adapter_path + ) + if spec is None or spec.loader is None: + return + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + engine_cls = getattr(mod, "KyvernoJsonEngine") + _register_builtin("kyverno-json", engine_cls) + except Exception: + pass + + +_autoload_kyverno_json() + + +if __name__ == "__main__": + eng = get_engine() + print(json.dumps({ + "engine": eng.name, + "is_configured": eng.is_configured(), + "policy_root": str(get_policy_root()), + }, indent=2)) \ No newline at end of file diff --git a/docs/METRICS.md b/docs/METRICS.md index f30d89b..21c8ba6 100644 --- a/docs/METRICS.md +++ b/docs/METRICS.md @@ -174,4 +174,18 @@ numbers. Every metric either has a real source or is explicitly deferred. | SLA / Unplanned Downtime | D-096 | `placeholder_sla_downtime.csv` | | Predictive vs Reactive Ratio | future emitter | `placeholder_predictive_reactive.csv` | -See `docs/METRICS_DEFERRED_ROADMAP.md` for the activation path for each. \ No newline at end of file +See `docs/METRICS_DEFERRED_ROADMAP.md` for the activation path for each. + +--- + +## v1.25 — Swappable Policy Engine + +The policy engine that produces the `PolicyCheckResult` records feeding +the confidence signal is **swappable** (NORTH_STAR Strategic Objective #2 +— provable trust via a replaceable substrate, not a vendor lock-in). +The `PolicyEngine` protocol (`core/policy_engine.py`) is the swap +boundary; `config.json.policy.engine` selects the active engine +(default `"kyverno-json"`). A future `OpaEngine` implements the same +protocol without touching the confidence signal, the PCR schema, or +the pipeline. See `.ciagent/ARCHITECTURE.md` §12.7 for the registry +diagram. \ No newline at end of file diff --git a/modules/STANDARDS.md b/modules/STANDARDS.md index 3b38437..895d0c3 100644 --- a/modules/STANDARDS.md +++ b/modules/STANDARDS.md @@ -611,4 +611,64 @@ must be checked before the module is registered and published. `stack.schema.json`). - [ ] For an L2, a test is added that the composition resolves to the expected set of L1 instances and that the adapter emits a root module - calling the L1 modules. \ No newline at end of file + calling the L1 modules. + +--- + +## 10. Policy Authoring Standard (v1.25) + +Module owners may ship per-module kyverno-json policies in +`modules//policies/` (future convention; v1.25 policies live +under `adapters/kyverno-json/policies/`). A policy file is a +`ValidatingPolicy` resource (YAML or JSON). + +### 10.1 Required fields + +- `apiVersion: json.kyverno.io/v1alpha1` +- `kind: ValidatingPolicy` +- `metadata.name` — matches the filename (e.g. `require-tags.json` → + `name: require-tags`). This becomes the `ruleId` prefix `KJ_`. +- `metadata.annotations["nova.cloudinit.dev/severity"]` — one of + `critical`, `high`, `medium`, `low`, `info`. Drives the confidence + signal's penalty mapping. +- `spec.rules[].validate.assert` — an `all` or `any` list of assertion + trees with JMESPath expressions. **No `forEach`, pattern operators, + anchors, or wildcards** — use the `~` projection modifier to iterate. + +### 10.2 Severity guidance + +| Severity | When to use | Confidence penalty | +| --- | --- | --- | +| `critical` | a violation makes the deploy unsafe (e.g. public ingress on a prod DB) | hard override (score = 0, block) | +| `high` | a violation is a security or compliance gap (e.g. plaintext secrets) | -0.20 | +| `medium` | a violation is a best-practice miss (e.g. missing tags) | -0.05 | +| `low` | a violation is a style or convention issue | -0.01 | +| `info` | a non-blocking observation (default) | 0.0 | + +### 10.3 Assertion-tree patterns + +- **Iterate an array:** use the `~` modifier on the array key: + ```yaml + check: + ~.resources: + (@ < `5`): true + ``` +- **Match a resource type:** use the `match.any` block: + ```yaml + match: + any: + - type: aws:s3:bucket + ``` +- **Binding for descendant access:** use `->name`: + ```yaml + (bar + bat)->sum: + ($sum): 10 + ``` + +### 10.4 Testing + +- Ship a fixture pair (`passing.json` + `failing.json`) under + `tests/fixtures//`. +- Add a test file `tests/test__policies.py` using the + `KyvernoJsonEngine` (skip-without-kj pattern). +- The regression gate (`pytest tests/`) must remain green. \ No newline at end of file diff --git a/schemas/README.md b/schemas/README.md index e56b7aa..0867a7d 100644 --- a/schemas/README.md +++ b/schemas/README.md @@ -15,6 +15,14 @@ Nova uses JSON Schema draft 2020-12 for all declarative contracts. Schemas are t | Nova PolicyCheckResult | `policy_check_result.schema.json` | Normalized policy check result schema (the contract between policy engines and the confidence signal) | `tests/conftest.py`, all adapter tests | | Nova Tagging Standard | `tagging-standard.json` | Required tag set for all taggable AWS resources | `adapters/terraform/policy/custom_rules/nova_tagging.py` | +> **v1.25 note (D-116):** the `engine` enum value `"kyverno"` is shared +> by the K8s-only Kyverno adapter (`adapters/kyverno/`) and the +> kyverno-json engine (`adapters/kyverno-json/`). The two are +> distinguished by `ruleId` prefix (`KYVERNO_` for the K8s adapter, +> `KJ_` for kyverno-json) and `evidence` payload shape. No new enum +> value was added — the `engine` field records the policy-engine +> family, not the specific binary. + ## How to Write a Schema 1. Use JSON Schema draft 2020-12: `"$schema": "https://json-schema.org/draft/2020-12/schema"`. diff --git a/scripts/install-kyverno-json.sh b/scripts/install-kyverno-json.sh new file mode 100644 index 0000000..1c446cb --- /dev/null +++ b/scripts/install-kyverno-json.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# scripts/install-kyverno-json.sh — install the kj CLI (v1.25, REQ-294) +# +# Installs the kyverno-json CLI (`kj`) via `go install` (D-115). The +# binary is a Go project — not a Python package. Cached via the Go +# module cache. +# +# Usage: bash scripts/install-kyverno-json.sh +# Exits 0 on success, 1 if Go is not installed, 2 if `kj version` fails. +set -euo pipefail + +if ! command -v go >/dev/null 2>&1; then + echo "ERROR: Go toolchain not found. Install Go (https://go.dev/dl/) first." >&2 + echo " kyverno-json is a Go binary — `go install` is the upstream-blessed path (D-115)." >&2 + exit 1 +fi + +echo "Installing kyverno-json CLI (kj) via go install..." +GOBIN="${GOBIN:-${HOME}/go/bin}" +go install github.com/kyverno/kyverno-json/cmd/kj@latest + +if ! command -v kj >/dev/null 2>&1; then + if [ -x "${GOBIN}/kj" ]; then + echo "kj installed to ${GOBIN}/kj (not on PATH)" + echo "add ${GOBIN} to PATH or symlink: ln -s ${GOBIN}/kj /usr/local/bin/kj" + "${GOBIN}/kj" version + exit 0 + fi + echo "ERROR: kj not found on PATH after go install (checked ${GOBIN})." >&2 + exit 2 +fi + +echo "kj installed:" +kj version +echo "DONE" \ No newline at end of file diff --git a/scripts/run_platform.sh b/scripts/run_platform.sh index b2db624..1a4e39f 100755 --- a/scripts/run_platform.sh +++ b/scripts/run_platform.sh @@ -518,8 +518,102 @@ for pcr in pcrs: marker = 'PASS' if res == 'pass' else 'FAIL' if res == 'fail' else 'SKIP' if res == 'skipped' else res.upper() print(f' [{marker}] {sev:8s} {rule:30s} {msg}') " - echo "" + +# ============================================================================ +# Step 5b: kyverno-json plan-JSON policy pass (v1.25, REQ-301) +# ============================================================================ +# After Checkov/Wiz produce raw PCRs (Step 5/6), run kyverno-json over the +# terraform plan JSON in parallel and merge the PCR lists. When `which kj` +# is absent, skip gracefully (the platform proceeds with the Checkov/Wiz +# list only — D-120 graceful degradation). +if command -v kj >/dev/null 2>&1; then + echo "=== Step 5b: kyverno-json plan-JSON policies (parallel with Checkov/Wiz) ===" + # Produce the terraform show JSON (kj scan --payload expects a JSON file). + if [ -f "$TF_DIR/tfplan" ]; then + terraform -chdir="$TF_DIR" show -json tfplan > "$WORK/tfshow.json" 2>/dev/null || true + if [ -s "$WORK/tfshow.json" ]; then + python3 - "$WORK/tfshow.json" "$CONTRACT_ID" <<'PY' > "$WORK/kj-pcr.json" 2>"$WORK/kj.err" || echo "[]" +import json, sys +from pathlib import Path +sys.path.insert(0, ".") +import importlib.util +_spec = importlib.util.spec_from_file_location("kj_engine", "adapters/kyverno-json/kyverno_json_engine.py") +_mod = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_mod) +_payload_path, _contract_id = sys.argv[1], sys.argv[2] +eng = _mod.KyvernoJsonEngine() +if not eng.is_configured(): + print("[]"); sys.exit(0) +out = eng.evaluate(json.load(open(_payload_path)), Path("adapters/kyverno-json/policies/plan-json"), _contract_id) +print(json.dumps(out)) +PY + if [ -s "$WORK/kj-pcr.json" ]; then + echo "kyverno-json plan-JSON summary: $(python3 -c "import json; d=json.load(open('$WORK/kj-pcr.json')); print(len([p for p in d if p.get('result')=='fail']), 'failed,', len([p for p in d if p.get('result')=='pass']), 'passed')")" + # Merge: concatenate the Checkov/Wiz PCRs + the kj PCRs into pcr.json. + python3 -c " +import json +ckv = json.load(open('$WORK/pcr.json')) +kj = json.load(open('$WORK/kj-pcr.json')) +json.dump(ckv + kj, open('$WORK/pcr.json', 'w')) +print(f'merged PCR list: {len(ckv)} checkov/wiz + {len(kj)} kyverno-json = {len(ckv)+len(kj)} total') +" + else + echo "kyverno-json produced no output; proceeding with Checkov/Wiz PCRs only" + fi + else + echo "terraform show -json produced no output; skipping kyverno-json plan-JSON policies" + fi + else + echo "tfplan not found; skipping kyverno-json plan-JSON policies" + fi +else + echo "=== Step 5b: kyverno-json not installed; skipping plan-JSON policies (D-120 graceful degradation) ===" +fi +echo "" + +# ============================================================================ +# Step 5c: kyverno-json meta-policies over the merged PCR list (v1.25, REQ-303) +# ============================================================================ +# After Step 5b merges the Checkov/Wiz + kj plan-JSON PCRs into pcr.json, run +# the meta-policies (block-on-any-critical, tagging-rules-agree) over the +# merged list. The meta-policy PCRs are appended to pcr.json before the +# confidence signal runs. The confidence_signal.py PENALTY["critical"]: None +# hard-override stays as defense-in-depth behind this declarative rule +# (D-119). Skips gracefully when kj is absent (D-120). +if command -v kj >/dev/null 2>&1 && [ -s "$WORK/pcr.json" ]; then + echo "=== Step 5c: kyverno-json meta-policies over the merged PCR list ===" + python3 - "$WORK/pcr.json" "$CONTRACT_ID" <<'PY' > "$WORK/meta-pcr.json" 2>"$WORK/meta.err" || echo "[]" +import json, sys +from pathlib import Path +sys.path.insert(0, ".") +import importlib.util +_spec = importlib.util.spec_from_file_location("kj_engine", "adapters/kyverno-json/kyverno_json_engine.py") +_mod = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_mod) +eng = _mod.KyvernoJsonEngine() +if not eng.is_configured(): + print("[]"); sys.exit(0) +pcrs = json.load(open(sys.argv[1])) +out = eng.evaluate(pcrs, Path("adapters/kyverno-json/policies/meta"), sys.argv[2]) +print(json.dumps(out)) +PY + if [ -s "$WORK/meta-pcr.json" ]; then + python3 -c " +import json +merged = json.load(open('$WORK/pcr.json')) +meta = json.load(open('$WORK/meta-pcr.json')) +json.dump(merged + meta, open('$WORK/pcr.json', 'w')) +print(f'meta-policies: {len(meta)} meta-PCRs appended; total PCR list now {len(merged)+len(meta)}') +" + else + echo "kyverno-json meta-policies produced no output; proceeding with the merged list only" + fi +else + echo "=== Step 5c: kj not installed or no merged PCR list; skipping meta-policies (D-120) ===" +fi +echo "" + echo "=== Step 7: confidence signal compute ===" python3 < "$WORK/signal.json" || fail "confidence signal failed" import json diff --git a/tests/fixtures/capability_inventory/clean.json b/tests/fixtures/capability_inventory/clean.json new file mode 100644 index 0000000..9248997 --- /dev/null +++ b/tests/fixtures/capability_inventory/clean.json @@ -0,0 +1,11 @@ +{ + "adapters": ["terraform", "checkov", "wiz", "kyverno-json"], + "metrics": [ + {"name": "MTTR", "status": "grounded"}, + {"name": "CloudSpend", "status": "derived"}, + {"name": "TouchlessResolution", "status": "deferred"} + ], + "deck": { + "beats": ["Problem", "Solution", "Proof", "Roadmap+Ask"] + } +} \ No newline at end of file diff --git a/tests/fixtures/capability_inventory/drifted.json b/tests/fixtures/capability_inventory/drifted.json new file mode 100644 index 0000000..1b3e1b7 --- /dev/null +++ b/tests/fixtures/capability_inventory/drifted.json @@ -0,0 +1,11 @@ +{ + "adapters": ["terraform", "checkov", "wiz", "terraform", "kyverno-json"], + "metrics": [ + {"name": "MTTR", "status": "grounded"}, + {"name": "CloudSpend", "status": "unknown"}, + {"name": "TouchlessResolution", "status": "deferred"} + ], + "deck": { + "beats": ["Problem", "Solution", "Proof"] + } +} \ No newline at end of file diff --git a/tests/fixtures/plan_json/failing.json b/tests/fixtures/plan_json/failing.json new file mode 100644 index 0000000..dbe30b7 --- /dev/null +++ b/tests/fixtures/plan_json/failing.json @@ -0,0 +1,35 @@ +{ + "planned_values": { + "root_module": { + "resources": [ + { + "address": "aws_db_instance.main", + "type": "aws_db_instance", + "name": "main", + "values": { + "password": "supersecret123", + "engine": "postgres" + } + }, + { + "address": "aws_iam_policy.bad", + "type": "aws_iam_policy", + "name": "bad", + "values": { + "policy_document": { + "Statement": [{"Action": "*", "Resource": "*", "Effect": "Allow"}] + } + } + }, + { + "address": "aws_kms_key.inline", + "type": "aws_kms_key", + "name": "inline", + "values": { + "description": "inline key with no alias" + } + } + ] + } + } +} \ No newline at end of file diff --git a/tests/fixtures/plan_json/passing.json b/tests/fixtures/plan_json/passing.json new file mode 100644 index 0000000..27b7317 --- /dev/null +++ b/tests/fixtures/plan_json/passing.json @@ -0,0 +1,27 @@ +{ + "planned_values": { + "root_module": { + "resources": [ + { + "address": "aws_s3_bucket.bucket", + "type": "aws_s3_bucket", + "name": "bucket", + "values": { + "bucket": "acdl-dev-msvc-bucket", + "tags": {"nova:owner": "team-a", "nova:environment": "dev"}, + "server_side_encryption_configuration": {"rule": {"apply_server_side_encryption_by_default": {"sse_algorithm": "AES256"}}} + } + }, + { + "address": "aws_kms_key.main", + "type": "aws_kms_key", + "name": "main", + "values": { + "key_id": "alias/nova-main", + "customer_master_key_spec": "SYMMETRIC_DEFAULT" + } + } + ] + } + } +} \ No newline at end of file diff --git a/tests/fixtures/stack_ir/failing.json b/tests/fixtures/stack_ir/failing.json new file mode 100644 index 0000000..ef03ba7 --- /dev/null +++ b/tests/fixtures/stack_ir/failing.json @@ -0,0 +1,34 @@ +{ + "version": "1.0.0", + "stack": { + "name": "bad", + "title": "failing stack", + "kind": "l1", + "depth": 1, + "environment": "dev" + }, + "resources": [ + { + "id": "bucket", + "type": "aws:s3:bucket", + "module": "s3@1.0.0", + "inputs": { + "bucket_name": "acdl-dev-bad-bucket", + "region": "us-east-1", + "tags": { + "nova:owner": "team-a" + } + } + }, + { + "id": "service", + "type": "aws:ecs:service", + "module": "microservice@1.0.0", + "inputs": { + "image": "nginx:latest", + "port": 80, + "public_ingress": true + } + } + ] +} \ No newline at end of file diff --git a/tests/fixtures/stack_ir/passing.json b/tests/fixtures/stack_ir/passing.json new file mode 100644 index 0000000..e4bf5bd --- /dev/null +++ b/tests/fixtures/stack_ir/passing.json @@ -0,0 +1,43 @@ +{ + "version": "1.0.0", + "stack": { + "name": "msvc", + "title": "microservice", + "kind": "l1", + "depth": 1, + "environment": "dev" + }, + "resources": [ + { + "id": "bucket", + "type": "aws:s3:bucket", + "module": "s3@1.0.0", + "inputs": { + "bucket_name": "acdl-dev-msvc-bucket", + "region": "us-east-1", + "bucket_encryption": {"rule": {"apply_server_side_encryption_by_default": {"sse_algorithm": "AES256"}}}, + "tags": { + "nova:owner": "team-a", + "nova:contract": "msvc", + "nova:environment": "dev", + "nova:cost-center": "cc-1" + } + } + }, + { + "id": "service", + "type": "aws:ecs:service", + "module": "microservice@1.0.0", + "inputs": { + "image": "nginx:latest", + "port": 80, + "tags": { + "nova:owner": "team-a", + "nova:contract": "msvc", + "nova:environment": "dev", + "nova:cost-center": "cc-1" + } + } + } + ] +} \ No newline at end of file diff --git a/tests/test_kyverno_json_engine.py b/tests/test_kyverno_json_engine.py new file mode 100644 index 0000000..48976cd --- /dev/null +++ b/tests/test_kyverno_json_engine.py @@ -0,0 +1,213 @@ +"""Tests for adapters/kyverno-json/kyverno_json_engine.py (REQ-309, v1.25). + +PCR schema validity (jsonschema validation), defensive parsing +(malformed output → error PCR, never exception), is_configured() +guard, severity annotation reading (G-Q10a), and pytest.skip when +kj is absent. +""" + +import json +import os +import sys +from pathlib import Path +from unittest import mock + +import jsonschema +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +# Load the engine module by file path (the dir has a hyphen). +import importlib.util +_ENGINE_PATH = Path(__file__).resolve().parent.parent / "adapters" / "kyverno-json" / "kyverno_json_engine.py" +_spec = importlib.util.spec_from_file_location("kyverno_json_engine", _ENGINE_PATH) +_mod = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_mod) +KyvernoJsonEngine = _mod.KyvernoJsonEngine +_to_pcr = _mod._to_pcr +_load_policy_severities = _mod._load_policy_severities + +PCR_SCHEMA_PATH = Path(__file__).resolve().parent.parent / "schemas" / "policy_check_result.schema.json" + + +def _load_pcr_schema(): + with open(PCR_SCHEMA_PATH, "r", encoding="utf-8") as fh: + return json.load(fh) + + +PCR_SCHEMA = _load_pcr_schema() + + +def _kj_installed() -> bool: + """Return True if the kj binary is on PATH.""" + return _mod._which_kj() is not None + + +def _smoke_policy_dir() -> Path: + return Path(__file__).resolve().parent.parent / "adapters" / "kyverno-json" / "policies" + + +class TestToPcr: + def test_pass_entry(self): + entry = {"policy": "require-contract-id", "rule": "require-id", + "result": "pass", "message": "ok", "resource": "res-1"} + pcr = _to_pcr(entry, "cid", "high") + assert pcr["contractId"] == "cid" + assert pcr["engine"] == "kyverno" + assert pcr["ruleId"] == "KJ_require-contract-id/require-id" + assert pcr["result"] == "pass" + assert pcr["severity"] == "high" + assert pcr["resourceRef"] == "res-1" + + def test_fail_entry(self): + entry = {"policy": "forbid-public-ingress", "rule": "no-public", + "result": "fail", "message": "public ingress not allowed", + "resource": "s3/x"} + pcr = _to_pcr(entry, "cid", "critical") + assert pcr["result"] == "fail" + assert pcr["severity"] == "critical" + assert pcr["message"] == "public ingress not allowed" + + def test_skip_entry(self): + entry = {"policy": "p", "rule": "r", "result": "skip"} + pcr = _to_pcr(entry, "cid", "info") + assert pcr["result"] == "skipped" + + def test_unknown_result_becomes_error(self): + entry = {"policy": "p", "rule": "r", "result": "garbled"} + pcr = _to_pcr(entry, "cid", "info") + assert pcr["result"] == "error" + + def test_pcr_validates_against_schema(self): + entry = {"policy": "p", "rule": "r", "result": "pass", + "message": "ok", "resource": "r"} + pcr = _to_pcr(entry, "cid-uuid", "medium") + jsonschema.validate(pcr, PCR_SCHEMA) + + +class TestSeverityAnnotation: + """G-Q10a: severity is read from the policy's metadata.annotation.""" + + def test_policy_with_severity_annotation(self, tmp_path): + policy = { + "apiVersion": "json.kyverno.io/v1alpha1", + "kind": "ValidatingPolicy", + "metadata": { + "name": "test-sev", + "annotations": {"nova.cloudinit.dev/severity": "high"}, + }, + "spec": {"rules": [{"name": "r", "validate": {"assert": {"all": []}}}]}, + } + p = tmp_path / "test-sev.json" + p.write_text(json.dumps(policy)) + sevs = _load_policy_severities(tmp_path) + assert sevs.get("test-sev") == "high" + + def test_policy_without_severity_defaults_info(self, tmp_path): + policy = { + "apiVersion": "json.kyverno.io/v1alpha1", + "kind": "ValidatingPolicy", + "metadata": {"name": "no-sev"}, + "spec": {"rules": [{"name": "r", "validate": {"assert": {"all": []}}}]}, + } + p = tmp_path / "no-sev.json" + p.write_text(json.dumps(policy)) + sevs = _load_policy_severities(tmp_path) + assert sevs.get("no-sev") == "info" + + def test_underscore_files_skipped(self, tmp_path): + # _smoke.json starts with _ — should be skipped. + (tmp_path / "_smoke.json").write_text("{}") + sevs = _load_policy_severities(tmp_path) + assert sevs == {} + + +class TestIsConfigured: + def test_is_configured_returns_bool(self): + eng = KyvernoJsonEngine() + assert isinstance(eng.is_configured(), bool) + + def test_is_configured_false_when_kj_absent(self, monkeypatch): + monkeypatch.setattr(_mod, "_which_kj", lambda: None) + eng = KyvernoJsonEngine() + assert eng.is_configured() is False + + +class TestEvaluateNotConfigured: + """When kj is absent, evaluate() returns KJ_ENGINE_NOT_CONFIGURED.""" + + def test_evaluate_returns_skipped_when_not_configured(self, monkeypatch): + monkeypatch.setattr(_mod, "_which_kj", lambda: None) + eng = KyvernoJsonEngine() + out = eng.evaluate({"id": "x"}, Path("/tmp/policies"), "cid-1") + assert len(out) == 1 + assert out[0]["ruleId"] == "KJ_ENGINE_NOT_CONFIGURED" + assert out[0]["result"] == "skipped" + jsonschema.validate(out[0], PCR_SCHEMA) + + +class TestEvaluateWithKj: + """Tests that run the real kj binary. Skip when kj is not installed.""" + + @pytest.fixture(autouse=True) + def _require_kj(self): + if not _kj_installed(): + pytest.skip("kj not installed (scripts/install-kyverno-json.sh)") + + def test_smoke_policy_round_trip(self, tmp_path): + eng = KyvernoJsonEngine() + if not eng.is_configured(): + pytest.skip("kj not configured") + # Use the real smoke policy dir. + out = eng.evaluate({"id": "msvc"}, _smoke_policy_dir(), "cid-smoke") + assert isinstance(out, list) + assert len(out) >= 1 + for pcr in out: + jsonschema.validate(pcr, PCR_SCHEMA) + assert pcr["engine"] == "kyverno" + assert pcr["contractId"] == "cid-smoke" + + def test_no_results_returns_pass(self, tmp_path): + # An empty policy dir → no results → KJ_NO_RESULTS pass PCR. + eng = KyvernoJsonEngine() + empty_dir = tmp_path / "empty" + empty_dir.mkdir() + out = eng.evaluate({"id": "x"}, empty_dir, "cid-empty") + assert len(out) == 1 + assert out[0]["ruleId"] == "KJ_NO_RESULTS" + assert out[0]["result"] == "pass" + + +class TestDefensiveParsing: + """Malformed kyverno-json output → error PCR, never exception.""" + + def test_malformed_output_produces_error_pcr(self, monkeypatch): + eng = KyvernoJsonEngine() + # Mock is_configured → True, then mock subprocess to return + # garbage output. + monkeypatch.setattr(_mod, "_which_kj", lambda: "/fake/kj") + monkeypatch.setattr(eng, "is_configured", lambda: True) + + class FakeProc: + returncode = 0 + stdout = "not valid json {" + stderr = "" + + def fake_run(*a, **kw): + return FakeProc() + + monkeypatch.setattr(_mod.subprocess, "run", fake_run) + out = eng.evaluate({"id": "x"}, _smoke_policy_dir(), "cid-bad") + assert len(out) == 1 + assert out[0]["result"] == "error" + assert out[0]["ruleId"] == "KJ_ENGINE_ERROR" + jsonschema.validate(out[0], PCR_SCHEMA) + + def test_missing_policy_dir_produces_error_pcr(self, monkeypatch): + eng = KyvernoJsonEngine() + monkeypatch.setattr(_mod, "_which_kj", lambda: "/fake/kj") + monkeypatch.setattr(eng, "is_configured", lambda: True) + out = eng.evaluate({"id": "x"}, Path("/nonexistent/dir"), "cid-miss") + assert len(out) == 1 + assert out[0]["result"] == "error" + assert "not found" in out[0]["message"] \ No newline at end of file diff --git a/tests/test_meta_policies.py b/tests/test_meta_policies.py new file mode 100644 index 0000000..df043d4 --- /dev/null +++ b/tests/test_meta_policies.py @@ -0,0 +1,84 @@ +"""Tests for meta-policies (REQ-303, v1.25). + +Tests block-on-any-critical + tagging-rules-agree over the merged PCR +list as payload. Skips when kj is absent. +""" + +import json +import os +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +import importlib.util +_ENGINE_PATH = Path(__file__).resolve().parent.parent / "adapters" / "kyverno-json" / "kyverno_json_engine.py" +_spec = importlib.util.spec_from_file_location("kyverno_json_engine", _ENGINE_PATH) +_mod = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_mod) +KyvernoJsonEngine = _mod.KyvernoJsonEngine + +POLICY_DIR = Path(__file__).resolve().parent.parent / "adapters" / "kyverno-json" / "policies" / "meta" + + +def _kj_installed() -> bool: + return _mod._which_kj() is not None + + +@pytest.fixture(autouse=True) +def _require_kj(): + if not _kj_installed(): + pytest.skip("kj not installed (scripts/install-kyverno-json.sh)") + + +class TestBlockOnAnyCritical: + def test_no_critical_passes(self): + pcrs = [ + {"severity": "high", "result": "fail", "ruleId": "X", "contractId": "c", + "message": "", "resourceRef": "", "engine": "kyverno", "evaluatedAt": "t", + "evidence": {}}, + {"severity": "info", "result": "pass", "ruleId": "Y", "contractId": "c", + "message": "", "resourceRef": "", "engine": "kyverno", "evaluatedAt": "t", + "evidence": {}}, + ] + eng = KyvernoJsonEngine() + out = eng.evaluate(pcrs, POLICY_DIR / "block-on-any-critical.json" + if (POLICY_DIR / "block-on-any-critical.json").is_file() else POLICY_DIR, + "cid") + assert isinstance(out, list) + + def test_critical_fail_present(self): + pcrs = [ + {"severity": "critical", "result": "fail", "ruleId": "Z", "contractId": "c", + "message": "critical!", "resourceRef": "", "engine": "kyverno", "evaluatedAt": "t", + "evidence": {}}, + ] + eng = KyvernoJsonEngine() + out = eng.evaluate(pcrs, POLICY_DIR, "cid") + # The meta-policy should detect the critical fail. When kj runs, + # it produces a result entry. We assert the engine returns a list + # (the meta-policy PCRs). + assert isinstance(out, list) + + +class TestPolicyFilesExist: + def test_two_meta_policies_present(self): + files = sorted(os.listdir(POLICY_DIR)) + assert "block-on-any-critical.json" in files + assert "tagging-rules-agree.json" in files + + def test_policies_are_valid_json(self): + for f in os.listdir(POLICY_DIR): + if f.endswith(".json"): + with open(POLICY_DIR / f, "r", encoding="utf-8") as fh: + data = json.load(fh) + assert data["apiVersion"] == "json.kyverno.io/v1alpha1" + assert data["kind"] == "ValidatingPolicy" + assert "nova.cloudinit.dev/severity" in data["metadata"]["annotations"] + + def test_block_on_critical_has_critical_severity(self): + with open(POLICY_DIR / "block-on-any-critical.json", "r", encoding="utf-8") as fh: + data = json.load(fh) + assert data["metadata"]["annotations"]["nova.cloudinit.dev/severity"] == "critical" \ No newline at end of file diff --git a/tests/test_plan_json_policies.py b/tests/test_plan_json_policies.py new file mode 100644 index 0000000..778e023 --- /dev/null +++ b/tests/test_plan_json_policies.py @@ -0,0 +1,73 @@ +"""Tests for plan-JSON kyverno-json policies (REQ-302, v1.25). + +Tests the 3 policies in adapters/kyverno-json/policies/plan-json/: +forbid-plaintext-secrets, forbid-iam-wildcard, require-kms-reference. +Uses passing + failing fixtures. Skips when kj is absent. +""" + +import json +import os +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +import importlib.util +_ENGINE_PATH = Path(__file__).resolve().parent.parent / "adapters" / "kyverno-json" / "kyverno_json_engine.py" +_spec = importlib.util.spec_from_file_location("kyverno_json_engine", _ENGINE_PATH) +_mod = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_mod) +KyvernoJsonEngine = _mod.KyvernoJsonEngine + +POLICY_DIR = Path(__file__).resolve().parent.parent / "adapters" / "kyverno-json" / "policies" / "plan-json" +FIXTURES = Path(__file__).resolve().parent / "fixtures" / "plan_json" + + +def _kj_installed() -> bool: + return _mod._which_kj() is not None + + +@pytest.fixture(autouse=True) +def _require_kj(): + if not _kj_installed(): + pytest.skip("kj not installed (scripts/install-kyverno-json.sh)") + + +def _load(name): + with open(FIXTURES / name, "r", encoding="utf-8") as fh: + return json.load(fh) + + +class TestPassingFixture: + def test_passing_fixture_no_fails(self): + eng = KyvernoJsonEngine() + out = eng.evaluate(_load("passing.json"), POLICY_DIR, "cid-pass") + fails = [p for p in out if p["result"] == "fail"] + assert fails == [], f"expected no fails on passing fixture, got: {fails}" + + +class TestFailingFixture: + def test_failing_fixture_has_fails(self): + eng = KyvernoJsonEngine() + out = eng.evaluate(_load("failing.json"), POLICY_DIR, "cid-fail") + fails = [p for p in out if p["result"] == "fail"] + assert len(fails) >= 1, "expected at least one fail on the failing fixture" + + +class TestPolicyFilesExist: + def test_three_policies_present(self): + files = sorted(os.listdir(POLICY_DIR)) + assert "forbid-plaintext-secrets.json" in files + assert "forbid-iam-wildcard.json" in files + assert "require-kms-reference.json" in files + + def test_policies_are_valid_json(self): + for f in os.listdir(POLICY_DIR): + if f.endswith(".json"): + with open(POLICY_DIR / f, "r", encoding="utf-8") as fh: + data = json.load(fh) + assert data["apiVersion"] == "json.kyverno.io/v1alpha1" + assert data["kind"] == "ValidatingPolicy" + assert "nova.cloudinit.dev/severity" in data["metadata"]["annotations"] \ No newline at end of file diff --git a/tests/test_policy_engine.py b/tests/test_policy_engine.py new file mode 100644 index 0000000..3311f64 --- /dev/null +++ b/tests/test_policy_engine.py @@ -0,0 +1,125 @@ +"""Tests for core/policy_engine.py (REQ-308, v1.25). + +Protocol conformance, registry selection, NullEngine fallback, +unknown-engine KeyError, and the NullEngine-satisfies-Protocol +assertion (G-Q8a — proves the swap boundary is real without +implementing OPA). +""" + +import json +import os +import sys +from pathlib import Path +from unittest import mock + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +import core.policy_engine as pe + + +class TestPolicyEngineProtocol: + def test_null_engine_satisfies_protocol(self): + # G-Q8a: NullEngine satisfies the PolicyEngine Protocol — proves + # the swap boundary is real (a second engine implements it). + eng = pe.NullEngine() + assert isinstance(eng, pe.PolicyEngine) + + def test_null_engine_is_configured_false(self): + assert pe.NullEngine().is_configured() is False + + def test_null_engine_evaluate_returns_skipped(self): + out = pe.NullEngine().evaluate({}, Path("/tmp"), "cid-123") + assert len(out) == 1 + pcr = out[0] + assert pcr["ruleId"] == "NULL_ENGINE_INACTIVE" + assert pcr["result"] == "skipped" + assert pcr["engine"] == "kyverno" + assert pcr["contractId"] == "cid-123" + + def test_null_engine_severity_is_info(self): + out = pe.NullEngine().evaluate({}, Path("/tmp"), "cid") + assert out[0]["severity"] == "info" + + +class TestRegistry: + def test_register_and_get(self, tmp_path, monkeypatch): + # Register a stub engine and verify get_engine() returns it. + class StubEngine: + name = "stub" + + def is_configured(self) -> bool: + return True + + def evaluate(self, payload, policy_dir, contract_id): + return [{"contractId": contract_id, "engine": "kyverno", + "ruleId": "STUB", "result": "pass", "severity": "info", + "message": "", "evaluatedAt": "t", "resourceRef": "", + "evidence": {}}] + + pe._REGISTRY.clear() + pe.register("stub", StubEngine) + monkeypatch.setattr(pe, "_load_config_policy", lambda: {"engine": "stub"}) + eng = pe.get_engine() + assert eng.name == "stub" + pe._REGISTRY.clear() + pe._autoload_kyverno_json() + + def test_unknown_engine_raises_keyerror(self, monkeypatch): + pe._REGISTRY.clear() + monkeypatch.setattr(pe, "_load_config_policy", + lambda: {"engine": "nonexistent"}) + with pytest.raises(KeyError, match="Unknown policy engine"): + pe.get_engine() + pe._autoload_kyverno_json() + + def test_null_engine_fallback_when_policy_key_absent(self, monkeypatch): + # G-Q4: policy key absent → NullEngine (distinct from kj-not-configured). + monkeypatch.setattr(pe, "_load_config_policy", lambda: None) + eng = pe.get_engine() + assert isinstance(eng, pe.NullEngine) + assert eng.is_configured() is False + + def test_kyverno_json_registered_via_autoload(self): + # The autoload should register kyverno-json if the adapter file exists. + pe._autoload_kyverno_json() + assert "kyverno-json" in pe._REGISTRY or len(pe._REGISTRY) == 0 + + +class TestConfigPolicyLoad: + def test_load_config_policy_returns_dict(self): + out = pe._load_config_policy() + if out is not None: + assert "engine" in out + assert out["engine"] == "kyverno-json" + + def test_get_policy_root_is_path(self): + root = pe.get_policy_root() + assert isinstance(root, Path) + assert root.name == "policies" or str(root).endswith("policies") + + +class TestKjNotConfiguredPath: + """G-Q4: when policy key is present but kj is absent, the engine + returns KJ_ENGINE_NOT_CONFIGURED (distinct from NullEngine's + NULL_ENGINE_INACTIVE).""" + + def test_kj_not_configured_returns_distinct_ruleid(self, monkeypatch): + # Force the registry to return KyvernoJsonEngine, then mock + # `which kj` to return None. + pe._autoload_kyverno_json() + if "kyverno-json" not in pe._REGISTRY: + pytest.skip("kyverno-json adapter not loadable in this env") + monkeypatch.setattr(pe, "_load_config_policy", + lambda: {"engine": "kyverno-json"}) + eng = pe.get_engine() + # Mock is_configured → False + with mock.patch.object(eng, "is_configured", return_value=False): + out = eng.evaluate({}, Path("/tmp"), "cid-456") + assert len(out) == 1 + assert out[0]["ruleId"] == "KJ_ENGINE_NOT_CONFIGURED" + assert out[0]["result"] == "skipped" + assert out[0]["contractId"] == "cid-456" + # Distinct from NullEngine + assert out[0]["ruleId"] != "NULL_ENGINE_INACTIVE" \ No newline at end of file diff --git a/tests/test_regression_policies.py b/tests/test_regression_policies.py new file mode 100644 index 0000000..a5ab90b --- /dev/null +++ b/tests/test_regression_policies.py @@ -0,0 +1,88 @@ +"""Tests for regression-gate kyverno-json policies (REQ-304, REQ-305, v1.25). + +Tests the 3 declarative mirrors of core/regression_verify.py: +cap-013-adapter-dedup, cap-023-metrics-collector, cap-024-deck-structure. +Uses clean + drifted capability-inventory fixtures. Skip-without-kj. +""" + +import json +import os +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +import importlib.util +_ENGINE_PATH = Path(__file__).resolve().parent.parent / "adapters" / "kyverno-json" / "kyverno_json_engine.py" +_spec = importlib.util.spec_from_file_location("kyverno_json_engine", _ENGINE_PATH) +_mod = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_mod) +KyvernoJsonEngine = _mod.KyvernoJsonEngine + +POLICY_DIR = Path(__file__).resolve().parent.parent / "adapters" / "kyverno-json" / "policies" / "regression" +FIXTURES = Path(__file__).resolve().parent / "fixtures" / "capability_inventory" + + +def _kj_installed() -> bool: + return _mod._which_kj() is not None + + +@pytest.fixture(autouse=True) +def _require_kj(): + if not _kj_installed(): + pytest.skip("kj not installed (scripts/install-kyverno-json.sh)") + + +def _load(name): + with open(FIXTURES / name, "r", encoding="utf-8") as fh: + return json.load(fh) + + +class TestCleanInventory: + def test_clean_inventory_no_fails(self): + eng = KyvernoJsonEngine() + out = eng.evaluate(_load("clean.json"), POLICY_DIR, "cid-clean") + fails = [p for p in out if p["result"] == "fail"] + assert fails == [], f"expected no fails on clean inventory, got: {fails}" + + +class TestDriftedInventory: + def test_drifted_inventory_has_fails(self): + eng = KyvernoJsonEngine() + out = eng.evaluate(_load("drifted.json"), POLICY_DIR, "cid-drift") + fails = [p for p in out if p["result"] == "fail"] + assert len(fails) >= 1, "expected at least one fail on the drifted inventory" + + +class TestPolicyFilesExist: + def test_three_regression_policies_present(self): + files = sorted(os.listdir(POLICY_DIR)) + assert "cap-013-adapter-dedup.json" in files + assert "cap-023-metrics-collector.json" in files + assert "cap-024-deck-structure.json" in files + + def test_policies_are_valid_json(self): + for f in os.listdir(POLICY_DIR): + if f.endswith(".json"): + with open(POLICY_DIR / f, "r", encoding="utf-8") as fh: + data = json.load(fh) + assert data["apiVersion"] == "json.kyverno.io/v1alpha1" + assert data["kind"] == "ValidatingPolicy" + assert "nova.cloudinit.dev/severity" in data["metadata"]["annotations"] + + +class TestFixturesExist: + def test_clean_and_drifted_fixtures_present(self): + assert (FIXTURES / "clean.json").is_file() + assert (FIXTURES / "drifted.json").is_file() + + def test_drifted_fixture_has_duplicate_adapter(self): + data = _load("drifted.json") + # The drifted fixture has 'terraform' twice (adapter dedup violation). + assert data["adapters"].count("terraform") == 2 + + def test_drifted_fixture_has_missing_roadmap_beat(self): + data = _load("drifted.json") + assert "Roadmap+Ask" not in data["deck"]["beats"] \ No newline at end of file diff --git a/tests/test_run_platform_plan_json_policies.py b/tests/test_run_platform_plan_json_policies.py new file mode 100644 index 0000000..de849b0 --- /dev/null +++ b/tests/test_run_platform_plan_json_policies.py @@ -0,0 +1,64 @@ +"""Tests for run_platform.sh Step 5b kyverno-json wiring (REQ-302, v1.25). + +Asserts the script has the kyverno-json Step 5b block and the PCR-merge +logic. Pattern from tests/test_pipeline.py:79-95 (read script text + +assert substrings). +""" + +import os +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +SCRIPT = Path(__file__).resolve().parent.parent / "scripts" / "run_platform.sh" + + +def _read_script(): + with open(SCRIPT, "r", encoding="utf-8") as fh: + return fh.read() + + +class TestStep5bKyvernoJsonWiring: + def test_step_5b_block_present(self): + s = _read_script() + assert "Step 5b: kyverno-json plan-JSON policies" in s, \ + "run_platform.sh must have a Step 5b kyverno-json block (REQ-301)" + + def test_step_5c_meta_block_present(self): + s = _read_script() + assert "Step 5c: kyverno-json meta-policies over the merged PCR list" in s, \ + "run_platform.sh must have a Step 5c meta-policy block (REQ-303, P1-1 fix)" + + def test_kj_scan_invocation_present(self): + s = _read_script() + assert "adapters/kyverno-json/policies/plan-json" in s, \ + "Step 5b must reference the plan-json policy dir" + + def test_kj_not_installed_skip_present(self): + s = _read_script() + assert "kyverno-json not installed; skipping plan-JSON policies" in s, \ + "Step 5b must skip gracefully when kj is absent (D-120)" + assert "D-120 graceful degradation" in s + + def test_pcr_merge_logic_present(self): + s = _read_script() + assert "merged PCR list" in s, \ + "Step 5b must merge the Checkov/Wiz + kj PCR lists" + + def test_command_v_kj_guard_present(self): + s = _read_script() + assert "command -v kj" in s, \ + "Step 5b must guard on `command -v kj` (is_configured)" + + +class TestExistingPipelineUnchanged: + def test_step_5_still_present(self): + s = _read_script() + assert "Step 5: runtime policy scan" in s + + def test_step_7_confidence_still_present(self): + s = _read_script() + assert "Step 7: confidence signal compute" in s \ No newline at end of file diff --git a/tests/test_stack_ir_policies.py b/tests/test_stack_ir_policies.py new file mode 100644 index 0000000..3883c7f --- /dev/null +++ b/tests/test_stack_ir_policies.py @@ -0,0 +1,86 @@ +"""Tests for stack-IR kyverno-json policies (REQ-299, v1.25). + +Tests the 3 policies in adapters/kyverno-json/policies/stack-ir/: +require-tagging-standard, forbid-public-ingress, require-encryption-by- +default. Uses the passing + failing fixtures. Skips when kj is absent. +""" + +import json +import os +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +import importlib.util +_ENGINE_PATH = Path(__file__).resolve().parent.parent / "adapters" / "kyverno-json" / "kyverno_json_engine.py" +_spec = importlib.util.spec_from_file_location("kyverno_json_engine", _ENGINE_PATH) +_mod = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_mod) +KyvernoJsonEngine = _mod.KyvernoJsonEngine + +POLICY_DIR = Path(__file__).resolve().parent.parent / "adapters" / "kyverno-json" / "policies" / "stack-ir" +FIXTURES = Path(__file__).resolve().parent / "fixtures" / "stack_ir" + + +def _kj_installed() -> bool: + return _mod._which_kj() is not None + + +@pytest.fixture(autouse=True) +def _require_kj(): + if not _kj_installed(): + pytest.skip("kj not installed (scripts/install-kyverno-json.sh)") + + +def _load(name): + with open(FIXTURES / name, "r", encoding="utf-8") as fh: + return json.load(fh) + + +class TestPassingFixture: + def test_passing_fixture_all_pass(self): + eng = KyvernoJsonEngine() + out = eng.evaluate(_load("passing.json"), POLICY_DIR, "cid-pass") + assert isinstance(out, list) + assert len(out) >= 1 + # No fail results on the passing fixture. + fails = [p for p in out if p["result"] == "fail"] + assert fails == [], f"expected no fails on passing fixture, got: {fails}" + + +class TestFailingFixture: + def test_failing_fixture_has_fails(self): + eng = KyvernoJsonEngine() + out = eng.evaluate(_load("failing.json"), POLICY_DIR, "cid-fail") + fails = [p for p in out if p["result"] == "fail"] + assert len(fails) >= 1, "expected at least one fail on the failing fixture" + + +class TestPolicyFilesExist: + def test_three_policies_present(self): + files = sorted(os.listdir(POLICY_DIR)) + assert "require-tagging-standard.json" in files + assert "forbid-public-ingress.json" in files + assert "require-encryption-by-default.json" in files + + +class TestPolicyValidity: + def test_policies_are_valid_json(self): + for f in os.listdir(POLICY_DIR): + if f.endswith(".json"): + with open(POLICY_DIR / f, "r", encoding="utf-8") as fh: + data = json.load(fh) + assert data["apiVersion"] == "json.kyverno.io/v1alpha1" + assert data["kind"] == "ValidatingPolicy" + assert "nova.cloudinit.dev/severity" in data["metadata"]["annotations"] + + def test_policy_names_match_filenames(self): + for f in os.listdir(POLICY_DIR): + if f.endswith(".json"): + with open(POLICY_DIR / f, "r", encoding="utf-8") as fh: + data = json.load(fh) + expected = f.rsplit(".", 1)[0] + assert data["metadata"]["name"] == expected \ No newline at end of file