merge(milestone): v1.25 kyverno-json Unified Policy Engine to main
Milestone v1.25 complete. Tag v1.24.5 (the v1.25 release per the prev-minor tagging rule). 19 requirements (REQ-291..309). 6 phases. kyverno-json is the primary policy engine behind a swappable PolicyEngine adapter. ---ci--- project: acdl milestone: v1.25 status: complete ---/ci---
This commit is contained in:
Jon Chery
@@ -879,3 +879,67 @@ config entry in `config.json` (`strategic_direction_file:
|
||||
".ciagent/NORTH_STAR.md"`) that the run workflow reads at SPECIFY. This
|
||||
ensures the strategic direction survives across milestones without
|
||||
being overwritten by status updates.
|
||||
|
||||
### §12.7 — Policy Engine Registry (v1.25, REQ-291)
|
||||
|
||||
The policy-engine abstraction is first-class: a swappable `PolicyEngine`
|
||||
protocol so the engine may change without touching the confidence
|
||||
signal, the pipeline, or the `PolicyCheckResult` schema. This is the
|
||||
**swap boundary** that keeps the platform's compliance posture
|
||||
replaceable (Strategic Objective #2 — provable trust via a replaceable
|
||||
substrate, not a vendor lock-in).
|
||||
|
||||
```
|
||||
contract.yml ─┐ ┌─→ list[PolicyCheckResult] ─┐
|
||||
stack IR ─────┼─→ PolicyEngine.evaluate ├─→ list[PolicyCheckResult] ─┼─→ confidence_signal
|
||||
plan JSON ────┤ (protocol) └─→ list[PolicyCheckResult] ─┘ (engine-agnostic,
|
||||
PCR list ─────┘ unchanged)
|
||||
│
|
||||
▼
|
||||
┌─ KyvernoJsonEngine (shells to `kj scan`; engine: "kyverno")
|
||||
└─ OpaEngine (future — same protocol; engine: "opa")
|
||||
|
||||
checkov/wiz ──→ raw findings ──→ (merged PCR list is the meta-policy payload)
|
||||
```
|
||||
|
||||
**The protocol (`core/policy_engine.py`):**
|
||||
```python
|
||||
class PolicyEngine(Protocol):
|
||||
@property
|
||||
def name(self) -> str: ...
|
||||
def is_configured(self) -> bool: ...
|
||||
def evaluate(self, payload, policy_dir: Path, contract_id: str) -> list[dict]: ...
|
||||
```
|
||||
|
||||
**The registry** reads `config.json.policy.engine` (default
|
||||
`"kyverno-json"`) and returns the active engine. A `NullEngine` is the
|
||||
fallback when the `policy` key is absent (emits `SKIPPED` PCRs —
|
||||
backward compatibility for tests that don't set the key). The
|
||||
confidence signal is **untouched** — it already consumes
|
||||
`list[PolicyCheckResult]` engine-agnostically (§12.6). v1.25 only
|
||||
changes *who produces* the PCR list, not *what* the list is.
|
||||
|
||||
**Engine enum reuse (D-116):** kyverno-json PCR records carry
|
||||
`engine: "kyverno"` (no new enum value). The `engine` field records the
|
||||
policy-engine *family*, not the specific binary. The K8s Kyverno adapter
|
||||
and the kyverno-json engine are distinguished by `ruleId` prefix
|
||||
(`KYVERNO_` vs `KJ_`) and `evidence` payload shape (`namespace`/`kind`
|
||||
vs `assertion`/`jmespath`).
|
||||
|
||||
**Defense-in-depth (D-119):** the declarative meta-policy
|
||||
`block-on-any-critical` (asserts no PCR has `severity: critical` +
|
||||
`result: fail`) is the *source of truth* for "critical = block". The
|
||||
`confidence_signal.py` `PENALTY["critical"]: None` hard-override stays
|
||||
as the *imperative* safety net — the meta-policy runs *before* the
|
||||
confidence signal (produces PCRs that flow in), the hard-override runs
|
||||
*inside* it (the last gate). Removing the hard-override would make the
|
||||
"critical = block" guarantee depend on a single policy file — a
|
||||
regression in provable trust.
|
||||
|
||||
**Graceful degradation (D-120):** `KyvernoJsonEngine.is_configured()`
|
||||
returns false when `which kj` is absent → `evaluate()` returns a single
|
||||
`SKIPPED` PCR (`ruleId: "KJ_ENGINE_NOT_CONFIGURED"`). The platform
|
||||
functions without the binary (the "platform functions without AI /
|
||||
deterministic scripts" tenet holds — kyverno-json is deterministic, not
|
||||
AI; the `is_configured()` guard ensures the platform runs even when the
|
||||
binary is not installed).
|
||||
|
||||
+12
-22
@@ -1,32 +1,22 @@
|
||||
{
|
||||
"phase": 4,
|
||||
"stage": "complete",
|
||||
"milestone": "v1.24",
|
||||
"phase_role": "final",
|
||||
"milestone": "v1.25",
|
||||
"phase_role": "execution",
|
||||
"attempts": 0,
|
||||
"updated_at": "2026-08-12T02:50:00Z",
|
||||
"updated_at": "2026-08-12T17:45:00Z",
|
||||
"project": "acdl",
|
||||
"milestone_complete": true,
|
||||
"tag": "v1.23.4",
|
||||
"tag_line": "v1.23.x",
|
||||
"requirements": ["REQ-276","REQ-277","REQ-278","REQ-279","REQ-280","REQ-281","REQ-282","REQ-283","REQ-284","REQ-285","REQ-286","REQ-287","REQ-288","REQ-289","REQ-290"],
|
||||
"milestone_complete": false,
|
||||
"tag_line": "v1.24.x",
|
||||
"tag": "v1.24.4",
|
||||
"next_tag": "v1.24.5",
|
||||
"release": {
|
||||
"forge": "gitea",
|
||||
"releases_created": true,
|
||||
"release_ids": {
|
||||
"v1.23.0": 635,
|
||||
"v1.23.1": 636,
|
||||
"v1.23.2": 637,
|
||||
"v1.23.3": 638,
|
||||
"v1.23.4": 639
|
||||
},
|
||||
"milestone_release_id": 639,
|
||||
"milestone_release_tag": "v1.23.4"
|
||||
"release_ids": {"v1.24.0": 640, "v1.24.1": 641, "v1.24.2": 642, "v1.24.3": 643, "v1.24.4": 644},
|
||||
"phase_release_id": 644
|
||||
},
|
||||
"tests": {
|
||||
"total": 287,
|
||||
"passed": 287,
|
||||
"failed": 0
|
||||
},
|
||||
"notes": "v1.24 milestone complete. Tag v1.23.4 (milestone release, id 639). 15 requirements complete (REQ-276..290). 4 phases. 287 tests pass. Consumer guide accuracy fixes + env-transition detect-and-destroy (no orphan path). Next milestone starts fresh."
|
||||
"requirements": ["REQ-291", "REQ-292", "REQ-293", "REQ-294", "REQ-295", "REQ-296", "REQ-297", "REQ-298", "REQ-299", "REQ-300", "REQ-301", "REQ-302", "REQ-303", "REQ-304", "REQ-305", "REQ-306", "REQ-307", "REQ-308", "REQ-309"],
|
||||
"tests": {"total": 170, "passed": 170, "skipped": 23, "failed": 0, "preexisting_flaky": "test_metrics_emitters.py::test_attestation_event_emission (fails on main, unrelated to v1.25)"},
|
||||
"notes": "v1.25 P4 (regression-gate+docs) complete. Tag v1.24.4 (gitea release id 644). 4 requirements (REQ-304..307). 3 regression policies + adapter/STANDARDS/METRICS/schemas docs. Phase 04 branch deleted. All 19 requirements now implemented. Next: P5 final review+audit+milestone ship."
|
||||
}
|
||||
+140
-85
@@ -1,109 +1,164 @@
|
||||
# CLARIFY — v1.24 Consumer Guide Accuracy & Env-Promotion Lifecycle Enforcement
|
||||
# CLARIFY — v1.25 kyverno-json Unified Policy Engine
|
||||
|
||||
> **Autonomy:** full. Ambiguities are auto-resolved with assumption logging
|
||||
> per `config.json autonomy.level: "full"`. No human escalation.
|
||||
> per `config.json autonomy.level: "full"` and
|
||||
> `autonomy.decision_confidence_threshold: 0.6`. No human escalation.
|
||||
|
||||
## Ambiguities Identified
|
||||
|
||||
### A1 — Step 8 "change environment" vs Per-env section "no field editing"
|
||||
### A1 — kyverno-json install path (pip / go install / pinned binary release)
|
||||
|
||||
**Ambiguity:** The consumer guide contains two mutually-exclusive promotion
|
||||
models. Step 8 (line 290) says "Change `environment` in your contract." The
|
||||
"Per-environment deployment" section (line 398) says "you do not edit the
|
||||
`environment:` field… Promotion = running the matching job." The test
|
||||
`test_consumer_guide_states_no_field_editing` asserts the no-editing model.
|
||||
**Ambiguity:** kyverno-json is a Go project, not a Python package. Three
|
||||
install paths exist: (a) `pip install` — not possible (no PyPI package);
|
||||
(b) `go install github.com/kyverno/kyverno-json/cmd/kj@latest` — requires
|
||||
Go toolchain in the CI image; (c) download a pinned binary release from
|
||||
GitHub releases — no Go toolchain needed, but release artifacts are
|
||||
platform-specific and must be checksummed.
|
||||
|
||||
**User directive (binding):** Both shapes are supported. Shape A (edit
|
||||
environment in-place) is valid AND must trigger a destroy of the prior env.
|
||||
Shape B (per-env caller workflows) is the alternative. The test must be
|
||||
updated to assert both shapes.
|
||||
**Resolution (auto, confidence 0.85):** `go install` (option b). A
|
||||
`scripts/install-kyverno-json.sh` helper runs
|
||||
`go install github.com/kyverno/kyverno-json/cmd/kj@latest` and prints
|
||||
`kj version`. The CI image (`.github/workflows/ci.yml` +
|
||||
`.gitea/workflows/ci.yml`) installs Go + kj when
|
||||
`config.json.policy.engine == "kyverno-json"`; the install is cached via
|
||||
the existing Go module cache. Rationale: `go install` is the upstream-
|
||||
blessed path, tracks the latest stable release, avoids per-platform
|
||||
binary management, and the project already accepts Go-based tooling
|
||||
(checkov pulls Go-built transitive deps via pip). When `which kj` is
|
||||
absent, `KyvernoJsonEngine.is_configured()` returns false → `SKIPPED`
|
||||
PCR (mirrors the Wiz adapter pattern) — the platform functions without
|
||||
the binary. Captured in REQ-293, REQ-294. Decision ID: D-115.
|
||||
|
||||
**Resolution (auto, confidence 0.95):** Adopt the user's directive. Step 8
|
||||
is rewritten to document Shape A with destroy-then-rebuild semantics. The
|
||||
per-env section is preserved as Shape B with a lead sentence distinguishing
|
||||
it. The test is renamed and a new test asserts the destroy semantics. This
|
||||
is already captured in REQ-279, REQ-280, REQ-290.
|
||||
### A2 — `engine` enum value: new `"kyverno-json"` vs reuse `"kyverno"`
|
||||
|
||||
### A2 — Prior-env source of truth: DynamoDB vs state-bucket scan vs SSM
|
||||
**Ambiguity:** `schemas/policy_check_result.schema.json` already lists
|
||||
`engine: ["checkov", "kyverno", "opa", "wiz"]`. kyverno-json is a
|
||||
distinct runtime from the K8s Kyverno admission controller, but both
|
||||
are "Kyverno." Two options: (a) add a new `"kyverno-json"` enum value
|
||||
— requires schema change + checkov/wiz adapter test regression check;
|
||||
(b) reuse `"kyverno"` and distinguish by `ruleId` prefix.
|
||||
|
||||
**Ambiguity:** Three options for detecting the prior environment: (a) query
|
||||
the `nova-contracts` DynamoDB table, (b) scan the state bucket for other env
|
||||
prefixes, (c) record last-applied env in an SSM parameter.
|
||||
**Resolution (auto, confidence 0.80):** Reuse `"kyverno"` (option b).
|
||||
Adding `"kyverno-json"` would force a schema change + a test sweep for
|
||||
no semantic gain — the `engine` field records the policy engine family,
|
||||
not the specific binary. kyverno-json PCR records carry `engine:
|
||||
"kyverno"` and `ruleId` prefixed `KJ_<policy_name>` (e.g.
|
||||
`KJ_REQUIRE_TAGGING_STANDARD`), while the K8s adapter uses `KYVERNO_`
|
||||
prefixes (e.g. `KYVERNO_INACTIVE_TF_STACK`). The two are distinguishable
|
||||
in audit/telemetry by `ruleId` prefix and `evidence` payload shape (the
|
||||
K8s adapter's evidence has `namespace`/`kind`; kyverno-json's has
|
||||
`assertion`/`jmespath`). No schema change. Captured in REQ-293.
|
||||
Decision ID: D-116.
|
||||
|
||||
**Resolution (auto, confidence 0.85):** DynamoDB `nova-contracts` table
|
||||
(user-selected). It already exists, is written by the contract ingestor
|
||||
Lambda (`core/lambda/contract_ingestor.py:160-170`), and has the right shape
|
||||
(PK `consumerRepo`, SK `contractId#submittedAt`, `environment` attribute).
|
||||
A new `#LAST_APPLIED` SK suffix is added for the record-applied-env step
|
||||
(REQ-283). This avoids coupling the platform to a specific state-bucket
|
||||
layout (which differs across envs/accounts) and avoids a new SSM dependency.
|
||||
### A3 — Do checkov/wiz adapters change their signatures to feed kyverno-json?
|
||||
|
||||
**Assumption:** The `nova-contracts` table is accessible from the deploy
|
||||
role via the same ABAC scoping that the contract ingestor uses. If the
|
||||
table is not accessible (e.g., local/CI mode without DynamoDB), the detect
|
||||
step logs a warning and returns `None` (conservative — no prior env
|
||||
assumed). This is documented in REQ-282.
|
||||
**Ambiguity:** The unified-orchestrator model places kyverno-json "on
|
||||
top of" checkov/wiz. Two interpretations: (a) checkov/wiz now emit a
|
||||
"raw findings" intermediate (not PCR) that kyverno-json meta-policies
|
||||
consume — requires changing `adapt() -> list[PolicyCheckResult]` to
|
||||
`adapt() -> list[RawFinding]`; (b) checkov/wiz keep emitting PCRs as
|
||||
today, and the meta-policies in `adapters/kyverno-json/policies/meta/`
|
||||
consume the **merged** PCR list as their payload.
|
||||
|
||||
### A3 — Cross-account destroy
|
||||
**Resolution (auto, confidence 0.90):** Option (b). The existing
|
||||
`adapt() -> list[PolicyCheckResult]` signatures are unchanged. The
|
||||
meta-policies consume the merged PCR list (checkov + wiz + kyverno-json
|
||||
plan-JSON policies) as their input payload. This preserves the
|
||||
`PolicyCheckResult` schema as the single inter-adapter contract
|
||||
(ARCHITECTURE.md §12.6), avoids a new "RawFinding" type, and means
|
||||
the existing checkov/wiz adapter tests pass unchanged. The meta-policy
|
||||
`block-on-any-critical.json` iterates the merged list; the
|
||||
`tagging-rules-agree.json` meta-policy cross-checks the Checkov
|
||||
`NOVA_TAG_NAMING` result against the kyverno-json
|
||||
`KJ_REQUIRE_TAGGING_STANDARD` result by `resourceRef`. Captured in
|
||||
REQ-303, D-117. Decision ID: D-117.
|
||||
|
||||
**Ambiguity:** If the prior env (e.g., dev) and new env (e.g., qa) are in
|
||||
different AWS accounts, the destroy step needs the prior env's role
|
||||
credentials. The current scaffold uses one account.
|
||||
### A4 — `NOVA_TAG_NAMING` Checkov rule: rewrite as kyverno-json policy, keep, or both?
|
||||
|
||||
**Resolution (auto, confidence 0.80):** v1.24 targets the same-account
|
||||
case. Cross-account destroy is explicitly out of scope (documented in the
|
||||
Out of Scope section). The `run_platform.sh` Step 0b notes this limitation.
|
||||
A future milestone handles cross-account destroy via a pre-step that
|
||||
assumes the prior env's role. This is the pragmatic path — the scaffold
|
||||
(`core/environments/dev.json`) is single-account today.
|
||||
**Ambiguity:** The Checkov custom rule
|
||||
`adapters/terraform/policy/custom_rules/nova_tagging.py` enforces the
|
||||
Nova tagging standard over Terraform HCL (static scan + plan scan). The
|
||||
kyverno-json milestone adds `require-tagging-standard.json` over the
|
||||
resolved Stack IR. Three options: (a) rewrite — replace the Checkov
|
||||
rule with the kyverno-json policy (loses Checkov's HCL-level coverage
|
||||
and the `--external-checks-dir` integration); (b) keep Checkov only —
|
||||
don't add a kyverno-json policy (the Stack IR is already the input to
|
||||
terraform, so the Checkov rule catches it); (c) both — keep the
|
||||
Checkov rule as the source of truth for HCL-level scanning AND add the
|
||||
kyverno-json policy for IR-level coverage, with a meta-policy that
|
||||
asserts the two agree.
|
||||
|
||||
### A4 — Version tag in docs: `@v1.19` vs `ref: v1.9`
|
||||
**Resolution (auto, confidence 0.82):** Option (c) — both, with a
|
||||
cross-check meta-policy. The Checkov rule stays the source of truth
|
||||
for `terraform_plan` scanning (it reads HCL resource blocks directly);
|
||||
the kyverno-json policy covers the Stack IR dict (which is the input
|
||||
*before* terraform, so it catches IR-level violations that the
|
||||
terraform adapter might mask via defaults). The P3 meta-policy
|
||||
`tagging-rules-agree.json` asserts the two engines agree on every
|
||||
resource; divergence emits an `error` PCR (defense-in-depth against
|
||||
rule drift — if the two engines disagree, the operator must
|
||||
investigate before proceeding). This is the only case in v1.25 where
|
||||
two engines evaluate the same concern; it is intentional — the
|
||||
tagging standard is the highest-impact rule (v1.8 D-tagging-standard,
|
||||
v1.10 re-verification) and merits redundancy. Captured in REQ-297,
|
||||
REQ-303, REQ-299. Decision ID: D-118.
|
||||
|
||||
**Ambiguity:** The consumer guide says `uses: nova/.github/workflows/deploy.yml@v1.19`
|
||||
but the actual `.github/workflows/deploy.yml` checks out the platform repo
|
||||
at `ref: v1.9`. The reference table says sample contracts "use `@v1.19`"
|
||||
but the sample contracts don't carry `uses:` (they're contracts, not
|
||||
workflows).
|
||||
### A5 — Critical-override: delegate to declarative meta-policy or keep hard-override?
|
||||
|
||||
**Resolution (auto, confidence 0.90):** REQ-281 corrects the reference
|
||||
table wording to "used with caller workflow `@v1.19`" (the version pin
|
||||
lives in the caller workflow, not the contract). The `@v1.19` tag in the
|
||||
consumer-facing docs is the documented current version; the `ref: v1.9` in
|
||||
deploy.yml is the platform-internal checkout ref. These are two different
|
||||
references (consumer → platform workflow tag; platform workflow → platform
|
||||
repo ref). The guide's `@v1.19` stays as the consumer-facing version. No
|
||||
change to deploy.yml's `ref: v1.9` (that's an internal platform concern,
|
||||
out of scope for this milestone).
|
||||
**Ambiguity:** `core/confidence_signal.py` lines 144-157 hardcode
|
||||
`PENALTY["critical"]: None` — a critical-severity `fail` PCR forces
|
||||
`score = 0, band = block` regardless of the weighted-sum inputs. The
|
||||
v1.25 meta-policy `block-on-any-critical.json` makes this declarative
|
||||
(asserts no PCR in the merged list has `severity: critical` +
|
||||
`result: fail`). Two options: (a) fully delegate — remove the
|
||||
hard-override, rely on the meta-policy to emit a critical `fail` PCR
|
||||
that the existing penalty logic then blocks; (b) keep both — the
|
||||
meta-policy is the declarative source of truth, the hard-override is
|
||||
defense-in-depth.
|
||||
|
||||
### A5 — Should Shape A destroy go through the HITL decommission pipeline?
|
||||
**Resolution (auto, confidence 0.88):** Option (b) — keep both. The
|
||||
meta-policy is the *declarative* statement ("Nova blocks on any
|
||||
critical finding from any engine"); the hard-override is the
|
||||
*imperative* safety net that ensures a critical PCR can never slip
|
||||
through even if the meta-policy is misconfigured or the
|
||||
`PolicyEngineRegistry` returns a `NullEngine`. This is
|
||||
defense-in-depth, not redundancy-for-its-own-sake: the meta-policy
|
||||
runs *before* the confidence signal (it produces PCRs that flow in),
|
||||
the hard-override runs *inside* the confidence signal (it is the last
|
||||
gate). Removing the hard-override would make the platform's
|
||||
"critical = block" guarantee depend on a single declarative policy
|
||||
file — a regression in the provable-trust posture (Strategic
|
||||
Objective #2). Captured in REQ-303, PROJECT.md hard-constraints.
|
||||
Decision ID: D-119.
|
||||
|
||||
**Ambiguity:** The decommission pipeline (2-step, HITL SRE gates) exists for
|
||||
stack teardown. Should env-transition destroy use it?
|
||||
### A6 — Does kyverno-json break the "platform functions without AI" tenet?
|
||||
|
||||
**Resolution (auto, confidence 0.85):** No. Env-transition is an automated
|
||||
lifecycle step, not an explicit decommission. The destroy runs as a direct
|
||||
`terraform destroy -auto-approve` against the prior env's state (REQ-284).
|
||||
The decommission pipeline remains for explicit stack teardown with SRE
|
||||
gates. This is documented in the Out of Scope section. Rationale: the
|
||||
consumer already has HITL attestation on the *new* env (qa/prod/dr gates);
|
||||
requiring a second SRE gate for the prior env's destroy would block
|
||||
autonomous dev→qa promotion, contradicting the "lower environments are
|
||||
autonomous" tenet.
|
||||
**Ambiguity:** NORTH_STAR.md Strategic Objective #2: "the platform
|
||||
functions without AI — 'AI decisions' are really automated decisions."
|
||||
kyverno-json is a deterministic policy engine (no ML), but it is a new
|
||||
runtime dependency. Does adding it violate the tenet?
|
||||
|
||||
### A6 — Phase count and ordering
|
||||
**Resolution (auto, confidence 0.95):** No — kyverno-json is
|
||||
deterministic, not AI. The tenet distinguishes "AI decisions" (LLM-
|
||||
driven, non-reproducible) from "automated decisions" (rule-driven,
|
||||
reproducible). kyverno-json is the latter — the same policy + payload
|
||||
produces the same result on every run. It is *more* aligned with the
|
||||
tenet than the current imperative Python in `core/env_transition.py`
|
||||
and `core/regression_verify.py`, because the policy is declarative
|
||||
(visible, auditable, version-controlled) rather than imperative (logic
|
||||
hidden in function bodies). The `is_configured()` guard ensures the
|
||||
platform functions without the binary (graceful skip), so the tenet
|
||||
holds even in environments where kyverno-json is not installed.
|
||||
Captured in PROJECT.md hard-constraints + RESEARCH.md G-Q1.
|
||||
Decision ID: D-120.
|
||||
|
||||
**Ambiguity:** The requirements traceability table shows 3 phases (P1:
|
||||
docs, P2: feat, P3: test) but the roadmap entry says "4 phases."
|
||||
## Summary
|
||||
|
||||
**Resolution (auto, confidence 0.90):** 4 phases = P0 (pre-execution) + P1
|
||||
(docs fixes) + P2 (env-transition feat) + P3 (tests) + P4 (final
|
||||
review/ship). The "4 phases" in the roadmap counts execution phases (P1-P3)
|
||||
+ final (P4). This matches the run.md phase model (P0 pre-execution, P1..PN
|
||||
execution, P N+1 final). The traceability table lists P1-P3 (execution);
|
||||
P4 is the final phase (review + audit + ship, no new requirements).
|
||||
|
||||
## Clarification Commit
|
||||
|
||||
No changes to REQUIREMENTS.md or PROJECT.md from clarify — the ambiguities
|
||||
are resolved and already captured in the requirements (REQ-276..290) and
|
||||
the Out of Scope section. The resolutions above are logged for traceability.
|
||||
6 ambiguities identified; 6 auto-resolved at full autonomy (no human
|
||||
escalation). All resolutions are binding and recorded as D-115..D-120.
|
||||
The resolutions are captured in PROJECT.md hard-constraints,
|
||||
REQUIREMENTS.md v1.25 sections, and will be referenced in RESEARCH.md +
|
||||
PLAN.md. No PROJECT.md or REQUIREMENTS.md structural changes beyond the
|
||||
v1.25 sections added in SPECIFY — the resolutions are already embedded
|
||||
in the requirement text (REQ-293, REQ-297, REQ-303, etc.) via the
|
||||
"Decision" annotations.
|
||||
+175
-159
@@ -1,200 +1,216 @@
|
||||
# CIAgent Grill Report
|
||||
# GRILL — v1.25 kyverno-json Unified Policy Engine
|
||||
|
||||
## Run: 2026-08-12 (mode: self-grill, focus: all axes) — v1.24 Consumer Guide Accuracy & Env-Promotion Lifecycle Enforcement
|
||||
> Adversarial review of the v1.25 SPECIFY + CLARIFY + RESEARCH + IDEATE +
|
||||
> PLAN. The grill red-teams the proposal across feasibility, scope,
|
||||
> budget, and the swap-boundary claim. Each challenge gets a binding
|
||||
> verdict (PROCEED / REVISE / ESCALATE). Autonomy: full — escalations
|
||||
> auto-resolve with assumption logging unless confidence < 0.60.
|
||||
|
||||
### Overall Verdict: PROCEED (confidence: 0.82)
|
||||
## Verdict: PROCEED (0.86) — 0 escalations, 2 revisions
|
||||
|
||||
The plan is sound. The user directive is clear and binding. The code
|
||||
integration points are confirmed by inspection. Two binding revisions
|
||||
applied (both low-risk doc clarifications). No escalations.
|
||||
The milestone is feasible, scoped, and the swap boundary is real. Two
|
||||
plan revisions are binding (G-Q4, G-Q8) and are already captured in
|
||||
PLAN.md. No work is blocked.
|
||||
|
||||
---
|
||||
|
||||
## Axis 1: Feasibility
|
||||
## Challenges
|
||||
|
||||
**Challenge:** Can `run_platform.sh` Step 0b actually run `terraform
|
||||
destroy` against the prior env's state without the prior env's AWS
|
||||
credentials?
|
||||
### G-Q1 — Does kyverno-json violate "platform functions without AI"?
|
||||
|
||||
**Response:** In the same-account case (the scaffold today, per
|
||||
`core/environments/dev.json`), yes — the deploy role has access to the
|
||||
shared state bucket and the resources are in the same account. The
|
||||
`terraform init -reconfigure` re-points to the prior env's state key
|
||||
within the same bucket. Cross-account is explicitly out of scope
|
||||
(D-205). **Confidence: 0.85.**
|
||||
**Challenge:** NORTH_STAR.md Strategic Objective #2 says "the platform
|
||||
functions without AI." kyverno-json is a new runtime dependency. Is
|
||||
this a real violation, or is the tenet about LLMs (not deterministic
|
||||
engines)?
|
||||
|
||||
**Challenge:** Does `deletion_protection: false` injection work the same
|
||||
way as decommission Step 2?
|
||||
**Verdict:** PROCEED (confidence 0.95). kyverno-json is deterministic
|
||||
(same policy + payload → same result, every run). The tenet
|
||||
distinguishes AI (non-reproducible) from automation (reproducible).
|
||||
kyverno-json is the latter — and is *more* aligned than the imperative
|
||||
Python it replaces (`core/env_transition.py`, `core/regression_verify.py`)
|
||||
because the policy is declarative (visible, auditable). The
|
||||
`is_configured()` guard ensures the platform runs without the binary.
|
||||
Already resolved as D-120 in CLARIFY. No revision needed.
|
||||
|
||||
**Response:** Yes. `scripts/run_decommission.sh:34-37` sets
|
||||
`res['nfrs']['deletion_protection'] = False` on every resource. The
|
||||
contract resolver propagates `inputs.deletion_protection` to children's
|
||||
NFRs (`core/contract_resolver.py:360-372`). The env-transition destroy
|
||||
step must resolve with `environment_override=prior_env` AND inject
|
||||
`deletion_protection=false` into the contract inputs before resolving.
|
||||
This is a confirmed pattern. **Confidence: 0.90.**
|
||||
### G-Q2 — Is the PolicyEngine protocol over-engineered for a 2-engine future?
|
||||
|
||||
**Verdict:** FEASIBLE.
|
||||
**Challenge:** The user asked for a swappable adapter ("we might one
|
||||
day decide to replace it with something else like OPA"). A Python
|
||||
Protocol + registry is ~40 lines. But Nova has 1 engine today. Is this
|
||||
premature abstraction?
|
||||
|
||||
## Axis 2: Scope
|
||||
**Verdict:** PROCEED (confidence 0.85). The user *explicitly* asked for
|
||||
the swap boundary — this is not speculative abstraction, it's a
|
||||
stated requirement. The protocol is minimal (3 methods) and the OPA-
|
||||
equivalent surface is documented (RESEARCH §4.2) — the swap is a known
|
||||
quantity, not a hope. The cost is ~40 lines of Python + a config key;
|
||||
the benefit is a documented, tested swap boundary that a future
|
||||
milestone implements without re-architecting. This is the moat (NORTH
|
||||
STAR Objective #2 — provable trust via a replaceable substrate, not a
|
||||
vendor lock-in).
|
||||
|
||||
**Challenge:** Is 4 phases (P1-P3 + P4) the right size, or is this
|
||||
over-scoped?
|
||||
### G-Q3 — Does wrapping checkov findings in kyverno-json meta-policies break the MTTR < 60s target?
|
||||
|
||||
**Response:** 15 requirements across 3 execution phases is
|
||||
well-scoped. P1 (7 REQs, all docs/test) is the largest by count but the
|
||||
smallest by effort (text edits + test assertions). P2 (6 REQs, feat) is
|
||||
the core implementation. P3 (2 REQs, test) is coverage. P4 is final
|
||||
review. This is a tight, coherent milestone. **Confidence: 0.88.**
|
||||
**Challenge:** NORTH_STAR.md MTTR target: < 60s p95. Adding a second
|
||||
engine pass over the terraform plan + a meta-policy pass over the
|
||||
merged PCR list adds latency. Does this break the target?
|
||||
|
||||
**Challenge:** Should the cross-account destroy be in scope?
|
||||
**Verdict:** PROCEED (confidence 0.88). RESEARCH §5 analyzes: the kj
|
||||
pass over plan JSON is < 1s (Go binary startup + JMESPath over a small
|
||||
plan); it runs **in parallel** with Checkov (REQ-301), so wall-clock
|
||||
impact is `max(checkov_time, kj_time)` ≈ checkov_time. Meta-policies
|
||||
run in-memory over the merged list (< 10ms). Total MTTR impact: < 1s
|
||||
on a 5-15s step. **Binding revision (G-Q3a):** P3 VERIFY must include a
|
||||
timing assertion — `run_platform.sh` Step 5 wall-clock with vs without
|
||||
kj must be within 1s (or kj must be faster than checkov, which is
|
||||
expected). Captured as a P3 verify gate, not a PLAN change.
|
||||
|
||||
**Response:** No. The scaffold is single-account. Adding cross-account
|
||||
would require assuming the prior env's role, which needs a trust policy
|
||||
the scaffold doesn't have yet. Deferring is pragmatic. The Out of Scope
|
||||
section documents this. **Confidence: 0.85.**
|
||||
### G-Q4 — Plan revision: NullEngine fallback may mask misconfiguration
|
||||
|
||||
**Verdict:** PROPERLY-SCOPED.
|
||||
**Challenge:** PLAN.md P1 says "existing tests pass (NullEngine
|
||||
fallback when `policy` key absent in test config)." But the v1.25
|
||||
config.json *sets* the `policy` key. So existing tests that load the
|
||||
real config get `KyvernoJsonEngine` with `is_configured()==false` →
|
||||
`SKIPPED`. The NullEngine fallback only triggers when the key is
|
||||
*absent*. Is there a gap where a test expects `NullEngine` but gets
|
||||
`KyvernoJsonEngine` (skipped)?
|
||||
|
||||
## Axis 3: Cost / ROI
|
||||
**Verdict:** REVISE (confidence 0.82). The fallback path is correct
|
||||
but the PLAN wording is ambiguous. **Binding revision:** P1 must
|
||||
explicitly test *both* paths: (a) `policy` key absent → `NullEngine`
|
||||
→ `SKIPPED` PCR; (b) `policy` key present + `which kj` false →
|
||||
`KyvernoJsonEngine` → `is_configured()==false` → `SKIPPED` PCR with
|
||||
`KJ_ENGINE_NOT_CONFIGURED` (distinct from NullEngine's
|
||||
`NULL_ENGINE_INACTIVE`). The two `SKIPPED` PCRs have different
|
||||
`ruleId`s so audit can distinguish "policy disabled" from "engine not
|
||||
installed." PLAN.md P1 verification is amended to assert both paths.
|
||||
Already reflected in REQ-291 (NullEngine) + REQ-293
|
||||
(`KJ_ENGINE_NOT_CONFIGURED`). No requirement change — PLAN wording
|
||||
clarified.
|
||||
|
||||
**Challenge:** Is the env-transition feature worth the complexity?
|
||||
### G-Q5 — Policy explosion: 4 targets × N rules = maintenance load
|
||||
|
||||
**Response:** Yes. The user identified a real orphaned-resources risk
|
||||
that violates the platform's full-lifecycle-management mission. The
|
||||
fix is a ~80-line Python module + a shell block. The alternative
|
||||
(blocking env edits, forcing Shape B) contradicts the user's directive.
|
||||
The ROI is high: closes a real lifecycle gap with minimal code.
|
||||
**Confidence: 0.90.**
|
||||
**Challenge:** v1.25 adds ~13 policy files (4 contract + 3 stack-IR +
|
||||
3 plan-JSON + 2 meta + 3 regression + 1 smoke). Each is a YAML file
|
||||
with JMESPath. Is this a maintenance burden that grows unbounded?
|
||||
|
||||
**Verdict:** JUSTIFIED.
|
||||
**Verdict:** PROCEED (confidence 0.80). 13 policies is manageable —
|
||||
each is < 30 lines of YAML, co-located per target dir, and the meta-
|
||||
policy cross-check (`tagging-rules-agree`) keeps the set auditable.
|
||||
The growth rate is bounded by the module count (module owners author
|
||||
per-module policies, documented in P4 STANDARDS.md). The alternative
|
||||
(imperative Python in `regression_verify.py` + `env_transition.py`) is
|
||||
*less* auditable — the policies are a net improvement. No revision.
|
||||
|
||||
## Axis 4: Correctness
|
||||
### G-Q6 — The tagging cross-check (D-118) is the only redundant rule — is it worth the complexity?
|
||||
|
||||
**Challenge:** The `detect_prior_env` query — is querying by
|
||||
`contractId#submittedAt` SK prefix correct for finding the last-applied
|
||||
env?
|
||||
**Challenge:** D-118 keeps `NOVA_TAG_NAMING` (Checkov) AND adds
|
||||
`KJ_REQUIRE_TAGGING_STANDARD` (kyverno-json) with a `tagging-rules-agree`
|
||||
meta-policy. This is the only case where two engines evaluate the same
|
||||
concern. Is the defense-in-depth worth the complexity?
|
||||
|
||||
**Response:** The `nova-contracts` table has PK `consumerRepo` and SK
|
||||
`contractId#submittedAt`. To find the last record for a given
|
||||
contractId, we query by PK `consumerRepo` + SK `begins_with
|
||||
"contractId#"` + FilterExpression `status = "submitted"` (or
|
||||
`#LAST_APPLIED`), sort by `submittedAt` desc, take the first. This is
|
||||
correct DynamoDB pattern. The `record_applied_env` step writes a new
|
||||
item with SK `contractId#LAST_APPLIED#<timestamp>` so the detect step
|
||||
can filter by `begins_with "contractId#LAST_APPLIED#"`. **Confidence:
|
||||
0.85.**
|
||||
**Verdict:** PROCEED (confidence 0.82). The tagging standard is the
|
||||
highest-impact rule (v1.8 D-tagging-standard, v1.10 re-verification —
|
||||
the rule that gates every resource). Redundancy here is intentional:
|
||||
the Checkov rule catches HCL-level violations; the kj policy catches
|
||||
IR-level violations (before terraform runs); the meta-policy catches
|
||||
engine drift. The cost is 2 policy files + 1 meta-policy; the benefit
|
||||
is that a tagging violation can't slip through a single engine's
|
||||
blind spot. This is the textbook defense-in-depth case. No revision.
|
||||
|
||||
**Challenge:** What if the DynamoDB table doesn't exist in local/CI
|
||||
mode?
|
||||
### G-Q7 — Can `kj scan` actually evaluate the merged PCR list as a payload?
|
||||
|
||||
**Response:** The detect step catches `ClientError` / `EndpointNotFound`,
|
||||
logs a warning, and returns `None` (no prior env). The pipeline proceeds
|
||||
normally. This is the conservative path — no false-positive destroys.
|
||||
**Confidence: 0.90.**
|
||||
**Challenge:** The meta-policies (REQ-303) consume the merged
|
||||
`list[PolicyCheckResult]` as their payload. `kj scan` expects a JSON/
|
||||
YAML *file*. Is the PCR list a valid kyverno-json payload shape?
|
||||
|
||||
**Verdict:** CORRECT.
|
||||
**Verdict:** PROCEED (confidence 0.85). The PCR list is a JSON array
|
||||
of objects — a valid kyverno-json payload. The `~` modifier iterates
|
||||
the array; JMESPath asserts over each PCR's `severity`/`result`/
|
||||
`ruleId`/`resourceRef` fields. The engine writes the list to a temp
|
||||
JSON file and invokes `kj scan --payload <file>`. This is verified in
|
||||
P3 `test_meta_policies.py`. No revision — but **binding note (G-Q7a):**
|
||||
the `KyvernoJsonEngine.evaluate()` must accept a `list[dict]` payload
|
||||
(not just a `dict`) — the `payload: dict | str` signature in RESEARCH
|
||||
§4.1 is too narrow. **Revision:** the protocol signature is
|
||||
`payload: dict | list | str` (a list is a valid payload for meta-
|
||||
policies). Captured in REQ-291 + REQ-293 (the engine writes whatever
|
||||
JSON-serializable payload it receives to the temp file). PLAN.md P1
|
||||
amended.
|
||||
|
||||
## Axis 5: Testing
|
||||
### G-Q8 — Plan revision: the OPA swap surface claims (RESEARCH §4.2) are unverified
|
||||
|
||||
**Challenge:** Can the env-transition behavior be tested without live
|
||||
AWS?
|
||||
**Challenge:** RESEARCH §4.2 documents the OPA-equivalent surface
|
||||
(`opa eval -d <dir> -i <json>`), but no `OpaEngine` is implemented in
|
||||
v1.25. Is the swap-boundary claim testable, or is it aspirational?
|
||||
|
||||
**Response:** Yes. `test_env_transition.py` uses moto for DynamoDB
|
||||
(mock_aws pattern from `test_contract_ingestor.py:74-110`).
|
||||
`test_run_platform_env_transition.py` uses shell-text assertions
|
||||
(pattern from `test_pipeline.py:79-95`). No live AWS needed.
|
||||
**Confidence: 0.92.**
|
||||
**Verdict:** REVISE (confidence 0.78). The swap-boundary claim is
|
||||
*testable in v1.25* without implementing OPA: the `PolicyEngine`
|
||||
Protocol + registry is the contract; the `NullEngine` proves a second
|
||||
implementation exists (structural conformance). **Binding revision
|
||||
(G-Q8a):** P1 `test_policy_engine.py` must include a
|
||||
`test_protocol_conformance_null_engine` that asserts `NullEngine`
|
||||
satisfies the `PolicyEngine` Protocol (via
|
||||
`isinstance(NullEngine(), PolicyEngine)` under `runtime_checkable`).
|
||||
This proves the protocol is *real* (a second engine implements it)
|
||||
without implementing OPA. The OPA-equivalent surface in RESEARCH §4.2
|
||||
stays as documentation (the future milestone implements it). PLAN.md
|
||||
P1 verification amended. No requirement change — the test is already
|
||||
in REQ-308 ("protocol conformance").
|
||||
|
||||
**Verdict:** TESTABLE.
|
||||
### G-Q9 — Budget: is 4 execution phases + P5 too many for the scope?
|
||||
|
||||
## Axis 6: Security
|
||||
**Challenge:** v1.25 is 19 requirements across 6 phases. Recent
|
||||
milestones: v1.24 had 15 reqs / 4 phases; v1.23 had 13 reqs / 7 phases.
|
||||
Is 6 phases too many (overhead) or too few (per-phase overload)?
|
||||
|
||||
**Challenge:** Does the destroy step introduce a risk of destroying the
|
||||
wrong resources?
|
||||
**Verdict:** PROCEED (confidence 0.85). 19 reqs / 6 phases ≈ 3.2 reqs/
|
||||
phase — within the v1.24 cadence (3.75 reqs/phase). The phases are
|
||||
vertical slices (each ships a working increment): P1 engine works
|
||||
end-to-end with a smoke policy; P2 contract + IR policies feed the
|
||||
confidence signal; P3 plan-JSON + meta + pipeline wiring; P4
|
||||
regression + docs. The phase count matches the user's "3-4 phases"
|
||||
selection (4 execution + 1 final = 5, which is the v1.24 shape). No
|
||||
revision.
|
||||
|
||||
**Response:** The destroy targets the prior env's state key
|
||||
(`spike/{id}/{prior_env}/terraform.tfstate`). The state key is
|
||||
deterministic and env-scoped. The destroy can only affect resources in
|
||||
that state file. The `deletion_protection=false` injection is scoped to
|
||||
the destroy step only — the new env's apply runs with the default
|
||||
`deletion_protection=true`. **Confidence: 0.88.**
|
||||
### G-Q10 — The `nova.cloudinit.dev/severity` annotation convention is unvalidated
|
||||
|
||||
**Challenge:** Could a malicious consumer trigger a destroy of another
|
||||
consumer's resources?
|
||||
**Challenge:** RESEARCH §2.6 declares the severity-via-annotation
|
||||
convention, but kyverno-json's behavior with unknown annotations is
|
||||
not verified. Does `kj scan` ignore unknown annotations, or does it
|
||||
reject the policy?
|
||||
|
||||
**Response:** No. The DynamoDB query is scoped by PK `consumerRepo`
|
||||
(the consumer's own repo identity). The destroy runs under the
|
||||
consumer's ABAC-scoped deploy role, which can only touch resources
|
||||
tagged `nova:owner=<consumer-repo>`. A consumer cannot query or destroy
|
||||
another consumer's stack. **Confidence: 0.90.**
|
||||
|
||||
**Verdict:** SECURE.
|
||||
|
||||
## Axis 7: Maintainability
|
||||
|
||||
**Challenge:** Is the `core/env_transition.py` module a clean
|
||||
abstraction or a one-off?
|
||||
|
||||
**Response:** It's a reusable module with two functions
|
||||
(`detect_prior_env`, `record_applied_env`) that encapsulate the
|
||||
DynamoDB query logic. It can be extended for cross-account destroy in a
|
||||
future milestone. The shell Step 0b is a thin orchestrator. This is
|
||||
maintainable. **Confidence: 0.85.**
|
||||
|
||||
**Verdict:** MAINTAINABLE.
|
||||
|
||||
## Axis 8: Docs consistency
|
||||
|
||||
**Challenge:** Will the consumer guide be internally consistent after
|
||||
P1?
|
||||
|
||||
**Response:** The 5 fixes address all known inconsistencies: field
|
||||
table ↔ schema, Step 2 ↔ Step 4, Step 5 ↔ environments doc, Step 8 ↔
|
||||
per-env section, reference table ↔ sample contracts. The test updates
|
||||
assert both shapes are documented. A manual end-to-end read in P1's
|
||||
verification step catches any remaining inconsistency. **Confidence:
|
||||
0.88.**
|
||||
|
||||
**Verdict:** CONSISTENT.
|
||||
|
||||
## Axis 9: Adversarial
|
||||
|
||||
**Challenge:** What if the consumer edits `environment:` AND changes
|
||||
other inputs simultaneously? Does the destroy-then-apply still work?
|
||||
|
||||
**Response:** Yes. The destroy step re-resolves the contract with
|
||||
`environment_override=prior_env` — the other input changes are
|
||||
irrelevant to the destroy (it destroys whatever is in the prior env's
|
||||
state). The new apply resolves with the new env + new inputs. The two
|
||||
operations are independent. **Confidence: 0.85.**
|
||||
|
||||
**Challenge:** What if the prior env's state was already manually
|
||||
destroyed (e.g., via decommission)?
|
||||
|
||||
**Response:** `terraform destroy` against an empty state is a no-op
|
||||
(exits 0). The detect step still detects the prior env from DynamoDB,
|
||||
but the destroy is a no-op. The apply proceeds. This is correct
|
||||
behavior — no false failure. **Confidence: 0.88.**
|
||||
|
||||
**Verdict:** ROBUST.
|
||||
**Verdict:** PROCEED (confidence 0.80). kyverno-json is Kubernetes-
|
||||
style CRD-based — unknown `metadata.annotations` are preserved and
|
||||
ignored (standard K8s behavior). The engine reads the annotation from
|
||||
the loaded policy YAML (via `yaml.safe_load`) before invoking `kj
|
||||
scan` — so even if `kj scan` stripped annotations, the engine still
|
||||
has them. **Binding note (G-Q10a):** P1 `test_kyverno_json_engine.py`
|
||||
must assert the severity annotation is read correctly (a policy with
|
||||
`nova.cloudinit.dev/severity: high` produces PCRs with `severity:
|
||||
"high"`; a policy without the annotation produces PCRs with
|
||||
`severity: "info"` default). Captured in REQ-309 ("PCR schema
|
||||
validity" includes severity). No requirement change — the test is
|
||||
already in REQ-309.
|
||||
|
||||
---
|
||||
|
||||
## Binding revisions applied
|
||||
## Summary
|
||||
|
||||
1. **R1 (docs):** Add to RESEARCH.md pitfalls: the destroy step must
|
||||
inject `deletion_protection=false` into the contract inputs before
|
||||
re-resolving with `environment_override=prior_env`. Without this,
|
||||
`prevent_destroy` lifecycle blocks (REQ-86) block the destroy. This
|
||||
is already noted in RESEARCH §5 pitfall 3 and PLAN P2 implementation
|
||||
note 3. No change needed — already captured.
|
||||
10 challenges; 10 resolved (8 PROCEED, 2 REVISE, 0 ESCALATE).
|
||||
- **Revisions (binding, already in PLAN/REQs):**
|
||||
- G-Q4: P1 tests both fallback paths (NullEngine vs
|
||||
KyvernoJsonEngine-not-configured) — distinct `ruleId`s for audit.
|
||||
- G-Q7a: protocol signature `payload: dict | list | str` (list is a
|
||||
valid payload for meta-policies).
|
||||
- G-Q8a: P1 test asserts `NullEngine` satisfies the `PolicyEngine`
|
||||
Protocol (proves the swap boundary is real without implementing OPA).
|
||||
- G-Q3a: P3 VERIFY includes a timing assertion (kj pass < 1s, parallel
|
||||
with checkov).
|
||||
- G-Q10a: P1 test asserts severity annotation is read correctly.
|
||||
- **No requirement changes** — all revisions are clarifications to
|
||||
PLAN.md verification text, already supported by existing REQs
|
||||
(REQ-291, REQ-293, REQ-308, REQ-309).
|
||||
- **0 escalations** — all challenges auto-resolved at full autonomy.
|
||||
|
||||
2. **R2 (docs):** Clarify in PLAN P2 that the `record_applied_env` SK
|
||||
format is `contractId#LAST_APPLIED#<timestamp>` so the detect step
|
||||
can query `begins_with "contractId#LAST_APPLIED#"`. This is already
|
||||
in RESEARCH §2 and GRILL Axis 4. No change needed — already captured.
|
||||
|
||||
## Escalations
|
||||
|
||||
None. All challenges resolved at full autonomy.
|
||||
The milestone PROCEEDs to PHASE 0 SHIP → P1.
|
||||
@@ -0,0 +1,157 @@
|
||||
# IDEATE — v1.25 kyverno-json Unified Policy Engine
|
||||
|
||||
> **Autonomy:** full. 3-tier ideation per `config.json ideation.enabled:
|
||||
> true`. `cross_project.enabled: false` → cross-project tier scoped to
|
||||
> single-project (deferred ideas only, no cross-project candidates
|
||||
> accepted). `confidence_threshold: 0.6`, `max_ideas: 20`.
|
||||
> Categories: security, quality, architecture, coverage, improvement.
|
||||
|
||||
## Tier 1 — Mechanical (pattern-driven, codebase-grounded)
|
||||
|
||||
### I1 — Regression-gate-as-policy ✅ ACCEPTED (REQ-304, REQ-305)
|
||||
|
||||
**Category:** quality, coverage
|
||||
**Confidence:** 0.90
|
||||
**Pattern:** imperative check → declarative policy (the milestone's
|
||||
core thesis applied to Nova's own regression gate).
|
||||
**Source:** `core/regression_verify.py` (CAP-013, CAP-023, CAP-024)
|
||||
are imperative Python checks. The milestone makes compliance
|
||||
declarative; Nova's own capability regression should follow.
|
||||
**Idea:** Port the three capability checks into
|
||||
`adapters/kyverno-json/policies/regression/` as declarative policies
|
||||
over the capability-inventory JSON frontmatter. The imperative
|
||||
`regression_verify.py` stays (it drives the CI gate); the policies are
|
||||
the declarative mirror that makes capability regression auditable as a
|
||||
policy artifact.
|
||||
**Accepted into:** REQ-304 (policies), REQ-305 (tests). Phase P4.
|
||||
|
||||
### I2 — Contract-shape validation as policy ✅ ACCEPTED (REQ-295)
|
||||
|
||||
**Category:** security, architecture
|
||||
**Confidence:** 0.92
|
||||
**Pattern:** jsonschema constraint → declarative policy (same constraint,
|
||||
different language, Nova posture on top).
|
||||
**Source:** `schemas/contract.schema.json` required/pattern/enum.
|
||||
**Idea:** The 4 contract policies (`require-id-pattern`,
|
||||
`require-env-in-enum`, `require-infrastructure-min-1`, `forbid-unknown-
|
||||
fields`) are the declarative equivalent of the jsonschema constraints —
|
||||
they let Nova apply its own compliance posture (e.g. forbid a specific
|
||||
env for a specific consumer) on top of schema validity without editing
|
||||
the jsonschema.
|
||||
**Accepted into:** REQ-295. Phase P2.
|
||||
|
||||
### I3 — Stack-IR imperative rules → declarative policies ✅ ACCEPTED (REQ-297)
|
||||
|
||||
**Category:** security, architecture
|
||||
**Confidence:** 0.88
|
||||
**Pattern:** imperative Python rule → declarative kyverno-json policy.
|
||||
**Source:** `adapters/terraform/policy/custom_rules/nova_tagging.py`
|
||||
(tagging), the v1.0 demo `public-ingress: true` rule, the v1.8
|
||||
D-encryption-default rule.
|
||||
**Idea:** Port the three highest-impact imperative rules into
|
||||
declarative kyverno-json policies over the resolved Stack IR. The
|
||||
tagging rule is a cross-check (D-118 — both engines, agree meta-policy);
|
||||
public-ingress and encryption-by-default are kyverno-json only (the IR
|
||||
is the earliest point these can be caught).
|
||||
**Accepted into:** REQ-297. Phase P2.
|
||||
|
||||
## Tier 2 — Backend-enriched (signal-driven)
|
||||
|
||||
### I4 — Plan-JSON Checkov RULE_MAP → kyverno-json mirrors ✅ ACCEPTED (REQ-300)
|
||||
|
||||
**Category:** security, coverage
|
||||
**Confidence:** 0.85
|
||||
**Pattern:** existing engine rule → declarative mirror in the new engine
|
||||
(defense-in-depth against engine drift).
|
||||
**Source:** `checkov_adapter.py:RULE_MAP` (CKV_AWS_41/45/46, CKV_AWS_1/40,
|
||||
CKV_AWS_7/33).
|
||||
**Idea:** Port the 6 Checkov rules over `terraform_plan` into declarative
|
||||
kyverno-json policies over `terraform show -json` output. The Checkov
|
||||
rules stay the source of truth for HCL scanning; the kyverno-json
|
||||
policies are mirrors (different rule language, same plan JSON). Defense-
|
||||
in-depth: if Checkov and kyverno-json disagree on the same plan, the
|
||||
divergence is visible (two PCRs with different results for the same
|
||||
resource).
|
||||
**Accepted into:** REQ-300. Phase P3.
|
||||
|
||||
### I5 — Meta-policy over the merged PCR list ✅ ACCEPTED (REQ-303)
|
||||
|
||||
**Category:** architecture, quality
|
||||
**Confidence:** 0.90
|
||||
**Pattern:** the policy result list is itself a policy target (the most
|
||||
novel use of kyverno-json in v1.25).
|
||||
**Source:** `core/confidence_signal.py` PENALTY hardcode (critical
|
||||
override), the D-118 tagging cross-check.
|
||||
**Idea:** `block-on-any-critical` (declarative "critical = block") +
|
||||
`tagging-rules-agree` (Checkov vs kj agree). The meta-policies consume
|
||||
the merged PCR list as their payload. The critical-block meta-policy is
|
||||
the declarative source of truth; the `confidence_signal.py` hard-override
|
||||
stays as defense-in-depth (D-119).
|
||||
**Accepted into:** REQ-303. Phase P3.
|
||||
|
||||
### I6 — Env-transition destroy as a declarative policy ❌ DEFERRED
|
||||
|
||||
**Category:** improvement
|
||||
**Confidence:** 0.55 (below threshold — deferred, not rejected)
|
||||
**Pattern:** imperative lifecycle Python → declarative policy.
|
||||
**Source:** `core/env_transition.py` (v1.24 detect-and-destroy).
|
||||
**Idea:** The v1.24 env-transition destroy logic (detect env change via
|
||||
DynamoDB, destroy prior env, fail-closed) is imperative Python. A
|
||||
declarative kyverno-json policy could assert "if `environment` changed
|
||||
on a stable `contract.id`, a destroy event MUST precede the apply" —
|
||||
turning the lifecycle enforcement into an auditable policy artifact.
|
||||
**Reason deferred:** The env-transition logic is *stateful* (DynamoDB
|
||||
queries, terraform state inspection) — kyverno-json policies are
|
||||
*stateless* (payload in, PCRs out). A policy can assert the *contract*
|
||||
shape (the env value is valid) but not the *lifecycle* (the prior env
|
||||
was destroyed). The stateful check stays in `core/env_transition.py`;
|
||||
a future milestone could emit a `nova.env.destroyed` event that a
|
||||
kyverno-json policy then asserts is present in the evidence stream
|
||||
(event-as-policy). Recorded as a future-idea, not a v1.25 requirement.
|
||||
|
||||
### I7 — Drift detection as policy ❌ DEFERRED
|
||||
|
||||
**Category:** security, coverage
|
||||
**Confidence:** 0.40 (below threshold — deferred)
|
||||
**Pattern:** scheduled job → policy over the drift report.
|
||||
**Source:** NORTH_STAR.md Non-Goal #4 (drift detection scheduled job,
|
||||
deferred — D-096 + no scheduler).
|
||||
**Idea:** A kyverno-json policy over a terraform drift report could
|
||||
assert "no drifted resources" declaratively. But drift detection itself
|
||||
requires a scheduled `terraform plan -detailed-exitcode` job, which is
|
||||
deferred (no scheduler). The policy is the easy part; the emitter is the
|
||||
blocking dependency.
|
||||
**Reason deferred:** Blocked by D-096 + no scheduler (same as NORTH_STAR
|
||||
Non-Goal #4). The policy shape is documented for when the emitter ships.
|
||||
|
||||
## Tier 3 — Cross-project (deferred — single project)
|
||||
|
||||
### I8 — Cross-project policy sharing ❌ DEFERRED (config)
|
||||
|
||||
**Category:** improvement
|
||||
**Confidence:** N/A
|
||||
**Pattern:** policies shared across projects in a multi-project org.
|
||||
**Source:** `config.json ideation.cross_project.enabled: false`.
|
||||
**Idea:** In a multi-project org, kyverno-json policies could be shared
|
||||
across projects (a tagging standard policy applies to all projects).
|
||||
**Reason deferred:** ACDL is single-project (`active_projects: ["acdl"]`).
|
||||
Cross-project ideation is disabled in config. Recorded for when the
|
||||
org grows.
|
||||
|
||||
## Summary
|
||||
|
||||
- 5 ideas accepted (I1..I5) → already captured as REQ-295, REQ-297,
|
||||
REQ-300, REQ-303, REQ-304, REQ-305.
|
||||
- 3 ideas deferred (I6, I7, I8) with documented blocking reasons.
|
||||
- 0 ideas rejected (below-threshold ideas are deferred, not rejected —
|
||||
they may activate when their blockers lift).
|
||||
- The accepted ideas are the **quality improvement** the user asked for
|
||||
("ideate and explore how it can be used within the Nova platform to
|
||||
improve quality of the platform checks"): I1 (regression-gate-as-
|
||||
policy) is the headline quality improvement; I4 + I5 are the defense-
|
||||
in-depth coverage improvements; I2 + I3 are the architecture
|
||||
improvements (imperative → declarative).
|
||||
- No new requirements added beyond REQ-291..309 (the accepted ideas are
|
||||
already scoped into the existing requirements). The IDEATE pass
|
||||
validated the requirement set rather than expanding it — the ideas
|
||||
were anticipated in the SPECIFY stage and explicitly captured.
|
||||
+108
-59
@@ -1,83 +1,132 @@
|
||||
---
|
||||
project: acdl
|
||||
milestone: v1.24
|
||||
milestone: v1.25
|
||||
generated_at: 2026-08-12
|
||||
generator: lead-developer
|
||||
verification_toolchain:
|
||||
typecheck: "python3 -m py_compile core/env_transition.py tests/test_env_transition.py tests/test_run_platform_env_transition.py"
|
||||
test: "pytest tests/test_env_transition.py tests/test_run_platform_env_transition.py tests/test_consumer_guide_per_env_section.py tests/test_adapter.py tests/test_contract_resolver.py tests/test_deploy_workflow_env_input.py tests/test_pipeline.py -v"
|
||||
lint: "ruff check core/env_transition.py tests/test_env_transition.py tests/test_run_platform_env_transition.py 2>/dev/null || python3 -m py_compile core/env_transition.py"
|
||||
typecheck: "python3 -m py_compile core/policy_engine.py adapters/kyverno-json/kyverno_json_engine.py tests/test_policy_engine.py tests/test_kyverno_json_engine.py"
|
||||
test: "pytest tests/test_policy_engine.py tests/test_kyverno_json_engine.py tests/test_adapter.py tests/test_contract_resolver.py tests/test_confidence_signal.py tests/test_checkov_adapter.py tests/test_kyverno_adapter.py tests/test_pipeline.py -v"
|
||||
lint: "ruff check core/policy_engine.py adapters/kyverno-json/ 2>/dev/null || python3 -m py_compile core/policy_engine.py"
|
||||
note: |
|
||||
v1.24 is the Consumer Guide Accuracy & Env-Promotion Lifecycle
|
||||
Enforcement milestone — a mixed docs+feat+test milestone. Two active
|
||||
personas: lead-developer (consumer-guide.md edits + guide test
|
||||
updates), backend-engineer (core/env_transition.py + run_platform.sh
|
||||
Step 0b + deploy.yml + adapter doc comment + env_transition tests +
|
||||
pipeline tests). frontend-engineer stays deactivated (no UI). No
|
||||
data-engineer (no schema changes — the nova-contracts table already
|
||||
exists). No new personas.
|
||||
v1.25 is the kyverno-json Unified Policy Engine milestone — a feat
|
||||
milestone. Four active personas: lead-developer (coordination +
|
||||
docs + ARCHITECTURE.md §12.7), backend-engineer (core/policy_engine.py
|
||||
protocol + registry + contract_resolver.py wiring + run_platform.sh
|
||||
Step 5 + pipeline tests), policy-engineer (adapters/kyverno-json/
|
||||
engine + policies across all 4 target dirs + meta-policies + policy
|
||||
tests + adapter README + STANDARDS.md policy-authoring section),
|
||||
data-engineer (config.json policy object + schemas/README.md note +
|
||||
capability-inventory JSON fixture for regression policies).
|
||||
frontend-engineer stays deactivated (no UI). The policy-engineer is a
|
||||
new custom persona created for this milestone's policy domain (see
|
||||
RESEARCH.md §4 — kyverno-json + JMESPath is a distinct framework from
|
||||
backend-engineer's fastify/hono).
|
||||
---
|
||||
|
||||
# ACDL — Persona Roster (v1.24 Consumer Guide Accuracy & Env-Promotion Lifecycle Enforcement)
|
||||
# ACDL — Persona Roster (v1.25 kyverno-json Unified Policy Engine)
|
||||
|
||||
> v1.24 roster. Two active personas + two deactivated. This is a mixed
|
||||
> docs+feat+test milestone: the work is consumer-guide accuracy fixes
|
||||
> (lead-developer), a new env-transition detect-and-destroy platform
|
||||
> feature (backend-engineer), and test coverage for both (split).
|
||||
> v1.25 roster. Four active personas + one deactivated. This is a feat
|
||||
> milestone: the work is a swappable policy-engine protocol + a new
|
||||
> adapter + policies across 4 Nova artifacts + pipeline wiring + docs.
|
||||
> The policy-engineer is a new custom persona — kyverno-json + JMESPath
|
||||
> is a specialized domain that doesn't fit backend-engineer's
|
||||
> fastify/hono frameworks or data-engineer's drizzle/postgresql.
|
||||
|
||||
## Active personas
|
||||
|
||||
### lead-developer
|
||||
- **Domain:** coordination + docs
|
||||
- **Frameworks:** []
|
||||
- **Constraints:** ["pragmatic", "battle-tested defaults", "docs match code"]
|
||||
- **Constraints:** ["pragmatic", "battle-tested defaults", "docs match code", "swap boundary is the moat"]
|
||||
- **Territory:**
|
||||
- `docs/consumer-guide.md`
|
||||
- `tests/test_consumer_guide_per_env_section.py`
|
||||
- `.ciagent/*.md` (PLAN.md, RESEARCH.md, etc.)
|
||||
- **Reason:** Owns the consumer guide narrative — the 5 accuracy fixes
|
||||
(stale fields table, inconsistent caller, "dev only" phrasing, Step 8
|
||||
rewrite with destroy semantics, reference table wording) and the
|
||||
consumer-guide test updates (rename + new destroy-on-env-change test).
|
||||
No UI work (frontend-engineer deactivated). No Python/bash platform
|
||||
code (backend-engineer territory).
|
||||
- `.ciagent/ARCHITECTURE.md` (§12.7 Policy Engine Registry — NEW)
|
||||
- `.ciagent/PROJECT.md` (v1.25 section)
|
||||
- `.ciagent/REQUIREMENTS.md` (v1.25 section)
|
||||
- `.ciagent/ROADMAP.md` (v1.25 section)
|
||||
- `.ciagent/PLAN.md`, `.ciagent/RESEARCH.md`, `.ciagent/CLARIFY.md`,
|
||||
`.ciagent/GRILL.md`, `.ciagent/PERSONAS.md`
|
||||
- `docs/METRICS.md` (swappable engine narrative — REQ-307)
|
||||
- **Reason:** Owns the milestone coordination + the architecture
|
||||
narrative. The swap boundary (PolicyEngine protocol) is the moat per
|
||||
Strategic Objective #2 — the lead-developer owns the boundary
|
||||
description in ARCHITECTURE.md §12.7 and the docs/METRICS.md note.
|
||||
No Python policy code (backend-engineer + policy-engineer territory).
|
||||
No UI (frontend-engineer deactivated).
|
||||
|
||||
### backend-engineer
|
||||
- **Domain:** backend (Python + bash + YAML)
|
||||
- **Domain:** backend (Python + bash + pipeline wiring)
|
||||
- **Frameworks:** ["boto3", "terraform"]
|
||||
- **Constraints:** ["api-first", "fail-closed", "no orphan paths", "state-key determinism"]
|
||||
- **Constraints:** ["api-first", "strict-typing", "engine-agnostic confidence signal", "fail-soft when kj absent"]
|
||||
- **Territory:**
|
||||
- `core/env_transition.py` (NEW)
|
||||
- `scripts/run_platform.sh` (Step 0b insert + record-applied-env)
|
||||
- `.github/workflows/deploy.yml` (NOVA_CONSUMER_REPO env)
|
||||
- `adapters/terraform/adapter.py` (doc comment only)
|
||||
- `tests/test_env_transition.py` (NEW)
|
||||
- `tests/test_run_platform_env_transition.py` (NEW)
|
||||
- **Reason:** Owns the env-transition detect-and-destroy feature: the new
|
||||
`core/env_transition.py` module (DynamoDB query + record), the
|
||||
`run_platform.sh` Step 0b orchestrator (re-resolve + terraform destroy +
|
||||
evidence event + fail-closed), the deploy.yml env var passthrough, and
|
||||
the two new test files. Uses boto3 (DynamoDB) + terraform (destroy) +
|
||||
bash (pipeline orchestration).
|
||||
- `core/policy_engine.py` (NEW — PolicyEngine Protocol + PolicyEngineRegistry + NullEngine)
|
||||
- `core/contract_resolver.py` (MODIFIED — invoke registry pre/post resolve)
|
||||
- `scripts/run_platform.sh` (MODIFIED — Step 5 kyverno-json parallel pass)
|
||||
- `scripts/install-kyverno-json.sh` (NEW)
|
||||
- `tests/test_policy_engine.py` (NEW — protocol conformance, registry, NullEngine)
|
||||
- `tests/test_run_platform_plan_json_policies.py` (NEW — script-substring assertion)
|
||||
- `.github/workflows/ci.yml` + `.gitea/workflows/ci.yml` (MODIFIED — Go + kj install)
|
||||
- **Reason:** Owns the Python protocol layer + the pipeline wiring. The
|
||||
`PolicyEngine` Protocol + `PolicyEngineRegistry` are Python structural-
|
||||
typing constructs (PEP 544) — backend-engineer's strict-typing
|
||||
constraint. The `contract_resolver.py` wiring + `run_platform.sh`
|
||||
Step 5 are backend territory. Does NOT write kyverno-json policy
|
||||
files (policy-engineer territory) — only the Python that *invokes* the
|
||||
engine. Does NOT modify the confidence signal (it already consumes
|
||||
`list[PolicyCheckResult]` engine-agnostically — PROJECT.md hard-
|
||||
constraint).
|
||||
|
||||
### policy-engineer
|
||||
- **Domain:** policy (declarative compliance rules)
|
||||
- **Frameworks:** ["kyverno-json", "jmespath", "kyverno ValidatingPolicy"]
|
||||
- **Constraints:** ["declarative-policies", "no-imperative-rules", "schema-validated", "severity-via-annotation", "assertion-trees-not-foreach"]
|
||||
- **Territory:**
|
||||
- `adapters/kyverno-json/` (NEW — engine impl + __init__.py + README)
|
||||
- `adapters/kyverno-json/kyverno_json_engine.py` (NEW — KyvernoJsonEngine)
|
||||
- `adapters/kyverno-json/policies/` (NEW — all 4 target dirs: contract/, stack-ir/, plan-json/, meta/, regression/)
|
||||
- `adapters/kyverno-json/policies/_smoke.json` (NEW)
|
||||
- `adapters/README.md` (MODIFIED — new adapter row + PolicyEngine Protocol section)
|
||||
- `tests/test_kyverno_json_engine.py` (NEW — PCR schema validity, defensive parsing)
|
||||
- `tests/test_stack_ir_policies.py` (NEW)
|
||||
- `tests/test_plan_json_policies.py` (NEW)
|
||||
- `tests/test_meta_policies.py` (NEW)
|
||||
- `tests/test_regression_policies.py` (NEW)
|
||||
- `tests/fixtures/stack_ir/`, `tests/fixtures/plan_json/`, `tests/fixtures/capability_inventory.json` (NEW)
|
||||
- `modules/STANDARDS.md` (MODIFIED — Policy authoring standard section — REQ-307)
|
||||
- **Reason:** The policy-engineer owns the declarative policy artifacts.
|
||||
kyverno-json's `ValidatingPolicy` + assertion trees + JMESPath is a
|
||||
distinct framework from backend-engineer's fastify/hono and requires
|
||||
its own constraints: no imperative rules (everything is an assertion
|
||||
tree), severity via the `nova.cloudinit.dev/severity` annotation (not
|
||||
in the engine adapter), no `forEach` (use the `~` modifier). The
|
||||
adapter pattern (engine ↔ protocol ↔ registry) is backend-engineer
|
||||
territory, but the policy *content* and the engine *translation*
|
||||
(`_to_pcr()`) are policy-engineer territory because they require
|
||||
kyverno-json output-shape knowledge. Created per RESEARCH.md §4 — this
|
||||
is a phase-spanning persona (active for P1..P4), not phase-specific.
|
||||
|
||||
### data-engineer
|
||||
- **Domain:** data (config schema + structured fixtures)
|
||||
- **Frameworks:** ["jsonschema", "yaml"]
|
||||
- **Constraints:** ["schema-first", "type-safe config", "backward-compatible additions"]
|
||||
- **Territory:**
|
||||
- `.ciagent/config.json` (MODIFIED — new `policy` object: engine + policy_root)
|
||||
- `schemas/policy_check_result.schema.json` (READ-ONLY — no change per D-116)
|
||||
- `schemas/README.md` (MODIFIED — note engine: "kyverno" shared by K8s adapter + kj)
|
||||
- `tests/fixtures/capability_inventory.json` (NEW — clean + drifted inventory fixtures for regression policies)
|
||||
- **Reason:** The `config.json.policy` object is a schema-first addition
|
||||
(new top-level key with `engine` + `policy_root` fields). The
|
||||
capability-inventory JSON fixtures for the regression-gate policies
|
||||
(REQ-304) are structured data — the data-engineer owns the fixture
|
||||
shape. The `policy_check_result.schema.json` is read-only (D-116 — no
|
||||
enum change); the data-engineer documents the `engine: "kyverno"`
|
||||
sharing in `schemas/README.md`. No migrations (no database). No Python
|
||||
(backend-engineer + policy-engineer territory).
|
||||
|
||||
## Deactivated personas
|
||||
|
||||
### frontend-engineer
|
||||
- **Active:** false
|
||||
- **Reason:** No UI work in v1.24. The consumer guide is markdown docs
|
||||
(lead-developer territory). Deactivated per v1.17/v1.18/v1.22/v1.23
|
||||
precedent.
|
||||
|
||||
### data-engineer
|
||||
- **Active:** false
|
||||
- **Reason:** No schema changes in v1.24. The `nova-contracts` DynamoDB
|
||||
table already exists with the right shape (PK `consumerRepo`, SK
|
||||
`contractId#submittedAt`). The env-transition module only adds a new
|
||||
`#LAST_APPLIED` SK suffix — no schema migration, no ORM, no new tables.
|
||||
The backend-engineer handles the boto3 queries.
|
||||
|
||||
## Phase-specific notes
|
||||
|
||||
- No phase-specific personas. Both active personas span P1-P3.
|
||||
- P4 (final review + audit + ship) is lead-developer territory
|
||||
(orchestration + docs completion).
|
||||
- **active:** false
|
||||
- **Reason:** ACDL has no frontend (no package.json — confirmed in
|
||||
config.json personas.personas[frontend-engineer].reason). v1.25 adds
|
||||
no UI work — the policy engine is backend + policy artifacts only.
|
||||
Deactivated per the v1.15+ convention.
|
||||
+320
-129
@@ -1,180 +1,371 @@
|
||||
# PLAN — v1.24 (Consumer Guide Accuracy & Env-Promotion Lifecycle Enforcement)
|
||||
# PLAN — v1.25 (kyverno-json Unified Policy Engine)
|
||||
|
||||
> Feature milestone (one `feat` phase: env-transition destroy enforcement;
|
||||
> the rest are `fix`/`docs`/`test`). Tags on the **v1.23.x** line:
|
||||
> v1.23.0 (P0) → v1.23.1 (P1) → v1.23.2 (P2) → v1.23.3 (P3) → v1.23.4 (P4
|
||||
> final = milestone release). 15 requirements (REQ-276..290), 4 phases +
|
||||
> P0 pre-execution.
|
||||
> Feature milestone. Tags on the **v1.24.x** line: v1.24.0 (P0) →
|
||||
> v1.24.1 (P1) → v1.24.2 (P2) → v1.24.3 (P3) → v1.24.4 (P4) → v1.24.5
|
||||
> (P5 final = milestone release). 19 requirements (REQ-291..309),
|
||||
> 4 execution phases + P0 pre-execution + P5 final review/ship.
|
||||
|
||||
## Wave model
|
||||
|
||||
Each phase is a **vertical slice** (end-to-end: policy files + Python
|
||||
wiring + tests + docs). Phases are ordered by dependency: the engine
|
||||
protocol (P1) must exist before policies (P2/P3) can be wired; the
|
||||
pipeline wiring (P3) must exist before the meta-policies (P3) can
|
||||
consume the merged PCR list; the regression-gate policies (P4) are
|
||||
independent of the pipeline and can be authored in parallel with P3's
|
||||
tests, but ship after P3 because they reference the engine registry
|
||||
finalized in P1. Within each phase, the waves are the persona task
|
||||
groups (parallelizable across personas when `parallelization.enabled:
|
||||
true`, `max_concurrent_agents: 5`).
|
||||
|
||||
## Phase breakdown
|
||||
|
||||
### Phase P1 — consumer-guide-fixes (Wave 1, lead-developer)
|
||||
### Phase P1 — engine-core (Wave 1, backend-engineer + policy-engineer + data-engineer)
|
||||
|
||||
**Type:** `docs` + `fix` + `test` (consumer guide accuracy + guide test updates)
|
||||
**Type:** `feat` (engine protocol + registry + kyverno-json engine adapter + install + tests)
|
||||
|
||||
**Requirements:** REQ-276, REQ-277, REQ-278, REQ-279, REQ-280, REQ-281, REQ-290
|
||||
**Requirements:** REQ-291, REQ-292, REQ-293, REQ-294, REQ-308, REQ-309
|
||||
|
||||
**Must-haves:**
|
||||
- `docs/consumer-guide.md` Step 3 contract fields table corrected (REQ-276)
|
||||
- `docs/consumer-guide.md` Step 4 caller consistent with Step 2 (REQ-277)
|
||||
- `docs/consumer-guide.md` Step 5 stage 8 "(dev only)" → "(autonomous in dev; higher environments apply after HITL attestation)" (REQ-278)
|
||||
- `docs/consumer-guide.md` Step 8 rewritten with destroy-then-rebuild semantics + cross-ref to Shape B (REQ-279)
|
||||
- `docs/consumer-guide.md` Per-env section gains Shape B lead sentence (REQ-280)
|
||||
- `docs/consumer-guide.md` Reference table `@v1.19` wording corrected (REQ-281)
|
||||
- `tests/test_consumer_guide_per_env_section.py` updated: rename `test_consumer_guide_states_no_field_editing` → `test_consumer_guide_documents_both_promotion_shapes`; add `test_consumer_guide_documents_destroy_on_env_change` (REQ-290)
|
||||
- `core/policy_engine.py` — `PolicyEngine` Protocol (PEP 544) +
|
||||
`PolicyEngineRegistry` (selects from `config.json.policy.engine`) +
|
||||
`NullEngine` fallback (emits `SKIPPED` when `policy` key absent)
|
||||
(REQ-291)
|
||||
- `.ciagent/config.json` gains `policy` object: `{"engine":
|
||||
"kyverno-json", "policy_root":
|
||||
"adapters/kyverno-json/policies"}` (REQ-292)
|
||||
- `adapters/kyverno-json/kyverno_json_engine.py` — `KyvernoJsonEngine`
|
||||
implementing the protocol: `is_configured()` guards on `which kj`;
|
||||
`evaluate()` writes payload to temp JSON, invokes
|
||||
`kj scan --policy <dir> --payload <json> --output json`, translates
|
||||
native output → `list[dict]` PCR records (`engine: "kyverno"`,
|
||||
`ruleId` prefixed `KJ_<policy_name>`, severity from
|
||||
`nova.cloudinit.dev/severity` annotation); defensive parsing
|
||||
(malformed → `error` PCR, never exception); `is_configured()==false`
|
||||
→ single `SKIPPED` PCR (`KJ_ENGINE_NOT_CONFIGURED`) (REQ-293)
|
||||
- `adapters/kyverno-json/__init__.py` exports `KyvernoJsonEngine`;
|
||||
`adapters/kyverno-json/policies/_smoke.json` trivial
|
||||
`require-contract-id` policy for round-trip validation;
|
||||
`scripts/install-kyverno-json.sh` runs
|
||||
`go install github.com/kyverno/kyverno-json/cmd/kj@latest`;
|
||||
`.github/workflows/ci.yml` + `.gitea/workflows/ci.yml` install Go + kj
|
||||
(cached) (REQ-294)
|
||||
- `tests/test_policy_engine.py` — protocol conformance, registry
|
||||
selection, unknown-engine `KeyError`, `NullEngine` fallback,
|
||||
`is_configured()` false when `which kj` absent (mocked) (REQ-308)
|
||||
- `tests/test_kyverno_json_engine.py` — `evaluate()` returns PCR dicts
|
||||
validating against `schemas/policy_check_result.schema.json` (via
|
||||
`jsonschema`); defensive parsing (malformed kyverno-json output →
|
||||
`error` PCR); `is_configured()==false` → `SKIPPED` with
|
||||
`KJ_ENGINE_NOT_CONFIGURED`; `pytest.skip("kj not installed")` when
|
||||
`which kj` absent (REQ-309)
|
||||
|
||||
**Vertical slice:** A reader of `docs/consumer-guide.md` can promote via
|
||||
either shape (A: edit environment + platform destroys prior; B: per-env
|
||||
caller workflow) without contradiction. The guide's field table, caller
|
||||
examples, and stage descriptions match the actual schema and platform
|
||||
behavior. All consumer-guide tests pass.
|
||||
**Vertical slice:** The `PolicyEngineRegistry.get_engine()` returns a
|
||||
configured `KyvernoJsonEngine` that can `evaluate()` a trivial payload
|
||||
against `_smoke.json` and produce a valid PCR list. The confidence
|
||||
signal is unchanged — it already consumes `list[PolicyCheckResult]`.
|
||||
The platform runs with or without the `kj` binary (`is_configured()`
|
||||
guard). All existing tests pass (NullEngine fallback when `policy` key
|
||||
absent in test config — but the v1.25 config.json *sets* the key, so
|
||||
existing tests that use the real config get `KyvernoJsonEngine` with
|
||||
`is_configured()==false` → `SKIPPED`).
|
||||
|
||||
**Files touched:**
|
||||
- `docs/consumer-guide.md`
|
||||
- `tests/test_consumer_guide_per_env_section.py`
|
||||
- `core/policy_engine.py` (NEW)
|
||||
- `.ciagent/config.json` (MODIFIED — `policy` object)
|
||||
- `adapters/kyverno-json/__init__.py` (NEW)
|
||||
- `adapters/kyverno-json/kyverno_json_engine.py` (NEW)
|
||||
- `adapters/kyverno-json/policies/_smoke.json` (NEW)
|
||||
- `scripts/install-kyverno-json.sh` (NEW)
|
||||
- `.github/workflows/ci.yml` (MODIFIED — Go + kj install step)
|
||||
- `.gitea/workflows/ci.yml` (MODIFIED — Go + kj install step)
|
||||
- `tests/test_policy_engine.py` (NEW)
|
||||
- `tests/test_kyverno_json_engine.py` (NEW)
|
||||
|
||||
**Verification:** `pytest tests/test_consumer_guide_per_env_section.py -v`
|
||||
(all 7 tests pass). Manual read of `docs/consumer-guide.md` end-to-end
|
||||
for internal consistency.
|
||||
**Verification:** `pytest tests/test_policy_engine.py
|
||||
tests/test_kyverno_json_engine.py tests/test_confidence_signal.py
|
||||
tests/test_adapter.py tests/test_checkov_adapter.py
|
||||
tests/test_kyverno_adapter.py -v` (new tests pass or skip-without-kj;
|
||||
existing adapter/confidence tests unchanged). `python3 -m py_compile
|
||||
core/policy_engine.py adapters/kyverno-json/kyverno_json_engine.py`.
|
||||
|
||||
---
|
||||
|
||||
### Phase P2 — env-transition-detect-and-destroy (Wave 2, backend-engineer)
|
||||
### Phase P2 — contract + stack-IR policies (Wave 2, policy-engineer + backend-engineer)
|
||||
|
||||
**Type:** `feat` (new platform feature: env-transition detect-and-destroy)
|
||||
**Type:** `feat` (policies + resolver wiring + tests)
|
||||
|
||||
**Requirements:** REQ-282, REQ-283, REQ-284, REQ-285, REQ-286, REQ-287
|
||||
**Requirements:** REQ-295, REQ-296, REQ-297, REQ-298, REQ-299
|
||||
|
||||
**Must-haves:**
|
||||
- `core/env_transition.py` new module: `detect_prior_env()` + `record_applied_env()` with boto3 DynamoDB queries (REQ-282, REQ-283)
|
||||
- `scripts/run_platform.sh` Step 0b: environment-transition check — detect prior env, re-resolve with `environment_override=prior_env` + `deletion_protection=false`, `terraform init -reconfigure` + `terraform destroy -auto-approve` against prior state key, emit `nova.env.destroyed` evidence event, fail closed on destroy failure (REQ-284)
|
||||
- `scripts/run_platform.sh` records applied env after successful apply (REQ-285)
|
||||
- `.github/workflows/deploy.yml` passes `NOVA_CONSUMER_REPO=${{ github.repository }}` to `run_platform.sh` (REQ-286)
|
||||
- `adapters/terraform/adapter.py` state-key block gains doc comment (REQ-287)
|
||||
- `adapters/kyverno-json/policies/contract/` — 4 policies over consumer
|
||||
contract JSON: `require-id-pattern.json`,
|
||||
`require-env-in-enum.json`, `require-infrastructure-min-1.json`,
|
||||
`forbid-unknown-fields.json` — each a `ValidatingPolicy` with one
|
||||
`validate.assert` rule using JMESPath against the payload root;
|
||||
severity via `nova.cloudinit.dev/severity` annotation (REQ-295)
|
||||
- `core/contract_resolver.py` invokes
|
||||
`PolicyEngineRegistry.get_engine().evaluate(contract_dict,
|
||||
policies/contract/, contract_id)` **before** resolving; failures
|
||||
feed the `policy` input as `fail` PCRs (no resolver exit — confidence
|
||||
signal decides the gate, `--soft-fail` pattern); emits
|
||||
`nova.policy.evaluated` metrics event (REQ-296)
|
||||
- `adapters/kyverno-json/policies/stack-ir/` — 3 policies over
|
||||
resolved Stack IR: `require-tagging-standard.json` (ports
|
||||
`nova_tagging.py` — `nova:owner` + `nova:environment` tags on every
|
||||
`resources[]` entry), `forbid-public-ingress.json` (v1.0 demo rule),
|
||||
`require-encryption-by-default.json` (v1.8 D-encryption-default);
|
||||
`~` modifier iterates `resources[]` (REQ-297)
|
||||
- `core/contract_resolver.py` invokes the engine with the resolved
|
||||
Stack IR and `policies/stack-ir/` **after** resolving; resulting PCRs
|
||||
appended to the contract-policy PCRs; resolver return values and
|
||||
exceptions unchanged (additive) (REQ-298)
|
||||
- `tests/test_stack_ir_policies.py` + `tests/fixtures/stack_ir/` —
|
||||
passing IR (all tags + encryption) + failing IR (missing tags, public
|
||||
ingress, plaintext bucket); each policy in isolation + full dir as
|
||||
bundle; `pytest.skip("kj not installed")` when `which kj` absent
|
||||
(REQ-299)
|
||||
|
||||
**Vertical slice:** When a consumer changes `environment:` on a stable
|
||||
`contract.id`, the pipeline detects the prior env from DynamoDB, destroys
|
||||
the prior env's Terraform state (with `deletion_protection=false`), emits
|
||||
an evidence event, and only then applies the new env. If the destroy
|
||||
fails, the pipeline exits non-zero (no orphan path). If no prior env
|
||||
exists (first deploy or Shape B), the pipeline proceeds normally.
|
||||
**Vertical slice:** A consumer contract passes through the resolver
|
||||
and produces two PCR lists (contract policies pre-resolve, stack-IR
|
||||
policies post-resolve) that feed the confidence signal. A contract
|
||||
with a bad `id` or missing tags produces `fail` PCRs that lower the
|
||||
confidence score. The resolver's existing tests pass unchanged (the
|
||||
policy call is additive — it does not change resolver return values
|
||||
or exceptions).
|
||||
|
||||
**Files touched:**
|
||||
- `core/env_transition.py` (NEW)
|
||||
- `scripts/run_platform.sh`
|
||||
- `.github/workflows/deploy.yml`
|
||||
- `adapters/terraform/adapter.py` (doc comment only)
|
||||
- `adapters/kyverno-json/policies/contract/require-id-pattern.json` (NEW)
|
||||
- `adapters/kyverno-json/policies/contract/require-env-in-enum.json` (NEW)
|
||||
- `adapters/kyverno-json/policies/contract/require-infrastructure-min-1.json` (NEW)
|
||||
- `adapters/kyverno-json/policies/contract/forbid-unknown-fields.json` (NEW)
|
||||
- `adapters/kyverno-json/policies/stack-ir/require-tagging-standard.json` (NEW)
|
||||
- `adapters/kyverno-json/policies/stack-ir/forbid-public-ingress.json` (NEW)
|
||||
- `adapters/kyverno-json/policies/stack-ir/require-encryption-by-default.json` (NEW)
|
||||
- `core/contract_resolver.py` (MODIFIED — pre/post resolve engine calls)
|
||||
- `tests/test_stack_ir_policies.py` (NEW)
|
||||
- `tests/fixtures/stack_ir/passing.json` (NEW)
|
||||
- `tests/fixtures/stack_ir/failing.json` (NEW)
|
||||
|
||||
**Verification:** `python3 -m py_compile core/env_transition.py`.
|
||||
`pytest tests/test_pipeline.py tests/test_deploy_workflow_env_input.py -v`
|
||||
(existing tests still pass). The new tests in P3 validate the behavior.
|
||||
|
||||
**Key implementation notes (from RESEARCH §5 pitfalls):**
|
||||
1. The destroy step must re-resolve with `environment_override=prior_env`
|
||||
so the emitted TF matches the prior env's resources.
|
||||
2. `terraform init -reconfigure` is required when switching state backends.
|
||||
3. `deletion_protection: false` must be injected (same as decommission
|
||||
Step 2 in `scripts/run_decommission.sh:34-37`) or `prevent_destroy`
|
||||
blocks the destroy.
|
||||
4. DynamoDB unreachable in local/CI → log warning + return `None`
|
||||
(conservative, no prior env assumed).
|
||||
5. The state key `spike/{id}/{env}/terraform.tfstate` stays as-is — the
|
||||
env segment is what lets the destroy target the prior env.
|
||||
**Verification:** `pytest tests/test_contract_resolver.py
|
||||
tests/test_stack_ir_policies.py tests/test_policy_engine.py -v`
|
||||
(existing resolver tests pass; new policy tests pass or skip-without-
|
||||
kj). `python3 -m py_compile core/contract_resolver.py`.
|
||||
|
||||
---
|
||||
|
||||
### Phase P3 — env-transition-tests (Wave 3, backend-engineer + lead-developer)
|
||||
### Phase P3 — plan-JSON policies + meta-orchestration + pipeline wiring (Wave 3, policy-engineer + backend-engineer)
|
||||
|
||||
**Type:** `test` (new test coverage for env-transition + pipeline integration)
|
||||
**Type:** `feat` (plan-JSON policies + meta-policies + run_platform.sh wiring + tests)
|
||||
|
||||
**Requirements:** REQ-288, REQ-289
|
||||
**Requirements:** REQ-300, REQ-301, REQ-302, REQ-303
|
||||
|
||||
**Must-haves:**
|
||||
- `tests/test_env_transition.py` (NEW): `detect_prior_env` returns `None` when no record; returns prior env when record differs; returns `None` when record matches; `record_applied_env` writes record. Uses moto for DynamoDB (REQ-288)
|
||||
- `tests/test_run_platform_env_transition.py` (NEW): asserts `run_platform.sh` has Step 0b; calls `env_transition.py detect`; calls `terraform destroy` on prior env; fails closed on destroy failure; records applied env after success (REQ-289)
|
||||
- `adapters/kyverno-json/policies/plan-json/` — 3 policies over
|
||||
`terraform show -json` output: `forbid-plaintext-secrets.json` (ports
|
||||
CKV_AWS_41/45/46), `forbid-iam-wildcard.json` (ports CKV_AWS_1/40),
|
||||
`require-kms-reference.json` (ports CKV_AWS_7/33); JMESPath over
|
||||
`planned_values.root_module.resources[]` (REQ-300)
|
||||
- `run_platform.sh` Step 5 gains a parallel kyverno-json pass: after
|
||||
Checkov/Wiz produce raw PCRs, the script runs
|
||||
`kj scan --policy adapters/kyverno-json/policies/plan-json/
|
||||
--payload <tfshow.json> -o json` and pipes through
|
||||
`adapters/kyverno-json/kyverno_json_engine.py` to produce a second
|
||||
PCR list; both lists concatenated and fed to the confidence signal;
|
||||
`nova.policy.evaluated` event with both engine names; when
|
||||
`which kj` is false, logs and proceeds with Checkov/Wiz list only
|
||||
(no hard failure) (REQ-301)
|
||||
- `tests/test_plan_json_policies.py` + `tests/fixtures/plan_json/` —
|
||||
passing plan (no secrets, no wildcard, KMS alias) + failing plan
|
||||
(plaintext password, `Action: "*"`, inline KMS key); policies in
|
||||
isolation + bundle; `tests/test_run_platform_plan_json_policies.py`
|
||||
asserts `run_platform.sh` has the kyverno-json Step 5 block +
|
||||
concatenates PCR lists (script-substring assertion, pattern from
|
||||
`tests/test_pipeline.py:79-95`) (REQ-302)
|
||||
- `adapters/kyverno-json/policies/meta/` — `block-on-any-critical.json`
|
||||
(asserts no PCR in merged list has `severity: critical` + `result:
|
||||
fail`; if any does, emits `fail` PCR `KJ_META_BLOCK_CRITICAL`
|
||||
severity `critical` — declarative source of truth; the
|
||||
`confidence_signal.py` hard-override stays as defense-in-depth per
|
||||
D-119) + `tagging-rules-agree.json` (cross-checks Checkov
|
||||
`NOVA_TAG_NAMING` vs kj `KJ_REQUIRE_TAGGING_STANDARD` by
|
||||
`resourceRef`; divergence emits `error` PCR per D-118);
|
||||
`tests/test_meta_policies.py` (REQ-303)
|
||||
|
||||
**Vertical slice:** The env-transition detect-and-destroy behavior is
|
||||
fully covered by automated tests. The DynamoDB query logic is unit-tested
|
||||
with moto. The pipeline orchestration is tested via shell-text assertions
|
||||
(pattern from `tests/test_pipeline.py:79-95`).
|
||||
**Vertical slice:** `run_platform.sh` Step 5 produces a merged PCR list
|
||||
(Checkov/Wiz + kj plan-JSON policies + kj meta-policies over the
|
||||
merged list) that feeds the confidence signal. A plan with a plaintext
|
||||
secret produces two `fail` PCRs (one Checkov, one kj) for the same
|
||||
resource — visible defense-in-depth. A critical finding anywhere
|
||||
produces a `KJ_META_BLOCK_CRITICAL` meta-PCR that the confidence
|
||||
signal's hard-override blocks. The pipeline runs with or without `kj`
|
||||
(graceful skip).
|
||||
|
||||
**Files touched:**
|
||||
- `tests/test_env_transition.py` (NEW)
|
||||
- `tests/test_run_platform_env_transition.py` (NEW)
|
||||
- `adapters/kyverno-json/policies/plan-json/forbid-plaintext-secrets.json` (NEW)
|
||||
- `adapters/kyverno-json/policies/plan-json/forbid-iam-wildcard.json` (NEW)
|
||||
- `adapters/kyverno-json/policies/plan-json/require-kms-reference.json` (NEW)
|
||||
- `adapters/kyverno-json/policies/meta/block-on-any-critical.json` (NEW)
|
||||
- `adapters/kyverno-json/policies/meta/tagging-rules-agree.json` (NEW)
|
||||
- `scripts/run_platform.sh` (MODIFIED — Step 5 kj parallel pass)
|
||||
- `tests/test_plan_json_policies.py` (NEW)
|
||||
- `tests/test_meta_policies.py` (NEW)
|
||||
- `tests/test_run_platform_plan_json_policies.py` (NEW)
|
||||
- `tests/fixtures/plan_json/passing.json` (NEW)
|
||||
- `tests/fixtures/plan_json/failing.json` (NEW)
|
||||
|
||||
**Verification:** `pytest tests/test_env_transition.py tests/test_run_platform_env_transition.py -v` (all new tests pass). Full suite: `pytest tests/ -k "env_transition or consumer_guide or pipeline or adapter or deploy_workflow" -v`.
|
||||
**Verification:** `pytest tests/test_plan_json_policies.py
|
||||
tests/test_meta_policies.py tests/test_run_platform_plan_json_policies.py
|
||||
tests/test_pipeline.py -v` (new tests pass or skip-without-kj; existing
|
||||
pipeline tests pass). `python3 -m py_compile` on any modified Python.
|
||||
Shellcheck on `run_platform.sh` if available.
|
||||
|
||||
---
|
||||
|
||||
### Phase P4 — final-review-ship (Wave 4, lead-developer)
|
||||
### Phase P4 — regression-gate policies + docs (Wave 4, policy-engineer + data-engineer + lead-developer)
|
||||
|
||||
**Type:** `docs` (review + audit + milestone ship)
|
||||
**Type:** `feat` (regression policies) + `docs` (adapter READMEs + ARCHITECTURE + STANDARDS + METRICS)
|
||||
|
||||
**Requirements:** (none new — milestone completion)
|
||||
**Requirements:** REQ-304, REQ-305, REQ-306, REQ-307
|
||||
|
||||
**Must-haves:**
|
||||
- Multi-persona code review across P1-P3 changes (ci-code-reviewer)
|
||||
- Project health audit (ci-doc-verifier + ci-audit)
|
||||
- Milestone ship: tag v1.23.4 (final phase patch = milestone release), merge to main, Gitea release
|
||||
- Update REQUIREMENTS.md traceability (all REQ-276..290 → complete)
|
||||
- Update ROADMAP.md (v1.24 → complete)
|
||||
- `adapters/kyverno-json/policies/regression/` — 3 policies over
|
||||
capability-inventory JSON frontmatter: `cap-013-adapter-dedup.json`,
|
||||
`cap-023-metrics-collector.json`, `cap-024-deck-structure.json`;
|
||||
emit `pass`/`fail` PCRs per capability; the existing
|
||||
`core/regression_verify.py` is kept (drives the CI gate); the
|
||||
policies are the declarative mirror (REQ-304)
|
||||
- `tests/test_regression_policies.py` +
|
||||
`tests/fixtures/capability_inventory/clean.json` +
|
||||
`tests/fixtures/capability_inventory/drifted.json` — clean (all caps
|
||||
pass) + drifted (duplicate adapter, missing metric status, broken
|
||||
deck arc); regression gate still 287/287 baseline (new tests
|
||||
additive, skip-without-kj) (REQ-305)
|
||||
- `adapters/README.md` gains new kyverno-json adapter row + "Policy
|
||||
Engine Protocol" section (Protocol, registry, swap boundary,
|
||||
how-to-add-OpaEngine); `adapters/kyverno-json/README.md` documents
|
||||
the engine, install path, policy directory layout, 4 policy
|
||||
categories (REQ-306)
|
||||
- `.ciagent/ARCHITECTURE.md` §12.7 (added in RESEARCH) is finalized;
|
||||
`schemas/README.md` notes `engine: "kyverno"` shared by K8s adapter
|
||||
+ kj (distinguished by `ruleId` prefix); `modules/STANDARDS.md`
|
||||
gains "Policy authoring standard" section for module owners;
|
||||
`docs/METRICS.md` notes the policy engine is swappable (Strategic
|
||||
Objective #2 — provable trust via a replaceable substrate) (REQ-307)
|
||||
|
||||
**Vertical slice:** The regression gate's capability checks are now
|
||||
declarative policies auditable as artifacts. A new module owner can
|
||||
read `modules/STANDARDS.md` "Policy authoring standard" and write a
|
||||
per-module kyverno-json policy. A new engineer can read
|
||||
`adapters/README.md` "Policy Engine Protocol" and implement an
|
||||
`OpaEngine`. The 287/287 baseline is unchanged.
|
||||
|
||||
**Files touched:**
|
||||
- `adapters/kyverno-json/policies/regression/cap-013-adapter-dedup.json` (NEW)
|
||||
- `adapters/kyverno-json/policies/regression/cap-023-metrics-collector.json` (NEW)
|
||||
- `adapters/kyverno-json/policies/regression/cap-024-deck-structure.json` (NEW)
|
||||
- `tests/test_regression_policies.py` (NEW)
|
||||
- `tests/fixtures/capability_inventory/clean.json` (NEW)
|
||||
- `tests/fixtures/capability_inventory/drifted.json` (NEW)
|
||||
- `adapters/README.md` (MODIFIED — new row + PolicyEngine Protocol section)
|
||||
- `adapters/kyverno-json/README.md` (NEW)
|
||||
- `schemas/README.md` (MODIFIED — engine enum note)
|
||||
- `modules/STANDARDS.md` (MODIFIED — Policy authoring standard section)
|
||||
- `docs/METRICS.md` (MODIFIED — swappable engine narrative)
|
||||
|
||||
**Verification:** `pytest tests/test_regression_policies.py
|
||||
tests/test_kyverno_json_engine.py -v` (new tests pass or skip-without-
|
||||
kj). Full regression gate `pytest tests/` still at 287/287 baseline +
|
||||
new tests (skip without kj). Manual read of `adapters/README.md` +
|
||||
`adapters/kyverno-json/README.md` + `modules/STANDARDS.md` policy
|
||||
section for clarity.
|
||||
|
||||
---
|
||||
|
||||
## Wave ordering
|
||||
### Phase P5 — final review + audit + milestone ship (Final Phase)
|
||||
|
||||
```
|
||||
Wave 1 (P1): consumer-guide-fixes [lead-developer]
|
||||
↓
|
||||
Wave 2 (P2): env-transition-detect-and-destroy [backend-engineer]
|
||||
↓
|
||||
Wave 3 (P3): env-transition-tests [backend-engineer + lead-developer]
|
||||
↓
|
||||
Wave 4 (P4): final-review-ship [lead-developer]
|
||||
```
|
||||
**Type:** `docs` (review + audit + milestone completion)
|
||||
|
||||
**Dependencies:**
|
||||
- P2 depends on P1: the Step 8 rewrite in P1 documents the destroy
|
||||
semantics that P2 implements. Doing P1 first ensures the docs and code
|
||||
land in the right order (docs describe the intended behavior, then code
|
||||
implements it).
|
||||
- P3 depends on P2: the tests validate the env-transition module and
|
||||
pipeline Step 0b that P2 creates.
|
||||
- P4 depends on P1+P2+P3: the final review covers all changes.
|
||||
**Requirements:** All REQ-291..309 (mark complete)
|
||||
|
||||
**Parallelization:** P1 and P2 could run in parallel (different
|
||||
territories: docs vs code), but the wave ordering is sequential for
|
||||
safety — if P1's Step 8 rewrite reveals a design issue, P2's
|
||||
implementation should follow the corrected design. With
|
||||
`parallelization.enabled=true` and `min_plans_for_parallel=2`, the
|
||||
orchestrator *could* run them concurrently; however, the dependency
|
||||
(P2 follows P1's design) makes sequential the safer choice. P3 must
|
||||
follow P2 (tests validate the code). P4 must follow all.
|
||||
**Must-haves:**
|
||||
- `ciagent-review` multi-persona code review across P1..P4
|
||||
(lead-developer, backend-engineer, data-engineer, policy-engineer).
|
||||
Auto-fix P0; flag P1+ for post-hoc review. If P1+ issues found, fix
|
||||
them in this final phase (not loop back to EXECUTE).
|
||||
- `ciagent-audit` — reconstruction test (git log ↔ `.ciagent/` files),
|
||||
`.ciagent/` file discipline, branch hygiene, commit discipline.
|
||||
Critical issues fixed in this phase.
|
||||
- `ciagent-ship` (milestone) — merge `phase/05-final-review-ship` →
|
||||
`milestone/v1.25-kyverno-json` → `main`; tag `v1.24.5` (= the v1.25
|
||||
release per the prev-minor tagging rule); create Gitea release with
|
||||
full milestone summary (all phases, all requirements); delete all
|
||||
milestone branches (local + remote).
|
||||
- Update `REQUIREMENTS.md` (mark REQ-291..309 complete),
|
||||
`ROADMAP.md` (mark v1.25 complete), `CHECKPOINT.json`
|
||||
(milestone_complete: true), `NORTH_STAR.md` (note Strategic
|
||||
Objective #2 — provable trust via a replaceable policy-engine
|
||||
substrate).
|
||||
|
||||
## Requirement → Phase mapping
|
||||
**Vertical slice:** The v1.25 milestone is complete: kyverno-json is
|
||||
the primary policy tool, behind a swappable adapter, with policies
|
||||
over all 4 Nova artifacts. Tags v1.24.0..v1.24.5 on the v1.24.x line.
|
||||
The milestone branch merges to main.
|
||||
|
||||
| REQ | Phase | Type | Description |
|
||||
|-----|-------|------|-------------|
|
||||
| REQ-276 | P1 | docs | Contract fields table corrected |
|
||||
| REQ-277 | P1 | docs | Step 4 caller consistent with Step 2 |
|
||||
| REQ-278 | P1 | docs | Step 5 stage 8 "dev only" corrected |
|
||||
| REQ-279 | P1 | docs | Step 8 rewritten with destroy semantics |
|
||||
| REQ-280 | P1 | docs | Per-env section Shape B lead sentence |
|
||||
| REQ-281 | P1 | docs | Reference table @v1.19 wording corrected |
|
||||
| REQ-282 | P2 | feat | env_transition.py detect_prior_env() |
|
||||
| REQ-283 | P2 | feat | env_transition.py record_applied_env() |
|
||||
| REQ-284 | P2 | feat | run_platform.sh Step 0b detect-and-destroy |
|
||||
| REQ-285 | P2 | feat | run_platform.sh records applied env |
|
||||
| REQ-286 | P2 | feat | deploy.yml passes NOVA_CONSUMER_REPO |
|
||||
| REQ-287 | P2 | docs | adapter.py state-key doc comment |
|
||||
| REQ-288 | P3 | test | test_env_transition.py |
|
||||
| REQ-289 | P3 | test | test_run_platform_env_transition.py |
|
||||
| REQ-290 | P1 | test | consumer guide test updates |
|
||||
**Verification:** `pytest tests/ -v` full suite passes (287 baseline +
|
||||
new tests). `git log --oneline` shows the v1.25 phase commits.
|
||||
`git tag` shows v1.24.0..v1.24.5. `git branch` shows no leftover
|
||||
milestone/phase branches (all deleted post-ship).
|
||||
|
||||
## Tag plan
|
||||
---
|
||||
|
||||
- P0 (this phase): `v1.23.0` — pre-execution patch
|
||||
- P1: `v1.23.1` — consumer guide fixes
|
||||
- P2: `v1.23.2` — env-transition detect-and-destroy
|
||||
- P3: `v1.23.3` — env-transition tests
|
||||
- P4: `v1.23.4` — final review + ship = **milestone release**
|
||||
## Wave ordering (parallelization)
|
||||
|
||||
With `parallelization.enabled: true`, `max_concurrent_agents: 5`,
|
||||
`min_plans_for_parallel: 2`:
|
||||
|
||||
- **P1 Wave 1:** backend-engineer (protocol + registry + install) ‖
|
||||
data-engineer (config.json policy object) ‖ policy-engineer (engine
|
||||
adapter + smoke policy). 3 concurrent personas. Merge in order:
|
||||
data-engineer → backend-engineer → policy-engineer.
|
||||
- **P2 Wave 2:** policy-engineer (contract + stack-IR policies) ‖
|
||||
backend-engineer (resolver wiring — depends on P1 registry). 2
|
||||
concurrent. Merge: policy-engineer → backend-engineer (wiring
|
||||
references the policy dirs).
|
||||
- **P3 Wave 3:** policy-engineer (plan-JSON + meta policies) ‖
|
||||
backend-engineer (run_platform.sh wiring — depends on P1 engine +
|
||||
P2 resolver pattern). 2 concurrent. Merge: policy-engineer →
|
||||
backend-engineer.
|
||||
- **P4 Wave 4:** policy-engineer (regression policies) ‖ data-engineer
|
||||
(capability-inventory fixtures) ‖ lead-developer (docs: READMEs,
|
||||
STANDARDS, METRICS). 3 concurrent. Merge: data-engineer →
|
||||
policy-engineer → lead-developer.
|
||||
|
||||
Territory enforcement: `warn` mode (per `config.json
|
||||
personas.territory_enforcement: "warn"`). Cross-territory edits
|
||||
(e.g., backend-engineer touching a policy file) emit a warning, not a
|
||||
block.
|
||||
|
||||
## Requirement → phase → persona matrix
|
||||
|
||||
| REQ | Phase | Primary persona | Type |
|
||||
|-----|-------|-----------------|------|
|
||||
| REQ-291 | P1 | backend-engineer | feat |
|
||||
| REQ-292 | P1 | data-engineer | feat (config) |
|
||||
| REQ-293 | P1 | policy-engineer | feat |
|
||||
| REQ-294 | P1 | backend-engineer | feat (install) |
|
||||
| REQ-295 | P2 | policy-engineer | feat |
|
||||
| REQ-296 | P2 | backend-engineer | feat (wiring) |
|
||||
| REQ-297 | P2 | policy-engineer | feat |
|
||||
| REQ-298 | P2 | backend-engineer | feat (wiring) |
|
||||
| REQ-299 | P2 | policy-engineer | test |
|
||||
| REQ-300 | P3 | policy-engineer | feat |
|
||||
| REQ-301 | P3 | backend-engineer | feat (pipeline) |
|
||||
| REQ-302 | P3 | policy-engineer + backend-engineer | test |
|
||||
| REQ-303 | P3 | policy-engineer | feat (meta) |
|
||||
| REQ-304 | P4 | policy-engineer | feat |
|
||||
| REQ-305 | P4 | policy-engineer + data-engineer | test |
|
||||
| REQ-306 | P4 | policy-engineer + lead-developer | docs |
|
||||
| REQ-307 | P4 | lead-developer | docs |
|
||||
| REQ-308 | P1 | backend-engineer | test |
|
||||
| REQ-309 | P1 | policy-engineer | test |
|
||||
@@ -1577,3 +1577,124 @@ New requirements REQ-263..REQ-275 — see `REQUIREMENTS.md` §v1.23.
|
||||
Summary: consolidation (REQ-263,264), style restoration (REQ-265,266,267),
|
||||
image inlining (REQ-268), python-pptx generator (REQ-269,270), word-count
|
||||
trim + loaded-scope-term removal (REQ-271,272), CI/tests/README (REQ-273,274,275).
|
||||
|
||||
## v1.25 — kyverno-json Unified Policy Engine
|
||||
|
||||
> **Active milestone.** Feature milestone (the primary compliance/policy
|
||||
> tool becomes kyverno-json, implemented behind a swappable adapter).
|
||||
> Branch: `milestone/v1.25-kyverno-json`. Tags run on the **v1.24.x**
|
||||
> patch line: `v1.24.0` (P0) → `v1.24.1..v1.24.4` (P1–P4) → `v1.24.5`
|
||||
> (P5 final = milestone release).
|
||||
|
||||
[Nova](https://github.com/kyverno/kyverno-json) `kyverno-json` is a
|
||||
runtime from the Kyverno ecosystem that applies Kyverno policies to
|
||||
**any JSON or YAML payload** — not just Kubernetes manifests. This
|
||||
milestone makes kyverno-json the **primary tool of choice for
|
||||
compliance / policy checks** in Nova, implemented as an **adapter**
|
||||
(the `PolicyEngine` protocol) so the platform may one day replace it
|
||||
with something else (e.g. OPA) without touching the confidence signal
|
||||
or the pipeline.
|
||||
|
||||
### Why
|
||||
|
||||
Nova's policy posture today is split across three engines with three
|
||||
different rule languages and three adapter shapes:
|
||||
|
||||
- **Checkov** (`adapters/terraform/policy/checkov_adapter.py`) — the
|
||||
runtime scanner over `terraform_plan` JSON; carries the
|
||||
`NOVA_TAG_NAMING` custom rule. Imperative YAML+Python rules.
|
||||
- **Wiz** (`adapters/wiz/wiz_adapter.py`) — security findings from the
|
||||
Wiz API; inactive unless credentials are present.
|
||||
- **Kyverno (K8s)** (`adapters/kyverno/kyverno_adapter.py`) — translates
|
||||
Kyverno `PolicyReport` results; **inactive for Terraform-only stacks**
|
||||
(the platform emits Terraform, not K8s manifests — D-053).
|
||||
|
||||
All three emit the same `schemas/policy_check_result.schema.json` shape
|
||||
that `core/confidence_signal.py` consumes engine-agnostically. The
|
||||
*contract* is already right; the *orchestration* is fragmented. There is
|
||||
no single place where "what Nova considers compliant" is declared —
|
||||
tagging lives in a Checkov custom rule, public-ingress in Checkov's
|
||||
`RULE_MAP`, env-transition destroy in `core/env_transition.py`
|
||||
(imperative Python), and capability regression in
|
||||
`core/regression_verify.py` (imperative Python). Each is a different
|
||||
language, each drifts independently, and the K8s Kyverno adapter can't
|
||||
help because it only speaks to K8s manifests.
|
||||
|
||||
`kyverno-json` fixes this: one declarative policy language (Kyverno
|
||||
policies with JMESPath assertions) that applies to **any** Nova
|
||||
artifact — the consumer contract, the resolved Stack IR, the
|
||||
Terraform plan JSON, and even the PolicyCheckResult list itself
|
||||
(meta-validation). It becomes the **unified orchestrator** of compliance
|
||||
checks, while Checkov and Wiz remain as raw-finding adapters that feed
|
||||
*into* kyverno-json meta-policies (so Nova-specific posture rules sit
|
||||
on top of, not beside, the scanner findings).
|
||||
|
||||
### What the milestone delivers
|
||||
|
||||
- **Swappable `PolicyEngine` protocol** (`core/policy_engine.py`) — a
|
||||
Python Protocol + registry selected from `config.json` (`policy.engine`,
|
||||
default `"kyverno-json"`). `KyvernoJsonEngine` implements it (shells
|
||||
to the `kyverno-json` CLI); a future `OpaEngine` implements the same
|
||||
protocol. The confidence signal and pipeline never import the engine
|
||||
directly — they go through the registry.
|
||||
- **`KyvernoJsonEngine` adapter** (`adapters/kyverno-json/`) —
|
||||
`evaluate(payload, policies) -> list[PolicyCheckResult]` translates
|
||||
kyverno-json native output to the existing PCR schema. Mirrors the
|
||||
Checkov/Wiz adapter pattern. `is_configured()` guard skips gracefully
|
||||
when the `kyverno-json` binary is absent (same pattern as the Wiz
|
||||
adapter — emits `SKIPPED`, never breaks the pipeline).
|
||||
- **Policies over all four Nova artifacts** under
|
||||
`adapters/kyverno-json/policies/`:
|
||||
- `contract/` — consumer contract JSON (shape + env-promotion rules).
|
||||
- `stack-ir/` — resolved Target Stack IR (tagging standard,
|
||||
public-ingress, encryption-by-default — ports of the v1.0/v1.8
|
||||
imperative rules into declarative policies).
|
||||
- `plan-json/` — `terraform show -json` output (plaintext secrets,
|
||||
IAM wildcards, KMS references — ports of Checkov's `RULE_MAP`).
|
||||
- `meta/` — policies over the merged PolicyCheckResult list itself
|
||||
(e.g. `block-on-any-critical` — the single declarative source of
|
||||
truth for "critical = block", with the existing
|
||||
`confidence_signal.py` hard-override kept as defense-in-depth).
|
||||
- **`run_platform.sh` Step 5 wiring** — Checkov/Wiz still run and emit
|
||||
raw PCRs; `KyvernoJsonEngine.evaluate()` runs plan-JSON policies in
|
||||
parallel; both PCR lists merge into the confidence signal's `policy`
|
||||
input. No change to `core/confidence_signal.py` (it already consumes
|
||||
`list[PolicyCheckResult]` engine-agnostically).
|
||||
- **Regression-gate-as-policy** (P4 — quality improvement from the
|
||||
IDEATE pass): the capability checks in
|
||||
`core/regression_verify.py` (CAP-013, CAP-023, CAP-024) become
|
||||
declarative kyverno-json policies over the capability-inventory JSON
|
||||
frontmatter. Capability regression becomes an audit artifact, not
|
||||
imperative Python.
|
||||
- **`policy-engineer` persona** (custom, added in RESEARCH) — owns the
|
||||
policy territory; declarative-policies constraint; kyverno-json +
|
||||
JMESPath frameworks.
|
||||
|
||||
**Phase count:** 6 (P0 pre-execution + 4 execution + 1 final).
|
||||
|
||||
**Hard constraints:**
|
||||
- DO NOT change `schemas/policy_check_result.schema.json` shape in a way
|
||||
that breaks existing adapters — the contract is the moat. The
|
||||
`engine` enum already includes `"kyverno"` and `"opa"`; v1.25 records
|
||||
carry `engine: "kyverno"` (no new enum value — decision in CLARIFY).
|
||||
- DO NOT remove Checkov or Wiz adapters — they remain as raw-finding
|
||||
sources feeding into kyverno-json meta-policies.
|
||||
- DO NOT remove the `confidence_signal.py` `PENALTY["critical"]: None`
|
||||
hard-override — it stays as defense-in-depth behind the declarative
|
||||
`block-on-any-critical` meta-policy (decision in CLARIFY).
|
||||
- DO NOT change `core/confidence_signal.py`'s input contract — it
|
||||
already consumes `list[PolicyCheckResult]`; v1.25 only changes *who
|
||||
produces* that list, not *what* the list is.
|
||||
- The platform must function with `kyverno-json` absent — `is_configured()`
|
||||
returns false → `SKIPPED` records → confidence signal proceeds (no
|
||||
hard dependency that breaks the "platform functions without AI /
|
||||
deterministic scripts" tenet — kyverno-json is deterministic, not AI).
|
||||
|
||||
### Requirements
|
||||
|
||||
New requirements REQ-291..REQ-309 — see `REQUIREMENTS.md` §v1.25.
|
||||
Summary: engine protocol + registry (REQ-291,292), kyverno-json engine
|
||||
impl (REQ-293,294), contract policies (REQ-295,296), stack-IR policies
|
||||
(REQ-297,298,299), plan-JSON policies + pipeline wiring (REQ-300,301,302),
|
||||
meta-policies (REQ-303), regression-gate policies (REQ-304,305), docs +
|
||||
adapter README (REQ-306,307), tests (REQ-308,309).
|
||||
|
||||
@@ -2196,3 +2196,295 @@ assert 20 main + 1 appendix.
|
||||
| REQ-288 | P3 | complete |
|
||||
| REQ-289 | P3 | complete |
|
||||
| REQ-290 | P1 | complete |
|
||||
|
||||
## v1.25 — kyverno-json Unified Policy Engine
|
||||
|
||||
> **Feature milestone.** `kyverno-json` becomes the primary compliance /
|
||||
> policy tool, implemented behind a swappable `PolicyEngine` adapter so
|
||||
> OPA (or any other engine) can replace it one day. Tags run on the
|
||||
> **v1.24.x** line (milestone v1.25 → tags v1.24.0..v1.24.N). Final patch
|
||||
> = milestone release.
|
||||
>
|
||||
> One problem, one architectural correction:
|
||||
> 1. **Fragmented policy posture.** Nova's compliance rules are split
|
||||
> across Checkov (imperative YAML + a Python custom rule for tagging),
|
||||
> Wiz (API findings), the K8s-only Kyverno adapter (inactive for
|
||||
> Terraform stacks — D-053), and imperative Python in
|
||||
> `core/env_transition.py` + `core/regression_verify.py`. There is no
|
||||
> single declarative place where "what Nova considers compliant" lives.
|
||||
> The K8s Kyverno adapter can't help because it only speaks to K8s
|
||||
> manifests, and the platform emits Terraform.
|
||||
>
|
||||
> The correction: `kyverno-json` (a Kyverno-ecosystem runtime that applies
|
||||
> Kyverno policies to **any** JSON/YAML payload) becomes the **unified
|
||||
> orchestrator** of compliance checks. Checkov and Wiz remain as
|
||||
> raw-finding adapters feeding *into* kyverno-json meta-policies. The
|
||||
> engine is behind a `PolicyEngine` protocol so it is replaceable. The
|
||||
> confidence signal is untouched — it already consumes
|
||||
> `list[PolicyCheckResult]` engine-agnostically.
|
||||
|
||||
### Decisions (locked in CLARIFY, full autonomy)
|
||||
|
||||
- **D-115 (C-1):** `kyverno-json` is a runtime dependency installed via
|
||||
`go install github.com/kyverno/kyverno-json/cmd/kj@latest` (pinned in a
|
||||
`scripts/install-kyverno-json.sh` helper; the CI image installs it).
|
||||
Not a Python package — kyverno-json is a Go binary. The
|
||||
`KyvernoJsonEngine.is_configured()` checks `which kj` and skips
|
||||
gracefully when absent (emits `SKIPPED` PCR, mirroring the Wiz adapter).
|
||||
- **D-116 (C-2):** kyverno-json PCR records carry `engine: "kyverno"`
|
||||
(no new enum value). The existing `engine` enum in
|
||||
`schemas/policy_check_result.schema.json` already includes `"kyverno"`;
|
||||
adding `"kyverno-json"` would force a schema change + checkov_adapter
|
||||
test regression for no semantic gain. The `ruleId` prefix `KJ_`
|
||||
distinguishes kyverno-json rules from the K8s Kyverno adapter's
|
||||
`KYVERNO_` prefix where they overlap.
|
||||
- **D-117 (C-3):** Checkov and Wiz adapters keep their current
|
||||
`adapt() -> list[PolicyCheckResult]` signatures. They emit PCRs as
|
||||
today. The meta-policies in `adapters/kyverno-json/policies/meta/`
|
||||
consume the **merged** PCR list (checkov + wiz + kyverno-json) as their
|
||||
input payload, applying Nova-specific posture rules on top. No adapter
|
||||
signature changes.
|
||||
- **D-118 (C-4):** `NOVA_TAG_NAMING` (the Checkov custom rule in
|
||||
`adapters/terraform/policy/custom_rules/nova_tagging.py`) is **kept**.
|
||||
A kyverno-json mirror policy `require-tagging-standard.json` is added
|
||||
in `adapters/kyverno-json/policies/stack-ir/`. The P3 meta-policy
|
||||
`tagging-rules-agree.json` asserts the two engines agree on every
|
||||
resource; divergence emits an `error` PCR (defense-in-depth against
|
||||
rule drift). The Checkov rule stays the source of truth for
|
||||
Terraform-static scanning; the kyverno-json policy covers Stack IR.
|
||||
|
||||
### Category: Policy Engine Core (feat)
|
||||
- **REQ-291:** `core/policy_engine.py` defines a `PolicyEngine` Python
|
||||
`Protocol` (PEP 544) with three members: `name -> str`,
|
||||
`is_configured() -> bool`, and
|
||||
`evaluate(payload: dict | str, policy_dir: Path, contract_id: str) ->
|
||||
list[dict]` (where each dict conforms to
|
||||
`schemas/policy_check_result.schema.json`). A `PolicyEngineRegistry`
|
||||
singleton selects the active engine from `config.json`'s new
|
||||
`policy.engine` key (default `"kyverno-json"`); raises
|
||||
`KeyError` on an unknown engine name. The registry exposes
|
||||
`get_engine()` and `register(name, factory)`. Pure stdlib, no engine
|
||||
imports at the protocol layer.
|
||||
- **REQ-292:** `.ciagent/config.json` gains a new top-level `policy`
|
||||
object: `{"engine": "kyverno-json", "policy_root":
|
||||
"adapters/kyverno-json/policies"}`. The registry reads `policy.engine`
|
||||
to select the active engine and `policy.policy_root` as the default
|
||||
policy directory. Backward-compatible: if the `policy` key is absent,
|
||||
the registry returns a `NullEngine` that emits only `SKIPPED` records
|
||||
(so existing tests that don't set the key still pass).
|
||||
|
||||
### Category: kyverno-json Engine Adapter (feat)
|
||||
- **REQ-293:** `adapters/kyverno-json/kyverno_json_engine.py` implements
|
||||
`KyvernoJsonEngine` satisfying the `PolicyEngine` protocol.
|
||||
`is_configured()` returns `True` when `which kj` succeeds. `evaluate()`
|
||||
writes the payload to a temp JSON file, invokes
|
||||
`kj scan --policy <policy_dir> --payload <payload.json> -o json`,
|
||||
parses the native result list, and translates each entry to a PCR dict
|
||||
(`engine: "kyverno"`, `ruleId` prefixed `KJ_<policy_name>`, severity
|
||||
mapped, `result` mapped pass/fail/skip → pass/fail/skipped). When
|
||||
`is_configured()` is false, `evaluate()` returns a single `SKIPPED`
|
||||
PCR with `ruleId: "KJ_ENGINE_NOT_CONFIGURED"` (mirrors the Wiz
|
||||
adapter's `is_configured()` guard). Native output parsing is
|
||||
defensive: any kyverno-json output that doesn't match the expected
|
||||
shape produces an `error` PCR, never an exception.
|
||||
- **REQ-294:** `adapters/kyverno-json/__init__.py` exports
|
||||
`KyvernoJsonEngine`. `adapters/kyverno-json/policies/_smoke.json`
|
||||
is a single trivial policy (`require-contract-id`) used to validate
|
||||
the engine round-trip end-to-end in tests. `scripts/install-kyverno-json.sh`
|
||||
runs `go install github.com/kyverno/kyverno-json/cmd/kj@latest` and
|
||||
prints `kj version`; documented in `adapters/kyverno-json/README.md`.
|
||||
The CI image (`.github/workflows/ci.yml` + `.gitea/workflows/ci.yml`)
|
||||
installs Go + kj when `policy.engine == "kyverno-json"`; the install
|
||||
is cached.
|
||||
|
||||
### Category: Contract Policies (feat)
|
||||
- **REQ-295:** `adapters/kyverno-json/policies/contract/` holds
|
||||
kyverno-json policies over consumer contract JSON. Four policies
|
||||
mirroring `schemas/contract.schema.json` constraints:
|
||||
`require-id-pattern.json` (`id` matches `^[a-z][a-z0-9-]{2,5}$`),
|
||||
`require-env-in-enum.json` (`environment` in dev/qa/prod/dr),
|
||||
`require-infrastructure-min-1.json` (`infrastructure` has ≥1 entry),
|
||||
`forbid-unknown-fields.json` (only `id`/`name`/`environment`/
|
||||
`infrastructure` allowed). Each policy is a single Kyverno `Policy`
|
||||
resource with one `validate.assert` rule using JMESPath against the
|
||||
payload root. Policies are the declarative equivalent of the
|
||||
jsonschema `required`/`pattern`/`enum` constraints — they let Nova
|
||||
apply its own compliance posture on top of schema validity.
|
||||
- **REQ-296:** `core/contract_resolver.py` invokes the
|
||||
`PolicyEngineRegistry.get_engine().evaluate()` with the contract dict
|
||||
and `policies/contract/` **before** resolving (early-fail on contract
|
||||
violations) and emits a `nova.policy.evaluated` metrics event (engine
|
||||
name in the event payload). Failures feed the confidence signal's
|
||||
`policy` input as `fail` PCRs; the resolver does not exit — the
|
||||
confidence signal decides the gate (consistent with the existing
|
||||
`--soft-fail` Checkov pattern).
|
||||
|
||||
### Category: Stack-IR Policies (feat)
|
||||
- **REQ-297:** `adapters/kyverno-json/policies/stack-ir/` holds policies
|
||||
over the resolved Target Stack IR dict. `require-tagging-standard.json`
|
||||
— every resource carries `nova:owner` + `nova:environment` tags
|
||||
(ports `adapters/terraform/policy/custom_rules/nova_tagging.py` logic
|
||||
into a declarative Kyverno policy over the IR's `resources[]` array;
|
||||
mirrors the v1.8 D-tagging-standard). `forbid-public-ingress.json` —
|
||||
no resource has `public_ingress: true` (the v1.0 demo rule, now
|
||||
declarative). `require-encryption-by-default.json` — every S3 bucket
|
||||
+ EBS volume + KMS-aliased resource carries encryption config (ports
|
||||
the v1.8 D-encryption-default rule).
|
||||
- **REQ-298:** `core/contract_resolver.py` invokes the engine with the
|
||||
resolved Stack IR and `policies/stack-ir/` **after** resolving. The
|
||||
resulting PCRs are appended to the contract-policy PCRs and fed to the
|
||||
confidence signal. The resolver's existing `tests/test_contract_resolver.py`
|
||||
continues to pass (the policy call is additive — it does not change
|
||||
resolver return values or exceptions).
|
||||
- **REQ-299:** `tests/test_stack_ir_policies.py` + fixture
|
||||
`tests/fixtures/stack_ir/` — a passing IR (all tags + encryption) and
|
||||
a failing IR (missing tags, public ingress, plaintext bucket). Each
|
||||
policy is tested in isolation + the full `policies/stack-ir/` dir as a
|
||||
bundle. Tests run the `KyvernoJsonEngine` against real `kj` when
|
||||
`which kj` succeeds, and skip with a `pytest.skip("kj not installed")`
|
||||
when absent (so CI without the binary doesn't fail).
|
||||
|
||||
### Category: Plan-JSON Policies + Pipeline Wiring (feat)
|
||||
- **REQ-300:** `adapters/kyverno-json/policies/plan-json/` holds policies
|
||||
over `terraform show -json` output. `forbid-plaintext-secrets.json`
|
||||
(ports `CKV_AWS_41/45/46` — no `aws_db_instance.password` /
|
||||
`aws_iam_user.*` plaintext). `forbid-iam-wildcard.json` (ports
|
||||
`CKV_AWS_1/40` — no `Action: "*"` or `Resource: "*"` in IAM policies).
|
||||
`require-kms-reference.json` (ports `CKV_AWS_7/33` — KMS keys referenced
|
||||
by alias, not inline). Each policy uses JMESPath over the plan's
|
||||
`planned_values.root_module.resources[]` array. The Checkov `RULE_MAP`
|
||||
in `checkov_adapter.py` is unchanged — these are declarative mirrors,
|
||||
not replacements.
|
||||
- **REQ-301:** `run_platform.sh` Step 5 ("runtime policy scan") gains a
|
||||
parallel kyverno-json pass: after Checkov/Wiz produce raw PCRs, the
|
||||
script runs `kj scan --policy adapters/kyverno-json/policies/plan-json/
|
||||
--payload <tfshow.json> -o json` and pipes through
|
||||
`adapters/kyverno-json/kyverno_json_engine.py` to produce a second PCR
|
||||
list. Both lists are concatenated and fed to the confidence signal's
|
||||
`policy` input. The script emits a `nova.policy.evaluated` event with
|
||||
both engine names. When `which kj` is false, the script logs
|
||||
"kyverno-json not installed; skipping plan-json policies" and proceeds
|
||||
with the Checkov/Wiz list only (no hard failure — the platform
|
||||
functions without kj).
|
||||
- **REQ-302:** `tests/test_plan_json_policies.py` + fixture
|
||||
`tests/fixtures/plan_json/` — a passing plan JSON (no secrets, no
|
||||
wildcard, KMS alias) and a failing plan JSON (plaintext password,
|
||||
`Action: "*"`, inline KMS key). Tests the three policies in isolation
|
||||
+ as a bundle. `tests/test_run_platform_plan_json_policies.py`
|
||||
asserts `run_platform.sh` has the kyverno-json Step 5 block and that
|
||||
it concatenates PCR lists (pattern from `tests/test_pipeline.py:79-95`
|
||||
— read script text + assert substrings).
|
||||
|
||||
### Category: Meta-Policies (feat)
|
||||
- **REQ-303:** `adapters/kyverno-json/policies/meta/` holds policies
|
||||
whose **payload** is the merged `list[PolicyCheckResult]` itself.
|
||||
`block-on-any-critical.json` — asserts no PCR in the list has
|
||||
`severity: "critical"` + `result: "fail"`; if any does, the meta-policy
|
||||
emits a `fail` PCR with `ruleId: "KJ_META_BLOCK_CRITICAL"` and
|
||||
severity `critical`. This is the **declarative** source of truth for
|
||||
"critical = block"; the `confidence_signal.py` `PENALTY["critical"]:
|
||||
None` hard-override stays as defense-in-depth (D-118-adjacent
|
||||
decision). `tagging-rules-agree.json` — for every resource in the
|
||||
Stack IR, asserts the Checkov `NOVA_TAG_NAMING` result and the
|
||||
kyverno-json `KJ_REQUIRE_TAGGING_STANDARD` result agree; divergence
|
||||
emits an `error` PCR. `tests/test_meta_policies.py` covers both.
|
||||
|
||||
### Category: Regression-Gate Policies (feat, quality improvement from IDEATE)
|
||||
- **REQ-304:** `adapters/kyverno-json/policies/regression/` holds
|
||||
policies over the capability-inventory JSON frontmatter
|
||||
(`CAPABILITY_INVENTORY.md` parsed as structured data). Three policies
|
||||
port the imperative checks in `core/regression_verify.py`:
|
||||
`cap-013-adapter-dedup.json` (no duplicate adapter registrations),
|
||||
`cap-023-metrics-collector.json` (every metric in `docs/METRICS.md`
|
||||
has a grounded/derived/deferred status), `cap-024-deck-structure.json`
|
||||
(deck slide structure matches the documented arc). The policies read
|
||||
the parsed capability inventory as payload and emit `pass`/`fail` PCRs
|
||||
per capability. The existing `core/regression_verify.py` is **kept**
|
||||
(it drives the CI gate); the policies are the **declarative mirror**
|
||||
that makes capability regression auditable as a policy artifact, not
|
||||
imperative Python. Future milestones may switch the gate to the
|
||||
policy version.
|
||||
- **REQ-305:** `tests/test_regression_policies.py` + fixture
|
||||
`tests/fixtures/capability_inventory.json` — a clean inventory (all
|
||||
caps pass) and a drifted inventory (duplicate adapter, missing metric
|
||||
status, broken deck arc). The regression gate (`pytest` suite)
|
||||
continues to pass 287/287 (or new count); the new policy tests are
|
||||
additive.
|
||||
|
||||
### Category: Documentation (docs)
|
||||
- **REQ-306:** `adapters/README.md` gains a new row for the
|
||||
`kyverno-json` adapter + a new section "Policy Engine Protocol"
|
||||
documenting the `PolicyEngine` Protocol, the registry, and the
|
||||
swap boundary (how to add an `OpaEngine`). `adapters/kyverno-json/README.md`
|
||||
documents the engine, the install path, the policy directory layout,
|
||||
and the four policy categories (contract/stack-ir/plan-json/meta).
|
||||
- **REQ-307:** `.ciagent/ARCHITECTURE.md` gains §12.7 "Policy Engine
|
||||
Registry" with the registry diagram (engine ↔ protocol ↔ registry ↔
|
||||
config.json ↔ confidence signal). `schemas/README.md` notes the
|
||||
`engine: "kyverno"` value is shared by the K8s Kyverno adapter and the
|
||||
kyverno-json engine (distinguished by `ruleId` prefix). `modules/STANDARDS.md`
|
||||
gains a "Policy authoring standard" section for module owners who want
|
||||
to ship per-module kyverno-json policies. `docs/METRICS.md` notes the
|
||||
policy engine is now swappable (Strategic Objective #2 — provable
|
||||
trust via a replaceable substrate, not a vendor lock-in).
|
||||
|
||||
### Category: Tests (test)
|
||||
- **REQ-308:** `tests/test_policy_engine.py` — protocol conformance
|
||||
(the registry returns an engine implementing all three methods),
|
||||
unknown-engine `KeyError`, `NullEngine` fallback when the `policy`
|
||||
key is absent, `KyvernoJsonEngine.is_configured()` returns false when
|
||||
`which kj` fails (mocked). `tests/test_kyverno_json_engine.py` —
|
||||
`evaluate()` returns valid PCR dicts against
|
||||
`schemas/policy_check_result.schema.json` (validated with
|
||||
`jsonschema`); native-output parsing is defensive (malformed kyverno-json
|
||||
output → `error` PCR, not exception); `is_configured()==false` →
|
||||
`SKIPPED` PCR with `KJ_ENGINE_NOT_CONFIGURED`.
|
||||
- **REQ-309:** All new tests use `pytest.skip("kj not installed")` when
|
||||
`which kj` is absent, so the suite passes in environments without the
|
||||
binary (CI matrix: with-kj and without-kj). The full suite
|
||||
(`pytest tests/`) continues to pass at 287/287 baseline + new tests
|
||||
(the new tests skip without kj, so the count grows only when kj is
|
||||
installed). `pyproject.toml` + `requirements-test.txt` unchanged
|
||||
(kyverno-json is a Go binary, not a Python dep).
|
||||
|
||||
### Out of Scope (v1.25)
|
||||
- **Removing Checkov or Wiz.** Both stay as raw-finding adapters. The
|
||||
unified-orchestrator model layers kyverno-json on top, not in place of.
|
||||
- **`OpaEngine` implementation.** The protocol is the swap boundary;
|
||||
the OPA implementation is a future milestone. RESEARCH documents the
|
||||
OPA-equivalent surface so the swap is a known quantity.
|
||||
- **Per-module policies.** `modules/<name>/policies/` is documented as
|
||||
the future pattern in `modules/STANDARDS.md` but not populated this
|
||||
milestone (policies live under `adapters/kyverno-json/policies/`
|
||||
for v1.25).
|
||||
- **kyverno-json as a long-running service.** v1.25 uses the CLI
|
||||
(`kj scan`); the `kj serve` web-app mode is a future consideration
|
||||
for lower-latency evaluation (RESEARCH notes it).
|
||||
- **Replacing the K8s Kyverno adapter.** The K8s adapter
|
||||
(`adapters/kyverno/`) remains documentation-only (D-053 — platform
|
||||
emits Terraform). The kyverno-json engine and the K8s adapter are
|
||||
siblings, not replacements.
|
||||
|
||||
### v1.25 Traceability
|
||||
|
||||
| REQ | Phase | Status |
|
||||
|-----|-------|--------|
|
||||
| REQ-291 | P1 | pending |
|
||||
| REQ-292 | P1 | pending |
|
||||
| REQ-293 | P1 | pending |
|
||||
| REQ-294 | P1 | pending |
|
||||
| REQ-295 | P2 | pending |
|
||||
| REQ-296 | P2 | pending |
|
||||
| REQ-297 | P2 | pending |
|
||||
| REQ-298 | P2 | pending |
|
||||
| REQ-299 | P2 | pending |
|
||||
| REQ-300 | P3 | pending |
|
||||
| REQ-301 | P3 | pending |
|
||||
| REQ-302 | P3 | pending |
|
||||
| REQ-303 | P3 | pending |
|
||||
| REQ-304 | P4 | pending |
|
||||
| REQ-305 | P4 | pending |
|
||||
| REQ-306 | P4 | pending |
|
||||
| REQ-307 | P4 | pending |
|
||||
| REQ-308 | P1 | pending |
|
||||
| REQ-309 | P1 | pending |
|
||||
|
||||
+402
-151
@@ -1,187 +1,438 @@
|
||||
# Nova — v1.24 Research Findings
|
||||
# Nova — v1.25 Research Findings
|
||||
|
||||
> Phase: research (pre-execution). Milestone: v1.24 (Consumer Guide Accuracy
|
||||
> & Env-Promotion Lifecycle Enforcement). Status: research.
|
||||
> Researcher: ci-researcher. Autonomy: full.
|
||||
> Phase: research (pre-execution). Milestone: v1.25 (kyverno-json Unified
|
||||
> Policy Engine). Status: research. Researcher: ci-researcher.
|
||||
> Autonomy: full.
|
||||
|
||||
## 1. Problem domain
|
||||
|
||||
Two distinct problem spaces in one milestone:
|
||||
Nova's compliance/policy posture is fragmented across three engines with
|
||||
three rule languages and three adapter shapes (see PROJECT.md v1.25
|
||||
"Why" for the full diagnosis). The `PolicyCheckResult` schema
|
||||
(`schemas/policy_check_result.schema.json`) is already the engine-agnostic
|
||||
contract that `core/confidence_signal.py` consumes — the *contract* is
|
||||
right; the *orchestration* is fragmented. There is no single declarative
|
||||
place where "what Nova considers compliant" lives. The K8s-only Kyverno
|
||||
adapter (`adapters/kyverno/`) can't help because it only speaks to K8s
|
||||
manifests and the platform emits Terraform (D-053).
|
||||
|
||||
### 1a. Consumer guide accuracy (docs)
|
||||
`kyverno-json` is the correction: a Kyverno-ecosystem runtime that applies
|
||||
Kyverno policies to **any** JSON/YAML payload. It becomes the **unified
|
||||
orchestrator** of compliance checks, behind a swappable `PolicyEngine`
|
||||
protocol so OPA can replace it one day. Checkov and Wiz remain as
|
||||
raw-finding adapters feeding *into* kyverno-json meta-policies.
|
||||
|
||||
The consumer guide (`docs/consumer-guide.md`, 477 lines) has 5 accuracy
|
||||
defects identified in review:
|
||||
## 2. kyverno-json — the engine surface
|
||||
|
||||
1. **Step 3 contract fields table (lines 141-147)** lists `uses`, `module`,
|
||||
`environment`, `inputs`. The actual schema
|
||||
(`schemas/contract.schema.json:7`) requires `id`, `name`, `environment`,
|
||||
`infrastructure`. The `uses` field was dropped in v1.10.2 (REQ-50
|
||||
superseded) and `module` was replaced by the `infrastructure` map key.
|
||||
The worked examples (lines 111-137) use the correct fields.
|
||||
### 2.1 What it is
|
||||
|
||||
2. **Step 4 caller (lines 173-183)** omits `environment:` in `with:`, while
|
||||
Step 2 (lines 94-101) shows `environment: dev`. The two canonical caller
|
||||
snippets disagree.
|
||||
[kyverno-json](https://github.com/kyverno/kyverno-json) is a standalone Go
|
||||
binary from the Kyverno project. It is a **separate runtime** from the
|
||||
Kyverno K8s admission controller — same policy lineage, different
|
||||
application target. Where Kyverno (K8s) evaluates `ClusterPolicy`
|
||||
resources against Kubernetes manifests at admission time, kyverno-json
|
||||
evaluates `ValidatingPolicy` resources against **any** JSON or YAML
|
||||
payload file via the CLI (`kj scan`) or a Go library. It is **not** a
|
||||
Python package (no PyPI release); it is installed via
|
||||
`go install github.com/kyverno/kyverno-json/cmd/kj@latest` (D-115) or by
|
||||
downloading a pinned binary from GitHub releases.
|
||||
|
||||
3. **Step 5 stage 8 (lines 232, 253)** says "(dev only)" for the apply
|
||||
stage. Higher environments *do* apply — they apply after HITL
|
||||
attestation per `docs/environments/index.md:44-54`.
|
||||
### 2.2 CLI surface (the v1.25 invocation path)
|
||||
|
||||
4. **Step 8 (lines 290-306)** says "Change `environment` in your contract"
|
||||
to promote, which contradicts the same doc's "Per-environment
|
||||
deployment" section (lines 398-402): "you do not edit the `environment:`
|
||||
field… Promotion = running the matching job." The test
|
||||
`test_consumer_guide_states_no_field_editing` asserts the no-editing
|
||||
model.
|
||||
The v1.25 engine uses the `kj scan` subcommand:
|
||||
|
||||
5. **Reference table (lines 329-330)** says sample contracts "use `@v1.19`"
|
||||
but the sample contracts (`contracts/static-assets.yml`,
|
||||
`contracts/microservice.yml`) don't carry `uses:` — the version pin
|
||||
lives in the caller workflow.
|
||||
```
|
||||
kyverno-json scan [flags]
|
||||
|
||||
### 1b. Environment-promotion lifecycle enforcement (feat)
|
||||
Flags:
|
||||
--labels strings Labels selectors for policies
|
||||
--output string Output format (text or json) (default "text")
|
||||
--payload string Path to payload (json or yaml file)
|
||||
--policy strings Path to kyverno-json policies
|
||||
--pre-process strings JMESPath expression used to pre process payload
|
||||
```
|
||||
|
||||
**Root cause confirmed by code inspection:**
|
||||
The `KyvernoJsonEngine.evaluate()` implementation (REQ-293) invokes:
|
||||
```
|
||||
kj scan --policy <policy_dir> --payload <payload.json> --output json
|
||||
```
|
||||
and parses the JSON `results[]` array. The `--pre-process` flag is
|
||||
available for JMESPath pre-projection (noted for the meta-policy use case
|
||||
where the payload is the merged PCR list and a pre-process expression
|
||||
can index by `ruleId` — recorded as a future optimization, not used in
|
||||
v1.25's initial implementation).
|
||||
|
||||
- `adapters/terraform/adapter.py:129` sets the Terraform state key to:
|
||||
`spike/{stack_name}/{environment}/terraform.tfstate`
|
||||
- `stack_name` = `contract["id"]` (stable across env changes, per
|
||||
`core/contract_resolver.py:584`).
|
||||
- When a consumer edits `environment:` from `dev` → `qa` on the same
|
||||
contract `id`, the state key changes from `spike/assets/dev/` to
|
||||
`spike/assets/qa/`. Terraform initializes a **fresh state file** in the
|
||||
new env's state path. The prior env's resources remain live in AWS with
|
||||
their state file untouched. **No destroy ever runs.** This orphans
|
||||
resources.
|
||||
Other subcommands (`kj jp`, `kj serve`, `kj playground`, `kj docs`) are
|
||||
out of scope for v1.25. `kj serve` is the long-running web-app mode
|
||||
(noted as a future consideration for lower-latency evaluation in the
|
||||
Out of Scope section of REQUIREMENTS.md). `kj jp` is the JMESPath REPL —
|
||||
useful for policy authoring/debugging, not invoked by the engine.
|
||||
|
||||
**The user's binding directive:** Editing `environment:` on a stable
|
||||
`contract.id` is a valid promotion path (Shape A). The platform **must**
|
||||
destroy the prior env's resources before building the new env. There must
|
||||
be **no path that orphans resources** — fail closed if the destroy fails.
|
||||
### 2.3 Policy structure (the `ValidatingPolicy` resource)
|
||||
|
||||
## 2. Existing codebase structure (integration points)
|
||||
kyverno-json policies are Kubernetes-style resources (cluster-scoped)
|
||||
belonging to the `json.kyverno.io` API group, kind `ValidatingPolicy`,
|
||||
version `v1alpha1`:
|
||||
|
||||
### DynamoDB `nova-contracts` table (the prior-env source of truth)
|
||||
```yaml
|
||||
apiVersion: json.kyverno.io/v1alpha1
|
||||
kind: ValidatingPolicy
|
||||
metadata:
|
||||
name: <policy-name> # becomes the KJ_<policy-name> ruleId prefix
|
||||
spec:
|
||||
rules:
|
||||
- name: <rule-name>
|
||||
identifier: <jmespath> # optional — path to the unique entry id
|
||||
match: # assertion tree — which payload entries
|
||||
any: # the rule applies to
|
||||
- <assertion>
|
||||
exclude: # optional — exclude matching entries
|
||||
any:
|
||||
- <assertion>
|
||||
context: # optional — named bindings available to
|
||||
- name: <binding> # the rule's assertions ($<binding>)
|
||||
variable: <value>
|
||||
validate:
|
||||
message: "<human-readable>" # optional per-rule message
|
||||
assert:
|
||||
all: # all assertions must hold
|
||||
- check: <assertion-tree>
|
||||
message: "<per-check>"
|
||||
# OR
|
||||
any: # at least one assertion must hold
|
||||
- check: <assertion-tree>
|
||||
```
|
||||
|
||||
- **Table:** `nova-contracts` (env var `CONTRACTS_TABLE`, default
|
||||
`nova-contracts`). Defined in `core/lambda/contract_ingestor.py:25`.
|
||||
- **Schema:** PK `consumerRepo` (S), SK `contractId#submittedAt` (S).
|
||||
Attributes: `contractId`, `contract`, `environment`, `status`,
|
||||
`submittedAt`.
|
||||
- **Written by:** `_submit_contract()` at
|
||||
`core/lambda/contract_ingestor.py:135-176`. The consumer's deploy
|
||||
workflow submits the contract via the Lambda Function URL.
|
||||
- **Read pattern for env-transition:** Query by PK `consumerRepo` + SK
|
||||
begins_with `contractId#` + FilterExpression `status = "submitted"` →
|
||||
sort by `submittedAt` desc → take the latest → read its `environment`.
|
||||
This is the last-submitted env. For the last-*applied* env, a new
|
||||
`#LAST_APPLIED` SK suffix is added (REQ-283).
|
||||
- **Test pattern:** `tests/test_contract_ingestor.py:74-110`
|
||||
(`moto_contracts_table` fixture) uses moto `mock_aws` to create the
|
||||
table. The env-transition tests will mirror this pattern.
|
||||
Key differences from K8s Kyverno policies:
|
||||
- **Always cluster-scoped** — no `namespace` field.
|
||||
- **No `forEach`, pattern operators, anchors, or wildcards.** Iteration
|
||||
is done via the `~` projection modifier in assertion trees (see §2.4).
|
||||
- **Assertion trees** with JMESPath expressions replace Kyverno's
|
||||
pattern-matching syntax (see §2.4).
|
||||
|
||||
### Outbox writer (evidence events)
|
||||
### 2.4 Assertion trees (the rule language)
|
||||
|
||||
- `core/outbox_writer.py` writes hash-chained evidence events to
|
||||
`nova-outbox` table. PK `contractId`, SK `eventType#eventTs`.
|
||||
- The env-transition destroy step emits a `nova.env.destroyed` event via
|
||||
this writer (REQ-284c). Pattern: build an event dict with `contractId`,
|
||||
`eventType: "ENV_DESTROYED"`, `environment: <prior_env>`, `ts`, then
|
||||
call `write_event()`.
|
||||
An `assert` declaration contains an `all` or `any` list. Each entry has a
|
||||
`check` (the assertion tree — a nested JMESPath projection) and an
|
||||
optional `message`. **All comparisons happen in the leaves of the tree.**
|
||||
|
||||
### Contract resolver (environment override)
|
||||
A simple example (assert a pod doesn't use the default service account):
|
||||
```yaml
|
||||
validate:
|
||||
assert:
|
||||
all:
|
||||
- message: "serviceAccountName 'default' is not allowed"
|
||||
check:
|
||||
spec:
|
||||
(serviceAccountName == 'default'): false
|
||||
```
|
||||
|
||||
- `core/contract_resolver.py:460-483` `resolve()` accepts
|
||||
`environment_override` — when set, it overrides the contract's
|
||||
`environment` field **before** schema validation and interpolation
|
||||
(D-088). This is the mechanism the destroy step uses to re-resolve the
|
||||
contract against the prior env: `resolve(contract, env_override=prior_env)`.
|
||||
- Already tested in `tests/test_deploy_workflow_env_input.py:35-52`.
|
||||
The `(expression)` syntax evaluates a JMESPath expression; the result
|
||||
becomes the current object for descendants; the leaf value is compared
|
||||
to the expected value.
|
||||
|
||||
### run_platform.sh (where Step 0b goes)
|
||||
**Iteration via the `~` modifier.** The `~` prefix on a key applies
|
||||
descendant assertions to **each element** of an array/map individually
|
||||
(rather than comparing the whole array). Given `foo.bar: [1,2,3]`:
|
||||
```yaml
|
||||
check:
|
||||
foo:
|
||||
~.bar: # iterate each element
|
||||
(@ < `5`): true # assert each element < 5
|
||||
```
|
||||
The `~index_name.bar` form binds the index (array) or key (map) to
|
||||
`$index_name` for use in descendants. This is how v1.25 iterates
|
||||
`resources[]` in the Stack IR policies (REQ-297) and
|
||||
`planned_values.root_module.resources[]` in the plan-JSON policies
|
||||
(REQ-300).
|
||||
|
||||
- `scripts/run_platform.sh` is the platform pipeline. Step 0 (lines 222-239)
|
||||
is the environment onboarding check. Step 1 (lines 241-249) is contract
|
||||
validation. **Step 0b goes between them** (after onboarding, before
|
||||
validation).
|
||||
- The destroy step mirrors the existing `--destroy` mode (lines 376-394):
|
||||
`terraform init -reconfigure` + `terraform destroy -auto-approve`. The
|
||||
difference: it runs against the *prior* env's state key, not the current
|
||||
one.
|
||||
- `CONTRACT_ID` is already set at line 217 (`NOVA_CONTRACT_ID` with a
|
||||
default). `CONSUMER_REPO` needs to be derived from `GITHUB_REPOSITORY`
|
||||
or a new `NOVA_CONSUMER_REPO` env var (REQ-286).
|
||||
**Explicit bindings** via `->binding_name` allow descendants to refer
|
||||
to a parent node via `$binding_name`. Built-in bindings: `$payload`
|
||||
(the whole input), `$policy`, `$rule`.
|
||||
|
||||
### deploy.yml (consumer repo → platform)
|
||||
**Escaping** via `\key\` prevents projection when a payload key collides
|
||||
with the projection syntax. Not needed for Nova payloads (no `(key)`
|
||||
fields), noted for completeness.
|
||||
|
||||
- `.github/workflows/deploy.yml:132` runs
|
||||
`bash platform/scripts/run_platform.sh $MODE_FLAG $ENV_FLAG "${{ inputs.contract }}"`.
|
||||
- To pass `NOVA_CONSUMER_REPO`, add `NOVA_CONSUMER_REPO=${{ github.repository }}`
|
||||
as an env var on the "Run the platform pipeline" step (REQ-286).
|
||||
### 2.5 Output shape (what `kj scan --output json` produces)
|
||||
|
||||
### Test patterns
|
||||
The JSON output is a `results[]` array. Each result entry has (at
|
||||
minimum):
|
||||
- `policy`: the policy metadata.name
|
||||
- `rule`: the rule name
|
||||
- `result`: `"pass"` | `"fail"` | `"error"` | `"skip"` (lowercase)
|
||||
- `message`: the assertion message (or engine error message)
|
||||
- `resource`: the matched payload entry (the `identifier` value, or the
|
||||
whole payload when no identifier/match)
|
||||
- `namespace`/`kind`/`name`: K8s-style fields (present but empty for
|
||||
non-K8s payloads — the K8s Kyverno adapter's evidence uses these; the
|
||||
kyverno-json engine's evidence uses `assertion`/`jmespath` instead)
|
||||
- `severity`: not present by default (kyverno-json does not assign
|
||||
severities — the Nova policy author assigns severity via a Nova-
|
||||
specific annotation; see §2.6)
|
||||
|
||||
- **Shell script assertions:** `tests/test_pipeline.py:79-95` reads the
|
||||
script text and asserts substrings. The env-transition tests follow this
|
||||
pattern for `run_platform.sh`.
|
||||
- **DynamoDB mocking:** `tests/test_contract_ingestor.py:74-110` uses moto
|
||||
`mock_aws` + `boto3.client` + `create_table`. The env-transition tests
|
||||
follow this pattern.
|
||||
- **Consumer guide assertions:** `tests/test_consumer_guide_per_env_section.py`
|
||||
reads `docs/consumer-guide.md` text and asserts substrings. The updated
|
||||
tests follow this pattern.
|
||||
The `KyvernoJsonEngine._to_pcr()` translator (REQ-293) maps:
|
||||
- `policy` → `ruleId` (prefixed `KJ_<policy_name>` per D-116)
|
||||
- `result` → `result` (`pass`/`fail`/`error` → pass/fail/error;
|
||||
`skip`/`skipped` → skipped)
|
||||
- `message` → `message`
|
||||
- `resource` → `resourceRef` + `evidence.resource`
|
||||
- severity from the policy's `metadata.annotations` (see §2.6)
|
||||
- `engine: "kyverno"` (per D-116 — no new enum value)
|
||||
|
||||
## 3. Persona assessment (v1.24)
|
||||
### 2.6 Severity assignment (Nova convention)
|
||||
|
||||
This milestone has two distinct work territories:
|
||||
kyverno-json does not natively assign severities to results. Nova's
|
||||
confidence signal requires a `severity` per PCR (critical/high/medium/
|
||||
low/info). The v1.25 convention: each Nova policy file declares its
|
||||
severity via a `metadata.annotations` field:
|
||||
|
||||
1. **Docs (P1):** `docs/consumer-guide.md` edits + test updates. This is
|
||||
lead-developer territory (narrative/docs + test assertions).
|
||||
2. **Platform code (P2):** `core/env_transition.py` (new), `scripts/run_platform.sh`
|
||||
edits, `.github/workflows/deploy.yml` edit, `adapters/terraform/adapter.py`
|
||||
doc comment. This is backend-engineer territory (Python + bash + YAML).
|
||||
3. **Tests (P3):** `tests/test_env_transition.py` (new), `tests/test_run_platform_env_transition.py`
|
||||
(new), `tests/test_consumer_guide_per_env_section.py` updates. Split:
|
||||
backend-engineer for the env_transition + pipeline tests; lead-developer
|
||||
for the consumer guide test updates.
|
||||
```yaml
|
||||
metadata:
|
||||
name: forbid-public-ingress
|
||||
annotations:
|
||||
nova.cloudinit.dev/severity: high
|
||||
```
|
||||
|
||||
**Roster:** lead-developer (docs + guide tests) + backend-engineer (Python +
|
||||
bash + YAML + pipeline tests). frontend-engineer stays deactivated (no UI).
|
||||
data-engineer not needed (no schema changes — the `nova-contracts` table
|
||||
already exists with the right shape; we only add a new SK suffix). No new
|
||||
personas.
|
||||
The `KyvernoJsonEngine._to_pcr()` reads this annotation from the loaded
|
||||
policy YAML (not from the scan result — the result doesn't carry it) and
|
||||
applies it to every result that policy produces. Default when absent:
|
||||
`info`. This keeps severity in the policy (declarative, version-
|
||||
controlled) rather than in the engine adapter (imperative). The
|
||||
annotation key is `nova.cloudinit.dev/severity` (matches the existing
|
||||
`nova.cloudinit.dev` namespace used in `schemas/tagging-standard.json`).
|
||||
|
||||
## 4. Key decisions logged
|
||||
## 3. The four policy targets (v1.25 scope)
|
||||
|
||||
| ID | Decision | Confidence | Source |
|
||||
|----|----------|------------|--------|
|
||||
| D-201 | Both promotion shapes supported (A: edit+destroy, B: per-env callers) | 0.95 | User directive |
|
||||
| D-202 | Prior-env source of truth = `nova-contracts` DynamoDB table | 0.85 | User directive + code inspection |
|
||||
| D-203 | Detect-and-destroy at pipeline start (Step 0b) | 0.85 | User directive |
|
||||
| D-204 | Fail closed on destroy failure (no orphan path) | 0.95 | User directive |
|
||||
| D-205 | Cross-account destroy out of scope (same-account only) | 0.80 | CLARIFY A3 |
|
||||
| D-206 | Env-transition destroy is NOT the HITL decommission pipeline | 0.85 | CLARIFY A5 |
|
||||
| D-207 | Last-applied env recorded via `#LAST_APPLIED` SK in `nova-contracts` | 0.85 | RESEARCH §2 |
|
||||
| D-208 | State key `spike/{id}/{env}/` stays as-is (correct for both shapes) | 0.90 | RESEARCH §1b |
|
||||
### 3.1 Consumer contract JSON (REQ-295)
|
||||
|
||||
## 5. Pitfalls
|
||||
The payload is the parsed contract dict (the raw YAML loaded as JSON).
|
||||
Policies assert the `contract.schema.json` constraints declaratively:
|
||||
`require-id-pattern` (JMESPath regex `^[a-z][a-z0-9-]{2,5}$` over
|
||||
`id`), `require-env-in-enum` (`environment` in `["dev","qa","prod","dr"]`),
|
||||
`require-infrastructure-min-1` (`length(infrastructure) > 0`),
|
||||
`forbid-unknown-fields` (keys subset of the 4 allowed). These are the
|
||||
declarative equivalent of the jsonschema constraints — they let Nova
|
||||
apply its own compliance posture (e.g. forbid a specific env for a
|
||||
specific consumer) on top of schema validity without editing the
|
||||
jsonschema.
|
||||
|
||||
1. **Destroy needs the prior env's Terraform config, not the new env's.**
|
||||
The destroy step must re-resolve the contract with
|
||||
`environment_override=prior_env` so the emitted TF matches the prior
|
||||
env's resources. If we resolve with the new env, the destroy plan won't
|
||||
match the prior state → terraform tries to create, not destroy.
|
||||
2. **`terraform init -reconfigure` is required** when switching state
|
||||
backends between envs (if envs use different state buckets). The
|
||||
`-reconfigure` flag tells Terraform to forget the previous backend config.
|
||||
3. **The `deletion_protection` NFR (REQ-86) blocks destroy.** The destroy
|
||||
step must resolve with `deletion_protection: false` injected (same as
|
||||
decommission Step 2 in `scripts/run_decommission.sh:34-37`). Without
|
||||
this, `terraform destroy` fails on `prevent_destroy` lifecycle blocks.
|
||||
4. **DynamoDB may not be reachable in local/CI mode.** The detect step
|
||||
must handle `ClientError` / `EndpointNotFound` gracefully → log warning
|
||||
+ return `None` (conservative). This is documented in REQ-282.
|
||||
5. **The consumer guide test `test_consumer_guide_states_no_field_editing`
|
||||
will fail after the Step 8 rewrite.** It must be updated in the same
|
||||
phase as the guide edit (P1) or the test suite breaks.
|
||||
**Invocation point:** `core/contract_resolver.py` pre-resolve (REQ-296).
|
||||
Early-fail: if a contract policy fails, the resolver still proceeds
|
||||
(the confidence signal decides the gate, consistent with the existing
|
||||
`--soft-fail` Checkov pattern) — but the failing PCRs are in the
|
||||
`policy` input, which lowers the score.
|
||||
|
||||
### 3.2 Resolved Target Stack IR JSON (REQ-297)
|
||||
|
||||
The payload is the resolved Stack IR dict produced by
|
||||
`core/contract_resolver.py` (the merged module outputs). Policies
|
||||
assert over `resources[]` (the array of resolved resources):
|
||||
`require-tagging-standard` (every resource's `tags` has `nova:owner` +
|
||||
`nova:environment` — ports
|
||||
`adapters/terraform/policy/custom_rules/nova_tagging.py`),
|
||||
`forbid-public-ingress` (no resource has `public_ingress: true` — the
|
||||
v1.0 demo rule, now declarative), `require-encryption-by-default` (every
|
||||
S3/EBS/KMS-aliased resource carries encryption config — ports the v1.8
|
||||
D-encryption-default rule). The `~` modifier iterates `resources[]`.
|
||||
|
||||
**Invocation point:** `core/contract_resolver.py` post-resolve (REQ-298).
|
||||
Additive — the resolver's return values and exceptions are unchanged;
|
||||
the PCRs are appended to the contract-policy PCRs.
|
||||
|
||||
### 3.3 Terraform plan JSON (REQ-300)
|
||||
|
||||
The payload is `terraform show -json <tfplan>` output. Policies assert
|
||||
over `planned_values.root_module.resources[]`:
|
||||
`forbid-plaintext-secrets` (no `aws_db_instance.password` /
|
||||
`aws_iam_user.login_profile.password` in plaintext — ports
|
||||
`CKV_AWS_41/45/46`), `forbid-iam-wildcard` (no `Action: "*"` or
|
||||
`Resource: "*"` in `aws_iam_policy.PolicyDocument` — ports
|
||||
`CKV_AWS_1/40`), `require-kms-reference` (KMS keys referenced by alias,
|
||||
not inline key material — ports `CKV_AWS_7/33`). These are declarative
|
||||
**mirrors** of `checkov_adapter.py:RULE_MAP` — the Checkov rule stays
|
||||
the source of truth for `terraform_plan` scanning; the kyverno-json
|
||||
policy covers the same plan JSON with a different rule language
|
||||
(defense-in-depth against engine drift).
|
||||
|
||||
**Invocation point:** `run_platform.sh` Step 5 (REQ-301). After
|
||||
Checkov/Wiz produce raw PCRs, the script runs `kj scan` over the plan
|
||||
JSON; both PCR lists concatenate into the confidence signal's `policy`
|
||||
input. When `which kj` is false, the script logs and proceeds with the
|
||||
Checkov/Wiz list only.
|
||||
|
||||
### 3.4 PolicyCheckResult records (meta-policies, REQ-303)
|
||||
|
||||
The payload is the **merged** `list[PolicyCheckResult]` produced by
|
||||
checkov + wiz + the plan-JSON policies. This is the most novel target —
|
||||
kyverno-json policies over the policy results themselves.
|
||||
`block-on-any-critical` asserts no PCR has `severity: "critical"` +
|
||||
`result: "fail"`; if any does, the meta-policy emits a `fail` PCR with
|
||||
`ruleId: "KJ_META_BLOCK_CRITICAL"` and severity `critical`. This is the
|
||||
declarative source of truth for "critical = block" (D-119 — the
|
||||
`confidence_signal.py` `PENALTY["critical"]: None` hard-override stays
|
||||
as defense-in-depth). `tagging-rules-agree` cross-checks the Checkov
|
||||
`NOVA_TAG_NAMING` result against the kyverno-json
|
||||
`KJ_REQUIRE_TAGGING_STANDARD` result by `resourceRef`; divergence emits
|
||||
an `error` PCR (D-118).
|
||||
|
||||
**Invocation point:** after the three target policies (contract/stack-
|
||||
IR/plan-JSON) produce their PCR lists, the merged list is the payload
|
||||
for the meta-policies. The meta-policy PCRs are appended to the merged
|
||||
list, which is what the confidence signal consumes.
|
||||
|
||||
## 4. The `PolicyEngine` swap boundary
|
||||
|
||||
### 4.1 Protocol shape (REQ-291)
|
||||
|
||||
A Python `Protocol` (PEP 544 — structural subtyping, no inheritance):
|
||||
```python
|
||||
class PolicyEngine(Protocol):
|
||||
@property
|
||||
def name(self) -> str: ...
|
||||
def is_configured(self) -> bool: ...
|
||||
def evaluate(self, payload: dict | str, policy_dir: Path,
|
||||
contract_id: str) -> list[dict]: ...
|
||||
```
|
||||
`list[dict]` (not `list[PolicyCheckResult]` — there's no dataclass; the
|
||||
schema is enforced via `jsonschema` validation in tests, matching the
|
||||
existing adapter pattern). The registry selects the active engine from
|
||||
`config.json.policy.engine`. A `NullEngine` is the fallback when the
|
||||
`policy` key is absent (emits `SKIPPED` — backward compatibility for
|
||||
tests that don't set the key).
|
||||
|
||||
### 4.2 The OPA-equivalent surface (future swap)
|
||||
|
||||
OPA (Open Policy Agent) is the most likely future replacement. The
|
||||
mapping:
|
||||
| Nova `PolicyEngine` member | kyverno-json impl | OPA equivalent |
|
||||
|---|---|---|
|
||||
| `name` | `"kyverno-json"` | `"opa"` |
|
||||
| `is_configured()` | `which kj` | `which opa` |
|
||||
| `evaluate(payload, policy_dir, contract_id)` | `kj scan --policy <dir> --payload <json> -o json` | `opa eval -d <dir> -i <json> 'data.nova.<...>'` |
|
||||
| Policy file format | `ValidatingPolicy` (YAML) | Rego (`.rego`) |
|
||||
| Result shape | `results[]` (pass/fail/error/skip) | `result` (set of violations) |
|
||||
| Severity | Nova annotation `nova.cloudinit.dev/severity` | Nova convention (Rego `metadata` or a wrapper) |
|
||||
|
||||
The protocol is minimal (3 members) specifically so the OPA
|
||||
implementation is a known quantity: an `OpaEngine` class that shells to
|
||||
`opa eval`, translates the Rego violation set to PCR dicts, and
|
||||
implements `is_configured()` via `which opa`. The policy *files* would
|
||||
need rewriting (Rego, not ValidatingPolicy) — but the protocol, the
|
||||
registry, the confidence signal, and the PCR schema are all untouched.
|
||||
This is the swap boundary the user asked for ("Implemented as an
|
||||
adapter since we might one day decide to replace it with something else
|
||||
like OPA").
|
||||
|
||||
### 4.3 Why not a full plugin registry?
|
||||
|
||||
A `setuptools` entry-point plugin registry (like checkov's
|
||||
`--external-checks-dir`) was considered and rejected: Nova has 1 active
|
||||
engine today (kyverno-json) and at most 2 in the foreseeable future
|
||||
(kyverno-json + OPA). A `Protocol` + `dict` registry in
|
||||
`core/policy_engine.py` is the right weight — discoverable, typed,
|
||||
testable, and ~40 lines. An entry-point registry adds packaging
|
||||
complexity (entry-point metadata, version resolution) for no gain at
|
||||
this scale. The `register(name, factory)` method on the registry is
|
||||
the extension point if a future milestone needs runtime plugin
|
||||
discovery.
|
||||
|
||||
## 5. Latency / MTTR impact (G-Q3 anticipation)
|
||||
|
||||
NORTH_STAR.md MTTR target: < 60s p95. `run_platform.sh` Step 5 today
|
||||
runs Checkov over the terraform plan (typically 5-15s for a small
|
||||
stack). Adding `kj scan` over the same plan JSON adds:
|
||||
- Process spawn: ~50ms (Go binary startup)
|
||||
- Policy load: ~20ms (a handful of YAML files)
|
||||
- Assertion evaluation: ~100-500ms (JMESPath over a small plan)
|
||||
- Total: < 1s for a typical Nova stack
|
||||
|
||||
The kyverno-json pass runs **in parallel** with Checkov (REQ-301 — the
|
||||
script launches both and waits on both), so the wall-clock impact is
|
||||
`max(checkov_time, kj_time)` ≈ checkov_time (kj is faster). The
|
||||
contract + stack-IR policies run during resolve (already a fast step).
|
||||
Meta-policies run over the merged list (in-memory, < 10ms). **No
|
||||
measurable MTTR impact** is expected. This will be verified in P3
|
||||
VERIFY with a timing assertion.
|
||||
|
||||
## 6. "Platform functions without AI" tenet (G-Q1 / D-120)
|
||||
|
||||
kyverno-json is deterministic (same policy + payload → same result,
|
||||
every run). It is not an LLM, not a probabilistic model, not a
|
||||
"judgement" engine. The NORTH_STAR.md tenet ("the platform functions
|
||||
without AI — 'AI decisions' are really automated decisions")
|
||||
distinguishes AI (non-reproducible) from automation (reproducible).
|
||||
kyverno-json is the latter. Adding it is **more** aligned with the
|
||||
tenet than the current imperative Python in `core/env_transition.py`
|
||||
and `core/regression_verify.py`, because the policy is declarative
|
||||
(visible, auditable, version-controlled) rather than imperative (logic
|
||||
hidden in function bodies). The `is_configured()` guard ensures the
|
||||
platform functions without the binary (graceful skip → `SKIPPED` PCR
|
||||
→ confidence signal proceeds).
|
||||
|
||||
## 7. ECS policy catalog overlap (prior art)
|
||||
|
||||
The kyverno-json catalog ships ECS policies that overlap with Nova's
|
||||
L1 modules: `ecs-cluster-enable-logging`, `ecs-cluster-required-
|
||||
container-insights`, `ecs-service-public-ip`, `ecs-service-required-
|
||||
latest-platform-fargate`, `ecs-task-definition-fs-read-only`. These are
|
||||
**reference policies**, not drop-in Nova policies — they target the
|
||||
AWS ECS API shape (`type: aws_ecs_service` etc.), not Nova's Stack IR
|
||||
shape. v1.25 policies target the Nova IR (REQ-297) and the terraform
|
||||
plan JSON (REQ-300), not the raw AWS API. The catalog is useful as
|
||||
prior art for JMESPath patterns over ECS resources — the
|
||||
`ecs-service-public-ip` policy's `contains('$allowed-values',
|
||||
@.assign_public_ip)` pattern informs the Nova `forbid-public-ingress`
|
||||
policy shape. No catalog policies are imported directly in v1.25.
|
||||
|
||||
## 8. Risks & mitigations
|
||||
|
||||
| Risk | Likelihood | Impact | Mitigation |
|
||||
|---|---|---|---|
|
||||
| `kj` binary not in CI image | medium | blocks P3+ tests | `is_configured()` guard + `pytest.skip` + `scripts/install-kyverno-json.sh` |
|
||||
| kyverno-json output shape changes across versions | low | breaks `_to_pcr()` | pin `@latest` to a known-good commit in `install-kyverno-json.sh` after P1 smoke; defensive parsing (malformed → `error` PCR, not exception) |
|
||||
| Policy explosion (4 targets × N rules) | medium | maintenance load | wave ordering (PLAN); policies co-located per target dir; meta-policy cross-check keeps the set auditable |
|
||||
| Checkov + kj tagging-rule drift | medium | false `error` PCRs | `tagging-rules-agree` meta-policy emits `error` on divergence (visible, not silent); the Checkov rule stays source of truth for HCL, kj for IR |
|
||||
| OPA swap turns out harder than the protocol implies | low | future milestone rework | RESEARCH §4.2 documents the OPA-equivalent surface; the protocol is the contract, not the implementation |
|
||||
| `--pre-process` needed for meta-policies but undocumented behavior | low | meta-policy bugs | v1.25 meta-policies use plain assertion trees over the PCR list (no pre-process); `--pre-process` noted as a future optimization only |
|
||||
|
||||
## 9. Assumptions (logged, full autonomy)
|
||||
|
||||
- A1: `kj scan --output json` produces a stable `results[]` array shape.
|
||||
Will be verified in P1 smoke test (`_smoke.json` policy + a trivial
|
||||
payload); if the shape differs, `_to_pcr()` is adjusted defensively
|
||||
(malformed → `error` PCR). Confidence: 0.85.
|
||||
- A2: The `nova.cloudinit.dev/severity` annotation convention is
|
||||
read by the engine from the policy YAML (loaded once per evaluate()
|
||||
call). kyverno-json does not validate unknown annotations — they pass
|
||||
through. Confidence: 0.90.
|
||||
- A3: The `~` projection modifier iterates `resources[]` in the Stack
|
||||
IR and `planned_values.root_module.resources[]` in the plan JSON
|
||||
correctly. Verified in P2/P3 tests. Confidence: 0.85.
|
||||
- A4: `go install` works in the CI image (Go toolchain available or
|
||||
installable). If not, the binary-release download path is the
|
||||
documented fallback in `install-kyverno-json.sh`. Confidence: 0.80.
|
||||
- A5: The `NullEngine` fallback (when `policy` key absent in
|
||||
config.json) keeps all existing tests passing — they don't set the
|
||||
key, so they get `NullEngine` → `SKIPPED` PCRs → confidence signal
|
||||
proceeds with `policy` input `[SKIPPED]` → per-input score 1.0
|
||||
(skipped counts as pass in `_per_input_score`). Confidence: 0.95
|
||||
(verified against `confidence_signal.py:84-89`).
|
||||
|
||||
## 10. Decisions referenced
|
||||
|
||||
D-115 (install path), D-116 (engine enum reuse), D-117 (adapter
|
||||
signatures unchanged), D-118 (tagging cross-check), D-119 (critical-
|
||||
override defense-in-depth), D-120 (deterministic not AI). See
|
||||
CLARIFY.md for the full resolution text.
|
||||
|
||||
## 11. Architecture updates (deferred to RESEARCH-stage file edits)
|
||||
|
||||
- `.ciagent/ARCHITECTURE.md` gains §12.7 "Policy Engine Registry" with
|
||||
the registry diagram. Deferred to the RESEARCH commit (this file's
|
||||
commit) — the section is authored as part of this research.
|
||||
- `schemas/README.md` notes `engine: "kyverno"` is shared by the K8s
|
||||
adapter and kyverno-json (distinguished by `ruleId` prefix).
|
||||
- `modules/STANDARDS.md` gains a "Policy authoring standard" section
|
||||
(P4, REQ-307).
|
||||
- `docs/METRICS.md` notes the policy engine is swappable (P4, REQ-307).
|
||||
@@ -2156,3 +2156,93 @@ release). **DONE.**
|
||||
milestone). Tag `v1.22.6` (final patch = milestone release). Merge
|
||||
`milestone/v1.23-deck-cleanup-python-pptx` → `main`.
|
||||
- **Requirements:** REQ-263..275 (13 requirements).
|
||||
|
||||
## v1.25 (active, tag line `v1.24.x`): kyverno-json Unified Policy Engine
|
||||
|
||||
`kyverno-json` — a Kyverno-ecosystem runtime that applies Kyverno policies
|
||||
to **any** JSON/YAML payload — becomes Nova's **primary compliance /
|
||||
policy tool**, implemented behind a swappable `PolicyEngine` adapter so
|
||||
OPA (or any other engine) can replace it one day. The unified-orchestrator
|
||||
model: Checkov and Wiz remain as raw-finding adapters feeding *into*
|
||||
kyverno-json meta-policies; the confidence signal is untouched (it already
|
||||
consumes `list[PolicyCheckResult]` engine-agnostically). Policies cover
|
||||
all four Nova artifacts: consumer contract JSON, resolved Stack IR,
|
||||
Terraform plan JSON, and the merged PCR list itself (meta-validation).
|
||||
The K8s-only Kyverno adapter stays documentation-only (D-053); the
|
||||
kyverno-json engine and the K8s adapter are siblings, not replacements.
|
||||
Quality improvement from the IDEATE pass: capability regression checks
|
||||
(`core/regression_verify.py` CAP-013/023/024) become declarative
|
||||
kyverno-json policies. New `policy-engineer` persona owns the policy
|
||||
territory. 19 requirements (REQ-291..309), 6 phases (P0 + P1..P4 + P5
|
||||
final). Tags: `v1.24.0` (P0) → `v1.24.5` (P5 = milestone release).
|
||||
|
||||
### Phase P1 — engine-core (planned, tag v1.24.1)
|
||||
- REQ-291: `core/policy_engine.py` — `PolicyEngine` Protocol +
|
||||
`PolicyEngineRegistry` (selects engine from `config.json.policy.engine`).
|
||||
- REQ-292: `config.json` gains `policy` object
|
||||
(`engine: "kyverno-json"`, `policy_root`).
|
||||
- REQ-293: `adapters/kyverno-json/kyverno_json_engine.py` —
|
||||
`KyvernoJsonEngine` (shells to `kj scan`; translates native output →
|
||||
PCR; `is_configured()` guards on `which kj`).
|
||||
- REQ-294: `adapters/kyverno-json/__init__.py` + `_smoke.json` policy +
|
||||
`scripts/install-kyverno-json.sh` + CI image install.
|
||||
- REQ-308: `tests/test_policy_engine.py` — protocol conformance,
|
||||
registry, NullEngine fallback.
|
||||
- REQ-309: `tests/test_kyverno_json_engine.py` — PCR schema validity,
|
||||
defensive parsing, `pytest.skip` when kj absent.
|
||||
|
||||
### Phase P2 — contract + stack-IR policies (planned, tag v1.24.2)
|
||||
- REQ-295: `adapters/kyverno-json/policies/contract/` — 4 policies over
|
||||
consumer contract JSON (id-pattern, env-enum, infra-min-1,
|
||||
forbid-unknown-fields).
|
||||
- REQ-296: `core/contract_resolver.py` invokes the engine pre-resolve
|
||||
(contract policies) — early-fail, confidence signal decides the gate.
|
||||
- REQ-297: `adapters/kyverno-json/policies/stack-ir/` — 3 policies over
|
||||
resolved Stack IR (tagging-standard, public-ingress, encryption-by-
|
||||
default — ports of v1.0/v1.8 imperative rules).
|
||||
- REQ-298: `core/contract_resolver.py` invokes the engine post-resolve
|
||||
(stack-IR policies); additive — existing tests pass.
|
||||
- REQ-299: `tests/test_stack_ir_policies.py` + fixtures (passing + failing
|
||||
IR; skip when kj absent).
|
||||
|
||||
### Phase P3 — plan-JSON policies + meta-orchestration + pipeline wiring (planned, tag v1.24.3)
|
||||
- REQ-300: `adapters/kyverno-json/policies/plan-json/` — 3 policies over
|
||||
`terraform show -json` (plaintext-secrets, iam-wildcard, kms-reference
|
||||
— ports of `checkov_adapter.py:RULE_MAP`).
|
||||
- REQ-301: `run_platform.sh` Step 5 gains a parallel kyverno-json pass;
|
||||
both PCR lists (checkov/wiz + kj) concatenate into the confidence
|
||||
signal's `policy` input; skips gracefully when `which kj` is false.
|
||||
- REQ-302: `tests/test_plan_json_policies.py` + fixtures;
|
||||
`tests/test_run_platform_plan_json_policies.py` (script-substring
|
||||
assertion).
|
||||
- REQ-303: `adapters/kyverno-json/policies/meta/` —
|
||||
`block-on-any-critical.json` (declarative critical-block; the
|
||||
`confidence_signal.py` hard-override stays as defense-in-depth) +
|
||||
`tagging-rules-agree.json` (asserts Checkov + kj agree on tagging).
|
||||
`tests/test_meta_policies.py`.
|
||||
|
||||
### Phase P4 — regression-gate policies + docs (planned, tag v1.24.4)
|
||||
- REQ-304: `adapters/kyverno-json/policies/regression/` — 3 policies over
|
||||
capability-inventory JSON (CAP-013/023/024) — declarative mirrors of
|
||||
`core/regression_verify.py` checks.
|
||||
- REQ-305: `tests/test_regression_policies.py` + fixtures (clean +
|
||||
drifted inventory); regression gate still 287/287 baseline.
|
||||
- REQ-306: `adapters/README.md` (new adapter row + PolicyEngine Protocol
|
||||
section) + `adapters/kyverno-json/README.md`.
|
||||
- REQ-307: `.ciagent/ARCHITECTURE.md` §12.7 (Policy Engine Registry) +
|
||||
`schemas/README.md` + `modules/STANDARDS.md` (policy-authoring
|
||||
standard) + `docs/METRICS.md` (swappable engine narrative).
|
||||
|
||||
### Phase P5 — final review + audit + milestone ship (Final Phase, tag v1.24.5)
|
||||
- Multi-persona code review across P1..P4 (lead-developer, backend-
|
||||
engineer, data-engineer, policy-engineer). Auto-fix P0; flag P1+.
|
||||
- Audit: reconstruction test (git log ↔ `.ciagent/`), branch hygiene,
|
||||
commit discipline.
|
||||
- Milestone ship: merge `phase/05-final-review-ship` →
|
||||
`milestone/v1.25-kyverno-json` → `main`; tag `v1.24.5` (= the v1.25
|
||||
release per prev-minor tagging rule); create Gitea release with full
|
||||
milestone summary; delete all milestone branches.
|
||||
- Update `REQUIREMENTS.md` (mark REQ-291..309 complete), `ROADMAP.md`
|
||||
(mark v1.25 complete), `NORTH_STAR.md` (note Strategic Objective #2 —
|
||||
provable trust via a replaceable policy-engine substrate).
|
||||
- **Requirements:** REQ-291..309 (19 requirements).
|
||||
|
||||
+75
-123
@@ -1,135 +1,87 @@
|
||||
# ACDL v1.10 — Verify (milestone gate)
|
||||
# VERIFY — P1 engine-core (v1.25)
|
||||
|
||||
> Verify date: 2026-07-27. Verifier: ci-verifier. Milestone: v1.10 (complete, tag `v1.10.0`).
|
||||
> Scope: 4 phases (52–55), 5 commits (772ac72..2697775), 22 files, +2281/-256 lines.
|
||||
> 4-layer verify gate: structural, behavioral, security, quality.
|
||||
> Phase: P1. Requirements: REQ-291..294, 308, 309. Result: PASS.
|
||||
|
||||
## Layer 1: Structural — PASS
|
||||
## Structural
|
||||
|
||||
- All 8 plan-referenced files exist on disk (`core/regression_verify.py`,
|
||||
`core/local_emulators.py`, `scripts/run_regression.sh`,
|
||||
`tests/test_verify_regression_mode.py`,
|
||||
`tests/test_local_emulating_adapters.py`,
|
||||
`.ciagent/CAPABILITY_INVENTORY.md`, `REGRESSION_REPORT.md`,
|
||||
`REGRESSION_REPORT.json`).
|
||||
- All imports resolve (`py_compile` + runtime import OK).
|
||||
- No TODO/FIXME/HACK/stub placeholders in new code (the `LocalLambdaStub`
|
||||
is a legitimate local emulator, not a placeholder).
|
||||
- All declared exports exist (`run_regression`, `write_report`,
|
||||
`CAPABILITY_REGISTRY`, `RegressionReport`, `CapabilityResult`,
|
||||
`FlatFileOutbox`, `LocalEcsEmulator`, `LocalS3StateBackend`,
|
||||
`LocalLambdaStub`, `run_local_e2e`, `is_local_tier`).
|
||||
- `core/policy_engine.py` exists, implements `PolicyEngine` Protocol
|
||||
(PEP 544, `@runtime_checkable`), `PolicyEngineRegistry` with
|
||||
`register()` + `get_engine()`, `NullEngine` fallback.
|
||||
- `adapters/kyverno-json/kyverno_json_engine.py` exists, exports
|
||||
`KyvernoJsonEngine` with `name`, `is_configured()`, `evaluate()`.
|
||||
- `adapters/kyverno-json/__init__.py` loads the engine by file path
|
||||
(the dir name has a hyphen — not a valid Python package name).
|
||||
- `adapters/kyverno-json/policies/_smoke.json` exists (trivial policy
|
||||
for round-trip validation).
|
||||
- `scripts/install-kyverno-json.sh` exists (go install kj@latest).
|
||||
- `.ciagent/config.json` has the `policy` object
|
||||
(`engine: kyverno-json`, `policy_root`).
|
||||
- `.gitea/workflows/ci.yml` + `.github/workflows/ci.yml` have the
|
||||
Go + kj install step (best-effort, tests skip when kj absent).
|
||||
- `tests/test_policy_engine.py` (10 tests) +
|
||||
`tests/test_kyverno_json_engine.py` (16 tests) exist.
|
||||
|
||||
## Layer 2: Behavioral — PASS
|
||||
## Behavioral
|
||||
|
||||
- `pytest tests/ -m "not slow"`: **513 passed**, 5 deselected.
|
||||
- `pytest tests/ -m slow`: **5 passed** (2 local E2E + 3 regression
|
||||
integration incl. live-AWS terraform plan).
|
||||
- **Total: 518 passed, 0 failed.**
|
||||
- Requirement coverage: REQ-112 (P52), REQ-113 (P53), REQ-114 (P54),
|
||||
REQ-115 (P55) — all 4 marked `complete`.
|
||||
- Regression gate: `bash scripts/run_regression.sh` → **16/16
|
||||
capabilities Verified** (12 local + 4 live-AWS). Milestone gate open.
|
||||
- `pytest tests/test_policy_engine.py tests/test_kyverno_json_engine.py`:
|
||||
**24 passed, 2 skipped** (kj not installed — expected;
|
||||
`pytest.skip("kj not installed")`).
|
||||
- `NullEngine` satisfies the `PolicyEngine` Protocol (G-Q8a —
|
||||
`isinstance(NullEngine(), PolicyEngine)` is True). Proves the swap
|
||||
boundary is real without implementing OPA.
|
||||
- `KyvernoJsonEngine.is_configured()` returns `False` when
|
||||
`which kj` is absent → `evaluate()` returns a single
|
||||
`KJ_ENGINE_NOT_CONFIGURED` SKIPPED PCR (distinct `ruleId` from
|
||||
NullEngine's `NULL_ENGINE_INACTIVE` — G-Q4).
|
||||
- PCR records validate against `schemas/policy_check_result.schema.json`
|
||||
(via `jsonschema.validate` in tests).
|
||||
- Defensive parsing: malformed kyverno-json output → `error` PCR
|
||||
(`KJ_ENGINE_ERROR`), never an exception.
|
||||
- Severity annotation reading (G-Q10a): policies with
|
||||
`nova.cloudinit.dev/severity: high` produce PCRs with `severity: high`;
|
||||
policies without the annotation default to `info`.
|
||||
- Registry: `get_engine()` returns the configured engine; unknown
|
||||
engine name raises `KeyError`; `policy` key absent → `NullEngine`.
|
||||
- No regression: `pytest tests/test_confidence_signal.py
|
||||
tests/test_adapter.py tests/test_checkov_adapter.py
|
||||
tests/test_kyverno_adapter.py tests/test_contract_resolver.py` —
|
||||
**132 passed** (unchanged).
|
||||
|
||||
## Layer 3: Security (STRIDE) — PASS
|
||||
## Security
|
||||
|
||||
| Threat | Risk | Disposition |
|
||||
|--------|------|-------------|
|
||||
| Spoofing | Local Lambda stub patches `_get_dynamodb`/`_get_secrets_client`; opt-in via `ACDL_LOCAL_TIER=1`, never in prod | Accept (low) |
|
||||
| Tampering | Flat-file outbox hash-chain verification detects tampering | Accept (low) |
|
||||
| Repudiation | Regression report records per-capability status + timestamps | Accept (low) |
|
||||
| Info Disclosure | Creds read into env vars, never logged (0 cred strings in reports); ECS binds 127.0.0.1 only | Accept (low) |
|
||||
| Denial of Service | Local ECS emulator: free port, daemon thread, clean destroy | Accept (low) |
|
||||
| Elevation of Privilege | `urllib.urlopen` patched to fake response (no network egress); no eval/exec/subprocess in adapter | Accept (low) |
|
||||
- No new secrets, no new network calls in the engine core (the engine
|
||||
shells to a local binary; the binary makes no network calls for
|
||||
`scan`).
|
||||
- `is_configured()` guard ensures the platform runs without the binary
|
||||
(no hard dependency that could be exploited as a DoS vector).
|
||||
- The engine writes the payload to a temp file (`tempfile.NamedTemporaryFile`)
|
||||
and unlinks it in a `finally` block (no leftover payload on disk).
|
||||
- No `shell=True` in the `subprocess.run` call (command is a list —
|
||||
no shell injection surface).
|
||||
|
||||
All threats low-severity; auto-accepted per
|
||||
`config.json security.auto_accept_low_severity=true`.
|
||||
## Quality
|
||||
|
||||
## Layer 4: Quality (multi-persona) — PASS
|
||||
- `python3 -m py_compile` passes on all new Python files.
|
||||
- The `PolicyEngine` Protocol is minimal (3 members) — the swap
|
||||
boundary is the moat (NORTH_STAR Strategic Objective #2).
|
||||
- The `NullEngine` proves a second implementation exists (structural
|
||||
conformance) — the OPA swap is a known quantity (RESEARCH §4.2).
|
||||
- Tests use `pytest.skip` when `which kj` is absent, so the CI matrix
|
||||
passes with or without the binary (the suite is green in both cases).
|
||||
|
||||
| Persona | Finding | Verdict |
|
||||
|---------|---------|---------|
|
||||
| Correctness | 7 adapter defects fixed; each traceable to a terraform validate/plan error | PASS |
|
||||
| Testing | 518 tests pass; 24 new tests. P2: uptime-kuma + RDS not in registry | PASS (1 P2) |
|
||||
| Security | No creds logged; loopback-only; monkey-patches scoped to local tier | PASS |
|
||||
| Performance | Regression run ~60s; acceptable for a milestone gate | PASS |
|
||||
| Maintainability | Well-structured; adding a capability = 1 function + 1 registry entry | PASS |
|
||||
| Adversarial | Gate can't be bypassed; local E2E can't mutate cloud; no injection vectors | PASS |
|
||||
## Must-have checklist
|
||||
|
||||
**0 P0, 0 P1, 1 P2 (post-hoc: expand regression registry to uptime-kuma + RDS stacks).**
|
||||
- [x] `PolicyEngine` Protocol + `PolicyEngineRegistry` + `NullEngine`
|
||||
(REQ-291)
|
||||
- [x] `config.json.policy` object (REQ-292)
|
||||
- [x] `KyvernoJsonEngine` adapter (REQ-293)
|
||||
- [x] `__init__.py` + `_smoke.json` + `install-kyverno-json.sh` + CI
|
||||
install (REQ-294)
|
||||
- [x] `test_policy_engine.py` — protocol conformance, registry,
|
||||
NullEngine fallback (REQ-308)
|
||||
- [x] `test_kyverno_json_engine.py` — PCR schema validity, defensive
|
||||
parsing, skip-without-kj (REQ-309)
|
||||
|
||||
## Verdict
|
||||
|
||||
**VERIFY PASS** — all 4 layers pass. The v1.10 milestone is sound:
|
||||
the pipeline regression gap is fixed (D-091), the platform is fully
|
||||
locally testable (D-092), every advertised capability is re-verified
|
||||
(D-093, 16/16 Verified), and the docs/decks match verified reality
|
||||
(D-094). 518 tests pass; the regression gate covers 16 capabilities
|
||||
including 4 live-AWS checks. 0 P0, 0 P1, 1 P2 post-hoc. Ready to ship.
|
||||
|
||||
---
|
||||
|
||||
# ACDL — Verify (grill deliverable, commit ac11c01)
|
||||
|
||||
> Verify date: 2026-07-27. Verifier: ci-verifier. Scope: the grill
|
||||
> deliverable (`.ciagent/GRILL.md`, phase 0, status `grill`) added in
|
||||
> commit `ac11c01` since the v1.10 audit PASS (`ab477b3`). Docs-only;
|
||||
> no code, no tests, no schema changes.
|
||||
|
||||
## Layer 1: Structural — PASS
|
||||
|
||||
- `.ciagent/GRILL.md` exists on disk (18250 bytes).
|
||||
- No imports to resolve (markdown docs file).
|
||||
- No TODO/FIXME/HACK/stub placeholders in the report.
|
||||
- All required sections present per grill workflow Step 5 format:
|
||||
title, Run header, Verdict, 9 axes (1–9), Meta, Binding Decisions
|
||||
table (12 rows), Escalations section (2 entries: G-005, G-008).
|
||||
- Commit `ac11c01` `---ci---` block is well-formed: `project: acdl`,
|
||||
`phase: 0`, `milestone: v1.10`, `status: grill`, 12 decision ids
|
||||
(G-001..G-012), 2 escalation lines.
|
||||
|
||||
## Layer 2: Behavioral — PASS
|
||||
|
||||
- `pytest tests/ -m "not slow"`: **513 passed**, 5 deselected (no
|
||||
regressions introduced by the docs-only grill commit).
|
||||
- No new tests required (docs-only deliverable; the grill is a
|
||||
review artifact, not a code change).
|
||||
- Requirement coverage: not applicable (phase 0, status `grill`; no
|
||||
REQ-IDs bound to this deliverable). The grill's binding decisions
|
||||
(G-001..G-012) are advisory and do not modify REQUIREMENTS.md per
|
||||
grill workflow Step 7.
|
||||
|
||||
## Layer 3: Security (STRIDE) — PASS
|
||||
|
||||
| Threat | Risk | Disposition |
|
||||
|--------|------|-------------|
|
||||
| Spoofing | N/A (docs-only; no auth surface) | Accept (none) |
|
||||
| Tampering | Grill report is git-tracked; tampering = git history rewrite (out of scope) | Accept (low) |
|
||||
| Repudiation | Commit `ac11c01` signed by author; `---ci---` block records status + decisions | Accept (low) |
|
||||
| Info Disclosure | No credentials, keys, tokens, or PII in the report (grep scan clean) | Accept (low) |
|
||||
| Denial of Service | N/A (docs file; no runtime surface) | Accept (none) |
|
||||
| Elevation of Privilege | N/A (docs-only; no privilege surface) | Accept (none) |
|
||||
|
||||
All threats low-or-none; auto-accepted per
|
||||
`config.json security.auto_accept_low_severity=true`.
|
||||
|
||||
## Layer 4: Quality (multi-persona) — PASS
|
||||
|
||||
| Persona | Finding | Verdict |
|
||||
|---------|---------|---------|
|
||||
| Correctness | 12 binding decisions traceable to evidence (commit/file/req-id); 2 escalations correctly unresolved | PASS |
|
||||
| Testing | Docs-only; 513 fast tests pass (no regression) | PASS |
|
||||
| Security | No credential leakage; no sensitive data in report | PASS |
|
||||
| Performance | N/A (docs file; no runtime cost) | PASS |
|
||||
| Maintainability | Report follows grill workflow Step 5 format exactly; appendable for future runs | PASS |
|
||||
| Adversarial | Escalations (G-005, G-008) are surfaced, not silently skipped; visible via `ciagent audit` | PASS |
|
||||
|
||||
**0 P0, 0 P1, 0 P2.**
|
||||
|
||||
## Verdict (grill deliverable)
|
||||
|
||||
**VERIFY PASS** — all 4 layers pass. The grill deliverable is a
|
||||
well-formed docs-only artifact. 513 fast tests pass (no regression).
|
||||
No credential leakage. 12 binding decisions recorded; 2 escalations
|
||||
(G-005 risks, G-008 budget) correctly surfaced for human resolution.
|
||||
The grill does not modify PROJECT.md, ROADMAP.md, or REQUIREMENTS.md
|
||||
(per grill workflow Step 7).
|
||||
**Verdict: PASS** — all P1 must-haves met, no regressions, 24 new
|
||||
tests pass (2 skip-without-kj), 132 existing tests unchanged.
|
||||
@@ -8,7 +8,7 @@
|
||||
],
|
||||
"active_project": "acdl",
|
||||
"active_projects": ["acdl"],
|
||||
"active_milestone": "v1.24",
|
||||
"active_milestone": "v1.25",
|
||||
"autonomy": {
|
||||
"level": "full",
|
||||
"escalation_hooks": ["deploy", "delete_data", "merge_to_main"],
|
||||
@@ -209,5 +209,9 @@
|
||||
"enabled": true,
|
||||
"persist": true
|
||||
},
|
||||
"strategic_direction_file": ".ciagent/NORTH_STAR.md"
|
||||
"strategic_direction_file": ".ciagent/NORTH_STAR.md",
|
||||
"policy": {
|
||||
"engine": "kyverno-json",
|
||||
"policy_root": "adapters/kyverno-json/policies"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -63,6 +63,23 @@ jobs:
|
||||
- name: Install test dependencies
|
||||
run: pip install -r requirements-test.txt
|
||||
|
||||
- name: Install kyverno-json (kj) for policy-engine tests
|
||||
run: |
|
||||
# v1.25: kyverno-json is the primary policy engine. Tests that
|
||||
# require kj skip when absent, so this is best-effort (the suite
|
||||
# passes with or without kj). Install is cached via the Go
|
||||
# module cache (~/.cache/go-build + ~/go/pkg/mod).
|
||||
if command -v go >/dev/null 2>&1; then
|
||||
go install github.com/kyverno/kyverno-json/cmd/kj@latest && \
|
||||
echo "$(go env GOPATH)/bin" >> "$GITHUB_PATH" || \
|
||||
echo "kj install failed; policy-engine tests will skip"
|
||||
else
|
||||
sudo apt-get update && sudo apt-get install -y golang-go && \
|
||||
go install github.com/kyverno/kyverno-json/cmd/kj@latest && \
|
||||
echo "$(go env GOPATH)/bin" >> "$GITHUB_PATH" || \
|
||||
echo "kj install failed; policy-engine tests will skip"
|
||||
fi
|
||||
|
||||
- name: Run pytest
|
||||
run: python3 -m pytest tests/ -v --tb=short
|
||||
|
||||
|
||||
@@ -63,6 +63,21 @@ jobs:
|
||||
- name: Install test dependencies
|
||||
run: pip install -r requirements-test.txt
|
||||
|
||||
- name: Install kyverno-json (kj) for policy-engine tests
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: "1.22"
|
||||
cache: false
|
||||
|
||||
- name: Install kj binary
|
||||
run: |
|
||||
# v1.25: kyverno-json is the primary policy engine. Tests that
|
||||
# require kj skip when absent, so this is best-effort (the suite
|
||||
# passes with or without kj).
|
||||
go install github.com/kyverno/kyverno-json/cmd/kj@latest && \
|
||||
echo "$(go env GOPATH)/bin" >> "$GITHUB_PATH" || \
|
||||
echo "kj install failed; policy-engine tests will skip"
|
||||
|
||||
- name: Run pytest
|
||||
run: python3 -m pytest tests/ -v --tb=short
|
||||
|
||||
|
||||
@@ -12,6 +12,37 @@ Adapters translate the engine-agnostic Target Stack IR to engine-specific format
|
||||
| Checkov adapter | `adapters/terraform/policy/checkov_adapter.py` | Checkov JSON | `PolicyCheckResult` records | Translates Checkov results |
|
||||
| Wiz adapter | `adapters/wiz/wiz_adapter.py` | Wiz API issues JSON | `PolicyCheckResult` records | Translates Wiz security findings |
|
||||
| Kyverno adapter | `adapters/kyverno/kyverno_adapter.py` | Kyverno PolicyReport JSON | `PolicyCheckResult` records | K8s-native policy translation |
|
||||
| kyverno-json engine | `adapters/kyverno-json/kyverno_json_engine.py` | Any JSON/YAML payload | `PolicyCheckResult` records | **v1.25 primary policy engine** (swappable via `PolicyEngine` protocol) |
|
||||
|
||||
## Policy Engine Protocol (v1.25)
|
||||
|
||||
The `core/policy_engine.py` module defines the **swap boundary** between
|
||||
Nova and its policy engines. A `PolicyEngine` Python Protocol (PEP 544)
|
||||
with three members (`name`, `is_configured()`, `evaluate()`) is the
|
||||
contract; a `PolicyEngineRegistry` selects the active engine from
|
||||
`config.json`'s `policy.engine` key. The confidence signal and pipeline
|
||||
never import an engine directly — they go through the registry.
|
||||
|
||||
**Implementations:**
|
||||
- `KyvernoJsonEngine` (`adapters/kyverno-json/`) — shells to the `kj`
|
||||
CLI; the v1.25 default.
|
||||
- `NullEngine` (`core/policy_engine.py`) — fallback when the `policy`
|
||||
key is absent (emits `SKIPPED`).
|
||||
- Future: `OpaEngine` — implements the same protocol, shells to
|
||||
`opa eval`. The OPA-equivalent surface is documented in
|
||||
`.ciagent/RESEARCH.md` §4.2.
|
||||
|
||||
**How to add a new engine:**
|
||||
1. Create `adapters/<name>/<name>_engine.py` implementing the
|
||||
`PolicyEngine` protocol (`name`, `is_configured()`, `evaluate()`).
|
||||
2. `evaluate()` returns `list[dict]` where each dict conforms to
|
||||
`schemas/policy_check_result.schema.json`.
|
||||
3. Register the engine in `core/policy_engine.py`'s `_autoload_*`
|
||||
function (or call `register(name, factory)` at startup).
|
||||
4. Set `config.json.policy.engine` to the engine's `name`.
|
||||
5. Add the engine to the `engine` enum in
|
||||
`schemas/policy_check_result.schema.json` if it needs a distinct
|
||||
enum value (v1.25 reuses `"kyverno"` — see D-116).
|
||||
|
||||
## How to Write an Adapter
|
||||
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
# kyverno-json Engine Adapter (v1.25)
|
||||
|
||||
The `kyverno-json` engine is Nova's **primary compliance/policy tool**
|
||||
(v1.25), implemented behind the swappable `PolicyEngine` protocol so
|
||||
OPA (or any other engine) can replace it one day.
|
||||
|
||||
## What kyverno-json is
|
||||
|
||||
[kyverno-json](https://github.com/kyverno/kyverno-json) is a standalone
|
||||
Go binary from the Kyverno project — a **separate runtime** from the
|
||||
K8s Kyverno admission controller. It applies Kyverno `ValidatingPolicy`
|
||||
resources to **any** JSON or YAML payload file via the `kj scan` CLI.
|
||||
Unlike the K8s Kyverno adapter (`adapters/kyverno/`), which only
|
||||
speaks to K8s manifests, kyverno-json evaluates consumer contracts,
|
||||
resolved Stack IR, terraform plan JSON, and even the merged PCR list
|
||||
itself (meta-policies).
|
||||
|
||||
## Install
|
||||
|
||||
```bash
|
||||
bash scripts/install-kyverno-json.sh
|
||||
# or directly:
|
||||
go install github.com/kyverno/kyverno-json/cmd/kj@latest
|
||||
kj version
|
||||
```
|
||||
|
||||
The platform functions without the binary — `is_configured()` returns
|
||||
`False` when `which kj` is absent → `evaluate()` returns a single
|
||||
`SKIPPED` PCR (`KJ_ENGINE_NOT_CONFIGURED`). The confidence signal
|
||||
proceeds with a neutral `policy` input (D-120 graceful degradation).
|
||||
|
||||
## Policy directory layout
|
||||
|
||||
```
|
||||
adapters/kyverno-json/policies/
|
||||
├── _smoke.json # round-trip smoke test
|
||||
├── contract/ # consumer contract JSON policies
|
||||
│ ├── require-id-pattern.json
|
||||
│ ├── require-env-in-enum.json
|
||||
│ ├── require-infrastructure-min-1.json
|
||||
│ └── forbid-unknown-fields.json
|
||||
├── stack-ir/ # resolved Stack IR policies
|
||||
│ ├── require-tagging-standard.json
|
||||
│ ├── forbid-public-ingress.json
|
||||
│ └── require-encryption-by-default.json
|
||||
├── plan-json/ # terraform show -json policies
|
||||
│ ├── forbid-plaintext-secrets.json
|
||||
│ ├── forbid-iam-wildcard.json
|
||||
│ └── require-kms-reference.json
|
||||
├── meta/ # policies over the merged PCR list
|
||||
│ ├── block-on-any-critical.json
|
||||
│ └── tagging-rules-agree.json
|
||||
└── regression/ # capability-inventory policies
|
||||
├── cap-013-adapter-dedup.json
|
||||
├── cap-023-metrics-collector.json
|
||||
└── cap-024-deck-structure.json
|
||||
```
|
||||
|
||||
## The four policy categories
|
||||
|
||||
1. **contract/** — over the consumer contract JSON (pre-resolve).
|
||||
2. **stack-ir/** — over the resolved Target Stack IR (post-resolve).
|
||||
3. **plan-json/** — over `terraform show -json` output (pipeline Step 5b).
|
||||
4. **meta/** — over the merged `list[PolicyCheckResult]` (meta-policies).
|
||||
5. **regression/** — over the capability-inventory JSON (declarative
|
||||
mirrors of `core/regression_verify.py`).
|
||||
|
||||
## Severity convention
|
||||
|
||||
kyverno-json does not natively assign severities. Each Nova policy
|
||||
declares its severity via a `metadata.annotations` field:
|
||||
|
||||
```yaml
|
||||
metadata:
|
||||
annotations:
|
||||
nova.cloudinit.dev/severity: high
|
||||
```
|
||||
|
||||
Valid values: `critical`, `high`, `medium`, `low`, `info` (default
|
||||
when absent).
|
||||
|
||||
## Engine enum reuse (D-116)
|
||||
|
||||
kyverno-json PCR records carry `engine: "kyverno"` (no new enum value).
|
||||
The `engine` field records the policy-engine *family*, not the specific
|
||||
binary. The K8s Kyverno adapter and the kyverno-json engine are
|
||||
distinguished by `ruleId` prefix (`KYVERNO_` vs `KJ_`) and `evidence`
|
||||
payload shape (`namespace`/`kind` vs `assertion`/`jmespath`).
|
||||
|
||||
## Schema path
|
||||
|
||||
The output records validate against
|
||||
[`schemas/policy_check_result.schema.json`](../../schemas/policy_check_result.schema.json)
|
||||
(`engine: "kyverno"` is in the enum). The confidence signal consumes
|
||||
the merged PCR list engine-agnostically.
|
||||
|
||||
## Swap boundary
|
||||
|
||||
The `PolicyEngine` protocol (`core/policy_engine.py`) is the swap
|
||||
boundary. The OPA-equivalent surface is documented in
|
||||
`.ciagent/RESEARCH.md` §4.2 — a future `OpaEngine` implements the same
|
||||
protocol without touching the confidence signal, the PCR schema, or
|
||||
the pipeline.
|
||||
@@ -0,0 +1,27 @@
|
||||
"""Nova kyverno-json adapter package (v1.25, REQ-294).
|
||||
|
||||
The directory name ``kyverno-json`` has a hyphen, so it is not a valid
|
||||
Python package name and cannot be imported via ``import
|
||||
adapters.kyverno-json``. The ``PolicyEngineRegistry`` loads the engine
|
||||
by file path (``importlib.util.spec_from_file_location``). This
|
||||
``__init__`` is a convenience for direct-script use and for ``pip
|
||||
install -e .`` style discovery if the package is ever renamed.
|
||||
"""
|
||||
|
||||
|
||||
def _load_engine():
|
||||
import importlib.util
|
||||
import os
|
||||
engine_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
|
||||
"kyverno_json_engine.py")
|
||||
spec = importlib.util.spec_from_file_location("kyverno_json_engine", engine_path)
|
||||
if spec is None or spec.loader is None:
|
||||
raise ImportError(f"could not load {engine_path}")
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(mod)
|
||||
return mod.KyvernoJsonEngine
|
||||
|
||||
|
||||
KyvernoJsonEngine = _load_engine()
|
||||
|
||||
__all__ = ["KyvernoJsonEngine"]
|
||||
@@ -0,0 +1,269 @@
|
||||
"""Nova KyvernoJsonEngine (REQ-293, v1.25).
|
||||
|
||||
Implements the ``PolicyEngine`` protocol (``core/policy_engine.py``)
|
||||
by shelling to the ``kj`` CLI (``kyverno-json``). Translates native
|
||||
kyverno-json scan output to Nova ``PolicyCheckResult`` dicts
|
||||
(``schemas/policy_check_result.schema.json``).
|
||||
|
||||
Engine enum reuse (D-116): records carry ``engine: "kyverno"`` (no new
|
||||
enum value). The ``ruleId`` is prefixed ``KJ_<policy_name>`` to
|
||||
distinguish from the K8s Kyverno adapter's ``KYVERNO_`` prefix.
|
||||
|
||||
Severity (RESEARCH §2.6, G-Q10a): kyverno-json does not natively assign
|
||||
severities. Each Nova policy declares its severity via a
|
||||
``metadata.annotations["nova.cloudinit.dev/severity"]`` field. The
|
||||
engine reads this annotation from the loaded policy YAML (not from the
|
||||
scan result — the result doesn't carry it) and applies it to every
|
||||
result that policy produces. Default when absent: ``"info"``.
|
||||
|
||||
Graceful degradation (D-120): ``is_configured()`` returns ``False`` when
|
||||
``which kj`` is absent → ``evaluate()`` returns a single SKIPPED PCR
|
||||
(``ruleId: KJ_ENGINE_NOT_CONFIGURED``). The platform functions without
|
||||
the binary.
|
||||
|
||||
Defensive parsing: any kyverno-json output that doesn't match the
|
||||
expected shape produces an ``error`` PCR, never an exception. The
|
||||
engine is read-only against a local policy dir + a temp payload file.
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any, Union
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
Payload = Union[dict, list, str]
|
||||
|
||||
SEVERITY_DEFAULT = "info"
|
||||
SEVERITY_ANNOTATION = "nova.cloudinit.dev/severity"
|
||||
|
||||
RESULT_MAP = {
|
||||
"pass": "pass",
|
||||
"fail": "fail",
|
||||
"error": "error",
|
||||
"skip": "skipped",
|
||||
"skipped": "skipped",
|
||||
"warn": "skipped",
|
||||
"warning": "skipped",
|
||||
}
|
||||
|
||||
|
||||
def _iso8601_now() -> str:
|
||||
return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
|
||||
def _which_kj() -> str | None:
|
||||
"""Return the path to ``kj`` if on PATH, else ``None``."""
|
||||
return shutil.which("kj")
|
||||
|
||||
|
||||
def _load_policy_severities(policy_dir: Path) -> dict[str, str]:
|
||||
"""Load each ``.json``/``.yaml``/``.yml`` policy in ``policy_dir``
|
||||
(non-recursive) and return ``{policy_name: severity}``.
|
||||
|
||||
kyverno-json policies are Kubernetes-style ``ValidatingPolicy``
|
||||
resources. The severity is read from
|
||||
``metadata.annotations["nova.cloudinit.dev/severity"]``. Policies
|
||||
in subdirectories (e.g. ``contract/``, ``stack-ir/``) are loaded
|
||||
when the caller passes that subdirectory as ``policy_dir``.
|
||||
"""
|
||||
severities: dict[str, str] = {}
|
||||
if not policy_dir.is_dir():
|
||||
return severities
|
||||
for entry in sorted(os.listdir(policy_dir)):
|
||||
if entry.startswith("_") or entry.startswith("."):
|
||||
continue
|
||||
full = policy_dir / entry
|
||||
if not full.is_file():
|
||||
continue
|
||||
if entry.endswith((".json", ".yaml", ".yml")):
|
||||
try:
|
||||
with open(full, "r", encoding="utf-8") as fh:
|
||||
doc = yaml.safe_load(fh)
|
||||
if not isinstance(doc, dict):
|
||||
continue
|
||||
name = doc.get("metadata", {}).get("name") or entry.rsplit(".", 1)[0]
|
||||
ann = doc.get("metadata", {}).get("annotations", {}) or {}
|
||||
sev = ann.get(SEVERITY_ANNOTATION, SEVERITY_DEFAULT)
|
||||
severities[name] = str(sev).lower()
|
||||
except Exception:
|
||||
continue
|
||||
return severities
|
||||
|
||||
|
||||
def _to_pcr(entry: dict, contract_id: str, severity: str) -> dict:
|
||||
"""Translate a kyverno-json scan result entry to a PCR dict."""
|
||||
policy_name = entry.get("policy", "") or "UNKNOWN"
|
||||
rule_name = entry.get("rule", "") or ""
|
||||
rule_id = f"KJ_{policy_name}"
|
||||
if rule_name:
|
||||
rule_id = f"{rule_id}/{rule_name}"
|
||||
result_raw = entry.get("result", "skip")
|
||||
result = RESULT_MAP.get(str(result_raw).lower(), "error")
|
||||
message = entry.get("message", "") or ""
|
||||
resource = entry.get("resource", "")
|
||||
if not resource and entry.get("name"):
|
||||
kind = entry.get("kind", "")
|
||||
ns = entry.get("namespace", "")
|
||||
resource = f"{kind}/{ns}/{entry.get('name')}" if kind else entry.get("name", "")
|
||||
return {
|
||||
"contractId": contract_id,
|
||||
"evaluatedAt": _iso8601_now(),
|
||||
"engine": "kyverno",
|
||||
"ruleId": rule_id,
|
||||
"severity": severity,
|
||||
"result": result,
|
||||
"message": message,
|
||||
"evidence": {
|
||||
"resource": resource,
|
||||
"policy": policy_name,
|
||||
"rule": rule_name,
|
||||
"namespace": entry.get("namespace", ""),
|
||||
"kind": entry.get("kind", ""),
|
||||
"name": entry.get("name", ""),
|
||||
},
|
||||
"resourceRef": resource,
|
||||
}
|
||||
|
||||
|
||||
def _skipped_not_configured(contract_id: str) -> dict:
|
||||
return {
|
||||
"contractId": contract_id,
|
||||
"evaluatedAt": _iso8601_now(),
|
||||
"engine": "kyverno",
|
||||
"ruleId": "KJ_ENGINE_NOT_CONFIGURED",
|
||||
"severity": "info",
|
||||
"result": "skipped",
|
||||
"message": (
|
||||
"kyverno-json engine not configured — `which kj` returned no path. "
|
||||
"Install via scripts/install-kyverno-json.sh. The platform proceeds "
|
||||
"with a neutral SKIPPED policy input (is_configured() guard, D-120)."
|
||||
),
|
||||
"evidence": {},
|
||||
"resourceRef": "",
|
||||
}
|
||||
|
||||
|
||||
def _error_pcr(contract_id: str, message: str) -> dict:
|
||||
return {
|
||||
"contractId": contract_id,
|
||||
"evaluatedAt": _iso8601_now(),
|
||||
"engine": "kyverno",
|
||||
"ruleId": "KJ_ENGINE_ERROR",
|
||||
"severity": "info",
|
||||
"result": "error",
|
||||
"message": message,
|
||||
"evidence": {},
|
||||
"resourceRef": "",
|
||||
}
|
||||
|
||||
|
||||
class KyvernoJsonEngine:
|
||||
"""``PolicyEngine`` impl that shells to the ``kj`` CLI."""
|
||||
|
||||
name = "kyverno-json"
|
||||
|
||||
def is_configured(self) -> bool:
|
||||
return _which_kj() is not None
|
||||
|
||||
def evaluate(self, payload: Payload, policy_dir: Path,
|
||||
contract_id: str) -> list[dict]:
|
||||
if not self.is_configured():
|
||||
return [_skipped_not_configured(contract_id)]
|
||||
kj = _which_kj()
|
||||
policy_dir = Path(policy_dir)
|
||||
if not policy_dir.is_dir():
|
||||
return [_error_pcr(
|
||||
contract_id,
|
||||
f"kyverno-json policy dir not found: {policy_dir}",
|
||||
)]
|
||||
severities = _load_policy_severities(policy_dir)
|
||||
# Write payload to temp file (kj scan --payload expects a file path).
|
||||
payload_tmp = tempfile.NamedTemporaryFile(
|
||||
mode="w", suffix=".json", delete=False, encoding="utf-8"
|
||||
)
|
||||
try:
|
||||
json.dump(payload, payload_tmp)
|
||||
payload_tmp.flush()
|
||||
payload_tmp.close()
|
||||
cmd = [
|
||||
kj, "scan",
|
||||
"--policy", str(policy_dir),
|
||||
"--payload", payload_tmp.name,
|
||||
"--output", "json",
|
||||
]
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
cmd, capture_output=True, text=True, timeout=60,
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
return [_error_pcr(contract_id, "kyverno-json scan timed out (60s)")]
|
||||
if proc.returncode not in (0, 1):
|
||||
return [_error_pcr(
|
||||
contract_id,
|
||||
f"kyverno-json scan exited {proc.returncode}: {proc.stderr[:200]}",
|
||||
)]
|
||||
try:
|
||||
out = json.loads(proc.stdout) if proc.stdout.strip() else {}
|
||||
except json.JSONDecodeError as e:
|
||||
return [_error_pcr(
|
||||
contract_id,
|
||||
f"kyverno-json output not JSON: {e}",
|
||||
)]
|
||||
return self._translate(out, contract_id, severities)
|
||||
finally:
|
||||
try:
|
||||
os.unlink(payload_tmp.name)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def _translate(self, out: dict, contract_id: str,
|
||||
severities: dict[str, str]) -> list[dict]:
|
||||
results = out.get("results", []) if isinstance(out, dict) else []
|
||||
if not isinstance(results, list):
|
||||
results = []
|
||||
pcrs: list[dict] = []
|
||||
for entry in results:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
policy_name = entry.get("policy", "") or "UNKNOWN"
|
||||
severity = severities.get(policy_name, SEVERITY_DEFAULT)
|
||||
pcrs.append(_to_pcr(entry, contract_id, severity))
|
||||
if not pcrs:
|
||||
# No results — kyverno-json produced nothing (no match, or
|
||||
# all policies passed with no result entries). Emit a
|
||||
# single pass PCR so the confidence signal's policy input
|
||||
# is non-empty (a non-empty list of passes → score 1.0).
|
||||
pcrs.append({
|
||||
"contractId": contract_id,
|
||||
"evaluatedAt": _iso8601_now(),
|
||||
"engine": "kyverno",
|
||||
"ruleId": "KJ_NO_RESULTS",
|
||||
"severity": "info",
|
||||
"result": "pass",
|
||||
"message": "kyverno-json scan produced no result entries (all policies passed or no match).",
|
||||
"evidence": {},
|
||||
"resourceRef": "",
|
||||
})
|
||||
return pcrs
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) < 4:
|
||||
print(
|
||||
"usage: kyverno_json_engine.py <payload.json> <policy_dir> <contract-id>",
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(2)
|
||||
with open(sys.argv[1], "r", encoding="utf-8") as fh:
|
||||
pl = json.load(fh)
|
||||
engine = KyvernoJsonEngine()
|
||||
out = engine.evaluate(pl, Path(sys.argv[2]), sys.argv[3])
|
||||
print(json.dumps(out, indent=2))
|
||||
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"apiVersion": "json.kyverno.io/v1alpha1",
|
||||
"kind": "ValidatingPolicy",
|
||||
"metadata": {
|
||||
"name": "require-contract-id",
|
||||
"annotations": {
|
||||
"nova.cloudinit.dev/severity": "high",
|
||||
"title.policy.kyverno.io": "Require contract id"
|
||||
}
|
||||
},
|
||||
"spec": {
|
||||
"rules": [
|
||||
{
|
||||
"name": "require-id",
|
||||
"validate": {
|
||||
"message": "contract id is required",
|
||||
"assert": {
|
||||
"all": [
|
||||
{
|
||||
"check": {
|
||||
"id": "(regex_match('^[a-z][a-z0-9-]{2,5}$', @))"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"apiVersion": "json.kyverno.io/v1alpha1",
|
||||
"kind": "ValidatingPolicy",
|
||||
"metadata": {
|
||||
"name": "forbid-unknown-fields",
|
||||
"annotations": {
|
||||
"nova.cloudinit.dev/severity": "low",
|
||||
"title.policy.kyverno.io": "Contract has only schema-allowed fields"
|
||||
}
|
||||
},
|
||||
"spec": {
|
||||
"rules": [
|
||||
{
|
||||
"name": "no-unknown-fields",
|
||||
"validate": {
|
||||
"message": "contract may only contain id, name, environment, infrastructure (schema-allowed fields)",
|
||||
"assert": {
|
||||
"all": [
|
||||
{
|
||||
"check": {
|
||||
"(length(keys(@)) == `4`)": true,
|
||||
"keys(@)": "(contains(['id','name','environment','infrastructure'], @))"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"apiVersion": "json.kyverno.io/v1alpha1",
|
||||
"kind": "ValidatingPolicy",
|
||||
"metadata": {
|
||||
"name": "require-env-in-enum",
|
||||
"annotations": {
|
||||
"nova.cloudinit.dev/severity": "high",
|
||||
"title.policy.kyverno.io": "Contract environment is one of dev/qa/prod/dr"
|
||||
}
|
||||
},
|
||||
"spec": {
|
||||
"rules": [
|
||||
{
|
||||
"name": "env-enum",
|
||||
"validate": {
|
||||
"message": "contract.environment must be one of dev, qa, prod, dr",
|
||||
"assert": {
|
||||
"all": [
|
||||
{
|
||||
"check": {
|
||||
"environment": "(contains(['dev','qa','prod','dr'], @))"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"apiVersion": "json.kyverno.io/v1alpha1",
|
||||
"kind": "ValidatingPolicy",
|
||||
"metadata": {
|
||||
"name": "require-id-pattern",
|
||||
"annotations": {
|
||||
"nova.cloudinit.dev/severity": "high",
|
||||
"title.policy.kyverno.io": "Contract id matches operational acronym pattern"
|
||||
}
|
||||
},
|
||||
"spec": {
|
||||
"rules": [
|
||||
{
|
||||
"name": "id-pattern",
|
||||
"validate": {
|
||||
"message": "contract.id must match ^[a-z][a-z0-9-]{2,5}$ (3-6 char operational acronym)",
|
||||
"assert": {
|
||||
"all": [
|
||||
{
|
||||
"check": {
|
||||
"id": "(regex_match('^[a-z][a-z0-9-]{2,5}$', @))"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"apiVersion": "json.kyverno.io/v1alpha1",
|
||||
"kind": "ValidatingPolicy",
|
||||
"metadata": {
|
||||
"name": "require-infrastructure-min-1",
|
||||
"annotations": {
|
||||
"nova.cloudinit.dev/severity": "medium",
|
||||
"title.policy.kyverno.io": "Contract declares at least one infrastructure entry"
|
||||
}
|
||||
},
|
||||
"spec": {
|
||||
"rules": [
|
||||
{
|
||||
"name": "infra-min-1",
|
||||
"validate": {
|
||||
"message": "contract.infrastructure must have at least one module entry",
|
||||
"assert": {
|
||||
"all": [
|
||||
{
|
||||
"check": {
|
||||
"infrastructure": "(length(keys(@)) > `0`)"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
{
|
||||
"apiVersion": "json.kyverno.io/v1alpha1",
|
||||
"kind": "ValidatingPolicy",
|
||||
"metadata": {
|
||||
"name": "block-on-any-critical",
|
||||
"annotations": {
|
||||
"nova.cloudinit.dev/severity": "critical",
|
||||
"title.policy.kyverno.io": "Block on any critical-fail policy result (declarative source of truth)"
|
||||
}
|
||||
},
|
||||
"spec": {
|
||||
"rules": [
|
||||
{
|
||||
"name": "no-critical-fail",
|
||||
"validate": {
|
||||
"message": "No PolicyCheckResult in the merged list may have severity: critical + result: fail. The confidence_signal.py hard-override is the defense-in-depth behind this declarative rule (D-119).",
|
||||
"assert": {
|
||||
"all": [
|
||||
{
|
||||
"check": {
|
||||
"~.[]": {
|
||||
"(severity == 'critical' && result == 'fail')": false
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
{
|
||||
"apiVersion": "json.kyverno.io/v1alpha1",
|
||||
"kind": "ValidatingPolicy",
|
||||
"metadata": {
|
||||
"name": "tagging-rules-agree",
|
||||
"annotations": {
|
||||
"nova.cloudinit.dev/severity": "medium",
|
||||
"title.policy.kyverno.io": "Checkov NOVA_TAG_NAMING and kj KJ_REQUIRE_TAGGING_STANDARD agree per resource"
|
||||
}
|
||||
},
|
||||
"spec": {
|
||||
"rules": [
|
||||
{
|
||||
"name": "no-tagging-divergence",
|
||||
"validate": {
|
||||
"message": "For every resource, the Checkov NOVA_TAG_NAMING result and the kyverno-json KJ_REQUIRE_TAGGING_STANDARD result must agree. Divergence emits an error PCR (D-118, defense-in-depth against rule drift).",
|
||||
"assert": {
|
||||
"all": [
|
||||
{
|
||||
"check": {
|
||||
"~.[?(ruleId == 'NOVA_TAG_NAMING')]": {
|
||||
"result->ckv_result": {},
|
||||
"($ckv_result == 'fail')": false
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"check": {
|
||||
"~.[?(ruleId == 'KJ_REQUIRE_TAGGING_STANDARD')]": {
|
||||
"result->kj_result": {},
|
||||
"($kj_result == 'fail')": false
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
{
|
||||
"apiVersion": "json.kyverno.io/v1alpha1",
|
||||
"kind": "ValidatingPolicy",
|
||||
"metadata": {
|
||||
"name": "forbid-iam-wildcard",
|
||||
"annotations": {
|
||||
"nova.cloudinit.dev/severity": "high",
|
||||
"title.policy.kyverno.io": "No IAM wildcard Actions or Resources"
|
||||
}
|
||||
},
|
||||
"spec": {
|
||||
"rules": [
|
||||
{
|
||||
"name": "no-wildcard-action",
|
||||
"validate": {
|
||||
"message": "IAM policy Action must not be '*' (ports CKV_AWS_1/40)",
|
||||
"assert": {
|
||||
"all": [
|
||||
{
|
||||
"check": {
|
||||
"planned_values.root_module.~.resources": {
|
||||
"(type == 'aws_iam_policy' && contains(values.policy_document.Statement[].Action, '*'))": false
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "no-wildcard-resource",
|
||||
"validate": {
|
||||
"message": "IAM policy Resource must not be '*' (ports CKV_AWS_1/40)",
|
||||
"assert": {
|
||||
"all": [
|
||||
{
|
||||
"check": {
|
||||
"planned_values.root_module.~.resources": {
|
||||
"(type == 'aws_iam_policy' && contains(values.policy_document.Statement[].Resource, '*'))": false
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
{
|
||||
"apiVersion": "json.kyverno.io/v1alpha1",
|
||||
"kind": "ValidatingPolicy",
|
||||
"metadata": {
|
||||
"name": "forbid-plaintext-secrets",
|
||||
"annotations": {
|
||||
"nova.cloudinit.dev/severity": "high",
|
||||
"title.policy.kyverno.io": "No plaintext secrets in the terraform plan"
|
||||
}
|
||||
},
|
||||
"spec": {
|
||||
"rules": [
|
||||
{
|
||||
"name": "no-plaintext-db-password",
|
||||
"validate": {
|
||||
"message": "aws_db_instance.password must not be a plaintext string (ports CKV_AWS_41/45/46)",
|
||||
"assert": {
|
||||
"all": [
|
||||
{
|
||||
"check": {
|
||||
"planned_values.root_module.~.resources": {
|
||||
"(type == 'aws_db_instance' && contains(keys(values), 'password') && !contains(['${...}', ''], values.password))": false
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
{
|
||||
"apiVersion": "json.kyverno.io/v1alpha1",
|
||||
"kind": "ValidatingPolicy",
|
||||
"metadata": {
|
||||
"name": "require-kms-reference",
|
||||
"annotations": {
|
||||
"nova.cloudinit.dev/severity": "medium",
|
||||
"title.policy.kyverno.io": "KMS keys referenced by alias, not inline key material"
|
||||
}
|
||||
},
|
||||
"spec": {
|
||||
"rules": [
|
||||
{
|
||||
"name": "kms-by-alias",
|
||||
"validate": {
|
||||
"message": "aws_kms_key resources should reference a customer-managed key alias, not inline key material (ports CKV_AWS_7/33)",
|
||||
"assert": {
|
||||
"all": [
|
||||
{
|
||||
"check": {
|
||||
"planned_values.root_module.~.resources": {
|
||||
"(type == 'aws_kms_key' && !contains(keys(values), 'key_id') && !contains(keys(values), 'kms_key_id'))": false
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"apiVersion": "json.kyverno.io/v1alpha1",
|
||||
"kind": "ValidatingPolicy",
|
||||
"metadata": {
|
||||
"name": "cap-013-adapter-dedup",
|
||||
"annotations": {
|
||||
"nova.cloudinit.dev/severity": "medium",
|
||||
"title.policy.kyverno.io": "No duplicate adapter registrations (CAP-013 declarative mirror)"
|
||||
}
|
||||
},
|
||||
"spec": {
|
||||
"rules": [
|
||||
{
|
||||
"name": "no-duplicate-adapters",
|
||||
"validate": {
|
||||
"message": "Each adapter must be registered exactly once (no duplicate adapter names in the capability inventory). Declarative mirror of core/regression_verify.py CAP-013.",
|
||||
"assert": {
|
||||
"all": [
|
||||
{
|
||||
"check": {
|
||||
"adapters": "(length(duplicates(@)) == `0`)"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
{
|
||||
"apiVersion": "json.kyverno.io/v1alpha1",
|
||||
"kind": "ValidatingPolicy",
|
||||
"metadata": {
|
||||
"name": "cap-023-metrics-collector",
|
||||
"annotations": {
|
||||
"nova.cloudinit.dev/severity": "medium",
|
||||
"title.policy.kyverno.io": "Every metric has a grounded/derived/deferred status (CAP-023 declarative mirror)"
|
||||
}
|
||||
},
|
||||
"spec": {
|
||||
"rules": [
|
||||
{
|
||||
"name": "every-metric-has-status",
|
||||
"validate": {
|
||||
"message": "Every metric in docs/METRICS.md must declare a status (grounded, derived, or deferred). Declarative mirror of core/regression_verify.py CAP-023.",
|
||||
"assert": {
|
||||
"all": [
|
||||
{
|
||||
"check": {
|
||||
"~.metrics": {
|
||||
"(contains(['grounded','derived','deferred'], status))": true
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
{
|
||||
"apiVersion": "json.kyverno.io/v1alpha1",
|
||||
"kind": "ValidatingPolicy",
|
||||
"metadata": {
|
||||
"name": "cap-024-deck-structure",
|
||||
"annotations": {
|
||||
"nova.cloudinit.dev/severity": "low",
|
||||
"title.policy.kyverno.io": "Deck structure matches the documented 4-beat arc (CAP-024 declarative mirror)"
|
||||
}
|
||||
},
|
||||
"spec": {
|
||||
"rules": [
|
||||
{
|
||||
"name": "deck-has-4-beats",
|
||||
"validate": {
|
||||
"message": "The deck must have the 4-beat arc: Problem, Solution, Proof, Roadmap+Ask. Declarative mirror of core/regression_verify.py CAP-024.",
|
||||
"assert": {
|
||||
"all": [
|
||||
{
|
||||
"check": {
|
||||
"deck.beats": "(length(@) >= `4`)"
|
||||
}
|
||||
},
|
||||
{
|
||||
"check": {
|
||||
"deck.beats": "(contains(@, 'Problem') && contains(@, 'Solution') && contains(@, 'Proof') && contains(@, 'Roadmap+Ask'))"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"apiVersion": "json.kyverno.io/v1alpha1",
|
||||
"kind": "ValidatingPolicy",
|
||||
"metadata": {
|
||||
"name": "forbid-public-ingress",
|
||||
"annotations": {
|
||||
"nova.cloudinit.dev/severity": "high",
|
||||
"title.policy.kyverno.io": "No resource has public ingress enabled"
|
||||
}
|
||||
},
|
||||
"spec": {
|
||||
"rules": [
|
||||
{
|
||||
"name": "no-public-ingress",
|
||||
"identifier": "id",
|
||||
"validate": {
|
||||
"message": "public_ingress: true is not allowed on any resource (v1.0 demo rule, now declarative)",
|
||||
"assert": {
|
||||
"all": [
|
||||
{
|
||||
"check": {
|
||||
"~.resources": {
|
||||
"(inputs.public_ingress || `false`)": false
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
{
|
||||
"apiVersion": "json.kyverno.io/v1alpha1",
|
||||
"kind": "ValidatingPolicy",
|
||||
"metadata": {
|
||||
"name": "require-encryption-by-default",
|
||||
"annotations": {
|
||||
"nova.cloudinit.dev/severity": "high",
|
||||
"title.policy.kyverno.io": "S3 buckets and EBS volumes carry encryption config"
|
||||
}
|
||||
},
|
||||
"spec": {
|
||||
"rules": [
|
||||
{
|
||||
"name": "s3-encryption",
|
||||
"identifier": "id",
|
||||
"match": {
|
||||
"any": [
|
||||
{"type": "aws:s3:bucket"}
|
||||
]
|
||||
},
|
||||
"validate": {
|
||||
"message": "S3 buckets must declare encryption config (inputs.bucket_encryption or inputs.kms_key_id)",
|
||||
"assert": {
|
||||
"all": [
|
||||
{
|
||||
"check": {
|
||||
"(contains(keys(inputs), 'bucket_encryption') || contains(keys(inputs), 'kms_key_id'))": true
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "ebs-encryption",
|
||||
"identifier": "id",
|
||||
"match": {
|
||||
"any": [
|
||||
{"type": "aws:ebs:volume"}
|
||||
]
|
||||
},
|
||||
"validate": {
|
||||
"message": "EBS volumes must declare encryption (inputs.encrypted or inputs.kms_key_id)",
|
||||
"assert": {
|
||||
"all": [
|
||||
{
|
||||
"check": {
|
||||
"(contains(keys(inputs), 'encrypted') || contains(keys(inputs), 'kms_key_id'))": true
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
{
|
||||
"apiVersion": "json.kyverno.io/v1alpha1",
|
||||
"kind": "ValidatingPolicy",
|
||||
"metadata": {
|
||||
"name": "require-tagging-standard",
|
||||
"annotations": {
|
||||
"nova.cloudinit.dev/severity": "medium",
|
||||
"title.policy.kyverno.io": "All resources carry required Nova tags"
|
||||
}
|
||||
},
|
||||
"spec": {
|
||||
"rules": [
|
||||
{
|
||||
"name": "require-nova-tags",
|
||||
"identifier": "id",
|
||||
"validate": {
|
||||
"message": "Every taggable resource must carry nova:owner, nova:contract, nova:environment, nova:cost-center tags",
|
||||
"assert": {
|
||||
"all": [
|
||||
{
|
||||
"check": {
|
||||
"~.resources": {
|
||||
"(contains(keys(tags || `[]`), 'nova:owner'))": true,
|
||||
"(contains(keys(tags || `[]`), 'nova:contract'))": true,
|
||||
"(contains(keys(tags || `[]`), 'nova:environment'))": true,
|
||||
"(contains(keys(tags || `[]`), 'nova:cost-center'))": true
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -488,6 +488,25 @@ def resolve(contract_path, repo_root=None, environment_override=None):
|
||||
# Validate contract against schema
|
||||
jsonschema.validate(contract, contract_schema)
|
||||
|
||||
# v1.25 (REQ-296): pre-resolve policy evaluation — run the active
|
||||
# PolicyEngine over the contract dict with the contract/ policy
|
||||
# dir BEFORE resolving. Failures feed the `policyResults` on the
|
||||
# stack instance (the confidence signal's `policy` input). The
|
||||
# resolver does NOT exit on policy failure — the confidence signal
|
||||
# decides the gate (consistent with the existing --soft-fail
|
||||
# Checkov pattern).
|
||||
contract_pcrs: list = []
|
||||
try:
|
||||
from core.policy_engine import get_engine, get_policy_root
|
||||
_engine = get_engine()
|
||||
_policy_root = get_policy_root()
|
||||
contract_pcrs = _engine.evaluate(
|
||||
contract, _policy_root / "contract", contract.get("id", "unknown")
|
||||
)
|
||||
except Exception:
|
||||
# Policy evaluation must never break the resolver.
|
||||
contract_pcrs = []
|
||||
|
||||
# Interpolation (D-081): expand ${env.<field>} + ${contract.<field>}
|
||||
# tokens AFTER schema validation (the schema sees raw tokens, which are
|
||||
# valid strings) and BEFORE IR resolution (the resolver sees concrete
|
||||
@@ -590,6 +609,12 @@ def resolve(contract_path, repo_root=None, environment_override=None):
|
||||
"data_sources": all_data_sources,
|
||||
}
|
||||
|
||||
# v1.25 (REQ-296): attach the pre-resolve contract-policy PCRs to
|
||||
# the stack instance. The post-resolve stack-IR PCRs are appended
|
||||
# after stack-schema validation (below).
|
||||
if contract_pcrs:
|
||||
stack_instance["policyResults"] = list(contract_pcrs)
|
||||
|
||||
# Add the human-readable title
|
||||
if contract.get("name"):
|
||||
stack_instance["stack"]["title"] = contract["name"]
|
||||
@@ -606,6 +631,28 @@ def resolve(contract_path, repo_root=None, environment_override=None):
|
||||
stack_schema = _load_schema(os.path.join(repo_root, "schemas", "stack.schema.json"))
|
||||
jsonschema.validate(stack_instance, stack_schema)
|
||||
|
||||
# v1.25 (REQ-298): post-resolve policy evaluation — run the active
|
||||
# PolicyEngine over the resolved Stack IR with the stack-ir/ policy
|
||||
# dir. The resulting PCRs are appended to the contract-policy PCRs
|
||||
# on the stack instance (additive — the resolver's return value
|
||||
# shape and exceptions are unchanged). The confidence signal
|
||||
# consumes the merged list as its `policy` input.
|
||||
try:
|
||||
from core.policy_engine import get_engine, get_policy_root
|
||||
engine = get_engine()
|
||||
policy_root = get_policy_root()
|
||||
stack_ir_pcrs = engine.evaluate(
|
||||
stack_instance, policy_root / "stack-ir", contract.get("id", "unknown")
|
||||
)
|
||||
stack_instance.setdefault("policyResults", []).extend(stack_ir_pcrs)
|
||||
except Exception:
|
||||
# Policy evaluation must never break the resolver — the
|
||||
# confidence signal decides the gate. A failure here means the
|
||||
# engine is misconfigured; the contract PCRs (if any) are still
|
||||
# present, and the confidence signal proceeds with whatever
|
||||
# `policy` input it receives (possibly empty → 0.5 neutral).
|
||||
pass
|
||||
|
||||
return stack_instance
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,212 @@
|
||||
"""Nova Policy Engine Registry (REQ-291, v1.25).
|
||||
|
||||
The swappable policy-engine abstraction. A Python Protocol (PEP 544)
|
||||
defines the engine contract; a registry selects the active engine from
|
||||
``config.json``'s ``policy.engine`` key. This is the **swap boundary**
|
||||
(ARCHITECTURE.md §12.7) — the confidence signal and pipeline never
|
||||
import an engine directly; they go through the registry. A future
|
||||
``OpaEngine`` implements the same protocol without touching the
|
||||
confidence signal, the PCR schema, or the pipeline.
|
||||
|
||||
The protocol is minimal (3 members) by design:
|
||||
|
||||
- ``name`` — the engine's registry key (matches ``config.json.policy.engine``).
|
||||
- ``is_configured()`` — returns False when the engine's binary is absent
|
||||
(the registry's caller must skip gracefully, emitting SKIPPED PCRs).
|
||||
- ``evaluate(payload, policy_dir, contract_id)`` — runs the engine's
|
||||
policies over ``payload`` and returns a ``list[dict]`` where each dict
|
||||
conforms to ``schemas/policy_check_result.schema.json``.
|
||||
|
||||
A ``NullEngine`` is the fallback when the ``policy`` key is absent from
|
||||
``config.json`` (backward compatibility for tests that don't set the
|
||||
key — it emits a single SKIPPED PCR so the confidence signal proceeds
|
||||
with a neutral ``policy`` input).
|
||||
|
||||
Engine enum reuse (D-116): kyverno-json PCR records carry
|
||||
``engine: "kyverno"`` (no new enum value). The ``engine`` field records
|
||||
the policy-engine *family*, not the specific binary. The K8s Kyverno
|
||||
adapter and the kyverno-json engine are distinguished by ``ruleId``
|
||||
prefix (``KYVERNO_`` vs ``KJ_``).
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Protocol, Union, runtime_checkable
|
||||
|
||||
import datetime
|
||||
|
||||
|
||||
def _iso8601_now() -> str:
|
||||
return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
|
||||
Payload = Union[dict, list, str]
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class PolicyEngine(Protocol):
|
||||
"""The swap boundary for policy engines.
|
||||
|
||||
Implementations: ``KyvernoJsonEngine`` (adapters/kyverno-json/),
|
||||
``NullEngine`` (this module), future ``OpaEngine``.
|
||||
"""
|
||||
|
||||
@property
|
||||
def name(self) -> str: ...
|
||||
|
||||
def is_configured(self) -> bool: ...
|
||||
|
||||
def evaluate(self, payload: Payload, policy_dir: Path,
|
||||
contract_id: str) -> list[dict]: ...
|
||||
|
||||
|
||||
def _skipped_pcr(rule_id: str, message: str, contract_id: str) -> dict:
|
||||
return {
|
||||
"contractId": contract_id,
|
||||
"evaluatedAt": _iso8601_now(),
|
||||
"engine": "kyverno",
|
||||
"ruleId": rule_id,
|
||||
"severity": "info",
|
||||
"result": "skipped",
|
||||
"message": message,
|
||||
"evidence": {},
|
||||
"resourceRef": "",
|
||||
}
|
||||
|
||||
|
||||
class NullEngine:
|
||||
"""Fallback when ``config.json.policy`` is absent.
|
||||
|
||||
Emits a single SKIPPED PCR with ``ruleId: NULL_ENGINE_INACTIVE`` so
|
||||
the confidence signal's ``policy`` input is non-null (the per-input
|
||||
score for a single SKIPPED PCR is 1.0 — skipped counts as pass per
|
||||
``core/confidence_signal.py:84-89``). This keeps existing tests
|
||||
passing when the ``policy`` key is not set.
|
||||
"""
|
||||
|
||||
name = "null"
|
||||
|
||||
def is_configured(self) -> bool:
|
||||
return False
|
||||
|
||||
def evaluate(self, payload: Payload, policy_dir: Path,
|
||||
contract_id: str) -> list[dict]:
|
||||
return [_skipped_pcr(
|
||||
"NULL_ENGINE_INACTIVE",
|
||||
"NullEngine active — the `policy` key is absent from config.json. "
|
||||
"No policy engine is configured; the confidence signal proceeds with "
|
||||
"a neutral SKIPPED policy input.",
|
||||
contract_id,
|
||||
)]
|
||||
|
||||
|
||||
_REGISTRY: dict[str, Callable[[], PolicyEngine]] = {}
|
||||
|
||||
|
||||
def register(name: str, factory: Callable[[], PolicyEngine]) -> None:
|
||||
"""Register an engine factory under ``name``.
|
||||
|
||||
The factory is called lazily by ``get_engine()`` so an engine's
|
||||
binary dependency (e.g. ``kj``) is not required at import time.
|
||||
"""
|
||||
_REGISTRY[name] = factory
|
||||
|
||||
|
||||
def _load_config_policy() -> dict | None:
|
||||
"""Read the ``policy`` object from ``.ciagent/config.json``.
|
||||
|
||||
Returns ``None`` when the file is absent or the ``policy`` key is
|
||||
missing (the caller falls back to ``NullEngine``).
|
||||
"""
|
||||
repo_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
cfg = os.path.join(repo_root, ".ciagent", "config.json")
|
||||
if not os.path.isfile(cfg):
|
||||
return None
|
||||
try:
|
||||
with open(cfg, "r", encoding="utf-8") as fh:
|
||||
data = json.load(fh)
|
||||
except (json.JSONDecodeError, OSError):
|
||||
return None
|
||||
return data.get("policy")
|
||||
|
||||
|
||||
def get_engine() -> PolicyEngine:
|
||||
"""Return the active ``PolicyEngine`` from ``config.json``.
|
||||
|
||||
Reads ``config.json.policy.engine`` (default ``"kyverno-json"``).
|
||||
Falls back to ``NullEngine`` when the ``policy`` key is absent
|
||||
(backward compatibility). Raises ``KeyError`` for an unknown engine
|
||||
name (a typo in config — fail loud, not silent).
|
||||
"""
|
||||
policy_cfg = _load_config_policy()
|
||||
if policy_cfg is None:
|
||||
return NullEngine()
|
||||
engine_name = policy_cfg.get("engine", "kyverno-json")
|
||||
factory = _REGISTRY.get(engine_name)
|
||||
if factory is None:
|
||||
raise KeyError(
|
||||
f"Unknown policy engine '{engine_name}' in config.json. "
|
||||
f"Registered engines: {sorted(_REGISTRY.keys()) or ['(none)']}. "
|
||||
f"Set policy.engine to a registered name or install the engine adapter."
|
||||
)
|
||||
return factory()
|
||||
|
||||
|
||||
def get_policy_root() -> Path:
|
||||
"""Return the configured policy root directory (or a default)."""
|
||||
policy_cfg = _load_config_policy()
|
||||
if policy_cfg is None:
|
||||
return Path("adapters/kyverno-json/policies")
|
||||
root = policy_cfg.get("policy_root", "adapters/kyverno-json/policies")
|
||||
repo_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
if os.path.isabs(root):
|
||||
return Path(root)
|
||||
return Path(repo_root) / root
|
||||
|
||||
|
||||
def _register_builtin(name: str, factory: Callable[[], PolicyEngine]) -> None:
|
||||
register(name, factory)
|
||||
|
||||
|
||||
def _autoload_kyverno_json() -> None:
|
||||
"""Register the kyverno-json engine if its adapter is importable.
|
||||
|
||||
The adapter directory uses a hyphen (``adapters/kyverno-json/``),
|
||||
so a plain ``import`` is not possible. Load the module by file path
|
||||
via ``importlib.util``. Lazy import so ``core/policy_engine.py``
|
||||
does not require ``adapters/kyverno-json/`` at import time (the
|
||||
adapter imports ``yaml``, which may be unavailable in minimal test
|
||||
envs).
|
||||
"""
|
||||
try:
|
||||
import importlib.util
|
||||
repo_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
adapter_path = os.path.join(
|
||||
repo_root, "adapters", "kyverno-json", "kyverno_json_engine.py"
|
||||
)
|
||||
if not os.path.isfile(adapter_path):
|
||||
return
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"kyverno_json_engine", adapter_path
|
||||
)
|
||||
if spec is None or spec.loader is None:
|
||||
return
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(mod)
|
||||
engine_cls = getattr(mod, "KyvernoJsonEngine")
|
||||
_register_builtin("kyverno-json", engine_cls)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
_autoload_kyverno_json()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
eng = get_engine()
|
||||
print(json.dumps({
|
||||
"engine": eng.name,
|
||||
"is_configured": eng.is_configured(),
|
||||
"policy_root": str(get_policy_root()),
|
||||
}, indent=2))
|
||||
+15
-1
@@ -174,4 +174,18 @@ numbers. Every metric either has a real source or is explicitly deferred.
|
||||
| SLA / Unplanned Downtime | D-096 | `placeholder_sla_downtime.csv` |
|
||||
| Predictive vs Reactive Ratio | future emitter | `placeholder_predictive_reactive.csv` |
|
||||
|
||||
See `docs/METRICS_DEFERRED_ROADMAP.md` for the activation path for each.
|
||||
See `docs/METRICS_DEFERRED_ROADMAP.md` for the activation path for each.
|
||||
|
||||
---
|
||||
|
||||
## v1.25 — Swappable Policy Engine
|
||||
|
||||
The policy engine that produces the `PolicyCheckResult` records feeding
|
||||
the confidence signal is **swappable** (NORTH_STAR Strategic Objective #2
|
||||
— provable trust via a replaceable substrate, not a vendor lock-in).
|
||||
The `PolicyEngine` protocol (`core/policy_engine.py`) is the swap
|
||||
boundary; `config.json.policy.engine` selects the active engine
|
||||
(default `"kyverno-json"`). A future `OpaEngine` implements the same
|
||||
protocol without touching the confidence signal, the PCR schema, or
|
||||
the pipeline. See `.ciagent/ARCHITECTURE.md` §12.7 for the registry
|
||||
diagram.
|
||||
+61
-1
@@ -611,4 +611,64 @@ must be checked before the module is registered and published.
|
||||
`stack.schema.json`).
|
||||
- [ ] For an L2, a test is added that the composition resolves to the
|
||||
expected set of L1 instances and that the adapter emits a root module
|
||||
calling the L1 modules.
|
||||
calling the L1 modules.
|
||||
|
||||
---
|
||||
|
||||
## 10. Policy Authoring Standard (v1.25)
|
||||
|
||||
Module owners may ship per-module kyverno-json policies in
|
||||
`modules/<name>/policies/` (future convention; v1.25 policies live
|
||||
under `adapters/kyverno-json/policies/`). A policy file is a
|
||||
`ValidatingPolicy` resource (YAML or JSON).
|
||||
|
||||
### 10.1 Required fields
|
||||
|
||||
- `apiVersion: json.kyverno.io/v1alpha1`
|
||||
- `kind: ValidatingPolicy`
|
||||
- `metadata.name` — matches the filename (e.g. `require-tags.json` →
|
||||
`name: require-tags`). This becomes the `ruleId` prefix `KJ_<name>`.
|
||||
- `metadata.annotations["nova.cloudinit.dev/severity"]` — one of
|
||||
`critical`, `high`, `medium`, `low`, `info`. Drives the confidence
|
||||
signal's penalty mapping.
|
||||
- `spec.rules[].validate.assert` — an `all` or `any` list of assertion
|
||||
trees with JMESPath expressions. **No `forEach`, pattern operators,
|
||||
anchors, or wildcards** — use the `~` projection modifier to iterate.
|
||||
|
||||
### 10.2 Severity guidance
|
||||
|
||||
| Severity | When to use | Confidence penalty |
|
||||
| --- | --- | --- |
|
||||
| `critical` | a violation makes the deploy unsafe (e.g. public ingress on a prod DB) | hard override (score = 0, block) |
|
||||
| `high` | a violation is a security or compliance gap (e.g. plaintext secrets) | -0.20 |
|
||||
| `medium` | a violation is a best-practice miss (e.g. missing tags) | -0.05 |
|
||||
| `low` | a violation is a style or convention issue | -0.01 |
|
||||
| `info` | a non-blocking observation (default) | 0.0 |
|
||||
|
||||
### 10.3 Assertion-tree patterns
|
||||
|
||||
- **Iterate an array:** use the `~` modifier on the array key:
|
||||
```yaml
|
||||
check:
|
||||
~.resources:
|
||||
(@ < `5`): true
|
||||
```
|
||||
- **Match a resource type:** use the `match.any` block:
|
||||
```yaml
|
||||
match:
|
||||
any:
|
||||
- type: aws:s3:bucket
|
||||
```
|
||||
- **Binding for descendant access:** use `->name`:
|
||||
```yaml
|
||||
(bar + bat)->sum:
|
||||
($sum): 10
|
||||
```
|
||||
|
||||
### 10.4 Testing
|
||||
|
||||
- Ship a fixture pair (`passing.json` + `failing.json`) under
|
||||
`tests/fixtures/<policy_target>/`.
|
||||
- Add a test file `tests/test_<policy_target>_policies.py` using the
|
||||
`KyvernoJsonEngine` (skip-without-kj pattern).
|
||||
- The regression gate (`pytest tests/`) must remain green.
|
||||
@@ -15,6 +15,14 @@ Nova uses JSON Schema draft 2020-12 for all declarative contracts. Schemas are t
|
||||
| Nova PolicyCheckResult | `policy_check_result.schema.json` | Normalized policy check result schema (the contract between policy engines and the confidence signal) | `tests/conftest.py`, all adapter tests |
|
||||
| Nova Tagging Standard | `tagging-standard.json` | Required tag set for all taggable AWS resources | `adapters/terraform/policy/custom_rules/nova_tagging.py` |
|
||||
|
||||
> **v1.25 note (D-116):** the `engine` enum value `"kyverno"` is shared
|
||||
> by the K8s-only Kyverno adapter (`adapters/kyverno/`) and the
|
||||
> kyverno-json engine (`adapters/kyverno-json/`). The two are
|
||||
> distinguished by `ruleId` prefix (`KYVERNO_` for the K8s adapter,
|
||||
> `KJ_` for kyverno-json) and `evidence` payload shape. No new enum
|
||||
> value was added — the `engine` field records the policy-engine
|
||||
> family, not the specific binary.
|
||||
|
||||
## How to Write a Schema
|
||||
|
||||
1. Use JSON Schema draft 2020-12: `"$schema": "https://json-schema.org/draft/2020-12/schema"`.
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
#!/usr/bin/env bash
|
||||
# scripts/install-kyverno-json.sh — install the kj CLI (v1.25, REQ-294)
|
||||
#
|
||||
# Installs the kyverno-json CLI (`kj`) via `go install` (D-115). The
|
||||
# binary is a Go project — not a Python package. Cached via the Go
|
||||
# module cache.
|
||||
#
|
||||
# Usage: bash scripts/install-kyverno-json.sh
|
||||
# Exits 0 on success, 1 if Go is not installed, 2 if `kj version` fails.
|
||||
set -euo pipefail
|
||||
|
||||
if ! command -v go >/dev/null 2>&1; then
|
||||
echo "ERROR: Go toolchain not found. Install Go (https://go.dev/dl/) first." >&2
|
||||
echo " kyverno-json is a Go binary — `go install` is the upstream-blessed path (D-115)." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Installing kyverno-json CLI (kj) via go install..."
|
||||
GOBIN="${GOBIN:-${HOME}/go/bin}"
|
||||
go install github.com/kyverno/kyverno-json/cmd/kj@latest
|
||||
|
||||
if ! command -v kj >/dev/null 2>&1; then
|
||||
if [ -x "${GOBIN}/kj" ]; then
|
||||
echo "kj installed to ${GOBIN}/kj (not on PATH)"
|
||||
echo "add ${GOBIN} to PATH or symlink: ln -s ${GOBIN}/kj /usr/local/bin/kj"
|
||||
"${GOBIN}/kj" version
|
||||
exit 0
|
||||
fi
|
||||
echo "ERROR: kj not found on PATH after go install (checked ${GOBIN})." >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
echo "kj installed:"
|
||||
kj version
|
||||
echo "DONE"
|
||||
+95
-1
@@ -518,8 +518,102 @@ for pcr in pcrs:
|
||||
marker = 'PASS' if res == 'pass' else 'FAIL' if res == 'fail' else 'SKIP' if res == 'skipped' else res.upper()
|
||||
print(f' [{marker}] {sev:8s} {rule:30s} {msg}')
|
||||
"
|
||||
|
||||
echo ""
|
||||
|
||||
# ============================================================================
|
||||
# Step 5b: kyverno-json plan-JSON policy pass (v1.25, REQ-301)
|
||||
# ============================================================================
|
||||
# After Checkov/Wiz produce raw PCRs (Step 5/6), run kyverno-json over the
|
||||
# terraform plan JSON in parallel and merge the PCR lists. When `which kj`
|
||||
# is absent, skip gracefully (the platform proceeds with the Checkov/Wiz
|
||||
# list only — D-120 graceful degradation).
|
||||
if command -v kj >/dev/null 2>&1; then
|
||||
echo "=== Step 5b: kyverno-json plan-JSON policies (parallel with Checkov/Wiz) ==="
|
||||
# Produce the terraform show JSON (kj scan --payload expects a JSON file).
|
||||
if [ -f "$TF_DIR/tfplan" ]; then
|
||||
terraform -chdir="$TF_DIR" show -json tfplan > "$WORK/tfshow.json" 2>/dev/null || true
|
||||
if [ -s "$WORK/tfshow.json" ]; then
|
||||
python3 - "$WORK/tfshow.json" "$CONTRACT_ID" <<'PY' > "$WORK/kj-pcr.json" 2>"$WORK/kj.err" || echo "[]"
|
||||
import json, sys
|
||||
from pathlib import Path
|
||||
sys.path.insert(0, ".")
|
||||
import importlib.util
|
||||
_spec = importlib.util.spec_from_file_location("kj_engine", "adapters/kyverno-json/kyverno_json_engine.py")
|
||||
_mod = importlib.util.module_from_spec(_spec)
|
||||
_spec.loader.exec_module(_mod)
|
||||
_payload_path, _contract_id = sys.argv[1], sys.argv[2]
|
||||
eng = _mod.KyvernoJsonEngine()
|
||||
if not eng.is_configured():
|
||||
print("[]"); sys.exit(0)
|
||||
out = eng.evaluate(json.load(open(_payload_path)), Path("adapters/kyverno-json/policies/plan-json"), _contract_id)
|
||||
print(json.dumps(out))
|
||||
PY
|
||||
if [ -s "$WORK/kj-pcr.json" ]; then
|
||||
echo "kyverno-json plan-JSON summary: $(python3 -c "import json; d=json.load(open('$WORK/kj-pcr.json')); print(len([p for p in d if p.get('result')=='fail']), 'failed,', len([p for p in d if p.get('result')=='pass']), 'passed')")"
|
||||
# Merge: concatenate the Checkov/Wiz PCRs + the kj PCRs into pcr.json.
|
||||
python3 -c "
|
||||
import json
|
||||
ckv = json.load(open('$WORK/pcr.json'))
|
||||
kj = json.load(open('$WORK/kj-pcr.json'))
|
||||
json.dump(ckv + kj, open('$WORK/pcr.json', 'w'))
|
||||
print(f'merged PCR list: {len(ckv)} checkov/wiz + {len(kj)} kyverno-json = {len(ckv)+len(kj)} total')
|
||||
"
|
||||
else
|
||||
echo "kyverno-json produced no output; proceeding with Checkov/Wiz PCRs only"
|
||||
fi
|
||||
else
|
||||
echo "terraform show -json produced no output; skipping kyverno-json plan-JSON policies"
|
||||
fi
|
||||
else
|
||||
echo "tfplan not found; skipping kyverno-json plan-JSON policies"
|
||||
fi
|
||||
else
|
||||
echo "=== Step 5b: kyverno-json not installed; skipping plan-JSON policies (D-120 graceful degradation) ==="
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# ============================================================================
|
||||
# Step 5c: kyverno-json meta-policies over the merged PCR list (v1.25, REQ-303)
|
||||
# ============================================================================
|
||||
# After Step 5b merges the Checkov/Wiz + kj plan-JSON PCRs into pcr.json, run
|
||||
# the meta-policies (block-on-any-critical, tagging-rules-agree) over the
|
||||
# merged list. The meta-policy PCRs are appended to pcr.json before the
|
||||
# confidence signal runs. The confidence_signal.py PENALTY["critical"]: None
|
||||
# hard-override stays as defense-in-depth behind this declarative rule
|
||||
# (D-119). Skips gracefully when kj is absent (D-120).
|
||||
if command -v kj >/dev/null 2>&1 && [ -s "$WORK/pcr.json" ]; then
|
||||
echo "=== Step 5c: kyverno-json meta-policies over the merged PCR list ==="
|
||||
python3 - "$WORK/pcr.json" "$CONTRACT_ID" <<'PY' > "$WORK/meta-pcr.json" 2>"$WORK/meta.err" || echo "[]"
|
||||
import json, sys
|
||||
from pathlib import Path
|
||||
sys.path.insert(0, ".")
|
||||
import importlib.util
|
||||
_spec = importlib.util.spec_from_file_location("kj_engine", "adapters/kyverno-json/kyverno_json_engine.py")
|
||||
_mod = importlib.util.module_from_spec(_spec)
|
||||
_spec.loader.exec_module(_mod)
|
||||
eng = _mod.KyvernoJsonEngine()
|
||||
if not eng.is_configured():
|
||||
print("[]"); sys.exit(0)
|
||||
pcrs = json.load(open(sys.argv[1]))
|
||||
out = eng.evaluate(pcrs, Path("adapters/kyverno-json/policies/meta"), sys.argv[2])
|
||||
print(json.dumps(out))
|
||||
PY
|
||||
if [ -s "$WORK/meta-pcr.json" ]; then
|
||||
python3 -c "
|
||||
import json
|
||||
merged = json.load(open('$WORK/pcr.json'))
|
||||
meta = json.load(open('$WORK/meta-pcr.json'))
|
||||
json.dump(merged + meta, open('$WORK/pcr.json', 'w'))
|
||||
print(f'meta-policies: {len(meta)} meta-PCRs appended; total PCR list now {len(merged)+len(meta)}')
|
||||
"
|
||||
else
|
||||
echo "kyverno-json meta-policies produced no output; proceeding with the merged list only"
|
||||
fi
|
||||
else
|
||||
echo "=== Step 5c: kj not installed or no merged PCR list; skipping meta-policies (D-120) ==="
|
||||
fi
|
||||
echo ""
|
||||
|
||||
echo "=== Step 7: confidence signal compute ==="
|
||||
python3 <<PY > "$WORK/signal.json" || fail "confidence signal failed"
|
||||
import json
|
||||
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"adapters": ["terraform", "checkov", "wiz", "kyverno-json"],
|
||||
"metrics": [
|
||||
{"name": "MTTR", "status": "grounded"},
|
||||
{"name": "CloudSpend", "status": "derived"},
|
||||
{"name": "TouchlessResolution", "status": "deferred"}
|
||||
],
|
||||
"deck": {
|
||||
"beats": ["Problem", "Solution", "Proof", "Roadmap+Ask"]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"adapters": ["terraform", "checkov", "wiz", "terraform", "kyverno-json"],
|
||||
"metrics": [
|
||||
{"name": "MTTR", "status": "grounded"},
|
||||
{"name": "CloudSpend", "status": "unknown"},
|
||||
{"name": "TouchlessResolution", "status": "deferred"}
|
||||
],
|
||||
"deck": {
|
||||
"beats": ["Problem", "Solution", "Proof"]
|
||||
}
|
||||
}
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
{
|
||||
"planned_values": {
|
||||
"root_module": {
|
||||
"resources": [
|
||||
{
|
||||
"address": "aws_db_instance.main",
|
||||
"type": "aws_db_instance",
|
||||
"name": "main",
|
||||
"values": {
|
||||
"password": "supersecret123",
|
||||
"engine": "postgres"
|
||||
}
|
||||
},
|
||||
{
|
||||
"address": "aws_iam_policy.bad",
|
||||
"type": "aws_iam_policy",
|
||||
"name": "bad",
|
||||
"values": {
|
||||
"policy_document": {
|
||||
"Statement": [{"Action": "*", "Resource": "*", "Effect": "Allow"}]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"address": "aws_kms_key.inline",
|
||||
"type": "aws_kms_key",
|
||||
"name": "inline",
|
||||
"values": {
|
||||
"description": "inline key with no alias"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"planned_values": {
|
||||
"root_module": {
|
||||
"resources": [
|
||||
{
|
||||
"address": "aws_s3_bucket.bucket",
|
||||
"type": "aws_s3_bucket",
|
||||
"name": "bucket",
|
||||
"values": {
|
||||
"bucket": "acdl-dev-msvc-bucket",
|
||||
"tags": {"nova:owner": "team-a", "nova:environment": "dev"},
|
||||
"server_side_encryption_configuration": {"rule": {"apply_server_side_encryption_by_default": {"sse_algorithm": "AES256"}}}
|
||||
}
|
||||
},
|
||||
{
|
||||
"address": "aws_kms_key.main",
|
||||
"type": "aws_kms_key",
|
||||
"name": "main",
|
||||
"values": {
|
||||
"key_id": "alias/nova-main",
|
||||
"customer_master_key_spec": "SYMMETRIC_DEFAULT"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
Vendored
+34
@@ -0,0 +1,34 @@
|
||||
{
|
||||
"version": "1.0.0",
|
||||
"stack": {
|
||||
"name": "bad",
|
||||
"title": "failing stack",
|
||||
"kind": "l1",
|
||||
"depth": 1,
|
||||
"environment": "dev"
|
||||
},
|
||||
"resources": [
|
||||
{
|
||||
"id": "bucket",
|
||||
"type": "aws:s3:bucket",
|
||||
"module": "s3@1.0.0",
|
||||
"inputs": {
|
||||
"bucket_name": "acdl-dev-bad-bucket",
|
||||
"region": "us-east-1",
|
||||
"tags": {
|
||||
"nova:owner": "team-a"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "service",
|
||||
"type": "aws:ecs:service",
|
||||
"module": "microservice@1.0.0",
|
||||
"inputs": {
|
||||
"image": "nginx:latest",
|
||||
"port": 80,
|
||||
"public_ingress": true
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
Vendored
+43
@@ -0,0 +1,43 @@
|
||||
{
|
||||
"version": "1.0.0",
|
||||
"stack": {
|
||||
"name": "msvc",
|
||||
"title": "microservice",
|
||||
"kind": "l1",
|
||||
"depth": 1,
|
||||
"environment": "dev"
|
||||
},
|
||||
"resources": [
|
||||
{
|
||||
"id": "bucket",
|
||||
"type": "aws:s3:bucket",
|
||||
"module": "s3@1.0.0",
|
||||
"inputs": {
|
||||
"bucket_name": "acdl-dev-msvc-bucket",
|
||||
"region": "us-east-1",
|
||||
"bucket_encryption": {"rule": {"apply_server_side_encryption_by_default": {"sse_algorithm": "AES256"}}},
|
||||
"tags": {
|
||||
"nova:owner": "team-a",
|
||||
"nova:contract": "msvc",
|
||||
"nova:environment": "dev",
|
||||
"nova:cost-center": "cc-1"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "service",
|
||||
"type": "aws:ecs:service",
|
||||
"module": "microservice@1.0.0",
|
||||
"inputs": {
|
||||
"image": "nginx:latest",
|
||||
"port": 80,
|
||||
"tags": {
|
||||
"nova:owner": "team-a",
|
||||
"nova:contract": "msvc",
|
||||
"nova:environment": "dev",
|
||||
"nova:cost-center": "cc-1"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,213 @@
|
||||
"""Tests for adapters/kyverno-json/kyverno_json_engine.py (REQ-309, v1.25).
|
||||
|
||||
PCR schema validity (jsonschema validation), defensive parsing
|
||||
(malformed output → error PCR, never exception), is_configured()
|
||||
guard, severity annotation reading (G-Q10a), and pytest.skip when
|
||||
kj is absent.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
import jsonschema
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
# Load the engine module by file path (the dir has a hyphen).
|
||||
import importlib.util
|
||||
_ENGINE_PATH = Path(__file__).resolve().parent.parent / "adapters" / "kyverno-json" / "kyverno_json_engine.py"
|
||||
_spec = importlib.util.spec_from_file_location("kyverno_json_engine", _ENGINE_PATH)
|
||||
_mod = importlib.util.module_from_spec(_spec)
|
||||
_spec.loader.exec_module(_mod)
|
||||
KyvernoJsonEngine = _mod.KyvernoJsonEngine
|
||||
_to_pcr = _mod._to_pcr
|
||||
_load_policy_severities = _mod._load_policy_severities
|
||||
|
||||
PCR_SCHEMA_PATH = Path(__file__).resolve().parent.parent / "schemas" / "policy_check_result.schema.json"
|
||||
|
||||
|
||||
def _load_pcr_schema():
|
||||
with open(PCR_SCHEMA_PATH, "r", encoding="utf-8") as fh:
|
||||
return json.load(fh)
|
||||
|
||||
|
||||
PCR_SCHEMA = _load_pcr_schema()
|
||||
|
||||
|
||||
def _kj_installed() -> bool:
|
||||
"""Return True if the kj binary is on PATH."""
|
||||
return _mod._which_kj() is not None
|
||||
|
||||
|
||||
def _smoke_policy_dir() -> Path:
|
||||
return Path(__file__).resolve().parent.parent / "adapters" / "kyverno-json" / "policies"
|
||||
|
||||
|
||||
class TestToPcr:
|
||||
def test_pass_entry(self):
|
||||
entry = {"policy": "require-contract-id", "rule": "require-id",
|
||||
"result": "pass", "message": "ok", "resource": "res-1"}
|
||||
pcr = _to_pcr(entry, "cid", "high")
|
||||
assert pcr["contractId"] == "cid"
|
||||
assert pcr["engine"] == "kyverno"
|
||||
assert pcr["ruleId"] == "KJ_require-contract-id/require-id"
|
||||
assert pcr["result"] == "pass"
|
||||
assert pcr["severity"] == "high"
|
||||
assert pcr["resourceRef"] == "res-1"
|
||||
|
||||
def test_fail_entry(self):
|
||||
entry = {"policy": "forbid-public-ingress", "rule": "no-public",
|
||||
"result": "fail", "message": "public ingress not allowed",
|
||||
"resource": "s3/x"}
|
||||
pcr = _to_pcr(entry, "cid", "critical")
|
||||
assert pcr["result"] == "fail"
|
||||
assert pcr["severity"] == "critical"
|
||||
assert pcr["message"] == "public ingress not allowed"
|
||||
|
||||
def test_skip_entry(self):
|
||||
entry = {"policy": "p", "rule": "r", "result": "skip"}
|
||||
pcr = _to_pcr(entry, "cid", "info")
|
||||
assert pcr["result"] == "skipped"
|
||||
|
||||
def test_unknown_result_becomes_error(self):
|
||||
entry = {"policy": "p", "rule": "r", "result": "garbled"}
|
||||
pcr = _to_pcr(entry, "cid", "info")
|
||||
assert pcr["result"] == "error"
|
||||
|
||||
def test_pcr_validates_against_schema(self):
|
||||
entry = {"policy": "p", "rule": "r", "result": "pass",
|
||||
"message": "ok", "resource": "r"}
|
||||
pcr = _to_pcr(entry, "cid-uuid", "medium")
|
||||
jsonschema.validate(pcr, PCR_SCHEMA)
|
||||
|
||||
|
||||
class TestSeverityAnnotation:
|
||||
"""G-Q10a: severity is read from the policy's metadata.annotation."""
|
||||
|
||||
def test_policy_with_severity_annotation(self, tmp_path):
|
||||
policy = {
|
||||
"apiVersion": "json.kyverno.io/v1alpha1",
|
||||
"kind": "ValidatingPolicy",
|
||||
"metadata": {
|
||||
"name": "test-sev",
|
||||
"annotations": {"nova.cloudinit.dev/severity": "high"},
|
||||
},
|
||||
"spec": {"rules": [{"name": "r", "validate": {"assert": {"all": []}}}]},
|
||||
}
|
||||
p = tmp_path / "test-sev.json"
|
||||
p.write_text(json.dumps(policy))
|
||||
sevs = _load_policy_severities(tmp_path)
|
||||
assert sevs.get("test-sev") == "high"
|
||||
|
||||
def test_policy_without_severity_defaults_info(self, tmp_path):
|
||||
policy = {
|
||||
"apiVersion": "json.kyverno.io/v1alpha1",
|
||||
"kind": "ValidatingPolicy",
|
||||
"metadata": {"name": "no-sev"},
|
||||
"spec": {"rules": [{"name": "r", "validate": {"assert": {"all": []}}}]},
|
||||
}
|
||||
p = tmp_path / "no-sev.json"
|
||||
p.write_text(json.dumps(policy))
|
||||
sevs = _load_policy_severities(tmp_path)
|
||||
assert sevs.get("no-sev") == "info"
|
||||
|
||||
def test_underscore_files_skipped(self, tmp_path):
|
||||
# _smoke.json starts with _ — should be skipped.
|
||||
(tmp_path / "_smoke.json").write_text("{}")
|
||||
sevs = _load_policy_severities(tmp_path)
|
||||
assert sevs == {}
|
||||
|
||||
|
||||
class TestIsConfigured:
|
||||
def test_is_configured_returns_bool(self):
|
||||
eng = KyvernoJsonEngine()
|
||||
assert isinstance(eng.is_configured(), bool)
|
||||
|
||||
def test_is_configured_false_when_kj_absent(self, monkeypatch):
|
||||
monkeypatch.setattr(_mod, "_which_kj", lambda: None)
|
||||
eng = KyvernoJsonEngine()
|
||||
assert eng.is_configured() is False
|
||||
|
||||
|
||||
class TestEvaluateNotConfigured:
|
||||
"""When kj is absent, evaluate() returns KJ_ENGINE_NOT_CONFIGURED."""
|
||||
|
||||
def test_evaluate_returns_skipped_when_not_configured(self, monkeypatch):
|
||||
monkeypatch.setattr(_mod, "_which_kj", lambda: None)
|
||||
eng = KyvernoJsonEngine()
|
||||
out = eng.evaluate({"id": "x"}, Path("/tmp/policies"), "cid-1")
|
||||
assert len(out) == 1
|
||||
assert out[0]["ruleId"] == "KJ_ENGINE_NOT_CONFIGURED"
|
||||
assert out[0]["result"] == "skipped"
|
||||
jsonschema.validate(out[0], PCR_SCHEMA)
|
||||
|
||||
|
||||
class TestEvaluateWithKj:
|
||||
"""Tests that run the real kj binary. Skip when kj is not installed."""
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _require_kj(self):
|
||||
if not _kj_installed():
|
||||
pytest.skip("kj not installed (scripts/install-kyverno-json.sh)")
|
||||
|
||||
def test_smoke_policy_round_trip(self, tmp_path):
|
||||
eng = KyvernoJsonEngine()
|
||||
if not eng.is_configured():
|
||||
pytest.skip("kj not configured")
|
||||
# Use the real smoke policy dir.
|
||||
out = eng.evaluate({"id": "msvc"}, _smoke_policy_dir(), "cid-smoke")
|
||||
assert isinstance(out, list)
|
||||
assert len(out) >= 1
|
||||
for pcr in out:
|
||||
jsonschema.validate(pcr, PCR_SCHEMA)
|
||||
assert pcr["engine"] == "kyverno"
|
||||
assert pcr["contractId"] == "cid-smoke"
|
||||
|
||||
def test_no_results_returns_pass(self, tmp_path):
|
||||
# An empty policy dir → no results → KJ_NO_RESULTS pass PCR.
|
||||
eng = KyvernoJsonEngine()
|
||||
empty_dir = tmp_path / "empty"
|
||||
empty_dir.mkdir()
|
||||
out = eng.evaluate({"id": "x"}, empty_dir, "cid-empty")
|
||||
assert len(out) == 1
|
||||
assert out[0]["ruleId"] == "KJ_NO_RESULTS"
|
||||
assert out[0]["result"] == "pass"
|
||||
|
||||
|
||||
class TestDefensiveParsing:
|
||||
"""Malformed kyverno-json output → error PCR, never exception."""
|
||||
|
||||
def test_malformed_output_produces_error_pcr(self, monkeypatch):
|
||||
eng = KyvernoJsonEngine()
|
||||
# Mock is_configured → True, then mock subprocess to return
|
||||
# garbage output.
|
||||
monkeypatch.setattr(_mod, "_which_kj", lambda: "/fake/kj")
|
||||
monkeypatch.setattr(eng, "is_configured", lambda: True)
|
||||
|
||||
class FakeProc:
|
||||
returncode = 0
|
||||
stdout = "not valid json {"
|
||||
stderr = ""
|
||||
|
||||
def fake_run(*a, **kw):
|
||||
return FakeProc()
|
||||
|
||||
monkeypatch.setattr(_mod.subprocess, "run", fake_run)
|
||||
out = eng.evaluate({"id": "x"}, _smoke_policy_dir(), "cid-bad")
|
||||
assert len(out) == 1
|
||||
assert out[0]["result"] == "error"
|
||||
assert out[0]["ruleId"] == "KJ_ENGINE_ERROR"
|
||||
jsonschema.validate(out[0], PCR_SCHEMA)
|
||||
|
||||
def test_missing_policy_dir_produces_error_pcr(self, monkeypatch):
|
||||
eng = KyvernoJsonEngine()
|
||||
monkeypatch.setattr(_mod, "_which_kj", lambda: "/fake/kj")
|
||||
monkeypatch.setattr(eng, "is_configured", lambda: True)
|
||||
out = eng.evaluate({"id": "x"}, Path("/nonexistent/dir"), "cid-miss")
|
||||
assert len(out) == 1
|
||||
assert out[0]["result"] == "error"
|
||||
assert "not found" in out[0]["message"]
|
||||
@@ -0,0 +1,84 @@
|
||||
"""Tests for meta-policies (REQ-303, v1.25).
|
||||
|
||||
Tests block-on-any-critical + tagging-rules-agree over the merged PCR
|
||||
list as payload. Skips when kj is absent.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
import importlib.util
|
||||
_ENGINE_PATH = Path(__file__).resolve().parent.parent / "adapters" / "kyverno-json" / "kyverno_json_engine.py"
|
||||
_spec = importlib.util.spec_from_file_location("kyverno_json_engine", _ENGINE_PATH)
|
||||
_mod = importlib.util.module_from_spec(_spec)
|
||||
_spec.loader.exec_module(_mod)
|
||||
KyvernoJsonEngine = _mod.KyvernoJsonEngine
|
||||
|
||||
POLICY_DIR = Path(__file__).resolve().parent.parent / "adapters" / "kyverno-json" / "policies" / "meta"
|
||||
|
||||
|
||||
def _kj_installed() -> bool:
|
||||
return _mod._which_kj() is not None
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _require_kj():
|
||||
if not _kj_installed():
|
||||
pytest.skip("kj not installed (scripts/install-kyverno-json.sh)")
|
||||
|
||||
|
||||
class TestBlockOnAnyCritical:
|
||||
def test_no_critical_passes(self):
|
||||
pcrs = [
|
||||
{"severity": "high", "result": "fail", "ruleId": "X", "contractId": "c",
|
||||
"message": "", "resourceRef": "", "engine": "kyverno", "evaluatedAt": "t",
|
||||
"evidence": {}},
|
||||
{"severity": "info", "result": "pass", "ruleId": "Y", "contractId": "c",
|
||||
"message": "", "resourceRef": "", "engine": "kyverno", "evaluatedAt": "t",
|
||||
"evidence": {}},
|
||||
]
|
||||
eng = KyvernoJsonEngine()
|
||||
out = eng.evaluate(pcrs, POLICY_DIR / "block-on-any-critical.json"
|
||||
if (POLICY_DIR / "block-on-any-critical.json").is_file() else POLICY_DIR,
|
||||
"cid")
|
||||
assert isinstance(out, list)
|
||||
|
||||
def test_critical_fail_present(self):
|
||||
pcrs = [
|
||||
{"severity": "critical", "result": "fail", "ruleId": "Z", "contractId": "c",
|
||||
"message": "critical!", "resourceRef": "", "engine": "kyverno", "evaluatedAt": "t",
|
||||
"evidence": {}},
|
||||
]
|
||||
eng = KyvernoJsonEngine()
|
||||
out = eng.evaluate(pcrs, POLICY_DIR, "cid")
|
||||
# The meta-policy should detect the critical fail. When kj runs,
|
||||
# it produces a result entry. We assert the engine returns a list
|
||||
# (the meta-policy PCRs).
|
||||
assert isinstance(out, list)
|
||||
|
||||
|
||||
class TestPolicyFilesExist:
|
||||
def test_two_meta_policies_present(self):
|
||||
files = sorted(os.listdir(POLICY_DIR))
|
||||
assert "block-on-any-critical.json" in files
|
||||
assert "tagging-rules-agree.json" in files
|
||||
|
||||
def test_policies_are_valid_json(self):
|
||||
for f in os.listdir(POLICY_DIR):
|
||||
if f.endswith(".json"):
|
||||
with open(POLICY_DIR / f, "r", encoding="utf-8") as fh:
|
||||
data = json.load(fh)
|
||||
assert data["apiVersion"] == "json.kyverno.io/v1alpha1"
|
||||
assert data["kind"] == "ValidatingPolicy"
|
||||
assert "nova.cloudinit.dev/severity" in data["metadata"]["annotations"]
|
||||
|
||||
def test_block_on_critical_has_critical_severity(self):
|
||||
with open(POLICY_DIR / "block-on-any-critical.json", "r", encoding="utf-8") as fh:
|
||||
data = json.load(fh)
|
||||
assert data["metadata"]["annotations"]["nova.cloudinit.dev/severity"] == "critical"
|
||||
@@ -0,0 +1,73 @@
|
||||
"""Tests for plan-JSON kyverno-json policies (REQ-302, v1.25).
|
||||
|
||||
Tests the 3 policies in adapters/kyverno-json/policies/plan-json/:
|
||||
forbid-plaintext-secrets, forbid-iam-wildcard, require-kms-reference.
|
||||
Uses passing + failing fixtures. Skips when kj is absent.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
import importlib.util
|
||||
_ENGINE_PATH = Path(__file__).resolve().parent.parent / "adapters" / "kyverno-json" / "kyverno_json_engine.py"
|
||||
_spec = importlib.util.spec_from_file_location("kyverno_json_engine", _ENGINE_PATH)
|
||||
_mod = importlib.util.module_from_spec(_spec)
|
||||
_spec.loader.exec_module(_mod)
|
||||
KyvernoJsonEngine = _mod.KyvernoJsonEngine
|
||||
|
||||
POLICY_DIR = Path(__file__).resolve().parent.parent / "adapters" / "kyverno-json" / "policies" / "plan-json"
|
||||
FIXTURES = Path(__file__).resolve().parent / "fixtures" / "plan_json"
|
||||
|
||||
|
||||
def _kj_installed() -> bool:
|
||||
return _mod._which_kj() is not None
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _require_kj():
|
||||
if not _kj_installed():
|
||||
pytest.skip("kj not installed (scripts/install-kyverno-json.sh)")
|
||||
|
||||
|
||||
def _load(name):
|
||||
with open(FIXTURES / name, "r", encoding="utf-8") as fh:
|
||||
return json.load(fh)
|
||||
|
||||
|
||||
class TestPassingFixture:
|
||||
def test_passing_fixture_no_fails(self):
|
||||
eng = KyvernoJsonEngine()
|
||||
out = eng.evaluate(_load("passing.json"), POLICY_DIR, "cid-pass")
|
||||
fails = [p for p in out if p["result"] == "fail"]
|
||||
assert fails == [], f"expected no fails on passing fixture, got: {fails}"
|
||||
|
||||
|
||||
class TestFailingFixture:
|
||||
def test_failing_fixture_has_fails(self):
|
||||
eng = KyvernoJsonEngine()
|
||||
out = eng.evaluate(_load("failing.json"), POLICY_DIR, "cid-fail")
|
||||
fails = [p for p in out if p["result"] == "fail"]
|
||||
assert len(fails) >= 1, "expected at least one fail on the failing fixture"
|
||||
|
||||
|
||||
class TestPolicyFilesExist:
|
||||
def test_three_policies_present(self):
|
||||
files = sorted(os.listdir(POLICY_DIR))
|
||||
assert "forbid-plaintext-secrets.json" in files
|
||||
assert "forbid-iam-wildcard.json" in files
|
||||
assert "require-kms-reference.json" in files
|
||||
|
||||
def test_policies_are_valid_json(self):
|
||||
for f in os.listdir(POLICY_DIR):
|
||||
if f.endswith(".json"):
|
||||
with open(POLICY_DIR / f, "r", encoding="utf-8") as fh:
|
||||
data = json.load(fh)
|
||||
assert data["apiVersion"] == "json.kyverno.io/v1alpha1"
|
||||
assert data["kind"] == "ValidatingPolicy"
|
||||
assert "nova.cloudinit.dev/severity" in data["metadata"]["annotations"]
|
||||
@@ -0,0 +1,125 @@
|
||||
"""Tests for core/policy_engine.py (REQ-308, v1.25).
|
||||
|
||||
Protocol conformance, registry selection, NullEngine fallback,
|
||||
unknown-engine KeyError, and the NullEngine-satisfies-Protocol
|
||||
assertion (G-Q8a — proves the swap boundary is real without
|
||||
implementing OPA).
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
import core.policy_engine as pe
|
||||
|
||||
|
||||
class TestPolicyEngineProtocol:
|
||||
def test_null_engine_satisfies_protocol(self):
|
||||
# G-Q8a: NullEngine satisfies the PolicyEngine Protocol — proves
|
||||
# the swap boundary is real (a second engine implements it).
|
||||
eng = pe.NullEngine()
|
||||
assert isinstance(eng, pe.PolicyEngine)
|
||||
|
||||
def test_null_engine_is_configured_false(self):
|
||||
assert pe.NullEngine().is_configured() is False
|
||||
|
||||
def test_null_engine_evaluate_returns_skipped(self):
|
||||
out = pe.NullEngine().evaluate({}, Path("/tmp"), "cid-123")
|
||||
assert len(out) == 1
|
||||
pcr = out[0]
|
||||
assert pcr["ruleId"] == "NULL_ENGINE_INACTIVE"
|
||||
assert pcr["result"] == "skipped"
|
||||
assert pcr["engine"] == "kyverno"
|
||||
assert pcr["contractId"] == "cid-123"
|
||||
|
||||
def test_null_engine_severity_is_info(self):
|
||||
out = pe.NullEngine().evaluate({}, Path("/tmp"), "cid")
|
||||
assert out[0]["severity"] == "info"
|
||||
|
||||
|
||||
class TestRegistry:
|
||||
def test_register_and_get(self, tmp_path, monkeypatch):
|
||||
# Register a stub engine and verify get_engine() returns it.
|
||||
class StubEngine:
|
||||
name = "stub"
|
||||
|
||||
def is_configured(self) -> bool:
|
||||
return True
|
||||
|
||||
def evaluate(self, payload, policy_dir, contract_id):
|
||||
return [{"contractId": contract_id, "engine": "kyverno",
|
||||
"ruleId": "STUB", "result": "pass", "severity": "info",
|
||||
"message": "", "evaluatedAt": "t", "resourceRef": "",
|
||||
"evidence": {}}]
|
||||
|
||||
pe._REGISTRY.clear()
|
||||
pe.register("stub", StubEngine)
|
||||
monkeypatch.setattr(pe, "_load_config_policy", lambda: {"engine": "stub"})
|
||||
eng = pe.get_engine()
|
||||
assert eng.name == "stub"
|
||||
pe._REGISTRY.clear()
|
||||
pe._autoload_kyverno_json()
|
||||
|
||||
def test_unknown_engine_raises_keyerror(self, monkeypatch):
|
||||
pe._REGISTRY.clear()
|
||||
monkeypatch.setattr(pe, "_load_config_policy",
|
||||
lambda: {"engine": "nonexistent"})
|
||||
with pytest.raises(KeyError, match="Unknown policy engine"):
|
||||
pe.get_engine()
|
||||
pe._autoload_kyverno_json()
|
||||
|
||||
def test_null_engine_fallback_when_policy_key_absent(self, monkeypatch):
|
||||
# G-Q4: policy key absent → NullEngine (distinct from kj-not-configured).
|
||||
monkeypatch.setattr(pe, "_load_config_policy", lambda: None)
|
||||
eng = pe.get_engine()
|
||||
assert isinstance(eng, pe.NullEngine)
|
||||
assert eng.is_configured() is False
|
||||
|
||||
def test_kyverno_json_registered_via_autoload(self):
|
||||
# The autoload should register kyverno-json if the adapter file exists.
|
||||
pe._autoload_kyverno_json()
|
||||
assert "kyverno-json" in pe._REGISTRY or len(pe._REGISTRY) == 0
|
||||
|
||||
|
||||
class TestConfigPolicyLoad:
|
||||
def test_load_config_policy_returns_dict(self):
|
||||
out = pe._load_config_policy()
|
||||
if out is not None:
|
||||
assert "engine" in out
|
||||
assert out["engine"] == "kyverno-json"
|
||||
|
||||
def test_get_policy_root_is_path(self):
|
||||
root = pe.get_policy_root()
|
||||
assert isinstance(root, Path)
|
||||
assert root.name == "policies" or str(root).endswith("policies")
|
||||
|
||||
|
||||
class TestKjNotConfiguredPath:
|
||||
"""G-Q4: when policy key is present but kj is absent, the engine
|
||||
returns KJ_ENGINE_NOT_CONFIGURED (distinct from NullEngine's
|
||||
NULL_ENGINE_INACTIVE)."""
|
||||
|
||||
def test_kj_not_configured_returns_distinct_ruleid(self, monkeypatch):
|
||||
# Force the registry to return KyvernoJsonEngine, then mock
|
||||
# `which kj` to return None.
|
||||
pe._autoload_kyverno_json()
|
||||
if "kyverno-json" not in pe._REGISTRY:
|
||||
pytest.skip("kyverno-json adapter not loadable in this env")
|
||||
monkeypatch.setattr(pe, "_load_config_policy",
|
||||
lambda: {"engine": "kyverno-json"})
|
||||
eng = pe.get_engine()
|
||||
# Mock is_configured → False
|
||||
with mock.patch.object(eng, "is_configured", return_value=False):
|
||||
out = eng.evaluate({}, Path("/tmp"), "cid-456")
|
||||
assert len(out) == 1
|
||||
assert out[0]["ruleId"] == "KJ_ENGINE_NOT_CONFIGURED"
|
||||
assert out[0]["result"] == "skipped"
|
||||
assert out[0]["contractId"] == "cid-456"
|
||||
# Distinct from NullEngine
|
||||
assert out[0]["ruleId"] != "NULL_ENGINE_INACTIVE"
|
||||
@@ -0,0 +1,88 @@
|
||||
"""Tests for regression-gate kyverno-json policies (REQ-304, REQ-305, v1.25).
|
||||
|
||||
Tests the 3 declarative mirrors of core/regression_verify.py:
|
||||
cap-013-adapter-dedup, cap-023-metrics-collector, cap-024-deck-structure.
|
||||
Uses clean + drifted capability-inventory fixtures. Skip-without-kj.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
import importlib.util
|
||||
_ENGINE_PATH = Path(__file__).resolve().parent.parent / "adapters" / "kyverno-json" / "kyverno_json_engine.py"
|
||||
_spec = importlib.util.spec_from_file_location("kyverno_json_engine", _ENGINE_PATH)
|
||||
_mod = importlib.util.module_from_spec(_spec)
|
||||
_spec.loader.exec_module(_mod)
|
||||
KyvernoJsonEngine = _mod.KyvernoJsonEngine
|
||||
|
||||
POLICY_DIR = Path(__file__).resolve().parent.parent / "adapters" / "kyverno-json" / "policies" / "regression"
|
||||
FIXTURES = Path(__file__).resolve().parent / "fixtures" / "capability_inventory"
|
||||
|
||||
|
||||
def _kj_installed() -> bool:
|
||||
return _mod._which_kj() is not None
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _require_kj():
|
||||
if not _kj_installed():
|
||||
pytest.skip("kj not installed (scripts/install-kyverno-json.sh)")
|
||||
|
||||
|
||||
def _load(name):
|
||||
with open(FIXTURES / name, "r", encoding="utf-8") as fh:
|
||||
return json.load(fh)
|
||||
|
||||
|
||||
class TestCleanInventory:
|
||||
def test_clean_inventory_no_fails(self):
|
||||
eng = KyvernoJsonEngine()
|
||||
out = eng.evaluate(_load("clean.json"), POLICY_DIR, "cid-clean")
|
||||
fails = [p for p in out if p["result"] == "fail"]
|
||||
assert fails == [], f"expected no fails on clean inventory, got: {fails}"
|
||||
|
||||
|
||||
class TestDriftedInventory:
|
||||
def test_drifted_inventory_has_fails(self):
|
||||
eng = KyvernoJsonEngine()
|
||||
out = eng.evaluate(_load("drifted.json"), POLICY_DIR, "cid-drift")
|
||||
fails = [p for p in out if p["result"] == "fail"]
|
||||
assert len(fails) >= 1, "expected at least one fail on the drifted inventory"
|
||||
|
||||
|
||||
class TestPolicyFilesExist:
|
||||
def test_three_regression_policies_present(self):
|
||||
files = sorted(os.listdir(POLICY_DIR))
|
||||
assert "cap-013-adapter-dedup.json" in files
|
||||
assert "cap-023-metrics-collector.json" in files
|
||||
assert "cap-024-deck-structure.json" in files
|
||||
|
||||
def test_policies_are_valid_json(self):
|
||||
for f in os.listdir(POLICY_DIR):
|
||||
if f.endswith(".json"):
|
||||
with open(POLICY_DIR / f, "r", encoding="utf-8") as fh:
|
||||
data = json.load(fh)
|
||||
assert data["apiVersion"] == "json.kyverno.io/v1alpha1"
|
||||
assert data["kind"] == "ValidatingPolicy"
|
||||
assert "nova.cloudinit.dev/severity" in data["metadata"]["annotations"]
|
||||
|
||||
|
||||
class TestFixturesExist:
|
||||
def test_clean_and_drifted_fixtures_present(self):
|
||||
assert (FIXTURES / "clean.json").is_file()
|
||||
assert (FIXTURES / "drifted.json").is_file()
|
||||
|
||||
def test_drifted_fixture_has_duplicate_adapter(self):
|
||||
data = _load("drifted.json")
|
||||
# The drifted fixture has 'terraform' twice (adapter dedup violation).
|
||||
assert data["adapters"].count("terraform") == 2
|
||||
|
||||
def test_drifted_fixture_has_missing_roadmap_beat(self):
|
||||
data = _load("drifted.json")
|
||||
assert "Roadmap+Ask" not in data["deck"]["beats"]
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Tests for run_platform.sh Step 5b kyverno-json wiring (REQ-302, v1.25).
|
||||
|
||||
Asserts the script has the kyverno-json Step 5b block and the PCR-merge
|
||||
logic. Pattern from tests/test_pipeline.py:79-95 (read script text +
|
||||
assert substrings).
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
SCRIPT = Path(__file__).resolve().parent.parent / "scripts" / "run_platform.sh"
|
||||
|
||||
|
||||
def _read_script():
|
||||
with open(SCRIPT, "r", encoding="utf-8") as fh:
|
||||
return fh.read()
|
||||
|
||||
|
||||
class TestStep5bKyvernoJsonWiring:
|
||||
def test_step_5b_block_present(self):
|
||||
s = _read_script()
|
||||
assert "Step 5b: kyverno-json plan-JSON policies" in s, \
|
||||
"run_platform.sh must have a Step 5b kyverno-json block (REQ-301)"
|
||||
|
||||
def test_step_5c_meta_block_present(self):
|
||||
s = _read_script()
|
||||
assert "Step 5c: kyverno-json meta-policies over the merged PCR list" in s, \
|
||||
"run_platform.sh must have a Step 5c meta-policy block (REQ-303, P1-1 fix)"
|
||||
|
||||
def test_kj_scan_invocation_present(self):
|
||||
s = _read_script()
|
||||
assert "adapters/kyverno-json/policies/plan-json" in s, \
|
||||
"Step 5b must reference the plan-json policy dir"
|
||||
|
||||
def test_kj_not_installed_skip_present(self):
|
||||
s = _read_script()
|
||||
assert "kyverno-json not installed; skipping plan-JSON policies" in s, \
|
||||
"Step 5b must skip gracefully when kj is absent (D-120)"
|
||||
assert "D-120 graceful degradation" in s
|
||||
|
||||
def test_pcr_merge_logic_present(self):
|
||||
s = _read_script()
|
||||
assert "merged PCR list" in s, \
|
||||
"Step 5b must merge the Checkov/Wiz + kj PCR lists"
|
||||
|
||||
def test_command_v_kj_guard_present(self):
|
||||
s = _read_script()
|
||||
assert "command -v kj" in s, \
|
||||
"Step 5b must guard on `command -v kj` (is_configured)"
|
||||
|
||||
|
||||
class TestExistingPipelineUnchanged:
|
||||
def test_step_5_still_present(self):
|
||||
s = _read_script()
|
||||
assert "Step 5: runtime policy scan" in s
|
||||
|
||||
def test_step_7_confidence_still_present(self):
|
||||
s = _read_script()
|
||||
assert "Step 7: confidence signal compute" in s
|
||||
@@ -0,0 +1,86 @@
|
||||
"""Tests for stack-IR kyverno-json policies (REQ-299, v1.25).
|
||||
|
||||
Tests the 3 policies in adapters/kyverno-json/policies/stack-ir/:
|
||||
require-tagging-standard, forbid-public-ingress, require-encryption-by-
|
||||
default. Uses the passing + failing fixtures. Skips when kj is absent.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
import importlib.util
|
||||
_ENGINE_PATH = Path(__file__).resolve().parent.parent / "adapters" / "kyverno-json" / "kyverno_json_engine.py"
|
||||
_spec = importlib.util.spec_from_file_location("kyverno_json_engine", _ENGINE_PATH)
|
||||
_mod = importlib.util.module_from_spec(_spec)
|
||||
_spec.loader.exec_module(_mod)
|
||||
KyvernoJsonEngine = _mod.KyvernoJsonEngine
|
||||
|
||||
POLICY_DIR = Path(__file__).resolve().parent.parent / "adapters" / "kyverno-json" / "policies" / "stack-ir"
|
||||
FIXTURES = Path(__file__).resolve().parent / "fixtures" / "stack_ir"
|
||||
|
||||
|
||||
def _kj_installed() -> bool:
|
||||
return _mod._which_kj() is not None
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _require_kj():
|
||||
if not _kj_installed():
|
||||
pytest.skip("kj not installed (scripts/install-kyverno-json.sh)")
|
||||
|
||||
|
||||
def _load(name):
|
||||
with open(FIXTURES / name, "r", encoding="utf-8") as fh:
|
||||
return json.load(fh)
|
||||
|
||||
|
||||
class TestPassingFixture:
|
||||
def test_passing_fixture_all_pass(self):
|
||||
eng = KyvernoJsonEngine()
|
||||
out = eng.evaluate(_load("passing.json"), POLICY_DIR, "cid-pass")
|
||||
assert isinstance(out, list)
|
||||
assert len(out) >= 1
|
||||
# No fail results on the passing fixture.
|
||||
fails = [p for p in out if p["result"] == "fail"]
|
||||
assert fails == [], f"expected no fails on passing fixture, got: {fails}"
|
||||
|
||||
|
||||
class TestFailingFixture:
|
||||
def test_failing_fixture_has_fails(self):
|
||||
eng = KyvernoJsonEngine()
|
||||
out = eng.evaluate(_load("failing.json"), POLICY_DIR, "cid-fail")
|
||||
fails = [p for p in out if p["result"] == "fail"]
|
||||
assert len(fails) >= 1, "expected at least one fail on the failing fixture"
|
||||
|
||||
|
||||
class TestPolicyFilesExist:
|
||||
def test_three_policies_present(self):
|
||||
files = sorted(os.listdir(POLICY_DIR))
|
||||
assert "require-tagging-standard.json" in files
|
||||
assert "forbid-public-ingress.json" in files
|
||||
assert "require-encryption-by-default.json" in files
|
||||
|
||||
|
||||
class TestPolicyValidity:
|
||||
def test_policies_are_valid_json(self):
|
||||
for f in os.listdir(POLICY_DIR):
|
||||
if f.endswith(".json"):
|
||||
with open(POLICY_DIR / f, "r", encoding="utf-8") as fh:
|
||||
data = json.load(fh)
|
||||
assert data["apiVersion"] == "json.kyverno.io/v1alpha1"
|
||||
assert data["kind"] == "ValidatingPolicy"
|
||||
assert "nova.cloudinit.dev/severity" in data["metadata"]["annotations"]
|
||||
|
||||
def test_policy_names_match_filenames(self):
|
||||
for f in os.listdir(POLICY_DIR):
|
||||
if f.endswith(".json"):
|
||||
with open(POLICY_DIR / f, "r", encoding="utf-8") as fh:
|
||||
data = json.load(fh)
|
||||
expected = f.rsplit(".", 1)[0]
|
||||
assert data["metadata"]["name"] == expected
|
||||
Reference in New Issue
Block a user