diff --git a/adapters/README.md b/adapters/README.md index 17900eb..6862b8f 100644 --- a/adapters/README.md +++ b/adapters/README.md @@ -12,6 +12,37 @@ Adapters translate the engine-agnostic Target Stack IR to engine-specific format | Checkov adapter | `adapters/terraform/policy/checkov_adapter.py` | Checkov JSON | `PolicyCheckResult` records | Translates Checkov results | | Wiz adapter | `adapters/wiz/wiz_adapter.py` | Wiz API issues JSON | `PolicyCheckResult` records | Translates Wiz security findings | | Kyverno adapter | `adapters/kyverno/kyverno_adapter.py` | Kyverno PolicyReport JSON | `PolicyCheckResult` records | K8s-native policy translation | +| kyverno-json engine | `adapters/kyverno-json/kyverno_json_engine.py` | Any JSON/YAML payload | `PolicyCheckResult` records | **v1.25 primary policy engine** (swappable via `PolicyEngine` protocol) | + +## Policy Engine Protocol (v1.25) + +The `core/policy_engine.py` module defines the **swap boundary** between +Nova and its policy engines. A `PolicyEngine` Python Protocol (PEP 544) +with three members (`name`, `is_configured()`, `evaluate()`) is the +contract; a `PolicyEngineRegistry` selects the active engine from +`config.json`'s `policy.engine` key. The confidence signal and pipeline +never import an engine directly — they go through the registry. + +**Implementations:** +- `KyvernoJsonEngine` (`adapters/kyverno-json/`) — shells to the `kj` + CLI; the v1.25 default. +- `NullEngine` (`core/policy_engine.py`) — fallback when the `policy` + key is absent (emits `SKIPPED`). +- Future: `OpaEngine` — implements the same protocol, shells to + `opa eval`. The OPA-equivalent surface is documented in + `.ciagent/RESEARCH.md` §4.2. + +**How to add a new engine:** +1. Create `adapters//_engine.py` implementing the + `PolicyEngine` protocol (`name`, `is_configured()`, `evaluate()`). +2. `evaluate()` returns `list[dict]` where each dict conforms to + `schemas/policy_check_result.schema.json`. +3. Register the engine in `core/policy_engine.py`'s `_autoload_*` + function (or call `register(name, factory)` at startup). +4. Set `config.json.policy.engine` to the engine's `name`. +5. Add the engine to the `engine` enum in + `schemas/policy_check_result.schema.json` if it needs a distinct + enum value (v1.25 reuses `"kyverno"` — see D-116). ## How to Write an Adapter diff --git a/adapters/kyverno-json/README.md b/adapters/kyverno-json/README.md new file mode 100644 index 0000000..fd156d1 --- /dev/null +++ b/adapters/kyverno-json/README.md @@ -0,0 +1,103 @@ +# kyverno-json Engine Adapter (v1.25) + +The `kyverno-json` engine is Nova's **primary compliance/policy tool** +(v1.25), implemented behind the swappable `PolicyEngine` protocol so +OPA (or any other engine) can replace it one day. + +## What kyverno-json is + +[kyverno-json](https://github.com/kyverno/kyverno-json) is a standalone +Go binary from the Kyverno project — a **separate runtime** from the +K8s Kyverno admission controller. It applies Kyverno `ValidatingPolicy` +resources to **any** JSON or YAML payload file via the `kj scan` CLI. +Unlike the K8s Kyverno adapter (`adapters/kyverno/`), which only +speaks to K8s manifests, kyverno-json evaluates consumer contracts, +resolved Stack IR, terraform plan JSON, and even the merged PCR list +itself (meta-policies). + +## Install + +```bash +bash scripts/install-kyverno-json.sh +# or directly: +go install github.com/kyverno/kyverno-json/cmd/kj@latest +kj version +``` + +The platform functions without the binary — `is_configured()` returns +`False` when `which kj` is absent → `evaluate()` returns a single +`SKIPPED` PCR (`KJ_ENGINE_NOT_CONFIGURED`). The confidence signal +proceeds with a neutral `policy` input (D-120 graceful degradation). + +## Policy directory layout + +``` +adapters/kyverno-json/policies/ +├── _smoke.json # round-trip smoke test +├── contract/ # consumer contract JSON policies +│ ├── require-id-pattern.json +│ ├── require-env-in-enum.json +│ ├── require-infrastructure-min-1.json +│ └── forbid-unknown-fields.json +├── stack-ir/ # resolved Stack IR policies +│ ├── require-tagging-standard.json +│ ├── forbid-public-ingress.json +│ └── require-encryption-by-default.json +├── plan-json/ # terraform show -json policies +│ ├── forbid-plaintext-secrets.json +│ ├── forbid-iam-wildcard.json +│ └── require-kms-reference.json +├── meta/ # policies over the merged PCR list +│ ├── block-on-any-critical.json +│ └── tagging-rules-agree.json +└── regression/ # capability-inventory policies + ├── cap-013-adapter-dedup.json + ├── cap-023-metrics-collector.json + └── cap-024-deck-structure.json +``` + +## The four policy categories + +1. **contract/** — over the consumer contract JSON (pre-resolve). +2. **stack-ir/** — over the resolved Target Stack IR (post-resolve). +3. **plan-json/** — over `terraform show -json` output (pipeline Step 5b). +4. **meta/** — over the merged `list[PolicyCheckResult]` (meta-policies). +5. **regression/** — over the capability-inventory JSON (declarative + mirrors of `core/regression_verify.py`). + +## Severity convention + +kyverno-json does not natively assign severities. Each Nova policy +declares its severity via a `metadata.annotations` field: + +```yaml +metadata: + annotations: + nova.cloudinit.dev/severity: high +``` + +Valid values: `critical`, `high`, `medium`, `low`, `info` (default +when absent). + +## Engine enum reuse (D-116) + +kyverno-json PCR records carry `engine: "kyverno"` (no new enum value). +The `engine` field records the policy-engine *family*, not the specific +binary. The K8s Kyverno adapter and the kyverno-json engine are +distinguished by `ruleId` prefix (`KYVERNO_` vs `KJ_`) and `evidence` +payload shape (`namespace`/`kind` vs `assertion`/`jmespath`). + +## Schema path + +The output records validate against +[`schemas/policy_check_result.schema.json`](../../schemas/policy_check_result.schema.json) +(`engine: "kyverno"` is in the enum). The confidence signal consumes +the merged PCR list engine-agnostically. + +## Swap boundary + +The `PolicyEngine` protocol (`core/policy_engine.py`) is the swap +boundary. The OPA-equivalent surface is documented in +`.ciagent/RESEARCH.md` §4.2 — a future `OpaEngine` implements the same +protocol without touching the confidence signal, the PCR schema, or +the pipeline. \ No newline at end of file diff --git a/adapters/kyverno-json/policies/regression/cap-013-adapter-dedup.json b/adapters/kyverno-json/policies/regression/cap-013-adapter-dedup.json new file mode 100644 index 0000000..1701640 --- /dev/null +++ b/adapters/kyverno-json/policies/regression/cap-013-adapter-dedup.json @@ -0,0 +1,30 @@ +{ + "apiVersion": "json.kyverno.io/v1alpha1", + "kind": "ValidatingPolicy", + "metadata": { + "name": "cap-013-adapter-dedup", + "annotations": { + "nova.cloudinit.dev/severity": "medium", + "title.policy.kyverno.io": "No duplicate adapter registrations (CAP-013 declarative mirror)" + } + }, + "spec": { + "rules": [ + { + "name": "no-duplicate-adapters", + "validate": { + "message": "Each adapter must be registered exactly once (no duplicate adapter names in the capability inventory). Declarative mirror of core/regression_verify.py CAP-013.", + "assert": { + "all": [ + { + "check": { + "adapters": "(length(duplicates(@)) == `0`)" + } + } + ] + } + } + } + ] + } +} \ No newline at end of file diff --git a/adapters/kyverno-json/policies/regression/cap-023-metrics-collector.json b/adapters/kyverno-json/policies/regression/cap-023-metrics-collector.json new file mode 100644 index 0000000..59d3e58 --- /dev/null +++ b/adapters/kyverno-json/policies/regression/cap-023-metrics-collector.json @@ -0,0 +1,32 @@ +{ + "apiVersion": "json.kyverno.io/v1alpha1", + "kind": "ValidatingPolicy", + "metadata": { + "name": "cap-023-metrics-collector", + "annotations": { + "nova.cloudinit.dev/severity": "medium", + "title.policy.kyverno.io": "Every metric has a grounded/derived/deferred status (CAP-023 declarative mirror)" + } + }, + "spec": { + "rules": [ + { + "name": "every-metric-has-status", + "validate": { + "message": "Every metric in docs/METRICS.md must declare a status (grounded, derived, or deferred). Declarative mirror of core/regression_verify.py CAP-023.", + "assert": { + "all": [ + { + "check": { + "~.metrics": { + "(contains(['grounded','derived','deferred'], status))": true + } + } + } + ] + } + } + } + ] + } +} \ No newline at end of file diff --git a/adapters/kyverno-json/policies/regression/cap-024-deck-structure.json b/adapters/kyverno-json/policies/regression/cap-024-deck-structure.json new file mode 100644 index 0000000..be39624 --- /dev/null +++ b/adapters/kyverno-json/policies/regression/cap-024-deck-structure.json @@ -0,0 +1,35 @@ +{ + "apiVersion": "json.kyverno.io/v1alpha1", + "kind": "ValidatingPolicy", + "metadata": { + "name": "cap-024-deck-structure", + "annotations": { + "nova.cloudinit.dev/severity": "low", + "title.policy.kyverno.io": "Deck structure matches the documented 4-beat arc (CAP-024 declarative mirror)" + } + }, + "spec": { + "rules": [ + { + "name": "deck-has-4-beats", + "validate": { + "message": "The deck must have the 4-beat arc: Problem, Solution, Proof, Roadmap+Ask. Declarative mirror of core/regression_verify.py CAP-024.", + "assert": { + "all": [ + { + "check": { + "deck.beats": "(length(@) >= `4`)" + } + }, + { + "check": { + "deck.beats": "(contains(@, 'Problem') && contains(@, 'Solution') && contains(@, 'Proof') && contains(@, 'Roadmap+Ask'))" + } + } + ] + } + } + } + ] + } +} \ No newline at end of file diff --git a/docs/METRICS.md b/docs/METRICS.md index f30d89b..21c8ba6 100644 --- a/docs/METRICS.md +++ b/docs/METRICS.md @@ -174,4 +174,18 @@ numbers. Every metric either has a real source or is explicitly deferred. | SLA / Unplanned Downtime | D-096 | `placeholder_sla_downtime.csv` | | Predictive vs Reactive Ratio | future emitter | `placeholder_predictive_reactive.csv` | -See `docs/METRICS_DEFERRED_ROADMAP.md` for the activation path for each. \ No newline at end of file +See `docs/METRICS_DEFERRED_ROADMAP.md` for the activation path for each. + +--- + +## v1.25 — Swappable Policy Engine + +The policy engine that produces the `PolicyCheckResult` records feeding +the confidence signal is **swappable** (NORTH_STAR Strategic Objective #2 +— provable trust via a replaceable substrate, not a vendor lock-in). +The `PolicyEngine` protocol (`core/policy_engine.py`) is the swap +boundary; `config.json.policy.engine` selects the active engine +(default `"kyverno-json"`). A future `OpaEngine` implements the same +protocol without touching the confidence signal, the PCR schema, or +the pipeline. See `.ciagent/ARCHITECTURE.md` §12.7 for the registry +diagram. \ No newline at end of file diff --git a/modules/STANDARDS.md b/modules/STANDARDS.md index 3b38437..895d0c3 100644 --- a/modules/STANDARDS.md +++ b/modules/STANDARDS.md @@ -611,4 +611,64 @@ must be checked before the module is registered and published. `stack.schema.json`). - [ ] For an L2, a test is added that the composition resolves to the expected set of L1 instances and that the adapter emits a root module - calling the L1 modules. \ No newline at end of file + calling the L1 modules. + +--- + +## 10. Policy Authoring Standard (v1.25) + +Module owners may ship per-module kyverno-json policies in +`modules//policies/` (future convention; v1.25 policies live +under `adapters/kyverno-json/policies/`). A policy file is a +`ValidatingPolicy` resource (YAML or JSON). + +### 10.1 Required fields + +- `apiVersion: json.kyverno.io/v1alpha1` +- `kind: ValidatingPolicy` +- `metadata.name` — matches the filename (e.g. `require-tags.json` → + `name: require-tags`). This becomes the `ruleId` prefix `KJ_`. +- `metadata.annotations["nova.cloudinit.dev/severity"]` — one of + `critical`, `high`, `medium`, `low`, `info`. Drives the confidence + signal's penalty mapping. +- `spec.rules[].validate.assert` — an `all` or `any` list of assertion + trees with JMESPath expressions. **No `forEach`, pattern operators, + anchors, or wildcards** — use the `~` projection modifier to iterate. + +### 10.2 Severity guidance + +| Severity | When to use | Confidence penalty | +| --- | --- | --- | +| `critical` | a violation makes the deploy unsafe (e.g. public ingress on a prod DB) | hard override (score = 0, block) | +| `high` | a violation is a security or compliance gap (e.g. plaintext secrets) | -0.20 | +| `medium` | a violation is a best-practice miss (e.g. missing tags) | -0.05 | +| `low` | a violation is a style or convention issue | -0.01 | +| `info` | a non-blocking observation (default) | 0.0 | + +### 10.3 Assertion-tree patterns + +- **Iterate an array:** use the `~` modifier on the array key: + ```yaml + check: + ~.resources: + (@ < `5`): true + ``` +- **Match a resource type:** use the `match.any` block: + ```yaml + match: + any: + - type: aws:s3:bucket + ``` +- **Binding for descendant access:** use `->name`: + ```yaml + (bar + bat)->sum: + ($sum): 10 + ``` + +### 10.4 Testing + +- Ship a fixture pair (`passing.json` + `failing.json`) under + `tests/fixtures//`. +- Add a test file `tests/test__policies.py` using the + `KyvernoJsonEngine` (skip-without-kj pattern). +- The regression gate (`pytest tests/`) must remain green. \ No newline at end of file diff --git a/schemas/README.md b/schemas/README.md index e56b7aa..0867a7d 100644 --- a/schemas/README.md +++ b/schemas/README.md @@ -15,6 +15,14 @@ Nova uses JSON Schema draft 2020-12 for all declarative contracts. Schemas are t | Nova PolicyCheckResult | `policy_check_result.schema.json` | Normalized policy check result schema (the contract between policy engines and the confidence signal) | `tests/conftest.py`, all adapter tests | | Nova Tagging Standard | `tagging-standard.json` | Required tag set for all taggable AWS resources | `adapters/terraform/policy/custom_rules/nova_tagging.py` | +> **v1.25 note (D-116):** the `engine` enum value `"kyverno"` is shared +> by the K8s-only Kyverno adapter (`adapters/kyverno/`) and the +> kyverno-json engine (`adapters/kyverno-json/`). The two are +> distinguished by `ruleId` prefix (`KYVERNO_` for the K8s adapter, +> `KJ_` for kyverno-json) and `evidence` payload shape. No new enum +> value was added — the `engine` field records the policy-engine +> family, not the specific binary. + ## How to Write a Schema 1. Use JSON Schema draft 2020-12: `"$schema": "https://json-schema.org/draft/2020-12/schema"`. diff --git a/tests/fixtures/capability_inventory/clean.json b/tests/fixtures/capability_inventory/clean.json new file mode 100644 index 0000000..9248997 --- /dev/null +++ b/tests/fixtures/capability_inventory/clean.json @@ -0,0 +1,11 @@ +{ + "adapters": ["terraform", "checkov", "wiz", "kyverno-json"], + "metrics": [ + {"name": "MTTR", "status": "grounded"}, + {"name": "CloudSpend", "status": "derived"}, + {"name": "TouchlessResolution", "status": "deferred"} + ], + "deck": { + "beats": ["Problem", "Solution", "Proof", "Roadmap+Ask"] + } +} \ No newline at end of file diff --git a/tests/fixtures/capability_inventory/drifted.json b/tests/fixtures/capability_inventory/drifted.json new file mode 100644 index 0000000..1b3e1b7 --- /dev/null +++ b/tests/fixtures/capability_inventory/drifted.json @@ -0,0 +1,11 @@ +{ + "adapters": ["terraform", "checkov", "wiz", "terraform", "kyverno-json"], + "metrics": [ + {"name": "MTTR", "status": "grounded"}, + {"name": "CloudSpend", "status": "unknown"}, + {"name": "TouchlessResolution", "status": "deferred"} + ], + "deck": { + "beats": ["Problem", "Solution", "Proof"] + } +} \ No newline at end of file diff --git a/tests/test_regression_policies.py b/tests/test_regression_policies.py new file mode 100644 index 0000000..a5ab90b --- /dev/null +++ b/tests/test_regression_policies.py @@ -0,0 +1,88 @@ +"""Tests for regression-gate kyverno-json policies (REQ-304, REQ-305, v1.25). + +Tests the 3 declarative mirrors of core/regression_verify.py: +cap-013-adapter-dedup, cap-023-metrics-collector, cap-024-deck-structure. +Uses clean + drifted capability-inventory fixtures. Skip-without-kj. +""" + +import json +import os +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +import importlib.util +_ENGINE_PATH = Path(__file__).resolve().parent.parent / "adapters" / "kyverno-json" / "kyverno_json_engine.py" +_spec = importlib.util.spec_from_file_location("kyverno_json_engine", _ENGINE_PATH) +_mod = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_mod) +KyvernoJsonEngine = _mod.KyvernoJsonEngine + +POLICY_DIR = Path(__file__).resolve().parent.parent / "adapters" / "kyverno-json" / "policies" / "regression" +FIXTURES = Path(__file__).resolve().parent / "fixtures" / "capability_inventory" + + +def _kj_installed() -> bool: + return _mod._which_kj() is not None + + +@pytest.fixture(autouse=True) +def _require_kj(): + if not _kj_installed(): + pytest.skip("kj not installed (scripts/install-kyverno-json.sh)") + + +def _load(name): + with open(FIXTURES / name, "r", encoding="utf-8") as fh: + return json.load(fh) + + +class TestCleanInventory: + def test_clean_inventory_no_fails(self): + eng = KyvernoJsonEngine() + out = eng.evaluate(_load("clean.json"), POLICY_DIR, "cid-clean") + fails = [p for p in out if p["result"] == "fail"] + assert fails == [], f"expected no fails on clean inventory, got: {fails}" + + +class TestDriftedInventory: + def test_drifted_inventory_has_fails(self): + eng = KyvernoJsonEngine() + out = eng.evaluate(_load("drifted.json"), POLICY_DIR, "cid-drift") + fails = [p for p in out if p["result"] == "fail"] + assert len(fails) >= 1, "expected at least one fail on the drifted inventory" + + +class TestPolicyFilesExist: + def test_three_regression_policies_present(self): + files = sorted(os.listdir(POLICY_DIR)) + assert "cap-013-adapter-dedup.json" in files + assert "cap-023-metrics-collector.json" in files + assert "cap-024-deck-structure.json" in files + + def test_policies_are_valid_json(self): + for f in os.listdir(POLICY_DIR): + if f.endswith(".json"): + with open(POLICY_DIR / f, "r", encoding="utf-8") as fh: + data = json.load(fh) + assert data["apiVersion"] == "json.kyverno.io/v1alpha1" + assert data["kind"] == "ValidatingPolicy" + assert "nova.cloudinit.dev/severity" in data["metadata"]["annotations"] + + +class TestFixturesExist: + def test_clean_and_drifted_fixtures_present(self): + assert (FIXTURES / "clean.json").is_file() + assert (FIXTURES / "drifted.json").is_file() + + def test_drifted_fixture_has_duplicate_adapter(self): + data = _load("drifted.json") + # The drifted fixture has 'terraform' twice (adapter dedup violation). + assert data["adapters"].count("terraform") == 2 + + def test_drifted_fixture_has_missing_roadmap_beat(self): + data = _load("drifted.json") + assert "Roadmap+Ask" not in data["deck"]["beats"] \ No newline at end of file