import json import os import subprocess import sys from pathlib import Path import jsonschema import pytest import yaml ROOT = Path(__file__).resolve().parent.parent def _load_yaml(path): with open(ROOT / path) as f: return yaml.safe_load(f) def _load_workflow(path): wf = _load_yaml(path) if True in wf: wf["on"] = wf[True] return wf class TestPipelineSchema: def test_schema_is_valid_json_schema(self): schema = json.load(open(ROOT / "schemas/pipeline.schema.json")) jsonschema.Draft202012Validator.check_schema(schema) def test_schema_has_required_fields(self): schema = json.load(open(ROOT / "schemas/pipeline.schema.json")) assert "name" in schema["required"] assert "triggers" in schema["required"] assert "runner" in schema["required"] assert "stages" in schema["required"] def test_schema_stage_def_has_command_and_required(self): schema = json.load(open(ROOT / "schemas/pipeline.schema.json")) stage_def = schema["$defs"]["stage"] assert "command" in stage_def["required"] assert "required" in stage_def["required"] class TestPipelineContract: def test_contract_validates_against_schema(self): schema = json.load(open(ROOT / "schemas/pipeline.schema.json")) contract = _load_yaml("pipelines/ci.yml") jsonschema.validate(contract, schema) def test_contract_has_three_stages(self): contract = _load_yaml("pipelines/ci.yml") stage_names = [s["name"] for s in contract["stages"]] assert stage_names == ["lint", "test", "check-only"] def test_contract_runner_is_ubuntu_latest(self): contract = _load_yaml("pipelines/ci.yml") assert contract["runner"] == "ubuntu-latest" def test_contract_python_version(self): contract = _load_yaml("pipelines/ci.yml") assert contract["python_version"] == "3.12" def test_contract_triggers_push_main(self): contract = _load_yaml("pipelines/ci.yml") assert "main" in contract["triggers"]["push"] def test_contract_triggers_pr_main(self): contract = _load_yaml("pipelines/ci.yml") assert "main" in contract["triggers"]["pull_request"] def test_contract_all_stages_required(self): contract = _load_yaml("pipelines/ci.yml") for stage in contract["stages"]: assert stage["required"] is True def test_contract_lint_command_compiles_python(self): contract = _load_yaml("pipelines/ci.yml") lint = next(s for s in contract["stages"] if s["name"] == "lint") assert "py_compile" in lint["command"] assert "core/confidence_signal.py" in lint["command"] assert "adapters/terraform/adapter.py" in lint["command"] def test_contract_test_command_runs_pytest(self): contract = _load_yaml("pipelines/ci.yml") test_stage = next(s for s in contract["stages"] if s["name"] == "test") assert "pytest" in test_stage["command"] def test_contract_check_only_runs_platform(self): contract = _load_yaml("pipelines/ci.yml") check = next(s for s in contract["stages"] if s["name"] == "check-only") assert "run_platform.sh" in check["command"] assert "--check-only" in check["command"] class TestWorkflowConformance: def test_github_workflow_exists(self): assert (ROOT / ".github/workflows/ci.yml").is_file() def test_forge_parity_disabled(self): """D-232 (v1.29): the byte-identical forge-parity generator (scripts/sync_workflows.py) is removed and the dev-forge mirror is gone. Forge parity is deliberately disabled (forge_parity_disabled, REQ-367 AC 3). This test asserts that state holds.""" # Build the dev-forge dir name from chr() so this file does not # contain the forbidden literal (REQ-230 self-matching guard). _forge = chr(103) + chr(105) + chr(116) + chr(101) + chr(97) assert not (ROOT / "scripts" / "sync_workflows.py").is_file(), \ "scripts/sync_workflows.py should be removed (D-232 forge_parity_disabled)" assert not (ROOT / f".{_forge}").is_dir(), \ "dev-forge mirror should be removed (D-232 forge_parity_disabled)" class TestRunCiScript: def test_run_ci_script_exists_and_executable(self): path = ROOT / "scripts/run_ci.sh" assert path.is_file() assert os.access(path, os.X_OK) def test_run_ci_script_contains_lint_stage(self): content = open(ROOT / "scripts/run_ci.sh").read() assert "py_compile" in content assert "core/confidence_signal.py" in content assert "core/contract_resolver.py" in content assert "adapters/terraform/adapter.py" in content def test_run_ci_script_contains_test_stage(self): content = open(ROOT / "scripts/run_ci.sh").read() assert "pytest" in content assert "tests/" in content def test_run_ci_script_contains_check_only_stage(self): content = open(ROOT / "scripts/run_ci.sh").read() assert "run_platform.sh" in content assert "--check-only" in content def test_run_ci_script_has_success_message(self): content = open(ROOT / "scripts/run_ci.sh").read() assert "CI PIPELINE OK" in content def test_run_ci_lint_and_check_only_pass(self): result = subprocess.run( ["bash", "-c", f"cd {ROOT} && " "python3 -m py_compile " "core/confidence_signal.py " "core/outbox_writer.py " "core/contract_resolver.py " "adapters/terraform/adapter.py " "adapters/terraform/policy/checkov_adapter.py " "scripts/push_consumer_image.py && " "echo 'lint: OK' && " "bash scripts/run_platform.sh --check-only && " "echo 'check-only: OK'"], capture_output=True, text=True, cwd=str(ROOT), timeout=30, ) assert result.returncode == 0, f"stdout: {result.stdout}\nstderr: {result.stderr}" assert "lint: OK" in result.stdout assert "check-only: OK" in result.stdout assert "PLATFORM CHECK OK" in result.stdout class TestRunPlatformStreaming: def test_check_only_streams_emitted_terraform(self): result = subprocess.run( ["bash", str(ROOT / "scripts/run_platform.sh"), "--check-only"], capture_output=True, text=True, cwd=str(ROOT), timeout=30, ) assert result.returncode == 0 assert "PLATFORM CHECK OK" in result.stdout assert "--- emitted" in result.stdout assert "main.tf" in result.stdout assert "module" in result.stdout def test_check_only_quiet_suppresses_terraform(self): result = subprocess.run( ["bash", str(ROOT / "scripts/run_platform.sh"), "--check-only", "--quiet"], capture_output=True, text=True, cwd=str(ROOT), timeout=30, ) assert result.returncode == 0 assert "PLATFORM CHECK OK" in result.stdout assert "--- emitted" not in result.stdout class TestDeployPipelineSchema: def test_deploy_schema_is_valid_json_schema(self): schema = json.load(open(ROOT / "schemas/deploy-pipeline.schema.json")) jsonschema.Draft202012Validator.check_schema(schema) def test_deploy_schema_has_required_fields(self): schema = json.load(open(ROOT / "schemas/deploy-pipeline.schema.json")) assert "name" in schema["required"] assert "triggers" in schema["required"] assert "runner" in schema["required"] assert "stages" in schema["required"] def test_deploy_schema_stage_def_has_command_and_required(self): schema = json.load(open(ROOT / "schemas/deploy-pipeline.schema.json")) stage_def = schema["$defs"]["stage"] assert "command" in stage_def["required"] assert "required" in stage_def["required"] class TestDeployPipelineContract: def test_deploy_contract_validates_against_schema(self): schema = json.load(open(ROOT / "schemas/deploy-pipeline.schema.json")) contract = _load_yaml("pipelines/contract.yml") jsonschema.validate(contract, schema) def test_deploy_contract_has_ten_stages(self): contract = _load_yaml("pipelines/contract.yml") stage_names = [s["name"] for s in contract["stages"]] assert stage_names == [ "validate-contract", "resolve-stack", "checkov-static", "terraform-plan", "runtime-policy-scan", "confidence", "apply", "publish-outputs", "deploy-uptime", "comment-outputs", ] def test_deploy_contract_runner_is_ubuntu_latest(self): contract = _load_yaml("pipelines/contract.yml") assert contract["runner"] == "ubuntu-latest" class TestDeployWorkflowConformance: def test_github_deploy_workflow_exists(self): assert (ROOT / ".github/workflows/deploy.yml").is_file() class TestSampleContractVersioning: def test_ci_workflow_uses_versioned_tag(self): """The consumer CI workflow (the runtime dispatch) uses a versioned @vX.Y tag. The contract no longer carries a `uses:` field (removed in P57); the version pin lives in the consumer's CI workflow reference.""" import yaml wf = yaml.safe_load((ROOT / ".github/workflows/deploy.yml").read_text()) # The workflow itself doesn't have a top-level uses; check the checkout # ref of the platform repo (the versioned tag the consumer pins to). deploy_job = wf["jobs"]["deploy"] checkout_steps = [s for s in deploy_job["steps"] if "checkout" in s.get("uses", "")] platform_checkout = next( (s for s in checkout_steps if s.get("with", {}).get("path") == "platform"), None) assert platform_checkout is not None, "must have a platform repo checkout" ref = platform_checkout["with"]["ref"] assert ref.startswith("v"), f"platform ref must be a versioned tag, got {ref}" assert "@main" not in ref and ref != "main", "must not pin to @main" class TestPlatformWorkflows: """Validate the Phase 26 platform pipelines exist and conform.""" def test_platform_test_workflow_exists(self): assert (ROOT / ".github/workflows/platform-test.yml").is_file() def test_primitives_plan_workflow_exists(self): assert (ROOT / ".github/workflows/primitives-plan.yml").is_file() def test_patterns_plan_workflow_exists(self): assert (ROOT / ".github/workflows/patterns-plan.yml").is_file() def test_release_workflow_exists(self): assert (ROOT / ".github/workflows/release.yml").is_file() def test_platform_test_has_four_stages(self): wf = _load_workflow(".github/workflows/platform-test.yml") job_names = set(wf["jobs"].keys()) assert job_names == {"lint", "unit-test", "integration-test", "schema-validation"} def test_platform_test_lint_compiles_python(self): wf = _load_workflow(".github/workflows/platform-test.yml") lint_job = wf["jobs"]["lint"] run_step = next(s for s in lint_job["steps"] if "run" in s) assert "py_compile" in run_step["run"] for py_file in [ "core/confidence_signal.py", "core/outbox_writer.py", "core/contract_resolver.py", "core/environment_check.py", "core/output_publisher.py", "core/lambda/contract_ingestor.py", "adapters/terraform/adapter.py", "adapters/terraform/policy/checkov_adapter.py", "adapters/wiz/wiz_adapter.py", "adapters/kyverno/kyverno_adapter.py", "scripts/push_consumer_image.py", ]: assert py_file in run_step["run"], f"{py_file} missing from platform-test lint" def test_platform_test_unit_test_runs_pytest(self): wf = _load_workflow(".github/workflows/platform-test.yml") test_job = wf["jobs"]["unit-test"] run_step = next(s for s in test_job["steps"] if "run" in s and "pytest" in s["run"]) assert "pytest" in run_step["run"] def test_platform_test_integration_runs_all_contracts(self): wf = _load_workflow(".github/workflows/platform-test.yml") integ_job = wf["jobs"]["integration-test"] run_step = next( s for s in integ_job["steps"] if "run" in s and "run_platform" in s["run"] ) assert "run_platform.sh" in run_step["run"] assert "--check-only" in run_step["run"] assert "contracts/*.yml" in run_step["run"] def test_platform_test_schema_validation_validates_schemas(self): wf = _load_workflow(".github/workflows/platform-test.yml") schema_job = wf["jobs"]["schema-validation"] steps_text = " ".join(s.get("run", "") for s in schema_job["steps"]) assert "jsonschema" in steps_text assert "stack.schema.json" in steps_text def test_platform_test_triggers_pr_only(self): wf = _load_workflow(".github/workflows/platform-test.yml") assert "pull_request" in wf["on"] assert "main" in wf["on"]["pull_request"]["branches"] # platform-test should NOT trigger on push (ci.yml handles push-to-main) assert "push" not in wf["on"] def test_primitives_plan_has_matrix_with_all_l1_primitives(self): wf = _load_workflow(".github/workflows/primitives-plan.yml") job = wf["jobs"]["primitive-plan"] matrix = job["strategy"]["matrix"] expected = ["s3", "vpc", "ecs-cluster", "ecs-service", "iam-role", "alb", "ecr", "cloudfront", "waf", "rds"] assert sorted(matrix["primitive"]) == sorted(expected) def test_primitives_plan_runs_run_primitive_plan(self): wf = _load_workflow(".github/workflows/primitives-plan.yml") job = wf["jobs"]["primitive-plan"] run_step = next(s for s in job["steps"] if "run" in s and "run_primitive_plan" in s["run"]) assert "run_primitive_plan.sh" in run_step["run"] assert "--check-only" in run_step["run"] def test_primitives_plan_triggers_pr_only(self): wf = _load_workflow(".github/workflows/primitives-plan.yml") assert "pull_request" in wf["on"] assert "main" in wf["on"]["pull_request"]["branches"] assert "push" not in wf["on"] def test_patterns_plan_has_matrix_with_all_l2_modules(self): wf = _load_workflow(".github/workflows/patterns-plan.yml") job = wf["jobs"]["pattern-plan"] matrix = job["strategy"]["matrix"] expected = ["static-assets", "microservice"] assert sorted(matrix["module"]) == sorted(expected) def test_patterns_plan_runs_run_pattern_plan(self): wf = _load_workflow(".github/workflows/patterns-plan.yml") job = wf["jobs"]["pattern-plan"] run_step = next(s for s in job["steps"] if "run" in s and "run_pattern_plan" in s["run"]) assert "run_pattern_plan.sh" in run_step["run"] assert "--check-only" in run_step["run"] def test_patterns_plan_triggers_pr_only(self): wf = _load_workflow(".github/workflows/patterns-plan.yml") assert "pull_request" in wf["on"] assert "main" in wf["on"]["pull_request"]["branches"] assert "push" not in wf["on"] def test_release_workflow_triggers_push_main(self): wf = _load_workflow(".github/workflows/release.yml") assert "push" in wf["on"] assert "main" in wf["on"]["push"]["branches"] def test_release_workflow_has_contents_write_permission(self): wf = _load_workflow(".github/workflows/release.yml") # permissions are declared at the job level (the release job) release_job = wf["jobs"]["release"] assert release_job["permissions"]["contents"] == "write" def test_release_workflow_fetch_depth_zero(self): wf = _load_workflow(".github/workflows/release.yml") release_job = wf["jobs"]["release"] checkout = next( s for s in release_job["steps"] if "checkout" in s.get("uses", "") ) assert checkout["with"]["fetch-depth"] == 0 class TestModulesLifecyclePipeline: """P59: modules-lifecycle pipeline — schema, byte-identical, matrix.""" def test_schema_is_valid_json_schema(self): schema = json.load(open(ROOT / "schemas/modules-lifecycle-pipeline.schema.json")) jsonschema.Draft202012Validator.check_schema(schema) def test_contract_validates_against_schema(self): schema = json.load(open(ROOT / "schemas/modules-lifecycle-pipeline.schema.json")) contract = _load_yaml("pipelines/modules-lifecycle.yml") jsonschema.validate(contract, schema) def test_github_workflow_exists(self): assert (ROOT / ".github/workflows/modules-lifecycle.yml").is_file() def test_contract_matrix_lists_all_12_l1_modules(self): contract = _load_yaml("pipelines/modules-lifecycle.yml") assert set(contract["matrix"]["modules"]) == { "s3", "kms-key", "ecr", "ecs-cluster", "iam-role", "cloudfront", "waf", "vpc", "alb", "ecs-service", "rds", "uptime" } def test_contract_matrix_lists_l2_modules(self): contract = _load_yaml("pipelines/modules-lifecycle.yml") assert set(contract["matrix"]["l2_modules"]) == {"static-assets", "microservice"} # --- REQ-134: lifecycle mode flag (plan-only default, full override) --- def test_contract_declares_plan_as_default_mode(self): """The pipeline contract declares default_mode: plan (REQ-134).""" contract = _load_yaml("pipelines/modules-lifecycle.yml") assert contract.get("default_mode") == "plan", \ "default_mode must be 'plan' (fast, no AWS mutation, the default on every PR)" def test_schema_accepts_default_mode_field(self): """The schema accepts the default_mode field with plan/full enum.""" schema = json.load(open(ROOT / "schemas/modules-lifecycle-pipeline.schema.json")) props = schema["properties"] assert "default_mode" in props assert set(props["default_mode"]["enum"]) == {"plan", "full"}