From 7ba72bf656debde79bc790383b11f2eecfccb698 Mon Sep 17 00:00:00 2001 From: Jon Chery Date: Tue, 28 Jul 2026 15:55:54 +0000 Subject: [PATCH] =?UTF-8?q?feat(P59):=20L1=20module=20lifecycle=20pipeline?= =?UTF-8?q?=20=E2=80=94=20author=20workflows=20+=20schema=20+=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit EXECUTE stage. Authors the modules-lifecycle pipeline that matrix-tests every L1 module's examples/{simple,complex}.yml contracts through apply→modify→destroy against live AWS. No per-module Python. New files: - pipelines/modules-lifecycle.yml: declarative contract (5 stages: platform-vpc-apply, lifecycle-apply, lifecycle-modify, lifecycle-destroy, platform-vpc-destroy). Matrix over 12 L1 modules. - .gitea/workflows/modules-lifecycle.yml + .github/workflows/modules-lifecycle.yml: byte-identical workflows. 3 jobs: platform-vpc-apply (prerequisite), lifecycle (matrix of 12 modules × apply/modify/destroy), platform-vpc-destroy (always runs, cleanup). Triggers: pull_request to main + workflow_dispatch. - schemas/modules-lifecycle-pipeline.schema.json: schema for the new pipeline shape (extends pipeline.schema.json with workflow_dispatch + matrix). Tests (tests/test_pipeline_contract.py): - TestModulesLifecyclePipeline: 12 tests (schema valid, contract validates, byte-identical, workflow name, 3 jobs, triggers, matrix lists all 12 L1 modules, apply/modify/destroy steps present, platform-vpc-destroy always runs). pipelines/README.md: added modules-lifecycle to the pipeline table. Regression: 479 passed, 0 skipped, 5 deselected (slow). ---ci--- project: acdl phase: P59 milestone: v1.11 status: execute ---/ci--- --- .gitea/workflows/modules-lifecycle.yml | 110 ++++++++++++++++++ .github/workflows/modules-lifecycle.yml | 110 ++++++++++++++++++ pipelines/README.md | 1 + pipelines/modules-lifecycle.yml | 55 +++++++++ .../modules-lifecycle-pipeline.schema.json | 92 +++++++++++++++ tests/test_pipeline_contract.py | 68 ++++++++++- 6 files changed, 435 insertions(+), 1 deletion(-) create mode 100644 .gitea/workflows/modules-lifecycle.yml create mode 100644 .github/workflows/modules-lifecycle.yml create mode 100644 pipelines/modules-lifecycle.yml create mode 100644 schemas/modules-lifecycle-pipeline.schema.json diff --git a/.gitea/workflows/modules-lifecycle.yml b/.gitea/workflows/modules-lifecycle.yml new file mode 100644 index 0000000..d9a8812 --- /dev/null +++ b/.gitea/workflows/modules-lifecycle.yml @@ -0,0 +1,110 @@ +# ACDL Modules Lifecycle Pipeline — Gitea Actions (dev environment) +# +# Matrix-runs each L1 module's examples/{simple,complex}.yml contracts through +# apply→modify→destroy against live AWS. No per-module Python. The "test" = +# the pipeline cell going green. +# +# This workflow implements pipelines/modules-lifecycle.yml (byte-identical +# in .gitea/workflows/ and .github/workflows/). +# +# VPC-dependent L1s (alb, ecs-service, rds, uptime) reference the platform VPC +# applied by a prerequisite job. Non-VPC L1s run independent. +name: acdl-modules-lifecycle + +on: + pull_request: + branches: [main] + workflow_dispatch: + +permissions: + id-token: write + contents: read + +jobs: + # Prerequisite: apply the shared platform VPC (needed by VPC-dependent L1s) + platform-vpc-apply: + name: Platform VPC apply + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install Terraform 1.9.* + run: | + wget -qO- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp.gpg + echo "deb [signed-by=/usr/share/keyrings/hashicorp.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list + sudo apt-get update && sudo apt-get install -y terraform=1.9.* + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: us-east-1 + access-key-id: ${{ secrets.ACDL_AWS_ACCESS_KEY_ID }} + secret-access-key: ${{ secrets.ACDL_AWS_SECRET_ACCESS_KEY }} + - name: Apply platform VPC + working-directory: terraform/platform + run: | + terraform init -input=false -lock=false + terraform apply -auto-approve -lock=false + + # L1 lifecycle matrix: apply simple → apply complex (modify) → destroy + lifecycle: + name: L1 lifecycle (${{ matrix.module }}) + needs: platform-vpc-apply + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + module: [s3, kms-key, ecr, ecs-cluster, iam-role, cloudfront, waf, vpc, alb, ecs-service, rds, uptime] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install dependencies + run: pip install jsonschema pyyaml boto3 + - name: Install Terraform 1.9.* + run: | + wget -qO- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp.gpg + echo "deb [signed-by=/usr/share/keyrings/hashicorp.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list + sudo apt-get update && sudo apt-get install -y terraform=1.9.* + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: us-east-1 + access-key-id: ${{ secrets.ACDL_AWS_ACCESS_KEY_ID }} + secret-access-key: ${{ secrets.ACDL_AWS_SECRET_ACCESS_KEY }} + - name: Apply (simple) + run: bash scripts/run_platform.sh --apply "modules/l1/${{ matrix.module }}/examples/simple.yml" + - name: Modify (complex) + run: bash scripts/run_platform.sh --apply "modules/l1/${{ matrix.module }}/examples/complex.yml" + - name: Destroy + run: bash scripts/run_platform.sh --destroy "modules/l1/${{ matrix.module }}/examples/complex.yml" + + # Cleanup: destroy the platform VPC (always runs, even if lifecycle fails) + platform-vpc-destroy: + name: Platform VPC destroy + needs: lifecycle + runs-on: ubuntu-latest + if: always() + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install Terraform 1.9.* + run: | + wget -qO- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp.gpg + echo "deb [signed-by=/usr/share/keyrings/hashicorp.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list + sudo apt-get update && sudo apt-get install -y terraform=1.9.* + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: us-east-1 + access-key-id: ${{ secrets.ACDL_AWS_ACCESS_KEY_ID }} + secret-access-key: ${{ secrets.ACDL_AWS_SECRET_ACCESS_KEY }} + - name: Destroy platform VPC + working-directory: terraform/platform + run: | + terraform init -input=false -lock=false + terraform destroy -auto-approve -lock=false \ No newline at end of file diff --git a/.github/workflows/modules-lifecycle.yml b/.github/workflows/modules-lifecycle.yml new file mode 100644 index 0000000..d9a8812 --- /dev/null +++ b/.github/workflows/modules-lifecycle.yml @@ -0,0 +1,110 @@ +# ACDL Modules Lifecycle Pipeline — Gitea Actions (dev environment) +# +# Matrix-runs each L1 module's examples/{simple,complex}.yml contracts through +# apply→modify→destroy against live AWS. No per-module Python. The "test" = +# the pipeline cell going green. +# +# This workflow implements pipelines/modules-lifecycle.yml (byte-identical +# in .gitea/workflows/ and .github/workflows/). +# +# VPC-dependent L1s (alb, ecs-service, rds, uptime) reference the platform VPC +# applied by a prerequisite job. Non-VPC L1s run independent. +name: acdl-modules-lifecycle + +on: + pull_request: + branches: [main] + workflow_dispatch: + +permissions: + id-token: write + contents: read + +jobs: + # Prerequisite: apply the shared platform VPC (needed by VPC-dependent L1s) + platform-vpc-apply: + name: Platform VPC apply + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install Terraform 1.9.* + run: | + wget -qO- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp.gpg + echo "deb [signed-by=/usr/share/keyrings/hashicorp.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list + sudo apt-get update && sudo apt-get install -y terraform=1.9.* + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: us-east-1 + access-key-id: ${{ secrets.ACDL_AWS_ACCESS_KEY_ID }} + secret-access-key: ${{ secrets.ACDL_AWS_SECRET_ACCESS_KEY }} + - name: Apply platform VPC + working-directory: terraform/platform + run: | + terraform init -input=false -lock=false + terraform apply -auto-approve -lock=false + + # L1 lifecycle matrix: apply simple → apply complex (modify) → destroy + lifecycle: + name: L1 lifecycle (${{ matrix.module }}) + needs: platform-vpc-apply + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + module: [s3, kms-key, ecr, ecs-cluster, iam-role, cloudfront, waf, vpc, alb, ecs-service, rds, uptime] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install dependencies + run: pip install jsonschema pyyaml boto3 + - name: Install Terraform 1.9.* + run: | + wget -qO- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp.gpg + echo "deb [signed-by=/usr/share/keyrings/hashicorp.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list + sudo apt-get update && sudo apt-get install -y terraform=1.9.* + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: us-east-1 + access-key-id: ${{ secrets.ACDL_AWS_ACCESS_KEY_ID }} + secret-access-key: ${{ secrets.ACDL_AWS_SECRET_ACCESS_KEY }} + - name: Apply (simple) + run: bash scripts/run_platform.sh --apply "modules/l1/${{ matrix.module }}/examples/simple.yml" + - name: Modify (complex) + run: bash scripts/run_platform.sh --apply "modules/l1/${{ matrix.module }}/examples/complex.yml" + - name: Destroy + run: bash scripts/run_platform.sh --destroy "modules/l1/${{ matrix.module }}/examples/complex.yml" + + # Cleanup: destroy the platform VPC (always runs, even if lifecycle fails) + platform-vpc-destroy: + name: Platform VPC destroy + needs: lifecycle + runs-on: ubuntu-latest + if: always() + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install Terraform 1.9.* + run: | + wget -qO- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp.gpg + echo "deb [signed-by=/usr/share/keyrings/hashicorp.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list + sudo apt-get update && sudo apt-get install -y terraform=1.9.* + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: us-east-1 + access-key-id: ${{ secrets.ACDL_AWS_ACCESS_KEY_ID }} + secret-access-key: ${{ secrets.ACDL_AWS_SECRET_ACCESS_KEY }} + - name: Destroy platform VPC + working-directory: terraform/platform + run: | + terraform init -input=false -lock=false + terraform destroy -auto-approve -lock=false \ No newline at end of file diff --git a/pipelines/README.md b/pipelines/README.md index 3fb4277..83783a6 100644 --- a/pipelines/README.md +++ b/pipelines/README.md @@ -10,6 +10,7 @@ ACDL uses declarative pipeline contracts (YAML) as the single source of truth. B | --- | --- | --- | --- | | ACDL CI | `ci.yml` | `lint`, `test`, `check-only` | push/PR to `main` | | ACDL Deploy | `contract.yml` | `validate-contract`, `resolve-stack`, `terraform-plan`, `checkov`, `confidence`, `apply`, `publish-outputs`, `deploy-uptime`, `comment-outputs` | push/PR to `main` (consumer repos via `workflow_call`) | +| ACDL Modules Lifecycle | `modules-lifecycle.yml` | `platform-vpc-apply`, `lifecycle-apply`, `lifecycle-modify`, `lifecycle-destroy`, `platform-vpc-destroy` | PR to `main` + `workflow_dispatch` | ## How to Write a Pipeline diff --git a/pipelines/modules-lifecycle.yml b/pipelines/modules-lifecycle.yml new file mode 100644 index 0000000..16860a6 --- /dev/null +++ b/pipelines/modules-lifecycle.yml @@ -0,0 +1,55 @@ +name: acdl-modules-lifecycle + +# ACDL Modules Lifecycle Pipeline — apply→modify→destroy against live AWS. +# +# Matrix-runs each L1 module's examples/{simple,complex}.yml contracts: +# 1. --apply simple.yml (terraform apply — creates resources) +# 2. --apply complex.yml (same state key — terraform modifies) +# 3. --destroy complex.yml (terraform destroy — cleans up) +# +# No per-module Python. The "test" = the pipeline cell going green. +# VPC-dependent L1s (alb, ecs-service, rds, uptime) reference the platform +# VPC applied by a prerequisite job. Non-VPC L1s run independent. +# +# Both Gitea (.gitea/workflows/modules-lifecycle.yml) and GitHub +# (.github/workflows/modules-lifecycle.yml) implement this contract +# byte-identically. + +triggers: + pull_request: [main] + workflow_dispatch: [] + +runner: ubuntu-latest +python_version: "3.12" +terraform_version: "1.9.*" + +stages: + - name: platform-vpc-apply + command: cd terraform/platform && terraform init -input=false && terraform apply -auto-approve -lock=false + required: true + description: "Apply the shared platform VPC (prerequisite for VPC-dependent L1s)" + + - name: lifecycle-apply + command: bash scripts/run_platform.sh --apply "modules/l1/${MODULE}/examples/simple.yml" + required: true + description: "Apply the module's simple example contract (terraform apply)" + + - name: lifecycle-modify + command: bash scripts/run_platform.sh --apply "modules/l1/${MODULE}/examples/complex.yml" + required: true + description: "Apply the module's complex example (same state key — terraform modifies)" + + - name: lifecycle-destroy + command: bash scripts/run_platform.sh --destroy "modules/l1/${MODULE}/examples/complex.yml" + required: true + description: "Destroy the module's resources (terraform destroy)" + + - name: platform-vpc-destroy + command: cd terraform/platform && terraform destroy -auto-approve -lock=false + required: false + description: "Destroy the shared platform VPC (cleanup — runs even if lifecycle steps fail)" + +matrix: + modules: [s3, kms-key, ecr, ecs-cluster, iam-role, cloudfront, waf, vpc, alb, ecs-service, rds, uptime] + vpc_dependent: [alb, ecs-service, rds, uptime] + independent: [s3, kms-key, ecr, ecs-cluster, iam-role, cloudfront, waf, vpc] \ No newline at end of file diff --git a/schemas/modules-lifecycle-pipeline.schema.json b/schemas/modules-lifecycle-pipeline.schema.json new file mode 100644 index 0000000..7bccb2d --- /dev/null +++ b/schemas/modules-lifecycle-pipeline.schema.json @@ -0,0 +1,92 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://acdl.cloudinit.dev/schemas/modules-lifecycle-pipeline.schema.json", + "title": "ACDL Modules Lifecycle Pipeline Contract", + "description": "Declarative contract for the modules-lifecycle pipeline. Matrix-runs each L1 module's examples/{simple,complex}.yml contracts through apply→modify→destroy against live AWS. Both Gitea Actions (.gitea/workflows/modules-lifecycle.yml) and GitHub Actions (.github/workflows/modules-lifecycle.yml) implement this contract byte-identically.", + "type": "object", + "required": ["name", "triggers", "runner", "python_version", "terraform_version", "stages", "matrix"], + "properties": { + "name": { + "type": "string", + "description": "Pipeline name (matches the workflow 'name:' field)." + }, + "triggers": { + "type": "object", + "required": ["pull_request", "workflow_dispatch"], + "properties": { + "pull_request": { + "type": "array", + "items": {"type": "string"}, + "description": "Branches that trigger the pipeline on PR." + }, + "workflow_dispatch": { + "type": "array", + "items": {"type": "string"}, + "description": "Allows manual dispatch (empty array = no params)." + } + } + }, + "runner": { + "type": "string", + "description": "Runner image (e.g. 'ubuntu-latest')." + }, + "python_version": { + "type": "string", + "description": "Python version for setup-python action." + }, + "terraform_version": { + "type": "string", + "description": "Terraform version constraint (e.g. '1.9.*')." + }, + "stages": { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/$defs/stage"} + }, + "matrix": { + "type": "object", + "required": ["modules"], + "properties": { + "modules": { + "type": "array", + "items": {"type": "string"}, + "description": "L1 module names to test." + }, + "vpc_dependent": { + "type": "array", + "items": {"type": "string"}, + "description": "Modules that require the platform VPC prerequisite." + }, + "independent": { + "type": "array", + "items": {"type": "string"}, + "description": "Modules that run without the platform VPC." + } + } + } + }, + "$defs": { + "stage": { + "type": "object", + "required": ["name", "command", "required"], + "properties": { + "name": { + "type": "string", + "description": "Stage name (maps to the workflow job/step name)." + }, + "command": { + "type": "string", + "description": "The shell command to run for this stage." + }, + "required": { + "type": "boolean", + "description": "If true, a non-zero exit code fails the pipeline." + }, + "description": { + "type": "string", + "description": "Optional: human-readable description of what this stage does." + } + } + } + } +} \ No newline at end of file diff --git a/tests/test_pipeline_contract.py b/tests/test_pipeline_contract.py index 1b51207..fe926d9 100644 --- a/tests/test_pipeline_contract.py +++ b/tests/test_pipeline_contract.py @@ -518,4 +518,70 @@ class TestPlatformWorkflows: checkout = next( s for s in release_job["steps"] if "checkout" in s.get("uses", "") ) - assert checkout["with"]["fetch-depth"] == 0 \ No newline at end of file + assert checkout["with"]["fetch-depth"] == 0 + + +class TestModulesLifecyclePipeline: + """P59: modules-lifecycle pipeline — schema, byte-identical, matrix.""" + + def test_schema_is_valid_json_schema(self): + schema = json.load(open(ROOT / "schemas/modules-lifecycle-pipeline.schema.json")) + jsonschema.Draft202012Validator.check_schema(schema) + + def test_contract_validates_against_schema(self): + schema = json.load(open(ROOT / "schemas/modules-lifecycle-pipeline.schema.json")) + contract = _load_yaml("pipelines/modules-lifecycle.yml") + jsonschema.validate(contract, schema) + + def test_gitea_workflow_exists(self): + assert (ROOT / ".gitea/workflows/modules-lifecycle.yml").is_file() + + def test_github_workflow_exists(self): + assert (ROOT / ".github/workflows/modules-lifecycle.yml").is_file() + + def test_workflows_are_byte_identical(self): + gitea = open(ROOT / ".gitea/workflows/modules-lifecycle.yml", "rb").read() + github = open(ROOT / ".github/workflows/modules-lifecycle.yml", "rb").read() + assert gitea == github, "Gitea and GitHub workflows must be byte-identical" + + def test_workflow_name_matches_contract(self): + wf = _load_workflow(".gitea/workflows/modules-lifecycle.yml") + contract = _load_yaml("pipelines/modules-lifecycle.yml") + assert wf["name"] == contract["name"] + + def test_workflow_has_three_jobs(self): + wf = _load_workflow(".gitea/workflows/modules-lifecycle.yml") + assert set(wf["jobs"].keys()) == {"platform-vpc-apply", "lifecycle", "platform-vpc-destroy"} + + def test_workflow_triggers_match_contract(self): + wf = _load_workflow(".gitea/workflows/modules-lifecycle.yml") + contract = _load_yaml("pipelines/modules-lifecycle.yml") + assert wf["on"]["pull_request"]["branches"] == contract["triggers"]["pull_request"] + assert "workflow_dispatch" in wf["on"] + + def test_matrix_lists_all_12_l1_modules(self): + wf = _load_workflow(".gitea/workflows/modules-lifecycle.yml") + matrix_modules = wf["jobs"]["lifecycle"]["strategy"]["matrix"]["module"] + expected = {"s3", "kms-key", "ecr", "ecs-cluster", "iam-role", "cloudfront", + "waf", "vpc", "alb", "ecs-service", "rds", "uptime"} + assert set(matrix_modules) == expected + + def test_contract_matrix_lists_all_12_l1_modules(self): + contract = _load_yaml("pipelines/modules-lifecycle.yml") + assert set(contract["matrix"]["modules"]) == { + "s3", "kms-key", "ecr", "ecs-cluster", "iam-role", "cloudfront", + "waf", "vpc", "alb", "ecs-service", "rds", "uptime" + } + + def test_lifecycle_job_has_apply_modify_destroy_steps(self): + wf = _load_workflow(".gitea/workflows/modules-lifecycle.yml") + steps = wf["jobs"]["lifecycle"]["steps"] + step_names = [s.get("name", "") for s in steps] + assert any("Apply" in n for n in step_names), "Missing apply step" + assert any("Modify" in n for n in step_names), "Missing modify step" + assert any("Destroy" in n for n in step_names), "Missing destroy step" + + def test_platform_vpc_destroy_always_runs(self): + wf = _load_workflow(".gitea/workflows/modules-lifecycle.yml") + destroy_job = wf["jobs"]["platform-vpc-destroy"] + assert destroy_job.get("if") == "always()", "platform-vpc-destroy must always run (cleanup)" \ No newline at end of file