feat(P61): L2 lifecycle pipeline — extend matrix + workflows + tests

Extend the modules-lifecycle pipeline with L2 composition modules (static-assets, microservice) per REQ-128: - pipelines/modules-lifecycle.yml: added l2-lifecycle-apply/modify/destroy stages + l2_modules matrix entry - .gitea/workflows/modules-lifecycle.yml + .github/workflows/modules-lifecycle.yml: added l2-lifecycle job (byte-identical), matrix over [static-assets, microservice], needs ci-vpc-apply, has apply/modify/destroy steps. ci-vpc-destroy now needs both [lifecycle, l2-lifecycle]. - schemas/modules-lifecycle-pipeline.schema.json: added l2_modules to matrix - scripts/run_l2_lifecycle_test.sh + run_l2_lifecycle_destroy.sh: L2 wrappers that set ACDL_REMOTE_STATE_KEY=spike/ci-vpc/terraform.tfstate so the microservice composition's terraform_remote_state reads from the CI VPC - adapters/terraform/adapter.py: parameterized remote_state key via ACDL_REMOTE_STATE_KEY env var (default: platform/terraform.tfstate) - modules/l2/static-assets/examples/complex.yml: fixed bucket_name to match simple (my-static-site) so terraform modifies in-place (adds CDN + WAF) - modules/l2/microservice/examples/complex.yml: fixed bucket_name to match simple (my-microservice-demo), added desired_count:2 (modify variant) - tests/test_pipeline_contract.py: 7 new L2 tests (l2 job exists, matrix lists both modules, apply/modify/destroy steps, needs ci-vpc-apply, ci-vpc-destroy needs both, contract matrix lists l2_modules) - pipelines/README.md: updated stages for L2 Regression: 485 passed, 5 deselected. Gitea + GitHub workflows byte-identical. ---ci--- project: acdl phase: P61 milestone: v1.11 status: execute ---/ci---
2026-07-28 20:17:27 +00:00
parent 0c5c4d1c40
commit 361fe600a9
11 changed files with 227 additions and 21 deletions
@@ -4,13 +4,18 @@
 # apply→modify→destroy against live AWS. No per-module Python. The "test" =
 # the pipeline cell going green.
 #
+# Also matrix-runs L2 composition modules (static-assets, microservice) through
+# the same apply→modify→destroy lifecycle. L2 = composition only (no L2
+# terraform files); the composition must be deterministic.
+#
 # This workflow implements pipelines/modules-lifecycle.yml (byte-identical
 # in .gitea/workflows/ and .github/workflows/).
 #
 # A short-lived CI VPC (terraform/ci-vpc/) is created before testing VPC-dependent
-# modules (alb, ecs-service, rds, uptime) and destroyed after all tests complete.
-# The CI VPC is separate from the long-lived platform VPC. Outputs are read
-# from the S3 state by each lifecycle job (no artifact passing needed).
+# modules (alb, ecs-service, rds, uptime, and L2 microservice) and destroyed
+# after all tests complete. The CI VPC is separate from the long-lived platform
+# VPC. Outputs are read from the S3 state by each lifecycle job (no artifact
+# passing needed).
 name: acdl-modules-lifecycle

 on:
@@ -22,7 +27,7 @@ permissions:
  contents: read

 jobs:
-  # Prerequisite: apply the short-lived CI VPC (needed by VPC-dependent L1s)
+  # Prerequisite: apply the short-lived CI VPC (needed by VPC-dependent L1s + L2 microservice)
  ci-vpc-apply:
    name: CI VPC apply
    runs-on: ubuntu-latest
@@ -92,10 +97,59 @@ jobs:
          AWS_DEFAULT_REGION: us-east-1
        run: bash scripts/run_lifecycle_destroy.sh ${{ matrix.module }} /tmp/ci-vpc-outputs.json

+  # L2 lifecycle matrix: apply simple → apply complex (modify) → destroy
+  l2-lifecycle:
+    name: L2 lifecycle (${{ matrix.module }})
+    needs: ci-vpc-apply
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        module: [static-assets, microservice]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Install dependencies
+        run: pip install jsonschema pyyaml boto3
+      - name: Install Terraform 1.9.*
+        run: |
+          wget -qO- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp.gpg
+          echo "deb [signed-by=/usr/share/keyrings/hashicorp.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
+          sudo apt-get update && sudo apt-get install -y terraform=1.9.*
+      - name: Read CI VPC outputs
+        working-directory: terraform/ci-vpc
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.ACDL_AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.ACDL_AWS_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: us-east-1
+        run: |
+          terraform init -input=false -lock=false
+          terraform output -json > /tmp/ci-vpc-outputs.json
+      - name: Apply (simple)
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.ACDL_AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.ACDL_AWS_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: us-east-1
+        run: bash scripts/run_l2_lifecycle_test.sh ${{ matrix.module }} simple /tmp/ci-vpc-outputs.json
+      - name: Modify (complex)
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.ACDL_AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.ACDL_AWS_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: us-east-1
+        run: bash scripts/run_l2_lifecycle_test.sh ${{ matrix.module }} complex /tmp/ci-vpc-outputs.json
+      - name: Destroy
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.ACDL_AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.ACDL_AWS_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: us-east-1
+        run: bash scripts/run_l2_lifecycle_destroy.sh ${{ matrix.module }} /tmp/ci-vpc-outputs.json
+
  # Cleanup: destroy the CI VPC (always runs, even if lifecycle fails)
  ci-vpc-destroy:
    name: CI VPC destroy
-    needs: lifecycle
+    needs: [lifecycle, l2-lifecycle]
    runs-on: ubuntu-latest
    if: always()
    steps:
@@ -4,13 +4,18 @@
 # apply→modify→destroy against live AWS. No per-module Python. The "test" =
 # the pipeline cell going green.
 #
+# Also matrix-runs L2 composition modules (static-assets, microservice) through
+# the same apply→modify→destroy lifecycle. L2 = composition only (no L2
+# terraform files); the composition must be deterministic.
+#
 # This workflow implements pipelines/modules-lifecycle.yml (byte-identical
 # in .gitea/workflows/ and .github/workflows/).
 #
 # A short-lived CI VPC (terraform/ci-vpc/) is created before testing VPC-dependent
-# modules (alb, ecs-service, rds, uptime) and destroyed after all tests complete.
-# The CI VPC is separate from the long-lived platform VPC. Outputs are read
-# from the S3 state by each lifecycle job (no artifact passing needed).
+# modules (alb, ecs-service, rds, uptime, and L2 microservice) and destroyed
+# after all tests complete. The CI VPC is separate from the long-lived platform
+# VPC. Outputs are read from the S3 state by each lifecycle job (no artifact
+# passing needed).
 name: acdl-modules-lifecycle

 on:
@@ -22,7 +27,7 @@ permissions:
  contents: read

 jobs:
-  # Prerequisite: apply the short-lived CI VPC (needed by VPC-dependent L1s)
+  # Prerequisite: apply the short-lived CI VPC (needed by VPC-dependent L1s + L2 microservice)
  ci-vpc-apply:
    name: CI VPC apply
    runs-on: ubuntu-latest
@@ -92,10 +97,59 @@ jobs:
          AWS_DEFAULT_REGION: us-east-1
        run: bash scripts/run_lifecycle_destroy.sh ${{ matrix.module }} /tmp/ci-vpc-outputs.json

+  # L2 lifecycle matrix: apply simple → apply complex (modify) → destroy
+  l2-lifecycle:
+    name: L2 lifecycle (${{ matrix.module }})
+    needs: ci-vpc-apply
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        module: [static-assets, microservice]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Install dependencies
+        run: pip install jsonschema pyyaml boto3
+      - name: Install Terraform 1.9.*
+        run: |
+          wget -qO- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp.gpg
+          echo "deb [signed-by=/usr/share/keyrings/hashicorp.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
+          sudo apt-get update && sudo apt-get install -y terraform=1.9.*
+      - name: Read CI VPC outputs
+        working-directory: terraform/ci-vpc
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.ACDL_AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.ACDL_AWS_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: us-east-1
+        run: |
+          terraform init -input=false -lock=false
+          terraform output -json > /tmp/ci-vpc-outputs.json
+      - name: Apply (simple)
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.ACDL_AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.ACDL_AWS_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: us-east-1
+        run: bash scripts/run_l2_lifecycle_test.sh ${{ matrix.module }} simple /tmp/ci-vpc-outputs.json
+      - name: Modify (complex)
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.ACDL_AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.ACDL_AWS_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: us-east-1
+        run: bash scripts/run_l2_lifecycle_test.sh ${{ matrix.module }} complex /tmp/ci-vpc-outputs.json
+      - name: Destroy
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.ACDL_AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.ACDL_AWS_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: us-east-1
+        run: bash scripts/run_l2_lifecycle_destroy.sh ${{ matrix.module }} /tmp/ci-vpc-outputs.json
+
  # Cleanup: destroy the CI VPC (always runs, even if lifecycle fails)
  ci-vpc-destroy:
    name: CI VPC destroy
-    needs: lifecycle
+    needs: [lifecycle, l2-lifecycle]
    runs-on: ubuntu-latest
    if: always()
    steps:
@@ -138,12 +138,13 @@ def adapt(stack_instance, out_dir):
    data_source_names = stack_instance.get("data_sources", [])
    data_blocks = []
    if data_source_names:
+        remote_state_key = os.environ.get("ACDL_REMOTE_STATE_KEY", "platform/terraform.tfstate")
        data_blocks.append(
            'data "terraform_remote_state" "platform" {\n'
            '  backend = "s3"\n'
            '  config = {\n'
            '    bucket = "acdl-tfstate-581513795199-us-east-1"\n'
-            '    key    = "platform/terraform.tfstate"\n'
+            f'    key    = "{remote_state_key}"\n'
            '    region = "us-east-1"\n'
            '  }\n'
            '}\n'
@@ -1,15 +1,18 @@
-# Complex microservice with ALB + env vars + health check
+# Complex microservice with ALB + env vars + desired_count=2
+# Modify variant: same bucket_name/family/image as simple (in-place modify:
+# adds ALB, env vars, scales desired_count 1->2)
 environment: dev
 id: msvc
 infrastructure:
  microservice:
    inputs:
-      bucket_name: my-production-microservice
+      bucket_name: my-microservice-demo
+      desired_count: 2
      env:
        ENVIRONMENT: production
        LOG_LEVEL: info
      image: public.ecr.aws/docker/library/nginx:latest
-      port: 8080
+      port: 80
      region: us-east-1
    version: 1.0.0
 name: microservice
@@ -1,11 +1,11 @@
 # Complex static-assets deployment (S3 + CloudFront + WAF)
-# The full production stack: S3 origin + CloudFront CDN edge + WAF protection.
+# Modify variant: same bucket_name as simple (in-place modify, adds CDN + WAF)
 environment: dev
 id: assets
 infrastructure:
  static-assets:
    inputs:
-      bucket_name: my-production-static-site
+      bucket_name: my-static-site
      default_ttl: 3600
      max_ttl: 86400
      price_class: PriceClass_100
@@ -10,7 +10,7 @@ ACDL uses declarative pipeline contracts (YAML) as the single source of truth. B
 | --- | --- | --- | --- |
 | ACDL CI | `ci.yml` | `lint`, `test`, `check-only` | push/PR to `main` |
 | ACDL Deploy | `contract.yml` | `validate-contract`, `resolve-stack`, `terraform-plan`, `checkov`, `confidence`, `apply`, `publish-outputs`, `deploy-uptime`, `comment-outputs` | push/PR to `main` (consumer repos via `workflow_call`) |
-| ACDL Modules Lifecycle | `modules-lifecycle.yml` | `platform-vpc-apply`, `lifecycle-apply`, `lifecycle-modify`, `lifecycle-destroy`, `platform-vpc-destroy` | PR to `main` + `workflow_dispatch` |
+| ACDL Modules Lifecycle | `modules-lifecycle.yml` | `platform-vpc-apply`, `lifecycle-apply`, `lifecycle-modify`, `lifecycle-destroy`, `l2-lifecycle-apply`, `l2-lifecycle-modify`, `l2-lifecycle-destroy`, `platform-vpc-destroy` | PR to `main` + `workflow_dispatch` |

 ## How to Write a Pipeline

@@ -49,7 +49,24 @@ stages:
    required: false
    description: "Destroy the shared platform VPC (cleanup — runs even if lifecycle steps fail)"

+  # L2 lifecycle stages (composition modules)
+  - name: l2-lifecycle-apply
+    command: bash scripts/run_platform.sh --apply "modules/l2/${L2_MODULE}/examples/simple.yml"
+    required: true
+    description: "Apply the L2 module's simple example contract (terraform apply)"
+
+  - name: l2-lifecycle-modify
+    command: bash scripts/run_platform.sh --apply "modules/l2/${L2_MODULE}/examples/complex.yml"
+    required: true
+    description: "Apply the L2 module's complex example (same state key — terraform modifies)"
+
+  - name: l2-lifecycle-destroy
+    command: bash scripts/run_platform.sh --destroy "modules/l2/${L2_MODULE}/examples/complex.yml"
+    required: true
+    description: "Destroy the L2 module's resources (terraform destroy)"
+
 matrix:
  modules: [s3, kms-key, ecr, ecs-cluster, iam-role, cloudfront, waf, vpc, alb, ecs-service, rds, uptime]
  vpc_dependent: [alb, ecs-service, rds, uptime]
-  independent: [s3, kms-key, ecr, ecs-cluster, iam-role, cloudfront, waf, vpc]
+  independent: [s3, kms-key, ecr, ecs-cluster, iam-role, cloudfront, waf, vpc]
+  l2_modules: [static-assets, microservice]
@@ -61,6 +61,11 @@
          "type": "array",
          "items": {"type": "string"},
          "description": "Modules that run without the platform VPC."
+        },
+        "l2_modules": {
+          "type": "array",
+          "items": {"type": "string"},
+          "description": "L2 composition module names to test."
        }
      }
    }
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# scripts/run_l2_lifecycle_destroy.sh — run a single L2 module lifecycle destroy.
+#
+# Usage: run_l2_lifecycle_destroy.sh <module> [ci-vpc-outputs.json]
+#
+# Wraps run_platform.sh for L2 composition modules in the modules-lifecycle
+# pipeline. Sets ACDL_REMOTE_STATE_KEY to point to the CI VPC state.
+set -euo pipefail
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+cd "$ROOT"
+
+MODULE="$1"
+
+CONTRACT="modules/l2/${MODULE}/examples/complex.yml"
+
+# Point terraform_remote_state to the CI VPC state (not the platform VPC)
+export ACDL_REMOTE_STATE_KEY="spike/ci-vpc/terraform.tfstate"
+
+# Run the platform lifecycle destroy command
+bash scripts/run_platform.sh --destroy "$CONTRACT"
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+# scripts/run_l2_lifecycle_test.sh — run a single L2 module lifecycle apply/modify.
+#
+# Usage: run_l2_lifecycle_test.sh <module> <example> [ci-vpc-outputs.json]
+#
+# Wraps run_platform.sh for L2 composition modules in the modules-lifecycle
+# pipeline. Sets ACDL_REMOTE_STATE_KEY to point to the CI VPC state so the
+# microservice composition's terraform_remote_state data source reads from
+# the short-lived CI VPC (not the long-lived platform VPC).
+set -euo pipefail
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+cd "$ROOT"
+
+MODULE="$1"
+EXAMPLE="$2"  # simple or complex
+
+CONTRACT="modules/l2/${MODULE}/examples/${EXAMPLE}.yml"
+
+# Point terraform_remote_state to the CI VPC state (not the platform VPC)
+export ACDL_REMOTE_STATE_KEY="spike/ci-vpc/terraform.tfstate"
+
+# Run the platform lifecycle command
+bash scripts/run_platform.sh --apply "$CONTRACT"
@@ -549,9 +549,9 @@ class TestModulesLifecyclePipeline:
        contract = _load_yaml("pipelines/modules-lifecycle.yml")
        assert wf["name"] == contract["name"]

-    def test_workflow_has_three_jobs(self):
+    def test_workflow_has_four_jobs(self):
        wf = _load_workflow(".gitea/workflows/modules-lifecycle.yml")
-        assert set(wf["jobs"].keys()) == {"ci-vpc-apply", "lifecycle", "ci-vpc-destroy"}
+        assert set(wf["jobs"].keys()) == {"ci-vpc-apply", "lifecycle", "l2-lifecycle", "ci-vpc-destroy"}

    def test_workflow_triggers_match_contract(self):
        wf = _load_workflow(".gitea/workflows/modules-lifecycle.yml")
@@ -584,4 +584,33 @@ class TestModulesLifecyclePipeline:
    def test_platform_vpc_destroy_always_runs(self):
        wf = _load_workflow(".gitea/workflows/modules-lifecycle.yml")
        destroy_job = wf["jobs"]["ci-vpc-destroy"]
-        assert destroy_job.get("if") == "always()", "ci-vpc-destroy must always run (cleanup)"
+        assert destroy_job.get("if") == "always()", "ci-vpc-destroy must always run (cleanup)"
+
+    def test_l2_lifecycle_job_exists(self):
+        wf = _load_workflow(".gitea/workflows/modules-lifecycle.yml")
+        assert "l2-lifecycle" in wf["jobs"]
+
+    def test_l2_matrix_lists_both_l2_modules(self):
+        wf = _load_workflow(".gitea/workflows/modules-lifecycle.yml")
+        matrix_modules = wf["jobs"]["l2-lifecycle"]["strategy"]["matrix"]["module"]
+        assert set(matrix_modules) == {"static-assets", "microservice"}
+
+    def test_l2_lifecycle_job_has_apply_modify_destroy_steps(self):
+        wf = _load_workflow(".gitea/workflows/modules-lifecycle.yml")
+        steps = wf["jobs"]["l2-lifecycle"]["steps"]
+        step_names = [s.get("name", "") for s in steps]
+        assert any("Apply" in n for n in step_names), "Missing L2 apply step"
+        assert any("Modify" in n for n in step_names), "Missing L2 modify step"
+        assert any("Destroy" in n for n in step_names), "Missing L2 destroy step"
+
+    def test_l2_lifecycle_job_needs_ci_vpc_apply(self):
+        wf = _load_workflow(".gitea/workflows/modules-lifecycle.yml")
+        assert wf["jobs"]["l2-lifecycle"]["needs"] == "ci-vpc-apply"
+
+    def test_ci_vpc_destroy_needs_both_lifecycle_and_l2(self):
+        wf = _load_workflow(".gitea/workflows/modules-lifecycle.yml")
+        assert set(wf["jobs"]["ci-vpc-destroy"]["needs"]) == {"lifecycle", "l2-lifecycle"}
+
+    def test_contract_matrix_lists_l2_modules(self):
+        contract = _load_yaml("pipelines/modules-lifecycle.yml")
+        assert set(contract["matrix"]["l2_modules"]) == {"static-assets", "microservice"}