From fda4564a7fa41dc757d37c1b8b7bca0a4c3d7de8 Mon Sep 17 00:00:00 2001 From: Jon Chery Date: Tue, 28 Jul 2026 15:46:42 +0000 Subject: [PATCH] feat(P58): single platform VPC + deterministic env-aware state keys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit EXECUTE stage. Fixes the 4-VPC bug: adds a single shared VPC to terraform/platform, drops the vpc child from the microservice composition (references the platform VPC via data source), and makes state keys env-aware (spike/{id}/{env}/terraform.tfstate — stable across lifecycle). Platform VPC (terraform/platform/main.tf): - aws_vpc.acdl_shared (10.0.0.0/16) + 2 subnets + IGW + route table + SG - Outputs: vpc_id, subnet_ids, ecs_security_group_id Microservice composition (modules/l2/microservice/composition.json): - Dropped the vpc child (no per-contract VPC ever again). - Added data_sources block: platform_vpc → terraform_remote_state (platform). - Wires: vpc.outputs.subnet_ids → platform_vpc.outputs.subnet_ids. - Wires: platform_vpc.outputs.vpc_id → alb.inputs.vpc_id. - Wires: platform_vpc.outputs.ecs_security_group_id → service.inputs.security_group. Contract resolver (core/contract_resolver.py): - Added environment to the stack instance (stack.environment). - Added data_sources handling: pseudo-children with outputs but no resources. - data_sources propagated through fragment merge to the final stack instance. Adapter (adapters/terraform/adapter.py): - State key: spike/{stack_name}/{environment}/terraform.tfstate (env-aware). - Emits data "terraform_remote_state" "platform" block when data_sources present. - ref:platform_vpc. → data.terraform_remote_state.platform.outputs.. Tests (tests/test_adapter.py): - test_adapt_env_aware_state_key: spike/msvc/prod/terraform.tfstate. - test_adapt_emits_data_source_block: data.terraform_remote_state.platform. - test_adapt_no_vpc_for_microservice: no resource "aws_vpc" in microservice output. - Updated existing state key assertion (spike/s3/dev/terraform.tfstate). Regression: 467 passed, 0 skipped, 5 deselected. run_platform.sh --check-only passes for both microservice (9 resources, no VPC) and static-assets (5 resources). ---ci--- project: acdl phase: P58 milestone: v1.11 status: execute ---/ci--- --- adapters/terraform/adapter.py | 42 ++++++--- core/contract_resolver.py | 17 ++++ modules/l2/microservice/composition.json | 16 ++-- terraform/platform/main.tf | 109 +++++++++++++++++++++++ tests/test_adapter.py | 43 ++++++++- 5 files changed, 208 insertions(+), 19 deletions(-) diff --git a/adapters/terraform/adapter.py b/adapters/terraform/adapter.py index bb7f81c..ae60e82 100644 --- a/adapters/terraform/adapter.py +++ b/adapters/terraform/adapter.py @@ -35,24 +35,29 @@ def _module_name(resource): return resource.get("module", "").split("@")[0] -def _ref_expr(value): - """Translate a `ref:.` string to a Terraform module output interpolation - `module..`. Returns None if the value is not a ref.""" +def _ref_expr(value, data_source_names=None): + """Translate a `ref:.` string to a Terraform interpolation. + + For module resources: `module..`. + For data sources (platform-owned): `data.terraform_remote_state.platform.outputs.`. + Returns None if the value is not a ref.""" if not isinstance(value, str) or not value.startswith("ref:"): return None body = value[len("ref:"):] rid, out_name = body.split(".", 1) + if data_source_names and rid in data_source_names: + return f"data.terraform_remote_state.platform.outputs.{out_name}" return f"module.{rid}.{out_name}" -def _tf_value(value): +def _tf_value(value, data_source_names=None): """Render a Python value as a Terraform expression fragment.""" if isinstance(value, bool): return "true" if value else "false" if isinstance(value, (int, float)) and not isinstance(value, bool): return str(value) if isinstance(value, str): - ref = _ref_expr(value) + ref = _ref_expr(value, data_source_names) if ref is not None: return ref stripped = value.lstrip() @@ -69,7 +74,7 @@ def _tf_value(value): raise ValueError(f"unsupported input value type {type(value).__name__}") -def _emit_module_block(resource, terraform_dirs, repo_root): +def _emit_module_block(resource, terraform_dirs, repo_root, data_source_names=None): """Emit a `module "" { source = ... ... }` block for one resource.""" rid = resource["id"] name = _module_name(resource) @@ -81,7 +86,7 @@ def _emit_module_block(resource, terraform_dirs, repo_root): for in_name, value in resource.get("inputs", {}).items(): if in_name == "region": continue - lines.append(f" {in_name} = {_tf_value(value)}") + lines.append(f" {in_name} = {_tf_value(value, data_source_names)}") lines.append("}") return "\n".join(lines) @@ -111,6 +116,7 @@ def adapt(stack_instance, out_dir): # --- terraform.tf: required_version + required_providers + S3 backend --- stack_name = stack.get("name", "spike") + environment = stack.get("environment", "dev") terraform_tf = ( 'terraform {\n' ' required_version = ">= 1.9, < 1.10"\n' @@ -122,14 +128,30 @@ def adapt(stack_instance, out_dir): ' }\n' ' backend "s3" {\n' ' bucket = "acdl-tfstate-581513795199-us-east-1"\n' - f' key = "spike/{stack_name}/terraform.tfstate"\n' + f' key = "spike/{stack_name}/{environment}/terraform.tfstate"\n' ' region = "us-east-1"\n' ' }\n' '}\n' ) - # --- main.tf: module instantiations + root outputs --- - parts = [_emit_module_block(r, terraform_dirs, repo_root) for r in resources] + # --- data sources: emit terraform_remote_state for platform-owned resources --- + data_source_names = stack_instance.get("data_sources", []) + data_blocks = [] + if data_source_names: + data_blocks.append( + 'data "terraform_remote_state" "platform" {\n' + ' backend = "s3"\n' + ' config = {\n' + ' bucket = "acdl-tfstate-581513795199-us-east-1"\n' + ' key = "platform/terraform.tfstate"\n' + ' region = "us-east-1"\n' + ' }\n' + '}\n' + ) + + # --- main.tf: data blocks + module instantiations + root outputs --- + parts = list(data_blocks) + parts.extend(_emit_module_block(r, terraform_dirs, repo_root, set(data_source_names)) for r in resources) for out_name, out_spec in stack_outputs.items(): if isinstance(out_spec, dict) and "from" in out_spec: rid = out_spec["from"] diff --git a/core/contract_resolver.py b/core/contract_resolver.py index 1115a0a..676ae8a 100644 --- a/core/contract_resolver.py +++ b/core/contract_resolver.py @@ -232,6 +232,9 @@ def _resolve_l2(module_name, version, inputs, registry, repo_root): # that actually declares that input (P1-1 — desired_count -> aws:ecs:service, # family -> aws:ecs:task_definition). child_input_map = {} + # data_source_names: set of child ids that are data sources (not modules) + # The adapter emits `data` blocks for these instead of `module` blocks. + data_source_names = set() resources = [] # Expand children to resources @@ -300,6 +303,15 @@ def _resolve_l2(module_name, version, inputs, registry, repo_root): child_outputs[child_id] = child_out_map child_input_map[child_id] = child_in_map + # P58: Process data_sources — pseudo-children that reference platform + # infrastructure via terraform_remote_state. They have outputs but no + # resources (the adapter emits `data` blocks, not `module` blocks). + for ds in composition.get("data_sources", []): + ds_name = ds["name"] + data_source_names.add(ds_name) + ds_outputs = ds.get("outputs", []) + child_outputs[ds_name] = {out: ds_name for out in ds_outputs} + # Resolve wires to populate inputs for wire in composition.get("wires", []): to_expr = wire["to"] @@ -378,6 +390,7 @@ def _resolve_l2(module_name, version, inputs, registry, repo_root): "resources": resources, "features": features, "outputs": stack_outputs, + "data_sources": list(data_source_names), } @@ -526,6 +539,7 @@ def resolve(contract_path, repo_root=None, environment_override=None): # Merge fragments into a single stack instance all_resources = [] + all_data_sources = [] max_depth = 1 any_l2 = False merged_features = {} @@ -538,6 +552,7 @@ def resolve(contract_path, repo_root=None, environment_override=None): any_l2 = True max_depth = max(max_depth, fragment["depth"]) merged_features.update(fragment.get("features", {})) + all_data_sources.extend(fragment.get("data_sources", [])) if multi_module: # Namespace resource IDs to avoid cross-module collisions @@ -569,8 +584,10 @@ def resolve(contract_path, repo_root=None, environment_override=None): "name": contract["id"], "kind": kind, "depth": max_depth, + "environment": contract.get("environment", "dev"), }, "resources": all_resources, + "data_sources": all_data_sources, } # Add the human-readable title diff --git a/modules/l2/microservice/composition.json b/modules/l2/microservice/composition.json index 7857aa4..8c7256f 100644 --- a/modules/l2/microservice/composition.json +++ b/modules/l2/microservice/composition.json @@ -3,9 +3,8 @@ "version": "1.0.0", "kind": "l2", "depth": 1, - "description": "A composition that references six L1 primitives to deploy an ECS Fargate microservice end-to-end.", + "description": "A composition that references five L1 primitives to deploy an ECS Fargate microservice end-to-end. The VPC is owned by the platform (terraform/platform) and referenced via data source — no per-contract VPC.", "children": [ - {"id": "vpc", "module": "vpc@1.0.0"}, {"id": "cluster", "module": "ecs-cluster@1.0.0"}, {"id": "ecr", "module": "ecr@1.0.0"}, {"id": "roles", "module": "iam-role@1.0.0"}, @@ -13,21 +12,22 @@ {"id": "service", "module": "ecs-service@1.0.0"}, {"id": "kms", "module": "kms-key@1.0.0"} ], + "data_sources": [ + {"name": "platform_vpc", "type": "terraform_remote_state", "source": "platform", "outputs": ["vpc_id", "subnet_ids", "ecs_security_group_id"]} + ], "wires": [ - {"from": "contract.inputs.bucket_name", "to": "vpc.inputs.cidr", "default": "10.0.0.0/16"}, - {"from": "contract.inputs.name", "to": "vpc.inputs.name", "default": "app"}, {"from": "contract.inputs.name", "to": "alb.inputs.name", "default": "app"}, - {"from": "contract.inputs.region", "to": "vpc.inputs.region"}, {"from": "contract.inputs.region", "to": "cluster.inputs.region"}, {"from": "contract.inputs.region", "to": "ecr.inputs.region"}, {"from": "contract.inputs.region", "to": "roles.inputs.region"}, {"from": "contract.inputs.region", "to": "alb.inputs.region"}, {"from": "contract.inputs.region", "to": "service.inputs.region"}, - {"from": "vpc.outputs.subnet_ids", "to": "alb.inputs.subnets"}, - {"from": "vpc.outputs.subnet_ids", "to": "service.inputs.subnets"}, + {"from": "platform_vpc.outputs.subnet_ids", "to": "alb.inputs.subnets"}, + {"from": "platform_vpc.outputs.subnet_ids", "to": "service.inputs.subnets"}, + {"from": "platform_vpc.outputs.vpc_id", "to": "alb.inputs.vpc_id"}, + {"from": "platform_vpc.outputs.ecs_security_group_id", "to": "service.inputs.security_group"}, {"from": "cluster.outputs.cluster_arn", "to": "service.inputs.cluster_arn"}, {"from": "ecr.outputs.repository_url", "to": "service.inputs.image"}, - {"from": "roles.outputs.role_arn", "to": "service.inputs.security_group"}, {"from": "alb.outputs.target_group_arn", "to": "service.inputs.lb_target_group_arn"}, {"from": "contract.inputs.region", "to": "kms.inputs.region"}, {"from": "kms.outputs.kms_key_arn", "to": "ecr.inputs.kms_key_arn"}, diff --git a/terraform/platform/main.tf b/terraform/platform/main.tf index e23eef7..fa3a116 100644 --- a/terraform/platform/main.tf +++ b/terraform/platform/main.tf @@ -238,3 +238,112 @@ resource "aws_sns_topic" "acdl_sod_halt" { output "acdl_sod_halt_topic_arn" { value = aws_sns_topic.acdl_sod_halt.arn } + +# --------------------------------------------------------------------------- +# P58: Single shared platform VPC — all consumer stacks reference this VPC +# via terraform_remote_state (data source). No per-contract VPC ever again. +# --------------------------------------------------------------------------- + +resource "aws_vpc" "acdl_shared" { + cidr_block = "10.0.0.0/16" + tags = { + Name = "acdl-shared" + acdl:owner = "acdl" + acdl:contract = "platform" + acdl:environment = "shared" + acdl:cost-center = "acdl-default" + } +} + +resource "aws_subnet" "acdl_shared" { + count = 2 + vpc_id = aws_vpc.acdl_shared.id + cidr_block = cidrsubnet(aws_vpc.acdl_shared.cidr_block, 8, count.index + 1) + availability_zone = data.aws_availability_zones.available.names[count.index] + tags = { + Name = "acdl-shared-subnet-${count.index}" + acdl:owner = "acdl" + acdl:contract = "platform" + acdl:environment = "shared" + acdl:cost-center = "acdl-default" + } +} + +data "aws_availability_zones" "available" { + state = "available" +} + +resource "aws_internet_gateway" "acdl_shared" { + vpc_id = aws_vpc.acdl_shared.id + tags = { + Name = "acdl-shared-igw" + acdl:owner = "acdl" + acdl:contract = "platform" + acdl:environment = "shared" + acdl:cost-center = "acdl-default" + } +} + +resource "aws_route_table" "acdl_shared" { + vpc_id = aws_vpc.acdl_shared.id + route { + cidr_block = "0.0.0.0/0" + gateway_id = aws_internet_gateway.acdl_shared.id + } + tags = { + Name = "acdl-shared-rt" + acdl:owner = "acdl" + acdl:contract = "platform" + acdl:environment = "shared" + acdl:cost-center = "acdl-default" + } +} + +resource "aws_route_table_association" "acdl_shared" { + count = 2 + subnet_id = aws_subnet.acdl_shared[count.index].id + route_table_id = aws_route_table.acdl_shared.id +} + +resource "aws_security_group" "ecs" { + name = "acdl-ecs-sg" + description = "Security group for ECS Fargate services (platform VPC)" + vpc_id = aws_vpc.acdl_shared.id + + ingress { + from_port = 80 + to_port = 80 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + tags = { + Name = "acdl-ecs-sg" + acdl:owner = "acdl" + acdl:contract = "platform" + acdl:environment = "shared" + acdl:cost-center = "acdl-default" + } +} + +output "vpc_id" { + value = aws_vpc.acdl_shared.id + description = "The shared platform VPC ID. Consumer stacks reference this via terraform_remote_state." +} + +output "subnet_ids" { + value = join(",", aws_subnet.acdl_shared[*].id) + description = "Comma-separated subnet IDs in the shared platform VPC." +} + +output "ecs_security_group_id" { + value = aws_security_group.ecs.id + description = "Security group ID for ECS Fargate services in the platform VPC." +} diff --git a/tests/test_adapter.py b/tests/test_adapter.py index b82c634..ffd0bde 100644 --- a/tests/test_adapter.py +++ b/tests/test_adapter.py @@ -88,7 +88,7 @@ class TestModuleAssembly: assert 'region = "us-east-1"' in providers_tf assert 'required_providers' in terraform_tf assert 'backend "s3"' in terraform_tf - assert 'spike/s3/terraform.tfstate' in terraform_tf + assert 'spike/s3/dev/terraform.tfstate' in terraform_tf def test_adapt_emits_root_outputs(self, tmp_path): instance = json.load(open(ROOT / "modules/l1/s3/instance.json")) @@ -122,6 +122,47 @@ class TestModuleAssembly: main_tf = (tmp_path / "main.tf").read_text() assert "kms_key_arn = module.src.bucket_arn" in main_tf + def test_adapt_env_aware_state_key(self, tmp_path): + """P58: state key includes environment — spike/{name}/{env}/terraform.tfstate.""" + instance = { + "version": "1.0.0", + "stack": {"name": "msvc", "kind": "l2", "depth": 1, "environment": "prod"}, + "resources": [ + {"id": "s3", "type": "aws:s3:bucket", "module": "s3@1.0.0", + "inputs": {"bucket_name": "test", "region": "us-east-1"}} + ], + } + adapt(instance, str(tmp_path)) + terraform_tf = (tmp_path / "terraform.tf").read_text() + assert "spike/msvc/prod/terraform.tfstate" in terraform_tf + + def test_adapt_emits_data_source_block(self, tmp_path): + """P58: when data_sources is present, emit terraform_remote_state block.""" + instance = { + "version": "1.0.0", + "stack": {"name": "msvc", "kind": "l2", "depth": 1, "environment": "dev"}, + "resources": [ + {"id": "alb", "type": "aws:elbv2:loadbalancer", "module": "alb@1.0.0", + "inputs": {"subnets": "ref:platform_vpc.subnet_ids", "region": "us-east-1"}} + ], + "data_sources": ["platform_vpc"], + } + adapt(instance, str(tmp_path)) + main_tf = (tmp_path / "main.tf").read_text() + assert 'data "terraform_remote_state" "platform"' in main_tf + assert "data.terraform_remote_state.platform.outputs.subnet_ids" in main_tf + + def test_adapt_no_vpc_for_microservice(self, tmp_path): + """P58: microservice contract resolves without inline VPC resources.""" + import sys + sys.path.insert(0, str(ROOT)) + from core.contract_resolver import resolve + stack = resolve(str(ROOT / "contracts/microservice.yml")) + adapt(stack, str(tmp_path)) + main_tf = (tmp_path / "main.tf").read_text() + assert 'resource "aws_vpc"' not in main_tf + assert 'data "terraform_remote_state" "platform"' in main_tf + class TestRefExpr: def test_ref_translates_to_module_output(self):