feat(P19): central pipeline contract + shell reproducibility + output streaming (v1.4.1)
---ci--- project: acdl phase: 19 milestone: v1.4 status: execute --- Add declarative pipeline contract (schemas/pipeline.schema.json + pipelines/ci.yaml) as single source of truth for both Gitea Actions (dev) and GitHub Actions (production) workflows. Both workflow files are byte-identical and validated against the contract by 32 new tests. Add scripts/run_ci.sh for shell reproducibility — mirrors the CI pipeline locally (lint → test → check-only), exits 0 with 'CI PIPELINE OK'. Update scripts/run_platform.sh to stream output by default: terraform init/validate/plan via tee, Checkov compliance results with per-record severity/rule/pass-fail, and emitted Terraform in --check-only. New --quiet flag for log-only mode. Requirements: REQ-43 (central pipeline contract), REQ-44 (shell reproducibility), REQ-45 (output streaming). 122 tests pass (90 + 32).
This commit is contained in:
Jon Chery
+67
-11
@@ -3,11 +3,16 @@
|
||||
#
|
||||
# Modes:
|
||||
# --check-only (offline, no AWS/Checkov/DynamoDB — for CI)
|
||||
# load IR -> adapter -> validate output structure -> exit 0
|
||||
# load IR -> adapter -> stream emitted TF -> validate structure -> exit 0
|
||||
# --plan-only (requires AWS creds, no Checkov/outbox)
|
||||
# load IR -> adapter -> terraform init/validate/plan -> exit 0
|
||||
# load IR -> adapter -> terraform init/validate/plan (streamed) -> exit 0
|
||||
# (default) (requires AWS creds + Checkov + DynamoDB)
|
||||
# load IR -> adapter -> terraform plan -> Checkov -> confidence -> outbox
|
||||
# load IR -> adapter -> terraform plan (streamed) -> Checkov (streamed) ->
|
||||
# confidence -> outbox
|
||||
#
|
||||
# Flags:
|
||||
# --quiet suppress terraform/checkov streaming (output to log only)
|
||||
# default: stream to stdout so the user sees what is happening
|
||||
#
|
||||
# NOTE: contract resolution (contract_resolver.py) was removed when the
|
||||
# thin-composition layer was taken out. The pipeline now starts from a
|
||||
@@ -22,16 +27,29 @@ cd "$ROOT"
|
||||
|
||||
CHECK_ONLY=0
|
||||
PLAN_ONLY=0
|
||||
QUIET=0
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--check-only) CHECK_ONLY=1 ;;
|
||||
--plan-only) PLAN_ONLY=1 ;;
|
||||
--quiet) QUIET=1 ;;
|
||||
*) echo "FAIL: unknown argument: $arg" >&2; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
fail() { echo "FAIL: $*" >&2; exit 1; }
|
||||
|
||||
# stream: pipe a command's stdout+stderr to both a log file and the
|
||||
# terminal (unless --quiet). Usage: stream <logfile> -- <command...>
|
||||
stream() {
|
||||
local log="$1"; shift
|
||||
if [ "$QUIET" = "1" ]; then
|
||||
"$@" > "$log" 2>&1
|
||||
else
|
||||
"$@" 2>&1 | tee "$log"
|
||||
fi
|
||||
}
|
||||
|
||||
CONTRACT_ID="11111111-1111-1111-1111-111111111111" # spike fixed UUID
|
||||
WORK="/tmp/spike_e2e"
|
||||
rm -rf "$WORK"; mkdir -p "$WORK"
|
||||
@@ -46,6 +64,13 @@ echo "=== Step 3: adapter compiles IR -> terraform/spike/*.tf (regenerate) ==="
|
||||
python3 adapters/terraform/adapter.py "$WORK/spike_ir.json" terraform/spike || fail "adapter failed"
|
||||
echo "adapter: emitted terraform/spike/{main.tf,terraform.tf,providers.tf}"
|
||||
|
||||
if [ "$QUIET" = "0" ]; then
|
||||
echo ""
|
||||
echo "--- emitted terraform/spike/main.tf ---"
|
||||
cat terraform/spike/main.tf
|
||||
echo "--- end main.tf ---"
|
||||
fi
|
||||
|
||||
if [ "$CHECK_ONLY" = "1" ]; then
|
||||
echo ""
|
||||
echo "=== Step 3b: validate adapter output structure (offline) ==="
|
||||
@@ -86,9 +111,20 @@ export AWS_DEFAULT_REGION="$AWS_DEFAULT_REGION"
|
||||
|
||||
echo "=== Step 4: terraform init + validate + plan -lock=false (real AWS) ==="
|
||||
cd terraform/spike
|
||||
terraform init -reconfigure -lock=false -input=false >> "$WORK/tf.log" 2>&1 || fail "terraform init failed"
|
||||
terraform validate >> "$WORK/tf.log" 2>&1 || fail "terraform validate failed"
|
||||
terraform plan -lock=false -input=false -out=tfplan >> "$WORK/tf.log" 2>&1 || fail "terraform plan failed"
|
||||
|
||||
echo ""
|
||||
echo "--- terraform init ---"
|
||||
stream "$WORK/tf-init.log" terraform init -reconfigure -lock=false -input=false || fail "terraform init failed"
|
||||
|
||||
echo ""
|
||||
echo "--- terraform validate ---"
|
||||
stream "$WORK/tf-validate.log" terraform validate || fail "terraform validate failed"
|
||||
|
||||
echo ""
|
||||
echo "--- terraform plan ---"
|
||||
stream "$WORK/tf-plan.log" terraform plan -lock=false -input=false -out=tfplan || fail "terraform plan failed"
|
||||
|
||||
echo ""
|
||||
echo "terraform plan OK (1 to add, 0 to change, 0 to destroy expected)"
|
||||
cd "$ROOT"
|
||||
|
||||
@@ -98,16 +134,35 @@ if [ "$PLAN_ONLY" = "1" ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "=== Step 5: run Checkov on terraform/spike/main.tf ==="
|
||||
checkov -f terraform/spike/main.tf --framework terraform -o json --soft-fail > "$WORK/checkov.json" 2> "$WORK/checkov.err"
|
||||
if [ "$QUIET" = "0" ]; then
|
||||
checkov -f terraform/spike/main.tf --framework terraform -o json --soft-fail 2>&1 | tee "$WORK/checkov.json"
|
||||
else
|
||||
checkov -f terraform/spike/main.tf --framework terraform -o json --soft-fail > "$WORK/checkov.json" 2> "$WORK/checkov.err"
|
||||
fi
|
||||
[ -s "$WORK/checkov.json" ] || fail "checkov produced no output"
|
||||
echo "checkov: $(python3 -c "import json; d=json.load(open('$WORK/checkov.json')); print(len(d.get('results',{}).get('failed_checks',[])), 'failed,', len(d.get('results',{}).get('passed_checks',[])), 'passed')")"
|
||||
echo ""
|
||||
echo "checkov summary: $(python3 -c "import json; d=json.load(open('$WORK/checkov.json')); print(len(d.get('results',{}).get('failed_checks',[])), 'failed,', len(d.get('results',{}).get('passed_checks',[])), 'passed')")"
|
||||
|
||||
echo "=== Step 6: Checkov adapter -> PolicyCheckResult list ==="
|
||||
echo ""
|
||||
echo "=== Step 6: Checkov adapter -> PolicyCheckResult (compliance details) ==="
|
||||
python3 adapters/terraform/policy/checkov_adapter.py "$WORK/checkov.json" "$CONTRACT_ID" > "$WORK/pcr.json" || fail "checkov adapter failed"
|
||||
PCR_COUNT=$(python3 -c "import json; print(len(json.load(open('$WORK/pcr.json'))))")
|
||||
echo "PolicyCheckResult: $PCR_COUNT record(s)"
|
||||
python3 -c "
|
||||
import json
|
||||
pcrs = json.load(open('$WORK/pcr.json'))
|
||||
print(f'PolicyCheckResult: {len(pcrs)} record(s)')
|
||||
print()
|
||||
for pcr in pcrs:
|
||||
sev = pcr.get('severity', 'info')
|
||||
res = pcr.get('result', 'unknown')
|
||||
rule = pcr.get('ruleId', 'unknown')
|
||||
msg = pcr.get('message', '')
|
||||
marker = 'PASS' if res == 'pass' else 'FAIL' if res == 'fail' else 'SKIP' if res == 'skipped' else res.upper()
|
||||
print(f' [{marker}] {sev:8s} {rule:30s} {msg}')
|
||||
"
|
||||
|
||||
echo ""
|
||||
echo "=== Step 7: confidence signal compute ==="
|
||||
python3 <<PY > "$WORK/signal.json" || fail "confidence signal failed"
|
||||
import json
|
||||
@@ -129,6 +184,7 @@ SCORE=$(python3 -c "import json; print(round(json.load(open('$WORK/signal.json')
|
||||
echo "confidence: score=$SCORE band=$BAND"
|
||||
[ "$BAND" = "pass" ] || fail "confidence band is $BAND, expected pass for dev"
|
||||
|
||||
echo ""
|
||||
echo "=== Step 8: write evidence event to DynamoDB outbox ==="
|
||||
python3 <<PY > "$WORK/event.json" || fail "event build failed"
|
||||
import json, datetime
|
||||
|
||||
Reference in New Issue
Block a user