From 72b359c9a902b035c8a5816437d38aba808b948e Mon Sep 17 00:00:00 2001 From: grimacing Date: Tue, 21 Jul 2026 13:42:25 +0000 Subject: [PATCH] ship: phase-04 pipeline-and-approval-gates (v1.0.4) Squash merge of phase/04-pipeline-and-approval-gates; pipeline.yml + issue-to-contract.yml + finalize_evidence.py; verify_phase04.sh green; 1 P0 fixed (shell injection). --- .ciagent/ARCHITECTURE.md | 41 ++++ .ciagent/PLAN.md | 118 +++++------ .ciagent/PROJECT.md | 6 +- .ciagent/REQUIREMENTS.md | 4 +- .ciagent/ROADMAP.md | 2 +- .gitea/workflows/pipeline.yml | 158 +++++++++++---- .../.gitea/workflows/issue-to-contract.yml | 161 ++++++++++++--- scripts/finalize_evidence.py | 182 +++++++++++++++++ scripts/verify_phase04.sh | 186 ++++++++++++++++++ 9 files changed, 720 insertions(+), 138 deletions(-) create mode 100755 scripts/finalize_evidence.py create mode 100755 scripts/verify_phase04.sh diff --git a/.ciagent/ARCHITECTURE.md b/.ciagent/ARCHITECTURE.md index d6696f4..de17d56 100644 --- a/.ciagent/ARCHITECTURE.md +++ b/.ciagent/ARCHITECTURE.md @@ -49,6 +49,47 @@ The demo is a three-repo, stub-driven system that simulates an autonomous cloud | `acdl-evidence` repo | Pages host for `audit.json` + `index.html` timeline | Read-only for the pipeline; written at finalize stage | evidence_writer.py output | | Reusable pipeline workflow | Dev → QA → Prod → Finalize stages with environment gates | Gitea Actions; calls core scripts | All core scripts | +## Phase 04 pipeline topology (research) + +Gitea Actions limitations (confirmed via research, supersedes any +GitHub-Actions assumptions): + +- `actions/upload-artifact@v3` / `download-artifact@v3` work; v4 is NOT supported by act_runner. +- Artifacts are scoped to a single workflow run; **re-dispatch starts a new run, so artifacts do NOT survive between dispatches**. +- `workflow_dispatch` API: `POST /api/v1/repos/{owner}/{repo}/actions/workflows/{filename}.yml/dispatches` with body `{ "ref": "", "inputs": {...} }`. +- `on: workflow_call` + `uses: //.gitea/workflows/@` works; pin to `@milestone/v1.0-initial`. +- `actions/checkout@v4` supports cross-repo (pass `repository:` + `ref:` + `token: ${{ secrets.GITEA_TOKEN }}` for private repos). +- File-contents API: POST to create (201), PUT to update (must include current `sha`, obtained via GET). +- `${{ secrets.GITEA_TOKEN }}` is a manually-created PAT secret on the `acdl` + `acdl-contracts` repos; the auto-injected token is current-repo only and cannot cross-repo. +- No native approval-gate UI; gates are `workflow_dispatch` inputs (`approve_qa`, `approve_prod`). + +### Approval-gate + state-persistence approach (D-027, D-028 refined) + +Because re-dispatch starts a new run and artifacts do not survive: + +1. The pipeline workflow has `workflow_dispatch` inputs: + - `contract-ref` (string; default `main`) — the ref on `acdl-contracts` carrying the contract. + - `approve_qa` (boolean; default `false`) — the human sets this to `true` to advance past QA. + - `approve_prod` (boolean; default `false`) — the human sets this to `true` to advance past Prod. + +2. Each stage job (`dev`, `qa-gate`, `prod-gate`, `finalize`) writes its evidence to `acdl-evidence` via the file-contents API (PUT `audit.json` with the new event appended). This is the persistent state across re-dispatches. + +3. **Dev stage** (always runs on dispatch): check out `acdl` + `acdl-contracts@`, run `policy_checker.py` + `confidence_signal.py`; if `score < 0.50`, write a `dev_rejected` evidence event and exit 1 (Act 4). Otherwise run `mock_executor.sh`, write a `dev_applied` evidence event, and exit 0. The run ends here. + +4. **QA gate** (next dispatch with `approve_qa=true`): check out, run `evidence_writer.py --stage qa --event "qa approved"`, commit updated `audit.json` to `acdl-evidence`. Exit 0. The run ends. + +5. **Prod gate** (next dispatch with `approve_prod=true`): same as QA but `--stage prod`. + +6. **Finalize** (same dispatch as Prod, chained via `needs: prod-gate`): write the `finalize` evidence event, commit final `audit.json` to `acdl-evidence`. The raw URL now serves the updated timeline. + +Because each stage is a separate dispatch, the workflow file uses `if:` conditions on each job: +- `dev` runs when `inputs.approve_qa != true && inputs.approve_prod != true` (the initial dispatch). +- `qa-gate` runs when `inputs.approve_qa == true && inputs.approve_prod != true`. +- `prod-gate` runs when `inputs.approve_prod == true`. +- `finalize` runs after `prod-gate` (`needs: prod-gate`). + +This means a full pipeline = 3 dispatches (initial, qa-approve, prod-approve). The human drives each via the Gitea UI or the dispatch API. + ## Data Flow 1. A `contract.yaml` arrives either by direct push (L3A) or by the issue workflow running `l3b_agent_stub.py` (L3B). diff --git a/.ciagent/PLAN.md b/.ciagent/PLAN.md index f8d91ce..cb0f5de 100644 --- a/.ciagent/PLAN.md +++ b/.ciagent/PLAN.md @@ -1,102 +1,88 @@ --- -phase: 03 -name: l2-modules-and-core-scripts +phase: 04 +name: pipeline-and-approval-gates milestone: v1.0 milestone_type: feature status: planned -requirements: [REQ-04, REQ-05, REQ-06, REQ-07, REQ-08, REQ-11] +requirements: [REQ-10, REQ-12] must_haves: - - "4 L2 module folders exist under modules/l2/ with exact names: l2-invoice-service, l2-commodity-price-feed, l2-energy-analytics-api, l2-regulatory-reporting" - - "Each L2 has a manifest.yaml matching the D-020 schema (name, kind: l2, description, l1s: list of {name, inputs: map})" - - "Each L2 references 5 L1s by name; all referenced L1 names exist in modules/l1/" - - "mock_executor.sh reads a contract.yaml, resolves the L2, invokes each L1 mock_apply.sh, writes state.json per D-022" - - "policy_checker.py exits 1 with 'POLICY_VIOLATION:PUBLIC_INGRESS' on public-ingress:true; exits 0 with 'POLICY_PASS' otherwise (D-025)" - - "confidence_signal.py prints {score, reason} JSON; score 0.90 on pass, 0.40 on policy fail (D-024)" - - "evidence_writer.py appends an event to audit.json with a valid canonical-JSON SHA-256 hash chain (D-023)" - - "l3b_agent_stub.py maps the Act 3 example issue text to l2-commodity-price-feed and emits a valid contract.yaml (D-026/D-008)" - - "scripts/verify_phase03.sh passes: validates all 4 L2s, runs each core script with a sample contract, confirms hash chain integrity" + - ".gitea/workflows/pipeline.yml is a real workflow (no placeholders) with workflow_dispatch inputs (contract-ref, approve_qa, approve_prod) and 4 jobs (dev, qa-gate, prod-gate, finalize) with correct if: conditions per D-027/D-028" + - "The dev job checks out acdl + acdl-contracts@contract-ref, runs policy_checker.py + confidence_signal.py + mock_executor.sh, and writes a dev_applied or dev_rejected evidence event to acdl-evidence via the file-contents API" + - "The qa-gate job (approve_qa=true) writes a qa_approved evidence event" + - "The prod-gate + finalize jobs (approve_prod=true) write prod_approved + finalize evidence events, committing the final audit.json to acdl-evidence main" + - "contracts-repo/.gitea/workflows/issue-to-contract.yml is a real workflow (no placeholders) that checks out l3b_agent_stub.py from acdl, parses the Issue body, commits contract.yaml to contract/ on acdl-contracts, and dispatches pipeline.yml with contract-ref=contract/" + - "scripts/finalize_evidence.py helper exists: given an audit.json file path, GETs the current file sha from acdl-evidence, PUTs the new content via the file-contents API. Used by the workflow steps to commit evidence updates." + - "scripts/verify_phase04.sh passes: validates the workflow YAML syntax, confirms job if: conditions, confirms issue-to-contract.yml references the dispatch API, runs a dry-run of finalize_evidence.py against a temp audit.json (mock API)" verification: - typecheck: "bash -n scripts/*.sh && python3 -m py_compile scripts/*.py && python3 -c 'import yaml, glob; [yaml.safe_load(open(f)) for f in glob.glob(\"modules/l2/*/manifest.yaml\")]'" - test: "scripts/verify_phase03.sh" + typecheck: "bash -n scripts/*.sh && python3 -m py_compile scripts/*.py && python3 -c 'import yaml; yaml.safe_load(open(\".gitea/workflows/pipeline.yml\")); yaml.safe_load(open(\"contracts-repo/.gitea/workflows/issue-to-contract.yml\"))'" + test: "scripts/verify_phase04.sh" build: no-op --- -# Phase 03 — l2-modules-and-core-scripts PLAN +# Phase 04 — pipeline-and-approval-gates PLAN ## Goal -Create the 4 L2 composition modules and the 5 core scripts. After this -phase, the demo has every primitive needed for Phase 04 to wire the -pipeline and Phase 05 to render the evidence UI. +Replace the Phase 01 workflow skeletons with real implementations. Wire +the Dev → QA → Prod → Finalize pipeline with the 3-dispatch approval-gate +pattern (D-027/D-028), and the issue-to-contract trigger (D-030). ## Requirements covered -- REQ-04: 4 L2 modules under modules/l2/ composing L1s -- REQ-05: L2s compose L1s, max depth 5 -- REQ-06: mock_executor.sh reads L2 + invokes L1s + writes state.json -- REQ-07: policy_checker.py fails on public-ingress:true -- REQ-08: confidence_signal.py 0.90/0.40 + gate ≥ 0.50 -- REQ-11: evidence_writer.py SHA-256 hash chain +- REQ-10: reusable pipeline runs Dev (autonomous), QA (approval), Prod (approval), Finalize (commits audit.json to acdl-evidence) +- REQ-12: opening an Issue in acdl-contracts runs l3b_agent_stub.py, commits a contract.yaml to a new branch, closes the Issue, and triggers the main pipeline ## Waves (vertical slices, domain priority order) -### Wave 1 — infra-stub-engineer (4 L2 manifests) +### Wave 1 — backend-engineer (pipeline + finalize helper + issue trigger) **Tasks:** -- **T-3.1** Create `modules/l2/l2-invoice-service/manifest.yaml` (L1s: l1-eks-fargate, l1-iam-role, l1-lambda, l1-sqs, l1-s3) -- **T-3.2** Create `modules/l2/l2-commodity-price-feed/manifest.yaml` (L1s: l1-eks-fargate, l1-lambda, l1-api-gateway, l1-eventbridge, l1-s3) -- **T-3.3** Create `modules/l2/l2-energy-analytics-api/manifest.yaml` (L1s: l1-eks-fargate, l1-api-gateway, l1-lambda, l1-s3, l1-cloudwatch) -- **T-3.4** Create `modules/l2/l2-regulatory-reporting/manifest.yaml` (L1s: l1-eks-fargate, l1-iam-role, l1-lambda, l1-sqs, l1-s3) +- **T-4.1** Create `scripts/finalize_evidence.py` — a helper that takes `--audit ` `--owner ` `--repo ` `--branch ` (defaults: continuous-intelligence, acdl-evidence, main) and `--token-env ` (default ACDL_GITEA_TOKEN). It GETs the current `audit.json` from the repo (to get the sha if it exists), reads the local `` file, PUTs (or POSTs if 404) the new content via the Gitea file-contents API with base64 + commit message "finalize: update audit.json (run )". Exit 0 on success, 1 on API failure. Used by the pipeline finalize + each stage to persist evidence to acdl-evidence. +- **T-4.2** Replace `.gitea/workflows/pipeline.yml` with the real implementation: + - `on: workflow_dispatch` with inputs `contract-ref` (string, default `main`), `approve_qa` (boolean, default `false`), `approve_prod` (boolean, default `false`). + - Job `dev`: `if: inputs.approve_qa != true && inputs.approve_prod != true`. Steps: checkout `acdl` (current repo) at `milestone/v1.0-initial`; checkout `acdl-contracts` at `inputs.contract-ref` into a sibling dir using `actions/checkout@v4` with `repository: continuous-intelligence/acdl-contracts`, `ref: ${{ inputs.contract-ref }}`, `token: ${{ secrets.GITEA_TOKEN }}`; run `python3 scripts/policy_checker.py ../acdl-contracts/contract.yaml`; run `python3 scripts/confidence_signal.py ../acdl-contracts/contract.yaml` and capture the score; if score < 0.50, run `python3 scripts/evidence_writer.py --stage dev --event "dev rejected: confidence < 0.50"` and `python3 scripts/finalize_evidence.py --audit audit.json` and `exit 1`; else run `bash scripts/mock_executor.sh ../acdl-contracts/contract.yaml`, then `python3 scripts/evidence_writer.py --stage dev --event "dev applied: "`, then `python3 scripts/finalize_evidence.py --audit audit.json`. + - Job `qa-gate`: `if: inputs.approve_qa == true && inputs.approve_prod != true`. Steps: checkout acdl; run `python3 scripts/evidence_writer.py --stage qa --event "qa approved"`; `python3 scripts/finalize_evidence.py --audit audit.json`. + - Job `prod-gate`: `if: inputs.approve_prod == true`. Steps: checkout acdl; run `python3 scripts/evidence_writer.py --stage prod --event "prod approved"`; `python3 scripts/finalize_evidence.py --audit audit.json`. + - Job `finalize`: `needs: prod-gate`. Steps: checkout acdl; run `python3 scripts/evidence_writer.py --stage finalize --event "pipeline complete: audit.json committed to acdl-evidence"`; `python3 scripts/finalize_evidence.py --audit audit.json`. + - All jobs `runs-on: ubuntu-latest`. Use `gitea.*` context where relevant. +- **T-4.3** Replace `contracts-repo/.gitea/workflows/issue-to-contract.yml` with the real implementation: + - `on: issues` with `types: [opened]`. + - Job `parse-and-trigger`: `runs-on: ubuntu-latest`. Steps: checkout `acdl` (to get `l3b_agent_stub.py`) at `milestone/v1.0-initial`; run `python3 scripts/l3b_agent_stub.py "${{ gitea.event.issue.body }}" -o contract.yaml`; parse the contract to confirm a stack was set; commit `contract.yaml` to a new branch `contract/${{ gitea.event.issue.number }}` on `acdl-contracts` via the file-contents API (POST); comment on the Issue with the contract summary; close the Issue via the issues API; dispatch the pipeline workflow via `POST /api/v1/repos/continuous-intelligence/acdl/actions/workflows/pipeline.yml/dispatches` with body `{ "ref": "milestone/v1.0-initial", "inputs": { "contract-ref": "contract/${{ gitea.event.issue.number }}" } }` using `secrets.GITEA_TOKEN`. -Each manifest declares plausible `inputs` per L1 (string map per D-017). -Remove `modules/l2/.gitkeep` in T-3.1. +**Files owned:** `scripts/finalize_evidence.py`, `.gitea/workflows/pipeline.yml`, `contracts-repo/.gitea/workflows/issue-to-contract.yml` -**Files owned:** `modules/l2/**` +**Commits:** one per task, `phase: 4, status: plan-as-execute, persona: backend-engineer, task: T-4.x, requirements.covered: [REQ-10 or REQ-12]`. -**Commits:** one per task, `phase: 3, status: plan-as-execute, persona: infra-stub-engineer, task: T-3.x, requirements.covered: [REQ-04, REQ-05]`. - -### Wave 2 — backend-engineer (5 core scripts) +### Wave 2 — lead-developer (verify script + traceability) **Tasks:** -- **T-3.5** Create `scripts/policy_checker.py` — reads contract.yaml (argv[1]); if `public-ingress: true`, print `POLICY_VIOLATION:PUBLIC_INGRESS` and exit 1; else print `POLICY_PASS` and exit 0. Use only stdlib (yaml is available). Idempotent, no side effects. -- **T-3.6** Create `scripts/confidence_signal.py` — reads contract.yaml (argv[1]); calls policy_checker as a subprocess; if pass → `{"score": 0.90, "reason": "POLICY_PASS"}`, if fail → `{"score": 0.40, "reason": "POLICY_VIOLATION:PUBLIC_INGRESS"}`. Print JSON to stdout. Exit 0 always. -- **T-3.7** Create `scripts/evidence_writer.py` — argv flags `--stage`, `--event`, `--audit ` (default `./audit.json`). Loads audit.json (or empty list), computes the new event with canonical-JSON SHA-256 hash chain per D-023, appends, writes back atomically (write tmp + rename). Prints `{"seq": N, "hash": "..."}` to stdout. Genesis event automatically inserted if the file is empty/missing. -- **T-3.8** Create `scripts/mock_executor.sh` — argv[1] = contract.yaml path. Reads contract.stack, resolves `modules/l2//manifest.yaml`, iterates `l1s`, invokes `modules/l1//mock_apply.sh` for each, captures exit code, writes `state.json` per D-022. Exit 0 if all L1s exit 0; non-zero otherwise. -- **T-3.9** Create `scripts/l3b_agent_stub.py` — argv[1] = issue body (or read stdin if absent); optional `-o ` (default stdout). Applies the D-008 keyword map; writes a contract.yaml (D-021 schema) with `stack` set to the mapped L2 name and a fixed `inputs:` map per stack. Exit 0 on success, 1 on empty input. +- **T-4.4** Create `scripts/verify_phase04.sh`. Checks: + 1. `.gitea/workflows/pipeline.yml` parses as YAML; has `on.workflow_dispatch` with inputs `contract-ref`, `approve_qa`, `approve_prod`; has 4 jobs `dev`, `qa-gate`, `prod-gate`, `finalize`; `dev.if` excludes approve_qa/approve_prod; `qa-gate.if` requires approve_qa; `prod-gate.if` requires approve_prod; `finalize.needs == prod-gate`. + 2. `contracts-repo/.gitea/workflows/issue-to-contract.yml` parses as YAML; has `on.issues` with `[opened]`; the job references `l3b_agent_stub.py` and the dispatch API endpoint `/actions/workflows/pipeline.yml/dispatches`. + 3. `scripts/finalize_evidence.py` is py_compile clean; `--help` exits 0; called with a non-existent token env fails cleanly (exit 1, not a stack trace). + 4. `bash -n` + `py_compile` + `yaml.safe_load` on all phase files. + 5. End-to-end mini-test: run `evidence_writer.py` to produce a temp `audit.json`, then call `finalize_evidence.py` against a mock Gitea API (set `GITEA_HOST=http://127.0.0.1:0` so the connection fails; verify the script exits 1 cleanly without crashing). +- **T-4.5** Update `.ciagent/REQUIREMENTS.md` (REQ-10/12 → covered pending VERIFY) and `.ciagent/ROADMAP.md` (Phase 04 → executing). -**Files owned:** `scripts/policy_checker.py`, `scripts/confidence_signal.py`, `scripts/evidence_writer.py`, `scripts/mock_executor.sh`, `scripts/l3b_agent_stub.py` +**Files owned:** `scripts/verify_phase04.sh`, `.ciagent/REQUIREMENTS.md`, `.ciagent/ROADMAP.md` -**Commits:** one per task, `phase: 3, status: plan-as-execute, persona: backend-engineer, task: T-3.x, requirements.covered: [REQ-06/07/08/11/12]`. - -### Wave 3 — lead-developer (verify script + traceability) - -**Tasks:** - -- **T-3.10** Create `scripts/verify_phase03.sh`. Checks: - 1. Exactly 4 L2 folders with the expected names. - 2. Each L2 manifest.yaml parses, name matches folder, kind=l2, l1s is a list of 5 entries, all referenced L1 names exist in modules/l1/. - 3. policy_checker.py on a passing contract → exit 0 + `POLICY_PASS`; on `public-ingress: true` contract → exit 1 + `POLICY_VIOLATION:PUBLIC_INGRESS`. - 4. confidence_signal.py on passing contract → `{"score": 0.90, ...}`; on failing contract → `{"score": 0.40, ...}`. Both exit 0. - 5. mock_executor.sh on a sample contract → writes state.json with l2 + l1s (all applied=true, exit_code=0) + contract fields. - 6. evidence_writer.py: append 3 events to a temp audit.json; verify seq increments 0/1/2, prev_hash chain links, each hash matches a recompute. - 7. l3b_agent_stub.py on the Act 3 example issue text ("We need to ingest natural gas prices from Platts...") → emits a contract.yaml with `stack: l2-commodity-price-feed`. -- **T-3.11** Update `.ciagent/REQUIREMENTS.md` (REQ-04/05/06/07/08/11 → covered pending VERIFY) and `.ciagent/ROADMAP.md` (Phase 03 → executing). - -**Files owned:** `scripts/verify_phase03.sh`, `.ciagent/REQUIREMENTS.md`, `.ciagent/ROADMAP.md` - -**Commits:** one per task, `phase: 3, status: plan-as-execute, persona: lead-developer, task: T-3.10/3.11`. +**Commits:** one per task, `phase: 4, status: plan-as-execute, persona: lead-developer, task: T-4.4/4.5`. ## Wave ordering -- Wave 1 (infra-stub-engineer) creates the 4 L2 manifests first so mock_executor.sh has something to resolve. -- Wave 2 (backend-engineer) builds the 5 core scripts. policy_checker + confidence_signal have no L2 dependency; mock_executor depends on Wave 1; l3b_agent_stub is independent. -- Wave 3 (lead-developer) wires the verify script after both Waves 1 and 2 are complete. +- Wave 1 (backend) builds the pipeline + helper + issue trigger. +- Wave 2 (lead-developer) wires the verify script after the workflows exist. -`data-engineer` and `frontend-engineer` have 0 tasks this phase. +`infra-stub-engineer`, `data-engineer`, `frontend-engineer` have 0 tasks. ## Dependencies -- Depends on Phase 02 (L1 modules exist so mock_executor can invoke them and verify_phase03 can confirm L2 references resolve). -- Phase 04 depends on this phase for the pipeline to call policy_checker, mock_executor, confidence_signal, evidence_writer, and for the issue workflow to call l3b_agent_stub. \ No newline at end of file +- Depends on Phase 03 (all 5 core scripts must exist for the workflow to invoke them). +- Phase 05 depends on this phase for the finalize step to produce `audit.json` on `acdl-evidence` that the UI fetches. + +## Risk notes + +- The full pipeline cannot be exercised end-to-end without an act_runner registered to the `acdl` repo (Phase 04 ships the workflow but cannot trigger a real run in this environment). verify_phase04.sh validates structure + syntax + a dry-run of finalize_evidence against a dead host; the end-to-end Act 2/3/4 dry run is Phase 05. +- `actions/checkout@v4` cross-repo requires the `GITEA_TOKEN` secret to be set on the `acdl` repo (out-of-band Gitea UI step). verify_phase04 cannot test this; documented in the workflow YAML comments. \ No newline at end of file diff --git a/.ciagent/PROJECT.md b/.ciagent/PROJECT.md index 693f59a..320dbbd 100644 --- a/.ciagent/PROJECT.md +++ b/.ciagent/PROJECT.md @@ -85,4 +85,8 @@ Build a runnable demo (Linux + GitHub/Gitea Actions) that walks executives throu | D-023 | `evidence_writer.py` appends events to `audit.json` (a JSON array of event objects). Each event: `{"seq": N, "ts": , "stage": "dev|qa|prod|finalize", "event": "", "prev_hash": "", "hash": ""}`. The genesis event has `prev_hash: "GENESIS"` and `seq: 0` | D-005 mandates hash-chained ledger; canonical JSON for deterministic hashing | Visible tamper-evidence without overengineering; Phase 05 UI reads the array | | D-024 | `confidence_signal.py` reads `contract.yaml`, calls `policy_checker.py` (as a subprocess or import), returns base 0.90 on pass and 0.40 with reason code on policy failure; prints `{"score": 0.90|0.40, "reason": ""}` to stdout; exit 0 always | REQ-08 literal: base 0.90, drops to 0.40, gate ≥ 0.50 | Deterministic JSON output for the pipeline to consume | | D-025 | `policy_checker.py` reads `contract.yaml`, fails with exit code 1 and stdout `POLICY_VIOLATION:PUBLIC_INGRESS` if `public-ingress: true`; otherwise exits 0 with stdout `POLICY_PASS` | REQ-07 literal | Single source of policy truth; called by confidence_signal and the pipeline directly | -| D-026 | `l3b_agent_stub.py` reads Issue body text from argv[1] (or stdin if no argv), applies the D-008 keyword map, writes a `contract.yaml` to stdout (or to `-o `). Output contract uses the D-021 schema with `stack:` set to the mapped L2 name and a fixed `inputs:` map per L2 | D-008 + Act 3 example; L3B must produce the same contract format as L3A | Deterministic keyword parser; no external APIs | \ No newline at end of file +| D-026 | `l3b_agent_stub.py` reads Issue body text from argv[1] (or stdin if no argv), applies the D-008 keyword map, writes a `contract.yaml` to stdout (or to `-o `). Output contract uses the D-021 schema with `stack:` set to the mapped L2 name and a fixed `inputs:` map per L2 | D-008 + Act 3 example; L3B must produce the same contract format as L3A | Deterministic keyword parser; no external APIs | +| D-027 | Phase 04 models the pipeline as TWO Gitea Actions workflows: (1) `acdl/.gitea/workflows/pipeline.yml` — `on: workflow_call` + `on: workflow_dispatch` (so it can be both called by the contracts-repo trigger AND manually re-dispatched for approvals); (2) `acdl-contracts/.gitea/workflows/issue-to-contract.yml` — `on: issues [opened]`. Approval gates are implemented as separate workflow_dispatch inputs (`approve_qa: bool`, `approve_prod: bool`) on the pipeline workflow, since Gitea ignores `environment:` blocks (D-013) | Gitea Actions has no environment reviewers, no `repository_dispatch`, no native approval UI | Pipeline can be re-dispatched by a human at each gate; the workflow_dispatch API call from a step (D-014) drives cross-repo triggering | +| D-028 | The pipeline workflow runs all 4 stages (dev, qa-gate, prod-gate, finalize) in a single workflow run, with each gate job checking a workflow_dispatch input (`approve_qa`/`approve_prod`). When the input is false (the default), the gate job fails with a clear "awaiting approval" message; the human re-dispatches with `approve_qa=true` to advance. State (state.json, audit.json, contract ref) is passed via workflow artifacts (upload/download between jobs) because Gitea Actions artifacts work the same as GitHub Actions | Gitea Actions supports `actions/upload-artifact` and `actions/download-artifact`; the alternative is committing state between jobs, which is heavier | Deterministic, observable pipeline; artifacts keep the audit trail within one run | +| D-029 | The finalize step commits `audit.json` to `acdl-evidence` main via the Gitea file-contents API (POST `/repos/{owner}/{repo}/contents/{path}` with the base64 content + a commit message referencing the pipeline run id), exactly like Phase 01's `gitea_setup.sh` does for `index.html`. It uses `${GITEA_TOKEN}` (a repo secret) for auth | D-012 raw-URL approach requires the file to be on main; the API is the only way to put it there from a workflow step | The evidence timeline (Phase 05 UI) fetches the raw URL after finalize completes | +| D-030 | The issue-to-contract workflow in `acdl-contracts` checks out `l3b_agent_stub.py` from the `acdl` repo (pinned to `@milestone/v1.0-initial` per the branch-pin rule), parses the Issue body, commits `contract.yaml` to a new branch `contract/` on `acdl-contracts`, then dispatches the pipeline workflow on the `acdl` repo via `curl POST /actions/workflows//dispatches` with `inputs: {contract-ref: contract/}` (D-014). The pipeline workflow checks out `acdl-contracts` at that ref to read the contract | Gitea Actions cannot trigger across repos without an explicit API call; the branch carries the contract ref | Reproducible Act 3: Issue → contract.yaml → pipeline run with the same contract as Act 2 | \ No newline at end of file diff --git a/.ciagent/REQUIREMENTS.md b/.ciagent/REQUIREMENTS.md index 7b26946..e76e1a0 100644 --- a/.ciagent/REQUIREMENTS.md +++ b/.ciagent/REQUIREMENTS.md @@ -66,9 +66,9 @@ | REQ-07 | 3 | complete (v1.0.3) | | REQ-08 | 3 | complete (v1.0.3) | | REQ-09 | 1 | complete (v1.0.1) | -| REQ-10 | 4 | partial (skeleton in Phase 01 v1.0.1; full impl in Phase 04) | +| REQ-10 | 4 | covered (pending VERIFY) | | REQ-11 | 3 | complete (v1.0.3) | -| REQ-12 | 4 | partial (l3b_agent_stub in Phase 03 v1.0.3; full trigger wiring in Phase 04) | +| REQ-12 | 4 | covered (pending VERIFY) | | REQ-13 | 5 | pending | | REQ-14 | 5 | pending | | REQ-15 | 5 | pending | \ No newline at end of file diff --git a/.ciagent/ROADMAP.md b/.ciagent/ROADMAP.md index 1ed0979..d01b702 100644 --- a/.ciagent/ROADMAP.md +++ b/.ciagent/ROADMAP.md @@ -39,7 +39,7 @@ Five-phase breakdown to take ACDL from empty repo to a reproducible 4-act execut ### Phase 04 — pipeline-and-approval-gates - **Description:** Build the reusable pipeline workflow in `acdl/.gitea/workflows/` (Dev → QA → Prod → Finalize) plus the issue-triggered L3B workflow in `acdl-contracts/.gitea/workflows/`. Wire environment protection for QA and Prod. -- **Status:** not_started +- **Status:** executing - **Depends on:** [3] - **Requirements:** REQ-08, REQ-09, REQ-10, REQ-12 - **Success Criteria:** diff --git a/.gitea/workflows/pipeline.yml b/.gitea/workflows/pipeline.yml index c8a9e26..71eccd2 100644 --- a/.gitea/workflows/pipeline.yml +++ b/.gitea/workflows/pipeline.yml @@ -1,75 +1,153 @@ -# ACDL reusable pipeline workflow (Phase 01 skeleton). +# ACDL pipeline workflow (Phase 04 implementation). # -# This workflow is called from acdl-contracts via: -# uses: continuous-intelligence/acdl/.gitea/workflows/pipeline.yml@milestone/v1.0-initial +# 3-dispatch approval-gate topology (D-027 / D-028; ARCHITECTURE.md +# "Phase 04 pipeline topology"): # -# Branch pinning rule (see .ciagent/ARCHITECTURE.md): the `acdl` repo's default -# branch is `milestone/v1.0-initial`, so `uses:` references must pin to -# `@milestone/v1.0-initial`, NOT `@main`. +# Dispatch 1 (initial): approve_qa=false, approve_prod=false +# -> runs the `dev` job (policy check, confidence +# gate, mock_executor, evidence + finalize). +# Dispatch 2 (QA approve): approve_qa=true, approve_prod=false +# -> runs the `qa-gate` job (records QA approval +# in the audit chain via evidence_writer + +# finalize_evidence). +# Dispatch 3 (Prod approve): approve_prod=true +# -> runs the `prod-gate` job, then the `finalize` +# job (needs: prod-gate) which writes the final +# evidence event and commits audit.json to +# acdl-evidence. # -# Phase 04 will implement the actual stage logic + approval gates (D-013: -# Gitea has no environments API; gates become workflow_dispatch approval -# inputs). +# Gitea Actions limitations driving this design: +# - No `repository_dispatch` trigger (D-014). +# - No environments API / `environment:` blocks are ignored (D-013). +# - Re-dispatch starts a NEW run; artifacts do NOT survive between runs, +# so state is persisted to acdl-evidence via the file-contents API +# (D-028 / finalize_evidence.py) instead of via artifacts. +# +# Branch-pin rule (ARCHITECTURE.md "Branch pinning rule"): +# This workflow lives on `acdl`'s default branch `milestone/v1.0-initial`. +# Cross-repo `uses:` references (e.g. the issue-trigger's checkout of +# l3b_agent_stub.py) MUST pin to `@milestone/v1.0-initial`, NOT `@main` +# (the `acdl` repo has no `main` branch). This workflow is invoked via +# the workflow_dispatch API (D-014), NOT via `workflow_call`, so the +# `uses:` rule applies to the issue-trigger's checkout of the acdl repo, +# not to this file itself. name: acdl-pipeline -on: - workflow_call: +"on": + workflow_dispatch: inputs: contract-ref: - description: "Ref on acdl-contracts that triggered the pipeline" + description: "Ref on acdl-contracts that carries the contract" required: false type: string default: main + approve_qa: + description: "Human approval to advance past QA" + required: false + type: boolean + default: false + approve_prod: + description: "Human approval to advance past Prod" + required: false + type: boolean + default: false jobs: dev: name: "Dev (autonomous)" + if: inputs.approve_qa != true && inputs.approve_prod != true runs-on: ubuntu-latest steps: - # Phase 04 will implement: checkout acdl + acdl-contracts, run - # policy_checker.py, mock_executor.sh, confidence_signal.py, write - # evidence via evidence_writer.py. - - name: "Dev stage placeholder" + - name: "Checkout acdl (this repo, pinned to milestone/v1.0-initial)" + uses: actions/checkout@v4 + with: + ref: milestone/v1.0-initial + + - name: "Checkout acdl-contracts at contract-ref" + uses: actions/checkout@v4 + with: + repository: continuous-intelligence/acdl-contracts + ref: ${{ inputs.contract-ref }} + token: ${{ secrets.GITEA_TOKEN }} + path: acdl-contracts + + - name: "Policy check" run: | - echo "Dev stage placeholder (Phase 01 skeleton)" - echo "Phase 04 will run policy_checker, mock_executor, confidence_signal, evidence_writer" - exit 0 + python3 scripts/policy_checker.py acdl-contracts/contract.yaml + + - name: "Confidence signal" + id: confidence + run: | + set +e + SCORE_JSON=$(python3 scripts/confidence_signal.py acdl-contracts/contract.yaml) + echo "$SCORE_JSON" + echo "score_json=$SCORE_JSON" >> "$GITHUB_OUTPUT" + + - name: "Apply or reject based on confidence (gate < 0.50)" + run: | + set +e + SCORE=$(python3 -c "import json,sys; print(json.load(sys.stdin)['score'])" <<< '${{ steps.confidence.outputs.score_json }}') + python3 -c "import sys; sys.exit(0 if float('${SCORE}') >= 0.50 else 1)" + THRESHOLD_RC=$? + if [ "$THRESHOLD_RC" -ne 0 ]; then + python3 scripts/evidence_writer.py --stage dev --event "dev rejected: confidence < 0.50" --audit audit.json + python3 scripts/finalize_evidence.py --audit audit.json + exit 1 + fi + STACK=$(python3 -c 'import yaml; print(yaml.safe_load(open("acdl-contracts/contract.yaml"))["stack"])') + bash scripts/mock_executor.sh acdl-contracts/contract.yaml + python3 scripts/evidence_writer.py --stage dev --event "dev applied: ${STACK}" --audit audit.json + python3 scripts/finalize_evidence.py --audit audit.json + + - name: "Upload dev state artifacts (best-effort)" + uses: actions/upload-artifact@v3 + with: + name: dev-state + path: | + audit.json + state.json qa-gate: name: "QA (manual approval)" - needs: dev + if: inputs.approve_qa == true && inputs.approve_prod != true runs-on: ubuntu-latest steps: - # Phase 04 will implement: gate via workflow_dispatch approval input - # (D-013 fallback; Gitea ignores jobs..environment). - - name: "QA gate placeholder" + - name: "Checkout acdl (this repo, pinned to milestone/v1.0-initial)" + uses: actions/checkout@v4 + with: + ref: milestone/v1.0-initial + + - name: "Record QA approval in evidence" run: | - echo "QA gate placeholder (Phase 01 skeleton)" - echo "Phase 04 will pause here for human approval via workflow_dispatch" - exit 0 + python3 scripts/evidence_writer.py --stage qa --event "qa approved" --audit audit.json + python3 scripts/finalize_evidence.py --audit audit.json prod-gate: name: "Prod (manual approval)" - needs: qa-gate + if: inputs.approve_prod == true runs-on: ubuntu-latest steps: - # Phase 04 will implement: same approval-input gate as qa-gate. - - name: "Prod gate placeholder" + - name: "Checkout acdl (this repo, pinned to milestone/v1.0-initial)" + uses: actions/checkout@v4 + with: + ref: milestone/v1.0-initial + + - name: "Record Prod approval in evidence" run: | - echo "Prod gate placeholder (Phase 01 skeleton)" - echo "Phase 04 will pause here for human approval via workflow_dispatch" - exit 0 + python3 scripts/evidence_writer.py --stage prod --event "prod approved" --audit audit.json + python3 scripts/finalize_evidence.py --audit audit.json finalize: name: "Finalize (publish evidence)" - needs: prod-gate + needs: [prod-gate] runs-on: ubuntu-latest steps: - # Phase 04/05 will implement: commit audit.json to acdl-evidence main - # via the Gitea file-contents API; raw URL republishes index.html + - # audit.json for the timeline UI (D-012). - - name: "Finalize placeholder" + - name: "Checkout acdl (this repo, pinned to milestone/v1.0-initial)" + uses: actions/checkout@v4 + with: + ref: milestone/v1.0-initial + + - name: "Write finalize event + commit audit.json to acdl-evidence" run: | - echo "Finalize placeholder (Phase 01 skeleton)" - echo "Phase 04/05 will commit audit.json to acdl-evidence main" - exit 0 \ No newline at end of file + python3 scripts/evidence_writer.py --stage finalize --event "pipeline complete: audit.json committed to acdl-evidence" --audit audit.json + python3 scripts/finalize_evidence.py --audit audit.json \ No newline at end of file diff --git a/contracts-repo/.gitea/workflows/issue-to-contract.yml b/contracts-repo/.gitea/workflows/issue-to-contract.yml index 651f79d..f96420d 100644 --- a/contracts-repo/.gitea/workflows/issue-to-contract.yml +++ b/contracts-repo/.gitea/workflows/issue-to-contract.yml @@ -1,18 +1,35 @@ -# ACDL issue-to-contract workflow (Phase 01 skeleton, reference copy). -# -# This file is the source-of-truth copy kept in the `acdl` repo under -# contracts-repo/.gitea/workflows/. Phase 04 will push it to the actual -# `acdl-contracts` repo under .gitea/workflows/ and implement the real -# step bodies. +# ACDL issue-to-contract workflow (Phase 04 implementation). # # Trigger: a new Issue is opened in acdl-contracts. The workflow runs -# l3b_agent_stub.py to map the Issue body to a contract.yaml, commits the -# contract to a new branch, closes the Issue, and triggers the main -# pipeline in the `acdl` repo via the workflow_dispatch API (D-014; Gitea -# Actions does not support repository_dispatch). +# l3b_agent_stub.py (checked out from the `acdl` repo, pinned to +# @milestone/v1.0-initial) to map the Issue body to a contract.yaml, commits +# the contract to a new branch `contract/` on acdl-contracts +# via the Gitea file-contents API, closes the Issue with a comment, and +# dispatches the main pipeline in the `acdl` repo via the workflow_dispatch +# API (D-014; Gitea Actions does not support repository_dispatch). +# +# Cross-repo trigger (D-014): +# The final step POSTs to +# /api/v1/repos/continuous-intelligence/acdl/actions/workflows/pipeline.yml/dispatches +# with body {"ref": "milestone/v1.0-initial", +# "inputs": {"contract-ref": "contract/"}}. +# +# Branch-pin rule (ARCHITECTURE.md): +# The `acdl` repo's default branch is `milestone/v1.0-initial`, so the +# checkout step pins `ref: milestone/v1.0-initial`. The pipeline dispatch +# also pins `ref: milestone/v1.0-initial` (the workflow file lives on +# that branch). The new `contract/` branch is created on acdl-contracts +# (whose default branch is `main`, per D-015). +# +# File-contents POST with `new_branch` (D-030): +# The POST to /repos/.../contents/contract.yaml includes +# `new_branch: contract/`, which tells Gitea to create the file on a +# NEW branch off the current head of `branch: main` instead of committing +# directly to main. This avoids a separate branch-create + commit round +# trip. name: issue-to-contract -on: +"on": issues: types: [opened] @@ -20,21 +37,109 @@ jobs: parse-and-trigger: runs-on: ubuntu-latest steps: - # Phase 04 will implement: - # 1. checkout acdl-contracts (so l3b_agent_stub.py is available). - # 2. run: python3 scripts/l3b_agent_stub.py "${{ gitea.event.issue.body }}" > contract.yaml - # 3. parse the generated contract; commit it to a new branch - # (e.g. contract/). - # 4. push the branch. - # 5. close the Issue with a comment linking to the pipeline run. - # 6. trigger the main pipeline: - # curl -X POST \ - # -H "Authorization: token ${GITEA_TOKEN}" \ - # https://git.cloudinit.dev/api/v1/repos/continuous-intelligence/acdl/actions/workflows//dispatches \ - # -d '{"ref":"milestone/v1.0-initial","inputs":{"contract-ref":""}}' - - name: "Issue-trigger placeholder" + - name: "Checkout acdl (pinned to milestone/v1.0-initial for l3b_agent_stub.py)" + uses: actions/checkout@v4 + with: + repository: continuous-intelligence/acdl + ref: milestone/v1.0-initial + token: ${{ secrets.GITEA_TOKEN }} + + - name: "Parse Issue body into contract.yaml" + env: + ISSUE_BODY: ${{ gitea.event.issue.body }} run: | - echo "issue-to-contract placeholder (Phase 01 skeleton)" - echo "Issue body: ${{ gitea.event.issue.body }}" - echo "Phase 04 will run l3b_agent_stub.py, commit contract.yaml, close issue, dispatch pipeline" - exit 0 \ No newline at end of file + # Pass the Issue body via an env var to avoid shell injection from + # arbitrary Issue text. l3b_agent_stub.py reads argv[1]; we pass + # the env var quoted so no metacharacter interpretation happens. + python3 scripts/l3b_agent_stub.py "$ISSUE_BODY" -o contract.yaml + echo "--- generated contract.yaml ---" + cat contract.yaml + + - name: "Commit contract.yaml to new branch contract/${{ gitea.event.issue.number }} on acdl-contracts" + env: + GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }} + run: | + set -euo pipefail + STACK=$(python3 -c 'import yaml; print(yaml.safe_load(open("contract.yaml"))["stack"])') + ISSUE_NUMBER="${{ gitea.event.issue.number }}" + BRANCH="contract/${ISSUE_NUMBER}" + HOST="https://git.cloudinit.dev" + API="${HOST}/api/v1/repos/continuous-intelligence/acdl-contracts/contents/contract.yaml" + B64=$(base64 -w 0 contract.yaml) + BODY=$(python3 -c " + import json + print(json.dumps({ + 'content': '${B64}', + 'message': 'l3b: contract for issue #${ISSUE_NUMBER}', + 'branch': 'main', + 'new_branch': '${BRANCH}' + })) + ") + STATUS=$(curl -sS -o /tmp/contract_post.json -w "%{http_code}" \ + -X POST \ + -H "Authorization: token ${GITEA_TOKEN}" \ + -H "Content-Type: application/json" \ + -d "$BODY" \ + "$API") + echo "POST contract.yaml -> HTTP ${STATUS}" + cat /tmp/contract_post.json || true + case "$STATUS" in + 201) echo "contract.yaml committed on branch ${BRANCH}" ;; + *) echo "ERROR: file-contents POST failed (HTTP ${STATUS})" >&2; exit 1 ;; + esac + echo "STACK=${STACK}" >> "$GITHUB_ENV" + echo "BRANCH=${BRANCH}" >> "$GITHUB_ENV" + + - name: "Comment on Issue + close it" + env: + GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }} + run: | + set -euo pipefail + ISSUE_NUMBER="${{ gitea.event.issue.number }}" + HOST="https://git.cloudinit.dev" + ISSUES_API="${HOST}/api/v1/repos/continuous-intelligence/acdl-contracts/issues/${ISSUE_NUMBER}" + COMMENT_BODY=$(python3 -c " + import json + print(json.dumps({'body': 'Generated contract.yaml for stack \`' + '${STACK}' + '\` on branch \`' + '${BRANCH}' + '\`. Pipeline dispatched.'})) + ") + curl -sS -o /tmp/comment.json -w "comment HTTP %{http_code}\n" \ + -X POST \ + -H "Authorization: token ${GITEA_TOKEN}" \ + -H "Content-Type: application/json" \ + -d "$COMMENT_BODY" \ + "${ISSUES_API}/comments" + CLOSE_BODY='{"state":"closed"}' + curl -sS -o /tmp/close.json -w "close HTTP %{http_code}\n" \ + -X PATCH \ + -H "Authorization: token ${GITEA_TOKEN}" \ + -H "Content-Type: application/json" \ + -d "$CLOSE_BODY" \ + "${ISSUES_API}" + + - name: "Dispatch the pipeline on acdl (contract-ref = contract/${{ gitea.event.issue.number }})" + env: + GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }} + run: | + set -euo pipefail + ISSUE_NUMBER="${{ gitea.event.issue.number }}" + HOST="https://git.cloudinit.dev" + DISPATCH_URL="${HOST}/api/v1/repos/continuous-intelligence/acdl/actions/workflows/pipeline.yml/dispatches" + BODY=$(python3 -c " + import json + print(json.dumps({ + 'ref': 'milestone/v1.0-initial', + 'inputs': {'contract-ref': 'contract/${ISSUE_NUMBER}'} + })) + ") + STATUS=$(curl -sS -o /tmp/dispatch.json -w "%{http_code}" \ + -X POST \ + -H "Authorization: token ${GITEA_TOKEN}" \ + -H "Content-Type: application/json" \ + -d "$BODY" \ + "$DISPATCH_URL") + echo "pipeline dispatch -> HTTP ${STATUS}" + cat /tmp/dispatch.json || true + case "$STATUS" in + 201|202|204) echo "pipeline dispatched (contract-ref=contract/${ISSUE_NUMBER})" ;; + *) echo "ERROR: pipeline dispatch failed (HTTP ${STATUS})" >&2; exit 1 ;; + esac \ No newline at end of file diff --git a/scripts/finalize_evidence.py b/scripts/finalize_evidence.py new file mode 100755 index 0000000..fdde84e --- /dev/null +++ b/scripts/finalize_evidence.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +"""finalize_evidence.py — REQ-10 / D-028 / D-029 + +Uploads (PUT or POST) a local `audit.json` to the `acdl-evidence` repo on +Gitea via the file-contents API. Used by the pipeline workflow steps to +persist the hash-chained audit trail to `acdl-evidence` between dispatches +(D-028 state-persistence across re-dispatches; D-029 finalize step). + +Uses only the Python standard library (urllib.request) so it has no +external dependency on `requests`. Auth header: `Authorization: token `. + +Input (argv flags): + --audit (required) local audit.json file to upload + --owner (optional, default continuous-intelligence) + --repo (optional, default acdl-evidence) + --branch (optional, default main) + --path (optional, default audit.json) path in the repo + --token-env (optional, default ACDL_GITEA_TOKEN) + --host (optional, default https://git.cloudinit.dev) + --message (optional, default chore(evidence): update audit.json) + +Behavior: + 1. Read the token from os.environ[token_env]. Missing -> stderr + exit 1. + 2. Read the local audit file; base64-encode it. + 3. GET the current file at .../contents/?ref= to discover + the existing `sha`. 200 -> capture sha (update mode). 404 -> no sha + (create mode). Other errors -> exit 1. + 4. If sha set: PUT with body {content, message, branch, sha}. + If no sha: POST with body {content, message, branch}. + 5. Print {"uploaded": true, "path": "", "sha": ""} to + stdout and exit 0. + 6. On any HTTP error: print + {"uploaded": false, "status": , "body": ""} to stdout + and exit 1. +""" +import argparse +import base64 +import json +import os +import sys +import urllib.error +import urllib.parse +import urllib.request + + +def _request(method: str, url: str, token: str, body: dict = None): + """Perform an HTTP request with the Gitea auth header. Returns + (status_code, response_body_text). Raises URLError on network failure.""" + data = None + headers = {"Authorization": f"token {token}", + "Accept": "application/json"} + if body is not None: + data = json.dumps(body).encode("utf-8") + headers["Content-Type"] = "application/json" + req = urllib.request.Request(url, data=data, method=method, headers=headers) + try: + with urllib.request.urlopen(req) as resp: + return resp.getcode(), resp.read().decode("utf-8", "replace") + except urllib.error.HTTPError as exc: + # HTTPError carries the response body + try: + body_text = exc.read().decode("utf-8", "replace") + except Exception: + body_text = "" + return exc.code, body_text + except urllib.error.URLError as exc: + # Network-level failure (connection refused, DNS, timeout). Return + # a synthetic 0 status + the reason so callers can report cleanly + # without a stack trace. + return 0, f"URLError: {exc.reason}" + + +def get_existing_sha(host: str, owner: str, repo: str, path: str, + branch: str, token: str): + """Return (sha-or-None, error_status_or_None). On 200 returns the sha. + On 404 returns (None, None). Other codes return (None, (status, body)).""" + qs = urllib.parse.urlencode({"ref": branch}) + url = f"{host}/api/v1/repos/{owner}/{repo}/contents/{path}?{qs}" + status, body = _request("GET", url, token) + if status == 200: + try: + data = json.loads(body) + return data.get("sha"), None + except (ValueError, TypeError): + return None, (status, body) + if status == 404: + return None, None + return None, (status, body) + + +def upload(host: str, owner: str, repo: str, path: str, branch: str, + message: str, content_b64: str, sha, token: str): + """PUT (update) or POST (create) the file. Returns (new_sha, None) on + success or (None, (status, body)) on HTTP error.""" + url = f"{host}/api/v1/repos/{owner}/{repo}/contents/{path}" + if sha: + body = {"content": content_b64, "message": message, + "branch": branch, "sha": sha} + status, resp = _request("PUT", url, token, body) + else: + body = {"content": content_b64, "message": message, "branch": branch} + status, resp = _request("POST", url, token, body) + if status in (200, 201): + try: + data = json.loads(resp) + # The file-contents API returns the new content object either at + # top-level `content` (POST create) or `content` (PUT update). + new_sha = None + if isinstance(data, dict): + content_obj = data.get("content") or data + if isinstance(content_obj, dict): + new_sha = content_obj.get("sha") + return new_sha, None + except (ValueError, TypeError): + return None, None + return None, (status, resp) + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Upload a local audit.json to the acdl-evidence Gitea " + "repo via the file-contents API (D-028/D-029).") + parser.add_argument("--audit", required=True, + help="Local audit.json file to upload") + parser.add_argument("--owner", default="continuous-intelligence", + help="Gitea org (default: continuous-intelligence)") + parser.add_argument("--repo", default="acdl-evidence", + help="Gitea repo (default: acdl-evidence)") + parser.add_argument("--branch", default="main", + help="Target branch (default: main)") + parser.add_argument("--path", default="audit.json", + help="Remote path in the repo (default: audit.json)") + parser.add_argument("--token-env", default="ACDL_GITEA_TOKEN", + help="Env var name holding the Gitea token " + "(default: ACDL_GITEA_TOKEN)") + parser.add_argument("--host", default="https://git.cloudinit.dev", + help="Gitea host URL (default: https://git.cloudinit.dev)") + parser.add_argument("--message", default="chore(evidence): update audit.json", + help="Commit message (default: chore(evidence): " + "update audit.json)") + args = parser.parse_args() + + token = os.environ.get(args.token_env) + if not token: + print(f"finalize_evidence: required env var {args.token_env} is not " + f"set", file=sys.stderr) + return 1 + + # Read + base64-encode the local audit file. Missing/unreadable file is + # a clean exit 1 (no stack trace). + try: + with open(args.audit, "rb") as fh: + raw = fh.read() + except OSError as exc: + print(f"finalize_evidence: cannot read {args.audit}: {exc}", + file=sys.stderr) + return 1 + content_b64 = base64.b64encode(raw).decode("ascii") + + # Discover existing sha (update vs create). + sha, err = get_existing_sha(args.host, args.owner, args.repo, + args.path, args.branch, token) + if err is not None: + status, body = err + print(json.dumps({"uploaded": False, "status": status, "body": body})) + return 1 + + # Upload (PUT if sha, POST otherwise). + new_sha, err = upload(args.host, args.owner, args.repo, args.path, + args.branch, args.message, content_b64, sha, token) + if err is not None: + status, body = err + print(json.dumps({"uploaded": False, "status": status, "body": body})) + return 1 + + print(json.dumps({"uploaded": True, "path": args.path, + "sha": new_sha})) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/scripts/verify_phase04.sh b/scripts/verify_phase04.sh new file mode 100755 index 0000000..be98497 --- /dev/null +++ b/scripts/verify_phase04.sh @@ -0,0 +1,186 @@ +#!/usr/bin/env bash +# Phase 04 verification script. +# Confirms the pipeline workflow + issue trigger + finalize_evidence.py +# conform to the Phase 04 plan and the Gitea Actions topology in +# ARCHITECTURE.md. Does NOT execute a real Gitea Actions run (act_runner +# is not registered in this environment); validates structure + syntax +# + a dry-run of finalize_evidence.py against a dead host. +# +# Usage: scripts/verify_phase04.sh +# Exit codes: 0 = all checks passed; 1 = one or more checks failed. + +set -uo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT" + +fail_count=0 +pass() { printf ' [PASS] %s\n' "$1"; } +fail() { printf ' [FAIL] %s\n' "$1"; fail_count=$((fail_count + 1)); } + +echo "== Phase 04 verification ==" +echo "Root: ${ROOT}" +echo + +# --- Check 1: typecheck --- +echo "-- Check 1: typecheck --" +if bash -n scripts/finalize_evidence.py 2>/dev/null || python3 -m py_compile scripts/finalize_evidence.py 2>/dev/null; then + pass "py_compile finalize_evidence.py" +else + fail "py_compile finalize_evidence.py" +fi +if python3 -c "import yaml; yaml.safe_load(open('.gitea/workflows/pipeline.yml')); yaml.safe_load(open('contracts-repo/.gitea/workflows/issue-to-contract.yml'))" 2>/dev/null; then + pass "yaml load both workflows" +else + fail "yaml load workflows" +fi + +# --- Check 2: pipeline.yml structure --- +echo "-- Check 2: pipeline.yml structure (D-027, D-028) --" +p_struct=$(python3 << 'PYEOF' +import yaml, sys +try: + d = yaml.safe_load(open('.gitea/workflows/pipeline.yml')) + on = d.get('on', d.get(True)) or {} + assert 'workflow_dispatch' in on, 'no workflow_dispatch trigger' + inputs = on['workflow_dispatch']['inputs'] + assert set(inputs.keys()) == {'contract-ref', 'approve_qa', 'approve_prod'}, f'inputs: {set(inputs.keys())}' + assert inputs['contract-ref']['type'] == 'string', 'contract-ref type' + assert inputs['approve_qa']['type'] == 'boolean', 'approve_qa type' + assert inputs['approve_prod']['type'] == 'boolean', 'approve_prod type' + jobs = d['jobs'] + assert set(jobs.keys()) == {'dev', 'qa-gate', 'prod-gate', 'finalize'}, f'jobs: {set(jobs.keys())}' + dev_if = jobs['dev'].get('if', '') + assert 'approve_qa' in dev_if and 'approve_prod' in dev_if, f'dev.if: {dev_if}' + qa_if = jobs['qa-gate'].get('if', '') + assert 'approve_qa' in qa_if, f'qa-gate.if: {qa_if}' + prod_if = jobs['prod-gate'].get('if', '') + assert 'approve_prod' in prod_if, f'prod-gate.if: {prod_if}' + fin_needs = jobs['finalize'].get('needs', []) + assert fin_needs == ['prod-gate'] or fin_needs == 'prod-gate', f'finalize.needs: {fin_needs}' + # All jobs runs-on ubuntu-latest + for name, job in jobs.items(): + assert job.get('runs-on') == 'ubuntu-latest', f'{name} runs-on: {job.get("runs-on")}' + print('OK') +except AssertionError as ex: + print(f'FAIL: {ex}') + sys.exit(1) +except Exception as ex: + print(f'FAIL: {ex}') + sys.exit(1) +PYEOF +) +if [ "$p_struct" = "OK" ]; then + pass "pipeline.yml: 3 inputs + 4 jobs + correct if: conditions + finalize.needs=prod-gate" +else + fail "pipeline.yml structure: $p_struct" +fi + +# --- Check 3: pipeline.yml references core scripts --- +echo "-- Check 3: pipeline.yml references core scripts (D-029) --" +text=$(cat .gitea/workflows/pipeline.yml) +missing="" +for ref in policy_checker.py confidence_signal.py mock_executor.sh evidence_writer.py finalize_evidence.py; do + if ! echo "$text" | grep -qF "$ref"; then + missing="$missing $ref" + fi +done +if [ -z "$missing" ]; then + pass "pipeline.yml references all 5 core scripts" +else + fail "pipeline.yml missing references:$missing" +fi +# Branch-pin documentation +if echo "$text" | grep -q 'milestone/v1.0-initial'; then + pass "pipeline.yml documents branch-pin to milestone/v1.0-initial" +else + fail "pipeline.yml missing branch-pin reference" +fi + +# --- Check 4: issue-to-contract.yml structure --- +echo "-- Check 4: issue-to-contract.yml structure (D-030) --" +i_struct=$(python3 << 'PYEOF' +import yaml, sys +try: + d = yaml.safe_load(open('contracts-repo/.gitea/workflows/issue-to-contract.yml')) + on = d.get('on', d.get(True)) or {} + assert 'issues' in on, 'no issues trigger' + assert on['issues']['types'] == ['opened'], f'types: {on["issues"]["types"]}' + assert 'parse-and-trigger' in d['jobs'], 'no parse-and-trigger job' + assert d['jobs']['parse-and-trigger'].get('runs-on') == 'ubuntu-latest', 'runs-on' + print('OK') +except AssertionError as ex: + print(f'FAIL: {ex}') + sys.exit(1) +PYEOF +) +if [ "$i_struct" = "OK" ]; then + pass "issue-to-contract.yml: issues[opened] + parse-and-trigger job" +else + fail "issue-to-contract.yml structure: $i_struct" +fi + +# --- Check 5: issue-to-contract.yml references + dispatch endpoint --- +echo "-- Check 5: issue-to-contract.yml references + dispatch (D-014, D-030) --" +text=$(cat contracts-repo/.gitea/workflows/issue-to-contract.yml) +missing="" +for ref in l3b_agent_stub.py 'actions/workflows/pipeline.yml/dispatches' 'contract-ref' 'gitea.event.issue.number' 'GITEA_TOKEN' 'new_branch'; do + if ! echo "$text" | grep -qF "$ref"; then + missing="$missing $ref" + fi +done +if [ -z "$missing" ]; then + pass "issue-to-contract.yml: l3b_agent_stub + dispatch + contract-ref + issue number + token + new_branch" +else + fail "issue-to-contract.yml missing references:$missing" +fi + +# --- Check 6: finalize_evidence.py --help + clean failure --- +echo "-- Check 6: finalize_evidence.py CLI + clean failure modes ---" +out=$(python3 scripts/finalize_evidence.py --help 2>&1); rc=$? +if [ "$rc" = "0" ] && echo "$out" | grep -qi 'usage\|--audit\|--owner'; then + pass "finalize_evidence.py --help exits 0 with usage" +else + fail "finalize_evidence.py --help: rc=$rc" +fi + +# Missing audit file (with a fake token so it gets past the env check) → exit 1, no stack trace +out=$(ACDL_GITEA_TOKEN=fake python3 scripts/finalize_evidence.py --audit /tmp/definitely_nonexistent_audit.json 2>&1); rc=$? +if [ "$rc" = "1" ] && ! echo "$out" | grep -q 'Traceback'; then + pass "finalize_evidence.py missing file → exit 1, no stack trace" +else + fail "finalize_evidence.py missing file: rc=$rc, out='$out'" +fi + +# Missing token env (audit file present) → exit 1, no stack trace +printf '[]\n' > /tmp/empty_audit.json +out=$(env -u ACDL_GITEA_TOKEN python3 scripts/finalize_evidence.py --audit /tmp/empty_audit.json 2>&1); rc=$? +if [ "$rc" = "1" ] && ! echo "$out" | grep -q 'Traceback'; then + pass "finalize_evidence.py missing token env → exit 1, no stack trace" +else + fail "finalize_evidence.py missing token: rc=$rc, out='$out'" +fi + +# --- Check 7: finalize_evidence.py dry-run against a dead host (clean failure) --- +echo "-- Check 7: finalize_evidence.py dry-run against dead host ---" +# Use a real audit.json but point at a host that will refuse the connection. +printf '[{"seq":0,"ts":"2026-07-21T00:00:00Z","stage":"genesis","event":"init","prev_hash":"GENESIS","hash":"x"}]\n' > /tmp/real_audit.json +out=$(ACDL_GITEA_TOKEN=fake GITEA_HOST=http://127.0.0.1:0 python3 scripts/finalize_evidence.py --audit /tmp/real_audit.json --host http://127.0.0.1:0 2>&1); rc=$? +if [ "$rc" = "1" ] && ! echo "$out" | grep -q 'Traceback'; then + pass "finalize_evidence.py dead host → exit 1, no stack trace (clean API failure)" +else + fail "finalize_evidence.py dead host: rc=$rc, out='$out'" +fi + +# Cleanup +rm -f /tmp/empty_audit.json /tmp/real_audit.json + +echo +echo "== Summary ==" +if [ "$fail_count" -eq 0 ]; then + echo "Phase 04 verification PASSED (pipeline + issue trigger + finalize helper, all checks ok)" + exit 0 +else + echo "Phase 04 verification FAILED (${fail_count} check(s) failed)" + exit 1 +fi \ No newline at end of file