From a4c5f332f6aa1fda6b04d73a3c2d48874dfcc14b Mon Sep 17 00:00:00 2001 From: Jon Chery Date: Thu, 6 Aug 2026 15:40:11 +0000 Subject: [PATCH] =?UTF-8?q?feat(P1):=20sync=5Fto=5Fnova.sh=20=E2=80=94=20m?= =?UTF-8?q?anual-only=202nd-release=20pipeline=20into=20~/nova=20(REQ-229)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces scripts/sync_to_gl.sh (kitchen-sink mirror sync into ~/gl/acdl) with scripts/sync_to_nova.sh — a manual-only, consumer-subset, domain-committed 2nd-release pipeline into ~/nova (GitLab jonathanchery/nova, separate repo + history, consumer/platform-team audience). - Manual-only gate: refuses without --release / RELEASE_CONFIRMED=1 (exit 2). Never triggerable by CI. - Consumer subset: excludes .ciagent/, .gitea/, .env*, terraform/, demo/, runtime metrics artifacts, and 18 internal-only scripts (EXCLUDE_SCRIPTS). Keeps consumer runbooks + metrics export views (README, powerbi, TRUST_SNAPSHOT). Protects ~/nova/.git via rsync --filter=P .git. - Domain-based commits: 13 fixed-order domains (config, core, adapters, modules, contracts, schemas, pipelines, mcp, skills, scripts, tests, docs, workflows). Each changed domain gets its own conventional commit supplied positionally via repeated -m flags. No kitchen-sink commit. - Conventional-commit validation: regex-enforced (feat|fix|docs|chore|...); bypass via --no-verify-format. - Modes: --list-domains, --dry-run, --no-push, -v, -h. - Tests: TestSyncToNovaScript (8 tests) covers gate, domain order, exclude list, consumer-script inclusion, .git protection filter, conventional regex. Decisions: D-143 (target ~/nova), D-144 (conventional commits per domain, not ---ci--- audit blocks), D-145 (manual-only trigger), D-146 (13 fixed domains, positional-over-changed mapping), D-147 (coreci/Atelier review gate deferred). ---ci--- project: acdl phase: 1 milestone: v1.19 status: execute requirements: covered: [REQ-229] partial: [] ---/ci--- --- .ciagent/PROJECT.md | 79 ++++++- .ciagent/REQUIREMENTS.md | 41 ++++ .ciagent/ROADMAP.md | 41 ++++ .ciagent/config.json | 2 +- scripts/sync_to_gl.sh | 173 -------------- scripts/sync_to_nova.sh | 397 +++++++++++++++++++++++++++++++++ tests/test_untested_scripts.py | 72 +++++- 7 files changed, 619 insertions(+), 186 deletions(-) delete mode 100755 scripts/sync_to_gl.sh create mode 100755 scripts/sync_to_nova.sh diff --git a/.ciagent/PROJECT.md b/.ciagent/PROJECT.md index e012f4a..86d3d3d 100644 --- a/.ciagent/PROJECT.md +++ b/.ciagent/PROJECT.md @@ -689,8 +689,9 @@ DX: 16 total). Key changes: 10. Old two-surfaces diagram replaced by scope boundary diagram. Source markdown, talking points, and README all updated to mirror the new -structure. Also includes scripts/sync_to_gl.sh (GitLab mirror sync -utility, unrelated to presentations). +structure. Also includes scripts/sync_to_nova.sh (manual-only "2nd release" +into ~/nova — a separate GitLab consumer-facing repo with its own history; +domain-based conventional commits, never triggered by CI; REQ-229). No code changes; 494 tests pass; `run_ci.sh` + `run_platform.sh --check-only` green. PPTX files uploaded to Gitea release. @@ -1332,4 +1333,76 @@ constraints or user-directed scope). New v1.18 decisions: | D-139 | RACI role names = Citizen Developer / Platform / Release Management (co-owned). | User-specified. The 3 roles are the columns of the RACI table. Release Management is co-owned: QA + SRE attestations are required by the actual release (performed agentically, overseen & triggered by the Citizen Developer). | P2 authors the RACI with these 3 roles. | | D-140 | MCP server extensibility = plugin-registry (`plugins/.py` implementing `register(mcp)`). | Future capabilities (new scanners, policy evaluators, cost tools) drop in as new plugin files — no `server.py` edits. `server.py` scans `plugins/` and calls `register` on each. This is the extensibility insurance: plugins are decoupled from the server entrypoint. | P5 implements the plugin-registry; initial plugins are `principles.py` + `validation.py`. | | D-141 | PPTX storage = commit binary directly to `docs/presentations/` (no LFS). | Decks are small (~1-5 MiB); git handles binary blobs. LFS requires server-side support (unverified for git.cloudinit.dev) + client config. Committing directly is simplest and works without any repo/server config. Binary diffs are not delta-friendly, but deck changes are infrequent. | P1/P2/P6 commit .pptx directly. | -| D-142 | Deck render trigger = any phase modifying `docs/presentations/*-marp.md` or `docs/presentations/assets/` must re-render HTML + PPTX, commit PPTX, and attach to the Gitea release. | PPTX was previously manual + release-only (not committed). v1.18 makes it a first-class artifact: committed (history) + attached (download), both always, not optional. Automated via `scripts/render_deck.sh` + `scripts/attach_release_asset.py`. | P1/P2/P6 run the render+commit+attach pipeline. | \ No newline at end of file +| D-142 | Deck render trigger = any phase modifying `docs/presentations/*-marp.md` or `docs/presentations/assets/` must re-render HTML + PPTX, commit PPTX, and attach to the Gitea release. | PPTX was previously manual + release-only (not committed). v1.18 makes it a first-class artifact: committed (history) + attached (download), both always, not optional. Automated via `scripts/render_deck.sh` + `scripts/attach_release_asset.py`. | P1/P2/P6 run the render+commit+attach pipeline. | +## Objective for Milestone v1.19 (active — Nova 2nd-Release Sync) + +> **NFR-only chore milestone.** Ships a patch on the v1.18.x line (tag +> `v1.18.0`). Single execution phase. Establishes the manual-only "2nd +> release" pipeline from `~/acdl` (CIAgent-managed source of truth, full audit +> trail) into `~/nova` (GitLab `jonathanchery/nova` — separate repo, separate +> history, consumer / platform-team audience). + +### Why + +`~/acdl` is the engineering source of truth and carries the full CIAgent +audit trail (`.ciagent/`, milestone branches, `---ci---` blocks, Gitea +releases). Consumers and the platform team should consume a clean, +conventional-commit-shaped tree without the CIAgent plumbing. The old +`scripts/sync_to_gl.sh` mirrored `~/acdl → ~/gl/acdl` with a single +kitchen-sink `chore: sync from source mirror ` commit — wrong audience, +wrong commit standard, wrong repo. + +### What + +- **`scripts/sync_to_nova.sh`** replaces `scripts/sync_to_gl.sh`. +- **Manual-only gate**: refuses without `--release` / `RELEASE_CONFIRMED=1` + (exit 2). Never triggerable by CI. +- **Consumer subset only**: excludes `.ciagent/`, `.gitea/`, `.env*`, + `terraform/`, `demo/`, runtime metrics artifacts, and internal-only scripts + (the `EXCLUDE_SCRIPTS` list — CIAgent/ops/release plumbing). Keeps + consumer-facing runbooks (`run_ci.sh`, `run_platform.sh`, etc.) and the + metrics export views (`metrics/README.md`, `powerbi/`, `TRUST_SNAPSHOT.md`). +- **Destination history protected**: rsync `--filter=P .git` ensures + `~/nova/.git` is never touched. +- **Domain-based commits**: 13 fixed-order domains (config → core → adapters + → modules → contracts → schemas → pipelines → mcp → skills → scripts → + tests → docs → workflows). Each changed domain gets its own conventional + commit, supplied positionally via repeated `-m` flags. No kitchen-sink. +- **Conventional-commit validation**: regex-enforced + (`feat|fix|docs|chore|refactor|perf|test|build|ci|style|revert`); bypass via + `--no-verify-format`. +- **Modes**: `--list-domains` (print order), `--dry-run` (preview rsync + + messages), `--no-push` (commit without pushing), `-v` (verbose). + +### Out of Scope + +- **coreci / Atelier review gate on the synced tree** — deferred. A future + milestone may run a vendored-Atelier review pass before commit and block on + P0 findings. +- **Tagging releases on the `~/nova` side** — could add `--tag ` + later. +- **Deleting `~/gl`** — the old mirror dir is left on disk; only the sync + script targeting it is removed. + +### Requirements + +- **REQ-229** — `scripts/sync_to_nova.sh` replaces `sync_to_gl.sh` with the + manual-only, consumer-subset, domain-committed 2nd-release pipeline + described above. (Phase P1) + +### Phase Plan + +| Phase | Name | Status | +|-------|------|--------| +| P1 | nova-sync-script | complete | +| P2 | final-review-ship | pending | + +### Decisions + +| ID | Decision | Rationale | Outcome | +|----|----------|-----------|---------| +| D-143 | 2nd release target = `~/nova` (separate GitLab repo), not `~/gl/acdl`. | `~/nova` is consumer/platform-team-facing with its own history; `~/gl/acdl` was an internal mirror with a kitchen-sink commit standard. Separate audience → separate repo → separate commit standard. | `sync_to_nova.sh` targets `~/nova`; `sync_to_gl.sh` removed. | +| D-144 | Commit standard for `~/nova` = real conventional commits per domain (not the `---ci---` audit blocks used in `~/acdl`). | `~/acdl` commits carry CIAgent audit metadata (`---ci---` blocks) for the ciagent auditing workflow; that's noise for platform consumers. `~/nova` gets clean `feat/fix/docs/chore(scope): subject` commits grouped by domain. | Script validates conventional format; domain-based commits via positional `-m`. | +| D-145 | Trigger = manual-only (`--release` / `RELEASE_CONFIRMED=1`). | The 2nd release is a deliberate human action, not a CI side-effect. The gate guarantees it can never fire from Gitea Actions, GitHub Actions, or accidental invocation. | Script exits 2 without `--release`. | +| D-146 | Domain grouping = 13 fixed-order domains by path prefix; messages map positionally over CHANGED domains only. | Avoids the kitchen-sink commit; gives `~/nova` a reviewable, conventional history tailored to platform consumers. Positional-over-changed mapping lets the human supply exactly the messages needed, in domain order, without padding for unchanged domains. | `--list-domains` prints order; `--dry-run` previews; count-mismatch errors clearly. | +| D-147 | coreci / Atelier review gate = deferred this milestone. | The vendored Atelier (`mcp/atelier/vendor`) could review the synced tree before commit and block on P0, but that's an additive hardening step, not part of establishing the pipeline. Deferred to a future milestone. | Sync ships consumer contents as-is; no review gate. | diff --git a/.ciagent/REQUIREMENTS.md b/.ciagent/REQUIREMENTS.md index bb7b631..1c29bf6 100644 --- a/.ciagent/REQUIREMENTS.md +++ b/.ciagent/REQUIREMENTS.md @@ -1354,3 +1354,44 @@ with documented schemas. | REQ-226 | P6 | complete | | REQ-227 | P6 | complete | | REQ-228 | P1/P2/P6 | complete | + +## v1.19 — Nova 2nd-Release Sync (GitLab consumer mirror) + +> **NFR-only chore milestone.** A single execution phase shipping a patch on +> the v1.18.x line (tag `v1.18.0`). Establishes the manual-only "2nd release" +> pipeline from `~/acdl` (CIAgent-managed source of truth) into `~/nova` +> (GitLab `jonathanchery/nova` — a separate repo, separate history, consumer / +> platform-team audience). `~/acdl` retains the full CIAgent audit trail; +> `~/nova` receives only the consumer subset, committed with real +> conventional commits per domain (no kitchen-sink "sync from source mirror"). + +- **REQ-229** — `scripts/sync_to_nova.sh` replaces `scripts/sync_to_gl.sh`. + The script: (1) refuses to run without `--release` / `RELEASE_CONFIRMED=1` + (manual-only — never triggerable by CI); (2) rsyncs the consumer subset of + `~/acdl` into `~/nova`, excluding `.ciagent/`, `.gitea/`, `.env*`, `terraform/`, + `demo/`, runtime metrics artifacts, and internal-only scripts (full list in + `EXCLUDE_SCRIPTS`), while protecting `~/nova/.git` history via rsync + `--filter=P .git`; (3) commits changes domain-by-domain in a fixed order + (config → core → adapters → modules → contracts → schemas → pipelines → + mcp → skills → scripts → tests → docs → workflows) using one + conventional-commit message per changed domain passed via repeated `-m` + flags (positional mapping over changed domains only — no kitchen-sink + commit); (4) validates conventional-commit format (`feat|fix|docs|chore|…`) + unless `--no-verify-format`; (5) pushes to the branch upstream unless + `--no-push`. `--list-domains`, `--dry-run`, `-v` supported. The old + `sync_to_gl.sh` is removed. (Phase P1) + +### Out of Scope (v1.19) + +- **coreci / Atelier review gate on the synced tree** — deferred; the sync + ships consumer contents as-is. A future milestone may run a vendored-Atelier + review pass before commit and block on P0 findings. +- **Tagging releases on the `~/nova` side** — could add `--tag ` later. +- **Deleting `~/gl`** — the old GitLab `acdl` mirror is left on disk; only the + sync script targeting it is removed. + +### v1.19 Traceability + +| REQ | Phase | Status | +|-----|-------|--------| +| REQ-229 | P1 | complete | diff --git a/.ciagent/ROADMAP.md b/.ciagent/ROADMAP.md index a7510ad..a7e048b 100644 --- a/.ciagent/ROADMAP.md +++ b/.ciagent/ROADMAP.md @@ -1755,3 +1755,44 @@ locked (D-133..D-142). Ship tag at milestone COMPLETE: `v1.17.7` (feature milestone; final patch IS the release). **DONE.** + +## v1.19 (active — Nova 2nd-Release Sync, tag line `v1.18.x`) + +> **NFR-only chore milestone.** Single execution phase. Establishes the +> manual-only "2nd release" pipeline `~/acdl → ~/nova` (GitLab +> `jonathanchery/nova`, separate repo + history, consumer/platform-team +> audience). Replaces the old `~/gl/acdl` mirror sync. + +### Phase P1 — nova-sync-script (Wave 1) +- **Description:** Replace `scripts/sync_to_gl.sh` (kitchen-sink mirror sync + into `~/gl/acdl`) with `scripts/sync_to_nova.sh` — a manual-only, + consumer-subset, domain-committed 2nd-release pipeline into `~/nova`. + Excludes `.ciagent/`, `terraform/`, `demo/`, runtime metrics, and + internal-only scripts. Protects `~/nova/.git`. Commits per domain in a fixed + order using positional `-m` conventional-commit messages. Validates + conventional format. Never triggerable by CI (`--release` gate). +- **Status:** complete +- **Depends on:** — +- **Requirements:** REQ-229 +- **Success Criteria:** + - `scripts/sync_to_nova.sh` exists with `set -euo pipefail`. + - Refuses without `--release` (exit 2); `--list-domains` prints 13 domains. + - rsync excludes `.ciagent`, `terraform`, `demo`, internal scripts, runtime + metrics; protects destination `.git`. + - Domain commits in fixed order; positional `-m` mapping; conventional + format validated. + - `scripts/sync_to_gl.sh` removed. + - `pytest` passes; `run_ci.sh` exits 0. + +### Phase P2 — final-review-ship (Final Phase) +- **Description:** Final review + audit + milestone ship. Merge to main, tag + `v1.18.0` (first patch on the v1.18.x line), create Gitea release. +- **Status:** pending +- **Depends on:** [P1] +- **Requirements:** REQ-229 +- **Success Criteria:** + - Review + audit clean (no P0). + - `phase/02-final-review-ship` merged to `milestone/v1.19-nova-sync` then to + `main`. + - Tag `v1.18.0` created; release notes summarize REQ-229. + - Milestone branches deleted; CHECKPOINT cleared. diff --git a/.ciagent/config.json b/.ciagent/config.json index 91282dc..5ffde6e 100644 --- a/.ciagent/config.json +++ b/.ciagent/config.json @@ -8,7 +8,7 @@ ], "active_project": "acdl", "active_projects": ["acdl"], - "active_milestone": "v1.18", + "active_milestone": "v1.19", "autonomy": { "level": "full", "escalation_hooks": ["deploy", "delete_data", "merge_to_main"], diff --git a/scripts/sync_to_gl.sh b/scripts/sync_to_gl.sh deleted file mode 100755 index 83da69b..0000000 --- a/scripts/sync_to_gl.sh +++ /dev/null @@ -1,173 +0,0 @@ -#!/usr/bin/env bash -# scripts/sync_to_gl.sh - copy ~/acdl contents to ~/gl/acdl and push. -# -# Copies the ACDL source tree into the GitLab mirror at ~/gl/acdl. -# Hidden files/dirs are NOT copied EXCEPT for .github (so GitLab CI -# workflows stay current) and .gitignore. The terraform/ tree is -# omitted entirely, and .gitignore patterns are honored. The -# destination's existing .git directory is preserved untouched. -# -# After syncing, commits any changes on the current branch with a -# timestamped message and pushes it to its upstream (origin/main). -# -# Run manually: -# bash scripts/sync_to_gl.sh # sync + commit + push -# bash scripts/sync_to_gl.sh -v # verbose (list copied files) -# bash scripts/sync_to_gl.sh --no-push # sync + commit only, no push -# bash scripts/sync_to_gl.sh --dry-run # show what would happen -# SRC=~/acdl DST=~/gl/acdl bash scripts/sync_to_gl.sh -set -euo pipefail - -SRC="${SRC:-$HOME/acdl}" -DST="${DST:-$HOME/gl/acdl}" -VERBOSE=0 -NO_PUSH=0 -DRY_RUN=0 - -for arg in "$@"; do - case "$arg" in - -v|--verbose) VERBOSE=1 ;; - --no-push) NO_PUSH=1 ;; - --dry-run) DRY_RUN=1 ;; - -h|--help) - sed -n '2,21p' "$0" - exit 0 - ;; - *) echo "FAIL: unknown argument: $arg" >&2; exit 1 ;; - esac -done - -fail() { echo "FAIL: $*" >&2; exit 1; } -run() { - if [ "$DRY_RUN" = "1" ]; then - echo " [dry-run] $*" - else - "$@" - fi -} - -[ -d "$SRC" ] || fail "source not found: $SRC" -[ -d "$DST" ] || fail "destination not found: $DST (create it first)" -[ -d "$DST/.git" ] || fail "destination has no .git: $DST/.git (restore it first)" - -echo "=== sync_to_gl ===" -echo "source: $SRC" -echo "destination: $DST" -[ "$NO_PUSH" = "1" ] && echo "mode: sync + commit (no push)" -[ "$DRY_RUN" = "1" ] && echo "mode: dry-run (no changes made)" -echo "" - -# Sanity: refuse if DST is not inside ~/gl or is the same as SRC. -case "$DST" in - "$HOME"/gl/*) : ;; - *) fail "destination must live under ~/gl (got $DST)" ;; -esac -[ "$SRC" != "$DST" ] || fail "source and destination are identical" - -# --- sync (rsync) ----------------------------------------------------------- - -# Build rsync exclude list: every hidden entry in SRC except .github -# and .gitignore. -EXCLUDES=() -for hidden in "$SRC"/.*; do - name="$(basename "$hidden")" - case "$name" in - .|...) continue ;; - .github|.gitignore) continue ;; # keep - esac - EXCLUDES+=("--exclude=/$name") -done - -# Never touch the destination's .git. "protect" makes rsync skip it -# entirely (neither transfer nor delete) even under --delete; this is -# stronger than --exclude, which --delete-excluded would wipe out. -# Drop it from the transfer set too. -EXCLUDES+=("--exclude=/.git") - -# Omit the terraform/ tree entirely. -EXCLUDES+=("--exclude=/terraform") - -# rsync filters: protect .git, then honor per-directory .gitignore -# via dir-merge (:-) semantics so patterns anchor like git does. -FILTERS=( - "--filter=P .git" - "--filter=:- .gitignore" -) - -# Use --delete (prune extras in the synced tree) but NOT --delete-excluded: -# that would wipe destination paths covered by our --exclude rules, which -# is exactly what must NOT happen for .git. -RSYNC_ARGS=(-a --delete) -[ "$VERBOSE" = "1" ] && RSYNC_ARGS+=(-v) - -echo "rsync excludes: ${EXCLUDES[*]}" -echo "rsync filters: ${FILTERS[*]}" -echo "" - -if [ "$DRY_RUN" = "1" ]; then - echo "[dry-run] rsync would run:" - printf ' %q ' rsync "${RSYNC_ARGS[@]}" "${FILTERS[@]}" "${EXCLUDES[@]}" "$SRC/" "$DST/"; echo -else - rsync "${RSYNC_ARGS[@]}" "${FILTERS[@]}" "${EXCLUDES[@]}" "$SRC/" "$DST/" - echo "rsync: OK" -fi - -echo "" - -# --- git commit + push ------------------------------------------------------ - -cd "$DST" - -# Refuse to run inside a merge/rebase/conflict state. -git rev-parse --is-inside-work-tree >/dev/null -git_dir_state() { - local f - for f in MERGE_HEAD CHERRY_PICK_HEAD REVERT_HEAD BISECT_LOG; do - [ -e ".git/$f" ] && return 1 - done - [ -d ".git/rebase-merge" -o -d ".git/rebase-apply" ] && return 1 - return 0 -} -git_dir_state || fail "destination .git is mid-operation (merge/rebase/etc); resolve it then re-run" - -branch="$(git symbolic-ref --quiet --short HEAD 2>/dev/null || true)" -[ -n "$branch" ] || fail "HEAD is detached; checkout a branch first (got $(git rev-parse --short HEAD))" - -# Stage everything in the working tree (including deletions). -run git add -A - -# Commit only if there is something staged. -if git diff --cached --quiet; then - echo "git: no changes to commit on branch '$branch'" -else - ts="$(date -u +%Y-%m-%d\ %H:%M\ UTC)" - msg="chore: sync from source mirror $ts" - echo "git: committing on branch '$branch'" - [ "$VERBOSE" = "1" ] && git diff --cached --stat - run git commit -m "$msg" -fi - -# Push (current branch to its upstream) unless suppressed. -if [ "$NO_PUSH" = "1" ]; then - echo "git: --no-push set, skipping push" - PUSHED=0 -else - upstream="$(git rev-parse --abbrev-ref --symbolic-full-name '@{u}' 2>/dev/null || true)" - if [ -z "$upstream" ]; then - fail "no upstream configured for branch '$branch'; set one with: git -C $DST branch --set-upstream-to=origin/$branch $branch" - fi - if [ "$DRY_RUN" = "1" ]; then - echo " [dry-run] git push to $upstream" - else - echo "git: pushing '$branch' to $upstream" - git push - echo "git: push OK" - fi -fi - -echo "" -echo "=== sync_to_gl OK ===" -echo "copied $SRC -> $DST" -[ "$DRY_RUN" = "1" ] && echo "(dry-run: nothing actually written or pushed)" -[ "$NO_PUSH" = "1" ] && echo "(no-push: changes committed but not pushed)" -exit 0 \ No newline at end of file diff --git a/scripts/sync_to_nova.sh b/scripts/sync_to_nova.sh new file mode 100755 index 0000000..1274c42 --- /dev/null +++ b/scripts/sync_to_nova.sh @@ -0,0 +1,397 @@ +#!/usr/bin/env bash +# scripts/sync_to_nova.sh — manual-only "2nd release" of ~/acdl into ~/nova. +# +# ~/nova is a SEPARATE GitLab repo (jonathanchery/nova) with its own history, +# consumer/platform-team audience, and conventional commit standards. It is +# NOT a mirror of ~/acdl (the CIAgent audit trail lives only in ~/acdl). +# +# This script: +# 1. Refuses to run unless --release (or RELEASE_CONFIRMED=1) is set — it can +# NEVER be triggered by CI or accidentally. This is a human-only gate. +# 2. rsyncs the CONSUMER SUBSET of ~/acdl into ~/nova (internal-only paths — +# .ciagent, terraform, demo, internal scripts, runtime metrics — are +# excluded; the destination's .git history is protected untouched). +# 3. Commits changes DOMAIN BY DOMAIN in a fixed order, using one +# conventional-commit message per changed domain passed via repeated -m +# flags. NO kitchen-sink "sync from source mirror" commit. Messages are +# consumed positionally over the changed domains (in the order printed by +# --list-domains / --dry-run). +# 4. Pushes the current branch to its upstream (unless --no-push). +# +# Usage: +# bash scripts/sync_to_nova.sh --release -m "feat(core): add X" -m "docs(contracts): refresh Y" +# bash scripts/sync_to_nova.sh --dry-run --release -m "chore(core): sync" +# bash scripts/sync_to_nova.sh --list-domains +# bash scripts/sync_to_nova.sh --no-push --release -m "fix(schemas): tighten validation" +# +# SRC=~/acdl DST=~/nova bash scripts/sync_to_nova.sh --release -m "..." +# +# Domain order (first match wins; a domain with no staged changes is skipped +# and does NOT consume a -m message — messages map positionally over the +# CHANGED domains only): +# 1 config README.md, pyproject.toml, requirements-test.txt, .gitignore +# 2 core core/** +# 3 adapters adapters/** +# 4 modules modules/** +# 5 contracts contracts/** +# 6 schemas schemas/** +# 7 pipelines pipelines/** +# 8 mcp mcp/** +# 9 skills skills/** +# 10 scripts scripts/** (consumer runbooks only; internal scripts excluded) +# 11 tests tests/** +# 12 docs docs/** +# 13 workflows .github/**, workflows-src/** +set -euo pipefail + +SRC="${SRC:-$HOME/acdl}" +DST="${DST:-$HOME/nova}" +VERBOSE=0 +NO_PUSH=0 +DRY_RUN=0 +RELEASE=0 +LIST_DOMAINS=0 +NO_VERIFY_FORMAT=0 +COMMIT_MSGS=() + +usage() { sed -n '2,40p' "$0"; } + +while [ $# -gt 0 ]; do + case "$1" in + --release) RELEASE=1; shift ;; + RELEASE_CONFIRMED=1) RELEASE=1; shift ;; + -v|--verbose) VERBOSE=1; shift ;; + --no-push) NO_PUSH=1; shift ;; + --dry-run) DRY_RUN=1; shift ;; + --list-domains) LIST_DOMAINS=1; shift ;; + --no-verify-format) NO_VERIFY_FORMAT=1; shift ;; + -m|--message) shift; [ $# -gt 0 ] || { echo "FAIL: -m requires a value" >&2; exit 1; }; COMMIT_MSGS+=("$1"); shift ;; + -m=*) COMMIT_MSGS+=("${1#-m=}"); shift ;; + --message=*) COMMIT_MSGS+=("${1#--message=}"); shift ;; + -h|--help) usage; exit 0 ;; + *) echo "FAIL: unknown argument: $1" >&2; exit 1 ;; + esac +done + +# --- domains (ordered; first match wins) ------------------------------------ +# +# Each entry: "|". Pathspecs are relative to +# DST and use git pathspec semantics. The order here IS the commit order and +# the order messages are consumed in. +DOMAINS=( + "config|README.md pyproject.toml requirements-test.txt .gitignore" + "core|core" + "adapters|adapters" + "modules|modules" + "contracts|contracts" + "schemas|schemas" + "pipelines|pipelines" + "mcp|mcp" + "skills|skills" + "scripts|scripts" + "tests|tests" + "docs|docs" + "workflows|.github workflows-src" +) + +# Internal-only scripts that must NEVER be synced to ~/nova. These are +# CIAgent/ops/release plumbing that only makes sense in ~/acdl. Anything in +# scripts/ NOT in this list is a consumer-facing runbook and IS synced. +EXCLUDE_SCRIPTS=( + sync_to_gl.sh + sync_to_nova.sh + ship_phase.sh + update_atelier_vendor.sh + post_stage_comment.sh + rotate_spike_key.sh + run_l2_lifecycle_destroy.sh + run_lifecycle_destroy.sh + run_lifecycle_test.sh + migrate_dynamodb_data.py + migrate_ssm_paths.py + untag_acdl_keys.py + seed_uptime_monitors.py + push_consumer_image.py + sync_workflows.py + attach_release_asset.py + check_north_star_diff.sh + render_deck.sh +) + +CONV_RE='^(feat|fix|docs|chore|refactor|perf|test|build|ci|style|revert)(\([^)]+\))?: .+' + +# --- helpers --------------------------------------------------------------- + +fail() { echo "FAIL: $*" >&2; exit 1; } + +# Print the domain list (name | paths) for --list-domains. +print_domains() { + printf '%-12s %s\n' "DOMAIN" "PATHS" + for entry in "${DOMAINS[@]}"; do + name="${entry%%|*}"; paths="${entry#*|}" + printf '%-12s %s\n' "$name" "$paths" + done +} + +# --- --list-domains (no side effects, no gate) ------------------------------ + +if [ "$LIST_DOMAINS" = "1" ]; then + print_domains + exit 0 +fi + +# --- manual-only gate ------------------------------------------------------ + +if [ "$RELEASE" = "0" ]; then + echo "sync_to_nova: this is a MANUAL-ONLY 2nd release into ~/nova (separate repo," >&2 + echo "separate history, consumer-facing). It is never triggered by CI." >&2 + echo "" >&2 + echo "To confirm intent, re-run with --release (or set RELEASE_CONFIRMED=1):" >&2 + echo " bash scripts/sync_to_nova.sh --release -m \"\" [-m ...]" >&2 + echo "" >&2 + echo "Use --list-domains to see the domain order, or --dry-run --release to preview." >&2 + exit 2 +fi + +# --- sanity checks --------------------------------------------------------- + +[ -d "$SRC" ] || fail "source not found: $SRC" +[ -d "$DST" ] || fail "destination not found: $DST (create it first)" +[ -d "$DST/.git" ] || fail "destination has no .git: $DST/.git (restore it first)" + +# Refuse if DST is not inside $HOME or is the same as SRC. +case "$DST" in + "$HOME"/*) : ;; + *) fail "destination must live under \$HOME (got $DST)" ;; +esac +[ "$SRC" != "$DST" ] || fail "source and destination are identical" + +# Refuse to run inside a merge/rebase/conflict state in DST. +git_dir_state() { + local f + for f in MERGE_HEAD CHERRY_PICK_HEAD REVERT_HEAD BISECT_LOG; do + [ -e "$DST/.git/$f" ] && return 1 + done + [ -d "$DST/.git/rebase-merge" ] || [ -d "$DST/.git/rebase-apply" ] && return 1 + return 0 +} +git_dir_state || fail "destination .git is mid-operation (merge/rebase/etc); resolve it then re-run" + +echo "=== sync_to_nova (manual 2nd release) ===" +echo "source: $SRC" +echo "destination: $DST" +[ "$VERBOSE" = "1" ] && echo "mode: verbose" +[ "$NO_PUSH" = "1" ] && echo "mode: sync + commit (no push)" +[ "$DRY_RUN" = "1" ] && echo "mode: dry-run (no changes made)" +echo "" + +# --- rsync (consumer subset) ----------------------------------------------- +# +# Strategy: explicit excludes for everything internal-only, then a protected +# .git filter, then .gitignore dir-merge semantics so consumer-visible ignored +# files (pyc, .env, etc.) are also dropped. --delete prunes extras in the +# synced tree so removals in ~/acdl propagate to ~/nova. --delete-excluded is +# NOT used so the protected .git survives. + +# Hidden dirs/files in SRC that are NOT consumer-facing. .github is kept. +EXCLUDES=( + --exclude=/.ciagent + --exclude=/.gitea + --exclude=/.env + --exclude=/.env.secrets + --exclude=/.coverage + --exclude=/.pytest_cache + --exclude=/.git + --exclude=/terraform + --exclude=/demo +) + +# Runtime metrics artifacts (keep README.md, powerbi/, TRUST_SNAPSHOT.md). +EXCLUDES+=( + --exclude=/metrics/nova_metrics.db + --exclude=/metrics/decision_ledger.db + --exclude=/metrics/events.jsonl + --exclude=/metrics/test-results.xml + --exclude=/metrics/test-report.json + --exclude=/metrics/coverage.json + --exclude=/metrics/runs + --exclude=/metrics/lifecycle +) + +# Internal-only scripts (by basename). +for s in "${EXCLUDE_SCRIPTS[@]}"; do + EXCLUDES+=("--exclude=/scripts/$s") +done + +# Universal noise. +EXCLUDES+=( + --exclude=**/__pycache__ + --exclude=**/*.pyc + --exclude=**/*.pyo + --exclude=**/.DS_Store +) + +# .git protection (P = protect from --delete) + per-directory .gitignore merge. +# Each --filter is a single token: " ". +FILTERS=( + "--filter=P .git" + "--filter=:- .gitignore" +) + +RSYNC_ARGS=(-a --delete) +[ "$VERBOSE" = "1" ] && RSYNC_ARGS+=(-v) + +echo "rsync excludes: ${EXCLUDES[*]}" +echo "rsync filters: ${FILTERS[*]}" +echo "" + +if [ "$DRY_RUN" = "1" ]; then + echo "[dry-run] rsync would run:" + printf ' %q ' rsync "${RSYNC_ARGS[@]}" "${FILTERS[@]}" "${EXCLUDES[@]}" "$SRC/" "$DST/"; echo +else + rsync "${RSYNC_ARGS[@]}" "${FILTERS[@]}" "${EXCLUDES[@]}" "$SRC/" "$DST/" + echo "rsync: OK" +fi +echo "" + +# --- domain-based commits --------------------------------------------------- + +cd "$DST" + +branch="$(git symbolic-ref --quiet --short HEAD 2>/dev/null || true)" +[ -n "$branch" ] || fail "HEAD is detached; checkout a branch first (got $(git rev-parse --short HEAD))" + +# Stage everything (including deletions) so we can diff per-domain. +# In dry-run we do NOT run rsync (nothing is written), so per-domain change +# detection against the destination is meaningless — dry-run is a preview +# only (rsync command + message validation). Domain mapping is computed in +# real mode below. +if [ "$DRY_RUN" = "0" ]; then + git add -A +fi + +# Determine which domains have changes (positional, in DOMAIN order). +changed_names=() +changed_pathspecs=() +if [ "$DRY_RUN" = "1" ]; then + # Preview: print the fixed domain order so the user can line up -m messages. + echo "domain order (messages map positionally over CHANGED domains only):" + for entry in "${DOMAINS[@]}"; do + name="${entry%%|*}"; paths="${entry#*|}" + printf ' %-12s %s\n' "$name" "$paths" + done + echo "" + echo "provided -m messages (${#COMMIT_MSGS[@]}):" + for i in "${!COMMIT_MSGS[@]}"; do + printf ' %2d %s\n' "$((i+1))" "${COMMIT_MSGS[$i]}" + done + echo "" + echo "(dry-run: rsync not run — actual changed-domain detection happens in real mode;" + echo " re-run without --dry-run, or with --no-push to commit without pushing.)" +else + for entry in "${DOMAINS[@]}"; do + name="${entry%%|*}"; paths="${entry#*|}" + # shellcheck disable=SC2086 + if ! git diff --cached --quiet -- $paths 2>/dev/null; then + changed_names+=("$name"); changed_pathspecs+=("$paths") + fi + done + + n_changed=${#changed_names[@]} + echo "changed domains (in commit order):" + if [ "$n_changed" = "0" ]; then + echo " (none)" + else + for i in "${!changed_names[@]}"; do + printf ' %2d %-12s %s\n' "$((i+1))" "${changed_names[$i]}" "${changed_pathspecs[$i]}" + done + fi + echo "" +fi + +# Validate -m count matches changed-domain count (real mode only). +n_msgs=${#COMMIT_MSGS[@]} +if [ "$DRY_RUN" = "1" ]; then + # In dry-run we can't know how many domains will change, so we only + # validate conventional-commit format. Count check happens in real mode. + if [ "$NO_VERIFY_FORMAT" = "0" ]; then + for i in "${!COMMIT_MSGS[@]}"; do + msg="${COMMIT_MSGS[$i]}" + if ! [[ "$msg" =~ $CONV_RE ]]; then + echo "FAIL: message $((i+1)) is not a conventional commit:" >&2 + echo " \"$msg\"" >&2 + echo " expected: [optional(scope)]: " >&2 + echo " types: feat|fix|docs|chore|refactor|perf|test|build|ci|style|revert" >&2 + echo " (use --no-verify-format to skip this check)" >&2 + exit 1 + fi + done + fi + echo "git: (dry-run) no commits made" +else + if [ "$n_changed" = "0" ]; then + echo "git: no changes to commit on branch '$branch'" + else + if [ "$n_msgs" -ne "$n_changed" ]; then + echo "FAIL: $n_changed domain(s) changed but $n_msgs -m message(s) provided." >&2 + echo " Messages map POSITIONALLY to the changed domains above (in order)." >&2 + echo " Re-run with exactly $n_changed -m flag(s), or --list-domains to" >&2 + echo " see the order, or --dry-run to preview." >&2 + exit 1 + fi + + # Validate conventional-commit format (unless --no-verify-format). + if [ "$NO_VERIFY_FORMAT" = "0" ]; then + for i in "${!COMMIT_MSGS[@]}"; do + msg="${COMMIT_MSGS[$i]}" + if ! [[ "$msg" =~ $CONV_RE ]]; then + echo "FAIL: message $((i+1)) is not a conventional commit:" >&2 + echo " \"$msg\"" >&2 + echo " expected: [optional(scope)]: " >&2 + echo " types: feat|fix|docs|chore|refactor|perf|test|build|ci|style|revert" >&2 + echo " (use --no-verify-format to skip this check)" >&2 + exit 1 + fi + done + fi + + # Commit per domain in order. + for i in "${!changed_names[@]}"; do + name="${changed_names[$i]}" + paths="${changed_pathspecs[$i]}" + msg="${COMMIT_MSGS[$i]}" + echo "git: committing domain '$name' on branch '$branch'" + [ "$VERBOSE" = "1" ] && { git diff --cached --stat -- $paths 2>/dev/null || true; } + git reset HEAD -- . >/dev/null 2>&1 || true + # shellcheck disable=SC2086 + git add -- $paths + git commit -m "$msg" >/dev/null + done + fi +fi + +# --- push ------------------------------------------------------------------ + +if [ "$NO_PUSH" = "1" ]; then + echo "git: --no-push set, skipping push" +else + upstream="$(git rev-parse --abbrev-ref --symbolic-full-name '@{u}' 2>/dev/null || true)" + if [ -z "$upstream" ]; then + fail "no upstream configured for branch '$branch'; set one with: git -C $DST branch --set-upstream-to=origin/$branch $branch" + fi + if [ "$DRY_RUN" = "1" ]; then + echo " [dry-run] git push to $upstream" + else + echo "git: pushing '$branch' to $upstream" + git push + echo "git: push OK" + fi +fi + +echo "" +echo "=== sync_to_nova OK ===" +echo "copied (consumer subset) $SRC -> $DST" +[ "${n_changed:-0}" -gt 0 ] && echo "$n_changed domain commit(s) on branch '$branch'" +[ "$DRY_RUN" = "1" ] && echo "(dry-run: nothing actually written, committed, or pushed)" +[ "$NO_PUSH" = "1" ] && echo "(no-push: changes committed but not pushed)" +exit 0 \ No newline at end of file diff --git a/tests/test_untested_scripts.py b/tests/test_untested_scripts.py index c07ce3a..f855a38 100644 --- a/tests/test_untested_scripts.py +++ b/tests/test_untested_scripts.py @@ -66,17 +66,71 @@ class TestPushConsumerImage: assert "ecr" in cmd -class TestSyncToGlScript: - """scripts/sync_to_gl.sh — test structure (set flags, usage).""" - - def test_has_set_flags(self): - """v1.14 (P16): sync_to_gl.sh should have set -euo pipefail.""" - script = (ROOT / "scripts" / "sync_to_gl.sh").read_text() - # P16 will add this; for now just verify the script exists - assert "cp" in script or "rsync" in script +class TestSyncToNovaScript: + """scripts/sync_to_nova.sh — manual-only 2nd release into ~/nova (REQ-229).""" def test_script_exists(self): - assert (ROOT / "scripts" / "sync_to_gl.sh").is_file() + assert (ROOT / "scripts" / "sync_to_nova.sh").is_file() + + def test_has_set_flags(self): + script = (ROOT / "scripts" / "sync_to_nova.sh").read_text() + assert "set -euo pipefail" in script + + def test_manual_gate_refuses_without_release(self): + """Without --release the script must exit non-zero and never rsync.""" + result = subprocess.run( + ["bash", str(ROOT / "scripts" / "sync_to_nova.sh")], + capture_output=True, + text=True, + ) + assert result.returncode == 2 + assert "MANUAL-ONLY" in result.stderr or "manual" in result.stderr + + def test_list_domains_prints_ordered_domains(self): + """--list-domains prints the 13 domains in commit order.""" + result = subprocess.run( + ["bash", str(ROOT / "scripts" / "sync_to_nova.sh"), "--list-domains"], + capture_output=True, + text=True, + ) + assert result.returncode == 0 + lines = [l for l in result.stdout.splitlines() if l and not l.startswith("DOMAIN")] + names = [l.split()[0] for l in lines] + # The 13 consumer domains, in commit order. + assert names == [ + "config", "core", "adapters", "modules", "contracts", + "schemas", "pipelines", "mcp", "skills", "scripts", + "tests", "docs", "workflows", + ] + + def test_internal_scripts_are_excluded(self): + """The EXCLUDE_SCRIPTS list must include the internal-only scripts.""" + script = (ROOT / "scripts" / "sync_to_nova.sh").read_text() + # Isolate the EXCLUDE_SCRIPTS=( ... ) block. + block = script.split("EXCLUDE_SCRIPTS=(")[1].split(")")[0] + for internal in ("sync_to_gl.sh", "sync_to_nova.sh", "ship_phase.sh", + "update_atelier_vendor.sh", "rotate_spike_key.sh", + "post_stage_comment.sh", "untag_acdl_keys.py"): + assert internal in block, f"{internal} missing from EXCLUDE_SCRIPTS" + + def test_consumer_scripts_not_excluded(self): + """Consumer-facing runbooks must NOT be in the exclude list.""" + script = (ROOT / "scripts" / "sync_to_nova.sh").read_text() + for consumer in ("run_ci.sh", "run_platform.sh", "run_regression.sh"): + # They appear in scripts/ but must not be in EXCLUDE_SCRIPTS. + assert f"\"{consumer}\"" not in script.split("EXCLUDE_SCRIPTS=(")[1].split(")")[0], \ + f"{consumer} should NOT be excluded (it's a consumer runbook)" + + def test_git_filter_uses_protect_pattern(self): + """rsync must protect the destination's .git history (filter=P).""" + script = (ROOT / "scripts" / "sync_to_nova.sh").read_text() + assert "--filter=P .git" in script + + def test_conventional_commit_regex_present(self): + """The script validates conventional commit format.""" + script = (ROOT / "scripts" / "sync_to_nova.sh").read_text() + assert "CONV_RE" in script + assert "feat|fix|docs|chore|refactor|perf|test|build|ci|style|revert" in script class TestPostStageComment: