#!/usr/bin/env bats # Bats tests for scripts/proxmox/lxc-deploy.sh orchestration (SLICE-09). # # Run: bats scripts/proxmox/test/lxc-deploy.bats # # lxc-deploy.sh orchestrates: stage-snippet → clone → config → start → # health-check → success. On ANY failure the EXIT trap fires rollback.sh. # The trap captures $? so a `set -e` child failure (e.g. health-check) # triggers rollback, not just INT/TERM. # # Idempotency (D-027): if the target VMID already exists + is healthy, # the deploy skips clone/config/start (idempotent re-deploy). If the CT # exists but is unhealthy, the operator must pass --recreate (rollback + # redeploy) or --reconfigure (re-PUT config + restart) — otherwise the # deploy errors with guidance and leaves the CT intact. # # These tests build a sandbox copy of lxc-deploy.sh with stub sibling # scripts + a stub api.sh + the REAL ct-exists.sh (P16) + a stub # timing.sh so the real orchestrator logic (trap, sequencing, # idempotency, flag parsing) is exercised without a live Proxmox # endpoint. # # Praxis v0.2 (vs coreci) key differences asserted here: # - NO proxy/backend-add/smoke-test steps (proxy tier removed) # - VMID auto-allocation via pve_nextid when PROXMOX_LXC_VMID unset # - hookscript snippet volid is local:snippets/praxis-firstboot.sh setup() { SCRIPT_DIR="$(cd "$(dirname "$BATS_TEST_FILENAME")/.." && pwd)" DEPLOY="${SCRIPT_DIR}/lxc-deploy.sh" STUB_DIR="$(mktemp -d)" export STUB_DIR LOG="${STUB_DIR}/calls.log" export CALL_LOG="$LOG" : > "$LOG" 2>/dev/null || true # Sandbox layout: # /lxc-deploy.sh (SCRIPT_DIR) # /api.sh (sourced) # /ct-exists.sh (REAL — sourced by lxc-deploy.sh) # /timing.sh (stubbed — sourced by lxc-deploy.sh) # /stage-snippet.sh (invoked) # /lxc-clone.sh (invoked) # /lxc-config.sh (invoked) # /lxc-start.sh (invoked) # /health-check.sh (invoked; exit overridable) # /rollback.sh (invoked on failure; records call) ROOT="${STUB_DIR}/root" mkdir -p "$ROOT" cp "$DEPLOY" "${ROOT}/lxc-deploy.sh" # ct-exists.sh (P16) — REAL, sourced by lxc-deploy.sh. cp "${SCRIPT_DIR}/ct-exists.sh" "${ROOT}/ct-exists.sh" # recording stub generator: logs ":" to $CALL_LOG, exits # with the given code (default 0). log_stub() { name="$1"; exit_var="$2" printf '#!/bin/sh\necho "%s:$*" >> "%s"\nexit ${%s:-0}\n' \ "$name" "$CALL_LOG" "$exit_var" > "${ROOT}/${name}.sh" chmod +x "${ROOT}/${name}.sh" } log_stub stage-snippet STUB_SNIPPET_EXIT log_stub lxc-clone STUB_CLONE_EXIT log_stub lxc-config STUB_CONFIG_EXIT log_stub lxc-start STUB_START_EXIT log_stub rollback STUB_ROLLBACK_EXIT # health-check stub: exit overridable; fails the FIRST call (the # idempotency probe) when STUB_HEALTH_FIRST_FAIL=1, then passes # subsequent calls (the post-remediation health-check). cat > "${ROOT}/health-check.sh" <<'HSTUB' #!/bin/sh echo "health-check:$*" >> "$CALL_LOG" count_file="${CALL_LOG}.hc" n=$(cat "$count_file" 2>/dev/null || echo 0) n=$((n + 1)) echo "$n" > "$count_file" if [ "${STUB_HEALTH_FIRST_FAIL:-0}" = "1" ] && [ "$n" -eq 1 ]; then exit 1 fi exit ${STUB_HEALTH_EXIT:-0} HSTUB chmod +x "${ROOT}/health-check.sh" # Mocked api.sh — pve_env no-op; pve_nextid returns STUB_NEXTID; # pve_get returns STUB_PVE_GET (empty by default → ct not found + # snippet-exists check finds nothing → stage-snippet runs); pve_curl # + pve_poll no-op. cat > "${ROOT}/api.sh" <<'ASTUB' pve_env() { :; } pve_nextid() { printf '%s\n' "${STUB_NEXTID:-200}"; } pve_get() { printf '%s\n' "${STUB_PVE_GET:-}"; } pve_curl() { :; } pve_poll() { :; } pve_tls_insecure() { :; } pve_auth_header() { :; } ASTUB chmod +x "${ROOT}/api.sh" # timing.sh — stubbed to no-op so the orchestrator logic is exercised # without the real helper; timing.sh itself is tested in timing.bats. cat > "${ROOT}/timing.sh" <<'EOF' timing_start() { :; } timing_end() { :; } EOF chmod +x "${ROOT}/timing.sh" export PROXMOX_API_URL="https://proxmox.test:8006/api2/json" export PROXMOX_API_TOKEN="root@pam!test=secret" export PROXMOX_NODE="testnode" export PROXMOX_STORAGE="local" export PROXMOX_TEMPLATE_VOLID="local:vztmpl/debian-12-template.tar.zst" export GITEA_TOKEN="gitea-test-token" export PROXMOX_LXC_VMID="200" # Isolate from the operator's real .env.secrets files: lxc-deploy.sh # sources ~/coreci/.ciagent/.env.secrets + .ciagent/.env.secrets, which # on a live deploy host would override the test's PROXMOX_LXC_VMID (and # other vars) with cluster values. Point HOME + the script's PROJ_ROOT # computation at the sandbox so neither secrets file is found (the # deploy script emits a warning + relies on the exported test env). export HOME="${STUB_DIR}" # Stub cd so PROJ_ROOT resolves inside the sandbox: lxc-deploy.sh uses # `cd "${SCRIPT_DIR}/../.."`. SCRIPT_DIR is the sandbox ; we make # /../.. resolve to by creating /.. (already # exists) — the default mktemp parent. No .ciagent/.env.secrets there. # Reset the health-check call counter between tests. rm -f "${CALL_LOG}.hc" 2>/dev/null || true } teardown() { [ -n "${STUB_DIR:-}" ] && rm -rf "$STUB_DIR" } # ── Happy path ─────────────────────────────────────────────────── @test "happy path: stage → clone → config → start → health → no rollback, success" { STUB_HEALTH_EXIT=0 export STUB_HEALTH_EXIT run "${ROOT}/lxc-deploy.sh" [ "$status" -eq 0 ] grep -q '^VMID=200$' <<< "$output" grep -q '^stage-snippet:' "$LOG" grep -q '^lxc-clone:200' "$LOG" grep -q '^lxc-config:200' "$LOG" grep -q '^lxc-start:200' "$LOG" grep -q '^health-check:200' "$LOG" # Rollback MUST NOT fire on success. ! grep -q '^rollback:' "$LOG" grep -q 'deploy: praxis deployed successfully to VMID 200' <<< "$output" } # ── Rollback on failure (trap fix: $? capture) ────────────────── @test "health-check fails (set -e) → rollback fires (trap fix: $? capture) → CT destroyed" { # THE TRAP FIX: a `set -e` child failure (health-check exits 1) # must trigger rollback. The trap captures $? so rc != 0 fires # rollback (not just INT/TERM). STUB_HEALTH_EXIT=1 export STUB_HEALTH_EXIT run "${ROOT}/lxc-deploy.sh" [ "$status" -ne 0 ] grep -q '^health-check:200' "$LOG" grep -q '^rollback:200' "$LOG" grep -q 'deploy: FAILED' <<< "$output" } @test "clone fails (set -e) → rollback fires (trap fix) → CT destroyed" { # Same trap fix, earlier failure: clone failure also fires rollback. STUB_CLONE_EXIT=1 export STUB_CLONE_EXIT run "${ROOT}/lxc-deploy.sh" [ "$status" -ne 0 ] grep -q '^lxc-clone:200' "$LOG" grep -q '^rollback:200' "$LOG" # config/start/health NOT reached. ! grep -q '^lxc-config:' "$LOG" ! grep -q '^health-check:' "$LOG" } @test "config fails (set -e) → rollback fires, start/health NOT reached" { STUB_CONFIG_EXIT=1 export STUB_CONFIG_EXIT run "${ROOT}/lxc-deploy.sh" [ "$status" -ne 0 ] grep -q '^lxc-config:200' "$LOG" grep -q '^rollback:200' "$LOG" ! grep -q '^lxc-start:' "$LOG" ! grep -q '^health-check:' "$LOG" } @test "start fails (set -e) → rollback fires, health NOT reached" { STUB_START_EXIT=1 export STUB_START_EXIT run "${ROOT}/lxc-deploy.sh" [ "$status" -ne 0 ] grep -q '^lxc-start:200' "$LOG" grep -q '^rollback:200' "$LOG" ! grep -q '^health-check:' "$LOG" } @test "stage-snippet fails (set -e) → exit non-zero, clone NOT reached (trap not yet installed)" { # NOTE: stage-snippet runs at step 0 (line 65), BEFORE the vmid is # resolved (line 69) + BEFORE the EXIT trap is installed (line 88). # So a stage-snippet failure exits at line 65 without firing # rollback (the trap isn't registered yet). This is a known # ordering: the snippet is staged before any CT is created, so # there's nothing to roll back. STUB_SNIPPET_EXIT=1 export STUB_SNIPPET_EXIT run "${ROOT}/lxc-deploy.sh" [ "$status" -ne 0 ] grep -q '^stage-snippet:' "$LOG" ! grep -q '^lxc-clone:' "$LOG" # No rollback: the trap isn't installed yet at this failure point. ! grep -q '^rollback:' "$LOG" } # ── VMID auto-allocation (D-027) ──────────────────────────────── @test "PROXMOX_LXC_VMID unset → auto-allocate via pve_nextid (STUB_NEXTID)" { STUB_HEALTH_EXIT=0 STUB_NEXTID=250 export STUB_HEALTH_EXIT STUB_NEXTID run env -u PROXMOX_LXC_VMID "${ROOT}/lxc-deploy.sh" [ "$status" -eq 0 ] grep -q 'deploy: auto-allocated VMID 250' <<< "$output" grep -q '^VMID=250$' <<< "$output" grep -q '^lxc-clone:250' "$LOG" } @test "PROXMOX_LXC_VMID set → use the configured VMID (no auto-allocate)" { STUB_HEALTH_EXIT=0 export STUB_HEALTH_EXIT PROXMOX_LXC_VMID=300 run "${ROOT}/lxc-deploy.sh" [ "$status" -eq 0 ] grep -q 'deploy: using configured VMID 300' <<< "$output" grep -q '^VMID=300$' <<< "$output" grep -q '^lxc-clone:300' "$LOG" } # ── Idempotency (D-027) ───────────────────────────────────────── @test "VMID not exists → clone proceeds (current path)" { STUB_HEALTH_EXIT=0 export STUB_HEALTH_EXIT # STUB_PVE_GET unset → empty → ct_exists false. run "${ROOT}/lxc-deploy.sh" [ "$status" -eq 0 ] grep -q '^VMID=200$' <<< "$output" grep -q '^lxc-clone:200' "$LOG" grep -q '^lxc-config:200' "$LOG" grep -q '^lxc-start:200' "$LOG" grep -q '^health-check:200' "$LOG" ! grep -q '^rollback:' "$LOG" } @test "VMID exists + running + healthy → skip clone/config/start (idempotent re-deploy)" { STUB_PVE_GET='{"status":"running","vmid":200}' STUB_HEALTH_EXIT=0 export STUB_PVE_GET STUB_HEALTH_EXIT run "${ROOT}/lxc-deploy.sh" [ "$status" -eq 0 ] grep -q 'already running + healthy — skipping clone/config/start (idempotent re-deploy)' <<< "$output" ! grep -q '^lxc-clone:' "$LOG" ! grep -q '^lxc-config:' "$LOG" ! grep -q '^lxc-start:' "$LOG" grep -q '^health-check:200' "$LOG" ! grep -q '^rollback:' "$LOG" grep -q '^VMID=200$' <<< "$output" } @test "VMID exists + unhealthy, no flag → exit 1 with guidance (--recreate / --reconfigure)" { STUB_PVE_GET='{"status":"running","vmid":200}' STUB_HEALTH_EXIT=1 export STUB_PVE_GET STUB_HEALTH_EXIT run "${ROOT}/lxc-deploy.sh" [ "$status" -eq 1 ] grep -q 'exists but is unhealthy' <<< "$output" grep -q -- '--recreate' <<< "$output" grep -q -- '--reconfigure' <<< "$output" grep -q 'No action taken' <<< "$output" ! grep -q '^lxc-clone:' "$LOG" ! grep -q '^rollback:' "$LOG" } @test "VMID exists but not running, no flag → exit 1 with guidance (not running counts as unhealthy)" { STUB_PVE_GET='{"status":"stopped","vmid":200}' STUB_HEALTH_EXIT=0 export STUB_PVE_GET STUB_HEALTH_EXIT run "${ROOT}/lxc-deploy.sh" [ "$status" -eq 1 ] grep -q 'exists but is unhealthy' <<< "$output" grep -q -- '--recreate' <<< "$output" ! grep -q '^lxc-clone:' "$LOG" ! grep -q '^rollback:' "$LOG" } @test "--recreate → rollback.sh called + redeploy proceeds (clone runs after destroy)" { STUB_PVE_GET='{"status":"running","vmid":200}' STUB_HEALTH_FIRST_FAIL=1 STUB_HEALTH_EXIT=0 export STUB_PVE_GET STUB_HEALTH_FIRST_FAIL STUB_HEALTH_EXIT run "${ROOT}/lxc-deploy.sh" --recreate [ "$status" -eq 0 ] grep -q -- '--recreate: rollback + redeploy' <<< "$output" grep -q '^rollback:200' "$LOG" grep -q '^lxc-clone:200' "$LOG" grep -q '^lxc-config:200' "$LOG" grep -q '^lxc-start:200' "$LOG" grep -q '^health-check:200' "$LOG" grep -q '^VMID=200$' <<< "$output" } @test "--reconfigure → lxc-config.sh re-PUT + lxc-start.sh restart (no clone)" { STUB_PVE_GET='{"status":"running","vmid":200}' STUB_HEALTH_FIRST_FAIL=1 STUB_HEALTH_EXIT=0 export STUB_PVE_GET STUB_HEALTH_FIRST_FAIL STUB_HEALTH_EXIT run "${ROOT}/lxc-deploy.sh" --reconfigure [ "$status" -eq 0 ] grep -q -- '--reconfigure: re-PUT config + restart' <<< "$output" grep -q '^lxc-config:200' "$LOG" grep -q '^lxc-start:200' "$LOG" ! grep -q '^lxc-clone:' "$LOG" ! grep -q '^rollback:' "$LOG" grep -q '^VMID=200$' <<< "$output" } # ── Flag parsing ──────────────────────────────────────────────── @test "unknown flag → exit 2 with error" { STUB_PVE_GET='{"status":"running","vmid":200}' STUB_HEALTH_EXIT=0 export STUB_PVE_GET STUB_HEALTH_EXIT run "${ROOT}/lxc-deploy.sh" --bogus [ "$status" -eq 2 ] grep -q 'unknown argument: --bogus' <<< "$output" } # ── Snippet-exists short-circuit ──────────────────────────────── @test "hookscript snippet already staged → stage-snippet.sh NOT re-run (idempotent)" { # The snippet-exists check calls pve_get /storage/.../content + jq. # Return a content array containing the praxis-firstboot.sh volid → # stage-snippet is skipped. The ct_exists check queries a DIFFERENT # path (/status/current), so we install a path-aware pve_get stub # that returns the content array for /storage/.../content and empty # for /status/current (CT not exists → clone proceeds). cat > "${ROOT}/api.sh" <<'ASTUB' pve_env() { :; } pve_nextid() { printf '%s\n' "${STUB_NEXTID:-200}"; } pve_get() { case "$1" in */storage/*/content) printf '%s\n' '[{"volid":"local:snippets/praxis-firstboot.sh"}]' ;; */lxc/*/status/current) printf '%s\n' '' ;; *) printf '%s\n' "${STUB_PVE_GET:-}" ;; esac } pve_curl() { :; } pve_poll() { :; } pve_tls_insecure() { :; } pve_auth_header() { :; } ASTUB chmod +x "${ROOT}/api.sh" STUB_HEALTH_EXIT=0 export STUB_HEALTH_EXIT run "${ROOT}/lxc-deploy.sh" [ "$status" -eq 0 ] grep -q 'hookscript snippet local:snippets/praxis-firstboot.sh already staged — skipping upload' <<< "$output" ! grep -q '^stage-snippet:' "$LOG" # clone/config/start/health still run (CT not exists). grep -q '^lxc-clone:200' "$LOG" grep -q '^health-check:200' "$LOG" ! grep -q '^rollback:' "$LOG" }