orca/scripts/orca-pull.sh

#!/usr/bin/env bash
# orca-pull.sh — lead-side transactional applier (P10a, v0.11).
# Implements the C-09 failure contract (idempotent re-run, bounded retry,
# deterministic state, structured syslog) and the C-23 cluster-wide vs
# namespace-scoped distinction.
#
# Usage:
#   orca-pull.sh --txn-dir <dir> [--namespace <ns> | --force --i-understand-the-risk | --force --yes]
#
# Exit codes:
#   0 = applied (or already-applied no-op)
#   1 = apply failure
#   2 = verify failure
#   3 = rollback failure
#   4 = invalid arguments
#   5 = already-applied no-op (re-run of a completed txn)
#   6 = drift detected (R-020; override with --force)
#
# This script is invoked by the Go-side txn.Apply over SSH on the lead
# peer. It wraps apply.sh / verify.sh / rollback.sh in the C-09 contract.

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=lib/orca-log.sh
. "$SCRIPT_DIR/lib/orca-log.sh"

# --- exit codes (C-09) ---
EXIT_OK=0
EXIT_APPLY_FAIL=1
EXIT_VERIFY_FAIL=2
EXIT_ROLLBACK_FAIL=3
EXIT_INVALID_ARGS=4
EXIT_ALREADY_APPLIED=5
EXIT_DRIFT_DETECTED=6

# --- bounded retry (C-09) ---
MAX_RETRIES=3
BACKOFF_SEQ=(1 2 4)

# --- args ---
TXN_DIR=""
NAMESPACE=""
FORCE=false
ACK_RISK=false
YES=false

usage() {
	cat >&2 <<EOF
usage: orca-pull.sh --txn-dir <dir> [--namespace <ns>]
                    [--force --i-understand-the-risk]
                    [--force --yes]
EOF
}

while [ "$#" -gt 0 ]; do
	case "$1" in
		--txn-dir)
			[ "$#" -ge 2 ] || { orca_log_error "orca-pull" "-" "failed" "missing --txn-dir value"; usage; exit "$EXIT_INVALID_ARGS"; }
			TXN_DIR="$2"; shift 2 ;;
		--namespace)
			[ "$#" -ge 2 ] || { orca_log_error "orca-pull" "-" "failed" "missing --namespace value"; usage; exit "$EXIT_INVALID_ARGS"; }
			NAMESPACE="$2"; shift 2 ;;
		--force)
			FORCE=true; shift ;;
		--i-understand-the-risk)
			ACK_RISK=true; shift ;;
		--yes)
			YES=true; shift ;;
		-h|--help)
			usage; exit "$EXIT_OK" ;;
		*)
			orca_log_error "orca-pull" "-" "failed" "unknown argument: $1"
			usage; exit "$EXIT_INVALID_ARGS" ;;
	esac
done

# --- arg validation ---
if [ -z "$TXN_DIR" ]; then
	orca_log_error "orca-pull" "-" "failed" "missing --txn-dir"
	usage; exit "$EXIT_INVALID_ARGS"
fi
if [ ! -d "$TXN_DIR" ]; then
	orca_log_error "orca-pull" "$TXN_DIR" "failed" "txn dir not found"
	exit "$EXIT_INVALID_ARGS"
fi

# --- C-23 cluster-wide vs namespace-scoped enforcement ---
if [ -z "$NAMESPACE" ]; then
	# Cluster-wide txn: requires --force + (--i-understand-the-risk | --yes).
	if [ "$FORCE" != "true" ]; then
		orca_log_error "orca-pull" "$TXN_DIR" "denied" "cluster-wide txn requires --force"
		echo "error: cluster-wide txn requires --force (C-23)" >&2
		exit "$EXIT_INVALID_ARGS"
	fi
	if [ "$ACK_RISK" != "true" ] && [ "$YES" != "true" ]; then
		orca_log_error "orca-pull" "$TXN_DIR" "denied" "cluster-wide --force requires --i-understand-the-risk (or --yes)"
		echo "error: cluster-wide --force requires --i-understand-the-risk (or --yes) (C-23)" >&2
		exit "$EXIT_INVALID_ARGS"
	fi
else
	# Namespace-scoped: --force not required. Drift in other namespaces
	# does not block this txn (C-23). We still honor --force if given
	# (it's a no-op for ns-scoped).
	:
fi

# --- pre-flight drift gate (R-020, REQ-110, P10b-T7) ---
# Before applying, check the lead-side aggregated drift state for the
# target namespace. If unacknowledged drift is detected, refuse with
# exit 6 (drift detected) unless --force is given. Per-namespace
# scoping: a drifted peer in ns-A does NOT block ns-B.
if [ "$FORCE" != "true" ]; then
	DRIFT_AGG_JSON="${ORCA_DRIFT_AGG_JSON:-/etc/orca/state/drift-events-aggregated.json}"
	if [ -f "$DRIFT_AGG_JSON" ]; then
		NS_FILTER="${NAMESPACE:-}"
		NS_REGEX="${NS_FILTER//\//.}"
		# REQ-131 / F18: use jq for drift-gate JSON parsing (not grep).
		if command -v jq >/dev/null 2>&1; then
			if [ -n "$NS_FILTER" ]; then
				DRIFT_HITS="$(jq -r --arg ns "$NS_FILTER" '[.events[]? | select((.path|test("/etc/orca/actual/\($ns)/")) and (.action != "acknowledged"))] | length' "$DRIFT_AGG_JSON" 2>/dev/null || true)"
			else
				DRIFT_HITS="$(jq -r '[.events[]? | select(.drift_confirmed == true and .action != "acknowledged")] | length' "$DRIFT_AGG_JSON" 2>/dev/null || true)"
			fi
		else
			# Fallback: grep (less accurate; the ack filter may not
			# match the same line as the path filter).
			if [ -n "$NS_FILTER" ]; then
				DRIFT_HITS="$(grep -o '"path"[[:space:]]*:[[:space:]]*"[^"]*"' "$DRIFT_AGG_JSON" 2>/dev/null | sed 's/.*: *"//;s/"//' | grep -E "/etc/orca/actual/${NS_REGEX}/" || true)"
			else
				DRIFT_HITS="$(grep -o '"drift_confirmed"[[:space:]]*:[[:space:]]*true' "$DRIFT_AGG_JSON" 2>/dev/null || true)"
			fi
		fi
		if [ -n "$DRIFT_HITS" ]; then
			orca_log_error "orca-pull" "$TXN_DIR" "drift-detected" "namespace=${NAMESPACE:-cluster-wide}"
			echo "error: drift detected (exit 6); use --force to override or acknowledge the drift (R-020)" >&2
			exit "$EXIT_DRIFT_DETECTED"
		fi
	fi
fi

# --- locate bundle files ---
APPLY="$TXN_DIR/apply.sh"
VERIFY="$TXN_DIR/verify.sh"
ROLLBACK="$TXN_DIR/rollback.sh"
MARKER="$TXN_DIR/.applied"

for f in "$APPLY" "$VERIFY" "$ROLLBACK"; do
	if [ ! -f "$f" ]; then
		orca_log_error "orca-pull" "$TXN_DIR" "failed" "missing bundle file: $f"
		echo "error: missing $f" >&2
		exit "$EXIT_INVALID_ARGS"
	fi
done

# --- idempotency: already-applied is a no-op (C-09) ---
if [ -f "$MARKER" ]; then
	orca_log_info "orca-pull" "$TXN_DIR" "already-applied" ""
	echo "already-applied"
	exit "$EXIT_ALREADY_APPLIED"
fi

# --- apply with bounded retry (C-09: 3 attempts, 1s/2s/4s backoff) ---
start_ns="$(date +%s%N)"

run_with_retry() {
	local script="$1" label="$2"
	local attempt=0
	local rc=0
	while [ "$attempt" -lt "$MAX_RETRIES" ]; do
		attempt=$((attempt + 1))
		set +e
		bash "$script"
		rc=$?
		set -e
		if [ "$rc" -eq 0 ]; then
			return 0
		fi
		if [ "$attempt" -eq "$MAX_RETRIES" ]; then
			break
		fi
		local wait_s="${BACKOFF_SEQ[$((attempt - 1))]}"
		orca_log_warn "orca-pull" "$TXN_DIR" "$label-retry" "attempt $attempt failed (rc=$rc), sleeping ${wait_s}s"
		sleep "$wait_s"
	done
	return "$rc"
}

# Apply phase.
if ! run_with_retry "$APPLY" "apply"; then
	apply_rc=$?
	end_ns="$(date +%s%N)"
	duration_ms=$(( (end_ns - start_ns) / 1000000 ))
	orca_log_error "orca-pull" "$TXN_DIR" "apply-failed" "rc=$apply_rc duration_ms=$duration_ms"
	# Run rollback on apply failure.
	if bash "$ROLLBACK"; then
		:
	else
		orca_log_error "orca-pull" "$TXN_DIR" "rollback-failed" "rollback after apply failure exited non-zero"
		exit "$EXIT_ROLLBACK_FAIL"
	fi
	exit "$EXIT_APPLY_FAIL"
fi

# Verify phase.
if ! run_with_retry "$VERIFY" "verify"; then
	verify_rc=$?
	end_ns="$(date +%s%N)"
	duration_ms=$(( (end_ns - start_ns) / 1000000 ))
	orca_log_error "orca-pull" "$TXN_DIR" "verify-failed" "rc=$verify_rc duration_ms=$duration_ms"
	# Run rollback on verify failure.
	if ! bash "$ROLLBACK"; then
		orca_log_error "orca-pull" "$TXN_DIR" "rollback-failed" "rollback after verify failure exited non-zero"
		exit "$EXIT_ROLLBACK_FAIL"
	fi
	exit "$EXIT_VERIFY_FAIL"
fi

end_ns="$(date +%s%N)"
duration_ms=$(( (end_ns - start_ns) / 1000000 ))
orca_log_info "orca-pull" "$TXN_DIR" "applied" "duration_ms=$duration_ms namespace=${NAMESPACE:-cluster-wide}"
echo "applied (duration=${duration_ms}ms)"
exit "$EXIT_OK"