Jon Chery
03f3585f16
internal/drift/drift.go: Detector (Watch via iter.Seq2, Aggregate,
Remediate with cooldown-on-success, Acknowledge), Config with tiered
cadence (critical 5s + Path units, standard 30s, default 60s).
internal/cli/drift.go: orca drift {show,watch,acknowledge,remediate,
config}. internal/emitter/drift_path.go: systemd Path+service unit
emitter (User=orca, ProtectSystem=strict). scripts/orca-drift-notify.sh
(sha256 event JSON), orca-remediate.sh (cooldown-on-success, transient
retry). Pre-flight gate (R-020, --force + per-ns scoping). orca
system user (REQ-111), NFS detection (D-233), orca job restart for
EnvironmentFile drift (D-235).
---ci---
project: orca
phase: 10b
milestone: v0.11
status: execute
---/ci---
129 lines
3.7 KiB
Bash
Executable File
129 lines
3.7 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# orca-remediate.sh — lead-side drift remediator (P10b, REQ-108, C4).
|
|
#
|
|
# Re-pushes the latest applied txn's per-peer render tree via rsync and
|
|
# runs the peer-side applier. Cooldown is 5 minutes per path and applies
|
|
# ONLY on successful remediation (C4 refinement from CLARIFY). Transient
|
|
# failures (SSH down, render tree missing) retry on the next aggregator
|
|
# tick WITHOUT entering cooldown.
|
|
#
|
|
# Usage: orca-remediate.sh <peer> <txn-id> [path]
|
|
#
|
|
# Cooldown state: /etc/orca/state/remediation-cooldown/<path-hash>
|
|
#
|
|
# Exit codes:
|
|
# 0 = remediated (or cooldown skipped)
|
|
# 1 = transient failure (no cooldown entered)
|
|
# 2 = cooldown active (caller may log)
|
|
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
# shellcheck source=lib/orca-log.sh
|
|
. "$SCRIPT_DIR/lib/orca-log.sh"
|
|
|
|
ORCA_LOG_ACTOR="spiffe://orca/cli/remediate"
|
|
|
|
STATE_DIR="${ORCA_STATE_DIR:-/etc/orca/state}"
|
|
APPLIED_DIR="${ORCA_APPLIED_DIR:-/etc/orca/state/applied}"
|
|
COOLDOWN_DIR="$STATE_DIR/remediation-cooldown"
|
|
COOLDOWN_SECONDS="${ORCA_REMEDIATE_COOLDOWN:-300}"
|
|
|
|
SSH_OPTS="${ORCA_SSH_OPTS:--o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=5}"
|
|
RSYNC_OPTS="${ORCA_RSYNC_OPTS:--a --quiet}"
|
|
|
|
if [ "$#" -lt 2 ]; then
|
|
orca_log_error "remediate" "-" "failed" "usage: orca-remediate.sh <peer> <txn-id> [path]"
|
|
echo "usage: orca-remediate.sh <peer> <txn-id> [path]" >&2
|
|
exit 1
|
|
fi
|
|
|
|
PEER="$1"
|
|
TXN_ID="$2"
|
|
DRIFT_PATH="${3:-}"
|
|
|
|
path_hash() {
|
|
local p="${1:-root}"
|
|
printf '%s' "$p" | sha256sum | awk '{print $1}'
|
|
}
|
|
|
|
PATH_HASH="$(path_hash "$DRIFT_PATH")"
|
|
COOLDOWN_FILE="$COOLDOWN_DIR/$PATH_HASH"
|
|
|
|
mkdir -p "$COOLDOWN_DIR"
|
|
|
|
check_cooldown() {
|
|
if [ -z "$DRIFT_PATH" ]; then
|
|
return 1
|
|
fi
|
|
if [ ! -f "$COOLDOWN_FILE" ]; then
|
|
return 1
|
|
fi
|
|
local now ts age
|
|
now="$(date +%s)"
|
|
ts="$(stat -c %Y "$COOLDOWN_FILE" 2>/dev/null || echo 0)"
|
|
age=$((now - ts))
|
|
if [ "$age" -lt "$COOLDOWN_SECONDS" ]; then
|
|
return 0
|
|
fi
|
|
return 1
|
|
}
|
|
|
|
mark_cooldown() {
|
|
if [ -z "$DRIFT_PATH" ]; then
|
|
return 0
|
|
fi
|
|
date -u +%Y-%m-%dT%H:%M:%S.%3NZ >"$COOLDOWN_FILE" 2>/dev/null || true
|
|
}
|
|
|
|
TXN_DIR="$APPLIED_DIR/$TXN_ID"
|
|
|
|
if [ ! -d "$TXN_DIR" ]; then
|
|
orca_log_error "remediate" "$PEER" "transient" "txn dir missing: $TXN_DIR"
|
|
echo "transient: txn dir missing: $TXN_DIR" >&2
|
|
exit 1
|
|
fi
|
|
|
|
if check_cooldown; then
|
|
orca_log_warn "remediate" "$PEER" "cooldown" "path=$DRIFT_PATH txn=$TXN_ID"
|
|
echo "cooldown: path=$DRIFT_PATH txn=$TXN_ID" >&2
|
|
exit 2
|
|
fi
|
|
|
|
RSYNC_FAILED=0
|
|
if rsync $RSYNC_OPTS "$TXN_DIR/" "$PEER:/run/orca/txns/$TXN_ID/" 2>/tmp/orca-remediate-rsync.err; then
|
|
:
|
|
else
|
|
RSYNC_FAILED=1
|
|
fi
|
|
|
|
if [ "$RSYNC_FAILED" -eq 1 ]; then
|
|
orca_log_warn "remediate" "$PEER" "transient" "rsync failed for txn=$TXN_ID path=$DRIFT_PATH err=$(tr '\n' ' ' < /tmp/orca-remediate-rsync.err 2>/dev/null)"
|
|
echo "transient: rsync failed" >&2
|
|
exit 1
|
|
fi
|
|
|
|
PULL_CMD="bash /run/orca/txns/$TXN_ID/orca-pull.sh --txn-dir /run/orca/txns/$TXN_ID --namespace _defaults"
|
|
if ! ssh $SSH_OPTS "$PEER" "$PULL_CMD" 2>/tmp/orca-remediate-pull.err; then
|
|
PULL_ERR="$(tr '\n' ' ' < /tmp/orca-remediate-pull.err 2>/dev/null)"
|
|
if echo "$PULL_ERR" | grep -q "already-applied"; then
|
|
orca_log_info "remediate" "$PEER" "ok" "already-applied txn=$TXN_ID path=$DRIFT_PATH"
|
|
mark_cooldown
|
|
echo "already-applied"
|
|
exit 0
|
|
fi
|
|
if echo "$PULL_ERR" | grep -Eq "connection refused|i/o timeout|no such host|connection reset|timeout|deadline exceeded|EOF"; then
|
|
orca_log_warn "remediate" "$PEER" "transient" "pull failed (transient): $PULL_ERR"
|
|
echo "transient: pull failed" >&2
|
|
exit 1
|
|
fi
|
|
orca_log_error "remediate" "$PEER" "failed" "pull failed: $PULL_ERR txn=$TXN_ID path=$DRIFT_PATH"
|
|
echo "failed: pull failed" >&2
|
|
exit 1
|
|
fi
|
|
|
|
mark_cooldown
|
|
orca_log_info "remediate" "$PEER" "ok" "remediated txn=$TXN_ID path=$DRIFT_PATH"
|
|
echo "remediated"
|
|
exit 0
|