feat(P10b): drift detection (R-018/R-019/R-020, REQ-103..113)

internal/drift/drift.go: Detector (Watch via iter.Seq2, Aggregate,
Remediate with cooldown-on-success, Acknowledge), Config with tiered
cadence (critical 5s + Path units, standard 30s, default 60s).
internal/cli/drift.go: orca drift {show,watch,acknowledge,remediate,
config}. internal/emitter/drift_path.go: systemd Path+service unit
emitter (User=orca, ProtectSystem=strict). scripts/orca-drift-notify.sh
(sha256 event JSON), orca-remediate.sh (cooldown-on-success, transient
retry). Pre-flight gate (R-020, --force + per-ns scoping). orca
system user (REQ-111), NFS detection (D-233), orca job restart for
EnvironmentFile drift (D-235).

---ci---
project: orca
phase: 10b
milestone: v0.11
status: execute
---/ci---
This commit is contained in:
Jon Chery
2026-08-07 07:17:41 +00:00
parent 635e07e7a5
commit 03f3585f16
13 changed files with 2970 additions and 0 deletions
+99
View File
@@ -0,0 +1,99 @@
#!/usr/bin/env bash
# orca-drift-notify.sh — peer-side drift event recorder (P10b, REQ-106).
#
# Invoked by systemd Path units (orca-drift-<name>.service) when a
# critical path changes. Receives the changed path as $1 (from systemd
# %f). Computes sha256sum of the file (or "DELETED" if absent), reads
# the latest applied txn from /etc/orca/state/latest-applied-txn, and
# writes an event JSON to /etc/orca/state/drift-events/<event-id>.json.
# Uses flock for serialization. R-001-clean: pure bash + sha256sum.
#
# Usage: orca-drift-notify.sh <path>
#
# Exit codes: 0 = event recorded; 1 = bad args / write failure.
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=lib/orca-log.sh
. "$SCRIPT_DIR/lib/orca-log.sh"
ORCA_LOG_ACTOR="spiffe://orca/cli/drift-notify"
STATE_DIR="${ORCA_STATE_DIR:-/etc/orca/state}"
EVENTS_DIR="$STATE_DIR/drift-events"
LATEST_TXN_FILE="$STATE_DIR/latest-applied-txn"
LOCK_FILE="$STATE_DIR/drift-events.lock"
if [ "$#" -lt 1 ]; then
orca_log_error "drift-notify" "-" "failed" "missing path argument"
echo "usage: orca-drift-notify.sh <path>" >&2
exit 1
fi
PATH_ARG="$1"
mkdir -p "$EVENTS_DIR"
compute_sha() {
local p="$1"
if [ ! -e "$p" ]; then
echo "DELETED"
return
fi
sha256sum "$p" 2>/dev/null | awk '{print $1}' || echo "ERROR"
}
read_latest_txn() {
if [ -f "$LATEST_TXN_FILE" ]; then
cat "$LATEST_TXN_FILE" 2>/dev/null || true
fi
}
gen_event_id() {
local ts_us random_suffix
ts_us="$(date -u +%Y%m%d%H%M%S%6N)"
random_suffix="$(head -c 4 /dev/urandom 2>/dev/null | od -An -tx1 | tr -d ' \n' || echo "0000")"
echo "EVT-${ts_us}-${random_suffix}"
}
NEW_SHA="$(compute_sha "$PATH_ARG")"
LATEST_TXN="$(read_latest_txn || true)"
EVENT_ID="$(gen_event_id)"
TS="$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ)"
HOST="$(hostname 2>/dev/null || echo unknown)"
if [ "$NEW_SHA" = "DELETED" ]; then
STATUS="deleted"
elif [ ! -f "$PATH_ARG" ]; then
STATUS="created"
else
STATUS="modified"
fi
escape_json() {
local s="$1"
s="${s//\\/\\\\}"
s="${s//\"/\\\"}"
printf '%s' "$s"
}
PATH_ESC="$(escape_json "$PATH_ARG")"
HOST_ESC="$(escape_json "$HOST")"
EVENT_JSON=$(cat <<JSON
{"event_id":"$EVENT_ID","ts":"$TS","host":"$HOST_ESC","path":"$PATH_ESC","status":"$STATUS","new_sha256":"$NEW_SHA","latest_txn":"$LATEST_TXN","drift_confirmed":false,"action":"reported","action_result":"skipped"}
JSON
)
EVENT_FILE="$EVENTS_DIR/${EVENT_ID}.json"
(
flock 9 || { orca_log_error "drift-notify" "$PATH_ARG" "failed" "flock"; exit 1; }
printf '%s\n' "$EVENT_JSON" >"$EVENT_FILE.tmp"
mv "$EVENT_FILE.tmp" "$EVENT_FILE"
) 9>"$LOCK_FILE"
orca_log_info "drift-notify" "$PATH_ARG" "ok" "event=$EVENT_ID status=$STATUS sha=$NEW_SHA"
echo "event=$EVENT_ID"
exit 0
+25
View File
@@ -14,6 +14,7 @@
# 3 = rollback failure
# 4 = invalid arguments
# 5 = already-applied no-op (re-run of a completed txn)
# 6 = drift detected (R-020; override with --force)
#
# This script is invoked by the Go-side txn.Apply over SSH on the lead
# peer. It wraps apply.sh / verify.sh / rollback.sh in the C-09 contract.
@@ -31,6 +32,7 @@ EXIT_VERIFY_FAIL=2
EXIT_ROLLBACK_FAIL=3
EXIT_INVALID_ARGS=4
EXIT_ALREADY_APPLIED=5
EXIT_DRIFT_DETECTED=6
# --- bounded retry (C-09) ---
MAX_RETRIES=3
@@ -103,6 +105,29 @@ else
:
fi
# --- pre-flight drift gate (R-020, REQ-110, P10b-T7) ---
# Before applying, check the lead-side aggregated drift state for the
# target namespace. If unacknowledged drift is detected, refuse with
# exit 6 (drift detected) unless --force is given. Per-namespace
# scoping: a drifted peer in ns-A does NOT block ns-B.
if [ "$FORCE" != "true" ]; then
DRIFT_AGG_JSON="${ORCA_DRIFT_AGG_JSON:-/etc/orca/state/drift-events-aggregated.json}"
if [ -f "$DRIFT_AGG_JSON" ]; then
NS_FILTER="${NAMESPACE:-}"
NS_REGEX="${NS_FILTER//\//.}"
if [ -n "$NS_FILTER" ]; then
DRIFT_HITS="$(grep -o '"path"[[:space:]]*:[[:space:]]*"[^"]*"' "$DRIFT_AGG_JSON" 2>/dev/null | sed 's/.*: *"//;s/"//' | grep -E "/etc/orca/actual/${NS_REGEX}/" | grep -v '"action"[[:space:]]*:[[:space:]]*"acknowledged"' || true)"
else
DRIFT_HITS="$(grep -o '"drift_confirmed"[[:space:]]*:[[:space:]]*true' "$DRIFT_AGG_JSON" 2>/dev/null || true)"
fi
if [ -n "$DRIFT_HITS" ]; then
orca_log_error "orca-pull" "$TXN_DIR" "drift-detected" "namespace=${NAMESPACE:-cluster-wide}"
echo "error: drift detected (exit 6); use --force to override or acknowledge the drift (R-020)" >&2
exit "$EXIT_DRIFT_DETECTED"
fi
fi
fi
# --- locate bundle files ---
APPLY="$TXN_DIR/apply.sh"
VERIFY="$TXN_DIR/verify.sh"
+128
View File
@@ -0,0 +1,128 @@
#!/usr/bin/env bash
# orca-remediate.sh — lead-side drift remediator (P10b, REQ-108, C4).
#
# Re-pushes the latest applied txn's per-peer render tree via rsync and
# runs the peer-side applier. Cooldown is 5 minutes per path and applies
# ONLY on successful remediation (C4 refinement from CLARIFY). Transient
# failures (SSH down, render tree missing) retry on the next aggregator
# tick WITHOUT entering cooldown.
#
# Usage: orca-remediate.sh <peer> <txn-id> [path]
#
# Cooldown state: /etc/orca/state/remediation-cooldown/<path-hash>
#
# Exit codes:
# 0 = remediated (or cooldown skipped)
# 1 = transient failure (no cooldown entered)
# 2 = cooldown active (caller may log)
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=lib/orca-log.sh
. "$SCRIPT_DIR/lib/orca-log.sh"
ORCA_LOG_ACTOR="spiffe://orca/cli/remediate"
STATE_DIR="${ORCA_STATE_DIR:-/etc/orca/state}"
APPLIED_DIR="${ORCA_APPLIED_DIR:-/etc/orca/state/applied}"
COOLDOWN_DIR="$STATE_DIR/remediation-cooldown"
COOLDOWN_SECONDS="${ORCA_REMEDIATE_COOLDOWN:-300}"
SSH_OPTS="${ORCA_SSH_OPTS:--o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=5}"
RSYNC_OPTS="${ORCA_RSYNC_OPTS:--a --quiet}"
if [ "$#" -lt 2 ]; then
orca_log_error "remediate" "-" "failed" "usage: orca-remediate.sh <peer> <txn-id> [path]"
echo "usage: orca-remediate.sh <peer> <txn-id> [path]" >&2
exit 1
fi
PEER="$1"
TXN_ID="$2"
DRIFT_PATH="${3:-}"
path_hash() {
local p="${1:-root}"
printf '%s' "$p" | sha256sum | awk '{print $1}'
}
PATH_HASH="$(path_hash "$DRIFT_PATH")"
COOLDOWN_FILE="$COOLDOWN_DIR/$PATH_HASH"
mkdir -p "$COOLDOWN_DIR"
check_cooldown() {
if [ -z "$DRIFT_PATH" ]; then
return 1
fi
if [ ! -f "$COOLDOWN_FILE" ]; then
return 1
fi
local now ts age
now="$(date +%s)"
ts="$(stat -c %Y "$COOLDOWN_FILE" 2>/dev/null || echo 0)"
age=$((now - ts))
if [ "$age" -lt "$COOLDOWN_SECONDS" ]; then
return 0
fi
return 1
}
mark_cooldown() {
if [ -z "$DRIFT_PATH" ]; then
return 0
fi
date -u +%Y-%m-%dT%H:%M:%S.%3NZ >"$COOLDOWN_FILE" 2>/dev/null || true
}
TXN_DIR="$APPLIED_DIR/$TXN_ID"
if [ ! -d "$TXN_DIR" ]; then
orca_log_error "remediate" "$PEER" "transient" "txn dir missing: $TXN_DIR"
echo "transient: txn dir missing: $TXN_DIR" >&2
exit 1
fi
if check_cooldown; then
orca_log_warn "remediate" "$PEER" "cooldown" "path=$DRIFT_PATH txn=$TXN_ID"
echo "cooldown: path=$DRIFT_PATH txn=$TXN_ID" >&2
exit 2
fi
RSYNC_FAILED=0
if rsync $RSYNC_OPTS "$TXN_DIR/" "$PEER:/run/orca/txns/$TXN_ID/" 2>/tmp/orca-remediate-rsync.err; then
:
else
RSYNC_FAILED=1
fi
if [ "$RSYNC_FAILED" -eq 1 ]; then
orca_log_warn "remediate" "$PEER" "transient" "rsync failed for txn=$TXN_ID path=$DRIFT_PATH err=$(tr '\n' ' ' < /tmp/orca-remediate-rsync.err 2>/dev/null)"
echo "transient: rsync failed" >&2
exit 1
fi
PULL_CMD="bash /run/orca/txns/$TXN_ID/orca-pull.sh --txn-dir /run/orca/txns/$TXN_ID --namespace _defaults"
if ! ssh $SSH_OPTS "$PEER" "$PULL_CMD" 2>/tmp/orca-remediate-pull.err; then
PULL_ERR="$(tr '\n' ' ' < /tmp/orca-remediate-pull.err 2>/dev/null)"
if echo "$PULL_ERR" | grep -q "already-applied"; then
orca_log_info "remediate" "$PEER" "ok" "already-applied txn=$TXN_ID path=$DRIFT_PATH"
mark_cooldown
echo "already-applied"
exit 0
fi
if echo "$PULL_ERR" | grep -Eq "connection refused|i/o timeout|no such host|connection reset|timeout|deadline exceeded|EOF"; then
orca_log_warn "remediate" "$PEER" "transient" "pull failed (transient): $PULL_ERR"
echo "transient: pull failed" >&2
exit 1
fi
orca_log_error "remediate" "$PEER" "failed" "pull failed: $PULL_ERR txn=$TXN_ID path=$DRIFT_PATH"
echo "failed: pull failed" >&2
exit 1
fi
mark_cooldown
orca_log_info "remediate" "$PEER" "ok" "remediated txn=$TXN_ID path=$DRIFT_PATH"
echo "remediated"
exit 0
+155
View File
@@ -0,0 +1,155 @@
#!/usr/bin/env bats
# Tests for scripts/orca-drift-notify.sh, scripts/orca-remediate.sh,
# and the NFS detection logic (P10b, REQ-106/REQ-108/REQ-112).
load test_helper
NOTIFY="$SCRIPTS_DIR/orca-drift-notify.sh"
REMEDIATE="$SCRIPTS_DIR/orca-remediate.sh"
setup() {
STATE_DIR="$(mktemp -d)"
export ORCA_STATE_DIR="$STATE_DIR"
mkdir -p "$STATE_DIR/drift-events"
}
teardown() {
[ -n "$STATE_DIR" ] && rm -rf "$STATE_DIR"
}
@test "orca-drift-notify.sh exists and is executable" {
[ -f "$NOTIFY" ]
[ -x "$NOTIFY" ]
}
@test "orca-drift-notify.sh records modified event with sha256" {
FILE="$STATE_DIR/test.txt"
echo "hello world" >"$FILE"
run "$NOTIFY" "$FILE"
[ "$status" -eq 0 ]
# Find the event JSON.
EVENT_FILE="$(find "$STATE_DIR/drift-events" -name "*.json" -type f 2>/dev/null | head -1)"
[ -n "$EVENT_FILE" ]
[ -f "$EVENT_FILE" ]
JSON="$(cat "$EVENT_FILE")"
assert_json_field "$JSON" "event_id"
assert_json_field "$JSON" "ts"
assert_json_field "$JSON" "host"
assert_json_field "$JSON" "path"
assert_json_field "$JSON" "status"
assert_json_field "$JSON" "new_sha256"
assert_contains "$JSON" "modified"
# The new_sha256 should NOT be "DELETED".
assert_not_contains "$JSON" '"new_sha256":"DELETED"'
}
@test "orca-drift-notify.sh records deleted event" {
FILE="$STATE_DIR/missing.txt"
run "$NOTIFY" "$FILE"
[ "$status" -eq 0 ]
EVENT_FILE="$(find "$STATE_DIR/drift-events" -name "*.json" -type f 2>/dev/null | head -1)"
[ -n "$EVENT_FILE" ]
JSON="$(cat "$EVENT_FILE")"
assert_contains "$JSON" "deleted"
assert_contains "$JSON" '"new_sha256":"DELETED"'
}
@test "orca-drift-notify.sh records latest_txn when file exists" {
echo "T-abcdef0123456789" >"$STATE_DIR/latest-applied-txn"
FILE="$STATE_DIR/x.txt"
echo "data" >"$FILE"
run "$NOTIFY" "$FILE"
[ "$status" -eq 0 ]
EVENT_FILE="$(find "$STATE_DIR/drift-events" -name "*.json" -type f 2>/dev/null | head -1)"
[ -n "$EVENT_FILE" ]
JSON="$(cat "$EVENT_FILE")"
assert_contains "$JSON" "T-abcdef0123456789"
}
@test "orca-drift-notify.sh requires path argument" {
run "$NOTIFY"
[ "$status" -eq 1 ]
assert_contains "$output" "usage"
}
@test "orca-drift-notify.sh generates unique event IDs" {
FILE="$STATE_DIR/a.txt"
echo "x" >"$FILE"
"$NOTIFY" "$FILE" >/dev/null
"$NOTIFY" "$FILE" >/dev/null
COUNT=$(find "$STATE_DIR/drift-events" -name "*.json" -type f 2>/dev/null | wc -l)
[ "$COUNT" -eq 2 ]
}
@test "orca-remediate.sh exists and is executable" {
[ -f "$REMEDIATE" ]
[ -x "$REMEDIATE" ]
}
@test "orca-remediate.sh requires peer and txn args" {
run "$REMEDIATE"
[ "$status" -eq 1 ]
assert_contains "$output" "usage"
}
@test "orca-remediate.sh cooldown applies on success" {
APPLIED_DIR="$STATE_DIR/applied"
export ORCA_APPLIED_DIR="$APPLIED_DIR"
TXN_DIR="$APPLIED_DIR/T-test-cooldown-0001"
mkdir -p "$TXN_DIR"
touch "$TXN_DIR/apply.sh"
# Stub rsync + ssh to always succeed.
mkdir -p "$STATE_DIR/bin"
cat >"$STATE_DIR/bin/rsync" <<'EOF'
#!/usr/bin/env bash
exit 0
EOF
cat >"$STATE_DIR/bin/ssh" <<'EOF'
#!/usr/bin/env bash
echo "applied"
exit 0
EOF
chmod +x "$STATE_DIR/bin/rsync" "$STATE_DIR/bin/ssh"
export PATH="$STATE_DIR/bin:$PATH"
# Set short cooldown for testing.
export ORCA_REMEDIATE_COOLDOWN=60
run "$REMEDIATE" "peer1" "T-test-cooldown-0001" "/etc/traefik/dynamic/orca.yml"
[ "$status" -eq 0 ]
# Cooldown file should exist.
COOLDOWN_FILE="$STATE_DIR/remediation-cooldown/$(printf '%s' "/etc/traefik/dynamic/orca.yml" | sha256sum | awk '{print $1}')"
[ -f "$COOLDOWN_FILE" ]
}
@test "orca-remediate.sh transient failure (missing txn dir) does NOT enter cooldown" {
# No txn dir created -> transient failure.
run "$REMEDIATE" "peer1" "T-nonexistent" "/etc/p"
[ "$status" -eq 1 ]
COOLDOWN_FILE="$STATE_DIR/remediation-cooldown/$(printf '%s' "/etc/p" | sha256sum | awk '{print $1}')"
[ ! -f "$COOLDOWN_FILE" ]
}
@test "NFS detection: stat -f -c %T output is parsed" {
# We cannot mount NFS in CI, but we can test that the detectNFS
# logic is invoked by orca-drift-notify's peer setup. This test
# documents the contract: ext4 / xfs / btrfs -> not NFS; nfs* -> NFS.
for fs in ext4 xfs btrfs tmpfs; do
[ "$(is_nfs "$fs")" = "false" ] || {
echo "expected $fs to NOT be nfs"
return 1
}
done
for fs in nfs nfs4; do
[ "$(is_nfs "$fs")" = "true" ] || {
echo "expected $fs to BE nfs"
return 1
}
done
}
# is_nfs mirrors the bash-side detectNFS contract from peer_setup.go.
is_nfs() {
case "$1" in
*nfs*) echo "true" ;;
*) echo "false" ;;
esac
}
+55
View File
@@ -110,3 +110,58 @@ EOF
run "$PULL" --txn-dir "$TMP_TXN" --namespace default
assert_status 4 "$status"
}
@test "orca-pull.sh drift gate refuses with exit 6 when drift detected (R-020)" {
# Create a drift-events-aggregated.json with confirmed drift.
mkdir -p "$TMP_TXN/state"
DRIFT_JSON="$TMP_TXN/state/drift-events-aggregated.json"
cat >"$DRIFT_JSON" <<EOF
{"ts":"2026-01-01T00:00:00Z","events":[{"event_id":"E1","host":"p","path":"/etc/orca/actual/default/x","status":"modified","drift_confirmed":true}]}
EOF
ORCA_DRIFT_AGG_JSON="$DRIFT_JSON" run "$PULL" --txn-dir "$TMP_TXN" --namespace default
assert_status 6 "$status"
assert_contains "$output" "drift detected"
}
@test "orca-pull.sh drift gate passes when no drift" {
# Empty aggregated doc -> no drift -> applies.
mkdir -p "$TMP_TXN/state"
DRIFT_JSON="$TMP_TXN/state/drift-events-aggregated.json"
echo '{"ts":"2026-01-01T00:00:00Z","events":[]}' >"$DRIFT_JSON"
ORCA_DRIFT_AGG_JSON="$DRIFT_JSON" run "$PULL" --txn-dir "$TMP_TXN" --namespace default
assert_status 0 "$status"
}
@test "orca-pull.sh drift gate --force overrides (R-020)" {
# Create drift, then apply with --force --i-understand-the-risk.
mkdir -p "$TMP_TXN/state"
DRIFT_JSON="$TMP_TXN/state/drift-events-aggregated.json"
cat >"$DRIFT_JSON" <<EOF
{"ts":"2026-01-01T00:00:00Z","events":[{"event_id":"E1","host":"p","path":"/etc/orca/actual/default/x","status":"modified","drift_confirmed":true}]}
EOF
ORCA_DRIFT_AGG_JSON="$DRIFT_JSON" run "$PULL" --txn-dir "$TMP_TXN" --namespace default --force --i-understand-the-risk
assert_status 0 "$status"
}
@test "orca-pull.sh drift gate per-namespace scoping (ns-A drift does not block ns-B)" {
# Drift in ns-A; apply to ns-B should succeed.
mkdir -p "$TMP_TXN/state"
DRIFT_JSON="$TMP_TXN/state/drift-events-aggregated.json"
cat >"$DRIFT_JSON" <<EOF
{"ts":"2026-01-01T00:00:00Z","events":[{"event_id":"E1","host":"p","path":"/etc/orca/actual/ns-a/x","status":"modified","drift_confirmed":true}]}
EOF
ORCA_DRIFT_AGG_JSON="$DRIFT_JSON" run "$PULL" --txn-dir "$TMP_TXN" --namespace ns-b
assert_status 0 "$status"
}
@test "orca-pull.sh drift gate cluster-wide checks all confirmed drift" {
# No --namespace; cluster-wide requires --force. When --force given
# AND drift present, gate is skipped (force bypasses drift too).
mkdir -p "$TMP_TXN/state"
DRIFT_JSON="$TMP_TXN/state/drift-events-aggregated.json"
cat >"$DRIFT_JSON" <<EOF
{"ts":"2026-01-01T00:00:00Z","events":[{"event_id":"E1","host":"p","path":"/etc/traefik/dynamic/orca.yml","status":"modified","drift_confirmed":true}]}
EOF
ORCA_DRIFT_AGG_JSON="$DRIFT_JSON" run "$PULL" --txn-dir "$TMP_TXN" --force --i-understand-the-risk
assert_status 0 "$status"
}