Jon Chery
5cbe3020d3
scripts/orca-aggregate.sh: 10s aggregator, cluster.json merge + drift-events rsync + remediation trigger (P10b stub). scripts/orca- watchdog.sh: C-11 starvation detection. internal/cli/collector.go: orca collector start/stop/status. Tests: CLI + bats. ---ci--- project: orca phase: 09 milestone: v0.11 status: execute ---/ci---
43 lines
1.4 KiB
Bash
Executable File
43 lines
1.4 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# orca-watchdog.sh — lead-side watchdog meta-timer (P09, C-11).
|
|
#
|
|
# Runs every 30s via orca-watchdog.timer. Checks whether
|
|
# orca-aggregate.sh has run in the last N seconds (default 30s — 3x the
|
|
# 10s cadence). If not: fires a structured alert via
|
|
# `logger -t orca-watchdog -p user.err "aggregator starvation detected"`.
|
|
# The check compares the mtime of /etc/orca/state/cluster.json to the
|
|
# current time.
|
|
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
# shellcheck source=lib/orca-log.sh
|
|
. "$SCRIPT_DIR/lib/orca-log.sh"
|
|
|
|
ORCA_LOG_ACTOR="spiffe://orca/cli/watchdog"
|
|
|
|
STATE_DIR="${ORCA_STATE_DIR:-/etc/orca/state}"
|
|
CLUSTER_JSON="$STATE_DIR/cluster.json"
|
|
STARVATION_THRESHOLD="${ORCA_WATCHDOG_THRESHOLD:-30}"
|
|
|
|
alert() {
|
|
logger -t orca-watchdog -p user.err "aggregator starvation detected: $1"
|
|
}
|
|
|
|
if [ ! -f "$CLUSTER_JSON" ]; then
|
|
alert "cluster.json absent at $CLUSTER_JSON"
|
|
orca_log_error "watchdog" "-" "starved" "cluster.json absent"
|
|
exit 0
|
|
fi
|
|
|
|
now="$(date +%s)"
|
|
mtime="$(stat -c %Y "$CLUSTER_JSON" 2>/dev/null || stat -f %m "$CLUSTER_JSON" 2>/dev/null || echo 0)"
|
|
age=$((now - mtime))
|
|
|
|
if [ "$age" -ge "$STARVATION_THRESHOLD" ]; then
|
|
alert "cluster.json age ${age}s >= threshold ${STARVATION_THRESHOLD}s"
|
|
orca_log_error "watchdog" "-" "starved" "age=${age}s threshold=${STARVATION_THRESHOLD}s"
|
|
else
|
|
orca_log_info "watchdog" "-" "ok" "age=${age}s threshold=${STARVATION_THRESHOLD}s"
|
|
fi
|