#!/usr/bin/env bash
# orca-remediate.sh — lead-side drift remediator (P10b, REQ-108, C4).
#
# Re-pushes the latest applied txn's per-peer render tree via rsync and
# runs the peer-side applier. Cooldown is 5 minutes per path and applies
# ONLY on successful remediation (C4 refinement from CLARIFY). Transient
# failures (SSH down, render tree missing) retry on the next aggregator
# tick WITHOUT entering cooldown.
#
# Usage: orca-remediate.sh <peer> <txn-id> [path]
#
# Cooldown state: /etc/orca/state/remediation-cooldown/<path-hash>
#
# Exit codes:
#   0 = remediated (or cooldown skipped)
#   1 = transient failure (no cooldown entered)
#   2 = cooldown active (caller may log)

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=lib/orca-log.sh
. "$SCRIPT_DIR/lib/orca-log.sh"

ORCA_LOG_ACTOR="spiffe://orca/cli/remediate"

STATE_DIR="${ORCA_STATE_DIR:-/etc/orca/state}"
APPLIED_DIR="${ORCA_APPLIED_DIR:-/etc/orca/state/applied}"
COOLDOWN_DIR="$STATE_DIR/remediation-cooldown"
COOLDOWN_SECONDS="${ORCA_REMEDIATE_COOLDOWN:-300}"

SSH_OPTS="${ORCA_SSH_OPTS:--o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=5}"
RSYNC_OPTS="${ORCA_RSYNC_OPTS:--a --quiet}"

if [ "$#" -lt 2 ]; then
	orca_log_error "remediate" "-" "failed" "usage: orca-remediate.sh <peer> <txn-id> [path]"
	echo "usage: orca-remediate.sh <peer> <txn-id> [path]" >&2
	exit 1
fi

PEER="$1"
TXN_ID="$2"
DRIFT_PATH="${3:-}"

path_hash() {
	local p="${1:-root}"
	printf '%s' "$p" | sha256sum | awk '{print $1}'
}

PATH_HASH="$(path_hash "$DRIFT_PATH")"
COOLDOWN_FILE="$COOLDOWN_DIR/$PATH_HASH"

mkdir -p "$COOLDOWN_DIR"

check_cooldown() {
	if [ -z "$DRIFT_PATH" ]; then
		return 1
	fi
	if [ ! -f "$COOLDOWN_FILE" ]; then
		return 1
	fi
	local now ts age
	now="$(date +%s)"
	ts="$(stat -c %Y "$COOLDOWN_FILE" 2>/dev/null || echo 0)"
	age=$((now - ts))
	if [ "$age" -lt "$COOLDOWN_SECONDS" ]; then
		return 0
	fi
	return 1
}

mark_cooldown() {
	if [ -z "$DRIFT_PATH" ]; then
		return 0
	fi
	date -u +%Y-%m-%dT%H:%M:%S.%3NZ >"$COOLDOWN_FILE" 2>/dev/null || true
}

TXN_DIR="$APPLIED_DIR/$TXN_ID"

if [ ! -d "$TXN_DIR" ]; then
	orca_log_error "remediate" "$PEER" "transient" "txn dir missing: $TXN_DIR"
	echo "transient: txn dir missing: $TXN_DIR" >&2
	exit 1
fi

if check_cooldown; then
	orca_log_warn "remediate" "$PEER" "cooldown" "path=$DRIFT_PATH txn=$TXN_ID"
	echo "cooldown: path=$DRIFT_PATH txn=$TXN_ID" >&2
	exit 2
fi

RSYNC_FAILED=0
if rsync $RSYNC_OPTS "$TXN_DIR/" "$PEER:/run/orca/txns/$TXN_ID/" 2>/tmp/orca-remediate-rsync.err; then
	:
else
	RSYNC_FAILED=1
fi

if [ "$RSYNC_FAILED" -eq 1 ]; then
	orca_log_warn "remediate" "$PEER" "transient" "rsync failed for txn=$TXN_ID path=$DRIFT_PATH err=$(tr '\n' ' ' < /tmp/orca-remediate-rsync.err 2>/dev/null)"
	echo "transient: rsync failed" >&2
	exit 1
fi

PULL_CMD="bash /run/orca/txns/$TXN_ID/orca-pull.sh --txn-dir /run/orca/txns/$TXN_ID --namespace _defaults"
if ! ssh $SSH_OPTS "$PEER" "$PULL_CMD" 2>/tmp/orca-remediate-pull.err; then
	PULL_ERR="$(tr '\n' ' ' < /tmp/orca-remediate-pull.err 2>/dev/null)"
	if echo "$PULL_ERR" | grep -q "already-applied"; then
		orca_log_info "remediate" "$PEER" "ok" "already-applied txn=$TXN_ID path=$DRIFT_PATH"
		mark_cooldown
		echo "already-applied"
		exit 0
	fi
	if echo "$PULL_ERR" | grep -Eq "connection refused|i/o timeout|no such host|connection reset|timeout|deadline exceeded|EOF"; then
		orca_log_warn "remediate" "$PEER" "transient" "pull failed (transient): $PULL_ERR"
		echo "transient: pull failed" >&2
		exit 1
	fi
	orca_log_error "remediate" "$PEER" "failed" "pull failed: $PULL_ERR txn=$TXN_ID path=$DRIFT_PATH"
	echo "failed: pull failed" >&2
	exit 1
fi

mark_cooldown
orca_log_info "remediate" "$PEER" "ok" "remediated txn=$TXN_ID path=$DRIFT_PATH"
echo "remediated"
exit 0