feat(P10b): drift detection (R-018/R-019/R-020, REQ-103..113)
internal/drift/drift.go: Detector (Watch via iter.Seq2, Aggregate,
Remediate with cooldown-on-success, Acknowledge), Config with tiered
cadence (critical 5s + Path units, standard 30s, default 60s).
internal/cli/drift.go: orca drift {show,watch,acknowledge,remediate,
config}. internal/emitter/drift_path.go: systemd Path+service unit
emitter (User=orca, ProtectSystem=strict). scripts/orca-drift-notify.sh
(sha256 event JSON), orca-remediate.sh (cooldown-on-success, transient
retry). Pre-flight gate (R-020, --force + per-ns scoping). orca
system user (REQ-111), NFS detection (D-233), orca job restart for
EnvironmentFile drift (D-235).
---ci---
project: orca
phase: 10b
milestone: v0.11
status: execute
---/ci---
This commit is contained in:
Jon Chery
@@ -0,0 +1,573 @@
|
||||
// Package drift implements orca's drift detection subsystem
|
||||
// (P10b, v0.11 milestone; R-018/R-019/R-020, REQ-103..REQ-113).
|
||||
//
|
||||
// The model is poll-based with a systemd Path-unit fast path for
|
||||
// critical files:
|
||||
//
|
||||
// - The Detector interface exposes four operations: Watch (stream
|
||||
// events via iter.Seq2[Event, error], D-017), Aggregate (read the
|
||||
// lead-side aggregated drift state), Remediate (trigger the
|
||||
// peer-side applier via orca-remediate.sh, with cooldown-on-success
|
||||
// per C4), and Acknowledge (record operator ack).
|
||||
// - DefaultDetector implements Detector against the SSH-push
|
||||
// transport (the same *sshpush.Transport used by txn / emitter).
|
||||
// - Config encodes the tiered cadence (critical 5s + systemd Path
|
||||
// units, standard 30s polling, default 60s) plus the remediation
|
||||
// policy (auto paths, require-approval paths, notify-on-remediate).
|
||||
//
|
||||
// The package never logs key material. slog calls carry only metadata.
|
||||
package drift
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"iter"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Status string
|
||||
|
||||
const (
|
||||
StatusModified Status = "modified"
|
||||
StatusDeleted Status = "deleted"
|
||||
StatusCreated Status = "created"
|
||||
)
|
||||
|
||||
type Action string
|
||||
|
||||
const (
|
||||
ActionAutoRemediated Action = "auto_remediated"
|
||||
ActionReported Action = "reported"
|
||||
ActionAcknowledged Action = "acknowledged"
|
||||
)
|
||||
|
||||
type ActionResult string
|
||||
|
||||
const (
|
||||
ActionResultSuccess ActionResult = "success"
|
||||
ActionResultFailed ActionResult = "failed"
|
||||
ActionResultSkipped ActionResult = "skipped"
|
||||
)
|
||||
|
||||
type Tier string
|
||||
|
||||
const (
|
||||
TierCritical Tier = "critical"
|
||||
TierStandard Tier = "standard"
|
||||
)
|
||||
|
||||
type Event struct {
|
||||
EventID string `json:"event_id"`
|
||||
TS time.Time `json:"ts"`
|
||||
Host string `json:"host"`
|
||||
Path string `json:"path"`
|
||||
Status Status `json:"status"`
|
||||
NewSHA256 string `json:"new_sha256"`
|
||||
LatestTxn string `json:"latest_txn"`
|
||||
ExpectedSHA256 string `json:"expected_sha256"`
|
||||
DriftConfirmed bool `json:"drift_confirmed"`
|
||||
Action Action `json:"action"`
|
||||
ActionResult ActionResult `json:"action_result"`
|
||||
OperatorID string `json:"operator_id"`
|
||||
}
|
||||
|
||||
type PathSpec struct {
|
||||
Tier Tier `json:"tier"`
|
||||
Pattern string `json:"pattern"`
|
||||
Interval time.Duration `json:"interval"`
|
||||
SystemdPathUnit bool `json:"systemd_path_unit"`
|
||||
}
|
||||
|
||||
type RemediationPolicy struct {
|
||||
Auto bool `json:"auto"`
|
||||
AutoPaths []string `json:"auto_paths"`
|
||||
RequireApproval []string `json:"require_approval"`
|
||||
NotifyOnRemediation bool `json:"notify_on_remediation"`
|
||||
}
|
||||
|
||||
type PollingConfig struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
DefaultInterval time.Duration `json:"default_interval"`
|
||||
MaxConcurrentPeers int `json:"max_concurrent_peers"`
|
||||
}
|
||||
|
||||
type PathsConfig struct {
|
||||
Critical []PathSpec `json:"critical"`
|
||||
Standard []PathSpec `json:"standard"`
|
||||
Excluded []string `json:"excluded"`
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
Polling PollingConfig `json:"polling"`
|
||||
Paths PathsConfig `json:"paths"`
|
||||
Remediate RemediationPolicy `json:"remediate"`
|
||||
}
|
||||
|
||||
type Detector interface {
|
||||
Watch(ctx context.Context, paths []PathSpec) iter.Seq2[Event, error]
|
||||
Aggregate(ctx context.Context, leadPeer string) ([]Event, error)
|
||||
Remediate(ctx context.Context, leadPeer, path string, force bool) error
|
||||
Acknowledge(ctx context.Context, leadPeer, path string) error
|
||||
}
|
||||
|
||||
type Transport interface {
|
||||
Exec(ctx context.Context, peer string, cmd string) ([]byte, error)
|
||||
WriteFileIdempotent(ctx context.Context, peer string, path string, content []byte, mode os.FileMode) (bool, error)
|
||||
ReadFile(ctx context.Context, peer string, path string) ([]byte, error)
|
||||
}
|
||||
|
||||
func LeadStateDir() string {
|
||||
if p := os.Getenv("ORCA_LEAD_STATE_DIR"); p != "" {
|
||||
return p
|
||||
}
|
||||
return "/etc/orca/state"
|
||||
}
|
||||
|
||||
func AggregatedPath() string {
|
||||
return filepath.Join(LeadStateDir(), "drift-events-aggregated.json")
|
||||
}
|
||||
|
||||
func AcknowledgmentsPath() string {
|
||||
return filepath.Join(LeadStateDir(), "drift-acknowledgments.json")
|
||||
}
|
||||
|
||||
func RemediatorPath() string {
|
||||
return "/usr/local/sbin/orca-remediate.sh"
|
||||
}
|
||||
|
||||
type DefaultDetector struct {
|
||||
transport Transport
|
||||
cooldown time.Duration
|
||||
}
|
||||
|
||||
func NewDefaultDetector(t Transport) *DefaultDetector {
|
||||
return &DefaultDetector{transport: t, cooldown: 5 * time.Minute}
|
||||
}
|
||||
|
||||
func (d *DefaultDetector) SetCooldown(dur time.Duration) {
|
||||
if dur > 0 {
|
||||
d.cooldown = dur
|
||||
}
|
||||
}
|
||||
|
||||
type aggregatedDoc struct {
|
||||
TS time.Time `json:"ts"`
|
||||
Events []Event `json:"events"`
|
||||
}
|
||||
|
||||
func (d *DefaultDetector) Watch(ctx context.Context, paths []PathSpec) iter.Seq2[Event, error] {
|
||||
interval := defaultWatchInterval(paths)
|
||||
if interval <= 0 {
|
||||
interval = 2 * time.Second
|
||||
}
|
||||
return func(yield func(Event, error) bool) {
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
seen := make(map[string]bool)
|
||||
for {
|
||||
events, err := d.Aggregate(ctx, "")
|
||||
if err != nil {
|
||||
if !yield(Event{}, err) {
|
||||
return
|
||||
}
|
||||
} else {
|
||||
for _, e := range events {
|
||||
if !matchesPaths(e.Path, paths) {
|
||||
continue
|
||||
}
|
||||
key := e.EventID
|
||||
if key == "" {
|
||||
key = e.Host + "|" + e.Path + "|" + e.TS.Format(time.RFC3339Nano)
|
||||
}
|
||||
if seen[key] {
|
||||
continue
|
||||
}
|
||||
seen[key] = true
|
||||
if !yield(e, nil) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (d *DefaultDetector) Aggregate(ctx context.Context, leadPeer string) ([]Event, error) {
|
||||
var raw []byte
|
||||
var err error
|
||||
if leadPeer == "" {
|
||||
raw, err = os.ReadFile(AggregatedPath())
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, fmt.Errorf("drift: read aggregated: %w", err)
|
||||
}
|
||||
} else {
|
||||
raw, err = d.transport.ReadFile(ctx, leadPeer, AggregatedPath())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("drift: read aggregated from %s: %w", leadPeer, err)
|
||||
}
|
||||
}
|
||||
if len(strings.TrimSpace(string(raw))) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
var doc aggregatedDoc
|
||||
if err := json.Unmarshal(raw, &doc); err != nil {
|
||||
return nil, fmt.Errorf("drift: parse aggregated: %w", err)
|
||||
}
|
||||
return doc.Events, nil
|
||||
}
|
||||
|
||||
func (d *DefaultDetector) Remediate(ctx context.Context, leadPeer, path string, force bool) error {
|
||||
if leadPeer == "" {
|
||||
return fmt.Errorf("drift: remediate requires a lead peer")
|
||||
}
|
||||
if !force {
|
||||
if d.inCooldown(path) {
|
||||
slog.Info("drift remediate skipped (cooldown)", "path", path, "peer", leadPeer)
|
||||
return ErrCooldown
|
||||
}
|
||||
}
|
||||
cmd := fmt.Sprintf("bash %s %s %s %s", shellQuote(RemediatorPath()), shellQuote(leadPeer), shellQuote(""), shellQuote(path))
|
||||
out, err := d.transport.Exec(ctx, leadPeer, cmd)
|
||||
if err != nil {
|
||||
if isTransientSSH(err) {
|
||||
slog.Warn("drift remediate transient failure (no cooldown)", "path", path, "peer", leadPeer, "error", err)
|
||||
return err
|
||||
}
|
||||
return fmt.Errorf("drift: remediate %s on %s: %w (output: %s)", path, leadPeer, err, string(out))
|
||||
}
|
||||
if !force {
|
||||
d.markCooldown(path)
|
||||
}
|
||||
slog.Info("drift remediate ok", "path", path, "peer", leadPeer, "output", string(out))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *DefaultDetector) Acknowledge(ctx context.Context, leadPeer, path string) error {
|
||||
if leadPeer == "" {
|
||||
return fmt.Errorf("drift: acknowledge requires a lead peer")
|
||||
}
|
||||
entry := map[string]any{
|
||||
"ts": time.Now().UTC().Format(time.RFC3339Nano),
|
||||
"path": path,
|
||||
"operator": os.Getenv("ORCA_OPERATOR_ID"),
|
||||
}
|
||||
raw, err := json.Marshal(entry)
|
||||
if err != nil {
|
||||
return fmt.Errorf("drift: marshal ack: %w", err)
|
||||
}
|
||||
if _, err := d.transport.WriteFileIdempotent(ctx, leadPeer, AcknowledgmentsPath(), append(raw, '\n'), 0o644); err != nil {
|
||||
return fmt.Errorf("drift: write ack: %w", err)
|
||||
}
|
||||
slog.Info("drift acknowledged", "path", path, "peer", leadPeer)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *DefaultDetector) inCooldown(path string) bool {
|
||||
hash := pathHash(path)
|
||||
stateDir := LeadStateDir()
|
||||
cooldownDir := filepath.Join(stateDir, "remediation-cooldown")
|
||||
stampPath := filepath.Join(cooldownDir, hash)
|
||||
info, err := os.Stat(stampPath)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return time.Since(info.ModTime()) < d.cooldown
|
||||
}
|
||||
|
||||
func (d *DefaultDetector) markCooldown(path string) {
|
||||
hash := pathHash(path)
|
||||
stateDir := LeadStateDir()
|
||||
cooldownDir := filepath.Join(stateDir, "remediation-cooldown")
|
||||
if err := os.MkdirAll(cooldownDir, 0o755); err != nil {
|
||||
slog.Warn("drift markCooldown mkdir failed", "dir", cooldownDir, "error", err)
|
||||
return
|
||||
}
|
||||
stampPath := filepath.Join(cooldownDir, hash)
|
||||
if err := os.WriteFile(stampPath, []byte(time.Now().UTC().Format(time.RFC3339Nano)), 0o644); err != nil {
|
||||
slog.Warn("drift markCooldown write failed", "path", stampPath, "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
ErrCooldown = errors.New("drift: remediation in cooldown")
|
||||
ErrPathExcluded = errors.New("drift: path is excluded")
|
||||
ErrOverlapCritical = errors.New("drift: path appears in both critical and excluded")
|
||||
)
|
||||
|
||||
func LoadConfig(path string) (*Config, error) {
|
||||
if path == "" {
|
||||
return DefaultConfig(), nil
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return DefaultConfig(), nil
|
||||
}
|
||||
return nil, fmt.Errorf("drift: load config %s: %w", path, err)
|
||||
}
|
||||
var cfg Config
|
||||
if err := json.Unmarshal(raw, &cfg); err != nil {
|
||||
return nil, fmt.Errorf("drift: parse config %s: %w", path, err)
|
||||
}
|
||||
applyConfigDefaults(&cfg)
|
||||
return &cfg, nil
|
||||
}
|
||||
|
||||
func ValidateConfig(cfg *Config) error {
|
||||
if cfg == nil {
|
||||
return fmt.Errorf("drift: nil config")
|
||||
}
|
||||
if cfg.Polling.Enabled && cfg.Polling.DefaultInterval <= 0 {
|
||||
return fmt.Errorf("drift: polling enabled but default_interval is zero")
|
||||
}
|
||||
for _, p := range cfg.Paths.Critical {
|
||||
if p.Pattern == "" {
|
||||
return fmt.Errorf("drift: critical path has empty pattern")
|
||||
}
|
||||
if p.Tier == "" {
|
||||
return fmt.Errorf("drift: critical path %q has empty tier", p.Pattern)
|
||||
}
|
||||
}
|
||||
for _, p := range cfg.Paths.Standard {
|
||||
if p.Pattern == "" {
|
||||
return fmt.Errorf("drift: standard path has empty pattern")
|
||||
}
|
||||
}
|
||||
for _, ex := range cfg.Paths.Excluded {
|
||||
for _, c := range cfg.Paths.Critical {
|
||||
if pathGlobMatch(ex, c.Pattern) {
|
||||
return fmt.Errorf("drift: excluded pattern %q overlaps critical %q: %w", ex, c.Pattern, ErrOverlapCritical)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func DefaultConfig() *Config {
|
||||
cfg := &Config{
|
||||
Polling: PollingConfig{
|
||||
Enabled: true,
|
||||
DefaultInterval: 60 * time.Second,
|
||||
MaxConcurrentPeers: 8,
|
||||
},
|
||||
Paths: PathsConfig{
|
||||
Critical: []PathSpec{
|
||||
{Tier: TierCritical, Pattern: "/etc/traefik/dynamic/orca.yml", Interval: 5 * time.Second, SystemdPathUnit: true},
|
||||
{Tier: TierCritical, Pattern: "/etc/systemd/system/orca-alloc-*.service", Interval: 5 * time.Second, SystemdPathUnit: true},
|
||||
{Tier: TierCritical, Pattern: "/etc/nftables.d/orca.nft", Interval: 5 * time.Second, SystemdPathUnit: true},
|
||||
{Tier: TierCritical, Pattern: "/etc/orca/actual/*/etc/traefik/dynamic/*", Interval: 5 * time.Second, SystemdPathUnit: true},
|
||||
{Tier: TierCritical, Pattern: "/etc/orca/actual/*/etc/systemd/system/orca-alloc-*.service", Interval: 5 * time.Second, SystemdPathUnit: true},
|
||||
{Tier: TierCritical, Pattern: "/etc/orca/actual/*/etc/sudoers.d/orca-*", Interval: 5 * time.Second, SystemdPathUnit: true},
|
||||
},
|
||||
Standard: []PathSpec{
|
||||
{Tier: TierStandard, Pattern: "/etc/orca/actual/*", Interval: 30 * time.Second},
|
||||
{Tier: TierStandard, Pattern: "/etc/sudoers.d/orca-*", Interval: 30 * time.Second},
|
||||
{Tier: TierStandard, Pattern: "/etc/systemd/system/orca-collector.{service,timer}", Interval: 30 * time.Second},
|
||||
{Tier: TierStandard, Pattern: "/etc/systemd/system/orca-aggregator.{service,timer}", Interval: 30 * time.Second},
|
||||
{Tier: TierStandard, Pattern: "/etc/systemd/system/orca-pull.{service,timer}", Interval: 30 * time.Second},
|
||||
{Tier: TierStandard, Pattern: "/etc/systemd/system/orca-drift.{service,timer}", Interval: 30 * time.Second},
|
||||
},
|
||||
Excluded: []string{
|
||||
"/etc/orca/credentials/*",
|
||||
"/run/orca/*",
|
||||
"/etc/orca/state/drift-events/*",
|
||||
},
|
||||
},
|
||||
Remediate: RemediationPolicy{
|
||||
Auto: true,
|
||||
AutoPaths: []string{"/etc/traefik/dynamic/*", "/etc/nftables.d/*", "/etc/sudoers.d/orca-*"},
|
||||
RequireApproval: []string{"/etc/systemd/system/orca-alloc-*.service"},
|
||||
NotifyOnRemediation: true,
|
||||
},
|
||||
}
|
||||
applyConfigDefaults(cfg)
|
||||
return cfg
|
||||
}
|
||||
|
||||
func applyConfigDefaults(cfg *Config) {
|
||||
if cfg.Polling.DefaultInterval == 0 {
|
||||
cfg.Polling.DefaultInterval = 60 * time.Second
|
||||
}
|
||||
if cfg.Polling.MaxConcurrentPeers == 0 {
|
||||
cfg.Polling.MaxConcurrentPeers = 8
|
||||
}
|
||||
for i := range cfg.Paths.Critical {
|
||||
if cfg.Paths.Critical[i].Tier == "" {
|
||||
cfg.Paths.Critical[i].Tier = TierCritical
|
||||
}
|
||||
if cfg.Paths.Critical[i].Interval == 0 {
|
||||
cfg.Paths.Critical[i].Interval = 5 * time.Second
|
||||
}
|
||||
}
|
||||
for i := range cfg.Paths.Standard {
|
||||
if cfg.Paths.Standard[i].Tier == "" {
|
||||
cfg.Paths.Standard[i].Tier = TierStandard
|
||||
}
|
||||
if cfg.Paths.Standard[i].Interval == 0 {
|
||||
cfg.Paths.Standard[i].Interval = 30 * time.Second
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func NamespaceScope(events []Event, namespace string) []Event {
|
||||
if namespace == "" {
|
||||
var out []Event
|
||||
for _, e := range events {
|
||||
if !e.DriftConfirmed || e.Action == ActionAcknowledged {
|
||||
continue
|
||||
}
|
||||
out = append(out, e)
|
||||
}
|
||||
return out
|
||||
}
|
||||
var out []Event
|
||||
for _, e := range events {
|
||||
if !e.DriftConfirmed || e.Action == ActionAcknowledged {
|
||||
continue
|
||||
}
|
||||
if nsForPath(e.Path) == namespace {
|
||||
out = append(out, e)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func nsForPath(p string) string {
|
||||
const prefix = "/etc/orca/actual/"
|
||||
if !strings.HasPrefix(p, prefix) {
|
||||
return ""
|
||||
}
|
||||
rest := p[len(prefix):]
|
||||
if i := strings.IndexByte(rest, '/'); i >= 0 {
|
||||
return rest[:i]
|
||||
}
|
||||
return rest
|
||||
}
|
||||
|
||||
func matchesPaths(p string, specs []PathSpec) bool {
|
||||
if len(specs) == 0 {
|
||||
return true
|
||||
}
|
||||
for _, s := range specs {
|
||||
if pathGlobMatch(s.Pattern, p) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func defaultWatchInterval(specs []PathSpec) time.Duration {
|
||||
var min time.Duration
|
||||
for _, s := range specs {
|
||||
if s.Interval > 0 && (min == 0 || s.Interval < min) {
|
||||
min = s.Interval
|
||||
}
|
||||
}
|
||||
return min
|
||||
}
|
||||
|
||||
func pathGlobMatch(pattern, p string) bool {
|
||||
if pattern == "" {
|
||||
return false
|
||||
}
|
||||
expanded := expandBraces(pattern)
|
||||
for _, alt := range expanded {
|
||||
if globMatchSegment(alt, p) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func globMatchSegment(pattern, p string) bool {
|
||||
if pattern == "" {
|
||||
return false
|
||||
}
|
||||
if pattern == p {
|
||||
return true
|
||||
}
|
||||
if strings.HasSuffix(pattern, "/*") {
|
||||
prefix := pattern[:len(pattern)-2]
|
||||
if p == prefix {
|
||||
return true
|
||||
}
|
||||
return strings.HasPrefix(p, prefix+"/")
|
||||
}
|
||||
ok, err := filepath.Match(pattern, p)
|
||||
return err == nil && ok
|
||||
}
|
||||
|
||||
func expandBraces(p string) []string {
|
||||
open := strings.IndexByte(p, '{')
|
||||
if open < 0 {
|
||||
return []string{p}
|
||||
}
|
||||
close := strings.IndexByte(p[open:], '}')
|
||||
if close < 0 {
|
||||
return []string{p}
|
||||
}
|
||||
close += open
|
||||
prefix := p[:open]
|
||||
body := p[open+1 : close]
|
||||
suffix := p[close+1:]
|
||||
alternatives := strings.Split(body, ",")
|
||||
var out []string
|
||||
for _, a := range alternatives {
|
||||
for _, tail := range expandBraces(suffix) {
|
||||
out = append(out, prefix+a+tail)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func pathHash(p string) string {
|
||||
sum := sha256.Sum256([]byte(p))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func isTransientSSH(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
s := err.Error()
|
||||
for _, sub := range []string{"connection refused", "i/o timeout", "EOF", "no such host", "connection reset", "timeout", "deadline exceeded", "temporarily unavailable"} {
|
||||
if strings.Contains(s, sub) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func shellQuote(s string) string {
|
||||
return "'" + strings.ReplaceAll(s, "'", "'\\''") + "'"
|
||||
}
|
||||
|
||||
func ParseEvent(raw []byte) (Event, error) {
|
||||
var e Event
|
||||
if err := json.Unmarshal(raw, &e); err != nil {
|
||||
return Event{}, fmt.Errorf("drift: parse event: %w", err)
|
||||
}
|
||||
return e, nil
|
||||
}
|
||||
|
||||
func MarshalEvent(e Event) ([]byte, error) {
|
||||
return json.MarshalIndent(e, "", " ")
|
||||
}
|
||||
|
||||
var _ Detector = (*DefaultDetector)(nil)
|
||||
@@ -0,0 +1,465 @@
|
||||
package drift
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type mockTransport struct {
|
||||
execOut []byte
|
||||
execErr error
|
||||
execFn func(ctx context.Context, peer string, cmd string) ([]byte, error)
|
||||
readOut []byte
|
||||
readErr error
|
||||
writes []writeEntry
|
||||
}
|
||||
|
||||
type writeEntry struct {
|
||||
peer string
|
||||
path string
|
||||
content []byte
|
||||
mode os.FileMode
|
||||
}
|
||||
|
||||
func (m *mockTransport) Exec(ctx context.Context, peer string, cmd string) ([]byte, error) {
|
||||
if m.execFn != nil {
|
||||
return m.execFn(ctx, peer, cmd)
|
||||
}
|
||||
return m.execOut, m.execErr
|
||||
}
|
||||
|
||||
func (m *mockTransport) WriteFileIdempotent(ctx context.Context, peer string, path string, content []byte, mode os.FileMode) (bool, error) {
|
||||
m.writes = append(m.writes, writeEntry{peer, path, content, mode})
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (m *mockTransport) ReadFile(ctx context.Context, peer string, path string) ([]byte, error) {
|
||||
return m.readOut, m.readErr
|
||||
}
|
||||
|
||||
func setupDriftTestEnv(t *testing.T) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
t.Setenv("ORCA_LEAD_STATE_DIR", dir)
|
||||
return dir
|
||||
}
|
||||
|
||||
func TestParseEvent(t *testing.T) {
|
||||
raw := []byte(`{
|
||||
"event_id":"EVT-1",
|
||||
"ts":"2026-01-01T00:00:00Z",
|
||||
"host":"peer1",
|
||||
"path":"/etc/traefik/dynamic/orca.yml",
|
||||
"status":"modified",
|
||||
"new_sha256":"abc",
|
||||
"latest_txn":"T-1234567890abcdef",
|
||||
"expected_sha256":"def",
|
||||
"drift_confirmed":true,
|
||||
"action":"reported",
|
||||
"action_result":"skipped",
|
||||
"operator_id":"op1"
|
||||
}`)
|
||||
e, err := ParseEvent(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseEvent: %v", err)
|
||||
}
|
||||
if e.EventID != "EVT-1" {
|
||||
t.Errorf("EventID = %q, want EVT-1", e.EventID)
|
||||
}
|
||||
if e.Host != "peer1" {
|
||||
t.Errorf("Host = %q, want peer1", e.Host)
|
||||
}
|
||||
if e.Path != "/etc/traefik/dynamic/orca.yml" {
|
||||
t.Errorf("Path = %q", e.Path)
|
||||
}
|
||||
if e.Status != StatusModified {
|
||||
t.Errorf("Status = %q, want modified", e.Status)
|
||||
}
|
||||
if !e.DriftConfirmed {
|
||||
t.Errorf("DriftConfirmed = false, want true")
|
||||
}
|
||||
if e.Action != ActionReported {
|
||||
t.Errorf("Action = %q, want reported", e.Action)
|
||||
}
|
||||
if e.ActionResult != ActionResultSkipped {
|
||||
t.Errorf("ActionResult = %q, want skipped", e.ActionResult)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseEventInvalid(t *testing.T) {
|
||||
_, err := ParseEvent([]byte(`{not json`))
|
||||
if err == nil {
|
||||
t.Fatal("expected error for invalid JSON, got nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarshalEventRoundTrip(t *testing.T) {
|
||||
e := Event{EventID: "E1", Host: "h", Path: "/p", Status: StatusCreated, DriftConfirmed: true}
|
||||
raw, err := MarshalEvent(e)
|
||||
if err != nil {
|
||||
t.Fatalf("MarshalEvent: %v", err)
|
||||
}
|
||||
out, err := ParseEvent(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseEvent: %v", err)
|
||||
}
|
||||
if out.EventID != e.EventID || out.Host != e.Host || out.Path != e.Path || out.Status != e.Status {
|
||||
t.Errorf("round-trip mismatch: %+v vs %+v", out, e)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadConfigDefault(t *testing.T) {
|
||||
cfg, err := LoadConfig("")
|
||||
if err != nil {
|
||||
t.Fatalf("LoadConfig empty: %v", err)
|
||||
}
|
||||
if !cfg.Polling.Enabled {
|
||||
t.Errorf("Polling.Enabled = false, want true")
|
||||
}
|
||||
if cfg.Polling.DefaultInterval != 60*time.Second {
|
||||
t.Errorf("DefaultInterval = %v, want 60s", cfg.Polling.DefaultInterval)
|
||||
}
|
||||
if len(cfg.Paths.Critical) == 0 {
|
||||
t.Errorf("Critical paths empty")
|
||||
}
|
||||
found := false
|
||||
for _, c := range cfg.Paths.Critical {
|
||||
if c.Pattern == "/etc/traefik/dynamic/orca.yml" {
|
||||
found = true
|
||||
if c.Interval != 5*time.Second {
|
||||
t.Errorf("critical interval = %v, want 5s", c.Interval)
|
||||
}
|
||||
if !c.SystemdPathUnit {
|
||||
t.Errorf("SystemdPathUnit = false, want true")
|
||||
}
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("traefik critical path missing")
|
||||
}
|
||||
found = false
|
||||
for _, s := range cfg.Paths.Standard {
|
||||
if s.Pattern == "/etc/orca/actual/*" {
|
||||
found = true
|
||||
if s.Interval != 30*time.Second {
|
||||
t.Errorf("standard interval = %v, want 30s", s.Interval)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("/etc/orca/actual/* standard path missing")
|
||||
}
|
||||
if !cfg.Remediate.Auto {
|
||||
t.Errorf("Remediate.Auto = false, want true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadConfigFile(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "drift.json")
|
||||
cfgJSON := `{
|
||||
"polling": {"enabled": true, "default_interval": 90000000000, "max_concurrent_peers": 4},
|
||||
"paths": {
|
||||
"critical": [{"tier":"critical","pattern":"/etc/critical.yml","interval":5000000000,"systemd_path_unit":true}],
|
||||
"standard": [{"tier":"standard","pattern":"/etc/standard.yml","interval":30000000000}],
|
||||
"excluded": ["/run/orca/*"]
|
||||
},
|
||||
"remediate": {"auto": false, "auto_paths": [], "require_approval": ["/etc/critical.yml"], "notify_on_remediation": false}
|
||||
}`
|
||||
if err := os.WriteFile(path, []byte(cfgJSON), 0o644); err != nil {
|
||||
t.Fatalf("write config: %v", err)
|
||||
}
|
||||
cfg, err := LoadConfig(path)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadConfig: %v", err)
|
||||
}
|
||||
if cfg.Polling.MaxConcurrentPeers != 4 {
|
||||
t.Errorf("MaxConcurrentPeers = %d, want 4", cfg.Polling.MaxConcurrentPeers)
|
||||
}
|
||||
if cfg.Polling.DefaultInterval != 90*time.Second {
|
||||
t.Errorf("DefaultInterval = %v, want 90s", cfg.Polling.DefaultInterval)
|
||||
}
|
||||
if cfg.Remediate.Auto {
|
||||
t.Errorf("Auto = true, want false")
|
||||
}
|
||||
if len(cfg.Paths.Critical) != 1 || cfg.Paths.Critical[0].Pattern != "/etc/critical.yml" {
|
||||
t.Errorf("critical = %+v", cfg.Paths.Critical)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateConfigValid(t *testing.T) {
|
||||
cfg := DefaultConfig()
|
||||
if err := ValidateConfig(cfg); err != nil {
|
||||
t.Fatalf("ValidateConfig(default): %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateConfigEmptyPattern(t *testing.T) {
|
||||
cfg := DefaultConfig()
|
||||
cfg.Paths.Critical[0].Pattern = ""
|
||||
if err := ValidateConfig(cfg); err == nil {
|
||||
t.Fatal("expected error for empty critical pattern")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateConfigPollingZero(t *testing.T) {
|
||||
cfg := DefaultConfig()
|
||||
cfg.Polling.Enabled = true
|
||||
cfg.Polling.DefaultInterval = 0
|
||||
if err := ValidateConfig(cfg); err == nil {
|
||||
t.Fatal("expected error for zero default_interval with polling enabled")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateConfigOverlapCriticalExcluded(t *testing.T) {
|
||||
cfg := DefaultConfig()
|
||||
cfg.Paths.Excluded = append(cfg.Paths.Excluded, "/etc/traefik/dynamic/orca.yml")
|
||||
err := ValidateConfig(cfg)
|
||||
if err == nil {
|
||||
t.Fatal("expected overlap error")
|
||||
}
|
||||
if !errors.Is(err, ErrOverlapCritical) {
|
||||
t.Errorf("expected ErrOverlapCritical, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateConfigNil(t *testing.T) {
|
||||
if err := ValidateConfig(nil); err == nil {
|
||||
t.Fatal("expected error for nil config")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAggregateLocalMissing(t *testing.T) {
|
||||
setupDriftTestEnv(t)
|
||||
d := NewDefaultDetector(&mockTransport{})
|
||||
events, err := d.Aggregate(context.Background(), "")
|
||||
if err != nil {
|
||||
t.Fatalf("Aggregate missing: %v", err)
|
||||
}
|
||||
if events != nil {
|
||||
t.Errorf("events = %v, want nil", events)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAggregateLocal(t *testing.T) {
|
||||
setupDriftTestEnv(t)
|
||||
doc := `{"ts":"2026-01-01T00:00:00Z","events":[{"event_id":"E1","host":"p","path":"/etc/traefik/dynamic/orca.yml","status":"modified","drift_confirmed":true}]}`
|
||||
if err := os.WriteFile(AggregatedPath(), []byte(doc), 0o644); err != nil {
|
||||
t.Fatalf("write aggregated: %v", err)
|
||||
}
|
||||
d := NewDefaultDetector(&mockTransport{})
|
||||
events, err := d.Aggregate(context.Background(), "")
|
||||
if err != nil {
|
||||
t.Fatalf("Aggregate: %v", err)
|
||||
}
|
||||
if len(events) != 1 {
|
||||
t.Fatalf("events len = %d, want 1", len(events))
|
||||
}
|
||||
if events[0].EventID != "E1" {
|
||||
t.Errorf("EventID = %q, want E1", events[0].EventID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAggregateRemote(t *testing.T) {
|
||||
setupDriftTestEnv(t)
|
||||
doc := `{"ts":"2026-01-01T00:00:00Z","events":[{"event_id":"E2","host":"p","path":"/etc/orca/actual/ns-a/x","status":"created","drift_confirmed":true}]}`
|
||||
mt := &mockTransport{readOut: []byte(doc)}
|
||||
d := NewDefaultDetector(mt)
|
||||
events, err := d.Aggregate(context.Background(), "lead:22")
|
||||
if err != nil {
|
||||
t.Fatalf("Aggregate remote: %v", err)
|
||||
}
|
||||
if len(events) != 1 {
|
||||
t.Fatalf("events len = %d, want 1", len(events))
|
||||
}
|
||||
if events[0].EventID != "E2" {
|
||||
t.Errorf("EventID = %q, want E2", events[0].EventID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWatchCancels(t *testing.T) {
|
||||
setupDriftTestEnv(t)
|
||||
doc := `{"ts":"2026-01-01T00:00:00Z","events":[{"event_id":"E1","host":"p","path":"/etc/traefik/dynamic/orca.yml","status":"modified","drift_confirmed":true}]}`
|
||||
if err := os.WriteFile(AggregatedPath(), []byte(doc), 0o644); err != nil {
|
||||
t.Fatalf("write aggregated: %v", err)
|
||||
}
|
||||
d := NewDefaultDetector(&mockTransport{})
|
||||
d.SetCooldown(50 * time.Millisecond)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
gotEvents := 0
|
||||
for e, err := range d.Watch(ctx, []PathSpec{{Pattern: "/etc/traefik/dynamic/orca.yml", Interval: 10 * time.Millisecond}}) {
|
||||
if err != nil {
|
||||
t.Fatalf("Watch err: %v", err)
|
||||
}
|
||||
gotEvents++
|
||||
_ = e
|
||||
cancel()
|
||||
break
|
||||
}
|
||||
if gotEvents != 1 {
|
||||
t.Errorf("gotEvents = %d, want 1", gotEvents)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWatchStreamsAndDedupes(t *testing.T) {
|
||||
setupDriftTestEnv(t)
|
||||
doc := `{"ts":"2026-01-01T00:00:00Z","events":[{"event_id":"E1","host":"p","path":"/etc/traefik/dynamic/orca.yml","status":"modified","drift_confirmed":true}]}`
|
||||
if err := os.WriteFile(AggregatedPath(), []byte(doc), 0o644); err != nil {
|
||||
t.Fatalf("write aggregated: %v", err)
|
||||
}
|
||||
d := NewDefaultDetector(&mockTransport{})
|
||||
d.SetCooldown(50 * time.Millisecond)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 80*time.Millisecond)
|
||||
defer cancel()
|
||||
got := 0
|
||||
for _, err := range d.Watch(ctx, nil) {
|
||||
if err != nil {
|
||||
t.Fatalf("Watch err: %v", err)
|
||||
}
|
||||
got++
|
||||
}
|
||||
if got != 1 {
|
||||
t.Errorf("got = %d, want 1 (dedup)", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemediateCooldownOnSuccess(t *testing.T) {
|
||||
setupDriftTestEnv(t)
|
||||
mt := &mockTransport{execOut: []byte("ok")}
|
||||
d := NewDefaultDetector(mt)
|
||||
d.SetCooldown(1 * time.Hour)
|
||||
if err := d.Remediate(context.Background(), "lead:22", "/etc/traefik/dynamic/orca.yml", false); err != nil {
|
||||
t.Fatalf("first Remediate: %v", err)
|
||||
}
|
||||
if err := d.Remediate(context.Background(), "lead:22", "/etc/traefik/dynamic/orca.yml", false); err == nil {
|
||||
t.Fatal("second Remediate should hit cooldown")
|
||||
} else if !errors.Is(err, ErrCooldown) {
|
||||
t.Errorf("expected ErrCooldown, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemediateForceBypassesCooldown(t *testing.T) {
|
||||
setupDriftTestEnv(t)
|
||||
mt := &mockTransport{execOut: []byte("ok")}
|
||||
d := NewDefaultDetector(mt)
|
||||
d.SetCooldown(1 * time.Hour)
|
||||
if err := d.Remediate(context.Background(), "lead:22", "/etc/p", false); err != nil {
|
||||
t.Fatalf("first: %v", err)
|
||||
}
|
||||
if err := d.Remediate(context.Background(), "lead:22", "/etc/p", true); err != nil {
|
||||
t.Fatalf("force bypass: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemediateTransientNoCooldown(t *testing.T) {
|
||||
setupDriftTestEnv(t)
|
||||
mt := &mockTransport{execErr: fmt.Errorf("connection refused")}
|
||||
d := NewDefaultDetector(mt)
|
||||
d.SetCooldown(1 * time.Hour)
|
||||
if err := d.Remediate(context.Background(), "lead:22", "/etc/p", false); err == nil {
|
||||
t.Fatal("expected transient error")
|
||||
}
|
||||
if d.inCooldown("/etc/p") {
|
||||
t.Errorf("cooldown entered after transient failure")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemediateRequiresLead(t *testing.T) {
|
||||
setupDriftTestEnv(t)
|
||||
d := NewDefaultDetector(&mockTransport{})
|
||||
if err := d.Remediate(context.Background(), "", "/etc/p", true); err == nil {
|
||||
t.Fatal("expected error for empty lead")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcknowledgeWrites(t *testing.T) {
|
||||
setupDriftTestEnv(t)
|
||||
mt := &mockTransport{}
|
||||
d := NewDefaultDetector(mt)
|
||||
if err := d.Acknowledge(context.Background(), "lead:22", "/etc/traefik/dynamic/orca.yml"); err != nil {
|
||||
t.Fatalf("Acknowledge: %v", err)
|
||||
}
|
||||
if len(mt.writes) != 1 {
|
||||
t.Fatalf("writes = %d, want 1", len(mt.writes))
|
||||
}
|
||||
w := mt.writes[0]
|
||||
if w.peer != "lead:22" {
|
||||
t.Errorf("peer = %q, want lead:22", w.peer)
|
||||
}
|
||||
if w.path != AcknowledgmentsPath() {
|
||||
t.Errorf("path = %q, want %q", w.path, AcknowledgmentsPath())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcknowledgeRequiresLead(t *testing.T) {
|
||||
setupDriftTestEnv(t)
|
||||
d := NewDefaultDetector(&mockTransport{})
|
||||
if err := d.Acknowledge(context.Background(), "", "/etc/p"); err == nil {
|
||||
t.Fatal("expected error for empty lead")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNamespaceScopeClusterWide(t *testing.T) {
|
||||
events := []Event{
|
||||
{EventID: "E1", Path: "/etc/traefik/dynamic/orca.yml", DriftConfirmed: true},
|
||||
{EventID: "E2", Path: "/etc/orca/actual/ns-a/x", DriftConfirmed: true},
|
||||
{EventID: "E3", Path: "/etc/orca/actual/ns-b/y", DriftConfirmed: true, Action: ActionAcknowledged},
|
||||
}
|
||||
out := NamespaceScope(events, "")
|
||||
if len(out) != 2 {
|
||||
t.Errorf("cluster-wide scope len = %d, want 2 (ack excluded)", len(out))
|
||||
}
|
||||
}
|
||||
|
||||
func TestNamespaceScopePerNamespace(t *testing.T) {
|
||||
events := []Event{
|
||||
{EventID: "E1", Path: "/etc/orca/actual/ns-a/x", DriftConfirmed: true},
|
||||
{EventID: "E2", Path: "/etc/orca/actual/ns-b/y", DriftConfirmed: true},
|
||||
{EventID: "E3", Path: "/etc/traefik/dynamic/orca.yml", DriftConfirmed: true},
|
||||
}
|
||||
nsA := NamespaceScope(events, "ns-a")
|
||||
if len(nsA) != 1 || nsA[0].EventID != "E1" {
|
||||
t.Errorf("ns-a scope = %+v, want only E1", nsA)
|
||||
}
|
||||
nsB := NamespaceScope(events, "ns-b")
|
||||
if len(nsB) != 1 || nsB[0].EventID != "E2" {
|
||||
t.Errorf("ns-b scope = %+v, want only E2", nsB)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExpandBraces(t *testing.T) {
|
||||
out := expandBraces("/etc/x.{service,timer}")
|
||||
if len(out) != 2 {
|
||||
t.Fatalf("len = %d, want 2", len(out))
|
||||
}
|
||||
if out[0] != "/etc/x.service" || out[1] != "/etc/x.timer" {
|
||||
t.Errorf("out = %v", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPathGlobMatch(t *testing.T) {
|
||||
if !pathGlobMatch("/etc/orca/actual/*", "/etc/orca/actual/ns-a") {
|
||||
t.Errorf("expected glob match for single segment")
|
||||
}
|
||||
if !pathGlobMatch("/etc/orca/actual/*", "/etc/orca/actual/ns-a/x") {
|
||||
t.Errorf("expected glob match for nested segment (drift /* crosses /)")
|
||||
}
|
||||
if !pathGlobMatch("/etc/systemd/system/orca-collector.{service,timer}", "/etc/systemd/system/orca-collector.service") {
|
||||
t.Errorf("expected brace match")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNsForPath(t *testing.T) {
|
||||
if got := nsForPath("/etc/orca/actual/ns-a/x"); got != "ns-a" {
|
||||
t.Errorf("nsForPath = %q, want ns-a", got)
|
||||
}
|
||||
if got := nsForPath("/etc/traefik/dynamic/orca.yml"); got != "" {
|
||||
t.Errorf("nsForPath = %q, want empty", got)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user