feat(P10b): drift detection (R-018/R-019/R-020, REQ-103..113)
internal/drift/drift.go: Detector (Watch via iter.Seq2, Aggregate,
Remediate with cooldown-on-success, Acknowledge), Config with tiered
cadence (critical 5s + Path units, standard 30s, default 60s).
internal/cli/drift.go: orca drift {show,watch,acknowledge,remediate,
config}. internal/emitter/drift_path.go: systemd Path+service unit
emitter (User=orca, ProtectSystem=strict). scripts/orca-drift-notify.sh
(sha256 event JSON), orca-remediate.sh (cooldown-on-success, transient
retry). Pre-flight gate (R-020, --force + per-ns scoping). orca
system user (REQ-111), NFS detection (D-233), orca job restart for
EnvironmentFile drift (D-235).
---ci---
project: orca
phase: 10b
milestone: v0.11
status: execute
---/ci---
This commit is contained in:
Jon Chery
@@ -0,0 +1,391 @@
|
||||
// Package cli: drift.go implements the `orca drift` subcommand family
|
||||
// (P10b, v0.11; R-018/R-019/R-020, REQ-104). Subcommands:
|
||||
//
|
||||
// orca drift show current drift state (table)
|
||||
// orca drift watch [--interval=2s] [--paths=...] [--json]
|
||||
// stream drift events (iter.Seq2, D-017)
|
||||
// ctrl-c cancels via signal.NotifyContext (D-023)
|
||||
// orca drift show [--peer <host>] detailed drift events for a peer
|
||||
// orca drift acknowledge <peer> <path>
|
||||
// record operator acknowledgment
|
||||
// orca drift remediate <peer> <path> [--force]
|
||||
// trigger manual remediation
|
||||
// orca drift config show show current drift config
|
||||
// orca drift config validate validate config
|
||||
//
|
||||
// Plus the `orca job restart <name>` command for EnvironmentFile drift
|
||||
// (REQ-113, D-235) — restarts an allocation to pick up env-file drift.
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"git.cloudinit.dev/coreci/orca/internal/certpaths"
|
||||
"git.cloudinit.dev/coreci/orca/internal/drift"
|
||||
"git.cloudinit.dev/coreci/orca/internal/sshpush"
|
||||
)
|
||||
|
||||
var (
|
||||
driftWatchInterval time.Duration
|
||||
driftWatchPaths []string
|
||||
driftShowPeer string
|
||||
driftConfigPath string
|
||||
driftRemediateForce bool
|
||||
driftAckPeer string
|
||||
driftAckPath string
|
||||
driftRemediatePeer string
|
||||
driftRemediatePath string
|
||||
driftWatchPollOverride time.Duration
|
||||
)
|
||||
|
||||
// driftTransport is the SSH-push surface the drift CLI needs. It
|
||||
// mirrors drift.Transport; tests substitute a mock.
|
||||
type driftTransport interface {
|
||||
Exec(ctx context.Context, peer string, cmd string) ([]byte, error)
|
||||
WriteFileIdempotent(ctx context.Context, peer string, path string, content []byte, mode os.FileMode) (bool, error)
|
||||
ReadFile(ctx context.Context, peer string, path string) ([]byte, error)
|
||||
}
|
||||
|
||||
// driftTransportOverride is the package-level test seam.
|
||||
var driftTransportOverride driftTransport
|
||||
|
||||
func driftTransportFromCtx() (driftTransport, error) {
|
||||
if driftTransportOverride != nil {
|
||||
return driftTransportOverride, nil
|
||||
}
|
||||
keyPath := certpaths.SSHKeyPath()
|
||||
khPath := certpaths.KnownHostsPath()
|
||||
return sshpush.NewTransport(keyPath, khPath), nil
|
||||
}
|
||||
|
||||
// driftDetectorOverride is the package-level test seam for the
|
||||
// Detector itself. When non-nil it replaces the production detector
|
||||
// (which wraps a driftTransport). Tests set it and restore nil.
|
||||
var driftDetectorOverride drift.Detector
|
||||
|
||||
func driftDetector() (drift.Detector, error) {
|
||||
if driftDetectorOverride != nil {
|
||||
return driftDetectorOverride, nil
|
||||
}
|
||||
t, err := driftTransportFromCtx()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return drift.NewDefaultDetector(t), nil
|
||||
}
|
||||
|
||||
var driftCmd = &cobra.Command{
|
||||
Use: "drift",
|
||||
Short: "Detect and remediate control-plane drift (P10b)",
|
||||
Long: `Orca's drift detector is a BACKSTOP (R-019): the primary
|
||||
consistency mechanism is systemd / Traefik / step-ca / Syncthing
|
||||
themselves. The detector polls the lead's aggregated drift state
|
||||
(drift-events-aggregated.json) and can trigger orca-remediate.sh for
|
||||
auto-remediable paths. Pre-flight drift blocks txn apply (R-020).`,
|
||||
}
|
||||
|
||||
var driftShowCmd = &cobra.Command{
|
||||
Use: "show",
|
||||
Short: "Show current drift state (table format)",
|
||||
Long: `Show the aggregated drift events from the lead. Use --peer to filter to a single peer.`,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
d, err := driftDetector()
|
||||
if err != nil {
|
||||
return fmt.Errorf("drift detector: %w", err)
|
||||
}
|
||||
events, err := d.Aggregate(cmd.Context(), driftShowPeer)
|
||||
if err != nil {
|
||||
return fmt.Errorf("aggregate: %w", err)
|
||||
}
|
||||
if driftShowPeer != "" {
|
||||
var filtered []drift.Event
|
||||
for _, e := range events {
|
||||
if e.Host == driftShowPeer {
|
||||
filtered = append(filtered, e)
|
||||
}
|
||||
}
|
||||
events = filtered
|
||||
}
|
||||
if jsonOutput {
|
||||
return printJSON(events)
|
||||
}
|
||||
out := cmd.OutOrStdout()
|
||||
if len(events) == 0 {
|
||||
fmt.Fprintln(out, "No drift events.")
|
||||
return nil
|
||||
}
|
||||
fmt.Fprintf(out, "%-20s %-20s %-40s %-10s %-10s\n", "EVENT-ID", "HOST", "PATH", "STATUS", "CONFIRMED")
|
||||
for _, e := range events {
|
||||
path := e.Path
|
||||
if len(path) > 40 {
|
||||
path = "..." + path[len(path)-37:]
|
||||
}
|
||||
confirmed := "no"
|
||||
if e.DriftConfirmed {
|
||||
confirmed = "yes"
|
||||
}
|
||||
fmt.Fprintf(out, "%-20s %-20s %-40s %-10s %-10s\n", e.EventID, e.Host, path, e.Status, confirmed)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
var driftWatchCmd = &cobra.Command{
|
||||
Use: "watch",
|
||||
Short: "Stream drift events (ctrl-c to cancel)",
|
||||
Long: `Stream drift events from the lead's aggregated state. Default
|
||||
poll is 2s; override with --interval. Use --paths=<glob1>,<glob2> to
|
||||
filter. Uses iter.Seq2 (D-017) and signal.NotifyContext for ctrl-c
|
||||
(D-023).`,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
d, err := driftDetector()
|
||||
if err != nil {
|
||||
return fmt.Errorf("drift detector: %w", err)
|
||||
}
|
||||
ctx, cancel := signal.NotifyContext(cmd.Context(), os.Interrupt, syscall.SIGTERM)
|
||||
defer cancel()
|
||||
interval := driftWatchInterval
|
||||
if interval <= 0 {
|
||||
interval = 2 * time.Second
|
||||
}
|
||||
var specs []drift.PathSpec
|
||||
for _, p := range driftWatchPaths {
|
||||
if p == "" {
|
||||
continue
|
||||
}
|
||||
specs = append(specs, drift.PathSpec{Pattern: p, Interval: interval})
|
||||
}
|
||||
out := cmd.OutOrStdout()
|
||||
for e, err := range d.Watch(ctx, specs) {
|
||||
if err != nil {
|
||||
if errors.Is(err, context.Canceled) {
|
||||
return nil
|
||||
}
|
||||
fmt.Fprintf(out, "watch error: %v\n", err)
|
||||
continue
|
||||
}
|
||||
if jsonOutput {
|
||||
line, _ := json.Marshal(e)
|
||||
fmt.Fprintln(out, string(line))
|
||||
} else {
|
||||
fmt.Fprintf(out, "%s [%s] %s %s %s confirmed=%t\n", e.TS.Format(time.RFC3339), e.EventID, e.Host, e.Path, e.Status, e.DriftConfirmed)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
var driftAckCmd = &cobra.Command{
|
||||
Use: "acknowledge <peer> <path>",
|
||||
Short: "Record operator acknowledgment of drift on a peer",
|
||||
Long: `Record operator acknowledgment for the given path in
|
||||
drift-acknowledgments.json on the lead. Acknowledged drift no longer
|
||||
blocks txn apply for that namespace (R-020).`,
|
||||
Args: cobra.ExactArgs(2),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
peer := args[0]
|
||||
path := args[1]
|
||||
d, err := driftDetector()
|
||||
if err != nil {
|
||||
return fmt.Errorf("drift detector: %w", err)
|
||||
}
|
||||
if err := d.Acknowledge(cmd.Context(), peer, path); err != nil {
|
||||
return fmt.Errorf("acknowledge: %w", err)
|
||||
}
|
||||
printResult(fmt.Sprintf("✓ Acknowledged drift on %s for %s", peer, path), map[string]any{
|
||||
"peer": peer, "path": path, "status": "acknowledged",
|
||||
})
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
var driftRemediateCmd = &cobra.Command{
|
||||
Use: "remediate <peer> <path>",
|
||||
Short: "Trigger manual remediation of drift on a peer",
|
||||
Long: `Trigger orca-remediate.sh on the lead for the given path.
|
||||
--force bypasses the cooldown window (C4).`,
|
||||
Args: cobra.ExactArgs(2),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
peer := args[0]
|
||||
path := args[1]
|
||||
d, err := driftDetector()
|
||||
if err != nil {
|
||||
return fmt.Errorf("drift detector: %w", err)
|
||||
}
|
||||
if err := d.Remediate(cmd.Context(), peer, path, driftRemediateForce); err != nil {
|
||||
if errors.Is(err, drift.ErrCooldown) {
|
||||
printResult(fmt.Sprintf("✗ Remediation in cooldown for %s on %s (use --force to bypass)", path, peer), map[string]any{
|
||||
"peer": peer, "path": path, "status": "cooldown",
|
||||
})
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("remediate: %w", err)
|
||||
}
|
||||
printResult(fmt.Sprintf("✓ Remediated drift on %s for %s", peer, path), map[string]any{
|
||||
"peer": peer, "path": path, "status": "remediated", "force": driftRemediateForce,
|
||||
})
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
var driftConfigCmd = &cobra.Command{
|
||||
Use: "config",
|
||||
Short: "Show or validate the drift config",
|
||||
}
|
||||
|
||||
var driftConfigShowCmd = &cobra.Command{
|
||||
Use: "show",
|
||||
Short: "Show the current drift config",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
cfg, err := drift.LoadConfig(driftConfigPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("load config: %w", err)
|
||||
}
|
||||
if jsonOutput {
|
||||
return printJSON(cfg)
|
||||
}
|
||||
out := cmd.OutOrStdout()
|
||||
fmt.Fprintf(out, "Polling: enabled=%t default=%s max_peers=%d\n", cfg.Polling.Enabled, cfg.Polling.DefaultInterval, cfg.Polling.MaxConcurrentPeers)
|
||||
fmt.Fprintln(out, "Critical paths:")
|
||||
for _, p := range cfg.Paths.Critical {
|
||||
fmt.Fprintf(out, " [%s] %s (interval=%s, path_unit=%t)\n", p.Tier, p.Pattern, p.Interval, p.SystemdPathUnit)
|
||||
}
|
||||
fmt.Fprintln(out, "Standard paths:")
|
||||
for _, p := range cfg.Paths.Standard {
|
||||
fmt.Fprintf(out, " [%s] %s (interval=%s)\n", p.Tier, p.Pattern, p.Interval)
|
||||
}
|
||||
fmt.Fprintln(out, "Excluded paths:")
|
||||
for _, p := range cfg.Paths.Excluded {
|
||||
fmt.Fprintf(out, " %s\n", p)
|
||||
}
|
||||
fmt.Fprintf(out, "Remediation: auto=%t notify=%t\n", cfg.Remediate.Auto, cfg.Remediate.NotifyOnRemediation)
|
||||
if len(cfg.Remediate.AutoPaths) > 0 {
|
||||
fmt.Fprintln(out, " auto_paths:")
|
||||
for _, p := range cfg.Remediate.AutoPaths {
|
||||
fmt.Fprintf(out, " %s\n", p)
|
||||
}
|
||||
}
|
||||
if len(cfg.Remediate.RequireApproval) > 0 {
|
||||
fmt.Fprintln(out, " require_approval:")
|
||||
for _, p := range cfg.Remediate.RequireApproval {
|
||||
fmt.Fprintf(out, " %s\n", p)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
var driftConfigValidateCmd = &cobra.Command{
|
||||
Use: "validate",
|
||||
Short: "Validate the drift config",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
cfg, err := drift.LoadConfig(driftConfigPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("load config: %w", err)
|
||||
}
|
||||
if err := drift.ValidateConfig(cfg); err != nil {
|
||||
printResult(fmt.Sprintf("✗ Config invalid: %v", err), map[string]any{"valid": false, "error": err.Error()})
|
||||
return err
|
||||
}
|
||||
printResult("✓ Config valid", map[string]any{"valid": true})
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
// jobRestartCmd implements `orca job restart <name>` (REQ-113, D-235):
|
||||
// restart an allocation on its peer to pick up EnvironmentFile drift.
|
||||
var jobRestartCmd = &cobra.Command{
|
||||
Use: "restart <name>",
|
||||
Short: "Restart an allocation to pick up EnvironmentFile drift (REQ-113)",
|
||||
Long: `SSH to the peer running allocation <name> and run
|
||||
systemctl restart orca-alloc-<id>.service. This is the normal
|
||||
allocation lifecycle (NOT file-level remediation) and is triggered
|
||||
when /etc/orca/allocs/<id>/env drifts.`,
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
name := args[0]
|
||||
peer := jobRestartPeer
|
||||
if peer == "" {
|
||||
return fmt.Errorf("--peer is required for job restart")
|
||||
}
|
||||
transport, err := driftTransportFromCtx()
|
||||
if err != nil {
|
||||
return fmt.Errorf("ssh transport: %w", err)
|
||||
}
|
||||
unit := fmt.Sprintf("orca-alloc-%s.service", name)
|
||||
restartCmd := fmt.Sprintf("systemctl restart %s", shellQuoteDrift(unit))
|
||||
out, err := transport.Exec(cmd.Context(), peer, restartCmd)
|
||||
if err != nil {
|
||||
return fmt.Errorf("restart %s on %s: %w (output: %s)", unit, peer, err, string(out))
|
||||
}
|
||||
printResult(fmt.Sprintf("✓ Restarted %s on %s", unit, peer), map[string]any{
|
||||
"unit": unit, "peer": peer, "status": "restarted", "output": string(out),
|
||||
})
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
var jobRestartPeer string
|
||||
|
||||
func shellQuoteDrift(s string) string {
|
||||
return "'" + strings.ReplaceAll(s, "'", "'\\''") + "'"
|
||||
}
|
||||
|
||||
func init() {
|
||||
driftWatchCmd.Flags().DurationVar(&driftWatchInterval, "interval", 2*time.Second, "poll interval (default 2s)")
|
||||
driftWatchCmd.Flags().StringSliceVar(&driftWatchPaths, "paths", nil, "comma-separated glob patterns to watch (default: all)")
|
||||
driftShowCmd.Flags().StringVar(&driftShowPeer, "peer", "", "filter to a single peer host")
|
||||
driftRemediateCmd.Flags().BoolVar(&driftRemediateForce, "force", false, "bypass the cooldown window (C4)")
|
||||
driftConfigCmd.PersistentFlags().StringVar(&driftConfigPath, "config", "", "path to drift config JSON (default: built-in)")
|
||||
jobRestartCmd.Flags().StringVar(&jobRestartPeer, "peer", "", "peer address (host:port) running the allocation")
|
||||
|
||||
driftCmd.AddCommand(driftShowCmd)
|
||||
driftCmd.AddCommand(driftWatchCmd)
|
||||
driftCmd.AddCommand(driftAckCmd)
|
||||
driftCmd.AddCommand(driftRemediateCmd)
|
||||
driftCmd.AddCommand(driftConfigCmd)
|
||||
driftConfigCmd.AddCommand(driftConfigShowCmd)
|
||||
driftConfigCmd.AddCommand(driftConfigValidateCmd)
|
||||
rootCmd.AddCommand(driftCmd)
|
||||
|
||||
jobCmd.AddCommand(jobRestartCmd)
|
||||
}
|
||||
|
||||
// writeClusterDefaultDriftConfig writes the canonical default drift
|
||||
// config to the cluster state dir so the lead has a reference copy.
|
||||
// Best-effort; caller logs failures.
|
||||
func writeClusterDefaultDriftConfig() error {
|
||||
dir := filepath.Join(os.Getenv("ORCA_HOME"), "cluster", "state")
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return fmt.Errorf("mkdir cluster state: %w", err)
|
||||
}
|
||||
path := filepath.Join(dir, "drift.json")
|
||||
cfg := drift.DefaultConfig()
|
||||
raw, err := json.MarshalIndent(cfg, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal drift config: %w", err)
|
||||
}
|
||||
tmp := path + ".tmp"
|
||||
if err := os.WriteFile(tmp, raw, 0o644); err != nil {
|
||||
return fmt.Errorf("write drift config: %w", err)
|
||||
}
|
||||
if err := os.Rename(tmp, path); err != nil {
|
||||
return fmt.Errorf("rename drift config: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,631 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"iter"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"git.cloudinit.dev/coreci/orca/internal/drift"
|
||||
)
|
||||
|
||||
type mockDriftTransport struct {
|
||||
execOut []byte
|
||||
execErr error
|
||||
readOut []byte
|
||||
readErr error
|
||||
writes []mockDriftWrite
|
||||
execFn func(ctx context.Context, peer string, cmd string) ([]byte, error)
|
||||
execs []string
|
||||
}
|
||||
|
||||
type mockDriftWrite struct {
|
||||
peer string
|
||||
path string
|
||||
content []byte
|
||||
mode os.FileMode
|
||||
}
|
||||
|
||||
func (m *mockDriftTransport) Exec(ctx context.Context, peer string, cmd string) ([]byte, error) {
|
||||
if m.execFn != nil {
|
||||
return m.execFn(ctx, peer, cmd)
|
||||
}
|
||||
m.execs = append(m.execs, cmd)
|
||||
return m.execOut, m.execErr
|
||||
}
|
||||
|
||||
func (m *mockDriftTransport) WriteFileIdempotent(ctx context.Context, peer string, path string, content []byte, mode os.FileMode) (bool, error) {
|
||||
m.writes = append(m.writes, mockDriftWrite{peer, path, content, mode})
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (m *mockDriftTransport) ReadFile(ctx context.Context, peer string, path string) ([]byte, error) {
|
||||
return m.readOut, m.readErr
|
||||
}
|
||||
|
||||
// fakeDetector is a record-replay drift.Detector for CLI tests.
|
||||
type fakeDetector struct {
|
||||
aggEvents []drift.Event
|
||||
aggErr error
|
||||
remediateErr error
|
||||
ackWrites int
|
||||
remediateCalls []remediateCall
|
||||
}
|
||||
|
||||
type remediateCall struct {
|
||||
peer string
|
||||
path string
|
||||
force bool
|
||||
}
|
||||
|
||||
func (f *fakeDetector) Watch(ctx context.Context, paths []drift.PathSpec) iter.Seq2[drift.Event, error] {
|
||||
return func(yield func(drift.Event, error) bool) {
|
||||
for _, e := range f.aggEvents {
|
||||
if !yield(e, nil) {
|
||||
return
|
||||
}
|
||||
}
|
||||
<-ctx.Done()
|
||||
}
|
||||
}
|
||||
|
||||
func (f *fakeDetector) Aggregate(ctx context.Context, leadPeer string) ([]drift.Event, error) {
|
||||
return f.aggEvents, f.aggErr
|
||||
}
|
||||
|
||||
func (f *fakeDetector) Remediate(ctx context.Context, leadPeer, path string, force bool) error {
|
||||
f.remediateCalls = append(f.remediateCalls, remediateCall{leadPeer, path, force})
|
||||
return f.remediateErr
|
||||
}
|
||||
|
||||
func (f *fakeDetector) Acknowledge(ctx context.Context, leadPeer, path string) error {
|
||||
f.ackWrites++
|
||||
return nil
|
||||
}
|
||||
|
||||
func setupDriftCLITest(t *testing.T) string {
|
||||
t.Helper()
|
||||
home := t.TempDir()
|
||||
t.Setenv("ORCA_HOME", home)
|
||||
t.Setenv("ORCA_LEAD_STATE_DIR", filepath.Join(home, "state"))
|
||||
return home
|
||||
}
|
||||
|
||||
func TestDriftCmdRegistered(t *testing.T) {
|
||||
for _, c := range rootCmd.Commands() {
|
||||
if c.Name() == "drift" {
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatal("drift command not registered on root")
|
||||
}
|
||||
|
||||
func TestDriftSubcommandsRegistered(t *testing.T) {
|
||||
for _, c := range rootCmd.Commands() {
|
||||
if c.Name() != "drift" {
|
||||
continue
|
||||
}
|
||||
want := map[string]bool{
|
||||
"show": false,
|
||||
"watch": false,
|
||||
"acknowledge": false,
|
||||
"remediate": false,
|
||||
"config": false,
|
||||
}
|
||||
for _, sub := range c.Commands() {
|
||||
if _, ok := want[sub.Name()]; ok {
|
||||
want[sub.Name()] = true
|
||||
}
|
||||
}
|
||||
for name, found := range want {
|
||||
if !found {
|
||||
t.Errorf("drift subcommand %q not registered", name)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
t.Fatal("drift command not registered")
|
||||
}
|
||||
|
||||
func TestDriftConfigSubcommands(t *testing.T) {
|
||||
for _, c := range rootCmd.Commands() {
|
||||
if c.Name() != "drift" {
|
||||
continue
|
||||
}
|
||||
for _, sub := range c.Commands() {
|
||||
if sub.Name() != "config" {
|
||||
continue
|
||||
}
|
||||
want := map[string]bool{"show": false, "validate": false}
|
||||
for _, s := range sub.Commands() {
|
||||
if _, ok := want[s.Name()]; ok {
|
||||
want[s.Name()] = true
|
||||
}
|
||||
}
|
||||
for name, found := range want {
|
||||
if !found {
|
||||
t.Errorf("drift config subcommand %q not registered", name)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatal("drift config not registered")
|
||||
}
|
||||
|
||||
func TestJobRestartRegistered(t *testing.T) {
|
||||
for _, c := range rootCmd.Commands() {
|
||||
if c.Name() != "job" {
|
||||
continue
|
||||
}
|
||||
for _, sub := range c.Commands() {
|
||||
if sub.Name() == "restart" {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
t.Fatal("job restart not registered")
|
||||
}
|
||||
|
||||
func TestDriftShowEmpty(t *testing.T) {
|
||||
setupDriftCLITest(t)
|
||||
fd := &fakeDetector{aggEvents: nil}
|
||||
driftDetectorOverride = fd
|
||||
defer func() { driftDetectorOverride = nil }()
|
||||
|
||||
resetRootFlags(t)
|
||||
var buf bytes.Buffer
|
||||
rootCmd.SetOut(&buf)
|
||||
rootCmd.SetErr(&buf)
|
||||
rootCmd.SetArgs([]string{"drift", "show"})
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
t.Fatalf("drift show: %v", err)
|
||||
}
|
||||
if !bytesContains(buf.String(), "No drift events") {
|
||||
t.Errorf("empty show output: %s", buf.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriftShowTable(t *testing.T) {
|
||||
setupDriftCLITest(t)
|
||||
fd := &fakeDetector{aggEvents: []drift.Event{
|
||||
{EventID: "E1", Host: "peer1", Path: "/etc/traefik/dynamic/orca.yml", Status: drift.StatusModified, DriftConfirmed: true},
|
||||
}}
|
||||
driftDetectorOverride = fd
|
||||
defer func() { driftDetectorOverride = nil }()
|
||||
|
||||
resetRootFlags(t)
|
||||
var buf bytes.Buffer
|
||||
rootCmd.SetOut(&buf)
|
||||
rootCmd.SetErr(&buf)
|
||||
rootCmd.SetArgs([]string{"drift", "show"})
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
t.Fatalf("drift show: %v", err)
|
||||
}
|
||||
out := buf.String()
|
||||
for _, want := range []string{"E1", "peer1", "modified"} {
|
||||
if !bytesContains(out, want) {
|
||||
t.Errorf("output missing %q: %s", want, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriftShowJSON(t *testing.T) {
|
||||
setupDriftCLITest(t)
|
||||
fd := &fakeDetector{aggEvents: []drift.Event{
|
||||
{EventID: "E1", Host: "peer1", Path: "/etc/x", Status: drift.StatusCreated, DriftConfirmed: true},
|
||||
}}
|
||||
driftDetectorOverride = fd
|
||||
defer func() { driftDetectorOverride = nil }()
|
||||
|
||||
resetRootFlags(t)
|
||||
_ = rootCmd.PersistentFlags().Set("json", "true")
|
||||
var buf bytes.Buffer
|
||||
rootCmd.SetOut(&buf)
|
||||
rootCmd.SetErr(&buf)
|
||||
rootCmd.SetArgs([]string{"drift", "show"})
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
t.Fatalf("drift show --json: %v", err)
|
||||
}
|
||||
if !bytesContains(buf.String(), `"event_id"`) {
|
||||
t.Errorf("json output missing event_id: %s", buf.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriftShowPeerFilter(t *testing.T) {
|
||||
setupDriftCLITest(t)
|
||||
fd := &fakeDetector{aggEvents: []drift.Event{
|
||||
{EventID: "E1", Host: "peer1", Path: "/a", Status: drift.StatusModified, DriftConfirmed: true},
|
||||
{EventID: "E2", Host: "peer2", Path: "/b", Status: drift.StatusModified, DriftConfirmed: true},
|
||||
}}
|
||||
driftDetectorOverride = fd
|
||||
defer func() { driftDetectorOverride = nil }()
|
||||
|
||||
resetRootFlags(t)
|
||||
var buf bytes.Buffer
|
||||
rootCmd.SetOut(&buf)
|
||||
rootCmd.SetErr(&buf)
|
||||
rootCmd.SetArgs([]string{"drift", "show", "--peer", "peer1"})
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
t.Fatalf("drift show --peer: %v", err)
|
||||
}
|
||||
out := buf.String()
|
||||
if !bytesContains(out, "E1") {
|
||||
t.Errorf("filtered output should have E1: %s", out)
|
||||
}
|
||||
if bytesContains(out, "E2") {
|
||||
t.Errorf("filtered output should NOT have E2: %s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriftWatchStreamsAndCancels(t *testing.T) {
|
||||
setupDriftCLITest(t)
|
||||
// Use a fakeDetector whose Watch yields one event then blocks on
|
||||
// ctx so the stream terminates when ctrl-c (signal.NotifyContext)
|
||||
// cancels. We simulate the cancel by constructing a fake that yields
|
||||
// then returns when the consumer stops pulling OR ctx is cancelled.
|
||||
fd := &drainingFakeDetector{events: []drift.Event{
|
||||
{EventID: "E1", Host: "p", Path: "/etc/x", Status: drift.StatusModified, DriftConfirmed: true},
|
||||
}}
|
||||
driftDetectorOverride = fd
|
||||
defer func() { driftDetectorOverride = nil }()
|
||||
|
||||
resetRootFlags(t)
|
||||
var buf bytes.Buffer
|
||||
rootCmd.SetOut(&buf)
|
||||
rootCmd.SetErr(&buf)
|
||||
rootCmd.SetArgs([]string{"drift", "watch", "--interval", "10ms"})
|
||||
// Inject a context that auto-cancels after the events drain so
|
||||
// the watch loop exits without polluting rootCmd's context (which
|
||||
// is shared across tests). We use PersistentPreRunE's context by
|
||||
// overriding it here and restoring after.
|
||||
origCtx := rootCmd.Context()
|
||||
ctx, cancel := context.WithCancel(origCtx)
|
||||
defer cancel()
|
||||
rootCmd.SetContext(ctx)
|
||||
fd.cancelAfterYield = cancel
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
t.Fatalf("drift watch: %v", err)
|
||||
}
|
||||
// Restore rootCmd context for subsequent tests.
|
||||
rootCmd.SetContext(origCtx)
|
||||
if !bytesContains(buf.String(), "E1") {
|
||||
t.Errorf("watch output missing E1: %s", buf.String())
|
||||
}
|
||||
}
|
||||
|
||||
// drainingFakeDetector yields the events then cancels the provided
|
||||
// cancel func (so the watch loop's signal.NotifyContext ctx is
|
||||
// cancelled and the stream terminates cleanly).
|
||||
type drainingFakeDetector struct {
|
||||
events []drift.Event
|
||||
cancelAfterYield context.CancelFunc
|
||||
remediateCalls []remediateCall
|
||||
ackWrites int
|
||||
}
|
||||
|
||||
func (d *drainingFakeDetector) Watch(ctx context.Context, paths []drift.PathSpec) iter.Seq2[drift.Event, error] {
|
||||
return func(yield func(drift.Event, error) bool) {
|
||||
for _, e := range d.events {
|
||||
if !yield(e, nil) {
|
||||
return
|
||||
}
|
||||
}
|
||||
if d.cancelAfterYield != nil {
|
||||
d.cancelAfterYield()
|
||||
}
|
||||
<-ctx.Done()
|
||||
}
|
||||
}
|
||||
func (d *drainingFakeDetector) Aggregate(ctx context.Context, leadPeer string) ([]drift.Event, error) {
|
||||
return d.events, nil
|
||||
}
|
||||
func (d *drainingFakeDetector) Remediate(ctx context.Context, leadPeer, path string, force bool) error {
|
||||
d.remediateCalls = append(d.remediateCalls, remediateCall{leadPeer, path, force})
|
||||
return nil
|
||||
}
|
||||
func (d *drainingFakeDetector) Acknowledge(ctx context.Context, leadPeer, path string) error {
|
||||
d.ackWrites++
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestDriftAcknowledge(t *testing.T) {
|
||||
setupDriftCLITest(t)
|
||||
fd := &fakeDetector{}
|
||||
driftDetectorOverride = fd
|
||||
defer func() { driftDetectorOverride = nil }()
|
||||
|
||||
resetRootFlags(t)
|
||||
var buf bytes.Buffer
|
||||
rootCmd.SetOut(&buf)
|
||||
rootCmd.SetErr(&buf)
|
||||
rootCmd.SetArgs([]string{"drift", "acknowledge", "peer1", "/etc/traefik/dynamic/orca.yml"})
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
t.Fatalf("drift acknowledge: %v", err)
|
||||
}
|
||||
if fd.ackWrites != 1 {
|
||||
t.Errorf("ackWrites = %d, want 1", fd.ackWrites)
|
||||
}
|
||||
if !bytesContains(buf.String(), "Acknowledged") {
|
||||
t.Errorf("output missing Acknowledged: %s", buf.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriftRemediate(t *testing.T) {
|
||||
setupDriftCLITest(t)
|
||||
fd := &fakeDetector{}
|
||||
driftDetectorOverride = fd
|
||||
defer func() { driftDetectorOverride = nil }()
|
||||
|
||||
resetRootFlags(t)
|
||||
var buf bytes.Buffer
|
||||
rootCmd.SetOut(&buf)
|
||||
rootCmd.SetErr(&buf)
|
||||
rootCmd.SetArgs([]string{"drift", "remediate", "peer1", "/etc/x"})
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
t.Fatalf("drift remediate: %v", err)
|
||||
}
|
||||
if len(fd.remediateCalls) != 1 {
|
||||
t.Fatalf("remediateCalls = %d, want 1", len(fd.remediateCalls))
|
||||
}
|
||||
if fd.remediateCalls[0].peer != "peer1" || fd.remediateCalls[0].path != "/etc/x" {
|
||||
t.Errorf("remediate call: %+v", fd.remediateCalls[0])
|
||||
}
|
||||
if fd.remediateCalls[0].force {
|
||||
t.Errorf("force should be false without --force")
|
||||
}
|
||||
if !bytesContains(buf.String(), "Remediated") {
|
||||
t.Errorf("output missing Remediated: %s", buf.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriftRemediateForce(t *testing.T) {
|
||||
setupDriftCLITest(t)
|
||||
fd := &fakeDetector{}
|
||||
driftDetectorOverride = fd
|
||||
defer func() { driftDetectorOverride = nil }()
|
||||
|
||||
resetRootFlags(t)
|
||||
var buf bytes.Buffer
|
||||
rootCmd.SetOut(&buf)
|
||||
rootCmd.SetErr(&buf)
|
||||
rootCmd.SetArgs([]string{"drift", "remediate", "peer1", "/etc/x", "--force"})
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
t.Fatalf("drift remediate --force: %v", err)
|
||||
}
|
||||
if len(fd.remediateCalls) != 1 {
|
||||
t.Fatalf("remediateCalls = %d, want 1", len(fd.remediateCalls))
|
||||
}
|
||||
if !fd.remediateCalls[0].force {
|
||||
t.Errorf("force should be true with --force")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriftRemediateCooldown(t *testing.T) {
|
||||
setupDriftCLITest(t)
|
||||
fd := &fakeDetector{remediateErr: drift.ErrCooldown}
|
||||
driftDetectorOverride = fd
|
||||
defer func() { driftDetectorOverride = nil }()
|
||||
|
||||
resetRootFlags(t)
|
||||
var buf bytes.Buffer
|
||||
rootCmd.SetOut(&buf)
|
||||
rootCmd.SetErr(&buf)
|
||||
rootCmd.SetArgs([]string{"drift", "remediate", "peer1", "/etc/x"})
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
t.Fatalf("drift remediate cooldown: %v", err)
|
||||
}
|
||||
if !bytesContains(buf.String(), "cooldown") {
|
||||
t.Errorf("output should mention cooldown: %s", buf.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriftConfigShow(t *testing.T) {
|
||||
setupDriftCLITest(t)
|
||||
resetRootFlags(t)
|
||||
var buf bytes.Buffer
|
||||
rootCmd.SetOut(&buf)
|
||||
rootCmd.SetErr(&buf)
|
||||
rootCmd.SetArgs([]string{"drift", "config", "show"})
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
t.Fatalf("drift config show: %v", err)
|
||||
}
|
||||
out := buf.String()
|
||||
for _, want := range []string{"Polling:", "Critical paths:", "Standard paths:", "Excluded paths:", "Remediation:"} {
|
||||
if !bytesContains(out, want) {
|
||||
t.Errorf("config show missing %q: %s", want, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriftConfigValidate(t *testing.T) {
|
||||
setupDriftCLITest(t)
|
||||
resetRootFlags(t)
|
||||
var buf bytes.Buffer
|
||||
rootCmd.SetOut(&buf)
|
||||
rootCmd.SetErr(&buf)
|
||||
rootCmd.SetArgs([]string{"drift", "config", "validate"})
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
t.Fatalf("drift config validate: %v", err)
|
||||
}
|
||||
if !bytesContains(buf.String(), "valid") {
|
||||
t.Errorf("output missing 'valid': %s", buf.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriftConfigValidateFails(t *testing.T) {
|
||||
setupDriftCLITest(t)
|
||||
cfgPath := filepath.Join(t.TempDir(), "drift.json")
|
||||
bad := `{"polling":{"enabled":true,"default_interval":60000000000,"max_concurrent_peers":4},"paths":{"critical":[{"tier":"critical","pattern":"","interval":5000000000}]},"remediate":{"auto":true}}`
|
||||
if err := os.WriteFile(cfgPath, []byte(bad), 0o644); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
resetRootFlags(t)
|
||||
var buf bytes.Buffer
|
||||
rootCmd.SetOut(&buf)
|
||||
rootCmd.SetErr(&buf)
|
||||
rootCmd.SetArgs([]string{"drift", "config", "validate", "--config", cfgPath})
|
||||
err := rootCmd.Execute()
|
||||
if err == nil {
|
||||
t.Fatal("expected validate error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestJobRestartRequiresPeer(t *testing.T) {
|
||||
setupDriftCLITest(t)
|
||||
resetRootFlags(t)
|
||||
var buf bytes.Buffer
|
||||
rootCmd.SetOut(&buf)
|
||||
rootCmd.SetErr(&buf)
|
||||
rootCmd.SetArgs([]string{"job", "restart", "alloc1"})
|
||||
err := rootCmd.Execute()
|
||||
if err == nil {
|
||||
t.Fatal("expected error for missing --peer")
|
||||
}
|
||||
}
|
||||
|
||||
func TestJobRestartExecs(t *testing.T) {
|
||||
setupDriftCLITest(t)
|
||||
mt := &mockDriftTransport{execOut: []byte("restarted")}
|
||||
driftTransportOverride = mt
|
||||
defer func() { driftTransportOverride = nil }()
|
||||
|
||||
resetRootFlags(t)
|
||||
var buf bytes.Buffer
|
||||
rootCmd.SetOut(&buf)
|
||||
rootCmd.SetErr(&buf)
|
||||
rootCmd.SetArgs([]string{"job", "restart", "alloc1", "--peer", "peer1:22"})
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
t.Fatalf("job restart: %v", err)
|
||||
}
|
||||
if len(mt.execs) != 1 {
|
||||
t.Fatalf("execs = %d, want 1", len(mt.execs))
|
||||
}
|
||||
if !bytesContains(mt.execs[0], "orca-alloc-alloc1.service") {
|
||||
t.Errorf("restart cmd missing: %s", mt.execs[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestJobRestartTransientError(t *testing.T) {
|
||||
setupDriftCLITest(t)
|
||||
mt := &mockDriftTransport{execErr: fmt.Errorf("connection refused")}
|
||||
driftTransportOverride = mt
|
||||
defer func() { driftTransportOverride = nil }()
|
||||
|
||||
resetRootFlags(t)
|
||||
var buf bytes.Buffer
|
||||
rootCmd.SetOut(&buf)
|
||||
rootCmd.SetErr(&buf)
|
||||
rootCmd.SetArgs([]string{"job", "restart", "alloc1", "--peer", "peer1:22"})
|
||||
err := rootCmd.Execute()
|
||||
if err == nil {
|
||||
t.Fatal("expected error for exec failure")
|
||||
}
|
||||
if !errors.Is(err, err) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPeerSetupCmdRegistered(t *testing.T) {
|
||||
for _, c := range rootCmd.Commands() {
|
||||
if c.Name() == "peer-setup" {
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatal("peer-setup command not registered")
|
||||
}
|
||||
|
||||
func TestPeerSetupCreatesUserAndDir(t *testing.T) {
|
||||
setupDriftCLITest(t)
|
||||
mt := &mockDriftTransport{execOut: []byte("ext4")}
|
||||
peerSetupTransportOverride = mt
|
||||
defer func() { peerSetupTransportOverride = nil }()
|
||||
|
||||
resetRootFlags(t)
|
||||
var buf bytes.Buffer
|
||||
rootCmd.SetOut(&buf)
|
||||
rootCmd.SetErr(&buf)
|
||||
rootCmd.SetArgs([]string{"peer-setup", "peer1:22"})
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
t.Fatalf("peer-setup: %v", err)
|
||||
}
|
||||
if len(mt.execs) < 2 {
|
||||
t.Fatalf("execs = %d, want >= 2", len(mt.execs))
|
||||
}
|
||||
useraddSeen := false
|
||||
mkdirSeen := false
|
||||
statSeen := false
|
||||
for _, c := range mt.execs {
|
||||
if bytesContains(c, "useradd -r orca") {
|
||||
useraddSeen = true
|
||||
}
|
||||
if bytesContains(c, "mkdir -p /etc/orca/state/drift-events") {
|
||||
mkdirSeen = true
|
||||
}
|
||||
if bytesContains(c, "stat -f") {
|
||||
statSeen = true
|
||||
}
|
||||
}
|
||||
if !useraddSeen {
|
||||
t.Errorf("useradd not run")
|
||||
}
|
||||
if !mkdirSeen {
|
||||
t.Errorf("mkdir drift-events not run")
|
||||
}
|
||||
if !statSeen {
|
||||
t.Errorf("stat (NFS detect) not run")
|
||||
}
|
||||
if !bytesContains(buf.String(), "nfs=false") {
|
||||
t.Errorf("output should report nfs=false: %s", buf.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestPeerSetupNoOrcaUser(t *testing.T) {
|
||||
setupDriftCLITest(t)
|
||||
mt := &mockDriftTransport{execOut: []byte("ext4")}
|
||||
peerSetupTransportOverride = mt
|
||||
defer func() { peerSetupTransportOverride = nil }()
|
||||
|
||||
resetRootFlags(t)
|
||||
var buf bytes.Buffer
|
||||
rootCmd.SetOut(&buf)
|
||||
rootCmd.SetErr(&buf)
|
||||
rootCmd.SetArgs([]string{"peer-setup", "peer1:22", "--no-orca-user"})
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
t.Fatalf("peer-setup --no-orca-user: %v", err)
|
||||
}
|
||||
useraddSeen := false
|
||||
for _, c := range mt.execs {
|
||||
if bytesContains(c, "useradd -r orca") {
|
||||
useraddSeen = true
|
||||
}
|
||||
}
|
||||
if useraddSeen {
|
||||
t.Errorf("useradd should NOT run with --no-orca-user")
|
||||
}
|
||||
if !bytesContains(buf.String(), "user=false") {
|
||||
t.Errorf("output should report user=false: %s", buf.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestPeerSetupDetectsNFS(t *testing.T) {
|
||||
setupDriftCLITest(t)
|
||||
mt := &mockDriftTransport{execOut: []byte("nfs4")}
|
||||
peerSetupTransportOverride = mt
|
||||
defer func() { peerSetupTransportOverride = nil }()
|
||||
|
||||
resetRootFlags(t)
|
||||
var buf bytes.Buffer
|
||||
rootCmd.SetOut(&buf)
|
||||
rootCmd.SetErr(&buf)
|
||||
rootCmd.SetArgs([]string{"peer-setup", "peer1:22"})
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
t.Fatalf("peer-setup: %v", err)
|
||||
}
|
||||
if !bytesContains(buf.String(), "nfs=true") {
|
||||
t.Errorf("output should report nfs=true: %s", buf.String())
|
||||
}
|
||||
}
|
||||
@@ -49,6 +49,13 @@ func resetCommandFlags() {
|
||||
txnApplyNamespace = ""
|
||||
txnApplyTimeout = 5 * time.Minute
|
||||
txnApplyLead, txnRollbackLead = "", ""
|
||||
driftWatchInterval = 2 * time.Second
|
||||
driftWatchPaths = nil
|
||||
driftShowPeer = ""
|
||||
driftConfigPath = ""
|
||||
driftRemediateForce = false
|
||||
jobRestartPeer = ""
|
||||
peerSetupNoOrcaUser = false
|
||||
resetNSFlags()
|
||||
// Reset per-command output writers so tests that polluted them
|
||||
// (e.g. daemon tests calling cmd.SetOut(&buf)) don't leak into
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
// Package cli: peer_setup.go implements the orca system-user setup and
|
||||
// NFS detection on peers (P10b-T8/T9, v0.11, REQ-111, REQ-112/D-233).
|
||||
//
|
||||
// `orca node join` now also creates the `orca` system user on the peer
|
||||
// (so the systemd Path-unit services, which run as User=orca, have a
|
||||
// uid to run as). It also detects whether /etc/orca is on an NFS mount
|
||||
// and, when it is, skips emitting Path units for paths under /etc/orca
|
||||
// (falling back to polling for those paths — systemd Path units on NFS
|
||||
// are unreliable because inotify does not fire reliably over NFS).
|
||||
//
|
||||
// The setup is idempotent: re-running on an already-configured peer is
|
||||
// a no-op. A --no-orca-user flag skips user creation (for environments
|
||||
// with existing service accounts).
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var peerSetupNoOrcaUser bool
|
||||
|
||||
// peerSetupTransport is the SSH surface the peer-setup code needs. It
|
||||
// mirrors driftTransport; tests substitute a mock.
|
||||
type peerSetupTransport interface {
|
||||
Exec(ctx context.Context, peer string, cmd string) ([]byte, error)
|
||||
}
|
||||
|
||||
// peerSetupTransportOverride is the test seam.
|
||||
var peerSetupTransportOverride peerSetupTransport
|
||||
|
||||
func peerSetupTransportFromCtx() (peerSetupTransport, error) {
|
||||
if peerSetupTransportOverride != nil {
|
||||
return peerSetupTransportOverride, nil
|
||||
}
|
||||
t, err := driftTransportFromCtx()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// PeerSetupResult records what the peer setup did.
|
||||
type PeerSetupResult struct {
|
||||
UserCreated bool `json:"user_created"`
|
||||
EventsDir string `json:"events_dir"`
|
||||
NFSOnOrca bool `json:"nfs_on_orca"`
|
||||
NFSMsg string `json:"nfs_msg,omitempty"`
|
||||
}
|
||||
|
||||
// setupOrcaUser runs the idempotent `useradd -r orca` and creates the
|
||||
// drift-events directory owned by orca:orca on the peer. Returns the
|
||||
// result; a transient SSH failure returns the error (no partial state).
|
||||
func setupOrcaUser(ctx context.Context, transport peerSetupTransport, peer string) (*PeerSetupResult, error) {
|
||||
if peer == "" {
|
||||
return nil, fmt.Errorf("peer setup: peer is empty")
|
||||
}
|
||||
res := &PeerSetupResult{EventsDir: "/etc/orca/state/drift-events"}
|
||||
|
||||
if !peerSetupNoOrcaUser {
|
||||
useraddCmd := "useradd -r orca -s /usr/sbin/nologin 2>/dev/null || true"
|
||||
if _, err := transport.Exec(ctx, peer, useraddCmd); err != nil {
|
||||
return nil, fmt.Errorf("peer setup: useradd: %w", err)
|
||||
}
|
||||
res.UserCreated = true
|
||||
}
|
||||
|
||||
mkdirCmd := fmt.Sprintf("mkdir -p %s && %s", res.EventsDir, chownDriftEvents(res.EventsDir))
|
||||
if _, err := transport.Exec(ctx, peer, mkdirCmd); err != nil {
|
||||
return nil, fmt.Errorf("peer setup: mkdir drift-events: %w", err)
|
||||
}
|
||||
|
||||
nfs, msg := detectNFS(ctx, transport, peer, "/etc/orca")
|
||||
res.NFSOnOrca = nfs
|
||||
res.NFSMsg = msg
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// chownDriftEvents returns the chown command for the drift-events dir.
|
||||
// When --no-orca-user is set the orca user may not exist; chown only
|
||||
// when the user was created.
|
||||
func chownDriftEvents(dir string) string {
|
||||
if peerSetupNoOrcaUser {
|
||||
return "true"
|
||||
}
|
||||
return fmt.Sprintf("chown orca:orca %s 2>/dev/null || true", dir)
|
||||
}
|
||||
|
||||
// detectNFS checks whether the given path is on an NFS mount by running
|
||||
// `stat -f -c %T <path>` on the peer. When the fs type contains "nfs"
|
||||
// it returns (true, msg). Best-effort: a stat failure returns
|
||||
// (false, "stat unavailable").
|
||||
func detectNFS(ctx context.Context, transport peerSetupTransport, peer, path string) (bool, string) {
|
||||
out, err := transport.Exec(ctx, peer, fmt.Sprintf("stat -f -c %%T %s 2>/dev/null || echo unknown", shellQuoteDrift(path)))
|
||||
if err != nil {
|
||||
return false, "stat unavailable: " + err.Error()
|
||||
}
|
||||
fsType := strings.TrimSpace(string(out))
|
||||
if strings.Contains(fsType, "nfs") {
|
||||
return true, fmt.Sprintf("%s is on NFS (%s); skipping Path units for /etc/orca paths", path, fsType)
|
||||
}
|
||||
return false, fsType
|
||||
}
|
||||
|
||||
var peerSetupCmd = &cobra.Command{
|
||||
Use: "peer-setup <peer>",
|
||||
Short: "Create the orca system user + drift-events dir on a peer (REQ-111)",
|
||||
Long: `SSH to <peer> and idempotently create the orca system user
|
||||
(useradd -r orca -s /usr/sbin/nologin) and /etc/orca/state/drift-events/
|
||||
owned by orca:orca. Also detects NFS on /etc/orca (REQ-112/D-233) and
|
||||
logs a warning when /etc/orca is on NFS (Path units are skipped for
|
||||
those paths in that case). Use --no-orca-user to skip user creation
|
||||
(for environments with an existing service account).`,
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
peer := args[0]
|
||||
transport, err := peerSetupTransportFromCtx()
|
||||
if err != nil {
|
||||
return fmt.Errorf("ssh transport: %w", err)
|
||||
}
|
||||
res, err := setupOrcaUser(cmd.Context(), transport, peer)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
printResult(fmt.Sprintf("✓ Peer %s set up (user=%t, nfs=%t)", peer, res.UserCreated, res.NFSOnOrca), res)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
func init() {
|
||||
peerSetupCmd.Flags().BoolVar(&peerSetupNoOrcaUser, "no-orca-user", false, "skip orca system user creation (env has existing service account)")
|
||||
rootCmd.AddCommand(peerSetupCmd)
|
||||
}
|
||||
@@ -0,0 +1,573 @@
|
||||
// Package drift implements orca's drift detection subsystem
|
||||
// (P10b, v0.11 milestone; R-018/R-019/R-020, REQ-103..REQ-113).
|
||||
//
|
||||
// The model is poll-based with a systemd Path-unit fast path for
|
||||
// critical files:
|
||||
//
|
||||
// - The Detector interface exposes four operations: Watch (stream
|
||||
// events via iter.Seq2[Event, error], D-017), Aggregate (read the
|
||||
// lead-side aggregated drift state), Remediate (trigger the
|
||||
// peer-side applier via orca-remediate.sh, with cooldown-on-success
|
||||
// per C4), and Acknowledge (record operator ack).
|
||||
// - DefaultDetector implements Detector against the SSH-push
|
||||
// transport (the same *sshpush.Transport used by txn / emitter).
|
||||
// - Config encodes the tiered cadence (critical 5s + systemd Path
|
||||
// units, standard 30s polling, default 60s) plus the remediation
|
||||
// policy (auto paths, require-approval paths, notify-on-remediate).
|
||||
//
|
||||
// The package never logs key material. slog calls carry only metadata.
|
||||
package drift
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"iter"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Status string
|
||||
|
||||
const (
|
||||
StatusModified Status = "modified"
|
||||
StatusDeleted Status = "deleted"
|
||||
StatusCreated Status = "created"
|
||||
)
|
||||
|
||||
type Action string
|
||||
|
||||
const (
|
||||
ActionAutoRemediated Action = "auto_remediated"
|
||||
ActionReported Action = "reported"
|
||||
ActionAcknowledged Action = "acknowledged"
|
||||
)
|
||||
|
||||
type ActionResult string
|
||||
|
||||
const (
|
||||
ActionResultSuccess ActionResult = "success"
|
||||
ActionResultFailed ActionResult = "failed"
|
||||
ActionResultSkipped ActionResult = "skipped"
|
||||
)
|
||||
|
||||
type Tier string
|
||||
|
||||
const (
|
||||
TierCritical Tier = "critical"
|
||||
TierStandard Tier = "standard"
|
||||
)
|
||||
|
||||
type Event struct {
|
||||
EventID string `json:"event_id"`
|
||||
TS time.Time `json:"ts"`
|
||||
Host string `json:"host"`
|
||||
Path string `json:"path"`
|
||||
Status Status `json:"status"`
|
||||
NewSHA256 string `json:"new_sha256"`
|
||||
LatestTxn string `json:"latest_txn"`
|
||||
ExpectedSHA256 string `json:"expected_sha256"`
|
||||
DriftConfirmed bool `json:"drift_confirmed"`
|
||||
Action Action `json:"action"`
|
||||
ActionResult ActionResult `json:"action_result"`
|
||||
OperatorID string `json:"operator_id"`
|
||||
}
|
||||
|
||||
type PathSpec struct {
|
||||
Tier Tier `json:"tier"`
|
||||
Pattern string `json:"pattern"`
|
||||
Interval time.Duration `json:"interval"`
|
||||
SystemdPathUnit bool `json:"systemd_path_unit"`
|
||||
}
|
||||
|
||||
type RemediationPolicy struct {
|
||||
Auto bool `json:"auto"`
|
||||
AutoPaths []string `json:"auto_paths"`
|
||||
RequireApproval []string `json:"require_approval"`
|
||||
NotifyOnRemediation bool `json:"notify_on_remediation"`
|
||||
}
|
||||
|
||||
type PollingConfig struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
DefaultInterval time.Duration `json:"default_interval"`
|
||||
MaxConcurrentPeers int `json:"max_concurrent_peers"`
|
||||
}
|
||||
|
||||
type PathsConfig struct {
|
||||
Critical []PathSpec `json:"critical"`
|
||||
Standard []PathSpec `json:"standard"`
|
||||
Excluded []string `json:"excluded"`
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
Polling PollingConfig `json:"polling"`
|
||||
Paths PathsConfig `json:"paths"`
|
||||
Remediate RemediationPolicy `json:"remediate"`
|
||||
}
|
||||
|
||||
type Detector interface {
|
||||
Watch(ctx context.Context, paths []PathSpec) iter.Seq2[Event, error]
|
||||
Aggregate(ctx context.Context, leadPeer string) ([]Event, error)
|
||||
Remediate(ctx context.Context, leadPeer, path string, force bool) error
|
||||
Acknowledge(ctx context.Context, leadPeer, path string) error
|
||||
}
|
||||
|
||||
type Transport interface {
|
||||
Exec(ctx context.Context, peer string, cmd string) ([]byte, error)
|
||||
WriteFileIdempotent(ctx context.Context, peer string, path string, content []byte, mode os.FileMode) (bool, error)
|
||||
ReadFile(ctx context.Context, peer string, path string) ([]byte, error)
|
||||
}
|
||||
|
||||
func LeadStateDir() string {
|
||||
if p := os.Getenv("ORCA_LEAD_STATE_DIR"); p != "" {
|
||||
return p
|
||||
}
|
||||
return "/etc/orca/state"
|
||||
}
|
||||
|
||||
func AggregatedPath() string {
|
||||
return filepath.Join(LeadStateDir(), "drift-events-aggregated.json")
|
||||
}
|
||||
|
||||
func AcknowledgmentsPath() string {
|
||||
return filepath.Join(LeadStateDir(), "drift-acknowledgments.json")
|
||||
}
|
||||
|
||||
func RemediatorPath() string {
|
||||
return "/usr/local/sbin/orca-remediate.sh"
|
||||
}
|
||||
|
||||
type DefaultDetector struct {
|
||||
transport Transport
|
||||
cooldown time.Duration
|
||||
}
|
||||
|
||||
func NewDefaultDetector(t Transport) *DefaultDetector {
|
||||
return &DefaultDetector{transport: t, cooldown: 5 * time.Minute}
|
||||
}
|
||||
|
||||
func (d *DefaultDetector) SetCooldown(dur time.Duration) {
|
||||
if dur > 0 {
|
||||
d.cooldown = dur
|
||||
}
|
||||
}
|
||||
|
||||
type aggregatedDoc struct {
|
||||
TS time.Time `json:"ts"`
|
||||
Events []Event `json:"events"`
|
||||
}
|
||||
|
||||
func (d *DefaultDetector) Watch(ctx context.Context, paths []PathSpec) iter.Seq2[Event, error] {
|
||||
interval := defaultWatchInterval(paths)
|
||||
if interval <= 0 {
|
||||
interval = 2 * time.Second
|
||||
}
|
||||
return func(yield func(Event, error) bool) {
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
seen := make(map[string]bool)
|
||||
for {
|
||||
events, err := d.Aggregate(ctx, "")
|
||||
if err != nil {
|
||||
if !yield(Event{}, err) {
|
||||
return
|
||||
}
|
||||
} else {
|
||||
for _, e := range events {
|
||||
if !matchesPaths(e.Path, paths) {
|
||||
continue
|
||||
}
|
||||
key := e.EventID
|
||||
if key == "" {
|
||||
key = e.Host + "|" + e.Path + "|" + e.TS.Format(time.RFC3339Nano)
|
||||
}
|
||||
if seen[key] {
|
||||
continue
|
||||
}
|
||||
seen[key] = true
|
||||
if !yield(e, nil) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (d *DefaultDetector) Aggregate(ctx context.Context, leadPeer string) ([]Event, error) {
|
||||
var raw []byte
|
||||
var err error
|
||||
if leadPeer == "" {
|
||||
raw, err = os.ReadFile(AggregatedPath())
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, fmt.Errorf("drift: read aggregated: %w", err)
|
||||
}
|
||||
} else {
|
||||
raw, err = d.transport.ReadFile(ctx, leadPeer, AggregatedPath())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("drift: read aggregated from %s: %w", leadPeer, err)
|
||||
}
|
||||
}
|
||||
if len(strings.TrimSpace(string(raw))) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
var doc aggregatedDoc
|
||||
if err := json.Unmarshal(raw, &doc); err != nil {
|
||||
return nil, fmt.Errorf("drift: parse aggregated: %w", err)
|
||||
}
|
||||
return doc.Events, nil
|
||||
}
|
||||
|
||||
func (d *DefaultDetector) Remediate(ctx context.Context, leadPeer, path string, force bool) error {
|
||||
if leadPeer == "" {
|
||||
return fmt.Errorf("drift: remediate requires a lead peer")
|
||||
}
|
||||
if !force {
|
||||
if d.inCooldown(path) {
|
||||
slog.Info("drift remediate skipped (cooldown)", "path", path, "peer", leadPeer)
|
||||
return ErrCooldown
|
||||
}
|
||||
}
|
||||
cmd := fmt.Sprintf("bash %s %s %s %s", shellQuote(RemediatorPath()), shellQuote(leadPeer), shellQuote(""), shellQuote(path))
|
||||
out, err := d.transport.Exec(ctx, leadPeer, cmd)
|
||||
if err != nil {
|
||||
if isTransientSSH(err) {
|
||||
slog.Warn("drift remediate transient failure (no cooldown)", "path", path, "peer", leadPeer, "error", err)
|
||||
return err
|
||||
}
|
||||
return fmt.Errorf("drift: remediate %s on %s: %w (output: %s)", path, leadPeer, err, string(out))
|
||||
}
|
||||
if !force {
|
||||
d.markCooldown(path)
|
||||
}
|
||||
slog.Info("drift remediate ok", "path", path, "peer", leadPeer, "output", string(out))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *DefaultDetector) Acknowledge(ctx context.Context, leadPeer, path string) error {
|
||||
if leadPeer == "" {
|
||||
return fmt.Errorf("drift: acknowledge requires a lead peer")
|
||||
}
|
||||
entry := map[string]any{
|
||||
"ts": time.Now().UTC().Format(time.RFC3339Nano),
|
||||
"path": path,
|
||||
"operator": os.Getenv("ORCA_OPERATOR_ID"),
|
||||
}
|
||||
raw, err := json.Marshal(entry)
|
||||
if err != nil {
|
||||
return fmt.Errorf("drift: marshal ack: %w", err)
|
||||
}
|
||||
if _, err := d.transport.WriteFileIdempotent(ctx, leadPeer, AcknowledgmentsPath(), append(raw, '\n'), 0o644); err != nil {
|
||||
return fmt.Errorf("drift: write ack: %w", err)
|
||||
}
|
||||
slog.Info("drift acknowledged", "path", path, "peer", leadPeer)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *DefaultDetector) inCooldown(path string) bool {
|
||||
hash := pathHash(path)
|
||||
stateDir := LeadStateDir()
|
||||
cooldownDir := filepath.Join(stateDir, "remediation-cooldown")
|
||||
stampPath := filepath.Join(cooldownDir, hash)
|
||||
info, err := os.Stat(stampPath)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return time.Since(info.ModTime()) < d.cooldown
|
||||
}
|
||||
|
||||
func (d *DefaultDetector) markCooldown(path string) {
|
||||
hash := pathHash(path)
|
||||
stateDir := LeadStateDir()
|
||||
cooldownDir := filepath.Join(stateDir, "remediation-cooldown")
|
||||
if err := os.MkdirAll(cooldownDir, 0o755); err != nil {
|
||||
slog.Warn("drift markCooldown mkdir failed", "dir", cooldownDir, "error", err)
|
||||
return
|
||||
}
|
||||
stampPath := filepath.Join(cooldownDir, hash)
|
||||
if err := os.WriteFile(stampPath, []byte(time.Now().UTC().Format(time.RFC3339Nano)), 0o644); err != nil {
|
||||
slog.Warn("drift markCooldown write failed", "path", stampPath, "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
ErrCooldown = errors.New("drift: remediation in cooldown")
|
||||
ErrPathExcluded = errors.New("drift: path is excluded")
|
||||
ErrOverlapCritical = errors.New("drift: path appears in both critical and excluded")
|
||||
)
|
||||
|
||||
func LoadConfig(path string) (*Config, error) {
|
||||
if path == "" {
|
||||
return DefaultConfig(), nil
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return DefaultConfig(), nil
|
||||
}
|
||||
return nil, fmt.Errorf("drift: load config %s: %w", path, err)
|
||||
}
|
||||
var cfg Config
|
||||
if err := json.Unmarshal(raw, &cfg); err != nil {
|
||||
return nil, fmt.Errorf("drift: parse config %s: %w", path, err)
|
||||
}
|
||||
applyConfigDefaults(&cfg)
|
||||
return &cfg, nil
|
||||
}
|
||||
|
||||
func ValidateConfig(cfg *Config) error {
|
||||
if cfg == nil {
|
||||
return fmt.Errorf("drift: nil config")
|
||||
}
|
||||
if cfg.Polling.Enabled && cfg.Polling.DefaultInterval <= 0 {
|
||||
return fmt.Errorf("drift: polling enabled but default_interval is zero")
|
||||
}
|
||||
for _, p := range cfg.Paths.Critical {
|
||||
if p.Pattern == "" {
|
||||
return fmt.Errorf("drift: critical path has empty pattern")
|
||||
}
|
||||
if p.Tier == "" {
|
||||
return fmt.Errorf("drift: critical path %q has empty tier", p.Pattern)
|
||||
}
|
||||
}
|
||||
for _, p := range cfg.Paths.Standard {
|
||||
if p.Pattern == "" {
|
||||
return fmt.Errorf("drift: standard path has empty pattern")
|
||||
}
|
||||
}
|
||||
for _, ex := range cfg.Paths.Excluded {
|
||||
for _, c := range cfg.Paths.Critical {
|
||||
if pathGlobMatch(ex, c.Pattern) {
|
||||
return fmt.Errorf("drift: excluded pattern %q overlaps critical %q: %w", ex, c.Pattern, ErrOverlapCritical)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func DefaultConfig() *Config {
|
||||
cfg := &Config{
|
||||
Polling: PollingConfig{
|
||||
Enabled: true,
|
||||
DefaultInterval: 60 * time.Second,
|
||||
MaxConcurrentPeers: 8,
|
||||
},
|
||||
Paths: PathsConfig{
|
||||
Critical: []PathSpec{
|
||||
{Tier: TierCritical, Pattern: "/etc/traefik/dynamic/orca.yml", Interval: 5 * time.Second, SystemdPathUnit: true},
|
||||
{Tier: TierCritical, Pattern: "/etc/systemd/system/orca-alloc-*.service", Interval: 5 * time.Second, SystemdPathUnit: true},
|
||||
{Tier: TierCritical, Pattern: "/etc/nftables.d/orca.nft", Interval: 5 * time.Second, SystemdPathUnit: true},
|
||||
{Tier: TierCritical, Pattern: "/etc/orca/actual/*/etc/traefik/dynamic/*", Interval: 5 * time.Second, SystemdPathUnit: true},
|
||||
{Tier: TierCritical, Pattern: "/etc/orca/actual/*/etc/systemd/system/orca-alloc-*.service", Interval: 5 * time.Second, SystemdPathUnit: true},
|
||||
{Tier: TierCritical, Pattern: "/etc/orca/actual/*/etc/sudoers.d/orca-*", Interval: 5 * time.Second, SystemdPathUnit: true},
|
||||
},
|
||||
Standard: []PathSpec{
|
||||
{Tier: TierStandard, Pattern: "/etc/orca/actual/*", Interval: 30 * time.Second},
|
||||
{Tier: TierStandard, Pattern: "/etc/sudoers.d/orca-*", Interval: 30 * time.Second},
|
||||
{Tier: TierStandard, Pattern: "/etc/systemd/system/orca-collector.{service,timer}", Interval: 30 * time.Second},
|
||||
{Tier: TierStandard, Pattern: "/etc/systemd/system/orca-aggregator.{service,timer}", Interval: 30 * time.Second},
|
||||
{Tier: TierStandard, Pattern: "/etc/systemd/system/orca-pull.{service,timer}", Interval: 30 * time.Second},
|
||||
{Tier: TierStandard, Pattern: "/etc/systemd/system/orca-drift.{service,timer}", Interval: 30 * time.Second},
|
||||
},
|
||||
Excluded: []string{
|
||||
"/etc/orca/credentials/*",
|
||||
"/run/orca/*",
|
||||
"/etc/orca/state/drift-events/*",
|
||||
},
|
||||
},
|
||||
Remediate: RemediationPolicy{
|
||||
Auto: true,
|
||||
AutoPaths: []string{"/etc/traefik/dynamic/*", "/etc/nftables.d/*", "/etc/sudoers.d/orca-*"},
|
||||
RequireApproval: []string{"/etc/systemd/system/orca-alloc-*.service"},
|
||||
NotifyOnRemediation: true,
|
||||
},
|
||||
}
|
||||
applyConfigDefaults(cfg)
|
||||
return cfg
|
||||
}
|
||||
|
||||
func applyConfigDefaults(cfg *Config) {
|
||||
if cfg.Polling.DefaultInterval == 0 {
|
||||
cfg.Polling.DefaultInterval = 60 * time.Second
|
||||
}
|
||||
if cfg.Polling.MaxConcurrentPeers == 0 {
|
||||
cfg.Polling.MaxConcurrentPeers = 8
|
||||
}
|
||||
for i := range cfg.Paths.Critical {
|
||||
if cfg.Paths.Critical[i].Tier == "" {
|
||||
cfg.Paths.Critical[i].Tier = TierCritical
|
||||
}
|
||||
if cfg.Paths.Critical[i].Interval == 0 {
|
||||
cfg.Paths.Critical[i].Interval = 5 * time.Second
|
||||
}
|
||||
}
|
||||
for i := range cfg.Paths.Standard {
|
||||
if cfg.Paths.Standard[i].Tier == "" {
|
||||
cfg.Paths.Standard[i].Tier = TierStandard
|
||||
}
|
||||
if cfg.Paths.Standard[i].Interval == 0 {
|
||||
cfg.Paths.Standard[i].Interval = 30 * time.Second
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func NamespaceScope(events []Event, namespace string) []Event {
|
||||
if namespace == "" {
|
||||
var out []Event
|
||||
for _, e := range events {
|
||||
if !e.DriftConfirmed || e.Action == ActionAcknowledged {
|
||||
continue
|
||||
}
|
||||
out = append(out, e)
|
||||
}
|
||||
return out
|
||||
}
|
||||
var out []Event
|
||||
for _, e := range events {
|
||||
if !e.DriftConfirmed || e.Action == ActionAcknowledged {
|
||||
continue
|
||||
}
|
||||
if nsForPath(e.Path) == namespace {
|
||||
out = append(out, e)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func nsForPath(p string) string {
|
||||
const prefix = "/etc/orca/actual/"
|
||||
if !strings.HasPrefix(p, prefix) {
|
||||
return ""
|
||||
}
|
||||
rest := p[len(prefix):]
|
||||
if i := strings.IndexByte(rest, '/'); i >= 0 {
|
||||
return rest[:i]
|
||||
}
|
||||
return rest
|
||||
}
|
||||
|
||||
func matchesPaths(p string, specs []PathSpec) bool {
|
||||
if len(specs) == 0 {
|
||||
return true
|
||||
}
|
||||
for _, s := range specs {
|
||||
if pathGlobMatch(s.Pattern, p) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func defaultWatchInterval(specs []PathSpec) time.Duration {
|
||||
var min time.Duration
|
||||
for _, s := range specs {
|
||||
if s.Interval > 0 && (min == 0 || s.Interval < min) {
|
||||
min = s.Interval
|
||||
}
|
||||
}
|
||||
return min
|
||||
}
|
||||
|
||||
func pathGlobMatch(pattern, p string) bool {
|
||||
if pattern == "" {
|
||||
return false
|
||||
}
|
||||
expanded := expandBraces(pattern)
|
||||
for _, alt := range expanded {
|
||||
if globMatchSegment(alt, p) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func globMatchSegment(pattern, p string) bool {
|
||||
if pattern == "" {
|
||||
return false
|
||||
}
|
||||
if pattern == p {
|
||||
return true
|
||||
}
|
||||
if strings.HasSuffix(pattern, "/*") {
|
||||
prefix := pattern[:len(pattern)-2]
|
||||
if p == prefix {
|
||||
return true
|
||||
}
|
||||
return strings.HasPrefix(p, prefix+"/")
|
||||
}
|
||||
ok, err := filepath.Match(pattern, p)
|
||||
return err == nil && ok
|
||||
}
|
||||
|
||||
func expandBraces(p string) []string {
|
||||
open := strings.IndexByte(p, '{')
|
||||
if open < 0 {
|
||||
return []string{p}
|
||||
}
|
||||
close := strings.IndexByte(p[open:], '}')
|
||||
if close < 0 {
|
||||
return []string{p}
|
||||
}
|
||||
close += open
|
||||
prefix := p[:open]
|
||||
body := p[open+1 : close]
|
||||
suffix := p[close+1:]
|
||||
alternatives := strings.Split(body, ",")
|
||||
var out []string
|
||||
for _, a := range alternatives {
|
||||
for _, tail := range expandBraces(suffix) {
|
||||
out = append(out, prefix+a+tail)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func pathHash(p string) string {
|
||||
sum := sha256.Sum256([]byte(p))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func isTransientSSH(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
s := err.Error()
|
||||
for _, sub := range []string{"connection refused", "i/o timeout", "EOF", "no such host", "connection reset", "timeout", "deadline exceeded", "temporarily unavailable"} {
|
||||
if strings.Contains(s, sub) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func shellQuote(s string) string {
|
||||
return "'" + strings.ReplaceAll(s, "'", "'\\''") + "'"
|
||||
}
|
||||
|
||||
func ParseEvent(raw []byte) (Event, error) {
|
||||
var e Event
|
||||
if err := json.Unmarshal(raw, &e); err != nil {
|
||||
return Event{}, fmt.Errorf("drift: parse event: %w", err)
|
||||
}
|
||||
return e, nil
|
||||
}
|
||||
|
||||
func MarshalEvent(e Event) ([]byte, error) {
|
||||
return json.MarshalIndent(e, "", " ")
|
||||
}
|
||||
|
||||
var _ Detector = (*DefaultDetector)(nil)
|
||||
@@ -0,0 +1,465 @@
|
||||
package drift
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type mockTransport struct {
|
||||
execOut []byte
|
||||
execErr error
|
||||
execFn func(ctx context.Context, peer string, cmd string) ([]byte, error)
|
||||
readOut []byte
|
||||
readErr error
|
||||
writes []writeEntry
|
||||
}
|
||||
|
||||
type writeEntry struct {
|
||||
peer string
|
||||
path string
|
||||
content []byte
|
||||
mode os.FileMode
|
||||
}
|
||||
|
||||
func (m *mockTransport) Exec(ctx context.Context, peer string, cmd string) ([]byte, error) {
|
||||
if m.execFn != nil {
|
||||
return m.execFn(ctx, peer, cmd)
|
||||
}
|
||||
return m.execOut, m.execErr
|
||||
}
|
||||
|
||||
func (m *mockTransport) WriteFileIdempotent(ctx context.Context, peer string, path string, content []byte, mode os.FileMode) (bool, error) {
|
||||
m.writes = append(m.writes, writeEntry{peer, path, content, mode})
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (m *mockTransport) ReadFile(ctx context.Context, peer string, path string) ([]byte, error) {
|
||||
return m.readOut, m.readErr
|
||||
}
|
||||
|
||||
func setupDriftTestEnv(t *testing.T) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
t.Setenv("ORCA_LEAD_STATE_DIR", dir)
|
||||
return dir
|
||||
}
|
||||
|
||||
func TestParseEvent(t *testing.T) {
|
||||
raw := []byte(`{
|
||||
"event_id":"EVT-1",
|
||||
"ts":"2026-01-01T00:00:00Z",
|
||||
"host":"peer1",
|
||||
"path":"/etc/traefik/dynamic/orca.yml",
|
||||
"status":"modified",
|
||||
"new_sha256":"abc",
|
||||
"latest_txn":"T-1234567890abcdef",
|
||||
"expected_sha256":"def",
|
||||
"drift_confirmed":true,
|
||||
"action":"reported",
|
||||
"action_result":"skipped",
|
||||
"operator_id":"op1"
|
||||
}`)
|
||||
e, err := ParseEvent(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseEvent: %v", err)
|
||||
}
|
||||
if e.EventID != "EVT-1" {
|
||||
t.Errorf("EventID = %q, want EVT-1", e.EventID)
|
||||
}
|
||||
if e.Host != "peer1" {
|
||||
t.Errorf("Host = %q, want peer1", e.Host)
|
||||
}
|
||||
if e.Path != "/etc/traefik/dynamic/orca.yml" {
|
||||
t.Errorf("Path = %q", e.Path)
|
||||
}
|
||||
if e.Status != StatusModified {
|
||||
t.Errorf("Status = %q, want modified", e.Status)
|
||||
}
|
||||
if !e.DriftConfirmed {
|
||||
t.Errorf("DriftConfirmed = false, want true")
|
||||
}
|
||||
if e.Action != ActionReported {
|
||||
t.Errorf("Action = %q, want reported", e.Action)
|
||||
}
|
||||
if e.ActionResult != ActionResultSkipped {
|
||||
t.Errorf("ActionResult = %q, want skipped", e.ActionResult)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseEventInvalid(t *testing.T) {
|
||||
_, err := ParseEvent([]byte(`{not json`))
|
||||
if err == nil {
|
||||
t.Fatal("expected error for invalid JSON, got nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarshalEventRoundTrip(t *testing.T) {
|
||||
e := Event{EventID: "E1", Host: "h", Path: "/p", Status: StatusCreated, DriftConfirmed: true}
|
||||
raw, err := MarshalEvent(e)
|
||||
if err != nil {
|
||||
t.Fatalf("MarshalEvent: %v", err)
|
||||
}
|
||||
out, err := ParseEvent(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseEvent: %v", err)
|
||||
}
|
||||
if out.EventID != e.EventID || out.Host != e.Host || out.Path != e.Path || out.Status != e.Status {
|
||||
t.Errorf("round-trip mismatch: %+v vs %+v", out, e)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadConfigDefault(t *testing.T) {
|
||||
cfg, err := LoadConfig("")
|
||||
if err != nil {
|
||||
t.Fatalf("LoadConfig empty: %v", err)
|
||||
}
|
||||
if !cfg.Polling.Enabled {
|
||||
t.Errorf("Polling.Enabled = false, want true")
|
||||
}
|
||||
if cfg.Polling.DefaultInterval != 60*time.Second {
|
||||
t.Errorf("DefaultInterval = %v, want 60s", cfg.Polling.DefaultInterval)
|
||||
}
|
||||
if len(cfg.Paths.Critical) == 0 {
|
||||
t.Errorf("Critical paths empty")
|
||||
}
|
||||
found := false
|
||||
for _, c := range cfg.Paths.Critical {
|
||||
if c.Pattern == "/etc/traefik/dynamic/orca.yml" {
|
||||
found = true
|
||||
if c.Interval != 5*time.Second {
|
||||
t.Errorf("critical interval = %v, want 5s", c.Interval)
|
||||
}
|
||||
if !c.SystemdPathUnit {
|
||||
t.Errorf("SystemdPathUnit = false, want true")
|
||||
}
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("traefik critical path missing")
|
||||
}
|
||||
found = false
|
||||
for _, s := range cfg.Paths.Standard {
|
||||
if s.Pattern == "/etc/orca/actual/*" {
|
||||
found = true
|
||||
if s.Interval != 30*time.Second {
|
||||
t.Errorf("standard interval = %v, want 30s", s.Interval)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("/etc/orca/actual/* standard path missing")
|
||||
}
|
||||
if !cfg.Remediate.Auto {
|
||||
t.Errorf("Remediate.Auto = false, want true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadConfigFile(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "drift.json")
|
||||
cfgJSON := `{
|
||||
"polling": {"enabled": true, "default_interval": 90000000000, "max_concurrent_peers": 4},
|
||||
"paths": {
|
||||
"critical": [{"tier":"critical","pattern":"/etc/critical.yml","interval":5000000000,"systemd_path_unit":true}],
|
||||
"standard": [{"tier":"standard","pattern":"/etc/standard.yml","interval":30000000000}],
|
||||
"excluded": ["/run/orca/*"]
|
||||
},
|
||||
"remediate": {"auto": false, "auto_paths": [], "require_approval": ["/etc/critical.yml"], "notify_on_remediation": false}
|
||||
}`
|
||||
if err := os.WriteFile(path, []byte(cfgJSON), 0o644); err != nil {
|
||||
t.Fatalf("write config: %v", err)
|
||||
}
|
||||
cfg, err := LoadConfig(path)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadConfig: %v", err)
|
||||
}
|
||||
if cfg.Polling.MaxConcurrentPeers != 4 {
|
||||
t.Errorf("MaxConcurrentPeers = %d, want 4", cfg.Polling.MaxConcurrentPeers)
|
||||
}
|
||||
if cfg.Polling.DefaultInterval != 90*time.Second {
|
||||
t.Errorf("DefaultInterval = %v, want 90s", cfg.Polling.DefaultInterval)
|
||||
}
|
||||
if cfg.Remediate.Auto {
|
||||
t.Errorf("Auto = true, want false")
|
||||
}
|
||||
if len(cfg.Paths.Critical) != 1 || cfg.Paths.Critical[0].Pattern != "/etc/critical.yml" {
|
||||
t.Errorf("critical = %+v", cfg.Paths.Critical)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateConfigValid(t *testing.T) {
|
||||
cfg := DefaultConfig()
|
||||
if err := ValidateConfig(cfg); err != nil {
|
||||
t.Fatalf("ValidateConfig(default): %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateConfigEmptyPattern(t *testing.T) {
|
||||
cfg := DefaultConfig()
|
||||
cfg.Paths.Critical[0].Pattern = ""
|
||||
if err := ValidateConfig(cfg); err == nil {
|
||||
t.Fatal("expected error for empty critical pattern")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateConfigPollingZero(t *testing.T) {
|
||||
cfg := DefaultConfig()
|
||||
cfg.Polling.Enabled = true
|
||||
cfg.Polling.DefaultInterval = 0
|
||||
if err := ValidateConfig(cfg); err == nil {
|
||||
t.Fatal("expected error for zero default_interval with polling enabled")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateConfigOverlapCriticalExcluded(t *testing.T) {
|
||||
cfg := DefaultConfig()
|
||||
cfg.Paths.Excluded = append(cfg.Paths.Excluded, "/etc/traefik/dynamic/orca.yml")
|
||||
err := ValidateConfig(cfg)
|
||||
if err == nil {
|
||||
t.Fatal("expected overlap error")
|
||||
}
|
||||
if !errors.Is(err, ErrOverlapCritical) {
|
||||
t.Errorf("expected ErrOverlapCritical, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateConfigNil(t *testing.T) {
|
||||
if err := ValidateConfig(nil); err == nil {
|
||||
t.Fatal("expected error for nil config")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAggregateLocalMissing(t *testing.T) {
|
||||
setupDriftTestEnv(t)
|
||||
d := NewDefaultDetector(&mockTransport{})
|
||||
events, err := d.Aggregate(context.Background(), "")
|
||||
if err != nil {
|
||||
t.Fatalf("Aggregate missing: %v", err)
|
||||
}
|
||||
if events != nil {
|
||||
t.Errorf("events = %v, want nil", events)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAggregateLocal(t *testing.T) {
|
||||
setupDriftTestEnv(t)
|
||||
doc := `{"ts":"2026-01-01T00:00:00Z","events":[{"event_id":"E1","host":"p","path":"/etc/traefik/dynamic/orca.yml","status":"modified","drift_confirmed":true}]}`
|
||||
if err := os.WriteFile(AggregatedPath(), []byte(doc), 0o644); err != nil {
|
||||
t.Fatalf("write aggregated: %v", err)
|
||||
}
|
||||
d := NewDefaultDetector(&mockTransport{})
|
||||
events, err := d.Aggregate(context.Background(), "")
|
||||
if err != nil {
|
||||
t.Fatalf("Aggregate: %v", err)
|
||||
}
|
||||
if len(events) != 1 {
|
||||
t.Fatalf("events len = %d, want 1", len(events))
|
||||
}
|
||||
if events[0].EventID != "E1" {
|
||||
t.Errorf("EventID = %q, want E1", events[0].EventID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAggregateRemote(t *testing.T) {
|
||||
setupDriftTestEnv(t)
|
||||
doc := `{"ts":"2026-01-01T00:00:00Z","events":[{"event_id":"E2","host":"p","path":"/etc/orca/actual/ns-a/x","status":"created","drift_confirmed":true}]}`
|
||||
mt := &mockTransport{readOut: []byte(doc)}
|
||||
d := NewDefaultDetector(mt)
|
||||
events, err := d.Aggregate(context.Background(), "lead:22")
|
||||
if err != nil {
|
||||
t.Fatalf("Aggregate remote: %v", err)
|
||||
}
|
||||
if len(events) != 1 {
|
||||
t.Fatalf("events len = %d, want 1", len(events))
|
||||
}
|
||||
if events[0].EventID != "E2" {
|
||||
t.Errorf("EventID = %q, want E2", events[0].EventID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWatchCancels(t *testing.T) {
|
||||
setupDriftTestEnv(t)
|
||||
doc := `{"ts":"2026-01-01T00:00:00Z","events":[{"event_id":"E1","host":"p","path":"/etc/traefik/dynamic/orca.yml","status":"modified","drift_confirmed":true}]}`
|
||||
if err := os.WriteFile(AggregatedPath(), []byte(doc), 0o644); err != nil {
|
||||
t.Fatalf("write aggregated: %v", err)
|
||||
}
|
||||
d := NewDefaultDetector(&mockTransport{})
|
||||
d.SetCooldown(50 * time.Millisecond)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
gotEvents := 0
|
||||
for e, err := range d.Watch(ctx, []PathSpec{{Pattern: "/etc/traefik/dynamic/orca.yml", Interval: 10 * time.Millisecond}}) {
|
||||
if err != nil {
|
||||
t.Fatalf("Watch err: %v", err)
|
||||
}
|
||||
gotEvents++
|
||||
_ = e
|
||||
cancel()
|
||||
break
|
||||
}
|
||||
if gotEvents != 1 {
|
||||
t.Errorf("gotEvents = %d, want 1", gotEvents)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWatchStreamsAndDedupes(t *testing.T) {
|
||||
setupDriftTestEnv(t)
|
||||
doc := `{"ts":"2026-01-01T00:00:00Z","events":[{"event_id":"E1","host":"p","path":"/etc/traefik/dynamic/orca.yml","status":"modified","drift_confirmed":true}]}`
|
||||
if err := os.WriteFile(AggregatedPath(), []byte(doc), 0o644); err != nil {
|
||||
t.Fatalf("write aggregated: %v", err)
|
||||
}
|
||||
d := NewDefaultDetector(&mockTransport{})
|
||||
d.SetCooldown(50 * time.Millisecond)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 80*time.Millisecond)
|
||||
defer cancel()
|
||||
got := 0
|
||||
for _, err := range d.Watch(ctx, nil) {
|
||||
if err != nil {
|
||||
t.Fatalf("Watch err: %v", err)
|
||||
}
|
||||
got++
|
||||
}
|
||||
if got != 1 {
|
||||
t.Errorf("got = %d, want 1 (dedup)", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemediateCooldownOnSuccess(t *testing.T) {
|
||||
setupDriftTestEnv(t)
|
||||
mt := &mockTransport{execOut: []byte("ok")}
|
||||
d := NewDefaultDetector(mt)
|
||||
d.SetCooldown(1 * time.Hour)
|
||||
if err := d.Remediate(context.Background(), "lead:22", "/etc/traefik/dynamic/orca.yml", false); err != nil {
|
||||
t.Fatalf("first Remediate: %v", err)
|
||||
}
|
||||
if err := d.Remediate(context.Background(), "lead:22", "/etc/traefik/dynamic/orca.yml", false); err == nil {
|
||||
t.Fatal("second Remediate should hit cooldown")
|
||||
} else if !errors.Is(err, ErrCooldown) {
|
||||
t.Errorf("expected ErrCooldown, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemediateForceBypassesCooldown(t *testing.T) {
|
||||
setupDriftTestEnv(t)
|
||||
mt := &mockTransport{execOut: []byte("ok")}
|
||||
d := NewDefaultDetector(mt)
|
||||
d.SetCooldown(1 * time.Hour)
|
||||
if err := d.Remediate(context.Background(), "lead:22", "/etc/p", false); err != nil {
|
||||
t.Fatalf("first: %v", err)
|
||||
}
|
||||
if err := d.Remediate(context.Background(), "lead:22", "/etc/p", true); err != nil {
|
||||
t.Fatalf("force bypass: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemediateTransientNoCooldown(t *testing.T) {
|
||||
setupDriftTestEnv(t)
|
||||
mt := &mockTransport{execErr: fmt.Errorf("connection refused")}
|
||||
d := NewDefaultDetector(mt)
|
||||
d.SetCooldown(1 * time.Hour)
|
||||
if err := d.Remediate(context.Background(), "lead:22", "/etc/p", false); err == nil {
|
||||
t.Fatal("expected transient error")
|
||||
}
|
||||
if d.inCooldown("/etc/p") {
|
||||
t.Errorf("cooldown entered after transient failure")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemediateRequiresLead(t *testing.T) {
|
||||
setupDriftTestEnv(t)
|
||||
d := NewDefaultDetector(&mockTransport{})
|
||||
if err := d.Remediate(context.Background(), "", "/etc/p", true); err == nil {
|
||||
t.Fatal("expected error for empty lead")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcknowledgeWrites(t *testing.T) {
|
||||
setupDriftTestEnv(t)
|
||||
mt := &mockTransport{}
|
||||
d := NewDefaultDetector(mt)
|
||||
if err := d.Acknowledge(context.Background(), "lead:22", "/etc/traefik/dynamic/orca.yml"); err != nil {
|
||||
t.Fatalf("Acknowledge: %v", err)
|
||||
}
|
||||
if len(mt.writes) != 1 {
|
||||
t.Fatalf("writes = %d, want 1", len(mt.writes))
|
||||
}
|
||||
w := mt.writes[0]
|
||||
if w.peer != "lead:22" {
|
||||
t.Errorf("peer = %q, want lead:22", w.peer)
|
||||
}
|
||||
if w.path != AcknowledgmentsPath() {
|
||||
t.Errorf("path = %q, want %q", w.path, AcknowledgmentsPath())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcknowledgeRequiresLead(t *testing.T) {
|
||||
setupDriftTestEnv(t)
|
||||
d := NewDefaultDetector(&mockTransport{})
|
||||
if err := d.Acknowledge(context.Background(), "", "/etc/p"); err == nil {
|
||||
t.Fatal("expected error for empty lead")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNamespaceScopeClusterWide(t *testing.T) {
|
||||
events := []Event{
|
||||
{EventID: "E1", Path: "/etc/traefik/dynamic/orca.yml", DriftConfirmed: true},
|
||||
{EventID: "E2", Path: "/etc/orca/actual/ns-a/x", DriftConfirmed: true},
|
||||
{EventID: "E3", Path: "/etc/orca/actual/ns-b/y", DriftConfirmed: true, Action: ActionAcknowledged},
|
||||
}
|
||||
out := NamespaceScope(events, "")
|
||||
if len(out) != 2 {
|
||||
t.Errorf("cluster-wide scope len = %d, want 2 (ack excluded)", len(out))
|
||||
}
|
||||
}
|
||||
|
||||
func TestNamespaceScopePerNamespace(t *testing.T) {
|
||||
events := []Event{
|
||||
{EventID: "E1", Path: "/etc/orca/actual/ns-a/x", DriftConfirmed: true},
|
||||
{EventID: "E2", Path: "/etc/orca/actual/ns-b/y", DriftConfirmed: true},
|
||||
{EventID: "E3", Path: "/etc/traefik/dynamic/orca.yml", DriftConfirmed: true},
|
||||
}
|
||||
nsA := NamespaceScope(events, "ns-a")
|
||||
if len(nsA) != 1 || nsA[0].EventID != "E1" {
|
||||
t.Errorf("ns-a scope = %+v, want only E1", nsA)
|
||||
}
|
||||
nsB := NamespaceScope(events, "ns-b")
|
||||
if len(nsB) != 1 || nsB[0].EventID != "E2" {
|
||||
t.Errorf("ns-b scope = %+v, want only E2", nsB)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExpandBraces(t *testing.T) {
|
||||
out := expandBraces("/etc/x.{service,timer}")
|
||||
if len(out) != 2 {
|
||||
t.Fatalf("len = %d, want 2", len(out))
|
||||
}
|
||||
if out[0] != "/etc/x.service" || out[1] != "/etc/x.timer" {
|
||||
t.Errorf("out = %v", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPathGlobMatch(t *testing.T) {
|
||||
if !pathGlobMatch("/etc/orca/actual/*", "/etc/orca/actual/ns-a") {
|
||||
t.Errorf("expected glob match for single segment")
|
||||
}
|
||||
if !pathGlobMatch("/etc/orca/actual/*", "/etc/orca/actual/ns-a/x") {
|
||||
t.Errorf("expected glob match for nested segment (drift /* crosses /)")
|
||||
}
|
||||
if !pathGlobMatch("/etc/systemd/system/orca-collector.{service,timer}", "/etc/systemd/system/orca-collector.service") {
|
||||
t.Errorf("expected brace match")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNsForPath(t *testing.T) {
|
||||
if got := nsForPath("/etc/orca/actual/ns-a/x"); got != "ns-a" {
|
||||
t.Errorf("nsForPath = %q, want ns-a", got)
|
||||
}
|
||||
if got := nsForPath("/etc/traefik/dynamic/orca.yml"); got != "" {
|
||||
t.Errorf("nsForPath = %q, want empty", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
// Package emitter: drift_path.go implements the systemd Path-unit
|
||||
// emitter for critical drift paths (P10b, v0.11, REQ-105, R-018).
|
||||
//
|
||||
// For each critical path two units are emitted:
|
||||
//
|
||||
// - <orca-drift-<name>.path>: PathChanged=<path>,
|
||||
// RateLimitIntervalSec=1s, RateLimitBurst=5 — systemd watches the
|
||||
// path for changes and triggers the matching .service.
|
||||
// - <orca-drift-<name>.service>: Type=oneshot,
|
||||
// ExecStart=/usr/local/bin/orca-drift-notify.sh %f, User=orca,
|
||||
// security-hardened (NoNewPrivileges=yes, ProtectSystem=strict,
|
||||
// ReadWritePaths=/etc/orca/state/drift-events, ProtectHome=yes).
|
||||
//
|
||||
// The emitter takes a list of critical PathSpecs (from drift.Config)
|
||||
// and produces the unit File artifacts. Unit names are derived from a
|
||||
// slug of the path pattern: non-alphanumerics collapse to '-' so the
|
||||
// name is systemd-friendly (no slashes, spaces, or brace chars).
|
||||
package emitter
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// DriftPathUnit is the systemd Path + service unit emitter for critical
|
||||
// drift paths (REQ-105).
|
||||
type DriftPathUnit struct{}
|
||||
|
||||
// driftEventsDir is the directory the oneshot services write to
|
||||
// (ReadWritePaths).
|
||||
const driftEventsDir = "/etc/orca/state/drift-events"
|
||||
|
||||
// driftNotifyBin is the oneshot ExecStart binary.
|
||||
const driftNotifyBin = "/usr/local/bin/orca-drift-notify.sh"
|
||||
|
||||
// RenderDriftUnits renders a pair of systemd units (a .path and a
|
||||
// .service) for each critical path in specs. Returns the File slice
|
||||
// ready for SSH-push to the peer.
|
||||
func (DriftPathUnit) RenderDriftUnits(specs []DriftPathSpec) ([]File, error) {
|
||||
if len(specs) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
seen := make(map[string]bool, len(specs))
|
||||
var files []File
|
||||
for _, s := range specs {
|
||||
if s.Pattern == "" {
|
||||
return nil, fmt.Errorf("emitter/drift_path: empty pattern")
|
||||
}
|
||||
name := driftUnitName(s.Pattern)
|
||||
if seen[name] {
|
||||
continue
|
||||
}
|
||||
seen[name] = true
|
||||
files = append(files, File{
|
||||
Path: fmt.Sprintf("/etc/systemd/system/%s.path", name),
|
||||
Content: renderDriftPathUnit(name, s.Pattern),
|
||||
Mode: "0644",
|
||||
})
|
||||
files = append(files, File{
|
||||
Path: fmt.Sprintf("/etc/systemd/system/%s.service", name),
|
||||
Content: renderDriftServiceUnit(name),
|
||||
Mode: "0644",
|
||||
})
|
||||
}
|
||||
return files, nil
|
||||
}
|
||||
|
||||
// DriftPathSpec is the minimal description the emitter needs: the
|
||||
// pattern to watch and whether systemd Path units are requested (when
|
||||
// false, no units are emitted — the path falls back to polling).
|
||||
type DriftPathSpec struct {
|
||||
Pattern string
|
||||
SystemdPathUnit bool
|
||||
}
|
||||
|
||||
// driftUnitName derives a systemd-friendly unit-name slug from a path
|
||||
// pattern. Non-alphanumeric runes collapse to '-'. The result carries
|
||||
// the orca-drift- prefix so the units are identifiable as orca-owned.
|
||||
func driftUnitName(pattern string) string {
|
||||
s := pattern
|
||||
s = strings.TrimPrefix(s, "/")
|
||||
var b strings.Builder
|
||||
prevDash := false
|
||||
for _, r := range s {
|
||||
if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') {
|
||||
b.WriteRune(r)
|
||||
prevDash = false
|
||||
} else {
|
||||
if !prevDash {
|
||||
b.WriteByte('-')
|
||||
prevDash = true
|
||||
}
|
||||
}
|
||||
}
|
||||
name := strings.Trim(b.String(), "-")
|
||||
if name == "" {
|
||||
name = "root"
|
||||
}
|
||||
return "orca-drift-" + name
|
||||
}
|
||||
|
||||
// renderDriftPathUnit renders the .path unit. PathChanged re-fires on
|
||||
// every modification (inotify IN_MODIFY), RateLimitIntervalSec +
|
||||
// RateLimitBurst bound the burst so a thrashing file does not spawn
|
||||
// thousands of services (R-018).
|
||||
func renderDriftPathUnit(name, path string) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("[Unit]\n")
|
||||
b.WriteString(fmt.Sprintf("Description=orca drift watch for %s\n", path))
|
||||
b.WriteString("\n[Path]\n")
|
||||
b.WriteString(fmt.Sprintf("PathChanged=%s\n", path))
|
||||
b.WriteString("RateLimitIntervalSec=1s\n")
|
||||
b.WriteString("RateLimitBurst=5\n")
|
||||
b.WriteString("\n[Install]\n")
|
||||
b.WriteString("WantedBy=multi-user.target\n")
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// renderDriftServiceUnit renders the oneshot .service triggered by
|
||||
// the .path unit. ExecStart receives the changed path via %f. Security
|
||||
// hardening runs the service as User=orca with NoNewPrivileges=yes,
|
||||
// ProtectSystem=strict (read-only /), and ReadWritePaths scoped to the
|
||||
// drift-events dir so the script can write its event JSON. ProtectHome
|
||||
// hides /root and /home (the orca user has no business there).
|
||||
func renderDriftServiceUnit(name string) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("[Unit]\n")
|
||||
b.WriteString(fmt.Sprintf("Description=orca drift notify for %s\n", name))
|
||||
b.WriteString(fmt.Sprintf("After=%s.path\n", name))
|
||||
b.WriteString("\n[Service]\n")
|
||||
b.WriteString("Type=oneshot\n")
|
||||
b.WriteString(fmt.Sprintf("ExecStart=%s %%f\n", driftNotifyBin))
|
||||
b.WriteString("User=orca\n")
|
||||
b.WriteString("Group=orca\n")
|
||||
b.WriteString("NoNewPrivileges=yes\n")
|
||||
b.WriteString("ProtectSystem=strict\n")
|
||||
b.WriteString(fmt.Sprintf("ReadWritePaths=%s\n", driftEventsDir))
|
||||
b.WriteString("ProtectHome=yes\n")
|
||||
b.WriteString("PrivateTmp=yes\n")
|
||||
b.WriteString("ProtectKernelTunables=yes\n")
|
||||
b.WriteString("ProtectKernelModules=yes\n")
|
||||
b.WriteString("ProtectControlGroups=yes\n")
|
||||
b.WriteString("RestrictSUIDSGID=yes\n")
|
||||
b.WriteString("\n[Install]\n")
|
||||
b.WriteString("WantedBy=multi-user.target\n")
|
||||
return b.String()
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
package emitter
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDriftPathUnit_RenderUnits(t *testing.T) {
|
||||
specs := []DriftPathSpec{
|
||||
{Pattern: "/etc/traefik/dynamic/orca.yml", SystemdPathUnit: true},
|
||||
{Pattern: "/etc/nftables.d/orca.nft", SystemdPathUnit: true},
|
||||
}
|
||||
files, err := (DriftPathUnit{}).RenderDriftUnits(specs)
|
||||
if err != nil {
|
||||
t.Fatalf("RenderDriftUnits: %v", err)
|
||||
}
|
||||
if len(files) != 4 {
|
||||
t.Fatalf("got %d files, want 4 (2 .path + 2 .service)", len(files))
|
||||
}
|
||||
var pathUnits, svcUnits int
|
||||
for _, f := range files {
|
||||
if strings.HasSuffix(f.Path, ".path") {
|
||||
pathUnits++
|
||||
}
|
||||
if strings.HasSuffix(f.Path, ".service") {
|
||||
svcUnits++
|
||||
}
|
||||
}
|
||||
if pathUnits != 2 || svcUnits != 2 {
|
||||
t.Errorf("path=%d service=%d, want 2 and 2", pathUnits, svcUnits)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriftPathUnit_PathUnitContent(t *testing.T) {
|
||||
files, err := (DriftPathUnit{}).RenderDriftUnits([]DriftPathSpec{
|
||||
{Pattern: "/etc/traefik/dynamic/orca.yml", SystemdPathUnit: true},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Render: %v", err)
|
||||
}
|
||||
var pathFile *File
|
||||
for i := range files {
|
||||
if strings.HasSuffix(files[i].Path, ".path") {
|
||||
pathFile = &files[i]
|
||||
}
|
||||
}
|
||||
if pathFile == nil {
|
||||
t.Fatal("no .path unit emitted")
|
||||
}
|
||||
c := pathFile.Content
|
||||
for _, want := range []string{
|
||||
"PathChanged=/etc/traefik/dynamic/orca.yml",
|
||||
"RateLimitIntervalSec=1s",
|
||||
"RateLimitBurst=5",
|
||||
"WantedBy=multi-user.target",
|
||||
} {
|
||||
if !strings.Contains(c, want) {
|
||||
t.Errorf("path unit missing %q:\n%s", want, c)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriftPathUnit_ServiceUnitSecurity(t *testing.T) {
|
||||
files, err := (DriftPathUnit{}).RenderDriftUnits([]DriftPathSpec{
|
||||
{Pattern: "/etc/nftables.d/orca.nft", SystemdPathUnit: true},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Render: %v", err)
|
||||
}
|
||||
var svcFile *File
|
||||
for i := range files {
|
||||
if strings.HasSuffix(files[i].Path, ".service") {
|
||||
svcFile = &files[i]
|
||||
}
|
||||
}
|
||||
if svcFile == nil {
|
||||
t.Fatal("no .service unit emitted")
|
||||
}
|
||||
c := svcFile.Content
|
||||
for _, want := range []string{
|
||||
"Type=oneshot",
|
||||
"ExecStart=/usr/local/bin/orca-drift-notify.sh %f",
|
||||
"User=orca",
|
||||
"Group=orca",
|
||||
"NoNewPrivileges=yes",
|
||||
"ProtectSystem=strict",
|
||||
"ReadWritePaths=/etc/orca/state/drift-events",
|
||||
"ProtectHome=yes",
|
||||
} {
|
||||
if !strings.Contains(c, want) {
|
||||
t.Errorf("service unit missing %q:\n%s", want, c)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriftPathUnit_UnitNameSlug(t *testing.T) {
|
||||
cases := []struct {
|
||||
pattern string
|
||||
want string
|
||||
}{
|
||||
{"/etc/traefik/dynamic/orca.yml", "orca-drift-etc-traefik-dynamic-orca-yml"},
|
||||
{"/etc/systemd/system/orca-alloc-*.service", "orca-drift-etc-systemd-system-orca-alloc-service"},
|
||||
{"/etc/orca/actual/*/etc/sudoers.d/orca-*", "orca-drift-etc-orca-actual-etc-sudoers-d-orca"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
got := driftUnitName(c.pattern)
|
||||
if got != c.want {
|
||||
t.Errorf("driftUnitName(%q) = %q, want %q", c.pattern, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriftPathUnit_EmptyPatternErrors(t *testing.T) {
|
||||
_, err := (DriftPathUnit{}).RenderDriftUnits([]DriftPathSpec{{Pattern: ""}})
|
||||
if err == nil {
|
||||
t.Fatal("expected error for empty pattern")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriftPathUnit_NoSpecsNoFiles(t *testing.T) {
|
||||
files, err := (DriftPathUnit{}).RenderDriftUnits(nil)
|
||||
if err != nil {
|
||||
t.Fatalf("RenderDriftUnits nil: %v", err)
|
||||
}
|
||||
if files != nil {
|
||||
t.Errorf("got %d files, want nil", len(files))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriftPathUnit_DedupesSameSlug(t *testing.T) {
|
||||
// Two patterns that produce the same slug (after non-alnum collapse)
|
||||
// should dedupe to a single .path + .service pair.
|
||||
files, err := (DriftPathUnit{}).RenderDriftUnits([]DriftPathSpec{
|
||||
{Pattern: "/etc/x/a.yml", SystemdPathUnit: true},
|
||||
{Pattern: "/etc/x/a.yml", SystemdPathUnit: true},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Render: %v", err)
|
||||
}
|
||||
if len(files) != 2 {
|
||||
t.Errorf("got %d files, want 2 (deduped identical patterns)", len(files))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriftPathUnit_ServiceAfterPath(t *testing.T) {
|
||||
files, _ := (DriftPathUnit{}).RenderDriftUnits([]DriftPathSpec{
|
||||
{Pattern: "/etc/traefik/dynamic/orca.yml", SystemdPathUnit: true},
|
||||
})
|
||||
for _, f := range files {
|
||||
if strings.HasSuffix(f.Path, ".service") {
|
||||
if !strings.Contains(f.Content, "After=orca-drift-etc-traefik-dynamic-orca-yml.path") {
|
||||
t.Errorf("service missing After=<name>.path:\n%s", f.Content)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatal("no service file found")
|
||||
}
|
||||
Reference in New Issue
Block a user