feat(P09): collector + aggregator (C-11/C-12/C-14) + drift aggregation (REQ-107)
scripts/orca-aggregate.sh: 10s aggregator, cluster.json merge + drift-events rsync + remediation trigger (P10b stub). scripts/orca- watchdog.sh: C-11 starvation detection. internal/cli/collector.go: orca collector start/stop/status. Tests: CLI + bats. ---ci--- project: orca phase: 09 milestone: v0.11 status: execute ---/ci---
This commit is contained in:
Jon Chery
@@ -0,0 +1,326 @@
|
||||
// Package cli: collector.go implements the `orca collector` subcommand
|
||||
// (P09, C-12 opt-in). The collector is the lead-side aggregator +
|
||||
// watchdog pair: `orca-aggregate.sh` runs every 10s (via a systemd
|
||||
// timer) merging per-peer state snapshots into cluster.json, and
|
||||
// `orca-watchdog.sh` runs every 30s detecting aggregator starvation
|
||||
// (C-11).
|
||||
//
|
||||
// `orca collector start` emits the scripts + systemd timers/services
|
||||
// to the lead and enables them. `orca collector stop` disables and
|
||||
// removes them. `orca collector status` reports whether the pair is
|
||||
// running.
|
||||
//
|
||||
// Paths default to the system layout (/etc/orca, /etc/systemd/system);
|
||||
// a `--root` flag (default "/") relocates every emitted path under
|
||||
// <root> for testability (tests use a temp dir).
|
||||
package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var collectorRoot string
|
||||
|
||||
const (
|
||||
collectorScriptDir = "etc/orca/collector"
|
||||
collectorUnitDir = "etc/systemd/system"
|
||||
collectorAggregateSh = "orca-aggregate.sh"
|
||||
collectorWatchdogSh = "orca-watchdog.sh"
|
||||
collectorAggregateSvc = "orca-aggregate.service"
|
||||
collectorAggregateTmr = "orca-aggregate.timer"
|
||||
collectorWatchdogSvc = "orca-watchdog.service"
|
||||
collectorWatchdogTmr = "orca-watchdog.timer"
|
||||
collectorStateDir = "etc/orca/state"
|
||||
)
|
||||
|
||||
var collectorCmd = &cobra.Command{
|
||||
Use: "collector",
|
||||
Short: "Manage the lead-side collector (aggregator + watchdog) (P09)",
|
||||
Long: `Manage the lead-side collector: the aggregator (orca-aggregate.sh,
|
||||
10s cadence, merges per-peer state into cluster.json + drift-event
|
||||
aggregation per REQ-107) and the watchdog (orca-watchdog.sh, 30s
|
||||
cadence, detects aggregator starvation per C-11). Opt-in (C-12).`,
|
||||
Args: cobra.NoArgs,
|
||||
}
|
||||
|
||||
var collectorStartCmd = &cobra.Command{
|
||||
Use: "start",
|
||||
Short: "Emit the collector scripts + systemd units and enable them",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
root, err := collectorResolveRoot()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := collectorEmit(root); err != nil {
|
||||
return err
|
||||
}
|
||||
if !collectorDryRun {
|
||||
if err := collectorEnable(root); err != nil {
|
||||
return fmt.Errorf("enable: %w", err)
|
||||
}
|
||||
}
|
||||
msg := "collector started"
|
||||
if collectorDryRun {
|
||||
msg = "collector scripts emitted (dry-run, not enabled)"
|
||||
}
|
||||
printResult(msg, map[string]string{"status": "started", "root": root})
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
var collectorStopCmd = &cobra.Command{
|
||||
Use: "stop",
|
||||
Short: "Disable and remove the collector scripts + systemd units",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
root, err := collectorResolveRoot()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !collectorDryRun {
|
||||
if err := collectorDisable(root); err != nil {
|
||||
return fmt.Errorf("disable: %w", err)
|
||||
}
|
||||
}
|
||||
if err := collectorRemove(root); err != nil {
|
||||
return err
|
||||
}
|
||||
msg := "collector stopped"
|
||||
if collectorDryRun {
|
||||
msg = "collector artifacts removed (dry-run, not disabled)"
|
||||
}
|
||||
printResult(msg, map[string]string{"status": "stopped", "root": root})
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
var collectorStatusCmd = &cobra.Command{
|
||||
Use: "status",
|
||||
Short: "Report whether the collector (aggregator + watchdog) is running",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
root, err := collectorResolveRoot()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
aggRunning, wdRunning := collectorRunning(root)
|
||||
overall := "running"
|
||||
if !aggRunning && !wdRunning {
|
||||
overall = "stopped"
|
||||
} else if !aggRunning || !wdRunning {
|
||||
overall = "partial"
|
||||
}
|
||||
printResult(
|
||||
fmt.Sprintf("collector: %s (aggregator=%t watchdog=%t)", overall, aggRunning, wdRunning),
|
||||
map[string]any{
|
||||
"status": overall,
|
||||
"aggregator": aggRunning,
|
||||
"watchdog": wdRunning,
|
||||
"root": root,
|
||||
},
|
||||
)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
var collectorDryRun bool
|
||||
|
||||
func init() {
|
||||
collectorCmd.PersistentFlags().StringVar(&collectorRoot, "root", "/", "install root for emitted paths (default: /; for testing use a temp dir)")
|
||||
collectorStartCmd.Flags().BoolVar(&collectorDryRun, "dry-run", false, "emit scripts/units without enabling or running systemctl")
|
||||
collectorStopCmd.Flags().BoolVar(&collectorDryRun, "dry-run", false, "remove scripts/units without disabling or running systemctl")
|
||||
collectorCmd.AddCommand(collectorStartCmd)
|
||||
collectorCmd.AddCommand(collectorStopCmd)
|
||||
collectorCmd.AddCommand(collectorStatusCmd)
|
||||
rootCmd.AddCommand(collectorCmd)
|
||||
}
|
||||
|
||||
func collectorResolveRoot() (string, error) {
|
||||
r := strings.TrimRight(collectorRoot, "/")
|
||||
if r == "" {
|
||||
r = "/"
|
||||
}
|
||||
if !filepath.IsAbs(r) {
|
||||
return "", fmt.Errorf("--root must be absolute, got %q", collectorRoot)
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
func collectorEmit(root string) error {
|
||||
dirs := []string{
|
||||
filepath.Join(root, collectorScriptDir),
|
||||
filepath.Join(root, collectorUnitDir),
|
||||
filepath.Join(root, collectorStateDir),
|
||||
}
|
||||
for _, d := range dirs {
|
||||
if err := os.MkdirAll(d, 0o755); err != nil {
|
||||
return fmt.Errorf("mkdir %s: %w", d, err)
|
||||
}
|
||||
}
|
||||
files := map[string]struct {
|
||||
Content string
|
||||
Mode os.FileMode
|
||||
}{
|
||||
filepath.Join(root, collectorScriptDir, collectorAggregateSh): {collectorAggregateScript, 0o755},
|
||||
filepath.Join(root, collectorScriptDir, collectorWatchdogSh): {collectorWatchdogScript, 0o755},
|
||||
filepath.Join(root, collectorUnitDir, collectorAggregateSvc): {collectorAggregateUnit, 0o644},
|
||||
filepath.Join(root, collectorUnitDir, collectorAggregateTmr): {collectorAggregateTimer, 0o644},
|
||||
filepath.Join(root, collectorUnitDir, collectorWatchdogSvc): {collectorWatchdogUnit, 0o644},
|
||||
filepath.Join(root, collectorUnitDir, collectorWatchdogTmr): {collectorWatchdogTimer, 0o644},
|
||||
}
|
||||
for path, f := range files {
|
||||
tmp := path + ".tmp"
|
||||
if err := os.WriteFile(tmp, []byte(f.Content), f.Mode); err != nil {
|
||||
return fmt.Errorf("write %s: %w", path, err)
|
||||
}
|
||||
if err := os.Rename(tmp, path); err != nil {
|
||||
return fmt.Errorf("rename %s: %w", path, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func collectorRemove(root string) error {
|
||||
paths := []string{
|
||||
filepath.Join(root, collectorScriptDir, collectorAggregateSh),
|
||||
filepath.Join(root, collectorScriptDir, collectorWatchdogSh),
|
||||
filepath.Join(root, collectorUnitDir, collectorAggregateSvc),
|
||||
filepath.Join(root, collectorUnitDir, collectorAggregateTmr),
|
||||
filepath.Join(root, collectorUnitDir, collectorWatchdogSvc),
|
||||
filepath.Join(root, collectorUnitDir, collectorWatchdogTmr),
|
||||
}
|
||||
for _, p := range paths {
|
||||
if err := os.Remove(p); err != nil && !os.IsNotExist(err) {
|
||||
return fmt.Errorf("remove %s: %w", p, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func collectorRunning(root string) (bool, bool) {
|
||||
aggRunning := fileExists(filepath.Join(root, collectorUnitDir, collectorAggregateSvc)) &&
|
||||
fileExists(filepath.Join(root, collectorScriptDir, collectorAggregateSh))
|
||||
wdRunning := fileExists(filepath.Join(root, collectorUnitDir, collectorWatchdogSvc)) &&
|
||||
fileExists(filepath.Join(root, collectorScriptDir, collectorWatchdogSh))
|
||||
return aggRunning, wdRunning
|
||||
}
|
||||
|
||||
func fileExists(p string) bool {
|
||||
_, err := os.Stat(p)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func collectorEnable(root string) error {
|
||||
if !commandAvailable("systemctl") {
|
||||
return nil
|
||||
}
|
||||
for _, u := range []string{collectorAggregateTmr, collectorWatchdogTmr} {
|
||||
_ = runSystemctl(root, "enable", u)
|
||||
_ = runSystemctl(root, "start", u)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func collectorDisable(root string) error {
|
||||
if !commandAvailable("systemctl") {
|
||||
return nil
|
||||
}
|
||||
for _, u := range []string{collectorAggregateTmr, collectorWatchdogTmr} {
|
||||
_ = runSystemctl(root, "stop", u)
|
||||
_ = runSystemctl(root, "disable", u)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func runSystemctl(root, action, unit string) error {
|
||||
args := []string{action, unit}
|
||||
if root != "/" {
|
||||
args = append([]string{"--root", root}, args...)
|
||||
}
|
||||
return runCmd("systemctl", args...)
|
||||
}
|
||||
|
||||
func commandAvailable(name string) bool {
|
||||
_, err := exec.LookPath(name)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func runCmd(name string, args ...string) error {
|
||||
cmd := exec.Command(name, args...)
|
||||
cmd.Stdout = os.Stdout
|
||||
cmd.Stderr = os.Stderr
|
||||
return cmd.Run()
|
||||
}
|
||||
|
||||
const collectorAggregateScript = `#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
# orca-aggregate.sh — emitted by orca collector start (P09).
|
||||
# Placeholder wrapper; the canonical copy lives at scripts/orca-aggregate.sh.
|
||||
exec /usr/local/bin/orca-aggregate.sh "$@"
|
||||
`
|
||||
|
||||
const collectorWatchdogScript = `#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
# orca-watchdog.sh — emitted by orca collector start (P09).
|
||||
# Placeholder wrapper; the canonical copy lives at scripts/orca-watchdog.sh.
|
||||
exec /usr/local/bin/orca-watchdog.sh "$@"
|
||||
`
|
||||
|
||||
const collectorAggregateUnit = `[Unit]
|
||||
Description=orca aggregator (P09, C-11/C-12)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/etc/orca/collector/orca-aggregate.sh
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
`
|
||||
|
||||
const collectorAggregateTimer = `[Unit]
|
||||
Description=orca aggregator 10s cadence (P09, C-11)
|
||||
|
||||
[Timer]
|
||||
OnBootSec=10s
|
||||
OnUnitActiveSec=10s
|
||||
AccuracySec=1s
|
||||
Unit=orca-aggregate.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
`
|
||||
|
||||
const collectorWatchdogUnit = `[Unit]
|
||||
Description=orca watchdog meta-timer (P09, C-11)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/etc/orca/collector/orca-watchdog.sh
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
`
|
||||
|
||||
const collectorWatchdogTimer = `[Unit]
|
||||
Description=orca watchdog 30s cadence (P09, C-11)
|
||||
|
||||
[Timer]
|
||||
OnBootSec=30s
|
||||
OnUnitActiveSec=30s
|
||||
AccuracySec=5s
|
||||
Unit=orca-watchdog.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
`
|
||||
@@ -0,0 +1,160 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCollectorCmdRegistered(t *testing.T) {
|
||||
found := false
|
||||
for _, cmd := range rootCmd.Commands() {
|
||||
if cmd.Name() == "collector" {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("collector command not registered on root")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollectorSubcommandsRegistered(t *testing.T) {
|
||||
want := map[string]bool{"start": false, "stop": false, "status": false}
|
||||
for _, cmd := range collectorCmd.Commands() {
|
||||
if _, ok := want[cmd.Name()]; ok {
|
||||
want[cmd.Name()] = true
|
||||
}
|
||||
}
|
||||
for name, found := range want {
|
||||
if !found {
|
||||
t.Errorf("collector subcommand %q not registered", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func runCollectorCmd(t *testing.T, root string, dryRun bool, args ...string) (string, error) {
|
||||
t.Helper()
|
||||
resetRootFlags(t)
|
||||
full := append([]string{"collector"}, args...)
|
||||
if root != "" {
|
||||
full = append(full, "--root", root)
|
||||
}
|
||||
if dryRun && (len(args) > 0 && (args[0] == "start" || args[0] == "stop")) {
|
||||
full = append(full, "--dry-run")
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
rootCmd.SetOut(&buf)
|
||||
rootCmd.SetErr(&buf)
|
||||
rootCmd.SetArgs(full)
|
||||
err := rootCmd.Execute()
|
||||
return buf.String(), err
|
||||
}
|
||||
|
||||
func TestCollectorStartEmitsArtifacts(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
out, err := runCollectorCmd(t, root, true, "start")
|
||||
if err != nil {
|
||||
t.Fatalf("orca collector start: %v\n%s", err, out)
|
||||
}
|
||||
wantFiles := []string{
|
||||
filepath.Join(root, collectorScriptDir, collectorAggregateSh),
|
||||
filepath.Join(root, collectorScriptDir, collectorWatchdogSh),
|
||||
filepath.Join(root, collectorUnitDir, collectorAggregateSvc),
|
||||
filepath.Join(root, collectorUnitDir, collectorAggregateTmr),
|
||||
filepath.Join(root, collectorUnitDir, collectorWatchdogSvc),
|
||||
filepath.Join(root, collectorUnitDir, collectorWatchdogTmr),
|
||||
}
|
||||
for _, p := range wantFiles {
|
||||
if _, err := os.Stat(p); err != nil {
|
||||
t.Errorf("expected emitted file %s: %v", p, err)
|
||||
}
|
||||
}
|
||||
for _, p := range []string{
|
||||
filepath.Join(root, collectorScriptDir, collectorAggregateSh),
|
||||
filepath.Join(root, collectorScriptDir, collectorWatchdogSh),
|
||||
} {
|
||||
info, err := os.Stat(p)
|
||||
if err != nil {
|
||||
t.Fatalf("stat %s: %v", p, err)
|
||||
}
|
||||
if perm := info.Mode().Perm(); perm&0o111 == 0 {
|
||||
t.Errorf("expected executable bit on %s, got %o", p, perm)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollectorStartStatusRunning(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
if _, err := runCollectorCmd(t, root, true, "start"); err != nil {
|
||||
t.Fatalf("start: %v", err)
|
||||
}
|
||||
agg, wd := collectorRunning(root)
|
||||
if !agg || !wd {
|
||||
t.Errorf("expected both running, got agg=%t wd=%t", agg, wd)
|
||||
}
|
||||
out, err := runCollectorCmd(t, root, true, "status")
|
||||
if err != nil {
|
||||
t.Fatalf("status: %v", err)
|
||||
}
|
||||
if !contains(out, "running") {
|
||||
t.Errorf("status output should say running, got %q", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollectorStopRemovesArtifacts(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
if _, err := runCollectorCmd(t, root, true, "start"); err != nil {
|
||||
t.Fatalf("start: %v", err)
|
||||
}
|
||||
out, err := runCollectorCmd(t, root, true, "stop")
|
||||
if err != nil {
|
||||
t.Fatalf("stop: %v\n%s", err, out)
|
||||
}
|
||||
wantFiles := []string{
|
||||
filepath.Join(root, collectorScriptDir, collectorAggregateSh),
|
||||
filepath.Join(root, collectorScriptDir, collectorWatchdogSh),
|
||||
filepath.Join(root, collectorUnitDir, collectorAggregateSvc),
|
||||
filepath.Join(root, collectorUnitDir, collectorAggregateTmr),
|
||||
filepath.Join(root, collectorUnitDir, collectorWatchdogSvc),
|
||||
filepath.Join(root, collectorUnitDir, collectorWatchdogTmr),
|
||||
}
|
||||
for _, p := range wantFiles {
|
||||
if _, err := os.Stat(p); !os.IsNotExist(err) {
|
||||
t.Errorf("expected %s removed, got %v", p, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollectorStatusWhenStopped(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
out, err := runCollectorCmd(t, root, true, "status")
|
||||
if err != nil {
|
||||
t.Fatalf("status: %v", err)
|
||||
}
|
||||
if !contains(out, "stopped") {
|
||||
t.Errorf("expected stopped, got %q", out)
|
||||
}
|
||||
agg, wd := collectorRunning(root)
|
||||
if agg || wd {
|
||||
t.Errorf("expected neither running, got agg=%t wd=%t", agg, wd)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollectorResolveRootRejectsRelative(t *testing.T) {
|
||||
collectorRoot = "tmp/relative"
|
||||
_, err := collectorResolveRoot()
|
||||
if err == nil {
|
||||
t.Error("expected error for relative root")
|
||||
}
|
||||
collectorRoot = "/"
|
||||
r, err := collectorResolveRoot()
|
||||
if err != nil || r != "/" {
|
||||
t.Errorf("expected / for default, got %q err=%v", r, err)
|
||||
}
|
||||
}
|
||||
|
||||
func contains(haystack, needle string) bool {
|
||||
return bytes.Contains([]byte(haystack), []byte(needle))
|
||||
}
|
||||
@@ -43,6 +43,8 @@ func resetCommandFlags() {
|
||||
backupOutPath, restoreInPath, restoreTargetDir = "", "", ""
|
||||
restoreForce = false
|
||||
restoreDryRun = false
|
||||
collectorRoot = "/"
|
||||
collectorDryRun = false
|
||||
resetNSFlags()
|
||||
// Reset per-command output writers so tests that polluted them
|
||||
// (e.g. daemon tests calling cmd.SetOut(&buf)) don't leak into
|
||||
|
||||
Reference in New Issue
Block a user