package cli import ( "context" "encoding/json" "fmt" "log/slog" "strings" "time" "github.com/spf13/cobra" "git.cloudinit.dev/coreci/orca/internal/emit" "git.cloudinit.dev/coreci/orca/internal/model" ) var noOrcaOnServerCmd = &cobra.Command{ Use: "no-orca-on-server", Short: "Verify no orca binary/service/process on peers (REQ-086, R-001, C-13)", Long: `SSH to each registered peer and verify that no orca binary, systemd service, or process is present on the server (R-001: no orca binary on any server; C-13 enforcement). Checks per peer: 1. command -v orca → must return nothing (no orca in PATH) 2. systemctl list-units 'orca*' (excluding orca-alloc-*) → must be empty 3. pgrep orca → must return nothing (no orca process) 4. /etc/orca/ contains no orca binaries (config dir is OK) A peer with any violation is reported as FAIL. The exit code is non-zero if any peer fails.`, Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { return runNoOrcaOnServer(cmd) }, } type noOrcaPeerResult struct { Node string `json:"node"` Peer string `json:"peer"` Pass bool `json:"pass"` Violations []string `json:"violations,omitempty"` } func runNoOrcaOnServer(cmd *cobra.Command) error { ctx, cancel := context.WithTimeout(cmd.Context(), 5*time.Minute) defer cancel() reg, closer, err := nodeRegistry() if err != nil { return err } defer closer() nodes, err := reg.List(ctx) if err != nil { return fmt.Errorf("list nodes: %w", err) } ex, err := drainExecFromCtx(cmd.Context()) if err != nil { return fmt.Errorf("ssh transport: %w", err) } log := newLogger() results := make([]noOrcaPeerResult, 0, len(nodes)) var failedNodes []string for i := range nodes { n := nodes[i] peer := peerAddrForNode(n) if peer == "" { continue } r := noOrcaPeerResult{Node: n.Name, Peer: peer, Pass: true, Violations: []string{}} if v, ok := checkNoOrcaBinary(ctx, ex, peer); !ok { r.Pass = false r.Violations = append(r.Violations, v) } if v, ok := checkNoOrcaService(ctx, ex, peer); !ok { r.Pass = false r.Violations = append(r.Violations, v) } if v, ok := checkNoOrcaProcess(ctx, ex, peer); !ok { r.Pass = false r.Violations = append(r.Violations, v) } if v, ok := checkNoOrcaBinInEtc(ctx, ex, peer); !ok { r.Pass = false r.Violations = append(r.Violations, v) } if !r.Pass { failedNodes = append(failedNodes, n.Name) log.Warn("no-orca-on-server: violations", slog.String("node", n.Name), slog.Any("violations", r.Violations)) } results = append(results, r) } summary := map[string]any{ "results": results, "failed": failedNodes, } if jsonOutput { return printJSON(summary) } out := cmd.OutOrStdout() for _, r := range results { status := "PASS" if !r.Pass { status = "FAIL" } fmt.Fprintf(out, "%-20s %-5s %s\n", r.Node, status, strings.Join(r.Violations, "; ")) } if len(failedNodes) > 0 { fmt.Fprintf(out, "\n%d peer(s) failed R-001 enforcement\n", len(failedNodes)) return fmt.Errorf("no-orca-on-server: %d peer(s) have violations", len(failedNodes)) } fmt.Fprintf(out, "\n✓ all peers clean (R-001 enforced)\n") return nil } func checkNoOrcaBinary(ctx context.Context, ex drainExecer, peer string) (string, bool) { out, err := ex.Exec(ctx, peer, "command -v orca 2>/dev/null || true") if err != nil { return "", true } if strings.TrimSpace(string(out)) != "" { return fmt.Sprintf("orca binary in PATH: %s", strings.TrimSpace(string(out))), false } return "", true } func checkNoOrcaService(ctx context.Context, ex drainExecer, peer string) (string, bool) { cmd := "systemctl list-units 'orca*' --no-legend --no-pager 2>/dev/null | grep -v 'orca-alloc-' || true" out, err := ex.Exec(ctx, peer, cmd) if err != nil { return "", true } trimmed := strings.TrimSpace(string(out)) if trimmed != "" { return fmt.Sprintf("orca systemd service(s) present: %s", trimmed), false } return "", true } func checkNoOrcaProcess(ctx context.Context, ex drainExecer, peer string) (string, bool) { out, err := ex.Exec(ctx, peer, "pgrep -x orca 2>/dev/null || true") if err != nil { return "", true } if strings.TrimSpace(string(out)) != "" { return fmt.Sprintf("orca process running: pid(s) %s", strings.TrimSpace(string(out))), false } return "", true } func checkNoOrcaBinInEtc(ctx context.Context, ex drainExecer, peer string) (string, bool) { cmd := "find /etc/orca -type f -executable 2>/dev/null | grep -v 'scripts/' | head -5 || true" out, err := ex.Exec(ctx, peer, cmd) if err != nil { return "", true } trimmed := strings.TrimSpace(string(out)) if trimmed != "" { return fmt.Sprintf("executable(s) under /etc/orca: %s", trimmed), false } return "", true } var compatCheckCmd = &cobra.Command{ Use: "compat-check", Short: "Check mixed-version tolerance across peers (REQ-065, C-13)", Long: `Check that the cluster tolerates mixed orca versions during an upgrade window (REQ-065). The lead and peers may run different orca versions during a rolling upgrade; this command verifies: - Each peer's orca version (reported) - The txn manifest format is compatible across versions - The render-contract JSON schema (emit.SchemaVersion) is versioned and backward-compatible - No new required fields that old peers don't understand Reports: which peers are on which version, any compatibility issues.`, Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { return runCompatCheck(cmd) }, } type compatPeerResult struct { Node string `json:"node"` Peer string `json:"peer"` Version string `json:"version"` LeadVersion string `json:"lead_version,omitempty"` Compatible bool `json:"compatible"` Issue string `json:"issue,omitempty"` } func runCompatCheck(cmd *cobra.Command) error { ctx, cancel := context.WithTimeout(cmd.Context(), 5*time.Minute) defer cancel() reg, closer, err := nodeRegistry() if err != nil { return err } defer closer() nodes, err := reg.List(ctx) if err != nil { return fmt.Errorf("list nodes: %w", err) } ex, err := drainExecFromCtx(cmd.Context()) if err != nil { return fmt.Errorf("ssh transport: %w", err) } leadVersion := version results := make([]compatPeerResult, 0, len(nodes)) var issues []string versionSet := map[string]int{} for i := range nodes { n := nodes[i] peer := peerAddrForNode(n) if peer == "" { continue } peerVersion := detectPeerOrcaVersion(ctx, ex, peer) versionSet[peerVersion]++ r := compatPeerResult{ Node: n.Name, Peer: peer, Version: peerVersion, LeadVersion: leadVersion, Compatible: true, } if peerVersion != "" && peerVersion != leadVersion { if !versionsCompatible(leadVersion, peerVersion) { r.Compatible = false r.Issue = fmt.Sprintf("peer %s (%s) incompatible with lead (%s)", n.Name, peerVersion, leadVersion) issues = append(issues, r.Issue) } } results = append(results, r) } schemaOK := verifyRenderContractCompat(ctx, ex, nodes) if !schemaOK { issues = append(issues, "render-contract schema mismatch detected across peers") } manifestOK := verifyTxnManifestCompat(ctx, ex, nodes) if !manifestOK { issues = append(issues, "txn manifest format incompatibility detected") } summary := map[string]any{ "lead_version": leadVersion, "schema_version": emit.SchemaVersion, "results": results, "versions_seen": versionSet, "issues": issues, "schema_ok": schemaOK, "manifest_ok": manifestOK, } if jsonOutput { return printJSON(summary) } out := cmd.OutOrStdout() fmt.Fprintf(out, "lead version: %s (schema %s)\n", leadVersion, emit.SchemaVersion) for _, r := range results { mark := "✓" if !r.Compatible { mark = "✗" } fmt.Fprintf(out, " %s %-20s %s\n", mark, r.Node, r.Version) if r.Issue != "" { fmt.Fprintf(out, " %s\n", r.Issue) } } if len(issues) > 0 { fmt.Fprintf(out, "\n%d compatibility issue(s) found\n", len(issues)) return fmt.Errorf("compat-check: %d issue(s)", len(issues)) } fmt.Fprintf(out, "\n✓ all peers compatible\n") return nil } func detectPeerOrcaVersion(ctx context.Context, ex drainExecer, peer string) string { out, err := ex.Exec(ctx, peer, "orca version --json 2>/dev/null || true") if err != nil { return "" } s := strings.TrimSpace(string(out)) if s == "" { return "" } var parsed map[string]any if err := json.Unmarshal([]byte(s), &parsed); err == nil { if v, ok := parsed["version"]; ok { if vs, ok := v.(string); ok && vs != "" { return vs } } } for _, line := range strings.Split(s, "\n") { line = strings.TrimSpace(line) if strings.Contains(line, "version") { fields := strings.Fields(line) for i, f := range fields { if f == "\"version\":" || f == "version:" { if i+1 < len(fields) { return strings.Trim(fields[i+1], "\",") } } } } } return s } func versionsCompatible(lead, peer string) bool { if lead == "" || peer == "" { return true } li := versionMinor(lead) pi := versionMinor(peer) if li == 0 || pi == 0 { return true } diff := li - pi if diff < 0 { diff = -diff } return diff <= 1 } func versionMinor(v string) int { s := strings.TrimPrefix(v, "v") parts := strings.Split(s, ".") if len(parts) < 2 { return 0 } var n int for _, c := range parts[1] { if c >= '0' && c <= '9' { n = n*10 + int(c-'0') } else { break } } return n } func verifyRenderContractCompat(ctx context.Context, ex drainExecer, nodes []*model.Node) bool { for i := range nodes { n := nodes[i] peer := peerAddrForNode(n) if peer == "" { continue } out, err := ex.Exec(ctx, peer, "test -f /etc/orca/cluster/render-contract.json && cat /etc/orca/cluster/render-contract.json || true") if err != nil { continue } s := strings.TrimSpace(string(out)) if s == "" { continue } if !strings.Contains(s, emit.SchemaVersion) && !strings.Contains(s, "schema_version") { return false } } return true } func verifyTxnManifestCompat(ctx context.Context, ex drainExecer, nodes []*model.Node) bool { for i := range nodes { n := nodes[i] peer := peerAddrForNode(n) if peer == "" { continue } out, err := ex.Exec(ctx, peer, "test -d /etc/orca/cluster/txns && ls /etc/orca/cluster/txns | head -1 || true") if err != nil { continue } first := strings.TrimSpace(string(out)) if first == "" { continue } man, err := ex.Exec(ctx, peer, fmt.Sprintf("cat /etc/orca/cluster/txns/%s/manifest.json 2>/dev/null || true", first)) if err != nil { continue } s := strings.TrimSpace(string(man)) if s == "" { continue } if !strings.Contains(s, "txn_id") || !strings.Contains(s, "files") { return false } } return true } func sshQuote(s string) string { return "'" + strings.ReplaceAll(s, "'", "'\\''") + "'" }