feat(P14b,P14c): daemon cutover + rotate-lead (REQ-114) + mixed-version tolerance (REQ-065, REQ-086, C-13)
P14b: orca cluster cutover (stop v0.8 daemon, adopt running allocs); orca cluster rotate-lead --to (R-003 enforcement, CA+master key copy, SSH key rotation). P14c: orca doctor no-orca-on-server (R-001 enforcement); orca cluster compat-check (mixed-version tolerance). ---ci--- project: orca phase: 14b milestone: v0.11 status: execute ---/ci---
This commit is contained in:
Jon Chery
@@ -0,0 +1,417 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"git.cloudinit.dev/coreci/orca/internal/emit"
|
||||
"git.cloudinit.dev/coreci/orca/internal/model"
|
||||
)
|
||||
|
||||
var noOrcaOnServerCmd = &cobra.Command{
|
||||
Use: "no-orca-on-server",
|
||||
Short: "Verify no orca binary/service/process on peers (REQ-086, R-001, C-13)",
|
||||
Long: `SSH to each registered peer and verify that no orca binary,
|
||||
systemd service, or process is present on the server (R-001: no orca
|
||||
binary on any server; C-13 enforcement).
|
||||
|
||||
Checks per peer:
|
||||
1. command -v orca → must return nothing (no orca in PATH)
|
||||
2. systemctl list-units 'orca*' (excluding orca-alloc-*) → must be empty
|
||||
3. pgrep orca → must return nothing (no orca process)
|
||||
4. /etc/orca/ contains no orca binaries (config dir is OK)
|
||||
|
||||
A peer with any violation is reported as FAIL. The exit code is non-zero
|
||||
if any peer fails.`,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
return runNoOrcaOnServer(cmd)
|
||||
},
|
||||
}
|
||||
|
||||
type noOrcaPeerResult struct {
|
||||
Node string `json:"node"`
|
||||
Peer string `json:"peer"`
|
||||
Pass bool `json:"pass"`
|
||||
Violations []string `json:"violations,omitempty"`
|
||||
}
|
||||
|
||||
func runNoOrcaOnServer(cmd *cobra.Command) error {
|
||||
ctx, cancel := context.WithTimeout(cmd.Context(), 5*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
reg, closer, err := nodeRegistry()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closer()
|
||||
|
||||
nodes, err := reg.List(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list nodes: %w", err)
|
||||
}
|
||||
|
||||
ex, err := drainExecFromCtx(cmd.Context())
|
||||
if err != nil {
|
||||
return fmt.Errorf("ssh transport: %w", err)
|
||||
}
|
||||
|
||||
log := newLogger()
|
||||
results := make([]noOrcaPeerResult, 0, len(nodes))
|
||||
var failedNodes []string
|
||||
|
||||
for i := range nodes {
|
||||
n := nodes[i]
|
||||
peer := peerAddrForNode(n)
|
||||
if peer == "" {
|
||||
continue
|
||||
}
|
||||
r := noOrcaPeerResult{Node: n.Name, Peer: peer, Pass: true, Violations: []string{}}
|
||||
|
||||
if v, ok := checkNoOrcaBinary(ctx, ex, peer); !ok {
|
||||
r.Pass = false
|
||||
r.Violations = append(r.Violations, v)
|
||||
}
|
||||
if v, ok := checkNoOrcaService(ctx, ex, peer); !ok {
|
||||
r.Pass = false
|
||||
r.Violations = append(r.Violations, v)
|
||||
}
|
||||
if v, ok := checkNoOrcaProcess(ctx, ex, peer); !ok {
|
||||
r.Pass = false
|
||||
r.Violations = append(r.Violations, v)
|
||||
}
|
||||
if v, ok := checkNoOrcaBinInEtc(ctx, ex, peer); !ok {
|
||||
r.Pass = false
|
||||
r.Violations = append(r.Violations, v)
|
||||
}
|
||||
|
||||
if !r.Pass {
|
||||
failedNodes = append(failedNodes, n.Name)
|
||||
log.Warn("no-orca-on-server: violations",
|
||||
slog.String("node", n.Name), slog.Any("violations", r.Violations))
|
||||
}
|
||||
results = append(results, r)
|
||||
}
|
||||
|
||||
summary := map[string]any{
|
||||
"results": results,
|
||||
"failed": failedNodes,
|
||||
}
|
||||
|
||||
if jsonOutput {
|
||||
return printJSON(summary)
|
||||
}
|
||||
out := cmd.OutOrStdout()
|
||||
for _, r := range results {
|
||||
status := "PASS"
|
||||
if !r.Pass {
|
||||
status = "FAIL"
|
||||
}
|
||||
fmt.Fprintf(out, "%-20s %-5s %s\n", r.Node, status, strings.Join(r.Violations, "; "))
|
||||
}
|
||||
if len(failedNodes) > 0 {
|
||||
fmt.Fprintf(out, "\n%d peer(s) failed R-001 enforcement\n", len(failedNodes))
|
||||
return fmt.Errorf("no-orca-on-server: %d peer(s) have violations", len(failedNodes))
|
||||
}
|
||||
fmt.Fprintf(out, "\n✓ all peers clean (R-001 enforced)\n")
|
||||
return nil
|
||||
}
|
||||
|
||||
func checkNoOrcaBinary(ctx context.Context, ex drainExecer, peer string) (string, bool) {
|
||||
out, err := ex.Exec(ctx, peer, "command -v orca 2>/dev/null || true")
|
||||
if err != nil {
|
||||
return "", true
|
||||
}
|
||||
if strings.TrimSpace(string(out)) != "" {
|
||||
return fmt.Sprintf("orca binary in PATH: %s", strings.TrimSpace(string(out))), false
|
||||
}
|
||||
return "", true
|
||||
}
|
||||
|
||||
func checkNoOrcaService(ctx context.Context, ex drainExecer, peer string) (string, bool) {
|
||||
cmd := "systemctl list-units 'orca*' --no-legend --no-pager 2>/dev/null | grep -v 'orca-alloc-' || true"
|
||||
out, err := ex.Exec(ctx, peer, cmd)
|
||||
if err != nil {
|
||||
return "", true
|
||||
}
|
||||
trimmed := strings.TrimSpace(string(out))
|
||||
if trimmed != "" {
|
||||
return fmt.Sprintf("orca systemd service(s) present: %s", trimmed), false
|
||||
}
|
||||
return "", true
|
||||
}
|
||||
|
||||
func checkNoOrcaProcess(ctx context.Context, ex drainExecer, peer string) (string, bool) {
|
||||
out, err := ex.Exec(ctx, peer, "pgrep -x orca 2>/dev/null || true")
|
||||
if err != nil {
|
||||
return "", true
|
||||
}
|
||||
if strings.TrimSpace(string(out)) != "" {
|
||||
return fmt.Sprintf("orca process running: pid(s) %s", strings.TrimSpace(string(out))), false
|
||||
}
|
||||
return "", true
|
||||
}
|
||||
|
||||
func checkNoOrcaBinInEtc(ctx context.Context, ex drainExecer, peer string) (string, bool) {
|
||||
cmd := "find /etc/orca -type f -executable 2>/dev/null | grep -v 'scripts/' | head -5 || true"
|
||||
out, err := ex.Exec(ctx, peer, cmd)
|
||||
if err != nil {
|
||||
return "", true
|
||||
}
|
||||
trimmed := strings.TrimSpace(string(out))
|
||||
if trimmed != "" {
|
||||
return fmt.Sprintf("executable(s) under /etc/orca: %s", trimmed), false
|
||||
}
|
||||
return "", true
|
||||
}
|
||||
|
||||
var compatCheckCmd = &cobra.Command{
|
||||
Use: "compat-check",
|
||||
Short: "Check mixed-version tolerance across peers (REQ-065, C-13)",
|
||||
Long: `Check that the cluster tolerates mixed orca versions during an
|
||||
upgrade window (REQ-065). The lead and peers may run different orca
|
||||
versions during a rolling upgrade; this command verifies:
|
||||
|
||||
- Each peer's orca version (reported)
|
||||
- The txn manifest format is compatible across versions
|
||||
- The render-contract JSON schema (emit.SchemaVersion) is versioned
|
||||
and backward-compatible
|
||||
- No new required fields that old peers don't understand
|
||||
|
||||
Reports: which peers are on which version, any compatibility issues.`,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
return runCompatCheck(cmd)
|
||||
},
|
||||
}
|
||||
|
||||
type compatPeerResult struct {
|
||||
Node string `json:"node"`
|
||||
Peer string `json:"peer"`
|
||||
Version string `json:"version"`
|
||||
LeadVersion string `json:"lead_version,omitempty"`
|
||||
Compatible bool `json:"compatible"`
|
||||
Issue string `json:"issue,omitempty"`
|
||||
}
|
||||
|
||||
func runCompatCheck(cmd *cobra.Command) error {
|
||||
ctx, cancel := context.WithTimeout(cmd.Context(), 5*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
reg, closer, err := nodeRegistry()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closer()
|
||||
|
||||
nodes, err := reg.List(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list nodes: %w", err)
|
||||
}
|
||||
|
||||
ex, err := drainExecFromCtx(cmd.Context())
|
||||
if err != nil {
|
||||
return fmt.Errorf("ssh transport: %w", err)
|
||||
}
|
||||
|
||||
leadVersion := version
|
||||
results := make([]compatPeerResult, 0, len(nodes))
|
||||
var issues []string
|
||||
versionSet := map[string]int{}
|
||||
|
||||
for i := range nodes {
|
||||
n := nodes[i]
|
||||
peer := peerAddrForNode(n)
|
||||
if peer == "" {
|
||||
continue
|
||||
}
|
||||
peerVersion := detectPeerOrcaVersion(ctx, ex, peer)
|
||||
versionSet[peerVersion]++
|
||||
r := compatPeerResult{
|
||||
Node: n.Name,
|
||||
Peer: peer,
|
||||
Version: peerVersion,
|
||||
LeadVersion: leadVersion,
|
||||
Compatible: true,
|
||||
}
|
||||
if peerVersion != "" && peerVersion != leadVersion {
|
||||
if !versionsCompatible(leadVersion, peerVersion) {
|
||||
r.Compatible = false
|
||||
r.Issue = fmt.Sprintf("peer %s (%s) incompatible with lead (%s)",
|
||||
n.Name, peerVersion, leadVersion)
|
||||
issues = append(issues, r.Issue)
|
||||
}
|
||||
}
|
||||
results = append(results, r)
|
||||
}
|
||||
|
||||
schemaOK := verifyRenderContractCompat(ctx, ex, nodes)
|
||||
if !schemaOK {
|
||||
issues = append(issues, "render-contract schema mismatch detected across peers")
|
||||
}
|
||||
manifestOK := verifyTxnManifestCompat(ctx, ex, nodes)
|
||||
if !manifestOK {
|
||||
issues = append(issues, "txn manifest format incompatibility detected")
|
||||
}
|
||||
|
||||
summary := map[string]any{
|
||||
"lead_version": leadVersion,
|
||||
"schema_version": emit.SchemaVersion,
|
||||
"results": results,
|
||||
"versions_seen": versionSet,
|
||||
"issues": issues,
|
||||
"schema_ok": schemaOK,
|
||||
"manifest_ok": manifestOK,
|
||||
}
|
||||
|
||||
if jsonOutput {
|
||||
return printJSON(summary)
|
||||
}
|
||||
out := cmd.OutOrStdout()
|
||||
fmt.Fprintf(out, "lead version: %s (schema %s)\n", leadVersion, emit.SchemaVersion)
|
||||
for _, r := range results {
|
||||
mark := "✓"
|
||||
if !r.Compatible {
|
||||
mark = "✗"
|
||||
}
|
||||
fmt.Fprintf(out, " %s %-20s %s\n", mark, r.Node, r.Version)
|
||||
if r.Issue != "" {
|
||||
fmt.Fprintf(out, " %s\n", r.Issue)
|
||||
}
|
||||
}
|
||||
if len(issues) > 0 {
|
||||
fmt.Fprintf(out, "\n%d compatibility issue(s) found\n", len(issues))
|
||||
return fmt.Errorf("compat-check: %d issue(s)", len(issues))
|
||||
}
|
||||
fmt.Fprintf(out, "\n✓ all peers compatible\n")
|
||||
return nil
|
||||
}
|
||||
|
||||
func detectPeerOrcaVersion(ctx context.Context, ex drainExecer, peer string) string {
|
||||
out, err := ex.Exec(ctx, peer, "orca version --json 2>/dev/null || true")
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
s := strings.TrimSpace(string(out))
|
||||
if s == "" {
|
||||
return ""
|
||||
}
|
||||
var parsed map[string]any
|
||||
if err := json.Unmarshal([]byte(s), &parsed); err == nil {
|
||||
if v, ok := parsed["version"]; ok {
|
||||
if vs, ok := v.(string); ok && vs != "" {
|
||||
return vs
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, line := range strings.Split(s, "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if strings.Contains(line, "version") {
|
||||
fields := strings.Fields(line)
|
||||
for i, f := range fields {
|
||||
if f == "\"version\":" || f == "version:" {
|
||||
if i+1 < len(fields) {
|
||||
return strings.Trim(fields[i+1], "\",")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func versionsCompatible(lead, peer string) bool {
|
||||
if lead == "" || peer == "" {
|
||||
return true
|
||||
}
|
||||
li := versionMinor(lead)
|
||||
pi := versionMinor(peer)
|
||||
if li == 0 || pi == 0 {
|
||||
return true
|
||||
}
|
||||
diff := li - pi
|
||||
if diff < 0 {
|
||||
diff = -diff
|
||||
}
|
||||
return diff <= 1
|
||||
}
|
||||
|
||||
func versionMinor(v string) int {
|
||||
s := strings.TrimPrefix(v, "v")
|
||||
parts := strings.Split(s, ".")
|
||||
if len(parts) < 2 {
|
||||
return 0
|
||||
}
|
||||
var n int
|
||||
for _, c := range parts[1] {
|
||||
if c >= '0' && c <= '9' {
|
||||
n = n*10 + int(c-'0')
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func verifyRenderContractCompat(ctx context.Context, ex drainExecer, nodes []*model.Node) bool {
|
||||
for i := range nodes {
|
||||
n := nodes[i]
|
||||
peer := peerAddrForNode(n)
|
||||
if peer == "" {
|
||||
continue
|
||||
}
|
||||
out, err := ex.Exec(ctx, peer, "test -f /etc/orca/cluster/render-contract.json && cat /etc/orca/cluster/render-contract.json || true")
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
s := strings.TrimSpace(string(out))
|
||||
if s == "" {
|
||||
continue
|
||||
}
|
||||
if !strings.Contains(s, emit.SchemaVersion) && !strings.Contains(s, "schema_version") {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func verifyTxnManifestCompat(ctx context.Context, ex drainExecer, nodes []*model.Node) bool {
|
||||
for i := range nodes {
|
||||
n := nodes[i]
|
||||
peer := peerAddrForNode(n)
|
||||
if peer == "" {
|
||||
continue
|
||||
}
|
||||
out, err := ex.Exec(ctx, peer, "test -d /etc/orca/cluster/txns && ls /etc/orca/cluster/txns | head -1 || true")
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
first := strings.TrimSpace(string(out))
|
||||
if first == "" {
|
||||
continue
|
||||
}
|
||||
man, err := ex.Exec(ctx, peer, fmt.Sprintf("cat /etc/orca/cluster/txns/%s/manifest.json 2>/dev/null || true", first))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
s := strings.TrimSpace(string(man))
|
||||
if s == "" {
|
||||
continue
|
||||
}
|
||||
if !strings.Contains(s, "txn_id") || !strings.Contains(s, "files") {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func sshQuote(s string) string {
|
||||
return "'" + strings.ReplaceAll(s, "'", "'\\''") + "'"
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user