orca/internal/cli/cluster_compat.go

package cli

import (
	"context"
	"encoding/json"
	"fmt"
	"log/slog"
	"strings"
	"time"

	"github.com/spf13/cobra"

	"git.cloudinit.dev/coreci/orca/internal/emit"
	"git.cloudinit.dev/coreci/orca/internal/model"
)

var noOrcaOnServerCmd = &cobra.Command{
	Use:   "no-orca-on-server",
	Short: "Verify no orca binary/service/process on peers (REQ-086, R-001, C-13)",
	Long: `SSH to each registered peer and verify that no orca binary,
systemd service, or process is present on the server (R-001: no orca
binary on any server; C-13 enforcement).

Checks per peer:
  1. command -v orca  → must return nothing (no orca in PATH)
  2. systemctl list-units 'orca*' (excluding orca-alloc-*) → must be empty
  3. pgrep orca → must return nothing (no orca process)
  4. /etc/orca/ contains no orca binaries (config dir is OK)

A peer with any violation is reported as FAIL. The exit code is non-zero
if any peer fails.`,
	Args: cobra.NoArgs,
	RunE: func(cmd *cobra.Command, args []string) error {
		return runNoOrcaOnServer(cmd)
	},
}

type noOrcaPeerResult struct {
	Node      string   `json:"node"`
	Peer      string   `json:"peer"`
	Pass      bool     `json:"pass"`
	Violations []string `json:"violations,omitempty"`
}

func runNoOrcaOnServer(cmd *cobra.Command) error {
	ctx, cancel := context.WithTimeout(cmd.Context(), 5*time.Minute)
	defer cancel()

	reg, closer, err := nodeRegistry()
	if err != nil {
		return err
	}
	defer closer()

	nodes, err := reg.List(ctx)
	if err != nil {
		return fmt.Errorf("list nodes: %w", err)
	}

	ex, err := drainExecFromCtx(cmd.Context())
	if err != nil {
		return fmt.Errorf("ssh transport: %w", err)
	}

	log := newLogger()
	results := make([]noOrcaPeerResult, 0, len(nodes))
	var failedNodes []string

	for i := range nodes {
		n := nodes[i]
		peer := peerAddrForNode(n)
		if peer == "" {
			continue
		}
		r := noOrcaPeerResult{Node: n.Name, Peer: peer, Pass: true, Violations: []string{}}

		if v, ok := checkNoOrcaBinary(ctx, ex, peer); !ok {
			r.Pass = false
			r.Violations = append(r.Violations, v)
		}
		if v, ok := checkNoOrcaService(ctx, ex, peer); !ok {
			r.Pass = false
			r.Violations = append(r.Violations, v)
		}
		if v, ok := checkNoOrcaProcess(ctx, ex, peer); !ok {
			r.Pass = false
			r.Violations = append(r.Violations, v)
		}
		if v, ok := checkNoOrcaBinInEtc(ctx, ex, peer); !ok {
			r.Pass = false
			r.Violations = append(r.Violations, v)
		}

		if !r.Pass {
			failedNodes = append(failedNodes, n.Name)
			log.Warn("no-orca-on-server: violations",
				slog.String("node", n.Name), slog.Any("violations", r.Violations))
		}
		results = append(results, r)
	}

	summary := map[string]any{
		"results": results,
		"failed":  failedNodes,
	}

	if jsonOutput {
		return printJSON(summary)
	}
	out := cmd.OutOrStdout()
	for _, r := range results {
		status := "PASS"
		if !r.Pass {
			status = "FAIL"
		}
		fmt.Fprintf(out, "%-20s %-5s  %s\n", r.Node, status, strings.Join(r.Violations, "; "))
	}
	if len(failedNodes) > 0 {
		fmt.Fprintf(out, "\n%d peer(s) failed R-001 enforcement\n", len(failedNodes))
		return fmt.Errorf("no-orca-on-server: %d peer(s) have violations", len(failedNodes))
	}
	fmt.Fprintf(out, "\n✓ all peers clean (R-001 enforced)\n")
	return nil
}

func checkNoOrcaBinary(ctx context.Context, ex drainExecer, peer string) (string, bool) {
	out, err := ex.Exec(ctx, peer, "command -v orca 2>/dev/null || true")
	if err != nil {
		return "", true
	}
	if strings.TrimSpace(string(out)) != "" {
		return fmt.Sprintf("orca binary in PATH: %s", strings.TrimSpace(string(out))), false
	}
	return "", true
}

func checkNoOrcaService(ctx context.Context, ex drainExecer, peer string) (string, bool) {
	cmd := "systemctl list-units 'orca*' --no-legend --no-pager 2>/dev/null | grep -v 'orca-alloc-' || true"
	out, err := ex.Exec(ctx, peer, cmd)
	if err != nil {
		return "", true
	}
	trimmed := strings.TrimSpace(string(out))
	if trimmed != "" {
		return fmt.Sprintf("orca systemd service(s) present: %s", trimmed), false
	}
	return "", true
}

func checkNoOrcaProcess(ctx context.Context, ex drainExecer, peer string) (string, bool) {
	out, err := ex.Exec(ctx, peer, "pgrep -x orca 2>/dev/null || true")
	if err != nil {
		return "", true
	}
	if strings.TrimSpace(string(out)) != "" {
		return fmt.Sprintf("orca process running: pid(s) %s", strings.TrimSpace(string(out))), false
	}
	return "", true
}

func checkNoOrcaBinInEtc(ctx context.Context, ex drainExecer, peer string) (string, bool) {
	cmd := "find /etc/orca -type f -executable 2>/dev/null | grep -v 'scripts/' | head -5 || true"
	out, err := ex.Exec(ctx, peer, cmd)
	if err != nil {
		return "", true
	}
	trimmed := strings.TrimSpace(string(out))
	if trimmed != "" {
		return fmt.Sprintf("executable(s) under /etc/orca: %s", trimmed), false
	}
	return "", true
}

var compatCheckCmd = &cobra.Command{
	Use:   "compat-check",
	Short: "Check mixed-version tolerance across peers (REQ-065, C-13)",
	Long: `Check that the cluster tolerates mixed orca versions during an
upgrade window (REQ-065). The lead and peers may run different orca
versions during a rolling upgrade; this command verifies:

  - Each peer's orca version (reported)
  - The txn manifest format is compatible across versions
  - The render-contract JSON schema (emit.SchemaVersion) is versioned
    and backward-compatible
  - No new required fields that old peers don't understand

Reports: which peers are on which version, any compatibility issues.`,
	Args: cobra.NoArgs,
	RunE: func(cmd *cobra.Command, args []string) error {
		return runCompatCheck(cmd)
	},
}

type compatPeerResult struct {
	Node            string `json:"node"`
	Peer            string `json:"peer"`
	Version         string `json:"version"`
	LeadVersion     string `json:"lead_version,omitempty"`
	Compatible      bool   `json:"compatible"`
	Issue           string `json:"issue,omitempty"`
}

func runCompatCheck(cmd *cobra.Command) error {
	ctx, cancel := context.WithTimeout(cmd.Context(), 5*time.Minute)
	defer cancel()

	reg, closer, err := nodeRegistry()
	if err != nil {
		return err
	}
	defer closer()

	nodes, err := reg.List(ctx)
	if err != nil {
		return fmt.Errorf("list nodes: %w", err)
	}

	ex, err := drainExecFromCtx(cmd.Context())
	if err != nil {
		return fmt.Errorf("ssh transport: %w", err)
	}

	leadVersion := version
	results := make([]compatPeerResult, 0, len(nodes))
	var issues []string
	versionSet := map[string]int{}

	for i := range nodes {
		n := nodes[i]
		peer := peerAddrForNode(n)
		if peer == "" {
			continue
		}
		peerVersion := detectPeerOrcaVersion(ctx, ex, peer)
		versionSet[peerVersion]++
		r := compatPeerResult{
			Node:        n.Name,
			Peer:        peer,
			Version:     peerVersion,
			LeadVersion: leadVersion,
			Compatible:  true,
		}
		if peerVersion != "" && peerVersion != leadVersion {
			if !versionsCompatible(leadVersion, peerVersion) {
				r.Compatible = false
				r.Issue = fmt.Sprintf("peer %s (%s) incompatible with lead (%s)",
					n.Name, peerVersion, leadVersion)
				issues = append(issues, r.Issue)
			}
		}
		results = append(results, r)
	}

	schemaOK := verifyRenderContractCompat(ctx, ex, nodes)
	if !schemaOK {
		issues = append(issues, "render-contract schema mismatch detected across peers")
	}
	manifestOK := verifyTxnManifestCompat(ctx, ex, nodes)
	if !manifestOK {
		issues = append(issues, "txn manifest format incompatibility detected")
	}

	summary := map[string]any{
		"lead_version":     leadVersion,
		"schema_version":   emit.SchemaVersion,
		"results":          results,
		"versions_seen":    versionSet,
		"issues":           issues,
		"schema_ok":        schemaOK,
		"manifest_ok":      manifestOK,
	}

	if jsonOutput {
		return printJSON(summary)
	}
	out := cmd.OutOrStdout()
	fmt.Fprintf(out, "lead version: %s (schema %s)\n", leadVersion, emit.SchemaVersion)
	for _, r := range results {
		mark := "✓"
		if !r.Compatible {
			mark = "✗"
		}
		fmt.Fprintf(out, "  %s %-20s %s\n", mark, r.Node, r.Version)
		if r.Issue != "" {
			fmt.Fprintf(out, "      %s\n", r.Issue)
		}
	}
	if len(issues) > 0 {
		fmt.Fprintf(out, "\n%d compatibility issue(s) found\n", len(issues))
		return fmt.Errorf("compat-check: %d issue(s)", len(issues))
	}
	fmt.Fprintf(out, "\n✓ all peers compatible\n")
	return nil
}

func detectPeerOrcaVersion(ctx context.Context, ex drainExecer, peer string) string {
	out, err := ex.Exec(ctx, peer, "orca version --json 2>/dev/null || true")
	if err != nil {
		return ""
	}
	s := strings.TrimSpace(string(out))
	if s == "" {
		return ""
	}
	var parsed map[string]any
	if err := json.Unmarshal([]byte(s), &parsed); err == nil {
		if v, ok := parsed["version"]; ok {
			if vs, ok := v.(string); ok && vs != "" {
				return vs
			}
		}
	}
	for _, line := range strings.Split(s, "\n") {
		line = strings.TrimSpace(line)
		if strings.Contains(line, "version") {
			fields := strings.Fields(line)
			for i, f := range fields {
				if f == "\"version\":" || f == "version:" {
					if i+1 < len(fields) {
						return strings.Trim(fields[i+1], "\",")
					}
				}
			}
		}
	}
	return s
}

func versionsCompatible(lead, peer string) bool {
	if lead == "" || peer == "" {
		return true
	}
	li := versionMinor(lead)
	pi := versionMinor(peer)
	if li == 0 || pi == 0 {
		return true
	}
	diff := li - pi
	if diff < 0 {
		diff = -diff
	}
	return diff <= 1
}

func versionMinor(v string) int {
	s := strings.TrimPrefix(v, "v")
	parts := strings.Split(s, ".")
	if len(parts) < 2 {
		return 0
	}
	var n int
	for _, c := range parts[1] {
		if c >= '0' && c <= '9' {
			n = n*10 + int(c-'0')
		} else {
			break
		}
	}
	return n
}

func verifyRenderContractCompat(ctx context.Context, ex drainExecer, nodes []*model.Node) bool {
	for i := range nodes {
		n := nodes[i]
		peer := peerAddrForNode(n)
		if peer == "" {
			continue
		}
		out, err := ex.Exec(ctx, peer, "test -f /etc/orca/cluster/render-contract.json && cat /etc/orca/cluster/render-contract.json || true")
		if err != nil {
			continue
		}
		s := strings.TrimSpace(string(out))
		if s == "" {
			continue
		}
		if !strings.Contains(s, emit.SchemaVersion) && !strings.Contains(s, "schema_version") {
			return false
		}
	}
	return true
}

func verifyTxnManifestCompat(ctx context.Context, ex drainExecer, nodes []*model.Node) bool {
	for i := range nodes {
		n := nodes[i]
		peer := peerAddrForNode(n)
		if peer == "" {
			continue
		}
		out, err := ex.Exec(ctx, peer, "test -d /etc/orca/cluster/txns && ls /etc/orca/cluster/txns | head -1 || true")
		if err != nil {
			continue
		}
		first := strings.TrimSpace(string(out))
		if first == "" {
			continue
		}
		man, err := ex.Exec(ctx, peer, fmt.Sprintf("cat /etc/orca/cluster/txns/%s/manifest.json 2>/dev/null || true", first))
		if err != nil {
			continue
		}
		s := strings.TrimSpace(string(man))
		if s == "" {
			continue
		}
		if !strings.Contains(s, "txn_id") || !strings.Contains(s, "files") {
			return false
		}
	}
	return true
}

func sshQuote(s string) string {
	return "'" + strings.ReplaceAll(s, "'", "'\\''") + "'"
}