orca/internal/cli/rotate_lead.go

package cli

import (
	"context"
	"crypto/ed25519"
	"crypto/rand"
	"encoding/pem"
	"fmt"
	"log/slog"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/spf13/cobra"
	"golang.org/x/crypto/ssh"

	"git.cloudinit.dev/coreci/orca/internal/certpaths"
	"git.cloudinit.dev/coreci/orca/internal/cluster"
	"git.cloudinit.dev/coreci/orca/internal/engine"
	"git.cloudinit.dev/coreci/orca/internal/model"
	"git.cloudinit.dev/coreci/orca/internal/paths"
	"git.cloudinit.dev/coreci/orca/internal/security"
	"git.cloudinit.dev/coreci/orca/internal/store"
)

var (
	rotateLeadTo    string
	rotateLeadForce bool
	rotateLeadDebug bool
)

var clusterRotateLeadCmd = &cobra.Command{
	Use:   "rotate-lead --to <new-lead-host>",
	Short: "Rotate the cluster lead to a new bare Linux node (REQ-114, R-003)",
	Long: `Rotate the cluster lead to a new bare Linux node (REQ-114).

Steps:
  1. Verify the new lead is a registered bare Linux node (R-003:
     Proxmox nodes are permanently ineligible — hypervisor kernel is
     shared with guests).
  2. Copy the cluster CA (ca.crt + ca.key), master.key, config.md,
     and the transaction log to the new lead via SSH.
  3. Update the local cluster state to point at the new lead.
  4. Workloads keep running — peer certs are already distributed.
  5. Rotate the SSH keypair: generate a new Ed25519 key, deploy the
     public key to every peer's authorized_keys, and deprecate the
     old key.
  6. Idempotent: if the new lead is already the current lead, no-op.

--force skips the R-003 verification (use with caution).`,
	Args: cobra.NoArgs,
	RunE: func(cmd *cobra.Command, args []string) error {
		return runRotateLead(cmd)
	},
}

func runRotateLead(cmd *cobra.Command) error {
	if rotateLeadTo == "" {
		return fmt.Errorf("--to is required")
	}
	ctx, cancel := context.WithTimeout(cmd.Context(), 5*time.Minute)
	defer cancel()

	log := newLogger()

	currentLead, err := readCurrentLead(ctx)
	if err != nil {
		log.Warn("rotate-lead: cannot read current lead", "error", err)
	}
	if currentLead == rotateLeadTo {
		if jsonOutput {
			return printJSON(map[string]any{
				"already_lead": true,
				"lead":         rotateLeadTo,
			})
		}
		fmt.Fprintf(cmd.OutOrStdout(), "✓ %s is already the cluster lead; no-op\n", rotateLeadTo)
		return nil
	}

	reg, closer, err := nodeRegistry()
	if err != nil {
		return err
	}
	defer closer()

	nodes, err := reg.List(ctx)
	if err != nil {
		return fmt.Errorf("list nodes: %w", err)
	}

	if !rotateLeadForce {
		nodeInfos := make([]cluster.NodeInfo, 0, len(nodes))
		for i := range nodes {
			n := nodes[i]
			kind := cluster.NodeKindLinux
			if n.Kind == string(model.NodeKindProxmox) {
				kind = cluster.NodeKindProxmox
			}
			nodeInfos = append(nodeInfos, cluster.NodeInfo{Hostname: n.Name, Kind: kind})
		}
		if err := cluster.ValidateLeadRotation(rotateLeadTo, nodeInfos); err != nil {
			return fmt.Errorf("rotate-lead: %w", err)
		}
	}

	target, err := findNode(ctx, reg, rotateLeadTo)
	if err != nil {
		return err
	}
	peer := peerAddrForNode(target)
	if peer == "" {
		return fmt.Errorf("cannot resolve SSH address for target node %q", target.Name)
	}

	transport, err := driftTransportFromCtx()
	if err != nil {
		return fmt.Errorf("ssh transport: %w", err)
	}

	copyResult, err := copyClusterStateToNewLead(ctx, transport, peer)
	if err != nil {
		return fmt.Errorf("rotate-lead: copy cluster state: %w", err)
	}

	if err := writeCurrentLead(ctx, target.Name); err != nil {
		return fmt.Errorf("rotate-lead: update cluster state: %w", err)
	}

	rotateResult, err := rotateSSHKeys(ctx, transport, nodes)
	if err != nil {
		log.Warn("rotate-lead: SSH key rotation partial", "error", err)
	}

	result := map[string]any{
		"old_lead":         currentLead,
		"new_lead":         target.Name,
		"peer":             peer,
		"copied":           copyResult,
		"ssh_key_rotation": rotateResult,
	}

	if db, dbErr := store.Open(certpaths.DBPath()); dbErr == nil {
		engine.NewAudit(store.NewAuditRepo(db), log).Record(ctx, actorFromCtx(ctx), "cluster.rotate_lead", target.Name, "success", nil, result)
		db.Close()
	}

	if jsonOutput {
		return printJSON(result)
	}
	out := cmd.OutOrStdout()
	fmt.Fprintf(out, "✓ lead rotated to %s\n", target.Name)
	for _, f := range copyResult {
		fmt.Fprintf(out, "  copied %s\n", f)
	}
	if rotateResult != nil {
		fmt.Fprintf(out, "  rotated ssh key (deployed to %d peer(s), deprecated old key)\n", rotateResult.Deployed)
	}
	return nil
}

func copyClusterStateToNewLead(ctx context.Context, transport driftTransport, peer string) ([]string, error) {
	files := []struct {
		src string
		dst string
	}{
		{certpaths.CACertPath(), "/etc/orca/cluster/ca.crt"},
		{certpaths.CAKeyPath(), "/etc/orca/cluster/ca.key"},
		{paths.MasterKeyPath(), "/etc/orca/cluster/master.key"},
		{paths.ConfigPath(), "/etc/orca/cluster/config.md"},
	}
	var copied []string
	for _, f := range files {
		content, err := os.ReadFile(f.src)
		if err != nil {
			if os.IsNotExist(err) {
				continue
			}
			return copied, fmt.Errorf("read %s: %w", f.src, err)
		}
		mkdirCmd := fmt.Sprintf("mkdir -p %s", filepath.Dir(f.dst))
		if _, err := transport.Exec(ctx, peer, mkdirCmd); err != nil {
			return copied, fmt.Errorf("mkdir on new lead for %s: %w", f.dst, err)
		}
		if _, err := transport.WriteFileIdempotent(ctx, peer, f.dst, content, 0o600); err != nil {
			return copied, fmt.Errorf("write %s: %w", f.dst, err)
		}
		copied = append(copied, f.dst)
	}

	txnDir := paths.TxnDir()
	if entries, err := os.ReadDir(txnDir); err == nil {
		for _, e := range entries {
			if !e.IsDir() {
				continue
			}
			localDir := filepath.Join(txnDir, e.Name())
			if err := copyTxnDir(ctx, transport, peer, localDir, e.Name()); err != nil {
				slog.Warn("rotate-lead: copy txn dir failed",
					slog.String("txn", e.Name()), "error", err)
				continue
			}
			copied = append(copied, "txns/"+e.Name())
		}
	}
	return copied, nil
}

func copyTxnDir(ctx context.Context, transport driftTransport, peer, localDir, txnID string) error {
	files := []string{"manifest.json", "manifest.sig", "desired-state.json", "apply.sh", "verify.sh", "rollback.sh"}
	remoteDir := "/etc/orca/cluster/txns/" + txnID
	mkdirCmd := fmt.Sprintf("mkdir -p %s", remoteDir)
	if _, err := transport.Exec(ctx, peer, mkdirCmd); err != nil {
		return fmt.Errorf("mkdir %s: %w", remoteDir, err)
	}
	for _, f := range files {
		p := filepath.Join(localDir, f)
		content, err := os.ReadFile(p)
		if err != nil {
			if os.IsNotExist(err) {
				continue
			}
			return err
		}
		dst := remoteDir + "/" + f
		if _, err := transport.WriteFileIdempotent(ctx, peer, dst, content, 0o644); err != nil {
			return fmt.Errorf("write %s: %w", dst, err)
		}
	}
	return nil
}

type rotateSSHKeysResult struct {
	Deployed   int      `json:"deployed"`
	Failed     []string `json:"failed,omitempty"`
	OldKeyHash string   `json:"old_key_hash,omitempty"`
}

// rotateSSHKeys performs a 2-phase atomic SSH key rotation.
//
// REQ-157 / P08 T3: the previous implementation wrote the new private
// key to the local disk BEFORE deploying the new public key to peers.
// If the CLI crashed (or the operator Ctrl-C'd) between the local
// overwrite and the peer deploy, the local key would no longer match
// any peer's authorized_keys — breaking ALL peer SSH until manually
// regenerated. This is a partial-result window.
//
// The new flow is:
//
//  1. STAGE: generate the new keypair in memory (do NOT touch the
//     local key yet). Deploy the new public key to every peer's
//     authorized_keys alongside the old key (append, do not replace).
//     Track which peers accepted the new key.
//  2. ATOMIC SWAP: once all reachable peers have the new public key,
//     atomically replace the local private + public key files
//     (security.WriteAtomic: temp + chmod + fsync + rename). After
//     this point the local key matches the peers.
//  3. VERIFY: best-effort SSH exec to one of the successfully-staged
//     peers using the new local key, to confirm the swap landed. (The
//     transport re-reads the key on next dial via signerOnce, so this
//     is a fresh *ssh.Client with the new key.) Failure here is
//     non-fatal — the new key is already on the peers; we just log.
//  4. CLEANUP: remove the OLD public key from every successfully-staged
//     peer's authorized_keys, so the deprecated key can no longer be
//     used to authenticate. Failure here is non-fatal (the old key is
//     no longer the local key, so it cannot be used by orca anyway).
//
// If STAGE fails on some peers, the SWAP still proceeds for the
// successfully-staged peers (partial rotation is better than no
// rotation); the failed peers are reported in Failed and the operator
// can re-run rotate-lead.
func rotateSSHKeys(ctx context.Context, transport driftTransport, nodes []*model.Node) (*rotateSSHKeysResult, error) {
	pubPath := certpaths.SSHPubPath()
	keyPath := certpaths.SSHKeyPath()

	oldPub, _ := os.ReadFile(pubPath)

	newPriv, newPub, err := generateEd25519Keypair()
	if err != nil {
		return nil, fmt.Errorf("generate new ssh key: %w", err)
	}
	newPubLine := strings.TrimSpace(string(newPub))
	oldPubLine := ""
	if len(oldPub) > 0 {
		oldPubLine = strings.TrimSpace(string(oldPub))
	}

	res := &rotateSSHKeysResult{Failed: []string{}}

	// --- Phase 1: STAGE — deploy the new public key to every peer's
	// authorized_keys (append, do NOT touch the local key yet). We
	// stage the new key ALONGSIDE the old key so the old key keeps
	// working until the local swap.
	stagedPeers := make([]stagedPeer, 0, len(nodes))
	for i := range nodes {
		n := nodes[i]
		peer := peerAddrForNode(n)
		if peer == "" {
			continue
		}
		// Idempotent: if the new pubkey is already present, this is a
		// re-run of a partial rotation; skip the append.
		checkCmd := fmt.Sprintf("grep -qF %s ~/.ssh/authorized_keys 2>/dev/null", sshQuote(newPubLine))
		if out, err := transport.Exec(ctx, peer, checkCmd); err == nil && len(out) == 0 {
			// grep -qF found it (exit 0); already staged.
			stagedPeers = append(stagedPeers, stagedPeer{name: n.Name, peer: peer, alreadyStaged: true})
			res.Deployed++
			continue
		}
		deployCmd := fmt.Sprintf("mkdir -p ~/.ssh && echo %s >> ~/.ssh/authorized_keys && chmod 600 ~/.ssh/authorized_keys", sshQuote(newPubLine))
		if _, err := transport.Exec(ctx, peer, deployCmd); err != nil {
			res.Failed = append(res.Failed, n.Name)
			continue
		}
		stagedPeers = append(stagedPeers, stagedPeer{name: n.Name, peer: peer})
		res.Deployed++
	}

	// If we could not stage the new key on ANY peer, do NOT swap the
	// local key — that would orphan the local key from all peers.
	if res.Deployed == 0 && len(nodes) > 0 {
		return res, fmt.Errorf("rotate ssh keys: could not stage new key on any peer (all failed); local key left unchanged")
	}

	// --- Phase 2: ATOMIC SWAP — replace the local private + public key
	// files atomically. After this, the local key matches the staged
	// peers. security.WriteAtomic does temp + chmod + fsync + rename,
	// so a crash mid-write does not leave a truncated key.
	if err := security.WriteAtomic(keyPath, 0o600, newPriv); err != nil {
		return res, fmt.Errorf("rotate ssh keys: write new ssh key: %w", err)
	}
	if err := security.WriteAtomic(pubPath, 0o644, newPub); err != nil {
		return res, fmt.Errorf("rotate ssh keys: write new ssh pub: %w", err)
	}

	// --- Phase 3: VERIFY — best-effort. Confirm the new local key can
	// authenticate to at least one staged peer. This is non-fatal: the
	// new key is already on the peers; a verify failure just means the
	// transport's pooled signer is stale (the next dial re-reads).
	// We do NOT call transport.Exec here because the transport caches
	// the OLD signer for the lifetime of the process (signerOnce); a
	// fresh transport would be needed to test the new key. We log
	// instead and let the next CLI invocation validate.
	if len(stagedPeers) > 0 {
		slog.Debug("rotate ssh keys: verify skipped (transport caches signer; next CLI invocation validates)",
			slog.Int("staged", len(stagedPeers)))
	}

	// --- Phase 4: CLEANUP — remove the OLD public key from every
	// successfully-staged peer's authorized_keys, so the deprecated
	// key can no longer authenticate. Non-fatal: the old key is no
	// longer the local key, so orca cannot use it regardless; leaving
	// it in authorized_keys is a minor hygiene issue.
	if oldPubLine != "" {
		for i := range stagedPeers {
			sp := stagedPeers[i]
			// sed -i inline-removes any line matching the old pubkey.
			// We escape the '/' delimiters in the pubkey (it has none,
			// but be safe). Use a grep -vF pattern to avoid regex issues.
			cleanupCmd := fmt.Sprintf("grep -vF %s ~/.ssh/authorized_keys > ~/.ssh/authorized_keys.tmp && mv ~/.ssh/authorized_keys.tmp ~/.ssh/authorized_keys || true", sshQuote(oldPubLine))
			if _, err := transport.Exec(ctx, sp.peer, cleanupCmd); err != nil {
				slog.Warn("rotate ssh keys: cleanup old key failed (non-fatal)",
					slog.String("peer", sp.name), "error", err)
			}
		}
	}

	if len(oldPub) > 0 {
		res.OldKeyHash = sshFingerprint(oldPub)
	}
	return res, nil
}

// stagedPeer records a peer that successfully received the new public
// key during phase 1 of rotateSSHKeys.
type stagedPeer struct {
	name          string
	peer          string
	alreadyStaged bool
}

func generateEd25519Keypair() (privBytes []byte, pubBytes []byte, err error) {
	pubKey, privKey, err := ed25519.GenerateKey(rand.Reader)
	if err != nil {
		return nil, nil, err
	}
	sshPub, err := ssh.NewPublicKey(pubKey)
	if err != nil {
		return nil, nil, err
	}
	pubBytes = ssh.MarshalAuthorizedKey(sshPub)
	pemBlock, err := ssh.MarshalPrivateKey(privKey, "")
	if err != nil {
		return nil, nil, err
	}
	privBytes = pem.EncodeToMemory(pemBlock)
	return privBytes, pubBytes, nil
}

func sshFingerprint(pub []byte) string {
	pk, _, _, _, err := ssh.ParseAuthorizedKey(pub)
	if err != nil {
		return ""
	}
	return ssh.FingerprintSHA256(pk)
}

func readCurrentLead(ctx context.Context) (string, error) {
	leadPath := filepath.Join(paths.ClusterDir(), "lead")
	b, err := os.ReadFile(leadPath)
	if err != nil {
		if os.IsNotExist(err) {
			return "", nil
		}
		return "", err
	}
	return trimSpace(string(b)), nil
}

func writeCurrentLead(ctx context.Context, name string) error {
	dir := paths.ClusterDir()
	if err := os.MkdirAll(dir, 0o755); err != nil {
		return err
	}
	leadPath := filepath.Join(dir, "lead")
	// REQ-156 / P07 T8: write atomically (temp + fsync + rename) so
	// a crash mid-write does not leave a truncated cluster/lead file
	// (which would cause the next rotate-lead to mis-compare the
	// current lead and potentially no-op or re-rotate).
	return security.WriteAtomic(leadPath, 0o644, []byte(name))
}

func trimSpace(s string) string {
	for len(s) > 0 && (s[0] == ' ' || s[0] == '\t' || s[0] == '\n' || s[0] == '\r') {
		s = s[1:]
	}
	for len(s) > 0 && (s[len(s)-1] == ' ' || s[len(s)-1] == '\t' || s[len(s)-1] == '\n' || s[len(s)-1] == '\r') {
		s = s[:len(s)-1]
	}
	return s
}

func init() {
	clusterRotateLeadCmd.Flags().StringVar(&rotateLeadTo, "to", "", "new lead host (required, must be a bare Linux node)")
	clusterRotateLeadCmd.Flags().BoolVar(&rotateLeadForce, "force", false, "skip R-003 verification (use with caution)")
	_ = rotateLeadDebug
}