orca/internal/cli/rotate_lead.go

package cli

import (
	"context"
	"crypto/ed25519"
	"crypto/rand"
	"encoding/pem"
	"fmt"
	"log/slog"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/spf13/cobra"
	"golang.org/x/crypto/ssh"

	"git.cloudinit.dev/coreci/orca/internal/certpaths"
	"git.cloudinit.dev/coreci/orca/internal/cluster"
	"git.cloudinit.dev/coreci/orca/internal/engine"
	"git.cloudinit.dev/coreci/orca/internal/model"
	"git.cloudinit.dev/coreci/orca/internal/paths"
	"git.cloudinit.dev/coreci/orca/internal/store"
)

var (
	rotateLeadTo    string
	rotateLeadForce bool
	rotateLeadDebug bool
)

var clusterRotateLeadCmd = &cobra.Command{
	Use:   "rotate-lead --to <new-lead-host>",
	Short: "Rotate the cluster lead to a new bare Linux node (REQ-114, R-003)",
	Long: `Rotate the cluster lead to a new bare Linux node (REQ-114).

Steps:
  1. Verify the new lead is a registered bare Linux node (R-003:
     Proxmox nodes are permanently ineligible — hypervisor kernel is
     shared with guests).
  2. Copy the cluster CA (ca.crt + ca.key), master.key, config.md,
     and the transaction log to the new lead via SSH.
  3. Update the local cluster state to point at the new lead.
  4. Workloads keep running — peer certs are already distributed.
  5. Rotate the SSH keypair: generate a new Ed25519 key, deploy the
     public key to every peer's authorized_keys, and deprecate the
     old key.
  6. Idempotent: if the new lead is already the current lead, no-op.

--force skips the R-003 verification (use with caution).`,
	Args: cobra.NoArgs,
	RunE: func(cmd *cobra.Command, args []string) error {
		return runRotateLead(cmd)
	},
}

func runRotateLead(cmd *cobra.Command) error {
	if rotateLeadTo == "" {
		return fmt.Errorf("--to is required")
	}
	ctx, cancel := context.WithTimeout(cmd.Context(), 5*time.Minute)
	defer cancel()

	log := newLogger()

	currentLead, err := readCurrentLead(ctx)
	if err != nil {
		log.Warn("rotate-lead: cannot read current lead", "error", err)
	}
	if currentLead == rotateLeadTo {
		if jsonOutput {
			return printJSON(map[string]any{
				"already_lead": true,
				"lead":         rotateLeadTo,
			})
		}
		fmt.Fprintf(cmd.OutOrStdout(), "✓ %s is already the cluster lead; no-op\n", rotateLeadTo)
		return nil
	}

	reg, closer, err := nodeRegistry()
	if err != nil {
		return err
	}
	defer closer()

	nodes, err := reg.List(ctx)
	if err != nil {
		return fmt.Errorf("list nodes: %w", err)
	}

	if !rotateLeadForce {
		nodeInfos := make([]cluster.NodeInfo, 0, len(nodes))
		for i := range nodes {
			n := nodes[i]
			kind := cluster.NodeKindLinux
			if n.Kind == string(model.NodeKindProxmox) {
				kind = cluster.NodeKindProxmox
			}
			nodeInfos = append(nodeInfos, cluster.NodeInfo{Hostname: n.Name, Kind: kind})
		}
		if err := cluster.ValidateLeadRotation(rotateLeadTo, nodeInfos); err != nil {
			return fmt.Errorf("rotate-lead: %w", err)
		}
	}

	target, err := findNode(ctx, reg, rotateLeadTo)
	if err != nil {
		return err
	}
	peer := peerAddrForNode(target)
	if peer == "" {
		return fmt.Errorf("cannot resolve SSH address for target node %q", target.Name)
	}

	transport, err := driftTransportFromCtx()
	if err != nil {
		return fmt.Errorf("ssh transport: %w", err)
	}

	copyResult, err := copyClusterStateToNewLead(ctx, transport, peer)
	if err != nil {
		return fmt.Errorf("rotate-lead: copy cluster state: %w", err)
	}

	if err := writeCurrentLead(ctx, target.Name); err != nil {
		return fmt.Errorf("rotate-lead: update cluster state: %w", err)
	}

	rotateResult, err := rotateSSHKeys(ctx, transport, nodes)
	if err != nil {
		log.Warn("rotate-lead: SSH key rotation partial", "error", err)
	}

	result := map[string]any{
		"old_lead":         currentLead,
		"new_lead":         target.Name,
		"peer":             peer,
		"copied":           copyResult,
		"ssh_key_rotation": rotateResult,
	}

	if db, dbErr := store.Open(certpaths.DBPath()); dbErr == nil {
		engine.NewAudit(store.NewAuditRepo(db), log).Record(ctx, "cli", "cluster.rotate_lead", target.Name, "success", nil, result)
		db.Close()
	}

	if jsonOutput {
		return printJSON(result)
	}
	out := cmd.OutOrStdout()
	fmt.Fprintf(out, "✓ lead rotated to %s\n", target.Name)
	for _, f := range copyResult {
		fmt.Fprintf(out, "  copied %s\n", f)
	}
	if rotateResult != nil {
		fmt.Fprintf(out, "  rotated ssh key (deployed to %d peer(s), deprecated old key)\n", rotateResult.Deployed)
	}
	return nil
}

func copyClusterStateToNewLead(ctx context.Context, transport driftTransport, peer string) ([]string, error) {
	files := []struct {
		src string
		dst string
	}{
		{certpaths.CACertPath(), "/etc/orca/cluster/ca.crt"},
		{certpaths.CAKeyPath(), "/etc/orca/cluster/ca.key"},
		{paths.MasterKeyPath(), "/etc/orca/cluster/master.key"},
		{paths.ConfigPath(), "/etc/orca/cluster/config.md"},
	}
	var copied []string
	for _, f := range files {
		content, err := os.ReadFile(f.src)
		if err != nil {
			if os.IsNotExist(err) {
				continue
			}
			return copied, fmt.Errorf("read %s: %w", f.src, err)
		}
		mkdirCmd := fmt.Sprintf("mkdir -p %s", filepath.Dir(f.dst))
		if _, err := transport.Exec(ctx, peer, mkdirCmd); err != nil {
			return copied, fmt.Errorf("mkdir on new lead for %s: %w", f.dst, err)
		}
		if _, err := transport.WriteFileIdempotent(ctx, peer, f.dst, content, 0o600); err != nil {
			return copied, fmt.Errorf("write %s: %w", f.dst, err)
		}
		copied = append(copied, f.dst)
	}

	txnDir := paths.TxnDir()
	if entries, err := os.ReadDir(txnDir); err == nil {
		for _, e := range entries {
			if !e.IsDir() {
				continue
			}
			localDir := filepath.Join(txnDir, e.Name())
			if err := copyTxnDir(ctx, transport, peer, localDir, e.Name()); err != nil {
				slog.Warn("rotate-lead: copy txn dir failed",
					slog.String("txn", e.Name()), "error", err)
				continue
			}
			copied = append(copied, "txns/"+e.Name())
		}
	}
	return copied, nil
}

func copyTxnDir(ctx context.Context, transport driftTransport, peer, localDir, txnID string) error {
	files := []string{"manifest.json", "manifest.sig", "desired-state.json", "apply.sh", "verify.sh", "rollback.sh"}
	remoteDir := "/etc/orca/cluster/txns/" + txnID
	mkdirCmd := fmt.Sprintf("mkdir -p %s", remoteDir)
	if _, err := transport.Exec(ctx, peer, mkdirCmd); err != nil {
		return fmt.Errorf("mkdir %s: %w", remoteDir, err)
	}
	for _, f := range files {
		p := filepath.Join(localDir, f)
		content, err := os.ReadFile(p)
		if err != nil {
			if os.IsNotExist(err) {
				continue
			}
			return err
		}
		dst := remoteDir + "/" + f
		if _, err := transport.WriteFileIdempotent(ctx, peer, dst, content, 0o644); err != nil {
			return fmt.Errorf("write %s: %w", dst, err)
		}
	}
	return nil
}

type rotateSSHKeysResult struct {
	Deployed   int      `json:"deployed"`
	Failed     []string `json:"failed,omitempty"`
	OldKeyHash string   `json:"old_key_hash,omitempty"`
}

func rotateSSHKeys(ctx context.Context, transport driftTransport, nodes []*model.Node) (*rotateSSHKeysResult, error) {
	pubPath := certpaths.SSHPubPath()
	keyPath := certpaths.SSHKeyPath()

	oldPub, _ := os.ReadFile(pubPath)

	newPriv, newPub, err := generateEd25519Keypair()
	if err != nil {
		return nil, fmt.Errorf("generate new ssh key: %w", err)
	}
	if err := os.WriteFile(keyPath, newPriv, 0o600); err != nil {
		return nil, fmt.Errorf("write new ssh key: %w", err)
	}
	if err := os.WriteFile(pubPath, newPub, 0o644); err != nil {
		return nil, fmt.Errorf("write new ssh pub: %w", err)
	}

	res := &rotateSSHKeysResult{Failed: []string{}}
	for i := range nodes {
		n := nodes[i]
		peer := peerAddrForNode(n)
		if peer == "" {
			continue
		}
		deployCmd := fmt.Sprintf("mkdir -p ~/.ssh && echo %s >> ~/.ssh/authorized_keys && chmod 600 ~/.ssh/authorized_keys", sshQuote(strings.TrimSpace(string(newPub))))
		if _, err := transport.Exec(ctx, peer, deployCmd); err != nil {
			res.Failed = append(res.Failed, n.Name)
			continue
		}
		res.Deployed++
	}

	if len(oldPub) > 0 {
		res.OldKeyHash = sshFingerprint(oldPub)
	}
	return res, nil
}

func generateEd25519Keypair() (privBytes []byte, pubBytes []byte, err error) {
	pubKey, privKey, err := ed25519.GenerateKey(rand.Reader)
	if err != nil {
		return nil, nil, err
	}
	sshPub, err := ssh.NewPublicKey(pubKey)
	if err != nil {
		return nil, nil, err
	}
	pubBytes = ssh.MarshalAuthorizedKey(sshPub)
	pemBlock, err := ssh.MarshalPrivateKey(privKey, "")
	if err != nil {
		return nil, nil, err
	}
	privBytes = pem.EncodeToMemory(pemBlock)
	return privBytes, pubBytes, nil
}

func sshFingerprint(pub []byte) string {
	pk, _, _, _, err := ssh.ParseAuthorizedKey(pub)
	if err != nil {
		return ""
	}
	return ssh.FingerprintSHA256(pk)
}

func readCurrentLead(ctx context.Context) (string, error) {
	leadPath := filepath.Join(paths.ClusterDir(), "lead")
	b, err := os.ReadFile(leadPath)
	if err != nil {
		if os.IsNotExist(err) {
			return "", nil
		}
		return "", err
	}
	return trimSpace(string(b)), nil
}

func writeCurrentLead(ctx context.Context, name string) error {
	dir := paths.ClusterDir()
	if err := os.MkdirAll(dir, 0o755); err != nil {
		return err
	}
	leadPath := filepath.Join(dir, "lead")
	return os.WriteFile(leadPath, []byte(name), 0o644)
}

func trimSpace(s string) string {
	for len(s) > 0 && (s[0] == ' ' || s[0] == '\t' || s[0] == '\n' || s[0] == '\r') {
		s = s[1:]
	}
	for len(s) > 0 && (s[len(s)-1] == ' ' || s[len(s)-1] == '\t' || s[len(s)-1] == '\n' || s[len(s)-1] == '\r') {
		s = s[:len(s)-1]
	}
	return s
}

func init() {
	clusterRotateLeadCmd.Flags().StringVar(&rotateLeadTo, "to", "", "new lead host (required, must be a bare Linux node)")
	clusterRotateLeadCmd.Flags().BoolVar(&rotateLeadForce, "force", false, "skip R-003 verification (use with caution)")
	_ = rotateLeadDebug
}