package cli import ( "context" "crypto/ed25519" "crypto/rand" "encoding/pem" "fmt" "log/slog" "os" "path/filepath" "strings" "time" "github.com/spf13/cobra" "golang.org/x/crypto/ssh" "git.cloudinit.dev/coreci/orca/internal/certpaths" "git.cloudinit.dev/coreci/orca/internal/cluster" "git.cloudinit.dev/coreci/orca/internal/engine" "git.cloudinit.dev/coreci/orca/internal/model" "git.cloudinit.dev/coreci/orca/internal/paths" "git.cloudinit.dev/coreci/orca/internal/store" ) var ( rotateLeadTo string rotateLeadForce bool rotateLeadDebug bool ) var clusterRotateLeadCmd = &cobra.Command{ Use: "rotate-lead --to ", Short: "Rotate the cluster lead to a new bare Linux node (REQ-114, R-003)", Long: `Rotate the cluster lead to a new bare Linux node (REQ-114). Steps: 1. Verify the new lead is a registered bare Linux node (R-003: Proxmox nodes are permanently ineligible — hypervisor kernel is shared with guests). 2. Copy the cluster CA (ca.crt + ca.key), master.key, config.md, and the transaction log to the new lead via SSH. 3. Update the local cluster state to point at the new lead. 4. Workloads keep running — peer certs are already distributed. 5. Rotate the SSH keypair: generate a new Ed25519 key, deploy the public key to every peer's authorized_keys, and deprecate the old key. 6. Idempotent: if the new lead is already the current lead, no-op. --force skips the R-003 verification (use with caution).`, Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { return runRotateLead(cmd) }, } func runRotateLead(cmd *cobra.Command) error { if rotateLeadTo == "" { return fmt.Errorf("--to is required") } ctx, cancel := context.WithTimeout(cmd.Context(), 5*time.Minute) defer cancel() log := newLogger() currentLead, err := readCurrentLead(ctx) if err != nil { log.Warn("rotate-lead: cannot read current lead", "error", err) } if currentLead == rotateLeadTo { if jsonOutput { return printJSON(map[string]any{ "already_lead": true, "lead": rotateLeadTo, }) } fmt.Fprintf(cmd.OutOrStdout(), "✓ %s is already the cluster lead; no-op\n", rotateLeadTo) return nil } reg, closer, err := nodeRegistry() if err != nil { return err } defer closer() nodes, err := reg.List(ctx) if err != nil { return fmt.Errorf("list nodes: %w", err) } if !rotateLeadForce { nodeInfos := make([]cluster.NodeInfo, 0, len(nodes)) for i := range nodes { n := nodes[i] kind := cluster.NodeKindLinux if n.Kind == string(model.NodeKindProxmox) { kind = cluster.NodeKindProxmox } nodeInfos = append(nodeInfos, cluster.NodeInfo{Hostname: n.Name, Kind: kind}) } if err := cluster.ValidateLeadRotation(rotateLeadTo, nodeInfos); err != nil { return fmt.Errorf("rotate-lead: %w", err) } } target, err := findNode(ctx, reg, rotateLeadTo) if err != nil { return err } peer := peerAddrForNode(target) if peer == "" { return fmt.Errorf("cannot resolve SSH address for target node %q", target.Name) } transport, err := driftTransportFromCtx() if err != nil { return fmt.Errorf("ssh transport: %w", err) } copyResult, err := copyClusterStateToNewLead(ctx, transport, peer) if err != nil { return fmt.Errorf("rotate-lead: copy cluster state: %w", err) } if err := writeCurrentLead(ctx, target.Name); err != nil { return fmt.Errorf("rotate-lead: update cluster state: %w", err) } rotateResult, err := rotateSSHKeys(ctx, transport, nodes) if err != nil { log.Warn("rotate-lead: SSH key rotation partial", "error", err) } result := map[string]any{ "old_lead": currentLead, "new_lead": target.Name, "peer": peer, "copied": copyResult, "ssh_key_rotation": rotateResult, } if db, dbErr := store.Open(certpaths.DBPath()); dbErr == nil { engine.NewAudit(store.NewAuditRepo(db), log).Record(ctx, actorFromCtx(ctx), "cluster.rotate_lead", target.Name, "success", nil, result) db.Close() } if jsonOutput { return printJSON(result) } out := cmd.OutOrStdout() fmt.Fprintf(out, "✓ lead rotated to %s\n", target.Name) for _, f := range copyResult { fmt.Fprintf(out, " copied %s\n", f) } if rotateResult != nil { fmt.Fprintf(out, " rotated ssh key (deployed to %d peer(s), deprecated old key)\n", rotateResult.Deployed) } return nil } func copyClusterStateToNewLead(ctx context.Context, transport driftTransport, peer string) ([]string, error) { files := []struct { src string dst string }{ {certpaths.CACertPath(), "/etc/orca/cluster/ca.crt"}, {certpaths.CAKeyPath(), "/etc/orca/cluster/ca.key"}, {paths.MasterKeyPath(), "/etc/orca/cluster/master.key"}, {paths.ConfigPath(), "/etc/orca/cluster/config.md"}, } var copied []string for _, f := range files { content, err := os.ReadFile(f.src) if err != nil { if os.IsNotExist(err) { continue } return copied, fmt.Errorf("read %s: %w", f.src, err) } mkdirCmd := fmt.Sprintf("mkdir -p %s", filepath.Dir(f.dst)) if _, err := transport.Exec(ctx, peer, mkdirCmd); err != nil { return copied, fmt.Errorf("mkdir on new lead for %s: %w", f.dst, err) } if _, err := transport.WriteFileIdempotent(ctx, peer, f.dst, content, 0o600); err != nil { return copied, fmt.Errorf("write %s: %w", f.dst, err) } copied = append(copied, f.dst) } txnDir := paths.TxnDir() if entries, err := os.ReadDir(txnDir); err == nil { for _, e := range entries { if !e.IsDir() { continue } localDir := filepath.Join(txnDir, e.Name()) if err := copyTxnDir(ctx, transport, peer, localDir, e.Name()); err != nil { slog.Warn("rotate-lead: copy txn dir failed", slog.String("txn", e.Name()), "error", err) continue } copied = append(copied, "txns/"+e.Name()) } } return copied, nil } func copyTxnDir(ctx context.Context, transport driftTransport, peer, localDir, txnID string) error { files := []string{"manifest.json", "manifest.sig", "desired-state.json", "apply.sh", "verify.sh", "rollback.sh"} remoteDir := "/etc/orca/cluster/txns/" + txnID mkdirCmd := fmt.Sprintf("mkdir -p %s", remoteDir) if _, err := transport.Exec(ctx, peer, mkdirCmd); err != nil { return fmt.Errorf("mkdir %s: %w", remoteDir, err) } for _, f := range files { p := filepath.Join(localDir, f) content, err := os.ReadFile(p) if err != nil { if os.IsNotExist(err) { continue } return err } dst := remoteDir + "/" + f if _, err := transport.WriteFileIdempotent(ctx, peer, dst, content, 0o644); err != nil { return fmt.Errorf("write %s: %w", dst, err) } } return nil } type rotateSSHKeysResult struct { Deployed int `json:"deployed"` Failed []string `json:"failed,omitempty"` OldKeyHash string `json:"old_key_hash,omitempty"` } func rotateSSHKeys(ctx context.Context, transport driftTransport, nodes []*model.Node) (*rotateSSHKeysResult, error) { pubPath := certpaths.SSHPubPath() keyPath := certpaths.SSHKeyPath() oldPub, _ := os.ReadFile(pubPath) newPriv, newPub, err := generateEd25519Keypair() if err != nil { return nil, fmt.Errorf("generate new ssh key: %w", err) } if err := os.WriteFile(keyPath, newPriv, 0o600); err != nil { return nil, fmt.Errorf("write new ssh key: %w", err) } if err := os.WriteFile(pubPath, newPub, 0o644); err != nil { return nil, fmt.Errorf("write new ssh pub: %w", err) } res := &rotateSSHKeysResult{Failed: []string{}} for i := range nodes { n := nodes[i] peer := peerAddrForNode(n) if peer == "" { continue } deployCmd := fmt.Sprintf("mkdir -p ~/.ssh && echo %s >> ~/.ssh/authorized_keys && chmod 600 ~/.ssh/authorized_keys", sshQuote(strings.TrimSpace(string(newPub)))) if _, err := transport.Exec(ctx, peer, deployCmd); err != nil { res.Failed = append(res.Failed, n.Name) continue } res.Deployed++ } if len(oldPub) > 0 { res.OldKeyHash = sshFingerprint(oldPub) } return res, nil } func generateEd25519Keypair() (privBytes []byte, pubBytes []byte, err error) { pubKey, privKey, err := ed25519.GenerateKey(rand.Reader) if err != nil { return nil, nil, err } sshPub, err := ssh.NewPublicKey(pubKey) if err != nil { return nil, nil, err } pubBytes = ssh.MarshalAuthorizedKey(sshPub) pemBlock, err := ssh.MarshalPrivateKey(privKey, "") if err != nil { return nil, nil, err } privBytes = pem.EncodeToMemory(pemBlock) return privBytes, pubBytes, nil } func sshFingerprint(pub []byte) string { pk, _, _, _, err := ssh.ParseAuthorizedKey(pub) if err != nil { return "" } return ssh.FingerprintSHA256(pk) } func readCurrentLead(ctx context.Context) (string, error) { leadPath := filepath.Join(paths.ClusterDir(), "lead") b, err := os.ReadFile(leadPath) if err != nil { if os.IsNotExist(err) { return "", nil } return "", err } return trimSpace(string(b)), nil } func writeCurrentLead(ctx context.Context, name string) error { dir := paths.ClusterDir() if err := os.MkdirAll(dir, 0o755); err != nil { return err } leadPath := filepath.Join(dir, "lead") return os.WriteFile(leadPath, []byte(name), 0o644) } func trimSpace(s string) string { for len(s) > 0 && (s[0] == ' ' || s[0] == '\t' || s[0] == '\n' || s[0] == '\r') { s = s[1:] } for len(s) > 0 && (s[len(s)-1] == ' ' || s[len(s)-1] == '\t' || s[len(s)-1] == '\n' || s[len(s)-1] == '\r') { s = s[:len(s)-1] } return s } func init() { clusterRotateLeadCmd.Flags().StringVar(&rotateLeadTo, "to", "", "new lead host (required, must be a bare Linux node)") clusterRotateLeadCmd.Flags().BoolVar(&rotateLeadForce, "force", false, "skip R-003 verification (use with caution)") _ = rotateLeadDebug }