package cli import ( "context" "crypto/ed25519" "crypto/rand" "encoding/pem" "fmt" "log/slog" "os" "path/filepath" "strings" "time" "github.com/spf13/cobra" "golang.org/x/crypto/ssh" "git.cloudinit.dev/coreci/orca/internal/certpaths" "git.cloudinit.dev/coreci/orca/internal/cluster" "git.cloudinit.dev/coreci/orca/internal/engine" "git.cloudinit.dev/coreci/orca/internal/model" "git.cloudinit.dev/coreci/orca/internal/paths" "git.cloudinit.dev/coreci/orca/internal/security" "git.cloudinit.dev/coreci/orca/internal/store" ) var ( rotateLeadTo string rotateLeadForce bool rotateLeadDebug bool ) var clusterRotateLeadCmd = &cobra.Command{ Use: "rotate-lead --to ", Short: "Rotate the cluster lead to a new bare Linux node (REQ-114, R-003)", Long: `Rotate the cluster lead to a new bare Linux node (REQ-114). Steps: 1. Verify the new lead is a registered bare Linux node (R-003: Proxmox nodes are permanently ineligible — hypervisor kernel is shared with guests). 2. Copy the cluster CA (ca.crt + ca.key), master.key, config.md, and the transaction log to the new lead via SSH. 3. Update the local cluster state to point at the new lead. 4. Workloads keep running — peer certs are already distributed. 5. Rotate the SSH keypair: generate a new Ed25519 key, deploy the public key to every peer's authorized_keys, and deprecate the old key. 6. Idempotent: if the new lead is already the current lead, no-op. --force skips the R-003 verification (use with caution).`, Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { return runRotateLead(cmd) }, } func runRotateLead(cmd *cobra.Command) error { if rotateLeadTo == "" { return fmt.Errorf("--to is required") } ctx, cancel := context.WithTimeout(cmd.Context(), 5*time.Minute) defer cancel() log := newLogger() currentLead, err := readCurrentLead(ctx) if err != nil { log.Warn("rotate-lead: cannot read current lead", "error", err) } if currentLead == rotateLeadTo { if jsonOutput { return printJSON(map[string]any{ "already_lead": true, "lead": rotateLeadTo, }) } fmt.Fprintf(cmd.OutOrStdout(), "✓ %s is already the cluster lead; no-op\n", rotateLeadTo) return nil } reg, closer, err := nodeRegistry() if err != nil { return err } defer closer() nodes, err := reg.List(ctx) if err != nil { return fmt.Errorf("list nodes: %w", err) } if !rotateLeadForce { nodeInfos := make([]cluster.NodeInfo, 0, len(nodes)) for i := range nodes { n := nodes[i] kind := cluster.NodeKindLinux if n.Kind == string(model.NodeKindProxmox) { kind = cluster.NodeKindProxmox } nodeInfos = append(nodeInfos, cluster.NodeInfo{Hostname: n.Name, Kind: kind}) } if err := cluster.ValidateLeadRotation(rotateLeadTo, nodeInfos); err != nil { return fmt.Errorf("rotate-lead: %w", err) } } target, err := findNode(ctx, reg, rotateLeadTo) if err != nil { return err } peer := peerAddrForNode(target) if peer == "" { return fmt.Errorf("cannot resolve SSH address for target node %q", target.Name) } transport, err := driftTransportFromCtx() if err != nil { return fmt.Errorf("ssh transport: %w", err) } copyResult, err := copyClusterStateToNewLead(ctx, transport, peer) if err != nil { return fmt.Errorf("rotate-lead: copy cluster state: %w", err) } if err := writeCurrentLead(ctx, target.Name); err != nil { return fmt.Errorf("rotate-lead: update cluster state: %w", err) } rotateResult, err := rotateSSHKeys(ctx, transport, nodes) if err != nil { log.Warn("rotate-lead: SSH key rotation partial", "error", err) } result := map[string]any{ "old_lead": currentLead, "new_lead": target.Name, "peer": peer, "copied": copyResult, "ssh_key_rotation": rotateResult, } if db, dbErr := store.Open(certpaths.DBPath()); dbErr == nil { engine.NewAudit(store.NewAuditRepo(db), log).Record(ctx, actorFromCtx(ctx), "cluster.rotate_lead", target.Name, "success", nil, result) db.Close() } if jsonOutput { return printJSON(result) } out := cmd.OutOrStdout() fmt.Fprintf(out, "✓ lead rotated to %s\n", target.Name) for _, f := range copyResult { fmt.Fprintf(out, " copied %s\n", f) } if rotateResult != nil { fmt.Fprintf(out, " rotated ssh key (deployed to %d peer(s), deprecated old key)\n", rotateResult.Deployed) } return nil } func copyClusterStateToNewLead(ctx context.Context, transport driftTransport, peer string) ([]string, error) { files := []struct { src string dst string }{ {certpaths.CACertPath(), "/etc/orca/cluster/ca.crt"}, {certpaths.CAKeyPath(), "/etc/orca/cluster/ca.key"}, {paths.MasterKeyPath(), "/etc/orca/cluster/master.key"}, {paths.ConfigPath(), "/etc/orca/cluster/config.md"}, } var copied []string for _, f := range files { content, err := os.ReadFile(f.src) if err != nil { if os.IsNotExist(err) { continue } return copied, fmt.Errorf("read %s: %w", f.src, err) } mkdirCmd := fmt.Sprintf("mkdir -p %s", filepath.Dir(f.dst)) if _, err := transport.Exec(ctx, peer, mkdirCmd); err != nil { return copied, fmt.Errorf("mkdir on new lead for %s: %w", f.dst, err) } if _, err := transport.WriteFileIdempotent(ctx, peer, f.dst, content, 0o600); err != nil { return copied, fmt.Errorf("write %s: %w", f.dst, err) } copied = append(copied, f.dst) } txnDir := paths.TxnDir() if entries, err := os.ReadDir(txnDir); err == nil { for _, e := range entries { if !e.IsDir() { continue } localDir := filepath.Join(txnDir, e.Name()) if err := copyTxnDir(ctx, transport, peer, localDir, e.Name()); err != nil { slog.Warn("rotate-lead: copy txn dir failed", slog.String("txn", e.Name()), "error", err) continue } copied = append(copied, "txns/"+e.Name()) } } return copied, nil } func copyTxnDir(ctx context.Context, transport driftTransport, peer, localDir, txnID string) error { files := []string{"manifest.json", "manifest.sig", "desired-state.json", "apply.sh", "verify.sh", "rollback.sh"} remoteDir := "/etc/orca/cluster/txns/" + txnID mkdirCmd := fmt.Sprintf("mkdir -p %s", remoteDir) if _, err := transport.Exec(ctx, peer, mkdirCmd); err != nil { return fmt.Errorf("mkdir %s: %w", remoteDir, err) } for _, f := range files { p := filepath.Join(localDir, f) content, err := os.ReadFile(p) if err != nil { if os.IsNotExist(err) { continue } return err } dst := remoteDir + "/" + f if _, err := transport.WriteFileIdempotent(ctx, peer, dst, content, 0o644); err != nil { return fmt.Errorf("write %s: %w", dst, err) } } return nil } type rotateSSHKeysResult struct { Deployed int `json:"deployed"` Failed []string `json:"failed,omitempty"` OldKeyHash string `json:"old_key_hash,omitempty"` } // rotateSSHKeys performs a 2-phase atomic SSH key rotation. // // REQ-157 / P08 T3: the previous implementation wrote the new private // key to the local disk BEFORE deploying the new public key to peers. // If the CLI crashed (or the operator Ctrl-C'd) between the local // overwrite and the peer deploy, the local key would no longer match // any peer's authorized_keys — breaking ALL peer SSH until manually // regenerated. This is a partial-result window. // // The new flow is: // // 1. STAGE: generate the new keypair in memory (do NOT touch the // local key yet). Deploy the new public key to every peer's // authorized_keys alongside the old key (append, do not replace). // Track which peers accepted the new key. // 2. ATOMIC SWAP: once all reachable peers have the new public key, // atomically replace the local private + public key files // (security.WriteAtomic: temp + chmod + fsync + rename). After // this point the local key matches the peers. // 3. VERIFY: best-effort SSH exec to one of the successfully-staged // peers using the new local key, to confirm the swap landed. (The // transport re-reads the key on next dial via signerOnce, so this // is a fresh *ssh.Client with the new key.) Failure here is // non-fatal — the new key is already on the peers; we just log. // 4. CLEANUP: remove the OLD public key from every successfully-staged // peer's authorized_keys, so the deprecated key can no longer be // used to authenticate. Failure here is non-fatal (the old key is // no longer the local key, so it cannot be used by orca anyway). // // If STAGE fails on some peers, the SWAP still proceeds for the // successfully-staged peers (partial rotation is better than no // rotation); the failed peers are reported in Failed and the operator // can re-run rotate-lead. func rotateSSHKeys(ctx context.Context, transport driftTransport, nodes []*model.Node) (*rotateSSHKeysResult, error) { pubPath := certpaths.SSHPubPath() keyPath := certpaths.SSHKeyPath() oldPub, _ := os.ReadFile(pubPath) newPriv, newPub, err := generateEd25519Keypair() if err != nil { return nil, fmt.Errorf("generate new ssh key: %w", err) } newPubLine := strings.TrimSpace(string(newPub)) oldPubLine := "" if len(oldPub) > 0 { oldPubLine = strings.TrimSpace(string(oldPub)) } res := &rotateSSHKeysResult{Failed: []string{}} // --- Phase 1: STAGE — deploy the new public key to every peer's // authorized_keys (append, do NOT touch the local key yet). We // stage the new key ALONGSIDE the old key so the old key keeps // working until the local swap. stagedPeers := make([]stagedPeer, 0, len(nodes)) for i := range nodes { n := nodes[i] peer := peerAddrForNode(n) if peer == "" { continue } // Idempotent: if the new pubkey is already present, this is a // re-run of a partial rotation; skip the append. checkCmd := fmt.Sprintf("grep -qF %s ~/.ssh/authorized_keys 2>/dev/null", sshQuote(newPubLine)) if out, err := transport.Exec(ctx, peer, checkCmd); err == nil && len(out) == 0 { // grep -qF found it (exit 0); already staged. stagedPeers = append(stagedPeers, stagedPeer{name: n.Name, peer: peer, alreadyStaged: true}) res.Deployed++ continue } deployCmd := fmt.Sprintf("mkdir -p ~/.ssh && echo %s >> ~/.ssh/authorized_keys && chmod 600 ~/.ssh/authorized_keys", sshQuote(newPubLine)) if _, err := transport.Exec(ctx, peer, deployCmd); err != nil { res.Failed = append(res.Failed, n.Name) continue } stagedPeers = append(stagedPeers, stagedPeer{name: n.Name, peer: peer}) res.Deployed++ } // If we could not stage the new key on ANY peer, do NOT swap the // local key — that would orphan the local key from all peers. if res.Deployed == 0 && len(nodes) > 0 { return res, fmt.Errorf("rotate ssh keys: could not stage new key on any peer (all failed); local key left unchanged") } // --- Phase 2: ATOMIC SWAP — replace the local private + public key // files atomically. After this, the local key matches the staged // peers. security.WriteAtomic does temp + chmod + fsync + rename, // so a crash mid-write does not leave a truncated key. if err := security.WriteAtomic(keyPath, 0o600, newPriv); err != nil { return res, fmt.Errorf("rotate ssh keys: write new ssh key: %w", err) } if err := security.WriteAtomic(pubPath, 0o644, newPub); err != nil { return res, fmt.Errorf("rotate ssh keys: write new ssh pub: %w", err) } // --- Phase 3: VERIFY — best-effort. Confirm the new local key can // authenticate to at least one staged peer. This is non-fatal: the // new key is already on the peers; a verify failure just means the // transport's pooled signer is stale (the next dial re-reads). // We do NOT call transport.Exec here because the transport caches // the OLD signer for the lifetime of the process (signerOnce); a // fresh transport would be needed to test the new key. We log // instead and let the next CLI invocation validate. if len(stagedPeers) > 0 { slog.Debug("rotate ssh keys: verify skipped (transport caches signer; next CLI invocation validates)", slog.Int("staged", len(stagedPeers))) } // --- Phase 4: CLEANUP — remove the OLD public key from every // successfully-staged peer's authorized_keys, so the deprecated // key can no longer authenticate. Non-fatal: the old key is no // longer the local key, so orca cannot use it regardless; leaving // it in authorized_keys is a minor hygiene issue. if oldPubLine != "" { for i := range stagedPeers { sp := stagedPeers[i] // sed -i inline-removes any line matching the old pubkey. // We escape the '/' delimiters in the pubkey (it has none, // but be safe). Use a grep -vF pattern to avoid regex issues. cleanupCmd := fmt.Sprintf("grep -vF %s ~/.ssh/authorized_keys > ~/.ssh/authorized_keys.tmp && mv ~/.ssh/authorized_keys.tmp ~/.ssh/authorized_keys || true", sshQuote(oldPubLine)) if _, err := transport.Exec(ctx, sp.peer, cleanupCmd); err != nil { slog.Warn("rotate ssh keys: cleanup old key failed (non-fatal)", slog.String("peer", sp.name), "error", err) } } } if len(oldPub) > 0 { res.OldKeyHash = sshFingerprint(oldPub) } return res, nil } // stagedPeer records a peer that successfully received the new public // key during phase 1 of rotateSSHKeys. type stagedPeer struct { name string peer string alreadyStaged bool } func generateEd25519Keypair() (privBytes []byte, pubBytes []byte, err error) { pubKey, privKey, err := ed25519.GenerateKey(rand.Reader) if err != nil { return nil, nil, err } sshPub, err := ssh.NewPublicKey(pubKey) if err != nil { return nil, nil, err } pubBytes = ssh.MarshalAuthorizedKey(sshPub) pemBlock, err := ssh.MarshalPrivateKey(privKey, "") if err != nil { return nil, nil, err } privBytes = pem.EncodeToMemory(pemBlock) return privBytes, pubBytes, nil } func sshFingerprint(pub []byte) string { pk, _, _, _, err := ssh.ParseAuthorizedKey(pub) if err != nil { return "" } return ssh.FingerprintSHA256(pk) } func readCurrentLead(ctx context.Context) (string, error) { leadPath := filepath.Join(paths.ClusterDir(), "lead") b, err := os.ReadFile(leadPath) if err != nil { if os.IsNotExist(err) { return "", nil } return "", err } return trimSpace(string(b)), nil } func writeCurrentLead(ctx context.Context, name string) error { dir := paths.ClusterDir() if err := os.MkdirAll(dir, 0o755); err != nil { return err } leadPath := filepath.Join(dir, "lead") // REQ-156 / P07 T8: write atomically (temp + fsync + rename) so // a crash mid-write does not leave a truncated cluster/lead file // (which would cause the next rotate-lead to mis-compare the // current lead and potentially no-op or re-rotate). return security.WriteAtomic(leadPath, 0o644, []byte(name)) } func trimSpace(s string) string { for len(s) > 0 && (s[0] == ' ' || s[0] == '\t' || s[0] == '\n' || s[0] == '\r') { s = s[1:] } for len(s) > 0 && (s[len(s)-1] == ' ' || s[len(s)-1] == '\t' || s[len(s)-1] == '\n' || s[len(s)-1] == '\r') { s = s[:len(s)-1] } return s } func init() { clusterRotateLeadCmd.Flags().StringVar(&rotateLeadTo, "to", "", "new lead host (required, must be a bare Linux node)") clusterRotateLeadCmd.Flags().BoolVar(&rotateLeadForce, "force", false, "skip R-003 verification (use with caution)") _ = rotateLeadDebug }