// Package proxmox implements the SSH-based bootstrap of a remote // Proxmox VE 8/9 host as an orca node (REQ-050, REQ-051). // // The bootstrap sequence (run via `orca node join --type proxmox`): // 1. Generate or load the orca SSH keypair (Ed25519, D-037) // 2. SSH dial with password auth + TOFU host-key capture (D-035) // 3. Deploy the orca pubkey to ~orca/.ssh/authorized_keys // 4. Create the `orca` Linux system user (config-overridable name) // 5. Create the OrcaOperator PVE role with least-privilege privileges // 6. Create the orca@pam PVE user (maps to the Linux system user) // 7. Assign the OrcaOperator role to orca@pam on path / // 8. Write /etc/sudoers.d/orca with NOEXEC on pct/qm, no NOEXEC on // apt-get/dpkg, and pvesh EXCLUDED (AD-020: pvesh can bypass NOEXEC // via the API execute endpoint) // 9. Validate the sudoers file with visudo -cf // 10. Return the node metadata for the caller to persist // // All steps are idempotent (D-036): re-running the bootstrap on an // already-configured host is a no-op. The password is never persisted // (D-031) — it is used only for the initial SSH auth and pubkey // deployment; subsequent orca→Proxmox access uses the deployed SSH key. package proxmox import ( "bytes" "context" "errors" "fmt" "log/slog" "net" "os" "strings" "time" "golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh/knownhosts" "git.cloudinit.dev/coreci/orca/internal/certpaths" "git.cloudinit.dev/coreci/orca/internal/security" ) // DefaultProxmoxUser is the default Linux system user created on the // Proxmox host. Overridable via Options.ProxmoxUser. const DefaultProxmoxUser = "orca" // DefaultProxmoxRole is the default PVE custom role created for the // orca user. Overridable via Options.ProxmoxRole. const DefaultProxmoxRole = "OrcaOperator" // DefaultSSHPort is the default SSH port for Proxmox hosts. const DefaultSSHPort = 22 // OrcaOperatorPrivileges is the least-privilege privilege set for the // OrcaOperator PVE role (D-033). Space-separated per pveum --privs // syntax. VM.Audit covers CTs as well (both live under /vms/{vmid}). const OrcaOperatorPrivileges = "VM.Audit Datastore.AllocateSpace SDN.Use" // Options configures a Proxmox bootstrap run. type Options struct { // Host is the Proxmox host address (IP or hostname, no port). Host string // SSHUser is the initial SSH username (default "root"). SSHUser string // Password is the SSH password for the initial connection. // NEVER persisted (D-031). The caller must zero this after use. Password string // ProxmoxUser is the Linux system user to create on the host // (default "orca"). Config-overridable. ProxmoxUser string // ProxmoxRole is the PVE custom role to create (default // "OrcaOperator"). Config-overridable. ProxmoxRole string // SSHPort is the SSH port (default 22). SSHPort int // HostKeyFingerprint is the operator-pinned SSH host key fingerprint // in `SHA256:base64` form (REQ-058, D-044). When non-empty, the // bootstrap dialer uses a pinned-host-key callback instead of the // TOFU known_hosts capture path. Empty falls back to TOFU. HostKeyFingerprint string // Logger receives audit-log entries. If nil, slog.Default() is used. Logger *slog.Logger } // Result is the outcome of a successful bootstrap. type Result struct { // NodeName is the name to use for the node in the orca registry // (typically the host address). NodeName string // NodeAddress is the orca daemon address on the Proxmox host // (host:8443 — the orca daemon port). NodeAddress string // HostKeyFingerprint is the SHA-256 fingerprint of the captured // SSH host key (for operator verification). HostKeyFingerprint string } // BootstrapProxmox runs the full SSH bootstrap sequence on a remote // Proxmox VE 8/9 host. All steps are idempotent. Returns a Result // describing the node to register, or an error if any step fails. func BootstrapProxmox(ctx context.Context, opts Options) (*Result, error) { if opts.Host == "" { return nil, fmt.Errorf("proxmox bootstrap: host is required") } if opts.Password == "" { return nil, fmt.Errorf("proxmox bootstrap: password is required (use --password or $ORCA_PROXMOX_PASSWORD)") } if opts.SSHUser == "" { opts.SSHUser = "root" } if opts.ProxmoxUser == "" { opts.ProxmoxUser = DefaultProxmoxUser } if opts.ProxmoxRole == "" { opts.ProxmoxRole = DefaultProxmoxRole } if opts.SSHPort == 0 { opts.SSHPort = DefaultSSHPort } log := opts.Logger if log == nil { log = slog.Default() } // Step 1: Generate or load the orca SSH keypair (D-037). // The key is deployed to the remote host's authorized_keys in step 3. _, pubLine, err := security.GenerateOrLoadSSHKey(certpaths.Dir()) if err != nil { return nil, fmt.Errorf("ssh key: %w", err) } // Step 2: SSH dial with password auth + host-key verification (D-035, // REQ-058). When opts.HostKeyFingerprint is set (D-044), use a pinned // callback that fails closed on mismatch (AD-028); otherwise use the // TOFU known_hosts capture callback (D-035). The TOFU wrapper fixes // the v0.6 ship-defect where knownhosts.New returned KeyError{Want:[]} // on first connect WITHOUT writing the captured key, so the first // `orca node join --type proxmox` always failed. sshAddr := fmt.Sprintf("%s:%d", opts.Host, opts.SSHPort) var capturedHostKey ssh.PublicKey var hostKeyCallback ssh.HostKeyCallback if opts.HostKeyFingerprint != "" { cb, err := pinnedHostKeyCallback(opts.HostKeyFingerprint, &capturedHostKey) if err != nil { return nil, fmt.Errorf("host-key fingerprint: %w", err) } hostKeyCallback = cb } else { cb, err := TOFUHostKeyCallback(sshAddr, &capturedHostKey) if err != nil { return nil, fmt.Errorf("tofu host-key callback: %w", err) } hostKeyCallback = cb } sshConfig := &ssh.ClientConfig{ User: opts.SSHUser, Auth: []ssh.AuthMethod{ssh.Password(opts.Password)}, HostKeyCallback: hostKeyCallback, Timeout: 10 * time.Second, } dialCtx, dialCancel := context.WithTimeout(ctx, 15*time.Second) defer dialCancel() conn, err := sshDialer.DialContext(dialCtx, "tcp", sshAddr, sshConfig) if err != nil { return nil, fmt.Errorf("ssh dial %s: %w", sshAddr, err) } defer conn.Close() if sessionRunner == nil { sessionRunner = &sshSessionRunner{client: conn} } hostKeyFP := "" if capturedHostKey != nil { hostKeyFP = security.SSHFingerprintSHA256(capturedHostKey) } log.Info("proxmox.ssh_connected", slog.String("event", "proxmox.ssh_connected"), slog.String("host", opts.Host), slog.String("ssh_user", opts.SSHUser), slog.String("host_key_fingerprint", hostKeyFP), ) // Step 3: Deploy orca pubkey to ~orca/.ssh/authorized_keys (idempotent). if err := deployPubKey(opts.ProxmoxUser, string(pubLine)); err != nil { return nil, fmt.Errorf("deploy pubkey: %w", err) } // Step 4: Create orca Linux system user (idempotent). if err := createLinuxUser(opts.ProxmoxUser); err != nil { return nil, fmt.Errorf("create user %s: %w", opts.ProxmoxUser, err) } // Step 5: Create OrcaOperator PVE role (idempotent). if err := createPVERole(opts.ProxmoxRole); err != nil { return nil, fmt.Errorf("create PVE role %s: %w", opts.ProxmoxRole, err) } // Step 6: Create orca@pam PVE user (idempotent). if err := createPVEUser(opts.ProxmoxUser); err != nil { return nil, fmt.Errorf("create PVE user %s@pam: %w", opts.ProxmoxUser, err) } // Step 7: Assign OrcaOperator role to orca@pam on path / (idempotent). if err := assignPVEACL(opts.ProxmoxUser, opts.ProxmoxRole); err != nil { return nil, fmt.Errorf("assign ACL: %w", err) } // Step 8: Write /etc/sudoers.d/orca (AD-020: NOEXEC on pct/qm, // no NOEXEC on apt-get/dpkg, pvesh EXCLUDED). if err := writeSudoers(opts.ProxmoxUser); err != nil { return nil, fmt.Errorf("write sudoers: %w", err) } // Step 9: Validate sudoers with visudo -cf. if err := validateSudoers(); err != nil { return nil, fmt.Errorf("validate sudoers: %w", err) } log.Info("proxmox.bootstrap_ok", slog.String("event", "proxmox.bootstrap_ok"), slog.String("host", opts.Host), slog.String("proxmox_user", opts.ProxmoxUser), slog.String("proxmox_role", opts.ProxmoxRole), ) return &Result{ NodeName: opts.Host, NodeAddress: opts.Host + ":8443", HostKeyFingerprint: hostKeyFP, }, nil } // sshDialer is the dialer used by BootstrapProxmox. It's a package-level // variable so tests can override it with a fake SSH server. var sshDialer sshDialerType = defaultSSHDialer{} // pinnedHostKeyCallback returns an ssh.HostKeyCallback that pins the // server's host key to the operator-supplied SHA256:base64 fingerprint // (REQ-058, AD-028). It validates the `SHA256:` prefix up front (D-045) // and fails closed on any mismatch. The capturedKey out-param records // the verified server key so the caller can populate Result. func pinnedHostKeyCallback(expectedSHA256Base64 string, capturedKey *ssh.PublicKey) (ssh.HostKeyCallback, error) { if !strings.HasPrefix(expectedSHA256Base64, "SHA256:") { return nil, fmt.Errorf("pinnedHostKeyCallback: fingerprint must be SHA256:-prefixed (D-045), got %q", expectedSHA256Base64) } return func(_ string, _ net.Addr, key ssh.PublicKey) error { got := security.SSHFingerprintSHA256(key) if got != expectedSHA256Base64 { return fmt.Errorf("REQ-058 host-key fingerprint mismatch: pinned=%s server=%s", expectedSHA256Base64, got) } if capturedKey != nil { *capturedKey = key } return nil }, nil } // TOFUHostKeyCallback returns an ssh.HostKeyCallback that wraps the // standard knownhosts.New verifier with TOFU first-connect capture // (D-035). On a host-unknown KeyError{Want:[]} it writes the // server-presented key to certpaths.KnownHostsPath() atomically // (security.WriteAtomic, AD-029) and allows the dial to proceed; on a // mismatch (Want non-empty) it fails closed (MITM detection). The // capturedKey out-param records the verified/captured server key so // the caller can populate Result. This fixes the v0.6 ship-defect // where knownhosts.New returned KeyError{Want:[]} on first connect // WITHOUT writing the captured key, so the first // `orca node join --type proxmox` always failed. // // Exported so the doctor proxmox probe (T02.9) can reuse the same // capture-fix wrapper for parity (GRILL condition #2). func TOFUHostKeyCallback(addr string, capturedKey *ssh.PublicKey) (ssh.HostKeyCallback, error) { cb, err := knownhosts.New(certpaths.KnownHostsPath()) if err != nil { return nil, err } return func(hostname string, remote net.Addr, key ssh.PublicKey) error { err := cb(hostname, remote, key) if err == nil { if capturedKey != nil { *capturedKey = key } return nil } var keyErr *knownhosts.KeyError if errors.As(err, &keyErr) && len(keyErr.Want) == 0 { line := knownhosts.Line([]string{knownhosts.Normalize(addr)}, key) path := certpaths.KnownHostsPath() release, lockErr := security.Flock(path) if lockErr != nil { return fmt.Errorf("tofu lock known_hosts: %w", lockErr) } defer release() existing, readErr := os.ReadFile(path) if readErr != nil && !os.IsNotExist(readErr) { return fmt.Errorf("tofu read known_hosts: %w", readErr) } if len(existing) > 0 && !bytes.HasSuffix(existing, []byte("\n")) { existing = append(existing, '\n') } updated := append(existing, []byte(line)...) if writeErr := security.WriteAtomic(path, 0o600, updated); writeErr != nil { return fmt.Errorf("tofu write known_hosts: %w", writeErr) } if capturedKey != nil { *capturedKey = key } return nil } return err }, nil } type sshDialerType interface { DialContext(ctx context.Context, network, addr string, config *ssh.ClientConfig) (*ssh.Client, error) } type defaultSSHDialer struct{} func (defaultSSHDialer) DialContext(ctx context.Context, network, addr string, config *ssh.ClientConfig) (*ssh.Client, error) { return ssh.Dial(network, addr, config) } type sessionRunnerType interface { CombinedOutput(cmd string) ([]byte, error) } var sessionRunner sessionRunnerType type sshSessionRunner struct { client *ssh.Client } func (r *sshSessionRunner) CombinedOutput(cmd string) ([]byte, error) { session, err := r.client.NewSession() if err != nil { return nil, fmt.Errorf("new session: %w", err) } defer session.Close() return session.CombinedOutput(cmd) } // runRemote runs a command over the SSH connection and returns its // combined output. Returns an error if the command exits non-zero. func runRemote(cmd string) ([]byte, error) { out, err := sessionRunner.CombinedOutput(cmd) if err != nil { return out, fmt.Errorf("run %q: %w (output: %s)", cmd, err, strings.TrimSpace(string(out))) } return out, nil } // deployPubKey appends the orca public key to the remote user's // authorized_keys file, creating the .ssh dir if needed. Idempotent: // if the key is already present, it is not re-appended. func deployPubKey(user, pubLine string) error { pubLine = strings.TrimSpace(pubLine) if pubLine == "" { return fmt.Errorf("deployPubKey: empty pub line") } home := "/home/" + user if user == "root" { home = "/root" } sshDir := home + "/.ssh" authFile := sshDir + "/authorized_keys" // Create .ssh dir, touch authorized_keys, set modes, append key if absent. cmd := fmt.Sprintf( "mkdir -p %s && touch %s && chmod 0700 %s && chmod 0600 %s && grep -qF '%s' %s || echo '%s' >> %s", sshDir, authFile, sshDir, authFile, pubLine, authFile, pubLine, authFile, ) if _, err := runRemote(cmd); err != nil { return err } return nil } // createLinuxUser creates the orca system user if it doesn't already // exist. Idempotent: `id -u` check before `useradd`. func createLinuxUser(user string) error { cmd := fmt.Sprintf("id -u %s 2>/dev/null || useradd -m -s /bin/bash %s", user, user) if _, err := runRemote(cmd); err != nil { return err } return nil } // createPVERole creates the OrcaOperator PVE role if it doesn't exist. // Idempotent: probes `pveum role list` before `pveum role add`. func createPVERole(role string) error { cmd := fmt.Sprintf( "pveum role list 2>/dev/null | grep -q '^%s' || pveum role add %s --privs '%s'", role, role, OrcaOperatorPrivileges, ) if _, err := runRemote(cmd); err != nil { return err } return nil } // createPVEUser creates the orca@pam PVE user if it doesn't exist. // Idempotent: probes `pveum user list` before `pveum user add`. // Uses @pam realm (AD-019) since orca creates a Linux system user. func createPVEUser(user string) error { pveUserID := user + "@pam" cmd := fmt.Sprintf( "pveum user list 2>/dev/null | grep -q '%s' || pveum user add %s -comment 'Orca automation user'", pveUserID, pveUserID, ) if _, err := runRemote(cmd); err != nil { return err } return nil } // assignPVEACL assigns the OrcaOperator role to orca@pam on path / // (cluster-wide). `pveum acl modify` is idempotent (creates or updates). func assignPVEACL(user, role string) error { pveUserID := user + "@pam" cmd := fmt.Sprintf("pveum acl modify / -user %s -role %s", pveUserID, role) if _, err := runRemote(cmd); err != nil { return err } return nil } // sudoersContent returns the /etc/sudoers.d/orca file content (AD-020). // NOEXEC on pct/qm (blocks shell escapes); no NOEXEC on apt-get/dpkg // (they need exec for maintainer scripts); pvesh EXCLUDED (API execute // bypasses NOEXEC). File must be mode 0440 per sudo requirements. func sudoersContent(user string) string { return fmt.Sprintf(`# /etc/sudoers.d/orca — Managed by orca; do not edit manually. # Least-privilege allowlist for the orca PVE operator user. # NOPASSWD: non-interactive SSH automation. NOEXEC: blocks shell escapes. # pvesh is EXCLUDED (AD-020: pvesh can bypass NOEXEC via API execute). %s ALL=(root) NOPASSWD: NOEXEC: /usr/bin/pct %s ALL=(root) NOPASSWD: NOEXEC: /usr/bin/qm %s ALL=(root) NOPASSWD: /usr/bin/apt-get %s ALL=(root) NOPASSWD: /usr/bin/dpkg `, user, user, user, user) } // writeSudoers writes the /etc/sudoers.d/orca file on the remote host // with mode 0440. Uses a heredoc via cat to avoid quoting issues. func writeSudoers(user string) error { content := sudoersContent(user) // Write via cat heredoc, then chmod 0440. cmd := fmt.Sprintf("cat > /etc/sudoers.d/%s <<'ORCA_SUDOERS_EOF'\n%s\nORCA_SUDOERS_EOF\nchmod 0440 /etc/sudoers.d/%s", user, content, user) if _, err := runRemote(cmd); err != nil { return err } return nil } // validateSudoers runs `visudo -cf` on the sudoers file. Aborts the // bootstrap if validation fails (prevents a broken sudoers from // locking the orca user out of sudo). func validateSudoers() error { cmd := "visudo -cf /etc/sudoers.d/orca" out, err := runRemote(cmd) if err != nil { return fmt.Errorf("visudo validation failed: %w (output: %s)", err, strings.TrimSpace(string(out))) } if !strings.Contains(string(out), "parsed OK") { return fmt.Errorf("visudo validation did not report OK: %s", strings.TrimSpace(string(out))) } return nil } // ResetHostKey removes all known_hosts entries for the given host from // certpaths.KnownHostsPath() (REQ-059, D-046, AD-029). It rewrites the // file atomically via security.WriteAtomic. LOCAL ONLY — it does NOT // touch the remote host's authorized_keys (D-046). The next connect // re-pins the host key via TOFU (T02.6) or the --host-key-fingerprint // pinned path (T02.5). // // A line matches when its first whitespace-delimited field (the host // pattern, normalized via knownhosts.Normalize) equals the normalized // target host. Comment/blank lines are preserved. func ResetHostKey(host string) error { if host == "" { return fmt.Errorf("ResetHostKey: host is required") } path := certpaths.KnownHostsPath() release, lockErr := security.Flock(path) if lockErr != nil { return fmt.Errorf("ResetHostKey: lock known_hosts: %w", lockErr) } defer release() existing, err := os.ReadFile(path) if err != nil { if os.IsNotExist(err) { return nil // nothing to reset } return fmt.Errorf("ResetHostKey: read known_hosts: %w", err) } target := knownhosts.Normalize(host) var kept []byte removed := 0 for _, line := range strings.Split(string(existing), "\n") { trimmed := strings.TrimSpace(line) if trimmed == "" || strings.HasPrefix(trimmed, "#") { kept = append(kept, []byte(line+"\n")...) continue } fields := strings.Fields(trimmed) if len(fields) == 0 { kept = append(kept, []byte(line+"\n")...) continue } if knownhosts.Normalize(fields[0]) == target { removed++ continue } kept = append(kept, []byte(line+"\n")...) } if removed == 0 { return nil } // Ensure the kept buffer ends with exactly one trailing newline. kept = bytes.TrimRight(kept, "\n") if len(kept) > 0 { kept = append(kept, '\n') } if err := security.WriteAtomic(path, 0o600, kept); err != nil { return fmt.Errorf("ResetHostKey: rewrite known_hosts: %w", err) } return nil }