orca/internal/proxmox/bootstrap.go

// Package proxmox implements the SSH-based bootstrap of a remote
// Proxmox VE 8/9 host as an orca node (REQ-050, REQ-051).
//
// The bootstrap sequence (run via `orca node join --type proxmox`):
//  1. Generate or load the orca SSH keypair (Ed25519, D-037)
//  2. SSH dial with password auth + TOFU host-key capture (D-035)
//  3. Deploy the orca pubkey to ~orca/.ssh/authorized_keys
//  4. Create the `orca` Linux system user (config-overridable name)
//  5. Create the OrcaOperator PVE role with least-privilege privileges
//  6. Create the orca@pam PVE user (maps to the Linux system user)
//  7. Assign the OrcaOperator role to orca@pam on path /
//  8. Write /etc/sudoers.d/orca with NOEXEC on pct/qm, no NOEXEC on
//     apt-get/dpkg, and pvesh EXCLUDED (AD-020: pvesh can bypass NOEXEC
//     via the API execute endpoint)
//  9. Validate the sudoers file with visudo -cf
//  10. Return the node metadata for the caller to persist
//
// All steps are idempotent (D-036): re-running the bootstrap on an
// already-configured host is a no-op. The password is never persisted
// (D-031) — it is used only for the initial SSH auth and pubkey
// deployment; subsequent orca→Proxmox access uses the deployed SSH key.
package proxmox

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"log/slog"
	"net"
	"os"
	"strings"
	"time"

	"golang.org/x/crypto/ssh"
	"golang.org/x/crypto/ssh/knownhosts"

	"git.cloudinit.dev/coreci/orca/internal/certpaths"
	"git.cloudinit.dev/coreci/orca/internal/security"
)

// DefaultProxmoxUser is the default Linux system user created on the
// Proxmox host. Overridable via Options.ProxmoxUser.
const DefaultProxmoxUser = "orca"

// DefaultProxmoxRole is the default PVE custom role created for the
// orca user. Overridable via Options.ProxmoxRole.
const DefaultProxmoxRole = "OrcaOperator"

// DefaultSSHPort is the default SSH port for Proxmox hosts.
const DefaultSSHPort = 22

// OrcaOperatorPrivileges is the least-privilege privilege set for the
// OrcaOperator PVE role (D-033). Space-separated per pveum --privs
// syntax. VM.Audit covers CTs as well (both live under /vms/{vmid}).
const OrcaOperatorPrivileges = "VM.Audit Datastore.AllocateSpace SDN.Use"

// Options configures a Proxmox bootstrap run.
type Options struct {
	// Host is the Proxmox host address (IP or hostname, no port).
	Host string
	// SSHUser is the initial SSH username (default "root").
	SSHUser string
	// SSHKeyPath is the path to the private SSH key for key-based auth
	// (R-021: no passwords). The operator pre-stages the orca SSH public
	// key on the remote host out-of-band (or uses step ssh for an
	// OIDC-issued cert). Required.
	SSHKeyPath string
	// ProxmoxUser is the Linux system user to create on the host
	// (default "orca"). Config-overridable.
	ProxmoxUser string
	// ProxmoxRole is the PVE custom role to create (default
	// "OrcaOperator"). Config-overridable.
	ProxmoxRole string
	// SSHPort is the SSH port (default 22).
	SSHPort int
	// HostKeyFingerprint is the operator-pinned SSH host key fingerprint
	// in `SHA256:base64` form (REQ-058, D-044). When non-empty, the
	// bootstrap dialer uses a pinned-host-key callback instead of the
	// TOFU known_hosts capture path. Empty falls back to TOFU.
	HostKeyFingerprint string
	// Logger receives audit-log entries. If nil, slog.Default() is used.
	Logger *slog.Logger
}

// Result is the outcome of a successful bootstrap.
type Result struct {
	// NodeName is the name to use for the node in the orca registry
	// (typically the host address).
	NodeName string
	// NodeAddress is the orca daemon address on the Proxmox host
	// (host:8443 — the orca daemon port).
	NodeAddress string
	// HostKeyFingerprint is the SHA-256 fingerprint of the captured
	// SSH host key (for operator verification).
	HostKeyFingerprint string
}

// BootstrapProxmox runs the full SSH bootstrap sequence on a remote
// Proxmox VE 8/9 host. All steps are idempotent. Returns a Result
// describing the node to register, or an error if any step fails.
func BootstrapProxmox(ctx context.Context, opts Options) (*Result, error) {
	if opts.Host == "" {
		return nil, fmt.Errorf("proxmox bootstrap: host is required")
	}
	if opts.SSHKeyPath == "" {
		return nil, fmt.Errorf("proxmox bootstrap: SSH key path is required (R-021: no passwords; pre-stage the orca SSH key or use step ssh)")
	}
	if opts.SSHUser == "" {
		opts.SSHUser = "root"
	}
	if opts.ProxmoxUser == "" {
		opts.ProxmoxUser = DefaultProxmoxUser
	}
	if opts.ProxmoxRole == "" {
		opts.ProxmoxRole = DefaultProxmoxRole
	}
	if opts.SSHPort == 0 {
		opts.SSHPort = DefaultSSHPort
	}
	log := opts.Logger
	if log == nil {
		log = slog.Default()
	}

	// Step 1: Generate or load the orca SSH keypair (D-037).
	// The key is deployed to the remote host's authorized_keys in step 3.
	_, pubLine, err := security.GenerateOrLoadSSHKey(certpaths.Dir())
	if err != nil {
		return nil, fmt.Errorf("ssh key: %w", err)
	}

	// Step 2: SSH dial with password auth + host-key verification (D-035,
	// REQ-058). When opts.HostKeyFingerprint is set (D-044), use a pinned
	// callback that fails closed on mismatch (AD-028); otherwise use the
	// TOFU known_hosts capture callback (D-035). The TOFU wrapper fixes
	// the v0.6 ship-defect where knownhosts.New returned KeyError{Want:[]}
	// on first connect WITHOUT writing the captured key, so the first
	// `orca node join --type proxmox` always failed.
	sshAddr := fmt.Sprintf("%s:%d", opts.Host, opts.SSHPort)
	var capturedHostKey ssh.PublicKey
	var hostKeyCallback ssh.HostKeyCallback
	if opts.HostKeyFingerprint != "" {
		cb, err := pinnedHostKeyCallback(opts.HostKeyFingerprint, &capturedHostKey)
		if err != nil {
			return nil, fmt.Errorf("host-key fingerprint: %w", err)
		}
		hostKeyCallback = cb
	} else {
		cb, err := TOFUHostKeyCallback(sshAddr, &capturedHostKey)
		if err != nil {
			return nil, fmt.Errorf("tofu host-key callback: %w", err)
		}
		hostKeyCallback = cb
	}

	// Load the SSH private key for key-based auth (R-021: no passwords).
	keyBytes, err := os.ReadFile(opts.SSHKeyPath)
	if err != nil {
		return nil, fmt.Errorf("read SSH key %s: %w", opts.SSHKeyPath, err)
	}
	signer, err := ssh.ParsePrivateKey(keyBytes)
	if err != nil {
		return nil, fmt.Errorf("parse SSH key %s: %w", opts.SSHKeyPath, err)
	}
	sshConfig := &ssh.ClientConfig{
		User:            opts.SSHUser,
		Auth:            []ssh.AuthMethod{ssh.PublicKeys(signer)},
		HostKeyCallback: hostKeyCallback,
		Timeout:         10 * time.Second,
	}

	dialCtx, dialCancel := context.WithTimeout(ctx, 15*time.Second)
	defer dialCancel()
	conn, err := sshDialer.DialContext(dialCtx, "tcp", sshAddr, sshConfig)
	if err != nil {
		return nil, fmt.Errorf("ssh dial %s: %w", sshAddr, err)
	}
	defer conn.Close()

	if sessionRunner == nil {
		sessionRunner = &sshSessionRunner{client: conn}
	}

	hostKeyFP := ""
	if capturedHostKey != nil {
		hostKeyFP = security.SSHFingerprintSHA256(capturedHostKey)
	}

	log.Info("proxmox.ssh_connected",
		slog.String("event", "proxmox.ssh_connected"),
		slog.String("host", opts.Host),
		slog.String("ssh_user", opts.SSHUser),
		slog.String("host_key_fingerprint", hostKeyFP),
	)

	// Step 3: Deploy orca pubkey to ~orca/.ssh/authorized_keys (idempotent).
	if err := deployPubKey(opts.ProxmoxUser, string(pubLine)); err != nil {
		return nil, fmt.Errorf("deploy pubkey: %w", err)
	}

	// Step 4: Create orca Linux system user (idempotent).
	if err := createLinuxUser(opts.ProxmoxUser); err != nil {
		return nil, fmt.Errorf("create user %s: %w", opts.ProxmoxUser, err)
	}

	// Step 5: Create OrcaOperator PVE role (idempotent).
	if err := createPVERole(opts.ProxmoxRole); err != nil {
		return nil, fmt.Errorf("create PVE role %s: %w", opts.ProxmoxRole, err)
	}

	// Step 6: Create orca@pam PVE user (idempotent).
	if err := createPVEUser(opts.ProxmoxUser); err != nil {
		return nil, fmt.Errorf("create PVE user %s@pam: %w", opts.ProxmoxUser, err)
	}

	// Step 7: Assign OrcaOperator role to orca@pam on path / (idempotent).
	if err := assignPVEACL(opts.ProxmoxUser, opts.ProxmoxRole); err != nil {
		return nil, fmt.Errorf("assign ACL: %w", err)
	}

	// Step 8: Write /etc/sudoers.d/orca (AD-020: NOEXEC on pct/qm,
	// no NOEXEC on apt-get/dpkg, pvesh EXCLUDED).
	if err := writeSudoers(opts.ProxmoxUser); err != nil {
		return nil, fmt.Errorf("write sudoers: %w", err)
	}

	// Step 9: Validate sudoers with visudo -cf.
	if err := validateSudoers(); err != nil {
		return nil, fmt.Errorf("validate sudoers: %w", err)
	}

	log.Info("proxmox.bootstrap_ok",
		slog.String("event", "proxmox.bootstrap_ok"),
		slog.String("host", opts.Host),
		slog.String("proxmox_user", opts.ProxmoxUser),
		slog.String("proxmox_role", opts.ProxmoxRole),
	)

	return &Result{
		NodeName:           opts.Host,
		NodeAddress:        opts.Host + ":8443",
		HostKeyFingerprint: hostKeyFP,
	}, nil
}

// sshDialer is the dialer used by BootstrapProxmox. It's a package-level
// variable so tests can override it with a fake SSH server.
var sshDialer sshDialerType = defaultSSHDialer{}

// pinnedHostKeyCallback returns an ssh.HostKeyCallback that pins the
// server's host key to the operator-supplied SHA256:base64 fingerprint
// (REQ-058, AD-028). It validates the `SHA256:` prefix up front (D-045)
// and fails closed on any mismatch. The capturedKey out-param records
// the verified server key so the caller can populate Result.
func pinnedHostKeyCallback(expectedSHA256Base64 string, capturedKey *ssh.PublicKey) (ssh.HostKeyCallback, error) {
	if !strings.HasPrefix(expectedSHA256Base64, "SHA256:") {
		return nil, fmt.Errorf("pinnedHostKeyCallback: fingerprint must be SHA256:-prefixed (D-045), got %q", expectedSHA256Base64)
	}
	return func(_ string, _ net.Addr, key ssh.PublicKey) error {
		got := security.SSHFingerprintSHA256(key)
		if got != expectedSHA256Base64 {
			return fmt.Errorf("REQ-058 host-key fingerprint mismatch: pinned=%s server=%s", expectedSHA256Base64, got)
		}
		if capturedKey != nil {
			*capturedKey = key
		}
		return nil
	}, nil
}

// TOFUHostKeyCallback returns an ssh.HostKeyCallback that wraps the
// standard knownhosts.New verifier with TOFU first-connect capture
// (D-035). On a host-unknown KeyError{Want:[]} it writes the
// server-presented key to certpaths.KnownHostsPath() atomically
// (security.WriteAtomic, AD-029) and allows the dial to proceed; on a
// mismatch (Want non-empty) it fails closed (MITM detection). The
// capturedKey out-param records the verified/captured server key so
// the caller can populate Result. This fixes the v0.6 ship-defect
// where knownhosts.New returned KeyError{Want:[]} on first connect
// WITHOUT writing the captured key, so the first
// `orca node join --type proxmox` always failed.
//
// Exported so the doctor proxmox probe (T02.9) can reuse the same
// capture-fix wrapper for parity (GRILL condition #2).
func TOFUHostKeyCallback(addr string, capturedKey *ssh.PublicKey) (ssh.HostKeyCallback, error) {
	cb, err := knownhosts.New(certpaths.KnownHostsPath())
	if err != nil {
		return nil, err
	}
	return func(hostname string, remote net.Addr, key ssh.PublicKey) error {
		err := cb(hostname, remote, key)
		if err == nil {
			if capturedKey != nil {
				*capturedKey = key
			}
			return nil
		}
		var keyErr *knownhosts.KeyError
		if errors.As(err, &keyErr) && len(keyErr.Want) == 0 {
			line := knownhosts.Line([]string{knownhosts.Normalize(addr)}, key)
			path := certpaths.KnownHostsPath()
			release, lockErr := security.Flock(path)
			if lockErr != nil {
				return fmt.Errorf("tofu lock known_hosts: %w", lockErr)
			}
			defer release()
			existing, readErr := os.ReadFile(path)
			if readErr != nil && !os.IsNotExist(readErr) {
				return fmt.Errorf("tofu read known_hosts: %w", readErr)
			}
			if len(existing) > 0 && !bytes.HasSuffix(existing, []byte("\n")) {
				existing = append(existing, '\n')
			}
			updated := append(existing, []byte(line)...)
			if writeErr := security.WriteAtomic(path, 0o600, updated); writeErr != nil {
				return fmt.Errorf("tofu write known_hosts: %w", writeErr)
			}
			if capturedKey != nil {
				*capturedKey = key
			}
			return nil
		}
		return err
	}, nil
}

type sshDialerType interface {
	DialContext(ctx context.Context, network, addr string, config *ssh.ClientConfig) (*ssh.Client, error)
}

type defaultSSHDialer struct{}

func (defaultSSHDialer) DialContext(ctx context.Context, network, addr string, config *ssh.ClientConfig) (*ssh.Client, error) {
	return ssh.Dial(network, addr, config)
}

type sessionRunnerType interface {
	CombinedOutput(cmd string) ([]byte, error)
}

var sessionRunner sessionRunnerType

type sshSessionRunner struct {
	client *ssh.Client
}

func (r *sshSessionRunner) CombinedOutput(cmd string) ([]byte, error) {
	session, err := r.client.NewSession()
	if err != nil {
		return nil, fmt.Errorf("new session: %w", err)
	}
	defer session.Close()
	return session.CombinedOutput(cmd)
}

// runRemote runs a command over the SSH connection and returns its
// combined output. Returns an error if the command exits non-zero.
func runRemote(cmd string) ([]byte, error) {
	out, err := sessionRunner.CombinedOutput(cmd)
	if err != nil {
		return out, fmt.Errorf("run %q: %w (output: %s)", cmd, err, strings.TrimSpace(string(out)))
	}
	return out, nil
}

// deployPubKey appends the orca public key to the remote user's
// authorized_keys file, creating the .ssh dir if needed. Idempotent:
// if the key is already present, it is not re-appended.
func deployPubKey(user, pubLine string) error {
	pubLine = strings.TrimSpace(pubLine)
	if pubLine == "" {
		return fmt.Errorf("deployPubKey: empty pub line")
	}
	home := "/home/" + user
	if user == "root" {
		home = "/root"
	}
	sshDir := home + "/.ssh"
	authFile := sshDir + "/authorized_keys"
	// Create .ssh dir, touch authorized_keys, set modes, append key if absent.
	cmd := fmt.Sprintf(
		"mkdir -p %s && touch %s && chmod 0700 %s && chmod 0600 %s && grep -qF '%s' %s || echo '%s' >> %s",
		sshDir, authFile, sshDir, authFile, pubLine, authFile, pubLine, authFile,
	)
	if _, err := runRemote(cmd); err != nil {
		return err
	}
	return nil
}

// createLinuxUser creates the orca system user if it doesn't already
// exist. Idempotent: `id -u` check before `useradd`.
func createLinuxUser(user string) error {
	cmd := fmt.Sprintf("id -u %s 2>/dev/null || useradd -m -s /bin/bash %s", user, user)
	if _, err := runRemote(cmd); err != nil {
		return err
	}
	return nil
}

// createPVERole creates the OrcaOperator PVE role if it doesn't exist.
// Idempotent: probes `pveum role list` before `pveum role add`.
func createPVERole(role string) error {
	cmd := fmt.Sprintf(
		"pveum role list 2>/dev/null | grep -q '^%s' || pveum role add %s --privs '%s'",
		role, role, OrcaOperatorPrivileges,
	)
	if _, err := runRemote(cmd); err != nil {
		return err
	}
	return nil
}

// createPVEUser creates the orca@pam PVE user if it doesn't exist.
// Idempotent: probes `pveum user list` before `pveum user add`.
// Uses @pam realm (AD-019) since orca creates a Linux system user.
func createPVEUser(user string) error {
	pveUserID := user + "@pam"
	cmd := fmt.Sprintf(
		"pveum user list 2>/dev/null | grep -q '%s' || pveum user add %s -comment 'Orca automation user'",
		pveUserID, pveUserID,
	)
	if _, err := runRemote(cmd); err != nil {
		return err
	}
	return nil
}

// assignPVEACL assigns the OrcaOperator role to orca@pam on path /
// (cluster-wide). `pveum acl modify` is idempotent (creates or updates).
func assignPVEACL(user, role string) error {
	pveUserID := user + "@pam"
	cmd := fmt.Sprintf("pveum acl modify / -user %s -role %s", pveUserID, role)
	if _, err := runRemote(cmd); err != nil {
		return err
	}
	return nil
}

// sudoersContent returns the /etc/sudoers.d/orca file content (AD-020).
// NOEXEC on pct/qm (blocks shell escapes); no NOEXEC on apt-get/dpkg
// (they need exec for maintainer scripts); pvesh EXCLUDED (API execute
// bypasses NOEXEC). File must be mode 0440 per sudo requirements.
func sudoersContent(user string) string {
	return fmt.Sprintf(`# /etc/sudoers.d/orca — Managed by orca; do not edit manually.
# Least-privilege allowlist for the orca PVE operator user.
# NOPASSWD: non-interactive SSH automation. NOEXEC: blocks shell escapes.
# pvesh is EXCLUDED (AD-020: pvesh can bypass NOEXEC via API execute).
%s ALL=(root) NOPASSWD: NOEXEC: /usr/bin/pct
%s ALL=(root) NOPASSWD: NOEXEC: /usr/bin/qm
%s ALL=(root) NOPASSWD: /usr/bin/apt-get
%s ALL=(root) NOPASSWD: /usr/bin/dpkg
`, user, user, user, user)
}

// writeSudoers writes the /etc/sudoers.d/orca file on the remote host
// with mode 0440. Uses a heredoc via cat to avoid quoting issues.
func writeSudoers(user string) error {
	content := sudoersContent(user)
	// Write via cat heredoc, then chmod 0440.
	cmd := fmt.Sprintf("cat > /etc/sudoers.d/%s <<'ORCA_SUDOERS_EOF'\n%s\nORCA_SUDOERS_EOF\nchmod 0440 /etc/sudoers.d/%s",
		user, content, user)
	if _, err := runRemote(cmd); err != nil {
		return err
	}
	return nil
}

// validateSudoers runs `visudo -cf` on the sudoers file. Aborts the
// bootstrap if validation fails (prevents a broken sudoers from
// locking the orca user out of sudo).
func validateSudoers() error {
	cmd := "visudo -cf /etc/sudoers.d/orca"
	out, err := runRemote(cmd)
	if err != nil {
		return fmt.Errorf("visudo validation failed: %w (output: %s)", err, strings.TrimSpace(string(out)))
	}
	if !strings.Contains(string(out), "parsed OK") {
		return fmt.Errorf("visudo validation did not report OK: %s", strings.TrimSpace(string(out)))
	}
	return nil
}

// ResetHostKey removes all known_hosts entries for the given host from
// certpaths.KnownHostsPath() (REQ-059, D-046, AD-029). It rewrites the
// file atomically via security.WriteAtomic. LOCAL ONLY — it does NOT
// touch the remote host's authorized_keys (D-046). The next connect
// re-pins the host key via TOFU (T02.6) or the --host-key-fingerprint
// pinned path (T02.5).
//
// A line matches when its first whitespace-delimited field (the host
// pattern, normalized via knownhosts.Normalize) equals the normalized
// target host. Comment/blank lines are preserved.
func ResetHostKey(host string) error {
	if host == "" {
		return fmt.Errorf("ResetHostKey: host is required")
	}
	path := certpaths.KnownHostsPath()
	release, lockErr := security.Flock(path)
	if lockErr != nil {
		return fmt.Errorf("ResetHostKey: lock known_hosts: %w", lockErr)
	}
	defer release()
	existing, err := os.ReadFile(path)
	if err != nil {
		if os.IsNotExist(err) {
			return nil // nothing to reset
		}
		return fmt.Errorf("ResetHostKey: read known_hosts: %w", err)
	}
	target := knownhosts.Normalize(host)
	var kept []byte
	removed := 0
	for _, line := range strings.Split(string(existing), "\n") {
		trimmed := strings.TrimSpace(line)
		if trimmed == "" || strings.HasPrefix(trimmed, "#") {
			kept = append(kept, []byte(line+"\n")...)
			continue
		}
		fields := strings.Fields(trimmed)
		if len(fields) == 0 {
			kept = append(kept, []byte(line+"\n")...)
			continue
		}
		if knownhosts.Normalize(fields[0]) == target {
			removed++
			continue
		}
		kept = append(kept, []byte(line+"\n")...)
	}
	if removed == 0 {
		return nil
	}
	// Ensure the kept buffer ends with exactly one trailing newline.
	kept = bytes.TrimRight(kept, "\n")
	if len(kept) > 0 {
		kept = append(kept, '\n')
	}
	if err := security.WriteAtomic(path, 0o600, kept); err != nil {
		return fmt.Errorf("ResetHostKey: rewrite known_hosts: %w", err)
	}
	return nil
}