diff --git a/internal/cli/node.go b/internal/cli/node.go index 0fe94ec..539514f 100644 --- a/internal/cli/node.go +++ b/internal/cli/node.go @@ -1,18 +1,22 @@ package cli import ( + "bufio" "context" "database/sql" "encoding/json" "fmt" "log/slog" + "net" "os" "os/signal" + "strings" "syscall" "time" "github.com/google/uuid" "github.com/spf13/cobra" + "golang.org/x/crypto/ssh" "git.cloudinit.dev/coreci/orca/internal/certpaths" "git.cloudinit.dev/coreci/orca/internal/engine" @@ -180,15 +184,85 @@ func joinProxmox(cmd *cobra.Command) error { return fmt.Errorf("SSH key path is required for --type proxmox (R-021: no passwords; use --ssh-key or pre-stage the orca key)") } - ctx, cancel := context.WithTimeout(cmd.Context(), 60*time.Second) - defer cancel() - // Default ingress mode to "native" if not specified (R-024). effectiveIngressMode := ingressMode if effectiveIngressMode == "" { - effectiveIngressMode = "native" + if !jsonOutput { + // Interactive mode: prompt for ingress mode. + fmt.Fprint(cmd.OutOrStdout(), "Ingress mode [native/floating-ip] (default native): ") + scanner := bufio.NewScanner(os.Stdin) + if scanner.Scan() { + input := strings.TrimSpace(scanner.Text()) + if input == "floating-ip" { + effectiveIngressMode = "floating-ip" + } else { + effectiveIngressMode = "native" + } + } else { + effectiveIngressMode = "native" + } + } else { + effectiveIngressMode = "native" + } } + // Floating-IP mode: prompt for params if not provided. + effectiveFloatingIP := floatingIP + effectiveGateway := gateway + effectiveMAC := macAddr + if effectiveIngressMode == "floating-ip" { + if effectiveFloatingIP == "" && !jsonOutput { + fmt.Fprint(cmd.OutOrStdout(), "Floating IP: ") + scanner := bufio.NewScanner(os.Stdin) + if scanner.Scan() { + effectiveFloatingIP = strings.TrimSpace(scanner.Text()) + } + } + if effectiveGateway == "" && !jsonOutput { + fmt.Fprint(cmd.OutOrStdout(), "Gateway: ") + scanner := bufio.NewScanner(os.Stdin) + if scanner.Scan() { + effectiveGateway = strings.TrimSpace(scanner.Text()) + } + } + if effectiveMAC == "" && !jsonOutput { + // D-261: auto-generate a random locally-administered MAC. + generated, err := proxmox.GenerateRandomMAC() + if err == nil { + fmt.Fprintf(cmd.OutOrStdout(), "Generated MAC: %s (press enter to accept, or type your own): ", generated) + scanner := bufio.NewScanner(os.Stdin) + if scanner.Scan() { + input := strings.TrimSpace(scanner.Text()) + if input != "" { + effectiveMAC = input + } else { + effectiveMAC = generated + } + } else { + effectiveMAC = generated + } + } + } + // Validate floating-IP mode params. + if effectiveIngressMode == "floating-ip" { + if net.ParseIP(effectiveFloatingIP) == nil { + return fmt.Errorf("--floating-ip %q is not a valid IP", effectiveFloatingIP) + } + if net.ParseIP(effectiveGateway) == nil { + return fmt.Errorf("--gateway %q is not a valid IP", effectiveGateway) + } + if _, err := net.ParseMAC(effectiveMAC); err != nil { + return fmt.Errorf("--mac %q is not a valid MAC: %w", effectiveMAC, err) + } + if netPrefix < 8 || netPrefix > 32 { + return fmt.Errorf("--net-prefix %d must be 8-32", netPrefix) + } + } + } + + ctx, cancel := context.WithTimeout(cmd.Context(), 180*time.Second) // 3min for LXC creation + defer cancel() + result, err := proxmox.BootstrapProxmox(ctx, proxmox.Options{ Host: joinHost, SSHUser: joinSSHUser, @@ -229,6 +303,54 @@ func joinProxmox(cmd *cobra.Command) error { if err := registry.Join(regCtx, node); err != nil { return fmt.Errorf("register proxmox node: %w", err) } + + // Floating-IP mode: provision the ingress LXC and register it as a + // linux node (R-024, REQ-176). The PVE host is registered as + // proxmox (above); the ingress LXC is registered as linux so + // `orca job run` pushes traefik dynamic config to it. + if effectiveIngressMode == "floating-ip" { + lxcLog := newLogger() + // Build a runRemote function from the proxmox bootstrap result. + // We need SSH access to the PVE host to run pct commands. + lxcCtx, lxcCancel := context.WithTimeout(ctx, 120*time.Second) + defer lxcCancel() + // The BootstrapProxmox result gives us the host; we need to + // re-establish the SSH connection for the LXC provisioning. + lxcExecFn, lxcErr := proxmoxRemoteExecFn(result, sshKeyPath, joinSSHUser, joinSSHPort) + if lxcErr != nil { + fmt.Fprintf(cmd.OutOrStdout(), "Warning: could not establish SSH for LXC provisioning: %v\n", lxcErr) + } else { + if err := proxmox.ProvisionIngressLXC(lxcCtx, lxcExecFn, proxmox.FloatingIPOptions{ + FloatingIP: effectiveFloatingIP, + Gateway: effectiveGateway, + MAC: effectiveMAC, + NetPrefix: netPrefix, + LXCTemplate: joinLXCTemplate, + }, lxcLog); err != nil { + fmt.Fprintf(cmd.OutOrStdout(), "Warning: ingress LXC provisioning failed: %v\n", err) + } else { + // Register the ingress LXC as a linux node. + ingressNode := &model.Node{ + ID: uuid.NewString(), + Name: "ingress", + Address: fmt.Sprintf("%s:8443", effectiveFloatingIP), + State: model.NodeStateReady, + JoinedAt: time.Now().UTC(), + LastSeen: time.Now().UTC(), + Kind: string(model.NodeKindLinux), + OS: "linux", + IngressMode: "floating-ip", + } + if err := registry.Join(regCtx, ingressNode); err != nil { + fmt.Fprintf(cmd.OutOrStdout(), "Warning: register ingress node: %v\n", err) + } + if !jsonOutput { + fmt.Fprintf(cmd.OutOrStdout(), "✓ Ingress LXC joined: %s (%s) at %s\n", ingressNode.ID, ingressNode.Name, ingressNode.Address) + } + } + } + } + // REQ-156 / P07 T5: invalidate the nodes cache. cacheInvalidate(cacheNodeClass) if jsonOutput { @@ -239,6 +361,46 @@ func joinProxmox(cmd *cobra.Command) error { return nil } +// proxmoxRemoteExecFn creates a RemoteExecFunc (func(string) ([]byte, +// error)) that runs commands on the PVE host via SSH. Used by the +// floating-IP LXC provisioning path (ProvisionIngressLXC). +func proxmoxRemoteExecFn(result *proxmox.Result, sshKeyPath, sshUser string, sshPort int) (func(string) ([]byte, error), error) { + cfg := &ssh.ClientConfig{ + User: sshUser, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + Timeout: 10 * time.Second, + } + if sshKeyPath != "" { + keyData, err := os.ReadFile(sshKeyPath) + if err != nil { + return nil, fmt.Errorf("read SSH key: %w", err) + } + signer, err := ssh.ParsePrivateKey(keyData) + if err != nil { + return nil, fmt.Errorf("parse SSH key: %w", err) + } + cfg.Auth = []ssh.AuthMethod{ssh.PublicKeys(signer)} + } + addr := result.NodeName + if sshPort != 22 { + addr = fmt.Sprintf("%s:%d", result.NodeName, sshPort) + } else { + addr = fmt.Sprintf("%s:%d", result.NodeName, sshPort) + } + client, err := ssh.Dial("tcp", addr, cfg) + if err != nil { + return nil, fmt.Errorf("ssh dial %s: %w", addr, err) + } + return func(cmd string) ([]byte, error) { + session, err := client.NewSession() + if err != nil { + return nil, err + } + defer session.Close() + return session.CombinedOutput(cmd) + }, nil +} + // joinLinux bootstraps a remote generic Linux worker via SSH and // registers it as an orca node (REQ-161, P12). Uses SSH key auth // (R-021: no passwords). diff --git a/internal/proxmox/ingress_lxc.go b/internal/proxmox/ingress_lxc.go new file mode 100644 index 0000000..c250972 --- /dev/null +++ b/internal/proxmox/ingress_lxc.go @@ -0,0 +1,171 @@ +package proxmox + +import ( + "context" + "crypto/rand" + "fmt" + "log/slog" + "os" + "strings" + "time" + + "git.cloudinit.dev/coreci/orca/internal/certpaths" + "git.cloudinit.dev/coreci/orca/internal/emitter" + "git.cloudinit.dev/coreci/orca/internal/traefik" +) + +// FloatingIPOptions carries the parameters for provisioning a +// floating-IP ingress LXC (R-024, REQ-176). +type FloatingIPOptions struct { + // FloatingIP is the public IP assigned to the LXC's eth0. + FloatingIP string + // Gateway is the default gateway for the LXC. + Gateway string + // MAC is the MAC address for the LXC's net0 interface. + MAC string + // NetPrefix is the CIDR prefix for the floating IP (8-32). + NetPrefix int + // LXCTemplate is the LXC template (default "ubuntu-24.04"). + LXCTemplate string + // VMID is the LXC container ID (default "201" for the ingress LXC). + VMID string +} + +// ProvisionIngressLXC creates an Ubuntu LXC named "ingress" that owns +// the floating IP, installs podman + orca-traefik inside it, applies nft +// DNAT+SNAT inside the LXC, and returns the LXC's IP for node +// registration (R-024, REQ-176). +// +// The LXC is created with: +// +// --unprivileged 1 --features nesting=1,keyctl=1,fuse=1 +// --net0 name=eth0,bridge=vmbr0,hwaddr=,ip=/,gw= +// --onboot 1 +// +// Inside the LXC, the complete ingress stack is set up: podman +// installed, traefik static config written, nft DNAT:443→127.0.0.1:8443 +// + postrouting masquerade applied, orca-traefik podman container +// running with --network host. +// +// Idempotent: if the LXC already exists, it is not re-created (C-53). +func ProvisionIngressLXC(ctx context.Context, runRemote func(string) ([]byte, error), opts FloatingIPOptions, log *slog.Logger) error { + template := opts.LXCTemplate + if template == "" { + template = "ubuntu-24.04" + } + vmid := opts.VMID + if vmid == "" { + vmid = "201" + } + if opts.NetPrefix == 0 { + opts.NetPrefix = 24 + } + + // Validate required fields. + if opts.FloatingIP == "" || opts.Gateway == "" || opts.MAC == "" { + return fmt.Errorf("ingress_lxc: floating-ip, gateway, and mac are required") + } + + // Ensure the template is downloaded. + _, _ = runRemote(fmt.Sprintf("pveam download local %s 2>/dev/null || true", shellQuote(template))) + + // Check if the LXC already exists (idempotent — C-53). + existOut, _ := runRemote(fmt.Sprintf("pct status %s 2>/dev/null || echo absent", vmid)) + existStr := strings.TrimSpace(string(existOut)) + if existStr == "absent" { + log.Info("ingress_lxc.creating", "vmid", vmid, "hostname", "ingress", "ip", opts.FloatingIP) + net0 := fmt.Sprintf("name=eth0,bridge=vmbr0,hwaddr=%s,ip=%s/%d,gw=%s", + opts.MAC, opts.FloatingIP, opts.NetPrefix, opts.Gateway) + createCmd := fmt.Sprintf( + "pct create %s local:vztmpl/%s --hostname ingress --unprivileged 1 --features nesting=1,keyctl=1,fuse=1 --net0 %s --onboot 1 --memory 2048 --swap 0 --rootfs local:8 2>&1", + vmid, shellQuote(template), net0, + ) + if out, err := runRemote(createCmd); err != nil { + return fmt.Errorf("pct create ingress LXC: %w (output: %s)", err, string(out)) + } + if out, err := runRemote(fmt.Sprintf("pct start %s", vmid)); err != nil { + return fmt.Errorf("pct start ingress LXC: %w (output: %s)", err, string(out)) + } + } + + // Wait for LXC network (retry for up to 60s). + for i := 0; i < 12; i++ { + ipOut, _ := runRemote(fmt.Sprintf("pct exec %s -- hostname -I 2>/dev/null", vmid)) + ipStr := strings.TrimSpace(string(ipOut)) + if ipStr != "" { + break + } + time.Sleep(5 * time.Second) + } + log.Info("ingress_lxc.network_ready", "vmid", vmid, "ip", opts.FloatingIP) + + // Install podman inside the LXC (C-53: idempotent). + _, _ = runRemote(fmt.Sprintf( + "pct exec %s -- bash -c 'command -v podman >/dev/null 2>&1 || (apt-get update -qq && apt-get install -y -qq podman conmon crun fuse-overlayfs nftables 2>&1)' 2>&1", + vmid, + )) + + // Enable podman-restart.service inside the LXC (research Topic 6). + _, _ = runRemote(fmt.Sprintf("pct exec %s -- systemctl enable --now podman-restart.service 2>/dev/null", vmid)) + + // Push step-ca root CA into the LXC (C-60: CACertPath). + caPath := certpaths.CACertPath() + caData, caErr := os.ReadFile(caPath) + if caErr != nil { + caData = []byte{} + } + caDelim := "EOF_CA" + _, _ = runRemote(fmt.Sprintf( + "pct exec %s -- bash -c 'mkdir -p /etc/orca && cat > /etc/orca/step-ca-root.crt <<%s\\n%s\\n%s'", + vmid, caDelim, string(caData), caDelim, + )) + + // Render + write traefik static config inside the LXC (C-58). + staticFiles, err := emitter.TraefikEmitter{}.RenderTraefikStaticConfig(emitter.TraefikStaticOpts{}) + if err == nil { + for _, f := range staticFiles { + delim := "EOF_TF" + _, _ = runRemote(fmt.Sprintf( + "pct exec %s -- bash -c 'mkdir -p /etc/traefik/dynamic && cat > %s <<%s\\n%s\\n%s'", + vmid, f.Path, delim, f.Content, delim, + )) + } + } + + // Render + apply nft DNAT+SNAT INSIDE the LXC (DNATTarget = + // 127.0.0.1 — traefik runs with --network host inside the LXC). + nftFiles, err := emitter.NftEmitter{}.RenderNftConfig(emitter.NftClusterConfig{}) + if err == nil { + for _, f := range nftFiles { + delim := "EOF_NF" + _, _ = runRemote(fmt.Sprintf( + "pct exec %s -- bash -c 'mkdir -p /etc/nftables.d && cat > %s <<%s\\n%s\\n%s'", + vmid, f.Path, delim, f.Content, delim, + )) + } + _, _ = runRemote(fmt.Sprintf("pct exec %s -- nft add table inet orca-ingress 2>/dev/null || true", vmid)) + _, _ = runRemote(fmt.Sprintf("pct exec %s -- nft -f /etc/nftables.d/orca.nft 2>&1", vmid)) + } + + // Ensure podman orca-traefik container inside the LXC. + traefikExecFn := func(cmd string) ([]byte, error) { + return runRemote(fmt.Sprintf("pct exec %s -- bash -c %s 2>&1", vmid, shellQuote(cmd))) + } + if err := traefik.EnsureTraefikContainerRemote(ctx, "", traefikExecFn); err != nil { + log.Warn("ingress_lxc.traefik_failed", "err", err) + } + + log.Info("ingress_lxc.provisioned", "vmid", vmid, "ip", opts.FloatingIP) + return nil +} + +// GenerateRandomMAC generates a random locally-administered MAC address +// (02:XX:XX:XX:XX:XX) for use as the LXC net0 hardware address when the +// operator does not provide one (D-261). +func GenerateRandomMAC() (string, error) { + b := make([]byte, 5) + if _, err := rand.Read(b); err != nil { + return "", fmt.Errorf("generate MAC: %w", err) + } + return fmt.Sprintf("02:%02x:%02x:%02x:%02x:%02x", b[0], b[1], b[2], b[3], b[4]), nil +}