diff --git a/internal/cli/node.go b/internal/cli/node.go index 731286c..0fe94ec 100644 --- a/internal/cli/node.go +++ b/internal/cli/node.go @@ -15,8 +15,8 @@ import ( "github.com/spf13/cobra" "git.cloudinit.dev/coreci/orca/internal/certpaths" - "git.cloudinit.dev/coreci/orca/internal/linux" "git.cloudinit.dev/coreci/orca/internal/engine" + "git.cloudinit.dev/coreci/orca/internal/linux" "git.cloudinit.dev/coreci/orca/internal/model" "git.cloudinit.dev/coreci/orca/internal/proxmox" "git.cloudinit.dev/coreci/orca/internal/security" @@ -46,20 +46,25 @@ func nodeRegistry() (*engine.NodeRegistry, func() error, error) { } var ( - joinName string - joinAddr string - joinCAFinger string - joinType string - joinHost string - joinSSHUser string - joinSSHKey string - joinSSHPort int - joinHostKeyFP string + joinName string + joinAddr string + joinCAFinger string + joinType string + joinHost string + joinSSHUser string + joinSSHKey string + joinSSHPort int + joinHostKeyFP string joinLXCTemplate string - proxmoxUser string - proxmoxRole string - leaveID string - nodeWatch bool + proxmoxUser string + proxmoxRole string + ingressMode string + floatingIP string + gateway string + macAddr string + netPrefix int + leaveID string + nodeWatch bool ) var nodeCmd = &cobra.Command{ @@ -178,6 +183,12 @@ func joinProxmox(cmd *cobra.Command) error { ctx, cancel := context.WithTimeout(cmd.Context(), 60*time.Second) defer cancel() + // Default ingress mode to "native" if not specified (R-024). + effectiveIngressMode := ingressMode + if effectiveIngressMode == "" { + effectiveIngressMode = "native" + } + result, err := proxmox.BootstrapProxmox(ctx, proxmox.Options{ Host: joinHost, SSHUser: joinSSHUser, @@ -188,6 +199,7 @@ func joinProxmox(cmd *cobra.Command) error { HostKeyFingerprint: joinHostKeyFP, Logger: newLogger(), LXCTemplate: joinLXCTemplate, + IngressMode: effectiveIngressMode, }) if err != nil { return fmt.Errorf("proxmox bootstrap: %w", err) @@ -204,14 +216,15 @@ func joinProxmox(cmd *cobra.Command) error { defer regCancel() node := &model.Node{ - ID: uuid.NewString(), - Name: result.NodeName, - Address: result.NodeAddress, - State: model.NodeStateReady, - JoinedAt: time.Now().UTC(), - LastSeen: time.Now().UTC(), - Kind: string(model.NodeKindProxmox), - OS: "pve", + ID: uuid.NewString(), + Name: result.NodeName, + Address: result.NodeAddress, + State: model.NodeStateReady, + JoinedAt: time.Now().UTC(), + LastSeen: time.Now().UTC(), + Kind: string(model.NodeKindProxmox), + OS: "pve", + IngressMode: effectiveIngressMode, } if err := registry.Join(regCtx, node); err != nil { return fmt.Errorf("register proxmox node: %w", err) @@ -252,7 +265,7 @@ func joinLinux(cmd *cobra.Command) error { SSHPort: joinSSHPort, HostKeyFingerprint: joinHostKeyFP, Logger: newLogger(), - }) + }) if err != nil { return fmt.Errorf("linux bootstrap: %w", err) } @@ -514,6 +527,11 @@ func init() { nodeJoinCmd.Flags().StringVar(&proxmoxRole, "proxmox-role", "OrcaOperator", "PVE custom role to create (config-overridable)") nodeJoinCmd.Flags().StringVar(&joinHostKeyFP, "host-key-fingerprint", "", "SSH host key SHA256:base64 fingerprint (pre-pin; supersedes TOFU for --type proxmox or --type linux)") nodeJoinCmd.Flags().StringVar(&joinLXCTemplate, "lxc-template", "ubuntu-24.04", "LXC template for Proxmox (default ubuntu-24.04; alternatives: alpine-3.20, debian-12)") + nodeJoinCmd.Flags().StringVar(&ingressMode, "ingress-mode", "", "proxmox ingress mode: native (default, traefik in LXC) or floating-ip (ingress LXC owns floating IP)") + nodeJoinCmd.Flags().StringVar(&floatingIP, "floating-ip", "", "floating public IP for the ingress LXC (required for --ingress-mode floating-ip)") + nodeJoinCmd.Flags().StringVar(&gateway, "gateway", "", "gateway for the ingress LXC (required for --ingress-mode floating-ip)") + nodeJoinCmd.Flags().StringVar(&macAddr, "mac", "", "MAC address for the ingress LXC net0 (required for --ingress-mode floating-ip in --json mode; auto-generated in interactive mode)") + nodeJoinCmd.Flags().IntVar(&netPrefix, "net-prefix", 24, "network prefix (CIDR) for the ingress LXC IP (default 24; valid 8-32)") nodeLeaveCmd.Flags().StringVar(&leaveID, "id", "", "node id") nodeListCmd.Flags().BoolVar(&nodeWatch, "watch", false, "stream nodes until Ctrl-C (table refresh or --json per-event)") diff --git a/internal/model/node.go b/internal/model/node.go index a0c028a..3be6c9d 100644 --- a/internal/model/node.go +++ b/internal/model/node.go @@ -39,4 +39,9 @@ type Node struct { // OS is the auto-detected OS identifier from /etc/os-release ID= // (ubuntu|debian|alpine|pve|linux). Empty for pre-0006 rows. OS string `json:"os,omitempty"` + // IngressMode is the ingress configuration for the node (R-024, + // v0.14). Values: "" (legacy/default for linux/localhost), + // "native" (proxmox native — traefik in LXC), "floating-ip" + // (proxmox floating-IP — ingress LXC owns the floating IP). + IngressMode string `json:"ingress_mode,omitempty"` } diff --git a/internal/proxmox/bootstrap.go b/internal/proxmox/bootstrap.go index c9f5f05..2ac3936 100644 --- a/internal/proxmox/bootstrap.go +++ b/internal/proxmox/bootstrap.go @@ -37,6 +37,7 @@ import ( "golang.org/x/crypto/ssh/knownhosts" "git.cloudinit.dev/coreci/orca/internal/certpaths" + "git.cloudinit.dev/coreci/orca/internal/emitter" "git.cloudinit.dev/coreci/orca/internal/security" "git.cloudinit.dev/coreci/orca/internal/traefik" ) @@ -86,6 +87,13 @@ type Options struct { // LXCTemplate is the LXC template to download during bootstrap // (default "ubuntu-24.04"; alternatives: "alpine-3.20", "debian-12"). LXCTemplate string + // IngressMode is the proxmox ingress mode (R-024, v0.14). + // "native" (default): traefik runs in an unprivileged LXC with + // nesting=1,keyctl=1,fuse=1 on the PVE host. nft on the PVE host + // DNATs to the LXC bridge IP. + // "floating-ip": a separate ingress LXC owns the floating IP; + // nft runs inside that LXC. See ProvisionIngressLXC (P6). + IngressMode string } // Result is the outcome of a successful bootstrap. @@ -247,23 +255,24 @@ func BootstrapProxmox(ctx context.Context, opts Options) (*Result, error) { return nil, fmt.Errorf("validate sudoers: %w", err) } - // Step 9a: Ensure orca-traefik podman container on the Proxmox host - // (REQ-172, R-024). Replaces v0.13 binary+systemd install. - // In native mode (default for v0.14 P5), the container runs inside - // an LXC with nesting. For now, this installs on the PVE host OS. - // Idempotent: no-op if container already running. - if err := traefik.EnsureTraefikContainerRemote(ctx, "", runRemote); err != nil { - log.Warn("proxmox.traefik_container_failed", "err", err) - } - - // Step 9b: Download default LXC template (REQ-167, Phase C). - // Default: ubuntu-24.04. Configurable via --lxc-template. + // Step 9a: Proxmox native ingress mode (R-024, REQ-175). + // Create an unprivileged LXC with nesting=1,keyctl=1,fuse=1 (research + // Topic 3), install podman inside it, and run the orca-traefik + // container. nft on the PVE host DNATs to the LXC bridge IP. + // Default mode is "native"; floating-ip mode is handled separately + // (P6 — ProvisionIngressLXC). template := opts.LXCTemplate if template == "" { template = "ubuntu-24.04" } _, _ = runRemote(fmt.Sprintf("pveam download local %s 2>/dev/null || true", shellQuote(template))) + if opts.IngressMode != "floating-ip" { + if err := provisionNativeIngressLXC(ctx, runRemote, template, log); err != nil { + log.Warn("proxmox.native_ingress_lxc_failed", "err", err) + } + } + log.Info("proxmox.bootstrap_ok", slog.String("event", "proxmox.bootstrap_ok"), slog.String("host", opts.Host), @@ -278,6 +287,119 @@ func BootstrapProxmox(ctx context.Context, opts Options) (*Result, error) { }, nil } +// provisionNativeIngressLXC creates an unprivileged LXC with +// nesting=1,keyctl=1,fuse=1 (research Topic 3), installs podman inside +// it, runs the orca-traefik container, and applies nft DNAT on the PVE +// host targeting the LXC's bridge IP (R-024, REQ-175). +// +// The LXC is named "orca-traefik" and uses a deterministic VMID derived +// from the host. It is idempotent: if the LXC already exists, it is +// not re-created (C-53: apt-get install is skipped if podman present). +func provisionNativeIngressLXC(ctx context.Context, runRemote func(string) ([]byte, error), template string, log *slog.Logger) error { + // Deterministic VMID for the native ingress LXC. + // Use a fixed VMID in the 200-299 range (Proxmox convention for CTs). + const vmid = "200" + const lxcName = "orca-traefik" + + // Check if the LXC already exists. + existOut, _ := runRemote(fmt.Sprintf("pct status %s 2>/dev/null || echo absent", vmid)) + existStr := strings.TrimSpace(string(existOut)) + if existStr == "absent" { + // Create the LXC (research Topic 3: nesting=1,keyctl=1,fuse=1). + log.Info("proxmox.creating_native_ingress_lxc", "vmid", vmid, "name", lxcName) + createCmd := fmt.Sprintf( + "pct create %s local:vztmpl/%s --hostname %s --unprivileged 1 --features nesting=1,keyctl=1,fuse=1 --onboot 1 --memory 2048 --swap 0 --rootfs local:8 2>&1", + vmid, shellQuote(template), lxcName, + ) + if out, err := runRemote(createCmd); err != nil { + return fmt.Errorf("pct create native ingress LXC: %w (output: %s)", err, string(out)) + } + if out, err := runRemote(fmt.Sprintf("pct start %s", vmid)); err != nil { + return fmt.Errorf("pct start native ingress LXC: %w (output: %s)", err, string(out)) + } + } + + // Wait for LXC network (retry for up to 60s). + lxcIP := "" + for i := 0; i < 12; i++ { + ipOut, _ := runRemote(fmt.Sprintf("pct exec %s -- hostname -I 2>/dev/null", vmid)) + ipStr := strings.TrimSpace(string(ipOut)) + if ipStr != "" { + fields := strings.Fields(ipStr) + if len(fields) > 0 { + lxcIP = fields[0] + break + } + } + time.Sleep(5 * time.Second) + } + if lxcIP == "" { + return fmt.Errorf("native ingress LXC: could not discover IP after 60s") + } + log.Info("proxmox.native_ingress_lxc_ip", "vmid", vmid, "ip", lxcIP) + + // Install podman inside the LXC (C-53: idempotent — check first). + _, _ = runRemote(fmt.Sprintf( + "pct exec %s -- bash -c 'command -v podman >/dev/null 2>&1 || (apt-get update -qq && apt-get install -y -qq podman conmon crun fuse-overlayfs nftables 2>&1)' 2>&1", + vmid, + )) + + // Enable podman-restart.service inside the LXC (research Topic 6). + _, _ = runRemote(fmt.Sprintf("pct exec %s -- systemctl enable --now podman-restart.service 2>/dev/null", vmid)) + + // Push step-ca root CA into the LXC (placeholder if absent locally). + caPath := certpaths.CACertPath() + caData, caErr := os.ReadFile(caPath) + if caErr != nil { + caData = []byte{} + } + // Write CA via pct exec heredoc. + caDelim := "EOF_CA" + _, _ = runRemote(fmt.Sprintf( + "pct exec %s -- bash -c 'mkdir -p /etc/orca && cat > /etc/orca/step-ca-root.crt <<%s\\n%s\\n%s'", + vmid, caDelim, string(caData), caDelim, + )) + + // Render + write traefik static config inside the LXC. + staticFiles, err := emitter.TraefikEmitter{}.RenderTraefikStaticConfig(emitter.TraefikStaticOpts{}) + if err == nil { + for _, f := range staticFiles { + delim := "EOF_TF" + _, _ = runRemote(fmt.Sprintf( + "pct exec %s -- bash -c 'mkdir -p /etc/traefik/dynamic && cat > %s <<%s\\n%s\\n%s'", + vmid, f.Path, delim, f.Content, delim, + )) + } + } + + // Ensure podman orca-traefik container inside the LXC. + traefikExecFn := func(cmd string) ([]byte, error) { + return runRemote(fmt.Sprintf("pct exec %s -- bash -c %s 2>&1", vmid, shellQuote(cmd))) + } + if err := traefik.EnsureTraefikContainerRemote(ctx, "", traefikExecFn); err != nil { + log.Warn("proxmox.native_ingress_lxc_traefik_failed", "err", err) + } + + // Render + apply nft on the PVE host with DNATTarget = LXC IP. + nftFiles, err := emitter.NftEmitter{}.RenderNftConfig(emitter.NftClusterConfig{ + DNATTarget: lxcIP, + }) + if err == nil { + for _, f := range nftFiles { + nftDelim := "EOF_NF" + _, _ = runRemote(fmt.Sprintf("mkdir -p /etc/nftables.d && cat > %s <<%s\\n%s\\n%s", + f.Path, nftDelim, f.Content, nftDelim)) + } + _, _ = runRemote("nft add table inet orca-ingress 2>/dev/null || true") + if out, err := runRemote("nft -f /etc/nftables.d/orca.nft 2>&1"); err != nil { + log.Warn("proxmox.native_ingress_nft_apply_failed", "err", err, "output", string(out)) + } + } + + log.Info("proxmox.native_ingress_lxc_ok", "vmid", vmid, "ip", lxcIP) + return nil +} + // sshDialer is the dialer used by BootstrapProxmox. It's a package-level // variable so tests can override it with a fake SSH server. var sshDialer sshDialerType = defaultSSHDialer{} diff --git a/internal/store/migrations/0009_ingress_mode.sql b/internal/store/migrations/0009_ingress_mode.sql new file mode 100644 index 0000000..f92ca47 --- /dev/null +++ b/internal/store/migrations/0009_ingress_mode.sql @@ -0,0 +1,6 @@ +-- REQ-175 / R-024: add ingress_mode column to nodes. +-- Values: '' (legacy/default), 'native' (proxmox native — traefik +-- in LXC), 'floating-ip' (proxmox floating-IP — ingress LXC owns +-- the floating IP). Defaults to empty string for backward +-- compatibility with pre-v0.14 nodes. +ALTER TABLE nodes ADD COLUMN ingress_mode TEXT NOT NULL DEFAULT ''; \ No newline at end of file