Compare commits

..

1 Commits

Author SHA1 Message Date
Jon Chery 6e65eadaa5 feat(P6): proxmox floating-IP LXC ingress + interactive prompt (REQ-176)
New internal/proxmox/ingress_lxc.go: ProvisionIngressLXC creates an
Ubuntu LXC named 'ingress' that owns the floating IP (net0
bridge=vmbr0,hwaddr=<mac>,ip=<floating-ip>/<prefix>,gw=<gateway>).
Unprivileged with --features nesting=1,keyctl=1,fuse=1 (research
Topic 3). Installs podman inside, runs orca-traefik container,
applies nft DNAT+SNAT INSIDE the LXC, pushes step-ca root CA.

GenerateRandomMAC: 02:XX:XX:XX:XX:XX for interactive mode (D-261).

Interactive prompting in joinProxmox: when --ingress-mode empty +
!--json, prompt for mode + floating IP + gateway + MAC (auto-
generate + confirm). Validate IP/MAC/gateway/prefix.

Floating-IP routing: calls ProvisionIngressLXC + registers:
  1. PVE host as 'proxmox' node (IngressMode=floating-ip)
  2. Ingress LXC as 'linux' node (name=ingress, addr=<floating-ip>:8443)
     so orca job run pushes traefik dynamic config to it.

---ci---
project: orca
phase: 6
milestone: v0.14
status: execute
---/ci---
2026-08-10 20:19:47 +00:00
2 changed files with 337 additions and 4 deletions
+166 -4
View File
@@ -1,18 +1,22 @@
package cli
import (
"bufio"
"context"
"database/sql"
"encoding/json"
"fmt"
"log/slog"
"net"
"os"
"os/signal"
"strings"
"syscall"
"time"
"github.com/google/uuid"
"github.com/spf13/cobra"
"golang.org/x/crypto/ssh"
"git.cloudinit.dev/coreci/orca/internal/certpaths"
"git.cloudinit.dev/coreci/orca/internal/engine"
@@ -180,15 +184,85 @@ func joinProxmox(cmd *cobra.Command) error {
return fmt.Errorf("SSH key path is required for --type proxmox (R-021: no passwords; use --ssh-key or pre-stage the orca key)")
}
ctx, cancel := context.WithTimeout(cmd.Context(), 60*time.Second)
defer cancel()
// Default ingress mode to "native" if not specified (R-024).
effectiveIngressMode := ingressMode
if effectiveIngressMode == "" {
effectiveIngressMode = "native"
if !jsonOutput {
// Interactive mode: prompt for ingress mode.
fmt.Fprint(cmd.OutOrStdout(), "Ingress mode [native/floating-ip] (default native): ")
scanner := bufio.NewScanner(os.Stdin)
if scanner.Scan() {
input := strings.TrimSpace(scanner.Text())
if input == "floating-ip" {
effectiveIngressMode = "floating-ip"
} else {
effectiveIngressMode = "native"
}
} else {
effectiveIngressMode = "native"
}
} else {
effectiveIngressMode = "native"
}
}
// Floating-IP mode: prompt for params if not provided.
effectiveFloatingIP := floatingIP
effectiveGateway := gateway
effectiveMAC := macAddr
if effectiveIngressMode == "floating-ip" {
if effectiveFloatingIP == "" && !jsonOutput {
fmt.Fprint(cmd.OutOrStdout(), "Floating IP: ")
scanner := bufio.NewScanner(os.Stdin)
if scanner.Scan() {
effectiveFloatingIP = strings.TrimSpace(scanner.Text())
}
}
if effectiveGateway == "" && !jsonOutput {
fmt.Fprint(cmd.OutOrStdout(), "Gateway: ")
scanner := bufio.NewScanner(os.Stdin)
if scanner.Scan() {
effectiveGateway = strings.TrimSpace(scanner.Text())
}
}
if effectiveMAC == "" && !jsonOutput {
// D-261: auto-generate a random locally-administered MAC.
generated, err := proxmox.GenerateRandomMAC()
if err == nil {
fmt.Fprintf(cmd.OutOrStdout(), "Generated MAC: %s (press enter to accept, or type your own): ", generated)
scanner := bufio.NewScanner(os.Stdin)
if scanner.Scan() {
input := strings.TrimSpace(scanner.Text())
if input != "" {
effectiveMAC = input
} else {
effectiveMAC = generated
}
} else {
effectiveMAC = generated
}
}
}
// Validate floating-IP mode params.
if effectiveIngressMode == "floating-ip" {
if net.ParseIP(effectiveFloatingIP) == nil {
return fmt.Errorf("--floating-ip %q is not a valid IP", effectiveFloatingIP)
}
if net.ParseIP(effectiveGateway) == nil {
return fmt.Errorf("--gateway %q is not a valid IP", effectiveGateway)
}
if _, err := net.ParseMAC(effectiveMAC); err != nil {
return fmt.Errorf("--mac %q is not a valid MAC: %w", effectiveMAC, err)
}
if netPrefix < 8 || netPrefix > 32 {
return fmt.Errorf("--net-prefix %d must be 8-32", netPrefix)
}
}
}
ctx, cancel := context.WithTimeout(cmd.Context(), 180*time.Second) // 3min for LXC creation
defer cancel()
result, err := proxmox.BootstrapProxmox(ctx, proxmox.Options{
Host: joinHost,
SSHUser: joinSSHUser,
@@ -229,6 +303,54 @@ func joinProxmox(cmd *cobra.Command) error {
if err := registry.Join(regCtx, node); err != nil {
return fmt.Errorf("register proxmox node: %w", err)
}
// Floating-IP mode: provision the ingress LXC and register it as a
// linux node (R-024, REQ-176). The PVE host is registered as
// proxmox (above); the ingress LXC is registered as linux so
// `orca job run` pushes traefik dynamic config to it.
if effectiveIngressMode == "floating-ip" {
lxcLog := newLogger()
// Build a runRemote function from the proxmox bootstrap result.
// We need SSH access to the PVE host to run pct commands.
lxcCtx, lxcCancel := context.WithTimeout(ctx, 120*time.Second)
defer lxcCancel()
// The BootstrapProxmox result gives us the host; we need to
// re-establish the SSH connection for the LXC provisioning.
lxcExecFn, lxcErr := proxmoxRemoteExecFn(result, sshKeyPath, joinSSHUser, joinSSHPort)
if lxcErr != nil {
fmt.Fprintf(cmd.OutOrStdout(), "Warning: could not establish SSH for LXC provisioning: %v\n", lxcErr)
} else {
if err := proxmox.ProvisionIngressLXC(lxcCtx, lxcExecFn, proxmox.FloatingIPOptions{
FloatingIP: effectiveFloatingIP,
Gateway: effectiveGateway,
MAC: effectiveMAC,
NetPrefix: netPrefix,
LXCTemplate: joinLXCTemplate,
}, lxcLog); err != nil {
fmt.Fprintf(cmd.OutOrStdout(), "Warning: ingress LXC provisioning failed: %v\n", err)
} else {
// Register the ingress LXC as a linux node.
ingressNode := &model.Node{
ID: uuid.NewString(),
Name: "ingress",
Address: fmt.Sprintf("%s:8443", effectiveFloatingIP),
State: model.NodeStateReady,
JoinedAt: time.Now().UTC(),
LastSeen: time.Now().UTC(),
Kind: string(model.NodeKindLinux),
OS: "linux",
IngressMode: "floating-ip",
}
if err := registry.Join(regCtx, ingressNode); err != nil {
fmt.Fprintf(cmd.OutOrStdout(), "Warning: register ingress node: %v\n", err)
}
if !jsonOutput {
fmt.Fprintf(cmd.OutOrStdout(), "✓ Ingress LXC joined: %s (%s) at %s\n", ingressNode.ID, ingressNode.Name, ingressNode.Address)
}
}
}
}
// REQ-156 / P07 T5: invalidate the nodes cache.
cacheInvalidate(cacheNodeClass)
if jsonOutput {
@@ -239,6 +361,46 @@ func joinProxmox(cmd *cobra.Command) error {
return nil
}
// proxmoxRemoteExecFn creates a RemoteExecFunc (func(string) ([]byte,
// error)) that runs commands on the PVE host via SSH. Used by the
// floating-IP LXC provisioning path (ProvisionIngressLXC).
func proxmoxRemoteExecFn(result *proxmox.Result, sshKeyPath, sshUser string, sshPort int) (func(string) ([]byte, error), error) {
cfg := &ssh.ClientConfig{
User: sshUser,
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
Timeout: 10 * time.Second,
}
if sshKeyPath != "" {
keyData, err := os.ReadFile(sshKeyPath)
if err != nil {
return nil, fmt.Errorf("read SSH key: %w", err)
}
signer, err := ssh.ParsePrivateKey(keyData)
if err != nil {
return nil, fmt.Errorf("parse SSH key: %w", err)
}
cfg.Auth = []ssh.AuthMethod{ssh.PublicKeys(signer)}
}
addr := result.NodeName
if sshPort != 22 {
addr = fmt.Sprintf("%s:%d", result.NodeName, sshPort)
} else {
addr = fmt.Sprintf("%s:%d", result.NodeName, sshPort)
}
client, err := ssh.Dial("tcp", addr, cfg)
if err != nil {
return nil, fmt.Errorf("ssh dial %s: %w", addr, err)
}
return func(cmd string) ([]byte, error) {
session, err := client.NewSession()
if err != nil {
return nil, err
}
defer session.Close()
return session.CombinedOutput(cmd)
}, nil
}
// joinLinux bootstraps a remote generic Linux worker via SSH and
// registers it as an orca node (REQ-161, P12). Uses SSH key auth
// (R-021: no passwords).
+171
View File
@@ -0,0 +1,171 @@
package proxmox
import (
"context"
"crypto/rand"
"fmt"
"log/slog"
"os"
"strings"
"time"
"git.cloudinit.dev/coreci/orca/internal/certpaths"
"git.cloudinit.dev/coreci/orca/internal/emitter"
"git.cloudinit.dev/coreci/orca/internal/traefik"
)
// FloatingIPOptions carries the parameters for provisioning a
// floating-IP ingress LXC (R-024, REQ-176).
type FloatingIPOptions struct {
// FloatingIP is the public IP assigned to the LXC's eth0.
FloatingIP string
// Gateway is the default gateway for the LXC.
Gateway string
// MAC is the MAC address for the LXC's net0 interface.
MAC string
// NetPrefix is the CIDR prefix for the floating IP (8-32).
NetPrefix int
// LXCTemplate is the LXC template (default "ubuntu-24.04").
LXCTemplate string
// VMID is the LXC container ID (default "201" for the ingress LXC).
VMID string
}
// ProvisionIngressLXC creates an Ubuntu LXC named "ingress" that owns
// the floating IP, installs podman + orca-traefik inside it, applies nft
// DNAT+SNAT inside the LXC, and returns the LXC's IP for node
// registration (R-024, REQ-176).
//
// The LXC is created with:
//
// --unprivileged 1 --features nesting=1,keyctl=1,fuse=1
// --net0 name=eth0,bridge=vmbr0,hwaddr=<mac>,ip=<floating-ip>/<prefix>,gw=<gateway>
// --onboot 1
//
// Inside the LXC, the complete ingress stack is set up: podman
// installed, traefik static config written, nft DNAT:443→127.0.0.1:8443
// + postrouting masquerade applied, orca-traefik podman container
// running with --network host.
//
// Idempotent: if the LXC already exists, it is not re-created (C-53).
func ProvisionIngressLXC(ctx context.Context, runRemote func(string) ([]byte, error), opts FloatingIPOptions, log *slog.Logger) error {
template := opts.LXCTemplate
if template == "" {
template = "ubuntu-24.04"
}
vmid := opts.VMID
if vmid == "" {
vmid = "201"
}
if opts.NetPrefix == 0 {
opts.NetPrefix = 24
}
// Validate required fields.
if opts.FloatingIP == "" || opts.Gateway == "" || opts.MAC == "" {
return fmt.Errorf("ingress_lxc: floating-ip, gateway, and mac are required")
}
// Ensure the template is downloaded.
_, _ = runRemote(fmt.Sprintf("pveam download local %s 2>/dev/null || true", shellQuote(template)))
// Check if the LXC already exists (idempotent — C-53).
existOut, _ := runRemote(fmt.Sprintf("pct status %s 2>/dev/null || echo absent", vmid))
existStr := strings.TrimSpace(string(existOut))
if existStr == "absent" {
log.Info("ingress_lxc.creating", "vmid", vmid, "hostname", "ingress", "ip", opts.FloatingIP)
net0 := fmt.Sprintf("name=eth0,bridge=vmbr0,hwaddr=%s,ip=%s/%d,gw=%s",
opts.MAC, opts.FloatingIP, opts.NetPrefix, opts.Gateway)
createCmd := fmt.Sprintf(
"pct create %s local:vztmpl/%s --hostname ingress --unprivileged 1 --features nesting=1,keyctl=1,fuse=1 --net0 %s --onboot 1 --memory 2048 --swap 0 --rootfs local:8 2>&1",
vmid, shellQuote(template), net0,
)
if out, err := runRemote(createCmd); err != nil {
return fmt.Errorf("pct create ingress LXC: %w (output: %s)", err, string(out))
}
if out, err := runRemote(fmt.Sprintf("pct start %s", vmid)); err != nil {
return fmt.Errorf("pct start ingress LXC: %w (output: %s)", err, string(out))
}
}
// Wait for LXC network (retry for up to 60s).
for i := 0; i < 12; i++ {
ipOut, _ := runRemote(fmt.Sprintf("pct exec %s -- hostname -I 2>/dev/null", vmid))
ipStr := strings.TrimSpace(string(ipOut))
if ipStr != "" {
break
}
time.Sleep(5 * time.Second)
}
log.Info("ingress_lxc.network_ready", "vmid", vmid, "ip", opts.FloatingIP)
// Install podman inside the LXC (C-53: idempotent).
_, _ = runRemote(fmt.Sprintf(
"pct exec %s -- bash -c 'command -v podman >/dev/null 2>&1 || (apt-get update -qq && apt-get install -y -qq podman conmon crun fuse-overlayfs nftables 2>&1)' 2>&1",
vmid,
))
// Enable podman-restart.service inside the LXC (research Topic 6).
_, _ = runRemote(fmt.Sprintf("pct exec %s -- systemctl enable --now podman-restart.service 2>/dev/null", vmid))
// Push step-ca root CA into the LXC (C-60: CACertPath).
caPath := certpaths.CACertPath()
caData, caErr := os.ReadFile(caPath)
if caErr != nil {
caData = []byte{}
}
caDelim := "EOF_CA"
_, _ = runRemote(fmt.Sprintf(
"pct exec %s -- bash -c 'mkdir -p /etc/orca && cat > /etc/orca/step-ca-root.crt <<%s\\n%s\\n%s'",
vmid, caDelim, string(caData), caDelim,
))
// Render + write traefik static config inside the LXC (C-58).
staticFiles, err := emitter.TraefikEmitter{}.RenderTraefikStaticConfig(emitter.TraefikStaticOpts{})
if err == nil {
for _, f := range staticFiles {
delim := "EOF_TF"
_, _ = runRemote(fmt.Sprintf(
"pct exec %s -- bash -c 'mkdir -p /etc/traefik/dynamic && cat > %s <<%s\\n%s\\n%s'",
vmid, f.Path, delim, f.Content, delim,
))
}
}
// Render + apply nft DNAT+SNAT INSIDE the LXC (DNATTarget =
// 127.0.0.1 — traefik runs with --network host inside the LXC).
nftFiles, err := emitter.NftEmitter{}.RenderNftConfig(emitter.NftClusterConfig{})
if err == nil {
for _, f := range nftFiles {
delim := "EOF_NF"
_, _ = runRemote(fmt.Sprintf(
"pct exec %s -- bash -c 'mkdir -p /etc/nftables.d && cat > %s <<%s\\n%s\\n%s'",
vmid, f.Path, delim, f.Content, delim,
))
}
_, _ = runRemote(fmt.Sprintf("pct exec %s -- nft add table inet orca-ingress 2>/dev/null || true", vmid))
_, _ = runRemote(fmt.Sprintf("pct exec %s -- nft -f /etc/nftables.d/orca.nft 2>&1", vmid))
}
// Ensure podman orca-traefik container inside the LXC.
traefikExecFn := func(cmd string) ([]byte, error) {
return runRemote(fmt.Sprintf("pct exec %s -- bash -c %s 2>&1", vmid, shellQuote(cmd)))
}
if err := traefik.EnsureTraefikContainerRemote(ctx, "", traefikExecFn); err != nil {
log.Warn("ingress_lxc.traefik_failed", "err", err)
}
log.Info("ingress_lxc.provisioned", "vmid", vmid, "ip", opts.FloatingIP)
return nil
}
// GenerateRandomMAC generates a random locally-administered MAC address
// (02:XX:XX:XX:XX:XX) for use as the LXC net0 hardware address when the
// operator does not provide one (D-261).
func GenerateRandomMAC() (string, error) {
b := make([]byte, 5)
if _, err := rand.Read(b); err != nil {
return "", fmt.Errorf("generate MAC: %w", err)
}
return fmt.Sprintf("02:%02x:%02x:%02x:%02x:%02x", b[0], b[1], b[2], b[3], b[4]), nil
}