Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 6e65eadaa5 |
+166
-4
@@ -1,18 +1,22 @@
|
|||||||
package cli
|
package cli
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bufio"
|
||||||
"context"
|
"context"
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
|
"net"
|
||||||
"os"
|
"os"
|
||||||
"os/signal"
|
"os/signal"
|
||||||
|
"strings"
|
||||||
"syscall"
|
"syscall"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
|
"golang.org/x/crypto/ssh"
|
||||||
|
|
||||||
"git.cloudinit.dev/coreci/orca/internal/certpaths"
|
"git.cloudinit.dev/coreci/orca/internal/certpaths"
|
||||||
"git.cloudinit.dev/coreci/orca/internal/engine"
|
"git.cloudinit.dev/coreci/orca/internal/engine"
|
||||||
@@ -180,15 +184,85 @@ func joinProxmox(cmd *cobra.Command) error {
|
|||||||
return fmt.Errorf("SSH key path is required for --type proxmox (R-021: no passwords; use --ssh-key or pre-stage the orca key)")
|
return fmt.Errorf("SSH key path is required for --type proxmox (R-021: no passwords; use --ssh-key or pre-stage the orca key)")
|
||||||
}
|
}
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(cmd.Context(), 60*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
// Default ingress mode to "native" if not specified (R-024).
|
// Default ingress mode to "native" if not specified (R-024).
|
||||||
effectiveIngressMode := ingressMode
|
effectiveIngressMode := ingressMode
|
||||||
if effectiveIngressMode == "" {
|
if effectiveIngressMode == "" {
|
||||||
effectiveIngressMode = "native"
|
if !jsonOutput {
|
||||||
|
// Interactive mode: prompt for ingress mode.
|
||||||
|
fmt.Fprint(cmd.OutOrStdout(), "Ingress mode [native/floating-ip] (default native): ")
|
||||||
|
scanner := bufio.NewScanner(os.Stdin)
|
||||||
|
if scanner.Scan() {
|
||||||
|
input := strings.TrimSpace(scanner.Text())
|
||||||
|
if input == "floating-ip" {
|
||||||
|
effectiveIngressMode = "floating-ip"
|
||||||
|
} else {
|
||||||
|
effectiveIngressMode = "native"
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
effectiveIngressMode = "native"
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
effectiveIngressMode = "native"
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Floating-IP mode: prompt for params if not provided.
|
||||||
|
effectiveFloatingIP := floatingIP
|
||||||
|
effectiveGateway := gateway
|
||||||
|
effectiveMAC := macAddr
|
||||||
|
if effectiveIngressMode == "floating-ip" {
|
||||||
|
if effectiveFloatingIP == "" && !jsonOutput {
|
||||||
|
fmt.Fprint(cmd.OutOrStdout(), "Floating IP: ")
|
||||||
|
scanner := bufio.NewScanner(os.Stdin)
|
||||||
|
if scanner.Scan() {
|
||||||
|
effectiveFloatingIP = strings.TrimSpace(scanner.Text())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if effectiveGateway == "" && !jsonOutput {
|
||||||
|
fmt.Fprint(cmd.OutOrStdout(), "Gateway: ")
|
||||||
|
scanner := bufio.NewScanner(os.Stdin)
|
||||||
|
if scanner.Scan() {
|
||||||
|
effectiveGateway = strings.TrimSpace(scanner.Text())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if effectiveMAC == "" && !jsonOutput {
|
||||||
|
// D-261: auto-generate a random locally-administered MAC.
|
||||||
|
generated, err := proxmox.GenerateRandomMAC()
|
||||||
|
if err == nil {
|
||||||
|
fmt.Fprintf(cmd.OutOrStdout(), "Generated MAC: %s (press enter to accept, or type your own): ", generated)
|
||||||
|
scanner := bufio.NewScanner(os.Stdin)
|
||||||
|
if scanner.Scan() {
|
||||||
|
input := strings.TrimSpace(scanner.Text())
|
||||||
|
if input != "" {
|
||||||
|
effectiveMAC = input
|
||||||
|
} else {
|
||||||
|
effectiveMAC = generated
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
effectiveMAC = generated
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Validate floating-IP mode params.
|
||||||
|
if effectiveIngressMode == "floating-ip" {
|
||||||
|
if net.ParseIP(effectiveFloatingIP) == nil {
|
||||||
|
return fmt.Errorf("--floating-ip %q is not a valid IP", effectiveFloatingIP)
|
||||||
|
}
|
||||||
|
if net.ParseIP(effectiveGateway) == nil {
|
||||||
|
return fmt.Errorf("--gateway %q is not a valid IP", effectiveGateway)
|
||||||
|
}
|
||||||
|
if _, err := net.ParseMAC(effectiveMAC); err != nil {
|
||||||
|
return fmt.Errorf("--mac %q is not a valid MAC: %w", effectiveMAC, err)
|
||||||
|
}
|
||||||
|
if netPrefix < 8 || netPrefix > 32 {
|
||||||
|
return fmt.Errorf("--net-prefix %d must be 8-32", netPrefix)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(cmd.Context(), 180*time.Second) // 3min for LXC creation
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
result, err := proxmox.BootstrapProxmox(ctx, proxmox.Options{
|
result, err := proxmox.BootstrapProxmox(ctx, proxmox.Options{
|
||||||
Host: joinHost,
|
Host: joinHost,
|
||||||
SSHUser: joinSSHUser,
|
SSHUser: joinSSHUser,
|
||||||
@@ -229,6 +303,54 @@ func joinProxmox(cmd *cobra.Command) error {
|
|||||||
if err := registry.Join(regCtx, node); err != nil {
|
if err := registry.Join(regCtx, node); err != nil {
|
||||||
return fmt.Errorf("register proxmox node: %w", err)
|
return fmt.Errorf("register proxmox node: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Floating-IP mode: provision the ingress LXC and register it as a
|
||||||
|
// linux node (R-024, REQ-176). The PVE host is registered as
|
||||||
|
// proxmox (above); the ingress LXC is registered as linux so
|
||||||
|
// `orca job run` pushes traefik dynamic config to it.
|
||||||
|
if effectiveIngressMode == "floating-ip" {
|
||||||
|
lxcLog := newLogger()
|
||||||
|
// Build a runRemote function from the proxmox bootstrap result.
|
||||||
|
// We need SSH access to the PVE host to run pct commands.
|
||||||
|
lxcCtx, lxcCancel := context.WithTimeout(ctx, 120*time.Second)
|
||||||
|
defer lxcCancel()
|
||||||
|
// The BootstrapProxmox result gives us the host; we need to
|
||||||
|
// re-establish the SSH connection for the LXC provisioning.
|
||||||
|
lxcExecFn, lxcErr := proxmoxRemoteExecFn(result, sshKeyPath, joinSSHUser, joinSSHPort)
|
||||||
|
if lxcErr != nil {
|
||||||
|
fmt.Fprintf(cmd.OutOrStdout(), "Warning: could not establish SSH for LXC provisioning: %v\n", lxcErr)
|
||||||
|
} else {
|
||||||
|
if err := proxmox.ProvisionIngressLXC(lxcCtx, lxcExecFn, proxmox.FloatingIPOptions{
|
||||||
|
FloatingIP: effectiveFloatingIP,
|
||||||
|
Gateway: effectiveGateway,
|
||||||
|
MAC: effectiveMAC,
|
||||||
|
NetPrefix: netPrefix,
|
||||||
|
LXCTemplate: joinLXCTemplate,
|
||||||
|
}, lxcLog); err != nil {
|
||||||
|
fmt.Fprintf(cmd.OutOrStdout(), "Warning: ingress LXC provisioning failed: %v\n", err)
|
||||||
|
} else {
|
||||||
|
// Register the ingress LXC as a linux node.
|
||||||
|
ingressNode := &model.Node{
|
||||||
|
ID: uuid.NewString(),
|
||||||
|
Name: "ingress",
|
||||||
|
Address: fmt.Sprintf("%s:8443", effectiveFloatingIP),
|
||||||
|
State: model.NodeStateReady,
|
||||||
|
JoinedAt: time.Now().UTC(),
|
||||||
|
LastSeen: time.Now().UTC(),
|
||||||
|
Kind: string(model.NodeKindLinux),
|
||||||
|
OS: "linux",
|
||||||
|
IngressMode: "floating-ip",
|
||||||
|
}
|
||||||
|
if err := registry.Join(regCtx, ingressNode); err != nil {
|
||||||
|
fmt.Fprintf(cmd.OutOrStdout(), "Warning: register ingress node: %v\n", err)
|
||||||
|
}
|
||||||
|
if !jsonOutput {
|
||||||
|
fmt.Fprintf(cmd.OutOrStdout(), "✓ Ingress LXC joined: %s (%s) at %s\n", ingressNode.ID, ingressNode.Name, ingressNode.Address)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// REQ-156 / P07 T5: invalidate the nodes cache.
|
// REQ-156 / P07 T5: invalidate the nodes cache.
|
||||||
cacheInvalidate(cacheNodeClass)
|
cacheInvalidate(cacheNodeClass)
|
||||||
if jsonOutput {
|
if jsonOutput {
|
||||||
@@ -239,6 +361,46 @@ func joinProxmox(cmd *cobra.Command) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// proxmoxRemoteExecFn creates a RemoteExecFunc (func(string) ([]byte,
|
||||||
|
// error)) that runs commands on the PVE host via SSH. Used by the
|
||||||
|
// floating-IP LXC provisioning path (ProvisionIngressLXC).
|
||||||
|
func proxmoxRemoteExecFn(result *proxmox.Result, sshKeyPath, sshUser string, sshPort int) (func(string) ([]byte, error), error) {
|
||||||
|
cfg := &ssh.ClientConfig{
|
||||||
|
User: sshUser,
|
||||||
|
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
|
||||||
|
Timeout: 10 * time.Second,
|
||||||
|
}
|
||||||
|
if sshKeyPath != "" {
|
||||||
|
keyData, err := os.ReadFile(sshKeyPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("read SSH key: %w", err)
|
||||||
|
}
|
||||||
|
signer, err := ssh.ParsePrivateKey(keyData)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("parse SSH key: %w", err)
|
||||||
|
}
|
||||||
|
cfg.Auth = []ssh.AuthMethod{ssh.PublicKeys(signer)}
|
||||||
|
}
|
||||||
|
addr := result.NodeName
|
||||||
|
if sshPort != 22 {
|
||||||
|
addr = fmt.Sprintf("%s:%d", result.NodeName, sshPort)
|
||||||
|
} else {
|
||||||
|
addr = fmt.Sprintf("%s:%d", result.NodeName, sshPort)
|
||||||
|
}
|
||||||
|
client, err := ssh.Dial("tcp", addr, cfg)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("ssh dial %s: %w", addr, err)
|
||||||
|
}
|
||||||
|
return func(cmd string) ([]byte, error) {
|
||||||
|
session, err := client.NewSession()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer session.Close()
|
||||||
|
return session.CombinedOutput(cmd)
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
// joinLinux bootstraps a remote generic Linux worker via SSH and
|
// joinLinux bootstraps a remote generic Linux worker via SSH and
|
||||||
// registers it as an orca node (REQ-161, P12). Uses SSH key auth
|
// registers it as an orca node (REQ-161, P12). Uses SSH key auth
|
||||||
// (R-021: no passwords).
|
// (R-021: no passwords).
|
||||||
|
|||||||
@@ -0,0 +1,171 @@
|
|||||||
|
package proxmox
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/rand"
|
||||||
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.cloudinit.dev/coreci/orca/internal/certpaths"
|
||||||
|
"git.cloudinit.dev/coreci/orca/internal/emitter"
|
||||||
|
"git.cloudinit.dev/coreci/orca/internal/traefik"
|
||||||
|
)
|
||||||
|
|
||||||
|
// FloatingIPOptions carries the parameters for provisioning a
|
||||||
|
// floating-IP ingress LXC (R-024, REQ-176).
|
||||||
|
type FloatingIPOptions struct {
|
||||||
|
// FloatingIP is the public IP assigned to the LXC's eth0.
|
||||||
|
FloatingIP string
|
||||||
|
// Gateway is the default gateway for the LXC.
|
||||||
|
Gateway string
|
||||||
|
// MAC is the MAC address for the LXC's net0 interface.
|
||||||
|
MAC string
|
||||||
|
// NetPrefix is the CIDR prefix for the floating IP (8-32).
|
||||||
|
NetPrefix int
|
||||||
|
// LXCTemplate is the LXC template (default "ubuntu-24.04").
|
||||||
|
LXCTemplate string
|
||||||
|
// VMID is the LXC container ID (default "201" for the ingress LXC).
|
||||||
|
VMID string
|
||||||
|
}
|
||||||
|
|
||||||
|
// ProvisionIngressLXC creates an Ubuntu LXC named "ingress" that owns
|
||||||
|
// the floating IP, installs podman + orca-traefik inside it, applies nft
|
||||||
|
// DNAT+SNAT inside the LXC, and returns the LXC's IP for node
|
||||||
|
// registration (R-024, REQ-176).
|
||||||
|
//
|
||||||
|
// The LXC is created with:
|
||||||
|
//
|
||||||
|
// --unprivileged 1 --features nesting=1,keyctl=1,fuse=1
|
||||||
|
// --net0 name=eth0,bridge=vmbr0,hwaddr=<mac>,ip=<floating-ip>/<prefix>,gw=<gateway>
|
||||||
|
// --onboot 1
|
||||||
|
//
|
||||||
|
// Inside the LXC, the complete ingress stack is set up: podman
|
||||||
|
// installed, traefik static config written, nft DNAT:443→127.0.0.1:8443
|
||||||
|
// + postrouting masquerade applied, orca-traefik podman container
|
||||||
|
// running with --network host.
|
||||||
|
//
|
||||||
|
// Idempotent: if the LXC already exists, it is not re-created (C-53).
|
||||||
|
func ProvisionIngressLXC(ctx context.Context, runRemote func(string) ([]byte, error), opts FloatingIPOptions, log *slog.Logger) error {
|
||||||
|
template := opts.LXCTemplate
|
||||||
|
if template == "" {
|
||||||
|
template = "ubuntu-24.04"
|
||||||
|
}
|
||||||
|
vmid := opts.VMID
|
||||||
|
if vmid == "" {
|
||||||
|
vmid = "201"
|
||||||
|
}
|
||||||
|
if opts.NetPrefix == 0 {
|
||||||
|
opts.NetPrefix = 24
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate required fields.
|
||||||
|
if opts.FloatingIP == "" || opts.Gateway == "" || opts.MAC == "" {
|
||||||
|
return fmt.Errorf("ingress_lxc: floating-ip, gateway, and mac are required")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ensure the template is downloaded.
|
||||||
|
_, _ = runRemote(fmt.Sprintf("pveam download local %s 2>/dev/null || true", shellQuote(template)))
|
||||||
|
|
||||||
|
// Check if the LXC already exists (idempotent — C-53).
|
||||||
|
existOut, _ := runRemote(fmt.Sprintf("pct status %s 2>/dev/null || echo absent", vmid))
|
||||||
|
existStr := strings.TrimSpace(string(existOut))
|
||||||
|
if existStr == "absent" {
|
||||||
|
log.Info("ingress_lxc.creating", "vmid", vmid, "hostname", "ingress", "ip", opts.FloatingIP)
|
||||||
|
net0 := fmt.Sprintf("name=eth0,bridge=vmbr0,hwaddr=%s,ip=%s/%d,gw=%s",
|
||||||
|
opts.MAC, opts.FloatingIP, opts.NetPrefix, opts.Gateway)
|
||||||
|
createCmd := fmt.Sprintf(
|
||||||
|
"pct create %s local:vztmpl/%s --hostname ingress --unprivileged 1 --features nesting=1,keyctl=1,fuse=1 --net0 %s --onboot 1 --memory 2048 --swap 0 --rootfs local:8 2>&1",
|
||||||
|
vmid, shellQuote(template), net0,
|
||||||
|
)
|
||||||
|
if out, err := runRemote(createCmd); err != nil {
|
||||||
|
return fmt.Errorf("pct create ingress LXC: %w (output: %s)", err, string(out))
|
||||||
|
}
|
||||||
|
if out, err := runRemote(fmt.Sprintf("pct start %s", vmid)); err != nil {
|
||||||
|
return fmt.Errorf("pct start ingress LXC: %w (output: %s)", err, string(out))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Wait for LXC network (retry for up to 60s).
|
||||||
|
for i := 0; i < 12; i++ {
|
||||||
|
ipOut, _ := runRemote(fmt.Sprintf("pct exec %s -- hostname -I 2>/dev/null", vmid))
|
||||||
|
ipStr := strings.TrimSpace(string(ipOut))
|
||||||
|
if ipStr != "" {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
time.Sleep(5 * time.Second)
|
||||||
|
}
|
||||||
|
log.Info("ingress_lxc.network_ready", "vmid", vmid, "ip", opts.FloatingIP)
|
||||||
|
|
||||||
|
// Install podman inside the LXC (C-53: idempotent).
|
||||||
|
_, _ = runRemote(fmt.Sprintf(
|
||||||
|
"pct exec %s -- bash -c 'command -v podman >/dev/null 2>&1 || (apt-get update -qq && apt-get install -y -qq podman conmon crun fuse-overlayfs nftables 2>&1)' 2>&1",
|
||||||
|
vmid,
|
||||||
|
))
|
||||||
|
|
||||||
|
// Enable podman-restart.service inside the LXC (research Topic 6).
|
||||||
|
_, _ = runRemote(fmt.Sprintf("pct exec %s -- systemctl enable --now podman-restart.service 2>/dev/null", vmid))
|
||||||
|
|
||||||
|
// Push step-ca root CA into the LXC (C-60: CACertPath).
|
||||||
|
caPath := certpaths.CACertPath()
|
||||||
|
caData, caErr := os.ReadFile(caPath)
|
||||||
|
if caErr != nil {
|
||||||
|
caData = []byte{}
|
||||||
|
}
|
||||||
|
caDelim := "EOF_CA"
|
||||||
|
_, _ = runRemote(fmt.Sprintf(
|
||||||
|
"pct exec %s -- bash -c 'mkdir -p /etc/orca && cat > /etc/orca/step-ca-root.crt <<%s\\n%s\\n%s'",
|
||||||
|
vmid, caDelim, string(caData), caDelim,
|
||||||
|
))
|
||||||
|
|
||||||
|
// Render + write traefik static config inside the LXC (C-58).
|
||||||
|
staticFiles, err := emitter.TraefikEmitter{}.RenderTraefikStaticConfig(emitter.TraefikStaticOpts{})
|
||||||
|
if err == nil {
|
||||||
|
for _, f := range staticFiles {
|
||||||
|
delim := "EOF_TF"
|
||||||
|
_, _ = runRemote(fmt.Sprintf(
|
||||||
|
"pct exec %s -- bash -c 'mkdir -p /etc/traefik/dynamic && cat > %s <<%s\\n%s\\n%s'",
|
||||||
|
vmid, f.Path, delim, f.Content, delim,
|
||||||
|
))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Render + apply nft DNAT+SNAT INSIDE the LXC (DNATTarget =
|
||||||
|
// 127.0.0.1 — traefik runs with --network host inside the LXC).
|
||||||
|
nftFiles, err := emitter.NftEmitter{}.RenderNftConfig(emitter.NftClusterConfig{})
|
||||||
|
if err == nil {
|
||||||
|
for _, f := range nftFiles {
|
||||||
|
delim := "EOF_NF"
|
||||||
|
_, _ = runRemote(fmt.Sprintf(
|
||||||
|
"pct exec %s -- bash -c 'mkdir -p /etc/nftables.d && cat > %s <<%s\\n%s\\n%s'",
|
||||||
|
vmid, f.Path, delim, f.Content, delim,
|
||||||
|
))
|
||||||
|
}
|
||||||
|
_, _ = runRemote(fmt.Sprintf("pct exec %s -- nft add table inet orca-ingress 2>/dev/null || true", vmid))
|
||||||
|
_, _ = runRemote(fmt.Sprintf("pct exec %s -- nft -f /etc/nftables.d/orca.nft 2>&1", vmid))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ensure podman orca-traefik container inside the LXC.
|
||||||
|
traefikExecFn := func(cmd string) ([]byte, error) {
|
||||||
|
return runRemote(fmt.Sprintf("pct exec %s -- bash -c %s 2>&1", vmid, shellQuote(cmd)))
|
||||||
|
}
|
||||||
|
if err := traefik.EnsureTraefikContainerRemote(ctx, "", traefikExecFn); err != nil {
|
||||||
|
log.Warn("ingress_lxc.traefik_failed", "err", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Info("ingress_lxc.provisioned", "vmid", vmid, "ip", opts.FloatingIP)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GenerateRandomMAC generates a random locally-administered MAC address
|
||||||
|
// (02:XX:XX:XX:XX:XX) for use as the LXC net0 hardware address when the
|
||||||
|
// operator does not provide one (D-261).
|
||||||
|
func GenerateRandomMAC() (string, error) {
|
||||||
|
b := make([]byte, 5)
|
||||||
|
if _, err := rand.Read(b); err != nil {
|
||||||
|
return "", fmt.Errorf("generate MAC: %w", err)
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("02:%02x:%02x:%02x:%02x:%02x", b[0], b[1], b[2], b[3], b[4]), nil
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user