Jon Chery
ea42a17474
Add ingress.BootstrapRemoteIngress: renders+writes traefik static config, renders+writes+applies nft DNAT/SNAT, pushes step-ca root CA, ensures podman traefik container — all over SSH exec. Uses a heredoc- based remoteWriteFile with a random delimiter (F9 injection guard). Wired into linux/bootstrap.go Step 4d, replacing the standalone EnsureTraefikContainerRemote call with the full ingress stack. C-60: uses certpaths.CACertPath() (not CAPath). C-58: mounts host-side traefik.yml (preserves REQ-100 opt-out). C-55: pre-creates nft table before nft -f. ---ci--- project: orca phase: 4 milestone: v0.14 status: execute ---/ci---
241 lines
9.6 KiB
Go
241 lines
9.6 KiB
Go
// Package ingress implements the R-024 ingress bootstrap: nft DNAT
|
|
// + SNAT/MASQUERADE + traefik podman container + step-ca root CA on
|
|
// every orca-managed node. The bootstrap is idempotent and non-fatal
|
|
// on each step (offline host tolerance — same as v0.13 traefik install).
|
|
//
|
|
// BootstrapLocalIngress runs on the lead (during `orca init`).
|
|
// BootstrapRemoteIngress runs on workers (during `orca node join`).
|
|
package ingress
|
|
|
|
import (
|
|
"context"
|
|
"crypto/rand"
|
|
"encoding/hex"
|
|
"fmt"
|
|
"log/slog"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strings"
|
|
|
|
"git.cloudinit.dev/coreci/orca/internal/certpaths"
|
|
"git.cloudinit.dev/coreci/orca/internal/emitter"
|
|
"git.cloudinit.dev/coreci/orca/internal/traefik"
|
|
)
|
|
|
|
// BootstrapLocalIngress ensures the complete ingress stack is running
|
|
// on the local host (lead node). It is called from `orca init` after
|
|
// EnsureTraefikContainerLocal.
|
|
//
|
|
// Steps (each non-fatal — logs a warning and continues):
|
|
// 1. mkdir -p /etc/traefik/dynamic /etc/orca
|
|
// 2. Push cluster root CA to /etc/orca/step-ca-root.crt (C-60:
|
|
// certpaths.CACertPath(), not CAPath)
|
|
// 3. Render static config via emitter.RenderTraefikStaticConfig to
|
|
// /etc/traefik/traefik.yml (preserves traefik-on-public-ip opt-out,
|
|
// C-58)
|
|
// 4. Render orca.nft via emitter.NftEmitter.RenderNftConfig + write
|
|
// to /etc/nftables.d/orca.nft
|
|
// 5. Pre-create nft table (C-55: avoids flush-table error on first
|
|
// apply)
|
|
// 6. Apply: nft -f /etc/nftables.d/orca.nft
|
|
func BootstrapLocalIngress(ctx context.Context, version string) error {
|
|
var errs []error
|
|
log := slog.Default()
|
|
|
|
// Step 1: ensure directories.
|
|
for _, dir := range []string{"/etc/traefik/dynamic", "/etc/orca"} {
|
|
if err := os.MkdirAll(dir, 0o755); err != nil {
|
|
log.Warn("ingress: mkdir failed", "dir", dir, "err", err)
|
|
errs = append(errs, fmt.Errorf("mkdir %s: %w", dir, err))
|
|
}
|
|
}
|
|
|
|
// Step 2: push cluster root CA (C-60: CACertPath, not CAPath).
|
|
caPath := certpaths.CACertPath()
|
|
if caData, err := os.ReadFile(caPath); err == nil {
|
|
if err := os.WriteFile("/etc/orca/step-ca-root.crt", caData, 0o644); err != nil {
|
|
log.Warn("ingress: step-ca root CA write failed", "err", err)
|
|
errs = append(errs, fmt.Errorf("write step-ca-root.crt: %w", err))
|
|
}
|
|
} else {
|
|
// CA may not exist yet (fresh init before step-ca). Write a
|
|
// placeholder so the podman volume mount doesn't fail.
|
|
_ = os.WriteFile("/etc/orca/step-ca-root.crt", []byte{}, 0o644)
|
|
log.Warn("ingress: step-ca root CA not found, wrote placeholder", "path", caPath)
|
|
}
|
|
|
|
// Step 3: render + write static config (C-58).
|
|
staticFiles, err := emitter.TraefikEmitter{}.RenderTraefikStaticConfig(emitter.TraefikStaticOpts{})
|
|
if err != nil {
|
|
log.Warn("ingress: render traefik static config failed", "err", err)
|
|
errs = append(errs, fmt.Errorf("render traefik static: %w", err))
|
|
} else {
|
|
for _, f := range staticFiles {
|
|
if err := os.WriteFile(f.Path, []byte(f.Content), 0o644); err != nil {
|
|
log.Warn("ingress: write traefik static config failed", "path", f.Path, "err", err)
|
|
errs = append(errs, fmt.Errorf("write %s: %w", f.Path, err))
|
|
}
|
|
}
|
|
}
|
|
|
|
// Step 4: render + write nft ruleset.
|
|
nftFiles, err := emitter.NftEmitter{}.RenderNftConfig(emitter.NftClusterConfig{})
|
|
if err != nil {
|
|
log.Warn("ingress: render nft config failed", "err", err)
|
|
errs = append(errs, fmt.Errorf("render nft: %w", err))
|
|
} else {
|
|
for _, f := range nftFiles {
|
|
dir := filepath.Dir(f.Path)
|
|
_ = os.MkdirAll(dir, 0o755)
|
|
if err := os.WriteFile(f.Path, []byte(f.Content), 0o644); err != nil {
|
|
log.Warn("ingress: write nft config failed", "path", f.Path, "err", err)
|
|
errs = append(errs, fmt.Errorf("write %s: %w", f.Path, err))
|
|
}
|
|
}
|
|
|
|
// Step 5: pre-create nft table (C-55: flush table on non-existent
|
|
// table errors — pre-create avoids the first-apply failure).
|
|
_ = exec.CommandContext(ctx, "nft", "add", "table", "inet", "orca-ingress").Run()
|
|
|
|
// Step 6: apply nft ruleset.
|
|
if out, err := exec.CommandContext(ctx, "nft", "-f", nftConfigPath).CombinedOutput(); err != nil {
|
|
log.Warn("ingress: nft apply failed", "err", err, "output", string(out))
|
|
errs = append(errs, fmt.Errorf("nft -f: %w (output: %s)", err, string(out)))
|
|
}
|
|
}
|
|
|
|
// Ensure the podman traefik container is running (Step 7 — C-50:
|
|
// installs podman if absent).
|
|
if err := traefik.EnsureTraefikContainerLocal(ctx, version); err != nil {
|
|
log.Warn("ingress: ensure traefik container failed", "err", err)
|
|
errs = append(errs, fmt.Errorf("ensure traefik container: %w", err))
|
|
}
|
|
|
|
if len(errs) > 0 {
|
|
return fmt.Errorf("ingress bootstrap: %d errors (first: %w)", len(errs), errs[0])
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// nftConfigPath mirrors the emitter constant.
|
|
const nftConfigPath = "/etc/nftables.d/orca.nft"
|
|
|
|
// RemoteExecFunc runs a command on a remote host and returns combined
|
|
// output. Same signature as traefik.RemoteExecFunc.
|
|
type RemoteExecFunc func(cmd string) ([]byte, error)
|
|
|
|
// BootstrapRemoteIngress ensures the complete ingress stack is running
|
|
// on a remote host (linux worker). It is called from
|
|
// `orca node join --type linux` after the user setup.
|
|
//
|
|
// Steps (each non-fatal — logs a warning and continues):
|
|
// 1. mkdir -p /etc/traefik/dynamic /etc/orca (remote)
|
|
// 2. Push step-ca root CA to remote /etc/orca/step-ca-root.crt
|
|
// (C-60: certpaths.CACertPath)
|
|
// 3. Render + write static config to remote /etc/traefik/traefik.yml
|
|
// (C-58: preserves traefik-on-public-ip opt-out)
|
|
// 4. Render + write nft ruleset to remote /etc/nftables.d/orca.nft
|
|
// 5. Pre-create nft table (C-55: avoids first-apply flush-table error)
|
|
// 6. Apply: nft -f (remote)
|
|
// 7. Ensure podman traefik container running (remote)
|
|
func BootstrapRemoteIngress(ctx context.Context, version string, execFn RemoteExecFunc) error {
|
|
var errs []error
|
|
log := slog.Default()
|
|
|
|
// Step 1: ensure directories.
|
|
if _, err := execFn("mkdir -p /etc/traefik/dynamic /etc/orca"); err != nil {
|
|
log.Warn("ingress: remote mkdir failed", "err", err)
|
|
errs = append(errs, fmt.Errorf("remote mkdir: %w", err))
|
|
}
|
|
|
|
// Step 2: push step-ca root CA (C-60: CACertPath, not CAPath).
|
|
if caData, err := os.ReadFile(certpaths.CACertPath()); err == nil {
|
|
if err := remoteWriteFile(execFn, "/etc/orca/step-ca-root.crt", caData, "0644"); err != nil {
|
|
log.Warn("ingress: remote step-ca CA write failed", "err", err)
|
|
errs = append(errs, fmt.Errorf("remote write step-ca-root.crt: %w", err))
|
|
}
|
|
} else {
|
|
// Write a placeholder so the podman volume mount doesn't fail.
|
|
_ = remoteWriteFile(execFn, "/etc/orca/step-ca-root.crt", []byte{}, "0644")
|
|
log.Warn("ingress: step-ca root CA not found locally, wrote remote placeholder")
|
|
}
|
|
|
|
// Step 3: render + write static config (C-58).
|
|
staticFiles, err := emitter.TraefikEmitter{}.RenderTraefikStaticConfig(emitter.TraefikStaticOpts{})
|
|
if err != nil {
|
|
log.Warn("ingress: render traefik static config failed", "err", err)
|
|
errs = append(errs, fmt.Errorf("render traefik static: %w", err))
|
|
} else {
|
|
for _, f := range staticFiles {
|
|
_, _ = execFn(fmt.Sprintf("mkdir -p %s", filepath.Dir(f.Path)))
|
|
if err := remoteWriteFile(execFn, f.Path, []byte(f.Content), f.Mode); err != nil {
|
|
log.Warn("ingress: remote write traefik static config failed", "path", f.Path, "err", err)
|
|
errs = append(errs, fmt.Errorf("remote write %s: %w", f.Path, err))
|
|
}
|
|
}
|
|
}
|
|
|
|
// Step 4: render + write nft ruleset.
|
|
nftFiles, err := emitter.NftEmitter{}.RenderNftConfig(emitter.NftClusterConfig{})
|
|
if err != nil {
|
|
log.Warn("ingress: render nft config failed", "err", err)
|
|
errs = append(errs, fmt.Errorf("render nft: %w", err))
|
|
} else {
|
|
for _, f := range nftFiles {
|
|
_, _ = execFn(fmt.Sprintf("mkdir -p %s", filepath.Dir(f.Path)))
|
|
if err := remoteWriteFile(execFn, f.Path, []byte(f.Content), f.Mode); err != nil {
|
|
log.Warn("ingress: remote write nft config failed", "path", f.Path, "err", err)
|
|
errs = append(errs, fmt.Errorf("remote write %s: %w", f.Path, err))
|
|
}
|
|
}
|
|
|
|
// Step 5: pre-create nft table (C-55).
|
|
_, _ = execFn("nft add table inet orca-ingress 2>/dev/null || true")
|
|
|
|
// Step 6: apply nft ruleset.
|
|
if out, err := execFn("nft -f /etc/nftables.d/orca.nft 2>&1"); err != nil {
|
|
log.Warn("ingress: remote nft apply failed", "err", err, "output", string(out))
|
|
errs = append(errs, fmt.Errorf("remote nft -f: %w (output: %s)", err, string(out)))
|
|
}
|
|
}
|
|
|
|
// Step 7: ensure podman traefik container (C-50).
|
|
traefikExecFn := traefik.RemoteExecFunc(execFn)
|
|
if err := traefik.EnsureTraefikContainerRemote(ctx, version, traefikExecFn); err != nil {
|
|
log.Warn("ingress: remote ensure traefik container failed", "err", err)
|
|
errs = append(errs, fmt.Errorf("remote ensure traefik container: %w", err))
|
|
}
|
|
|
|
if len(errs) > 0 {
|
|
return fmt.Errorf("remote ingress bootstrap: %d errors (first: %w)", len(errs), errs[0])
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// remoteWriteFile writes content to a remote path via a heredoc
|
|
// (same pattern as sshpush.idempotency.writeFile). The heredoc
|
|
// delimiter is a random hex string verified absent from the content
|
|
// (F9 injection guard).
|
|
func remoteWriteFile(execFn RemoteExecFunc, path string, content []byte, mode string) error {
|
|
// Generate a random delimiter unlikely to be in the content.
|
|
delim := "EOF_"
|
|
for {
|
|
b := make([]byte, 8)
|
|
if _, err := rand.Read(b); err != nil {
|
|
return fmt.Errorf("rand: %w", err)
|
|
}
|
|
delim = "EOF_" + hex.EncodeToString(b)
|
|
if !strings.Contains(string(content), delim) {
|
|
break
|
|
}
|
|
}
|
|
dir := filepath.Dir(path)
|
|
cmd := fmt.Sprintf("mkdir -p %s && cat > %s <<'%s'\n%s\n%s\nchmod %s %s",
|
|
dir, path, delim, string(content), delim, mode, path)
|
|
if out, err := execFn(cmd); err != nil {
|
|
return fmt.Errorf("remote write %s: %w (output: %s)", path, err, string(out))
|
|
}
|
|
return nil
|
|
}
|