From ecdba833d9228489f9b727494a28f16037cb480f Mon Sep 17 00:00:00 2001 From: Jon Chery Date: Mon, 10 Aug 2026 20:24:13 +0000 Subject: [PATCH] feat(P7): doctor ingress + docs + integration tests (REQ-177,178,179) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New 'orca doctor ingress' command: verifies podman orca-traefik container running, nft DNAT+SNAT, /etc/traefik/dynamic exists, step-ca root CA present. UAT signoff script: replaced assertion 36 (systemd → podman container), added assertions 40-46 (nft table, DNAT, SNAT, dynamic dir, step-ca CA, traefik.yml, doctor ingress pass). docs/ingress.md: R-024 podman traefik section — three topologies, container config, nft ruleset, doctor ingress, Dockerfile.traefik. TLS model updated (drop certResolver, tls:{} for v0.14, mTLS v0.15). ARCHITECTURE.md: v0.14 deltas section — R-024, three topologies, nft emitter changes, TLS model, migration 0009, new CLI. Integration tests (tests/ingress_bootstrap_test.go): nft postrouting + DNATTarget, priority -10, traefik TLS model (tls:{} no certResolver), image ref resolution, floating-IP LXC provisioning commands (pct create with hwaddr/ip/gw/features), MAC generation. ---ci--- project: orca phase: 7 milestone: v0.14 status: execute ---/ci--- --- .ciagent/ARCHITECTURE.md | 67 +++++++++++++ docs/ingress.md | 78 +++++++++++++-- internal/cli/doctor_ingress.go | 115 ++++++++++++++++++++++ scripts/uat-signoff.sh | 27 +++++- tests/ingress_bootstrap_test.go | 163 ++++++++++++++++++++++++++++++++ 5 files changed, 441 insertions(+), 9 deletions(-) create mode 100644 internal/cli/doctor_ingress.go create mode 100644 tests/ingress_bootstrap_test.go diff --git a/.ciagent/ARCHITECTURE.md b/.ciagent/ARCHITECTURE.md index 344b722..b8b8c9f 100644 --- a/.ciagent/ARCHITECTURE.md +++ b/.ciagent/ARCHITECTURE.md @@ -872,3 +872,70 @@ scheduler.Schedule(spec, nodes) → emitter.Render(unit) → sshpush.Deploy(targ - IPv6 `net.JoinHostPort` in all SSH dial paths. - Explicit timeouts on all SSH commands. - Root SIGINT/SIGTERM handler for clean exit on non-watch commands. + +## v0.14 Deltas — Ingress Bootstrap Completeness (R-024) + +### R-024: Traefik as Podman Container + +Traefik runs exclusively as a podman container, deployed from the +custom `orca-traefik` image (published per release via `Dockerfile.traefik` ++ `scripts/release.sh` + `.coreci.yml container-publish-traefik`). + +The v0.13 binary+systemd install (`internal/traefik/install.go`) is +replaced by an idempotent podman container reconciler +(`EnsureTraefikContainerLocal`/`Remote`). The container runs with +`--network host`, `--restart=unless-stopped`, and volume mounts for +`traefik.yml` (static config), `dynamic` (dynamic config), and +`step-ca-root.crt` (future mTLS). No SELinux `:Z` flag. + +### Three Ingress Topologies + +1. **Linux** (`orca init` / `orca node join --type linux`): + host → nft DNAT → podman traefik (host network). + `internal/ingress/bootstrap.go` → `BootstrapLocalIngress` / + `BootstrapRemoteIngress`. + +2. **Proxmox Native** (`--ingress-mode native`, default): + PVE host → nft DNAT (target = LXC bridge IP) → LXC + (`--features nesting=1,keyctl=1,fuse=1`) → podman traefik. + `internal/proxmox/bootstrap.go` → `provisionNativeIngressLXC`. + +3. **Proxmox Floating-IP** (`--ingress-mode floating-ip`): + LXC owns the floating IP (`net0 bridge=vmbr0,hwaddr=, + ip=/,gw=`) → nft inside LXC → + podman traefik. The ingress LXC is registered as a `linux` node + (name=`ingress`) so `orca job run` pushes traefik dynamic config. + `internal/proxmox/ingress_lxc.go` → `ProvisionIngressLXC`. + +### nft Emitter Changes + +`internal/emitter/nft.go`: +- `DNATTarget` field (C-51: validated via `net.ParseIP`). Default + `127.0.0.1`; proxmox native uses LXC bridge IP. +- `EnableSNAT` field + postrouting masquerade chain: `ip saddr + 127.0.0.0/8 oifname != "lo" masquerade` (research Topic 1). +- Input/forward chain priority shifted from `filter` (=0) to `-10` + (research Topic 2: pve-firewall coexistence — avoids same-priority + undefined evaluation order). + +### TLS Model + +v0.14 drops `certResolver: orca` from the dynamic config (traefik v3.3 +only supports `acme`/`tailscale` resolvers, not CA-file-based). The +dynamic config emits `tls: {}` (traefik default cert). Real mTLS via +`tls.certificates` + `tls.options.default.clientAuth.caFiles` is +deferred to v0.15 (grill G-003, confidence 0.55 < 0.60). + +### Migration 0009 + +`ALTER TABLE nodes ADD COLUMN ingress_mode TEXT NOT NULL DEFAULT '';` +Values: `""` (legacy), `"native"`, `"floating-ip"`. `IngressMode` field +on `model.Node`. + +### New CLI + +- `orca doctor ingress` — verifies podman container running, nft + DNAT+SNAT, dynamic dir, step-ca root CA. +- `--ingress-mode` flag on `orca node join --type proxmox`. +- `--floating-ip`, `--gateway`, `--mac`, `--net-prefix` flags for + floating-IP mode. diff --git a/docs/ingress.md b/docs/ingress.md index 850057a..1d81877 100644 --- a/docs/ingress.md +++ b/docs/ingress.md @@ -138,13 +138,77 @@ restore traffic). ## TLS -- **certResolver**: `orca` (references the Traefik ACME/step-ca - certificate resolver configured in Traefik's static config). -- **Trust domain**: `cluster.orca.local` (placeholder in v0.9; step-ca - provisioner in v0.11 overrides with the real cluster trust domain). -- **SPIFFE SVIDs**: workload identity via SPIFFE SVIDs minted at submit - time via step-ca (v0.11-P01.5, gate C-08). The SVID is a URI SAN in - the workload's X.509 cert. +- **v0.14 model**: `tls: {}` in dynamic config (no certResolver). + Traefik v3.3 `certificatesResolvers` only supports `acme` and + `tailscale` — not CA-file-based. The `certResolver: orca` reference + from v0.11 was broken (research finding). v0.14 emits `tls: {}` + (traefik uses its default self-signed cert). Real mTLS via dynamic + `tls.certificates` + `tls.options.default.clientAuth.caFiles` is + deferred to v0.15. +- **Step-ca root CA**: mounted at `/etc/orca/step-ca-root.crt` in the + traefik container. v0.14 does not use it for TLS termination (it's + a placeholder for v0.15 mTLS). + +## R-024: Podman Traefik Container (v0.14) + +As of v0.14, Traefik runs as a **podman container** from the custom +`orca-traefik` image (published per release). The v0.13 binary+systemd +install is replaced. + +### Three topologies + +1. **Linux**: host → nft DNAT → `podman run orca-traefik` (`--network host`) +2. **Proxmox Native** (`--ingress-mode native`, default): PVE host → + nft DNAT → LXC (nesting=1,keyctl=1,fuse=1) → `podman run orca-traefik` +3. **Proxmox Floating-IP** (`--ingress-mode floating-ip`): LXC owns + the floating IP → nft inside LXC → `podman run orca-traefik` + +### Container configuration + +```bash +podman run -d --name orca-traefik --restart=unless-stopped \ + --network host \ + -v /etc/traefik/traefik.yml:/etc/traefik/traefik.yml:ro \ + -v /etc/traefik/dynamic:/etc/traefik/dynamic:ro \ + -v /etc/orca/step-ca-root.crt:/etc/orca/step-ca-root.crt:ro \ + git.cloudinit.dev/coreci/orca-traefik: +``` + +- `--network host`: traefik binds 127.0.0.1:8080/8443 on host/LXC loopback +- `--restart=unless-stopped`: survives reboot via `podman-restart.service` +- No `:Z` SELinux flag (research Topic 7) +- Static config mounted `:ro` (overrides baked image default, preserves + `traefik-on-public-ip` opt-out, REQ-100) + +### nft ruleset + +The nft emitter (`internal/emitter/nft.go`) renders `/etc/nftables.d/orca.nft`: + +- DNAT `:443` → `:8443` (default 127.0.0.1; LXC IP for native) +- DNAT `:80` → `:8080` +- SNAT/MASQUERADE: `ip saddr 127.0.0.0/8 oifname != "lo" masquerade` +- Input/forward chains at priority -10 (pve-firewall coexistence) + +### `orca doctor ingress` + +```bash +orca doctor ingress # check localhost +orca doctor ingress --peer # check remote peer +``` + +Verifies: podman container running, nft DNAT+SNAT, dynamic dir exists, +step-ca root CA present. + +### Dockerfile.traefik + +```dockerfile +FROM traefik:v3.3.0 +COPY docker/orca-traefik/traefik.yml /etc/traefik/traefik.yml +CMD ["--configFile=/etc/traefik/traefik.yml"] +``` + +Built + published per release alongside the orca image +(`scripts/release.sh` + `.coreci.yml container-publish-traefik`). ## Health checks diff --git a/internal/cli/doctor_ingress.go b/internal/cli/doctor_ingress.go new file mode 100644 index 0000000..606f4fd --- /dev/null +++ b/internal/cli/doctor_ingress.go @@ -0,0 +1,115 @@ +// Package cli: doctor_ingress.go implements `orca doctor ingress` +// (R-024, v0.14). The check verifies the podman traefik container is +// running, nft DNAT+SNAT is applied, the dynamic config directory +// exists, and the step-ca root CA is mounted. +package cli + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/spf13/cobra" +) + +var doctorIngressCmd = &cobra.Command{ + Use: "ingress", + Short: "Check the ingress stack (R-024: podman traefik + nft + CA)", + Long: `Verify the orca ingress data plane is healthy: + 1. orca-traefik podman container is running + 2. nft DNAT + SNAT masquerade applied + 3. /etc/traefik/dynamic directory exists + 4. step-ca root CA mounted at /etc/orca/step-ca-root.crt + +For remote peers, use --peer .`, + RunE: func(cmd *cobra.Command, args []string) error { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + results := runIngressChecks(ctx) + if jsonOutput { + return printJSON(results) + } + allPass := true + for _, r := range results { + status := "✓" + if r.Result != "PASS" { + status = "✗" + allPass = false + } + fmt.Fprintf(cmd.OutOrStdout(), "%s %s: %s\n", status, r.Name, r.Message) + } + if !allPass { + return fmt.Errorf("ingress checks failed") + } + return nil + }, +} + +// ingressCheckResult is one line of `orca doctor ingress` output. +type ingressCheckResult struct { + Name string `json:"name"` + Result string `json:"result"` + Message string `json:"message"` +} + +func runIngressChecks(ctx context.Context) []ingressCheckResult { + t, err := nftTransportFromCtx() + if err != nil { + return []ingressCheckResult{{Name: "ingress:transport", Result: "FAIL", Message: err.Error()}} + } + peer := nftLeadPeer() + var results []ingressCheckResult + + // 1. Check podman orca-traefik container is running. + out, err := t.Exec(ctx, peer, "podman inspect --format '{{.State.Running}}' orca-traefik 2>/dev/null") + if err != nil { + results = append(results, ingressCheckResult{Name: "ingress:container", Result: "FAIL", Message: fmt.Sprintf("podman inspect: %v", err)}) + } else { + v := strings.TrimSpace(string(out)) + if v == "true" { + results = append(results, ingressCheckResult{Name: "ingress:container", Result: "PASS", Message: "orca-traefik container running"}) + } else if v == "false" { + results = append(results, ingressCheckResult{Name: "ingress:container", Result: "FAIL", Message: "orca-traefik container is stopped"}) + } else { + results = append(results, ingressCheckResult{Name: "ingress:container", Result: "FAIL", Message: "orca-traefik container not found"}) + } + } + + // 2. Check nft DNAT + SNAT (reuse the nft table output). + tableOut, tableErr := t.Exec(ctx, peer, "nft list table inet orca-ingress 2>/dev/null") + if tableErr != nil { + results = append(results, ingressCheckResult{Name: "ingress:nft", Result: "FAIL", Message: "nft table orca-ingress missing"}) + } else { + tableStr := string(tableOut) + hasDNAT := strings.Contains(tableStr, "dnat to") + hasSNAT := strings.Contains(tableStr, "masquerade") + if hasDNAT && hasSNAT { + results = append(results, ingressCheckResult{Name: "ingress:nft", Result: "PASS", Message: "nft DNAT + SNAT masquerade present"}) + } else if hasDNAT { + results = append(results, ingressCheckResult{Name: "ingress:nft", Result: "WARN", Message: "DNAT present but SNAT masquerade missing"}) + } else { + results = append(results, ingressCheckResult{Name: "ingress:nft", Result: "FAIL", Message: "nft DNAT missing"}) + } + } + + // 3. Check /etc/traefik/dynamic directory exists. + if _, err := t.Exec(ctx, peer, "test -d /etc/traefik/dynamic"); err != nil { + results = append(results, ingressCheckResult{Name: "ingress:dynamic-dir", Result: "FAIL", Message: "/etc/traefik/dynamic directory missing"}) + } else { + results = append(results, ingressCheckResult{Name: "ingress:dynamic-dir", Result: "PASS", Message: "/etc/traefik/dynamic exists"}) + } + + // 4. Check step-ca root CA is mounted/present. + if _, err := t.Exec(ctx, peer, "test -f /etc/orca/step-ca-root.crt"); err != nil { + results = append(results, ingressCheckResult{Name: "ingress:ca", Result: "WARN", Message: "/etc/orca/step-ca-root.crt missing (TLS not configured)"}) + } else { + results = append(results, ingressCheckResult{Name: "ingress:ca", Result: "PASS", Message: "step-ca root CA present"}) + } + + return results +} + +func init() { + doctorCmd.AddCommand(doctorIngressCmd) +} diff --git a/scripts/uat-signoff.sh b/scripts/uat-signoff.sh index 1eef636..38a65a7 100755 --- a/scripts/uat-signoff.sh +++ b/scripts/uat-signoff.sh @@ -145,8 +145,8 @@ assert "34 type_linux_available" \ assert "35 status_deprecated" \ '$ORCA status 2>&1 | grep -qi "deprecated"' -assert "36 traefik_installed" \ - 'systemctl is-active orca-traefik 2>/dev/null | grep -q "active" || exit 77' +assert "36 traefik_container_running" \ + 'podman inspect --format "{{.State.Running}}" orca-traefik 2>/dev/null | grep -q "true" || exit 77' assert "37 known_hosts_exists" \ 'test -f "$ORCA_HOME/known_hosts" || test -f "$ORCA_HOME/cluster/known_hosts"' @@ -154,6 +154,29 @@ assert "37 known_hosts_exists" \ assert "38 master_key_exists" \ 'test -f "$ORCA_HOME/cluster/master.key" || test -f "$ORCA_HOME/cluster/master.key.sealed"' +# --- v0.14 ingress bootstrap assertions (R-024) --- + +assert "40 ingress_nft_table" \ + 'nft list table inet orca-ingress 2>/dev/null | grep -q "chain prerouting"' + +assert "41 ingress_nft_dnat" \ + 'nft list table inet orca-ingress 2>/dev/null | grep -q "dnat to"' + +assert "42 ingress_nft_snat" \ + 'nft list table inet orca-ingress 2>/dev/null | grep -q "masquerade"' + +assert "43 ingress_dynamic_dir" \ + 'test -d /etc/traefik/dynamic' + +assert "44 ingress_step_ca" \ + 'test -f /etc/orca/step-ca-root.crt' + +assert "45 ingress_traefik_yml" \ + 'test -f /etc/traefik/traefik.yml' + +assert "46 ingress_doctor_pass" \ + '$ORCA doctor ingress 2>&1 | grep -q "PASS"' + # --- Report --- echo "==========================================" diff --git a/tests/ingress_bootstrap_test.go b/tests/ingress_bootstrap_test.go new file mode 100644 index 0000000..07b9243 --- /dev/null +++ b/tests/ingress_bootstrap_test.go @@ -0,0 +1,163 @@ +package tests + +import ( + "log/slog" + "strings" + "testing" + + "git.cloudinit.dev/coreci/orca/internal/emitter" + "git.cloudinit.dev/coreci/orca/internal/jobspec" + "git.cloudinit.dev/coreci/orca/internal/proxmox" + "git.cloudinit.dev/coreci/orca/internal/traefik" +) + +// TestNftEmitter_PostroutingAndDNATTarget (REQ-173) verifies the nft +// emitter renders the postrouting masquerade chain and supports +// DNATTarget substitution. +func TestNftEmitter_PostroutingAndDNATTarget(t *testing.T) { + files, err := emitter.NftEmitter{}.RenderNftConfig(emitter.NftClusterConfig{ + DNATTarget: "10.99.0.10", + EnableSNAT: true, + }) + if err != nil { + t.Fatalf("RenderNftConfig: %v", err) + } + c := files[0].Content + if !strings.Contains(c, "chain postrouting") { + t.Errorf("missing postrouting chain:\n%s", c) + } + if !strings.Contains(c, "masquerade") { + t.Errorf("missing masquerade rule:\n%s", c) + } + if !strings.Contains(c, "dnat to 10.99.0.10:8443") { + t.Errorf("missing custom DNAT target:\n%s", c) + } +} + +// TestNftEmitter_PriorityMinus10 (research Topic 2) verifies the input +// and forward chains use priority -10 for pve-firewall coexistence. +func TestNftEmitter_PriorityMinus10(t *testing.T) { + files, _ := emitter.NftEmitter{}.RenderNftConfig(emitter.NftClusterConfig{}) + c := files[0].Content + if !strings.Contains(c, "hook input priority -10;") { + t.Errorf("input chain should use priority -10:\n%s", c) + } + if !strings.Contains(c, "hook forward priority -10;") { + t.Errorf("forward chain should use priority -10:\n%s", c) + } +} + +// TestTraefikEmitter_TLSModel (REQ-172) verifies the dynamic config +// emits tls: {} and does NOT contain certResolver (dropped in v0.14). +func TestTraefikEmitter_TLSModel(t *testing.T) { + spec := &jobspec.WorkloadSpec{ + Name: "test-svc", + Kind: "Service", + Ports: []jobspec.PortSpec{{Name: "http"}}, + } + node := &emitter.Node{ + Hostname: "test-node", + } + files, err := emitter.TraefikEmitter{}.Render(spec, node) + if err != nil { + t.Fatalf("Render: %v", err) + } + c := files[0].Content + if !strings.Contains(c, "tls: {}") { + t.Errorf("missing tls: {} (v0.14 model):\n%s", c) + } + if strings.Contains(c, "certResolver: orca") { + t.Errorf("certResolver: orca should be removed (v0.14):\n%s", c) + } +} + +// TestTraefikImageRef verifies the image reference resolution for the +// orca-traefik podman container. +func TestTraefikImageRef(t *testing.T) { + ref := traefik.ImageRef("v0.13.7") + want := "git.cloudinit.dev/coreci/orca-traefik:v0.13.7" + if ref != want { + t.Errorf("ImageRef(v0.13.7) = %q, want %q", ref, want) + } + // Dev build falls back to latest. + ref = traefik.ImageRef("dev") + if ref != "git.cloudinit.dev/coreci/orca-traefik:latest" { + t.Errorf("ImageRef(dev) = %q, want latest", ref) + } +} + +// TestProxmox_FloatingIP_LXC_ProvisioningCommands (REQ-176) verifies +// the ProvisionIngressLXC function sends the correct pct create +// command with the right net0 parameters. +func TestProxmox_FloatingIP_LXC_ProvisioningCommands(t *testing.T) { + var cmds []string + execFn := func(cmd string) ([]byte, error) { + cmds = append(cmds, cmd) + // Simulate: pct status returns "absent" on first call, then OK. + if strings.Contains(cmd, "pct status 201") { + return []byte("absent\n"), nil + } + if strings.Contains(cmd, "pct create") { + return []byte(""), nil + } + if strings.Contains(cmd, "pct start 201") { + return []byte(""), nil + } + if strings.Contains(cmd, "hostname -I") { + return []byte("203.0.113.10\n"), nil + } + return []byte(""), nil + } + err := proxmox.ProvisionIngressLXC(nil, execFn, proxmox.FloatingIPOptions{ + FloatingIP: "203.0.113.10", + Gateway: "203.0.113.1", + MAC: "02:01:02:03:04:05", + NetPrefix: 24, + LXCTemplate: "ubuntu-24.04", + }, slog.Default()) + if err != nil { + t.Fatalf("ProvisionIngressLXC: %v", err) + } + // Verify pct create has the right net0 params. + foundCreate := false + for _, c := range cmds { + if strings.Contains(c, "pct create") { + foundCreate = true + if !strings.Contains(c, "hostname ingress") { + t.Errorf("pct create missing hostname ingress: %s", c) + } + if !strings.Contains(c, "hwaddr=02:01:02:03:04:05") { + t.Errorf("pct create missing hwaddr: %s", c) + } + if !strings.Contains(c, "ip=203.0.113.10/24") { + t.Errorf("pct create missing ip: %s", c) + } + if !strings.Contains(c, "gw=203.0.113.1") { + t.Errorf("pct create missing gw: %s", c) + } + if !strings.Contains(c, "nesting=1,keyctl=1,fuse=1") { + t.Errorf("pct create missing features (research Topic 3): %s", c) + } + } + } + if !foundCreate { + t.Errorf("pct create command not sent\ncommands: %v", cmds) + } +} + +// TestProxmox_GenerateRandomMAC (D-261) verifies MAC generation produces +// a valid locally-administered MAC. +func TestProxmox_GenerateRandomMAC(t *testing.T) { + mac, err := proxmox.GenerateRandomMAC() + if err != nil { + t.Fatalf("GenerateRandomMAC: %v", err) + } + if !strings.HasPrefix(mac, "02:") { + t.Errorf("MAC should start with 02: (locally administered): %s", mac) + } + // Verify it's 6 octets. + parts := strings.Split(mac, ":") + if len(parts) != 6 { + t.Errorf("MAC should have 6 octets: %s", mac) + } +}