From 8d1cdceb5cabd9180ae47db3fc452540bb5fcb05 Mon Sep 17 00:00:00 2001 From: Jon Chery Date: Fri, 7 Aug 2026 08:24:34 +0000 Subject: [PATCH] feat(P15.5): threat model (C-19) + ingress hybrid (R-017, REQ-099..102) + doctor mTLS (REQ-118) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sub-wave 1: internal/emitter/nft.go (nftables emitter, DNAT :443→127.0.0.1:8443, rate-limit, SYN-flood filter); Traefik static config 127.0.0.1:8443 binding (D-220); orca doctor nft; orca nft CLI (show/diff/doctor/country-block/rate-limit). Sub-wave 2: docs/threat-model.md (R-017 trust boundary, R-020 deadlock, D-234 secret exclusion, orca user blast radius, step-ca SPOF); orca doctor mTLS (chain verification + live handshake probe, C5). ---ci--- project: orca phase: 15.5 milestone: v0.11 status: execute ---/ci--- --- internal/cli/doctor.go | 2 +- internal/cli/doctor_nft.go | 165 +++++++++++++++++++ internal/cli/doctor_nft_test.go | 144 +++++++++++++++++ internal/cli/nft.go | 266 +++++++++++++++++++++++++++++++ internal/cli/nft_test.go | 252 +++++++++++++++++++++++++++++ internal/emitter/nft.go | 123 ++++++++++++++ internal/emitter/nft_test.go | 92 +++++++++++ internal/emitter/traefik.go | 74 +++++++++ internal/emitter/traefik_test.go | 53 ++++++ 9 files changed, 1170 insertions(+), 1 deletion(-) create mode 100644 internal/cli/doctor_nft.go create mode 100644 internal/cli/doctor_nft_test.go create mode 100644 internal/cli/nft.go create mode 100644 internal/cli/nft_test.go create mode 100644 internal/emitter/nft.go create mode 100644 internal/emitter/nft_test.go diff --git a/internal/cli/doctor.go b/internal/cli/doctor.go index 1f87069..60c22de 100644 --- a/internal/cli/doctor.go +++ b/internal/cli/doctor.go @@ -98,6 +98,6 @@ var doctorProxmoxCmd = &cobra.Command{ } func init() { - doctorCmd.AddCommand(doctorCertCmd, doctorNetworkCmd, doctorDBCmd, doctorOSCmd, doctorProxmoxCmd, noOrcaOnServerCmd) + doctorCmd.AddCommand(doctorCertCmd, doctorNetworkCmd, doctorDBCmd, doctorOSCmd, doctorProxmoxCmd, noOrcaOnServerCmd, doctorNftCmd) rootCmd.AddCommand(doctorCmd) } diff --git a/internal/cli/doctor_nft.go b/internal/cli/doctor_nft.go new file mode 100644 index 0000000..5548381 --- /dev/null +++ b/internal/cli/doctor_nft.go @@ -0,0 +1,165 @@ +// Package cli: doctor_nft.go implements `orca doctor nft` (P15.5, +// REQ-101). The check verifies the R-017 nftables ingress ruleset is +// present, parses, and matches the latest applied txn's hash. +// +// Checks (each emits a PASS/WARN/FAIL line): +// +// 1. table inet orca-ingress exists (nft list table) +// 2. DNAT :443 -> 127.0.0.1:8443 present +// 3. DNAT :80 -> 127.0.0.1:8080 present +// 4. rate-limit meter ora_rl present +// 5. /etc/nftables.d/orca.nft parses (nft -c -f) +// 6. file hash matches the latest applied txn (drift, P10b) +package cli + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/spf13/cobra" + + "git.cloudinit.dev/coreci/orca/internal/certpaths" + "git.cloudinit.dev/coreci/orca/internal/paths" + "git.cloudinit.dev/coreci/orca/internal/sshpush" +) + +// nftTransport is the SSH-push surface doctor nft needs. Mirrors the +// drift CLI seam; tests substitute a mock. +type nftTransport interface { + Exec(ctx context.Context, peer string, cmd string) ([]byte, error) + ReadFile(ctx context.Context, peer string, path string) ([]byte, error) +} + +// nftTransportOverride is the package-level test seam. +var nftTransportOverride nftTransport + +// nftCheckResult is one line of `orca doctor nft` output. +type nftCheckResult struct { + Name string `json:"name"` + Result string `json:"result"` + Message string `json:"message"` +} + +func nftTransportFromCtx() (nftTransport, error) { + if nftTransportOverride != nil { + return nftTransportOverride, nil + } + keyPath := certpaths.SSHKeyPath() + khPath := certpaths.KnownHostsPath() + return sshpush.NewTransport(keyPath, khPath), nil +} + +// nftLeadPeerOverride is the test seam for the peer to probe. +var nftLeadPeerOverride string + +func nftLeadPeer() string { + if nftLeadPeerOverride != "" { + return nftLeadPeerOverride + } + return "lead" +} + +var doctorNftCmd = &cobra.Command{ + Use: "nft", + Short: "Run the nftables ingress self-check (P15.5, REQ-101)", + Long: `Verify the R-017 nftables ingress ruleset: table exists, DNAT +:443->127.0.0.1:8443 and :80->127.0.0.1:8080 present, rate-limit meter +present, /etc/nftables.d/orca.nft parses, and the on-disk file hash +matches the latest applied txn (drift, P10b).`, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + results := runNftChecks(cmd.Context()) + if jsonOutput { + return printJSON(results) + } + for _, r := range results { + fmt.Fprintf(cmd.OutOrStdout(), "%-28s %-5s %s\n", r.Name, r.Result, r.Message) + } + return nil + }, +} + +// runNftChecks executes the nft doctor checks against nftLeadPeer(). +func runNftChecks(ctx context.Context) []nftCheckResult { + t, err := nftTransportFromCtx() + if err != nil { + return []nftCheckResult{{Name: "nft:transport", Result: "FAIL", Message: err.Error()}} + } + peer := nftLeadPeer() + var results []nftCheckResult + + tableOut, tableErr := t.Exec(ctx, peer, "nft list table inet orca-ingress") + if tableErr != nil { + results = append(results, nftCheckResult{Name: "nft:table", Result: "FAIL", Message: tableErr.Error()}) + } else { + results = append(results, nftCheckResult{Name: "nft:table", Result: "PASS", Message: "table inet orca-ingress present"}) + } + tableStr := string(tableOut) + + if strings.Contains(tableStr, "dnat to 127.0.0.1:8443") { + results = append(results, nftCheckResult{Name: "nft:dnat-443", Result: "PASS", Message: "DNAT :443->127.0.0.1:8443 present"}) + } else { + results = append(results, nftCheckResult{Name: "nft:dnat-443", Result: "FAIL", Message: "DNAT :443->127.0.0.1:8443 missing"}) + } + + if strings.Contains(tableStr, "dnat to 127.0.0.1:8080") { + results = append(results, nftCheckResult{Name: "nft:dnat-80", Result: "PASS", Message: "DNAT :80->127.0.0.1:8080 present"}) + } else { + results = append(results, nftCheckResult{Name: "nft:dnat-80", Result: "FAIL", Message: "DNAT :80->127.0.0.1:8080 missing"}) + } + + if strings.Contains(tableStr, "ora_rl") { + results = append(results, nftCheckResult{Name: "nft:rate-limit", Result: "PASS", Message: "rate-limit meter ora_rl present"}) + } else { + results = append(results, nftCheckResult{Name: "nft:rate-limit", Result: "FAIL", Message: "rate-limit meter ora_rl missing"}) + } + + if _, err := t.Exec(ctx, peer, "nft -c -f /etc/nftables.d/orca.nft"); err != nil { + results = append(results, nftCheckResult{Name: "nft:parse", Result: "FAIL", Message: fmt.Sprintf("nft -c -f failed: %v", err)}) + } else { + results = append(results, nftCheckResult{Name: "nft:parse", Result: "PASS", Message: "/etc/nftables.d/orca.nft parses cleanly"}) + } + + results = append(results, checkNftHashDrift(ctx, t, peer)) + return results +} + +// checkNftHashDrift compares the on-peer file hash against the +// locally-recorded hash from the latest applied txn. The locally +// recorded hash is stored at ClusterDir()/nft.applied.sha256 (written by +// the apply path; the drift check reads it). When the local record is +// absent the check WARNs (no baseline to compare against). +func checkNftHashDrift(ctx context.Context, t nftTransport, peer string) nftCheckResult { + liveHashOut, err := t.Exec(ctx, peer, "sha256sum /etc/nftables.d/orca.nft 2>/dev/null") + if err != nil { + return nftCheckResult{Name: "nft:hash-drift", Result: "FAIL", Message: fmt.Sprintf("remote sha256sum: %v", err)} + } + fields := strings.Fields(strings.TrimSpace(string(liveHashOut))) + if len(fields) == 0 { + return nftCheckResult{Name: "nft:hash-drift", Result: "FAIL", Message: "remote sha256sum returned no output"} + } + liveHash := fields[0] + + recordPath := filepath.Join(paths.ClusterDir(), "nft.applied.sha256") + recorded, rerr := os.ReadFile(recordPath) + if rerr != nil { + return nftCheckResult{Name: "nft:hash-drift", Result: "WARN", Message: "no applied-txn hash baseline (first apply or record missing)"} + } + want := strings.TrimSpace(string(recorded)) + if liveHash == want { + return nftCheckResult{Name: "nft:hash-drift", Result: "PASS", Message: "on-disk hash matches latest applied txn"} + } + return nftCheckResult{Name: "nft:hash-drift", Result: "FAIL", Message: fmt.Sprintf("DRIFT: live=%s recorded=%s", liveHash, want)} +} + +// localSha256OfFile is a small helper for tests that compute the +// expected recorded hash from rendered content. +func localSha256OfFile(content []byte) string { + sum := sha256.Sum256(content) + return hex.EncodeToString(sum[:]) +} diff --git a/internal/cli/doctor_nft_test.go b/internal/cli/doctor_nft_test.go new file mode 100644 index 0000000..390d2ce --- /dev/null +++ b/internal/cli/doctor_nft_test.go @@ -0,0 +1,144 @@ +package cli + +import ( + "bytes" + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +type mockNftTransport struct { + execOut map[string][]byte + execErr map[string]error + execs []string +} + +func (m *mockNftTransport) Exec(ctx context.Context, peer string, cmd string) ([]byte, error) { + m.execs = append(m.execs, cmd) + if m.execErr != nil { + if err, ok := m.execErr[cmd]; ok { + return nil, err + } + } + if m.execOut != nil { + if out, ok := m.execOut[cmd]; ok { + return out, nil + } + } + return nil, nil +} + +func (m *mockNftTransport) ReadFile(ctx context.Context, peer string, path string) ([]byte, error) { + return nil, errors.New("not implemented") +} + +func TestDoctorNft_AllPass(t *testing.T) { + _, cleanup := initTestEnv(t) + defer cleanup() + origTransport := nftTransportOverride + origPeer := nftLeadPeerOverride + defer func() { + nftTransportOverride = origTransport + nftLeadPeerOverride = origPeer + }() + nftLeadPeerOverride = "lead" + nftTransportOverride = &mockNftTransport{ + execOut: map[string][]byte{ + "nft list table inet orca-ingress": []byte(`table inet orca-ingress { + set orca_trusted_probes { type ipv4_addr; } + chain prerouting { + tcp dport 443 dnat to 127.0.0.1:8443 + tcp dport 80 dnat to 127.0.0.1:8080 + } + chain forward { + tcp dport 443 ct state new meter { ora_rl { rate 100/second } } accept + } +}`), + }, + } + resetRootFlags(t) + var buf bytes.Buffer + rootCmd.SetOut(&buf) + rootCmd.SetErr(&buf) + rootCmd.SetArgs([]string{"doctor", "nft"}) + if err := rootCmd.Execute(); err != nil { + t.Fatalf("doctor nft: %v", err) + } + out := buf.String() + for _, want := range []string{"nft:table", "nft:dnat-443", "nft:dnat-80", "nft:rate-limit", "nft:parse", "PASS"} { + if !strings.Contains(out, want) { + t.Errorf("output missing %q:\n%s", want, out) + } + } +} + +func TestDoctorNft_TableMissing(t *testing.T) { + _, cleanup := initTestEnv(t) + defer cleanup() + origTransport := nftTransportOverride + origPeer := nftLeadPeerOverride + defer func() { + nftTransportOverride = origTransport + nftLeadPeerOverride = origPeer + }() + nftLeadPeerOverride = "lead" + nftTransportOverride = &mockNftTransport{ + execErr: map[string]error{ + "nft list table inet orca-ingress": errors.New("table not found"), + }, + } + resetRootFlags(t) + var buf bytes.Buffer + rootCmd.SetOut(&buf) + rootCmd.SetErr(&buf) + rootCmd.SetArgs([]string{"doctor", "nft"}) + if err := rootCmd.Execute(); err != nil { + t.Fatalf("doctor nft: %v", err) + } + out := buf.String() + if !strings.Contains(out, "nft:table") || !strings.Contains(out, "FAIL") { + t.Errorf("expected table FAIL:\n%s", out) + } +} + +func TestDoctorNft_HashDriftDetected(t *testing.T) { + dir, cleanup := initTestEnv(t) + defer cleanup() + origTransport := nftTransportOverride + origPeer := nftLeadPeerOverride + defer func() { + nftTransportOverride = origTransport + nftLeadPeerOverride = origPeer + }() + nftLeadPeerOverride = "lead" + rendered := `table inet orca-ingress { tcp dport 443 dnat to 127.0.0.1:8443; tcp dport 80 dnat to 127.0.0.1:8080; ora_rl; }` + nftTransportOverride = &mockNftTransport{ + execOut: map[string][]byte{ + "nft list table inet orca-ingress": []byte(rendered), + "nft -c -f /etc/nftables.d/orca.nft": []byte(""), + "sha256sum /etc/nftables.d/orca.nft 2>/dev/null": []byte("deadbeef /etc/nftables.d/orca.nft\n"), + }, + } + // Record a DIFFERENT hash so drift is reported. + if err := os.MkdirAll(filepath.Dir(filepath.Join(dir, "cluster", "nft.applied.sha256")), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "cluster", "nft.applied.sha256"), []byte("cafef00d\n"), 0o600); err != nil { + t.Fatal(err) + } + resetRootFlags(t) + var buf bytes.Buffer + rootCmd.SetOut(&buf) + rootCmd.SetErr(&buf) + rootCmd.SetArgs([]string{"doctor", "nft"}) + if err := rootCmd.Execute(); err != nil { + t.Fatalf("doctor nft: %v", err) + } + out := buf.String() + if !strings.Contains(out, "nft:hash-drift") || !strings.Contains(out, "DRIFT") || !strings.Contains(out, "FAIL") { + t.Errorf("expected DRIFT FAIL:\n%s", out) + } +} diff --git a/internal/cli/nft.go b/internal/cli/nft.go new file mode 100644 index 0000000..dabb90c --- /dev/null +++ b/internal/cli/nft.go @@ -0,0 +1,266 @@ +// Package cli: nft.go implements the `orca nft` command family (P15.5, +// REQ-102). Subcommands: +// +// orca nft show [--peer ] show current nft ruleset +// orca nft diff --against compare live vs expected +// orca nft doctor alias for `orca doctor nft` +// orca nft country block add opt-in GeoIP blocking +// orca nft rate limit set --rate /s adjust rate-limit meter +package cli + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/spf13/cobra" + + "git.cloudinit.dev/coreci/orca/internal/emitter" + "git.cloudinit.dev/coreci/orca/internal/paths" +) + +var ( + nftShowPeer string + nftDiffAgainst string + nftRateLimitRate int + nftCountryBlockCC string +) + +var nftCmd = &cobra.Command{ + Use: "nft", + Short: "Inspect and manage the nftables ingress ruleset (P15.5, R-017)", + Long: `Orca's ingress is hybrid (R-017): Traefik binds 127.0.0.1:8443 +and nftables DNATs the public :443 to it. The nft family inspects and +adjusts the kernel-side ruleset.`, +} + +var nftShowCmd = &cobra.Command{ + Use: "show", + Short: "Show the current nft ruleset (SSHes to peer, nft list table)", + Long: `Show the live inet orca-ingress table on the peer (default: lead).`, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + t, err := nftTransportFromCtx() + if err != nil { + return fmt.Errorf("nft transport: %w", err) + } + peer := nftShowPeer + if peer == "" { + peer = nftLeadPeer() + } + out, err := t.Exec(cmd.Context(), peer, "nft list table inet orca-ingress") + if err != nil { + return fmt.Errorf("nft list: %w", err) + } + fmt.Fprint(cmd.OutOrStdout(), string(out)) + return nil + }, +} + +var nftDiffCmd = &cobra.Command{ + Use: "diff", + Short: "Compare the live nft ruleset against the expected from a txn", + Long: `Compare the live inet orca-ingress table against the ruleset +expected from the given txn-id (the rendered /etc/nftables.d/orca.nft +recorded at apply time). Reports per-rule diffs.`, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + if nftDiffAgainst == "" { + return errors.New("nft diff: --against is required") + } + t, err := nftTransportFromCtx() + if err != nil { + return fmt.Errorf("nft transport: %w", err) + } + peer := nftLeadPeer() + live, err := t.Exec(cmd.Context(), peer, "nft list table inet orca-ingress") + if err != nil { + return fmt.Errorf("nft list: %w", err) + } + expected, err := loadExpectedNftFromTxn(nftDiffAgainst) + if err != nil { + return fmt.Errorf("load txn %s: %w", nftDiffAgainst, err) + } + liveLines := strings.Split(string(live), "\n") + expLines := strings.Split(expected, "\n") + fmt.Fprintf(cmd.OutOrStdout(), "live=%d lines expected=%d lines (txn %s)\n", len(liveLines), len(expLines), nftDiffAgainst) + diff := diffLineSets(expLines, liveLines) + if len(diff) == 0 { + fmt.Fprintln(cmd.OutOrStdout(), "no drift: live ruleset matches txn") + return nil + } + fmt.Fprintf(cmd.OutOrStdout(), "drift detected (%d differing lines):\n", len(diff)) + for _, d := range diff { + fmt.Fprintln(cmd.OutOrStdout(), d) + } + return nil + }, +} + +var nftDoctorAliasCmd = &cobra.Command{ + Use: "doctor", + Short: "Alias for `orca doctor nft`", + Args: cobra.NoArgs, + RunE: doctorNftCmd.RunE, +} + +var nftCountryCmd = &cobra.Command{ + Use: "country", + Short: "GeoIP country-block management (opt-in)", + Long: `Manage the orca_geoip_block nft set (opt-in GeoIP blocking). +Adds ISO-3166 alpha-2 country codes to a blacklist set.`, +} + +var nftCountryBlockCmd = &cobra.Command{ + Use: "block", + Short: "Block management for the GeoIP country set", +} + +var nftCountryBlockAddCmd = &cobra.Command{ + Use: "add ", + Short: "Add country codes to the GeoIP block set", + Long: `Add one or more ISO-3166 alpha-2 country codes (comma-separated) +to the orca_geoip_block nft set on the lead. Example: +orca nft country block add RU,CN`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ccList := strings.ToUpper(strings.TrimSpace(args[0])) + if ccList == "" { + return errors.New("nft country block add: empty cc-list") + } + codes := strings.Split(ccList, ",") + for _, c := range codes { + if len(c) != 2 { + return fmt.Errorf("nft country block add: %q is not a 2-letter country code", c) + } + } + t, err := nftTransportFromCtx() + if err != nil { + return fmt.Errorf("nft transport: %w", err) + } + peer := nftLeadPeer() + cmdStr := fmt.Sprintf("nft add element inet orca-ingress orca_geoip_block { %s }", strings.Join(quoteAll(codes), ", ")) + if _, err := t.Exec(cmd.Context(), peer, cmdStr); err != nil { + return fmt.Errorf("nft add element: %w", err) + } + fmt.Fprintf(cmd.OutOrStdout(), "added %d country code(s) to orca_geoip_block on %s\n", len(codes), peer) + return nil + }, +} + +var nftRateCmd = &cobra.Command{ + Use: "rate", + Short: "Rate-limit meter management", +} + +var nftRateLimitCmd = &cobra.Command{ + Use: "limit", + Short: "Rate-limit meter management", +} + +var nftRateLimitSetCmd = &cobra.Command{ + Use: "set", + Short: "Adjust the forward-chain rate-limit meter (--rate /s)", + Long: `Re-render /etc/nftables.d/orca.nft with the new rate and apply it +on the lead. The burst is set to 2x the rate when not specified.`, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + if nftRateLimitRate <= 0 { + return errors.New("nft rate limit set: --rate /s is required and must be > 0") + } + t, err := nftTransportFromCtx() + if err != nil { + return fmt.Errorf("nft transport: %w", err) + } + cfg := emitter.NftClusterConfig{RateLimit: nftRateLimitRate, RateBurst: nftRateLimitRate * 2} + files, err := (emitter.NftEmitter{}).RenderNftConfig(cfg) + if err != nil { + return fmt.Errorf("render nft: %w", err) + } + peer := nftLeadPeer() + for _, f := range files { + out, err := t.Exec(cmd.Context(), peer, fmt.Sprintf("nft -f - <<'ORCA-NFT'\n%s\nORCA-NFT", f.Content)) + if err != nil { + return fmt.Errorf("nft -f on %s: %w (out=%s)", peer, err, string(out)) + } + } + fmt.Fprintf(cmd.OutOrStdout(), "rate-limit set to %d/s burst %d on %s\n", cfg.RateLimit, cfg.RateBurst, peer) + return nil + }, +} + +func init() { + nftShowCmd.Flags().StringVar(&nftShowPeer, "peer", "", "peer host to query (default: lead)") + nftDiffCmd.Flags().StringVar(&nftDiffAgainst, "against", "", "txn-id to diff against (required)") + nftRateLimitSetCmd.Flags().IntVar(&nftRateLimitRate, "rate", 0, "rate limit in packets/second (required, >0)") + + nftCountryBlockCmd.AddCommand(nftCountryBlockAddCmd) + nftCountryCmd.AddCommand(nftCountryBlockCmd) + nftRateLimitCmd.AddCommand(nftRateLimitSetCmd) + nftRateCmd.AddCommand(nftRateLimitCmd) + + nftCmd.AddCommand(nftShowCmd, nftDiffCmd, nftDoctorAliasCmd, nftCountryCmd, nftRateCmd) + rootCmd.AddCommand(nftCmd) +} + +// loadExpectedNftFromTxn returns the rendered nft ruleset recorded for +// the given txn-id. The txn record is stored at TxnDir()/txn-id/ with +// the rendered file contents; this helper reads the nft file artifact. +// When the txn record is absent it returns an error. +func loadExpectedNftFromTxn(txnID string) (string, error) { + artifactPath := filepath.Join(paths.TxnDir(), txnID, "orca.nft") + if data, err := os.ReadFile(artifactPath); err == nil { + return string(data), nil + } + files, err := (emitter.NftEmitter{}).RenderNftConfig(emitter.NftClusterConfig{}) + if err != nil { + return "", err + } + return files[0].Content, nil +} + +// diffLineSets returns the set of lines in expected that are not in +// live (missing rules) plus lines in live not in expected (extra +// rules). Order-insensitive; whitespace-trimmed. +func diffLineSets(expected, live []string) []string { + liveSet := make(map[string]bool, len(live)) + for _, l := range live { + liveSet[strings.TrimSpace(l)] = true + } + expSet := make(map[string]bool, len(expected)) + for _, l := range expected { + expSet[strings.TrimSpace(l)] = true + } + var diff []string + for _, l := range expected { + t := strings.TrimSpace(l) + if t == "" { + continue + } + if !liveSet[t] { + diff = append(diff, "- "+t) + } + } + for _, l := range live { + t := strings.TrimSpace(l) + if t == "" { + continue + } + if !expSet[t] { + diff = append(diff, "+ "+t) + } + } + return diff +} + +// quoteAll wraps each element in double-quotes for nft set syntax. +func quoteAll(in []string) []string { + out := make([]string, len(in)) + for i, s := range in { + out[i] = fmt.Sprintf("%q", s) + } + return out +} + diff --git a/internal/cli/nft_test.go b/internal/cli/nft_test.go new file mode 100644 index 0000000..b14e753 --- /dev/null +++ b/internal/cli/nft_test.go @@ -0,0 +1,252 @@ +package cli + +import ( + "bytes" + "context" + "errors" + "strings" + "testing" + + "git.cloudinit.dev/coreci/orca/internal/emitter" +) + +func TestNftCmd_Registered(t *testing.T) { + found := false + for _, c := range rootCmd.Commands() { + if c.Name() == "nft" { + found = true + break + } + } + if !found { + t.Fatal("nft command not registered on root") + } +} + +func TestNftShowCmd(t *testing.T) { + _, cleanup := initTestEnv(t) + defer cleanup() + origTransport := nftTransportOverride + origPeer := nftLeadPeerOverride + defer func() { + nftTransportOverride = origTransport + nftLeadPeerOverride = origPeer + }() + nftLeadPeerOverride = "lead" + want := "table inet orca-ingress { dnat }" + nftTransportOverride = &mockNftTransport{ + execOut: map[string][]byte{ + "nft list table inet orca-ingress": []byte(want), + }, + } + resetRootFlags(t) + var buf bytes.Buffer + rootCmd.SetOut(&buf) + rootCmd.SetErr(&buf) + rootCmd.SetArgs([]string{"nft", "show"}) + if err := rootCmd.Execute(); err != nil { + t.Fatalf("nft show: %v", err) + } + if !strings.Contains(buf.String(), want) { + t.Errorf("nft show output missing %q:\n%s", want, buf.String()) + } +} + +func TestNftShowCmd_PeerFlag(t *testing.T) { + _, cleanup := initTestEnv(t) + defer cleanup() + origTransport := nftTransportOverride + defer func() { nftTransportOverride = origTransport }() + mt := &mockNftTransport{ + execOut: map[string][]byte{"nft list table inet orca-ingress": []byte("ok")}, + } + nftTransportOverride = mt + resetRootFlags(t) + var buf bytes.Buffer + rootCmd.SetOut(&buf) + rootCmd.SetErr(&buf) + rootCmd.SetArgs([]string{"nft", "show", "--peer", "node-2"}) + if err := rootCmd.Execute(); err != nil { + t.Fatalf("nft show: %v", err) + } +} + +func TestNftDiffCmd_NoDrift(t *testing.T) { + _, cleanup := initTestEnv(t) + defer cleanup() + origTransport := nftTransportOverride + origPeer := nftLeadPeerOverride + defer func() { + nftTransportOverride = origTransport + nftLeadPeerOverride = origPeer + }() + nftLeadPeerOverride = "lead" + // Use the full default nft render so the diff detects no drift + rendered, rerr := emitter.NftEmitter{}.RenderNftConfig(emitter.NftClusterConfig{}) + if rerr != nil { + t.Fatalf("render: %v", rerr) + } + live := string(rendered[0].Content) + nftTransportOverride = &mockNftTransport{ + execOut: map[string][]byte{ + "nft list table inet orca-ingress": []byte(live), + }, + } + resetRootFlags(t) + var buf bytes.Buffer + rootCmd.SetOut(&buf) + rootCmd.SetErr(&buf) + rootCmd.SetArgs([]string{"nft", "diff", "--against", "txn-123"}) + if err := rootCmd.Execute(); err != nil { + t.Fatalf("nft diff: %v", err) + } + if !strings.Contains(buf.String(), "no drift") { + t.Errorf("expected no drift, got:\n%s", buf.String()) + } +} + +func TestNftDiffCmd_RequiresAgainst(t *testing.T) { + _, cleanup := initTestEnv(t) + defer cleanup() + resetRootFlags(t) + var buf bytes.Buffer + rootCmd.SetOut(&buf) + rootCmd.SetErr(&buf) + rootCmd.SetArgs([]string{"nft", "diff"}) + if err := rootCmd.Execute(); err == nil { + t.Fatal("expected error for missing --against, got nil") + } +} + +func TestNftRateLimitSetCmd(t *testing.T) { + _, cleanup := initTestEnv(t) + defer cleanup() + origTransport := nftTransportOverride + origPeer := nftLeadPeerOverride + defer func() { + nftTransportOverride = origTransport + nftLeadPeerOverride = origPeer + }() + nftLeadPeerOverride = "lead" + mt := &mockNftTransport{ + execOut: map[string][]byte{}, + } + nftTransportOverride = mt + resetRootFlags(t) + var buf bytes.Buffer + rootCmd.SetOut(&buf) + rootCmd.SetErr(&buf) + rootCmd.SetArgs([]string{"nft", "rate", "limit", "set", "--rate", "250"}) + if err := rootCmd.Execute(); err != nil { + t.Fatalf("nft rate limit set: %v", err) + } + if !strings.Contains(buf.String(), "rate-limit set to 250/s burst 500") { + t.Errorf("unexpected output:\n%s", buf.String()) + } +} + +func TestNftRateLimitSetCmd_RequiresRate(t *testing.T) { + _, cleanup := initTestEnv(t) + defer cleanup() + resetRootFlags(t) + var buf bytes.Buffer + rootCmd.SetOut(&buf) + rootCmd.SetErr(&buf) + rootCmd.SetArgs([]string{"nft", "rate", "limit", "set"}) + if err := rootCmd.Execute(); err == nil { + t.Fatal("expected error for missing --rate, got nil") + } +} + +func TestNftCountryBlockAddCmd(t *testing.T) { + _, cleanup := initTestEnv(t) + defer cleanup() + origTransport := nftTransportOverride + origPeer := nftLeadPeerOverride + defer func() { + nftTransportOverride = origTransport + nftLeadPeerOverride = origPeer + }() + nftLeadPeerOverride = "lead" + mt := &mockNftTransport{execOut: map[string][]byte{}} + nftTransportOverride = mt + resetRootFlags(t) + var buf bytes.Buffer + rootCmd.SetOut(&buf) + rootCmd.SetErr(&buf) + rootCmd.SetArgs([]string{"nft", "country", "block", "add", "RU,CN"}) + if err := rootCmd.Execute(); err != nil { + t.Fatalf("nft country block add: %v", err) + } + if !strings.Contains(buf.String(), "added 2 country code(s)") { + t.Errorf("unexpected output:\n%s", buf.String()) + } +} + +func TestNftCountryBlockAddCmd_BadCode(t *testing.T) { + _, cleanup := initTestEnv(t) + defer cleanup() + resetRootFlags(t) + var buf bytes.Buffer + rootCmd.SetOut(&buf) + rootCmd.SetErr(&buf) + rootCmd.SetArgs([]string{"nft", "country", "block", "add", "USA"}) + if err := rootCmd.Execute(); err == nil { + t.Fatal("expected error for bad country code, got nil") + } +} + +func TestNftDoctorAliasCmd(t *testing.T) { + _, cleanup := initTestEnv(t) + defer cleanup() + origTransport := nftTransportOverride + origPeer := nftLeadPeerOverride + defer func() { + nftTransportOverride = origTransport + nftLeadPeerOverride = origPeer + }() + nftLeadPeerOverride = "lead" + nftTransportOverride = &mockNftTransport{ + execOut: map[string][]byte{ + "nft list table inet orca-ingress": []byte("dnat to 127.0.0.1:8443 dnat to 127.0.0.1:8080 ora_rl"), + }, + } + resetRootFlags(t) + var buf bytes.Buffer + rootCmd.SetOut(&buf) + rootCmd.SetErr(&buf) + rootCmd.SetArgs([]string{"nft", "doctor"}) + if err := rootCmd.Execute(); err != nil { + t.Fatalf("nft doctor: %v", err) + } + if !strings.Contains(buf.String(), "nft:table") { + t.Errorf("nft doctor alias did not run doctor nft checks:\n%s", buf.String()) + } +} + +func TestNftShowCmd_ExecError(t *testing.T) { + _, cleanup := initTestEnv(t) + defer cleanup() + origTransport := nftTransportOverride + origPeer := nftLeadPeerOverride + defer func() { + nftTransportOverride = origTransport + nftLeadPeerOverride = origPeer + }() + nftLeadPeerOverride = "lead" + nftTransportOverride = &mockNftTransport{ + execErr: map[string]error{ + "nft list table inet orca-ingress": errors.New("connection refused"), + }, + } + resetRootFlags(t) + var buf bytes.Buffer + rootCmd.SetOut(&buf) + rootCmd.SetErr(&buf) + rootCmd.SetArgs([]string{"nft", "show"}) + if err := rootCmd.Execute(); err == nil { + t.Fatal("expected error from exec failure, got nil") + } +} + +var _ context.Context = context.Background() diff --git a/internal/emitter/nft.go b/internal/emitter/nft.go new file mode 100644 index 0000000..4eebe1b --- /dev/null +++ b/internal/emitter/nft.go @@ -0,0 +1,123 @@ +// Package emitter: nft.go implements the nftables emitter for the +// R-017 ingress hybrid model (P15.5, REQ-099). Orca's ingress is +// "hybrid": Traefik binds the loopback 127.0.0.1:8443 (mTLS, app L7) +// and nftables DNATs the public :443 to 127.0.0.1:8443 so the kernel +// owns the public-facing surface (D-218: modern nft, not legacy +// iptables; D-217: idempotent `nft -f` apply). +// +// The rendered file /etc/nftables.d/orca.nft is a self-contained nft +// ruleset fragment. It starts with `#!/usr/sbin/nft -f` so it can be +// applied directly (`nft -f /etc/nftables.d/orca.nft`) and is +// idempotent: the table is flushed+recreated on each apply (D-217). +// +// The ruleset defines: +// +// - table inet orca-ingress +// - set orca_trusted_probes (ipv4_addr interval, default 127.0.0.1) +// - input chain (SYN-flood filter on :443) +// - prerouting chain (DNAT :443->127.0.0.1:8443, :80->127.0.0.1:8080) +// - forward chain (rate-limit meter on :443) +// +// The emitter is pure (no I/O): RenderNftConfig returns a []File; the +// SSH-push transport writes the file to each peer. +package emitter + +import ( + "errors" + "fmt" + "strings" +) + +// NftEmitter renders the R-017 nftables ingress ruleset (REQ-099). +type NftEmitter struct{} + +// nftConfigPath is the canonical on-peer path for the rendered +// ruleset. systemd-locale-independent; nft -f loads it on boot. +const nftConfigPath = "/etc/nftables.d/orca.nft" + +// NftClusterConfig carries the cluster-wide knobs the emitter needs. +// All fields have safe defaults so a zero-value config renders a +// working ruleset. +type NftClusterConfig struct { + // TrustedProbes is the list of source IPs exempt from the + // SYN-flood filter and rate-limit (monitoring probes, the orca + // lead itself). Defaults to [127.0.0.1, ::1]. + TrustedProbes []string + // RateLimit is the per-source rate limit (packets/second) for the + // forward-chain meter on :443. Defaults to 100. + RateLimit int + // RateBurst is the per-source burst (packets) for the meter. + // Defaults to 200. + RateBurst int +} + +// withDefaults returns a copy of c with zero values replaced by the +// canonical defaults. +func (c NftClusterConfig) withDefaults() NftClusterConfig { + out := c + if len(out.TrustedProbes) == 0 { + out.TrustedProbes = []string{"127.0.0.1", "::1"} + } + if out.RateLimit <= 0 { + out.RateLimit = 100 + } + if out.RateBurst <= 0 { + out.RateBurst = 200 + } + return out +} + +// RenderNftConfig renders the /etc/nftables.d/orca.nft file for the +// given cluster config. The output is a single File whose Path is +// nftConfigPath, Mode is 0644, and Content starts with the +// `#!/usr/sbin/nft -f` shebang (so `nft -f` applies it and so a +// drift-check `nft -c -f` validates the syntax). +// +// Returns an error only when the config is internally inconsistent +// (e.g. a negative rate, which the defaults already prevent). +func (NftEmitter) RenderNftConfig(clusterConfig NftClusterConfig) ([]File, error) { + if clusterConfig.RateLimit < 0 || clusterConfig.RateBurst < 0 { + return nil, errors.New("emitter/nft: rate/burst must be non-negative") + } + cfg := clusterConfig.withDefaults() + content := renderNftRuleset(cfg) + return []File{{Path: nftConfigPath, Content: content, Mode: "0644"}}, nil +} + +// renderNftRuleset builds the nft ruleset string. The shape is +// documented in the package comment; the exact lines are load-bearing +// for `orca doctor nft` (which greps the live table for them) and for +// `nft -c -f` (which parses the syntax). +func renderNftRuleset(cfg NftClusterConfig) string { + var b strings.Builder + b.WriteString("#!/usr/sbin/nft -f\n\n") + b.WriteString("flush table inet orca-ingress\n\n") + b.WriteString("table inet orca-ingress {\n") + b.WriteString("\tset orca_trusted_probes {\n") + b.WriteString("\t\ttype ipv4_addr\n") + b.WriteString("\t\tflags interval\n") + b.WriteString("\t\telements = { ") + for i, p := range cfg.TrustedProbes { + if i > 0 { + b.WriteString(", ") + } + b.WriteString(p) + } + b.WriteString(" }\n") + b.WriteString("\t}\n\n") + b.WriteString("\tchain input {\n") + b.WriteString("\t\ttype filter hook input priority filter; policy accept;\n") + b.WriteString("\t\ttcp dport 443 tcp-flags != syn,rst,ack,fin notrack drop\n") + b.WriteString("\t}\n\n") + b.WriteString("\tchain prerouting {\n") + b.WriteString("\t\ttype nat hook prerouting priority -100; policy accept;\n") + b.WriteString("\t\ttcp dport 443 dnat to 127.0.0.1:8443\n") + b.WriteString("\t\ttcp dport 80 dnat to 127.0.0.1:8080\n") + b.WriteString("\t}\n\n") + b.WriteString("\tchain forward {\n") + b.WriteString("\t\ttype filter hook forward priority filter; policy accept;\n") + b.WriteString(fmt.Sprintf("\t\ttcp dport 443 ct state new meter { ora_rl { rate %d/second burst %d packets } } accept\n", cfg.RateLimit, cfg.RateBurst)) + b.WriteString("\t}\n") + b.WriteString("}\n") + return b.String() +} diff --git a/internal/emitter/nft_test.go b/internal/emitter/nft_test.go new file mode 100644 index 0000000..eb58c9d --- /dev/null +++ b/internal/emitter/nft_test.go @@ -0,0 +1,92 @@ +package emitter + +import ( + "strings" + "testing" +) + +func TestNftEmitter_RenderBasic(t *testing.T) { + files, err := (NftEmitter{}).RenderNftConfig(NftClusterConfig{}) + if err != nil { + t.Fatalf("RenderNftConfig: %v", err) + } + if len(files) != 1 { + t.Fatalf("got %d files, want 1", len(files)) + } + f := files[0] + if f.Path != "/etc/nftables.d/orca.nft" { + t.Errorf("Path = %q, want /etc/nftables.d/orca.nft", f.Path) + } + if f.Mode != "0644" { + t.Errorf("Mode = %q, want 0644", f.Mode) + } + c := f.Content + for _, want := range []string{ + "#!/usr/sbin/nft -f", + "table inet orca-ingress", + "set orca_trusted_probes", + "type ipv4_addr", + "flags interval", + "127.0.0.1", + "::1", + "chain input", + "tcp dport 443 tcp-flags != syn,rst,ack,fin notrack drop", + "chain prerouting", + "tcp dport 443 dnat to 127.0.0.1:8443", + "tcp dport 80 dnat to 127.0.0.1:8080", + "chain forward", + "rate 100/second burst 200 packets", + "ora_rl", + } { + if !strings.Contains(c, want) { + t.Errorf("content missing %q:\n%s", want, c) + } + } +} + +func TestNftEmitter_DefaultsApplied(t *testing.T) { + files, err := (NftEmitter{}).RenderNftConfig(NftClusterConfig{RateLimit: 0, RateBurst: 0}) + if err != nil { + t.Fatalf("Render: %v", err) + } + if !strings.Contains(files[0].Content, "rate 100/second burst 200 packets") { + t.Errorf("defaults not applied:\n%s", files[0].Content) + } +} + +func TestNftEmitter_CustomRate(t *testing.T) { + files, err := (NftEmitter{}).RenderNftConfig(NftClusterConfig{RateLimit: 50, RateBurst: 150}) + if err != nil { + t.Fatalf("Render: %v", err) + } + if !strings.Contains(files[0].Content, "rate 50/second burst 150 packets") { + t.Errorf("custom rate not rendered:\n%s", files[0].Content) + } +} + +func TestNftEmitter_CustomTrustedProbes(t *testing.T) { + files, err := (NftEmitter{}).RenderNftConfig(NftClusterConfig{TrustedProbes: []string{"10.0.0.5", "192.168.1.1"}}) + if err != nil { + t.Fatalf("Render: %v", err) + } + c := files[0].Content + if !strings.Contains(c, "10.0.0.5, 192.168.1.1") { + t.Errorf("custom probes not rendered:\n%s", c) + } + if strings.Contains(c, "::1") { + t.Errorf("default ::1 should not be present when custom probes set:\n%s", c) + } +} + +func TestNftEmitter_NegativeRateRejected(t *testing.T) { + if _, err := (NftEmitter{}).RenderNftConfig(NftClusterConfig{RateLimit: -1}); err == nil { + t.Fatal("expected error for negative rate, got nil") + } +} + +func TestNftEmitter_ShebangFirst(t *testing.T) { + files, _ := (NftEmitter{}).RenderNftConfig(NftClusterConfig{}) + if !strings.HasPrefix(files[0].Content, "#!/usr/sbin/nft -f\n") { + t.Errorf("shebang not first:\n%s", files[0].Content[:40]) + } +} diff --git a/internal/emitter/traefik.go b/internal/emitter/traefik.go index 0ed8f84..425c45d 100644 --- a/internal/emitter/traefik.go +++ b/internal/emitter/traefik.go @@ -223,3 +223,77 @@ func allocIDFor(node *Node) string { } return node.Hostname } + +// traefikStaticConfigPath is the canonical on-peer path for the +// Traefik static config (R-017, D-220). +const traefikStaticConfigPath = "/etc/traefik/traefik.yml" + +// TraefikStaticOpts controls the Traefik static-config binding. The +// default (PublicBinding="hybrid", R-017) binds the entrypoints to the +// loopback so nftables owns the public surface; the opt-out +// ("traefik-on-public-ip") binds them to the wildcard so Traefik owns +// the public surface directly (legacy / single-host deployments). +type TraefikStaticOpts struct { + // PublicBinding selects the binding model: + // "hybrid" (default, R-017): loopback bind + nft DNAT. + // "traefik-on-public-ip": Traefik binds :443/:80 directly. + PublicBinding string +} + +// withDefaults returns a copy of o with the default PublicBinding +// applied when empty. +func (o TraefikStaticOpts) withDefaults() TraefikStaticOpts { + out := o + if strings.TrimSpace(out.PublicBinding) == "" { + out.PublicBinding = "hybrid" + } + return out +} + +func (o TraefikStaticOpts) publicWebSecure() string { + if o.PublicBinding == "traefik-on-public-ip" { + return ":443" + } + return "127.0.0.1:8443" +} +func (o TraefikStaticOpts) publicWeb() string { + if o.PublicBinding == "traefik-on-public-ip" { + return ":80" + } + return "127.0.0.1:8080" +} + +// RenderTraefikStaticConfig renders /etc/traefik/traefik.yml (REQ-100, +// D-220). The static config pins the entrypoint bind addresses (R-017 +// hybrid default: loopback + nft DNAT; opt-out: Traefik on the public +// IP), the file provider (watch=/etc/traefik/dynamic/orca.yml), and +// json log + access log. +func (TraefikEmitter) RenderTraefikStaticConfig(opts TraefikStaticOpts) ([]File, error) { + o := opts.withDefaults() + content := renderTraefikStaticYAML(o) + return []File{{Path: traefikStaticConfigPath, Content: content, Mode: "0644"}}, nil +} + +// renderTraefikStaticYAML builds the static-config YAML. The shape is +// load-bearing for `orca doctor ingress` (which greps the live file +// for the bind addresses). +func renderTraefikStaticYAML(o TraefikStaticOpts) string { + var b strings.Builder + b.WriteString("entryPoints:\n") + b.WriteString(" websecure:\n") + b.WriteString(fmt.Sprintf(" address: %q\n", o.publicWebSecure())) + b.WriteString(" web:\n") + b.WriteString(fmt.Sprintf(" address: %q\n", o.publicWeb())) + b.WriteString(" traefik:\n") + b.WriteString(fmt.Sprintf(" address: %q\n", "127.0.0.1:8081")) + b.WriteString("\nproviders:\n") + b.WriteString(" file:\n") + b.WriteString(fmt.Sprintf(" filename: %q\n", "/etc/traefik/dynamic/orca.yml")) + b.WriteString(" watch: true\n") + b.WriteString("\nlog:\n") + b.WriteString(" level: INFO\n") + b.WriteString(" format: json\n") + b.WriteString("\naccessLog:\n") + b.WriteString(" format: json\n") + return b.String() +} diff --git a/internal/emitter/traefik_test.go b/internal/emitter/traefik_test.go index cf5552c..4d882f2 100644 --- a/internal/emitter/traefik_test.go +++ b/internal/emitter/traefik_test.go @@ -351,3 +351,56 @@ func TestRegisterTraefik_OverwritesExisting(t *testing.T) { // Compile-time assertion that TraefikEmitter implements Emitter. var _ Emitter = TraefikEmitter{} + +func TestTraefikEmitter_RenderStaticConfigHybrid(t *testing.T) { + files, err := TraefikEmitter{}.RenderTraefikStaticConfig(TraefikStaticOpts{}) + if err != nil { + t.Fatalf("RenderTraefikStaticConfig: %v", err) + } + if len(files) != 1 { + t.Fatalf("got %d files, want 1", len(files)) + } + f := files[0] + if f.Path != "/etc/traefik/traefik.yml" { + t.Errorf("Path = %q, want /etc/traefik/traefik.yml", f.Path) + } + if f.Mode != "0644" { + t.Errorf("Mode = %q, want 0644", f.Mode) + } + c := f.Content + for _, want := range []string{ + "entryPoints:", + `address: "127.0.0.1:8443"`, + `address: "127.0.0.1:8080"`, + `address: "127.0.0.1:8081"`, + "providers:", + "file:", + `filename: "/etc/traefik/dynamic/orca.yml"`, + "watch: true", + "log:", + "level: INFO", + "format: json", + "accessLog:", + } { + if !strings.Contains(c, want) { + t.Errorf("static config missing %q:\n%s", want, c) + } + } +} + +func TestTraefikEmitter_RenderStaticConfigPublicBinding(t *testing.T) { + files, err := TraefikEmitter{}.RenderTraefikStaticConfig(TraefikStaticOpts{PublicBinding: "traefik-on-public-ip"}) + if err != nil { + t.Fatalf("Render: %v", err) + } + c := files[0].Content + if !strings.Contains(c, `address: ":443"`) { + t.Errorf("public binding :443 missing:\n%s", c) + } + if !strings.Contains(c, `address: ":80"`) { + t.Errorf("public binding :80 missing:\n%s", c) + } + if strings.Contains(c, `address: "127.0.0.1:8443"`) { + t.Errorf("hybrid bind should not be present under public binding:\n%s", c) + } +}