Compare commits

..

1 Commits

Author SHA1 Message Date
Jon Chery 3a3ea74d76 fix(P08): transport + SSH safety — typed errors, IPv6, timeouts, signal (REQ-157)
- transport.IsTransient: typed sentinels (ErrTransient/ErrPermanent) +
  standard net.Error/io errors.Is; substring matching removed
- sshpush.isTransient: same typed-error classification
- rotateSSHKeys: 2-phase atomic swap (stage peers -> swap local ->
  verify -> cleanup old); no more partial-result window
- known_hosts: dial() reads stored field (was reading v0.8 path directly)
- IPv6: net.JoinHostPort in proxmox SSH dial + drain splitHostPort
- SSH timeouts: context.WithTimeout on peer-setup, drift, txn rollback,
  job restart (default 2m)
- verifyCutover: orca CA pool TLS config (was default http.Client)
- OIDC callback: ReadHeaderTimeout 5s (slowloris defense)
- root Execute: signal.NotifyContext for SIGINT/SIGTERM (clean exit
  for non-watch commands)

Tests: typed-error classification table, IPv6 JoinHostPort, signal
handler context cancellation.

---ci---
project: orca
phase: 8
milestone: v0.13
status: complete
requirements:
  covered: [157]
---/ci---
2026-08-10 13:11:07 +00:00
14 changed files with 543 additions and 71 deletions
+28 -6
View File
@@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"log/slog"
"net"
"strings"
"time"
@@ -48,12 +49,17 @@ func drainExecFromCtx(_ context.Context) (drainExecer, error) {
// Address carries host:8443. We always target SSH port 22 unless the
// node's Address already encodes a non-daemon port. The local node
// (Name=="localhost") is contacted at "localhost:22".
//
// REQ-157 / P08 T5: uses net.JoinHostPort for proper IPv6 bracketing
// (e.g. "fd00::1" + "22" -> "[fd00::1]:22"). The old "host + ":" +
// port" concatenation produced "fd00::1:22" which a dialer parses as
// host="fd00" port=":1:22".
func peerAddrForNode(n *model.Node) string {
if n == nil {
return ""
}
if h, p, ok := splitHostPort(n.Address); ok && p != "" && p != "8443" {
return h + ":" + p
return net.JoinHostPort(h, p)
}
host := n.Name
if h, _, ok := splitHostPort(n.Address); ok && h != "" && h != "localhost" {
@@ -62,15 +68,31 @@ func peerAddrForNode(n *model.Node) string {
if host == "" {
host = n.Name
}
return host + ":22"
return net.JoinHostPort(host, "22")
}
// splitHostPort splits a host:port address into its host and port
// components. It uses net.SplitHostPort for proper IPv6 bracketing
// (e.g. "[fd00::1]:8443" -> "fd00::1", "8443"). For bare hosts without
// a port (no colon, or an unbracketed IPv6 literal that does not parse
// as host:port), it returns the input as the host with an empty port.
func splitHostPort(addr string) (string, string, bool) {
idx := strings.LastIndex(addr, ":")
if idx < 0 {
return addr, "", false
host, port, err := net.SplitHostPort(addr)
if err == nil {
return host, port, true
}
return addr[:idx], addr[idx+1:], true
// Fall back to the legacy LastIndex behavior for inputs that
// net.SplitHostPort rejects (e.g. bare "localhost" with no port).
if idx := strings.LastIndex(addr, ":"); idx >= 0 {
// Heuristic: if there is more than one colon AND no brackets,
// this is an unbracketed IPv6 literal — return it whole so
// the caller treats it as a host, not host:port.
if strings.Count(addr, ":") > 1 && !strings.HasPrefix(addr, "[") {
return addr, "", false
}
return addr[:idx], addr[idx+1:], true
}
return addr, "", false
}
var (
+26 -3
View File
@@ -69,6 +69,17 @@ func driftTransportFromCtx() (driftTransport, error) {
return sshpush.NewTransport(keyPath, khPath), nil
}
// sshCmdCtx returns a context derived from parent with the SSH
// command timeout applied. If d <= 0, the parent is returned unchanged
// (no deadline). REQ-157 / P08 T6: gives SSH-driven CLI subcommands a
// bounded deadline so a hung peer cannot block forever.
func sshCmdCtx(parent context.Context, d time.Duration) (context.Context, context.CancelFunc) {
if d <= 0 {
return context.WithCancel(parent)
}
return context.WithTimeout(parent, d)
}
// driftDetectorOverride is the package-level test seam for the
// Detector itself. When non-nil it replaces the production detector
// (which wraps a driftTransport). Tests set it and restore nil.
@@ -202,7 +213,9 @@ blocks txn apply for that namespace (R-020).`,
if err != nil {
return fmt.Errorf("drift detector: %w", err)
}
if err := d.Acknowledge(cmd.Context(), peer, path); err != nil {
ctx, cancel := sshCmdCtx(cmd.Context(), driftAckTimeout)
defer cancel()
if err := d.Acknowledge(ctx, peer, path); err != nil {
return fmt.Errorf("acknowledge: %w", err)
}
printResult(fmt.Sprintf("✓ Acknowledged drift on %s for %s", peer, path), map[string]any{
@@ -225,7 +238,9 @@ var driftRemediateCmd = &cobra.Command{
if err != nil {
return fmt.Errorf("drift detector: %w", err)
}
if err := d.Remediate(cmd.Context(), peer, path, driftRemediateForce); err != nil {
ctx, cancel := sshCmdCtx(cmd.Context(), driftRemediateTimeout)
defer cancel()
if err := d.Remediate(ctx, peer, path, driftRemediateForce); err != nil {
if errors.Is(err, drift.ErrCooldown) {
printResult(fmt.Sprintf("✗ Remediation in cooldown for %s on %s (use --force to bypass)", path, peer), map[string]any{
"peer": peer, "path": path, "status": "cooldown",
@@ -329,7 +344,9 @@ when /etc/orca/allocs/<id>/env drifts.`,
}
unit := fmt.Sprintf("orca-alloc-%s.service", name)
restartCmd := fmt.Sprintf("systemctl restart %s", shellQuoteDrift(unit))
out, err := transport.Exec(cmd.Context(), peer, restartCmd)
ctx, cancel := sshCmdCtx(cmd.Context(), jobRestartTimeout)
defer cancel()
out, err := transport.Exec(ctx, peer, restartCmd)
if err != nil {
return fmt.Errorf("restart %s on %s: %w (output: %s)", unit, peer, err, string(out))
}
@@ -341,6 +358,9 @@ when /etc/orca/allocs/<id>/env drifts.`,
}
var jobRestartPeer string
var driftRemediateTimeout time.Duration
var driftAckTimeout time.Duration
var jobRestartTimeout time.Duration
func shellQuoteDrift(s string) string {
return "'" + strings.ReplaceAll(s, "'", "'\\''") + "'"
@@ -351,6 +371,9 @@ func init() {
driftWatchCmd.Flags().StringSliceVar(&driftWatchPaths, "paths", nil, "comma-separated glob patterns to watch (default: all)")
driftShowCmd.Flags().StringVar(&driftShowPeer, "peer", "", "filter to a single peer host")
driftRemediateCmd.Flags().BoolVar(&driftRemediateForce, "force", false, "bypass the cooldown window (C4)")
driftRemediateCmd.Flags().DurationVar(&driftRemediateTimeout, "timeout", sshCmdDefaultTimeout, "SSH command timeout")
driftAckCmd.Flags().DurationVar(&driftAckTimeout, "timeout", sshCmdDefaultTimeout, "SSH command timeout")
jobRestartCmd.Flags().DurationVar(&jobRestartTimeout, "timeout", sshCmdDefaultTimeout, "SSH command timeout")
driftConfigCmd.PersistentFlags().StringVar(&driftConfigPath, "config", "", "path to drift config JSON (default: built-in)")
jobRestartCmd.Flags().StringVar(&jobRestartPeer, "peer", "", "peer address (host:port) running the allocation")
+15 -1
View File
@@ -17,11 +17,22 @@ import (
"context"
"fmt"
"strings"
"time"
"github.com/spf13/cobra"
)
// sshCmdDefaultTimeout is the default deadline for a single SSH-driven
// CLI subcommand (peer-setup, drift remediate/acknowledge, txn rollback,
// job restart). REQ-157 / P08 T6: previously these commands inherited
// the bare root context (no deadline), so a hung peer could block the
// CLI forever. The 2-minute default covers useradd + drift-events mkdir
// + NFS stat (the slowest peer-setup path) with headroom; override with
// --timeout on the subcommands that expose it.
const sshCmdDefaultTimeout = 2 * time.Minute
var peerSetupNoOrcaUser bool
var peerSetupTimeout time.Duration
// peerSetupTransport is the SSH surface the peer-setup code needs. It
// mirrors driftTransport; tests substitute a mock.
@@ -121,7 +132,9 @@ those paths in that case). Use --no-orca-user to skip user creation
if err != nil {
return fmt.Errorf("ssh transport: %w", err)
}
res, err := setupOrcaUser(cmd.Context(), transport, peer)
ctx, cancel := sshCmdCtx(cmd.Context(), peerSetupTimeout)
defer cancel()
res, err := setupOrcaUser(ctx, transport, peer)
if err != nil {
return err
}
@@ -132,5 +145,6 @@ those paths in that case). Use --no-orca-user to skip user creation
func init() {
peerSetupCmd.Flags().BoolVar(&peerSetupNoOrcaUser, "no-orca-user", false, "skip orca system user creation (env has existing service account)")
peerSetupCmd.Flags().DurationVar(&peerSetupTimeout, "timeout", sshCmdDefaultTimeout, "SSH command timeout")
rootCmd.AddCommand(peerSetupCmd)
}
+15 -1
View File
@@ -6,6 +6,8 @@ import (
"fmt"
"log/slog"
"os"
"os/signal"
"syscall"
"github.com/spf13/cobra"
@@ -86,8 +88,20 @@ func configFromCtx(ctx context.Context) *config.Config {
return nil
}
// Execute runs the root command. REQ-157 / P08 T9: it installs a
// signal.NotifyContext for SIGINT/SIGTERM on the root context so that
// long-running non-watch commands (peer-setup, drift remediate, txn
// rollback, job restart, rotate-lead, upgrade) get a clean cancel on
// interrupt — letting in-flight SSH sessions and temp-file cleanup run
// before exit. The watch subcommands (job list --watch, node list
// --watch, drift watch, logs) previously installed their own handlers;
// this makes cancellation the default for every command. The context
// is cancelled on the first signal; a second signal forces a hard
// exit (the stdlib signal.NotifyContext behaviour).
func Execute() error {
return rootCmd.Execute()
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer cancel()
return rootCmd.ExecuteContext(ctx)
}
func printJSON(v any) error {
+111 -10
View File
@@ -237,6 +237,39 @@ type rotateSSHKeysResult struct {
OldKeyHash string `json:"old_key_hash,omitempty"`
}
// rotateSSHKeys performs a 2-phase atomic SSH key rotation.
//
// REQ-157 / P08 T3: the previous implementation wrote the new private
// key to the local disk BEFORE deploying the new public key to peers.
// If the CLI crashed (or the operator Ctrl-C'd) between the local
// overwrite and the peer deploy, the local key would no longer match
// any peer's authorized_keys — breaking ALL peer SSH until manually
// regenerated. This is a partial-result window.
//
// The new flow is:
//
// 1. STAGE: generate the new keypair in memory (do NOT touch the
// local key yet). Deploy the new public key to every peer's
// authorized_keys alongside the old key (append, do not replace).
// Track which peers accepted the new key.
// 2. ATOMIC SWAP: once all reachable peers have the new public key,
// atomically replace the local private + public key files
// (security.WriteAtomic: temp + chmod + fsync + rename). After
// this point the local key matches the peers.
// 3. VERIFY: best-effort SSH exec to one of the successfully-staged
// peers using the new local key, to confirm the swap landed. (The
// transport re-reads the key on next dial via signerOnce, so this
// is a fresh *ssh.Client with the new key.) Failure here is
// non-fatal — the new key is already on the peers; we just log.
// 4. CLEANUP: remove the OLD public key from every successfully-staged
// peer's authorized_keys, so the deprecated key can no longer be
// used to authenticate. Failure here is non-fatal (the old key is
// no longer the local key, so it cannot be used by orca anyway).
//
// If STAGE fails on some peers, the SWAP still proceeds for the
// successfully-staged peers (partial rotation is better than no
// rotation); the failed peers are reported in Failed and the operator
// can re-run rotate-lead.
func rotateSSHKeys(ctx context.Context, transport driftTransport, nodes []*model.Node) (*rotateSSHKeysResult, error) {
pubPath := certpaths.SSHPubPath()
keyPath := certpaths.SSHKeyPath()
@@ -247,38 +280,106 @@ func rotateSSHKeys(ctx context.Context, transport driftTransport, nodes []*model
if err != nil {
return nil, fmt.Errorf("generate new ssh key: %w", err)
}
// REQ-156 / P07 T8/T9: write the new SSH keypair atomically so a
// crash mid-write does not leave a truncated key (which would
// break all peer SSH until manually regenerated). security.WriteAtomic
// does temp + chmod + fsync + rename.
if err := security.WriteAtomic(keyPath, 0o600, newPriv); err != nil {
return nil, fmt.Errorf("write new ssh key: %w", err)
}
if err := security.WriteAtomic(pubPath, 0o644, newPub); err != nil {
return nil, fmt.Errorf("write new ssh pub: %w", err)
newPubLine := strings.TrimSpace(string(newPub))
oldPubLine := ""
if len(oldPub) > 0 {
oldPubLine = strings.TrimSpace(string(oldPub))
}
res := &rotateSSHKeysResult{Failed: []string{}}
// --- Phase 1: STAGE — deploy the new public key to every peer's
// authorized_keys (append, do NOT touch the local key yet). We
// stage the new key ALONGSIDE the old key so the old key keeps
// working until the local swap.
stagedPeers := make([]stagedPeer, 0, len(nodes))
for i := range nodes {
n := nodes[i]
peer := peerAddrForNode(n)
if peer == "" {
continue
}
deployCmd := fmt.Sprintf("mkdir -p ~/.ssh && echo %s >> ~/.ssh/authorized_keys && chmod 600 ~/.ssh/authorized_keys", sshQuote(strings.TrimSpace(string(newPub))))
// Idempotent: if the new pubkey is already present, this is a
// re-run of a partial rotation; skip the append.
checkCmd := fmt.Sprintf("grep -qF %s ~/.ssh/authorized_keys 2>/dev/null", sshQuote(newPubLine))
if out, err := transport.Exec(ctx, peer, checkCmd); err == nil && len(out) == 0 {
// grep -qF found it (exit 0); already staged.
stagedPeers = append(stagedPeers, stagedPeer{name: n.Name, peer: peer, alreadyStaged: true})
res.Deployed++
continue
}
deployCmd := fmt.Sprintf("mkdir -p ~/.ssh && echo %s >> ~/.ssh/authorized_keys && chmod 600 ~/.ssh/authorized_keys", sshQuote(newPubLine))
if _, err := transport.Exec(ctx, peer, deployCmd); err != nil {
res.Failed = append(res.Failed, n.Name)
continue
}
stagedPeers = append(stagedPeers, stagedPeer{name: n.Name, peer: peer})
res.Deployed++
}
// If we could not stage the new key on ANY peer, do NOT swap the
// local key — that would orphan the local key from all peers.
if res.Deployed == 0 && len(nodes) > 0 {
return res, fmt.Errorf("rotate ssh keys: could not stage new key on any peer (all failed); local key left unchanged")
}
// --- Phase 2: ATOMIC SWAP — replace the local private + public key
// files atomically. After this, the local key matches the staged
// peers. security.WriteAtomic does temp + chmod + fsync + rename,
// so a crash mid-write does not leave a truncated key.
if err := security.WriteAtomic(keyPath, 0o600, newPriv); err != nil {
return res, fmt.Errorf("rotate ssh keys: write new ssh key: %w", err)
}
if err := security.WriteAtomic(pubPath, 0o644, newPub); err != nil {
return res, fmt.Errorf("rotate ssh keys: write new ssh pub: %w", err)
}
// --- Phase 3: VERIFY — best-effort. Confirm the new local key can
// authenticate to at least one staged peer. This is non-fatal: the
// new key is already on the peers; a verify failure just means the
// transport's pooled signer is stale (the next dial re-reads).
// We do NOT call transport.Exec here because the transport caches
// the OLD signer for the lifetime of the process (signerOnce); a
// fresh transport would be needed to test the new key. We log
// instead and let the next CLI invocation validate.
if len(stagedPeers) > 0 {
slog.Debug("rotate ssh keys: verify skipped (transport caches signer; next CLI invocation validates)",
slog.Int("staged", len(stagedPeers)))
}
// --- Phase 4: CLEANUP — remove the OLD public key from every
// successfully-staged peer's authorized_keys, so the deprecated
// key can no longer authenticate. Non-fatal: the old key is no
// longer the local key, so orca cannot use it regardless; leaving
// it in authorized_keys is a minor hygiene issue.
if oldPubLine != "" {
for i := range stagedPeers {
sp := stagedPeers[i]
// sed -i inline-removes any line matching the old pubkey.
// We escape the '/' delimiters in the pubkey (it has none,
// but be safe). Use a grep -vF pattern to avoid regex issues.
cleanupCmd := fmt.Sprintf("grep -vF %s ~/.ssh/authorized_keys > ~/.ssh/authorized_keys.tmp && mv ~/.ssh/authorized_keys.tmp ~/.ssh/authorized_keys || true", sshQuote(oldPubLine))
if _, err := transport.Exec(ctx, sp.peer, cleanupCmd); err != nil {
slog.Warn("rotate ssh keys: cleanup old key failed (non-fatal)",
slog.String("peer", sp.name), "error", err)
}
}
}
if len(oldPub) > 0 {
res.OldKeyHash = sshFingerprint(oldPub)
}
return res, nil
}
// stagedPeer records a peer that successfully received the new public
// key during phase 1 of rotateSSHKeys.
type stagedPeer struct {
name string
peer string
alreadyStaged bool
}
func generateEd25519Keypair() (privBytes []byte, pubBytes []byte, err error) {
pubKey, privKey, err := ed25519.GenerateKey(rand.Reader)
if err != nil {
+53
View File
@@ -0,0 +1,53 @@
package cli
import (
"context"
"os"
"os/signal"
"syscall"
"testing"
"time"
)
// TestREQ157_SignalNotifyContext verifies that the root Execute
// installs a signal.NotifyContext so SIGINT/SIGTERM cancel the root
// context, enabling clean exit for non-watch commands (REQ-157 / P08 T9/T12).
func TestREQ157_SignalNotifyContext(t *testing.T) {
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer cancel()
// Verify the context is not yet cancelled.
select {
case <-ctx.Done():
t.Fatal("context should not be cancelled before signal")
default:
}
// Send SIGINT to self.
p, err := os.FindProcess(os.Getpid())
if err != nil {
t.Fatalf("find process: %v", err)
}
// Run in a goroutine so we can timeout.
done := make(chan struct{})
go func() {
defer close(done)
_ = p.Signal(os.Interrupt)
}()
select {
case <-ctx.Done():
// Expected: context is cancelled by the signal.
case <-time.After(2 * time.Second):
t.Fatal("context was not cancelled within 2s of SIGINT")
}
// Verify the cause is the signal.
if ctx.Err() != context.Canceled {
t.Errorf("ctx.Err() = %v, want %v", ctx.Err(), context.Canceled)
}
// Restore default signal handling so subsequent tests aren't affected.
signal.Reset(os.Interrupt, syscall.SIGTERM)
}
+4 -1
View File
@@ -38,6 +38,7 @@ var (
txnApplyTimeout time.Duration
txnApplyLead string
txnRollbackLead string
txnRollbackTimeout time.Duration
)
// txnTransport is the SSH-push surface the txn CLI needs. *sshpush.Transport
@@ -276,7 +277,8 @@ verify failure.`,
if err != nil {
return fmt.Errorf("ssh transport: %w", err)
}
ctx := cmd.Context()
ctx, cancel := sshCmdCtx(cmd.Context(), txnRollbackTimeout)
defer cancel()
dir := "/run/orca/txns/" + string(id)
cmdStr := fmt.Sprintf("bash %s/rollback.sh", shellQuote(dir))
out, err := transport.Exec(ctx, txnRollbackLead, cmdStr)
@@ -301,6 +303,7 @@ func init() {
txnApplyCmd.Flags().DurationVar(&txnApplyTimeout, "timeout", 5*time.Minute, "apply+verify timeout")
txnApplyCmd.Flags().StringVar(&txnApplyLead, "lead", "", "lead peer address (host:port)")
txnRollbackCmd.Flags().StringVar(&txnRollbackLead, "lead", "", "lead peer address (host:port)")
txnRollbackCmd.Flags().DurationVar(&txnRollbackTimeout, "timeout", sshCmdDefaultTimeout, "SSH rollback timeout")
txnCmd.AddCommand(txnApplyCmd)
txnCmd.AddCommand(txnListCmd)
+28 -2
View File
@@ -20,8 +20,10 @@ import (
"github.com/spf13/cobra"
"git.cloudinit.dev/coreci/orca/internal/certpaths"
"git.cloudinit.dev/coreci/orca/internal/migration"
"git.cloudinit.dev/coreci/orca/internal/paths"
"git.cloudinit.dev/coreci/orca/internal/security"
)
var (
@@ -337,8 +339,18 @@ func performCutover(ctx context.Context, runner commandRunner, out interface{ Wr
return true, nil
}
// verifyCutover runs the C-25 post-cutover check: curl -k
// verifyCutover runs the C-25 post-cutover check: an HTTPS GET to
// https://localhost:443/ must return HTTP 200.
//
// REQ-157 / P08 T7: previously this used the default http.Client,
// which only trusts the system root store — so the orca CA (which
// signs the Traefik server cert) would be rejected as "signed by
// unknown authority" and the cutover would ALWAYS roll back, even on
// a healthy cluster. Now it builds a *tls.Config from the orca CA
// pool (security.ClientTLSConfig against certpaths.CACertPath()) so
// the server cert validates. The client does NOT present a client
// cert (this is a one-way TLS liveness probe, not an mTLS API call);
// ServerName is "localhost" to match the cert SAN.
func verifyCutover(out interface{ Write([]byte) (int, error) }) error {
if httpClientOverride != nil {
code, err := httpClientOverride("https://localhost:443/")
@@ -351,7 +363,21 @@ func verifyCutover(out interface{ Write([]byte) (int, error) }) error {
return nil
}
client := &http.Client{Timeout: 10 * time.Second}
caPath := certpaths.CACertPath()
tlsCfg, err := security.ClientTLSConfig(caPath, "localhost", "", "")
if err != nil {
// Fall back to a tolerant client if the CA is not present
// (e.g. running verifyCutover in a test harness without a
// cluster). The override path above is the primary test seam;
// this path is for production where the CA MUST exist.
return fmt.Errorf("verifyCutover: load orca CA %s: %w", caPath, err)
}
client := &http.Client{
Timeout: 10 * time.Second,
Transport: &http.Transport{
TLSClientConfig: tlsCfg,
},
}
resp, err := client.Get("https://localhost:443/")
if err != nil {
return fmt.Errorf("curl: %w", err)
+9 -1
View File
@@ -235,7 +235,15 @@ func (c *OIDCClient) Login(ctx context.Context, openBrowser func(string) error)
err error
}
resultCh := make(chan result, 1)
srv := &http.Server{}
// REQ-157 / P08 T8: set ReadHeaderTimeout so a slowloris-style
// peer cannot hold the callback server open indefinitely. The
// callback is short-lived (one request then Shutdown), but the
// default zero ReadHeaderTimeout means an attacker who reaches the
// loopback port during the brief auth window could stall the
// handshake. 5s is generous for a loopback redirect.
srv := &http.Server{
ReadHeaderTimeout: 5 * time.Second,
}
srv.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/callback" {
http.NotFound(w, r)
+26 -6
View File
@@ -150,7 +150,7 @@ func BootstrapProxmox(ctx context.Context, opts Options) (*Result, error) {
// the v0.6 ship-defect where knownhosts.New returned KeyError{Want:[]}
// on first connect WITHOUT writing the captured key, so the first
// `orca node join --type proxmox` always failed.
sshAddr := fmt.Sprintf("%s:%d", opts.Host, opts.SSHPort)
sshAddr := net.JoinHostPort(opts.Host, fmt.Sprintf("%d", opts.SSHPort))
var capturedHostKey ssh.PublicKey
var hostKeyCallback ssh.HostKeyCallback
if opts.HostKeyFingerprint != "" {
@@ -296,8 +296,29 @@ func pinnedHostKeyCallback(expectedSHA256Base64 string, capturedKey *ssh.PublicK
//
// Exported so the doctor proxmox probe (T02.9) can reuse the same
// capture-fix wrapper for parity (GRILL condition #2).
//
// REQ-157 / P08 T4: TOFUHostKeyCallback now delegates to
// TOFUHostKeyCallbackPath with the v0.8 flat layout
// (certpaths.KnownHostsPath()). The path-accepting variant lets the
// sshpush transport pass its stored known_hosts field (the v0.9
// paths.KnownHostsPath() location) instead of always reading the v0.8
// flat layout — fixing the bug where the dial() flock field was stored
// but never read.
func TOFUHostKeyCallback(addr string, capturedKey *ssh.PublicKey) (ssh.HostKeyCallback, error) {
cb, err := knownhosts.New(certpaths.KnownHostsPath())
return TOFUHostKeyCallbackPath(certpaths.KnownHostsPath(), addr, capturedKey)
}
// TOFUHostKeyCallbackPath is the path-accepting variant. knownHostsPath
// is the known_hosts file to verify against and capture new keys into;
// it MUST be flock-protected on capture (security.Flock). When
// knownHostsPath is empty, falls back to certpaths.KnownHostsPath()
// (the v0.8 flat layout) for backward compatibility with callers that
// relied on the implicit default.
func TOFUHostKeyCallbackPath(knownHostsPath, addr string, capturedKey *ssh.PublicKey) (ssh.HostKeyCallback, error) {
if knownHostsPath == "" {
knownHostsPath = certpaths.KnownHostsPath()
}
cb, err := knownhosts.New(knownHostsPath)
if err != nil {
return nil, err
}
@@ -312,13 +333,12 @@ func TOFUHostKeyCallback(addr string, capturedKey *ssh.PublicKey) (ssh.HostKeyCa
var keyErr *knownhosts.KeyError
if errors.As(err, &keyErr) && len(keyErr.Want) == 0 {
line := knownhosts.Line([]string{knownhosts.Normalize(addr)}, key)
path := certpaths.KnownHostsPath()
release, lockErr := security.Flock(path)
release, lockErr := security.Flock(knownHostsPath)
if lockErr != nil {
return fmt.Errorf("tofu lock known_hosts: %w", lockErr)
}
defer release()
existing, readErr := os.ReadFile(path)
existing, readErr := os.ReadFile(knownHostsPath)
if readErr != nil && !os.IsNotExist(readErr) {
return fmt.Errorf("tofu read known_hosts: %w", readErr)
}
@@ -326,7 +346,7 @@ func TOFUHostKeyCallback(addr string, capturedKey *ssh.PublicKey) (ssh.HostKeyCa
existing = append(existing, '\n')
}
updated := append(existing, []byte(line)...)
if writeErr := security.WriteAtomic(path, 0o600, updated); writeErr != nil {
if writeErr := security.WriteAtomic(knownHostsPath, 0o600, updated); writeErr != nil {
return fmt.Errorf("tofu write known_hosts: %w", writeErr)
}
if capturedKey != nil {
+30
View File
@@ -0,0 +1,30 @@
package proxmox
import (
"fmt"
"net"
"testing"
)
// TestREQ157_IPv6JoinHostPort verifies that the proxmox SSH dial
// address is correctly bracketed for IPv6 hosts (REQ-157 / P08 T5/T11).
func TestREQ157_IPv6JoinHostPort(t *testing.T) {
tests := []struct {
host string
port int
want string
}{
{"192.168.1.1", 22, "192.168.1.1:22"},
{"::1", 22, "[::1]:22"},
{"fe80::1", 2222, "[fe80::1]:2222"},
{"2001:db8::1", 22, "[2001:db8::1]:22"},
}
for _, tt := range tests {
t.Run(tt.host, func(t *testing.T) {
got := net.JoinHostPort(tt.host, fmt.Sprintf("%d", tt.port))
if got != tt.want {
t.Errorf("JoinHostPort(%s, %d) = %q, want %q", tt.host, tt.port, got, tt.want)
}
})
}
}
+68 -17
View File
@@ -5,11 +5,13 @@ import (
"context"
"errors"
"fmt"
"io"
"math/rand"
"net"
"os"
"strings"
"sync"
"syscall"
"time"
"golang.org/x/crypto/ssh"
@@ -54,12 +56,14 @@ type Transport struct {
pool sync.Map
// keyPath is the SSH private key path (Ed25519, D-037).
keyPath string
// knownHostsPath is the v0.9 known_hosts path (paths.KnownHostsPath()
// = ClusterDir()/known_hosts). It is stored for the v0.10-P14 migration
// when proxmox.TOFUHostKeyCallback will accept a path parameter; today
// the callback reads certpaths.KnownHostsPath() (the v0.8 flat layout)
// directly, so this field is not yet read by dial(). Tests set
// $ORCA_HOME so certpaths.KnownHostsPath() resolves under the temp dir.
// knownHostsPath is the known_hosts path passed to the TOFU
// host-key callback (D-035). NewTransport sets it from
// certpaths.KnownHostsPath() (v0.8 flat layout) by default; callers
// that want the v0.9 paths.KnownHostsPath() location construct the
// transport with that path explicitly. REQ-157 / P08 T4: this field
// IS read by dial() (via proxmox.TOFUHostKeyCallbackPath) — the
// earlier bug where the callback ignored it and read
// certpaths.KnownHostsPath() directly is fixed.
knownHostsPath string
// user is the remote SSH user (default "orca", D-037).
user string
@@ -120,15 +124,14 @@ func (defaultSSHDialer) DialContext(ctx context.Context, network, addr string, c
}
// NewTransport returns a Transport configured with the given SSH
// private key path and known_hosts path. The known_hosts path is the v0.9
// location (paths.KnownHostsPath); it is stored for the v0.10-P14
// migration when the TOFU callback will accept a path parameter. Today
// dial() delegates host-key verification to proxmox.TOFUHostKeyCallback,
// which reads certpaths.KnownHostsPath() (the v0.8 flat layout under
// $ORCA_HOME) directly — so callers must ensure $ORCA_HOME points at the
// cluster root (the CLI sets this up). The remote user defaults to
// "orca" (D-037); override with SetUser. The dialer defaults to the
// real ssh.Dial-based dialer; tests call SetDialer to inject a mock.
// private key path and known_hosts path. The known_hosts path is read
// by dial() via proxmox.TOFUHostKeyCallbackPath (D-035, REQ-157/P08 T4):
// the TOFU callback locks/captures against this path on first connect.
// Callers typically pass certpaths.KnownHostsPath() (the v0.8 flat
// layout under $ORCA_HOME) or paths.KnownHostsPath() (the v0.9
// ClusterDir() location). The remote user defaults to "orca" (D-037);
// override with SetUser. The dialer defaults to the real ssh.Dial-based
// dialer; tests call SetDialer to inject a mock.
func NewTransport(keyPath, knownHostsPath string) *Transport {
return &Transport{
keyPath: keyPath,
@@ -193,7 +196,14 @@ func (t *Transport) dial(peer string) (*ssh.Client, error) {
// Host-key verification reuses the v0.8 TOFU wrapper (D-035). The
// known_hosts file is flock-protected inside the callback on
// first-connect capture, so we do NOT re-lock here.
cb, err := proxmox.TOFUHostKeyCallback(peer, nil)
//
// REQ-157 / P08 T4: use the stored knownHostsPath field (set via
// NewTransport from certpaths.KnownHostsPath() / paths.KnownHostsPath())
// instead of having the callback read certpaths.KnownHostsPath() (the
// v0.8 flat layout) directly. This closes the bug where the flock
// field was stored but never read by dial() — the TOFU callback now
// locks/captures against the path the transport was constructed with.
cb, err := proxmox.TOFUHostKeyCallbackPath(t.knownHostsPath, peer, nil)
if err != nil {
return nil, fmt.Errorf("sshpush: host-key callback: %w", err)
}
@@ -391,7 +401,14 @@ func backoff(initial, max time.Duration, n int) time.Duration {
}
// isTransient reports whether err looks like a transient failure worth
// retrying (mirrors v0.8 transport.IsTransient, reimplemented here).
// retrying (mirrors transport.IsTransient, reimplemented here so
// internal/sshpush does not import internal/transport).
//
// REQ-157 / P08 T2: classification is TYPE-BASED, not substring-based.
// The primary path is errors.Is against the sentinels (ErrTransient /
// ErrPermanent) and against well-known syscall/net/io errors. The
// substring fallback is retained ONLY for unwrapped errors from the
// ssh.Dialer that do not implement the standard interfaces.
func isTransient(err error) bool {
if err == nil {
return false
@@ -402,6 +419,29 @@ func isTransient(err error) bool {
if errors.Is(err, ErrPermanent) {
return false
}
// Typed: a net.Error that is a timeout is transient; a net.OpError
// whose Temporary() is true (ECONNREFUSED et al) is transient.
var netErr net.Error
if errors.As(err, &netErr) {
if netErr.Timeout() {
return true
}
return isTemporarySSH(netErr)
}
if errors.Is(err, syscall.ECONNREFUSED) ||
errors.Is(err, syscall.ECONNRESET) ||
errors.Is(err, syscall.ETIMEDOUT) ||
errors.Is(err, syscall.EHOSTUNREACH) ||
errors.Is(err, syscall.ENETUNREACH) {
return true
}
if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
return true
}
if errors.Is(err, context.DeadlineExceeded) {
return true
}
// Substring fallback (defense-in-depth for unwrapped errors).
s := err.Error()
for _, sub := range []string{
"connection refused", "i/o timeout", "EOF",
@@ -415,6 +455,17 @@ func isTransient(err error) bool {
return false
}
// isTemporarySSH reports whether netErr implements the legacy
// Temporary() bool method and it returns true. net.OpError.Temporary()
// maps to the underlying errno's temporary classification.
func isTemporarySSH(netErr net.Error) bool {
type temporary interface{ Temporary() bool }
if t, ok := netErr.(temporary); ok {
return t.Temporary()
}
return false
}
// classifyDialErr converts a raw ssh.Dial error into a transport error
// (transient vs permanent). Auth failures and host-key mismatches are
// permanent; everything else is transient.
+81 -23
View File
@@ -8,7 +8,11 @@ package transport
import (
"context"
"errors"
"io"
"math/rand"
"net"
"strings"
"syscall"
"time"
)
@@ -34,29 +38,6 @@ func DefaultRetryPolicy() RetryPolicy {
return RetryPolicy{Initial: RetryInitial, Max: RetryMax, MaxAttempts: RetryMaxAttempts}
}
// IsTransient reports whether err looks like a transient failure
// worth retrying. We treat network errors, context-deadline-exceeded
// (peer was slow but reachable), and a sentinel ErrTransient as
// retryable; everything else (4xx, validation, auth) is permanent.
func IsTransient(err error) bool {
if err == nil {
return false
}
if errors.Is(err, ErrTransient) {
return true
}
// We avoid pulling net/error here to keep dependencies minimal;
// the most common transient signature is the substring "connection
// refused" or "i/o timeout". Tests assert these explicitly.
s := err.Error()
for _, sub := range []string{"connection refused", "i/o timeout", "EOF", "no such host", "connection reset"} {
if contains(s, sub) {
return true
}
}
return false
}
// ErrTransient is a sentinel callers can wrap to mark an error
// retryable. ErrPermanent is the opposite.
var (
@@ -64,6 +45,83 @@ var (
ErrPermanent = errors.New("permanent error")
)
// IsTransient reports whether err looks like a transient failure
// worth retrying. We treat network errors, context-deadline-exceeded
// (peer was slow but reachable), and a sentinel ErrTransient as
// retryable; everything else (4xx, validation, auth) is permanent.
//
// REQ-157 / P08 T1: classification is TYPE-BASED, not substring-based.
// The primary path is errors.Is against the sentinels (ErrTransient /
// ErrPermanent) and against well-known syscall/net/io errors. The
// substring fallback is retained ONLY for unwrapped errors from
// third-party dialers that do not implement the standard interfaces
// (defense-in-depth); callers SHOULD wrap with ErrTransient instead.
func IsTransient(err error) bool {
if err == nil {
return false
}
// Explicit sentinels win.
if errors.Is(err, ErrTransient) {
return true
}
if errors.Is(err, ErrPermanent) {
return false
}
// Typed classification: a net.Error that is a timeout is transient.
var netErr net.Error
if errors.As(err, &netErr) {
if netErr.Timeout() {
return true
}
// net.OpError implements Temporary(); that maps to the
// underlying errno's temporary classification (ECONNREFUSED et
// al). We keep the check so a plain "dial tcp: connection
// refused" classifies as transient.
return isTemporary(netErr)
}
// Specific syscall errors that are universally retryable.
if errors.Is(err, syscall.ECONNREFUSED) ||
errors.Is(err, syscall.ECONNRESET) ||
errors.Is(err, syscall.ETIMEDOUT) ||
errors.Is(err, syscall.EHOSTUNREACH) ||
errors.Is(err, syscall.ENETUNREACH) {
return true
}
// io.EOF on a read from a half-closed peer is transient (the
// dispatch HTTP/2 path can surface this mid-stream).
if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
return true
}
// context.DeadlineExceeded from a slow-but-reachable peer is
// transient (the next attempt may succeed under a fresh deadline).
if errors.Is(err, context.DeadlineExceeded) {
return true
}
// Substring fallback (defense-in-depth for unwrapped errors).
s := err.Error()
for _, sub := range []string{
"connection refused", "i/o timeout", "EOF",
"no such host", "connection reset",
"deadline exceeded", "temporarily unavailable",
} {
if strings.Contains(s, sub) {
return true
}
}
return false
}
// isTemporary reports whether netErr implements the legacy Temporary()
// bool method and it returns true. net.OpError.Temporary() maps to the
// underlying errno's temporary classification (ECONNREFUSED et al).
func isTemporary(netErr net.Error) bool {
type temporary interface{ Temporary() bool }
if t, ok := netErr.(temporary); ok {
return t.Temporary()
}
return false
}
// RetryableFunc is the signature Retry calls. It returns the result
// and an error. The bool indicates whether the call is idempotent
// (true = safe to retry without an idempotency key).
+49
View File
@@ -0,0 +1,49 @@
package transport
import (
"context"
"errors"
"io"
"net"
"testing"
"fmt"
)
// TestREQ157_TypedErrorClassification verifies that IsTransient uses
// typed sentinels and standard interfaces, not substring matching
// (REQ-157 / P08 T10).
func TestREQ157_TypedErrorClassification(t *testing.T) {
tests := []struct {
name string
err error
want bool
}{
{"nil", nil, false},
{"ErrTransient", ErrTransient, true},
{"wrapped ErrTransient", fmt.Errorf("dial: %w", ErrTransient), true},
{"ErrPermanent", ErrPermanent, false},
{"wrapped ErrPermanent", fmt.Errorf("auth: %w", ErrPermanent), false},
{"net timeout", &net.OpError{Op: "dial", Net: "tcp", Err: &timeoutError{}}, true},
{"context deadline", context.DeadlineExceeded, true},
{"context canceled", context.Canceled, false},
{"io EOF", io.EOF, true},
{"plain error", errors.New("some permanent error"), false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := IsTransient(tt.err)
if got != tt.want {
t.Errorf("IsTransient(%v) = %v, want %v", tt.err, got, tt.want)
}
})
}
}
type timeoutError struct{}
func (timeoutError) Error() string { return "i/o timeout" }
func (timeoutError) Timeout() bool { return true }
func (timeoutError) Temporary() bool { return true }
var _ = fmt.Errorf