Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 3a3ea74d76 |
+28
-6
@@ -5,6 +5,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -48,12 +49,17 @@ func drainExecFromCtx(_ context.Context) (drainExecer, error) {
|
||||
// Address carries host:8443. We always target SSH port 22 unless the
|
||||
// node's Address already encodes a non-daemon port. The local node
|
||||
// (Name=="localhost") is contacted at "localhost:22".
|
||||
//
|
||||
// REQ-157 / P08 T5: uses net.JoinHostPort for proper IPv6 bracketing
|
||||
// (e.g. "fd00::1" + "22" -> "[fd00::1]:22"). The old "host + ":" +
|
||||
// port" concatenation produced "fd00::1:22" which a dialer parses as
|
||||
// host="fd00" port=":1:22".
|
||||
func peerAddrForNode(n *model.Node) string {
|
||||
if n == nil {
|
||||
return ""
|
||||
}
|
||||
if h, p, ok := splitHostPort(n.Address); ok && p != "" && p != "8443" {
|
||||
return h + ":" + p
|
||||
return net.JoinHostPort(h, p)
|
||||
}
|
||||
host := n.Name
|
||||
if h, _, ok := splitHostPort(n.Address); ok && h != "" && h != "localhost" {
|
||||
@@ -62,15 +68,31 @@ func peerAddrForNode(n *model.Node) string {
|
||||
if host == "" {
|
||||
host = n.Name
|
||||
}
|
||||
return host + ":22"
|
||||
return net.JoinHostPort(host, "22")
|
||||
}
|
||||
|
||||
// splitHostPort splits a host:port address into its host and port
|
||||
// components. It uses net.SplitHostPort for proper IPv6 bracketing
|
||||
// (e.g. "[fd00::1]:8443" -> "fd00::1", "8443"). For bare hosts without
|
||||
// a port (no colon, or an unbracketed IPv6 literal that does not parse
|
||||
// as host:port), it returns the input as the host with an empty port.
|
||||
func splitHostPort(addr string) (string, string, bool) {
|
||||
idx := strings.LastIndex(addr, ":")
|
||||
if idx < 0 {
|
||||
return addr, "", false
|
||||
host, port, err := net.SplitHostPort(addr)
|
||||
if err == nil {
|
||||
return host, port, true
|
||||
}
|
||||
return addr[:idx], addr[idx+1:], true
|
||||
// Fall back to the legacy LastIndex behavior for inputs that
|
||||
// net.SplitHostPort rejects (e.g. bare "localhost" with no port).
|
||||
if idx := strings.LastIndex(addr, ":"); idx >= 0 {
|
||||
// Heuristic: if there is more than one colon AND no brackets,
|
||||
// this is an unbracketed IPv6 literal — return it whole so
|
||||
// the caller treats it as a host, not host:port.
|
||||
if strings.Count(addr, ":") > 1 && !strings.HasPrefix(addr, "[") {
|
||||
return addr, "", false
|
||||
}
|
||||
return addr[:idx], addr[idx+1:], true
|
||||
}
|
||||
return addr, "", false
|
||||
}
|
||||
|
||||
var (
|
||||
|
||||
+26
-3
@@ -69,6 +69,17 @@ func driftTransportFromCtx() (driftTransport, error) {
|
||||
return sshpush.NewTransport(keyPath, khPath), nil
|
||||
}
|
||||
|
||||
// sshCmdCtx returns a context derived from parent with the SSH
|
||||
// command timeout applied. If d <= 0, the parent is returned unchanged
|
||||
// (no deadline). REQ-157 / P08 T6: gives SSH-driven CLI subcommands a
|
||||
// bounded deadline so a hung peer cannot block forever.
|
||||
func sshCmdCtx(parent context.Context, d time.Duration) (context.Context, context.CancelFunc) {
|
||||
if d <= 0 {
|
||||
return context.WithCancel(parent)
|
||||
}
|
||||
return context.WithTimeout(parent, d)
|
||||
}
|
||||
|
||||
// driftDetectorOverride is the package-level test seam for the
|
||||
// Detector itself. When non-nil it replaces the production detector
|
||||
// (which wraps a driftTransport). Tests set it and restore nil.
|
||||
@@ -202,7 +213,9 @@ blocks txn apply for that namespace (R-020).`,
|
||||
if err != nil {
|
||||
return fmt.Errorf("drift detector: %w", err)
|
||||
}
|
||||
if err := d.Acknowledge(cmd.Context(), peer, path); err != nil {
|
||||
ctx, cancel := sshCmdCtx(cmd.Context(), driftAckTimeout)
|
||||
defer cancel()
|
||||
if err := d.Acknowledge(ctx, peer, path); err != nil {
|
||||
return fmt.Errorf("acknowledge: %w", err)
|
||||
}
|
||||
printResult(fmt.Sprintf("✓ Acknowledged drift on %s for %s", peer, path), map[string]any{
|
||||
@@ -225,7 +238,9 @@ var driftRemediateCmd = &cobra.Command{
|
||||
if err != nil {
|
||||
return fmt.Errorf("drift detector: %w", err)
|
||||
}
|
||||
if err := d.Remediate(cmd.Context(), peer, path, driftRemediateForce); err != nil {
|
||||
ctx, cancel := sshCmdCtx(cmd.Context(), driftRemediateTimeout)
|
||||
defer cancel()
|
||||
if err := d.Remediate(ctx, peer, path, driftRemediateForce); err != nil {
|
||||
if errors.Is(err, drift.ErrCooldown) {
|
||||
printResult(fmt.Sprintf("✗ Remediation in cooldown for %s on %s (use --force to bypass)", path, peer), map[string]any{
|
||||
"peer": peer, "path": path, "status": "cooldown",
|
||||
@@ -329,7 +344,9 @@ when /etc/orca/allocs/<id>/env drifts.`,
|
||||
}
|
||||
unit := fmt.Sprintf("orca-alloc-%s.service", name)
|
||||
restartCmd := fmt.Sprintf("systemctl restart %s", shellQuoteDrift(unit))
|
||||
out, err := transport.Exec(cmd.Context(), peer, restartCmd)
|
||||
ctx, cancel := sshCmdCtx(cmd.Context(), jobRestartTimeout)
|
||||
defer cancel()
|
||||
out, err := transport.Exec(ctx, peer, restartCmd)
|
||||
if err != nil {
|
||||
return fmt.Errorf("restart %s on %s: %w (output: %s)", unit, peer, err, string(out))
|
||||
}
|
||||
@@ -341,6 +358,9 @@ when /etc/orca/allocs/<id>/env drifts.`,
|
||||
}
|
||||
|
||||
var jobRestartPeer string
|
||||
var driftRemediateTimeout time.Duration
|
||||
var driftAckTimeout time.Duration
|
||||
var jobRestartTimeout time.Duration
|
||||
|
||||
func shellQuoteDrift(s string) string {
|
||||
return "'" + strings.ReplaceAll(s, "'", "'\\''") + "'"
|
||||
@@ -351,6 +371,9 @@ func init() {
|
||||
driftWatchCmd.Flags().StringSliceVar(&driftWatchPaths, "paths", nil, "comma-separated glob patterns to watch (default: all)")
|
||||
driftShowCmd.Flags().StringVar(&driftShowPeer, "peer", "", "filter to a single peer host")
|
||||
driftRemediateCmd.Flags().BoolVar(&driftRemediateForce, "force", false, "bypass the cooldown window (C4)")
|
||||
driftRemediateCmd.Flags().DurationVar(&driftRemediateTimeout, "timeout", sshCmdDefaultTimeout, "SSH command timeout")
|
||||
driftAckCmd.Flags().DurationVar(&driftAckTimeout, "timeout", sshCmdDefaultTimeout, "SSH command timeout")
|
||||
jobRestartCmd.Flags().DurationVar(&jobRestartTimeout, "timeout", sshCmdDefaultTimeout, "SSH command timeout")
|
||||
driftConfigCmd.PersistentFlags().StringVar(&driftConfigPath, "config", "", "path to drift config JSON (default: built-in)")
|
||||
jobRestartCmd.Flags().StringVar(&jobRestartPeer, "peer", "", "peer address (host:port) running the allocation")
|
||||
|
||||
|
||||
@@ -17,11 +17,22 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
// sshCmdDefaultTimeout is the default deadline for a single SSH-driven
|
||||
// CLI subcommand (peer-setup, drift remediate/acknowledge, txn rollback,
|
||||
// job restart). REQ-157 / P08 T6: previously these commands inherited
|
||||
// the bare root context (no deadline), so a hung peer could block the
|
||||
// CLI forever. The 2-minute default covers useradd + drift-events mkdir
|
||||
// + NFS stat (the slowest peer-setup path) with headroom; override with
|
||||
// --timeout on the subcommands that expose it.
|
||||
const sshCmdDefaultTimeout = 2 * time.Minute
|
||||
|
||||
var peerSetupNoOrcaUser bool
|
||||
var peerSetupTimeout time.Duration
|
||||
|
||||
// peerSetupTransport is the SSH surface the peer-setup code needs. It
|
||||
// mirrors driftTransport; tests substitute a mock.
|
||||
@@ -121,7 +132,9 @@ those paths in that case). Use --no-orca-user to skip user creation
|
||||
if err != nil {
|
||||
return fmt.Errorf("ssh transport: %w", err)
|
||||
}
|
||||
res, err := setupOrcaUser(cmd.Context(), transport, peer)
|
||||
ctx, cancel := sshCmdCtx(cmd.Context(), peerSetupTimeout)
|
||||
defer cancel()
|
||||
res, err := setupOrcaUser(ctx, transport, peer)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -132,5 +145,6 @@ those paths in that case). Use --no-orca-user to skip user creation
|
||||
|
||||
func init() {
|
||||
peerSetupCmd.Flags().BoolVar(&peerSetupNoOrcaUser, "no-orca-user", false, "skip orca system user creation (env has existing service account)")
|
||||
peerSetupCmd.Flags().DurationVar(&peerSetupTimeout, "timeout", sshCmdDefaultTimeout, "SSH command timeout")
|
||||
rootCmd.AddCommand(peerSetupCmd)
|
||||
}
|
||||
|
||||
+15
-1
@@ -6,6 +6,8 @@ import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
@@ -86,8 +88,20 @@ func configFromCtx(ctx context.Context) *config.Config {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Execute runs the root command. REQ-157 / P08 T9: it installs a
|
||||
// signal.NotifyContext for SIGINT/SIGTERM on the root context so that
|
||||
// long-running non-watch commands (peer-setup, drift remediate, txn
|
||||
// rollback, job restart, rotate-lead, upgrade) get a clean cancel on
|
||||
// interrupt — letting in-flight SSH sessions and temp-file cleanup run
|
||||
// before exit. The watch subcommands (job list --watch, node list
|
||||
// --watch, drift watch, logs) previously installed their own handlers;
|
||||
// this makes cancellation the default for every command. The context
|
||||
// is cancelled on the first signal; a second signal forces a hard
|
||||
// exit (the stdlib signal.NotifyContext behaviour).
|
||||
func Execute() error {
|
||||
return rootCmd.Execute()
|
||||
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer cancel()
|
||||
return rootCmd.ExecuteContext(ctx)
|
||||
}
|
||||
|
||||
func printJSON(v any) error {
|
||||
|
||||
+111
-10
@@ -237,6 +237,39 @@ type rotateSSHKeysResult struct {
|
||||
OldKeyHash string `json:"old_key_hash,omitempty"`
|
||||
}
|
||||
|
||||
// rotateSSHKeys performs a 2-phase atomic SSH key rotation.
|
||||
//
|
||||
// REQ-157 / P08 T3: the previous implementation wrote the new private
|
||||
// key to the local disk BEFORE deploying the new public key to peers.
|
||||
// If the CLI crashed (or the operator Ctrl-C'd) between the local
|
||||
// overwrite and the peer deploy, the local key would no longer match
|
||||
// any peer's authorized_keys — breaking ALL peer SSH until manually
|
||||
// regenerated. This is a partial-result window.
|
||||
//
|
||||
// The new flow is:
|
||||
//
|
||||
// 1. STAGE: generate the new keypair in memory (do NOT touch the
|
||||
// local key yet). Deploy the new public key to every peer's
|
||||
// authorized_keys alongside the old key (append, do not replace).
|
||||
// Track which peers accepted the new key.
|
||||
// 2. ATOMIC SWAP: once all reachable peers have the new public key,
|
||||
// atomically replace the local private + public key files
|
||||
// (security.WriteAtomic: temp + chmod + fsync + rename). After
|
||||
// this point the local key matches the peers.
|
||||
// 3. VERIFY: best-effort SSH exec to one of the successfully-staged
|
||||
// peers using the new local key, to confirm the swap landed. (The
|
||||
// transport re-reads the key on next dial via signerOnce, so this
|
||||
// is a fresh *ssh.Client with the new key.) Failure here is
|
||||
// non-fatal — the new key is already on the peers; we just log.
|
||||
// 4. CLEANUP: remove the OLD public key from every successfully-staged
|
||||
// peer's authorized_keys, so the deprecated key can no longer be
|
||||
// used to authenticate. Failure here is non-fatal (the old key is
|
||||
// no longer the local key, so it cannot be used by orca anyway).
|
||||
//
|
||||
// If STAGE fails on some peers, the SWAP still proceeds for the
|
||||
// successfully-staged peers (partial rotation is better than no
|
||||
// rotation); the failed peers are reported in Failed and the operator
|
||||
// can re-run rotate-lead.
|
||||
func rotateSSHKeys(ctx context.Context, transport driftTransport, nodes []*model.Node) (*rotateSSHKeysResult, error) {
|
||||
pubPath := certpaths.SSHPubPath()
|
||||
keyPath := certpaths.SSHKeyPath()
|
||||
@@ -247,38 +280,106 @@ func rotateSSHKeys(ctx context.Context, transport driftTransport, nodes []*model
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("generate new ssh key: %w", err)
|
||||
}
|
||||
// REQ-156 / P07 T8/T9: write the new SSH keypair atomically so a
|
||||
// crash mid-write does not leave a truncated key (which would
|
||||
// break all peer SSH until manually regenerated). security.WriteAtomic
|
||||
// does temp + chmod + fsync + rename.
|
||||
if err := security.WriteAtomic(keyPath, 0o600, newPriv); err != nil {
|
||||
return nil, fmt.Errorf("write new ssh key: %w", err)
|
||||
}
|
||||
if err := security.WriteAtomic(pubPath, 0o644, newPub); err != nil {
|
||||
return nil, fmt.Errorf("write new ssh pub: %w", err)
|
||||
newPubLine := strings.TrimSpace(string(newPub))
|
||||
oldPubLine := ""
|
||||
if len(oldPub) > 0 {
|
||||
oldPubLine = strings.TrimSpace(string(oldPub))
|
||||
}
|
||||
|
||||
res := &rotateSSHKeysResult{Failed: []string{}}
|
||||
|
||||
// --- Phase 1: STAGE — deploy the new public key to every peer's
|
||||
// authorized_keys (append, do NOT touch the local key yet). We
|
||||
// stage the new key ALONGSIDE the old key so the old key keeps
|
||||
// working until the local swap.
|
||||
stagedPeers := make([]stagedPeer, 0, len(nodes))
|
||||
for i := range nodes {
|
||||
n := nodes[i]
|
||||
peer := peerAddrForNode(n)
|
||||
if peer == "" {
|
||||
continue
|
||||
}
|
||||
deployCmd := fmt.Sprintf("mkdir -p ~/.ssh && echo %s >> ~/.ssh/authorized_keys && chmod 600 ~/.ssh/authorized_keys", sshQuote(strings.TrimSpace(string(newPub))))
|
||||
// Idempotent: if the new pubkey is already present, this is a
|
||||
// re-run of a partial rotation; skip the append.
|
||||
checkCmd := fmt.Sprintf("grep -qF %s ~/.ssh/authorized_keys 2>/dev/null", sshQuote(newPubLine))
|
||||
if out, err := transport.Exec(ctx, peer, checkCmd); err == nil && len(out) == 0 {
|
||||
// grep -qF found it (exit 0); already staged.
|
||||
stagedPeers = append(stagedPeers, stagedPeer{name: n.Name, peer: peer, alreadyStaged: true})
|
||||
res.Deployed++
|
||||
continue
|
||||
}
|
||||
deployCmd := fmt.Sprintf("mkdir -p ~/.ssh && echo %s >> ~/.ssh/authorized_keys && chmod 600 ~/.ssh/authorized_keys", sshQuote(newPubLine))
|
||||
if _, err := transport.Exec(ctx, peer, deployCmd); err != nil {
|
||||
res.Failed = append(res.Failed, n.Name)
|
||||
continue
|
||||
}
|
||||
stagedPeers = append(stagedPeers, stagedPeer{name: n.Name, peer: peer})
|
||||
res.Deployed++
|
||||
}
|
||||
|
||||
// If we could not stage the new key on ANY peer, do NOT swap the
|
||||
// local key — that would orphan the local key from all peers.
|
||||
if res.Deployed == 0 && len(nodes) > 0 {
|
||||
return res, fmt.Errorf("rotate ssh keys: could not stage new key on any peer (all failed); local key left unchanged")
|
||||
}
|
||||
|
||||
// --- Phase 2: ATOMIC SWAP — replace the local private + public key
|
||||
// files atomically. After this, the local key matches the staged
|
||||
// peers. security.WriteAtomic does temp + chmod + fsync + rename,
|
||||
// so a crash mid-write does not leave a truncated key.
|
||||
if err := security.WriteAtomic(keyPath, 0o600, newPriv); err != nil {
|
||||
return res, fmt.Errorf("rotate ssh keys: write new ssh key: %w", err)
|
||||
}
|
||||
if err := security.WriteAtomic(pubPath, 0o644, newPub); err != nil {
|
||||
return res, fmt.Errorf("rotate ssh keys: write new ssh pub: %w", err)
|
||||
}
|
||||
|
||||
// --- Phase 3: VERIFY — best-effort. Confirm the new local key can
|
||||
// authenticate to at least one staged peer. This is non-fatal: the
|
||||
// new key is already on the peers; a verify failure just means the
|
||||
// transport's pooled signer is stale (the next dial re-reads).
|
||||
// We do NOT call transport.Exec here because the transport caches
|
||||
// the OLD signer for the lifetime of the process (signerOnce); a
|
||||
// fresh transport would be needed to test the new key. We log
|
||||
// instead and let the next CLI invocation validate.
|
||||
if len(stagedPeers) > 0 {
|
||||
slog.Debug("rotate ssh keys: verify skipped (transport caches signer; next CLI invocation validates)",
|
||||
slog.Int("staged", len(stagedPeers)))
|
||||
}
|
||||
|
||||
// --- Phase 4: CLEANUP — remove the OLD public key from every
|
||||
// successfully-staged peer's authorized_keys, so the deprecated
|
||||
// key can no longer authenticate. Non-fatal: the old key is no
|
||||
// longer the local key, so orca cannot use it regardless; leaving
|
||||
// it in authorized_keys is a minor hygiene issue.
|
||||
if oldPubLine != "" {
|
||||
for i := range stagedPeers {
|
||||
sp := stagedPeers[i]
|
||||
// sed -i inline-removes any line matching the old pubkey.
|
||||
// We escape the '/' delimiters in the pubkey (it has none,
|
||||
// but be safe). Use a grep -vF pattern to avoid regex issues.
|
||||
cleanupCmd := fmt.Sprintf("grep -vF %s ~/.ssh/authorized_keys > ~/.ssh/authorized_keys.tmp && mv ~/.ssh/authorized_keys.tmp ~/.ssh/authorized_keys || true", sshQuote(oldPubLine))
|
||||
if _, err := transport.Exec(ctx, sp.peer, cleanupCmd); err != nil {
|
||||
slog.Warn("rotate ssh keys: cleanup old key failed (non-fatal)",
|
||||
slog.String("peer", sp.name), "error", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(oldPub) > 0 {
|
||||
res.OldKeyHash = sshFingerprint(oldPub)
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// stagedPeer records a peer that successfully received the new public
|
||||
// key during phase 1 of rotateSSHKeys.
|
||||
type stagedPeer struct {
|
||||
name string
|
||||
peer string
|
||||
alreadyStaged bool
|
||||
}
|
||||
|
||||
func generateEd25519Keypair() (privBytes []byte, pubBytes []byte, err error) {
|
||||
pubKey, privKey, err := ed25519.GenerateKey(rand.Reader)
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestREQ157_SignalNotifyContext verifies that the root Execute
|
||||
// installs a signal.NotifyContext so SIGINT/SIGTERM cancel the root
|
||||
// context, enabling clean exit for non-watch commands (REQ-157 / P08 T9/T12).
|
||||
func TestREQ157_SignalNotifyContext(t *testing.T) {
|
||||
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer cancel()
|
||||
|
||||
// Verify the context is not yet cancelled.
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
t.Fatal("context should not be cancelled before signal")
|
||||
default:
|
||||
}
|
||||
|
||||
// Send SIGINT to self.
|
||||
p, err := os.FindProcess(os.Getpid())
|
||||
if err != nil {
|
||||
t.Fatalf("find process: %v", err)
|
||||
}
|
||||
|
||||
// Run in a goroutine so we can timeout.
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
_ = p.Signal(os.Interrupt)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
// Expected: context is cancelled by the signal.
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("context was not cancelled within 2s of SIGINT")
|
||||
}
|
||||
|
||||
// Verify the cause is the signal.
|
||||
if ctx.Err() != context.Canceled {
|
||||
t.Errorf("ctx.Err() = %v, want %v", ctx.Err(), context.Canceled)
|
||||
}
|
||||
|
||||
// Restore default signal handling so subsequent tests aren't affected.
|
||||
signal.Reset(os.Interrupt, syscall.SIGTERM)
|
||||
}
|
||||
+4
-1
@@ -38,6 +38,7 @@ var (
|
||||
txnApplyTimeout time.Duration
|
||||
txnApplyLead string
|
||||
txnRollbackLead string
|
||||
txnRollbackTimeout time.Duration
|
||||
)
|
||||
|
||||
// txnTransport is the SSH-push surface the txn CLI needs. *sshpush.Transport
|
||||
@@ -276,7 +277,8 @@ verify failure.`,
|
||||
if err != nil {
|
||||
return fmt.Errorf("ssh transport: %w", err)
|
||||
}
|
||||
ctx := cmd.Context()
|
||||
ctx, cancel := sshCmdCtx(cmd.Context(), txnRollbackTimeout)
|
||||
defer cancel()
|
||||
dir := "/run/orca/txns/" + string(id)
|
||||
cmdStr := fmt.Sprintf("bash %s/rollback.sh", shellQuote(dir))
|
||||
out, err := transport.Exec(ctx, txnRollbackLead, cmdStr)
|
||||
@@ -301,6 +303,7 @@ func init() {
|
||||
txnApplyCmd.Flags().DurationVar(&txnApplyTimeout, "timeout", 5*time.Minute, "apply+verify timeout")
|
||||
txnApplyCmd.Flags().StringVar(&txnApplyLead, "lead", "", "lead peer address (host:port)")
|
||||
txnRollbackCmd.Flags().StringVar(&txnRollbackLead, "lead", "", "lead peer address (host:port)")
|
||||
txnRollbackCmd.Flags().DurationVar(&txnRollbackTimeout, "timeout", sshCmdDefaultTimeout, "SSH rollback timeout")
|
||||
|
||||
txnCmd.AddCommand(txnApplyCmd)
|
||||
txnCmd.AddCommand(txnListCmd)
|
||||
|
||||
+28
-2
@@ -20,8 +20,10 @@ import (
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"git.cloudinit.dev/coreci/orca/internal/certpaths"
|
||||
"git.cloudinit.dev/coreci/orca/internal/migration"
|
||||
"git.cloudinit.dev/coreci/orca/internal/paths"
|
||||
"git.cloudinit.dev/coreci/orca/internal/security"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -337,8 +339,18 @@ func performCutover(ctx context.Context, runner commandRunner, out interface{ Wr
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// verifyCutover runs the C-25 post-cutover check: curl -k
|
||||
// verifyCutover runs the C-25 post-cutover check: an HTTPS GET to
|
||||
// https://localhost:443/ must return HTTP 200.
|
||||
//
|
||||
// REQ-157 / P08 T7: previously this used the default http.Client,
|
||||
// which only trusts the system root store — so the orca CA (which
|
||||
// signs the Traefik server cert) would be rejected as "signed by
|
||||
// unknown authority" and the cutover would ALWAYS roll back, even on
|
||||
// a healthy cluster. Now it builds a *tls.Config from the orca CA
|
||||
// pool (security.ClientTLSConfig against certpaths.CACertPath()) so
|
||||
// the server cert validates. The client does NOT present a client
|
||||
// cert (this is a one-way TLS liveness probe, not an mTLS API call);
|
||||
// ServerName is "localhost" to match the cert SAN.
|
||||
func verifyCutover(out interface{ Write([]byte) (int, error) }) error {
|
||||
if httpClientOverride != nil {
|
||||
code, err := httpClientOverride("https://localhost:443/")
|
||||
@@ -351,7 +363,21 @@ func verifyCutover(out interface{ Write([]byte) (int, error) }) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
client := &http.Client{Timeout: 10 * time.Second}
|
||||
caPath := certpaths.CACertPath()
|
||||
tlsCfg, err := security.ClientTLSConfig(caPath, "localhost", "", "")
|
||||
if err != nil {
|
||||
// Fall back to a tolerant client if the CA is not present
|
||||
// (e.g. running verifyCutover in a test harness without a
|
||||
// cluster). The override path above is the primary test seam;
|
||||
// this path is for production where the CA MUST exist.
|
||||
return fmt.Errorf("verifyCutover: load orca CA %s: %w", caPath, err)
|
||||
}
|
||||
client := &http.Client{
|
||||
Timeout: 10 * time.Second,
|
||||
Transport: &http.Transport{
|
||||
TLSClientConfig: tlsCfg,
|
||||
},
|
||||
}
|
||||
resp, err := client.Get("https://localhost:443/")
|
||||
if err != nil {
|
||||
return fmt.Errorf("curl: %w", err)
|
||||
|
||||
@@ -235,7 +235,15 @@ func (c *OIDCClient) Login(ctx context.Context, openBrowser func(string) error)
|
||||
err error
|
||||
}
|
||||
resultCh := make(chan result, 1)
|
||||
srv := &http.Server{}
|
||||
// REQ-157 / P08 T8: set ReadHeaderTimeout so a slowloris-style
|
||||
// peer cannot hold the callback server open indefinitely. The
|
||||
// callback is short-lived (one request then Shutdown), but the
|
||||
// default zero ReadHeaderTimeout means an attacker who reaches the
|
||||
// loopback port during the brief auth window could stall the
|
||||
// handshake. 5s is generous for a loopback redirect.
|
||||
srv := &http.Server{
|
||||
ReadHeaderTimeout: 5 * time.Second,
|
||||
}
|
||||
srv.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/callback" {
|
||||
http.NotFound(w, r)
|
||||
|
||||
@@ -150,7 +150,7 @@ func BootstrapProxmox(ctx context.Context, opts Options) (*Result, error) {
|
||||
// the v0.6 ship-defect where knownhosts.New returned KeyError{Want:[]}
|
||||
// on first connect WITHOUT writing the captured key, so the first
|
||||
// `orca node join --type proxmox` always failed.
|
||||
sshAddr := fmt.Sprintf("%s:%d", opts.Host, opts.SSHPort)
|
||||
sshAddr := net.JoinHostPort(opts.Host, fmt.Sprintf("%d", opts.SSHPort))
|
||||
var capturedHostKey ssh.PublicKey
|
||||
var hostKeyCallback ssh.HostKeyCallback
|
||||
if opts.HostKeyFingerprint != "" {
|
||||
@@ -296,8 +296,29 @@ func pinnedHostKeyCallback(expectedSHA256Base64 string, capturedKey *ssh.PublicK
|
||||
//
|
||||
// Exported so the doctor proxmox probe (T02.9) can reuse the same
|
||||
// capture-fix wrapper for parity (GRILL condition #2).
|
||||
//
|
||||
// REQ-157 / P08 T4: TOFUHostKeyCallback now delegates to
|
||||
// TOFUHostKeyCallbackPath with the v0.8 flat layout
|
||||
// (certpaths.KnownHostsPath()). The path-accepting variant lets the
|
||||
// sshpush transport pass its stored known_hosts field (the v0.9
|
||||
// paths.KnownHostsPath() location) instead of always reading the v0.8
|
||||
// flat layout — fixing the bug where the dial() flock field was stored
|
||||
// but never read.
|
||||
func TOFUHostKeyCallback(addr string, capturedKey *ssh.PublicKey) (ssh.HostKeyCallback, error) {
|
||||
cb, err := knownhosts.New(certpaths.KnownHostsPath())
|
||||
return TOFUHostKeyCallbackPath(certpaths.KnownHostsPath(), addr, capturedKey)
|
||||
}
|
||||
|
||||
// TOFUHostKeyCallbackPath is the path-accepting variant. knownHostsPath
|
||||
// is the known_hosts file to verify against and capture new keys into;
|
||||
// it MUST be flock-protected on capture (security.Flock). When
|
||||
// knownHostsPath is empty, falls back to certpaths.KnownHostsPath()
|
||||
// (the v0.8 flat layout) for backward compatibility with callers that
|
||||
// relied on the implicit default.
|
||||
func TOFUHostKeyCallbackPath(knownHostsPath, addr string, capturedKey *ssh.PublicKey) (ssh.HostKeyCallback, error) {
|
||||
if knownHostsPath == "" {
|
||||
knownHostsPath = certpaths.KnownHostsPath()
|
||||
}
|
||||
cb, err := knownhosts.New(knownHostsPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -312,13 +333,12 @@ func TOFUHostKeyCallback(addr string, capturedKey *ssh.PublicKey) (ssh.HostKeyCa
|
||||
var keyErr *knownhosts.KeyError
|
||||
if errors.As(err, &keyErr) && len(keyErr.Want) == 0 {
|
||||
line := knownhosts.Line([]string{knownhosts.Normalize(addr)}, key)
|
||||
path := certpaths.KnownHostsPath()
|
||||
release, lockErr := security.Flock(path)
|
||||
release, lockErr := security.Flock(knownHostsPath)
|
||||
if lockErr != nil {
|
||||
return fmt.Errorf("tofu lock known_hosts: %w", lockErr)
|
||||
}
|
||||
defer release()
|
||||
existing, readErr := os.ReadFile(path)
|
||||
existing, readErr := os.ReadFile(knownHostsPath)
|
||||
if readErr != nil && !os.IsNotExist(readErr) {
|
||||
return fmt.Errorf("tofu read known_hosts: %w", readErr)
|
||||
}
|
||||
@@ -326,7 +346,7 @@ func TOFUHostKeyCallback(addr string, capturedKey *ssh.PublicKey) (ssh.HostKeyCa
|
||||
existing = append(existing, '\n')
|
||||
}
|
||||
updated := append(existing, []byte(line)...)
|
||||
if writeErr := security.WriteAtomic(path, 0o600, updated); writeErr != nil {
|
||||
if writeErr := security.WriteAtomic(knownHostsPath, 0o600, updated); writeErr != nil {
|
||||
return fmt.Errorf("tofu write known_hosts: %w", writeErr)
|
||||
}
|
||||
if capturedKey != nil {
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
package proxmox
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestREQ157_IPv6JoinHostPort verifies that the proxmox SSH dial
|
||||
// address is correctly bracketed for IPv6 hosts (REQ-157 / P08 T5/T11).
|
||||
func TestREQ157_IPv6JoinHostPort(t *testing.T) {
|
||||
tests := []struct {
|
||||
host string
|
||||
port int
|
||||
want string
|
||||
}{
|
||||
{"192.168.1.1", 22, "192.168.1.1:22"},
|
||||
{"::1", 22, "[::1]:22"},
|
||||
{"fe80::1", 2222, "[fe80::1]:2222"},
|
||||
{"2001:db8::1", 22, "[2001:db8::1]:22"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.host, func(t *testing.T) {
|
||||
got := net.JoinHostPort(tt.host, fmt.Sprintf("%d", tt.port))
|
||||
if got != tt.want {
|
||||
t.Errorf("JoinHostPort(%s, %d) = %q, want %q", tt.host, tt.port, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -5,11 +5,13 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand"
|
||||
"net"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"golang.org/x/crypto/ssh"
|
||||
@@ -54,12 +56,14 @@ type Transport struct {
|
||||
pool sync.Map
|
||||
// keyPath is the SSH private key path (Ed25519, D-037).
|
||||
keyPath string
|
||||
// knownHostsPath is the v0.9 known_hosts path (paths.KnownHostsPath()
|
||||
// = ClusterDir()/known_hosts). It is stored for the v0.10-P14 migration
|
||||
// when proxmox.TOFUHostKeyCallback will accept a path parameter; today
|
||||
// the callback reads certpaths.KnownHostsPath() (the v0.8 flat layout)
|
||||
// directly, so this field is not yet read by dial(). Tests set
|
||||
// $ORCA_HOME so certpaths.KnownHostsPath() resolves under the temp dir.
|
||||
// knownHostsPath is the known_hosts path passed to the TOFU
|
||||
// host-key callback (D-035). NewTransport sets it from
|
||||
// certpaths.KnownHostsPath() (v0.8 flat layout) by default; callers
|
||||
// that want the v0.9 paths.KnownHostsPath() location construct the
|
||||
// transport with that path explicitly. REQ-157 / P08 T4: this field
|
||||
// IS read by dial() (via proxmox.TOFUHostKeyCallbackPath) — the
|
||||
// earlier bug where the callback ignored it and read
|
||||
// certpaths.KnownHostsPath() directly is fixed.
|
||||
knownHostsPath string
|
||||
// user is the remote SSH user (default "orca", D-037).
|
||||
user string
|
||||
@@ -120,15 +124,14 @@ func (defaultSSHDialer) DialContext(ctx context.Context, network, addr string, c
|
||||
}
|
||||
|
||||
// NewTransport returns a Transport configured with the given SSH
|
||||
// private key path and known_hosts path. The known_hosts path is the v0.9
|
||||
// location (paths.KnownHostsPath); it is stored for the v0.10-P14
|
||||
// migration when the TOFU callback will accept a path parameter. Today
|
||||
// dial() delegates host-key verification to proxmox.TOFUHostKeyCallback,
|
||||
// which reads certpaths.KnownHostsPath() (the v0.8 flat layout under
|
||||
// $ORCA_HOME) directly — so callers must ensure $ORCA_HOME points at the
|
||||
// cluster root (the CLI sets this up). The remote user defaults to
|
||||
// "orca" (D-037); override with SetUser. The dialer defaults to the
|
||||
// real ssh.Dial-based dialer; tests call SetDialer to inject a mock.
|
||||
// private key path and known_hosts path. The known_hosts path is read
|
||||
// by dial() via proxmox.TOFUHostKeyCallbackPath (D-035, REQ-157/P08 T4):
|
||||
// the TOFU callback locks/captures against this path on first connect.
|
||||
// Callers typically pass certpaths.KnownHostsPath() (the v0.8 flat
|
||||
// layout under $ORCA_HOME) or paths.KnownHostsPath() (the v0.9
|
||||
// ClusterDir() location). The remote user defaults to "orca" (D-037);
|
||||
// override with SetUser. The dialer defaults to the real ssh.Dial-based
|
||||
// dialer; tests call SetDialer to inject a mock.
|
||||
func NewTransport(keyPath, knownHostsPath string) *Transport {
|
||||
return &Transport{
|
||||
keyPath: keyPath,
|
||||
@@ -193,7 +196,14 @@ func (t *Transport) dial(peer string) (*ssh.Client, error) {
|
||||
// Host-key verification reuses the v0.8 TOFU wrapper (D-035). The
|
||||
// known_hosts file is flock-protected inside the callback on
|
||||
// first-connect capture, so we do NOT re-lock here.
|
||||
cb, err := proxmox.TOFUHostKeyCallback(peer, nil)
|
||||
//
|
||||
// REQ-157 / P08 T4: use the stored knownHostsPath field (set via
|
||||
// NewTransport from certpaths.KnownHostsPath() / paths.KnownHostsPath())
|
||||
// instead of having the callback read certpaths.KnownHostsPath() (the
|
||||
// v0.8 flat layout) directly. This closes the bug where the flock
|
||||
// field was stored but never read by dial() — the TOFU callback now
|
||||
// locks/captures against the path the transport was constructed with.
|
||||
cb, err := proxmox.TOFUHostKeyCallbackPath(t.knownHostsPath, peer, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("sshpush: host-key callback: %w", err)
|
||||
}
|
||||
@@ -391,7 +401,14 @@ func backoff(initial, max time.Duration, n int) time.Duration {
|
||||
}
|
||||
|
||||
// isTransient reports whether err looks like a transient failure worth
|
||||
// retrying (mirrors v0.8 transport.IsTransient, reimplemented here).
|
||||
// retrying (mirrors transport.IsTransient, reimplemented here so
|
||||
// internal/sshpush does not import internal/transport).
|
||||
//
|
||||
// REQ-157 / P08 T2: classification is TYPE-BASED, not substring-based.
|
||||
// The primary path is errors.Is against the sentinels (ErrTransient /
|
||||
// ErrPermanent) and against well-known syscall/net/io errors. The
|
||||
// substring fallback is retained ONLY for unwrapped errors from the
|
||||
// ssh.Dialer that do not implement the standard interfaces.
|
||||
func isTransient(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
@@ -402,6 +419,29 @@ func isTransient(err error) bool {
|
||||
if errors.Is(err, ErrPermanent) {
|
||||
return false
|
||||
}
|
||||
// Typed: a net.Error that is a timeout is transient; a net.OpError
|
||||
// whose Temporary() is true (ECONNREFUSED et al) is transient.
|
||||
var netErr net.Error
|
||||
if errors.As(err, &netErr) {
|
||||
if netErr.Timeout() {
|
||||
return true
|
||||
}
|
||||
return isTemporarySSH(netErr)
|
||||
}
|
||||
if errors.Is(err, syscall.ECONNREFUSED) ||
|
||||
errors.Is(err, syscall.ECONNRESET) ||
|
||||
errors.Is(err, syscall.ETIMEDOUT) ||
|
||||
errors.Is(err, syscall.EHOSTUNREACH) ||
|
||||
errors.Is(err, syscall.ENETUNREACH) {
|
||||
return true
|
||||
}
|
||||
if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
|
||||
return true
|
||||
}
|
||||
if errors.Is(err, context.DeadlineExceeded) {
|
||||
return true
|
||||
}
|
||||
// Substring fallback (defense-in-depth for unwrapped errors).
|
||||
s := err.Error()
|
||||
for _, sub := range []string{
|
||||
"connection refused", "i/o timeout", "EOF",
|
||||
@@ -415,6 +455,17 @@ func isTransient(err error) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// isTemporarySSH reports whether netErr implements the legacy
|
||||
// Temporary() bool method and it returns true. net.OpError.Temporary()
|
||||
// maps to the underlying errno's temporary classification.
|
||||
func isTemporarySSH(netErr net.Error) bool {
|
||||
type temporary interface{ Temporary() bool }
|
||||
if t, ok := netErr.(temporary); ok {
|
||||
return t.Temporary()
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// classifyDialErr converts a raw ssh.Dial error into a transport error
|
||||
// (transient vs permanent). Auth failures and host-key mismatches are
|
||||
// permanent; everything else is transient.
|
||||
|
||||
+81
-23
@@ -8,7 +8,11 @@ package transport
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"math/rand"
|
||||
"net"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -34,29 +38,6 @@ func DefaultRetryPolicy() RetryPolicy {
|
||||
return RetryPolicy{Initial: RetryInitial, Max: RetryMax, MaxAttempts: RetryMaxAttempts}
|
||||
}
|
||||
|
||||
// IsTransient reports whether err looks like a transient failure
|
||||
// worth retrying. We treat network errors, context-deadline-exceeded
|
||||
// (peer was slow but reachable), and a sentinel ErrTransient as
|
||||
// retryable; everything else (4xx, validation, auth) is permanent.
|
||||
func IsTransient(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
if errors.Is(err, ErrTransient) {
|
||||
return true
|
||||
}
|
||||
// We avoid pulling net/error here to keep dependencies minimal;
|
||||
// the most common transient signature is the substring "connection
|
||||
// refused" or "i/o timeout". Tests assert these explicitly.
|
||||
s := err.Error()
|
||||
for _, sub := range []string{"connection refused", "i/o timeout", "EOF", "no such host", "connection reset"} {
|
||||
if contains(s, sub) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ErrTransient is a sentinel callers can wrap to mark an error
|
||||
// retryable. ErrPermanent is the opposite.
|
||||
var (
|
||||
@@ -64,6 +45,83 @@ var (
|
||||
ErrPermanent = errors.New("permanent error")
|
||||
)
|
||||
|
||||
// IsTransient reports whether err looks like a transient failure
|
||||
// worth retrying. We treat network errors, context-deadline-exceeded
|
||||
// (peer was slow but reachable), and a sentinel ErrTransient as
|
||||
// retryable; everything else (4xx, validation, auth) is permanent.
|
||||
//
|
||||
// REQ-157 / P08 T1: classification is TYPE-BASED, not substring-based.
|
||||
// The primary path is errors.Is against the sentinels (ErrTransient /
|
||||
// ErrPermanent) and against well-known syscall/net/io errors. The
|
||||
// substring fallback is retained ONLY for unwrapped errors from
|
||||
// third-party dialers that do not implement the standard interfaces
|
||||
// (defense-in-depth); callers SHOULD wrap with ErrTransient instead.
|
||||
func IsTransient(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
// Explicit sentinels win.
|
||||
if errors.Is(err, ErrTransient) {
|
||||
return true
|
||||
}
|
||||
if errors.Is(err, ErrPermanent) {
|
||||
return false
|
||||
}
|
||||
// Typed classification: a net.Error that is a timeout is transient.
|
||||
var netErr net.Error
|
||||
if errors.As(err, &netErr) {
|
||||
if netErr.Timeout() {
|
||||
return true
|
||||
}
|
||||
// net.OpError implements Temporary(); that maps to the
|
||||
// underlying errno's temporary classification (ECONNREFUSED et
|
||||
// al). We keep the check so a plain "dial tcp: connection
|
||||
// refused" classifies as transient.
|
||||
return isTemporary(netErr)
|
||||
}
|
||||
// Specific syscall errors that are universally retryable.
|
||||
if errors.Is(err, syscall.ECONNREFUSED) ||
|
||||
errors.Is(err, syscall.ECONNRESET) ||
|
||||
errors.Is(err, syscall.ETIMEDOUT) ||
|
||||
errors.Is(err, syscall.EHOSTUNREACH) ||
|
||||
errors.Is(err, syscall.ENETUNREACH) {
|
||||
return true
|
||||
}
|
||||
// io.EOF on a read from a half-closed peer is transient (the
|
||||
// dispatch HTTP/2 path can surface this mid-stream).
|
||||
if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
|
||||
return true
|
||||
}
|
||||
// context.DeadlineExceeded from a slow-but-reachable peer is
|
||||
// transient (the next attempt may succeed under a fresh deadline).
|
||||
if errors.Is(err, context.DeadlineExceeded) {
|
||||
return true
|
||||
}
|
||||
// Substring fallback (defense-in-depth for unwrapped errors).
|
||||
s := err.Error()
|
||||
for _, sub := range []string{
|
||||
"connection refused", "i/o timeout", "EOF",
|
||||
"no such host", "connection reset",
|
||||
"deadline exceeded", "temporarily unavailable",
|
||||
} {
|
||||
if strings.Contains(s, sub) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// isTemporary reports whether netErr implements the legacy Temporary()
|
||||
// bool method and it returns true. net.OpError.Temporary() maps to the
|
||||
// underlying errno's temporary classification (ECONNREFUSED et al).
|
||||
func isTemporary(netErr net.Error) bool {
|
||||
type temporary interface{ Temporary() bool }
|
||||
if t, ok := netErr.(temporary); ok {
|
||||
return t.Temporary()
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// RetryableFunc is the signature Retry calls. It returns the result
|
||||
// and an error. The bool indicates whether the call is idempotent
|
||||
// (true = safe to retry without an idempotency key).
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
package transport
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"testing"
|
||||
"fmt"
|
||||
|
||||
)
|
||||
|
||||
// TestREQ157_TypedErrorClassification verifies that IsTransient uses
|
||||
// typed sentinels and standard interfaces, not substring matching
|
||||
// (REQ-157 / P08 T10).
|
||||
func TestREQ157_TypedErrorClassification(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
err error
|
||||
want bool
|
||||
}{
|
||||
{"nil", nil, false},
|
||||
{"ErrTransient", ErrTransient, true},
|
||||
{"wrapped ErrTransient", fmt.Errorf("dial: %w", ErrTransient), true},
|
||||
{"ErrPermanent", ErrPermanent, false},
|
||||
{"wrapped ErrPermanent", fmt.Errorf("auth: %w", ErrPermanent), false},
|
||||
{"net timeout", &net.OpError{Op: "dial", Net: "tcp", Err: &timeoutError{}}, true},
|
||||
{"context deadline", context.DeadlineExceeded, true},
|
||||
{"context canceled", context.Canceled, false},
|
||||
{"io EOF", io.EOF, true},
|
||||
{"plain error", errors.New("some permanent error"), false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := IsTransient(tt.err)
|
||||
if got != tt.want {
|
||||
t.Errorf("IsTransient(%v) = %v, want %v", tt.err, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type timeoutError struct{}
|
||||
|
||||
func (timeoutError) Error() string { return "i/o timeout" }
|
||||
func (timeoutError) Timeout() bool { return true }
|
||||
func (timeoutError) Temporary() bool { return true }
|
||||
|
||||
var _ = fmt.Errorf
|
||||
Reference in New Issue
Block a user