package sshpush import ( "bytes" "context" "errors" "fmt" "io" "math/rand" "net" "os" "strings" "sync" "syscall" "time" "golang.org/x/crypto/ssh" "git.cloudinit.dev/coreci/orca/internal/proxmox" ) // Default timeouts and retry parameters (REQ-073, I-B-001). const ( // ExecTimeout is the default per-exec timeout for a single SSH // command (I-B-001). ExecTimeout = 10 * time.Second // SCPTimeout is the default per-SCP timeout for a single file // transfer (I-B-001). SCPTimeout = 30 * time.Second // DialTimeout is the default SSH dial timeout. DialTimeout = 15 * time.Second // RetryInitial is the first backoff interval (v0.8 transport/retry.go). RetryInitial = 100 * time.Millisecond // RetryMax is the cap on backoff between attempts. RetryMax = 5 * time.Second // RetryMaxAttempts is the total attempt count (including the first). RetryMaxAttempts = 5 ) // Sentinel errors. ErrTransient marks a transient failure worth // retrying; ErrPermanent marks a non-retryable failure (auth, host-key // mismatch, validation). These mirror the v0.8 transport sentinels // (reimplemented here since internal/transport is not imported). var ( ErrTransient = errors.New("sshpush: transient error") ErrPermanent = errors.New("sshpush: permanent error") ErrNotConnected = errors.New("sshpush: not connected") ) // Transport is the SSH-push transport (REQ-073). It reuses one // *ssh.Client per peer across multiple operations within a single CLI // invocation (I-B-001). The zero value is NOT usable; construct one with // NewTransport. type Transport struct { // pool caches *ssh.Client per peer address ("host:port"). pool sync.Map // keyPath is the SSH private key path (Ed25519, D-037). keyPath string // knownHostsPath is the known_hosts path passed to the TOFU // host-key callback (D-035). NewTransport sets it from // certpaths.KnownHostsPath() (v0.8 flat layout) by default; callers // that want the v0.9 paths.KnownHostsPath() location construct the // transport with that path explicitly. REQ-157 / P08 T4: this field // IS read by dial() (via proxmox.TOFUHostKeyCallbackPath) — the // earlier bug where the callback ignored it and read // certpaths.KnownHostsPath() directly is fixed. knownHostsPath string // user is the remote SSH user (default "orca", D-037). user string // signer is the parsed SSH private key signer, set lazily on first // dial. signer ssh.Signer signErr error // signerOnce guards signer initialization. signerOnce sync.Once // dialer is the SSH dialer. Tests override it to inject a mock // server. The default uses ssh.DialContext via the context-aware // wrapper. dialer sshDialer // sessionFactory returns a new session for a given client. Tests // override it to inject mock sessions without a real *ssh.Client. // When nil, the default (*ssh.Client).NewSession is used. sessionFactory func(*ssh.Client) (sshSession, error) // mu guards the closed flag (pool iteration is sync.Map.Range). closed bool mu sync.Mutex } // sshSession is the minimal *ssh.Session surface the transport uses. // It lets tests substitute a mock without a real SSH server. type sshSession interface { CombinedOutput(cmd string) ([]byte, error) Close() error } // sshDialer is the SSH dialer interface (mirrors proxmox.sshDialerType). // The default uses ssh.Dial; tests inject mocks that return a fake // *ssh.Client or an error. type sshDialer interface { DialContext(ctx context.Context, network, addr string, config *ssh.ClientConfig) (*ssh.Client, error) } // defaultSSHDialer wraps ssh.Dial with a context-aware connect timeout. type defaultSSHDialer struct{} func (defaultSSHDialer) DialContext(ctx context.Context, network, addr string, config *ssh.ClientConfig) (*ssh.Client, error) { d := net.Dialer{Timeout: config.Timeout} if d.Timeout == 0 { d.Timeout = DialTimeout } conn, err := d.DialContext(ctx, network, addr) if err != nil { return nil, err } sshConn, chans, reqs, err := ssh.NewClientConn(conn, addr, config) if err != nil { _ = conn.Close() return nil, err } return ssh.NewClient(sshConn, chans, reqs), nil } // NewTransport returns a Transport configured with the given SSH // private key path and known_hosts path. The known_hosts path is read // by dial() via proxmox.TOFUHostKeyCallbackPath (D-035, REQ-157/P08 T4): // the TOFU callback locks/captures against this path on first connect. // Callers typically pass certpaths.KnownHostsPath() (the v0.8 flat // layout under $ORCA_HOME) or paths.KnownHostsPath() (the v0.9 // ClusterDir() location). The remote user defaults to "orca" (D-037); // override with SetUser. The dialer defaults to the real ssh.Dial-based // dialer; tests call SetDialer to inject a mock. func NewTransport(keyPath, knownHostsPath string) *Transport { return &Transport{ keyPath: keyPath, knownHostsPath: knownHostsPath, user: "orca", dialer: defaultSSHDialer{}, } } // SetUser overrides the remote SSH user (default "orca"). func (t *Transport) SetUser(user string) { if user != "" { t.user = user } } // SetDialer overrides the SSH dialer (for tests). func (t *Transport) SetDialer(d sshDialer) { if d != nil { t.dialer = d } } // SetSessionFactory overrides the session factory (for tests). The // factory is called per-exec/write/read to obtain a fresh session; it // must close the session when the test mock is done, or the transport // will call Close on the returned session. func (t *Transport) SetSessionFactory(f func(*ssh.Client) (sshSession, error)) { t.sessionFactory = f } // dial returns the cached *ssh.Client for peer, dialing and caching on // first use (I-B-001 connection pooling). Returns an error if the dial // fails or the transport is closed. func (t *Transport) dial(peer string) (*ssh.Client, error) { t.mu.Lock() if t.closed { t.mu.Unlock() return nil, ErrPermanent } t.mu.Unlock() if c, ok := t.pool.Load(peer); ok { return c.(*ssh.Client), nil } // Lazily parse the private key signer (once across all dials). t.signerOnce.Do(func() { keyBytes, err := os.ReadFile(t.keyPath) if err != nil { t.signErr = fmt.Errorf("sshpush: read key %s: %w", t.keyPath, err) return } s, err := ssh.ParsePrivateKey(keyBytes) if err != nil { t.signErr = fmt.Errorf("sshpush: parse key: %w", err) return } t.signer = s }) if t.signErr != nil { return nil, t.signErr } // Host-key verification reuses the v0.8 TOFU wrapper (D-035). The // known_hosts file is flock-protected inside the callback on // first-connect capture, so we do NOT re-lock here. // // REQ-157 / P08 T4: use the stored knownHostsPath field (set via // NewTransport from certpaths.KnownHostsPath() / paths.KnownHostsPath()) // instead of having the callback read certpaths.KnownHostsPath() (the // v0.8 flat layout) directly. This closes the bug where the flock // field was stored but never read by dial() — the TOFU callback now // locks/captures against the path the transport was constructed with. cb, err := proxmox.TOFUHostKeyCallbackPath(t.knownHostsPath, peer, nil) if err != nil { return nil, fmt.Errorf("sshpush: host-key callback: %w", err) } config := &ssh.ClientConfig{ User: t.user, Auth: []ssh.AuthMethod{ssh.PublicKeys(t.signer)}, HostKeyCallback: cb, Timeout: DialTimeout, } ctx, cancel := context.WithTimeout(context.Background(), DialTimeout) defer cancel() client, err := t.dialer.DialContext(ctx, "tcp", peer, config) if err != nil { return nil, classifyDialErr(err) } // Race: two goroutines dialing the same peer concurrently both // create a client. Last-wins; the loser is closed. This is rare // (dial is rare and the pool hit short-circuits) and harmless. if existing, loaded := t.pool.LoadOrStore(peer, client); loaded { _ = client.Close() return existing.(*ssh.Client), nil } return client, nil } // Exec runs cmd on peer over SSH and returns its combined output. The // default per-exec timeout is ExecTimeout (I-B-001); override by // passing a context with a shorter deadline. Transient failures are // retried with exponential backoff (100ms ×2, cap 5s, max 5 attempts — // the v0.8 transport/retry.go pattern, reimplemented here). func (t *Transport) Exec(ctx context.Context, peer string, cmd string) ([]byte, error) { return t.execWithRetry(ctx, peer, cmd, true) } // execWithRetry runs the exec with retry. exec is treated as // idempotent (read-only) for retry purposes; the idempotency helpers // (WriteFileIdempotent) handle writes. func (t *Transport) execWithRetry(ctx context.Context, peer string, cmd string, idempotent bool) ([]byte, error) { var lastErr error for attempt := 1; attempt <= RetryMaxAttempts; attempt++ { if err := ctx.Err(); err != nil { return nil, err } out, err := t.execOnce(ctx, peer, cmd) if err == nil { return out, nil } if errors.Is(err, ErrPermanent) { return nil, err } lastErr = err if attempt == RetryMaxAttempts { break } if !isTransient(err) { return nil, err } wait := backoff(RetryInitial, RetryMax, attempt) timer := time.NewTimer(wait) select { case <-ctx.Done(): timer.Stop() return nil, ctx.Err() case <-timer.C: } } return nil, lastErr } // execOnce runs the command a single time against peer. func (t *Transport) execOnce(ctx context.Context, peer string, cmd string) ([]byte, error) { client, err := t.dial(peer) if err != nil { return nil, classifyDialErr(err) } sess, err := t.newSession(client) if err != nil { return nil, fmt.Errorf("sshpush: new session: %w", err) } defer sess.Close() type result struct { out []byte err error } ch := make(chan result, 1) go func() { out, err := sess.CombinedOutput(cmd) ch <- result{out, err} }() timeout := ExecTimeout if dl, ok := ctx.Deadline(); ok { if remaining := time.Until(dl); remaining > 0 && remaining < timeout { timeout = remaining } } select { case <-ctx.Done(): return nil, ctx.Err() case <-time.After(timeout): return nil, fmt.Errorf("sshpush: exec timeout after %s: %w", timeout, ErrTransient) case r := <-ch: if r.err != nil { return r.out, classifyExecErr(r.err) } return r.out, nil } } // newSession returns a session for client, using the override factory // when set (tests), otherwise the real *ssh.Client.NewSession. func (t *Transport) newSession(client *ssh.Client) (sshSession, error) { if t.sessionFactory != nil { return t.sessionFactory(client) } s, err := client.NewSession() if err != nil { return nil, err } return &realSession{Session: s}, nil } // realSession wraps *ssh.Session to satisfy the sshSession interface. type realSession struct { *ssh.Session } func (r *realSession) CombinedOutput(cmd string) ([]byte, error) { return r.Session.CombinedOutput(cmd) } // WriteFile SCPs content to peer:path atomically (write-to-tmp + mv, // REQ-074). The default per-SCP timeout is SCPTimeout (I-B-001). // Idempotency: if the file already exists with the same SHA-256, the // write is skipped (C-18). Use WriteFileIdempotent for the explicit // written/skipped result. func (t *Transport) WriteFile(ctx context.Context, peer string, path string, content []byte, mode os.FileMode) error { _, err := t.WriteFileIdempotent(ctx, peer, path, content, mode) return err } // ReadFile reads the file at peer:path via SSH cat. func (t *Transport) ReadFile(ctx context.Context, peer string, path string) ([]byte, error) { cmd := fmt.Sprintf("cat %s", shellQuote(path)) out, err := t.Exec(ctx, peer, cmd) if err != nil { return nil, err } return out, nil } // Close closes all pooled SSH clients (REQ-073). Safe to call // multiple times; subsequent calls are no-ops. func (t *Transport) Close() error { t.mu.Lock() if t.closed { t.mu.Unlock() return nil } t.closed = true t.mu.Unlock() var firstErr error t.pool.Range(func(key, value any) bool { if c, ok := value.(*ssh.Client); ok { if err := c.Close(); err != nil && firstErr == nil { firstErr = err } } t.pool.Delete(key) return true }) return firstErr } // backoff returns the wait duration for the n-th attempt (1-indexed). // Formula: min(Initial * 2^(n-1), Max), with up to 25% jitter (matches // v0.8 transport/retry.go). func backoff(initial, max time.Duration, n int) time.Duration { d := initial for i := 1; i < n; i++ { d *= 2 if d > max { d = max break } } if d <= 0 { return 0 } jitter := time.Duration(rand.Int63n(int64(d) / 2)) d = d - d/4 + jitter if d < 0 { d = 0 } return d } // isTransient reports whether err looks like a transient failure worth // retrying (mirrors transport.IsTransient, reimplemented here so // internal/sshpush does not import internal/transport). // // REQ-157 / P08 T2: classification is TYPE-BASED, not substring-based. // The primary path is errors.Is against the sentinels (ErrTransient / // ErrPermanent) and against well-known syscall/net/io errors. The // substring fallback is retained ONLY for unwrapped errors from the // ssh.Dialer that do not implement the standard interfaces. func isTransient(err error) bool { if err == nil { return false } if errors.Is(err, ErrTransient) { return true } if errors.Is(err, ErrPermanent) { return false } // Typed: a net.Error that is a timeout is transient; a net.OpError // whose Temporary() is true (ECONNREFUSED et al) is transient. var netErr net.Error if errors.As(err, &netErr) { if netErr.Timeout() { return true } return isTemporarySSH(netErr) } if errors.Is(err, syscall.ECONNREFUSED) || errors.Is(err, syscall.ECONNRESET) || errors.Is(err, syscall.ETIMEDOUT) || errors.Is(err, syscall.EHOSTUNREACH) || errors.Is(err, syscall.ENETUNREACH) { return true } if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { return true } if errors.Is(err, context.DeadlineExceeded) { return true } // Substring fallback (defense-in-depth for unwrapped errors). s := err.Error() for _, sub := range []string{ "connection refused", "i/o timeout", "EOF", "no such host", "connection reset", "timeout", "deadline exceeded", "temporarily unavailable", } { if strings.Contains(s, sub) { return true } } return false } // isTemporarySSH reports whether netErr implements the legacy // Temporary() bool method and it returns true. net.OpError.Temporary() // maps to the underlying errno's temporary classification. func isTemporarySSH(netErr net.Error) bool { type temporary interface{ Temporary() bool } if t, ok := netErr.(temporary); ok { return t.Temporary() } return false } // classifyDialErr converts a raw ssh.Dial error into a transport error // (transient vs permanent). Auth failures and host-key mismatches are // permanent; everything else is transient. func classifyDialErr(err error) error { if err == nil { return nil } s := err.Error() if strings.Contains(s, "unable to authenticate") || strings.Contains(s, "handshake failed") { return fmt.Errorf("%w: %v", ErrPermanent, err) } if strings.Contains(s, "host key") && strings.Contains(s, "mismatch") { return fmt.Errorf("%w: %v", ErrPermanent, err) } if strings.Contains(s, "knownhosts") { return fmt.Errorf("%w: %v", ErrPermanent, err) } return fmt.Errorf("%w: %v", ErrTransient, err) } // classifyExecErr converts a raw session exec error into a transport // error. Non-zero exit codes are NOT transient (the command ran; the // failure is logical, not network). Session-creation failures and // network-level errors are transient. func classifyExecErr(err error) error { if err == nil { return nil } var exitErr *ssh.ExitError if errors.As(err, &exitErr) { return fmt.Errorf("%w: exit %d", ErrPermanent, exitErr.ExitStatus()) } s := err.Error() for _, sub := range []string{"EOF", "session closed", "channel closed"} { if strings.Contains(s, sub) { return fmt.Errorf("%w: %v", ErrTransient, err) } } return fmt.Errorf("%w: %v", ErrPermanent, err) } // shellQuote single-quotes a path for safe shell interpolation. It // escapes embedded single-quotes via the standard '\” idiom. func shellQuote(s string) string { return "'" + strings.ReplaceAll(s, "'", "'\\''") + "'" } // remoteSHA256 returns the SHA-256 of the file at peer:path via SSH // `sha256sum`, or ("", error) if the file is missing or the command // fails. The returned hash is the hex digest (lowercase, no filename). func (t *Transport) remoteSHA256(ctx context.Context, peer string, path string) (string, error) { cmd := fmt.Sprintf("sha256sum %s 2>/dev/null", shellQuote(path)) out, err := t.execWithRetry(ctx, peer, cmd, true) if err != nil { return "", err } out = bytes.TrimSpace(out) if len(out) == 0 { return "", nil } fields := strings.Fields(string(out)) if len(fields) == 0 { return "", nil } return fields[0], nil }