orca/internal/sshpush/transport.go

package sshpush

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"io"
	"math/rand"
	"net"
	"os"
	"strings"
	"sync"
	"syscall"
	"time"

	"golang.org/x/crypto/ssh"

	"git.cloudinit.dev/coreci/orca/internal/proxmox"
)

// Default timeouts and retry parameters (REQ-073, I-B-001).
const (
	// ExecTimeout is the default per-exec timeout for a single SSH
	// command (I-B-001).
	ExecTimeout = 10 * time.Second
	// SCPTimeout is the default per-SCP timeout for a single file
	// transfer (I-B-001).
	SCPTimeout = 30 * time.Second
	// DialTimeout is the default SSH dial timeout.
	DialTimeout = 15 * time.Second
	// RetryInitial is the first backoff interval (v0.8 transport/retry.go).
	RetryInitial = 100 * time.Millisecond
	// RetryMax is the cap on backoff between attempts.
	RetryMax = 5 * time.Second
	// RetryMaxAttempts is the total attempt count (including the first).
	RetryMaxAttempts = 5
)

// Sentinel errors. ErrTransient marks a transient failure worth
// retrying; ErrPermanent marks a non-retryable failure (auth, host-key
// mismatch, validation). These mirror the v0.8 transport sentinels
// (reimplemented here since internal/transport is not imported).
var (
	ErrTransient    = errors.New("sshpush: transient error")
	ErrPermanent    = errors.New("sshpush: permanent error")
	ErrNotConnected = errors.New("sshpush: not connected")
)

// Transport is the SSH-push transport (REQ-073). It reuses one
// *ssh.Client per peer across multiple operations within a single CLI
// invocation (I-B-001). The zero value is NOT usable; construct one with
// NewTransport.
type Transport struct {
	// pool caches *ssh.Client per peer address ("host:port").
	pool sync.Map
	// keyPath is the SSH private key path (Ed25519, D-037).
	keyPath string
	// knownHostsPath is the known_hosts path passed to the TOFU
	// host-key callback (D-035). NewTransport sets it from
	// certpaths.KnownHostsPath() (v0.8 flat layout) by default; callers
	// that want the v0.9 paths.KnownHostsPath() location construct the
	// transport with that path explicitly. REQ-157 / P08 T4: this field
	// IS read by dial() (via proxmox.TOFUHostKeyCallbackPath) — the
	// earlier bug where the callback ignored it and read
	// certpaths.KnownHostsPath() directly is fixed.
	knownHostsPath string
	// user is the remote SSH user (default "orca", D-037).
	user string
	// signer is the parsed SSH private key signer, set lazily on first
	// dial.
	signer  ssh.Signer
	signErr error
	// signerOnce guards signer initialization.
	signerOnce sync.Once

	// dialer is the SSH dialer. Tests override it to inject a mock
	// server. The default uses ssh.DialContext via the context-aware
	// wrapper.
	dialer sshDialer

	// sessionFactory returns a new session for a given client. Tests
	// override it to inject mock sessions without a real *ssh.Client.
	// When nil, the default (*ssh.Client).NewSession is used.
	sessionFactory func(*ssh.Client) (sshSession, error)

	// mu guards the closed flag (pool iteration is sync.Map.Range).
	closed bool
	mu     sync.Mutex
}

// sshSession is the minimal *ssh.Session surface the transport uses.
// It lets tests substitute a mock without a real SSH server.
type sshSession interface {
	CombinedOutput(cmd string) ([]byte, error)
	Close() error
}

// sshDialer is the SSH dialer interface (mirrors proxmox.sshDialerType).
// The default uses ssh.Dial; tests inject mocks that return a fake
// *ssh.Client or an error.
type sshDialer interface {
	DialContext(ctx context.Context, network, addr string, config *ssh.ClientConfig) (*ssh.Client, error)
}

// defaultSSHDialer wraps ssh.Dial with a context-aware connect timeout.
type defaultSSHDialer struct{}

func (defaultSSHDialer) DialContext(ctx context.Context, network, addr string, config *ssh.ClientConfig) (*ssh.Client, error) {
	d := net.Dialer{Timeout: config.Timeout}
	if d.Timeout == 0 {
		d.Timeout = DialTimeout
	}
	conn, err := d.DialContext(ctx, network, addr)
	if err != nil {
		return nil, err
	}
	sshConn, chans, reqs, err := ssh.NewClientConn(conn, addr, config)
	if err != nil {
		_ = conn.Close()
		return nil, err
	}
	return ssh.NewClient(sshConn, chans, reqs), nil
}

// NewTransport returns a Transport configured with the given SSH
// private key path and known_hosts path. The known_hosts path is read
// by dial() via proxmox.TOFUHostKeyCallbackPath (D-035, REQ-157/P08 T4):
// the TOFU callback locks/captures against this path on first connect.
// Callers typically pass certpaths.KnownHostsPath() (the v0.8 flat
// layout under $ORCA_HOME) or paths.KnownHostsPath() (the v0.9
// ClusterDir() location). The remote user defaults to "orca" (D-037);
// override with SetUser. The dialer defaults to the real ssh.Dial-based
// dialer; tests call SetDialer to inject a mock.
func NewTransport(keyPath, knownHostsPath string) *Transport {
	return &Transport{
		keyPath:        keyPath,
		knownHostsPath: knownHostsPath,
		user:           "orca",
		dialer:         defaultSSHDialer{},
	}
}

// SetUser overrides the remote SSH user (default "orca").
func (t *Transport) SetUser(user string) {
	if user != "" {
		t.user = user
	}
}

// SetDialer overrides the SSH dialer (for tests).
func (t *Transport) SetDialer(d sshDialer) {
	if d != nil {
		t.dialer = d
	}
}

// SetSessionFactory overrides the session factory (for tests). The
// factory is called per-exec/write/read to obtain a fresh session; it
// must close the session when the test mock is done, or the transport
// will call Close on the returned session.
func (t *Transport) SetSessionFactory(f func(*ssh.Client) (sshSession, error)) {
	t.sessionFactory = f
}

// dial returns the cached *ssh.Client for peer, dialing and caching on
// first use (I-B-001 connection pooling). Returns an error if the dial
// fails or the transport is closed.
func (t *Transport) dial(peer string) (*ssh.Client, error) {
	t.mu.Lock()
	if t.closed {
		t.mu.Unlock()
		return nil, ErrPermanent
	}
	t.mu.Unlock()
	if c, ok := t.pool.Load(peer); ok {
		return c.(*ssh.Client), nil
	}
	// Lazily parse the private key signer (once across all dials).
	t.signerOnce.Do(func() {
		keyBytes, err := os.ReadFile(t.keyPath)
		if err != nil {
			t.signErr = fmt.Errorf("sshpush: read key %s: %w", t.keyPath, err)
			return
		}
		s, err := ssh.ParsePrivateKey(keyBytes)
		if err != nil {
			t.signErr = fmt.Errorf("sshpush: parse key: %w", err)
			return
		}
		t.signer = s
	})
	if t.signErr != nil {
		return nil, t.signErr
	}
	// Host-key verification reuses the v0.8 TOFU wrapper (D-035). The
	// known_hosts file is flock-protected inside the callback on
	// first-connect capture, so we do NOT re-lock here.
	//
	// REQ-157 / P08 T4: use the stored knownHostsPath field (set via
	// NewTransport from certpaths.KnownHostsPath() / paths.KnownHostsPath())
	// instead of having the callback read certpaths.KnownHostsPath() (the
	// v0.8 flat layout) directly. This closes the bug where the flock
	// field was stored but never read by dial() — the TOFU callback now
	// locks/captures against the path the transport was constructed with.
	cb, err := proxmox.TOFUHostKeyCallbackPath(t.knownHostsPath, peer, nil)
	if err != nil {
		return nil, fmt.Errorf("sshpush: host-key callback: %w", err)
	}
	config := &ssh.ClientConfig{
		User:            t.user,
		Auth:            []ssh.AuthMethod{ssh.PublicKeys(t.signer)},
		HostKeyCallback: cb,
		Timeout:         DialTimeout,
	}
	ctx, cancel := context.WithTimeout(context.Background(), DialTimeout)
	defer cancel()
	client, err := t.dialer.DialContext(ctx, "tcp", peer, config)
	if err != nil {
		return nil, classifyDialErr(err)
	}
	// Race: two goroutines dialing the same peer concurrently both
	// create a client. Last-wins; the loser is closed. This is rare
	// (dial is rare and the pool hit short-circuits) and harmless.
	if existing, loaded := t.pool.LoadOrStore(peer, client); loaded {
		_ = client.Close()
		return existing.(*ssh.Client), nil
	}
	return client, nil
}

// Exec runs cmd on peer over SSH and returns its combined output. The
// default per-exec timeout is ExecTimeout (I-B-001); override by
// passing a context with a shorter deadline. Transient failures are
// retried with exponential backoff (100ms ×2, cap 5s, max 5 attempts —
// the v0.8 transport/retry.go pattern, reimplemented here).
func (t *Transport) Exec(ctx context.Context, peer string, cmd string) ([]byte, error) {
	return t.execWithRetry(ctx, peer, cmd, true)
}

// execWithRetry runs the exec with retry. exec is treated as
// idempotent (read-only) for retry purposes; the idempotency helpers
// (WriteFileIdempotent) handle writes.
func (t *Transport) execWithRetry(ctx context.Context, peer string, cmd string, idempotent bool) ([]byte, error) {
	var lastErr error
	for attempt := 1; attempt <= RetryMaxAttempts; attempt++ {
		if err := ctx.Err(); err != nil {
			return nil, err
		}
		out, err := t.execOnce(ctx, peer, cmd)
		if err == nil {
			return out, nil
		}
		if errors.Is(err, ErrPermanent) {
			return nil, err
		}
		lastErr = err
		if attempt == RetryMaxAttempts {
			break
		}
		if !isTransient(err) {
			return nil, err
		}
		wait := backoff(RetryInitial, RetryMax, attempt)
		timer := time.NewTimer(wait)
		select {
		case <-ctx.Done():
			timer.Stop()
			return nil, ctx.Err()
		case <-timer.C:
		}
	}
	return nil, lastErr
}

// execOnce runs the command a single time against peer.
func (t *Transport) execOnce(ctx context.Context, peer string, cmd string) ([]byte, error) {
	client, err := t.dial(peer)
	if err != nil {
		return nil, classifyDialErr(err)
	}
	sess, err := t.newSession(client)
	if err != nil {
		return nil, fmt.Errorf("sshpush: new session: %w", err)
	}
	defer sess.Close()
	type result struct {
		out []byte
		err error
	}
	ch := make(chan result, 1)
	go func() {
		out, err := sess.CombinedOutput(cmd)
		ch <- result{out, err}
	}()
	timeout := ExecTimeout
	if dl, ok := ctx.Deadline(); ok {
		if remaining := time.Until(dl); remaining > 0 && remaining < timeout {
			timeout = remaining
		}
	}
	select {
	case <-ctx.Done():
		return nil, ctx.Err()
	case <-time.After(timeout):
		return nil, fmt.Errorf("sshpush: exec timeout after %s: %w", timeout, ErrTransient)
	case r := <-ch:
		if r.err != nil {
			return r.out, classifyExecErr(r.err)
		}
		return r.out, nil
	}
}

// newSession returns a session for client, using the override factory
// when set (tests), otherwise the real *ssh.Client.NewSession.
func (t *Transport) newSession(client *ssh.Client) (sshSession, error) {
	if t.sessionFactory != nil {
		return t.sessionFactory(client)
	}
	s, err := client.NewSession()
	if err != nil {
		return nil, err
	}
	return &realSession{Session: s}, nil
}

// realSession wraps *ssh.Session to satisfy the sshSession interface.
type realSession struct {
	*ssh.Session
}

func (r *realSession) CombinedOutput(cmd string) ([]byte, error) {
	return r.Session.CombinedOutput(cmd)
}

// WriteFile SCPs content to peer:path atomically (write-to-tmp + mv,
// REQ-074). The default per-SCP timeout is SCPTimeout (I-B-001).
// Idempotency: if the file already exists with the same SHA-256, the
// write is skipped (C-18). Use WriteFileIdempotent for the explicit
// written/skipped result.
func (t *Transport) WriteFile(ctx context.Context, peer string, path string, content []byte, mode os.FileMode) error {
	_, err := t.WriteFileIdempotent(ctx, peer, path, content, mode)
	return err
}

// ReadFile reads the file at peer:path via SSH cat.
func (t *Transport) ReadFile(ctx context.Context, peer string, path string) ([]byte, error) {
	cmd := fmt.Sprintf("cat %s", shellQuote(path))
	out, err := t.Exec(ctx, peer, cmd)
	if err != nil {
		return nil, err
	}
	return out, nil
}

// Close closes all pooled SSH clients (REQ-073). Safe to call
// multiple times; subsequent calls are no-ops.
func (t *Transport) Close() error {
	t.mu.Lock()
	if t.closed {
		t.mu.Unlock()
		return nil
	}
	t.closed = true
	t.mu.Unlock()
	var firstErr error
	t.pool.Range(func(key, value any) bool {
		if c, ok := value.(*ssh.Client); ok {
			if err := c.Close(); err != nil && firstErr == nil {
				firstErr = err
			}
		}
		t.pool.Delete(key)
		return true
	})
	return firstErr
}

// backoff returns the wait duration for the n-th attempt (1-indexed).
// Formula: min(Initial * 2^(n-1), Max), with up to 25% jitter (matches
// v0.8 transport/retry.go).
func backoff(initial, max time.Duration, n int) time.Duration {
	d := initial
	for i := 1; i < n; i++ {
		d *= 2
		if d > max {
			d = max
			break
		}
	}
	if d <= 0 {
		return 0
	}
	jitter := time.Duration(rand.Int63n(int64(d) / 2))
	d = d - d/4 + jitter
	if d < 0 {
		d = 0
	}
	return d
}

// isTransient reports whether err looks like a transient failure worth
// retrying (mirrors transport.IsTransient, reimplemented here so
// internal/sshpush does not import internal/transport).
//
// REQ-157 / P08 T2: classification is TYPE-BASED, not substring-based.
// The primary path is errors.Is against the sentinels (ErrTransient /
// ErrPermanent) and against well-known syscall/net/io errors. The
// substring fallback is retained ONLY for unwrapped errors from the
// ssh.Dialer that do not implement the standard interfaces.
func isTransient(err error) bool {
	if err == nil {
		return false
	}
	if errors.Is(err, ErrTransient) {
		return true
	}
	if errors.Is(err, ErrPermanent) {
		return false
	}
	// Typed: a net.Error that is a timeout is transient; a net.OpError
	// whose Temporary() is true (ECONNREFUSED et al) is transient.
	var netErr net.Error
	if errors.As(err, &netErr) {
		if netErr.Timeout() {
			return true
		}
		return isTemporarySSH(netErr)
	}
	if errors.Is(err, syscall.ECONNREFUSED) ||
		errors.Is(err, syscall.ECONNRESET) ||
		errors.Is(err, syscall.ETIMEDOUT) ||
		errors.Is(err, syscall.EHOSTUNREACH) ||
		errors.Is(err, syscall.ENETUNREACH) {
		return true
	}
	if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
		return true
	}
	if errors.Is(err, context.DeadlineExceeded) {
		return true
	}
	// Substring fallback (defense-in-depth for unwrapped errors).
	s := err.Error()
	for _, sub := range []string{
		"connection refused", "i/o timeout", "EOF",
		"no such host", "connection reset", "timeout",
		"deadline exceeded", "temporarily unavailable",
	} {
		if strings.Contains(s, sub) {
			return true
		}
	}
	return false
}

// isTemporarySSH reports whether netErr implements the legacy
// Temporary() bool method and it returns true. net.OpError.Temporary()
// maps to the underlying errno's temporary classification.
func isTemporarySSH(netErr net.Error) bool {
	type temporary interface{ Temporary() bool }
	if t, ok := netErr.(temporary); ok {
		return t.Temporary()
	}
	return false
}

// classifyDialErr converts a raw ssh.Dial error into a transport error
// (transient vs permanent). Auth failures and host-key mismatches are
// permanent; everything else is transient.
func classifyDialErr(err error) error {
	if err == nil {
		return nil
	}
	s := err.Error()
	if strings.Contains(s, "unable to authenticate") || strings.Contains(s, "handshake failed") {
		return fmt.Errorf("%w: %v", ErrPermanent, err)
	}
	if strings.Contains(s, "host key") && strings.Contains(s, "mismatch") {
		return fmt.Errorf("%w: %v", ErrPermanent, err)
	}
	if strings.Contains(s, "knownhosts") {
		return fmt.Errorf("%w: %v", ErrPermanent, err)
	}
	return fmt.Errorf("%w: %v", ErrTransient, err)
}

// classifyExecErr converts a raw session exec error into a transport
// error. Non-zero exit codes are NOT transient (the command ran; the
// failure is logical, not network). Session-creation failures and
// network-level errors are transient.
func classifyExecErr(err error) error {
	if err == nil {
		return nil
	}
	var exitErr *ssh.ExitError
	if errors.As(err, &exitErr) {
		return fmt.Errorf("%w: exit %d", ErrPermanent, exitErr.ExitStatus())
	}
	s := err.Error()
	for _, sub := range []string{"EOF", "session closed", "channel closed"} {
		if strings.Contains(s, sub) {
			return fmt.Errorf("%w: %v", ErrTransient, err)
		}
	}
	return fmt.Errorf("%w: %v", ErrPermanent, err)
}

// shellQuote single-quotes a path for safe shell interpolation. It
// escapes embedded single-quotes via the standard '\” idiom.
func shellQuote(s string) string {
	return "'" + strings.ReplaceAll(s, "'", "'\\''") + "'"
}

// remoteSHA256 returns the SHA-256 of the file at peer:path via SSH
// `sha256sum`, or ("", error) if the file is missing or the command
// fails. The returned hash is the hex digest (lowercase, no filename).
func (t *Transport) remoteSHA256(ctx context.Context, peer string, path string) (string, error) {
	cmd := fmt.Sprintf("sha256sum %s 2>/dev/null", shellQuote(path))
	out, err := t.execWithRetry(ctx, peer, cmd, true)
	if err != nil {
		return "", err
	}
	out = bytes.TrimSpace(out)
	if len(out) == 0 {
		return "", nil
	}
	fields := strings.Fields(string(out))
	if len(fields) == 0 {
		return "", nil
	}
	return fields[0], nil
}