orca/internal/transport/retry.go

// Package transport — retry.go implements exponential backoff with
// jitter for cross-node dispatch retries. Per the P02 plan: 100ms
// initial, x2, 5s cap, max 5 attempts. Auto-retry only when the call
// is idempotent (X-Orca-Idempotency-Key header present, or the verb
// is intrinsically idempotent like GET/HEAD).
package transport

import (
	"context"
	"errors"
	"io"
	"math/rand"
	"net"
	"strings"
	"syscall"
	"time"
)

const (
	// RetryInitial is the first backoff interval.
	RetryInitial = 100 * time.Millisecond
	// RetryMax is the cap on backoff between attempts.
	RetryMax = 5 * time.Second
	// RetryMaxAttempts is the total attempt count (including the first).
	RetryMaxAttempts = 5
)

// RetryPolicy carries the backoff configuration. Zero value is the
// default (100ms / 5s / 5 attempts).
type RetryPolicy struct {
	Initial     time.Duration
	Max         time.Duration
	MaxAttempts int
}

// DefaultRetryPolicy returns the P02 default.
func DefaultRetryPolicy() RetryPolicy {
	return RetryPolicy{Initial: RetryInitial, Max: RetryMax, MaxAttempts: RetryMaxAttempts}
}

// ErrTransient is a sentinel callers can wrap to mark an error
// retryable. ErrPermanent is the opposite.
var (
	ErrTransient = errors.New("transient error")
	ErrPermanent = errors.New("permanent error")
)

// IsTransient reports whether err looks like a transient failure
// worth retrying. We treat network errors, context-deadline-exceeded
// (peer was slow but reachable), and a sentinel ErrTransient as
// retryable; everything else (4xx, validation, auth) is permanent.
//
// REQ-157 / P08 T1: classification is TYPE-BASED, not substring-based.
// The primary path is errors.Is against the sentinels (ErrTransient /
// ErrPermanent) and against well-known syscall/net/io errors. The
// substring fallback is retained ONLY for unwrapped errors from
// third-party dialers that do not implement the standard interfaces
// (defense-in-depth); callers SHOULD wrap with ErrTransient instead.
func IsTransient(err error) bool {
	if err == nil {
		return false
	}
	// Explicit sentinels win.
	if errors.Is(err, ErrTransient) {
		return true
	}
	if errors.Is(err, ErrPermanent) {
		return false
	}
	// Typed classification: a net.Error that is a timeout is transient.
	var netErr net.Error
	if errors.As(err, &netErr) {
		if netErr.Timeout() {
			return true
		}
		// net.OpError implements Temporary(); that maps to the
		// underlying errno's temporary classification (ECONNREFUSED et
		// al). We keep the check so a plain "dial tcp: connection
		// refused" classifies as transient.
		return isTemporary(netErr)
	}
	// Specific syscall errors that are universally retryable.
	if errors.Is(err, syscall.ECONNREFUSED) ||
		errors.Is(err, syscall.ECONNRESET) ||
		errors.Is(err, syscall.ETIMEDOUT) ||
		errors.Is(err, syscall.EHOSTUNREACH) ||
		errors.Is(err, syscall.ENETUNREACH) {
		return true
	}
	// io.EOF on a read from a half-closed peer is transient (the
	// dispatch HTTP/2 path can surface this mid-stream).
	if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
		return true
	}
	// context.DeadlineExceeded from a slow-but-reachable peer is
	// transient (the next attempt may succeed under a fresh deadline).
	if errors.Is(err, context.DeadlineExceeded) {
		return true
	}
	// Substring fallback (defense-in-depth for unwrapped errors).
	s := err.Error()
	for _, sub := range []string{
		"connection refused", "i/o timeout", "EOF",
		"no such host", "connection reset",
		"deadline exceeded", "temporarily unavailable",
	} {
		if strings.Contains(s, sub) {
			return true
		}
	}
	return false
}

// isTemporary reports whether netErr implements the legacy Temporary()
// bool method and it returns true. net.OpError.Temporary() maps to the
// underlying errno's temporary classification (ECONNREFUSED et al).
func isTemporary(netErr net.Error) bool {
	type temporary interface{ Temporary() bool }
	if t, ok := netErr.(temporary); ok {
		return t.Temporary()
	}
	return false
}

// RetryableFunc is the signature Retry calls. It returns the result
// and an error. The bool indicates whether the call is idempotent
// (true = safe to retry without an idempotency key).
type RetryableFunc[T any] func(ctx context.Context, attempt int) (T, bool, error)

// Do runs fn with backoff according to policy. It retries only if
// (a) the call is idempotent, OR (b) ctx carries an idempotency key
// (set via WithIdempotencyKey). Otherwise a transient error on the
// first attempt is returned immediately (REQ-037: no retry without
// the key).
//
// The generic result T lets callers reuse this for jobIDs, status
// responses, etc. without boxing through `any`.
func Do[T any](ctx context.Context, p RetryPolicy, fn RetryableFunc[T]) (T, error) {
	var zero T
	if p.MaxAttempts <= 0 {
		p = DefaultRetryPolicy()
	}
	hasKey := IdempotencyKeyFromContext(ctx) != ""
	for attempt := 1; attempt <= p.MaxAttempts; attempt++ {
		if err := ctx.Err(); err != nil {
			return zero, err
		}
		v, idempotent, err := fn(ctx, attempt)
		if err == nil {
			return v, nil
		}
		// Permanent errors never retry.
		if errors.Is(err, ErrPermanent) {
			return zero, err
		}
		// Last attempt — surface the error.
		if attempt == p.MaxAttempts {
			return zero, err
		}
		// Transient + no idempotency + not idempotent verb: no retry.
		if IsTransient(err) && !idempotent && !hasKey {
			return zero, err
		}
		// Wait with jittered backoff, but respect ctx cancellation.
		wait := backoff(p.Initial, p.Max, attempt)
		t := time.NewTimer(wait)
		select {
		case <-ctx.Done():
			t.Stop()
			return zero, ctx.Err()
		case <-t.C:
		}
	}
	return zero, errors.New("retry.Do: exhausted attempts without error (impossible)")
}

// backoff returns the wait duration for the n-th attempt (1-indexed).
// Formula: min(Initial * 2^(n-1), Max), with up to 25% jitter.
func backoff(initial, max time.Duration, n int) time.Duration {
	d := initial
	for i := 1; i < n; i++ {
		d *= 2
		if d > max {
			d = max
			break
		}
	}
	// Jitter: ±25% of d.
	jitter := time.Duration(rand.Int63n(int64(d) / 2))
	d = d - d/4 + jitter
	if d < 0 {
		d = 0
	}
	return d
}

// contains is a tiny substring helper (avoids pulling strings for one
// call site; this is hot-path retry classification).
func contains(s, sub string) bool {
	if len(sub) == 0 {
		return true
	}
	for i := 0; i+len(sub) <= len(s); i++ {
		if s[i:i+len(sub)] == sub {
			return true
		}
	}
	return false
}