// Package transport — retry.go implements exponential backoff with // jitter for cross-node dispatch retries. Per the P02 plan: 100ms // initial, x2, 5s cap, max 5 attempts. Auto-retry only when the call // is idempotent (X-Orca-Idempotency-Key header present, or the verb // is intrinsically idempotent like GET/HEAD). package transport import ( "context" "errors" "math/rand" "time" ) const ( // RetryInitial is the first backoff interval. RetryInitial = 100 * time.Millisecond // RetryMax is the cap on backoff between attempts. RetryMax = 5 * time.Second // RetryMaxAttempts is the total attempt count (including the first). RetryMaxAttempts = 5 ) // RetryPolicy carries the backoff configuration. Zero value is the // default (100ms / 5s / 5 attempts). type RetryPolicy struct { Initial time.Duration Max time.Duration MaxAttempts int } // DefaultRetryPolicy returns the P02 default. func DefaultRetryPolicy() RetryPolicy { return RetryPolicy{Initial: RetryInitial, Max: RetryMax, MaxAttempts: RetryMaxAttempts} } // IsTransient reports whether err looks like a transient failure // worth retrying. We treat network errors, context-deadline-exceeded // (peer was slow but reachable), and a sentinel ErrTransient as // retryable; everything else (4xx, validation, auth) is permanent. func IsTransient(err error) bool { if err == nil { return false } if errors.Is(err, ErrTransient) { return true } // We avoid pulling net/error here to keep dependencies minimal; // the most common transient signature is the substring "connection // refused" or "i/o timeout". Tests assert these explicitly. s := err.Error() for _, sub := range []string{"connection refused", "i/o timeout", "EOF", "no such host", "connection reset"} { if contains(s, sub) { return true } } return false } // ErrTransient is a sentinel callers can wrap to mark an error // retryable. ErrPermanent is the opposite. var ( ErrTransient = errors.New("transient error") ErrPermanent = errors.New("permanent error") ) // RetryableFunc is the signature Retry calls. It returns the result // and an error. The bool indicates whether the call is idempotent // (true = safe to retry without an idempotency key). type RetryableFunc[T any] func(ctx context.Context, attempt int) (T, bool, error) // Do runs fn with backoff according to policy. It retries only if // (a) the call is idempotent, OR (b) ctx carries an idempotency key // (set via WithIdempotencyKey). Otherwise a transient error on the // first attempt is returned immediately (REQ-037: no retry without // the key). // // The generic result T lets callers reuse this for jobIDs, status // responses, etc. without boxing through `any`. func Do[T any](ctx context.Context, p RetryPolicy, fn RetryableFunc[T]) (T, error) { var zero T if p.MaxAttempts <= 0 { p = DefaultRetryPolicy() } hasKey := IdempotencyKeyFromContext(ctx) != "" for attempt := 1; attempt <= p.MaxAttempts; attempt++ { if err := ctx.Err(); err != nil { return zero, err } v, idempotent, err := fn(ctx, attempt) if err == nil { return v, nil } // Permanent errors never retry. if errors.Is(err, ErrPermanent) { return zero, err } // Last attempt — surface the error. if attempt == p.MaxAttempts { return zero, err } // Transient + no idempotency + not idempotent verb: no retry. if IsTransient(err) && !idempotent && !hasKey { return zero, err } // Wait with jittered backoff, but respect ctx cancellation. wait := backoff(p.Initial, p.Max, attempt) t := time.NewTimer(wait) select { case <-ctx.Done(): t.Stop() return zero, ctx.Err() case <-t.C: } } return zero, errors.New("retry.Do: exhausted attempts without error (impossible)") } // backoff returns the wait duration for the n-th attempt (1-indexed). // Formula: min(Initial * 2^(n-1), Max), with up to 25% jitter. func backoff(initial, max time.Duration, n int) time.Duration { d := initial for i := 1; i < n; i++ { d *= 2 if d > max { d = max break } } // Jitter: ±25% of d. jitter := time.Duration(rand.Int63n(int64(d) / 2)) d = d - d/4 + jitter if d < 0 { d = 0 } return d } // contains is a tiny substring helper (avoids pulling strings for one // call site; this is hot-path retry classification). func contains(s, sub string) bool { if len(sub) == 0 { return true } for i := 0; i+len(sub) <= len(s); i++ { if s[i:i+len(sub)] == sub { return true } } return false }