Scenario: Payment provider goes down at 14:00:00
Services with naive retry (sleep 1000ms, retry up to 3 times):

14:00:00 — 10,000 requests fail simultaneously
14:00:01 — 10,000 retry #1 at exactly T+1s (all wake up together)
            Provider is still recovering — all fail
14:00:02 — 10,000 retry #2 at exactly T+2s
            Provider's recovery is interrupted by the storm — fails again
14:00:03 — 10,000 retry #3 at exactly T+3s
            Provider almost recovered — storm hits again — fails
14:00:03 — All retries exhausted — 10,000 errors to callers

Provider's actual recovery window: 45 seconds
Effective downtime created by the retry storm: 3 minutes

// Exponential backoff: base × 2^attempt, capped at a maximum
function exponentialBackoffMs(attempt: number, baseMs = 100, capMs = 30_000): number {
  return Math.min(capMs, baseMs * Math.pow(2, attempt))
}

// Attempt 0: min(30000, 100 × 2^0) = 100ms
// Attempt 1: min(30000, 100 × 2^1) = 200ms
// Attempt 2: min(30000, 100 × 2^2) = 400ms
// Attempt 3: min(30000, 100 × 2^3) = 800ms
// Attempt 4: min(30000, 100 × 2^4) = 1600ms
// Attempt 5: min(30000, 100 × 2^5) = 3200ms
// Attempt 10: min(30000, 100 × 2^10) = capped at 30000ms

// Without jitter: 10,000 services all compute the same backoff and retry together

// Three jitter strategies — Full Jitter is the production recommendation

// ❌ No jitter — synchronized storm
function noJitter(attempt: number): number {
  return exponentialBackoffMs(attempt)
}
// All callers sleep for exactly the same duration → synchronized retry

// ⚠️ Equal jitter — better, but still clusters
function equalJitter(attempt: number): number {
  const backoff = exponentialBackoffMs(attempt)
  const half = backoff / 2
  return half + Math.random() * half  // Range: [backoff/2, backoff]
  // Never waits less than half the backoff — limits minimum spread
}

// ✅ Full jitter — recommended for distributed systems
function fullJitter(attempt: number, baseMs = 100, capMs = 30_000): number {
  const backoff = Math.min(capMs, baseMs * Math.pow(2, attempt))
  return Math.random() * backoff  // Range: [0, backoff]
  // Spreads retries uniformly across the ENTIRE backoff window
  // Some callers retry very quickly, some after the full backoff
  // Load is spread uniformly — no synchronized storm
}

// ✅ Decorrelated jitter — maximum spread across attempts
let previousSleepMs = 100
function decorrelatedJitter(baseMs = 100, capMs = 30_000): number {
  const sleepMs = Math.min(capMs, Math.random() * (previousSleepMs * 3 - baseMs) + baseMs)
  previousSleepMs = sleepMs
  return sleepMs
  // Each sleep is drawn from [base, previous × 3] — decorrelated from attempt count
  // Prevents accidental synchronization even with similar attempt histories
}

// ❌ Arbitrary retry count — violates the caller's SLA silently
const MAX_RETRIES = 5  // "Seems reasonable"

// If each attempt takes up to 500ms (including timeout):
// 5 attempts × 500ms = 2500ms maximum
// If the caller's SLA is 800ms: this configuration breaks it on the 2nd attempt
// The service appears to work in isolation; breaks the SLA in the real system

// ✅ Derive retry budget from the caller's SLA
function calculateRetryBudget(config: {
  callerSlaMs: number,       // Our SLA to OUR caller
  perAttemptTimeoutMs: number, // Max time for each individual attempt
  backoffBaseMs: number,
  backoffCapMs: number,
}) {
  const { callerSlaMs, perAttemptTimeoutMs, backoffBaseMs, backoffCapMs } = config

  let totalBudgetMs = callerSlaMs
  let maxAttempts = 0

  // Simulate worst-case retry sequence
  for (let attempt = 0; attempt < 20; attempt++) {
    const attemptCost = perAttemptTimeoutMs + fullJitter(attempt, backoffBaseMs, backoffCapMs)
    if (totalBudgetMs < attemptCost) break  // Can't afford another attempt
    totalBudgetMs -= attemptCost
    maxAttempts++
  }

  return maxAttempts
}

// Example:
const budget = calculateRetryBudget({
  callerSlaMs: 800,           // Our caller expects a response within 800ms
  perAttemptTimeoutMs: 300,   // We give each attempt 300ms before timing out
  backoffBaseMs: 50,
  backoffCapMs: 500,
})
// Result: maxAttempts = 2 (first attempt = 300ms, 500ms remaining = 1 more attempt)
// Setting maxAttempts = 5 silently breaks the 800ms SLA

// ✅ Classify errors BEFORE retrying — never retry client errors
async function retryableRequest<T>(
  operation: () => Promise<T>,
  maxAttempts: number,
  context: string,
): Promise<T> {
  let attempt = 0

  while (true) {
    try {
      return await operation()
    } catch (err) {
      const statusCode = (err as any).status ?? (err as any).statusCode

      // 4xx errors: the CLIENT sent a bad request — retrying with the same payload will always fail
      if (statusCode >= 400 && statusCode < 500) {
        if (statusCode === 429) {
          // 429 IS retryable — but ONLY after respecting Retry-After
          const retryAfterSeconds = parseInt((err as any).headers?.['retry-after'] ?? '60', 10)
          if (attempt >= maxAttempts) throw err
          await sleep(retryAfterSeconds * 1000)  // Respect the provider's backoff
          attempt++
          continue
        }
        // 400, 401, 403, 404, 409, 422 — never retry
        throw err  // Fail immediately — the client must fix the request
      }

      // 5xx errors: potentially transient — retry with backoff (if idempotent)
      if (attempt >= maxAttempts) {
        logger.error(`${context}: exhausted ${maxAttempts} retry attempts`, { statusCode })
        throw err
      }

      const backoffMs = fullJitter(attempt, 100, 10_000)
      logger.warn(`${context}: attempt ${attempt + 1} failed, retrying in ${backoffMs}ms`, { statusCode })
      await sleep(backoffMs)
      attempt++
    }
  }
}

// ✅ Server-side idempotency key deduplication — safe retry guarantee
// (Detailed key schema covered in Distributed Architecture P2; summary here)

const idempotencyCache = new Map<string, { result: unknown; expiresAt: number }>()

async function createOrder(
  payload: CreateOrderRequest,
  idempotencyKey: string,  // Client generates: crypto.randomUUID() before first attempt
): Promise<OrderResult> {
  // Check if this key was already processed
  const cached = idempotencyCache.get(idempotencyKey)
  if (cached) {
    if (Date.now() < cached.expiresAt) {
      return cached.result as OrderResult  // Return same result — idempotent
    }
  }

  // Process the request
  const result = await processOrderCreation(payload)

  // Cache the result for 24 hours (key TTL should match max retry window + buffer)
  idempotencyCache.set(idempotencyKey, {
    result,
    expiresAt: Date.now() + 86_400_000,
  })

  return result
}

// Client: generate key ONCE before all attempts
const idempotencyKey = crypto.randomUUID()

// Retry loop — same key on every attempt
for (let attempt = 0; attempt < maxAttempts; attempt++) {
  try {
    return await orderService.createOrder(payload, idempotencyKey)
    // First success: creates order, caches result
    // Subsequent calls (if first response was lost): returns cached result
    // No duplicate orders, regardless of how many times the network retries
  } catch (err) {
    if (attempt < maxAttempts - 1) await sleep(fullJitter(attempt))
    else throw err
  }
}

// ✅ Kafka retry topic pattern — exponential retry with DLQ escalation
// Topics: orders.events → orders.retry.1m → orders.retry.10m → orders.retry.60m → orders.dlq

interface RetryMessage {
  originalTopic: string
  originalPartition: number
  originalOffset: string
  payload: unknown
  attemptCount: number
  firstAttemptAt: string
  lastAttemptAt: string
  lastError: string
}

async function processWithRetry(
  message: RetryMessage,
  producer: KafkaProducer,
) {
  try {
    await processPayload(message.payload)
  } catch (err) {
    const nextAttempt = message.attemptCount + 1
    const retryTopic = getRetryTopic(nextAttempt)  // 1m, 10m, 60m, or DLQ

    await producer.send({
      topic: retryTopic,
      messages: [{
        value: JSON.stringify({
          ...message,
          attemptCount: nextAttempt,
          lastAttemptAt: new Date().toISOString(),
          lastError: (err as Error).message,
        } satisfies RetryMessage),
        headers: {
          'retry-after': retryTopic === 'orders.dlq' ? '0' : getDelay(nextAttempt).toString(),
        },
      }]
    })
  }
}

function getRetryTopic(attempt: number): string {
  const topics = ['orders.retry.1m', 'orders.retry.10m', 'orders.retry.60m', 'orders.dlq']
  return topics[Math.min(attempt - 1, topics.length - 1)]
}

// ✅ SQS delay queue — AWS native retry with visibility timeout
import { SQSClient, SendMessageCommand, ChangeMessageVisibilityCommand } from '@aws-sdk/client-sqs'

const sqs = new SQSClient({ region: 'us-east-1' })

async function requeueWithDelay(
  message: SQSMessage,
  delaySeconds: number,
): Promise<void> {
  if (delaySeconds > 900) {
    // SQS max delay is 15 minutes — use DLQ for longer delays
    await sqs.send(new SendMessageCommand({
      QueueUrl: process.env.DLQ_URL,
      MessageBody: message.Body!,
      MessageAttributes: {
        RetryCount: {
          DataType: 'Number',
          StringValue: ((parseInt(message.MessageAttributes?.RetryCount?.StringValue ?? '0')) + 1).toString(),
        },
        LastError: { DataType: 'String', StringValue: 'max_delay_exceeded' },
        OriginalEnqueueTime: {
          DataType: 'String',
          StringValue: message.MessageAttributes?.OriginalEnqueueTime?.StringValue ?? new Date().toISOString(),
        },
      },
    }))
  } else {
    // Change visibility timeout — message becomes visible again after delaySeconds
    await sqs.send(new ChangeMessageVisibilityCommand({
      QueueUrl: process.env.QUEUE_URL,
      ReceiptHandle: message.ReceiptHandle!,
      VisibilityTimeout: delaySeconds,
    }))
  }
}

// ✅ Retry-After compliance — the most impactful single implementation
// RFC 9110: Retry-After can be a delay in seconds or an HTTP date

async function handleRateLimitError(
  response: Response,
  attempt: number,
  maxAttempts: number,
): Promise<void> {
  if (attempt >= maxAttempts) {
    throw new Error(`Rate limit exceeded: exhausted ${maxAttempts} attempts`)
  }

  const retryAfter = response.headers.get('retry-after')
  let waitMs: number

  if (retryAfter) {
    const retryAfterSeconds = parseInt(retryAfter, 10)

    if (isNaN(retryAfterSeconds)) {
      // HTTP date format: "Wed, 06 Sep 2026 17:43:00 GMT"
      const retryDate = new Date(retryAfter).getTime()
      waitMs = Math.max(0, retryDate - Date.now())
    } else {
      waitMs = retryAfterSeconds * 1000
    }

    // Add small jitter to the provider-specified wait
    // (prevents synchronized retry even when provider specifies a fixed delay)
    waitMs += Math.random() * 1000
  } else {
    // Provider didn't send Retry-After — use exponential backoff with full jitter
    // Start with a generous base (60s for rate limiting scenarios)
    waitMs = fullJitter(attempt, 60_000, 300_000)
  }

  logger.warn('Rate limited by provider', {
    attempt, maxAttempts, waitMs, retryAfter,
    trace_id: trace.getActiveSpan()?.spanContext().traceId,
  })

  await sleep(waitMs)
}