This article builds the multi-provider AI gateway: a token-bucket rate limiter with org-level quotas, a circuit breaker that automatically reroutes traffic away from degraded providers, and a hedged request strategy that sends the same request to two providers simultaneously and returns whichever responds first — compressing P99 latency by 60%+ on high-percentile requests.
Who this is for: Engineers running AI features in production who have had a single provider outage take down their entire service. After this article, a single provider going offline requires zero manual intervention.
Different customers have different consumption agreements. A token-bucket limiter allows short bursts while enforcing long-run average rate compliance.
import { createClient } from 'redis'
const redis = createClient({ url: process.env.REDIS_URL })
interface RateLimitConfig {
tokensPerMinute: number
burstCapacity: number
refillRatePerMs: number
}
const ORG_LIMITS: Record<string, RateLimitConfig> = {
free: { tokensPerMinute: 10_000, burstCapacity: 20_000, refillRatePerMs: 10_000/60000 },
pro: { tokensPerMinute: 100_000, burstCapacity: 200_000, refillRatePerMs: 100_000/60000 },
enterprise: { tokensPerMinute: 500_000, burstCapacity: 1_000_000, refillRatePerMs: 500_000/60000 },
}
export async function checkRateLimit(
orgId: string,
tier: keyof typeof ORG_LIMITS,
requestedTokens: number,
): Promise<{ allowed: boolean; remainingTokens: number; retryAfterMs: number }> {
const config = ORG_LIMITS[tier]
const key = `ratelimit:${orgId}`
const now = Date.now()
const result = await redis.eval(
`
local key = KEYS[1]
local now = tonumber(ARGV[1])
local requested = tonumber(ARGV[2])
local capacity = tonumber(ARGV[3])
local refillRate = tonumber(ARGV[4])
local bucket = redis.call('HMGET', key, 'tokens', 'lastRefill')
local tokens = tonumber(bucket[1]) or capacity
local lastRefill = tonumber(bucket[2]) or now
-- Refill based on elapsed time
local elapsed = now - lastRefill
tokens = math.min(capacity, tokens + elapsed * refillRate)
local allowed = tokens >= requested
if allowed then tokens = tokens - requested end
redis.call('HMSET', key, 'tokens', tokens, 'lastRefill', now)
redis.call('EXPIRE', key, 3600)
return {allowed and 1 or 0, math.floor(tokens), allowed and 0 or math.floor((requested - tokens) / refillRate)}
`,
[key],
[now, requestedTokens, config.burstCapacity, config.refillRatePerMs],
) as [number, number, number]
return {
allowed: result[0] === 1,
remainingTokens: result[1],
retryAfterMs: result[2],
}
}
type CircuitState = 'CLOSED' | 'OPEN' | 'HALF_OPEN'
interface CircuitConfig {
failureThreshold: number
successThreshold: number
timeoutMs: number
monitoringWindowMs: number
}
export class CircuitBreaker {
private state: CircuitState = 'CLOSED'
private failures = 0
private successes = 0
private lastFailureAt = 0
constructor(
private readonly providerName: string,
private readonly config: CircuitConfig = {
failureThreshold: 5,
successThreshold: 2,
timeoutMs: 60_000, // 60 seconds in OPEN
monitoringWindowMs: 30_000,
},
) {}
async execute<T>(fn: () => Promise<T>): Promise<T> {
this.maybeMoveToHalfOpen()
if (this.state === 'OPEN') {
throw new CircuitOpenError(`Circuit OPEN for provider: ${this.providerName}`)
}
try {
const result = await fn()
this.onSuccess()
return result
} catch (error) {
this.onFailure()
throw error
}
}
private maybeMoveToHalfOpen(): void {
if (this.state === 'OPEN' && Date.now() - this.lastFailureAt > this.config.timeoutMs) {
this.state = 'HALF_OPEN'
this.successes = 0
}
}
private onSuccess(): void {
if (this.state === 'HALF_OPEN') {
this.successes++
if (this.successes >= this.config.successThreshold) {
this.state = 'CLOSED'
this.failures = 0
}
} else {
this.failures = 0
}
}
private onFailure(): void {
this.lastFailureAt = Date.now()
this.failures++
if (this.failures >= this.config.failureThreshold) {
this.state = 'OPEN'
}
if (this.state === 'HALF_OPEN') {
this.state = 'OPEN'
}
}
get isOpen(): boolean { return this.state === 'OPEN' }
}
export class CircuitOpenError extends Error {
readonly name = 'CircuitOpenError' as const
}
import Anthropic from '@anthropic-ai/sdk'
import OpenAI from 'openai'
interface ProviderEntry {
name: string
breaker: CircuitBreaker
priority: number
}
const PROVIDERS: ProviderEntry[] = [
{ name: 'anthropic', breaker: new CircuitBreaker('anthropic'), priority: 1 },
{ name: 'openai', breaker: new CircuitBreaker('openai'), priority: 2 },
{ name: 'deepseek', breaker: new CircuitBreaker('deepseek'), priority: 3 },
]
export class MultiProviderGateway {
async *streamWithFallback(
request: GatewayRequest,
signal: AbortSignal,
): AsyncGenerator<WorkspaceAction> {
const available = PROVIDERS
.filter(p => !p.breaker.isOpen)
.sort((a, b) => a.priority - b.priority)
if (available.length === 0) {
throw new Error('All AI providers unavailable (circuit breakers OPEN)')
}
let lastError: Error | null = null
for (const provider of available) {
try {
yield* await provider.breaker.execute(async () =>
this.streamFromProvider(provider.name, request, signal)
)
return
} catch (error) {
lastError = error as Error
if (error instanceof CircuitOpenError) continue
if (error instanceof GatewaySchemaError) throw error
console.warn(`[Gateway] Provider ${provider.name} failed: ${(error as Error).message}`)
}
}
throw lastError ?? new Error('All providers failed')
}
}
Hedging sends the same request to two providers simultaneously and returns the first response — discarding (and aborting) the slower one. This is a deliberate trade: slightly higher average cost in exchange for dramatically compressed P99 latency.
export class HedgedGateway {
async *streamHedged(
request: GatewayRequest,
primaryProvider: string,
hedgeProvider: string,
hedgeDelayMs = 1000,
): AsyncGenerator<WorkspaceAction> {
const primaryController = new AbortController()
const hedgeController = new AbortController()
const primary = this.streamFromProvider(primaryProvider, request, primaryController.signal)
const hedgePromise = new Promise<AsyncGenerator<WorkspaceAction>>((resolve) => {
setTimeout(() => {
resolve(this.streamFromProvider(hedgeProvider, request, hedgeController.signal))
}, hedgeDelayMs)
})
const winner = await Promise.race([
primary.next().then(first => ({ source: 'primary', first, gen: primary })),
hedgePromise.then(gen => gen.next().then(first => ({ source: 'hedge', first, gen }))),
])
if (winner.source === 'primary') {
hedgeController.abort()
} else {
primaryController.abort()
}
if (!winner.first.done) yield winner.first.value
for await (const action of winner.gen) {
yield action
}
}
}