GET /realtime HTTP/1.1
Host: api.example.com
Upgrade: websocket
Connection: Upgrade
Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==
Sec-WebSocket-Version: 13

HTTP/1.1 101 Switching Protocols
Upgrade: websocket
Connection: Upgrade
Sec-WebSocket-Accept: s3pPLMBiTxaQ9kYGzzhZRbK+xOo=

// ❌ No connection limit — memory exhausts silently at ~65k connections
import { WebSocketServer } from 'ws'

const wss = new WebSocketServer({ port: 8080 })
wss.on('connection', (ws) => {
  // Each connection: ~4-8KB OS TCP buffer + application state
  // At 50,000 connections: 200–400MB memory consumed by sockets alone
  // Plus Node.js event loop entries, listeners, per-connection data structures
  ws.on('message', (data) => { /* ... */ })
})

// ✅ Explicit capacity planning with connection limit
const MAX_CONNECTIONS = parseInt(process.env.MAX_WS_CONNECTIONS ?? '10000', 10)
const wss = new WebSocketServer({ port: 8080, maxPayload: 65536 }) // 64KB max message

wss.on('connection', (ws, req) => {
  if (wss.clients.size > MAX_CONNECTIONS) {
    ws.close(1008, 'Connection limit reached')
    return
  }

  const connectionId = crypto.randomUUID()
  const userId = req.headers['x-user-id'] as string

  // Heartbeat: detect dead connections (TCP half-open)
  ws.isAlive = true
  ws.on('pong', () => { ws.isAlive = true })

  ws.on('close', () => {
    // Clean up per-connection state immediately — no GC dependency
    connectionRegistry.delete(connectionId)
  })
})

// Heartbeat interval — terminate connections that stop responding
const heartbeat = setInterval(() => {
  wss.clients.forEach((ws) => {
    if (!ws.isAlive) {
      ws.terminate()  // SIGKILL the connection — frees the file descriptor immediately
      return
    }
    ws.isAlive = false
    ws.ping()  // Expects a pong within the next interval
  })
}, 30_000)

wss.on('close', () => clearInterval(heartbeat))

Node 1: 3,000 connections (users A, B, C, ...)
Node 2: 3,000 connections (users D, E, F, ...)
Node 3: 4,000 connections (users G, H, I, ...)

Event: "Order shipped for user B" — processed by Order Service → publishes to Node 1
Node 1 delivers to user B's WebSocket ✅

Event: "Order shipped for user D" — processed by Order Service → publishes to Node 1
Node 1 looks for user D's connection → not found ❌ (user D is on Node 2)
Message is dropped silently.

// ✅ Redis PubSub — all nodes subscribe to the same channel
import { createClient } from 'redis'
import { WebSocketServer } from 'ws'

const publisher = createClient({ url: process.env.REDIS_URL })
const subscriber = publisher.duplicate()

await Promise.all([publisher.connect(), subscriber.connect()])

// Each WebSocket node subscribes to all relevant channels
const connectionsByUserId = new Map<string, Set<WebSocket>>()

await subscriber.subscribe('orders:updates', (message) => {
  const event = JSON.parse(message) as { userId: string; payload: unknown }

  // Find this user's connections on THIS node
  const userConnections = connectionsByUserId.get(event.userId)
  if (!userConnections) return // User not connected to this node — no-op, another node handles it

  userConnections.forEach((ws) => {
    if (ws.readyState === WebSocket.OPEN) {
      ws.send(JSON.stringify(event.payload))
    }
  })
})

// When the Order Service ships an order, it publishes to Redis
// ALL WebSocket nodes receive the message and deliver to their local connections for that user
async function publishOrderUpdate(userId: string, orderEvent: OrderEvent) {
  await publisher.publish('orders:updates', JSON.stringify({ userId, payload: orderEvent }))
}

// ✅ STOMP server with @stomp/stompjs on the server side
import { Server } from '@stomp/stompjs'
import { WebSocketServer } from 'ws'

const wss = new WebSocketServer({ port: 8080 })

const stompServer = new Server({
  webSocketServer: wss,

  onConnect(frame) {
    const userId = frame.headers['x-user-id']
    console.log(`STOMP CONNECT from user ${userId}`)
    return { headers: { 'server': 'OrderService/1.0' } }
  },

  onSubscribe(subscription, frame) {
    // Client subscribes to /user/orders or /topic/announcements
    const destination = frame.headers['destination']

    if (destination.startsWith('/user/')) {
      // Per-user subscription — route to this connection only
      registerUserSubscription(subscription, destination)
    } else if (destination.startsWith('/topic/')) {
      // Broadcast subscription — route to all subscribers
      registerTopicSubscription(subscription, destination)
    }
  },
})

// ✅ SSE endpoint — standard HTTP response, no WebSocket upgrade needed
app.get('/events/orders/:userId', (req, res) => {
  const userId = req.params.userId

  // SSE requires these exact headers
  res.setHeader('Content-Type', 'text/event-stream')
  res.setHeader('Cache-Control', 'no-cache')
  res.setHeader('Connection', 'keep-alive')
  res.setHeader('X-Accel-Buffering', 'no')  // Disable NGINX buffering — critical for SSE
  res.flushHeaders()  // Send headers immediately — opens the stream

  // Send initial comment to establish connection (also resets reconnect timeout)
  res.write(': connected\n\n')

  // Send a named event with a last-event-id for reconnect support
  let lastEventId = 0

  function sendEvent(eventType: string, data: unknown) {
    lastEventId++
    res.write(`id: ${lastEventId}\n`)
    res.write(`event: ${eventType}\n`)
    res.write(`data: ${JSON.stringify(data)}\n\n`) // Double newline terminates the event
  }

  // Subscribe to Redis for this user's events
  const redisSubscriber = redis.duplicate()
  redisSubscriber.subscribe(`sse:user:${userId}`, (message) => {
    const event = JSON.parse(message)
    sendEvent(event.type, event.payload)
  })

  // Heartbeat — prevents proxy timeouts (many proxies close idle connections after 60s)
  const heartbeat = setInterval(() => {
    res.write(': heartbeat\n\n')
  }, 25_000)

  req.on('close', () => {
    clearInterval(heartbeat)
    redisSubscriber.unsubscribe()
    redisSubscriber.quit()
  })
})

// Browser client — EventSource handles reconnection automatically
const es = new EventSource('/events/orders/usr_abc')

es.addEventListener('order.shipped', (event) => {
  console.log('Order shipped:', JSON.parse(event.data))
})

es.addEventListener('order.delivered', (event) => {
  console.log('Order delivered:', JSON.parse(event.data))
})

// On reconnect, the browser sends:
// GET /events/orders/usr_abc
// Last-Event-ID: 42
// The server uses this to replay events after event ID 42
// Enabling exactly-once delivery across reconnects

HTTP/2 over TCP: Multiplexed streams on a single TCP connection

Stream 1: [frame 1] [frame 2] [frame 3] ...
Stream 2: [frame 1] [frame 2] [frame 3] ...
Stream 3: [frame 1] [frame 2] [frame 3] ...

If frame 2 of Stream 1 is lost in transit:
- TCP must wait for retransmission of the lost frame
- ALL streams (1, 2, 3) are blocked — even though Streams 2 and 3 have no missing data
- This is TCP head-of-line blocking at the transport layer
- HTTP/2 multiplexing works at the application layer; TCP serializes bytes at the transport layer

HTTP/3 over QUIC: Independent streams, each with its own flow control

Stream 1: [frame 1] [frame 2 LOST] [frame 3] ...
Stream 2: [frame 1] [frame 2] [frame 3] ...
Stream 3: [frame 1] [frame 2] [frame 3] ...

If frame 2 of Stream 1 is lost:
- Only Stream 1 is retransmitted — Streams 2 and 3 continue without interruption
- Each QUIC stream has independent sequencing and flow control
- Loss on one stream does not affect any other stream on the same connection

HTTP/1.1 / HTTP/2 over TCP: 1–3 RTTs before data
  RTT 1: TCP SYN / SYN-ACK
  RTT 2: TLS ClientHello / ServerHello
  RTT 3: TLS Finished / Application Data begins

HTTP/3 over QUIC + TLS 1.3 (first connection): 1 RTT
  RTT 1: QUIC Initial (combines TCP SYN + TLS ClientHello)
          Server sends QUIC Handshake + CertificateVerify
  RTT 0 (data): Client can send 0-RTT data with session resumption
               (server verifies before processing — replay protection)

Mobile client transitions from WiFi (192.168.1.5) to LTE (10.0.0.12):

HTTP/2 over TCP: Connection is identified by (src-ip, src-port, dst-ip, dst-port)
  IP change → connection ID changes → TCP connection drops → reconnect required
  All in-flight requests are lost; TLS handshake must be repeated

HTTP/3 over QUIC: Connection is identified by a stable Connection ID (random bytes)
  IP change → client sends new packet with the same Connection ID → server accepts
  Connection migrates transparently — no reconnect, no handshake repeat
  In-flight requests continue uninterrupted