Traces  → Answer: "What happened and where did it happen?"
          Request ABC triggered order-service → payment-service → stripe-api
          Stripe call took 2.4s; total request took 3.1s

Metrics → Answer: "How much? How often? Is this normal?"
          payment-service: 99th percentile latency is 2.8s (baseline: 340ms)
          payment-service error rate: 12% (baseline: 0.2%)

Logs    → Answer: "What exactly happened? What was the state?"
          [ERROR] Payment charge failed: stripe_error=card_declined,
          amount=4999, currency=USD, user_id=usr_abc, trace_id=abc123, span_id=span003

Without traces: you know something failed but cannot find where in the distributed call chain.
Without metrics: you cannot tell if this is isolated or affecting 12% of users.
Without logs: you cannot see the exact payload, error code, or state at the moment of failure.
All three are required. Each answers a question the others cannot.

// ✅ Auto-instrumentation — zero code change for HTTP and DB spans
import { NodeSDK } from '@opentelemetry/sdk-node'
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http'
import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node'

const sdk = new NodeSDK({
  serviceName: 'order-service',
  serviceVersion: '2.1.0',
  traceExporter: new OTLPTraceExporter({
    url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT, // e.g. http://otel-collector:4318/v1/traces
  }),
  instrumentations: [
    getNodeAutoInstrumentations({
      '@opentelemetry/instrumentation-http': { enabled: true },    // Inbound HTTP spans
      '@opentelemetry/instrumentation-express': { enabled: true }, // Express route spans
      '@opentelemetry/instrumentation-pg': { enabled: true },      // PostgreSQL query spans
      '@opentelemetry/instrumentation-redis': { enabled: true },   // Redis command spans
      '@opentelemetry/instrumentation-grpc': { enabled: true },    // gRPC call spans
    }),
  ],
})

// Must start BEFORE requiring application code
sdk.start()
// Auto-instrumentation creates spans for:
// - Every incoming HTTP request (server span)
// - Every outgoing HTTP request (client span)
// - Every PostgreSQL query (with sanitized SQL statement)
// - Every Redis command
// - Every gRPC call

// ✅ Manual span for business logic that auto-instrumentation cannot see
import { trace, context, SpanStatusCode } from '@opentelemetry/api'

const tracer = trace.getTracer('order-service', '2.1.0')

async function processPayment(orderId: string, amountCents: number): Promise<PaymentResult> {
  // Create a child span within the current trace context
  return tracer.startActiveSpan('payment.process', async (span) => {
    try {
      span.setAttribute('order.id', orderId)              // Low cardinality — OK as attribute
      span.setAttribute('payment.currency', 'USD')        // Low cardinality — OK
      span.setAttribute('payment.provider', 'stripe')

      // ❌ High cardinality as attribute — never do this
      // span.setAttribute('user.id', userId)  // Millions of unique values → collector OOM

      // ✅ High cardinality as span event — correct approach
      span.addEvent('payment.attempt', {
        'user.id': userId,           // Per-event attribute — not aggregated across traces
        'amount.cents': amountCents,
        'idempotency.key': idempotencyKey,
      })

      const result = await stripeClient.charges.create({ amount: amountCents, currency: 'usd' })

      span.setAttribute('payment.status', 'succeeded')
      span.setAttribute('stripe.charge_id', result.id)
      span.setStatus({ code: SpanStatusCode.OK })
      return result
    } catch (err) {
      span.recordException(err as Error)  // Captures stack trace + message as span event
      span.setStatus({ code: SpanStatusCode.ERROR, message: (err as Error).message })
      throw err
    } finally {
      span.end()  // ALWAYS call end() — even on error
    }
  })
}

# traceparent header format (W3C Trace Context specification)
traceparent: 00-{traceId}-{parentSpanId}-{flags}

# Example:
traceparent: 00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01

# Fields:
# 00                              = version (always 00)
# 4bf92f3577b34da6a3ce929d0e0e4736 = 128-bit trace ID (hex, 32 chars)
# 00f067aa0ba902b7               = 64-bit parent span ID (hex, 16 chars)
# 01                              = flags (01 = sampled, 00 = not sampled)

// ❌ HTTP client that does not forward trace context — breaks the trace chain
const response = await fetch('http://payment-service:3001/charges', {
  method: 'POST',
  body: JSON.stringify(payload),
  // No traceparent header forwarded — payment-service starts a NEW root trace
  // The order-service trace and payment-service trace are disconnected
  // Jaeger shows two separate traces with no causal link
})

// ✅ OpenTelemetry auto-instrumentation propagates automatically
// BUT only if you use the instrumented HTTP client (Node fetch or axios with OTEL)
import { context, propagation } from '@opentelemetry/api'

async function callPaymentService(payload: ChargeRequest): Promise<ChargeResult> {
  const headers: Record<string, string> = { 'Content-Type': 'application/json' }

  // Inject current trace context into outgoing headers
  propagation.inject(context.active(), headers)
  // headers now contains: { 'traceparent': '00-{traceId}-{spanId}-01' }

  const response = await fetch('http://payment-service:3001/charges', {
    method: 'POST',
    headers,
    body: JSON.stringify(payload),
  })
  return response.json()
}

// Head-based sampling — decision made at the root span (before any child spans exist)
import { TraceIdRatioBased } from '@opentelemetry/sdk-trace-base'

const sampler = new TraceIdRatioBased(0.01) // Sample 1% of all traces
// Decision is made at the first service based on the Trace ID hash
// ALL spans in the trace share the same sampling decision
// Problem: A 500 error occurring in 0.1% of requests has a ~1% chance of being sampled
// Most errors are thrown away by head-based sampling

# OpenTelemetry Collector config — tail-based sampling policy
processors:
  tail_sampling:
    decision_wait: 10s    # Buffer spans for 10s before making the sampling decision
    num_traces: 50000     # Buffer up to 50k traces in memory
    expected_new_traces_per_sec: 1000

    policies:
      - name: errors-policy
        type: status_code
        status_code: { status_codes: [ERROR] }
        # Keep 100% of traces that contain any error span

      - name: high-latency-policy
        type: latency
        latency: { threshold_ms: 1000 }
        # Keep 100% of traces where total duration > 1 second

      - name: low-rate-baseline
        type: probabilistic
        probabilistic: { sampling_percentage: 1 }
        # Keep 1% of remaining (normal, fast) traces for baseline metrics

# Result: ~100% of error traces retained, ~100% of slow traces, ~1% of normal traces
# Storage cost: dramatically lower than 100% retention
# Visibility into problems: dramatically higher than head-based

// ✅ OpenTelemetry Semantic Conventions — standard attribute names
// Use these instead of inventing ad-hoc names

// HTTP server span (auto-instrumented by OTEL):
span.setAttribute('http.method', 'POST')          // not 'method' or 'httpMethod'
span.setAttribute('http.route', '/orders/:id')    // not 'route' or 'path'
span.setAttribute('http.status_code', 201)        // not 'statusCode' or 'status'
span.setAttribute('http.target', '/orders/42')

// Database span (auto-instrumented by OTEL):
span.setAttribute('db.system', 'postgresql')
span.setAttribute('db.name', 'orders_db')
span.setAttribute('db.operation', 'SELECT')
span.setAttribute('db.statement', 'SELECT id, status FROM orders WHERE id = $1')
// Note: Sanitize bind parameters — never include actual values in db.statement

// RPC span:
span.setAttribute('rpc.system', 'grpc')
span.setAttribute('rpc.service', 'PaymentService')
span.setAttribute('rpc.method', 'CreateCharge')
span.setAttribute('rpc.grpc.status_code', 0)  // 0 = OK

// ❌ High-cardinality attributes — never as span attributes
// span.setAttribute('user.id', userId)     // Millions of unique values
// span.setAttribute('order.id', orderId)   // Millions of unique values
// span.setAttribute('request.body', JSON.stringify(body))  // Unbounded size

// ✅ High-cardinality data belongs in span events (not span attributes)
// Span events are attached to a specific point in time within a span
// They are NOT aggregated across spans — no cardinality explosion

span.addEvent('order.validation.failed', {
  timestamp: Date.now(),
  attributes: {
    'user.id': userId,          // High cardinality — OK in event attributes
    'validation.rule': 'inventory_check',
    'sku.id': sku,              // High cardinality — OK in event
    'quantity.requested': qty,
    'quantity.available': available,
  }
})

// ✅ Structured JSON log — every field is queryable
import { createLogger, format, transports } from 'winston'
import { trace, context } from '@opentelemetry/api'

const logger = createLogger({
  format: format.combine(
    format.timestamp(),
    format.json(),
  ),
  transports: [new transports.Console()],
})

function getStructuredLogger(req?: Request) {
  const span = trace.getActiveSpan()
  const spanContext = span?.spanContext()

  return {
    info: (message: string, extra: Record<string, unknown> = {}) =>
      logger.info({
        message,
        // Correlation identifiers — mandatory for all logs
        trace_id: spanContext?.traceId,      // Links to distributed trace
        span_id: spanContext?.spanId,        // Links to exact span in trace
        service: process.env.SERVICE_NAME,
        service_version: process.env.SERVICE_VERSION,
        // Request context
        request_id: req?.headers['x-request-id'],
        user_id: req?.headers['x-user-id'],
        // Structured payload — never string interpolation
        ...extra,
      }),
    error: (message: string, err: Error, extra: Record<string, unknown> = {}) =>
      logger.error({
        message,
        trace_id: spanContext?.traceId,
        span_id: spanContext?.spanId,
        service: process.env.SERVICE_NAME,
        error: {
          name: err.name,
          message: err.message,
          stack: err.stack,
        },
        ...extra,
      }),
  }
}

{
  "level": "error",
  "message": "Payment charge failed",
  "timestamp": "2026-09-06T17:43:21.847Z",
  "trace_id": "4bf92f3577b34da6a3ce929d0e0e4736",
  "span_id": "00f067aa0ba902b7",
  "service": "payment-service",
  "service_version": "2.1.0",
  "user_id": "usr_abc",
  "error": {
    "name": "StripeError",
    "message": "Your card was declined.",
    "code": "card_declined",
    "decline_code": "insufficient_funds"
  },
  "payment": {
    "amount_cents": 4999,
    "currency": "USD",
    "idempotency_key": "order-ord_xyz-charge"
  }
}

Incident workflow:
1. Alert fires: payment-service error rate > 5%
2. Open Grafana → Tempo → search by time window
3. Find any trace with ERROR status → click → see waterfall timeline
4. Identify failing span: payment-service, span003, 2.4s, ERROR
5. Copy span_id: 00f067aa0ba902b7
6. Open Loki → query: {service="payment-service"} | json | span_id="00f067aa0ba902b7"
7. Exact log line with StripeError + decline_code: insufficient_funds
MTTR: 4 minutes. Without correlation: 45 minutes.

# Prometheus alerting rules — RED method per service
groups:
  - name: service-slos
    rules:
      # Rate (requests per second) — detect traffic anomalies
      - alert: UnusuallyLowRequestRate
        expr: |
          rate(http_server_requests_total{service="order-service"}[5m]) < 10
        for: 5m
        annotations:
          summary: "Order service receiving <10 req/s — possible upstream issue or deployment failure"

      # Errors — detect error rate spike
      - alert: HighErrorRate
        expr: |
          rate(http_server_requests_total{service="order-service",status=~"5.."}[5m])
          /
          rate(http_server_requests_total{service="order-service"}[5m]) > 0.05
        for: 2m
        annotations:
          summary: "Order service error rate > 5% (current: {{ $value | humanizePercentage }})"

      # Duration — detect latency regression
      - alert: HighLatencyP99
        expr: |
          histogram_quantile(0.99,
            rate(http_server_request_duration_seconds_bucket{service="order-service"}[5m])
          ) > 2.0
        for: 3m
        annotations:
          summary: "Order service P99 latency > 2s (current: {{ $value }}s)"