"How do I implement rate limiting in Node.js?"
→ [0.023, -0.412, 0.887, ..., 0.031]  // 1536 dimensions (text-embedding-3-large)

"What is the best approach for throttling API requests in Express?"
→ [0.019, -0.398, 0.901, ..., 0.028]  // Very close in 1536-dimensional space

"What is the capital of France?"
→ [0.741, 0.123, -0.302, ..., 0.618]  // Far away

// ✅ Normalise at ingestion — do once, benefit every query
// file: packages/db/src/embeddings.ts

export function l2Normalize(vector: number[]): number[] {
  const magnitude = Math.sqrt(vector.reduce((sum, v) => sum + v * v, 0))
  if (magnitude === 0) throw new Error('Cannot normalise zero vector')
  return vector.map(v => v / magnitude)
}

export async function embedAndNormalise(text: string): Promise<number[]> {
  const response = await openai.embeddings.create({
    model: 'text-embedding-3-large',
    input: text,
    dimensions: 1536,  // Matryoshka: reducible to 256 for fast ANN (Part 9)
  })
  return l2Normalize(response.data[0].embedding)
}

// ❌ Anti-Pattern — naive character chunking destroys semantic coherence
function naiveChunk(text: string, size = 500): string[] {
  const chunks: string[] = []
  for (let i = 0; i < text.length; i += size) {
    chunks.push(text.slice(i, i + size))
  }
  return chunks
}

// Input: A Markdown section about pgvector HNSW indexing
// Output chunk boundary might split:
// Chunk 3: "...The HNSW index uses a layered graph where each no"
// Chunk 4: "de connects to M nearest neighbours. The ef_construction pa..."
// → Neither chunk makes sense in isolation

// ✅ AST-aware Markdown chunker
// file: packages/db/src/chunker/MarkdownChunker.ts

import { remark } from 'remark'
import type { Root, Node, Parent } from 'mdast'
import { createHash } from 'node:crypto'

export interface DocumentChunk {
  content: string
  heading: string      // Nearest parent heading — provides context
  chunkIndex: number
  sha256: string       // Content hash for deduplication
  tokenCount: number   // Pre-computed for context budget scheduling
}

const MAX_CHUNK_TOKENS = 400  // Leave room for contextual prefix (Part 12)
const MIN_CHUNK_TOKENS = 50   // Avoid micro-chunks that carry no information

export class MarkdownChunker {
  async chunk(markdown: string): Promise<DocumentChunk[]> {
    const ast = remark().parse(markdown) as Root
    const chunks: DocumentChunk[] = []
    let currentContent = ''
    let currentHeading = 'Introduction'
    let chunkIndex = 0

    const flush = () => {
      const trimmed = currentContent.trim()
      if (!trimmed) return
      const tokens = this.estimateTokens(trimmed)
      if (tokens < MIN_CHUNK_TOKENS) return  // Skip trivially small chunks

      chunks.push({
        content: trimmed,
        heading: currentHeading,
        chunkIndex: chunkIndex++,
        sha256: createHash('sha256').update(trimmed).digest('hex'),
        tokenCount: tokens,
      })
      currentContent = ''
    }

    for (const node of ast.children) {
      if (node.type === 'heading') {
        flush()  // Each heading boundary = chunk boundary
        currentHeading = this.extractText(node as Parent)
      } else if (node.type === 'code') {
        // Code blocks are always their own chunk — never split
        flush()
        const codeContent = `\`\`\`${(node as any).lang ?? ''}\n${(node as any).value}\n\`\`\``
        chunks.push({
          content: codeContent,
          heading: currentHeading,
          chunkIndex: chunkIndex++,
          sha256: createHash('sha256').update(codeContent).digest('hex'),
          tokenCount: this.estimateTokens(codeContent),
        })
      } else {
        const nodeText = this.extractText(node as Parent)
        currentContent += '\n' + nodeText
        // If accumulated content exceeds token limit, flush mid-section
        if (this.estimateTokens(currentContent) >= MAX_CHUNK_TOKENS) {
          flush()
        }
      }
    }
    flush()  // Final chunk

    return chunks
  }

  private estimateTokens(text: string): number {
    // Approximation: ~4 chars per token for English prose.
    // For code-heavy documents (2–3 chars/token), lower MAX_CHUNK_TOKENS to ~300
    // to avoid oversized chunks. Use TokenAwareRequestBuilder (Part 2) in the
    // ingestion pipeline when token accuracy matters more than startup cost.
    return Math.ceil(text.length / 4)
  }

  private extractText(node: Parent): string {
    if (!node.children) return ''
    return node.children
      .map((child: any) => child.value ?? this.extractText(child))
      .join('')
  }
}

// ✅ Deduplication-aware ingestion pipeline
// file: packages/db/src/ingestion/DocumentIngester.ts

import { db } from '../drizzle'
import { chunks } from '../schema'
import { eq } from 'drizzle-orm'
import { MarkdownChunker } from '../chunker/MarkdownChunker'
import { embedAndNormalise } from '../embeddings'

export async function ingestDocument(
  documentId: string,
  content: string,
): Promise<{ created: number; skipped: number }> {
  const chunker = new MarkdownChunker()
  const newChunks = await chunker.chunk(content)

  // Fetch existing hashes for this document
  const existingHashes = new Set(
    (await db.select({ sha256: chunks.sha256 })
      .from(chunks)
      .where(eq(chunks.documentId, documentId)))
      .map(r => r.sha256)
  )

  let created = 0
  let skipped = 0

  // Process in batches of 20 to respect rate limits
  for (let i = 0; i < newChunks.length; i += 20) {
    const batch = newChunks.slice(i, i + 20)
    const toEmbed = batch.filter(c => !existingHashes.has(c.sha256))
    const alreadyExist = batch.filter(c => existingHashes.has(c.sha256))

    skipped += alreadyExist.length

    if (toEmbed.length === 0) continue

    // Batch embed only genuinely new chunks
    const embeddings = await Promise.all(
      toEmbed.map(c => embedAndNormalise(c.content))
    )

    await db.insert(chunks).values(
      toEmbed.map((chunk, idx) => ({
        documentId,
        content: chunk.content,
        heading: chunk.heading,
        chunkIndex: chunk.chunkIndex,
        sha256: chunk.sha256,
        tokenCount: chunk.tokenCount,
        embedding: embeddings[idx],  // pgvector stores as float4[]
      }))
    )

    created += toEmbed.length
  }

  return { created, skipped }
}

// ✅ Drizzle schema — chunks table with pgvector column
// file: packages/db/src/schema/chunks.ts

import { pgTable, uuid, text, integer, vector, index } from 'drizzle-orm/pg-core'

export const chunks = pgTable('chunks', {
  id:          uuid('id').primaryKey().defaultRandom(),
  documentId:  uuid('document_id').notNull(),
  orgId:       uuid('org_id').notNull(),         // Multi-tenant partition key
  content:     text('content').notNull(),
  heading:     text('heading').notNull(),
  chunkIndex:  integer('chunk_index').notNull(),
  sha256:      text('sha256').notNull(),          // Deduplication key
  tokenCount:  integer('token_count').notNull(),
  // 1536-dimensional normalised float vector (text-embedding-3-large)
  embedding:   vector('embedding', { dimensions: 1536 }).notNull(),
}, (table) => ({
  // HNSW index — built in Part 9
  embeddingIdx: index('chunks_embedding_hnsw_idx')
    .using('hnsw', table.embedding.op('vector_ip_ops'))
    .with({ m: 16, ef_construction: 64 }),
  // Compound index for tenant-scoped lookups
  orgDocIdx: index('chunks_org_doc_idx').on(table.orgId, table.documentId),
  // Unique constraint prevents duplicate chunk content per document
  sha256DocIdx: index('chunks_sha256_doc_idx').on(table.sha256, table.documentId),
}))