// ✅ Canonical evaluation dataset schema
// file: tests/evals/fixtures/rag-eval-dataset.ts

export interface EvalCase {
  id: string
  question: string
  groundTruthAnswer: string    // Written by human domain expert
  groundTruthChunks?: string[] // Expected relevant chunks (for Context Recall)
  metadata: {
    difficulty: 'easy' | 'medium' | 'hard'
    topic: string
    addedAt: string
  }
}

export interface EvalResult {
  caseId: string
  question: string
  retrievedChunks: string[]
  generatedAnswer: string
  faithfulnessScore: number     // 0.0 – 1.0
  answerRelevanceScore: number  // 0.0 – 1.0
  contextRecallScore: number    // 0.0 – 1.0 (when groundTruthChunks provided)
}

// Minimum acceptable scores — failure below these blocks the PR
export const QUALITY_THRESHOLDS = {
  faithfulness: 0.88,
  answerRelevance: 0.82,
  contextRecall: 0.80,
}

// ✅ Faithfulness evaluator — LLM-as-a-Judge
// file: tests/evals/evaluators/Faithfulness.ts

import { generateObject } from 'ai'
import { createAnthropic } from '@ai-sdk/anthropic'
import { z } from 'zod'

const anthropic = createAnthropic()

const ClaimsSchema = z.object({
  claims: z.array(z.object({
    claim: z.string(),
    supportedByContext: z.boolean(),
    evidence: z.string().optional(),
  })),
  faithfulnessScore: z.number().min(0).max(1),
})

export async function scoreFaithfulness(
  question: string,
  answer: string,
  context: string[],
): Promise<{ score: number; breakdown: z.infer<typeof ClaimsSchema> }> {
  const contextText = context.map((c, i) => `[${i + 1}] ${c}`).join('\n\n')

  const { object } = await generateObject({
    model: anthropic('claude-3-5-sonnet-20241022'),
    schema: ClaimsSchema,
    temperature: 0,
    prompt: `Evaluate the faithfulness of the following answer to the retrieved context.

QUESTION: ${question}

RETRIEVED CONTEXT:
${contextText}

ANSWER TO EVALUATE:
${answer}

For each distinct factual claim in the answer:
1. Identify the claim
2. Determine if it is directly supported by the retrieved context
3. If supported, cite the evidence from context

Then compute faithfulnessScore = (supported claims) / (total claims).
If the answer makes no factual claims, score 1.0.`,
  })

  return { score: object.faithfulnessScore, breakdown: object }
}

// ✅ Answer Relevance — back-generation method
// file: tests/evals/evaluators/AnswerRelevance.ts

import { generateObject } from 'ai'
import { createAnthropic } from '@ai-sdk/anthropic'
import { z } from 'zod'
import { embedAndNormalise } from '../../../packages/db/src/embeddings'

const anthropic = createAnthropic()

export async function scoreAnswerRelevance(
  originalQuestion: string,
  answer: string,
  n = 3,  // Number of back-generated questions
): Promise<number> {
  const { object } = await generateObject({
    model: anthropic('claude-3-haiku-20240307'),
    schema: z.object({ questions: z.array(z.string()).length(n) }),
    temperature: 0.2,
    prompt: `Given this answer, generate ${n} different questions that this answer could be responding to.
ANSWER: ${answer}
Generate exactly ${n} questions.`,
  })

  // Embed original question and all back-generated questions
  const [origEmbedding, ...genEmbeddings] = await Promise.all([
    embedAndNormalise(originalQuestion),
    ...object.questions.map(q => embedAndNormalise(q)),
  ])

  // Compute cosine similarity (dot product of normalised vectors)
  const similarities = genEmbeddings.map(gen =>
    gen.reduce((sum, val, i) => sum + val * origEmbedding[i], 0)
  )

  return similarities.reduce((sum, s) => sum + s, 0) / similarities.length
}

# .github/workflows/rag-evals.yml
name: RAG Quality Gate

on:
  pull_request:
    paths:
      - 'src/gateway/**'
      - 'packages/db/src/retrieval/**'
      - 'tests/evals/**'
      - 'content/**'

jobs:
  rag-evals:
    runs-on: ubuntu-latest
    timeout-minutes: 15

    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-node@v4
        with: { node-version: '22' }

      - run: npm ci --workspace packages/db

      - name: Run RAG evaluation suite
        run: node tests/evals/run-evals.mjs
        env:
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          DATABASE_URL: ${{ secrets.CI_DATABASE_URL }}
          EVAL_DATASET: tests/evals/fixtures/rag-eval-dataset.json

      - name: Upload eval results
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: rag-eval-results
          path: tests/evals/results/latest.json

// ✅ Eval runner — blocks CI on threshold violation
// file: tests/evals/run-evals.mjs

import { dataset } from './fixtures/rag-eval-dataset.json' with { type: 'json' }
import { retrieve } from '../../packages/db/src/retrieval/RetrievalPipeline.ts'
import { scoreFaithfulness } from './evaluators/Faithfulness.ts'
import { scoreAnswerRelevance } from './evaluators/AnswerRelevance.ts'
import { QUALITY_THRESHOLDS } from './fixtures/rag-eval-dataset.ts'

const results = []
let passed = 0
let failed = 0

for (const evalCase of dataset) {
  const chunks = await retrieve('ci-org', evalCase.question, { finalCount: 5 })
  const answer = await generateAnswer(evalCase.question, chunks)

  const [faithfulness, relevance] = await Promise.all([
    scoreFaithfulness(evalCase.question, answer, chunks.map(c => c.content)),
    scoreAnswerRelevance(evalCase.question, answer),
  ])

  const caseResult = {
    id: evalCase.id,
    faithfulness: faithfulness.score,
    answerRelevance: relevance,
    passed: faithfulness.score >= QUALITY_THRESHOLDS.faithfulness &&
            relevance >= QUALITY_THRESHOLDS.answerRelevance,
  }

  results.push(caseResult)
  if (caseResult.passed) passed++; else failed++

  console.log(`[${caseResult.passed ? '✓' : '✗'}] ${evalCase.id}: ` +
    `F=${caseResult.faithfulness.toFixed(2)} AR=${caseResult.answerRelevance.toFixed(2)}`)
}

const avgFaithfulness = results.reduce((s, r) => s + r.faithfulness, 0) / results.length
const avgRelevance = results.reduce((s, r) => s + r.answerRelevance, 0) / results.length

console.log(`\nSummary: ${passed}/${results.length} passed`)
console.log(`Avg Faithfulness: ${avgFaithfulness.toFixed(3)} (threshold: ${QUALITY_THRESHOLDS.faithfulness})`)
console.log(`Avg Answer Relevance: ${avgRelevance.toFixed(3)} (threshold: ${QUALITY_THRESHOLDS.answerRelevance})`)

if (failed > 0 || avgFaithfulness < QUALITY_THRESHOLDS.faithfulness) {
  console.error('\n❌ RAG quality gate FAILED')
  process.exit(1)
}

console.log('\n✅ RAG quality gate PASSED')