Evaluating Your AI Application

import os
import json
from dataclasses import dataclass, field
from openai import OpenAI

client = OpenAI(api_key=os.environ.get("OPENAI_API_KEY"))

# --- SYSTEM UNDER TEST ---
# Simple Q&A system that we're evaluating

def run_qa_system(question: str) -> str:
  """The AI app we're testing."""
  response = client.chat.completions.create(
      model="gpt-4o-mini",
      messages=[
          {
              "role": "system",
              "content": "You are a helpful assistant. Answer concisely in 1-2 sentences."
          },
          {"role": "user", "content": question}
      ],
      max_tokens=150,
      temperature=0,
  )
  return response.choices[0].message.content

# --- TEST CASES ---

@dataclass
class TestCase:
  id: str
  input: str
  must_contain: list[str] = field(default_factory=list)
  must_not_contain: list[str] = field(default_factory=list)
  rubric: str = ""

TEST_CASES = [
  TestCase(
      id="tc-001",
      input="What is the capital of France?",
      must_contain=["Paris"],
      must_not_contain=["Berlin", "London", "Madrid"],
      rubric="The answer must correctly identify Paris as the capital of France.",
  ),
  TestCase(
      id="tc-002",
      input="What are the primary colors in painting?",
      must_contain=["red", "blue", "yellow"],
      rubric="The answer must name red, blue, and yellow as the three primary colors.",
  ),
  TestCase(
      id="tc-003",
      input="Explain what an API is in one sentence.",
      must_not_contain=["I cannot", "I don't know"],
      rubric=(
          "Score 1-5 on: (1) accuracy - correctly describes what an API is, "
          "(2) conciseness - fits in 1-2 sentences, "
          "(3) clarity - understandable to a non-technical reader."
      ),
  ),
]

# --- ASSERTION EVAL ---

def run_assertion_eval(tc: TestCase, output: str) -> dict:
  failures = []

  for required in tc.must_contain:
      if required.lower() not in output.lower():
          failures.append(f"Missing required term: '{required}'")

  for forbidden in tc.must_not_contain:
      if forbidden.lower() in output.lower():
          failures.append(f"Contains forbidden term: '{forbidden}'")

  return {
      "passed": len(failures) == 0,
      "failures": failures,
  }

# --- LLM-AS-JUDGE EVAL ---

JUDGE_PROMPT = """You are an AI evaluation judge. Score the following AI response.

Question: {question}
AI Response: {response}

Rubric: {rubric}

Respond with JSON only:
{{
"score": <integer 1-5>,
"reasoning": "<one sentence explanation>"
}}"""

def run_llm_judge(tc: TestCase, output: str) -> dict:
  if not tc.rubric:
      return {"score": None, "reasoning": "No rubric defined"}

  prompt = JUDGE_PROMPT.format(
      question=tc.input,
      response=output,
      rubric=tc.rubric,
  )

  response = client.chat.completions.create(
      model="gpt-4o",   # Stronger judge model
      messages=[{"role": "user", "content": prompt}],
      max_tokens=200,
      temperature=0,
      response_format={"type": "json_object"},
  )

  try:
      return json.loads(response.choices[0].message.content)
  except json.JSONDecodeError:
      return {"score": None, "reasoning": "Judge returned invalid JSON"}

# --- RUN THE EVAL SUITE ---

def run_eval_suite(test_cases: list[TestCase]) -> dict:
  results = []
  total_score = 0
  total_assertion_passes = 0

  print(f"Running {len(test_cases)} test cases...\n{'='*50}")

  for tc in test_cases:
      output = run_qa_system(tc.input)
      assertion_result = run_assertion_eval(tc, output)
      judge_result = run_llm_judge(tc, output)

      if assertion_result["passed"]:
          total_assertion_passes += 1

      if judge_result.get("score"):
          total_score += judge_result["score"]

      result = {
          "test_id": tc.id,
          "input": tc.input,
          "output": output,
          "assertion_passed": assertion_result["passed"],
          "assertion_failures": assertion_result["failures"],
          "judge_score": judge_result.get("score"),
          "judge_reasoning": judge_result.get("reasoning"),
      }
      results.append(result)

      status = "PASS" if assertion_result["passed"] else "FAIL"
      score_str = str(judge_result.get("score", "N/A"))
      print(f"[{tc.id}] Assertion: {status} | Judge: {score_str}/5")
      if assertion_result["failures"]:
          for f in assertion_result["failures"]:
              print(f"  ✗ {f}")

  n = len(test_cases)
  summary = {
      "total": n,
      "assertion_pass_rate": total_assertion_passes / n,
      "avg_judge_score": total_score / n if total_score else None,
      "results": results,
  }

  print(f"\n{'='*50}")
  print(f"Assertion pass rate: {summary['assertion_pass_rate']*100:.0f}%")
  if summary["avg_judge_score"]:
      print(f"Avg judge score: {summary['avg_judge_score']:.1f}/5")

  return summary

summary = run_eval_suite(TEST_CASES)