AI Testing Strategies for QA Engineers

import re
import json
from dataclasses import dataclass, field
from typing import List, Optional, Callable

@dataclass
class AITestCase:
  test_id: str
  input_text: str
  required_contains: List[str] = field(default_factory=list)
  forbidden_contains: List[str] = field(default_factory=list)
  max_words: Optional[int] = None
  must_be_valid_json: bool = False
  custom_checks: List[Callable] = field(default_factory=list)

def run_property_tests(test_case: AITestCase, response: str) -> dict:
  """Run all property checks against a response."""
  results = {"test_id": test_case.test_id, "passed": [], "failed": []}
  
  def check(condition: bool, description: str):
      if condition:
          results["passed"].append(description)
      else:
          results["failed"].append(description)
  
  # Required content checks
  for term in test_case.required_contains:
      check(term.lower() in response.lower(), f"contains '{term}'")
  
  # Forbidden content checks
  for term in test_case.forbidden_contains:
      check(term.lower() not in response.lower(), f"does not contain '{term}'")
  
  # Length check
  if test_case.max_words:
      word_count = len(response.split())
      check(word_count <= test_case.max_words, f"max words ({word_count}/{test_case.max_words})")
  
  # JSON validity check
  if test_case.must_be_valid_json:
      try:
          json.loads(response)
          results["passed"].append("valid JSON")
      except json.JSONDecodeError as e:
          results["failed"].append(f"invalid JSON: {str(e)[:50]}")
  
  # Custom property checks
  for custom_fn in test_case.custom_checks:
      try:
          passed, msg = custom_fn(response)
          check(passed, msg)
      except Exception as e:
          results["failed"].append(f"custom check error: {str(e)}")
  
  total = len(results["passed"]) + len(results["failed"])
  results["pass_rate"] = len(results["passed"]) / total if total > 0 else 0
  results["status"] = "PASS" if len(results["failed"]) == 0 else "FAIL"
  return results

# Define your test suite
test_suite = [
  AITestCase(
      test_id="tc_001_financial_summary",
      input_text="Summarize: Q3 revenue was $4.2M, up 32% YoY. Net margin improved to 18%.",
      required_contains=["4.2M", "32%"],
      forbidden_contains=["billion", "loss"],
      max_words=50,
  ),
  AITestCase(
      test_id="tc_002_json_extraction",
      input_text="Extract to JSON: Customer Alice Smith, ID cust_12345, purchased 3 items.",
      required_contains=["Alice Smith", "cust_12345"],
      must_be_valid_json=True,
  ),
  AITestCase(
      test_id="tc_003_no_pii_leak",
      input_text="What products do we offer?",
      forbidden_contains=["@", "555-", "ssn"],  # No PII patterns
      max_words=100,
  ),
]

# Mock responses for demo
mock_responses = {
  "tc_001_financial_summary": "Q3 revenue reached $4.2M, a 32% year-over-year increase, with net margin at 18%.",
  "tc_002_json_extraction": '{"name": "Alice Smith", "customer_id": "cust_12345", "items_purchased": 3}',
  "tc_003_no_pii_leak": "We offer cloud storage, API services, and enterprise analytics solutions.",
}

print("=== AI TEST SUITE RESULTS ===\n")
total_pass = total_fail = 0
for tc in test_suite:
  response = mock_responses[tc.test_id]
  result = run_property_tests(tc, response)
  icon = "✅" if result["status"] == "PASS" else "❌"
  print(f"{icon} {tc.test_id}: {result['status']}")
  for f in result["failed"]:
      print(f"   ↳ FAILED: {f}")
  if result["status"] == "PASS":
      total_pass += 1
  else:
      total_fail += 1

print(f"\nResults: {total_pass} passed, {total_fail} failed")
print("CI Gate: PASS" if total_fail == 0 else "CI Gate: FAIL  -  block deployment")