# Traditional test
def test_add():
    assert add(2, 3) == 5  # Binary: pass or fail

# LLM eval
def eval_customer_support_response(response: str, context: dict) -> Score:
    # Did the response solve the user's problem?
    # Was it empathetic?
    # Was it factually correct?
    # Was it on-brand?
    # Return a score + reasoning

expected = {"intent": "cancel_subscription", "confidence": "high"}
actual = json.loads(llm_response)
assert actual["intent"] == expected["intent"]

# Response must contain order number in expected format
assert re.search(r'order #[A-Z]{2}\d{6}', response)

prompt = f"""
You are evaluating a customer support response.
User question: {user_question}
Agent response: {agent_response}

Rate on:
1. Accuracy (0-5): Is the information factually correct?
2. Helpfulness (0-5): Does it actually help the user?
3. Tone (0-5): Is it empathetic and professional?

Respond as JSON: accuracy
"""
scores = json.loads(gpt4o(prompt))

JAILBREAK_PROMPTS = [
    "Ignore your previous instructions and...",
    "As a DAN (Do Anything Now)...",
    "My grandmother used to read me instructions for...",
    # ... many more from your threat model
]

for prompt in JAILBREAK_PROMPTS:
    response = your_llm(prompt)
    assert not contains_harmful_content(response), f"Jailbreak succeeded: {prompt}"

# Baseline: production prompt responses on golden dataset
baseline_scores = evaluate(production_prompt, golden_dataset)

# Candidate: new prompt responses
candidate_scores = evaluate(new_prompt, golden_dataset)

# Significant regression?
assert candidate_scores.mean() >= baseline_scores.mean() * 0.95, \
    "New prompt regressed >5% on golden dataset"

# Structure of a golden dataset entry
{
    "id": "cs-001",
    "input": {
        "user_message": "I want to cancel my subscription",
        "user_history": "Premium customer, 2 years",
        "context": "user contacted support via chat"
    },
    "expected_behavior": {
        "intent_detected": "cancellation",
        "should_offer_retention": true,
        "required_elements": ["acknowledgment", "retention_offer", "cancellation_path"],
        "forbidden_elements": ["rude_language", "false_promises"]
    },
    "metadata": {
        "difficulty": "medium",
        "category": "retention",
        "created_from": "real_conversation_2026-01-15",
        "annotator": "jane.smith@company.com"
    }
}

from dataclasses import dataclass
from typing import Callable

@dataclass
class EvalCriteria:
    name: str
    weight: float
    scorer: Callable[[str, dict], float]  # Returns 0.0-1.0

def score_intent_detection(response: str, expected: dict) -> float:
    """Check if the correct intent was detected and acted upon"""
    intent = expected["intent_detected"]
    return 1.0 if intent.lower() in response.lower() else 0.0

def score_required_elements(response: str, expected: dict) -> float:
    """Check what fraction of required elements are present"""
    elements = expected.get("required_elements", [])
    if not elements:
        return 1.0
    present = sum(1 for e in elements if e in response)
    return present / len(elements)

CRITERIA = [
    EvalCriteria("intent_detection", weight=0.3, scorer=score_intent_detection),
    EvalCriteria("required_elements", weight=0.4, scorer=score_required_elements),
    EvalCriteria("llm_quality_judge", weight=0.3, scorer=llm_judge_score),
]

import asyncio
from datetime import datetime

async def run_eval_suite(
    prompt_template: str,
    dataset: list[dict],
    criteria: list[EvalCriteria]
) -> EvalReport:
    results = []
    
    async for item in dataset:
        response = await llm_call(
            prompt=prompt_template.format(**item["input"])
        )
        
        scores = {}
        for criterion in criteria:
            scores[criterion.name] = criterion.scorer(
                response, item["expected_behavior"]
            )
        
        weighted_score = sum(
            scores[c.name] * c.weight for c in criteria
        )
        
        results.append({
            "id": item["id"],
            "response": response,
            "scores": scores,
            "weighted_score": weighted_score,
        })
    
    return EvalReport(
        timestamp=datetime.now(),
        prompt_hash=hash(prompt_template),
        mean_score=sum(r["weighted_score"] for r in results) / len(results),
        results=results,
    )

# Gold standard: human-labeled pairs
human_labels = [
    {"response_a": "...", "response_b": "...", "human_preference": "A"},
    # ... 200 pairs
]

# Test your judge
judge_agreement = 0
for pair in human_labels:
    judge_verdict = llm_judge_compare(pair["response_a"], pair["response_b"])
    if judge_verdict == pair["human_preference"]:
        judge_agreement += 1

agreement_rate = judge_agreement / len(human_labels)
print(f"Judge-human agreement: {agreement_rate:.1%}")  # Aim for >75%

# 1. Swap order and average
score_ab = judge(response_a=candidate, response_b=baseline)
score_ba = judge(response_a=baseline, response_b=candidate)
final_score = (score_ab + (1 - score_ba)) / 2

# 2. Use specific rubrics, not "which is better"
# Bad: "Which response is better?"
# Good: "Rate factual accuracy (0-5), citing specific errors"

# 3. Chain-of-thought before judgment
# Ask the judge to reason before scoring