Define What "Correct"
Actually Means.

Represent examples as typed records with structured expectations and metadata.

from __future__ import annotations
from dataclasses import dataclass, field
import json


@dataclass(frozen=True)
class GoldenExample:
    """A single verified input paired with its expected outcome.

    Args:
        id: Stable identifier that survives dataset versioning.
        input: The exact input handed to the agent under test.
        expected: Structured acceptance criteria (not just a string).
        category: Behavioral bucket used by the coverage layer.
        difficulty: Relative hardness, useful for slicing results.
    """
    id: str
    input: str
    expected: dict
    category: str = "general"
    difficulty: str = "medium"


def load_golden_set(path: str) -> list[GoldenExample]:
    """Load a JSONL golden set from disk.

    Args:
        path: Path to a newline-delimited JSON file.

    Returns:
        A list of :class:`GoldenExample` records.
    """
    examples: list[GoldenExample] = []
    with open(path, encoding="utf-8") as fh:
        for line in fh:
            line = line.strip()
            if line:
                examples.append(GoldenExample(**json.loads(line)))
    return examples

Exact-string matching is too brittle for LLMs. Combine criteria-based and semantic matchers so valid rephrasings still pass.

from typing import Protocol


class Matcher(Protocol):
    """Scores how well an actual output satisfies an expectation."""
    def __call__(self, actual: str, expected: dict) -> float: ...


def criteria_match(actual: str, expected: dict) -> float:
    """Fraction of required phrases present in ``actual`` (0.0–1.0)."""
    required = [p.lower() for p in expected.get("must_mention", [])]
    if not required:
        return 1.0
    hits = sum(1 for phrase in required if phrase in actual.lower())
    return hits / len(required)


def decision_match(actual_decision: str, expected: dict) -> float:
    """Return 1.0 when a categorical decision matches exactly."""
    return float(actual_decision == expected.get("decision"))

A small runner executes the agent over every example and aggregates a pass rate plus per-item detail.

from dataclasses import dataclass
from typing import Callable
from .models import GoldenExample


@dataclass
class ItemResult:
    """Outcome of grading one golden example."""
    id: str
    score: float
    passed: bool


def evaluate_golden_set(
    examples: list[GoldenExample],
    agent: Callable[[str], str],
    matcher: Callable[[str, dict], float],
    threshold: float = 1.0,
) -> dict:
    """Run ``agent`` over every example and grade with ``matcher``.

    Args:
        examples: The golden set to evaluate against.
        agent: Callable mapping an input string to the agent's output.
        matcher: Scoring function ``(actual, expected) -> float``.
        threshold: Minimum per-item score to count as a pass.

    Returns:
        A summary dict with overall ``pass_rate`` and per-item ``results``.
    """
    results = []
    for ex in examples:
        score = matcher(agent(ex.input), ex.expected)
        results.append(ItemResult(ex.id, score, score >= threshold))
    passed = sum(1 for r in results if r.passed)
    return {"pass_rate": passed / max(len(results), 1), "results": results}

Fail the build if the pass rate drops below your agreed baseline.

from golden.models import load_golden_set
from golden.matchers import criteria_match
from golden.runner import evaluate_golden_set


def test_agent_meets_golden_baseline():
    """The agent must satisfy at least 95% of the golden set."""
    examples = load_golden_set("golden/support.jsonl")
    report = evaluate_golden_set(examples, build_agent(), criteria_match)
    failing = [r.id for r in report["results"] if not r.passed]
    assert report["pass_rate"] >= 0.95, f"Regressions: {failing}"