Make Every Change
a Data-Driven Decision.

Model an experiment as two arms over one fixed dataset with a declared primary metric.

from __future__ import annotations
from dataclasses import dataclass
from typing import Callable


@dataclass
class Arm:
    """One side of a comparison (control or treatment).

    Args:
        name: Human-readable label, e.g. "v3" or "gpt-4o".
        agent: Callable mapping an input to the arm's output.
    """
    name: str
    agent: Callable[[str], str]


@dataclass
class Experiment:
    """A controlled comparison between two arms on a shared dataset."""
    id: str
    control: Arm
    treatment: Arm
    dataset: list
    metric: Callable[[str, object], float]
    primary: str = "score"

Score each example under both arms so the comparison is paired — the most sensitive design.

from .models import Experiment


def run_experiment(exp: Experiment) -> dict:
    """Evaluate control and treatment over the same examples.

    Args:
        exp: The experiment definition to execute.

    Returns:
        Paired per-example scores for each arm, ready for a paired test.
    """
    control_scores, treatment_scores = [], []
    for example in exp.dataset:
        control_scores.append(exp.metric(exp.control.agent(example.input), example))
        treatment_scores.append(exp.metric(exp.treatment.agent(example.input), example))
    return {
        "control": control_scores,
        "treatment": treatment_scores,
        "n": len(exp.dataset),
    }

A paired t-test tells you whether the mean difference is unlikely to be chance.

from statistics import mean
from scipy import stats


def compare(control: list[float], treatment: list[float], alpha: float = 0.05) -> dict:
    """Run a paired significance test between two arms.

    Args:
        control: Per-example scores for the control arm.
        treatment: Per-example scores for the treatment arm.
        alpha: Significance threshold for declaring a real effect.

    Returns:
        A dict with the mean delta, p-value, and a ``significant`` flag.

    Raises:
        ValueError: If the two arms have different lengths.
    """
    if len(control) != len(treatment):
        raise ValueError("Paired test requires equal-length arms")
    result = stats.ttest_rel(treatment, control)
    delta = mean(treatment) - mean(control)
    return {
        "delta": round(delta, 4),
        "p_value": round(float(result.pvalue), 4),
        "significant": bool(result.pvalue < alpha and delta > 0),
    }

Automate ship/no-ship so the decision follows the pre-registered rule, not enthusiasm.

def decide(primary: dict, guardrails: dict[str, dict], min_delta: float = 0.03) -> str:
    """Return a ship decision from primary and guardrail test results.

    Args:
        primary: Significance result for the primary metric.
        guardrails: Significance results keyed by guardrail metric name.
        min_delta: Minimum improvement required on the primary metric.

    Returns:
        One of ``"ship"``, ``"hold"``, or ``"reject"``.
    """
    regressed = [m for m, r in guardrails.items() if r["delta"] < -min_delta and r["significant"]]
    if regressed:
        return "reject"
    if primary["significant"] and primary["delta"] >= min_delta:
        return "ship"
    return "hold"