Skip to content

calibrated_agent_evaluation.sampling

calibrated_agent_evaluation.sampling

Frame-assembly engine.

Turns a taxonomy grid into two dataset frames: - Discovery frame -- pairwise covering array (every 2-way interaction at least 1x). - Production-representative frame -- proportional stratified sampling.

Public API re-exports from submodules.

DeffWarning

Bases: UserWarning

Non-fatal design-effect warning when Deff exceeds a threshold.

FrameContaminationError

Bases: CalibratedAgentEvaluationDataError

Attempt to use a discovery frame where production-representative is required.

SamplingConfigError

Bases: CalibratedAgentEvaluationDataError

Invalid grid or distribution input.

FrameResult dataclass

FrameResult(
    rows, frame, coverage_strength=2, deff=None, n_eff=None
)

Output of a frame generation operation.

rows instance-attribute

rows

frame instance-attribute

frame

coverage_strength class-attribute instance-attribute

coverage_strength = 2

deff class-attribute instance-attribute

deff = None

n_eff class-attribute instance-attribute

n_eff = None

FrameRow dataclass

FrameRow(cell, frame, weight=1.0)

One row in a generated frame: a specific taxonomy-cell assignment.

cell instance-attribute

cell

frame instance-attribute

frame

weight class-attribute instance-attribute

weight = 1.0

TaxonomyGrid dataclass

TaxonomyGrid(dimensions)

Input grid: dimension name → list of possible values.

dimensions instance-attribute

dimensions

TrafficDistribution dataclass

TrafficDistribution(strata_weights)

Declared stratum weights for production-representative sampling.

strata_weights instance-attribute

strata_weights

discovery_frame

discovery_frame(
    grid,
    *,
    strength=2,
    filter_func=None,
    previously_tested=None,
)

Produce a covering-array discovery frame from a taxonomy grid.

Uses allpairspy's AllPairs as the base generator and supplements with additional rows to guarantee full t-way interaction coverage.

Source code in src/calibrated_agent_evaluation/sampling/discovery.py
def discovery_frame(
    grid: TaxonomyGrid,
    *,
    strength: int = 2,
    filter_func: Callable[[list[str]], bool] | None = None,
    previously_tested: list[list[str]] | None = None,
) -> FrameResult:
    """Produce a covering-array discovery frame from a taxonomy grid.

    Uses allpairspy's ``AllPairs`` as the base generator and supplements with
    additional rows to guarantee full t-way interaction coverage.
    """
    # Imported here, not at module top: `calibrated_agent_evaluation.run` reaches this package for
    # the frame check, and `calibrated-agent-evaluation run` must work with only the [stats] extra
    # installed.
    from allpairspy import AllPairs

    validate_grid(grid)

    dim_names = list(grid.dimensions.keys())
    dim_values = list(grid.dimensions.values())

    if strength > len(dim_names):
        raise SamplingConfigError(
            f"Cannot achieve {strength}-way coverage with only {len(dim_names)} dimensions"
        )

    # Wrap the allpairspy generator: any failure crossing this public surface becomes a typed
    # SamplingConfigError so callers can catch the CalibratedAgentEvaluationDataError family
    # (conventions: "wrap external errors at the boundary"; no raw vendor exception escapes).
    try:
        tuples: list[list[str]] = [
            list(row)
            for row in AllPairs(
                dim_values,
                filter_func=filter_func if filter_func is not None else (lambda _: True),
                previously_tested=previously_tested,
                n=strength,
            )
        ]
    except SamplingConfigError:
        raise
    except Exception as exc:  # allpairspy raised on the (pre-validated) grid; surface it typed
        raise SamplingConfigError(f"covering-array generation failed: {exc}") from exc

    if not tuples and filter_func is not None:
        raise SamplingConfigError("no valid combinations remain after filtering")

    if strength > 2 and tuples:
        tuples += _supplement_coverage(
            tuples, dim_values, strength, filter_func, previously_tested or []
        )

    rows = [
        FrameRow(
            cell={dim_names[i]: t[i] for i in range(len(dim_names))},
            frame="discovery",
            weight=1.0,
        )
        for t in tuples
    ]

    return FrameResult(rows=rows, frame="discovery", coverage_strength=strength)

production_frame

production_frame(
    grid,
    distribution,
    n,
    *,
    oversample=None,
    deff_threshold=DEFAULT_DEFF_THRESHOLD,
)

Produce a proportional stratified sample.

The strata are the values of one grid dimension. Rows are allocated in proportion to the declared distribution, at least one per stratum, and every row carries the design weight of its stratum's realised allocation, so the weighted shares match the declared distribution even where rounding, the 1-row minimum or oversample moved the row counts. When any weight differs from 1 the Kish design effect is reported, with its warning threshold forwarded via deff_threshold.

Raises:

Type Description
SamplingConfigError

The grid or distribution is invalid, a stratum key names no value or a value in two dimensions, the keys come from more than one dimension, or n is smaller than the number of strata.

Source code in src/calibrated_agent_evaluation/sampling/production.py
def production_frame(
    grid: TaxonomyGrid,
    distribution: TrafficDistribution,
    n: int,
    *,
    oversample: dict[str, float] | None = None,
    deff_threshold: float = DEFAULT_DEFF_THRESHOLD,
) -> FrameResult:
    """Produce a proportional stratified sample.

    The strata are the values of one grid dimension. Rows are allocated in proportion to the
    declared distribution, at least one per stratum, and every row carries the design weight of its
    stratum's realised allocation, so the weighted shares match the declared distribution even
    where rounding, the 1-row minimum or *oversample* moved the row counts. When any weight differs
    from 1 the Kish design effect is reported, with its warning threshold forwarded via
    *deff_threshold*.

    Raises:
        SamplingConfigError: The grid or distribution is invalid, a stratum key names no value or
            a value in two dimensions, the keys come from more than one dimension, or ``n`` is
            smaller than the number of strata.
    """
    validate_grid(grid)
    validate_distribution(distribution)

    num_strata = len(distribution.strata_weights)
    if n < num_strata:
        raise SamplingConfigError(
            f"n={n} is less than the number of strata ({num_strata}); "
            f"minimum 1 row per stratum required"
        )
    strata_to_cells = _strata_cells(grid, distribution)

    # Proportional allocation (largest-remainder, >=1 per stratum, sums to exactly n), then the
    # oversampling multipliers.
    allocation = _proportional_allocation(distribution.strata_weights, n)
    if oversample:
        allocation = {
            stratum: max(1, round(count * oversample.get(stratum, 1.0)))
            for stratum, count in allocation.items()
        }
    weights = _stratum_weights(distribution, allocation)

    rows: list[FrameRow] = []
    for stratum in sorted(distribution.strata_weights):
        cells = strata_to_cells[stratum]
        weight = weights[stratum]
        rows.extend(
            FrameRow(cell=cells[i % len(cells)], frame="production-representative", weight=weight)
            for i in range(allocation[stratum])
        )

    deff_val: float | None = None
    n_eff_val: float | None = None
    if any(w != 1.0 for w in weights.values()):
        deff_val, n_eff_val = kish_deff([r.weight for r in rows], threshold=deff_threshold)

    return FrameResult(
        rows=rows,
        frame="production-representative",
        deff=deff_val,
        n_eff=n_eff_val,
    )

kish_deff

kish_deff(weights, *, threshold=DEFAULT_DEFF_THRESHOLD)

Compute Kish design effect and effective sample size.

Deff = 1 + CV²(weights) = (sum(w²) * n) / (sum(w))² n_eff = n / Deff

Emits DeffWarning when deff exceeds threshold.

Source code in src/calibrated_agent_evaluation/sampling/stats.py
def kish_deff(
    weights: Sequence[float], *, threshold: float = DEFAULT_DEFF_THRESHOLD
) -> tuple[float, float]:
    """Compute Kish design effect and effective sample size.

    Deff = 1 + CV²(weights) = (sum(w²) * n) / (sum(w))²
    n_eff = n / Deff

    Emits ``DeffWarning`` when deff exceeds *threshold*.
    """
    n = len(weights)
    if n == 0:
        raise SamplingConfigError("Cannot compute Kish deff on an empty weight list")
    sum_w = sum(weights)
    if sum_w == 0.0:
        raise SamplingConfigError("Cannot compute Kish deff: all weights are zero")

    sum_w_sq = sum(w * w for w in weights)
    deff = (sum_w_sq * n) / (sum_w * sum_w)
    n_eff = n / deff

    if deff > threshold:
        warnings.warn(
            f"Deff={deff:.2f}, n_eff={n_eff:.1f}; effective sample critically low",
            DeffWarning,
            stacklevel=2,
        )

    return (deff, n_eff)

assert_production_frame

assert_production_frame(dataset)

Raise if any row has frame != 'production-representative'.

Source code in src/calibrated_agent_evaluation/sampling/validate.py
def assert_production_frame(dataset: Sequence[FrameRow]) -> None:
    """Raise if any row has frame != 'production-representative'."""
    for row in dataset:
        if row.frame != "production-representative":
            raise FrameContaminationError(
                f"Headline rates require frame='production-representative'; "
                f"this dataset contains row(s) tagged '{row.frame}'"
            )

validate_distribution

validate_distribution(dist)

Raise SamplingConfigError unless weights are finite, non-negative and sum to 1 ±0.01.

Source code in src/calibrated_agent_evaluation/sampling/validate.py
def validate_distribution(dist: TrafficDistribution) -> None:
    """Raise ``SamplingConfigError`` unless weights are finite, non-negative and sum to 1 ±0.01."""
    bad = {s: w for s, w in dist.strata_weights.items() if not math.isfinite(w) or w < 0}
    if bad:
        raise SamplingConfigError(
            f"Distribution weights must be finite and non-negative; got {bad}"
        )
    total = sum(dist.strata_weights.values())
    if abs(total - 1.0) > 0.01:
        raise SamplingConfigError(f"Distribution weights sum to {total:.4f}, expected 1.0 ±0.01")

validate_grid

validate_grid(grid)

Raise on < 2 dims or any dim with < 2 values.

Source code in src/calibrated_agent_evaluation/sampling/validate.py
def validate_grid(grid: TaxonomyGrid) -> None:
    """Raise on < 2 dims or any dim with < 2 values."""
    dims = grid.dimensions
    if not dims:
        raise SamplingConfigError(
            "Grid must have ≥2 dimensions each with ≥2 values; got an empty grid"
        )
    if len(dims) < 2:
        raise SamplingConfigError(
            f"Grid must have ≥2 dimensions each with ≥2 values; got {len(dims)} dimension(s)"
        )
    for name, values in dims.items():
        if len(values) < 2:
            raise SamplingConfigError(
                f"Grid must have ≥2 dimensions each with ≥2 values; "
                f"dimension {name!r} has {len(values)} value(s)"
            )