veridict 0.19.7

Statistical regression gate for A/B and hypothesis testing: compare candidate vs baseline results with win-rate, Elo, SPRT, and bootstrap confidence intervals.
Documentation
/// All errors the library can return. Every variant represents bad input or
/// bad configuration; the CLI maps every one of them to exit code 3.
#[derive(Debug, thiserror::Error)]
pub enum VeridictError {
    #[error("invalid JSON on line {line}: {source}")]
    InvalidJson {
        line: usize,
        #[source]
        source: serde_json::Error,
    },

    #[error("invalid CSV on line {line}: {source}")]
    InvalidCsv {
        line: usize,
        #[source]
        source: csv::Error,
    },

    /// `context` names what was being computed (e.g. "metric winrate",
    /// "sprt") when a record turned out to carry none of the fields that
    /// computation understands. A plain label rather than `MetricKind`
    /// because not every consumer of this error is a `MetricKind` (SPRT
    /// isn't one of the `compare` metrics).
    #[error("line {line}: record incompatible with {context}: {detail}")]
    SchemaMismatch {
        line: usize,
        context: &'static str,
        detail: String,
    },

    #[error("duplicate id '{id}' in paired numeric input (line {line})")]
    DuplicateId { id: String, line: usize },

    #[error("input contains no records")]
    EmptyInput,

    #[error("invalid confidence level {0}: must be finite and in (0, 1)")]
    InvalidConfidence(f64),

    #[error("invalid bootstrap resample count {0}: must be at least 1")]
    InvalidResamples(usize),

    #[error("invalid threshold configuration: {0}")]
    InvalidThreshold(String),

    #[error(
        "--ci-method {method} is only supported for metric winrate or sign-test (got {metric}); \
         elo metrics use fractional scores and mean-diff isn't a binomial proportion, so neither \
         an exact nor a Jeffreys binomial credible interval applies"
    )]
    IncompatibleCiMethod {
        method: &'static str,
        metric: &'static str,
    },

    #[error(
        "--failure-policy {policy} is only supported for metric winrate, elo, or paired-elo (got {metric}); \
         mean-diff/sign-test have no win/loss/draw outcome for a failed trial to become, so \
         there's no principled way to exclude or lose a numeric trial"
    )]
    IncompatibleFailurePolicy {
        policy: &'static str,
        metric: &'static str,
    },

    #[error(
        "--bootstrap-method {method} is not available for {metric}; the sample quantile is a \
         non-smooth statistic, so BCa's jackknife acceleration term has no solid asymptotic \
         footing the way it does for mean-diff - use percentile or basic instead (see \
         docs/metrics.md's quantile-diff section)"
    )]
    IncompatibleBootstrapMethod {
        method: &'static str,
        metric: &'static str,
    },

    #[error("invalid --quantile {0}: must be finite and in (0, 1)")]
    InvalidQuantile(f64),

    #[error("--quantile requires --metric quantile-diff to be one of the requested metrics")]
    QuantileRequiresQuantileDiffMetric,

    #[error(
        "--metric paired-elo requires --paired-by-id: every id must identify exactly two usable \
         trials so the five pair-score buckets remain well-defined"
    )]
    PairedEloRequiresPairedById,

    #[error(
        "power estimation for metric {metric} isn't supported via this flat (kind, ci_method) \
         constructor; winrate/sign-test/elo take a ci_method here, but mean-diff needs an assumed \
         standard deviation instead - construct PowerMetric::MeanDiff {{ assume_sd, sd_source }} \
         directly (see docs/metrics.md's power section)"
    )]
    UnsupportedPowerMetric { metric: &'static str },

    #[error(
        "power estimation isn't supported for metric paired-elo: its independent observation is \
         a complete pair and its uncertainty comes from the empirical five-bucket pair-score \
         distribution, which is unavailable before data collection"
    )]
    PowerUnsupportedForPairedElo,

    #[error(
        "power estimation isn't supported for metric quantile-diff; it needs an order-statistic \
         asymptotic variance (or a density estimate at the quantile), a separate research \
         problem from mean-diff's closed-form power - see docs/research-map.md"
    )]
    PowerUnsupportedForQuantileDiff,

    #[error(
        "power estimation isn't supported for metric relative-diff this round; it would need an \
         assumed standard deviation of relative (not absolute) observations, and --pilot would \
         need to relative-transform each record (validating baseline > 0) before estimating one - \
         structurally close to mean-diff's existing --assume-sd/--pilot design but not yet audited \
         on its own terms, so it isn't reused sight-unseen - see docs/research-map.md"
    )]
    PowerUnsupportedForRelativeDiff,

    #[error(
        "--assume-effect ({assume_effect}) must be strictly greater than --min-effect \
         ({min_effect}); power evaluated with the true effect equal to the pass threshold is \
         just the interval's own miscoverage at that boundary (~1-confidence), not something \
         more trials climbs toward a target power - see docs/metrics.md's power section"
    )]
    PowerRequiresEffectGap { min_effect: f64, assume_effect: f64 },

    #[error("invalid --target-power {0}: must be finite and in (0, 1)")]
    InvalidTargetPower(f64),

    #[error(
        "power search exceeded {cap} trials without finding a stable n at target_power \
         {target_power}; the requested effect gap (min_effect={min_effect}, \
         assume_effect={assume_effect}) is too small relative to --confidence/--target-power for \
         this to be a practical experiment size"
    )]
    PowerSearchExceededCap {
        cap: u64,
        min_effect: f64,
        assume_effect: f64,
        target_power: f64,
    },

    #[error("invalid --assume-sd {0}: must be finite and greater than 0")]
    InvalidAssumeSd(f64),

    #[error(
        "--pilot '{path}' produced only {count} usable paired diff(s); at least 2 are needed to \
         estimate a standard deviation"
    )]
    InsufficientPilotData { path: String, count: usize },

    #[error(
        "--pilot '{path}' data has zero variance ({count} identical paired diff(s)) - a sample \
         standard deviation of 0 isn't a usable power-analysis input; pass --assume-sd directly \
         with a real assumed value instead"
    )]
    ZeroVariancePilotData { path: String, count: usize },

    #[error(
        "--metric mean-diff requires exactly one of --assume-sd or --pilot FILE (there's no real \
         data pre-experiment to estimate a standard deviation from otherwise) - see \
         docs/metrics.md's power section"
    )]
    MeanDiffPowerRequiresSd,

    #[error(
        "--assume-sd/--pilot are only used with --metric mean-diff (got {metric}); the other \
         metrics' power search doesn't need an assumed standard deviation"
    )]
    AssumeSdOnlyForMeanDiff { metric: &'static str },

    #[error(
        "line {line}: invalid numeric value in field '{field}': {value} (NaN/Infinity not allowed)"
    )]
    InvalidValue {
        line: usize,
        field: &'static str,
        value: f64,
    },

    #[error(
        "{metric} produced a non-finite derived field '{field}': {value}; rescale numeric inputs \
         or choose less extreme settings so every reported statistic remains representable"
    )]
    NonFiniteStatistic {
        metric: &'static str,
        field: &'static str,
        value: f64,
    },

    #[error("unsupported SPRT checkpoint version {found}; supported version is {supported}")]
    UnsupportedSprtCheckpointVersion { found: u32, supported: u32 },

    #[error("SPRT checkpoint does not match the requested resume context: {detail}")]
    SprtCheckpointMismatch { detail: String },

    #[error(
        "SPRT accumulator line {line} is not after the previous line {previous}; resumed input \
         must continue in strictly increasing stream order"
    )]
    NonMonotonicSprtLine { line: usize, previous: usize },

    #[error(
        "line {line}: relative-diff requires baseline > 0 (got {baseline}); this isn't only about \
         avoiding division by zero - a percentage change relative to a negative baseline doesn't \
         have a stable interpretation (the same underlying improvement can read as a positive or \
         negative ratio depending on which side of zero the baseline sits on), so it's rejected \
         rather than silently computed against abs(baseline) or a shifted denominator. Use \
         --metric mean-diff instead if some baselines are zero or negative"
    )]
    RelativeDiffRequiresPositiveBaseline { line: usize, baseline: f64 },

    #[error(
        "line {line}: unrecognized status '{value}' in field '{field}' (expected ok|timeout|crash|invalid)"
    )]
    UnrecognizedStatus {
        line: usize,
        field: &'static str,
        value: String,
    },

    #[error("line {line}: unrecognized result '{value}' (expected {expected})")]
    UnrecognizedOutcome {
        line: usize,
        value: String,
        expected: &'static str,
    },

    #[error("io error reading '{path}': {source}")]
    Io {
        path: String,
        #[source]
        source: std::io::Error,
    },

    #[error(
        "--cluster-by-id is only supported for metric winrate or elo (got {metric}); mean-diff/\
         sign-test/quantile-diff cluster-robust support is a separate, deferred piece of work \
         (see docs/research-map.md)"
    )]
    IncompatibleClusterById { metric: &'static str },

    #[error(
        "--cluster-by-id and --paired-by-id are mutually exclusive: pairing nets exactly two \
         records sharing an id into one observation, while clustering keeps every record in an \
         id group as its own resampling unit - the two describe incompatible ways to treat a \
         repeated id"
    )]
    ClusterByIdConflictsWithPairedById,

    #[error(
        "--claim-correction (or its deprecated alias --correction) is not supported together \
         with --cluster-by-id: achieved_alpha rebuilds a plain i.i.d. Wilson CI from the raw \
         record count, not the cluster bootstrap CI the report actually used - under positive \
         intra-cluster correlation that reconstruction is narrower than the true cluster-robust \
         CI, so correction could under-downgrade a pass instead of catching it, the opposite of \
         this project's own false-pass-is-worse-than-inconclusive bias. Rejected outright rather \
         than silently applying a mismatched correction; cluster-aware correction is a separate, \
         deferred piece of work (see docs/research-map.md)"
    )]
    CorrectionConflictsWithClusterById,

    #[error(
        "--claim-correction (or its deprecated alias --correction) is not supported for a family \
         that includes {metric}: there is no closed-form CI-at-a-hypothetical-confidence function \
         for its bootstrap CI (same reason estimated_additional_trials/power special-case it), so \
         this metric's own claim can't get a real, method-consistent family-wise guarantee. \
         Rejected outright rather than leaving it uncorrected while still counting it toward \
         family_size (today's `--correction` behavior) - bootstrap-aware correction is a \
         separate, deferred piece of work (see docs/research-map.md)"
    )]
    CorrectionRequiresClosedFormCi { metric: &'static str },

    #[error(
        "Bradley-Terry MM solver did not converge after {iterations} iterations \
         (largest relative rating change {last_relative_change:.3e} still above the \
         {threshold:.0e} threshold; competitor index {worst_competitor} changed most). \
         This input already passed the strong-connectivity check, so it is not \
         literally disconnected - it usually means a technically-connected but heavily \
         lopsided graph (e.g. one subgroup beat another all-but-once). Inspect win/loss \
         tallies for near-total sweeps between subgroups before rerunning."
    )]
    BradleyTerryDidNotConverge {
        iterations: usize,
        last_relative_change: f64,
        threshold: f64,
        worst_competitor: usize,
    },

    #[error("invalid TOML in '{path}': {source}")]
    InvalidToml {
        path: String,
        #[source]
        source: toml::de::Error,
    },

    /// Whole-file JSON config parse failure - distinct from `InvalidJson` (which is line-
    /// oriented, for JSONL trial records): a `--reward-schedule` file is one JSON document, not
    /// one record per line, the same distinction `InvalidToml` already draws for `manifest.toml`.
    #[error("invalid JSON in '{path}': {source}")]
    InvalidJsonFile {
        path: String,
        #[source]
        source: serde_json::Error,
    },

    #[error(
        "manifest_schema_version {found} is not supported by this build (supports {supported}); \
         regenerate the manifest with a matching veridict version rather than trusting an \
         unrecognized shape"
    )]
    UnsupportedManifestSchemaVersion { found: u32, supported: u32 },

    #[error(
        "manifest declares nothing for verify-run to check: schedule, experiment_id, and all of \
         dataset_sha256/binary_sha256/weight_sha256/config_sha256 are absent. The three \
         manifest-independent checks (pair completeness, global index uniqueness, role \
         consistency) still ran regardless and don't count toward this - populate at least one \
         manifest-dependent field, or there is nothing meaningful for this command to verify"
    )]
    ManifestDeclaresNothingToVerify,

    #[error(
        "no EDO stationary action exists for time_scale={time_scale}: the moment equation \
         sup_a E_p1[phi(a,x)^eta] = exp(1/time_scale) has a solution eta in (0,1) only when \
         exp(1/time_scale) < p1/p0 = {ratio:.6}, i.e. time_scale > {min_time_scale:.6}; the \
         requested time_scale decays too fast for any bet to be more informative on average \
         than the null itself - use --policy gro, or increase --time-scale"
    )]
    EdoRootDoesNotExist {
        p0: f64,
        p1: f64,
        time_scale: f64,
        ratio: f64,
        min_time_scale: f64,
    },

    #[error(
        "estimated Bellman grid cost (horizon={horizon} * wealth_grid_size={wealth_grid_size} * \
         action_grid_size={action_grid_size} = {estimated_ops}) exceeds the configured cap of \
         {cap}; this would silently run for a long time rather than fail fast - lower \
         --horizon/--deadline, --wealth-grid-size, or --action-grid-size"
    )]
    TimeSensitiveGridTooLarge {
        horizon: u64,
        wealth_grid_size: usize,
        action_grid_size: usize,
        estimated_ops: u128,
        cap: u128,
    },
}