1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
//! Percentile/basic/BCa bootstrap confidence interval on `(candidate - baseline) / baseline` for
//! paired numeric records - proportional change relative to the baseline, rather than `mean-diff`'s
//! absolute `candidate - baseline`. See `docs/metrics.md`'s `relative-diff` section for why this is
//! an independent metric (different units, different baseline constraint, non-symmetric under
//! swapping the arms) rather than a display mode of `mean-diff`.
use crate::error::VeridictError;
use crate::input::Record;
use crate::metrics::common::{DiffCollector, compute_scale_diagnostics};
use crate::metrics::{FailureBreakdown, MetricAggregator, MetricOutput, metric_label};
use crate::stats::bootstrap;
use crate::{BootstrapMethod, MetricKind, TrialStatus};
pub(crate) struct RelativeDiffAggregator {
collector: DiffCollector,
confidence: f64,
resamples: usize,
seed: u64,
bootstrap_method: BootstrapMethod,
/// Raw baseline values, in ingestion order, for `scale_diagnostics` - see `MeanDiffAggregator`'s
/// identically-purposed field. Every value pushed here is already validated `> 0` and finite (a
/// non-positive or non-finite baseline is rejected before reaching this point), so
/// `non_positive_baseline_count` in this metric's own `ScaleDiagnostics` is always `0`.
baselines: Vec<f64>,
/// Count of ingested records where `candidate == baseline` exactly - counted here, at ingest,
/// rather than post-`DiffCollector::finish()`, so a `--paired-by-id` netting of two opposite
/// nonzero ratios to `0.0` is never mistaken for a genuine exact-match tie. See `MetricOutput.
/// tied_count`.
tied_count: u64,
}
impl RelativeDiffAggregator {
pub(crate) fn new(
confidence: f64,
resamples: usize,
seed: u64,
paired_by_id: bool,
bootstrap_method: BootstrapMethod,
) -> Self {
Self {
collector: DiffCollector::new(paired_by_id),
confidence,
resamples,
seed,
bootstrap_method,
baselines: Vec::new(),
tied_count: 0,
}
}
}
impl MetricAggregator for RelativeDiffAggregator {
fn ingest(
&mut self,
line: usize,
record: &Record,
baseline_status: Option<TrialStatus>,
candidate_status: Option<TrialStatus>,
) -> Result<(), VeridictError> {
let mut used = baseline_status.is_some() || candidate_status.is_some();
if let (Some(b), Some(c)) = (record.baseline, record.candidate) {
if !b.is_finite() {
return Err(VeridictError::InvalidValue {
line,
field: "baseline",
value: b,
});
}
// Zero and negative baselines are both rejected here, not just zero - see
// `VeridictError::RelativeDiffRequiresPositiveBaseline`'s doc for why a negative
// baseline is rejected too, rather than silently dividing by `abs(baseline)` (a
// different effect size that would quietly change what the number means) or adding an
// epsilon to the denominator (which would fabricate a number rather than reporting
// that this record's percentage change isn't well-defined).
if b <= 0.0 {
return Err(VeridictError::RelativeDiffRequiresPositiveBaseline {
line,
baseline: b,
});
}
if !c.is_finite() {
return Err(VeridictError::InvalidValue {
line,
field: "candidate",
value: c,
});
}
used = true;
self.baselines.push(b);
if c == b {
self.tied_count += 1;
}
let difference = c - b;
// Preserve the established arithmetic for ordinary values. Dividing first is
// algebraically equivalent and avoids an overflowing subtraction at extreme scales.
let relative = if difference.is_finite() {
difference / b
} else {
c / b - 1.0
};
// `b > 0` and `c` finite still do not guarantee a finite ratio: a large finite `c`
// over a tiny finite `b` (e.g. candidate=1e300, baseline=1e-300) can overflow.
if !relative.is_finite() {
return Err(VeridictError::InvalidValue {
line,
field: "relative_diff",
value: relative,
});
}
self.collector
.record(line, record.id.as_deref(), relative)?;
}
if !used {
return Err(VeridictError::SchemaMismatch {
line,
context: metric_label(MetricKind::RelativeDiff),
detail: "record has no fields usable by this metric and no status fields"
.to_string(),
});
}
Ok(())
}
fn finish(self: Box<Self>, failures: &FailureBreakdown) -> Result<MetricOutput, VeridictError> {
let diffs = self.collector.finish()?;
let timeouts = failures.baseline.timeout + failures.candidate.timeout;
let crashes = failures.baseline.crash + failures.candidate.crash;
let invalid = failures.baseline.invalid + failures.candidate.invalid;
if diffs.is_empty() {
return Ok(MetricOutput {
effect: 0.0,
ci_low: 0.0,
ci_high: 0.0,
baseline_count: 0,
candidate_count: 0,
paired_count: 0,
timeouts,
crashes,
invalid,
failures: *failures,
warning: Some(
"no paired numeric trials to compute relative difference".to_string(),
),
records_with_id: 0,
max_id_count: 0,
quantile: None,
cluster_count: None,
max_cluster_size: None,
effective_sample_size: None,
design_effect: None,
scale_diagnostics: None,
tied_count: None,
pair_score_counts: None,
});
}
// The effect is `mean(relative_diff_i)` - the average of each pair's own ratio, not
// `sum(candidate_i) / sum(baseline_i) - 1` (a total-volume ratio). Averaging per-pair ratios
// treats every benchmark case as one equally-weighted observation, matching how `mean-diff`
// already treats every pair - a large-baseline case doesn't get more influence over the
// effect just because its absolute numbers are bigger. The two quantities are generally
// different whenever baselines vary in scale, which is precisely the situation this metric
// exists for.
let effect = bootstrap::mean(&diffs);
let (ci_low, ci_high) = match self.bootstrap_method {
BootstrapMethod::Percentile => bootstrap::bootstrap_mean_diff_ci(
&diffs,
self.confidence,
self.resamples,
self.seed,
),
BootstrapMethod::Bca => bootstrap::bootstrap_mean_diff_ci_bca(
&diffs,
self.confidence,
self.resamples,
self.seed,
),
BootstrapMethod::Basic => bootstrap::bootstrap_mean_diff_ci_basic(
&diffs,
self.confidence,
self.resamples,
self.seed,
),
};
Ok(MetricOutput {
effect,
ci_low,
ci_high,
baseline_count: diffs.len() as u64,
candidate_count: diffs.len() as u64,
paired_count: diffs.len() as u64,
timeouts,
crashes,
invalid,
failures: *failures,
warning: None,
records_with_id: 0,
max_id_count: 0,
quantile: None,
cluster_count: None,
max_cluster_size: None,
effective_sample_size: None,
design_effect: None,
scale_diagnostics: Some(compute_scale_diagnostics(&self.baselines)),
tied_count: Some(self.tied_count),
pair_score_counts: None,
})
}
}