Skip to main content

stegobench_metrics/
lib.rs

1// Author:  Daniel Iwugo
2// Comment: Christ is King
3// SPDX-License-Identifier: AGPL-3.0-or-later
4// Copyright (C) 2026 Daniel Iwugo
5//
6// Migrated from Stegcore's `stegcore-ops` crate, where it was dual licensed.
7// It is relicensed AGPL-only here by its author. The code is unchanged apart
8// from this header: it arrives with its thirteen tests, and a metric that has
9// been producing published numbers is not the place to start rewriting.
10//
11// It lives in the benchmark rather than in Stegcore because the benchmark must
12// not be built out of the product it judges.
13
14//! Detection metrics for the comparative benchmark.
15//!
16//! Pure functions over labels and scores: a binary confusion matrix (positive
17//! = stego), the rank-based ROC AUC (Mann-Whitney, tie-aware), and the ROC
18//! curve points. These are deliberately storage- and tool-agnostic so the
19//! same code scores Stegcore's ensemble, any single detector, or an external
20//! comparator.
21
22use std::cmp::Ordering;
23use std::fmt;
24
25/// Why a confusion count could not be taken honestly.
26///
27/// Hand written rather than derived, because this crate has no dependencies on
28/// purpose: a metric whose number needs an audit of somebody else's crate before
29/// it can be trusted is not the metric this benchmark wants to publish.
30#[derive(Debug, Clone, Copy, PartialEq, Eq)]
31pub enum TallyError {
32    /// The two slices were built from different record sets.
33    LengthMismatch {
34        /// How many labels the caller handed over.
35        labels: usize,
36        /// How many predictions the caller handed over.
37        predicted_positive: usize,
38    },
39    /// Nothing was classified at all.
40    Empty,
41}
42
43impl fmt::Display for TallyError {
44    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
45        match self {
46            TallyError::LengthMismatch {
47                labels,
48                predicted_positive,
49            } => write!(
50                f,
51                "{labels} labels against {predicted_positive} predictions: the \
52                 two were built from different record sets, so any count over \
53                 them would describe images nobody named"
54            ),
55            TallyError::Empty => write!(
56                f,
57                "nothing to count: an empty confusion matrix reads as a \
58                 detector that got everything wrong rather than one that was \
59                 never run"
60            ),
61        }
62    }
63}
64
65impl std::error::Error for TallyError {}
66
67/// Binary classification counts. Positive is "stego".
68#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
69pub struct Confusion {
70    pub tp: u64,
71    pub fp: u64,
72    pub tn: u64,
73    pub fn_: u64,
74}
75
76impl Confusion {
77    /// Tally counts from per-sample labels and predicted-positive flags,
78    /// **truncating to the shorter slice**.
79    ///
80    /// Prefer [`Confusion::try_tally`], which refuses that case instead. This
81    /// one walks the two slices pairwise and stops at whichever runs out first,
82    /// so a caller that built the labels and the predictions from different
83    /// record sets gets a count over a subset nobody named, with nothing in the
84    /// return value to say so. It is kept, undeprecated, only because it is
85    /// published and callers depend on the signature.
86    pub fn tally(labels: &[bool], predicted_positive: &[bool]) -> Self {
87        let mut c = Confusion::default();
88        for (&label, &pred) in labels.iter().zip(predicted_positive) {
89            match (label, pred) {
90                (true, true) => c.tp += 1,
91                (false, true) => c.fp += 1,
92                (false, false) => c.tn += 1,
93                (true, false) => c.fn_ += 1,
94            }
95        }
96        c
97    }
98
99    /// Tally counts from per-sample labels and predicted-positive flags,
100    /// refusing anything that cannot honestly produce a count.
101    ///
102    /// This is [`Confusion::tally`] with the truncation taken out. A length
103    /// mismatch means the labels and the predictions came from different record
104    /// sets, and walking the shorter of the two publishes a figure measured on
105    /// images nobody named. An empty pair of slices is refused too: every rate
106    /// on [`Confusion`] answers 0.0 on a zero denominator, so an empty count
107    /// reads as a detector that caught nothing rather than one that was never
108    /// run.
109    ///
110    /// A single class is **not** refused, unlike [`roc_auc`] and [`roc_curve`].
111    /// A ranking needs both a positive and a negative to rank against each
112    /// other; a confusion count does not, and a one-sided run is a real
113    /// measurement here. The `must_clear` half of a detector's self test is
114    /// exactly that: clean images only, where the whole question is how many
115    /// false positives came back.
116    ///
117    /// NaN has no analogue to refuse: both inputs are `bool`, so there is no
118    /// unordered or absent value a detector could hand over. A detector that
119    /// failed to score an image must be left out of both slices rather than
120    /// given a placeholder flag, and the length check is what catches a caller
121    /// that dropped it from one slice and not the other.
122    pub fn try_tally(labels: &[bool], predicted_positive: &[bool]) -> Result<Self, TallyError> {
123        if labels.len() != predicted_positive.len() {
124            return Err(TallyError::LengthMismatch {
125                labels: labels.len(),
126                predicted_positive: predicted_positive.len(),
127            });
128        }
129        if labels.is_empty() {
130            return Err(TallyError::Empty);
131        }
132        Ok(Self::tally(labels, predicted_positive))
133    }
134
135    fn ratio(num: u64, den: u64) -> f64 {
136        if den == 0 {
137            0.0
138        } else {
139            num as f64 / den as f64
140        }
141    }
142
143    /// True-positive rate (detection rate / recall): tp / (tp + fn).
144    pub fn tpr(&self) -> f64 {
145        Self::ratio(self.tp, self.tp + self.fn_)
146    }
147
148    /// False-positive rate: fp / (fp + tn).
149    pub fn fpr(&self) -> f64 {
150        Self::ratio(self.fp, self.fp + self.tn)
151    }
152
153    /// Precision: tp / (tp + fp).
154    pub fn precision(&self) -> f64 {
155        Self::ratio(self.tp, self.tp + self.fp)
156    }
157
158    /// Accuracy: (tp + tn) / total.
159    pub fn accuracy(&self) -> f64 {
160        Self::ratio(self.tp + self.tn, self.tp + self.tn + self.fp + self.fn_)
161    }
162
163    /// F1 score: harmonic mean of precision and recall; 0 when both are 0.
164    pub fn f1(&self) -> f64 {
165        let (p, r) = (self.precision(), self.tpr());
166        if p + r == 0.0 {
167            0.0
168        } else {
169            2.0 * p * r / (p + r)
170        }
171    }
172}
173
174fn cmp_f64(a: f64, b: f64) -> Ordering {
175    a.partial_cmp(&b).unwrap_or(Ordering::Equal)
176}
177
178/// Whether these two slices can be ranked against each other at all.
179///
180/// A length mismatch means the caller built the scores and the labels from
181/// different record sets, and a ranking over the shorter of the two is a number
182/// measured on a subset nobody named. A score that is not a number cannot be
183/// ordered, so every comparison against it is false and the ranking silently
184/// stops meaning anything. Both answer `None` rather than a figure, because in
185/// a measurement tool a quietly wrong number is worse than no number: it gets
186/// published.
187fn rankable(scores: &[f64], labels: &[bool]) -> bool {
188    scores.len() == labels.len() && scores.iter().all(|s| !s.is_nan())
189}
190
191/// An AUC with the uncertainty that belongs beside it.
192///
193/// A bare AUC printed to sixteen digits invites a reader to compare two
194/// numbers that differ in the third, on corpora small enough that one image
195/// moves the figure further than that. The interval is the tool saying how
196/// much of its own answer is real.
197#[derive(Debug, Clone, Copy, PartialEq)]
198pub struct AucInterval {
199    /// The point estimate, identical to [`roc_auc`].
200    pub auc: f64,
201    /// DeLong's standard error.
202    pub standard_error: f64,
203    /// Lower bound, clamped into `[0, 1]` because AUC cannot leave it.
204    pub low: f64,
205    /// Upper bound, clamped the same way.
206    pub high: f64,
207    /// How many standard errors wide, so a reader knows what was asked for.
208    pub z: f64,
209}
210
211/// 1.959964, the two-sided normal quantile for 95 per cent.
212///
213/// Written out rather than computed: there is no inverse normal in the
214/// standard library, and this crate has no dependencies on purpose (a metric
215/// with a dependency graph is one somebody has to audit before trusting a
216/// number).
217pub const Z_95: f64 = 1.959_963_984_540_054;
218
219/// Average ranks of `values` among themselves, 1-based, ties sharing the mean
220/// of the positions they span.
221///
222/// Split out because DeLong needs the same computation three times over three
223/// different sets, and a second copy of a tie rule is how two of them come to
224/// disagree.
225fn midranks(values: &[f64]) -> Vec<f64> {
226    let mut idx: Vec<usize> = (0..values.len()).collect();
227    idx.sort_by(|&a, &b| cmp_f64(values[a], values[b]));
228    let mut ranks = vec![0.0f64; values.len()];
229    let mut i = 0;
230    while i < idx.len() {
231        let mut j = i;
232        while j + 1 < idx.len() && values[idx[j + 1]] == values[idx[i]] {
233            j += 1;
234        }
235        let avg = ((i + 1) + (j + 1)) as f64 / 2.0;
236        for &k in &idx[i..=j] {
237            ranks[k] = avg;
238        }
239        i = j + 1;
240    }
241    ranks
242}
243
244/// AUC with a confidence interval, by DeLong's method.
245///
246/// WHY DELONG AND NOT HANLEY AND MCNEIL
247///
248/// The Hanley and McNeil standard error is two lines and assumes the scores
249/// are exponentially distributed within each class. Detector scores are not:
250/// several of the tools in this registry emit a bounded statistic, one emits
251/// a count, and one answers the same number to everything. Under those the
252/// parametric approximation is wrong in a direction nobody can predict from
253/// the output. DeLong's estimator makes no such assumption; it is computed
254/// from the data's own ranks and is what the statistical literature treats as
255/// the default for comparing classifiers.
256///
257/// Computed through midranks rather than the O(n_pos * n_neg) definition, so
258/// it stays O(n log n) and a Core tier run pays sorting rather than a hundred
259/// billion comparisons.
260///
261/// `z` is how many standard errors wide the interval is: [`Z_95`] for the
262/// usual 95 per cent.
263///
264/// Returns `None` for everything [`roc_auc`] returns `None` for, and also
265/// when either class has fewer than two members, because a variance over one
266/// observation is not an estimate of anything.
267pub fn roc_auc_interval(scores: &[f64], labels: &[bool], z: f64) -> Option<AucInterval> {
268    let auc = roc_auc(scores, labels)?;
269    if !z.is_finite() || z < 0.0 {
270        return None;
271    }
272
273    let pos: Vec<f64> = scores
274        .iter()
275        .zip(labels)
276        .filter(|(_, &l)| l)
277        .map(|(&s, _)| s)
278        .collect();
279    let neg: Vec<f64> = scores
280        .iter()
281        .zip(labels)
282        .filter(|(_, &l)| !l)
283        .map(|(&s, _)| s)
284        .collect();
285    let (m, n) = (pos.len(), neg.len());
286    if m < 2 || n < 2 {
287        return None;
288    }
289
290    // Ranks of each class within itself, and of everything within everything.
291    let t_x = midranks(&pos);
292    let t_y = midranks(&neg);
293    let mut all = Vec::with_capacity(m + n);
294    all.extend_from_slice(&pos);
295    all.extend_from_slice(&neg);
296    let t_z = midranks(&all);
297
298    // DeLong's structural components. V10 is, for each positive, the fraction
299    // of negatives it beats; V01 the mirror. Both fall out of the difference
300    // between a point's rank among everything and its rank among its own
301    // class, which is why this costs a sort rather than a product.
302    let v10: Vec<f64> = (0..m).map(|i| (t_z[i] - t_x[i]) / n as f64).collect();
303    let v01: Vec<f64> = (0..n)
304        .map(|j| 1.0 - (t_z[m + j] - t_y[j]) / m as f64)
305        .collect();
306
307    let s10 = sample_variance(&v10)?;
308    let s01 = sample_variance(&v01)?;
309    let var = s10 / m as f64 + s01 / n as f64;
310    if !var.is_finite() || var < 0.0 {
311        return None;
312    }
313    let se = var.sqrt();
314
315    // A WIDTH OF ZERO IS NOT A MEASUREMENT OF PRECISION
316    //
317    // DeLong's variance is built from how much each observation moves the
318    // statistic. When the classes separate completely every positive beats
319    // every negative, each component is identical, the sample variance is
320    // exactly zero and the interval collapses onto the estimate. The same
321    // happens when a detector answers identically every time, which gives
322    // 0.5 with no width.
323    //
324    // Both cases printed a 95% interval of zero width on as few as four
325    // observations, which is the most quotable and least defensible line the
326    // tool could produce. There is no width to report here, so none is
327    // reported: the estimator has nothing to say at the boundary, and saying
328    // so is honest where inventing a bound from a method this crate does not
329    // implement would not be.
330    if se == 0.0 {
331        return None;
332    }
333
334    Some(AucInterval {
335        auc,
336        standard_error: se,
337        // Clamped, because the normal interval runs past the ends of the
338        // scale near 1.0 and an upper bound of 1.03 is not a thing anybody
339        // should print beside a measurement.
340        low: (auc - z * se).clamp(0.0, 1.0),
341        high: (auc + z * se).clamp(0.0, 1.0),
342        z,
343    })
344}
345
346/// Unbiased sample variance, or `None` for fewer than two observations.
347fn sample_variance(v: &[f64]) -> Option<f64> {
348    if v.len() < 2 {
349        return None;
350    }
351    let n = v.len() as f64;
352    let mean = v.iter().sum::<f64>() / n;
353    Some(v.iter().map(|x| (x - mean).powi(2)).sum::<f64>() / (n - 1.0))
354}
355
356/// Rank-based ROC AUC (equivalent to the Mann-Whitney U statistic), tie-aware
357/// via average ranks.
358///
359/// Returns `None` when one class is absent, since AUC is undefined without both
360/// a positive and a negative sample; when the two slices are of different
361/// lengths; and when any score is NaN.
362pub fn roc_auc(scores: &[f64], labels: &[bool]) -> Option<f64> {
363    if !rankable(scores, labels) {
364        return None;
365    }
366    let n_pos = labels.iter().filter(|&&l| l).count();
367    let n_neg = labels.len() - n_pos;
368    if n_pos == 0 || n_neg == 0 {
369        return None;
370    }
371
372    // Indices sorted by ascending score.
373    let mut idx: Vec<usize> = (0..scores.len()).collect();
374    idx.sort_by(|&a, &b| cmp_f64(scores[a], scores[b]));
375
376    // Average ranks (1-based), tie groups share the mean of their positions.
377    let mut ranks = vec![0.0f64; scores.len()];
378    let mut i = 0;
379    while i < idx.len() {
380        let mut j = i;
381        while j + 1 < idx.len() && scores[idx[j + 1]] == scores[idx[i]] {
382            j += 1;
383        }
384        let avg = ((i + 1) + (j + 1)) as f64 / 2.0;
385        for &k in &idx[i..=j] {
386            ranks[k] = avg;
387        }
388        i = j + 1;
389    }
390
391    let sum_pos: f64 = labels
392        .iter()
393        .zip(&ranks)
394        .filter(|(&l, _)| l)
395        .map(|(_, &r)| r)
396        .sum();
397    let auc = (sum_pos - (n_pos * (n_pos + 1)) as f64 / 2.0) / (n_pos as f64 * n_neg as f64);
398    Some(auc)
399}
400
401/// ROC curve as `(fpr, tpr)` points, swept from the highest score downward
402/// (predicted positive when `score >= threshold`). Begins at `(0, 0)` and ends
403/// at `(1, 1)`.
404///
405/// Empty when either class is absent, when the two slices are of different
406/// lengths, and when any score is NaN. See [`roc_auc`].
407pub fn roc_curve(scores: &[f64], labels: &[bool]) -> Vec<(f64, f64)> {
408    // Derived from `roc_points` rather than computed again, so the curve and the
409    // thresholds that produced it cannot drift apart. A second copy of this
410    // sweep is exactly the "arithmetic that quietly disagrees" this crate exists
411    // to avoid.
412    roc_points(scores, labels)
413        .into_iter()
414        .map(|p| (p.fpr, p.tpr))
415        .collect()
416}
417
418/// One vertex of the ROC curve, including the score that produced it.
419///
420/// WHY THE THRESHOLD IS CARRIED AND NOT THROWN AWAY
421///
422/// A developer wiring detection into a product needs one number: the score to
423/// compare against. Every other figure here says how GOOD the detector is at a
424/// false-alarm budget; only this one says what to type into the `if`.
425///
426/// It used to be computed and discarded. A journey walked by a developer whose
427/// whole goal was picking a cutoff searched the result document for it, found
428/// nothing, and wrote his own ROC code against the records file to get it. His
429/// figures reproduced these to within tie handling, which is the good news and
430/// also the point: the tool had already done the arithmetic and made him do it
431/// again, and a second implementation of a metric is the thing this crate was
432/// built to prevent.
433#[derive(Debug, Clone, Copy, PartialEq)]
434pub struct RocPoint {
435    /// Flag an image when its score is greater than or equal to this.
436    ///
437    /// `None` at the origin, where nothing is flagged at all. That vertex is
438    /// real and belongs on the curve, and it has no threshold because there is
439    /// no score high enough to be worth flagging; reporting a number there
440    /// would be inventing one.
441    pub threshold: Option<f64>,
442    pub fpr: f64,
443    pub tpr: f64,
444}
445
446/// The ROC curve as vertices carrying the threshold each one sits at.
447///
448/// Swept from the highest score downward, predicting positive when
449/// `score >= threshold`. Begins at the origin, where nothing is flagged, and
450/// ends at `(1, 1)`, where everything is.
451///
452/// Empty under exactly the conditions [`roc_curve`] is empty.
453pub fn roc_points(scores: &[f64], labels: &[bool]) -> Vec<RocPoint> {
454    if !rankable(scores, labels) {
455        return Vec::new();
456    }
457    let n_pos = labels.iter().filter(|&&l| l).count() as f64;
458    let n_neg = labels.len() as f64 - n_pos;
459    if n_pos == 0.0 || n_neg == 0.0 {
460        return Vec::new();
461    }
462
463    let mut pairs: Vec<(f64, bool)> = scores.iter().copied().zip(labels.iter().copied()).collect();
464    pairs.sort_by(|a, b| cmp_f64(b.0, a.0)); // descending score
465
466    let mut curve = vec![RocPoint {
467        threshold: None,
468        fpr: 0.0,
469        tpr: 0.0,
470    }];
471    let (mut tp, mut fp) = (0.0f64, 0.0f64);
472    let mut i = 0;
473    while i < pairs.len() {
474        let thr = pairs[i].0;
475        // Advance over every sample at this threshold before recording a point,
476        // so tied scores collapse to a single ROC vertex.
477        while i < pairs.len() && pairs[i].0 == thr {
478            if pairs[i].1 {
479                tp += 1.0;
480            } else {
481                fp += 1.0;
482            }
483            i += 1;
484        }
485        // The threshold is the score of the group just consumed, because every
486        // sample at it has now been counted as flagged, which is what
487        // `score >= threshold` means.
488        curve.push(RocPoint {
489            threshold: Some(thr),
490            fpr: fp / n_neg,
491            tpr: tp / n_pos,
492        });
493    }
494    curve
495}
496
497/// The best true-positive rate this detector reaches without exceeding
498/// `max_fpr`.
499///
500/// `None` when the curve cannot be drawn (see [`roc_curve`]) and when `max_fpr`
501/// is not a false-alarm budget: a negative, a NaN or a figure above 1 is a
502/// caller's mistake rather than a strict budget, and answering 0.0 to it would
503/// read as a detector that caught nothing.
504///
505/// ONE SWEEP, NOT ONE PASS PER THRESHOLD
506///
507/// The obvious shape of this is a loop over every distinct score that counts
508/// the hits above it, and a copy of this metric elsewhere in the project was
509/// written that way. It is quadratic, and the scores are floats a detector
510/// produced, so "distinct" means nearly all of them: an arm of 344,357 images
511/// made it around a hundred billion comparisons, which is not slow, it is a
512/// run that never ends. [`roc_curve`] walks the scores once in descending
513/// order and accumulates counts, which gives the same answer, ties included,
514/// for the cost of the sort.
515///
516/// This is the headline number for an external evaluation and accuracy is not.
517/// A detector facing a corpus that is mostly clean can score 95% accuracy by
518/// answering "clean" every time, and a false-positive rate chosen after seeing
519/// the results is not a measurement. Pinning the false-positive budget first and
520/// asking what detection it buys is the comparison that survives review.
521pub fn tpr_at_fpr(scores: &[f64], labels: &[bool], max_fpr: f64) -> Option<f64> {
522    if !(0.0..=1.0).contains(&max_fpr) {
523        return None;
524    }
525    let curve = roc_curve(scores, labels);
526    if curve.is_empty() {
527        return None;
528    }
529    // The curve is a step function, so the answer is the highest point whose
530    // false-positive rate is still inside the budget. The epsilon absorbs the
531    // representation error in ratios like 1/3, which would otherwise drop an
532    // operating point that is exactly on the limit.
533    let best = curve
534        .iter()
535        .filter(|&&(fpr, _)| fpr <= max_fpr + 1e-12)
536        .map(|&(_, tpr)| tpr)
537        .fold(f64::NEG_INFINITY, f64::max);
538    Some(if best.is_finite() { best } else { 0.0 })
539}
540
541/// The point on the curve that [`tpr_at_fpr`] actually reported.
542///
543/// WHY THIS EXISTS, AND IT IS NOT A CONVENIENCE
544///
545/// A false-alarm budget finer than one clean image cannot be spent. With six
546/// clean images the only false-alarm rates that exist are 0, 1/6, 2/6 and so
547/// on, so a budget of 0.01 buys exactly what a budget of 0 buys, and
548/// `tpr_at_fpr` correctly returns the zero-budget answer. What it cannot do
549/// is say so, and the caller then prints that number under the heading
550/// "TPR@1%FA", which claims a resolution the measurement never had.
551///
552/// An engineer picking a review threshold reads that column and nothing else.
553/// So this returns the rate that was actually achieved beside the one that
554/// was asked for, and the two differing is the signal that the corpus is too
555/// small for the question.
556#[derive(Debug, Clone, Copy, PartialEq)]
557pub struct OperatingPoint {
558    /// The detection rate at this point.
559    pub tpr: f64,
560    /// The false-alarm rate this point actually sits at, which is a multiple
561    /// of `1 / n_clean` and is never above `requested`.
562    pub achieved_fpr: f64,
563    /// The budget the caller asked for.
564    pub requested_fpr: f64,
565    /// Flag an image when its score is greater than or equal to this.
566    ///
567    /// The number a developer actually needs, and the one this struct used to
568    /// compute and discard. See [`RocPoint::threshold`] for why `None` is a real
569    /// answer rather than a missing one: at the origin nothing is flagged, so
570    /// there is no cutoff worth printing.
571    ///
572    /// It is a raw detector score and carries that detector's units and scale.
573    /// It does not transfer to another tool, or to another version of the same
574    /// tool, which is a property of the thing rather than a limitation here.
575    pub threshold: Option<f64>,
576}
577
578impl OperatingPoint {
579    /// Whether the corpus could express the budget that was asked for.
580    ///
581    /// False means the answer is real but the label is not: the figure is the
582    /// detection rate at `achieved_fpr`, not at `requested_fpr`.
583    pub fn budget_was_expressible(&self, n_clean: usize) -> bool {
584        n_clean > 0 && fpr_resolution(n_clean) <= self.requested_fpr + 1e-12
585    }
586}
587
588/// The smallest false-alarm rate a corpus of this many clean images can show.
589///
590/// One clean image wrongly flagged out of `n_clean`. Anything finer is a
591/// budget the sample cannot spend.
592pub fn fpr_resolution(n_clean: usize) -> f64 {
593    if n_clean == 0 {
594        return f64::INFINITY;
595    }
596    1.0 / n_clean as f64
597}
598
599/// [`tpr_at_fpr`], plus the rate the answer actually came from.
600pub fn operating_point(scores: &[f64], labels: &[bool], max_fpr: f64) -> Option<OperatingPoint> {
601    if !(0.0..=1.0).contains(&max_fpr) {
602        return None;
603    }
604    let curve = roc_points(scores, labels);
605    if curve.is_empty() {
606        return None;
607    }
608    // The same point `tpr_at_fpr` picks: highest detection rate inside the
609    // budget. Ties on tpr take the lowest fpr, because two points with the
610    // same detection rate are the same answer bought more or less cheaply,
611    // and reporting the dearer one would overstate what the budget cost.
612    let mut best: Option<RocPoint> = None;
613    for &point in curve.iter().filter(|p| p.fpr <= max_fpr + 1e-12) {
614        best = Some(match best {
615            None => point,
616            Some(b) if point.tpr > b.tpr || (point.tpr == b.tpr && point.fpr < b.fpr) => point,
617            Some(b) => b,
618        });
619    }
620    let chosen = best.unwrap_or(RocPoint {
621        threshold: None,
622        fpr: 0.0,
623        tpr: 0.0,
624    });
625    Some(OperatingPoint {
626        tpr: chosen.tpr,
627        achieved_fpr: chosen.fpr,
628        requested_fpr: max_fpr,
629        threshold: chosen.threshold,
630    })
631}
632
633/// Something about a set of scores that makes the number beside it misleading.
634///
635/// A finding carries the facts and no wording. This crate has no dependencies
636/// and no opinion about presentation: two callers print in two registers, and a
637/// sentence baked in here would be the one that is wrong in both. The caller
638/// writes the sentence; the enum is what stops the two callers disagreeing
639/// about when to write one.
640#[derive(Debug, Clone, Copy, PartialEq)]
641pub enum Finding {
642    /// Every score was the same number, so the AUC is 0.5 by construction.
643    ///
644    /// A detector that answers one thing to everything and a coin flip produce
645    /// the identical figure, and they mean completely different things: one
646    /// could not tell the images apart, the other ranked them no better than
647    /// chance.
648    EveryScoreIdentical {
649        /// How many answers were the same.
650        count: usize,
651    },
652    /// The AUC is below 0.5, which is a sign-flipped adapter far more often
653    /// than it is a detector that sees nothing.
654    ///
655    /// A ranking that is wrong this consistently carries as much signal as one
656    /// that is right; it is being read upside down. The commonest way to wire
657    /// a new detector into this benchmark incorrectly.
658    ScoresRunBackwards {
659        /// The AUC as measured.
660        auc: f64,
661        /// What the same ranking would score read the other way round, which
662        /// is the figure that says how much signal is actually there.
663        inverted: f64,
664    },
665}
666
667/// Every [`Finding`] that applies to these scores and labels.
668///
669/// Empty for a healthy detector, and empty for anything [`roc_auc`] refuses:
670/// a set that cannot be ranked has no AUC to describe, and the caller is
671/// already refusing it for a named reason it would only muddy.
672pub fn findings(scores: &[f64], labels: &[bool]) -> Vec<Finding> {
673    let Some(auc) = roc_auc(scores, labels) else {
674        return Vec::new();
675    };
676    let mut out = Vec::new();
677    if scores.len() > 1 && scores.windows(2).all(|w| w[0] == w[1]) {
678        out.push(Finding::EveryScoreIdentical {
679            count: scores.len(),
680        });
681    }
682    if auc < 0.5 {
683        out.push(Finding::ScoresRunBackwards {
684            auc,
685            inverted: 1.0 - auc,
686        });
687    }
688    out
689}
690
691#[cfg(test)]
692mod finding_tests {
693    use super::*;
694
695    #[test]
696    fn a_detector_that_separates_the_images_is_not_flagged() {
697        // The half that stops the check being made to pass by always firing.
698        assert!(findings(&[0.1, 0.2, 0.8, 0.9], &[false, false, true, true]).is_empty());
699        // And a genuine chance-level ranking, which is 0.5 by measurement
700        // rather than by construction, is a result and not a finding.
701        assert_eq!(
702            roc_auc(&[0.1, 0.2, 0.8, 0.9], &[true, false, false, true]),
703            Some(0.5)
704        );
705        assert!(findings(&[0.1, 0.2, 0.8, 0.9], &[true, false, false, true]).is_empty());
706    }
707
708    #[test]
709    fn one_answer_to_everything_is_flagged_with_its_count() {
710        let got = findings(
711            &[7.0; 8],
712            &[true, false, true, false, true, false, true, false],
713        );
714        assert_eq!(got, vec![Finding::EveryScoreIdentical { count: 8 }]);
715    }
716
717    #[test]
718    fn an_inverted_ranking_is_flagged_with_both_figures() {
719        let got = findings(&[0.9, 0.8, 0.2, 0.1], &[false, false, true, true]);
720        assert_eq!(
721            got,
722            vec![Finding::ScoresRunBackwards {
723                auc: 0.0,
724                inverted: 1.0
725            }]
726        );
727        // And a ranking only slightly the wrong way round is flagged too: the
728        // sign error is the same one whether or not it is total.
729        let got = findings(&[0.3, 0.4, 0.35, 0.2], &[false, false, true, true]);
730        assert_eq!(
731            got,
732            vec![Finding::ScoresRunBackwards {
733                auc: 0.25,
734                inverted: 0.75
735            }]
736        );
737    }
738
739    #[test]
740    fn a_single_score_is_not_an_identical_set() {
741        // One answer cannot be inconsistent with itself, and calling it
742        // degenerate would fire on every one-image run.
743        assert!(findings(&[0.5, 0.5], &[true, false])
744            .contains(&Finding::EveryScoreIdentical { count: 2 }));
745        assert!(findings(&[0.5], &[true]).is_empty());
746    }
747
748    #[test]
749    fn a_set_that_cannot_be_ranked_produces_nothing() {
750        // Each of these is refused by name somewhere above; a finding here
751        // would describe an AUC that was never computed.
752        assert!(findings(&[0.5, 0.5], &[true, true]).is_empty());
753        assert!(findings(&[0.5, 0.5, 0.5], &[true, false]).is_empty());
754        assert!(findings(&[f64::NAN, f64::NAN], &[true, false]).is_empty());
755        assert!(findings(&[], &[]).is_empty());
756    }
757}
758
759#[cfg(test)]
760mod tests {
761    use super::*;
762
763    #[test]
764    fn a_budget_finer_than_one_clean_image_is_reported_as_not_expressible() {
765        // The finding this exists for. Six clean images, a 1% budget, and the
766        // honest answer is the zero-budget answer: 1/6 is 16.7%, so 1% buys
767        // nothing that 0% did not already buy. The figure is correct and the
768        // label "TPR@1%FA" is not.
769        let scores = [
770            0.9, 0.8, 0.7, 0.6, 0.1, 0.2, 0.3, 0.4, 0.5, 0.05, 0.06, 0.07,
771        ];
772        let labels = [
773            true, true, true, true, true, true, false, false, false, false, false, false,
774        ];
775        let p = operating_point(&scores, &labels, 0.01).expect("a point");
776        assert_eq!(p.requested_fpr, 0.01);
777        assert_eq!(
778            p.achieved_fpr, 0.0,
779            "1% cannot be spent on six clean images"
780        );
781        assert!(!p.budget_was_expressible(6));
782        // And the same corpus CAN express a budget of one image in six.
783        let coarse = operating_point(&scores, &labels, 1.0 / 6.0).expect("a point");
784        assert!(coarse.budget_was_expressible(6));
785    }
786
787    #[test]
788    fn a_budget_the_corpus_can_spend_is_reported_as_expressible() {
789        // Two hundred clean images make 1% a real budget: two of them.
790        assert!(fpr_resolution(200) <= 0.01);
791        let mut scores = Vec::new();
792        let mut labels = Vec::new();
793        for i in 0..200 {
794            scores.push(i as f64 / 200.0);
795            labels.push(false);
796        }
797        for i in 0..200 {
798            scores.push(1.0 + i as f64);
799            labels.push(true);
800        }
801        let p = operating_point(&scores, &labels, 0.01).expect("a point");
802        assert!(p.budget_was_expressible(200));
803        assert_eq!(p.tpr, 1.0);
804    }
805
806    #[test]
807    fn the_operating_point_agrees_with_the_figure_tpr_at_fpr_reports() {
808        // Two ways of asking one question must not drift apart.
809        let scores = [0.9, 0.4, 0.5, 0.1];
810        let labels = [true, true, false, false];
811        for budget in [0.0, 0.25, 0.5, 1.0] {
812            let a = tpr_at_fpr(&scores, &labels, budget);
813            let b = operating_point(&scores, &labels, budget).map(|p| p.tpr);
814            assert_eq!(a, b, "at a budget of {budget}");
815        }
816    }
817
818    #[test]
819    fn the_achieved_rate_never_exceeds_the_budget_that_was_asked_for() {
820        let scores = [0.9, 0.4, 0.5, 0.1];
821        let labels = [true, true, false, false];
822        for budget in [0.0, 0.1, 0.25, 0.4, 0.5, 0.75, 1.0] {
823            let p = operating_point(&scores, &labels, budget).expect("a point");
824            assert!(
825                p.achieved_fpr <= budget + 1e-12,
826                "spent {} of a {budget} budget",
827                p.achieved_fpr
828            );
829        }
830    }
831
832    // ---- the threshold, which is the number a developer came for ---------
833
834    #[test]
835    fn the_curve_and_the_points_it_is_derived_from_agree_exactly() {
836        // `roc_curve` is now a projection of `roc_points`, and this is what
837        // stops a future edit making them two implementations again.
838        let scores = [0.9, 0.8, 0.8, 0.4, 0.1, 0.1, 0.7];
839        let labels = [true, true, false, true, false, false, false];
840        let from_points: Vec<(f64, f64)> = roc_points(&scores, &labels)
841            .into_iter()
842            .map(|p| (p.fpr, p.tpr))
843            .collect();
844        assert_eq!(roc_curve(&scores, &labels), from_points);
845    }
846
847    #[test]
848    fn the_origin_carries_no_threshold_because_nothing_is_flagged_there() {
849        let points = roc_points(&[0.9, 0.1], &[true, false]);
850        assert_eq!(points[0].threshold, None);
851        assert_eq!((points[0].fpr, points[0].tpr), (0.0, 0.0));
852        // Every other vertex has one.
853        assert!(points[1..].iter().all(|p| p.threshold.is_some()));
854    }
855
856    #[test]
857    fn the_threshold_at_a_vertex_flags_exactly_the_images_that_vertex_counts() {
858        // The contract is `score >= threshold`. Checked by applying it rather
859        // than by trusting the sweep: this is the number somebody puts in an
860        // `if`, so it has to mean what it says.
861        let scores = [0.9, 0.75, 0.75, 0.6, 0.2, 0.05];
862        let labels = [true, true, false, true, false, false];
863        let n_pos = labels.iter().filter(|l| **l).count() as f64;
864        let n_neg = labels.len() as f64 - n_pos;
865        for point in roc_points(&scores, &labels) {
866            let Some(threshold) = point.threshold else {
867                continue;
868            };
869            let flagged: Vec<bool> = scores.iter().map(|s| *s >= threshold).collect();
870            let tp = flagged
871                .iter()
872                .zip(labels)
873                .filter(|(f, l)| **f && *l)
874                .count() as f64;
875            let fp = flagged
876                .iter()
877                .zip(labels)
878                .filter(|(f, l)| **f && !*l)
879                .count() as f64;
880            assert_eq!(
881                (fp / n_neg, tp / n_pos),
882                (point.fpr, point.tpr),
883                "threshold {threshold} does not produce the rates its vertex claims"
884            );
885        }
886    }
887
888    #[test]
889    fn an_operating_point_reports_the_threshold_that_bought_its_rates() {
890        let scores = [0.9, 0.8, 0.7, 0.6, 0.5, 0.4];
891        let labels = [true, true, true, false, false, false];
892        let point = operating_point(&scores, &labels, 0.0).expect("a point");
893        // Perfect separation, so a zero budget buys everything, and the cutoff
894        // is the lowest stego score rather than the highest clean one.
895        assert_eq!(point.tpr, 1.0);
896        assert_eq!(point.achieved_fpr, 0.0);
897        assert_eq!(point.threshold, Some(0.7));
898    }
899
900    #[test]
901    fn the_threshold_moves_down_as_the_budget_is_loosened() {
902        let scores = [0.9, 0.5, 0.8, 0.4, 0.7, 0.3];
903        let labels = [true, true, true, false, false, false];
904        let tight = operating_point(&scores, &labels, 0.0).expect("tight");
905        let loose = operating_point(&scores, &labels, 1.0).expect("loose");
906        let (Some(t), Some(l)) = (tight.threshold, loose.threshold) else {
907            panic!("both budgets should name a cutoff here");
908        };
909        assert!(
910            l <= t,
911            "loosening the budget raised the cutoff: {t} then {l}"
912        );
913        assert!(loose.tpr >= tight.tpr);
914    }
915
916    #[test]
917    fn the_threshold_agrees_with_the_rate_tpr_at_fpr_reports() {
918        // The two must not disagree: one is the figure published and the other
919        // is the cutoff published beside it.
920        let scores = [0.31, 0.62, 0.11, 0.88, 0.47, 0.05, 0.73, 0.22];
921        let labels = [false, true, false, true, true, false, true, false];
922        for budget in [0.0, 0.25, 0.5, 1.0] {
923            let point = operating_point(&scores, &labels, budget).expect("a point");
924            assert_eq!(
925                Some(point.tpr),
926                tpr_at_fpr(&scores, &labels, budget),
927                "operating_point and tpr_at_fpr disagree at budget {budget}"
928            );
929        }
930    }
931
932    #[test]
933    fn a_detector_answering_one_number_to_everything_has_one_usable_threshold() {
934        // Every score identical: the curve is the origin and one vertex, and the
935        // cutoff flags everything. Worth pinning because this is the shape a
936        // broken adapter produces and it must not panic or invent a number.
937        let points = roc_points(&[0.5; 4], &[true, true, false, false]);
938        assert_eq!(points.len(), 2);
939        assert_eq!(points[1].threshold, Some(0.5));
940        assert_eq!((points[1].fpr, points[1].tpr), (1.0, 1.0));
941    }
942
943    #[test]
944    fn an_unrankable_set_yields_no_points_and_therefore_no_threshold() {
945        assert!(roc_points(&[f64::NAN, 0.1], &[true, false]).is_empty());
946        assert!(roc_points(&[0.1], &[true]).is_empty());
947        assert!(operating_point(&[f64::NAN, 0.1], &[true, false], 0.5).is_none());
948    }
949
950    #[test]
951    fn the_resolution_of_a_corpus_with_no_clean_images_is_not_a_number_to_divide_by() {
952        assert!(fpr_resolution(0).is_infinite());
953        // Which means no budget is ever expressible, rather than all of them.
954        let p = OperatingPoint {
955            tpr: 1.0,
956            achieved_fpr: 0.0,
957            requested_fpr: 0.5,
958            threshold: Some(0.9),
959        };
960        assert!(!p.budget_was_expressible(0));
961    }
962
963    #[test]
964    fn tpr_at_fpr_perfect_separation_is_one_at_zero_budget() {
965        let scores = [0.1, 0.2, 0.8, 0.9];
966        let labels = [false, false, true, true];
967        assert_eq!(tpr_at_fpr(&scores, &labels, 0.0), Some(1.0));
968    }
969
970    #[test]
971    fn tpr_at_fpr_tightens_as_the_budget_shrinks() {
972        // 2 pos (0.9, 0.4), 2 neg (0.5, 0.1). Catching the 0.4 positive means
973        // first admitting the 0.5 negative, so a zero-FPR budget buys half.
974        let scores = [0.9, 0.4, 0.5, 0.1];
975        let labels = [true, true, false, false];
976        assert_eq!(tpr_at_fpr(&scores, &labels, 0.0), Some(0.5));
977        assert_eq!(tpr_at_fpr(&scores, &labels, 0.5), Some(1.0));
978    }
979
980    #[test]
981    fn tpr_at_fpr_is_zero_when_the_budget_buys_nothing() {
982        // Every negative outranks every positive: no detection at zero FPR.
983        let scores = [0.1, 0.2, 0.8, 0.9];
984        let labels = [true, true, false, false];
985        assert_eq!(tpr_at_fpr(&scores, &labels, 0.0), Some(0.0));
986    }
987
988    #[test]
989    fn tpr_at_fpr_undefined_with_single_class() {
990        assert_eq!(tpr_at_fpr(&[0.1, 0.2], &[true, true], 0.01), None);
991    }
992
993    /// A detector that only answers yes or no has two operating points, so the
994    /// budget either admits its false positives or refuses them outright. This
995    /// is exactly why `Detector::is_graded` exists: the number is real, but it
996    /// cannot be traded off the way a graded detector's can.
997    #[test]
998    fn tpr_at_fpr_handles_a_binary_detector() {
999        let scores = [1.0, 1.0, 1.0, 0.0];
1000        let labels = [true, true, false, false];
1001        assert_eq!(tpr_at_fpr(&scores, &labels, 0.0), Some(0.0));
1002        assert_eq!(tpr_at_fpr(&scores, &labels, 0.5), Some(1.0));
1003    }
1004
1005    #[test]
1006    fn confusion_counts_and_rates() {
1007        // labels:  S S S C C   (3 stego, 2 clean)
1008        // preds:   1 1 0 1 0
1009        let labels = [true, true, true, false, false];
1010        let preds = [true, true, false, true, false];
1011        let c = Confusion::tally(&labels, &preds);
1012        assert_eq!((c.tp, c.fn_, c.fp, c.tn), (2, 1, 1, 1));
1013        assert!((c.tpr() - 2.0 / 3.0).abs() < 1e-12);
1014        assert!((c.fpr() - 0.5).abs() < 1e-12);
1015        assert!((c.precision() - 2.0 / 3.0).abs() < 1e-12);
1016        assert!((c.accuracy() - 3.0 / 5.0).abs() < 1e-12);
1017        assert!((c.f1() - 2.0 / 3.0).abs() < 1e-12);
1018    }
1019
1020    #[test]
1021    fn confusion_zero_denominators_are_zero_not_nan() {
1022        let c = Confusion::default();
1023        assert_eq!(c.tpr(), 0.0);
1024        assert_eq!(c.fpr(), 0.0);
1025        assert_eq!(c.precision(), 0.0);
1026        assert_eq!(c.accuracy(), 0.0);
1027        assert_eq!(c.f1(), 0.0);
1028    }
1029
1030    /// The honest counterpart agrees with the truncating one whenever the
1031    /// truncating one had nothing to truncate.
1032    #[test]
1033    fn try_tally_matches_tally_on_well_formed_input() {
1034        let labels = [true, true, true, false, false];
1035        let preds = [true, true, false, true, false];
1036        assert_eq!(
1037            Confusion::try_tally(&labels, &preds),
1038            Ok(Confusion::tally(&labels, &preds))
1039        );
1040    }
1041
1042    /// Labels and predictions of different lengths came from different record
1043    /// sets. `tally` walks the shorter one; this refuses, which is the whole
1044    /// reason for having both.
1045    #[test]
1046    fn try_tally_refuses_a_length_mismatch_in_either_direction() {
1047        let labels = [true, true, false, false];
1048        assert_eq!(
1049            Confusion::try_tally(&labels, &[true, false, true]),
1050            Err(TallyError::LengthMismatch {
1051                labels: 4,
1052                predicted_positive: 3
1053            })
1054        );
1055        assert_eq!(
1056            Confusion::try_tally(&labels[..3], &[true, false, true, false]),
1057            Err(TallyError::LengthMismatch {
1058                labels: 3,
1059                predicted_positive: 4
1060            })
1061        );
1062    }
1063
1064    /// Zero samples is not a score of zero. Every rate on `Confusion` answers
1065    /// 0.0 on a zero denominator, so an empty count is indistinguishable from a
1066    /// detector that missed everything.
1067    #[test]
1068    fn try_tally_refuses_an_empty_run() {
1069        assert_eq!(Confusion::try_tally(&[], &[]), Err(TallyError::Empty));
1070    }
1071
1072    /// One class is a real measurement here, unlike a ranking. `must_clear` in
1073    /// a detector's self test hands over clean images only, and the false
1074    /// positives that come back are the answer it wanted.
1075    #[test]
1076    fn try_tally_accepts_a_single_class() {
1077        let clean_only = Confusion::try_tally(&[false, false, false], &[true, false, false]);
1078        assert_eq!(
1079            clean_only,
1080            Ok(Confusion {
1081                tp: 0,
1082                fp: 1,
1083                tn: 2,
1084                fn_: 0
1085            })
1086        );
1087        let stego_only = Confusion::try_tally(&[true, true], &[true, false]);
1088        assert_eq!(
1089            stego_only,
1090            Ok(Confusion {
1091                tp: 1,
1092                fp: 0,
1093                tn: 0,
1094                fn_: 1
1095            })
1096        );
1097    }
1098
1099    /// The point of adding a second function rather than changing the first:
1100    /// `tally` still truncates, and a caller depending on that still gets it.
1101    #[test]
1102    fn tally_still_truncates_exactly_as_before() {
1103        let labels = [true, true, false, false];
1104        let preds = [true, false, true];
1105        let c = Confusion::tally(&labels, &preds);
1106        assert_eq!((c.tp, c.fn_, c.fp, c.tn), (1, 1, 1, 0));
1107        assert_eq!(Confusion::tally(&[], &[]), Confusion::default());
1108        assert_eq!(
1109            Confusion::tally(&labels, &[]),
1110            Confusion::default(),
1111            "an empty prediction slice truncates to nothing rather than panicking"
1112        );
1113    }
1114
1115    /// The error has to survive being turned into a message a user reads, so
1116    /// each variant says which condition fired and names the figures.
1117    #[test]
1118    fn a_tally_error_says_which_condition_fired() {
1119        let mismatch = TallyError::LengthMismatch {
1120            labels: 9_882,
1121            predicted_positive: 10_000,
1122        };
1123        let text = mismatch.to_string();
1124        assert!(text.contains("9882"), "{text}");
1125        assert!(text.contains("10000"), "{text}");
1126        assert!(text.contains("different record sets"), "{text}");
1127
1128        let empty = TallyError::Empty.to_string();
1129        assert!(empty.contains("nothing to count"), "{empty}");
1130
1131        // Usable as a `std::error::Error`, so a caller can box it like any
1132        // other failure rather than matching on it by hand.
1133        let boxed: Box<dyn std::error::Error> = Box::new(mismatch);
1134        assert_eq!(boxed.to_string(), text);
1135    }
1136
1137    #[test]
1138    fn auc_perfect_separation_is_one() {
1139        let scores = [0.1, 0.2, 0.8, 0.9];
1140        let labels = [false, false, true, true];
1141        assert_eq!(roc_auc(&scores, &labels), Some(1.0));
1142    }
1143
1144    #[test]
1145    fn auc_reversed_is_zero_and_random_is_half() {
1146        let scores = [0.9, 0.8, 0.2, 0.1];
1147        let labels = [false, false, true, true];
1148        assert_eq!(roc_auc(&scores, &labels), Some(0.0));
1149
1150        // Perfectly interleaved with ties handled by average rank → 0.5.
1151        let s = [0.5, 0.5, 0.5, 0.5];
1152        let l = [true, false, true, false];
1153        assert_eq!(roc_auc(&s, &l), Some(0.5));
1154    }
1155
1156    #[test]
1157    fn auc_undefined_with_single_class() {
1158        assert_eq!(roc_auc(&[0.1, 0.2], &[true, true]), None);
1159        assert_eq!(roc_auc(&[0.1, 0.2], &[false, false]), None);
1160    }
1161
1162    #[test]
1163    fn auc_matches_manual_small_case() {
1164        // 2 pos, 2 neg; one pos below one neg → AUC = 3/4.
1165        let scores = [0.3, 0.4, 0.35, 0.2];
1166        let labels = [true, true, false, false];
1167        let auc = roc_auc(&scores, &labels).unwrap();
1168        assert!((auc - 0.75).abs() < 1e-12, "auc={auc}");
1169    }
1170
1171    #[test]
1172    fn roc_curve_reaches_corners() {
1173        let scores = [0.1, 0.2, 0.8, 0.9];
1174        let labels = [false, false, true, true];
1175        let curve = roc_curve(&scores, &labels);
1176        assert_eq!(curve.first(), Some(&(0.0, 0.0)));
1177        assert_eq!(curve.last(), Some(&(1.0, 1.0)));
1178        // Perfect separation: tpr hits 1.0 before any fpr accrues.
1179        assert!(curve.iter().any(|&(fpr, tpr)| fpr == 0.0 && tpr == 1.0));
1180    }
1181
1182    #[test]
1183    fn roc_curve_empty_without_both_classes() {
1184        assert!(roc_curve(&[0.1, 0.2], &[true, true]).is_empty());
1185    }
1186
1187    /// Scores and labels of different lengths mean they were built from
1188    /// different record sets. Walking the shorter of the two would answer with a
1189    /// figure measured on a subset nobody named, and the caller would have no
1190    /// way of telling that from a real one.
1191    #[test]
1192    fn a_length_mismatch_is_refused_rather_than_truncated() {
1193        let scores = [0.9, 0.8, 0.2, 0.1];
1194        assert_eq!(roc_auc(&scores, &[true, false, true]), None);
1195        assert_eq!(roc_auc(&scores[..3], &[true, false, true, false]), None);
1196        assert!(roc_curve(&scores, &[true, false, true]).is_empty());
1197        assert_eq!(tpr_at_fpr(&scores, &[true, false, true], 0.01), None);
1198    }
1199
1200    /// A detector that answers NaN for an image has not scored it. NaN compares
1201    /// false against everything, so it lands wherever the sort happens to leave
1202    /// it and the rank sum built on top of that is arbitrary.
1203    #[test]
1204    fn a_nan_score_is_refused_rather_than_ranked() {
1205        let scores = [0.9, f64::NAN, 0.2, 0.1];
1206        let labels = [true, true, false, false];
1207        assert_eq!(roc_auc(&scores, &labels), None);
1208        assert!(roc_curve(&scores, &labels).is_empty());
1209        assert_eq!(tpr_at_fpr(&scores, &labels, 0.01), None);
1210    }
1211
1212    /// A budget that is not a rate is a caller's mistake, and 0.0 would read as
1213    /// a detector that caught nothing at a budget it was never asked about.
1214    #[test]
1215    fn a_budget_that_is_not_a_rate_is_refused() {
1216        let scores = [0.9, 0.4, 0.5, 0.1];
1217        let labels = [true, true, false, false];
1218        assert_eq!(tpr_at_fpr(&scores, &labels, -0.1), None);
1219        assert_eq!(tpr_at_fpr(&scores, &labels, 1.5), None);
1220        assert_eq!(tpr_at_fpr(&scores, &labels, f64::NAN), None);
1221        // The ends of the range are budgets, not mistakes.
1222        assert_eq!(tpr_at_fpr(&scores, &labels, 0.0), Some(0.5));
1223        assert_eq!(tpr_at_fpr(&scores, &labels, 1.0), Some(1.0));
1224    }
1225}
1226
1227#[cfg(test)]
1228mod delong_tests {
1229    use super::*;
1230
1231    /// Hanley and McNeil's 1982 worked example, which is the one every
1232    /// implementation is checked against.
1233    ///
1234    /// Ratings 1 to 5 for 58 normal and 51 abnormal cases. The published AUC
1235    /// is 0.893 and the published standard error 0.029, computed by their own
1236    /// parametric method; DeLong on the same data gives an SE close to but
1237    /// not identical with it, which is the point of preferring DeLong. The
1238    /// assertion is therefore that the AUC matches to three places and the SE
1239    /// lands in the range every published DeLong implementation agrees on.
1240    fn hanley_mcneil() -> (Vec<f64>, Vec<bool>) {
1241        // (rating, n_normal, n_abnormal)
1242        let table = [
1243            (1.0, 33, 3),
1244            (2.0, 6, 2),
1245            (3.0, 6, 2),
1246            (4.0, 11, 11),
1247            (5.0, 2, 33),
1248        ];
1249        let mut scores = Vec::new();
1250        let mut labels = Vec::new();
1251        for (rating, n_norm, n_abn) in table {
1252            for _ in 0..n_norm {
1253                scores.push(rating);
1254                labels.push(false);
1255            }
1256            for _ in 0..n_abn {
1257                scores.push(rating);
1258                labels.push(true);
1259            }
1260        }
1261        (scores, labels)
1262    }
1263
1264    #[test]
1265    fn the_published_worked_example_comes_out_where_it_is_published() {
1266        let (scores, labels) = hanley_mcneil();
1267        let got = roc_auc_interval(&scores, &labels, Z_95).expect("both classes, enough of each");
1268        assert!(
1269            (got.auc - 0.893).abs() < 0.001,
1270            "AUC {} is not the published 0.893",
1271            got.auc
1272        );
1273        assert!(
1274            (0.025..0.035).contains(&got.standard_error),
1275            "standard error {} is outside what every DeLong implementation gives here",
1276            got.standard_error
1277        );
1278        assert!(got.low < got.auc && got.auc < got.high);
1279    }
1280
1281    /// This test used to assert the opposite, and the opposite was wrong.
1282    ///
1283    /// Six observations that happen to separate cleanly say nothing about
1284    /// how the detector behaves on the seventh, so "95% confident the AUC is
1285    /// between 1.0 and 1.0" is a claim no sample of this size supports. It
1286    /// was also the single most quotable line the tool produced.
1287    #[test]
1288    fn a_perfect_separation_reports_no_interval_rather_than_a_width_of_zero() {
1289        let scores = vec![1.0, 2.0, 3.0, 10.0, 11.0, 12.0];
1290        let labels = vec![false, false, false, true, true, true];
1291        assert_eq!(roc_auc(&scores, &labels), Some(1.0));
1292        assert_eq!(roc_auc_interval(&scores, &labels, Z_95), None);
1293    }
1294
1295    /// The same collapse from the other direction.
1296    ///
1297    /// Every comparison is a tie, so every structural component is identical
1298    /// and the variance is zero. The estimator is not certain here; it has
1299    /// nothing to say. What the detector did is reported by the identical
1300    /// answers finding, not by a zero width interval.
1301    #[test]
1302    fn one_answer_to_everything_reports_no_interval_either() {
1303        let scores = vec![7.0; 8];
1304        let labels = vec![true, false, true, false, true, false, true, false];
1305        assert_eq!(roc_auc(&scores, &labels), Some(0.5));
1306        assert_eq!(roc_auc_interval(&scores, &labels, Z_95), None);
1307    }
1308
1309    /// A real sample still gets a real interval, so the guard above cannot
1310    /// be quietly swallowing every case.
1311    #[test]
1312    fn an_ordinary_sample_still_has_a_width() {
1313        let scores = vec![1.0, 4.0, 2.0, 9.0, 3.0, 8.0, 5.0, 7.0];
1314        let labels = vec![false, true, false, true, false, true, false, true];
1315        let got = roc_auc_interval(&scores, &labels, Z_95).expect("an interval");
1316        assert!(got.standard_error > 0.0);
1317        assert!(got.low < got.high, "[{}, {}]", got.low, got.high);
1318    }
1319
1320    #[test]
1321    fn the_interval_is_clamped_to_the_scale_it_is_measured_on() {
1322        // Near 1.0 the normal interval runs off the end, and an upper bound
1323        // of 1.04 beside a measurement is nonsense a reader would quote.
1324        let mut scores: Vec<f64> = (0..20).map(|i| i as f64).collect();
1325        let mut labels = vec![false; 20];
1326        scores.extend((0..20).map(|i| 100.0 + i as f64));
1327        labels.extend(vec![true; 20]);
1328        scores[0] = 200.0; // one negative above everything, so AUC is high but not 1
1329        let got = roc_auc_interval(&scores, &labels, Z_95).expect("valid");
1330        assert!(got.high <= 1.0, "upper bound {} left the scale", got.high);
1331        assert!(got.low >= 0.0, "lower bound {} left the scale", got.low);
1332    }
1333
1334    #[test]
1335    fn one_of_a_class_is_refused_because_a_variance_needs_two() {
1336        let scores = vec![1.0, 2.0, 3.0];
1337        let labels = vec![true, false, false];
1338        assert_eq!(roc_auc_interval(&scores, &labels, Z_95), None);
1339        // And the point estimate is still available, because that one IS
1340        // defined with a single positive.
1341        assert!(roc_auc(&scores, &labels).is_some());
1342    }
1343
1344    #[test]
1345    fn everything_roc_auc_refuses_this_refuses_too() {
1346        assert_eq!(roc_auc_interval(&[1.0, 2.0], &[true], Z_95), None);
1347        assert_eq!(
1348            roc_auc_interval(
1349                &[1.0, f64::NAN, 3.0, 4.0],
1350                &[true, true, false, false],
1351                Z_95
1352            ),
1353            None
1354        );
1355        assert_eq!(roc_auc_interval(&[1.0, 2.0], &[true, true], Z_95), None);
1356    }
1357
1358    #[test]
1359    fn a_nonsense_width_is_refused_rather_than_producing_a_nonsense_interval() {
1360        let scores = vec![1.0, 2.0, 3.0, 4.0];
1361        let labels = vec![false, false, true, true];
1362        assert_eq!(roc_auc_interval(&scores, &labels, f64::NAN), None);
1363        assert_eq!(roc_auc_interval(&scores, &labels, -1.0), None);
1364    }
1365
1366    #[test]
1367    fn a_wider_z_gives_a_wider_interval_around_the_same_point() {
1368        let (scores, labels) = hanley_mcneil();
1369        let narrow = roc_auc_interval(&scores, &labels, 1.0).expect("valid");
1370        let wide = roc_auc_interval(&scores, &labels, Z_95).expect("valid");
1371        assert_eq!(narrow.auc, wide.auc);
1372        assert_eq!(narrow.standard_error, wide.standard_error);
1373        assert!(wide.high - wide.low > narrow.high - narrow.low);
1374    }
1375
1376    #[test]
1377    fn the_point_estimate_is_the_same_number_roc_auc_gives() {
1378        // Two implementations of one metric is the thing this crate exists to
1379        // avoid, so the interval must not quietly compute its own AUC.
1380        let (scores, labels) = hanley_mcneil();
1381        let plain = roc_auc(&scores, &labels).expect("valid");
1382        let with_ci = roc_auc_interval(&scores, &labels, Z_95).expect("valid");
1383        assert_eq!(plain, with_ci.auc);
1384    }
1385}