stegobench_metrics/lib.rs
1// Author: Daniel Iwugo
2// Comment: Christ is King
3// SPDX-License-Identifier: AGPL-3.0-or-later
4// Copyright (C) 2026 Daniel Iwugo
5//
6// Migrated from Stegcore's `stegcore-ops` crate, where it was dual licensed.
7// It is relicensed AGPL-only here by its author. The code is unchanged apart
8// from this header: it arrives with its thirteen tests, and a metric that has
9// been producing published numbers is not the place to start rewriting.
10//
11// It lives in the benchmark rather than in Stegcore because the benchmark must
12// not be built out of the product it judges.
13
14//! Detection metrics for the comparative benchmark.
15//!
16//! Pure functions over labels and scores: a binary confusion matrix (positive
17//! = stego), the rank-based ROC AUC (Mann-Whitney, tie-aware), and the ROC
18//! curve points. These are deliberately storage- and tool-agnostic so the
19//! same code scores Stegcore's ensemble, any single detector, or an external
20//! comparator.
21
22use std::cmp::Ordering;
23use std::fmt;
24
25/// Why a confusion count could not be taken honestly.
26///
27/// Hand written rather than derived, because this crate has no dependencies on
28/// purpose: a metric whose number needs an audit of somebody else's crate before
29/// it can be trusted is not the metric this benchmark wants to publish.
30#[derive(Debug, Clone, Copy, PartialEq, Eq)]
31pub enum TallyError {
32 /// The two slices were built from different record sets.
33 LengthMismatch {
34 /// How many labels the caller handed over.
35 labels: usize,
36 /// How many predictions the caller handed over.
37 predicted_positive: usize,
38 },
39 /// Nothing was classified at all.
40 Empty,
41}
42
43impl fmt::Display for TallyError {
44 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
45 match self {
46 TallyError::LengthMismatch {
47 labels,
48 predicted_positive,
49 } => write!(
50 f,
51 "{labels} labels against {predicted_positive} predictions: the \
52 two were built from different record sets, so any count over \
53 them would describe images nobody named"
54 ),
55 TallyError::Empty => write!(
56 f,
57 "nothing to count: an empty confusion matrix reads as a \
58 detector that got everything wrong rather than one that was \
59 never run"
60 ),
61 }
62 }
63}
64
65impl std::error::Error for TallyError {}
66
67/// Binary classification counts. Positive is "stego".
68#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
69pub struct Confusion {
70 pub tp: u64,
71 pub fp: u64,
72 pub tn: u64,
73 pub fn_: u64,
74}
75
76impl Confusion {
77 /// Tally counts from per-sample labels and predicted-positive flags,
78 /// **truncating to the shorter slice**.
79 ///
80 /// Prefer [`Confusion::try_tally`], which refuses that case instead. This
81 /// one walks the two slices pairwise and stops at whichever runs out first,
82 /// so a caller that built the labels and the predictions from different
83 /// record sets gets a count over a subset nobody named, with nothing in the
84 /// return value to say so. It is kept, undeprecated, only because it is
85 /// published and callers depend on the signature.
86 pub fn tally(labels: &[bool], predicted_positive: &[bool]) -> Self {
87 let mut c = Confusion::default();
88 for (&label, &pred) in labels.iter().zip(predicted_positive) {
89 match (label, pred) {
90 (true, true) => c.tp += 1,
91 (false, true) => c.fp += 1,
92 (false, false) => c.tn += 1,
93 (true, false) => c.fn_ += 1,
94 }
95 }
96 c
97 }
98
99 /// Tally counts from per-sample labels and predicted-positive flags,
100 /// refusing anything that cannot honestly produce a count.
101 ///
102 /// This is [`Confusion::tally`] with the truncation taken out. A length
103 /// mismatch means the labels and the predictions came from different record
104 /// sets, and walking the shorter of the two publishes a figure measured on
105 /// images nobody named. An empty pair of slices is refused too: every rate
106 /// on [`Confusion`] answers 0.0 on a zero denominator, so an empty count
107 /// reads as a detector that caught nothing rather than one that was never
108 /// run.
109 ///
110 /// A single class is **not** refused, unlike [`roc_auc`] and [`roc_curve`].
111 /// A ranking needs both a positive and a negative to rank against each
112 /// other; a confusion count does not, and a one-sided run is a real
113 /// measurement here. The `must_clear` half of a detector's self test is
114 /// exactly that: clean images only, where the whole question is how many
115 /// false positives came back.
116 ///
117 /// NaN has no analogue to refuse: both inputs are `bool`, so there is no
118 /// unordered or absent value a detector could hand over. A detector that
119 /// failed to score an image must be left out of both slices rather than
120 /// given a placeholder flag, and the length check is what catches a caller
121 /// that dropped it from one slice and not the other.
122 pub fn try_tally(labels: &[bool], predicted_positive: &[bool]) -> Result<Self, TallyError> {
123 if labels.len() != predicted_positive.len() {
124 return Err(TallyError::LengthMismatch {
125 labels: labels.len(),
126 predicted_positive: predicted_positive.len(),
127 });
128 }
129 if labels.is_empty() {
130 return Err(TallyError::Empty);
131 }
132 Ok(Self::tally(labels, predicted_positive))
133 }
134
135 fn ratio(num: u64, den: u64) -> f64 {
136 if den == 0 {
137 0.0
138 } else {
139 num as f64 / den as f64
140 }
141 }
142
143 /// True-positive rate (detection rate / recall): tp / (tp + fn).
144 pub fn tpr(&self) -> f64 {
145 Self::ratio(self.tp, self.tp + self.fn_)
146 }
147
148 /// False-positive rate: fp / (fp + tn).
149 pub fn fpr(&self) -> f64 {
150 Self::ratio(self.fp, self.fp + self.tn)
151 }
152
153 /// Precision: tp / (tp + fp).
154 pub fn precision(&self) -> f64 {
155 Self::ratio(self.tp, self.tp + self.fp)
156 }
157
158 /// Accuracy: (tp + tn) / total.
159 pub fn accuracy(&self) -> f64 {
160 Self::ratio(self.tp + self.tn, self.tp + self.tn + self.fp + self.fn_)
161 }
162
163 /// F1 score: harmonic mean of precision and recall; 0 when both are 0.
164 pub fn f1(&self) -> f64 {
165 let (p, r) = (self.precision(), self.tpr());
166 if p + r == 0.0 {
167 0.0
168 } else {
169 2.0 * p * r / (p + r)
170 }
171 }
172}
173
174fn cmp_f64(a: f64, b: f64) -> Ordering {
175 a.partial_cmp(&b).unwrap_or(Ordering::Equal)
176}
177
178/// Whether these two slices can be ranked against each other at all.
179///
180/// A length mismatch means the caller built the scores and the labels from
181/// different record sets, and a ranking over the shorter of the two is a number
182/// measured on a subset nobody named. A score that is not a number cannot be
183/// ordered, so every comparison against it is false and the ranking silently
184/// stops meaning anything. Both answer `None` rather than a figure, because in
185/// a measurement tool a quietly wrong number is worse than no number: it gets
186/// published.
187fn rankable(scores: &[f64], labels: &[bool]) -> bool {
188 scores.len() == labels.len() && scores.iter().all(|s| !s.is_nan())
189}
190
191/// An AUC with the uncertainty that belongs beside it.
192///
193/// A bare AUC printed to sixteen digits invites a reader to compare two
194/// numbers that differ in the third, on corpora small enough that one image
195/// moves the figure further than that. The interval is the tool saying how
196/// much of its own answer is real.
197#[derive(Debug, Clone, Copy, PartialEq)]
198pub struct AucInterval {
199 /// The point estimate, identical to [`roc_auc`].
200 pub auc: f64,
201 /// DeLong's standard error.
202 pub standard_error: f64,
203 /// Lower bound, clamped into `[0, 1]` because AUC cannot leave it.
204 pub low: f64,
205 /// Upper bound, clamped the same way.
206 pub high: f64,
207 /// How many standard errors wide, so a reader knows what was asked for.
208 pub z: f64,
209}
210
211/// 1.959964, the two-sided normal quantile for 95 per cent.
212///
213/// Written out rather than computed: there is no inverse normal in the
214/// standard library, and this crate has no dependencies on purpose (a metric
215/// with a dependency graph is one somebody has to audit before trusting a
216/// number).
217pub const Z_95: f64 = 1.959_963_984_540_054;
218
219/// Average ranks of `values` among themselves, 1-based, ties sharing the mean
220/// of the positions they span.
221///
222/// Split out because DeLong needs the same computation three times over three
223/// different sets, and a second copy of a tie rule is how two of them come to
224/// disagree.
225fn midranks(values: &[f64]) -> Vec<f64> {
226 let mut idx: Vec<usize> = (0..values.len()).collect();
227 idx.sort_by(|&a, &b| cmp_f64(values[a], values[b]));
228 let mut ranks = vec![0.0f64; values.len()];
229 let mut i = 0;
230 while i < idx.len() {
231 let mut j = i;
232 while j + 1 < idx.len() && values[idx[j + 1]] == values[idx[i]] {
233 j += 1;
234 }
235 let avg = ((i + 1) + (j + 1)) as f64 / 2.0;
236 for &k in &idx[i..=j] {
237 ranks[k] = avg;
238 }
239 i = j + 1;
240 }
241 ranks
242}
243
244/// AUC with a confidence interval, by DeLong's method.
245///
246/// WHY DELONG AND NOT HANLEY AND MCNEIL
247///
248/// The Hanley and McNeil standard error is two lines and assumes the scores
249/// are exponentially distributed within each class. Detector scores are not:
250/// several of the tools in this registry emit a bounded statistic, one emits
251/// a count, and one answers the same number to everything. Under those the
252/// parametric approximation is wrong in a direction nobody can predict from
253/// the output. DeLong's estimator makes no such assumption; it is computed
254/// from the data's own ranks and is what the statistical literature treats as
255/// the default for comparing classifiers.
256///
257/// Computed through midranks rather than the O(n_pos * n_neg) definition, so
258/// it stays O(n log n) and a Core tier run pays sorting rather than a hundred
259/// billion comparisons.
260///
261/// `z` is how many standard errors wide the interval is: [`Z_95`] for the
262/// usual 95 per cent.
263///
264/// Returns `None` for everything [`roc_auc`] returns `None` for, and also
265/// when either class has fewer than two members, because a variance over one
266/// observation is not an estimate of anything.
267pub fn roc_auc_interval(scores: &[f64], labels: &[bool], z: f64) -> Option<AucInterval> {
268 let auc = roc_auc(scores, labels)?;
269 if !z.is_finite() || z < 0.0 {
270 return None;
271 }
272
273 let pos: Vec<f64> = scores
274 .iter()
275 .zip(labels)
276 .filter(|(_, &l)| l)
277 .map(|(&s, _)| s)
278 .collect();
279 let neg: Vec<f64> = scores
280 .iter()
281 .zip(labels)
282 .filter(|(_, &l)| !l)
283 .map(|(&s, _)| s)
284 .collect();
285 let (m, n) = (pos.len(), neg.len());
286 if m < 2 || n < 2 {
287 return None;
288 }
289
290 // Ranks of each class within itself, and of everything within everything.
291 let t_x = midranks(&pos);
292 let t_y = midranks(&neg);
293 let mut all = Vec::with_capacity(m + n);
294 all.extend_from_slice(&pos);
295 all.extend_from_slice(&neg);
296 let t_z = midranks(&all);
297
298 // DeLong's structural components. V10 is, for each positive, the fraction
299 // of negatives it beats; V01 the mirror. Both fall out of the difference
300 // between a point's rank among everything and its rank among its own
301 // class, which is why this costs a sort rather than a product.
302 let v10: Vec<f64> = (0..m).map(|i| (t_z[i] - t_x[i]) / n as f64).collect();
303 let v01: Vec<f64> = (0..n)
304 .map(|j| 1.0 - (t_z[m + j] - t_y[j]) / m as f64)
305 .collect();
306
307 let s10 = sample_variance(&v10)?;
308 let s01 = sample_variance(&v01)?;
309 let var = s10 / m as f64 + s01 / n as f64;
310 if !var.is_finite() || var < 0.0 {
311 return None;
312 }
313 let se = var.sqrt();
314
315 // A WIDTH OF ZERO IS NOT A MEASUREMENT OF PRECISION
316 //
317 // DeLong's variance is built from how much each observation moves the
318 // statistic. When the classes separate completely every positive beats
319 // every negative, each component is identical, the sample variance is
320 // exactly zero and the interval collapses onto the estimate. The same
321 // happens when a detector answers identically every time, which gives
322 // 0.5 with no width.
323 //
324 // Both cases printed a 95% interval of zero width on as few as four
325 // observations, which is the most quotable and least defensible line the
326 // tool could produce. There is no width to report here, so none is
327 // reported: the estimator has nothing to say at the boundary, and saying
328 // so is honest where inventing a bound from a method this crate does not
329 // implement would not be.
330 if se == 0.0 {
331 return None;
332 }
333
334 Some(AucInterval {
335 auc,
336 standard_error: se,
337 // Clamped, because the normal interval runs past the ends of the
338 // scale near 1.0 and an upper bound of 1.03 is not a thing anybody
339 // should print beside a measurement.
340 low: (auc - z * se).clamp(0.0, 1.0),
341 high: (auc + z * se).clamp(0.0, 1.0),
342 z,
343 })
344}
345
346/// Unbiased sample variance, or `None` for fewer than two observations.
347fn sample_variance(v: &[f64]) -> Option<f64> {
348 if v.len() < 2 {
349 return None;
350 }
351 let n = v.len() as f64;
352 let mean = v.iter().sum::<f64>() / n;
353 Some(v.iter().map(|x| (x - mean).powi(2)).sum::<f64>() / (n - 1.0))
354}
355
356/// Rank-based ROC AUC (equivalent to the Mann-Whitney U statistic), tie-aware
357/// via average ranks.
358///
359/// Returns `None` when one class is absent, since AUC is undefined without both
360/// a positive and a negative sample; when the two slices are of different
361/// lengths; and when any score is NaN.
362pub fn roc_auc(scores: &[f64], labels: &[bool]) -> Option<f64> {
363 if !rankable(scores, labels) {
364 return None;
365 }
366 let n_pos = labels.iter().filter(|&&l| l).count();
367 let n_neg = labels.len() - n_pos;
368 if n_pos == 0 || n_neg == 0 {
369 return None;
370 }
371
372 // Indices sorted by ascending score.
373 let mut idx: Vec<usize> = (0..scores.len()).collect();
374 idx.sort_by(|&a, &b| cmp_f64(scores[a], scores[b]));
375
376 // Average ranks (1-based), tie groups share the mean of their positions.
377 let mut ranks = vec![0.0f64; scores.len()];
378 let mut i = 0;
379 while i < idx.len() {
380 let mut j = i;
381 while j + 1 < idx.len() && scores[idx[j + 1]] == scores[idx[i]] {
382 j += 1;
383 }
384 let avg = ((i + 1) + (j + 1)) as f64 / 2.0;
385 for &k in &idx[i..=j] {
386 ranks[k] = avg;
387 }
388 i = j + 1;
389 }
390
391 let sum_pos: f64 = labels
392 .iter()
393 .zip(&ranks)
394 .filter(|(&l, _)| l)
395 .map(|(_, &r)| r)
396 .sum();
397 let auc = (sum_pos - (n_pos * (n_pos + 1)) as f64 / 2.0) / (n_pos as f64 * n_neg as f64);
398 Some(auc)
399}
400
401/// ROC curve as `(fpr, tpr)` points, swept from the highest score downward
402/// (predicted positive when `score >= threshold`). Begins at `(0, 0)` and ends
403/// at `(1, 1)`.
404///
405/// Empty when either class is absent, when the two slices are of different
406/// lengths, and when any score is NaN. See [`roc_auc`].
407pub fn roc_curve(scores: &[f64], labels: &[bool]) -> Vec<(f64, f64)> {
408 // Derived from `roc_points` rather than computed again, so the curve and the
409 // thresholds that produced it cannot drift apart. A second copy of this
410 // sweep is exactly the "arithmetic that quietly disagrees" this crate exists
411 // to avoid.
412 roc_points(scores, labels)
413 .into_iter()
414 .map(|p| (p.fpr, p.tpr))
415 .collect()
416}
417
418/// One vertex of the ROC curve, including the score that produced it.
419///
420/// WHY THE THRESHOLD IS CARRIED AND NOT THROWN AWAY
421///
422/// A developer wiring detection into a product needs one number: the score to
423/// compare against. Every other figure here says how GOOD the detector is at a
424/// false-alarm budget; only this one says what to type into the `if`.
425///
426/// It used to be computed and discarded. A journey walked by a developer whose
427/// whole goal was picking a cutoff searched the result document for it, found
428/// nothing, and wrote his own ROC code against the records file to get it. His
429/// figures reproduced these to within tie handling, which is the good news and
430/// also the point: the tool had already done the arithmetic and made him do it
431/// again, and a second implementation of a metric is the thing this crate was
432/// built to prevent.
433#[derive(Debug, Clone, Copy, PartialEq)]
434pub struct RocPoint {
435 /// Flag an image when its score is greater than or equal to this.
436 ///
437 /// `None` at the origin, where nothing is flagged at all. That vertex is
438 /// real and belongs on the curve, and it has no threshold because there is
439 /// no score high enough to be worth flagging; reporting a number there
440 /// would be inventing one.
441 pub threshold: Option<f64>,
442 pub fpr: f64,
443 pub tpr: f64,
444}
445
446/// The ROC curve as vertices carrying the threshold each one sits at.
447///
448/// Swept from the highest score downward, predicting positive when
449/// `score >= threshold`. Begins at the origin, where nothing is flagged, and
450/// ends at `(1, 1)`, where everything is.
451///
452/// Empty under exactly the conditions [`roc_curve`] is empty.
453pub fn roc_points(scores: &[f64], labels: &[bool]) -> Vec<RocPoint> {
454 if !rankable(scores, labels) {
455 return Vec::new();
456 }
457 let n_pos = labels.iter().filter(|&&l| l).count() as f64;
458 let n_neg = labels.len() as f64 - n_pos;
459 if n_pos == 0.0 || n_neg == 0.0 {
460 return Vec::new();
461 }
462
463 let mut pairs: Vec<(f64, bool)> = scores.iter().copied().zip(labels.iter().copied()).collect();
464 pairs.sort_by(|a, b| cmp_f64(b.0, a.0)); // descending score
465
466 let mut curve = vec![RocPoint {
467 threshold: None,
468 fpr: 0.0,
469 tpr: 0.0,
470 }];
471 let (mut tp, mut fp) = (0.0f64, 0.0f64);
472 let mut i = 0;
473 while i < pairs.len() {
474 let thr = pairs[i].0;
475 // Advance over every sample at this threshold before recording a point,
476 // so tied scores collapse to a single ROC vertex.
477 while i < pairs.len() && pairs[i].0 == thr {
478 if pairs[i].1 {
479 tp += 1.0;
480 } else {
481 fp += 1.0;
482 }
483 i += 1;
484 }
485 // The threshold is the score of the group just consumed, because every
486 // sample at it has now been counted as flagged, which is what
487 // `score >= threshold` means.
488 curve.push(RocPoint {
489 threshold: Some(thr),
490 fpr: fp / n_neg,
491 tpr: tp / n_pos,
492 });
493 }
494 curve
495}
496
497/// The best true-positive rate this detector reaches without exceeding
498/// `max_fpr`.
499///
500/// `None` when the curve cannot be drawn (see [`roc_curve`]) and when `max_fpr`
501/// is not a false-alarm budget: a negative, a NaN or a figure above 1 is a
502/// caller's mistake rather than a strict budget, and answering 0.0 to it would
503/// read as a detector that caught nothing.
504///
505/// ONE SWEEP, NOT ONE PASS PER THRESHOLD
506///
507/// The obvious shape of this is a loop over every distinct score that counts
508/// the hits above it, and a copy of this metric elsewhere in the project was
509/// written that way. It is quadratic, and the scores are floats a detector
510/// produced, so "distinct" means nearly all of them: an arm of 344,357 images
511/// made it around a hundred billion comparisons, which is not slow, it is a
512/// run that never ends. [`roc_curve`] walks the scores once in descending
513/// order and accumulates counts, which gives the same answer, ties included,
514/// for the cost of the sort.
515///
516/// This is the headline number for an external evaluation and accuracy is not.
517/// A detector facing a corpus that is mostly clean can score 95% accuracy by
518/// answering "clean" every time, and a false-positive rate chosen after seeing
519/// the results is not a measurement. Pinning the false-positive budget first and
520/// asking what detection it buys is the comparison that survives review.
521pub fn tpr_at_fpr(scores: &[f64], labels: &[bool], max_fpr: f64) -> Option<f64> {
522 if !(0.0..=1.0).contains(&max_fpr) {
523 return None;
524 }
525 let curve = roc_curve(scores, labels);
526 if curve.is_empty() {
527 return None;
528 }
529 // The curve is a step function, so the answer is the highest point whose
530 // false-positive rate is still inside the budget. The epsilon absorbs the
531 // representation error in ratios like 1/3, which would otherwise drop an
532 // operating point that is exactly on the limit.
533 let best = curve
534 .iter()
535 .filter(|&&(fpr, _)| fpr <= max_fpr + 1e-12)
536 .map(|&(_, tpr)| tpr)
537 .fold(f64::NEG_INFINITY, f64::max);
538 Some(if best.is_finite() { best } else { 0.0 })
539}
540
541/// The point on the curve that [`tpr_at_fpr`] actually reported.
542///
543/// WHY THIS EXISTS, AND IT IS NOT A CONVENIENCE
544///
545/// A false-alarm budget finer than one clean image cannot be spent. With six
546/// clean images the only false-alarm rates that exist are 0, 1/6, 2/6 and so
547/// on, so a budget of 0.01 buys exactly what a budget of 0 buys, and
548/// `tpr_at_fpr` correctly returns the zero-budget answer. What it cannot do
549/// is say so, and the caller then prints that number under the heading
550/// "TPR@1%FA", which claims a resolution the measurement never had.
551///
552/// An engineer picking a review threshold reads that column and nothing else.
553/// So this returns the rate that was actually achieved beside the one that
554/// was asked for, and the two differing is the signal that the corpus is too
555/// small for the question.
556#[derive(Debug, Clone, Copy, PartialEq)]
557pub struct OperatingPoint {
558 /// The detection rate at this point.
559 pub tpr: f64,
560 /// The false-alarm rate this point actually sits at, which is a multiple
561 /// of `1 / n_clean` and is never above `requested`.
562 pub achieved_fpr: f64,
563 /// The budget the caller asked for.
564 pub requested_fpr: f64,
565 /// Flag an image when its score is greater than or equal to this.
566 ///
567 /// The number a developer actually needs, and the one this struct used to
568 /// compute and discard. See [`RocPoint::threshold`] for why `None` is a real
569 /// answer rather than a missing one: at the origin nothing is flagged, so
570 /// there is no cutoff worth printing.
571 ///
572 /// It is a raw detector score and carries that detector's units and scale.
573 /// It does not transfer to another tool, or to another version of the same
574 /// tool, which is a property of the thing rather than a limitation here.
575 pub threshold: Option<f64>,
576}
577
578impl OperatingPoint {
579 /// Whether the corpus could express the budget that was asked for.
580 ///
581 /// False means the answer is real but the label is not: the figure is the
582 /// detection rate at `achieved_fpr`, not at `requested_fpr`.
583 pub fn budget_was_expressible(&self, n_clean: usize) -> bool {
584 n_clean > 0 && fpr_resolution(n_clean) <= self.requested_fpr + 1e-12
585 }
586}
587
588/// The smallest false-alarm rate a corpus of this many clean images can show.
589///
590/// One clean image wrongly flagged out of `n_clean`. Anything finer is a
591/// budget the sample cannot spend.
592pub fn fpr_resolution(n_clean: usize) -> f64 {
593 if n_clean == 0 {
594 return f64::INFINITY;
595 }
596 1.0 / n_clean as f64
597}
598
599/// [`tpr_at_fpr`], plus the rate the answer actually came from.
600pub fn operating_point(scores: &[f64], labels: &[bool], max_fpr: f64) -> Option<OperatingPoint> {
601 if !(0.0..=1.0).contains(&max_fpr) {
602 return None;
603 }
604 let curve = roc_points(scores, labels);
605 if curve.is_empty() {
606 return None;
607 }
608 // The same point `tpr_at_fpr` picks: highest detection rate inside the
609 // budget. Ties on tpr take the lowest fpr, because two points with the
610 // same detection rate are the same answer bought more or less cheaply,
611 // and reporting the dearer one would overstate what the budget cost.
612 let mut best: Option<RocPoint> = None;
613 for &point in curve.iter().filter(|p| p.fpr <= max_fpr + 1e-12) {
614 best = Some(match best {
615 None => point,
616 Some(b) if point.tpr > b.tpr || (point.tpr == b.tpr && point.fpr < b.fpr) => point,
617 Some(b) => b,
618 });
619 }
620 let chosen = best.unwrap_or(RocPoint {
621 threshold: None,
622 fpr: 0.0,
623 tpr: 0.0,
624 });
625 Some(OperatingPoint {
626 tpr: chosen.tpr,
627 achieved_fpr: chosen.fpr,
628 requested_fpr: max_fpr,
629 threshold: chosen.threshold,
630 })
631}
632
633/// Something about a set of scores that makes the number beside it misleading.
634///
635/// A finding carries the facts and no wording. This crate has no dependencies
636/// and no opinion about presentation: two callers print in two registers, and a
637/// sentence baked in here would be the one that is wrong in both. The caller
638/// writes the sentence; the enum is what stops the two callers disagreeing
639/// about when to write one.
640#[derive(Debug, Clone, Copy, PartialEq)]
641pub enum Finding {
642 /// Every score was the same number, so the AUC is 0.5 by construction.
643 ///
644 /// A detector that answers one thing to everything and a coin flip produce
645 /// the identical figure, and they mean completely different things: one
646 /// could not tell the images apart, the other ranked them no better than
647 /// chance.
648 EveryScoreIdentical {
649 /// How many answers were the same.
650 count: usize,
651 },
652 /// The AUC is below 0.5, which is a sign-flipped adapter far more often
653 /// than it is a detector that sees nothing.
654 ///
655 /// A ranking that is wrong this consistently carries as much signal as one
656 /// that is right; it is being read upside down. The commonest way to wire
657 /// a new detector into this benchmark incorrectly.
658 ScoresRunBackwards {
659 /// The AUC as measured.
660 auc: f64,
661 /// What the same ranking would score read the other way round, which
662 /// is the figure that says how much signal is actually there.
663 inverted: f64,
664 },
665}
666
667/// Every [`Finding`] that applies to these scores and labels.
668///
669/// Empty for a healthy detector, and empty for anything [`roc_auc`] refuses:
670/// a set that cannot be ranked has no AUC to describe, and the caller is
671/// already refusing it for a named reason it would only muddy.
672pub fn findings(scores: &[f64], labels: &[bool]) -> Vec<Finding> {
673 let Some(auc) = roc_auc(scores, labels) else {
674 return Vec::new();
675 };
676 let mut out = Vec::new();
677 if scores.len() > 1 && scores.windows(2).all(|w| w[0] == w[1]) {
678 out.push(Finding::EveryScoreIdentical {
679 count: scores.len(),
680 });
681 }
682 if auc < 0.5 {
683 out.push(Finding::ScoresRunBackwards {
684 auc,
685 inverted: 1.0 - auc,
686 });
687 }
688 out
689}
690
691#[cfg(test)]
692mod finding_tests {
693 use super::*;
694
695 #[test]
696 fn a_detector_that_separates_the_images_is_not_flagged() {
697 // The half that stops the check being made to pass by always firing.
698 assert!(findings(&[0.1, 0.2, 0.8, 0.9], &[false, false, true, true]).is_empty());
699 // And a genuine chance-level ranking, which is 0.5 by measurement
700 // rather than by construction, is a result and not a finding.
701 assert_eq!(
702 roc_auc(&[0.1, 0.2, 0.8, 0.9], &[true, false, false, true]),
703 Some(0.5)
704 );
705 assert!(findings(&[0.1, 0.2, 0.8, 0.9], &[true, false, false, true]).is_empty());
706 }
707
708 #[test]
709 fn one_answer_to_everything_is_flagged_with_its_count() {
710 let got = findings(
711 &[7.0; 8],
712 &[true, false, true, false, true, false, true, false],
713 );
714 assert_eq!(got, vec![Finding::EveryScoreIdentical { count: 8 }]);
715 }
716
717 #[test]
718 fn an_inverted_ranking_is_flagged_with_both_figures() {
719 let got = findings(&[0.9, 0.8, 0.2, 0.1], &[false, false, true, true]);
720 assert_eq!(
721 got,
722 vec![Finding::ScoresRunBackwards {
723 auc: 0.0,
724 inverted: 1.0
725 }]
726 );
727 // And a ranking only slightly the wrong way round is flagged too: the
728 // sign error is the same one whether or not it is total.
729 let got = findings(&[0.3, 0.4, 0.35, 0.2], &[false, false, true, true]);
730 assert_eq!(
731 got,
732 vec![Finding::ScoresRunBackwards {
733 auc: 0.25,
734 inverted: 0.75
735 }]
736 );
737 }
738
739 #[test]
740 fn a_single_score_is_not_an_identical_set() {
741 // One answer cannot be inconsistent with itself, and calling it
742 // degenerate would fire on every one-image run.
743 assert!(findings(&[0.5, 0.5], &[true, false])
744 .contains(&Finding::EveryScoreIdentical { count: 2 }));
745 assert!(findings(&[0.5], &[true]).is_empty());
746 }
747
748 #[test]
749 fn a_set_that_cannot_be_ranked_produces_nothing() {
750 // Each of these is refused by name somewhere above; a finding here
751 // would describe an AUC that was never computed.
752 assert!(findings(&[0.5, 0.5], &[true, true]).is_empty());
753 assert!(findings(&[0.5, 0.5, 0.5], &[true, false]).is_empty());
754 assert!(findings(&[f64::NAN, f64::NAN], &[true, false]).is_empty());
755 assert!(findings(&[], &[]).is_empty());
756 }
757}
758
759#[cfg(test)]
760mod tests {
761 use super::*;
762
763 #[test]
764 fn a_budget_finer_than_one_clean_image_is_reported_as_not_expressible() {
765 // The finding this exists for. Six clean images, a 1% budget, and the
766 // honest answer is the zero-budget answer: 1/6 is 16.7%, so 1% buys
767 // nothing that 0% did not already buy. The figure is correct and the
768 // label "TPR@1%FA" is not.
769 let scores = [
770 0.9, 0.8, 0.7, 0.6, 0.1, 0.2, 0.3, 0.4, 0.5, 0.05, 0.06, 0.07,
771 ];
772 let labels = [
773 true, true, true, true, true, true, false, false, false, false, false, false,
774 ];
775 let p = operating_point(&scores, &labels, 0.01).expect("a point");
776 assert_eq!(p.requested_fpr, 0.01);
777 assert_eq!(
778 p.achieved_fpr, 0.0,
779 "1% cannot be spent on six clean images"
780 );
781 assert!(!p.budget_was_expressible(6));
782 // And the same corpus CAN express a budget of one image in six.
783 let coarse = operating_point(&scores, &labels, 1.0 / 6.0).expect("a point");
784 assert!(coarse.budget_was_expressible(6));
785 }
786
787 #[test]
788 fn a_budget_the_corpus_can_spend_is_reported_as_expressible() {
789 // Two hundred clean images make 1% a real budget: two of them.
790 assert!(fpr_resolution(200) <= 0.01);
791 let mut scores = Vec::new();
792 let mut labels = Vec::new();
793 for i in 0..200 {
794 scores.push(i as f64 / 200.0);
795 labels.push(false);
796 }
797 for i in 0..200 {
798 scores.push(1.0 + i as f64);
799 labels.push(true);
800 }
801 let p = operating_point(&scores, &labels, 0.01).expect("a point");
802 assert!(p.budget_was_expressible(200));
803 assert_eq!(p.tpr, 1.0);
804 }
805
806 #[test]
807 fn the_operating_point_agrees_with_the_figure_tpr_at_fpr_reports() {
808 // Two ways of asking one question must not drift apart.
809 let scores = [0.9, 0.4, 0.5, 0.1];
810 let labels = [true, true, false, false];
811 for budget in [0.0, 0.25, 0.5, 1.0] {
812 let a = tpr_at_fpr(&scores, &labels, budget);
813 let b = operating_point(&scores, &labels, budget).map(|p| p.tpr);
814 assert_eq!(a, b, "at a budget of {budget}");
815 }
816 }
817
818 #[test]
819 fn the_achieved_rate_never_exceeds_the_budget_that_was_asked_for() {
820 let scores = [0.9, 0.4, 0.5, 0.1];
821 let labels = [true, true, false, false];
822 for budget in [0.0, 0.1, 0.25, 0.4, 0.5, 0.75, 1.0] {
823 let p = operating_point(&scores, &labels, budget).expect("a point");
824 assert!(
825 p.achieved_fpr <= budget + 1e-12,
826 "spent {} of a {budget} budget",
827 p.achieved_fpr
828 );
829 }
830 }
831
832 // ---- the threshold, which is the number a developer came for ---------
833
834 #[test]
835 fn the_curve_and_the_points_it_is_derived_from_agree_exactly() {
836 // `roc_curve` is now a projection of `roc_points`, and this is what
837 // stops a future edit making them two implementations again.
838 let scores = [0.9, 0.8, 0.8, 0.4, 0.1, 0.1, 0.7];
839 let labels = [true, true, false, true, false, false, false];
840 let from_points: Vec<(f64, f64)> = roc_points(&scores, &labels)
841 .into_iter()
842 .map(|p| (p.fpr, p.tpr))
843 .collect();
844 assert_eq!(roc_curve(&scores, &labels), from_points);
845 }
846
847 #[test]
848 fn the_origin_carries_no_threshold_because_nothing_is_flagged_there() {
849 let points = roc_points(&[0.9, 0.1], &[true, false]);
850 assert_eq!(points[0].threshold, None);
851 assert_eq!((points[0].fpr, points[0].tpr), (0.0, 0.0));
852 // Every other vertex has one.
853 assert!(points[1..].iter().all(|p| p.threshold.is_some()));
854 }
855
856 #[test]
857 fn the_threshold_at_a_vertex_flags_exactly_the_images_that_vertex_counts() {
858 // The contract is `score >= threshold`. Checked by applying it rather
859 // than by trusting the sweep: this is the number somebody puts in an
860 // `if`, so it has to mean what it says.
861 let scores = [0.9, 0.75, 0.75, 0.6, 0.2, 0.05];
862 let labels = [true, true, false, true, false, false];
863 let n_pos = labels.iter().filter(|l| **l).count() as f64;
864 let n_neg = labels.len() as f64 - n_pos;
865 for point in roc_points(&scores, &labels) {
866 let Some(threshold) = point.threshold else {
867 continue;
868 };
869 let flagged: Vec<bool> = scores.iter().map(|s| *s >= threshold).collect();
870 let tp = flagged
871 .iter()
872 .zip(labels)
873 .filter(|(f, l)| **f && *l)
874 .count() as f64;
875 let fp = flagged
876 .iter()
877 .zip(labels)
878 .filter(|(f, l)| **f && !*l)
879 .count() as f64;
880 assert_eq!(
881 (fp / n_neg, tp / n_pos),
882 (point.fpr, point.tpr),
883 "threshold {threshold} does not produce the rates its vertex claims"
884 );
885 }
886 }
887
888 #[test]
889 fn an_operating_point_reports_the_threshold_that_bought_its_rates() {
890 let scores = [0.9, 0.8, 0.7, 0.6, 0.5, 0.4];
891 let labels = [true, true, true, false, false, false];
892 let point = operating_point(&scores, &labels, 0.0).expect("a point");
893 // Perfect separation, so a zero budget buys everything, and the cutoff
894 // is the lowest stego score rather than the highest clean one.
895 assert_eq!(point.tpr, 1.0);
896 assert_eq!(point.achieved_fpr, 0.0);
897 assert_eq!(point.threshold, Some(0.7));
898 }
899
900 #[test]
901 fn the_threshold_moves_down_as_the_budget_is_loosened() {
902 let scores = [0.9, 0.5, 0.8, 0.4, 0.7, 0.3];
903 let labels = [true, true, true, false, false, false];
904 let tight = operating_point(&scores, &labels, 0.0).expect("tight");
905 let loose = operating_point(&scores, &labels, 1.0).expect("loose");
906 let (Some(t), Some(l)) = (tight.threshold, loose.threshold) else {
907 panic!("both budgets should name a cutoff here");
908 };
909 assert!(
910 l <= t,
911 "loosening the budget raised the cutoff: {t} then {l}"
912 );
913 assert!(loose.tpr >= tight.tpr);
914 }
915
916 #[test]
917 fn the_threshold_agrees_with_the_rate_tpr_at_fpr_reports() {
918 // The two must not disagree: one is the figure published and the other
919 // is the cutoff published beside it.
920 let scores = [0.31, 0.62, 0.11, 0.88, 0.47, 0.05, 0.73, 0.22];
921 let labels = [false, true, false, true, true, false, true, false];
922 for budget in [0.0, 0.25, 0.5, 1.0] {
923 let point = operating_point(&scores, &labels, budget).expect("a point");
924 assert_eq!(
925 Some(point.tpr),
926 tpr_at_fpr(&scores, &labels, budget),
927 "operating_point and tpr_at_fpr disagree at budget {budget}"
928 );
929 }
930 }
931
932 #[test]
933 fn a_detector_answering_one_number_to_everything_has_one_usable_threshold() {
934 // Every score identical: the curve is the origin and one vertex, and the
935 // cutoff flags everything. Worth pinning because this is the shape a
936 // broken adapter produces and it must not panic or invent a number.
937 let points = roc_points(&[0.5; 4], &[true, true, false, false]);
938 assert_eq!(points.len(), 2);
939 assert_eq!(points[1].threshold, Some(0.5));
940 assert_eq!((points[1].fpr, points[1].tpr), (1.0, 1.0));
941 }
942
943 #[test]
944 fn an_unrankable_set_yields_no_points_and_therefore_no_threshold() {
945 assert!(roc_points(&[f64::NAN, 0.1], &[true, false]).is_empty());
946 assert!(roc_points(&[0.1], &[true]).is_empty());
947 assert!(operating_point(&[f64::NAN, 0.1], &[true, false], 0.5).is_none());
948 }
949
950 #[test]
951 fn the_resolution_of_a_corpus_with_no_clean_images_is_not_a_number_to_divide_by() {
952 assert!(fpr_resolution(0).is_infinite());
953 // Which means no budget is ever expressible, rather than all of them.
954 let p = OperatingPoint {
955 tpr: 1.0,
956 achieved_fpr: 0.0,
957 requested_fpr: 0.5,
958 threshold: Some(0.9),
959 };
960 assert!(!p.budget_was_expressible(0));
961 }
962
963 #[test]
964 fn tpr_at_fpr_perfect_separation_is_one_at_zero_budget() {
965 let scores = [0.1, 0.2, 0.8, 0.9];
966 let labels = [false, false, true, true];
967 assert_eq!(tpr_at_fpr(&scores, &labels, 0.0), Some(1.0));
968 }
969
970 #[test]
971 fn tpr_at_fpr_tightens_as_the_budget_shrinks() {
972 // 2 pos (0.9, 0.4), 2 neg (0.5, 0.1). Catching the 0.4 positive means
973 // first admitting the 0.5 negative, so a zero-FPR budget buys half.
974 let scores = [0.9, 0.4, 0.5, 0.1];
975 let labels = [true, true, false, false];
976 assert_eq!(tpr_at_fpr(&scores, &labels, 0.0), Some(0.5));
977 assert_eq!(tpr_at_fpr(&scores, &labels, 0.5), Some(1.0));
978 }
979
980 #[test]
981 fn tpr_at_fpr_is_zero_when_the_budget_buys_nothing() {
982 // Every negative outranks every positive: no detection at zero FPR.
983 let scores = [0.1, 0.2, 0.8, 0.9];
984 let labels = [true, true, false, false];
985 assert_eq!(tpr_at_fpr(&scores, &labels, 0.0), Some(0.0));
986 }
987
988 #[test]
989 fn tpr_at_fpr_undefined_with_single_class() {
990 assert_eq!(tpr_at_fpr(&[0.1, 0.2], &[true, true], 0.01), None);
991 }
992
993 /// A detector that only answers yes or no has two operating points, so the
994 /// budget either admits its false positives or refuses them outright. This
995 /// is exactly why `Detector::is_graded` exists: the number is real, but it
996 /// cannot be traded off the way a graded detector's can.
997 #[test]
998 fn tpr_at_fpr_handles_a_binary_detector() {
999 let scores = [1.0, 1.0, 1.0, 0.0];
1000 let labels = [true, true, false, false];
1001 assert_eq!(tpr_at_fpr(&scores, &labels, 0.0), Some(0.0));
1002 assert_eq!(tpr_at_fpr(&scores, &labels, 0.5), Some(1.0));
1003 }
1004
1005 #[test]
1006 fn confusion_counts_and_rates() {
1007 // labels: S S S C C (3 stego, 2 clean)
1008 // preds: 1 1 0 1 0
1009 let labels = [true, true, true, false, false];
1010 let preds = [true, true, false, true, false];
1011 let c = Confusion::tally(&labels, &preds);
1012 assert_eq!((c.tp, c.fn_, c.fp, c.tn), (2, 1, 1, 1));
1013 assert!((c.tpr() - 2.0 / 3.0).abs() < 1e-12);
1014 assert!((c.fpr() - 0.5).abs() < 1e-12);
1015 assert!((c.precision() - 2.0 / 3.0).abs() < 1e-12);
1016 assert!((c.accuracy() - 3.0 / 5.0).abs() < 1e-12);
1017 assert!((c.f1() - 2.0 / 3.0).abs() < 1e-12);
1018 }
1019
1020 #[test]
1021 fn confusion_zero_denominators_are_zero_not_nan() {
1022 let c = Confusion::default();
1023 assert_eq!(c.tpr(), 0.0);
1024 assert_eq!(c.fpr(), 0.0);
1025 assert_eq!(c.precision(), 0.0);
1026 assert_eq!(c.accuracy(), 0.0);
1027 assert_eq!(c.f1(), 0.0);
1028 }
1029
1030 /// The honest counterpart agrees with the truncating one whenever the
1031 /// truncating one had nothing to truncate.
1032 #[test]
1033 fn try_tally_matches_tally_on_well_formed_input() {
1034 let labels = [true, true, true, false, false];
1035 let preds = [true, true, false, true, false];
1036 assert_eq!(
1037 Confusion::try_tally(&labels, &preds),
1038 Ok(Confusion::tally(&labels, &preds))
1039 );
1040 }
1041
1042 /// Labels and predictions of different lengths came from different record
1043 /// sets. `tally` walks the shorter one; this refuses, which is the whole
1044 /// reason for having both.
1045 #[test]
1046 fn try_tally_refuses_a_length_mismatch_in_either_direction() {
1047 let labels = [true, true, false, false];
1048 assert_eq!(
1049 Confusion::try_tally(&labels, &[true, false, true]),
1050 Err(TallyError::LengthMismatch {
1051 labels: 4,
1052 predicted_positive: 3
1053 })
1054 );
1055 assert_eq!(
1056 Confusion::try_tally(&labels[..3], &[true, false, true, false]),
1057 Err(TallyError::LengthMismatch {
1058 labels: 3,
1059 predicted_positive: 4
1060 })
1061 );
1062 }
1063
1064 /// Zero samples is not a score of zero. Every rate on `Confusion` answers
1065 /// 0.0 on a zero denominator, so an empty count is indistinguishable from a
1066 /// detector that missed everything.
1067 #[test]
1068 fn try_tally_refuses_an_empty_run() {
1069 assert_eq!(Confusion::try_tally(&[], &[]), Err(TallyError::Empty));
1070 }
1071
1072 /// One class is a real measurement here, unlike a ranking. `must_clear` in
1073 /// a detector's self test hands over clean images only, and the false
1074 /// positives that come back are the answer it wanted.
1075 #[test]
1076 fn try_tally_accepts_a_single_class() {
1077 let clean_only = Confusion::try_tally(&[false, false, false], &[true, false, false]);
1078 assert_eq!(
1079 clean_only,
1080 Ok(Confusion {
1081 tp: 0,
1082 fp: 1,
1083 tn: 2,
1084 fn_: 0
1085 })
1086 );
1087 let stego_only = Confusion::try_tally(&[true, true], &[true, false]);
1088 assert_eq!(
1089 stego_only,
1090 Ok(Confusion {
1091 tp: 1,
1092 fp: 0,
1093 tn: 0,
1094 fn_: 1
1095 })
1096 );
1097 }
1098
1099 /// The point of adding a second function rather than changing the first:
1100 /// `tally` still truncates, and a caller depending on that still gets it.
1101 #[test]
1102 fn tally_still_truncates_exactly_as_before() {
1103 let labels = [true, true, false, false];
1104 let preds = [true, false, true];
1105 let c = Confusion::tally(&labels, &preds);
1106 assert_eq!((c.tp, c.fn_, c.fp, c.tn), (1, 1, 1, 0));
1107 assert_eq!(Confusion::tally(&[], &[]), Confusion::default());
1108 assert_eq!(
1109 Confusion::tally(&labels, &[]),
1110 Confusion::default(),
1111 "an empty prediction slice truncates to nothing rather than panicking"
1112 );
1113 }
1114
1115 /// The error has to survive being turned into a message a user reads, so
1116 /// each variant says which condition fired and names the figures.
1117 #[test]
1118 fn a_tally_error_says_which_condition_fired() {
1119 let mismatch = TallyError::LengthMismatch {
1120 labels: 9_882,
1121 predicted_positive: 10_000,
1122 };
1123 let text = mismatch.to_string();
1124 assert!(text.contains("9882"), "{text}");
1125 assert!(text.contains("10000"), "{text}");
1126 assert!(text.contains("different record sets"), "{text}");
1127
1128 let empty = TallyError::Empty.to_string();
1129 assert!(empty.contains("nothing to count"), "{empty}");
1130
1131 // Usable as a `std::error::Error`, so a caller can box it like any
1132 // other failure rather than matching on it by hand.
1133 let boxed: Box<dyn std::error::Error> = Box::new(mismatch);
1134 assert_eq!(boxed.to_string(), text);
1135 }
1136
1137 #[test]
1138 fn auc_perfect_separation_is_one() {
1139 let scores = [0.1, 0.2, 0.8, 0.9];
1140 let labels = [false, false, true, true];
1141 assert_eq!(roc_auc(&scores, &labels), Some(1.0));
1142 }
1143
1144 #[test]
1145 fn auc_reversed_is_zero_and_random_is_half() {
1146 let scores = [0.9, 0.8, 0.2, 0.1];
1147 let labels = [false, false, true, true];
1148 assert_eq!(roc_auc(&scores, &labels), Some(0.0));
1149
1150 // Perfectly interleaved with ties handled by average rank → 0.5.
1151 let s = [0.5, 0.5, 0.5, 0.5];
1152 let l = [true, false, true, false];
1153 assert_eq!(roc_auc(&s, &l), Some(0.5));
1154 }
1155
1156 #[test]
1157 fn auc_undefined_with_single_class() {
1158 assert_eq!(roc_auc(&[0.1, 0.2], &[true, true]), None);
1159 assert_eq!(roc_auc(&[0.1, 0.2], &[false, false]), None);
1160 }
1161
1162 #[test]
1163 fn auc_matches_manual_small_case() {
1164 // 2 pos, 2 neg; one pos below one neg → AUC = 3/4.
1165 let scores = [0.3, 0.4, 0.35, 0.2];
1166 let labels = [true, true, false, false];
1167 let auc = roc_auc(&scores, &labels).unwrap();
1168 assert!((auc - 0.75).abs() < 1e-12, "auc={auc}");
1169 }
1170
1171 #[test]
1172 fn roc_curve_reaches_corners() {
1173 let scores = [0.1, 0.2, 0.8, 0.9];
1174 let labels = [false, false, true, true];
1175 let curve = roc_curve(&scores, &labels);
1176 assert_eq!(curve.first(), Some(&(0.0, 0.0)));
1177 assert_eq!(curve.last(), Some(&(1.0, 1.0)));
1178 // Perfect separation: tpr hits 1.0 before any fpr accrues.
1179 assert!(curve.iter().any(|&(fpr, tpr)| fpr == 0.0 && tpr == 1.0));
1180 }
1181
1182 #[test]
1183 fn roc_curve_empty_without_both_classes() {
1184 assert!(roc_curve(&[0.1, 0.2], &[true, true]).is_empty());
1185 }
1186
1187 /// Scores and labels of different lengths mean they were built from
1188 /// different record sets. Walking the shorter of the two would answer with a
1189 /// figure measured on a subset nobody named, and the caller would have no
1190 /// way of telling that from a real one.
1191 #[test]
1192 fn a_length_mismatch_is_refused_rather_than_truncated() {
1193 let scores = [0.9, 0.8, 0.2, 0.1];
1194 assert_eq!(roc_auc(&scores, &[true, false, true]), None);
1195 assert_eq!(roc_auc(&scores[..3], &[true, false, true, false]), None);
1196 assert!(roc_curve(&scores, &[true, false, true]).is_empty());
1197 assert_eq!(tpr_at_fpr(&scores, &[true, false, true], 0.01), None);
1198 }
1199
1200 /// A detector that answers NaN for an image has not scored it. NaN compares
1201 /// false against everything, so it lands wherever the sort happens to leave
1202 /// it and the rank sum built on top of that is arbitrary.
1203 #[test]
1204 fn a_nan_score_is_refused_rather_than_ranked() {
1205 let scores = [0.9, f64::NAN, 0.2, 0.1];
1206 let labels = [true, true, false, false];
1207 assert_eq!(roc_auc(&scores, &labels), None);
1208 assert!(roc_curve(&scores, &labels).is_empty());
1209 assert_eq!(tpr_at_fpr(&scores, &labels, 0.01), None);
1210 }
1211
1212 /// A budget that is not a rate is a caller's mistake, and 0.0 would read as
1213 /// a detector that caught nothing at a budget it was never asked about.
1214 #[test]
1215 fn a_budget_that_is_not_a_rate_is_refused() {
1216 let scores = [0.9, 0.4, 0.5, 0.1];
1217 let labels = [true, true, false, false];
1218 assert_eq!(tpr_at_fpr(&scores, &labels, -0.1), None);
1219 assert_eq!(tpr_at_fpr(&scores, &labels, 1.5), None);
1220 assert_eq!(tpr_at_fpr(&scores, &labels, f64::NAN), None);
1221 // The ends of the range are budgets, not mistakes.
1222 assert_eq!(tpr_at_fpr(&scores, &labels, 0.0), Some(0.5));
1223 assert_eq!(tpr_at_fpr(&scores, &labels, 1.0), Some(1.0));
1224 }
1225}
1226
1227#[cfg(test)]
1228mod delong_tests {
1229 use super::*;
1230
1231 /// Hanley and McNeil's 1982 worked example, which is the one every
1232 /// implementation is checked against.
1233 ///
1234 /// Ratings 1 to 5 for 58 normal and 51 abnormal cases. The published AUC
1235 /// is 0.893 and the published standard error 0.029, computed by their own
1236 /// parametric method; DeLong on the same data gives an SE close to but
1237 /// not identical with it, which is the point of preferring DeLong. The
1238 /// assertion is therefore that the AUC matches to three places and the SE
1239 /// lands in the range every published DeLong implementation agrees on.
1240 fn hanley_mcneil() -> (Vec<f64>, Vec<bool>) {
1241 // (rating, n_normal, n_abnormal)
1242 let table = [
1243 (1.0, 33, 3),
1244 (2.0, 6, 2),
1245 (3.0, 6, 2),
1246 (4.0, 11, 11),
1247 (5.0, 2, 33),
1248 ];
1249 let mut scores = Vec::new();
1250 let mut labels = Vec::new();
1251 for (rating, n_norm, n_abn) in table {
1252 for _ in 0..n_norm {
1253 scores.push(rating);
1254 labels.push(false);
1255 }
1256 for _ in 0..n_abn {
1257 scores.push(rating);
1258 labels.push(true);
1259 }
1260 }
1261 (scores, labels)
1262 }
1263
1264 #[test]
1265 fn the_published_worked_example_comes_out_where_it_is_published() {
1266 let (scores, labels) = hanley_mcneil();
1267 let got = roc_auc_interval(&scores, &labels, Z_95).expect("both classes, enough of each");
1268 assert!(
1269 (got.auc - 0.893).abs() < 0.001,
1270 "AUC {} is not the published 0.893",
1271 got.auc
1272 );
1273 assert!(
1274 (0.025..0.035).contains(&got.standard_error),
1275 "standard error {} is outside what every DeLong implementation gives here",
1276 got.standard_error
1277 );
1278 assert!(got.low < got.auc && got.auc < got.high);
1279 }
1280
1281 /// This test used to assert the opposite, and the opposite was wrong.
1282 ///
1283 /// Six observations that happen to separate cleanly say nothing about
1284 /// how the detector behaves on the seventh, so "95% confident the AUC is
1285 /// between 1.0 and 1.0" is a claim no sample of this size supports. It
1286 /// was also the single most quotable line the tool produced.
1287 #[test]
1288 fn a_perfect_separation_reports_no_interval_rather_than_a_width_of_zero() {
1289 let scores = vec![1.0, 2.0, 3.0, 10.0, 11.0, 12.0];
1290 let labels = vec![false, false, false, true, true, true];
1291 assert_eq!(roc_auc(&scores, &labels), Some(1.0));
1292 assert_eq!(roc_auc_interval(&scores, &labels, Z_95), None);
1293 }
1294
1295 /// The same collapse from the other direction.
1296 ///
1297 /// Every comparison is a tie, so every structural component is identical
1298 /// and the variance is zero. The estimator is not certain here; it has
1299 /// nothing to say. What the detector did is reported by the identical
1300 /// answers finding, not by a zero width interval.
1301 #[test]
1302 fn one_answer_to_everything_reports_no_interval_either() {
1303 let scores = vec![7.0; 8];
1304 let labels = vec![true, false, true, false, true, false, true, false];
1305 assert_eq!(roc_auc(&scores, &labels), Some(0.5));
1306 assert_eq!(roc_auc_interval(&scores, &labels, Z_95), None);
1307 }
1308
1309 /// A real sample still gets a real interval, so the guard above cannot
1310 /// be quietly swallowing every case.
1311 #[test]
1312 fn an_ordinary_sample_still_has_a_width() {
1313 let scores = vec![1.0, 4.0, 2.0, 9.0, 3.0, 8.0, 5.0, 7.0];
1314 let labels = vec![false, true, false, true, false, true, false, true];
1315 let got = roc_auc_interval(&scores, &labels, Z_95).expect("an interval");
1316 assert!(got.standard_error > 0.0);
1317 assert!(got.low < got.high, "[{}, {}]", got.low, got.high);
1318 }
1319
1320 #[test]
1321 fn the_interval_is_clamped_to_the_scale_it_is_measured_on() {
1322 // Near 1.0 the normal interval runs off the end, and an upper bound
1323 // of 1.04 beside a measurement is nonsense a reader would quote.
1324 let mut scores: Vec<f64> = (0..20).map(|i| i as f64).collect();
1325 let mut labels = vec![false; 20];
1326 scores.extend((0..20).map(|i| 100.0 + i as f64));
1327 labels.extend(vec![true; 20]);
1328 scores[0] = 200.0; // one negative above everything, so AUC is high but not 1
1329 let got = roc_auc_interval(&scores, &labels, Z_95).expect("valid");
1330 assert!(got.high <= 1.0, "upper bound {} left the scale", got.high);
1331 assert!(got.low >= 0.0, "lower bound {} left the scale", got.low);
1332 }
1333
1334 #[test]
1335 fn one_of_a_class_is_refused_because_a_variance_needs_two() {
1336 let scores = vec![1.0, 2.0, 3.0];
1337 let labels = vec![true, false, false];
1338 assert_eq!(roc_auc_interval(&scores, &labels, Z_95), None);
1339 // And the point estimate is still available, because that one IS
1340 // defined with a single positive.
1341 assert!(roc_auc(&scores, &labels).is_some());
1342 }
1343
1344 #[test]
1345 fn everything_roc_auc_refuses_this_refuses_too() {
1346 assert_eq!(roc_auc_interval(&[1.0, 2.0], &[true], Z_95), None);
1347 assert_eq!(
1348 roc_auc_interval(
1349 &[1.0, f64::NAN, 3.0, 4.0],
1350 &[true, true, false, false],
1351 Z_95
1352 ),
1353 None
1354 );
1355 assert_eq!(roc_auc_interval(&[1.0, 2.0], &[true, true], Z_95), None);
1356 }
1357
1358 #[test]
1359 fn a_nonsense_width_is_refused_rather_than_producing_a_nonsense_interval() {
1360 let scores = vec![1.0, 2.0, 3.0, 4.0];
1361 let labels = vec![false, false, true, true];
1362 assert_eq!(roc_auc_interval(&scores, &labels, f64::NAN), None);
1363 assert_eq!(roc_auc_interval(&scores, &labels, -1.0), None);
1364 }
1365
1366 #[test]
1367 fn a_wider_z_gives_a_wider_interval_around_the_same_point() {
1368 let (scores, labels) = hanley_mcneil();
1369 let narrow = roc_auc_interval(&scores, &labels, 1.0).expect("valid");
1370 let wide = roc_auc_interval(&scores, &labels, Z_95).expect("valid");
1371 assert_eq!(narrow.auc, wide.auc);
1372 assert_eq!(narrow.standard_error, wide.standard_error);
1373 assert!(wide.high - wide.low > narrow.high - narrow.low);
1374 }
1375
1376 #[test]
1377 fn the_point_estimate_is_the_same_number_roc_auc_gives() {
1378 // Two implementations of one metric is the thing this crate exists to
1379 // avoid, so the interval must not quietly compute its own AUC.
1380 let (scores, labels) = hanley_mcneil();
1381 let plain = roc_auc(&scores, &labels).expect("valid");
1382 let with_ci = roc_auc_interval(&scores, &labels, Z_95).expect("valid");
1383 assert_eq!(plain, with_ci.auc);
1384 }
1385}