use super::{MetricError, validate_binary, validate_finite, validate_lengths};
pub(super) fn ascending_score_order(scores: &[f32]) -> Vec<usize> {
let mut order = (0..scores.len()).collect::<Vec<_>>();
order.sort_unstable_by(|&left, &right| {
scores[left]
.total_cmp(&scores[right])
.then_with(|| left.cmp(&right))
});
order
}
pub(super) fn tie_groups(scores: &[f32], order: &[usize]) -> Vec<(usize, usize)> {
let mut groups = Vec::new();
let mut start = 0;
while start < order.len() {
let mut end = start + 1;
while end < order.len() && scores[order[end]] == scores[order[start]] {
end += 1;
}
groups.push((start, end));
start = end;
}
groups
}
fn class_counts(expected: &[u8], scores: &[f32]) -> Result<(f64, f64), MetricError> {
validate_lengths(expected.len(), scores.len())?;
validate_binary(expected, 0)?;
validate_finite(scores, 1)?;
let positives = expected.iter().filter(|&&value| value == 1).count();
let negatives = expected.len() - positives;
if positives == 0 || negatives == 0 {
return Err(MetricError::Undefined);
}
Ok((positives as f64, negatives as f64))
}
#[derive(Clone, Debug, PartialEq)]
pub struct RocCurve {
thresholds: Vec<f32>,
false_positive_rates: Vec<f64>,
true_positive_rates: Vec<f64>,
}
impl RocCurve {
pub fn thresholds(&self) -> &[f32] {
&self.thresholds
}
pub fn false_positive_rates(&self) -> &[f64] {
&self.false_positive_rates
}
pub fn true_positive_rates(&self) -> &[f64] {
&self.true_positive_rates
}
pub fn len(&self) -> usize {
self.thresholds.len()
}
pub fn is_empty(&self) -> bool {
self.thresholds.is_empty()
}
}
#[derive(Clone, Debug, PartialEq)]
pub struct PrecisionRecallCurve {
thresholds: Vec<f32>,
precisions: Vec<f64>,
recalls: Vec<f64>,
}
impl PrecisionRecallCurve {
pub fn thresholds(&self) -> &[f32] {
&self.thresholds
}
pub fn precisions(&self) -> &[f64] {
&self.precisions
}
pub fn recalls(&self) -> &[f64] {
&self.recalls
}
pub fn len(&self) -> usize {
self.thresholds.len()
}
pub fn is_empty(&self) -> bool {
self.thresholds.is_empty()
}
}
pub fn roc_curve(expected: &[u8], scores: &[f32]) -> Result<RocCurve, MetricError> {
let (positives, negatives) = class_counts(expected, scores)?;
let order = ascending_score_order(scores);
let groups = tie_groups(scores, &order);
let mut thresholds = Vec::with_capacity(groups.len() + 1);
let mut false_positive_rates = Vec::with_capacity(groups.len() + 1);
let mut true_positive_rates = Vec::with_capacity(groups.len() + 1);
thresholds.push(f32::INFINITY);
false_positive_rates.push(0.0);
true_positive_rates.push(0.0);
let mut true_positives = 0.0_f64;
let mut false_positives = 0.0_f64;
for &(start, end) in groups.iter().rev() {
for &index in &order[start..end] {
if expected[index] == 1 {
true_positives += 1.0;
} else {
false_positives += 1.0;
}
}
thresholds.push(scores[order[start]]);
false_positive_rates.push(false_positives / negatives);
true_positive_rates.push(true_positives / positives);
}
Ok(RocCurve {
thresholds,
false_positive_rates,
true_positive_rates,
})
}
pub fn precision_recall_curve(
expected: &[u8],
scores: &[f32],
) -> Result<PrecisionRecallCurve, MetricError> {
validate_lengths(expected.len(), scores.len())?;
validate_binary(expected, 0)?;
validate_finite(scores, 1)?;
let positives = expected.iter().filter(|&&value| value == 1).count() as f64;
if positives == 0.0 {
return Err(MetricError::Undefined);
}
let order = ascending_score_order(scores);
let groups = tie_groups(scores, &order);
let mut thresholds = Vec::with_capacity(groups.len());
let mut precisions = Vec::with_capacity(groups.len());
let mut recalls = Vec::with_capacity(groups.len());
let mut true_positives = 0.0_f64;
let mut predicted = 0.0_f64;
for &(start, end) in groups.iter().rev() {
for &index in &order[start..end] {
if expected[index] == 1 {
true_positives += 1.0;
}
predicted += 1.0;
}
thresholds.push(scores[order[start]]);
precisions.push(true_positives / predicted);
recalls.push(true_positives / positives);
}
Ok(PrecisionRecallCurve {
thresholds,
precisions,
recalls,
})
}
pub fn average_precision_score(expected: &[u8], scores: &[f32]) -> Result<f64, MetricError> {
let curve = precision_recall_curve(expected, scores)?;
let mut area = 0.0_f64;
let mut previous_recall = 0.0_f64;
for (&precision, &recall) in curve.precisions.iter().zip(&curve.recalls) {
area += (recall - previous_recall) * precision;
previous_recall = recall;
}
Ok(area)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::metrics::roc_auc_score;
fn assert_near(actual: &[f64], expected: &[f64]) {
assert_eq!(actual.len(), expected.len(), "{actual:?} vs {expected:?}");
for (index, (&actual, &expected)) in actual.iter().zip(expected).enumerate() {
assert!(
(actual - expected).abs() < 1.0e-12,
"point {index}: expected {expected}, got {actual}"
);
}
}
#[test]
fn the_roc_curve_matches_the_reference_operating_points() {
let curve = roc_curve(&[0, 0, 1, 1], &[0.1, 0.4, 0.35, 0.8]).unwrap();
assert_eq!(curve.thresholds(), &[f32::INFINITY, 0.8, 0.4, 0.35, 0.1]);
assert_near(curve.false_positive_rates(), &[0.0, 0.0, 0.5, 0.5, 1.0]);
assert_near(curve.true_positive_rates(), &[0.0, 0.5, 0.5, 1.0, 1.0]);
assert_eq!(curve.len(), 5);
assert!(!curve.is_empty());
let tied = roc_curve(&[0, 1, 1, 0, 1], &[0.5, 0.5, 0.5, 0.2, 0.9]).unwrap();
assert_eq!(tied.thresholds(), &[f32::INFINITY, 0.9, 0.5, 0.2]);
assert_near(tied.false_positive_rates(), &[0.0, 0.0, 0.5, 1.0]);
assert_near(
tied.true_positive_rates(),
&[0.0, 0.333_333_333_333_333_3, 1.0, 1.0],
);
}
#[test]
fn the_roc_curve_is_monotone_and_ends_at_the_unit_corner() {
for scores in [
vec![0.9_f32, 0.1, 0.4, 0.4, 0.7, 0.2],
vec![0.5; 6],
vec![-3.0, 2.0, 2.0, 0.0, 1.0, -1.0],
] {
let expected = [0_u8, 1, 0, 1, 1, 0];
let curve = roc_curve(&expected, &scores).unwrap();
assert_eq!(curve.false_positive_rates()[0], 0.0);
assert_eq!(curve.true_positive_rates()[0], 0.0);
assert_eq!(*curve.false_positive_rates().last().unwrap(), 1.0);
assert_eq!(*curve.true_positive_rates().last().unwrap(), 1.0);
assert!(
curve.thresholds().windows(2).all(|pair| pair[0] > pair[1]),
"{:?}",
curve.thresholds()
);
for rates in [curve.false_positive_rates(), curve.true_positive_rates()] {
assert!(rates.windows(2).all(|pair| pair[0] <= pair[1]), "{rates:?}");
}
}
}
#[test]
fn trapezoidal_area_under_the_roc_curve_equals_the_auc_score() {
for (expected, scores) in [
(vec![0_u8, 0, 1, 1], vec![0.1_f32, 0.4, 0.35, 0.8]),
(vec![0, 1, 1, 0, 1], vec![0.5, 0.5, 0.5, 0.2, 0.9]),
(vec![1, 0, 1, 0, 0, 1], vec![0.9, 0.9, 0.2, 0.2, 0.5, 0.5]),
(vec![0, 1], vec![0.3, 0.3]),
] {
let curve = roc_curve(&expected, &scores).unwrap();
let area = curve
.false_positive_rates()
.windows(2)
.zip(curve.true_positive_rates().windows(2))
.map(|(false_rate, true_rate)| {
(false_rate[1] - false_rate[0]) * (true_rate[0] + true_rate[1]) / 2.0
})
.sum::<f64>();
let score = roc_auc_score(&expected, &scores).unwrap();
assert!(
(area - score).abs() < 1.0e-12,
"curve area {area} disagrees with AUC {score}"
);
}
}
#[test]
fn the_precision_recall_curve_matches_the_reference_operating_points() {
let curve = precision_recall_curve(&[0, 0, 1, 1], &[0.1, 0.4, 0.35, 0.8]).unwrap();
assert_eq!(curve.thresholds(), &[0.8, 0.4, 0.35, 0.1]);
assert_near(
curve.precisions(),
&[1.0, 0.5, 0.666_666_666_666_666_6, 0.5],
);
assert_near(curve.recalls(), &[0.5, 0.5, 1.0, 1.0]);
let tied = precision_recall_curve(&[0, 1, 1, 0, 1], &[0.5, 0.5, 0.5, 0.2, 0.9]).unwrap();
assert_eq!(tied.thresholds(), &[0.9, 0.5, 0.2]);
assert_near(tied.precisions(), &[1.0, 0.75, 0.6]);
assert_near(tied.recalls(), &[0.333_333_333_333_333_3, 1.0, 1.0]);
assert!(tied.recalls().windows(2).all(|pair| pair[0] <= pair[1]));
assert_eq!(*tied.recalls().last().unwrap(), 1.0);
assert_eq!(tied.len(), 3);
assert!(!tied.is_empty());
}
#[test]
fn average_precision_is_the_recall_weighted_step_area() {
assert_eq!(
average_precision_score(&[0, 0, 1, 1], &[0.1, 0.4, 0.35, 0.8]),
Ok(0.833_333_333_333_333_3)
);
assert_eq!(
average_precision_score(&[0, 1, 1, 0, 1], &[0.5, 0.5, 0.5, 0.2, 0.9]),
Ok(0.833_333_333_333_333_3)
);
assert_eq!(average_precision_score(&[0, 1], &[0.3, 0.3]), Ok(0.5));
assert_eq!(
average_precision_score(&[0, 0, 1, 1], &[0.1, 0.2, 0.8, 0.9]),
Ok(1.0)
);
assert_eq!(average_precision_score(&[1, 0, 0, 0], &[0.5; 4]), Ok(0.25));
}
#[test]
fn ranked_metrics_reject_absent_classes_and_invalid_inputs() {
assert_eq!(
roc_curve(&[1, 1], &[0.1, 0.9]).unwrap_err(),
MetricError::Undefined
);
assert_eq!(
roc_curve(&[0, 0], &[0.1, 0.9]).unwrap_err(),
MetricError::Undefined
);
assert!(precision_recall_curve(&[1, 1], &[0.1, 0.9]).is_ok());
assert_eq!(
precision_recall_curve(&[0, 0], &[0.1, 0.9]).unwrap_err(),
MetricError::Undefined
);
assert_eq!(
average_precision_score(&[0], &[]),
Err(MetricError::LengthMismatch {
expected: 1,
actual: 0,
})
);
assert_eq!(average_precision_score(&[], &[]), Err(MetricError::Empty));
assert_eq!(
roc_curve(&[0, 2], &[0.1, 0.9]),
Err(MetricError::InvalidBinaryTarget {
input: 0,
index: 1,
value: 2,
})
);
assert_eq!(
precision_recall_curve(&[0, 1], &[0.1, f32::NAN]),
Err(MetricError::NonFiniteValue { input: 1, index: 1 })
);
}
#[test]
fn the_shared_order_is_deterministic_under_ties_and_signed_zero() {
let scores = [1.0_f32, 0.0, -0.0, 1.0, f32::MIN, 0.0];
let order = ascending_score_order(&scores);
assert_eq!(order, ascending_score_order(&scores));
assert!(
order
.windows(2)
.all(|pair| scores[pair[0]].total_cmp(&scores[pair[1]]).is_le())
);
assert_eq!(order, vec![4, 2, 1, 5, 0, 3]);
assert_eq!(tie_groups(&scores, &order), vec![(0, 1), (1, 4), (4, 6)]);
}
}