use super::{Average, Averaging, MetricError, ZeroDivision, validate_lengths};
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
pub struct ClassCounts {
true_positives: u64,
false_positives: u64,
false_negatives: u64,
}
impl ClassCounts {
pub const fn true_positives(self) -> u64 {
self.true_positives
}
pub const fn false_positives(self) -> u64 {
self.false_positives
}
pub const fn false_negatives(self) -> u64 {
self.false_negatives
}
pub const fn support(self) -> u64 {
self.true_positives + self.false_negatives
}
pub const fn predicted(self) -> u64 {
self.true_positives + self.false_positives
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ConfusionMatrix {
labels: Vec<u8>,
counts: Vec<u64>,
total: u64,
}
impl ConfusionMatrix {
pub fn new(expected: &[u8], predicted: &[u8]) -> Result<Self, MetricError> {
validate_lengths(expected.len(), predicted.len())?;
let mut observed = [false; 256];
for &label in expected.iter().chain(predicted) {
observed[label as usize] = true;
}
let labels = (0..=u8::MAX)
.filter(|&label| observed[label as usize])
.collect::<Vec<_>>();
let mut position = [0_usize; 256];
for (index, &label) in labels.iter().enumerate() {
position[label as usize] = index;
}
let mut counts = vec![0_u64; labels.len() * labels.len()];
for (&expected, &predicted) in expected.iter().zip(predicted) {
let row = position[expected as usize];
let column = position[predicted as usize];
counts[row * labels.len() + column] += 1;
}
Ok(Self {
labels,
counts,
total: expected.len() as u64,
})
}
pub fn labels(&self) -> &[u8] {
&self.labels
}
pub fn n_labels(&self) -> usize {
self.labels.len()
}
pub fn counts(&self) -> &[u64] {
&self.counts
}
pub const fn total(&self) -> u64 {
self.total
}
pub fn class_counts(&self, index: usize) -> Option<ClassCounts> {
if index >= self.labels.len() {
return None;
}
Some(self.counts_at(index))
}
pub fn accuracy(&self) -> f64 {
let correct = (0..self.labels.len())
.map(|index| self.counts[index * self.labels.len() + index])
.sum::<u64>();
correct as f64 / self.total as f64
}
pub fn balanced_accuracy(&self) -> f64 {
let mut total = 0.0_f64;
let mut classes = 0.0_f64;
for index in 0..self.labels.len() {
let counts = self.counts_at(index);
if counts.support() == 0 {
continue;
}
total += counts.true_positives() as f64 / counts.support() as f64;
classes += 1.0;
}
debug_assert!(classes > 0.0, "a non-empty matrix has a class with support");
total / classes
}
pub fn matthews_correlation(&self) -> Result<f64, MetricError> {
let labels = self.labels.len();
let total = self.total as f64;
let mut correct = 0.0_f64;
let mut agreement = 0.0_f64;
let mut expected_squares = 0.0_f64;
let mut predicted_squares = 0.0_f64;
for index in 0..labels {
let counts = self.counts_at(index);
let support = counts.support() as f64;
let predicted = counts.predicted() as f64;
correct += counts.true_positives() as f64;
agreement += support * predicted;
expected_squares += support * support;
predicted_squares += predicted * predicted;
}
let expected_variance = total * total - expected_squares;
let predicted_variance = total * total - predicted_squares;
if expected_variance <= 0.0 || predicted_variance <= 0.0 {
return Err(MetricError::Undefined);
}
Ok((correct * total - agreement) / (expected_variance * predicted_variance).sqrt())
}
pub fn precision(&self, averaging: impl Into<Averaging>) -> Result<f64, MetricError> {
self.average(averaging.into(), Score::Precision)
}
pub fn recall(&self, averaging: impl Into<Averaging>) -> Result<f64, MetricError> {
self.average(averaging.into(), Score::Recall)
}
pub fn f1(&self, averaging: impl Into<Averaging>) -> Result<f64, MetricError> {
self.fbeta(1.0, averaging)
}
pub fn fbeta(&self, beta: f64, averaging: impl Into<Averaging>) -> Result<f64, MetricError> {
if !beta.is_finite() || beta <= 0.0 {
return Err(MetricError::InvalidBeta);
}
self.average(
averaging.into(),
Score::FBeta {
squared: beta * beta,
},
)
}
fn counts_at(&self, index: usize) -> ClassCounts {
let labels = self.labels.len();
let true_positives = self.counts[index * labels + index];
let row = self.counts[index * labels..(index + 1) * labels]
.iter()
.sum::<u64>();
let column = (0..labels)
.map(|other| self.counts[other * labels + index])
.sum::<u64>();
ClassCounts {
true_positives,
false_positives: column - true_positives,
false_negatives: row - true_positives,
}
}
fn average(&self, averaging: Averaging, score: Score) -> Result<f64, MetricError> {
match averaging.average() {
Average::Binary => {
if self.labels.last().is_some_and(|&label| label > 1) {
return Err(MetricError::NotBinary {
labels: self.labels.len(),
});
}
let positive = self.labels.iter().position(|&label| label == 1);
let counts = positive.map_or(ClassCounts::default(), |index| self.counts_at(index));
score.of(counts).ok_or(MetricError::Undefined)
}
Average::Micro => {
let pooled =
(0..self.labels.len()).fold(ClassCounts::default(), |mut sum, index| {
let counts = self.counts_at(index);
sum.true_positives += counts.true_positives;
sum.false_positives += counts.false_positives;
sum.false_negatives += counts.false_negatives;
sum
});
score.of(pooled).ok_or(MetricError::Undefined)
}
Average::Macro => self.combine(score, averaging.zero_division(), |_| Some(1.0)),
Average::Weighted => self.combine(score, averaging.zero_division(), |counts| {
(counts.support() > 0).then(|| counts.support() as f64)
}),
}
}
fn combine(
&self,
score: Score,
zero_division: ZeroDivision,
weight: impl Fn(ClassCounts) -> Option<f64>,
) -> Result<f64, MetricError> {
let mut total = 0.0_f64;
let mut divisor = 0.0_f64;
for index in 0..self.labels.len() {
let counts = self.counts_at(index);
let Some(weight) = weight(counts) else {
continue;
};
let value = match score.of(counts) {
Some(value) => value,
None => match zero_division {
ZeroDivision::Error => return Err(MetricError::Undefined),
ZeroDivision::Zero => 0.0,
ZeroDivision::Skip => continue,
},
};
total += weight * value;
divisor += weight;
}
if divisor == 0.0 {
return Err(MetricError::Undefined);
}
Ok(total / divisor)
}
}
#[derive(Clone, Copy, Debug)]
enum Score {
Precision,
Recall,
FBeta { squared: f64 },
}
impl Score {
fn of(self, counts: ClassCounts) -> Option<f64> {
let (numerator, denominator) = match self {
Self::Precision => (counts.true_positives() as f64, counts.predicted() as f64),
Self::Recall => (counts.true_positives() as f64, counts.support() as f64),
Self::FBeta { squared } => {
let scale = 1.0 + squared;
(
scale * counts.true_positives() as f64,
scale * counts.true_positives() as f64
+ squared * counts.false_negatives() as f64
+ counts.false_positives() as f64,
)
}
};
(denominator > 0.0).then(|| numerator / denominator)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::metrics::{f1_score, precision_score, recall_score};
const THREE_CLASS_EXPECTED: [u8; 8] = [0, 1, 2, 2, 1, 0, 1, 2];
const THREE_CLASS_PREDICTED: [u8; 8] = [0, 2, 2, 2, 1, 0, 0, 1];
fn three_class() -> ConfusionMatrix {
ConfusionMatrix::new(&THREE_CLASS_EXPECTED, &THREE_CLASS_PREDICTED).unwrap()
}
fn assert_near(actual: Result<f64, MetricError>, expected: f64) {
let actual = actual.expect("score is defined");
assert!(
(actual - expected).abs() < 1.0e-12,
"expected {expected}, got {actual}"
);
}
#[test]
fn the_matrix_counts_the_observed_label_union_in_sorted_order() {
let matrix = three_class();
assert_eq!(matrix.labels(), &[0, 1, 2]);
assert_eq!(matrix.n_labels(), 3);
assert_eq!(matrix.counts(), &[2, 0, 0, 1, 1, 1, 0, 1, 2]);
assert_eq!(matrix.total(), 8);
assert_eq!(matrix.counts().iter().sum::<u64>(), matrix.total());
let class_one = matrix.class_counts(1).unwrap();
assert_eq!(class_one.true_positives(), 1);
assert_eq!(class_one.false_positives(), 1);
assert_eq!(class_one.false_negatives(), 2);
assert_eq!(class_one.support(), 3);
assert_eq!(class_one.predicted(), 2);
assert_eq!(matrix.class_counts(3), None);
let sparse = ConfusionMatrix::new(&[0, 7], &[7, 7]).unwrap();
assert_eq!(sparse.labels(), &[0, 7]);
assert_eq!(sparse.counts(), &[0, 1, 0, 1]);
}
#[test]
fn micro_averaging_equals_accuracy_for_single_label_predictions() {
for (expected, predicted) in [
(&THREE_CLASS_EXPECTED[..], &THREE_CLASS_PREDICTED[..]),
(&[0, 1, 2][..], &[0, 1, 1][..]),
(&[0, 0, 1][..], &[0, 2, 1][..]),
(&[3, 3, 3][..], &[3, 3, 3][..]),
(&[0, 1][..], &[1, 0][..]),
] {
let matrix = ConfusionMatrix::new(expected, predicted).unwrap();
let accuracy = matrix.accuracy();
assert_eq!(matrix.precision(Average::Micro), Ok(accuracy));
assert_eq!(matrix.recall(Average::Micro), Ok(accuracy));
assert_eq!(matrix.f1(Average::Micro), Ok(accuracy));
assert_eq!(matrix.fbeta(2.0, Average::Micro), Ok(accuracy));
assert_eq!(matrix.fbeta(0.5, Average::Micro), Ok(accuracy));
}
}
#[test]
fn binary_averaging_reproduces_the_binary_metrics_exactly() {
for (expected, predicted) in [
(&[0, 0, 1, 1][..], &[0, 1, 1, 1][..]),
(&[0, 0, 0, 1, 1, 1][..], &[0, 1, 0, 1, 0, 1][..]),
(&[1, 1][..], &[0, 1][..]),
(&[0, 0][..], &[0, 1][..]),
(&[0, 0][..], &[0, 0][..]),
] {
let matrix = ConfusionMatrix::new(expected, predicted).unwrap();
assert_eq!(
matrix.precision(Average::Binary),
precision_score(expected, predicted)
);
assert_eq!(
matrix.recall(Average::Binary),
recall_score(expected, predicted)
);
assert_eq!(matrix.f1(Average::Binary), f1_score(expected, predicted));
assert_eq!(matrix.accuracy(), matrix.precision(Average::Micro).unwrap());
}
}
#[test]
fn a_wider_label_set_rejects_binary_averaging_instead_of_scoring_one_class() {
let matrix = three_class();
assert_eq!(
matrix.precision(Average::Binary),
Err(MetricError::NotBinary { labels: 3 })
);
assert_eq!(
matrix.f1(Average::Binary),
Err(MetricError::NotBinary { labels: 3 })
);
let negative = ConfusionMatrix::new(&[0, 0], &[0, 0]).unwrap();
assert_eq!(
negative.recall(Average::Binary),
Err(MetricError::Undefined)
);
}
#[test]
fn macro_and_weighted_averages_match_the_reference_conventions() {
let matrix = three_class();
assert_near(matrix.precision(Average::Macro), 0.611_111_111_111_111);
assert_near(matrix.recall(Average::Macro), 0.666_666_666_666_666_6);
assert_near(matrix.f1(Average::Macro), 0.622_222_222_222_222_2);
assert_near(matrix.fbeta(2.0, Average::Macro), 0.644_300_144_300_144_3);
assert_near(matrix.precision(Average::Weighted), 0.604_166_666_666_666_6);
assert_near(matrix.recall(Average::Weighted), 0.625);
assert_near(matrix.f1(Average::Weighted), 0.6);
assert_near(
matrix.fbeta(2.0, Average::Weighted),
0.611_201_298_701_298_7,
);
let absent_truth = ConfusionMatrix::new(&[0, 0, 1], &[0, 2, 1]).unwrap();
assert_near(absent_truth.precision(Average::Weighted), 1.0);
assert_near(absent_truth.f1(Average::Weighted), 0.777_777_777_777_777_7);
assert_near(
absent_truth.precision(Average::Macro),
0.666_666_666_666_666_6,
);
assert_near(absent_truth.f1(Average::Macro), 0.555_555_555_555_555_5);
}
#[test]
fn an_empty_denominator_is_an_error_until_the_caller_says_otherwise() {
let matrix = ConfusionMatrix::new(&[0, 1, 2], &[0, 1, 1]).unwrap();
assert_eq!(
matrix.precision(Average::Macro),
Err(MetricError::Undefined)
);
assert_eq!(
matrix.precision(Average::Weighted),
Err(MetricError::Undefined)
);
for average in [Average::Macro, Average::Weighted] {
assert_near(
matrix.precision(Averaging::new(average).with_zero_division(ZeroDivision::Skip)),
0.75,
);
assert_near(
matrix.precision(Averaging::new(average).with_zero_division(ZeroDivision::Zero)),
0.5,
);
}
assert_near(matrix.recall(Average::Macro), 0.666_666_666_666_666_6);
assert_near(matrix.f1(Average::Macro), 0.555_555_555_555_555_5);
let nothing_predicted = ConfusionMatrix::new(&[0, 0], &[0, 0]).unwrap();
assert_eq!(
nothing_predicted
.recall(Averaging::new(Average::Macro).with_zero_division(ZeroDivision::Skip)),
Ok(1.0)
);
let absent = ConfusionMatrix::new(&[0, 0], &[1, 1]).unwrap();
assert_eq!(
absent.recall(Averaging::new(Average::Macro).with_zero_division(ZeroDivision::Skip)),
Ok(0.0)
);
}
#[test]
fn beta_selects_between_precision_and_recall_and_rejects_invalid_values() {
let matrix = ConfusionMatrix::new(&[0, 0, 1, 1], &[0, 1, 1, 1]).unwrap();
let precision_heavy = matrix.fbeta(0.5, Average::Binary).unwrap();
let balanced = matrix.f1(Average::Binary).unwrap();
let recall_heavy = matrix.fbeta(2.0, Average::Binary).unwrap();
assert!(precision_heavy < balanced && balanced < recall_heavy);
assert_near(Ok(recall_heavy), 0.909_090_909_090_909_1);
assert_near(Ok(precision_heavy), 0.714_285_714_285_714_2);
for beta in [0.0, -1.0, f64::NAN, f64::INFINITY] {
assert_eq!(
matrix.fbeta(beta, Average::Binary),
Err(MetricError::InvalidBeta)
);
}
}
#[test]
fn a_perfect_result_scores_one_under_every_average() {
let matrix = ConfusionMatrix::new(&[0, 1, 2], &[0, 1, 2]).unwrap();
for average in [Average::Micro, Average::Macro, Average::Weighted] {
assert_eq!(matrix.precision(average), Ok(1.0));
assert_eq!(matrix.recall(average), Ok(1.0));
assert_eq!(matrix.f1(average), Ok(1.0));
}
assert_eq!(matrix.accuracy(), 1.0);
}
#[test]
fn balanced_accuracy_is_macro_recall_over_the_classes_that_occur() {
let matrix = three_class();
assert_near(Ok(matrix.balanced_accuracy()), 0.666_666_666_666_666_6);
assert_eq!(
Ok(matrix.balanced_accuracy()),
matrix.recall(Average::Macro)
);
let absent_truth = ConfusionMatrix::new(&[0, 0, 1], &[0, 2, 1]).unwrap();
assert_eq!(
absent_truth.recall(Average::Macro),
Err(MetricError::Undefined)
);
assert_near(Ok(absent_truth.balanced_accuracy()), 0.75);
let mut expected = vec![0_u8; 90];
expected.extend(std::iter::repeat_n(1, 10));
let majority = ConfusionMatrix::new(&expected, &[0_u8; 100]).unwrap();
assert_eq!(majority.accuracy(), 0.9);
assert_eq!(majority.balanced_accuracy(), 0.5);
}
#[test]
fn matthews_correlation_spans_disagreement_to_agreement() {
assert_near(
three_class().matthews_correlation(),
0.452_380_952_380_952_4,
);
assert_near(
ConfusionMatrix::new(&[0, 0, 1, 1], &[0, 1, 1, 1])
.unwrap()
.matthews_correlation(),
0.577_350_269_189_625_8,
);
assert_eq!(
ConfusionMatrix::new(&[0, 1, 2], &[0, 1, 2])
.unwrap()
.matthews_correlation(),
Ok(1.0)
);
assert_eq!(
ConfusionMatrix::new(&[0, 1, 0, 1], &[1, 0, 1, 0])
.unwrap()
.matthews_correlation(),
Ok(-1.0)
);
for (expected, predicted) in [
(&[0, 0][..], &[0, 0][..]),
(&[0, 1][..], &[1, 1][..]),
(&[1, 1][..], &[0, 1][..]),
] {
assert_eq!(
ConfusionMatrix::new(expected, predicted)
.unwrap()
.matthews_correlation(),
Err(MetricError::Undefined)
);
}
}
#[test]
fn matrix_construction_validates_before_counting() {
assert_eq!(
ConfusionMatrix::new(&[0], &[]),
Err(MetricError::LengthMismatch {
expected: 1,
actual: 0,
})
);
assert_eq!(ConfusionMatrix::new(&[], &[]), Err(MetricError::Empty));
}
}