use std::error::Error;
use std::fmt;
use crate::api::{Classifier, ModelError, Regressor};
use crate::data::{BinaryTargets, MatrixView, RegressionTargets};
use crate::metrics::{
MetricError, accuracy_score, brier_score, f1_score, log_loss, mean_absolute_error,
mean_squared_error, precision_score, r2_score, recall_score, roc_auc_score,
root_mean_squared_error,
};
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[non_exhaustive]
pub enum ClassificationScorer {
Accuracy,
Precision,
Recall,
F1,
Brier,
LogLoss,
RocAuc,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[non_exhaustive]
pub enum RegressionScorer {
MeanAbsoluteError,
MeanSquaredError,
RootMeanSquaredError,
R2,
}
#[derive(Clone, Debug, PartialEq)]
#[non_exhaustive]
pub enum ScoringError {
TargetLength {
rows: usize,
targets: usize,
},
UnsupportedClasses,
Prediction(ModelError),
Metric(MetricError),
}
impl fmt::Display for ScoringError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::TargetLength { rows, targets } => {
write!(
f,
"target length {targets} does not match {rows} evaluated rows"
)
}
Self::UnsupportedClasses => {
f.write_str("binary scoring requires classes [0], [1], or [0, 1]")
}
Self::Prediction(error) => write!(f, "prediction failed: {error}"),
Self::Metric(error) => write!(f, "metric failed: {error}"),
}
}
}
impl Error for ScoringError {
fn source(&self) -> Option<&(dyn Error + 'static)> {
match self {
Self::Prediction(error) => Some(error),
Self::Metric(error) => Some(error),
_ => None,
}
}
}
pub fn score_classifier(
classifier: &dyn Classifier,
data: &MatrixView<'_>,
targets: &BinaryTargets,
scorer: ClassificationScorer,
) -> Result<f64, ScoringError> {
validate_target_length(data.rows(), targets.len())?;
match scorer {
ClassificationScorer::Accuracy => {
let predicted = classifier.predict(data).map_err(ScoringError::Prediction)?;
accuracy_score(targets.as_slice(), &predicted).map_err(ScoringError::Metric)
}
ClassificationScorer::Precision => {
let predicted = classifier.predict(data).map_err(ScoringError::Prediction)?;
precision_score(targets.as_slice(), &predicted).map_err(ScoringError::Metric)
}
ClassificationScorer::Recall => {
let predicted = classifier.predict(data).map_err(ScoringError::Prediction)?;
recall_score(targets.as_slice(), &predicted).map_err(ScoringError::Metric)
}
ClassificationScorer::F1 => {
let predicted = classifier.predict(data).map_err(ScoringError::Prediction)?;
f1_score(targets.as_slice(), &predicted).map_err(ScoringError::Metric)
}
ClassificationScorer::Brier => {
let probabilities = positive_probabilities(classifier, data)?;
brier_score(targets.as_slice(), &probabilities).map_err(ScoringError::Metric)
}
ClassificationScorer::LogLoss => {
let probabilities = positive_probabilities(classifier, data)?;
log_loss(targets.as_slice(), &probabilities).map_err(ScoringError::Metric)
}
ClassificationScorer::RocAuc => {
let probabilities = positive_probabilities(classifier, data)?;
roc_auc_score(targets.as_slice(), &probabilities).map_err(ScoringError::Metric)
}
}
}
pub fn score_regressor(
regressor: &dyn Regressor,
data: &MatrixView<'_>,
targets: &RegressionTargets,
scorer: RegressionScorer,
) -> Result<f64, ScoringError> {
validate_target_length(data.rows(), targets.len())?;
let predicted = regressor.predict(data).map_err(ScoringError::Prediction)?;
match scorer {
RegressionScorer::MeanAbsoluteError => {
mean_absolute_error(targets.as_slice(), &predicted).map_err(ScoringError::Metric)
}
RegressionScorer::MeanSquaredError => {
mean_squared_error(targets.as_slice(), &predicted).map_err(ScoringError::Metric)
}
RegressionScorer::RootMeanSquaredError => {
root_mean_squared_error(targets.as_slice(), &predicted).map_err(ScoringError::Metric)
}
RegressionScorer::R2 => {
r2_score(targets.as_slice(), &predicted).map_err(ScoringError::Metric)
}
}
}
fn validate_target_length(rows: usize, targets: usize) -> Result<(), ScoringError> {
if rows != targets {
return Err(ScoringError::TargetLength { rows, targets });
}
Ok(())
}
fn positive_probabilities(
classifier: &dyn Classifier,
data: &MatrixView<'_>,
) -> Result<Vec<f32>, ScoringError> {
match classifier.classes() {
[0] => Ok(vec![0.0; data.rows()]),
[1] => Ok(vec![1.0; data.rows()]),
[0, 1] => classifier
.predict_class_proba(data, 1)
.map_err(ScoringError::Prediction),
_ => Err(ScoringError::UnsupportedClasses),
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::api::{AnyClassifier, AnyRegressor};
use crate::data::DenseMatrix;
use crate::ensemble::{
MaxFeatures, RandomForestClassifier, RandomForestClassifierParams, RandomForestRegressor,
RandomForestRegressorParams,
};
fn matrix() -> DenseMatrix {
DenseMatrix::new(vec![0.0, 1.0, 2.0, 3.0], 4, 1).unwrap()
}
fn classifier(targets: &BinaryTargets) -> RandomForestClassifier {
RandomForestClassifier::fit(
&matrix().as_view(),
targets,
RandomForestClassifierParams::default()
.with_n_estimators(1)
.with_bootstrap(false)
.with_max_features(MaxFeatures::All),
)
.unwrap()
}
#[test]
fn classifier_scorers_equal_direct_metric_calls_for_concrete_and_erased_models() {
let data = matrix();
let targets = BinaryTargets::new(vec![0, 0, 1, 1]).unwrap();
let concrete = classifier(&targets);
let labels = concrete.predict(&data.as_view()).unwrap();
let probabilities = concrete.predict_class_proba(&data.as_view(), 1).unwrap();
assert_eq!(
score_classifier(
&concrete,
&data.as_view(),
&targets,
ClassificationScorer::Accuracy
),
accuracy_score(targets.as_slice(), &labels).map_err(ScoringError::Metric)
);
assert_eq!(
score_classifier(
&concrete,
&data.as_view(),
&targets,
ClassificationScorer::Brier
),
brier_score(targets.as_slice(), &probabilities).map_err(ScoringError::Metric)
);
let erased: AnyClassifier = concrete.into();
for scorer in [
ClassificationScorer::Accuracy,
ClassificationScorer::Precision,
ClassificationScorer::Recall,
ClassificationScorer::F1,
ClassificationScorer::Brier,
ClassificationScorer::LogLoss,
ClassificationScorer::RocAuc,
] {
assert!(score_classifier(&erased, &data.as_view(), &targets, scorer).is_ok());
}
}
#[test]
fn singleton_class_probability_scores_are_explicit() {
let data = matrix();
for (label, expected_brier) in [(0, 0.0), (1, 0.0)] {
let targets = BinaryTargets::new(vec![label; 4]).unwrap();
let model = classifier(&targets);
assert_eq!(
score_classifier(
&model,
&data.as_view(),
&targets,
ClassificationScorer::Brier
),
Ok(expected_brier)
);
assert_eq!(
score_classifier(
&model,
&data.as_view(),
&targets,
ClassificationScorer::RocAuc
),
Err(ScoringError::Metric(MetricError::Undefined))
);
}
}
#[test]
fn regressor_scorers_equal_direct_metrics_and_runtime_dispatch() {
let data = matrix();
let targets = RegressionTargets::new(vec![0.0, 1.0, 4.0, 9.0]).unwrap();
let concrete = RandomForestRegressor::fit(
&data.as_view(),
&targets,
RandomForestRegressorParams::default()
.with_n_estimators(1)
.with_bootstrap(false),
)
.unwrap();
let erased: AnyRegressor = concrete.into();
for scorer in [
RegressionScorer::MeanAbsoluteError,
RegressionScorer::MeanSquaredError,
RegressionScorer::RootMeanSquaredError,
RegressionScorer::R2,
] {
assert!(score_regressor(&erased, &data.as_view(), &targets, scorer).is_ok());
}
}
#[test]
fn scoring_validates_targets_before_prediction_and_preserves_metric_errors() {
let data = matrix();
let targets = BinaryTargets::new(vec![0, 1]).unwrap();
let fitted_targets = BinaryTargets::new(vec![0, 0, 1, 1]).unwrap();
let model = classifier(&fitted_targets);
assert_eq!(
score_classifier(
&model,
&data.as_view(),
&targets,
ClassificationScorer::Accuracy
),
Err(ScoringError::TargetLength {
rows: 4,
targets: 2,
})
);
let all_negative = BinaryTargets::new(vec![0; 4]).unwrap();
assert_eq!(
score_classifier(
&model,
&data.as_view(),
&all_negative,
ClassificationScorer::Recall
),
Err(ScoringError::Metric(MetricError::Undefined))
);
}
}