use std::error::Error;
use std::fmt;
use crate::api::{Classifier, ModelError, ProbabilisticClassifier, Regressor};
use crate::data::{BinaryTargets, ClassTargets, MatrixView, RegressionTargets};
use crate::metrics::{
MetricError, accuracy_score, brier_score, f1_score, log_loss, mean_absolute_error,
mean_squared_error, multiclass_brier_score, multiclass_log_loss, precision_score, r2_score,
recall_score, roc_auc_score, root_mean_squared_error,
};
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[non_exhaustive]
pub enum ClassificationScorer {
Accuracy,
Precision,
Recall,
F1,
Brier,
LogLoss,
RocAuc,
MulticlassLogLoss,
MulticlassBrier,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[non_exhaustive]
pub enum RegressionScorer {
MeanAbsoluteError,
MeanSquaredError,
RootMeanSquaredError,
R2,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[non_exhaustive]
pub enum ClassifierOutputKind {
Labels,
PositiveProbabilities,
ProbabilityMatrix,
}
#[derive(Clone, Copy, Debug, PartialEq)]
#[non_exhaustive]
pub enum ClassifierOutput<'a> {
Labels(&'a [u8]),
PositiveProbabilities(&'a [f32]),
ProbabilityMatrix {
classes: &'a [u8],
probabilities: &'a [f32],
},
}
impl ClassifierOutput<'_> {
pub const fn kind(&self) -> ClassifierOutputKind {
match self {
Self::Labels(_) => ClassifierOutputKind::Labels,
Self::PositiveProbabilities(_) => ClassifierOutputKind::PositiveProbabilities,
Self::ProbabilityMatrix { .. } => ClassifierOutputKind::ProbabilityMatrix,
}
}
}
pub trait ClassificationScore {
fn output_kind(&self) -> ClassifierOutputKind;
fn greater_is_better(&self) -> bool;
fn score(&self, expected: &[u8], output: ClassifierOutput<'_>) -> Result<f64, ScoringError>;
}
pub trait RegressionScore {
fn greater_is_better(&self) -> bool;
fn score(&self, expected: &[f32], predicted: &[f32]) -> Result<f64, ScoringError>;
}
impl<S: ClassificationScore + ?Sized> ClassificationScore for &S {
fn output_kind(&self) -> ClassifierOutputKind {
(**self).output_kind()
}
fn greater_is_better(&self) -> bool {
(**self).greater_is_better()
}
fn score(&self, expected: &[u8], output: ClassifierOutput<'_>) -> Result<f64, ScoringError> {
(**self).score(expected, output)
}
}
impl<S: RegressionScore + ?Sized> RegressionScore for &S {
fn greater_is_better(&self) -> bool {
(**self).greater_is_better()
}
fn score(&self, expected: &[f32], predicted: &[f32]) -> Result<f64, ScoringError> {
(**self).score(expected, predicted)
}
}
impl ClassificationScore for ClassificationScorer {
fn output_kind(&self) -> ClassifierOutputKind {
match self {
Self::Accuracy | Self::Precision | Self::Recall | Self::F1 => {
ClassifierOutputKind::Labels
}
Self::Brier | Self::LogLoss | Self::RocAuc => {
ClassifierOutputKind::PositiveProbabilities
}
Self::MulticlassLogLoss | Self::MulticlassBrier => {
ClassifierOutputKind::ProbabilityMatrix
}
}
}
fn greater_is_better(&self) -> bool {
!matches!(
self,
Self::Brier | Self::LogLoss | Self::MulticlassLogLoss | Self::MulticlassBrier
)
}
fn score(&self, expected: &[u8], output: ClassifierOutput<'_>) -> Result<f64, ScoringError> {
let value = match (self, output) {
(Self::Accuracy, ClassifierOutput::Labels(predicted)) => {
accuracy_score(expected, predicted)
}
(Self::Precision, ClassifierOutput::Labels(predicted)) => {
precision_score(expected, predicted)
}
(Self::Recall, ClassifierOutput::Labels(predicted)) => {
recall_score(expected, predicted)
}
(Self::F1, ClassifierOutput::Labels(predicted)) => f1_score(expected, predicted),
(Self::Brier, ClassifierOutput::PositiveProbabilities(probabilities)) => {
brier_score(expected, probabilities)
}
(Self::LogLoss, ClassifierOutput::PositiveProbabilities(probabilities)) => {
log_loss(expected, probabilities)
}
(Self::RocAuc, ClassifierOutput::PositiveProbabilities(probabilities)) => {
roc_auc_score(expected, probabilities)
}
(
Self::MulticlassLogLoss,
ClassifierOutput::ProbabilityMatrix {
classes,
probabilities,
},
) => multiclass_log_loss(expected, classes, probabilities),
(
Self::MulticlassBrier,
ClassifierOutput::ProbabilityMatrix {
classes,
probabilities,
},
) => multiclass_brier_score(expected, classes, probabilities),
(_, supplied) => {
return Err(ScoringError::UnsupportedOutput {
required: self.output_kind(),
supplied: supplied.kind(),
});
}
};
value.map_err(ScoringError::Metric)
}
}
impl RegressionScore for RegressionScorer {
fn greater_is_better(&self) -> bool {
matches!(self, Self::R2)
}
fn score(&self, expected: &[f32], predicted: &[f32]) -> Result<f64, ScoringError> {
match self {
Self::MeanAbsoluteError => mean_absolute_error(expected, predicted),
Self::MeanSquaredError => mean_squared_error(expected, predicted),
Self::RootMeanSquaredError => root_mean_squared_error(expected, predicted),
Self::R2 => r2_score(expected, predicted),
}
.map_err(ScoringError::Metric)
}
}
#[derive(Clone, Debug, Default)]
pub struct ScoringWorkspace {
labels: Vec<u8>,
values: Vec<f32>,
}
impl ScoringWorkspace {
pub fn new() -> Self {
Self::default()
}
fn labels(&mut self, rows: usize) -> &mut [u8] {
self.labels.resize(rows, 0);
&mut self.labels
}
fn values(&mut self, rows: usize) -> &mut [f32] {
self.values.resize(rows, 0.0);
&mut self.values
}
fn matrix(&mut self, rows: usize, columns: usize) -> Result<&mut [f32], ScoringError> {
let len = rows.checked_mul(columns).ok_or(ScoringError::Prediction(
ModelError::OutputShapeOverflow { rows, columns },
))?;
Ok(self.values(len))
}
}
#[derive(Clone, Debug, PartialEq)]
#[non_exhaustive]
pub enum ScoringError {
TargetLength {
rows: usize,
targets: usize,
},
UnsupportedClasses,
UnsupportedOutput {
required: ClassifierOutputKind,
supplied: ClassifierOutputKind,
},
Prediction(ModelError),
Metric(MetricError),
}
impl fmt::Display for ScoringError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::TargetLength { rows, targets } => {
write!(
f,
"target length {targets} does not match {rows} evaluated rows"
)
}
Self::UnsupportedClasses => {
f.write_str("binary scoring requires classes [0], [1], or [0, 1]")
}
Self::UnsupportedOutput { required, supplied } => {
write!(f, "score reads {required:?} but was given {supplied:?}")
}
Self::Prediction(error) => write!(f, "prediction failed: {error}"),
Self::Metric(error) => write!(f, "metric failed: {error}"),
}
}
}
impl Error for ScoringError {
fn source(&self) -> Option<&(dyn Error + 'static)> {
match self {
Self::Prediction(error) => Some(error),
Self::Metric(error) => Some(error),
_ => None,
}
}
}
#[derive(Clone, Copy)]
pub struct ScorableClassifier<'a> {
labels: &'a dyn Classifier,
probabilities: Option<&'a dyn ProbabilisticClassifier>,
}
impl<'a> ScorableClassifier<'a> {
#[must_use]
pub fn probabilistic(model: &'a dyn ProbabilisticClassifier) -> Self {
Self {
labels: model,
probabilities: Some(model),
}
}
#[must_use]
pub const fn labels_only(model: &'a dyn Classifier) -> Self {
Self {
labels: model,
probabilities: None,
}
}
fn probabilities(
self,
required: ClassifierOutputKind,
) -> Result<&'a dyn ProbabilisticClassifier, ScoringError> {
self.probabilities.ok_or(ScoringError::UnsupportedOutput {
required,
supplied: ClassifierOutputKind::Labels,
})
}
}
pub fn score_classifier<S: ClassificationScore>(
classifier: ScorableClassifier<'_>,
data: &MatrixView<'_>,
targets: &BinaryTargets,
scorer: S,
) -> Result<f64, ScoringError> {
score_classifier_with(
classifier,
data,
targets,
scorer,
&mut ScoringWorkspace::new(),
)
}
pub fn score_classifier_with<S: ClassificationScore>(
classifier: ScorableClassifier<'_>,
data: &MatrixView<'_>,
targets: &BinaryTargets,
scorer: S,
workspace: &mut ScoringWorkspace,
) -> Result<f64, ScoringError> {
score_labelled(classifier, data, targets.as_slice(), scorer, workspace)
}
pub fn score_multiclass_classifier<S: ClassificationScore>(
classifier: ScorableClassifier<'_>,
data: &MatrixView<'_>,
targets: &ClassTargets,
scorer: S,
) -> Result<f64, ScoringError> {
score_multiclass_classifier_with(
classifier,
data,
targets,
scorer,
&mut ScoringWorkspace::new(),
)
}
pub fn score_multiclass_classifier_with<S: ClassificationScore>(
classifier: ScorableClassifier<'_>,
data: &MatrixView<'_>,
targets: &ClassTargets,
scorer: S,
workspace: &mut ScoringWorkspace,
) -> Result<f64, ScoringError> {
score_labelled(classifier, data, targets.as_slice(), scorer, workspace)
}
pub(crate) fn score_labelled<S: ClassificationScore>(
classifier: ScorableClassifier<'_>,
data: &MatrixView<'_>,
targets: &[u8],
scorer: S,
workspace: &mut ScoringWorkspace,
) -> Result<f64, ScoringError> {
validate_target_length(data.rows(), targets.len())?;
let output = match scorer.output_kind() {
ClassifierOutputKind::Labels => {
let labels = workspace.labels(data.rows());
classifier
.labels
.predict_into(data, labels)
.map_err(ScoringError::Prediction)?;
ClassifierOutput::Labels(labels)
}
ClassifierOutputKind::PositiveProbabilities => {
let model = classifier.probabilities(ClassifierOutputKind::PositiveProbabilities)?;
let probabilities = workspace.values(data.rows());
match model.classes() {
[0] => probabilities.fill(0.0),
[1] => probabilities.fill(1.0),
[0, 1] => model
.predict_class_proba_into(data, 1, probabilities)
.map_err(ScoringError::Prediction)?,
_ => return Err(ScoringError::UnsupportedClasses),
}
ClassifierOutput::PositiveProbabilities(probabilities)
}
ClassifierOutputKind::ProbabilityMatrix => {
let model = classifier.probabilities(ClassifierOutputKind::ProbabilityMatrix)?;
let classes = model.classes();
let probabilities = workspace.matrix(data.rows(), classes.len())?;
model
.predict_proba_into(data, probabilities)
.map_err(ScoringError::Prediction)?;
ClassifierOutput::ProbabilityMatrix {
classes,
probabilities,
}
}
};
scorer.score(targets, output)
}
pub fn score_regressor<S: RegressionScore>(
regressor: &dyn Regressor,
data: &MatrixView<'_>,
targets: &RegressionTargets,
scorer: S,
) -> Result<f64, ScoringError> {
score_regressor_with(
regressor,
data,
targets,
scorer,
&mut ScoringWorkspace::new(),
)
}
pub fn score_regressor_with<S: RegressionScore>(
regressor: &dyn Regressor,
data: &MatrixView<'_>,
targets: &RegressionTargets,
scorer: S,
workspace: &mut ScoringWorkspace,
) -> Result<f64, ScoringError> {
validate_target_length(data.rows(), targets.len())?;
let predicted = workspace.values(data.rows());
regressor
.predict_into(data, predicted)
.map_err(ScoringError::Prediction)?;
scorer.score(targets.as_slice(), predicted)
}
fn validate_target_length(rows: usize, targets: usize) -> Result<(), ScoringError> {
if rows != targets {
return Err(ScoringError::TargetLength { rows, targets });
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use crate::api::{AnyClassifier, AnyRegressor};
use crate::data::{ClassTargets, DenseMatrix};
use crate::ensemble::{
RandomForestClassifier, RandomForestClassifierParams, RandomForestRegressor,
RandomForestRegressorParams,
};
use crate::metrics::{Average, ConfusionMatrix};
use crate::tree::MaxFeatures;
fn matrix() -> DenseMatrix {
DenseMatrix::new(vec![0.0, 1.0, 2.0, 3.0], 4, 1).unwrap()
}
fn classifier(targets: &BinaryTargets) -> RandomForestClassifier {
RandomForestClassifier::fit(
&matrix().as_view(),
targets,
RandomForestClassifierParams::default()
.with_n_estimators(1)
.with_bootstrap(false)
.with_max_features(MaxFeatures::All),
)
.unwrap()
}
struct MacroF1;
impl ClassificationScore for MacroF1 {
fn output_kind(&self) -> ClassifierOutputKind {
ClassifierOutputKind::Labels
}
fn greater_is_better(&self) -> bool {
true
}
fn score(
&self,
expected: &[u8],
output: ClassifierOutput<'_>,
) -> Result<f64, ScoringError> {
let ClassifierOutput::Labels(predicted) = output else {
return Err(ScoringError::UnsupportedOutput {
required: self.output_kind(),
supplied: output.kind(),
});
};
ConfusionMatrix::new(expected, predicted)
.and_then(|matrix| matrix.f1(Average::Macro))
.map_err(ScoringError::Metric)
}
}
#[test]
fn classifier_scorers_equal_direct_metric_calls_for_concrete_and_erased_models() {
let data = matrix();
let targets = BinaryTargets::new(vec![0, 0, 1, 1]).unwrap();
let concrete = classifier(&targets);
let labels = concrete.predict(&data.as_view()).unwrap();
let probabilities = concrete.predict_class_proba(&data.as_view(), 1).unwrap();
assert_eq!(
score_classifier(
ScorableClassifier::probabilistic(&concrete),
&data.as_view(),
&targets,
ClassificationScorer::Accuracy
),
accuracy_score(targets.as_slice(), &labels).map_err(ScoringError::Metric)
);
assert_eq!(
score_classifier(
ScorableClassifier::probabilistic(&concrete),
&data.as_view(),
&targets,
ClassificationScorer::Brier
),
brier_score(targets.as_slice(), &probabilities).map_err(ScoringError::Metric)
);
let erased: AnyClassifier = concrete.into();
for scorer in [
ClassificationScorer::Accuracy,
ClassificationScorer::Precision,
ClassificationScorer::Recall,
ClassificationScorer::F1,
ClassificationScorer::Brier,
ClassificationScorer::LogLoss,
ClassificationScorer::RocAuc,
] {
assert!(
score_classifier(
ScorableClassifier::probabilistic(
erased
.as_probabilistic()
.expect("every shipped variant produces probabilities")
),
&data.as_view(),
&targets,
scorer
)
.is_ok()
);
}
}
#[test]
fn singleton_class_probability_scores_are_explicit() {
let data = matrix();
for (label, expected_brier) in [(0, 0.0), (1, 0.0)] {
let targets = BinaryTargets::new(vec![label; 4]).unwrap();
let model = classifier(&targets);
assert_eq!(
score_classifier(
ScorableClassifier::probabilistic(&model),
&data.as_view(),
&targets,
ClassificationScorer::Brier
),
Ok(expected_brier)
);
assert_eq!(
score_classifier(
ScorableClassifier::probabilistic(&model),
&data.as_view(),
&targets,
ClassificationScorer::RocAuc
),
Err(ScoringError::Metric(MetricError::Undefined))
);
}
}
#[test]
fn regressor_scorers_equal_direct_metrics_and_runtime_dispatch() {
let data = matrix();
let targets = RegressionTargets::new(vec![0.0, 1.0, 4.0, 9.0]).unwrap();
let concrete = RandomForestRegressor::fit(
&data.as_view(),
&targets,
RandomForestRegressorParams::default()
.with_n_estimators(1)
.with_bootstrap(false),
)
.unwrap();
let erased: AnyRegressor = concrete.into();
for scorer in [
RegressionScorer::MeanAbsoluteError,
RegressionScorer::MeanSquaredError,
RegressionScorer::RootMeanSquaredError,
RegressionScorer::R2,
] {
assert!(score_regressor(&erased, &data.as_view(), &targets, scorer).is_ok());
}
}
#[test]
fn scoring_validates_targets_before_prediction_and_preserves_metric_errors() {
let data = matrix();
let targets = BinaryTargets::new(vec![0, 1]).unwrap();
let fitted_targets = BinaryTargets::new(vec![0, 0, 1, 1]).unwrap();
let model = classifier(&fitted_targets);
assert_eq!(
score_classifier(
ScorableClassifier::probabilistic(&model),
&data.as_view(),
&targets,
ClassificationScorer::Accuracy
),
Err(ScoringError::TargetLength {
rows: 4,
targets: 2,
})
);
let all_negative = BinaryTargets::new(vec![0; 4]).unwrap();
assert_eq!(
score_classifier(
ScorableClassifier::probabilistic(&model),
&data.as_view(),
&all_negative,
ClassificationScorer::Recall
),
Err(ScoringError::Metric(MetricError::Undefined))
);
}
#[test]
fn the_workspace_form_agrees_with_the_allocating_one_and_can_be_reused() {
let data = matrix();
let binary = BinaryTargets::new(vec![0, 0, 1, 1]).unwrap();
let model = classifier(&binary);
let mut workspace = ScoringWorkspace::new();
for scorer in [
ClassificationScorer::Accuracy,
ClassificationScorer::F1,
ClassificationScorer::Brier,
ClassificationScorer::RocAuc,
] {
let allocating = score_classifier(
ScorableClassifier::probabilistic(&model),
&data.as_view(),
&binary,
scorer,
);
let reused = score_classifier_with(
ScorableClassifier::probabilistic(&model),
&data.as_view(),
&binary,
scorer,
&mut workspace,
);
assert_eq!(allocating, reused, "{scorer:?}");
assert_eq!(
reused,
score_classifier_with(
ScorableClassifier::probabilistic(&model),
&data.as_view(),
&binary,
scorer,
&mut workspace
)
);
}
let regression = RegressionTargets::new(vec![0.0, 1.0, 4.0, 9.0]).unwrap();
let regressor = RandomForestRegressor::fit(
&data.as_view(),
®ression,
RandomForestRegressorParams::default()
.with_n_estimators(1)
.with_bootstrap(false),
)
.unwrap();
for scorer in [RegressionScorer::MeanSquaredError, RegressionScorer::R2] {
assert_eq!(
score_regressor(®ressor, &data.as_view(), ®ression, scorer),
score_regressor_with(
®ressor,
&data.as_view(),
®ression,
scorer,
&mut workspace
)
);
}
}
#[test]
fn a_caller_defined_score_is_consumed_exactly_like_a_built_in_one() {
let data = matrix();
let binary = BinaryTargets::new(vec![0, 0, 1, 1]).unwrap();
let model = classifier(&binary);
let labels = model.predict(&data.as_view()).unwrap();
let expected = ConfusionMatrix::new(binary.as_slice(), &labels)
.unwrap()
.f1(Average::Macro)
.unwrap();
assert_eq!(
score_classifier(
ScorableClassifier::probabilistic(&model),
&data.as_view(),
&binary,
MacroF1
),
Ok(expected)
);
let scorer = MacroF1;
assert_eq!(
score_classifier_with(
ScorableClassifier::probabilistic(&model),
&data.as_view(),
&binary,
&scorer,
&mut ScoringWorkspace::new()
),
Ok(expected)
);
}
#[test]
fn orientation_is_declared_by_the_score_itself() {
for scorer in [
ClassificationScorer::Accuracy,
ClassificationScorer::Precision,
ClassificationScorer::Recall,
ClassificationScorer::F1,
ClassificationScorer::RocAuc,
] {
assert!(scorer.greater_is_better(), "{scorer:?}");
}
for scorer in [ClassificationScorer::Brier, ClassificationScorer::LogLoss] {
assert!(!scorer.greater_is_better(), "{scorer:?}");
}
assert!(RegressionScorer::R2.greater_is_better());
for scorer in [
RegressionScorer::MeanAbsoluteError,
RegressionScorer::MeanSquaredError,
RegressionScorer::RootMeanSquaredError,
] {
assert!(!scorer.greater_is_better(), "{scorer:?}");
}
}
#[test]
fn a_score_given_the_wrong_output_reports_it_instead_of_guessing() {
let labels = [0_u8, 1];
let probabilities = [0.2_f32, 0.9];
assert_eq!(
ClassificationScorer::Accuracy.score(
&labels,
ClassifierOutput::PositiveProbabilities(&probabilities)
),
Err(ScoringError::UnsupportedOutput {
required: ClassifierOutputKind::Labels,
supplied: ClassifierOutputKind::PositiveProbabilities,
})
);
assert_eq!(
ClassificationScorer::RocAuc.score(&labels, ClassifierOutput::Labels(&labels)),
Err(ScoringError::UnsupportedOutput {
required: ClassifierOutputKind::PositiveProbabilities,
supplied: ClassifierOutputKind::Labels,
})
);
assert_eq!(
ClassifierOutput::Labels(&labels).kind(),
ClassifierOutputKind::Labels
);
let classes = [0_u8, 1];
assert_eq!(
ClassificationScorer::MulticlassLogLoss
.score(&labels, ClassifierOutput::Labels(&labels)),
Err(ScoringError::UnsupportedOutput {
required: ClassifierOutputKind::ProbabilityMatrix,
supplied: ClassifierOutputKind::Labels,
})
);
assert_eq!(
ClassificationScorer::Brier.score(
&labels,
ClassifierOutput::ProbabilityMatrix {
classes: &classes,
probabilities: &[0.8, 0.2, 0.1, 0.9],
}
),
Err(ScoringError::UnsupportedOutput {
required: ClassifierOutputKind::PositiveProbabilities,
supplied: ClassifierOutputKind::ProbabilityMatrix,
})
);
}
fn multiclass_forest(targets: &ClassTargets) -> RandomForestClassifier {
RandomForestClassifier::fit_multiclass(
&matrix().as_view(),
targets,
RandomForestClassifierParams::default()
.with_n_estimators(3)
.with_bootstrap(false)
.with_max_features(MaxFeatures::All),
)
.unwrap()
}
#[test]
fn probability_matrix_scores_read_every_column_for_any_class_set() {
let data = matrix();
let targets = ClassTargets::new(vec![3, 3, 7, 10]).unwrap();
let model = multiclass_forest(&targets);
let probabilities = model.predict_proba(&data.as_view()).unwrap();
assert_eq!(model.classes(), &[3, 7, 10]);
for (scorer, expected) in [
(
ClassificationScorer::MulticlassLogLoss,
multiclass_log_loss(targets.as_slice(), model.classes(), &probabilities),
),
(
ClassificationScorer::MulticlassBrier,
multiclass_brier_score(targets.as_slice(), model.classes(), &probabilities),
),
] {
assert_eq!(
score_multiclass_classifier(
ScorableClassifier::probabilistic(&model),
&data.as_view(),
&targets,
scorer
),
expected.map_err(ScoringError::Metric),
"{scorer:?}"
);
}
assert_eq!(
score_multiclass_classifier(
ScorableClassifier::probabilistic(&model),
&data.as_view(),
&targets,
ClassificationScorer::Brier
),
Err(ScoringError::UnsupportedClasses)
);
}
#[test]
fn probability_matrix_scores_reach_binary_targets_through_the_same_path() {
let data = matrix();
let binary = BinaryTargets::new(vec![0, 0, 1, 1]).unwrap();
let model = classifier(&binary);
let probabilities = model.predict_proba(&data.as_view()).unwrap();
let positive = model.predict_class_proba(&data.as_view(), 1).unwrap();
let multiclass = score_classifier(
ScorableClassifier::probabilistic(&model),
&data.as_view(),
&binary,
ClassificationScorer::MulticlassLogLoss,
)
.unwrap();
let binary_log_loss = score_classifier(
ScorableClassifier::probabilistic(&model),
&data.as_view(),
&binary,
ClassificationScorer::LogLoss,
)
.unwrap();
assert!((multiclass - binary_log_loss).abs() <= 1.0e-12);
assert_eq!(
multiclass,
multiclass_log_loss(binary.as_slice(), model.classes(), &probabilities).unwrap()
);
let multiclass_brier = score_classifier(
ScorableClassifier::probabilistic(&model),
&data.as_view(),
&binary,
ClassificationScorer::MulticlassBrier,
)
.unwrap();
let binary_brier = brier_score(binary.as_slice(), &positive).unwrap();
assert!((multiclass_brier - 2.0 * binary_brier).abs() <= 1.0e-12);
}
#[test]
fn the_probability_matrix_workspace_is_reused_and_agrees_with_the_allocating_form() {
let data = matrix();
let targets = ClassTargets::new(vec![3, 3, 7, 10]).unwrap();
let model = multiclass_forest(&targets);
let mut workspace = ScoringWorkspace::new();
for scorer in [
ClassificationScorer::MulticlassLogLoss,
ClassificationScorer::MulticlassBrier,
] {
let allocating = score_multiclass_classifier(
ScorableClassifier::probabilistic(&model),
&data.as_view(),
&targets,
scorer,
);
let reused = score_multiclass_classifier_with(
ScorableClassifier::probabilistic(&model),
&data.as_view(),
&targets,
scorer,
&mut workspace,
);
assert_eq!(allocating, reused, "{scorer:?}");
assert_eq!(
reused,
score_multiclass_classifier_with(
ScorableClassifier::probabilistic(&model),
&data.as_view(),
&targets,
scorer,
&mut workspace
)
);
}
assert_eq!(
score_multiclass_classifier_with(
ScorableClassifier::probabilistic(&model),
&data.as_view(),
&targets,
ClassificationScorer::Accuracy,
&mut workspace
),
score_multiclass_classifier(
ScorableClassifier::probabilistic(&model),
&data.as_view(),
&targets,
ClassificationScorer::Accuracy
)
);
}
#[test]
fn multiclass_scoring_validates_targets_before_prediction() {
let data = matrix();
let fitted = ClassTargets::new(vec![3, 3, 7, 10]).unwrap();
let model = multiclass_forest(&fitted);
let short = ClassTargets::new(vec![3, 7]).unwrap();
assert_eq!(
score_multiclass_classifier(
ScorableClassifier::probabilistic(&model),
&data.as_view(),
&short,
ClassificationScorer::MulticlassLogLoss
),
Err(ScoringError::TargetLength {
rows: 4,
targets: 2,
})
);
let unknown = ClassTargets::new(vec![3, 3, 7, 11]).unwrap();
assert_eq!(
score_multiclass_classifier(
ScorableClassifier::probabilistic(&model),
&data.as_view(),
&unknown,
ClassificationScorer::MulticlassLogLoss
),
Err(ScoringError::Metric(MetricError::UnknownClass {
index: 3,
value: 11,
}))
);
}
}