use crate::api::{
Capabilities, Classifier, Estimator, HasCapabilities, ModelError, ProbabilisticClassifier,
};
use crate::data::{BinaryTargets, MatrixView};
use super::{
Calibrator, IsotonicRegression, IsotonicRegressionParams, PlattCalibrator, PlattParams,
};
#[derive(Clone, Debug, PartialEq)]
pub struct CalibratedClassifier<C, K> {
inner: C,
calibrator: K,
}
impl<C: ProbabilisticClassifier, K: Calibrator> CalibratedClassifier<C, K> {
pub fn new(inner: C, calibrator: K) -> Result<Self, ModelError> {
validate_binary_inner(&inner)?;
Ok(Self { inner, calibrator })
}
pub const fn inner(&self) -> &C {
&self.inner
}
pub const fn calibrator(&self) -> &K {
&self.calibrator
}
pub fn into_parts(self) -> (C, K) {
(self.inner, self.calibrator)
}
pub fn predict_into_with(
&self,
data: &MatrixView<'_>,
scores: &mut [f32],
output: &mut [u8],
) -> Result<(), ModelError> {
if output.len() != data.rows() {
return Err(ModelError::OutputLength {
expected: data.rows(),
actual: output.len(),
});
}
self.positive_probabilities_into(data, scores)?;
for (label, &probability) in output.iter_mut().zip(scores.iter()) {
*label = u8::from(probability > 1.0 - probability);
}
Ok(())
}
fn check_batch_width(&self, data: &MatrixView<'_>) -> Result<(), ModelError> {
let expected = self.inner.n_features_in();
if data.columns() != expected {
return Err(ModelError::FeatureDimension {
expected,
actual: data.columns(),
});
}
Ok(())
}
fn positive_probabilities_into(
&self,
data: &MatrixView<'_>,
output: &mut [f32],
) -> Result<(), ModelError> {
self.inner.predict_class_proba_into(data, 1, output)?;
self.calibrator.calibrate_in_place(output);
for slot in output.iter_mut() {
*slot = slot.clamp(0.0, 1.0);
}
Ok(())
}
}
impl<C: ProbabilisticClassifier> CalibratedClassifier<C, IsotonicRegression> {
pub fn fit_isotonic(
inner: C,
data: &MatrixView<'_>,
targets: &BinaryTargets,
params: IsotonicRegressionParams,
) -> Result<Self, ModelError> {
let scores = calibration_scores(&inner, data, targets)?;
let calibrator = IsotonicRegression::fit_calibration(&scores, targets, params)?;
Ok(Self { inner, calibrator })
}
}
impl<C: ProbabilisticClassifier> CalibratedClassifier<C, PlattCalibrator> {
pub fn fit_platt(
inner: C,
data: &MatrixView<'_>,
targets: &BinaryTargets,
params: PlattParams,
) -> Result<Self, ModelError> {
let scores = calibration_scores(&inner, data, targets)?;
let calibrator = PlattCalibrator::fit(&scores, targets, params)?;
Ok(Self { inner, calibrator })
}
pub fn decision_function_into(
&self,
data: &MatrixView<'_>,
output: &mut [f32],
) -> Result<(), ModelError> {
self.inner.predict_class_proba_into(data, 1, output)?;
for slot in output.iter_mut() {
*slot = self.calibrator.decision_score(*slot);
}
Ok(())
}
pub fn decision_function(&self, data: &MatrixView<'_>) -> Result<Vec<f32>, ModelError> {
self.check_batch_width(data)?;
let mut output = vec![0.0; data.rows()];
self.decision_function_into(data, &mut output)?;
Ok(output)
}
}
fn calibration_scores<C: ProbabilisticClassifier>(
inner: &C,
data: &MatrixView<'_>,
targets: &BinaryTargets,
) -> Result<Vec<f32>, ModelError> {
validate_binary_inner(inner)?;
if data.rows() == 0 || data.columns() == 0 {
return Err(ModelError::EmptyData);
}
if data.columns() != inner.n_features_in() {
return Err(ModelError::FeatureDimension {
expected: inner.n_features_in(),
actual: data.columns(),
});
}
if data.rows() != targets.len() {
return Err(ModelError::TargetLength {
rows: data.rows(),
targets: targets.len(),
});
}
let mut scores = vec![0.0; data.rows()];
inner.predict_class_proba_into(data, 1, &mut scores)?;
Ok(scores)
}
fn validate_binary_inner<C: Classifier>(inner: &C) -> Result<(), ModelError> {
match inner.classes() {
[0, 1] => Ok(()),
classes if classes.len() < 2 => Err(ModelError::RequiresTwoClasses),
classes if classes.len() > 2 => Err(ModelError::MulticlassOutput {
columns: classes.len(),
}),
_ => Err(ModelError::UnknownClass { class: 1 }),
}
}
impl<C: ProbabilisticClassifier, K: Calibrator> Estimator for CalibratedClassifier<C, K> {
fn n_features_in(&self) -> usize {
self.inner.n_features_in()
}
}
impl<C: ProbabilisticClassifier> HasCapabilities for CalibratedClassifier<C, IsotonicRegression> {
const CAPABILITIES: Capabilities = Capabilities::NONE.with_probability(true);
}
impl<C: ProbabilisticClassifier> HasCapabilities for CalibratedClassifier<C, PlattCalibrator> {
const CAPABILITIES: Capabilities = Capabilities::NONE
.with_decision_function(true)
.with_probability(true);
}
impl<C: ProbabilisticClassifier, K: Calibrator> Classifier for CalibratedClassifier<C, K> {
fn classes(&self) -> &[u8] {
self.inner.classes()
}
fn predict_into(&self, data: &MatrixView<'_>, output: &mut [u8]) -> Result<(), ModelError> {
if output.len() != data.rows() {
return Err(ModelError::OutputLength {
expected: data.rows(),
actual: output.len(),
});
}
self.check_batch_width(data)?;
let mut scores = vec![0.0; data.rows()];
self.predict_into_with(data, &mut scores, output)
}
}
impl<C: ProbabilisticClassifier, K: Calibrator> ProbabilisticClassifier
for CalibratedClassifier<C, K>
{
fn predict_proba_into(
&self,
data: &MatrixView<'_>,
output: &mut [f32],
) -> Result<(), ModelError> {
let rows = data.rows();
let expected = rows
.checked_mul(2)
.ok_or(ModelError::OutputShapeOverflow { rows, columns: 2 })?;
if output.len() != expected {
return Err(ModelError::OutputLength {
expected,
actual: output.len(),
});
}
self.positive_probabilities_into(data, &mut output[rows..])?;
for index in 0..rows {
let positive = output[rows + index];
output[2 * index] = 1.0 - positive;
output[2 * index + 1] = positive;
}
Ok(())
}
fn predict_class_proba_into(
&self,
data: &MatrixView<'_>,
class: u8,
output: &mut [f32],
) -> Result<(), ModelError> {
self.check_batch_width(data)?;
if class > 1 {
return Err(ModelError::UnknownClass { class });
}
self.positive_probabilities_into(data, output)?;
if class == 0 {
for slot in output.iter_mut() {
*slot = 1.0 - *slot;
}
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::data::{DenseMatrix, RegressionTargets};
use crate::dummy::{DummyClassifier, DummyClassifierParams};
#[derive(Clone, Debug, PartialEq)]
struct Overconfident {
classes: Vec<u8>,
}
impl Overconfident {
fn binary() -> Self {
Self {
classes: vec![0, 1],
}
}
fn probability(row: &[f32]) -> f32 {
(row[0] * 4.0 - 2.0).clamp(0.0, 1.0)
}
}
impl Estimator for Overconfident {
fn n_features_in(&self) -> usize {
1
}
}
impl Classifier for Overconfident {
fn classes(&self) -> &[u8] {
&self.classes
}
fn predict_into(&self, data: &MatrixView<'_>, output: &mut [u8]) -> Result<(), ModelError> {
self.validate(data, output.len())?;
for (slot, row) in output.iter_mut().zip(data.iter_rows()) {
*slot = u8::from(Self::probability(row) > 0.5);
}
Ok(())
}
}
impl ProbabilisticClassifier for Overconfident {
fn predict_proba_into(
&self,
data: &MatrixView<'_>,
output: &mut [f32],
) -> Result<(), ModelError> {
self.validate(data, output.len() / 2)?;
if output.len() != data.rows() * 2 {
return Err(ModelError::OutputLength {
expected: data.rows() * 2,
actual: output.len(),
});
}
for (slots, row) in output.chunks_exact_mut(2).zip(data.iter_rows()) {
let positive = Self::probability(row);
slots[0] = 1.0 - positive;
slots[1] = positive;
}
Ok(())
}
fn predict_class_proba_into(
&self,
data: &MatrixView<'_>,
class: u8,
output: &mut [f32],
) -> Result<(), ModelError> {
self.validate(data, output.len())?;
if !self.classes.contains(&class) {
return Err(ModelError::UnknownClass { class });
}
for (slot, row) in output.iter_mut().zip(data.iter_rows()) {
let positive = Self::probability(row);
*slot = if class == 1 { positive } else { 1.0 - positive };
}
Ok(())
}
}
impl Overconfident {
fn validate(&self, data: &MatrixView<'_>, rows: usize) -> Result<(), ModelError> {
if data.columns() != self.n_features_in() {
return Err(ModelError::FeatureDimension {
expected: self.n_features_in(),
actual: data.columns(),
});
}
if rows != data.rows() {
return Err(ModelError::OutputLength {
expected: data.rows(),
actual: rows,
});
}
Ok(())
}
}
fn sample() -> (DenseMatrix, BinaryTargets) {
let values: Vec<f32> = (0..24).map(|step| step as f32 / 23.0).collect();
let labels: Vec<u8> = (0..24).map(|step| u8::from(step % 3 != 0)).collect();
(
DenseMatrix::new(values, 24, 1).unwrap(),
BinaryTargets::new(labels).unwrap(),
)
}
#[test]
fn calibrated_probabilities_form_valid_rows_and_agree_with_their_columns() {
let (data, labels) = sample();
let model = CalibratedClassifier::fit_isotonic(
Overconfident::binary(),
&data.as_view(),
&labels,
IsotonicRegressionParams,
)
.unwrap();
assert_eq!(model.classes(), &[0, 1]);
assert_eq!(model.n_features_in(), 1);
let matrix = model.predict_proba(&data.as_view()).unwrap();
let positive = model.predict_class_proba(&data.as_view(), 1).unwrap();
let negative = model.predict_class_proba(&data.as_view(), 0).unwrap();
let predicted = model.predict(&data.as_view()).unwrap();
assert_eq!(matrix.len(), data.rows() * 2);
for (index, row) in matrix.chunks_exact(2).enumerate() {
assert_eq!(row[1], positive[index]);
assert_eq!(row[0], negative[index]);
assert!((row[0] + row[1] - 1.0).abs() <= 1.0e-6, "row {index}");
assert!((0.0..=1.0).contains(&row[1]));
let expected = u8::from(row[1] > row[0]);
assert_eq!(predicted[index], expected, "row {index}");
}
}
#[test]
fn the_allocation_free_paths_agree_with_the_allocating_ones() {
let (data, labels) = sample();
let model = CalibratedClassifier::fit_platt(
Overconfident::binary(),
&data.as_view(),
&labels,
PlattParams::default(),
)
.unwrap();
let view = data.as_view();
let mut matrix = vec![f32::MAX; data.rows() * 2];
model.predict_proba_into(&view, &mut matrix).unwrap();
assert_eq!(matrix, model.predict_proba(&view).unwrap());
let mut labels_into = vec![u8::MAX; data.rows()];
let mut scores = vec![f32::MAX; data.rows()];
model
.predict_into_with(&view, &mut scores, &mut labels_into)
.unwrap();
assert_eq!(labels_into, model.predict(&view).unwrap());
assert_eq!(scores, model.predict_class_proba(&view, 1).unwrap());
let mut decisions = vec![f32::MAX; data.rows()];
model.decision_function_into(&view, &mut decisions).unwrap();
assert_eq!(decisions, model.decision_function(&view).unwrap());
for (&decision, &probability) in decisions.iter().zip(&scores) {
assert_eq!(crate::numeric::sigmoid_f32(decision), probability);
}
}
#[test]
fn every_prediction_path_validates_before_writing() {
let (data, labels) = sample();
let model = CalibratedClassifier::fit_isotonic(
Overconfident::binary(),
&data.as_view(),
&labels,
IsotonicRegressionParams,
)
.unwrap();
let wide = DenseMatrix::new(vec![0.5; data.rows() * 2], data.rows(), 2).unwrap();
let mut sentinel = vec![f32::MAX; data.rows() * 2];
assert_eq!(
model.predict_proba_into(&wide.as_view(), &mut sentinel),
Err(ModelError::FeatureDimension {
expected: 1,
actual: 2,
})
);
assert!(sentinel.iter().all(|value| *value == f32::MAX));
assert_eq!(
model.predict_proba_into(&data.as_view(), &mut sentinel[..3]),
Err(ModelError::OutputLength {
expected: 48,
actual: 3,
})
);
assert!(sentinel.iter().all(|value| *value == f32::MAX));
let mut labels_sentinel = vec![u8::MAX; data.rows()];
assert_eq!(
model.predict_into(&data.as_view(), &mut labels_sentinel[..2]),
Err(ModelError::OutputLength {
expected: 24,
actual: 2,
})
);
assert!(labels_sentinel.iter().all(|value| *value == u8::MAX));
assert_eq!(
model.predict_class_proba_into(&data.as_view(), 2, &mut sentinel[..data.rows()]),
Err(ModelError::UnknownClass { class: 2 })
);
assert!(sentinel.iter().all(|value| *value == f32::MAX));
}
#[test]
fn a_wrapped_model_the_map_cannot_address_is_rejected_at_construction() {
let (data, labels) = sample();
let single = DummyClassifier::fit(
&data.as_view(),
&BinaryTargets::new(vec![1; data.rows()]).unwrap(),
DummyClassifierParams,
)
.unwrap();
assert_eq!(single.classes(), &[1]);
assert_eq!(
CalibratedClassifier::fit_isotonic(
single,
&data.as_view(),
&labels,
IsotonicRegressionParams
)
.unwrap_err(),
ModelError::RequiresTwoClasses
);
let wide = Overconfident {
classes: vec![0, 1, 2],
};
assert_eq!(
CalibratedClassifier::fit_isotonic(
wide,
&data.as_view(),
&labels,
IsotonicRegressionParams
)
.unwrap_err(),
ModelError::MulticlassOutput { columns: 3 }
);
let relabelled = Overconfident {
classes: vec![3, 7],
};
assert_eq!(
CalibratedClassifier::fit_isotonic(
relabelled,
&data.as_view(),
&labels,
IsotonicRegressionParams
)
.unwrap_err(),
ModelError::UnknownClass { class: 1 }
);
}
#[test]
fn fitting_validates_the_calibration_sample_before_predicting_on_it() {
let (data, labels) = sample();
let wide = DenseMatrix::new(vec![0.5; data.rows() * 2], data.rows(), 2).unwrap();
assert_eq!(
CalibratedClassifier::fit_isotonic(
Overconfident::binary(),
&wide.as_view(),
&labels,
IsotonicRegressionParams
)
.unwrap_err(),
ModelError::FeatureDimension {
expected: 1,
actual: 2,
}
);
assert_eq!(
CalibratedClassifier::fit_isotonic(
Overconfident::binary(),
&data.as_view(),
&BinaryTargets::new(vec![0, 1]).unwrap(),
IsotonicRegressionParams,
)
.unwrap_err(),
ModelError::TargetLength {
rows: 24,
targets: 2,
}
);
assert_eq!(
CalibratedClassifier::fit_isotonic(
Overconfident::binary(),
&data.as_view(),
&BinaryTargets::new(vec![0; data.rows()]).unwrap(),
IsotonicRegressionParams,
)
.unwrap_err(),
ModelError::RequiresTwoClasses
);
}
#[test]
fn refitting_the_same_calibration_sample_reproduces_the_same_model() {
let (data, labels) = sample();
for _ in 0..2 {
assert_eq!(
CalibratedClassifier::fit_isotonic(
Overconfident::binary(),
&data.as_view(),
&labels,
IsotonicRegressionParams,
)
.unwrap(),
CalibratedClassifier::fit_isotonic(
Overconfident::binary(),
&data.as_view(),
&labels,
IsotonicRegressionParams,
)
.unwrap()
);
assert_eq!(
CalibratedClassifier::fit_platt(
Overconfident::binary(),
&data.as_view(),
&labels,
PlattParams::default()
)
.unwrap(),
CalibratedClassifier::fit_platt(
Overconfident::binary(),
&data.as_view(),
&labels,
PlattParams::default()
)
.unwrap()
);
}
}
#[test]
fn a_calibrator_that_is_not_a_probability_map_is_saturated_not_forwarded() {
let (data, _) = sample();
let knots = DenseMatrix::new(vec![0.0_f32, 0.25, 0.5, 0.75, 1.0], 5, 1).unwrap();
let unbounded = RegressionTargets::new(vec![-40.0_f32, -12.0, 0.5, 17.0, 49.0]).unwrap();
let calibrator =
IsotonicRegression::fit(&knots.as_view(), &unbounded, IsotonicRegressionParams)
.unwrap();
assert!(calibrator.map(0.0) < 0.0 && calibrator.map(1.0) > 1.0);
let model = CalibratedClassifier::new(Overconfident::binary(), calibrator).unwrap();
let rows = data.rows();
let probabilities = model.predict_proba(&data.as_view()).unwrap();
assert_eq!(probabilities.len(), rows * 2);
for (row, slots) in probabilities.chunks_exact(2).enumerate() {
for (column, &value) in slots.iter().enumerate() {
assert!(
(0.0..=1.0).contains(&value),
"row {row} column {column} is {value}, which is not a probability"
);
}
assert!((slots.iter().sum::<f32>() - 1.0).abs() <= 2.0 * f32::EPSILON);
}
for class in [0_u8, 1] {
let column = model.predict_class_proba(&data.as_view(), class).unwrap();
for (row, &value) in column.iter().enumerate() {
assert!(
(0.0..=1.0).contains(&value),
"class {class} row {row} is {value}"
);
}
for (row, &value) in column.iter().enumerate() {
assert_eq!(value, probabilities[2 * row + usize::from(class)]);
}
}
let labels = model.predict(&data.as_view()).unwrap();
for (row, &label) in labels.iter().enumerate() {
let positive = probabilities[2 * row + 1];
assert_eq!(label, u8::from(positive > 1.0 - positive), "row {row}");
}
}
#[test]
fn saturation_does_not_move_a_calibrator_that_was_already_in_range() {
let (data, labels) = sample();
let isotonic = CalibratedClassifier::fit_isotonic(
Overconfident::binary(),
&data.as_view(),
&labels,
IsotonicRegressionParams,
)
.unwrap();
let platt = CalibratedClassifier::fit_platt(
Overconfident::binary(),
&data.as_view(),
&labels,
PlattParams::default(),
)
.unwrap();
for (name, positives) in [
(
"isotonic",
isotonic.predict_class_proba(&data.as_view(), 1).unwrap(),
),
(
"platt",
platt.predict_class_proba(&data.as_view(), 1).unwrap(),
),
] {
let mut raw = vec![0.0_f32; data.rows()];
Overconfident::binary()
.predict_class_proba_into(&data.as_view(), 1, &mut raw)
.unwrap();
for (row, &value) in positives.iter().enumerate() {
let uncalibrated = if name == "isotonic" {
isotonic.calibrator.calibrate(raw[row])
} else {
platt.calibrator.calibrate(raw[row])
};
assert_eq!(
value.to_bits(),
uncalibrated.to_bits(),
"{name} row {row}: saturation moved a value that was in range"
);
}
}
}
#[test]
fn the_composition_declares_only_what_it_really_offers() {
assert_eq!(
<CalibratedClassifier<Overconfident, IsotonicRegression> as HasCapabilities>::CAPABILITIES,
Capabilities::NONE.with_probability(true)
);
assert_eq!(
<CalibratedClassifier<Overconfident, PlattCalibrator> as HasCapabilities>::CAPABILITIES,
Capabilities::NONE
.with_decision_function(true)
.with_probability(true)
);
}
}