use crate::api::{Capabilities, Estimator, HasCapabilities, HasParams, ModelError, Transformer};
use crate::artifact::{ArtifactError, ROBUST_SCALER_ARTIFACT_KIND, StageArtifact};
use crate::data::MatrixView;
use crate::numeric::{QuantileRule, quantile_sorted, sort_for_quantiles};
use super::scaling::{
BASE_PAYLOAD_VERSION, ScalerHeader, ScalerParameters, decode_flag, decode_scaler_artifact,
encode_scaler_artifact, inverse_transform_allocating, substituted_divisor,
transform_preflighted, validate_inverse_request, validate_transform_request,
};
const RULE: QuantileRule = QuantileRule::Linear;
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct RobustScalerParams {
quantile_low: f64,
quantile_high: f64,
with_centering: bool,
with_scaling: bool,
}
impl Default for RobustScalerParams {
fn default() -> Self {
Self {
quantile_low: 25.0,
quantile_high: 75.0,
with_centering: true,
with_scaling: true,
}
}
}
impl RobustScalerParams {
#[must_use]
pub const fn with_quantile_range(mut self, low: f64, high: f64) -> Self {
self.quantile_low = low;
self.quantile_high = high;
self
}
#[must_use]
pub const fn with_centering(mut self, with_centering: bool) -> Self {
self.with_centering = with_centering;
self
}
#[must_use]
pub const fn with_scaling(mut self, with_scaling: bool) -> Self {
self.with_scaling = with_scaling;
self
}
#[must_use]
pub const fn quantile_range(&self) -> (f64, f64) {
(self.quantile_low, self.quantile_high)
}
#[must_use]
pub const fn centering_enabled(&self) -> bool {
self.with_centering
}
#[must_use]
pub const fn scaling_enabled(&self) -> bool {
self.with_scaling
}
fn validate(&self) -> Result<(), ModelError> {
let (low, high) = self.quantile_range();
if !(low.is_finite() && high.is_finite())
|| !(0.0..=100.0).contains(&low)
|| !(0.0..=100.0).contains(&high)
|| low > high
{
return Err(ModelError::InvalidQuantileRange);
}
Ok(())
}
}
#[derive(Clone, Debug, PartialEq)]
pub struct RobustScaler {
n_features_in: usize,
params: RobustScalerParams,
centers: Vec<f64>,
spreads: Vec<f64>,
scales: Vec<f64>,
}
impl RobustScaler {
pub fn fit(data: &MatrixView<'_>, params: RobustScalerParams) -> Result<Self, ModelError> {
params.validate()?;
let (low, high) = params.quantile_range();
let columns = data.columns();
let mut column = vec![0.0_f64; data.rows()];
let mut centers = Vec::with_capacity(columns);
let mut spreads = Vec::with_capacity(columns);
let mut scales = Vec::with_capacity(columns);
for index in 0..columns {
for (slot, row) in column.iter_mut().zip(data.iter_rows()) {
*slot = f64::from(row[index]);
}
sort_for_quantiles(&mut column);
centers.push(quantile_sorted(&column, 50.0, RULE));
let spread = quantile_sorted(&column, high, RULE) - quantile_sorted(&column, low, RULE);
spreads.push(spread);
scales.push(substituted_divisor(spread));
}
Ok(Self {
n_features_in: columns,
params,
centers,
spreads,
scales,
})
}
#[must_use]
pub fn centers(&self) -> &[f64] {
&self.centers
}
#[must_use]
pub fn spreads(&self) -> &[f64] {
&self.spreads
}
#[must_use]
pub fn scales(&self) -> &[f64] {
&self.scales
}
#[must_use]
pub const fn n_features_in(&self) -> usize {
self.n_features_in
}
#[must_use]
pub const fn n_features_out(&self) -> usize {
self.n_features_in
}
#[must_use]
pub const fn get_params(&self) -> &RobustScalerParams {
&self.params
}
pub fn transform_into<'output>(
&self,
data: &MatrixView<'_>,
output: &'output mut [f32],
) -> Result<MatrixView<'output>, ModelError> {
<Self as Transformer>::transform_into(self, data, output)
}
pub fn transform(&self, data: &MatrixView<'_>) -> Result<crate::data::DenseMatrix, ModelError> {
<Self as Transformer>::transform(self, data)
}
pub fn inverse_transform_into<'output>(
&self,
data: &MatrixView<'_>,
output: &'output mut [f32],
) -> Result<MatrixView<'output>, ModelError> {
validate_inverse_request(self.n_features_in, data, output)?;
match (self.params.with_centering, self.params.with_scaling) {
(false, false) => transform_preflighted(data, output, |value, _| value),
(true, false) => transform_preflighted(data, output, |value, column| {
(f64::from(value) + self.centers[column]) as f32
}),
(false, true) => transform_preflighted(data, output, |value, column| {
(f64::from(value) * self.scales[column]) as f32
}),
(true, true) => transform_preflighted(data, output, |value, column| {
(f64::from(value) * self.scales[column] + self.centers[column]) as f32
}),
}
}
pub fn inverse_transform(
&self,
data: &MatrixView<'_>,
) -> Result<crate::data::DenseMatrix, ModelError> {
inverse_transform_allocating(self.n_features_in, data, |batch, output| {
self.inverse_transform_into(batch, output).map(|_| ())
})
}
}
impl StageArtifact for RobustScaler {
const ARTIFACT_KIND: u16 = ROBUST_SCALER_ARTIFACT_KIND;
fn to_artifact(
&self,
input_schema: [u8; 32],
transformed_schema: [u8; 32],
) -> Result<Vec<u8>, ArtifactError> {
let (low, high) = self.params.quantile_range();
encode_scaler_artifact(
Self::ARTIFACT_KIND,
input_schema,
transformed_schema,
self.n_features_in,
ScalerParameters {
version: BASE_PAYLOAD_VERSION,
flags: &[
u32::from(self.params.with_centering),
u32::from(self.params.with_scaling),
],
reals: &[low, high],
},
2,
|feature, state| {
state.f64(self.centers[feature]);
state.f64(self.spreads[feature]);
},
)
}
fn from_artifact(
bytes: &[u8],
input_schema: [u8; 32],
transformed_schema: [u8; 32],
) -> Result<Self, ArtifactError> {
let ScalerHeader {
n_features_in,
flags,
parameters,
mut state,
} = decode_scaler_artifact(
bytes,
Self::ARTIFACT_KIND,
input_schema,
transformed_schema,
BASE_PAYLOAD_VERSION,
2,
2,
)?;
let params = RobustScalerParams {
quantile_low: parameters[0],
quantile_high: parameters[1],
with_centering: decode_flag(flags[0])?,
with_scaling: decode_flag(flags[1])?,
};
params
.validate()
.map_err(|_| ArtifactError::InvalidPayload)?;
let capacity = state.bounded_capacity(n_features_in, 2 * 8);
let mut centers = Vec::with_capacity(capacity);
let mut spreads = Vec::with_capacity(capacity);
let mut scales = Vec::with_capacity(capacity);
for _ in 0..n_features_in {
let center = state.f64()?;
let spread = state.f64()?;
if !center.is_finite() || !spread.is_finite() || spread < 0.0 {
return Err(ArtifactError::InvalidPayload);
}
centers.push(center);
spreads.push(spread);
scales.push(substituted_divisor(spread));
}
if !state.is_empty() {
return Err(ArtifactError::TrailingBytes);
}
Ok(Self {
n_features_in,
params,
centers,
spreads,
scales,
})
}
}
impl Estimator for RobustScaler {
fn n_features_in(&self) -> usize {
self.n_features_in
}
}
impl HasCapabilities for RobustScaler {
const CAPABILITIES: Capabilities = Capabilities::NONE.with_artifact(true);
}
impl HasParams for RobustScaler {
type Params = RobustScalerParams;
fn get_params(&self) -> &Self::Params {
&self.params
}
}
impl Transformer for RobustScaler {
fn n_features_out(&self) -> usize {
self.n_features_in
}
fn transform_into<'output>(
&self,
data: &MatrixView<'_>,
output: &'output mut [f32],
) -> Result<MatrixView<'output>, ModelError> {
validate_transform_request(self.n_features_in, data, output)?;
match (self.params.with_centering, self.params.with_scaling) {
(false, false) => transform_preflighted(data, output, |value, _| value),
(true, false) => transform_preflighted(data, output, |value, column| {
(f64::from(value) - self.centers[column]) as f32
}),
(false, true) => transform_preflighted(data, output, |value, column| {
(f64::from(value) / self.scales[column]) as f32
}),
(true, true) => transform_preflighted(data, output, |value, column| {
((f64::from(value) - self.centers[column]) / self.scales[column]) as f32
}),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::data::DenseMatrix;
fn matrix(values: &[f32], rows: usize, columns: usize) -> DenseMatrix {
DenseMatrix::new(values.to_vec(), rows, columns).unwrap()
}
fn worked_examples() -> DenseMatrix {
matrix(
&[
0.0, 0.0, -3.0, 1.0, 1.0, 0.5, 1.0, 10.0, 2.0, 1.0, 10.0, 11.0,
],
4,
3,
)
}
fn fitted(data: &DenseMatrix, params: RobustScalerParams) -> RobustScaler {
RobustScaler::fit(&data.as_view(), params).unwrap()
}
#[test]
fn fits_the_median_and_the_interquartile_range_per_column() {
let data = matrix(&[0.0, 1.0, 2.0, 10.0], 4, 1);
let scaler = fitted(&data, RobustScalerParams::default());
assert_eq!(scaler.centers(), &[1.5]);
assert_eq!(scaler.spreads(), &[3.25]);
assert_eq!(scaler.scales(), &[3.25]);
assert_eq!(
scaler.transform(&data.as_view()).unwrap().as_slice(),
&[-0.461_538_46, -0.153_846_16, 0.153_846_16, 2.615_384_6_f32]
);
}
#[test]
fn columns_are_fitted_independently_of_each_other() {
let scaler = fitted(&worked_examples(), RobustScalerParams::default());
assert_eq!(scaler.centers(), &[1.0, 5.5, 1.25]);
assert_eq!(scaler.spreads(), &[0.25, 9.25, 4.625]);
let alone = fitted(&matrix(&[-3.0, 0.5, 2.0, 11.0], 4, 1), Default::default());
assert_eq!(alone.centers(), &[scaler.centers()[2]]);
assert_eq!(alone.spreads(), &[scaler.spreads()[2]]);
}
#[test]
fn the_quantile_range_selects_which_spread_is_removed() {
let data = matrix(&[0.0, 1.0, 2.0, 10.0], 4, 1);
let full = fitted(
&data,
RobustScalerParams::default().with_quantile_range(0.0, 100.0),
);
assert_eq!(full.spreads(), &[10.0], "the full range is the extrema");
let narrow = fitted(
&data,
RobustScalerParams::default().with_quantile_range(40.0, 60.0),
);
assert_eq!(narrow.centers(), &[1.5]);
assert!(narrow.spreads()[0] < full.spreads()[0]);
}
#[test]
fn an_empty_quantile_range_is_accepted_and_leaves_the_column_uncentred_only() {
let data = matrix(&[0.0, 1.0, 2.0, 10.0], 4, 1);
let scaler = fitted(
&data,
RobustScalerParams::default().with_quantile_range(50.0, 50.0),
);
assert_eq!(scaler.spreads(), &[0.0]);
assert_eq!(scaler.scales(), &[1.0]);
assert_eq!(
scaler.transform(&data.as_view()).unwrap().as_slice(),
&[-1.5, -0.5, 0.5, 8.5]
);
}
#[test]
fn an_invalid_quantile_range_is_rejected_before_any_column_work() {
let data = worked_examples();
for (low, high) in [
(75.0, 25.0),
(-1.0, 50.0),
(50.0, 101.0),
(f64::NAN, 50.0),
(0.0, f64::INFINITY),
] {
let params = RobustScalerParams::default().with_quantile_range(low, high);
assert_eq!(
RobustScaler::fit(&data.as_view(), params).unwrap_err(),
ModelError::InvalidQuantileRange,
"range ({low}, {high})"
);
}
assert!(
RobustScaler::fit(
&data.as_view(),
RobustScalerParams::default().with_quantile_range(0.0, 100.0)
)
.is_ok()
);
}
#[test]
fn a_constant_column_keeps_a_divisor_of_one_and_transforms_to_zero() {
let data = matrix(&[3.0, 3.0, 3.0, 3.0], 4, 1);
let scaler = fitted(&data, RobustScalerParams::default());
assert_eq!(scaler.centers(), &[3.0]);
assert_eq!(scaler.spreads(), &[0.0]);
assert_eq!(scaler.scales(), &[1.0]);
assert_eq!(
scaler.transform(&data.as_view()).unwrap().as_slice(),
&[0.0, 0.0, 0.0, 0.0]
);
}
#[test]
fn a_zero_spread_column_that_is_not_constant_passes_its_tails_through_raw() {
let values = [0.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 100.0];
let data = matrix(&values, 9, 1);
let scaler = fitted(&data, RobustScalerParams::default());
assert_eq!(scaler.centers(), &[5.0]);
assert_eq!(scaler.spreads(), &[0.0]);
assert_eq!(scaler.scales(), &[1.0]);
let transformed = scaler.transform(&data.as_view()).unwrap();
assert_eq!(transformed.get(0, 0), Some(-5.0));
assert_eq!(transformed.get(8, 0), Some(95.0));
}
#[test]
fn a_single_row_fit_is_constant_in_every_column() {
let data = matrix(&[7.0, -3.0], 1, 2);
let scaler = fitted(&data, RobustScalerParams::default());
assert_eq!(scaler.centers(), &[7.0, -3.0]);
assert_eq!(scaler.scales(), &[1.0, 1.0]);
assert_eq!(
scaler.transform(&data.as_view()).unwrap().as_slice(),
&[0.0, 0.0]
);
}
#[test]
fn a_repeated_single_value_is_degenerate_however_many_rows_it_has() {
let data = matrix(&[7.5; 64], 64, 1);
let scaler = fitted(&data, RobustScalerParams::default());
assert_eq!(scaler.centers(), &[7.5]);
assert_eq!(scaler.scales(), &[1.0]);
}
#[test]
fn a_tiny_spread_is_scaled_rather_than_substituted() {
let step = f32::MIN_POSITIVE;
let data = matrix(&[0.0, step, 2.0 * step, 3.0 * step], 4, 1);
let scaler = fitted(&data, RobustScalerParams::default());
assert!(scaler.spreads()[0] > 0.0, "the spread is small, not zero");
assert_eq!(scaler.scales()[0], scaler.spreads()[0]);
let transformed = scaler.transform(&data.as_view()).unwrap();
assert!(
transformed.as_slice().iter().all(|value| value.is_finite()),
"a small spread still produces finite values here"
);
}
#[test]
fn an_overflowing_scale_is_reported_before_anything_is_written() {
let tiny = matrix(&[0.0, f32::MIN_POSITIVE, f32::MIN_POSITIVE, 0.0], 4, 1);
let scaler = fitted(&tiny, RobustScalerParams::default());
let extreme = matrix(&[0.0, f32::MAX], 2, 1);
let mut output = [73.0; 2];
assert_eq!(
scaler
.transform_into(&extreme.as_view(), &mut output)
.unwrap_err(),
ModelError::NonFiniteTransform { row: 1, column: 0 }
);
assert_eq!(output, [73.0; 2]);
}
#[test]
fn both_toggles_select_which_statistic_is_removed() {
let data = matrix(&[0.0, 1.0, 2.0, 10.0], 4, 1);
let base = RobustScalerParams::default();
let raw = [0.0_f32, 1.0, 2.0, 10.0];
let centred = fitted(&data, base.with_scaling(false));
assert_eq!(
centred.transform(&data.as_view()).unwrap().as_slice(),
&[-1.5, -0.5, 0.5, 8.5]
);
let scaled = fitted(&data, base.with_centering(false));
let expected: Vec<f32> = raw.iter().map(|value| value / 3.25).collect();
assert_eq!(
scaled.transform(&data.as_view()).unwrap().as_slice(),
expected.as_slice()
);
let neither = fitted(&data, base.with_centering(false).with_scaling(false));
assert_eq!(
neither.transform(&data.as_view()).unwrap().as_slice(),
&raw,
"with both statistics disabled the transform is the identity"
);
assert_eq!(neither.centers(), centred.centers());
assert_eq!(neither.scales(), scaled.scales());
}
#[test]
fn refitting_the_same_batch_is_deterministic() {
let data = worked_examples();
let params = RobustScalerParams::default()
.with_quantile_range(10.0, 90.0)
.with_centering(false);
let first = fitted(&data, params);
let second = fitted(&data, params);
assert_eq!(first, second);
assert_eq!(first.get_params(), ¶ms);
}
#[test]
fn row_order_does_not_change_the_fitted_statistics() {
let forward = fitted(&worked_examples(), RobustScalerParams::default());
let reversed = matrix(
&[
1.0, 10.0, 11.0, 1.0, 10.0, 2.0, 1.0, 1.0, 0.5, 0.0, 0.0, -3.0,
],
4,
3,
);
let backward = fitted(&reversed, RobustScalerParams::default());
assert_eq!(forward.centers(), backward.centers());
assert_eq!(forward.spreads(), backward.spreads());
}
#[test]
fn the_inverse_recovers_out_of_sample_rows_and_round_trips_them_exactly() {
let scaler = fitted(&worked_examples(), RobustScalerParams::default());
let unseen = matrix(&[100.0, -50.0, 7.5, 0.25, 0.0, -1.0], 2, 3);
let recovered = scaler.inverse_transform(&unseen.as_view()).unwrap();
let back = scaler.transform(&recovered.as_view()).unwrap();
assert_eq!(
back.as_slice(),
unseen.as_slice(),
"transform after inverse_transform is exact on this probe"
);
}
#[test]
fn the_round_trip_is_exact_wherever_the_arithmetic_is() {
let data = worked_examples();
let base = RobustScalerParams::default();
let params = base.with_centering(false).with_scaling(false);
let scaler = fitted(&data, params);
let transformed = scaler.transform(&data.as_view()).unwrap();
let recovered = scaler.inverse_transform(&transformed.as_view()).unwrap();
assert_eq!(recovered.as_slice(), data.as_slice());
let constant = matrix(&[3.0, 3.0, 3.0, 3.0], 4, 1);
let scaler = fitted(&constant, base);
let transformed = scaler.transform(&constant.as_view()).unwrap();
let recovered = scaler.inverse_transform(&transformed.as_view()).unwrap();
assert_eq!(
recovered.as_slice(),
constant.as_slice(),
"a degenerate column divides by the substituted one, so it round \
trips exactly"
);
}
#[test]
fn dividing_and_multiplying_back_is_not_an_identity() {
let data = worked_examples();
let scaler = fitted(&data, RobustScalerParams::default().with_centering(false));
let transformed = scaler.transform(&data.as_view()).unwrap();
let recovered = scaler.inverse_transform(&transformed.as_view()).unwrap();
assert_ne!(
recovered.as_slice(),
data.as_slice(),
"scaling alone loses bits on this data, and the docs say so"
);
}
#[test]
fn a_general_round_trip_stays_within_a_bounded_envelope() {
let data = worked_examples();
let scaler = fitted(&data, RobustScalerParams::default());
let transformed = scaler.transform(&data.as_view()).unwrap();
let recovered = scaler.inverse_transform(&transformed.as_view()).unwrap();
for (original, recovered) in data.as_slice().iter().zip(recovered.as_slice()) {
let tolerance = 8.0 * f32::EPSILON * original.abs().max(1.0);
assert!(
(original - recovered).abs() <= tolerance,
"{original} recovered as {recovered}"
);
}
}
#[test]
fn the_inverse_validates_width_and_output_before_writing() {
let scaler = fitted(&worked_examples(), RobustScalerParams::default());
let mut short = [91.0; 11];
assert_eq!(
scaler
.inverse_transform_into(&worked_examples().as_view(), &mut short)
.unwrap_err(),
ModelError::OutputLength {
expected: 12,
actual: 11
}
);
assert_eq!(short, [91.0; 11]);
}
#[test]
fn validates_width_and_workspace_before_writing() {
let data = worked_examples();
let scaler = fitted(&data, RobustScalerParams::default());
let mut short = [91.0; 11];
assert_eq!(
scaler
.transform_into(&data.as_view(), &mut short)
.unwrap_err(),
ModelError::OutputLength {
expected: 12,
actual: 11
}
);
assert_eq!(short, [91.0; 11]);
let narrow = matrix(&[1.0, 2.0], 1, 2);
let mut narrow_output = [91.0; 2];
assert_eq!(
scaler
.transform_into(&narrow.as_view(), &mut narrow_output)
.unwrap_err(),
ModelError::FeatureDimension {
expected: 3,
actual: 2
}
);
assert_eq!(narrow_output, [91.0; 2]);
}
#[test]
fn artifact_is_deterministic_and_schema_bound() {
let scaler = fitted(
&worked_examples(),
RobustScalerParams::default()
.with_quantile_range(10.0, 90.0)
.with_centering(false),
);
let bytes = scaler.to_artifact([1; 32], [2; 32]).unwrap();
assert_eq!(bytes, scaler.to_artifact([1; 32], [2; 32]).unwrap());
assert_eq!(
RobustScaler::from_artifact(&bytes, [1; 32], [2; 32]).unwrap(),
scaler
);
assert_eq!(
RobustScaler::from_artifact(&bytes, [3; 32], [2; 32]).unwrap_err(),
ArtifactError::FeatureSchemaMismatch
);
assert_eq!(
RobustScaler::from_artifact(&bytes, [1; 32], [9; 32]).unwrap_err(),
ArtifactError::FeatureSchemaMismatch
);
}
#[test]
fn a_degenerate_column_round_trips_as_the_spread_it_really_had() {
let data = matrix(&[3.0, 3.0, 3.0, 3.0], 4, 1);
let scaler = fitted(&data, RobustScalerParams::default());
let bytes = scaler.to_artifact([1; 32], [2; 32]).unwrap();
let decoded = RobustScaler::from_artifact(&bytes, [1; 32], [2; 32]).unwrap();
assert_eq!(decoded.spreads(), &[0.0]);
assert_eq!(decoded.scales(), &[1.0]);
assert_eq!(decoded, scaler);
}
#[test]
fn artifact_rejects_a_truncation_and_a_range_no_fit_could_produce() {
let scaler = fitted(&worked_examples(), RobustScalerParams::default());
let bytes = scaler.to_artifact([1; 32], [2; 32]).unwrap();
assert_eq!(
RobustScaler::from_artifact(&bytes[..bytes.len() - 1], [1; 32], [2; 32]).unwrap_err(),
ArtifactError::ChecksumMismatch
);
let inverted = RobustScaler {
n_features_in: 1,
params: RobustScalerParams {
quantile_low: 75.0,
quantile_high: 25.0,
with_centering: true,
with_scaling: true,
},
centers: vec![1.0],
spreads: vec![2.0],
scales: vec![2.0],
};
let bytes = inverted.to_artifact([1; 32], [2; 32]).unwrap();
assert_eq!(
RobustScaler::from_artifact(&bytes, [1; 32], [2; 32]).unwrap_err(),
ArtifactError::InvalidPayload
);
let negative = RobustScaler {
n_features_in: 1,
params: RobustScalerParams::default(),
centers: vec![1.0],
spreads: vec![-2.0],
scales: vec![-2.0],
};
let bytes = negative.to_artifact([1; 32], [2; 32]).unwrap();
assert_eq!(
RobustScaler::from_artifact(&bytes, [1; 32], [2; 32]).unwrap_err(),
ArtifactError::InvalidPayload
);
}
#[test]
fn the_caller_owned_path_matches_the_allocating_one() {
let data = worked_examples();
let scaler = fitted(&data, RobustScalerParams::default());
let allocating = scaler.transform(&data.as_view()).unwrap();
let mut into = vec![f32::MAX; allocating.as_slice().len()];
let view = scaler.transform_into(&data.as_view(), &mut into).unwrap();
assert_eq!(view.as_slice(), allocating.as_slice());
assert_eq!(view.rows(), 4);
assert_eq!(view.columns(), 3);
}
}