use ndarray::{Array1, Array2, ArrayView1, ArrayView2, Axis};
use solow_core::{numeric::compensated_mean_and_var, Error, Result};
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[derive(Clone, Debug, PartialEq)]
pub struct StandardScaler {
pub mean: Array1<f64>,
pub scale: Array1<f64>,
biased: bool,
with_mean: bool,
with_std: bool,
}
impl StandardScaler {
pub fn fit(x: ArrayView2<'_, f64>) -> Result<Self> {
Self::fit_with(x, true, true, false)
}
pub fn fit_with(
x: ArrayView2<'_, f64>,
with_mean: bool,
with_std: bool,
biased: bool,
) -> Result<Self> {
if x.nrows() == 0 || x.ncols() == 0 {
return Err(Error::Value(
"StandardScaler::fit: x must have at least one row and one column".into(),
));
}
let d = x.ncols();
let n = x.nrows();
let mut mean = Array1::<f64>::zeros(d);
let mut scale = Array1::<f64>::ones(d);
for j in 0..d {
let (m, v) = compensated_mean_and_var(x.column(j).iter().copied());
mean[j] = m;
let var = if biased && n > 1 {
v * (n as f64 - 1.0) / n as f64
} else {
v
};
let s = var.sqrt();
scale[j] = if s > 0.0 && s.is_finite() { s } else { 1.0 };
}
Ok(Self {
mean,
scale,
biased,
with_mean,
with_std,
})
}
pub fn population(mut self) -> Self {
self.biased = true;
self
}
pub fn fit_transform(x: ArrayView2<'_, f64>) -> Result<(Self, Array2<f64>)> {
let s = Self::fit(x)?;
let t = s.transform(x)?;
Ok((s, t))
}
pub fn transform(&self, x: ArrayView2<'_, f64>) -> Result<Array2<f64>> {
if x.ncols() != self.mean.len() {
return Err(Error::Shape(format!(
"StandardScaler::transform: expected {} columns, got {}",
self.mean.len(),
x.ncols()
)));
}
let mut out = x.to_owned();
for (j, mut col) in out.axis_iter_mut(Axis(1)).enumerate() {
for v in col.iter_mut() {
if self.with_mean {
*v -= self.mean[j];
}
if self.with_std {
*v /= self.scale[j];
}
}
}
Ok(out)
}
pub fn inverse_transform(&self, x: ArrayView2<'_, f64>) -> Result<Array2<f64>> {
if x.ncols() != self.mean.len() {
return Err(Error::Shape(format!(
"StandardScaler::inverse_transform: expected {} columns, got {}",
self.mean.len(),
x.ncols()
)));
}
let mut out = x.to_owned();
for (j, mut col) in out.axis_iter_mut(Axis(1)).enumerate() {
for v in col.iter_mut() {
if self.with_std {
*v *= self.scale[j];
}
if self.with_mean {
*v += self.mean[j];
}
}
}
Ok(out)
}
}
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[derive(Clone, Debug, PartialEq)]
pub struct MinMaxScaler {
pub data_min: Array1<f64>,
pub data_max: Array1<f64>,
pub scale: Array1<f64>,
pub min: Array1<f64>,
pub low: f64,
pub high: f64,
}
impl MinMaxScaler {
pub fn fit(x: ArrayView2<'_, f64>) -> Result<Self> {
Self::fit_range(x, 0.0, 1.0)
}
pub fn fit_range(x: ArrayView2<'_, f64>, low: f64, high: f64) -> Result<Self> {
if x.nrows() == 0 || x.ncols() == 0 {
return Err(Error::Value(
"MinMaxScaler::fit_range: x must have at least one row and one column".into(),
));
}
if !(low.is_finite() && high.is_finite() && low < high) {
return Err(Error::Value(format!(
"MinMaxScaler::fit_range: require low < high (got low={low}, high={high})"
)));
}
let d = x.ncols();
let mut data_min = Array1::<f64>::zeros(d);
let mut data_max = Array1::<f64>::zeros(d);
let mut scale = Array1::<f64>::ones(d);
let mut min = Array1::<f64>::zeros(d);
for j in 0..d {
let mut mn = f64::INFINITY;
let mut mx = f64::NEG_INFINITY;
for &v in x.column(j).iter() {
if !v.is_finite() {
return Err(Error::Value(format!(
"MinMaxScaler::fit_range: column {j} contains non-finite values"
)));
}
if v < mn {
mn = v;
}
if v > mx {
mx = v;
}
}
data_min[j] = mn;
data_max[j] = mx;
let range = mx - mn;
if range > 0.0 {
scale[j] = (high - low) / range;
min[j] = low - mn * scale[j];
} else {
scale[j] = 0.0;
min[j] = 0.5 * (low + high);
}
}
Ok(Self {
data_min,
data_max,
scale,
min,
low,
high,
})
}
pub fn fit_transform(x: ArrayView2<'_, f64>) -> Result<(Self, Array2<f64>)> {
let s = Self::fit(x)?;
let t = s.transform(x)?;
Ok((s, t))
}
pub fn transform(&self, x: ArrayView2<'_, f64>) -> Result<Array2<f64>> {
if x.ncols() != self.scale.len() {
return Err(Error::Shape(format!(
"MinMaxScaler::transform: expected {} columns, got {}",
self.scale.len(),
x.ncols()
)));
}
let mut out = x.to_owned();
for (j, mut col) in out.axis_iter_mut(Axis(1)).enumerate() {
for v in col.iter_mut() {
*v = *v * self.scale[j] + self.min[j];
}
}
Ok(out)
}
pub fn inverse_transform(&self, x: ArrayView2<'_, f64>) -> Result<Array2<f64>> {
if x.ncols() != self.scale.len() {
return Err(Error::Shape(format!(
"MinMaxScaler::inverse_transform: expected {} columns, got {}",
self.scale.len(),
x.ncols()
)));
}
let mut out = x.to_owned();
for (j, mut col) in out.axis_iter_mut(Axis(1)).enumerate() {
if self.scale[j] > 0.0 {
for v in col.iter_mut() {
*v = (*v - self.min[j]) / self.scale[j];
}
} else {
for v in col.iter_mut() {
*v = self.data_min[j];
}
}
}
Ok(out)
}
}
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[derive(Clone, Debug, PartialEq)]
pub struct RobustScaler {
pub center: Array1<f64>,
pub scale: Array1<f64>,
pub q_low: f64,
pub q_high: f64,
with_center: bool,
with_scale: bool,
}
impl RobustScaler {
pub fn fit(x: ArrayView2<'_, f64>) -> Result<Self> {
Self::fit_range(x, 0.25, 0.75, true, true)
}
pub fn fit_range(
x: ArrayView2<'_, f64>,
q_low: f64,
q_high: f64,
with_center: bool,
with_scale: bool,
) -> Result<Self> {
if !(0.0 <= q_low && q_low < q_high && q_high <= 1.0) {
return Err(Error::Value(format!(
"RobustScaler::fit_range: require 0 ≤ q_low < q_high ≤ 1 (got {q_low}, {q_high})"
)));
}
if x.nrows() == 0 || x.ncols() == 0 {
return Err(Error::Value(
"RobustScaler::fit_range: x must have at least one row and one column".into(),
));
}
let d = x.ncols();
let mut center = Array1::<f64>::zeros(d);
let mut scale = Array1::<f64>::ones(d);
for j in 0..d {
let mut col: Vec<f64> = x.column(j).iter().copied().collect();
col.sort_by(|a, b| a.partial_cmp(b).unwrap());
center[j] = quantile_sorted(&col, 0.5);
let iqr = quantile_sorted(&col, q_high) - quantile_sorted(&col, q_low);
scale[j] = if iqr > 0.0 && iqr.is_finite() {
iqr
} else {
1.0
};
}
Ok(Self {
center,
scale,
q_low,
q_high,
with_center,
with_scale,
})
}
pub fn quantile_range(mut self, q_low: f64, q_high: f64) -> Self {
self.q_low = q_low;
self.q_high = q_high;
self
}
pub fn fit_transform(x: ArrayView2<'_, f64>) -> Result<(Self, Array2<f64>)> {
let s = Self::fit(x)?;
let t = s.transform(x)?;
Ok((s, t))
}
pub fn transform(&self, x: ArrayView2<'_, f64>) -> Result<Array2<f64>> {
if x.ncols() != self.center.len() {
return Err(Error::Shape(format!(
"RobustScaler::transform: expected {} columns, got {}",
self.center.len(),
x.ncols()
)));
}
let mut out = x.to_owned();
for (j, mut col) in out.axis_iter_mut(Axis(1)).enumerate() {
for v in col.iter_mut() {
if self.with_center {
*v -= self.center[j];
}
if self.with_scale {
*v /= self.scale[j];
}
}
}
Ok(out)
}
pub fn inverse_transform(&self, x: ArrayView2<'_, f64>) -> Result<Array2<f64>> {
if x.ncols() != self.center.len() {
return Err(Error::Shape(format!(
"RobustScaler::inverse_transform: expected {} columns, got {}",
self.center.len(),
x.ncols()
)));
}
let mut out = x.to_owned();
for (j, mut col) in out.axis_iter_mut(Axis(1)).enumerate() {
for v in col.iter_mut() {
if self.with_scale {
*v *= self.scale[j];
}
if self.with_center {
*v += self.center[j];
}
}
}
Ok(out)
}
}
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[derive(Clone, Debug, PartialEq)]
pub struct MaxAbsScaler {
pub max_abs: Array1<f64>,
pub scale: Array1<f64>,
}
impl MaxAbsScaler {
pub fn fit(x: ArrayView2<'_, f64>) -> Result<Self> {
if x.nrows() == 0 || x.ncols() == 0 {
return Err(Error::Value(
"MaxAbsScaler::fit: x must have at least one row and one column".into(),
));
}
let d = x.ncols();
let mut max_abs = Array1::<f64>::zeros(d);
let mut scale = Array1::<f64>::ones(d);
for j in 0..d {
let m = x
.column(j)
.iter()
.copied()
.map(f64::abs)
.fold(0.0, f64::max);
max_abs[j] = m;
scale[j] = if m > 0.0 { m } else { 1.0 };
}
Ok(Self { max_abs, scale })
}
pub fn fit_transform(x: ArrayView2<'_, f64>) -> Result<(Self, Array2<f64>)> {
let s = Self::fit(x)?;
let t = s.transform(x)?;
Ok((s, t))
}
pub fn transform(&self, x: ArrayView2<'_, f64>) -> Result<Array2<f64>> {
if x.ncols() != self.scale.len() {
return Err(Error::Shape(format!(
"MaxAbsScaler::transform: expected {} columns, got {}",
self.scale.len(),
x.ncols()
)));
}
let mut out = x.to_owned();
for (j, mut col) in out.axis_iter_mut(Axis(1)).enumerate() {
for v in col.iter_mut() {
*v /= self.scale[j];
}
}
Ok(out)
}
pub fn inverse_transform(&self, x: ArrayView2<'_, f64>) -> Result<Array2<f64>> {
if x.ncols() != self.scale.len() {
return Err(Error::Shape(format!(
"MaxAbsScaler::inverse_transform: expected {} columns, got {}",
self.scale.len(),
x.ncols()
)));
}
let mut out = x.to_owned();
for (j, mut col) in out.axis_iter_mut(Axis(1)).enumerate() {
for v in col.iter_mut() {
*v *= self.scale[j];
}
}
Ok(out)
}
}
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
pub enum NormKind {
L1,
L2,
Max,
}
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[derive(Copy, Clone, Debug, PartialEq)]
pub struct Normalizer {
pub kind: NormKind,
}
impl Normalizer {
pub fn new(kind: NormKind) -> Self {
Self { kind }
}
pub fn default() -> Self {
Self { kind: NormKind::L2 }
}
pub fn transform(&self, x: ArrayView2<'_, f64>) -> Result<Array2<f64>> {
let mut out = x.to_owned();
for mut row in out.axis_iter_mut(Axis(0)) {
let n = match self.kind {
NormKind::L1 => row.iter().map(|v| v.abs()).sum::<f64>(),
NormKind::L2 => row.iter().map(|v| v * v).sum::<f64>().sqrt(),
NormKind::Max => row.iter().map(|v| v.abs()).fold(0.0, f64::max),
};
if n > 0.0 {
for v in row.iter_mut() {
*v /= n;
}
}
}
Ok(out)
}
pub fn fit_transform(&self, x: ArrayView2<'_, f64>) -> Result<Array2<f64>> {
self.transform(x)
}
}
fn quantile_sorted(sorted: &[f64], q: f64) -> f64 {
let n = sorted.len();
if n == 0 {
return f64::NAN;
}
let h = (n as f64 - 1.0) * q;
let lo = h.floor() as usize;
let hi = (lo + 1).min(n - 1);
let frac = h - lo as f64;
(1.0 - frac) * sorted[lo] + frac * sorted[hi]
}
#[allow(dead_code)]
fn _unused(_v: ArrayView1<'_, f64>) {}
#[cfg(test)]
mod tests {
use super::*;
use approx::assert_abs_diff_eq;
use ndarray::array;
#[test]
fn standard_scaler_zeroes_mean_and_units_variance() {
let x = array![[1.0, 10.0], [2.0, 20.0], [3.0, 30.0], [4.0, 40.0]];
let scaler = StandardScaler::fit(x.view()).unwrap();
assert_abs_diff_eq!(scaler.mean[0], 2.5, epsilon = 1e-12);
assert_abs_diff_eq!(scaler.mean[1], 25.0, epsilon = 1e-12);
let t = scaler.transform(x.view()).unwrap();
for j in 0..2 {
let m: f64 = t.column(j).iter().sum::<f64>() / t.nrows() as f64;
assert_abs_diff_eq!(m, 0.0, epsilon = 1e-12);
}
let back = scaler.inverse_transform(t.view()).unwrap();
for (a, b) in back.iter().zip(x.iter()) {
assert_abs_diff_eq!(a, b, epsilon = 1e-12);
}
}
#[test]
fn minmax_scaler_maps_to_target_range() {
let x = array![[-1.0, 100.0], [0.0, 0.0], [1.0, -100.0]];
let scaler = MinMaxScaler::fit_range(x.view(), 0.0, 1.0).unwrap();
let t = scaler.transform(x.view()).unwrap();
for j in 0..2 {
let mn = t.column(j).iter().cloned().fold(f64::INFINITY, f64::min);
let mx = t
.column(j)
.iter()
.cloned()
.fold(f64::NEG_INFINITY, f64::max);
assert_abs_diff_eq!(mn, 0.0, epsilon = 1e-12);
assert_abs_diff_eq!(mx, 1.0, epsilon = 1e-12);
}
let back = scaler.inverse_transform(t.view()).unwrap();
for (a, b) in back.iter().zip(x.iter()) {
assert_abs_diff_eq!(a, b, epsilon = 1e-12);
}
}
#[test]
fn robust_scaler_is_immune_to_a_lone_outlier() {
let x = array![[1e6, 1.0], [1.0, 2.0], [2.0, 3.0], [3.0, 4.0], [4.0, 5.0]];
let robust = RobustScaler::fit(x.view()).unwrap();
assert_abs_diff_eq!(robust.center[0], 3.0, epsilon = 1e-12);
assert_abs_diff_eq!(robust.scale[0], 2.0, epsilon = 1e-12);
}
#[test]
fn maxabs_scaler_preserves_sign() {
let x = array![[2.0, -4.0], [-1.0, 2.0], [0.0, 0.0]];
let scaler = MaxAbsScaler::fit(x.view()).unwrap();
let t = scaler.transform(x.view()).unwrap();
assert_abs_diff_eq!(t[[0, 0]], 1.0, epsilon = 1e-12);
assert_abs_diff_eq!(t[[1, 0]], -0.5, epsilon = 1e-12);
assert_abs_diff_eq!(t[[0, 1]], -1.0, epsilon = 1e-12);
}
#[test]
fn normalizer_l2_gives_unit_norm_rows() {
let x = array![[3.0, 4.0], [1.0, 0.0], [0.0, 0.0]];
let n = Normalizer::default();
let t = n.transform(x.view()).unwrap();
assert_abs_diff_eq!(
(t.row(0).iter().map(|v| v * v).sum::<f64>()).sqrt(),
1.0,
epsilon = 1e-12
);
assert_abs_diff_eq!(t[[2, 0]], 0.0, epsilon = 1e-12); }
#[test]
fn standard_scaler_handles_constant_columns() {
let x = array![[1.0, 5.0], [1.0, 6.0], [1.0, 7.0]];
let scaler = StandardScaler::fit(x.view()).unwrap();
assert_eq!(scaler.scale[0], 1.0); let t = scaler.transform(x.view()).unwrap();
assert_abs_diff_eq!(t[[0, 0]], 0.0, epsilon = 1e-12);
}
}