ferricml 0.2.0

A lean, pure-Rust toolkit for classical machine learning
Documentation
use crate::api::ModelError;
use crate::data::{MatrixView, SampleWeights};
use nalgebra::{DMatrix, DVector};

pub(super) struct LeastSquaresFit {
    pub(super) coefficients: Vec<f64>,
    pub(super) intercept: f64,
    pub(super) rank: usize,
}

pub(super) struct DenseLinearFit {
    pub(super) coefficients: Vec<f64>,
    pub(super) intercept: f64,
}

pub(super) fn fit_dense(
    data: &MatrixView<'_>,
    targets: &[f32],
    sample_weights: Option<&SampleWeights>,
    fit_intercept: bool,
    tolerance: f32,
) -> Result<LeastSquaresFit, ModelError> {
    debug_assert_eq!(data.rows(), targets.len());
    debug_assert!(sample_weights.is_none_or(|weights| weights.len() == data.rows()));

    let preprocessed = preprocess(data, targets, sample_weights, fit_intercept);
    let rows = data.rows();
    let columns = data.columns();
    let svd = preprocessed.matrix.svd(true, true);
    let largest_singular = svd.singular_values.iter().copied().fold(0.0_f64, f64::max);
    if !largest_singular.is_finite() {
        return Err(ModelError::LinearSolveFailed);
    }
    let numerical_floor = f64::EPSILON * rows.max(columns) as f64 * largest_singular;
    let cutoff = (f64::from(tolerance) * largest_singular).max(numerical_floor);
    let rank = svd
        .singular_values
        .iter()
        .filter(|&&value| value > cutoff)
        .count();
    let coefficients = if rank == 0 {
        vec![0.0; columns]
    } else {
        svd.solve(&preprocessed.targets, cutoff)
            .map_err(|_| ModelError::LinearSolveFailed)?
            .iter()
            .copied()
            .collect::<Vec<_>>()
    };
    let intercept = preprocessed.target_mean
        - preprocessed
            .feature_means
            .iter()
            .zip(&coefficients)
            .map(|(&mean, &coefficient)| mean * coefficient)
            .sum::<f64>();
    if coefficients.iter().any(|value| !value.is_finite()) || !intercept.is_finite() {
        return Err(ModelError::LinearSolveFailed);
    }
    Ok(LeastSquaresFit {
        coefficients,
        intercept,
        rank,
    })
}

pub(super) fn fit_ridge_dense(
    data: &MatrixView<'_>,
    targets: &[f32],
    sample_weights: Option<&SampleWeights>,
    fit_intercept: bool,
    alpha: f32,
) -> Result<DenseLinearFit, ModelError> {
    if alpha == 0.0 {
        let fit = fit_dense(data, targets, sample_weights, fit_intercept, 0.0)?;
        return Ok(DenseLinearFit {
            coefficients: fit.coefficients,
            intercept: fit.intercept,
        });
    }
    let preprocessed = preprocess(data, targets, sample_weights, fit_intercept);
    let transpose = preprocessed.matrix.transpose();
    let mut gram = &transpose * &preprocessed.matrix;
    for index in 0..gram.nrows() {
        gram[(index, index)] += f64::from(alpha);
    }
    let right = transpose * preprocessed.targets;
    let coefficients = gram
        .cholesky()
        .ok_or(ModelError::LinearSolveFailed)?
        .solve(&right)
        .iter()
        .copied()
        .collect::<Vec<_>>();
    let intercept = preprocessed.target_mean
        - preprocessed
            .feature_means
            .iter()
            .zip(&coefficients)
            .map(|(&mean, &coefficient)| mean * coefficient)
            .sum::<f64>();
    if coefficients.iter().any(|value| !value.is_finite()) || !intercept.is_finite() {
        return Err(ModelError::LinearSolveFailed);
    }
    Ok(DenseLinearFit {
        coefficients,
        intercept,
    })
}

/// Centered, weight-scaled dense storage shared by every dense linear fit.
///
/// `matrix` is column-major, which is the layout nalgebra builds and the one a
/// coordinate sweep wants: a coordinate step touches one whole column and no
/// part of any other.
pub(super) struct PreprocessedDense {
    pub(super) matrix: DMatrix<f64>,
    pub(super) targets: DVector<f64>,
    pub(super) feature_means: Vec<f64>,
    pub(super) target_mean: f64,
}

/// Centers by the weighted means when an intercept is fitted and scales each
/// row by the square root of its weight.
///
/// Stated once for the whole family. Sharing it is what makes "the penalty
/// applies to raw-scale coefficients, and the intercept is recovered from the
/// centering" one definition rather than one per estimator.
pub(super) fn preprocess(
    data: &MatrixView<'_>,
    targets: &[f32],
    sample_weights: Option<&SampleWeights>,
    fit_intercept: bool,
) -> PreprocessedDense {
    let rows = data.rows();
    let columns = data.columns();
    let total_weight = sample_weights.map_or(rows as f64, SampleWeights::total);
    let mut feature_means = vec![0.0_f64; columns];
    let mut target_mean = 0.0_f64;
    if fit_intercept {
        for (row_index, (row, &target)) in data.iter_rows().zip(targets).enumerate() {
            let weight = sample_weight(sample_weights, row_index);
            for (column, &value) in row.iter().enumerate() {
                feature_means[column] += weight * f64::from(value);
            }
            target_mean += weight * f64::from(target);
        }
        for mean in &mut feature_means {
            *mean /= total_weight;
        }
        target_mean /= total_weight;
    }
    let weight_sqrts = sample_weights.map(|weights| {
        weights
            .as_slice()
            .iter()
            .map(|&weight| f64::from(weight).sqrt())
            .collect::<Vec<_>>()
    });
    let row_scale = |row: usize| weight_sqrts.as_ref().map_or(1.0, |weights| weights[row]);
    let mut matrix_values = Vec::with_capacity(rows * columns);
    for (column, &mean) in feature_means.iter().enumerate() {
        for row in 0..rows {
            let value = data.as_slice()[row * columns + column];
            matrix_values.push(row_scale(row) * (f64::from(value) - mean));
        }
    }
    let matrix = DMatrix::from_vec(rows, columns, matrix_values);
    let target_vector = DVector::from_iterator(
        rows,
        targets
            .iter()
            .enumerate()
            .map(|(row, &target)| row_scale(row) * (f64::from(target) - target_mean)),
    );
    PreprocessedDense {
        matrix,
        targets: target_vector,
        feature_means,
        target_mean,
    }
}

fn sample_weight(sample_weights: Option<&SampleWeights>, row: usize) -> f64 {
    sample_weights.map_or(1.0, |weights| f64::from(weights.as_slice()[row]))
}