oxidelm 0.2.0

Bare-metal pure-Rust Transformer engine with reverse-mode autograd, BPE tokenization, and wgpu acceleration
Documentation
use rand::prelude::*;

pub fn normal_values(length: usize, standard_deviation: f32, seed: u64) -> Vec<f32> {
    assert!(standard_deviation.is_finite() && standard_deviation > 0.0);
    let mut rng = StdRng::seed_from_u64(seed);
    normal_with_rng(length, standard_deviation, &mut rng)
}

pub fn xavier_uniform_values(length: usize, fan_in: usize, fan_out: usize, seed: u64) -> Vec<f32> {
    assert!(fan_in > 0 && fan_out > 0);
    let mut rng = StdRng::seed_from_u64(seed);
    xavier_uniform_with_rng(length, fan_in, fan_out, &mut rng)
}

pub fn kaiming_uniform_values(length: usize, fan_in: usize, seed: u64) -> Vec<f32> {
    assert!(fan_in > 0);
    let mut rng = StdRng::seed_from_u64(seed);
    let limit = (6.0 / fan_in as f32).sqrt();
    (0..length)
        .map(|_| rng.random_range(-limit..=limit))
        .collect()
}

pub(crate) fn xavier_uniform_with_rng(
    length: usize,
    fan_in: usize,
    fan_out: usize,
    rng: &mut StdRng,
) -> Vec<f32> {
    let limit = (6.0 / (fan_in + fan_out) as f32).sqrt();
    (0..length)
        .map(|_| rng.random_range(-limit..=limit))
        .collect()
}

pub(crate) fn normal_with_rng(
    length: usize,
    standard_deviation: f32,
    rng: &mut StdRng,
) -> Vec<f32> {
    let mut values = Vec::with_capacity(length);
    while values.len() < length {
        let uniform_radius = rng.random::<f32>().max(f32::MIN_POSITIVE);
        let angle = std::f32::consts::TAU * rng.random::<f32>();
        let radius = (-2.0 * uniform_radius.ln()).sqrt() * standard_deviation;
        values.push(radius * angle.cos());
        if values.len() < length {
            values.push(radius * angle.sin())
        }
    }
    values
}

#[cfg(test)]
mod tests {
    use super::{normal_values, xavier_uniform_values};

    #[test]
    fn normal_statistics_match_configured_scale() {
        let standard_deviation = 0.02;
        let values = normal_values(100_000, standard_deviation, 7);
        let mean = values.iter().sum::<f32>() / values.len() as f32;
        let variance = values
            .iter()
            .map(|value| (value - mean).powi(2))
            .sum::<f32>()
            / values.len() as f32;

        assert!(mean.abs() < standard_deviation * 0.03);
        assert!((variance.sqrt() - standard_deviation).abs() < standard_deviation * 0.03);
    }

    #[test]
    fn xavier_statistics_match_expected_scale() {
        let fan_in = 64;
        let fan_out = 32;
        let values = xavier_uniform_values(100_000, fan_in, fan_out, 7);
        let mean = values.iter().sum::<f32>() / values.len() as f32;
        let variance = values
            .iter()
            .map(|value| (value - mean).powi(2))
            .sum::<f32>()
            / values.len() as f32;
        let expected_variance = 2.0 / (fan_in + fan_out) as f32;

        assert!(mean.abs() < 0.01);
        assert!((variance - expected_variance).abs() < expected_variance * 0.08);
    }
}