bunsen 0.41.0

bunsen is a batteries included common library for burn
Documentation
use burn::{
    Tensor,
    tensor::Distribution,
};

/// Applies the random dropout op on the input tensor.
///
/// ```ignore
/// p_keep = 1.0 - p;
/// input * input.random_like(Bernoulli(p_keep)) / p_keep
/// ```
///
/// At `prob == 1.0` every element is dropped, and the result is `input * 0`
/// (zeros, for finite input), as in `PyTorch`'s `dropout`; there is no
/// `1 / p_keep` to apply.
///
/// # Arguments
/// * `prob` - the drop probability.
/// * `input` - the input tensor.
///
/// # Panics
///
/// If `prob` is not in `[0, 1]`.
pub fn dropout<const D: usize>(
    prob: f64,
    input: Tensor<D>,
) -> Tensor<D> {
    if prob == 0.0 {
        return input;
    }
    if !(0.0..=1.0).contains(&prob) {
        panic!("Dropout probability should be between 0 and 1, but got {prob}");
    }
    if prob == 1.0 {
        return input.mul_scalar(0.0);
    }

    let prob_keep = 1.0 - prob;
    let random = input.random_like(Distribution::Bernoulli(prob_keep));
    let x = input * random;

    x * (1.0 / prob_keep)
}

#[cfg(test)]
mod tests {
    use burn::tensor::Distribution;
    use serial_test::serial;

    use super::*;
    use crate::{
        ops::drop::dropout,
        support::testing::{
            DeviceMemoryGuard,
            performance_device,
        },
    };

    #[test]
    #[serial]
    fn dropout_prob_0_should_return_input() {
        let device = performance_device();
        let _memory = DeviceMemoryGuard::new(&device);
        let input = Tensor::<2>::random([10, 3], Distribution::Default, &device);

        let output = dropout(0., input.clone());

        output.to_data().assert_eq(&input.to_data(), true);
    }

    #[test]
    #[serial]
    fn dropout_prob_1_should_return_zeros() {
        let device = performance_device();
        let _memory = DeviceMemoryGuard::new(&device);
        let input = Tensor::<2>::random([10, 3], Distribution::Default, &device);

        let output = dropout(1., input.clone());

        output
            .to_data()
            .assert_eq(&input.zeros_like().to_data(), true);
    }

    #[test]
    #[serial]
    fn dropout_rates_stochastic_test() {
        let device = performance_device();
        let _memory = DeviceMemoryGuard::new(&device);
        device.seed(0);

        let input = Tensor::<2>::ones([10, 10], &device);
        let num_elem = input.shape().num_elements();

        let prob = 0.25;
        let output = dropout(prob, input.clone());

        let prob_keep = 1.0 - prob;
        let keep_value = 1.0 / prob_keep;

        let drop_count: f32 = output.clone().equal_elem(0.0).float().sum().into_scalar();
        let keep_count: f32 = output
            .clone()
            .equal_elem(keep_value)
            .float()
            .sum()
            .into_scalar();

        assert_eq!(keep_count, num_elem as f32 - drop_count);

        let drop_rate = drop_count / num_elem as f32;
        assert!((drop_rate - prob as f32).abs() < 0.1);
    }

    #[test]
    #[serial]
    #[should_panic = "Dropout probability should be between 0 and 1,"]
    fn dropout_prob_invalid() {
        let device = performance_device();
        let _memory = DeviceMemoryGuard::new(&device);

        let input = Tensor::<1>::ones([10], &device);
        let _ = dropout(-10., input);
    }
}