dfdx 0.13.0

Ergonomic auto differentiation in Rust, with pytorch like apis.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134

mod cpu_kernel;

#[cfg(feature = "cuda")]
mod cuda_kernel;

use crate::{
    shapes::*,
    tensor::{PutTape, RandomU64, SplitTape, Storage, Tape, Tensor},
};

#[repr(C)]
#[derive(Debug, Clone, Copy)]
pub struct DropoutKernelOp {
    pub seed: u64,
    pub prob: f64,
}

pub trait DropoutKernel<E: Dtype>: Storage<E> + RandomU64 {
    fn forward<S: Shape>(
        &self,
        op: DropoutKernelOp,
        inp: &Tensor<S, E, Self>,
    ) -> Result<Tensor<S, E, Self>, Self::Err>;
    fn backward<S: Shape>(
        &self,
        op: DropoutKernelOp,
        inp: &Tensor<S, E, Self>,
        grad_inp: &mut Self::Vec,
        grad_out: &Self::Vec,
    ) -> Result<(), Self::Err>;
}

/// Zeros elements with probability `p` and scales all elements by `1 / (1 - p)`.
///
/// Described in paper: [Improving neural networks by preventing co-adaptation of feature detectors](https://arxiv.org/abs/1207.0580)
///
/// Example:
/// ```rust
/// # use dfdx::prelude::*;
/// # let dev: Cpu = Default::default();
/// let t = dev.tensor([1.0f32, 2.0, 3.0, 4.0]);
/// let r = t.dropout(0.5);
/// assert_eq!(r.array(), [2.0, 0.0, 6.0, 0.0]);
/// ```
///
/// ### Implementation details:
///
/// To reduce memory usage, this function first samples a u64 seed from `rng`,
/// and then instantiates two identical [rand::rngs::StdRng] with that seed. These rngs
/// are used in both the forward pass and backward pass to generate identical
/// random numbers, so the masking is the same for both.
pub fn dropout<S: Shape, E: Dtype, D: DropoutKernel<E>, T: Tape<E, D>>(
    t: Tensor<S, E, D, T>,
    prob: impl Into<f64>,
) -> Tensor<S, E, D, T> {
    t.dropout(prob)
}

impl<S: Shape, E: Dtype, D: DropoutKernel<E>, T: Tape<E, D>> Tensor<S, E, D, T> {
    /// See [dropout]
    pub fn dropout(self, prob: impl Into<f64>) -> Self {
        self.try_dropout(prob).unwrap()
    }
    /// See [dropout]
    pub fn try_dropout(self, prob: impl Into<f64>) -> Result<Self, D::Err> {
        let seed = self.device.random_u64();
        let prob = prob.into();
        let op = DropoutKernelOp { seed, prob };
        let (inp, mut tape) = self.split_tape();
        let out = inp.device.forward(op, &inp)?;
        let inp_ghost = inp.ghost();
        let out_ghost = out.ghost();
        tape.add_backward_op(move |grads| {
            grads.try_alloc_for(&inp_ghost)?;
            grads.try_alloc_for(&out_ghost)?;
            let (grad_inp, grad_out) = grads.mut_and_ref(&inp_ghost, &out_ghost);
            inp.device.backward(op, &inp, grad_inp, grad_out)
        });
        Ok(out.put_tape(tape))
    }
}

#[cfg(test)]
mod tests {
    use crate::{tensor::*, tensor_ops::*, tests::*};

    #[test]
    fn test_dropout_all_0d() {
        let dev: TestDevice = Default::default();
        let t = dev.tensor([3.0, 4.0, 5.0]).to_dtype::<TestDtype>();
        let r = t.leaky_trace().dropout(1.0);
        assert_close_to_literal!(r, [0.0, 0.0, 0.0]);
        let g = r.sum().backward();
        assert_close_to_literal!(g.get(&t), [0.0, 0.0, 0.0]);
    }

    #[test]
    fn test_dropout_none_0d() {
        let dev: TestDevice = Default::default();
        let t = dev.tensor([3.0, 4.0, 5.0]).to_dtype::<TestDtype>();
        let r = t.leaky_trace().dropout(0.0);
        assert_close_to_literal!(r, [3.0, 4.0, 5.0]);
        let g = r.sum().backward();
        assert_close_to_literal!(g.get(&t), [1.0, 1.0, 1.0]);
    }

    #[test]
    fn test_dropout_1d_with_non_positive_values() {
        let dev: TestDevice = TestDevice::seed_from_u64(1);
        let t = dev
            .tensor([-0.0, 2.0, -3.0, -4.0, -0.0])
            .to_dtype::<TestDtype>();
        let r = t.leaky_trace().dropout(0.5);
        assert_close_to_literal!(r, [0.0, 4.0, -6.0, -8.0, 0.0]);
        let g = r.mean().backward();
        assert_close_to_literal!(g.get(&t), [0.4, 0.4, 0.4, 0.4, 0.0]);
    }

    #[test]
    fn test_dropout_2d() {
        let dev: TestDevice = Default::default();
        let t = dev
            .tensor([[0.05, 0.1, -0.2], [0.3, -0.4, 0.5]])
            .to_dtype::<TestDtype>();
        let r = t.leaky_trace().dropout(0.6);
        assert_close_to_literal!(r, [[0.125, 0.0, -0.5], [0.0, -1.0, 0.0]]);
        // NOTE: .exp() so we ensure result grad is used properly
        let g = r.exp().mean().backward();
        assert_close_to_literal!(
            g.get(&t),
            [[0.4721452, 0.0, 0.25272113], [0.0, 0.1532831, 0.0]]
        );
    }
}