Skip to main content

incin_core/nn/
dropout.rs

1use crate::dist::placement::Local;
2use crate::err::{Error, Result};
3use crate::exec::capability::Capabilities;
4use crate::exec::catalog::{DropoutAttributes, op};
5use crate::exec::dispatch;
6use crate::exec::request::TensorHandle;
7use crate::nn::{Module, TrainMode};
8use crate::shapes::{DynShape, Shape};
9use crate::tensor::backend::Execute;
10use crate::tensor::backend::SupportsDType;
11use crate::tensor::base::Tensor;
12use crate::tensor::device::ConstDevice;
13use crate::tensor::dtype::BuiltinDType;
14use crate::tensor::grad::RequiresGrad;
15
16/// A Dropout layer.
17///
18/// Randomly zeroes some of the elements of the input tensor with probability `p` using samples
19/// from a uniform distribution. The elements to zero are randomized on every forward call during training.
20///
21/// This has proven to be an effective technique for regularization and preventing the co-adaptation of neurons.
22/// Furthermore, the outputs are scaled by a factor of `1 / (1 - p)` during training. This means that during
23/// evaluation the module simply computes an identity function.
24#[derive(Debug, Clone)]
25pub struct Dropout {
26    /// The probability of an element to be zeroed.
27    pub p: f32,
28    /// Whether the module is in training mode.
29    /// If false, dropout acts as an identity function.
30    pub is_training: bool,
31}
32
33impl<B: crate::tensor::backend::VariableBackend> crate::nn::VisitParameters<B> for Dropout {
34    fn visit_parameters<V: crate::nn::ParameterVisitor<B>>(
35        &self,
36        _: &crate::nn::StatePath,
37        _: &mut V,
38    ) -> Result<()> {
39        Ok(())
40    }
41}
42
43impl Dropout {
44    /// Creates a new Dropout module with the specified probability.
45    /// By default, `is_training` is set to true.
46    pub fn new(p: f32) -> Self {
47        Self {
48            p,
49            is_training: true,
50        }
51    }
52}
53
54impl<B: crate::tensor::backend::VariableBackend> crate::nn::VisitState<B> for Dropout {
55    fn visit_state<V: crate::nn::StateVisitor<B>>(
56        &self,
57        _: &crate::nn::StatePath,
58        _: &mut V,
59    ) -> Result<()> {
60        Ok(())
61    }
62}
63
64impl<B: crate::tensor::backend::VariableBackend> crate::nn::VisitStateMut<B> for Dropout {
65    fn visit_state_mut<V: crate::nn::StateMutVisitor<B>>(
66        &mut self,
67        _: &crate::nn::StatePath,
68        _: &mut V,
69    ) -> Result<()> {
70        Ok(())
71    }
72}
73
74impl crate::nn::NamedLayers for Dropout {
75    fn layer_structure(&self, prefix: &str) -> alloc::vec::Vec<crate::nn::LayerNode> {
76        alloc::vec![crate::nn::LayerNode {
77            name: alloc::string::String::from(prefix),
78            type_name: alloc::string::String::from("Dropout"),
79            shape_info: alloc::format!("p={}", self.p),
80            children: alloc::vec![],
81        }]
82    }
83}
84
85impl crate::nn::ShapeInfo for Dropout {
86    fn shape_info(&self) -> Option<alloc::string::String> {
87        None
88    }
89}
90
91impl<B: crate::tensor::backend::VariableBackend, NewD: crate::tensor::device::Device>
92    crate::tensor::transfer::ToDevice<B, NewD> for Dropout
93{
94    type Output = Dropout;
95    fn to_device(self, _arg: &NewD::Arg) -> Result<Self::Output> {
96        Ok(self)
97    }
98}
99
100impl TrainMode for Dropout {
101    /// Directly sets `is_training` - `Dropout`'s own forward already reads
102    /// this flag to decide identity-vs-random-zeroing behavior.
103    fn set_training(&mut self, training: bool) {
104        self.is_training = training;
105    }
106}
107
108impl<
109    S: Shape + DynShape,
110    B: crate::tensor::backend::VariableBackend,
111    K: BuiltinDType,
112    G: RequiresGrad,
113> Module<Tensor<S, B, K, G>> for Dropout
114where
115    B: SupportsDType<K> + Capabilities + Execute<op::Dropout>,
116    B::Device: ConstDevice,
117    <B as Execute<op::Dropout>>::Output: Into<B::Storage<K>>,
118{
119    type Output = Tensor<S, B, K, G>;
120    type Error = Error;
121
122    #[inline]
123    fn forward(&self, x: Tensor<S, B, K, G>) -> core::result::Result<Tensor<S, B, K, G>, Error> {
124        if !self.is_training || self.p <= 0.0 {
125            return Ok(x);
126        }
127
128        let input = TensorHandle::from_storage::<B, K, Local>(&x.inner);
129        let context = crate::tensor::grad::execution_context::<B, G>(&x._grad)
130            .with_training(self.is_training);
131        let output = dispatch::execute_shaped::<op::Dropout, B, S>(
132            &context,
133            DropoutAttributes {
134                probability: self.p as f64,
135                training: self.is_training,
136            },
137            &[input],
138            &x._shape,
139        )
140        .map_err(Error::from)?
141        .into();
142        Tensor::from_shape_value(
143            output,
144            x._shape.clone(),
145            x._dtype.clone(),
146            x._device.clone(),
147            x._grad,
148        )
149    }
150}