burn-optim 0.22.0

Optimizer building blocks for the Burn deep learning framework
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127

use burn_core as burn;

use burn::{
    Tensor,
    tensor::{Device, Gradients, container::TensorContainer},
};

use burn::module::{Module, ParamId};

use super::visitor::{GradientsParamsChangeDevice, GradientsParamsConverter};

/// Data type that contains gradients for parameters.
#[derive(Default, Debug)]
pub struct GradientsParams {
    container: TensorContainer<ParamId>,
}

impl GradientsParams {
    /// Creates a new [GradientsParams](GradientsParams).
    pub fn new() -> Self {
        Self::default()
    }

    /// Extract each tensor gradients for the given [module](Module).
    ///
    /// Note: This consumes the gradients. See ['from_module'] to extract gradients only for
    ///  a specific module.
    pub fn from_grads<M: Module>(grads: Gradients, module: &M) -> Self {
        let mut grads = grads;
        Self::from_module(&mut grads, module)
    }

    /// Extract each tensor gradients for the given [module](Module).
    pub fn from_module<M: Module>(grads: &mut Gradients, module: &M) -> Self {
        let mut grads_params = GradientsParams::new();
        let mut visitor = GradientsParamsConverter::<M>::new(grads, &mut grads_params, None);
        module.visit(&mut visitor);
        grads_params
    }

    /// Extract tensor gradients for the given [module](Module) and given parameters.
    pub fn from_params<M: Module>(grads: &mut Gradients, module: &M, params: &[ParamId]) -> Self {
        let mut grads_params = GradientsParams::new();
        let mut visitor =
            GradientsParamsConverter::<M>::new(grads, &mut grads_params, Some(params.to_vec()));
        module.visit(&mut visitor);
        grads_params
    }

    /// Get the gradients for the given [parameter id](ParamId).
    ///
    /// # Notes
    ///
    /// You should use [remove](GradientsParams::remove) if you want to get the gradients
    /// only one time.
    pub fn get<const D: usize>(&self, id: ParamId) -> Option<Tensor<D>> {
        self.container.get(&id)
    }

    /// Remove the gradients for the given [parameter id](ParamId).
    pub fn remove<const D: usize>(&mut self, id: ParamId) -> Option<Tensor<D>> {
        self.container.remove(&id)
    }

    /// Register a gradients tensor for the given [parameter id](ParamId).
    ///
    /// # Notes
    ///
    /// If a tensor is already registered for the given [parameter id](ParamId), it will be replaced.
    pub fn register<const D: usize>(&mut self, id: ParamId, value: Tensor<D>) {
        // TODO: always call value.inner() to make sure?
        self.container.register(id, value)
    }

    /// The number of gradients tensors registered.
    pub fn len(&self) -> usize {
        self.container.len()
    }

    /// If any tensor is contained.
    pub fn is_empty(&self) -> bool {
        self.len() == 0
    }

    /// Change the device of each tensor gradients registered for the given [module](Module).
    pub fn to_device<M: Module>(mut self, device: &Device, module: &M) -> Self {
        let mut visitor = GradientsParamsChangeDevice::<M>::new(device, &mut self);
        module.visit(&mut visitor);
        self
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use burn::module::{Module, ParamGroup};
    use burn::tensor::Distribution;
    use burn_nn::{Linear, LinearConfig};

    #[test]
    fn test_convert_grads() {
        let device = Device::default().autodiff();
        let layer_1 = layer(&device);
        let mut layer_2 = layer_1.clone();
        layer_2 = layer_2.fork(&device);
        let loss_1 = layer_1.forward(random_tensor(&device));
        let loss_2 = layer_2.forward(random_tensor(&device));
        let grads_1 = GradientsParams::from_grads(loss_1.backward(), &layer_1);
        let grads_2 = GradientsParams::from_grads(loss_2.backward(), &layer_2);

        let group = ParamGroup::ids_from_module(layer_1.clone());
        let bias_2 = layer_2.bias.as_ref().unwrap();

        assert!(group.matches(&layer_2.weight.id, None));
        assert!(group.matches(&bias_2.id, None));
        assert_eq!(grads_1.len(), 2);
        assert_eq!(grads_2.len(), 2);
    }

    fn layer(device: &Device) -> Linear {
        LinearConfig::new(20, 20).init(device)
    }

    fn random_tensor(device: &Device) -> Tensor<2> {
        Tensor::<2>::random([2, 20], Distribution::Default, device)
    }
}