burn_optim/optim/
grads.rs

1use burn_core as burn;
2
3use burn::{
4    Tensor,
5    tensor::{Device, Gradients, container::TensorContainer},
6};
7
8use burn::module::{Module, ParamId};
9
10use super::visitor::{GradientsParamsChangeDevice, GradientsParamsConverter};
11
12/// Data type that contains gradients for parameters.
13#[derive(Default, Debug)]
14pub struct GradientsParams {
15    container: TensorContainer<ParamId>,
16}
17
18impl GradientsParams {
19    /// Creates a new [GradientsParams](GradientsParams).
20    pub fn new() -> Self {
21        Self::default()
22    }
23
24    /// Extract each tensor gradients for the given [module](Module).
25    ///
26    /// Note: This consumes the gradients. See ['from_module'] to extract gradients only for
27    ///  a specific module.
28    pub fn from_grads<M: Module>(grads: Gradients, module: &M) -> Self {
29        let mut grads = grads;
30        Self::from_module(&mut grads, module)
31    }
32
33    /// Extract each tensor gradients for the given [module](Module).
34    pub fn from_module<M: Module>(grads: &mut Gradients, module: &M) -> Self {
35        let mut grads_params = GradientsParams::new();
36        let mut visitor = GradientsParamsConverter::<M>::new(grads, &mut grads_params, None);
37        module.visit(&mut visitor);
38        grads_params
39    }
40
41    /// Extract tensor gradients for the given [module](Module) and given parameters.
42    pub fn from_params<M: Module>(grads: &mut Gradients, module: &M, params: &[ParamId]) -> Self {
43        let mut grads_params = GradientsParams::new();
44        let mut visitor =
45            GradientsParamsConverter::<M>::new(grads, &mut grads_params, Some(params.to_vec()));
46        module.visit(&mut visitor);
47        grads_params
48    }
49
50    /// Get the gradients for the given [parameter id](ParamId).
51    ///
52    /// # Notes
53    ///
54    /// You should use [remove](GradientsParams::remove) if you want to get the gradients
55    /// only one time.
56    pub fn get<const D: usize>(&self, id: ParamId) -> Option<Tensor<D>> {
57        self.container.get(&id)
58    }
59
60    /// Remove the gradients for the given [parameter id](ParamId).
61    pub fn remove<const D: usize>(&mut self, id: ParamId) -> Option<Tensor<D>> {
62        self.container.remove(&id)
63    }
64
65    /// Register a gradients tensor for the given [parameter id](ParamId).
66    ///
67    /// # Notes
68    ///
69    /// If a tensor is already registered for the given [parameter id](ParamId), it will be replaced.
70    pub fn register<const D: usize>(&mut self, id: ParamId, value: Tensor<D>) {
71        // TODO: always call value.inner() to make sure?
72        self.container.register(id, value)
73    }
74
75    /// The number of gradients tensors registered.
76    pub fn len(&self) -> usize {
77        self.container.len()
78    }
79
80    /// If any tensor is contained.
81    pub fn is_empty(&self) -> bool {
82        self.len() == 0
83    }
84
85    /// Change the device of each tensor gradients registered for the given [module](Module).
86    pub fn to_device<M: Module>(mut self, device: &Device, module: &M) -> Self {
87        let mut visitor = GradientsParamsChangeDevice::<M>::new(device, &mut self);
88        module.visit(&mut visitor);
89        self
90    }
91}
92
93#[cfg(test)]
94mod tests {
95    use super::*;
96    use burn::module::{Module, ParamGroup};
97    use burn::tensor::Distribution;
98    use burn_nn::{Linear, LinearConfig};
99
100    #[test]
101    fn test_convert_grads() {
102        let device = Device::default().autodiff();
103        let layer_1 = layer(&device);
104        let mut layer_2 = layer_1.clone();
105        layer_2 = layer_2.fork(&device);
106        let loss_1 = layer_1.forward(random_tensor(&device));
107        let loss_2 = layer_2.forward(random_tensor(&device));
108        let grads_1 = GradientsParams::from_grads(loss_1.backward(), &layer_1);
109        let grads_2 = GradientsParams::from_grads(loss_2.backward(), &layer_2);
110
111        let group = ParamGroup::ids_from_module(layer_1.clone());
112        let bias_2 = layer_2.bias.as_ref().unwrap();
113
114        assert!(group.matches(&layer_2.weight.id, None));
115        assert!(group.matches(&bias_2.id, None));
116        assert_eq!(grads_1.len(), 2);
117        assert_eq!(grads_2.len(), 2);
118    }
119
120    fn layer(device: &Device) -> Linear {
121        LinearConfig::new(20, 20).init(device)
122    }
123
124    fn random_tensor(device: &Device) -> Tensor<2> {
125        Tensor::<2>::random([2, 20], Distribution::Default, device)
126    }
127}
burn_optim/optim/grads.rs

burn_optim/optim/
grads.rs
