Skip to main content

coeus_nn/
linear.rs

1use crate::init::InitializationError;
2use crate::module::{Module, ModuleError};
3use coeus_autograd::Var;
4use coeus_core::{Float, MoiraiBackend, Scalar};
5use coeus_ops::RandomInitOps;
6use coeus_tensor::Tensor;
7
8/// Fully-connected linear layer.
9#[derive(Clone)]
10pub struct Linear<T: Scalar, B: coeus_ops::BackendOps<T> + Default = MoiraiBackend> {
11    /// Weight matrix: `[out_features × in_features]`.
12    pub weight: Var<T, B>,
13    /// Bias vector: `[out_features]`.
14    pub bias: Option<Var<T, B>>,
15}
16
17/// Construction needs more of `T` and `B` than holding a layer does -- the
18/// weights are drawn, so the scalar must be sampleable and the backend must be
19/// able to sample it. The bounds sit here rather than on the type so
20/// `Linear<T, B>` stays nameable for any scalar.
21impl<T, B> Linear<T, B>
22where
23    T: Scalar + Float + coeus_cpu::RealScalar,
24    B: coeus_ops::BackendOps<T> + RandomInitOps<T> + Default,
25{
26    /// Create a Linear layer with given input/output features.
27    ///
28    /// Weights are drawn Kaiming-uniform over `in_features` from a fixed seed,
29    /// so a layer built the same way twice is the same layer; use
30    /// [`Linear::with_seed`] to choose the draw. Biases are zero.
31    ///
32    /// # Errors
33    ///
34    /// Returns [`InitializationError`] when `in_features` is zero, or when the
35    /// backend's draw fails.
36    pub fn new(
37        in_features: usize,
38        out_features: usize,
39        bias: bool,
40    ) -> Result<Self, InitializationError<B::Error>> {
41        Self::build(in_features, out_features, bias, None)
42    }
43
44    /// Create a Linear layer whose weights are drawn from `seed`.
45    ///
46    /// # Errors
47    ///
48    /// As [`Linear::new`].
49    pub fn with_seed(
50        in_features: usize,
51        out_features: usize,
52        bias: bool,
53        seed: u64,
54    ) -> Result<Self, InitializationError<B::Error>> {
55        Self::build(in_features, out_features, bias, Some(seed))
56    }
57
58    fn build(
59        in_features: usize,
60        out_features: usize,
61        bias: bool,
62        seed: Option<u64>,
63    ) -> Result<Self, InitializationError<B::Error>> {
64        let backend = B::default();
65        let w_tensor = Tensor::zeros_on([out_features, in_features], &backend)?;
66        let mut weight = Var::new(w_tensor, true)?;
67
68        // Every weight was 1.0 here. Each unit in the layer then computed the
69        // same value from the same input, took the same gradient and applied
70        // the same update -- identical at step zero and identical forever, so a
71        // layer of any width had the capacity of one unit. See ADR 0067.
72        match seed {
73            Some(seed) => crate::init::kaiming_uniform_with_seed(&mut weight, in_features, seed)?,
74            None => crate::init::kaiming_uniform(&mut weight, in_features)?,
75        }
76
77        let bias_var = if bias {
78            Some(Var::new(Tensor::zeros_on([out_features], &backend)?, true)?)
79        } else {
80            None
81        };
82
83        Ok(Self {
84            weight,
85            bias: bias_var,
86        })
87    }
88}
89
90impl<T: Scalar + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default> Module<T, B>
91    for Linear<T, B>
92{
93    fn parameters(&self) -> Vec<Var<T, B>> {
94        let mut params = vec![self.weight.clone()];
95        if let Some(ref b) = self.bias {
96            params.push(b.clone());
97        }
98        params
99    }
100
101    fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
102        let input_shape = input.tensor.shape();
103        if input_shape.len() < 2 {
104            return Err(ModuleError::InvalidRank {
105                module: "Linear",
106                expected: "at least 2",
107                actual: input_shape.len(),
108            });
109        }
110        let in_features = self.weight.tensor.shape()[1];
111        let actual_features = input_shape[input_shape.len() - 1];
112        if actual_features != in_features {
113            return Err(ModuleError::ShapeMismatch {
114                module: "Linear",
115                parameter: "input trailing dimension",
116                expected: vec![in_features],
117                actual: vec![actual_features],
118            });
119        }
120
121        let rows = input_shape[..input_shape.len() - 1]
122            .iter()
123            .copied()
124            .product::<usize>();
125        let w_t = coeus_autograd::transpose_2d(&self.weight)?;
126        let projected = if input_shape.len() == 2 {
127            coeus_autograd::matmul(input, &w_t)?
128        } else {
129            let flattened = coeus_autograd::reshape(input, [rows, in_features])?;
130            coeus_autograd::matmul(&flattened, &w_t)?
131        };
132        let projected = if let Some(ref bias) = self.bias {
133            coeus_autograd::add(&projected, bias)?
134        } else {
135            projected
136        };
137
138        let output = if input_shape.len() == 2 {
139            projected
140        } else {
141            let mut output_shape = input_shape.to_vec();
142            *output_shape
143                .last_mut()
144                .expect("invariant: rank was validated as at least two") =
145                self.weight.tensor.shape()[0];
146            coeus_autograd::reshape(&projected, output_shape)?
147        };
148        Ok(output)
149    }
150
151    fn load_parameters(&mut self, params: &[Var<T, B>]) {
152        self.weight = params[0].clone();
153        if self.bias.is_some() {
154            self.bias = Some(params[1].clone());
155        }
156    }
157}