Skip to main content

coeus_nn/rnn/
lstm.rs

1// ── LSTMCell ──
2
3use crate::linear::Linear;
4use crate::module::{prefixed_parameters, Module, ModuleError, ModuleResult};
5use crate::rnn::validation;
6use crate::rnn::VarPair;
7use coeus_autograd::Var;
8use coeus_core::{Float, MoiraiBackend};
9use coeus_tensor::Tensor;
10
11/// Sequence output: stacked hidden states plus the final hidden/cell state.
12type SequenceOutput<T, B> = (Var<T, B>, VarPair<T, B>);
13
14/// Single-timestep Long Short-Term Memory cell.
15///
16/// Gate computation:
17/// ```text
18/// gates = x @ W_ih.T + b_ih + h @ W_hh.T + b_hh   // [batch, 4*H]
19/// i = sigmoid(gates[:, 0:H])        — input gate
20/// f = sigmoid(gates[:, H:2H])       — forget gate
21/// g = tanh(gates[:, 2H:3H])         — cell gate
22/// o = sigmoid(gates[:, 3H:4H])      — output gate
23///
24/// c_new = f ⊙ c + i ⊙ g
25/// h_new = o ⊙ tanh(c_new)
26/// ```
27#[derive(Clone)]
28pub struct LSTMCell<T: Float, B: coeus_ops::BackendOps<T> + Default = MoiraiBackend> {
29    /// Input-to-hidden projection: `[input_size, 4*hidden_size]`.
30    pub w_ih: Linear<T, B>,
31    /// Hidden-to-hidden projection: `[hidden_size, 4*hidden_size]`.
32    pub w_hh: Linear<T, B>,
33    /// Number of input features per timestep.
34    pub input_size: usize,
35    /// Number of hidden features.
36    pub hidden_size: usize,
37}
38
39impl<T: Float + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default> LSTMCell<T, B> {
40    /// Create with Kaiming-initialized weights and zero biases.
41    ///
42    /// # Errors
43    ///
44    /// Returns [`crate::init::InitializationError`] when a size is zero or the
45    /// backend's draw fails.
46    pub fn new(
47        input_size: usize,
48        hidden_size: usize,
49    ) -> Result<Self, crate::init::InitializationError<B::Error>>
50    where
51        B: coeus_ops::RandomInitOps<T>,
52    {
53        let w_ih = Linear::new(input_size, 4 * hidden_size, true)?;
54        let w_hh = Linear::new(hidden_size, 4 * hidden_size, true)?;
55        Ok(Self {
56            w_ih,
57            w_hh,
58            input_size,
59            hidden_size,
60        })
61    }
62
63    /// Forward step.
64    ///
65    /// - `x`: `[batch, input_size]`
66    /// - `h`: `[batch, hidden_size]`
67    /// - `c`: `[batch, hidden_size]`
68    ///
69    /// Returns `(h_new, c_new)`, both `[batch, hidden_size]`.
70    pub fn step(
71        &self,
72        x: &Var<T, B>,
73        h: &Var<T, B>,
74        c: &Var<T, B>,
75    ) -> ModuleResult<VarPair<T, B>, B> {
76        let batch = validation::cell_input(x.tensor.shape(), self.input_size, "LSTMCell")?;
77        validation::state(
78            h.tensor.shape(),
79            batch,
80            self.hidden_size,
81            "LSTMCell",
82            "hidden state",
83        )?;
84        validation::state(
85            c.tensor.shape(),
86            batch,
87            self.hidden_size,
88            "LSTMCell",
89            "cell state",
90        )?;
91        self.step_validated(x, h, c, batch)
92    }
93
94    fn step_validated(
95        &self,
96        x: &Var<T, B>,
97        h: &Var<T, B>,
98        c: &Var<T, B>,
99        batch: usize,
100    ) -> ModuleResult<VarPair<T, B>, B> {
101        let hs = self.hidden_size;
102        let input = self.w_ih.forward(x)?;
103        let hidden = self.w_hh.forward(h)?;
104        let gates = coeus_autograd::add(&input, &hidden)?;
105
106        let slice = |start: usize, end: usize| -> Result<Var<T, B>, B::Error> {
107            coeus_autograd::slice(&gates, &[(0, batch), (start, end)])
108        };
109
110        let i_g = coeus_autograd::sigmoid(&slice(0, hs)?)?;
111        let f_g = coeus_autograd::sigmoid(&slice(hs, 2 * hs)?)?;
112        let g_g = coeus_autograd::tanh(&slice(2 * hs, 3 * hs)?)?;
113        let o_g = coeus_autograd::sigmoid(&slice(3 * hs, 4 * hs)?)?;
114
115        let c_new = coeus_autograd::add(
116            &coeus_autograd::mul(&f_g, c)?,
117            &coeus_autograd::mul(&i_g, &g_g)?,
118        )?;
119        let h_new = coeus_autograd::mul(&o_g, &coeus_autograd::tanh(&c_new)?)?;
120        Ok((h_new, c_new))
121    }
122}
123
124impl<T: Float + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default> Module<T, B>
125    for LSTMCell<T, B>
126{
127    fn parameters(&self) -> Vec<Var<T, B>> {
128        let mut p = self.w_ih.parameters();
129        p.extend(self.w_hh.parameters());
130        p
131    }
132
133    fn named_parameters(&self) -> Vec<coeus_autograd::Parameter<T, B>> {
134        let mut parameters = prefixed_parameters("input", &self.w_ih);
135        parameters.extend(prefixed_parameters("hidden", &self.w_hh));
136        parameters
137    }
138
139    fn forward(&self, x: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
140        let batch = validation::cell_input(x.tensor.shape(), self.input_size, "LSTMCell")?;
141        let backend = B::default();
142        let h = Var::new(
143            Tensor::zeros_on([batch, self.hidden_size], &backend)?,
144            false,
145        )?;
146        let c = Var::new(
147            Tensor::zeros_on([batch, self.hidden_size], &backend)?,
148            false,
149        )?;
150        Ok(self.step_validated(x, &h, &c, batch)?.0)
151    }
152}
153
154// ── Lstm (sequence-level) ──
155
156/// Sequence-level LSTM module: unrolls [`LSTMCell`] across the time axis.
157///
158/// Input layout: `[batch, seq_len, input_size]`.
159/// Output layout: `[batch, seq_len, hidden_size]`.
160/// Final state: `(h_n, c_n)`, each `[batch, hidden_size]`.
161///
162/// Initial hidden and cell states default to zeros; use [`Lstm::forward_seq`]
163/// for full `(output, (h_n, c_n))` when the final state is needed, or
164/// `Module::forward` when only the output sequence is needed.
165///
166/// # Example
167/// ```text
168/// zeros input ⟹ all gate pre-activations = 0
169///   i = f = o = σ(0) = 0.5,  g = tanh(0) = 0
170///   c_new = f·c + i·g = 0,   h_new = o·tanh(c_new) = 0
171/// ∴ output is all zeros for any sequence length.
172/// ```
173#[derive(Clone)]
174pub struct Lstm<T: Float, B: coeus_ops::BackendOps<T> + Default = MoiraiBackend> {
175    cell: LSTMCell<T, B>,
176    /// Number of input features per timestep.
177    pub input_size: usize,
178    /// Number of hidden features.
179    pub hidden_size: usize,
180}
181
182impl<T: Float + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default> Lstm<T, B>
183where
184    B::DeviceBuffer<T>:
185        coeus_core::CpuAddressableStorage<T> + coeus_core::CpuAddressableStorageMut<T>,
186{
187    /// Create with Xavier-initialized weights and zero biases.
188    ///
189    /// # Errors
190    ///
191    /// Returns [`crate::init::InitializationError`] when a size is zero or the
192    /// backend's draw fails.
193    pub fn new(
194        input_size: usize,
195        hidden_size: usize,
196    ) -> Result<Self, crate::init::InitializationError<B::Error>>
197    where
198        B: coeus_ops::RandomInitOps<T>,
199    {
200        Ok(Self {
201            cell: LSTMCell::new(input_size, hidden_size)?,
202            input_size,
203            hidden_size,
204        })
205    }
206
207    /// Unroll over the sequence dimension with zero initial state.
208    ///
209    /// Returns `(output, (h_n, c_n))`:
210    /// - `output`: `[batch, seq_len, hidden_size]` — all hidden states stacked.
211    /// - `h_n`, `c_n`: `[batch, hidden_size]` — final hidden and cell state.
212    pub fn forward_seq(&self, x: &Var<T, B>) -> ModuleResult<SequenceOutput<T, B>, B> {
213        let (batch, seq_len) =
214            validation::sequence_input(x.tensor.shape(), self.input_size, "Lstm")?;
215        let backend = B::default();
216
217        let mut h = Var::new(
218            Tensor::zeros_on([batch, self.hidden_size], &backend)?,
219            false,
220        )?;
221        let mut c = Var::new(
222            Tensor::zeros_on([batch, self.hidden_size], &backend)?,
223            false,
224        )?;
225
226        let mut outputs: Vec<Var<T, B>> = Vec::with_capacity(seq_len);
227        for t in 0..seq_len {
228            let x_t_3d = coeus_autograd::slice(x, &[(0, batch), (t, t + 1), (0, self.input_size)])?;
229            let x_t = coeus_autograd::reshape(&x_t_3d, vec![batch, self.input_size])?;
230            let (h_new, c_new) = self.cell.step_validated(&x_t, &h, &c, batch)?;
231            outputs.push(coeus_autograd::reshape(
232                &h_new,
233                vec![batch, 1, self.hidden_size],
234            )?);
235            h = h_new;
236            c = c_new;
237        }
238
239        let refs: Vec<&Var<T, B>> = outputs.iter().collect();
240        let output = coeus_autograd::cat(&refs, 1)?;
241        Ok((output, (h, c)))
242    }
243}
244
245impl<T: Float + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default> Module<T, B>
246    for Lstm<T, B>
247where
248    B::DeviceBuffer<T>:
249        coeus_core::CpuAddressableStorage<T> + coeus_core::CpuAddressableStorageMut<T>,
250{
251    fn parameters(&self) -> Vec<Var<T, B>> {
252        self.cell.parameters()
253    }
254
255    fn named_parameters(&self) -> Vec<coeus_autograd::Parameter<T, B>> {
256        prefixed_parameters("cell", &self.cell)
257    }
258
259    /// Returns `output` of shape `[batch, seq_len, hidden_size]`.
260    fn forward(&self, x: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
261        Ok(self.forward_seq(x)?.0)
262    }
263}