Skip to main content

coeus_nn/rnn/
vanilla.rs

1// ── Vanilla (Elman) RNN ──
2
3use crate::linear::Linear;
4use crate::module::{prefixed_parameters, Module, ModuleError, ModuleResult};
5use crate::rnn::validation;
6use crate::rnn::VarPair;
7use coeus_autograd::Var;
8use coeus_core::{Float, MoiraiBackend};
9use coeus_tensor::Tensor;
10
11/// Pointwise nonlinearity applied to a vanilla RNN cell's pre-activation.
12/// Mirrors PyTorch `RNNCell(nonlinearity=...)`.
13#[derive(Clone, Copy, Debug, PartialEq, Eq)]
14pub enum RnnNonlinearity {
15    /// `tanh` (PyTorch default).
16    Tanh,
17    /// `relu`.
18    Relu,
19}
20
21/// Single-timestep Elman (vanilla) RNN cell.
22///
23/// Step computation (PyTorch `RNNCell`):
24/// ```text
25/// h_new = f(x @ W_ih.T + b_ih + h @ W_hh.T + b_hh)
26/// ```
27/// where `f` is [`RnnNonlinearity`] (`tanh` or `relu`).
28#[derive(Clone)]
29pub struct RNNCell<T: Float, B: coeus_ops::BackendOps<T> + Default = MoiraiBackend> {
30    /// Input-to-hidden projection: `[input_size, hidden_size]`.
31    pub w_ih: Linear<T, B>,
32    /// Hidden-to-hidden projection: `[hidden_size, hidden_size]`.
33    pub w_hh: Linear<T, B>,
34    /// Pointwise nonlinearity.
35    pub nonlinearity: RnnNonlinearity,
36    /// Number of input features per timestep.
37    pub input_size: usize,
38    /// Number of hidden features.
39    pub hidden_size: usize,
40}
41
42impl<T: Float + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default> RNNCell<T, B> {
43    /// Create with Kaiming-initialized weights, zero biases, and the given nonlinearity.
44    ///
45    /// # Errors
46    ///
47    /// Returns [`crate::init::InitializationError`] when a size is zero or the
48    /// backend's draw fails.
49    pub fn new(
50        input_size: usize,
51        hidden_size: usize,
52        nonlinearity: RnnNonlinearity,
53    ) -> Result<Self, crate::init::InitializationError<B::Error>>
54    where
55        B: coeus_ops::RandomInitOps<T>,
56    {
57        Ok(Self {
58            w_ih: Linear::new(input_size, hidden_size, true)?,
59            w_hh: Linear::new(hidden_size, hidden_size, true)?,
60            nonlinearity,
61            input_size,
62            hidden_size,
63        })
64    }
65
66    /// Forward step.
67    ///
68    /// - `x`: `[batch, input_size]`
69    /// - `h`: `[batch, hidden_size]`
70    ///
71    /// Returns `h_new` of shape `[batch, hidden_size]`.
72    pub fn step(&self, x: &Var<T, B>, h: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
73        let batch = validation::cell_input(x.tensor.shape(), self.input_size, "RNNCell")?;
74        validation::state(
75            h.tensor.shape(),
76            batch,
77            self.hidden_size,
78            "RNNCell",
79            "hidden state",
80        )?;
81        self.step_validated(x, h)
82    }
83
84    fn step_validated(
85        &self,
86        x: &Var<T, B>,
87        h: &Var<T, B>,
88    ) -> Result<Var<T, B>, ModuleError<B::Error>> {
89        let input = self.w_ih.forward(x)?;
90        let hidden = self.w_hh.forward(h)?;
91        let pre = coeus_autograd::add(&input, &hidden)?;
92        Ok(match self.nonlinearity {
93            RnnNonlinearity::Tanh => coeus_autograd::tanh(&pre)?,
94            RnnNonlinearity::Relu => coeus_autograd::relu(&pre)?,
95        })
96    }
97}
98
99impl<T: Float + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default> Module<T, B>
100    for RNNCell<T, B>
101{
102    fn parameters(&self) -> Vec<Var<T, B>> {
103        let mut p = self.w_ih.parameters();
104        p.extend(self.w_hh.parameters());
105        p
106    }
107
108    fn named_parameters(&self) -> Vec<coeus_autograd::Parameter<T, B>> {
109        let mut parameters = prefixed_parameters("input", &self.w_ih);
110        parameters.extend(prefixed_parameters("hidden", &self.w_hh));
111        parameters
112    }
113
114    fn forward(&self, x: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
115        let batch = validation::cell_input(x.tensor.shape(), self.input_size, "RNNCell")?;
116        let backend = B::default();
117        let h = Var::new(
118            Tensor::zeros_on([batch, self.hidden_size], &backend)?,
119            false,
120        )?;
121        self.step_validated(x, &h)
122    }
123}
124
125// ── Rnn (sequence-level) ──
126
127/// Sequence-level vanilla RNN: unrolls [`RNNCell`] across the time axis.
128///
129/// Input layout: `[batch, seq_len, input_size]`.
130/// Output layout: `[batch, seq_len, hidden_size]`.
131/// Final hidden state: `h_n` of shape `[batch, hidden_size]`.
132#[derive(Clone)]
133pub struct Rnn<T: Float, B: coeus_ops::BackendOps<T> + Default = MoiraiBackend> {
134    cell: RNNCell<T, B>,
135    /// Number of input features per timestep.
136    pub input_size: usize,
137    /// Number of hidden features.
138    pub hidden_size: usize,
139}
140
141impl<T: Float + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default> Rnn<T, B>
142where
143    B::DeviceBuffer<T>:
144        coeus_core::CpuAddressableStorage<T> + coeus_core::CpuAddressableStorageMut<T>,
145{
146    /// Create with Kaiming-initialized weights, zero biases, and the given nonlinearity.
147    ///
148    /// # Errors
149    ///
150    /// Returns [`crate::init::InitializationError`] when a size is zero or the
151    /// backend's draw fails.
152    pub fn new(
153        input_size: usize,
154        hidden_size: usize,
155        nonlinearity: RnnNonlinearity,
156    ) -> Result<Self, crate::init::InitializationError<B::Error>>
157    where
158        B: coeus_ops::RandomInitOps<T>,
159    {
160        Ok(Self {
161            cell: RNNCell::new(input_size, hidden_size, nonlinearity)?,
162            input_size,
163            hidden_size,
164        })
165    }
166
167    /// Unroll over the sequence dimension with a zero initial hidden state.
168    ///
169    /// Returns `(output, h_n)`:
170    /// - `output`: `[batch, seq_len, hidden_size]` — all hidden states stacked.
171    /// - `h_n`: `[batch, hidden_size]` — final hidden state.
172    pub fn forward_seq(&self, x: &Var<T, B>) -> ModuleResult<VarPair<T, B>, B> {
173        let (batch, seq_len) =
174            validation::sequence_input(x.tensor.shape(), self.input_size, "Rnn")?;
175        let backend = B::default();
176
177        let mut h = Var::new(
178            Tensor::zeros_on([batch, self.hidden_size], &backend)?,
179            false,
180        )?;
181        let mut outputs: Vec<Var<T, B>> = Vec::with_capacity(seq_len);
182        for t in 0..seq_len {
183            let x_t_3d = coeus_autograd::slice(x, &[(0, batch), (t, t + 1), (0, self.input_size)])?;
184            let x_t = coeus_autograd::reshape(&x_t_3d, vec![batch, self.input_size])?;
185            let h_new = self.cell.step_validated(&x_t, &h)?;
186            outputs.push(coeus_autograd::reshape(
187                &h_new,
188                vec![batch, 1, self.hidden_size],
189            )?);
190            h = h_new;
191        }
192
193        let refs: Vec<&Var<T, B>> = outputs.iter().collect();
194        let output = coeus_autograd::cat(&refs, 1)?;
195        Ok((output, h))
196    }
197}
198
199impl<T: Float + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default> Module<T, B>
200    for Rnn<T, B>
201where
202    B::DeviceBuffer<T>:
203        coeus_core::CpuAddressableStorage<T> + coeus_core::CpuAddressableStorageMut<T>,
204{
205    fn parameters(&self) -> Vec<Var<T, B>> {
206        self.cell.parameters()
207    }
208
209    fn named_parameters(&self) -> Vec<coeus_autograd::Parameter<T, B>> {
210        prefixed_parameters("cell", &self.cell)
211    }
212
213    /// Returns `output` of shape `[batch, seq_len, hidden_size]`.
214    fn forward(&self, x: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
215        Ok(self.forward_seq(x)?.0)
216    }
217}