Skip to main content

coeus_nn/rnn/
gru.rs

1// ── GRUCell ──
2
3use crate::linear::Linear;
4use crate::module::{prefixed_parameters, Module, ModuleError, ModuleResult};
5use crate::rnn::validation;
6use crate::rnn::VarPair;
7use coeus_autograd::Var;
8use coeus_core::{Float, MoiraiBackend};
9use coeus_tensor::Tensor;
10
11/// Single-timestep Gated Recurrent Unit cell.
12///
13/// Gate computation:
14/// ```text
15/// gates_r = sigmoid(x @ W_ih_r.T + h @ W_hh_r.T + b_r)   — reset gate
16/// gates_z = sigmoid(x @ W_ih_z.T + h @ W_hh_z.T + b_z)   — update gate
17/// n       = tanh(x @ W_ih_n.T + r ⊙ (h @ W_hh_n.T) + b_n) — new gate
18///
19/// h_new = (1 - z) ⊙ n + z ⊙ h
20/// ```
21///
22/// Fused implementation uses a single `[3*H]` projection for efficiency.
23#[derive(Clone)]
24pub struct GRUCell<T: Float, B: coeus_ops::BackendOps<T> + Default = MoiraiBackend> {
25    /// Input-to-hidden projection: `[input_size, 3*hidden_size]`.
26    pub w_ih: Linear<T, B>,
27    /// Hidden-to-hidden projection: `[hidden_size, 3*hidden_size]`.
28    pub w_hh: Linear<T, B>,
29    /// Number of input features per timestep.
30    pub input_size: usize,
31    /// Number of hidden features.
32    pub hidden_size: usize,
33}
34
35impl<T: Float + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default> GRUCell<T, B> {
36    /// Create with Kaiming-initialized weights and zero biases.
37    ///
38    /// # Errors
39    ///
40    /// Returns [`crate::init::InitializationError`] when a size is zero or the
41    /// backend's draw fails.
42    pub fn new(
43        input_size: usize,
44        hidden_size: usize,
45    ) -> Result<Self, crate::init::InitializationError<B::Error>>
46    where
47        B: coeus_ops::RandomInitOps<T>,
48    {
49        let w_ih = Linear::new(input_size, 3 * hidden_size, true)?;
50        let w_hh = Linear::new(hidden_size, 3 * hidden_size, true)?;
51        Ok(Self {
52            w_ih,
53            w_hh,
54            input_size,
55            hidden_size,
56        })
57    }
58
59    /// Forward step.
60    ///
61    /// - `x`: `[batch, input_size]`
62    /// - `h`: `[batch, hidden_size]`
63    ///
64    /// Returns `h_new` of shape `[batch, hidden_size]`.
65    pub fn step(&self, x: &Var<T, B>, h: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
66        let batch = validation::cell_input(x.tensor.shape(), self.input_size, "GRUCell")?;
67        validation::state(
68            h.tensor.shape(),
69            batch,
70            self.hidden_size,
71            "GRUCell",
72            "hidden state",
73        )?;
74        self.step_validated(x, h, batch)
75    }
76
77    fn step_validated(
78        &self,
79        x: &Var<T, B>,
80        h: &Var<T, B>,
81        batch: usize,
82    ) -> Result<Var<T, B>, ModuleError<B::Error>> {
83        let hs = self.hidden_size;
84        let ih = self.w_ih.forward(x)?;
85        let hh = self.w_hh.forward(h)?;
86
87        let slice_ih = |start: usize, end: usize| -> Result<Var<T, B>, B::Error> {
88            coeus_autograd::slice(&ih, &[(0, batch), (start, end)])
89        };
90        let slice_hh = |start: usize, end: usize| -> Result<Var<T, B>, B::Error> {
91            coeus_autograd::slice(&hh, &[(0, batch), (start, end)])
92        };
93
94        let r =
95            coeus_autograd::sigmoid(&coeus_autograd::add(&slice_ih(0, hs)?, &slice_hh(0, hs)?)?)?;
96        let z = coeus_autograd::sigmoid(&coeus_autograd::add(
97            &slice_ih(hs, 2 * hs)?,
98            &slice_hh(hs, 2 * hs)?,
99        )?)?;
100        // n = tanh(ih_n + r * hh_n)
101        let n = coeus_autograd::tanh(&coeus_autograd::add(
102            &slice_ih(2 * hs, 3 * hs)?,
103            &coeus_autograd::mul(&r, &slice_hh(2 * hs, 3 * hs)?)?,
104        )?)?;
105
106        // h_new = (1 - z) * n + z * h
107        let backend = B::default();
108        let ones = Var::new(
109            coeus_tensor::Tensor::ones_on(z.tensor.shape_cloned(), &backend)?,
110            false,
111        )?;
112        let one_minus_z = coeus_autograd::sub(&ones, &z)?;
113        Ok(coeus_autograd::add(
114            &coeus_autograd::mul(&one_minus_z, &n)?,
115            &coeus_autograd::mul(&z, h)?,
116        )?)
117    }
118}
119
120impl<T: Float + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default> Module<T, B>
121    for GRUCell<T, B>
122{
123    fn parameters(&self) -> Vec<Var<T, B>> {
124        let mut p = self.w_ih.parameters();
125        p.extend(self.w_hh.parameters());
126        p
127    }
128
129    fn named_parameters(&self) -> Vec<coeus_autograd::Parameter<T, B>> {
130        let mut parameters = prefixed_parameters("input", &self.w_ih);
131        parameters.extend(prefixed_parameters("hidden", &self.w_hh));
132        parameters
133    }
134
135    fn forward(&self, x: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
136        let batch = validation::cell_input(x.tensor.shape(), self.input_size, "GRUCell")?;
137        let backend = B::default();
138        let h = Var::new(
139            Tensor::zeros_on([batch, self.hidden_size], &backend)?,
140            false,
141        )?;
142        self.step_validated(x, &h, batch)
143    }
144}
145
146// ── Gru (sequence-level) ──
147
148/// Sequence-level GRU module: unrolls [`GRUCell`] across the time axis.
149///
150/// Input layout: `[batch, seq_len, input_size]`.
151/// Output layout: `[batch, seq_len, hidden_size]`.
152/// Final hidden state: `h_n` of shape `[batch, hidden_size]`.
153///
154/// Initial hidden state defaults to zeros; use [`Gru::forward_seq`]
155/// for full `(output, h_n)` when the final state is needed, or
156/// `Module::forward` when only the output sequence is needed.
157///
158/// # Example
159/// ```text
160/// zeros input ⟹ all gate pre-activations = 0
161///   r = z = σ(0) = 0.5,  n = tanh(0 + 0.5·0) = 0
162///   h_new = (1−z)·n + z·h = 0
163/// ∴ output is all zeros for any sequence length.
164/// ```
165#[derive(Clone)]
166pub struct Gru<T: Float, B: coeus_ops::BackendOps<T> + Default = MoiraiBackend> {
167    cell: GRUCell<T, B>,
168    /// Number of input features per timestep.
169    pub input_size: usize,
170    /// Number of hidden features.
171    pub hidden_size: usize,
172}
173
174impl<T: Float + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default> Gru<T, B>
175where
176    B::DeviceBuffer<T>:
177        coeus_core::CpuAddressableStorage<T> + coeus_core::CpuAddressableStorageMut<T>,
178{
179    /// Create with Kaiming-initialized weights and zero biases.
180    ///
181    /// # Errors
182    ///
183    /// Returns [`crate::init::InitializationError`] when a size is zero or the
184    /// backend's draw fails.
185    pub fn new(
186        input_size: usize,
187        hidden_size: usize,
188    ) -> Result<Self, crate::init::InitializationError<B::Error>>
189    where
190        B: coeus_ops::RandomInitOps<T>,
191    {
192        Ok(Self {
193            cell: GRUCell::new(input_size, hidden_size)?,
194            input_size,
195            hidden_size,
196        })
197    }
198
199    /// Wrap an existing cell, taking the sizes from the cell itself.
200    ///
201    /// Used by bindings that own the weights as Python tensors and rebuild the
202    /// cell around them on each call, so the unrolled forward runs over the
203    /// caller's parameters rather than a fresh initialization.
204    pub fn from_cell(cell: GRUCell<T, B>) -> Self {
205        Self {
206            input_size: cell.input_size,
207            hidden_size: cell.hidden_size,
208            cell,
209        }
210    }
211
212    /// Unroll over the sequence dimension with zero initial hidden state.
213    ///
214    /// Returns `(output, h_n)`:
215    /// - `output`: `[batch, seq_len, hidden_size]` — all hidden states stacked.
216    /// - `h_n`: `[batch, hidden_size]` — final hidden state.
217    pub fn forward_seq(&self, x: &Var<T, B>) -> ModuleResult<VarPair<T, B>, B> {
218        let (batch, seq_len) =
219            validation::sequence_input(x.tensor.shape(), self.input_size, "Gru")?;
220        let backend = B::default();
221
222        let mut h = Var::new(
223            Tensor::zeros_on([batch, self.hidden_size], &backend)?,
224            false,
225        )?;
226
227        let mut outputs: Vec<Var<T, B>> = Vec::with_capacity(seq_len);
228        for t in 0..seq_len {
229            let x_t_3d = coeus_autograd::slice(x, &[(0, batch), (t, t + 1), (0, self.input_size)])?;
230            let x_t = coeus_autograd::reshape(&x_t_3d, vec![batch, self.input_size])?;
231            let h_new = self.cell.step_validated(&x_t, &h, batch)?;
232            outputs.push(coeus_autograd::reshape(
233                &h_new,
234                vec![batch, 1, self.hidden_size],
235            )?);
236            h = h_new;
237        }
238
239        let refs: Vec<&Var<T, B>> = outputs.iter().collect();
240        let output = coeus_autograd::cat(&refs, 1)?;
241        Ok((output, h))
242    }
243}
244
245impl<T: Float + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default> Module<T, B>
246    for Gru<T, B>
247where
248    B::DeviceBuffer<T>:
249        coeus_core::CpuAddressableStorage<T> + coeus_core::CpuAddressableStorageMut<T>,
250{
251    fn parameters(&self) -> Vec<Var<T, B>> {
252        self.cell.parameters()
253    }
254
255    fn named_parameters(&self) -> Vec<coeus_autograd::Parameter<T, B>> {
256        prefixed_parameters("cell", &self.cell)
257    }
258
259    /// Returns `output` of shape `[batch, seq_len, hidden_size]`.
260    fn forward(&self, x: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
261        Ok(self.forward_seq(x)?.0)
262    }
263}