1use crate::linear::Linear;
4use crate::module::{prefixed_parameters, Module, ModuleError, ModuleResult};
5use crate::rnn::validation;
6use crate::rnn::VarPair;
7use coeus_autograd::Var;
8use coeus_core::{Float, MoiraiBackend};
9use coeus_tensor::Tensor;
10
11#[derive(Clone)]
24pub struct GRUCell<T: Float, B: coeus_ops::BackendOps<T> + Default = MoiraiBackend> {
25 pub w_ih: Linear<T, B>,
27 pub w_hh: Linear<T, B>,
29 pub input_size: usize,
31 pub hidden_size: usize,
33}
34
35impl<T: Float + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default> GRUCell<T, B> {
36 pub fn new(
43 input_size: usize,
44 hidden_size: usize,
45 ) -> Result<Self, crate::init::InitializationError<B::Error>>
46 where
47 B: coeus_ops::RandomInitOps<T>,
48 {
49 let w_ih = Linear::new(input_size, 3 * hidden_size, true)?;
50 let w_hh = Linear::new(hidden_size, 3 * hidden_size, true)?;
51 Ok(Self {
52 w_ih,
53 w_hh,
54 input_size,
55 hidden_size,
56 })
57 }
58
59 pub fn step(&self, x: &Var<T, B>, h: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
66 let batch = validation::cell_input(x.tensor.shape(), self.input_size, "GRUCell")?;
67 validation::state(
68 h.tensor.shape(),
69 batch,
70 self.hidden_size,
71 "GRUCell",
72 "hidden state",
73 )?;
74 self.step_validated(x, h, batch)
75 }
76
77 fn step_validated(
78 &self,
79 x: &Var<T, B>,
80 h: &Var<T, B>,
81 batch: usize,
82 ) -> Result<Var<T, B>, ModuleError<B::Error>> {
83 let hs = self.hidden_size;
84 let ih = self.w_ih.forward(x)?;
85 let hh = self.w_hh.forward(h)?;
86
87 let slice_ih = |start: usize, end: usize| -> Result<Var<T, B>, B::Error> {
88 coeus_autograd::slice(&ih, &[(0, batch), (start, end)])
89 };
90 let slice_hh = |start: usize, end: usize| -> Result<Var<T, B>, B::Error> {
91 coeus_autograd::slice(&hh, &[(0, batch), (start, end)])
92 };
93
94 let r =
95 coeus_autograd::sigmoid(&coeus_autograd::add(&slice_ih(0, hs)?, &slice_hh(0, hs)?)?)?;
96 let z = coeus_autograd::sigmoid(&coeus_autograd::add(
97 &slice_ih(hs, 2 * hs)?,
98 &slice_hh(hs, 2 * hs)?,
99 )?)?;
100 let n = coeus_autograd::tanh(&coeus_autograd::add(
102 &slice_ih(2 * hs, 3 * hs)?,
103 &coeus_autograd::mul(&r, &slice_hh(2 * hs, 3 * hs)?)?,
104 )?)?;
105
106 let backend = B::default();
108 let ones = Var::new(
109 coeus_tensor::Tensor::ones_on(z.tensor.shape_cloned(), &backend)?,
110 false,
111 )?;
112 let one_minus_z = coeus_autograd::sub(&ones, &z)?;
113 Ok(coeus_autograd::add(
114 &coeus_autograd::mul(&one_minus_z, &n)?,
115 &coeus_autograd::mul(&z, h)?,
116 )?)
117 }
118}
119
120impl<T: Float + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default> Module<T, B>
121 for GRUCell<T, B>
122{
123 fn parameters(&self) -> Vec<Var<T, B>> {
124 let mut p = self.w_ih.parameters();
125 p.extend(self.w_hh.parameters());
126 p
127 }
128
129 fn named_parameters(&self) -> Vec<coeus_autograd::Parameter<T, B>> {
130 let mut parameters = prefixed_parameters("input", &self.w_ih);
131 parameters.extend(prefixed_parameters("hidden", &self.w_hh));
132 parameters
133 }
134
135 fn forward(&self, x: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
136 let batch = validation::cell_input(x.tensor.shape(), self.input_size, "GRUCell")?;
137 let backend = B::default();
138 let h = Var::new(
139 Tensor::zeros_on([batch, self.hidden_size], &backend)?,
140 false,
141 )?;
142 self.step_validated(x, &h, batch)
143 }
144}
145
146#[derive(Clone)]
166pub struct Gru<T: Float, B: coeus_ops::BackendOps<T> + Default = MoiraiBackend> {
167 cell: GRUCell<T, B>,
168 pub input_size: usize,
170 pub hidden_size: usize,
172}
173
174impl<T: Float + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default> Gru<T, B>
175where
176 B::DeviceBuffer<T>:
177 coeus_core::CpuAddressableStorage<T> + coeus_core::CpuAddressableStorageMut<T>,
178{
179 pub fn new(
186 input_size: usize,
187 hidden_size: usize,
188 ) -> Result<Self, crate::init::InitializationError<B::Error>>
189 where
190 B: coeus_ops::RandomInitOps<T>,
191 {
192 Ok(Self {
193 cell: GRUCell::new(input_size, hidden_size)?,
194 input_size,
195 hidden_size,
196 })
197 }
198
199 pub fn from_cell(cell: GRUCell<T, B>) -> Self {
205 Self {
206 input_size: cell.input_size,
207 hidden_size: cell.hidden_size,
208 cell,
209 }
210 }
211
212 pub fn forward_seq(&self, x: &Var<T, B>) -> ModuleResult<VarPair<T, B>, B> {
218 let (batch, seq_len) =
219 validation::sequence_input(x.tensor.shape(), self.input_size, "Gru")?;
220 let backend = B::default();
221
222 let mut h = Var::new(
223 Tensor::zeros_on([batch, self.hidden_size], &backend)?,
224 false,
225 )?;
226
227 let mut outputs: Vec<Var<T, B>> = Vec::with_capacity(seq_len);
228 for t in 0..seq_len {
229 let x_t_3d = coeus_autograd::slice(x, &[(0, batch), (t, t + 1), (0, self.input_size)])?;
230 let x_t = coeus_autograd::reshape(&x_t_3d, vec![batch, self.input_size])?;
231 let h_new = self.cell.step_validated(&x_t, &h, batch)?;
232 outputs.push(coeus_autograd::reshape(
233 &h_new,
234 vec![batch, 1, self.hidden_size],
235 )?);
236 h = h_new;
237 }
238
239 let refs: Vec<&Var<T, B>> = outputs.iter().collect();
240 let output = coeus_autograd::cat(&refs, 1)?;
241 Ok((output, h))
242 }
243}
244
245impl<T: Float + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default> Module<T, B>
246 for Gru<T, B>
247where
248 B::DeviceBuffer<T>:
249 coeus_core::CpuAddressableStorage<T> + coeus_core::CpuAddressableStorageMut<T>,
250{
251 fn parameters(&self) -> Vec<Var<T, B>> {
252 self.cell.parameters()
253 }
254
255 fn named_parameters(&self) -> Vec<coeus_autograd::Parameter<T, B>> {
256 prefixed_parameters("cell", &self.cell)
257 }
258
259 fn forward(&self, x: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
261 Ok(self.forward_seq(x)?.0)
262 }
263}