1use crate::linear::Linear;
4use crate::module::{prefixed_parameters, Module, ModuleError, ModuleResult};
5use crate::rnn::validation;
6use crate::rnn::VarPair;
7use coeus_autograd::Var;
8use coeus_core::{Float, MoiraiBackend};
9use coeus_tensor::Tensor;
10
11#[derive(Clone, Copy, Debug, PartialEq, Eq)]
14pub enum RnnNonlinearity {
15 Tanh,
17 Relu,
19}
20
21#[derive(Clone)]
29pub struct RNNCell<T: Float, B: coeus_ops::BackendOps<T> + Default = MoiraiBackend> {
30 pub w_ih: Linear<T, B>,
32 pub w_hh: Linear<T, B>,
34 pub nonlinearity: RnnNonlinearity,
36 pub input_size: usize,
38 pub hidden_size: usize,
40}
41
42impl<T: Float + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default> RNNCell<T, B> {
43 pub fn new(
50 input_size: usize,
51 hidden_size: usize,
52 nonlinearity: RnnNonlinearity,
53 ) -> Result<Self, crate::init::InitializationError<B::Error>>
54 where
55 B: coeus_ops::RandomInitOps<T>,
56 {
57 Ok(Self {
58 w_ih: Linear::new(input_size, hidden_size, true)?,
59 w_hh: Linear::new(hidden_size, hidden_size, true)?,
60 nonlinearity,
61 input_size,
62 hidden_size,
63 })
64 }
65
66 pub fn step(&self, x: &Var<T, B>, h: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
73 let batch = validation::cell_input(x.tensor.shape(), self.input_size, "RNNCell")?;
74 validation::state(
75 h.tensor.shape(),
76 batch,
77 self.hidden_size,
78 "RNNCell",
79 "hidden state",
80 )?;
81 self.step_validated(x, h)
82 }
83
84 fn step_validated(
85 &self,
86 x: &Var<T, B>,
87 h: &Var<T, B>,
88 ) -> Result<Var<T, B>, ModuleError<B::Error>> {
89 let input = self.w_ih.forward(x)?;
90 let hidden = self.w_hh.forward(h)?;
91 let pre = coeus_autograd::add(&input, &hidden)?;
92 Ok(match self.nonlinearity {
93 RnnNonlinearity::Tanh => coeus_autograd::tanh(&pre)?,
94 RnnNonlinearity::Relu => coeus_autograd::relu(&pre)?,
95 })
96 }
97}
98
99impl<T: Float + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default> Module<T, B>
100 for RNNCell<T, B>
101{
102 fn parameters(&self) -> Vec<Var<T, B>> {
103 let mut p = self.w_ih.parameters();
104 p.extend(self.w_hh.parameters());
105 p
106 }
107
108 fn named_parameters(&self) -> Vec<coeus_autograd::Parameter<T, B>> {
109 let mut parameters = prefixed_parameters("input", &self.w_ih);
110 parameters.extend(prefixed_parameters("hidden", &self.w_hh));
111 parameters
112 }
113
114 fn forward(&self, x: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
115 let batch = validation::cell_input(x.tensor.shape(), self.input_size, "RNNCell")?;
116 let backend = B::default();
117 let h = Var::new(
118 Tensor::zeros_on([batch, self.hidden_size], &backend)?,
119 false,
120 )?;
121 self.step_validated(x, &h)
122 }
123}
124
125#[derive(Clone)]
133pub struct Rnn<T: Float, B: coeus_ops::BackendOps<T> + Default = MoiraiBackend> {
134 cell: RNNCell<T, B>,
135 pub input_size: usize,
137 pub hidden_size: usize,
139}
140
141impl<T: Float + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default> Rnn<T, B>
142where
143 B::DeviceBuffer<T>:
144 coeus_core::CpuAddressableStorage<T> + coeus_core::CpuAddressableStorageMut<T>,
145{
146 pub fn new(
153 input_size: usize,
154 hidden_size: usize,
155 nonlinearity: RnnNonlinearity,
156 ) -> Result<Self, crate::init::InitializationError<B::Error>>
157 where
158 B: coeus_ops::RandomInitOps<T>,
159 {
160 Ok(Self {
161 cell: RNNCell::new(input_size, hidden_size, nonlinearity)?,
162 input_size,
163 hidden_size,
164 })
165 }
166
167 pub fn forward_seq(&self, x: &Var<T, B>) -> ModuleResult<VarPair<T, B>, B> {
173 let (batch, seq_len) =
174 validation::sequence_input(x.tensor.shape(), self.input_size, "Rnn")?;
175 let backend = B::default();
176
177 let mut h = Var::new(
178 Tensor::zeros_on([batch, self.hidden_size], &backend)?,
179 false,
180 )?;
181 let mut outputs: Vec<Var<T, B>> = Vec::with_capacity(seq_len);
182 for t in 0..seq_len {
183 let x_t_3d = coeus_autograd::slice(x, &[(0, batch), (t, t + 1), (0, self.input_size)])?;
184 let x_t = coeus_autograd::reshape(&x_t_3d, vec![batch, self.input_size])?;
185 let h_new = self.cell.step_validated(&x_t, &h)?;
186 outputs.push(coeus_autograd::reshape(
187 &h_new,
188 vec![batch, 1, self.hidden_size],
189 )?);
190 h = h_new;
191 }
192
193 let refs: Vec<&Var<T, B>> = outputs.iter().collect();
194 let output = coeus_autograd::cat(&refs, 1)?;
195 Ok((output, h))
196 }
197}
198
199impl<T: Float + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default> Module<T, B>
200 for Rnn<T, B>
201where
202 B::DeviceBuffer<T>:
203 coeus_core::CpuAddressableStorage<T> + coeus_core::CpuAddressableStorageMut<T>,
204{
205 fn parameters(&self) -> Vec<Var<T, B>> {
206 self.cell.parameters()
207 }
208
209 fn named_parameters(&self) -> Vec<coeus_autograd::Parameter<T, B>> {
210 prefixed_parameters("cell", &self.cell)
211 }
212
213 fn forward(&self, x: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
215 Ok(self.forward_seq(x)?.0)
216 }
217}