1use crate::linear::Linear;
4use crate::module::{prefixed_parameters, Module, ModuleError, ModuleResult};
5use crate::rnn::validation;
6use crate::rnn::VarPair;
7use coeus_autograd::Var;
8use coeus_core::{Float, MoiraiBackend};
9use coeus_tensor::Tensor;
10
11type SequenceOutput<T, B> = (Var<T, B>, VarPair<T, B>);
13
14#[derive(Clone)]
28pub struct LSTMCell<T: Float, B: coeus_ops::BackendOps<T> + Default = MoiraiBackend> {
29 pub w_ih: Linear<T, B>,
31 pub w_hh: Linear<T, B>,
33 pub input_size: usize,
35 pub hidden_size: usize,
37}
38
39impl<T: Float + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default> LSTMCell<T, B> {
40 pub fn new(
47 input_size: usize,
48 hidden_size: usize,
49 ) -> Result<Self, crate::init::InitializationError<B::Error>>
50 where
51 B: coeus_ops::RandomInitOps<T>,
52 {
53 let w_ih = Linear::new(input_size, 4 * hidden_size, true)?;
54 let w_hh = Linear::new(hidden_size, 4 * hidden_size, true)?;
55 Ok(Self {
56 w_ih,
57 w_hh,
58 input_size,
59 hidden_size,
60 })
61 }
62
63 pub fn step(
71 &self,
72 x: &Var<T, B>,
73 h: &Var<T, B>,
74 c: &Var<T, B>,
75 ) -> ModuleResult<VarPair<T, B>, B> {
76 let batch = validation::cell_input(x.tensor.shape(), self.input_size, "LSTMCell")?;
77 validation::state(
78 h.tensor.shape(),
79 batch,
80 self.hidden_size,
81 "LSTMCell",
82 "hidden state",
83 )?;
84 validation::state(
85 c.tensor.shape(),
86 batch,
87 self.hidden_size,
88 "LSTMCell",
89 "cell state",
90 )?;
91 self.step_validated(x, h, c, batch)
92 }
93
94 fn step_validated(
95 &self,
96 x: &Var<T, B>,
97 h: &Var<T, B>,
98 c: &Var<T, B>,
99 batch: usize,
100 ) -> ModuleResult<VarPair<T, B>, B> {
101 let hs = self.hidden_size;
102 let input = self.w_ih.forward(x)?;
103 let hidden = self.w_hh.forward(h)?;
104 let gates = coeus_autograd::add(&input, &hidden)?;
105
106 let slice = |start: usize, end: usize| -> Result<Var<T, B>, B::Error> {
107 coeus_autograd::slice(&gates, &[(0, batch), (start, end)])
108 };
109
110 let i_g = coeus_autograd::sigmoid(&slice(0, hs)?)?;
111 let f_g = coeus_autograd::sigmoid(&slice(hs, 2 * hs)?)?;
112 let g_g = coeus_autograd::tanh(&slice(2 * hs, 3 * hs)?)?;
113 let o_g = coeus_autograd::sigmoid(&slice(3 * hs, 4 * hs)?)?;
114
115 let c_new = coeus_autograd::add(
116 &coeus_autograd::mul(&f_g, c)?,
117 &coeus_autograd::mul(&i_g, &g_g)?,
118 )?;
119 let h_new = coeus_autograd::mul(&o_g, &coeus_autograd::tanh(&c_new)?)?;
120 Ok((h_new, c_new))
121 }
122}
123
124impl<T: Float + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default> Module<T, B>
125 for LSTMCell<T, B>
126{
127 fn parameters(&self) -> Vec<Var<T, B>> {
128 let mut p = self.w_ih.parameters();
129 p.extend(self.w_hh.parameters());
130 p
131 }
132
133 fn named_parameters(&self) -> Vec<coeus_autograd::Parameter<T, B>> {
134 let mut parameters = prefixed_parameters("input", &self.w_ih);
135 parameters.extend(prefixed_parameters("hidden", &self.w_hh));
136 parameters
137 }
138
139 fn forward(&self, x: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
140 let batch = validation::cell_input(x.tensor.shape(), self.input_size, "LSTMCell")?;
141 let backend = B::default();
142 let h = Var::new(
143 Tensor::zeros_on([batch, self.hidden_size], &backend)?,
144 false,
145 )?;
146 let c = Var::new(
147 Tensor::zeros_on([batch, self.hidden_size], &backend)?,
148 false,
149 )?;
150 Ok(self.step_validated(x, &h, &c, batch)?.0)
151 }
152}
153
154#[derive(Clone)]
174pub struct Lstm<T: Float, B: coeus_ops::BackendOps<T> + Default = MoiraiBackend> {
175 cell: LSTMCell<T, B>,
176 pub input_size: usize,
178 pub hidden_size: usize,
180}
181
182impl<T: Float + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default> Lstm<T, B>
183where
184 B::DeviceBuffer<T>:
185 coeus_core::CpuAddressableStorage<T> + coeus_core::CpuAddressableStorageMut<T>,
186{
187 pub fn new(
194 input_size: usize,
195 hidden_size: usize,
196 ) -> Result<Self, crate::init::InitializationError<B::Error>>
197 where
198 B: coeus_ops::RandomInitOps<T>,
199 {
200 Ok(Self {
201 cell: LSTMCell::new(input_size, hidden_size)?,
202 input_size,
203 hidden_size,
204 })
205 }
206
207 pub fn forward_seq(&self, x: &Var<T, B>) -> ModuleResult<SequenceOutput<T, B>, B> {
213 let (batch, seq_len) =
214 validation::sequence_input(x.tensor.shape(), self.input_size, "Lstm")?;
215 let backend = B::default();
216
217 let mut h = Var::new(
218 Tensor::zeros_on([batch, self.hidden_size], &backend)?,
219 false,
220 )?;
221 let mut c = Var::new(
222 Tensor::zeros_on([batch, self.hidden_size], &backend)?,
223 false,
224 )?;
225
226 let mut outputs: Vec<Var<T, B>> = Vec::with_capacity(seq_len);
227 for t in 0..seq_len {
228 let x_t_3d = coeus_autograd::slice(x, &[(0, batch), (t, t + 1), (0, self.input_size)])?;
229 let x_t = coeus_autograd::reshape(&x_t_3d, vec![batch, self.input_size])?;
230 let (h_new, c_new) = self.cell.step_validated(&x_t, &h, &c, batch)?;
231 outputs.push(coeus_autograd::reshape(
232 &h_new,
233 vec![batch, 1, self.hidden_size],
234 )?);
235 h = h_new;
236 c = c_new;
237 }
238
239 let refs: Vec<&Var<T, B>> = outputs.iter().collect();
240 let output = coeus_autograd::cat(&refs, 1)?;
241 Ok((output, (h, c)))
242 }
243}
244
245impl<T: Float + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default> Module<T, B>
246 for Lstm<T, B>
247where
248 B::DeviceBuffer<T>:
249 coeus_core::CpuAddressableStorage<T> + coeus_core::CpuAddressableStorageMut<T>,
250{
251 fn parameters(&self) -> Vec<Var<T, B>> {
252 self.cell.parameters()
253 }
254
255 fn named_parameters(&self) -> Vec<coeus_autograd::Parameter<T, B>> {
256 prefixed_parameters("cell", &self.cell)
257 }
258
259 fn forward(&self, x: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
261 Ok(self.forward_seq(x)?.0)
262 }
263}