1use crate::err::Error;
2use crate::exec::catalog::op;
3use crate::nn::init::Init;
4use crate::nn::linear::LinearShape;
5use crate::nn::optional::{False, True};
6use crate::nn::param::{Frozen, TrainState, Trainable};
7use crate::nn::{Linear, Module, VisitParameters};
8use crate::shapes::shape::{DimCons, Nil};
9use crate::shapes::{Dim, Dyn, DynShape, Shape, ShapeValue};
10use crate::tensor::backend::Execute;
11use crate::tensor::base::Tensor;
12use crate::tensor::device::ConstDevice;
13use crate::tensor::dtype::{ConstDType, DType};
14use crate::tensor::grad::{Grad, GradJoin, JoinedGrad, RequiresGrad};
15use alloc::string::ToString;
16use alloc::vec::Vec;
17
18type D2<A, B> = DimCons<A, DimCons<B, Nil>>;
19type D3<A, B, C> = DimCons<A, DimCons<B, DimCons<C, Nil>>>;
20
21pub trait RnnShape: Shape + DynShape {
27 type In: Dim;
29 type Out: Dim;
31 type IhShape: LinearShape<InF = Self::In, OutF = Self::Out>;
33 type HhShape: LinearShape<InF = Self::Out, OutF = Self::Out>;
35}
36
37impl<In: Dim, Out: Dim> RnnShape
38 for crate::shapes::shape::DimCons<
39 In,
40 crate::shapes::shape::DimCons<Out, crate::shapes::shape::Nil>,
41 >
42{
43 type In = In;
44 type Out = Out;
45 type IhShape = crate::shapes::shape::DimCons<
46 In,
47 crate::shapes::shape::DimCons<Out, crate::shapes::shape::Nil>,
48 >;
49 type HhShape = crate::shapes::shape::DimCons<
50 Out,
51 crate::shapes::shape::DimCons<Out, crate::shapes::shape::Nil>,
52 >;
53}
54
55impl RnnShape for Dyn {
56 type In = usize;
58 type Out = usize;
60 type IhShape = Dyn;
61 type HhShape = Dyn;
62}
63
64pub struct RNNCellBuilder<
74 S: RnnShape,
75 BiasIh: crate::nn::optional::OptionalField = True,
76 BiasHh: crate::nn::optional::OptionalField = True,
77 Train: TrainState = Trainable,
78> {
79 pub shape: ShapeValue<S>,
81 pub input_weight_init: Init,
83 pub hidden_weight_init: Init,
85 pub input_bias_init: Init,
87 pub hidden_bias_init: Init,
89 pub _phantom: core::marker::PhantomData<(BiasIh, BiasHh, Train)>,
91}
92
93impl<
94 S: RnnShape,
95 BiasIh: crate::nn::optional::OptionalField,
96 BiasHh: crate::nn::optional::OptionalField,
97 Train: TrainState,
98> RNNCellBuilder<S, BiasIh, BiasHh, Train>
99{
100 pub fn shape(&self) -> &ShapeValue<S> {
102 &self.shape
103 }
104
105 pub fn no_input_bias(self) -> RNNCellBuilder<S, False, BiasHh, Train> {
107 RNNCellBuilder {
108 shape: self.shape,
109 input_weight_init: self.input_weight_init,
110 hidden_weight_init: self.hidden_weight_init,
111 input_bias_init: self.input_bias_init,
112 hidden_bias_init: self.hidden_bias_init,
113 _phantom: core::marker::PhantomData,
114 }
115 }
116
117 pub fn no_hidden_bias(self) -> RNNCellBuilder<S, BiasIh, False, Train> {
119 RNNCellBuilder {
120 shape: self.shape,
121 input_weight_init: self.input_weight_init,
122 hidden_weight_init: self.hidden_weight_init,
123 input_bias_init: self.input_bias_init,
124 hidden_bias_init: self.hidden_bias_init,
125 _phantom: core::marker::PhantomData,
126 }
127 }
128
129 pub fn no_bias(self) -> RNNCellBuilder<S, False, False, Train> {
131 RNNCellBuilder {
132 shape: self.shape,
133 input_weight_init: self.input_weight_init,
134 hidden_weight_init: self.hidden_weight_init,
135 input_bias_init: self.input_bias_init,
136 hidden_bias_init: self.hidden_bias_init,
137 _phantom: core::marker::PhantomData,
138 }
139 }
140
141 pub fn frozen(self) -> RNNCellBuilder<S, BiasIh, BiasHh, Frozen> {
143 RNNCellBuilder {
144 shape: self.shape,
145 input_weight_init: self.input_weight_init,
146 hidden_weight_init: self.hidden_weight_init,
147 input_bias_init: self.input_bias_init,
148 hidden_bias_init: self.hidden_bias_init,
149 _phantom: core::marker::PhantomData,
150 }
151 }
152
153 pub fn input_weight_init(mut self, init: Init) -> Self {
155 self.input_weight_init = init;
156 self
157 }
158
159 pub fn hidden_weight_init(mut self, init: Init) -> Self {
161 self.hidden_weight_init = init;
162 self
163 }
164
165 pub fn input_bias_init(mut self, init: Init) -> Self {
167 self.input_bias_init = init;
168 self
169 }
170
171 pub fn hidden_bias_init(mut self, init: Init) -> Self {
173 self.hidden_bias_init = init;
174 self
175 }
176}
177
178pub fn rnn_cell<S: RnnShape>(shape: ShapeValue<S>) -> RNNCellBuilder<S> {
180 let init = crate::nn::init::kaiming_uniform();
181 RNNCellBuilder {
182 shape,
183 input_weight_init: init,
184 hidden_weight_init: init,
185 input_bias_init: init,
186 hidden_bias_init: init,
187 _phantom: core::marker::PhantomData,
188 }
189}
190
191pub struct RNNBuilder<
199 S: RnnShape,
200 BiasIh: crate::nn::optional::OptionalField = True,
201 BiasHh: crate::nn::optional::OptionalField = True,
202 Train: TrainState = Trainable,
203> {
204 pub cell: RNNCellBuilder<S, BiasIh, BiasHh, Train>,
206}
207
208impl<
209 S: RnnShape,
210 BiasIh: crate::nn::optional::OptionalField,
211 BiasHh: crate::nn::optional::OptionalField,
212 Train: TrainState,
213> RNNBuilder<S, BiasIh, BiasHh, Train>
214{
215 pub fn shape(&self) -> &ShapeValue<S> {
217 self.cell.shape()
218 }
219
220 pub fn no_input_bias(self) -> RNNBuilder<S, False, BiasHh, Train> {
222 RNNBuilder {
223 cell: self.cell.no_input_bias(),
224 }
225 }
226
227 pub fn no_hidden_bias(self) -> RNNBuilder<S, BiasIh, False, Train> {
229 RNNBuilder {
230 cell: self.cell.no_hidden_bias(),
231 }
232 }
233
234 pub fn no_bias(self) -> RNNBuilder<S, False, False, Train> {
236 RNNBuilder {
237 cell: self.cell.no_bias(),
238 }
239 }
240
241 pub fn frozen(self) -> RNNBuilder<S, BiasIh, BiasHh, Frozen> {
243 RNNBuilder {
244 cell: self.cell.frozen(),
245 }
246 }
247
248 pub fn input_weight_init(mut self, init: Init) -> Self {
250 self.cell = self.cell.input_weight_init(init);
251 self
252 }
253
254 pub fn hidden_weight_init(mut self, init: Init) -> Self {
256 self.cell = self.cell.hidden_weight_init(init);
257 self
258 }
259
260 pub fn input_bias_init(mut self, init: Init) -> Self {
262 self.cell = self.cell.input_bias_init(init);
263 self
264 }
265
266 pub fn hidden_bias_init(mut self, init: Init) -> Self {
268 self.cell = self.cell.hidden_bias_init(init);
269 self
270 }
271}
272
273pub fn rnn<S: RnnShape>(shape: ShapeValue<S>) -> RNNBuilder<S> {
275 RNNBuilder {
276 cell: rnn_cell(shape),
277 }
278}
279
280#[derive(Debug, Clone)]
292pub struct RNNCell<
293 S: RnnShape,
294 B: crate::tensor::backend::VariableBackend,
295 BiasIh: crate::nn::optional::OptionalField = True,
296 BiasHh: crate::nn::optional::OptionalField = True,
297 K: DType = f32,
298 Train: TrainState = Trainable,
299> where
300 S::IhShape: LinearShape,
301 S::HhShape: LinearShape,
302{
303 pub wi: Linear<S::IhShape, B, BiasIh, K, Train>,
305 pub wh: Linear<S::HhShape, B, BiasHh, K, Train>,
307}
308
309impl<
310 S: RnnShape,
311 B: crate::tensor::backend::VariableBackend,
312 BiasIh: crate::nn::optional::OptionalField,
313 BiasHh: crate::nn::optional::OptionalField,
314 K: DType,
315 Train: TrainState,
316> RNNCell<S, B, BiasIh, BiasHh, K, Train>
317{
318 pub fn new(
320 wi: Linear<S::IhShape, B, BiasIh, K, Train>,
321 wh: Linear<S::HhShape, B, BiasHh, K, Train>,
322 ) -> Self {
323 Self { wi, wh }
324 }
325
326 pub fn freeze(self) -> RNNCell<S, B, BiasIh, BiasHh, K, Frozen> {
328 RNNCell {
329 wi: self.wi.freeze(),
330 wh: self.wh.freeze(),
331 }
332 }
333
334 pub fn unfreeze(self) -> RNNCell<S, B, BiasIh, BiasHh, K, Trainable> {
336 RNNCell {
337 wi: self.wi.unfreeze(),
338 wh: self.wh.unfreeze(),
339 }
340 }
341}
342
343impl<
344 S: RnnShape,
345 B: crate::tensor::backend::VariableBackend,
346 BiasIh: crate::nn::optional::OptionalField,
347 BiasHh: crate::nn::optional::OptionalField,
348 K: DType,
349 Train: TrainState,
350> VisitParameters<B> for RNNCell<S, B, BiasIh, BiasHh, K, Train>
351where
352 Linear<S::IhShape, B, BiasIh, K, Train>: VisitParameters<B>,
353 Linear<S::HhShape, B, BiasHh, K, Train>: VisitParameters<B>,
354{
355 fn visit_parameters<V: crate::nn::ParameterVisitor<B>>(
356 &self,
357 path: &crate::nn::StatePath,
358 visitor: &mut V,
359 ) -> crate::err::Result<()> {
360 self.wi.visit_parameters(&path.try_child("wi")?, visitor)?;
361 self.wh.visit_parameters(&path.try_child("wh")?, visitor)
362 }
363}
364
365impl<
366 S: RnnShape,
367 Batch: Dim,
368 B: crate::tensor::backend::VariableBackend + Execute<op::Add> + Execute<op::Tanh>,
369 BiasIh: crate::nn::optional::OptionalField,
370 BiasHh: crate::nn::optional::OptionalField,
371 K: DType,
372 Train: TrainState,
373 G: RequiresGrad,
374>
375 Module<(
376 Tensor<D2<Batch, S::In>, B, K, G>,
377 Tensor<D2<Batch, S::Out>, B, K, G>,
378 )> for RNNCell<S, B, BiasIh, BiasHh, K, Train>
379where
380 Linear<S::IhShape, B, BiasIh, K, Train>: Module<
381 Tensor<D2<Batch, S::In>, B, K, G>,
382 Output = Tensor<D2<Batch, S::Out>, B, K, JoinedGrad<G, Train::TensorGrad>>,
383 Error = Error,
384 >,
385 Linear<S::HhShape, B, BiasHh, K, Train>: Module<
386 Tensor<D2<Batch, S::Out>, B, K, G>,
387 Output = Tensor<D2<Batch, S::Out>, B, K, JoinedGrad<G, Train::TensorGrad>>,
388 Error = Error,
389 >,
390 <B as Execute<op::Add>>::Output: Into<B::Storage<K>>,
391 <B as Execute<op::Tanh>>::Output: Into<B::Storage<K>>,
392 G: GradJoin<Train::TensorGrad>,
393{
394 type Output = Tensor<D2<Batch, S::Out>, B, K, JoinedGrad<G, Train::TensorGrad>>;
396 type Error = Error;
398
399 #[inline]
400 fn forward(
402 &self,
403 (x, h_prev): (
404 Tensor<D2<Batch, S::In>, B, K, G>,
405 Tensor<D2<Batch, S::Out>, B, K, G>,
406 ),
407 ) -> core::result::Result<Self::Output, Error> {
408 let i = self.wi.forward(x)?;
409 let h = self.wh.forward(h_prev)?;
410 let sum = i.add_exact(&h)?;
412 sum.tanh()
413 }
414}
415
416#[derive(Debug, Clone)]
452pub struct RNN<
453 S: RnnShape,
454 B: crate::tensor::backend::VariableBackend,
455 BiasIh: crate::nn::optional::OptionalField = True,
456 BiasHh: crate::nn::optional::OptionalField = True,
457 K: DType = f32,
458 Train: TrainState = Trainable,
459 G: RequiresGrad = Grad,
460> {
461 pub cell: RNNCell<S, B, BiasIh, BiasHh, K, Train>,
463 _grad: core::marker::PhantomData<G>,
464}
465
466impl<
467 S: RnnShape,
468 B: crate::tensor::backend::VariableBackend,
469 BiasIh: crate::nn::optional::OptionalField,
470 BiasHh: crate::nn::optional::OptionalField,
471 K: DType,
472 Train: TrainState,
473 G: RequiresGrad,
474> RNN<S, B, BiasIh, BiasHh, K, Train, G>
475{
476 pub fn new(cell: RNNCell<S, B, BiasIh, BiasHh, K, Train>) -> Self {
478 Self {
479 cell,
480 _grad: core::marker::PhantomData,
481 }
482 }
483
484 pub fn freeze(self) -> RNN<S, B, BiasIh, BiasHh, K, Frozen, G> {
486 RNN {
487 cell: self.cell.freeze(),
488 _grad: core::marker::PhantomData,
489 }
490 }
491
492 pub fn unfreeze(self) -> RNN<S, B, BiasIh, BiasHh, K, Trainable, G> {
494 RNN {
495 cell: self.cell.unfreeze(),
496 _grad: core::marker::PhantomData,
497 }
498 }
499}
500
501impl<
502 S: RnnShape,
503 B: crate::tensor::backend::VariableBackend,
504 BiasIh: crate::nn::optional::OptionalField,
505 BiasHh: crate::nn::optional::OptionalField,
506 K: DType,
507 Train: TrainState,
508 G: RequiresGrad,
509> VisitParameters<B> for RNN<S, B, BiasIh, BiasHh, K, Train, G>
510where
511 RNNCell<S, B, BiasIh, BiasHh, K, Train>: VisitParameters<B>,
512{
513 fn visit_parameters<V: crate::nn::ParameterVisitor<B>>(
514 &self,
515 path: &crate::nn::StatePath,
516 visitor: &mut V,
517 ) -> crate::err::Result<()> {
518 self.cell
519 .visit_parameters(&path.try_child("cell")?, visitor)
520 }
521}
522
523impl<
524 S: RnnShape,
525 Batch: Dim<Arg = ()>,
526 Seq: Dim<Arg = ()>,
527 B: crate::tensor::backend::VariableBackend
528 + Execute<op::StackExact>
529 + Execute<op::Narrow>
530 + Execute<op::SqueezeExact>
531 + crate::exec::Capabilities,
532 BiasIh: crate::nn::optional::OptionalField,
533 BiasHh: crate::nn::optional::OptionalField,
534 K: ConstDType,
535 G: RequiresGrad,
536 Train: TrainState,
537>
538 Module<(
539 Tensor<D3<Batch, Seq, S::In>, B, K, G>,
540 Tensor<D2<Batch, S::Out>, B, K, G>,
541 )> for RNN<S, B, BiasIh, BiasHh, K, Train, G>
542where
543 <B as Execute<op::StackExact>>::Output: Into<B::Storage<K>>,
544 <B as Execute<op::Narrow>>::Output: Into<B::Storage<K>>,
545 <B as Execute<op::SqueezeExact>>::Output: Into<B::Storage<K>>,
546 S::In: Dim<Arg = ()>,
547 S::Out: Dim<Arg = ()>,
548 K: ConstDType,
549 B::Device: ConstDevice,
550 G: GradJoin<Train::TensorGrad, Output = G>,
551 RNNCell<S, B, BiasIh, BiasHh, K, Train>: Module<
552 (
553 Tensor<D2<Batch, S::In>, B, K, G>,
554 Tensor<D2<Batch, S::Out>, B, K, G>,
555 ),
556 Output = Tensor<D2<Batch, S::Out>, B, K, G>,
557 Error = Error,
558 >,
559{
560 type Output = (
562 Tensor<D3<Batch, Seq, S::Out>, B, K, G>,
563 Tensor<D2<Batch, S::Out>, B, K, G>,
564 );
565 type Error = Error;
567
568 #[inline]
569 fn forward(
571 &self,
572 (x, mut h): (
573 Tensor<D3<Batch, Seq, S::In>, B, K, G>,
574 Tensor<D2<Batch, S::Out>, B, K, G>,
575 ),
576 ) -> core::result::Result<Self::Output, Error> {
577 let seq_len = Seq::static_size().map_err(Error::Shape)?;
578 let mut outputs = Vec::with_capacity(seq_len);
579
580 for i in 0..seq_len {
581 let x_step = x.clone().try_narrow(1isize, i, 1)?.try_squeeze(1isize)?;
582 let x_step_static: Tensor<D2<Batch, S::In>, B, K, G> = x_step.into_shape()?;
583 h = self.cell.forward((x_step_static, h))?;
584 outputs.push(h.clone().into_shape::<Dyn>()?);
585 }
586
587 let refs: Vec<&Tensor<Dyn, B, K, G>> = outputs.iter().collect();
588 let stacked_dyn = crate::tensor::ops::manipulation::try_stack_tensors(&refs, 1)?;
589 let stacked: Tensor<D3<Batch, Seq, S::Out>, B, K, G> = stacked_dyn.into_shape()?;
590
591 Ok((stacked, h))
592 }
593}
594
595impl<
596 S: RnnShape,
597 B: crate::tensor::backend::VariableBackend,
598 BiasIh: crate::nn::optional::OptionalField,
599 BiasHh: crate::nn::optional::OptionalField,
600 K: DType,
601 Train: TrainState,
602> crate::nn::VisitState<B> for RNNCell<S, B, BiasIh, BiasHh, K, Train>
603where
604 Linear<S::IhShape, B, BiasIh, K, Train>: crate::nn::VisitState<B>,
605 Linear<S::HhShape, B, BiasHh, K, Train>: crate::nn::VisitState<B>,
606{
607 fn visit_state<V: crate::nn::StateVisitor<B>>(
608 &self,
609 path: &crate::nn::StatePath,
610 visitor: &mut V,
611 ) -> crate::err::Result<()> {
612 self.wi.visit_state(&path.try_child("wi")?, visitor)?;
613 self.wh.visit_state(&path.try_child("wh")?, visitor)
614 }
615}
616
617impl<
618 S: RnnShape,
619 B: crate::tensor::backend::VariableBackend,
620 BiasIh: crate::nn::optional::OptionalField,
621 BiasHh: crate::nn::optional::OptionalField,
622 K: DType,
623 Train: TrainState,
624> crate::nn::VisitStateMut<B> for RNNCell<S, B, BiasIh, BiasHh, K, Train>
625where
626 Linear<S::IhShape, B, BiasIh, K, Train>: crate::nn::VisitStateMut<B>,
627 Linear<S::HhShape, B, BiasHh, K, Train>: crate::nn::VisitStateMut<B>,
628{
629 fn visit_state_mut<V: crate::nn::StateMutVisitor<B>>(
630 &mut self,
631 path: &crate::nn::StatePath,
632 visitor: &mut V,
633 ) -> crate::err::Result<()> {
634 self.wi.visit_state_mut(&path.try_child("wi")?, visitor)?;
635 self.wh.visit_state_mut(&path.try_child("wh")?, visitor)
636 }
637}
638
639impl<
640 S: RnnShape,
641 B: crate::tensor::backend::VariableBackend,
642 BiasIh: crate::nn::optional::OptionalField,
643 BiasHh: crate::nn::optional::OptionalField,
644 K: DType,
645 Train: TrainState,
646> crate::nn::VisitState<B> for RNN<S, B, BiasIh, BiasHh, K, Train>
647where
648 RNNCell<S, B, BiasIh, BiasHh, K, Train>: crate::nn::VisitState<B>,
649{
650 fn visit_state<V: crate::nn::StateVisitor<B>>(
651 &self,
652 path: &crate::nn::StatePath,
653 visitor: &mut V,
654 ) -> crate::err::Result<()> {
655 self.cell.visit_state(&path.try_child("cell")?, visitor)
656 }
657}
658
659impl<
660 S: RnnShape,
661 B: crate::tensor::backend::VariableBackend,
662 BiasIh: crate::nn::optional::OptionalField,
663 BiasHh: crate::nn::optional::OptionalField,
664 K: DType,
665 Train: TrainState,
666> crate::nn::VisitStateMut<B> for RNN<S, B, BiasIh, BiasHh, K, Train>
667where
668 RNNCell<S, B, BiasIh, BiasHh, K, Train>: crate::nn::VisitStateMut<B>,
669{
670 fn visit_state_mut<V: crate::nn::StateMutVisitor<B>>(
671 &mut self,
672 path: &crate::nn::StatePath,
673 visitor: &mut V,
674 ) -> crate::err::Result<()> {
675 self.cell.visit_state_mut(&path.try_child("cell")?, visitor)
676 }
677}
678
679impl<
680 S: RnnShape,
681 B: crate::tensor::backend::VariableBackend,
682 BiasIh: crate::nn::optional::OptionalField,
683 BiasHh: crate::nn::optional::OptionalField,
684 K: DType,
685 Train: TrainState,
686> crate::nn::module::NamedLayers for RNNCell<S, B, BiasIh, BiasHh, K, Train>
687where
688 Linear<S::IhShape, B, BiasIh, K, Train>: crate::nn::module::NamedLayers,
689 Linear<S::HhShape, B, BiasHh, K, Train>: crate::nn::module::NamedLayers,
690{
691 fn layer_structure(&self, prefix: &str) -> Vec<crate::nn::module::LayerNode> {
693 let mut children = Vec::new();
694 let p_wi = if prefix.is_empty() {
695 alloc::string::String::from("wi")
696 } else {
697 format!("{}.wi", prefix)
698 };
699 children.extend(self.wi.layer_structure(&p_wi));
700 let p_wh = if prefix.is_empty() {
701 alloc::string::String::from("wh")
702 } else {
703 format!("{}.wh", prefix)
704 };
705 children.extend(self.wh.layer_structure(&p_wh));
706
707 let node_name = if prefix.is_empty() {
708 alloc::string::String::from("RNNCell")
709 } else {
710 prefix.to_string()
711 };
712 vec![crate::nn::module::LayerNode {
713 name: node_name,
714 type_name: alloc::string::String::from("RNNCell"),
715 shape_info: "".to_string(),
716 children,
717 }]
718 }
719}
720
721impl<
722 S: RnnShape,
723 B: crate::tensor::backend::VariableBackend,
724 BiasIh: crate::nn::optional::OptionalField,
725 BiasHh: crate::nn::optional::OptionalField,
726 K: DType,
727 Train: TrainState,
728> crate::nn::module::NamedLayers for RNN<S, B, BiasIh, BiasHh, K, Train>
729where
730 RNNCell<S, B, BiasIh, BiasHh, K, Train>: crate::nn::module::NamedLayers,
731{
732 fn layer_structure(&self, prefix: &str) -> Vec<crate::nn::module::LayerNode> {
734 let mut children = Vec::new();
735 let p_cell = if prefix.is_empty() {
736 alloc::string::String::from("cell")
737 } else {
738 format!("{}.cell", prefix)
739 };
740 children.extend(self.cell.layer_structure(&p_cell));
741
742 let node_name = if prefix.is_empty() {
743 alloc::string::String::from("RNN")
744 } else {
745 prefix.to_string()
746 };
747 vec![crate::nn::module::LayerNode {
748 name: node_name,
749 type_name: alloc::string::String::from("RNN"),
750 shape_info: "".to_string(),
751 children,
752 }]
753 }
754}