Skip to main content

incin_core/nn/
lstm.rs

1use crate::backend_authoring::SupportsDType;
2use crate::err::{Error, Result};
3use crate::exec::catalog::op;
4use crate::nn::init::Init;
5use crate::nn::linear::LinearShape;
6use crate::nn::optional::{False, True};
7use crate::nn::param::{Frozen, TrainState, Trainable};
8use crate::nn::{Linear, Module, VisitParameters};
9use crate::shapes::shape::{DimCons, Nil};
10use crate::shapes::{Dim, Dyn, DynShape, Shape, ShapeValue};
11use crate::tensor::backend::Execute;
12use crate::tensor::base::Tensor;
13use crate::tensor::device::Device;
14use crate::tensor::dtype::DType;
15use alloc::string::ToString;
16use alloc::vec::Vec;
17
18type D2<A, B> = DimCons<A, DimCons<B, Nil>>;
19type D3<A, B, C> = DimCons<A, DimCons<B, DimCons<C, Nil>>>;
20
21/// A shape marker trait specifying the input and output features of an [`LSTMCell`] / [`LSTM`].
22///
23/// Supply `(In, Out)` for fully static dimensions, or [`Dyn`] for fully runtime sizes.
24pub trait LstmShape: Shape + DynShape {
25    /// `In`.
26    type In: Dim;
27    /// `Out`.
28    type Out: Dim;
29    /// Input-to-hidden projection geometry.
30    type IhShape: LinearShape<InF = Self::In, OutF = Self::Out>;
31    /// Hidden-to-hidden recurrence geometry.
32    type HhShape: LinearShape<InF = Self::Out, OutF = Self::Out>;
33}
34
35impl<In: Dim, Out: Dim> LstmShape
36    for crate::shapes::shape::DimCons<
37        In,
38        crate::shapes::shape::DimCons<Out, crate::shapes::shape::Nil>,
39    >
40{
41    type In = In;
42    type Out = Out;
43    type IhShape = crate::shapes::shape::DimCons<
44        In,
45        crate::shapes::shape::DimCons<Out, crate::shapes::shape::Nil>,
46    >;
47    type HhShape = crate::shapes::shape::DimCons<
48        Out,
49        crate::shapes::shape::DimCons<Out, crate::shapes::shape::Nil>,
50    >;
51}
52
53impl LstmShape for Dyn {
54    /// `In`.
55    type In = usize;
56    /// `Out`.
57    type Out = usize;
58    type IhShape = Dyn;
59    type HhShape = Dyn;
60}
61
62// ---------------------------------------------------------------------------
63// LSTMCellBuilder: typestate builder
64// ---------------------------------------------------------------------------
65
66/// A builder for constructing an [`LSTMCell`] before target-based initialization.
67///
68/// Stores the layer geometry ([`ShapeValue`]), weight and bias initializer policies
69/// (grouped semantically as input and hidden), and compile-time typestate parameters
70/// for bias presence and trainability.
71pub struct LSTMCellBuilder<
72    S: LstmShape,
73    BiasIh: crate::nn::optional::OptionalField = True,
74    BiasHh: crate::nn::optional::OptionalField = True,
75    Train: TrainState = Trainable,
76> {
77    /// Shape specification (encodes `[in_features, out_features]`).
78    pub shape: ShapeValue<S>,
79    /// Initializer for all input-to-hidden weight matrices (`W_ih_i`, `W_ih_f`, etc.).
80    pub input_weight_init: Init,
81    /// Initializer for all hidden-to-hidden weight matrices (`W_hh_i`, `W_hh_f`, etc.).
82    pub hidden_weight_init: Init,
83    /// Initializer for all input-to-hidden bias vectors.
84    pub input_bias_init: Init,
85    /// Initializer for all hidden-to-hidden bias vectors.
86    pub hidden_bias_init: Init,
87    /// Bias-presence and train-state markers.
88    pub _phantom: core::marker::PhantomData<(BiasIh, BiasHh, Train)>,
89}
90
91impl<
92    S: LstmShape,
93    BiasIh: crate::nn::optional::OptionalField,
94    BiasHh: crate::nn::optional::OptionalField,
95    Train: TrainState,
96> LSTMCellBuilder<S, BiasIh, BiasHh, Train>
97{
98    /// Returns a reference to the shape specification of this builder.
99    pub fn shape(&self) -> &ShapeValue<S> {
100        &self.shape
101    }
102
103    /// Removes input-to-hidden biases from the built cell.
104    pub fn no_input_bias(self) -> LSTMCellBuilder<S, False, BiasHh, Train> {
105        LSTMCellBuilder {
106            shape: self.shape,
107            input_weight_init: self.input_weight_init,
108            hidden_weight_init: self.hidden_weight_init,
109            input_bias_init: self.input_bias_init,
110            hidden_bias_init: self.hidden_bias_init,
111            _phantom: core::marker::PhantomData,
112        }
113    }
114
115    /// Removes hidden-to-hidden biases from the built cell.
116    pub fn no_hidden_bias(self) -> LSTMCellBuilder<S, BiasIh, False, Train> {
117        LSTMCellBuilder {
118            shape: self.shape,
119            input_weight_init: self.input_weight_init,
120            hidden_weight_init: self.hidden_weight_init,
121            input_bias_init: self.input_bias_init,
122            hidden_bias_init: self.hidden_bias_init,
123            _phantom: core::marker::PhantomData,
124        }
125    }
126
127    /// Removes all biases from the built cell.
128    pub fn no_bias(self) -> LSTMCellBuilder<S, False, False, Train> {
129        LSTMCellBuilder {
130            shape: self.shape,
131            input_weight_init: self.input_weight_init,
132            hidden_weight_init: self.hidden_weight_init,
133            input_bias_init: self.input_bias_init,
134            hidden_bias_init: self.hidden_bias_init,
135            _phantom: core::marker::PhantomData,
136        }
137    }
138
139    /// Marks the created cell parameters as frozen (non-trainable).
140    pub fn frozen(self) -> LSTMCellBuilder<S, BiasIh, BiasHh, Frozen> {
141        LSTMCellBuilder {
142            shape: self.shape,
143            input_weight_init: self.input_weight_init,
144            hidden_weight_init: self.hidden_weight_init,
145            input_bias_init: self.input_bias_init,
146            hidden_bias_init: self.hidden_bias_init,
147            _phantom: core::marker::PhantomData,
148        }
149    }
150
151    /// Sets the initializer used for all input-to-hidden weight matrices.
152    pub fn input_weight_init(mut self, init: Init) -> Self {
153        self.input_weight_init = init;
154        self
155    }
156
157    /// Sets the initializer used for all hidden-to-hidden weight matrices.
158    pub fn hidden_weight_init(mut self, init: Init) -> Self {
159        self.hidden_weight_init = init;
160        self
161    }
162
163    /// Sets the initializer used for all input-to-hidden bias vectors.
164    pub fn input_bias_init(mut self, init: Init) -> Self {
165        self.input_bias_init = init;
166        self
167    }
168
169    /// Sets the initializer used for all hidden-to-hidden bias vectors.
170    pub fn hidden_bias_init(mut self, init: Init) -> Self {
171        self.hidden_bias_init = init;
172        self
173    }
174}
175
176/// Free constructor for a backend-independent [`LSTMCellBuilder`].
177pub fn lstm_cell<S: LstmShape>(shape: ShapeValue<S>) -> LSTMCellBuilder<S> {
178    let init = crate::nn::init::kaiming_uniform();
179    LSTMCellBuilder {
180        shape,
181        input_weight_init: init,
182        hidden_weight_init: init,
183        input_bias_init: init,
184        hidden_bias_init: init,
185        _phantom: core::marker::PhantomData,
186    }
187}
188
189// ---------------------------------------------------------------------------
190// LSTMBuilder: typestate builder for the multi-step LSTM
191// ---------------------------------------------------------------------------
192
193/// A builder for constructing an [`LSTM`] before target-based initialization.
194///
195/// Wraps an [`LSTMCellBuilder`] and exposes the same bias/trainability controls.
196pub struct LSTMBuilder<
197    S: LstmShape,
198    BiasIh: crate::nn::optional::OptionalField = True,
199    BiasHh: crate::nn::optional::OptionalField = True,
200    Train: TrainState = Trainable,
201> {
202    /// The inner cell builder.
203    pub cell: LSTMCellBuilder<S, BiasIh, BiasHh, Train>,
204}
205
206impl<
207    S: LstmShape,
208    BiasIh: crate::nn::optional::OptionalField,
209    BiasHh: crate::nn::optional::OptionalField,
210    Train: TrainState,
211> LSTMBuilder<S, BiasIh, BiasHh, Train>
212{
213    /// Returns a reference to the shape specification of this builder.
214    pub fn shape(&self) -> &ShapeValue<S> {
215        self.cell.shape()
216    }
217
218    /// Removes input-to-hidden biases from the built LSTM.
219    pub fn no_input_bias(self) -> LSTMBuilder<S, False, BiasHh, Train> {
220        LSTMBuilder {
221            cell: self.cell.no_input_bias(),
222        }
223    }
224
225    /// Removes hidden-to-hidden biases from the built LSTM.
226    pub fn no_hidden_bias(self) -> LSTMBuilder<S, BiasIh, False, Train> {
227        LSTMBuilder {
228            cell: self.cell.no_hidden_bias(),
229        }
230    }
231
232    /// Removes all biases from the built LSTM.
233    pub fn no_bias(self) -> LSTMBuilder<S, False, False, Train> {
234        LSTMBuilder {
235            cell: self.cell.no_bias(),
236        }
237    }
238
239    /// Marks the created LSTM parameters as frozen (non-trainable).
240    pub fn frozen(self) -> LSTMBuilder<S, BiasIh, BiasHh, Frozen> {
241        LSTMBuilder {
242            cell: self.cell.frozen(),
243        }
244    }
245
246    /// Sets the initializer used for all input-to-hidden weight matrices.
247    pub fn input_weight_init(mut self, init: Init) -> Self {
248        self.cell = self.cell.input_weight_init(init);
249        self
250    }
251
252    /// Sets the initializer used for all hidden-to-hidden weight matrices.
253    pub fn hidden_weight_init(mut self, init: Init) -> Self {
254        self.cell = self.cell.hidden_weight_init(init);
255        self
256    }
257
258    /// Sets the initializer used for all input-to-hidden bias vectors.
259    pub fn input_bias_init(mut self, init: Init) -> Self {
260        self.cell = self.cell.input_bias_init(init);
261        self
262    }
263
264    /// Sets the initializer used for all hidden-to-hidden bias vectors.
265    pub fn hidden_bias_init(mut self, init: Init) -> Self {
266        self.cell = self.cell.hidden_bias_init(init);
267        self
268    }
269}
270
271/// Free constructor for a backend-independent [`LSTMBuilder`].
272pub fn lstm<S: LstmShape>(shape: ShapeValue<S>) -> LSTMBuilder<S> {
273    LSTMBuilder {
274        cell: lstm_cell(shape),
275    }
276}
277
278// ---------------------------------------------------------------------------
279// LSTMCell: concrete BiasIh / BiasHh combinations
280// ---------------------------------------------------------------------------
281
282/// A single LSTM step cell implementing the standard 4-gate recurrence.
283///
284/// * `S`: [`LstmShape`]: `(In, Out)` static or [`Dyn`] for runtime sizes.
285/// * `BiasIh`: whether input-to-hidden biases exist: [`True`], [`False`].
286/// * `BiasHh`: whether hidden-to-hidden biases exist: [`True`], [`False`].
287/// * `K`: parameter dtype (default: `f32`).
288/// * `Train`: trainability typestate (default: [`Trainable`]).
289///
290/// ## Examples
291///
292/// ```rust
293/// # extern crate incin_core as incin;
294/// # fn main() -> incin::prelude::Result<()> {
295/// # type DefaultBackend = incin_backends::cpu::CpuBackendImpl;
296/// use incin::prelude::*;
297///
298/// // Fully static, both biases present (default)
299/// let cell = LSTMCell::<s![10, 20], DefaultBackend>::build(())?;
300///
301/// // Fully static, input bias present, hidden bias absent
302/// let cell = LSTMCell::<s![10, 20], DefaultBackend, True, False>::build(())?;
303///
304/// // Fully dynamic shape, both biases always present
305/// let cell = LSTMCell::<Dyn, DefaultBackend>::build((10, 20))?;
306///
307/// // Fully dynamic shape, no biases
308/// let cell = LSTMCell::<Dyn, DefaultBackend, False, False>::build((10, 20))?;
309/// # Ok(()) }
310/// ```
311#[derive(Debug, Clone)]
312pub struct LSTMCell<
313    S: LstmShape,
314    B: crate::tensor::backend::VariableBackend,
315    BiasIh: crate::nn::optional::OptionalField = True,
316    BiasHh: crate::nn::optional::OptionalField = True,
317    K: DType = f32,
318    Train: TrainState = Trainable,
319> where
320    S::IhShape: LinearShape,
321    S::HhShape: LinearShape,
322{
323    /// `wi_i`.
324    pub wi_i: Linear<S::IhShape, B, BiasIh, K, Train>,
325    /// `wi_f`.
326    pub wi_f: Linear<S::IhShape, B, BiasIh, K, Train>,
327    /// `wi_g`.
328    pub wi_g: Linear<S::IhShape, B, BiasIh, K, Train>,
329    /// `wi_o`.
330    pub wi_o: Linear<S::IhShape, B, BiasIh, K, Train>,
331    /// `wh_i`.
332    pub wh_i: Linear<S::HhShape, B, BiasHh, K, Train>,
333    /// `wh_f`.
334    pub wh_f: Linear<S::HhShape, B, BiasHh, K, Train>,
335    /// `wh_g`.
336    pub wh_g: Linear<S::HhShape, B, BiasHh, K, Train>,
337    /// `wh_o`.
338    pub wh_o: Linear<S::HhShape, B, BiasHh, K, Train>,
339}
340
341impl<S, B, BiasIh, BiasHh, K: DType, Train: TrainState> LSTMCell<S, B, BiasIh, BiasHh, K, Train>
342where
343    S: LstmShape,
344    B: crate::tensor::backend::VariableBackend
345        + SupportsDType<K>
346        + crate::tensor::backend::SupportsDType<K>
347        + crate::nn::param::ParameterInit<K>,
348    BiasIh: crate::nn::optional::OptionalField,
349    BiasHh: crate::nn::optional::OptionalField,
350    <K as DType>::Arg: Clone,
351    <S::In as Dim>::Arg: Clone,
352    <S::Out as Dim>::Arg: Clone,
353    <B::Device as Device>::Arg: Clone,
354    <BiasIh as crate::nn::optional::OptionalField>::Arg: Clone,
355    <BiasHh as crate::nn::optional::OptionalField>::Arg: Clone,
356{
357    /// Constructs the cell directly from explicit arguments (the "build" path).
358    ///
359    /// Prefer the target-aware `LSTMCellBuilder::init()` path for new code.
360    pub fn build<A>(args: A) -> Result<Self>
361    where
362        A: crate::tensor::arg_into::LayerArgInto<(
363                <S::In as Dim>::Arg,
364                <S::Out as Dim>::Arg,
365                <K as DType>::Arg,
366                <B::Device as Device>::Arg,
367                <BiasIh as crate::nn::optional::OptionalField>::Arg,
368                <BiasHh as crate::nn::optional::OptionalField>::Arg,
369            )>,
370    {
371        let (input, output, dtype, device, bias_ih, bias_hh) = args.into_layer_arg();
372        Ok(Self {
373            wi_i: Linear::<_, _, _, K, Train>::build_full(
374                input.clone(),
375                output.clone(),
376                dtype.clone(),
377                device.clone(),
378                bias_ih.clone(),
379            )?,
380            wi_f: Linear::<_, _, _, K, Train>::build_full(
381                input.clone(),
382                output.clone(),
383                dtype.clone(),
384                device.clone(),
385                bias_ih.clone(),
386            )?,
387            wi_g: Linear::<_, _, _, K, Train>::build_full(
388                input.clone(),
389                output.clone(),
390                dtype.clone(),
391                device.clone(),
392                bias_ih.clone(),
393            )?,
394            wi_o: Linear::<_, _, _, K, Train>::build_full(
395                input,
396                output.clone(),
397                dtype.clone(),
398                device.clone(),
399                bias_ih,
400            )?,
401            wh_i: Linear::<_, _, _, K, Train>::build_full(
402                output.clone(),
403                output.clone(),
404                dtype.clone(),
405                device.clone(),
406                bias_hh.clone(),
407            )?,
408            wh_f: Linear::<_, _, _, K, Train>::build_full(
409                output.clone(),
410                output.clone(),
411                dtype.clone(),
412                device.clone(),
413                bias_hh.clone(),
414            )?,
415            wh_g: Linear::<_, _, _, K, Train>::build_full(
416                output.clone(),
417                output.clone(),
418                dtype.clone(),
419                device.clone(),
420                bias_hh.clone(),
421            )?,
422            wh_o: Linear::<_, _, _, K, Train>::build_full(
423                output.clone(),
424                output,
425                dtype,
426                device,
427                bias_hh,
428            )?,
429        })
430    }
431
432    /// Converts this cell's parameters to frozen typestate.
433    pub fn freeze(self) -> LSTMCell<S, B, BiasIh, BiasHh, K, Frozen> {
434        LSTMCell {
435            wi_i: self.wi_i.freeze(),
436            wi_f: self.wi_f.freeze(),
437            wi_g: self.wi_g.freeze(),
438            wi_o: self.wi_o.freeze(),
439            wh_i: self.wh_i.freeze(),
440            wh_f: self.wh_f.freeze(),
441            wh_g: self.wh_g.freeze(),
442            wh_o: self.wh_o.freeze(),
443        }
444    }
445
446    /// Converts this cell's parameters to trainable typestate.
447    pub fn unfreeze(self) -> LSTMCell<S, B, BiasIh, BiasHh, K, Trainable> {
448        LSTMCell {
449            wi_i: self.wi_i.unfreeze(),
450            wi_f: self.wi_f.unfreeze(),
451            wi_g: self.wi_g.unfreeze(),
452            wi_o: self.wi_o.unfreeze(),
453            wh_i: self.wh_i.unfreeze(),
454            wh_f: self.wh_f.unfreeze(),
455            wh_g: self.wh_g.unfreeze(),
456            wh_o: self.wh_o.unfreeze(),
457        }
458    }
459}
460
461// ---------------------------------------------------------------------------
462// Parameter visitor (generic over all BiasIh/BiasHh/K/Train combinations)
463// ---------------------------------------------------------------------------
464
465impl<
466    In: Dim,
467    Out: Dim,
468    B: crate::tensor::backend::VariableBackend,
469    BiasIh: crate::nn::optional::OptionalField,
470    BiasHh: crate::nn::optional::OptionalField,
471    K: DType,
472    Train: TrainState,
473> VisitParameters<B> for LSTMCell<D2<In, Out>, B, BiasIh, BiasHh, K, Train>
474where
475    Linear<D2<In, Out>, B, BiasIh, K, Train>: VisitParameters<B>,
476    Linear<D2<Out, Out>, B, BiasHh, K, Train>: VisitParameters<B>,
477{
478    fn visit_parameters<V: crate::nn::ParameterVisitor<B>>(
479        &self,
480        path: &crate::nn::StatePath,
481        visitor: &mut V,
482    ) -> crate::err::Result<()> {
483        self.wi_i
484            .visit_parameters(&path.try_child("wi_i")?, visitor)?;
485        self.wi_f
486            .visit_parameters(&path.try_child("wi_f")?, visitor)?;
487        self.wi_g
488            .visit_parameters(&path.try_child("wi_g")?, visitor)?;
489        self.wi_o
490            .visit_parameters(&path.try_child("wi_o")?, visitor)?;
491        self.wh_i
492            .visit_parameters(&path.try_child("wh_i")?, visitor)?;
493        self.wh_f
494            .visit_parameters(&path.try_child("wh_f")?, visitor)?;
495        self.wh_g
496            .visit_parameters(&path.try_child("wh_g")?, visitor)?;
497        self.wh_o
498            .visit_parameters(&path.try_child("wh_o")?, visitor)
499    }
500}
501
502// ---------------------------------------------------------------------------
503// Module (forward) impl: static shapes
504// ---------------------------------------------------------------------------
505
506impl<
507    In: Dim,
508    Out: Dim,
509    Batch: Dim,
510    B: crate::tensor::backend::VariableBackend
511        + Execute<op::Add>
512        + Execute<op::Mul>
513        + Execute<op::Sigmoid>
514        + Execute<op::Tanh>,
515    BiasIh: crate::nn::optional::OptionalField,
516    BiasHh: crate::nn::optional::OptionalField,
517    K: DType,
518    Train: TrainState,
519>
520    Module<(
521        Tensor<D2<Batch, In>, B, K>,
522        (Tensor<D2<Batch, Out>, B, K>, Tensor<D2<Batch, Out>, B, K>),
523    )> for LSTMCell<D2<In, Out>, B, BiasIh, BiasHh, K, Train>
524where
525    Linear<D2<In, Out>, B, BiasIh, K, Train>:
526        Module<Tensor<D2<Batch, In>, B, K>, Output = Tensor<D2<Batch, Out>, B, K>, Error = Error>,
527    Linear<D2<Out, Out>, B, BiasHh, K, Train>:
528        Module<Tensor<D2<Batch, Out>, B, K>, Output = Tensor<D2<Batch, Out>, B, K>, Error = Error>,
529    <B as Execute<op::Add>>::Output: Into<B::Storage<K>>,
530    <B as Execute<op::Mul>>::Output: Into<B::Storage<K>>,
531    <B as Execute<op::Sigmoid>>::Output: Into<B::Storage<K>>,
532    <B as Execute<op::Tanh>>::Output: Into<B::Storage<K>>,
533{
534    /// The output tensor type produced by this module's forward pass.
535    type Output = (Tensor<D2<Batch, Out>, B, K>, Tensor<D2<Batch, Out>, B, K>);
536    /// The error type returned if the forward pass fails.
537    type Error = Error;
538
539    #[inline]
540    /// Runs the forward pass of this module on the given input.
541    fn forward(
542        &self,
543        (x, (h_prev, c_prev)): (
544            Tensor<D2<Batch, In>, B, K>,
545            (Tensor<D2<Batch, Out>, B, K>, Tensor<D2<Batch, Out>, B, K>),
546        ),
547    ) -> core::result::Result<Self::Output, Error> {
548        let i = self
549            .wi_i
550            .forward(x.clone())?
551            .add_exact(&self.wh_i.forward(h_prev.clone())?)?
552            .sigmoid()?;
553        let f = self
554            .wi_f
555            .forward(x.clone())?
556            .add_exact(&self.wh_f.forward(h_prev.clone())?)?
557            .sigmoid()?;
558        let g = self
559            .wi_g
560            .forward(x.clone())?
561            .add_exact(&self.wh_g.forward(h_prev.clone())?)?
562            .tanh()?;
563        let o = self
564            .wi_o
565            .forward(x)?
566            .add_exact(&self.wh_o.forward(h_prev)?)?
567            .sigmoid()?;
568        let c = f.mul_exact(&c_prev)?.add_exact(&i.mul_exact(&g)?)?;
569        let h = o.mul_exact(&c.clone().tanh()?)?;
570        Ok((h, c))
571    }
572}
573
574// ---------------------------------------------------------------------------
575// LSTM (multi-step wrapper)
576// ---------------------------------------------------------------------------
577
578#[derive(Debug, Clone)]
579/// `LSTM`.
580#[allow(clippy::upper_case_acronyms)]
581pub struct LSTM<
582    S: LstmShape,
583    B: crate::tensor::backend::VariableBackend,
584    BiasIh: crate::nn::optional::OptionalField = True,
585    BiasHh: crate::nn::optional::OptionalField = True,
586    K: DType = f32,
587    Train: TrainState = Trainable,
588> {
589    /// `cell`.
590    pub cell: LSTMCell<S, B, BiasIh, BiasHh, K, Train>,
591}
592
593impl<
594    S: LstmShape,
595    B: crate::tensor::backend::VariableBackend,
596    BiasIh: crate::nn::optional::OptionalField,
597    BiasHh: crate::nn::optional::OptionalField,
598    K: DType,
599    Train: TrainState,
600> LSTM<S, B, BiasIh, BiasHh, K, Train>
601{
602    /// Creates a new instance from a pre-built cell.
603    pub fn new(cell: LSTMCell<S, B, BiasIh, BiasHh, K, Train>) -> Self {
604        Self { cell }
605    }
606
607    /// Converts this LSTM's parameters to frozen typestate.
608    pub fn freeze(self) -> LSTM<S, B, BiasIh, BiasHh, K, Frozen>
609    where
610        LSTMCell<S, B, BiasIh, BiasHh, K, Train>: Sized,
611        S: LstmShape,
612        B: crate::tensor::backend::VariableBackend
613            + SupportsDType<K>
614            + crate::tensor::backend::SupportsDType<K>
615            + crate::nn::param::ParameterInit<K>,
616        <K as DType>::Arg: Clone,
617        <S::In as Dim>::Arg: Clone,
618        <S::Out as Dim>::Arg: Clone,
619        <B::Device as Device>::Arg: Clone,
620        <BiasIh as crate::nn::optional::OptionalField>::Arg: Clone,
621        <BiasHh as crate::nn::optional::OptionalField>::Arg: Clone,
622    {
623        LSTM {
624            cell: self.cell.freeze(),
625        }
626    }
627
628    /// Converts this LSTM's parameters to trainable typestate.
629    pub fn unfreeze(self) -> LSTM<S, B, BiasIh, BiasHh, K, Trainable>
630    where
631        LSTMCell<S, B, BiasIh, BiasHh, K, Train>: Sized,
632        S: LstmShape,
633        B: crate::tensor::backend::VariableBackend
634            + SupportsDType<K>
635            + crate::tensor::backend::SupportsDType<K>
636            + crate::nn::param::ParameterInit<K>,
637        <K as DType>::Arg: Clone,
638        <S::In as Dim>::Arg: Clone,
639        <S::Out as Dim>::Arg: Clone,
640        <B::Device as Device>::Arg: Clone,
641        <BiasIh as crate::nn::optional::OptionalField>::Arg: Clone,
642        <BiasHh as crate::nn::optional::OptionalField>::Arg: Clone,
643    {
644        LSTM {
645            cell: self.cell.unfreeze(),
646        }
647    }
648}
649
650impl<
651    In: Dim,
652    Out: Dim,
653    B: crate::tensor::backend::VariableBackend,
654    BiasIh: crate::nn::optional::OptionalField,
655    BiasHh: crate::nn::optional::OptionalField,
656    K: DType,
657    Train: TrainState,
658> VisitParameters<B> for LSTM<D2<In, Out>, B, BiasIh, BiasHh, K, Train>
659where
660    LSTMCell<D2<In, Out>, B, BiasIh, BiasHh, K, Train>: VisitParameters<B>,
661{
662    fn visit_parameters<V: crate::nn::ParameterVisitor<B>>(
663        &self,
664        path: &crate::nn::StatePath,
665        visitor: &mut V,
666    ) -> crate::err::Result<()> {
667        self.cell
668            .visit_parameters(&path.try_child("cell")?, visitor)
669    }
670}
671
672impl<
673    In: Dim<Arg = ()>,
674    Out: Dim<Arg = ()>,
675    Batch: Dim<Arg = ()>,
676    Seq: Dim<Arg = ()>,
677    B: crate::tensor::backend::VariableBackend
678        + Execute<op::StackExact>
679        + Execute<op::Narrow>
680        + Execute<op::SqueezeExact>
681        + crate::exec::Capabilities,
682    BiasIh: crate::nn::optional::OptionalField,
683    BiasHh: crate::nn::optional::OptionalField,
684    K: DType,
685    Train: TrainState,
686>
687    Module<(
688        Tensor<D3<Batch, Seq, In>, B, K>,
689        (Tensor<D2<Batch, Out>, B, K>, Tensor<D2<Batch, Out>, B, K>),
690    )> for LSTM<D2<In, Out>, B, BiasIh, BiasHh, K, Train>
691where
692    <B as Execute<op::StackExact>>::Output: Into<B::Storage<K>>,
693    <B as Execute<op::Narrow>>::Output: Into<B::Storage<K>>,
694    <B as Execute<op::SqueezeExact>>::Output: Into<B::Storage<K>>,
695    LSTMCell<D2<In, Out>, B, BiasIh, BiasHh, K, Train>: Module<
696            (
697                Tensor<D2<Batch, In>, B, K>,
698                (Tensor<D2<Batch, Out>, B, K>, Tensor<D2<Batch, Out>, B, K>),
699            ),
700            Output = (Tensor<D2<Batch, Out>, B, K>, Tensor<D2<Batch, Out>, B, K>),
701            Error = Error,
702        >,
703{
704    /// The output tensor type produced by this module's forward pass.
705    type Output = (
706        Tensor<D3<Batch, Seq, Out>, B, K>,
707        (Tensor<D2<Batch, Out>, B, K>, Tensor<D2<Batch, Out>, B, K>),
708    );
709    /// The error type returned if the forward pass fails.
710    type Error = Error;
711
712    #[inline]
713    /// Runs the forward pass of this module on the given input.
714    fn forward(
715        &self,
716        (x, (mut h, mut c)): (
717            Tensor<D3<Batch, Seq, In>, B, K>,
718            (Tensor<D2<Batch, Out>, B, K>, Tensor<D2<Batch, Out>, B, K>),
719        ),
720    ) -> core::result::Result<Self::Output, Error> {
721        let seq_len = Seq::static_size().map_err(Error::Shape)?;
722        let mut outputs = Vec::with_capacity(seq_len);
723
724        for i in 0..seq_len {
725            let x_step = x.clone().try_narrow(1isize, i, 1)?.try_squeeze(1isize)?;
726            let x_step_static: Tensor<D2<Batch, In>, B, K> = x_step.into_shape()?;
727            let (h_next, c_next) = self.cell.forward((x_step_static, (h, c)))?;
728            h = h_next;
729            c = c_next;
730            outputs.push(h.clone().into_shape::<Dyn>()?);
731        }
732
733        let refs: Vec<&Tensor<Dyn, B, K>> = outputs.iter().collect();
734        let stacked_dyn = crate::tensor::ops::manipulation::try_stack_tensors(&refs, 1)?;
735        let stacked: Tensor<D3<Batch, Seq, Out>, B, K> = stacked_dyn.into_shape()?;
736        Ok((stacked, (h, c)))
737    }
738}
739
740impl<
741    S: LstmShape,
742    B: crate::tensor::backend::VariableBackend,
743    BiasIh: crate::nn::optional::OptionalField,
744    BiasHh: crate::nn::optional::OptionalField,
745    K: DType,
746    Train: TrainState,
747> crate::nn::module::NamedLayers for LSTMCell<S, B, BiasIh, BiasHh, K, Train>
748where
749    Linear<S::IhShape, B, BiasIh, K, Train>: crate::nn::module::NamedLayers,
750    Linear<S::HhShape, B, BiasHh, K, Train>: crate::nn::module::NamedLayers,
751{
752    /// Returns the layer hierarchy rooted at this module for visualization.
753    fn layer_structure(&self, prefix: &str) -> Vec<crate::nn::module::LayerNode> {
754        let mut children = Vec::new();
755
756        let p_wi_i = if prefix.is_empty() {
757            alloc::string::String::from("wi_i")
758        } else {
759            format!("{}.wi_i", prefix)
760        };
761        children.extend(self.wi_i.layer_structure(&p_wi_i));
762        let p_wi_f = if prefix.is_empty() {
763            alloc::string::String::from("wi_f")
764        } else {
765            format!("{}.wi_f", prefix)
766        };
767        children.extend(self.wi_f.layer_structure(&p_wi_f));
768        let p_wi_g = if prefix.is_empty() {
769            alloc::string::String::from("wi_g")
770        } else {
771            format!("{}.wi_g", prefix)
772        };
773        children.extend(self.wi_g.layer_structure(&p_wi_g));
774        let p_wi_o = if prefix.is_empty() {
775            alloc::string::String::from("wi_o")
776        } else {
777            format!("{}.wi_o", prefix)
778        };
779        children.extend(self.wi_o.layer_structure(&p_wi_o));
780        let p_wh_i = if prefix.is_empty() {
781            alloc::string::String::from("wh_i")
782        } else {
783            format!("{}.wh_i", prefix)
784        };
785        children.extend(self.wh_i.layer_structure(&p_wh_i));
786        let p_wh_f = if prefix.is_empty() {
787            alloc::string::String::from("wh_f")
788        } else {
789            format!("{}.wh_f", prefix)
790        };
791        children.extend(self.wh_f.layer_structure(&p_wh_f));
792        let p_wh_g = if prefix.is_empty() {
793            alloc::string::String::from("wh_g")
794        } else {
795            format!("{}.wh_g", prefix)
796        };
797        children.extend(self.wh_g.layer_structure(&p_wh_g));
798        let p_wh_o = if prefix.is_empty() {
799            alloc::string::String::from("wh_o")
800        } else {
801            format!("{}.wh_o", prefix)
802        };
803        children.extend(self.wh_o.layer_structure(&p_wh_o));
804
805        let node_name = if prefix.is_empty() {
806            alloc::string::String::from("LSTMCell")
807        } else {
808            prefix.to_string()
809        };
810        vec![crate::nn::module::LayerNode {
811            name: node_name,
812            type_name: alloc::string::String::from("LSTMCell"),
813            shape_info: "".to_string(),
814            children,
815        }]
816    }
817}
818
819impl<
820    S: LstmShape,
821    B: crate::tensor::backend::VariableBackend,
822    BiasIh: crate::nn::optional::OptionalField,
823    BiasHh: crate::nn::optional::OptionalField,
824    K: DType,
825    Train: TrainState,
826> crate::nn::module::NamedLayers for LSTM<S, B, BiasIh, BiasHh, K, Train>
827where
828    LSTMCell<S, B, BiasIh, BiasHh, K, Train>: crate::nn::module::NamedLayers,
829{
830    /// Returns the layer hierarchy rooted at this module for visualization.
831    fn layer_structure(&self, prefix: &str) -> Vec<crate::nn::module::LayerNode> {
832        let mut children = Vec::new();
833        let p_cell = if prefix.is_empty() {
834            alloc::string::String::from("cell")
835        } else {
836            format!("{}.cell", prefix)
837        };
838        children.extend(self.cell.layer_structure(&p_cell));
839
840        let node_name = if prefix.is_empty() {
841            alloc::string::String::from("LSTM")
842        } else {
843            prefix.to_string()
844        };
845        vec![crate::nn::module::LayerNode {
846            name: node_name,
847            type_name: alloc::string::String::from("LSTM"),
848            shape_info: "".to_string(),
849            children,
850        }]
851    }
852}