Skip to main content

coeus_nn/transformer/
decoder.rs

1// ── Transformer Decoder stack ──
2
3use super::decoder_layer::TransformerDecoderLayer;
4use crate::module::ModuleError;
5use coeus_autograd::{AttentionMask, CausalMask, NullMask, Var};
6use coeus_core::MoiraiBackend;
7
8/// Stack of N `TransformerDecoderLayer`s.
9///
10/// # Type parameters
11/// - `H` — attention heads per layer (const generic)
12/// - `N` — number of stacked decoder layers (const generic)
13/// - `SelfM` — self-attention masking strategy ZST
14/// - `CrossM` — cross-attention masking strategy ZST
15pub struct TransformerDecoder<
16    T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
17    B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default = MoiraiBackend,
18    const H: usize = 8,
19    const N: usize = 6,
20    SelfM: AttentionMask = CausalMask,
21    CrossM: AttentionMask = NullMask,
22> {
23    /// Fixed-size array of decoder layers — stack-allocated container.
24    pub layers: [TransformerDecoderLayer<T, B, H, SelfM, CrossM>; N],
25}
26
27impl<
28        T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
29        B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
30        const H: usize,
31        const N: usize,
32        SelfM: AttentionMask,
33        CrossM: AttentionMask,
34    > TransformerDecoder<T, B, H, N, SelfM, CrossM>
35where
36    TransformerDecoderLayer<T, B, H, SelfM, CrossM>: Clone,
37{
38    /// Construct N independently-initialized decoder layers.
39    ///
40    /// # Errors
41    /// Returns an initialization error when `N == 0` or a layer cannot be
42    /// initialized by the selected backend.
43    pub fn new(
44        d_model: usize,
45        d_ff: usize,
46        dropout_p: f64,
47    ) -> Result<Self, crate::init::InitializationError<B::Error>>
48    where
49        T: coeus_cpu::RealScalar,
50        B: coeus_ops::RandomInitOps<T>,
51    {
52        if N == 0 {
53            return Err(crate::init::InitializationError::InvalidLayerCount { layers: N });
54        }
55        let layers = (0..N)
56            .map(|_| {
57                TransformerDecoderLayer::<T, B, H, SelfM, CrossM>::new(d_model, d_ff, dropout_p)
58            })
59            .collect::<Result<Vec<_>, _>>()?
60            .try_into()
61            .map_err(|_| crate::init::InitializationError::InvalidLayerCount { layers: N })?;
62        Ok(Self { layers })
63    }
64
65    /// Forward decoder sequentially through the stack.
66    ///
67    /// Input/output shape: `[batch, seq_tgt, d_model]`.
68    pub fn forward_decoder(
69        &self,
70        tgt: &Var<T, B>,
71        memory: &Var<T, B>,
72    ) -> Result<Var<T, B>, ModuleError<B::Error>> {
73        let mut x = tgt.clone();
74        for layer in &self.layers {
75            x = layer.forward_decoder(&x, memory)?;
76        }
77        Ok(x)
78    }
79}
80
81impl<
82        T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
83        B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
84        const H: usize,
85        const N: usize,
86        SelfM: AttentionMask,
87        CrossM: AttentionMask,
88    > TransformerDecoder<T, B, H, N, SelfM, CrossM>
89{
90    /// Collect all trainable decoder-stack parameters.
91    pub fn parameters(&self) -> Vec<Var<T, B>> {
92        self.layers.iter().flat_map(|l| l.parameters()).collect()
93    }
94
95    /// Collect trainable parameters with stable hierarchical names.
96    pub fn named_parameters(&self) -> Vec<coeus_autograd::Parameter<T, B>> {
97        self.layers
98            .iter()
99            .enumerate()
100            .flat_map(|(index, layer)| {
101                let prefix = format!("layers.{index}");
102                layer
103                    .named_parameters()
104                    .into_iter()
105                    .map(move |parameter| parameter.with_prefix(&prefix))
106            })
107            .collect()
108    }
109}
110
111/// Manual Clone impl.
112impl<
113        T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
114        B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
115        const H: usize,
116        const N: usize,
117        SelfM: AttentionMask,
118        CrossM: AttentionMask,
119    > Clone for TransformerDecoder<T, B, H, N, SelfM, CrossM>
120where
121    TransformerDecoderLayer<T, B, H, SelfM, CrossM>: Clone,
122{
123    fn clone(&self) -> Self {
124        Self {
125            layers: self.layers.clone(),
126        }
127    }
128}