Skip to main content

coeus_nn/transformer/
encoder.rs

1// ── Transformer Encoder stack ──
2//
3// N independently-initialized encoder layers in a fixed-size array.
4// `N` is a const generic: `[TransformerEncoderLayer; N]` is stack-allocated with
5// no heap overhead for the container. Monomorphization emits one specialization
6// per `(T, B, H, N, M)`.
7
8use super::encoder_layer::TransformerEncoderLayer;
9use crate::module::{prefixed_parameters, Module, ModuleError};
10use coeus_autograd::{AttentionMask, Var};
11use coeus_core::MoiraiBackend;
12
13/// Stack of N `TransformerEncoderLayer`s.
14///
15/// # Type parameters
16/// - `H` — attention heads per layer (const generic, uniform across layers)
17/// - `N` — number of stacked encoder layers (const generic)
18/// - `M` — masking strategy ZST
19#[derive(Clone)]
20pub struct TransformerEncoder<
21    T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
22    B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default = MoiraiBackend,
23    const H: usize = 8,
24    const N: usize = 6,
25    M: AttentionMask = coeus_autograd::NullMask,
26> {
27    /// Fixed-size array of encoder layers — no heap allocation for the stack.
28    pub layers: [TransformerEncoderLayer<T, B, H, M>; N],
29}
30
31impl<
32        T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
33        B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
34        const H: usize,
35        const N: usize,
36        M: AttentionMask,
37    > TransformerEncoder<T, B, H, N, M>
38where
39    TransformerEncoderLayer<T, B, H, M>: Clone,
40{
41    /// Construct N independently-initialized encoder layers.
42    ///
43    /// # Errors
44    /// Returns an initialization error when `N == 0` or a layer cannot be
45    /// initialized by the selected backend.
46    pub fn new(
47        d_model: usize,
48        d_ff: usize,
49        dropout_p: f64,
50    ) -> Result<Self, crate::init::InitializationError<B::Error>>
51    where
52        T: coeus_cpu::RealScalar,
53        B: coeus_ops::RandomInitOps<T>,
54    {
55        if N == 0 {
56            return Err(crate::init::InitializationError::InvalidLayerCount { layers: N });
57        }
58        let layers = (0..N)
59            .map(|_| TransformerEncoderLayer::<T, B, H, M>::new(d_model, d_ff, dropout_p))
60            .collect::<Result<Vec<_>, _>>()?
61            .try_into()
62            .map_err(|_| crate::init::InitializationError::InvalidLayerCount { layers: N })?;
63        Ok(Self { layers })
64    }
65
66    /// Forward through all N layers sequentially with optional key padding mask.
67    ///
68    /// Input/output shape: `[batch, seq, d_model]`.
69    /// `key_padding_mask` shape: `[batch, seq]` (or backend-supported broadcast form).
70    pub fn forward_with_mask(
71        &self,
72        input: &Var<T, B>,
73        key_padding_mask: Option<&Var<T, B>>,
74    ) -> Result<Var<T, B>, ModuleError<B::Error>> {
75        self.layers.iter().try_fold(input.clone(), |x, layer| {
76            layer.forward_with_mask(&x, key_padding_mask)
77        })
78    }
79}
80
81impl<
82        T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
83        B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
84        const H: usize,
85        const N: usize,
86        M: AttentionMask,
87    > Module<T, B> for TransformerEncoder<T, B, H, N, M>
88{
89    fn parameters(&self) -> Vec<Var<T, B>> {
90        self.layers.iter().flat_map(|l| l.parameters()).collect()
91    }
92
93    fn named_parameters(&self) -> Vec<coeus_autograd::Parameter<T, B>> {
94        self.layers
95            .iter()
96            .enumerate()
97            .flat_map(|(index, layer)| prefixed_parameters(&format!("layers.{index}"), layer))
98            .collect()
99    }
100
101    /// Forward through all N layers sequentially.
102    ///
103    /// Input/output shape: `[batch, seq, d_model]`.
104    fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
105        self.forward_with_mask(input, None)
106    }
107}