coeus-nn 0.14.0

Neural-network layers and operators for Coeus.
Documentation
// ── Transformer Encoder stack ──
//
// N independently-initialized encoder layers in a fixed-size array.
// `N` is a const generic: `[TransformerEncoderLayer; N]` is stack-allocated with
// no heap overhead for the container. Monomorphization emits one specialization
// per `(T, B, H, N, M)`.

use super::encoder_layer::TransformerEncoderLayer;
use crate::module::{prefixed_parameters, Module, ModuleError};
use coeus_autograd::{AttentionMask, Var};
use coeus_core::MoiraiBackend;

/// Stack of N `TransformerEncoderLayer`s.
///
/// # Type parameters
/// - `H` — attention heads per layer (const generic, uniform across layers)
/// - `N` — number of stacked encoder layers (const generic)
/// - `M` — masking strategy ZST
#[derive(Clone)]
pub struct TransformerEncoder<
    T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
    B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default = MoiraiBackend,
    const H: usize = 8,
    const N: usize = 6,
    M: AttentionMask = coeus_autograd::NullMask,
> {
    /// Fixed-size array of encoder layers — no heap allocation for the stack.
    pub layers: [TransformerEncoderLayer<T, B, H, M>; N],
}

impl<
        T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
        B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
        const H: usize,
        const N: usize,
        M: AttentionMask,
    > TransformerEncoder<T, B, H, N, M>
where
    TransformerEncoderLayer<T, B, H, M>: Clone,
{
    /// Construct N independently-initialized encoder layers.
    ///
    /// # Errors
    /// Returns an initialization error when `N == 0` or a layer cannot be
    /// initialized by the selected backend.
    pub fn new(
        d_model: usize,
        d_ff: usize,
        dropout_p: f64,
    ) -> Result<Self, crate::init::InitializationError<B::Error>>
    where
        T: coeus_cpu::RealScalar,
        B: coeus_ops::RandomInitOps<T>,
    {
        if N == 0 {
            return Err(crate::init::InitializationError::InvalidLayerCount { layers: N });
        }
        let layers = (0..N)
            .map(|_| TransformerEncoderLayer::<T, B, H, M>::new(d_model, d_ff, dropout_p))
            .collect::<Result<Vec<_>, _>>()?
            .try_into()
            .map_err(|_| crate::init::InitializationError::InvalidLayerCount { layers: N })?;
        Ok(Self { layers })
    }

    /// Forward through all N layers sequentially with optional key padding mask.
    ///
    /// Input/output shape: `[batch, seq, d_model]`.
    /// `key_padding_mask` shape: `[batch, seq]` (or backend-supported broadcast form).
    pub fn forward_with_mask(
        &self,
        input: &Var<T, B>,
        key_padding_mask: Option<&Var<T, B>>,
    ) -> Result<Var<T, B>, ModuleError<B::Error>> {
        self.layers.iter().try_fold(input.clone(), |x, layer| {
            layer.forward_with_mask(&x, key_padding_mask)
        })
    }
}

impl<
        T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
        B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
        const H: usize,
        const N: usize,
        M: AttentionMask,
    > Module<T, B> for TransformerEncoder<T, B, H, N, M>
{
    fn parameters(&self) -> Vec<Var<T, B>> {
        self.layers.iter().flat_map(|l| l.parameters()).collect()
    }

    fn named_parameters(&self) -> Vec<coeus_autograd::Parameter<T, B>> {
        self.layers
            .iter()
            .enumerate()
            .flat_map(|(index, layer)| prefixed_parameters(&format!("layers.{index}"), layer))
            .collect()
    }

    /// Forward through all N layers sequentially.
    ///
    /// Input/output shape: `[batch, seq, d_model]`.
    fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
        self.forward_with_mask(input, None)
    }
}