use super::encoder_layer::TransformerEncoderLayer;
use crate::module::{prefixed_parameters, Module, ModuleError};
use coeus_autograd::{AttentionMask, Var};
use coeus_core::MoiraiBackend;
#[derive(Clone)]
pub struct TransformerEncoder<
T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default = MoiraiBackend,
const H: usize = 8,
const N: usize = 6,
M: AttentionMask = coeus_autograd::NullMask,
> {
pub layers: [TransformerEncoderLayer<T, B, H, M>; N],
}
impl<
T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
const H: usize,
const N: usize,
M: AttentionMask,
> TransformerEncoder<T, B, H, N, M>
where
TransformerEncoderLayer<T, B, H, M>: Clone,
{
pub fn new(
d_model: usize,
d_ff: usize,
dropout_p: f64,
) -> Result<Self, crate::init::InitializationError<B::Error>>
where
T: coeus_cpu::RealScalar,
B: coeus_ops::RandomInitOps<T>,
{
if N == 0 {
return Err(crate::init::InitializationError::InvalidLayerCount { layers: N });
}
let layers = (0..N)
.map(|_| TransformerEncoderLayer::<T, B, H, M>::new(d_model, d_ff, dropout_p))
.collect::<Result<Vec<_>, _>>()?
.try_into()
.map_err(|_| crate::init::InitializationError::InvalidLayerCount { layers: N })?;
Ok(Self { layers })
}
pub fn forward_with_mask(
&self,
input: &Var<T, B>,
key_padding_mask: Option<&Var<T, B>>,
) -> Result<Var<T, B>, ModuleError<B::Error>> {
self.layers.iter().try_fold(input.clone(), |x, layer| {
layer.forward_with_mask(&x, key_padding_mask)
})
}
}
impl<
T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
const H: usize,
const N: usize,
M: AttentionMask,
> Module<T, B> for TransformerEncoder<T, B, H, N, M>
{
fn parameters(&self) -> Vec<Var<T, B>> {
self.layers.iter().flat_map(|l| l.parameters()).collect()
}
fn named_parameters(&self) -> Vec<coeus_autograd::Parameter<T, B>> {
self.layers
.iter()
.enumerate()
.flat_map(|(index, layer)| prefixed_parameters(&format!("layers.{index}"), layer))
.collect()
}
fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
self.forward_with_mask(input, None)
}
}