coeus_nn/transformer/
encoder.rs1use super::encoder_layer::TransformerEncoderLayer;
9use crate::module::{prefixed_parameters, Module, ModuleError};
10use coeus_autograd::{AttentionMask, Var};
11use coeus_core::MoiraiBackend;
12
13#[derive(Clone)]
20pub struct TransformerEncoder<
21 T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
22 B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default = MoiraiBackend,
23 const H: usize = 8,
24 const N: usize = 6,
25 M: AttentionMask = coeus_autograd::NullMask,
26> {
27 pub layers: [TransformerEncoderLayer<T, B, H, M>; N],
29}
30
31impl<
32 T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
33 B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
34 const H: usize,
35 const N: usize,
36 M: AttentionMask,
37 > TransformerEncoder<T, B, H, N, M>
38where
39 TransformerEncoderLayer<T, B, H, M>: Clone,
40{
41 pub fn new(
47 d_model: usize,
48 d_ff: usize,
49 dropout_p: f64,
50 ) -> Result<Self, crate::init::InitializationError<B::Error>>
51 where
52 T: coeus_cpu::RealScalar,
53 B: coeus_ops::RandomInitOps<T>,
54 {
55 if N == 0 {
56 return Err(crate::init::InitializationError::InvalidLayerCount { layers: N });
57 }
58 let layers = (0..N)
59 .map(|_| TransformerEncoderLayer::<T, B, H, M>::new(d_model, d_ff, dropout_p))
60 .collect::<Result<Vec<_>, _>>()?
61 .try_into()
62 .map_err(|_| crate::init::InitializationError::InvalidLayerCount { layers: N })?;
63 Ok(Self { layers })
64 }
65
66 pub fn forward_with_mask(
71 &self,
72 input: &Var<T, B>,
73 key_padding_mask: Option<&Var<T, B>>,
74 ) -> Result<Var<T, B>, ModuleError<B::Error>> {
75 self.layers.iter().try_fold(input.clone(), |x, layer| {
76 layer.forward_with_mask(&x, key_padding_mask)
77 })
78 }
79}
80
81impl<
82 T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
83 B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
84 const H: usize,
85 const N: usize,
86 M: AttentionMask,
87 > Module<T, B> for TransformerEncoder<T, B, H, N, M>
88{
89 fn parameters(&self) -> Vec<Var<T, B>> {
90 self.layers.iter().flat_map(|l| l.parameters()).collect()
91 }
92
93 fn named_parameters(&self) -> Vec<coeus_autograd::Parameter<T, B>> {
94 self.layers
95 .iter()
96 .enumerate()
97 .flat_map(|(index, layer)| prefixed_parameters(&format!("layers.{index}"), layer))
98 .collect()
99 }
100
101 fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
105 self.forward_with_mask(input, None)
106 }
107}