Skip to main content

coeus_nn/transformer/
encoder_layer.rs

1// ── Transformer Encoder Layer (Pre-LayerNorm) ──
2
3use super::ffn::{feed_forward_with_training, FeedForward};
4use crate::attention::{
5    multi_head_attention_cross, MhaProjectionParams, MultiHeadAttention, NullMask,
6};
7use crate::dropout::Dropout;
8use crate::module::{prefixed_parameters, Module, ModuleError};
9use crate::normalization::LayerNorm;
10use coeus_autograd::{AttentionMask, Var};
11use coeus_core::MoiraiBackend;
12
13/// Borrowed parameters for functional transformer encoder-layer execution.
14pub struct TransformerEncoderLayerParams<
15    'a,
16    T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
17    B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
18> {
19    /// LayerNorm1 gamma `[d_model]`.
20    pub norm1_weight: &'a Var<T, B>,
21    /// LayerNorm1 beta `[d_model]`.
22    pub norm1_bias: &'a Var<T, B>,
23    /// Self-attention projections.
24    pub self_attn: MhaProjectionParams<'a, T, B>,
25    /// LayerNorm2 gamma `[d_model]`.
26    pub norm2_weight: &'a Var<T, B>,
27    /// LayerNorm2 beta `[d_model]`.
28    pub norm2_bias: &'a Var<T, B>,
29    /// FFN linear1 weight `[d_ff, d_model]`.
30    pub ffn_w1: &'a Var<T, B>,
31    /// FFN linear1 bias `[d_ff]`.
32    pub ffn_b1: Option<&'a Var<T, B>>,
33    /// FFN linear2 weight `[d_model, d_ff]`.
34    pub ffn_w2: &'a Var<T, B>,
35    /// FFN linear2 bias `[d_model]`.
36    pub ffn_b2: Option<&'a Var<T, B>>,
37    /// Dropout probability after self-attention.
38    pub attn_residual_dropout_p: f64,
39    /// Training mode for dropout after self-attention.
40    pub attn_residual_training: bool,
41    /// Dropout probability inside FFN hidden projection.
42    pub ffn_hidden_dropout_p: f64,
43    /// Training mode for FFN hidden dropout.
44    pub ffn_hidden_training: bool,
45    /// Dropout probability after FFN residual branch.
46    pub ffn_residual_dropout_p: f64,
47    /// Training mode for dropout after FFN.
48    pub ffn_residual_training: bool,
49}
50
51fn apply_dropout<
52    T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
53    B: coeus_ops::BackendOps<T> + Default,
54>(
55    x: &Var<T, B>,
56    p: f64,
57    is_training: bool,
58) -> Result<Var<T, B>, ModuleError<B::Error>> {
59    let mut dropout = Dropout::new(p);
60    dropout.set_training(is_training);
61    dropout.forward(x)
62}
63
64/// Functional (stateless) TransformerEncoderLayer forward.
65///
66/// Computes pre-LN encoder sublayers:
67/// `x1 = x + Dropout(SelfAttn(LN1(x)))`
68/// `x2 = x1 + Dropout(Linear2(Dropout(GELU(Linear1(LN2(x1))))))`.
69pub fn transformer_encoder_layer<
70    T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
71    B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
72    const H: usize,
73    M: AttentionMask,
74>(
75    input: &Var<T, B>,
76    key_padding_mask: Option<&Var<T, B>>,
77    params: TransformerEncoderLayerParams<'_, T, B>,
78) -> Result<Var<T, B>, ModuleError<B::Error>> {
79    const MODULE: &str = "TransformerEncoderLayer";
80    let [_, _, d_model] = super::validation::rank_three(MODULE, input)?;
81    super::validation::affine_vector(MODULE, "norm1 weight", params.norm1_weight, d_model)?;
82    super::validation::affine_vector(MODULE, "norm1 bias", params.norm1_bias, d_model)?;
83    super::validation::affine_vector(MODULE, "norm2 weight", params.norm2_weight, d_model)?;
84    super::validation::affine_vector(MODULE, "norm2 bias", params.norm2_bias, d_model)?;
85    super::validation::feed_forward(
86        MODULE,
87        d_model,
88        params.ffn_w1,
89        params.ffn_b1,
90        params.ffn_w2,
91        params.ffn_b2,
92    )?;
93    let normed1 = super::normalization::layer_norm_three_dimensional(
94        MODULE,
95        input,
96        params.norm1_weight,
97        params.norm1_bias,
98        1e-5,
99    )?;
100    let attn_out = multi_head_attention_cross::<T, B, H, M>(
101        &normed1,
102        &normed1,
103        &normed1,
104        params.self_attn,
105        key_padding_mask,
106    )?;
107    let dropped1 = apply_dropout(
108        &attn_out,
109        params.attn_residual_dropout_p,
110        params.attn_residual_training,
111    )?;
112    let x = coeus_autograd::add(input, &dropped1)?;
113
114    let normed2 = super::normalization::layer_norm_three_dimensional(
115        MODULE,
116        &x,
117        params.norm2_weight,
118        params.norm2_bias,
119        1e-5,
120    )?;
121    let ff_out = feed_forward_with_training(
122        &normed2,
123        params.ffn_w1,
124        params.ffn_b1,
125        params.ffn_w2,
126        params.ffn_b2,
127        params.ffn_hidden_dropout_p,
128        params.ffn_hidden_training,
129    )?;
130    let dropped2 = apply_dropout(
131        &ff_out,
132        params.ffn_residual_dropout_p,
133        params.ffn_residual_training,
134    )?;
135    Ok(coeus_autograd::add(&x, &dropped2)?)
136}
137
138/// Single Transformer encoder layer.
139///
140/// Pre-LayerNorm sub-layer composition:
141/// ```text
142///   x₁ = x + Dropout(SelfAttn(LayerNorm(x)))
143///   x₂ = x₁ + Dropout(FFN(LayerNorm(x₁)))
144/// ```
145pub struct TransformerEncoderLayer<
146    T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
147    B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default = MoiraiBackend,
148    const H: usize = 8,
149    M: AttentionMask = NullMask,
150> {
151    /// Pre-LayerNorm before self-attention.
152    pub norm1: LayerNorm<T, B>,
153    /// Self-attention sub-layer.
154    pub self_attn: MultiHeadAttention<T, B, H, M>,
155    /// Dropout applied after self-attention.
156    pub dropout1: Dropout,
157    /// Pre-LayerNorm before FFN.
158    pub norm2: LayerNorm<T, B>,
159    /// Feed-forward sub-layer.
160    pub ffn: FeedForward<T, B>,
161    /// Dropout applied after FFN.
162    pub dropout2: Dropout,
163}
164
165impl<
166        T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
167        B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
168        const H: usize,
169        M: AttentionMask,
170    > TransformerEncoderLayer<T, B, H, M>
171{
172    /// Construct an encoder layer.
173    ///
174    /// # Errors
175    ///
176    /// Returns a typed error when attention dimensions are invalid or the
177    /// selected backend cannot initialize projection weights.
178    pub fn new(
179        d_model: usize,
180        d_ff: usize,
181        dropout_p: f64,
182    ) -> Result<Self, crate::init::InitializationError<B::Error>>
183    where
184        T: coeus_cpu::RealScalar,
185        B: coeus_ops::RandomInitOps<T>,
186    {
187        Ok(Self {
188            norm1: LayerNorm::new(d_model, 1e-5)?,
189            self_attn: MultiHeadAttention::new(d_model, true)?,
190            dropout1: Dropout::new(dropout_p),
191            norm2: LayerNorm::new(d_model, 1e-5)?,
192            ffn: FeedForward::new(d_model, d_ff, dropout_p)?,
193            dropout2: Dropout::new(dropout_p),
194        })
195    }
196
197    /// Pre-LayerNorm forward with optional key padding mask.
198    ///
199    /// Input/output shape: `[batch, seq, d_model]`.
200    /// `key_padding_mask` shape: `[batch, seq]` (or backend-supported broadcast form).
201    pub fn forward_with_mask(
202        &self,
203        input: &Var<T, B>,
204        key_padding_mask: Option<&Var<T, B>>,
205    ) -> Result<Var<T, B>, ModuleError<B::Error>> {
206        transformer_encoder_layer::<T, B, H, M>(
207            input,
208            key_padding_mask,
209            TransformerEncoderLayerParams {
210                norm1_weight: &self.norm1.weight,
211                norm1_bias: &self.norm1.bias,
212                self_attn: MhaProjectionParams {
213                    w_q: &self.self_attn.w_q,
214                    b_q: self.self_attn.b_q.as_ref(),
215                    w_k: &self.self_attn.w_k,
216                    b_k: self.self_attn.b_k.as_ref(),
217                    w_v: &self.self_attn.w_v,
218                    b_v: self.self_attn.b_v.as_ref(),
219                    w_o: &self.self_attn.w_o,
220                    b_o: self.self_attn.b_o.as_ref(),
221                },
222                norm2_weight: &self.norm2.weight,
223                norm2_bias: &self.norm2.bias,
224                ffn_w1: &self.ffn.linear1.weight,
225                ffn_b1: self.ffn.linear1.bias.as_ref(),
226                ffn_w2: &self.ffn.linear2.weight,
227                ffn_b2: self.ffn.linear2.bias.as_ref(),
228                attn_residual_dropout_p: self.dropout1.p,
229                attn_residual_training: self.dropout1.is_training,
230                ffn_hidden_dropout_p: self.ffn.dropout.p,
231                ffn_hidden_training: self.ffn.dropout.is_training,
232                ffn_residual_dropout_p: self.dropout2.p,
233                ffn_residual_training: self.dropout2.is_training,
234            },
235        )
236    }
237}
238
239impl<
240        T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
241        B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
242        const H: usize,
243        M: AttentionMask,
244    > Module<T, B> for TransformerEncoderLayer<T, B, H, M>
245{
246    fn parameters(&self) -> Vec<Var<T, B>> {
247        let mut p = self.norm1.parameters();
248        p.extend(self.self_attn.parameters());
249        p.extend(self.norm2.parameters());
250        p.extend(self.ffn.parameters());
251        p
252    }
253
254    fn named_parameters(&self) -> Vec<coeus_autograd::Parameter<T, B>> {
255        let mut parameters = prefixed_parameters("norm1", &self.norm1);
256        parameters.extend(prefixed_parameters("self_attention", &self.self_attn));
257        parameters.extend(prefixed_parameters("norm2", &self.norm2));
258        parameters.extend(prefixed_parameters("feed_forward", &self.ffn));
259        parameters
260    }
261
262    /// Pre-LayerNorm forward. Input/output: `[batch, seq, d_model]`.
263    fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
264        self.forward_with_mask(input, None)
265    }
266}
267
268/// Manual Clone impl.
269impl<
270        T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
271        B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
272        const H: usize,
273        M: AttentionMask,
274    > Clone for TransformerEncoderLayer<T, B, H, M>
275where
276    LayerNorm<T, B>: Clone,
277    MultiHeadAttention<T, B, H, M>: Clone,
278    FeedForward<T, B>: Clone,
279{
280    fn clone(&self) -> Self {
281        Self {
282            norm1: self.norm1.clone(),
283            self_attn: self.self_attn.clone(),
284            dropout1: self.dropout1.clone(),
285            norm2: self.norm2.clone(),
286            ffn: self.ffn.clone(),
287            dropout2: self.dropout2.clone(),
288        }
289    }
290}