Skip to main content

coeus_nn/transformer/
decoder_layer.rs

1// ── Transformer Decoder Layer (Pre-LayerNorm) ──
2
3use super::ffn::{feed_forward_with_training, FeedForward};
4use crate::attention::{multi_head_attention_cross, MhaProjectionParams, MultiHeadAttention};
5use crate::dropout::Dropout;
6use crate::module::{prefixed_parameters, Module, ModuleError};
7use crate::normalization::LayerNorm;
8use coeus_autograd::{AttentionMask, CausalMask, NullMask, Var};
9use coeus_core::MoiraiBackend;
10
11/// Borrowed parameters for functional transformer decoder-layer execution.
12pub struct TransformerDecoderLayerParams<
13    'a,
14    T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
15    B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
16> {
17    /// LayerNorm1 gamma `[d_model]`.
18    pub norm1_weight: &'a Var<T, B>,
19    /// LayerNorm1 beta `[d_model]`.
20    pub norm1_bias: &'a Var<T, B>,
21    /// Masked self-attention projections.
22    pub self_attn: MhaProjectionParams<'a, T, B>,
23    /// LayerNorm2 gamma `[d_model]`.
24    pub norm2_weight: &'a Var<T, B>,
25    /// LayerNorm2 beta `[d_model]`.
26    pub norm2_bias: &'a Var<T, B>,
27    /// Cross-attention projections.
28    pub cross_attn: MhaProjectionParams<'a, T, B>,
29    /// LayerNorm3 gamma `[d_model]`.
30    pub norm3_weight: &'a Var<T, B>,
31    /// LayerNorm3 beta `[d_model]`.
32    pub norm3_bias: &'a Var<T, B>,
33    /// FFN linear1 weight `[d_ff, d_model]`.
34    pub ffn_w1: &'a Var<T, B>,
35    /// FFN linear1 bias `[d_ff]`.
36    pub ffn_b1: Option<&'a Var<T, B>>,
37    /// FFN linear2 weight `[d_model, d_ff]`.
38    pub ffn_w2: &'a Var<T, B>,
39    /// FFN linear2 bias `[d_model]`.
40    pub ffn_b2: Option<&'a Var<T, B>>,
41    /// Dropout probability after masked self-attention.
42    pub self_attn_residual_dropout_p: f64,
43    /// Training mode for self-attention residual dropout.
44    pub self_attn_residual_training: bool,
45    /// Dropout probability after cross-attention.
46    pub cross_attn_residual_dropout_p: f64,
47    /// Training mode for cross-attention residual dropout.
48    pub cross_attn_residual_training: bool,
49    /// Dropout probability inside FFN hidden projection.
50    pub ffn_hidden_dropout_p: f64,
51    /// Training mode for FFN hidden dropout.
52    pub ffn_hidden_training: bool,
53    /// Dropout probability after FFN residual branch.
54    pub ffn_residual_dropout_p: f64,
55    /// Training mode for FFN residual dropout.
56    pub ffn_residual_training: bool,
57}
58
59fn apply_dropout<
60    T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
61    B: coeus_ops::BackendOps<T> + Default,
62>(
63    x: &Var<T, B>,
64    p: f64,
65    is_training: bool,
66) -> Result<Var<T, B>, ModuleError<B::Error>> {
67    let mut dropout = Dropout::new(p);
68    dropout.set_training(is_training);
69    dropout.forward(x)
70}
71
72/// Functional (stateless) TransformerDecoderLayer forward.
73///
74/// Computes pre-LN decoder sublayers:
75/// `x1 = tgt + Dropout(SelfAttn(LN1(tgt)))`
76/// `x2 = x1 + Dropout(CrossAttn(LN2(x1), memory, memory))`
77/// `x3 = x2 + Dropout(Linear2(Dropout(GELU(Linear1(LN3(x2))))))`.
78pub fn transformer_decoder_layer<
79    T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
80    B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
81    const H: usize,
82    SelfM: AttentionMask,
83    CrossM: AttentionMask,
84>(
85    tgt: &Var<T, B>,
86    memory: &Var<T, B>,
87    params: TransformerDecoderLayerParams<'_, T, B>,
88) -> Result<Var<T, B>, ModuleError<B::Error>> {
89    const MODULE: &str = "TransformerDecoderLayer";
90    let ([_, _, d_model], _) = super::validation::decoder_inputs(MODULE, tgt, memory)?;
91    for (parameter, value) in [
92        ("norm1 weight", params.norm1_weight),
93        ("norm1 bias", params.norm1_bias),
94        ("norm2 weight", params.norm2_weight),
95        ("norm2 bias", params.norm2_bias),
96        ("norm3 weight", params.norm3_weight),
97        ("norm3 bias", params.norm3_bias),
98    ] {
99        super::validation::affine_vector(MODULE, parameter, value, d_model)?;
100    }
101    super::validation::feed_forward(
102        MODULE,
103        d_model,
104        params.ffn_w1,
105        params.ffn_b1,
106        params.ffn_w2,
107        params.ffn_b2,
108    )?;
109    let normed1 = super::normalization::layer_norm_three_dimensional(
110        MODULE,
111        tgt,
112        params.norm1_weight,
113        params.norm1_bias,
114        1e-5,
115    )?;
116    let self_attn_out = multi_head_attention_cross::<T, B, H, SelfM>(
117        &normed1,
118        &normed1,
119        &normed1,
120        params.self_attn,
121        None,
122    )?;
123    let dropped1 = apply_dropout(
124        &self_attn_out,
125        params.self_attn_residual_dropout_p,
126        params.self_attn_residual_training,
127    )?;
128    let x = coeus_autograd::add(tgt, &dropped1)?;
129
130    let normed2 = super::normalization::layer_norm_three_dimensional(
131        MODULE,
132        &x,
133        params.norm2_weight,
134        params.norm2_bias,
135        1e-5,
136    )?;
137    let cross_attn_out = multi_head_attention_cross::<T, B, H, CrossM>(
138        &normed2,
139        memory,
140        memory,
141        params.cross_attn,
142        None,
143    )?;
144    let dropped2 = apply_dropout(
145        &cross_attn_out,
146        params.cross_attn_residual_dropout_p,
147        params.cross_attn_residual_training,
148    )?;
149    let x = coeus_autograd::add(&x, &dropped2)?;
150
151    let normed3 = super::normalization::layer_norm_three_dimensional(
152        MODULE,
153        &x,
154        params.norm3_weight,
155        params.norm3_bias,
156        1e-5,
157    )?;
158    let ff_out = feed_forward_with_training(
159        &normed3,
160        params.ffn_w1,
161        params.ffn_b1,
162        params.ffn_w2,
163        params.ffn_b2,
164        params.ffn_hidden_dropout_p,
165        params.ffn_hidden_training,
166    )?;
167    let dropped3 = apply_dropout(
168        &ff_out,
169        params.ffn_residual_dropout_p,
170        params.ffn_residual_training,
171    )?;
172    Ok(coeus_autograd::add(&x, &dropped3)?)
173}
174
175/// Single Transformer decoder layer.
176///
177/// Pre-LayerNorm sub-layer composition:
178/// ```text
179///   x₁ = x + Dropout(SelfAttn(LayerNorm(x)))
180///   x₂ = x₁ + Dropout(CrossAttn(LayerNorm(x₁), memory, memory))
181///   x₃ = x₂ + Dropout(FFN(LayerNorm(x₂)))
182/// ```
183pub struct TransformerDecoderLayer<
184    T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
185    B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default = MoiraiBackend,
186    const H: usize = 8,
187    SelfM: AttentionMask = CausalMask,
188    CrossM: AttentionMask = NullMask,
189> {
190    /// Pre-LayerNorm before self-attention.
191    pub norm1: LayerNorm<T, B>,
192    /// Masked self-attention sub-layer.
193    pub self_attn: MultiHeadAttention<T, B, H, SelfM>,
194    /// Dropout applied after self-attention.
195    pub dropout1: Dropout,
196    /// Pre-LayerNorm before cross-attention.
197    pub norm2: LayerNorm<T, B>,
198    /// Cross-attention sub-layer attending to encoder memory.
199    pub cross_attn: MultiHeadAttention<T, B, H, CrossM>,
200    /// Dropout applied after cross-attention.
201    pub dropout2: Dropout,
202    /// Pre-LayerNorm before FFN.
203    pub norm3: LayerNorm<T, B>,
204    /// Feed-forward sub-layer.
205    pub ffn: FeedForward<T, B>,
206    /// Dropout applied after FFN.
207    pub dropout3: Dropout,
208}
209
210impl<
211        T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
212        B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
213        const H: usize,
214        SelfM: AttentionMask,
215        CrossM: AttentionMask,
216    > TransformerDecoderLayer<T, B, H, SelfM, CrossM>
217{
218    /// Construct a decoder layer.
219    ///
220    /// # Errors
221    ///
222    /// Returns a typed error when attention dimensions are invalid or the
223    /// selected backend cannot initialize projection weights.
224    pub fn new(
225        d_model: usize,
226        d_ff: usize,
227        dropout_p: f64,
228    ) -> Result<Self, crate::init::InitializationError<B::Error>>
229    where
230        T: coeus_cpu::RealScalar,
231        B: coeus_ops::RandomInitOps<T>,
232    {
233        Ok(Self {
234            norm1: LayerNorm::new(d_model, 1e-5)?,
235            self_attn: MultiHeadAttention::new(d_model, true)?,
236            dropout1: Dropout::new(dropout_p),
237            norm2: LayerNorm::new(d_model, 1e-5)?,
238            cross_attn: MultiHeadAttention::new(d_model, true)?,
239            dropout2: Dropout::new(dropout_p),
240            norm3: LayerNorm::new(d_model, 1e-5)?,
241            ffn: FeedForward::new(d_model, d_ff, dropout_p)?,
242            dropout3: Dropout::new(dropout_p),
243        })
244    }
245
246    /// Decode forward pass with cross-attention.
247    ///
248    /// - `tgt`:    target inputs `[batch, seq_tgt, d_model]`
249    /// - `memory`: encoder outputs `[batch, seq_src, d_model]`
250    pub fn forward_decoder(
251        &self,
252        tgt: &Var<T, B>,
253        memory: &Var<T, B>,
254    ) -> Result<Var<T, B>, ModuleError<B::Error>> {
255        transformer_decoder_layer::<T, B, H, SelfM, CrossM>(
256            tgt,
257            memory,
258            TransformerDecoderLayerParams {
259                norm1_weight: &self.norm1.weight,
260                norm1_bias: &self.norm1.bias,
261                self_attn: MhaProjectionParams {
262                    w_q: &self.self_attn.w_q,
263                    b_q: self.self_attn.b_q.as_ref(),
264                    w_k: &self.self_attn.w_k,
265                    b_k: self.self_attn.b_k.as_ref(),
266                    w_v: &self.self_attn.w_v,
267                    b_v: self.self_attn.b_v.as_ref(),
268                    w_o: &self.self_attn.w_o,
269                    b_o: self.self_attn.b_o.as_ref(),
270                },
271                norm2_weight: &self.norm2.weight,
272                norm2_bias: &self.norm2.bias,
273                cross_attn: MhaProjectionParams {
274                    w_q: &self.cross_attn.w_q,
275                    b_q: self.cross_attn.b_q.as_ref(),
276                    w_k: &self.cross_attn.w_k,
277                    b_k: self.cross_attn.b_k.as_ref(),
278                    w_v: &self.cross_attn.w_v,
279                    b_v: self.cross_attn.b_v.as_ref(),
280                    w_o: &self.cross_attn.w_o,
281                    b_o: self.cross_attn.b_o.as_ref(),
282                },
283                norm3_weight: &self.norm3.weight,
284                norm3_bias: &self.norm3.bias,
285                ffn_w1: &self.ffn.linear1.weight,
286                ffn_b1: self.ffn.linear1.bias.as_ref(),
287                ffn_w2: &self.ffn.linear2.weight,
288                ffn_b2: self.ffn.linear2.bias.as_ref(),
289                self_attn_residual_dropout_p: self.dropout1.p,
290                self_attn_residual_training: self.dropout1.is_training,
291                cross_attn_residual_dropout_p: self.dropout2.p,
292                cross_attn_residual_training: self.dropout2.is_training,
293                ffn_hidden_dropout_p: self.ffn.dropout.p,
294                ffn_hidden_training: self.ffn.dropout.is_training,
295                ffn_residual_dropout_p: self.dropout3.p,
296                ffn_residual_training: self.dropout3.is_training,
297            },
298        )
299    }
300}
301
302impl<
303        T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
304        B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
305        const H: usize,
306        SelfM: AttentionMask,
307        CrossM: AttentionMask,
308    > TransformerDecoderLayer<T, B, H, SelfM, CrossM>
309{
310    /// Collect all trainable decoder-layer parameters.
311    pub fn parameters(&self) -> Vec<Var<T, B>> {
312        let mut p = self.norm1.parameters();
313        p.extend(self.self_attn.parameters());
314        p.extend(self.norm2.parameters());
315        p.extend(self.cross_attn.parameters());
316        p.extend(self.norm3.parameters());
317        p.extend(self.ffn.parameters());
318        p
319    }
320
321    /// Collect trainable parameters with stable hierarchical names.
322    pub fn named_parameters(&self) -> Vec<coeus_autograd::Parameter<T, B>> {
323        let mut parameters = prefixed_parameters("norm1", &self.norm1);
324        parameters.extend(prefixed_parameters("self_attention", &self.self_attn));
325        parameters.extend(prefixed_parameters("norm2", &self.norm2));
326        parameters.extend(prefixed_parameters("cross_attention", &self.cross_attn));
327        parameters.extend(prefixed_parameters("norm3", &self.norm3));
328        parameters.extend(prefixed_parameters("feed_forward", &self.ffn));
329        parameters
330    }
331}
332
333/// Manual Clone impl.
334impl<
335        T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
336        B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
337        const H: usize,
338        SelfM: AttentionMask,
339        CrossM: AttentionMask,
340    > Clone for TransformerDecoderLayer<T, B, H, SelfM, CrossM>
341where
342    LayerNorm<T, B>: Clone,
343    MultiHeadAttention<T, B, H, SelfM>: Clone,
344    MultiHeadAttention<T, B, H, CrossM>: Clone,
345    FeedForward<T, B>: Clone,
346{
347    fn clone(&self) -> Self {
348        Self {
349            norm1: self.norm1.clone(),
350            self_attn: self.self_attn.clone(),
351            dropout1: self.dropout1.clone(),
352            norm2: self.norm2.clone(),
353            cross_attn: self.cross_attn.clone(),
354            dropout2: self.dropout2.clone(),
355            norm3: self.norm3.clone(),
356            ffn: self.ffn.clone(),
357            dropout3: self.dropout3.clone(),
358        }
359    }
360}