1use super::ffn::{feed_forward_with_training, FeedForward};
4use crate::attention::{multi_head_attention_cross, MhaProjectionParams, MultiHeadAttention};
5use crate::dropout::Dropout;
6use crate::module::{prefixed_parameters, Module, ModuleError};
7use crate::normalization::LayerNorm;
8use coeus_autograd::{AttentionMask, CausalMask, NullMask, Var};
9use coeus_core::MoiraiBackend;
10
11pub struct TransformerDecoderLayerParams<
13 'a,
14 T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
15 B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
16> {
17 pub norm1_weight: &'a Var<T, B>,
19 pub norm1_bias: &'a Var<T, B>,
21 pub self_attn: MhaProjectionParams<'a, T, B>,
23 pub norm2_weight: &'a Var<T, B>,
25 pub norm2_bias: &'a Var<T, B>,
27 pub cross_attn: MhaProjectionParams<'a, T, B>,
29 pub norm3_weight: &'a Var<T, B>,
31 pub norm3_bias: &'a Var<T, B>,
33 pub ffn_w1: &'a Var<T, B>,
35 pub ffn_b1: Option<&'a Var<T, B>>,
37 pub ffn_w2: &'a Var<T, B>,
39 pub ffn_b2: Option<&'a Var<T, B>>,
41 pub self_attn_residual_dropout_p: f64,
43 pub self_attn_residual_training: bool,
45 pub cross_attn_residual_dropout_p: f64,
47 pub cross_attn_residual_training: bool,
49 pub ffn_hidden_dropout_p: f64,
51 pub ffn_hidden_training: bool,
53 pub ffn_residual_dropout_p: f64,
55 pub ffn_residual_training: bool,
57}
58
59fn apply_dropout<
60 T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
61 B: coeus_ops::BackendOps<T> + Default,
62>(
63 x: &Var<T, B>,
64 p: f64,
65 is_training: bool,
66) -> Result<Var<T, B>, ModuleError<B::Error>> {
67 let mut dropout = Dropout::new(p);
68 dropout.set_training(is_training);
69 dropout.forward(x)
70}
71
72pub fn transformer_decoder_layer<
79 T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
80 B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
81 const H: usize,
82 SelfM: AttentionMask,
83 CrossM: AttentionMask,
84>(
85 tgt: &Var<T, B>,
86 memory: &Var<T, B>,
87 params: TransformerDecoderLayerParams<'_, T, B>,
88) -> Result<Var<T, B>, ModuleError<B::Error>> {
89 const MODULE: &str = "TransformerDecoderLayer";
90 let ([_, _, d_model], _) = super::validation::decoder_inputs(MODULE, tgt, memory)?;
91 for (parameter, value) in [
92 ("norm1 weight", params.norm1_weight),
93 ("norm1 bias", params.norm1_bias),
94 ("norm2 weight", params.norm2_weight),
95 ("norm2 bias", params.norm2_bias),
96 ("norm3 weight", params.norm3_weight),
97 ("norm3 bias", params.norm3_bias),
98 ] {
99 super::validation::affine_vector(MODULE, parameter, value, d_model)?;
100 }
101 super::validation::feed_forward(
102 MODULE,
103 d_model,
104 params.ffn_w1,
105 params.ffn_b1,
106 params.ffn_w2,
107 params.ffn_b2,
108 )?;
109 let normed1 = super::normalization::layer_norm_three_dimensional(
110 MODULE,
111 tgt,
112 params.norm1_weight,
113 params.norm1_bias,
114 1e-5,
115 )?;
116 let self_attn_out = multi_head_attention_cross::<T, B, H, SelfM>(
117 &normed1,
118 &normed1,
119 &normed1,
120 params.self_attn,
121 None,
122 )?;
123 let dropped1 = apply_dropout(
124 &self_attn_out,
125 params.self_attn_residual_dropout_p,
126 params.self_attn_residual_training,
127 )?;
128 let x = coeus_autograd::add(tgt, &dropped1)?;
129
130 let normed2 = super::normalization::layer_norm_three_dimensional(
131 MODULE,
132 &x,
133 params.norm2_weight,
134 params.norm2_bias,
135 1e-5,
136 )?;
137 let cross_attn_out = multi_head_attention_cross::<T, B, H, CrossM>(
138 &normed2,
139 memory,
140 memory,
141 params.cross_attn,
142 None,
143 )?;
144 let dropped2 = apply_dropout(
145 &cross_attn_out,
146 params.cross_attn_residual_dropout_p,
147 params.cross_attn_residual_training,
148 )?;
149 let x = coeus_autograd::add(&x, &dropped2)?;
150
151 let normed3 = super::normalization::layer_norm_three_dimensional(
152 MODULE,
153 &x,
154 params.norm3_weight,
155 params.norm3_bias,
156 1e-5,
157 )?;
158 let ff_out = feed_forward_with_training(
159 &normed3,
160 params.ffn_w1,
161 params.ffn_b1,
162 params.ffn_w2,
163 params.ffn_b2,
164 params.ffn_hidden_dropout_p,
165 params.ffn_hidden_training,
166 )?;
167 let dropped3 = apply_dropout(
168 &ff_out,
169 params.ffn_residual_dropout_p,
170 params.ffn_residual_training,
171 )?;
172 Ok(coeus_autograd::add(&x, &dropped3)?)
173}
174
175pub struct TransformerDecoderLayer<
184 T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
185 B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default = MoiraiBackend,
186 const H: usize = 8,
187 SelfM: AttentionMask = CausalMask,
188 CrossM: AttentionMask = NullMask,
189> {
190 pub norm1: LayerNorm<T, B>,
192 pub self_attn: MultiHeadAttention<T, B, H, SelfM>,
194 pub dropout1: Dropout,
196 pub norm2: LayerNorm<T, B>,
198 pub cross_attn: MultiHeadAttention<T, B, H, CrossM>,
200 pub dropout2: Dropout,
202 pub norm3: LayerNorm<T, B>,
204 pub ffn: FeedForward<T, B>,
206 pub dropout3: Dropout,
208}
209
210impl<
211 T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
212 B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
213 const H: usize,
214 SelfM: AttentionMask,
215 CrossM: AttentionMask,
216 > TransformerDecoderLayer<T, B, H, SelfM, CrossM>
217{
218 pub fn new(
225 d_model: usize,
226 d_ff: usize,
227 dropout_p: f64,
228 ) -> Result<Self, crate::init::InitializationError<B::Error>>
229 where
230 T: coeus_cpu::RealScalar,
231 B: coeus_ops::RandomInitOps<T>,
232 {
233 Ok(Self {
234 norm1: LayerNorm::new(d_model, 1e-5)?,
235 self_attn: MultiHeadAttention::new(d_model, true)?,
236 dropout1: Dropout::new(dropout_p),
237 norm2: LayerNorm::new(d_model, 1e-5)?,
238 cross_attn: MultiHeadAttention::new(d_model, true)?,
239 dropout2: Dropout::new(dropout_p),
240 norm3: LayerNorm::new(d_model, 1e-5)?,
241 ffn: FeedForward::new(d_model, d_ff, dropout_p)?,
242 dropout3: Dropout::new(dropout_p),
243 })
244 }
245
246 pub fn forward_decoder(
251 &self,
252 tgt: &Var<T, B>,
253 memory: &Var<T, B>,
254 ) -> Result<Var<T, B>, ModuleError<B::Error>> {
255 transformer_decoder_layer::<T, B, H, SelfM, CrossM>(
256 tgt,
257 memory,
258 TransformerDecoderLayerParams {
259 norm1_weight: &self.norm1.weight,
260 norm1_bias: &self.norm1.bias,
261 self_attn: MhaProjectionParams {
262 w_q: &self.self_attn.w_q,
263 b_q: self.self_attn.b_q.as_ref(),
264 w_k: &self.self_attn.w_k,
265 b_k: self.self_attn.b_k.as_ref(),
266 w_v: &self.self_attn.w_v,
267 b_v: self.self_attn.b_v.as_ref(),
268 w_o: &self.self_attn.w_o,
269 b_o: self.self_attn.b_o.as_ref(),
270 },
271 norm2_weight: &self.norm2.weight,
272 norm2_bias: &self.norm2.bias,
273 cross_attn: MhaProjectionParams {
274 w_q: &self.cross_attn.w_q,
275 b_q: self.cross_attn.b_q.as_ref(),
276 w_k: &self.cross_attn.w_k,
277 b_k: self.cross_attn.b_k.as_ref(),
278 w_v: &self.cross_attn.w_v,
279 b_v: self.cross_attn.b_v.as_ref(),
280 w_o: &self.cross_attn.w_o,
281 b_o: self.cross_attn.b_o.as_ref(),
282 },
283 norm3_weight: &self.norm3.weight,
284 norm3_bias: &self.norm3.bias,
285 ffn_w1: &self.ffn.linear1.weight,
286 ffn_b1: self.ffn.linear1.bias.as_ref(),
287 ffn_w2: &self.ffn.linear2.weight,
288 ffn_b2: self.ffn.linear2.bias.as_ref(),
289 self_attn_residual_dropout_p: self.dropout1.p,
290 self_attn_residual_training: self.dropout1.is_training,
291 cross_attn_residual_dropout_p: self.dropout2.p,
292 cross_attn_residual_training: self.dropout2.is_training,
293 ffn_hidden_dropout_p: self.ffn.dropout.p,
294 ffn_hidden_training: self.ffn.dropout.is_training,
295 ffn_residual_dropout_p: self.dropout3.p,
296 ffn_residual_training: self.dropout3.is_training,
297 },
298 )
299 }
300}
301
302impl<
303 T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
304 B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
305 const H: usize,
306 SelfM: AttentionMask,
307 CrossM: AttentionMask,
308 > TransformerDecoderLayer<T, B, H, SelfM, CrossM>
309{
310 pub fn parameters(&self) -> Vec<Var<T, B>> {
312 let mut p = self.norm1.parameters();
313 p.extend(self.self_attn.parameters());
314 p.extend(self.norm2.parameters());
315 p.extend(self.cross_attn.parameters());
316 p.extend(self.norm3.parameters());
317 p.extend(self.ffn.parameters());
318 p
319 }
320
321 pub fn named_parameters(&self) -> Vec<coeus_autograd::Parameter<T, B>> {
323 let mut parameters = prefixed_parameters("norm1", &self.norm1);
324 parameters.extend(prefixed_parameters("self_attention", &self.self_attn));
325 parameters.extend(prefixed_parameters("norm2", &self.norm2));
326 parameters.extend(prefixed_parameters("cross_attention", &self.cross_attn));
327 parameters.extend(prefixed_parameters("norm3", &self.norm3));
328 parameters.extend(prefixed_parameters("feed_forward", &self.ffn));
329 parameters
330 }
331}
332
333impl<
335 T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
336 B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
337 const H: usize,
338 SelfM: AttentionMask,
339 CrossM: AttentionMask,
340 > Clone for TransformerDecoderLayer<T, B, H, SelfM, CrossM>
341where
342 LayerNorm<T, B>: Clone,
343 MultiHeadAttention<T, B, H, SelfM>: Clone,
344 MultiHeadAttention<T, B, H, CrossM>: Clone,
345 FeedForward<T, B>: Clone,
346{
347 fn clone(&self) -> Self {
348 Self {
349 norm1: self.norm1.clone(),
350 self_attn: self.self_attn.clone(),
351 dropout1: self.dropout1.clone(),
352 norm2: self.norm2.clone(),
353 cross_attn: self.cross_attn.clone(),
354 dropout2: self.dropout2.clone(),
355 norm3: self.norm3.clone(),
356 ffn: self.ffn.clone(),
357 dropout3: self.dropout3.clone(),
358 }
359 }
360}