1use super::ffn::{feed_forward_with_training, FeedForward};
4use crate::attention::{
5 multi_head_attention_cross, MhaProjectionParams, MultiHeadAttention, NullMask,
6};
7use crate::dropout::Dropout;
8use crate::module::{prefixed_parameters, Module, ModuleError};
9use crate::normalization::LayerNorm;
10use coeus_autograd::{AttentionMask, Var};
11use coeus_core::MoiraiBackend;
12
13pub struct TransformerEncoderLayerParams<
15 'a,
16 T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
17 B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
18> {
19 pub norm1_weight: &'a Var<T, B>,
21 pub norm1_bias: &'a Var<T, B>,
23 pub self_attn: MhaProjectionParams<'a, T, B>,
25 pub norm2_weight: &'a Var<T, B>,
27 pub norm2_bias: &'a Var<T, B>,
29 pub ffn_w1: &'a Var<T, B>,
31 pub ffn_b1: Option<&'a Var<T, B>>,
33 pub ffn_w2: &'a Var<T, B>,
35 pub ffn_b2: Option<&'a Var<T, B>>,
37 pub attn_residual_dropout_p: f64,
39 pub attn_residual_training: bool,
41 pub ffn_hidden_dropout_p: f64,
43 pub ffn_hidden_training: bool,
45 pub ffn_residual_dropout_p: f64,
47 pub ffn_residual_training: bool,
49}
50
51fn apply_dropout<
52 T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
53 B: coeus_ops::BackendOps<T> + Default,
54>(
55 x: &Var<T, B>,
56 p: f64,
57 is_training: bool,
58) -> Result<Var<T, B>, ModuleError<B::Error>> {
59 let mut dropout = Dropout::new(p);
60 dropout.set_training(is_training);
61 dropout.forward(x)
62}
63
64pub fn transformer_encoder_layer<
70 T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
71 B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
72 const H: usize,
73 M: AttentionMask,
74>(
75 input: &Var<T, B>,
76 key_padding_mask: Option<&Var<T, B>>,
77 params: TransformerEncoderLayerParams<'_, T, B>,
78) -> Result<Var<T, B>, ModuleError<B::Error>> {
79 const MODULE: &str = "TransformerEncoderLayer";
80 let [_, _, d_model] = super::validation::rank_three(MODULE, input)?;
81 super::validation::affine_vector(MODULE, "norm1 weight", params.norm1_weight, d_model)?;
82 super::validation::affine_vector(MODULE, "norm1 bias", params.norm1_bias, d_model)?;
83 super::validation::affine_vector(MODULE, "norm2 weight", params.norm2_weight, d_model)?;
84 super::validation::affine_vector(MODULE, "norm2 bias", params.norm2_bias, d_model)?;
85 super::validation::feed_forward(
86 MODULE,
87 d_model,
88 params.ffn_w1,
89 params.ffn_b1,
90 params.ffn_w2,
91 params.ffn_b2,
92 )?;
93 let normed1 = super::normalization::layer_norm_three_dimensional(
94 MODULE,
95 input,
96 params.norm1_weight,
97 params.norm1_bias,
98 1e-5,
99 )?;
100 let attn_out = multi_head_attention_cross::<T, B, H, M>(
101 &normed1,
102 &normed1,
103 &normed1,
104 params.self_attn,
105 key_padding_mask,
106 )?;
107 let dropped1 = apply_dropout(
108 &attn_out,
109 params.attn_residual_dropout_p,
110 params.attn_residual_training,
111 )?;
112 let x = coeus_autograd::add(input, &dropped1)?;
113
114 let normed2 = super::normalization::layer_norm_three_dimensional(
115 MODULE,
116 &x,
117 params.norm2_weight,
118 params.norm2_bias,
119 1e-5,
120 )?;
121 let ff_out = feed_forward_with_training(
122 &normed2,
123 params.ffn_w1,
124 params.ffn_b1,
125 params.ffn_w2,
126 params.ffn_b2,
127 params.ffn_hidden_dropout_p,
128 params.ffn_hidden_training,
129 )?;
130 let dropped2 = apply_dropout(
131 &ff_out,
132 params.ffn_residual_dropout_p,
133 params.ffn_residual_training,
134 )?;
135 Ok(coeus_autograd::add(&x, &dropped2)?)
136}
137
138pub struct TransformerEncoderLayer<
146 T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
147 B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default = MoiraiBackend,
148 const H: usize = 8,
149 M: AttentionMask = NullMask,
150> {
151 pub norm1: LayerNorm<T, B>,
153 pub self_attn: MultiHeadAttention<T, B, H, M>,
155 pub dropout1: Dropout,
157 pub norm2: LayerNorm<T, B>,
159 pub ffn: FeedForward<T, B>,
161 pub dropout2: Dropout,
163}
164
165impl<
166 T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
167 B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
168 const H: usize,
169 M: AttentionMask,
170 > TransformerEncoderLayer<T, B, H, M>
171{
172 pub fn new(
179 d_model: usize,
180 d_ff: usize,
181 dropout_p: f64,
182 ) -> Result<Self, crate::init::InitializationError<B::Error>>
183 where
184 T: coeus_cpu::RealScalar,
185 B: coeus_ops::RandomInitOps<T>,
186 {
187 Ok(Self {
188 norm1: LayerNorm::new(d_model, 1e-5)?,
189 self_attn: MultiHeadAttention::new(d_model, true)?,
190 dropout1: Dropout::new(dropout_p),
191 norm2: LayerNorm::new(d_model, 1e-5)?,
192 ffn: FeedForward::new(d_model, d_ff, dropout_p)?,
193 dropout2: Dropout::new(dropout_p),
194 })
195 }
196
197 pub fn forward_with_mask(
202 &self,
203 input: &Var<T, B>,
204 key_padding_mask: Option<&Var<T, B>>,
205 ) -> Result<Var<T, B>, ModuleError<B::Error>> {
206 transformer_encoder_layer::<T, B, H, M>(
207 input,
208 key_padding_mask,
209 TransformerEncoderLayerParams {
210 norm1_weight: &self.norm1.weight,
211 norm1_bias: &self.norm1.bias,
212 self_attn: MhaProjectionParams {
213 w_q: &self.self_attn.w_q,
214 b_q: self.self_attn.b_q.as_ref(),
215 w_k: &self.self_attn.w_k,
216 b_k: self.self_attn.b_k.as_ref(),
217 w_v: &self.self_attn.w_v,
218 b_v: self.self_attn.b_v.as_ref(),
219 w_o: &self.self_attn.w_o,
220 b_o: self.self_attn.b_o.as_ref(),
221 },
222 norm2_weight: &self.norm2.weight,
223 norm2_bias: &self.norm2.bias,
224 ffn_w1: &self.ffn.linear1.weight,
225 ffn_b1: self.ffn.linear1.bias.as_ref(),
226 ffn_w2: &self.ffn.linear2.weight,
227 ffn_b2: self.ffn.linear2.bias.as_ref(),
228 attn_residual_dropout_p: self.dropout1.p,
229 attn_residual_training: self.dropout1.is_training,
230 ffn_hidden_dropout_p: self.ffn.dropout.p,
231 ffn_hidden_training: self.ffn.dropout.is_training,
232 ffn_residual_dropout_p: self.dropout2.p,
233 ffn_residual_training: self.dropout2.is_training,
234 },
235 )
236 }
237}
238
239impl<
240 T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
241 B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
242 const H: usize,
243 M: AttentionMask,
244 > Module<T, B> for TransformerEncoderLayer<T, B, H, M>
245{
246 fn parameters(&self) -> Vec<Var<T, B>> {
247 let mut p = self.norm1.parameters();
248 p.extend(self.self_attn.parameters());
249 p.extend(self.norm2.parameters());
250 p.extend(self.ffn.parameters());
251 p
252 }
253
254 fn named_parameters(&self) -> Vec<coeus_autograd::Parameter<T, B>> {
255 let mut parameters = prefixed_parameters("norm1", &self.norm1);
256 parameters.extend(prefixed_parameters("self_attention", &self.self_attn));
257 parameters.extend(prefixed_parameters("norm2", &self.norm2));
258 parameters.extend(prefixed_parameters("feed_forward", &self.ffn));
259 parameters
260 }
261
262 fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
264 self.forward_with_mask(input, None)
265 }
266}
267
268impl<
270 T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
271 B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
272 const H: usize,
273 M: AttentionMask,
274 > Clone for TransformerEncoderLayer<T, B, H, M>
275where
276 LayerNorm<T, B>: Clone,
277 MultiHeadAttention<T, B, H, M>: Clone,
278 FeedForward<T, B>: Clone,
279{
280 fn clone(&self) -> Self {
281 Self {
282 norm1: self.norm1.clone(),
283 self_attn: self.self_attn.clone(),
284 dropout1: self.dropout1.clone(),
285 norm2: self.norm2.clone(),
286 ffn: self.ffn.clone(),
287 dropout2: self.dropout2.clone(),
288 }
289 }
290}