use super::ffn::{feed_forward_with_training, FeedForward};
use crate::attention::{
multi_head_attention_cross, MhaProjectionParams, MultiHeadAttention, NullMask,
};
use crate::dropout::Dropout;
use crate::module::{prefixed_parameters, Module, ModuleError};
use crate::normalization::LayerNorm;
use coeus_autograd::{AttentionMask, Var};
use coeus_core::MoiraiBackend;
pub struct TransformerEncoderLayerParams<
'a,
T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
> {
pub norm1_weight: &'a Var<T, B>,
pub norm1_bias: &'a Var<T, B>,
pub self_attn: MhaProjectionParams<'a, T, B>,
pub norm2_weight: &'a Var<T, B>,
pub norm2_bias: &'a Var<T, B>,
pub ffn_w1: &'a Var<T, B>,
pub ffn_b1: Option<&'a Var<T, B>>,
pub ffn_w2: &'a Var<T, B>,
pub ffn_b2: Option<&'a Var<T, B>>,
pub attn_residual_dropout_p: f64,
pub attn_residual_training: bool,
pub ffn_hidden_dropout_p: f64,
pub ffn_hidden_training: bool,
pub ffn_residual_dropout_p: f64,
pub ffn_residual_training: bool,
}
fn apply_dropout<
T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
B: coeus_ops::BackendOps<T> + Default,
>(
x: &Var<T, B>,
p: f64,
is_training: bool,
) -> Result<Var<T, B>, ModuleError<B::Error>> {
let mut dropout = Dropout::new(p);
dropout.set_training(is_training);
dropout.forward(x)
}
pub fn transformer_encoder_layer<
T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
const H: usize,
M: AttentionMask,
>(
input: &Var<T, B>,
key_padding_mask: Option<&Var<T, B>>,
params: TransformerEncoderLayerParams<'_, T, B>,
) -> Result<Var<T, B>, ModuleError<B::Error>> {
const MODULE: &str = "TransformerEncoderLayer";
let [_, _, d_model] = super::validation::rank_three(MODULE, input)?;
super::validation::affine_vector(MODULE, "norm1 weight", params.norm1_weight, d_model)?;
super::validation::affine_vector(MODULE, "norm1 bias", params.norm1_bias, d_model)?;
super::validation::affine_vector(MODULE, "norm2 weight", params.norm2_weight, d_model)?;
super::validation::affine_vector(MODULE, "norm2 bias", params.norm2_bias, d_model)?;
super::validation::feed_forward(
MODULE,
d_model,
params.ffn_w1,
params.ffn_b1,
params.ffn_w2,
params.ffn_b2,
)?;
let normed1 = super::normalization::layer_norm_three_dimensional(
MODULE,
input,
params.norm1_weight,
params.norm1_bias,
1e-5,
)?;
let attn_out = multi_head_attention_cross::<T, B, H, M>(
&normed1,
&normed1,
&normed1,
params.self_attn,
key_padding_mask,
)?;
let dropped1 = apply_dropout(
&attn_out,
params.attn_residual_dropout_p,
params.attn_residual_training,
)?;
let x = coeus_autograd::add(input, &dropped1)?;
let normed2 = super::normalization::layer_norm_three_dimensional(
MODULE,
&x,
params.norm2_weight,
params.norm2_bias,
1e-5,
)?;
let ff_out = feed_forward_with_training(
&normed2,
params.ffn_w1,
params.ffn_b1,
params.ffn_w2,
params.ffn_b2,
params.ffn_hidden_dropout_p,
params.ffn_hidden_training,
)?;
let dropped2 = apply_dropout(
&ff_out,
params.ffn_residual_dropout_p,
params.ffn_residual_training,
)?;
Ok(coeus_autograd::add(&x, &dropped2)?)
}
pub struct TransformerEncoderLayer<
T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default = MoiraiBackend,
const H: usize = 8,
M: AttentionMask = NullMask,
> {
pub norm1: LayerNorm<T, B>,
pub self_attn: MultiHeadAttention<T, B, H, M>,
pub dropout1: Dropout,
pub norm2: LayerNorm<T, B>,
pub ffn: FeedForward<T, B>,
pub dropout2: Dropout,
}
impl<
T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
const H: usize,
M: AttentionMask,
> TransformerEncoderLayer<T, B, H, M>
{
pub fn new(
d_model: usize,
d_ff: usize,
dropout_p: f64,
) -> Result<Self, crate::init::InitializationError<B::Error>>
where
T: coeus_cpu::RealScalar,
B: coeus_ops::RandomInitOps<T>,
{
Ok(Self {
norm1: LayerNorm::new(d_model, 1e-5)?,
self_attn: MultiHeadAttention::new(d_model, true)?,
dropout1: Dropout::new(dropout_p),
norm2: LayerNorm::new(d_model, 1e-5)?,
ffn: FeedForward::new(d_model, d_ff, dropout_p)?,
dropout2: Dropout::new(dropout_p),
})
}
pub fn forward_with_mask(
&self,
input: &Var<T, B>,
key_padding_mask: Option<&Var<T, B>>,
) -> Result<Var<T, B>, ModuleError<B::Error>> {
transformer_encoder_layer::<T, B, H, M>(
input,
key_padding_mask,
TransformerEncoderLayerParams {
norm1_weight: &self.norm1.weight,
norm1_bias: &self.norm1.bias,
self_attn: MhaProjectionParams {
w_q: &self.self_attn.w_q,
b_q: self.self_attn.b_q.as_ref(),
w_k: &self.self_attn.w_k,
b_k: self.self_attn.b_k.as_ref(),
w_v: &self.self_attn.w_v,
b_v: self.self_attn.b_v.as_ref(),
w_o: &self.self_attn.w_o,
b_o: self.self_attn.b_o.as_ref(),
},
norm2_weight: &self.norm2.weight,
norm2_bias: &self.norm2.bias,
ffn_w1: &self.ffn.linear1.weight,
ffn_b1: self.ffn.linear1.bias.as_ref(),
ffn_w2: &self.ffn.linear2.weight,
ffn_b2: self.ffn.linear2.bias.as_ref(),
attn_residual_dropout_p: self.dropout1.p,
attn_residual_training: self.dropout1.is_training,
ffn_hidden_dropout_p: self.ffn.dropout.p,
ffn_hidden_training: self.ffn.dropout.is_training,
ffn_residual_dropout_p: self.dropout2.p,
ffn_residual_training: self.dropout2.is_training,
},
)
}
}
impl<
T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
const H: usize,
M: AttentionMask,
> Module<T, B> for TransformerEncoderLayer<T, B, H, M>
{
fn parameters(&self) -> Vec<Var<T, B>> {
let mut p = self.norm1.parameters();
p.extend(self.self_attn.parameters());
p.extend(self.norm2.parameters());
p.extend(self.ffn.parameters());
p
}
fn named_parameters(&self) -> Vec<coeus_autograd::Parameter<T, B>> {
let mut parameters = prefixed_parameters("norm1", &self.norm1);
parameters.extend(prefixed_parameters("self_attention", &self.self_attn));
parameters.extend(prefixed_parameters("norm2", &self.norm2));
parameters.extend(prefixed_parameters("feed_forward", &self.ffn));
parameters
}
fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
self.forward_with_mask(input, None)
}
}
impl<
T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
const H: usize,
M: AttentionMask,
> Clone for TransformerEncoderLayer<T, B, H, M>
where
LayerNorm<T, B>: Clone,
MultiHeadAttention<T, B, H, M>: Clone,
FeedForward<T, B>: Clone,
{
fn clone(&self) -> Self {
Self {
norm1: self.norm1.clone(),
self_attn: self.self_attn.clone(),
dropout1: self.dropout1.clone(),
norm2: self.norm2.clone(),
ffn: self.ffn.clone(),
dropout2: self.dropout2.clone(),
}
}
}