use super::ffn::{feed_forward_with_training, FeedForward};
use crate::attention::{multi_head_attention_cross, MhaProjectionParams, MultiHeadAttention};
use crate::dropout::Dropout;
use crate::module::{prefixed_parameters, Module, ModuleError};
use crate::normalization::LayerNorm;
use coeus_autograd::{AttentionMask, CausalMask, NullMask, Var};
use coeus_core::MoiraiBackend;
pub struct TransformerDecoderLayerParams<
'a,
T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
> {
pub norm1_weight: &'a Var<T, B>,
pub norm1_bias: &'a Var<T, B>,
pub self_attn: MhaProjectionParams<'a, T, B>,
pub norm2_weight: &'a Var<T, B>,
pub norm2_bias: &'a Var<T, B>,
pub cross_attn: MhaProjectionParams<'a, T, B>,
pub norm3_weight: &'a Var<T, B>,
pub norm3_bias: &'a Var<T, B>,
pub ffn_w1: &'a Var<T, B>,
pub ffn_b1: Option<&'a Var<T, B>>,
pub ffn_w2: &'a Var<T, B>,
pub ffn_b2: Option<&'a Var<T, B>>,
pub self_attn_residual_dropout_p: f64,
pub self_attn_residual_training: bool,
pub cross_attn_residual_dropout_p: f64,
pub cross_attn_residual_training: bool,
pub ffn_hidden_dropout_p: f64,
pub ffn_hidden_training: bool,
pub ffn_residual_dropout_p: f64,
pub ffn_residual_training: bool,
}
fn apply_dropout<
T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
B: coeus_ops::BackendOps<T> + Default,
>(
x: &Var<T, B>,
p: f64,
is_training: bool,
) -> Result<Var<T, B>, ModuleError<B::Error>> {
let mut dropout = Dropout::new(p);
dropout.set_training(is_training);
dropout.forward(x)
}
pub fn transformer_decoder_layer<
T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
const H: usize,
SelfM: AttentionMask,
CrossM: AttentionMask,
>(
tgt: &Var<T, B>,
memory: &Var<T, B>,
params: TransformerDecoderLayerParams<'_, T, B>,
) -> Result<Var<T, B>, ModuleError<B::Error>> {
const MODULE: &str = "TransformerDecoderLayer";
let ([_, _, d_model], _) = super::validation::decoder_inputs(MODULE, tgt, memory)?;
for (parameter, value) in [
("norm1 weight", params.norm1_weight),
("norm1 bias", params.norm1_bias),
("norm2 weight", params.norm2_weight),
("norm2 bias", params.norm2_bias),
("norm3 weight", params.norm3_weight),
("norm3 bias", params.norm3_bias),
] {
super::validation::affine_vector(MODULE, parameter, value, d_model)?;
}
super::validation::feed_forward(
MODULE,
d_model,
params.ffn_w1,
params.ffn_b1,
params.ffn_w2,
params.ffn_b2,
)?;
let normed1 = super::normalization::layer_norm_three_dimensional(
MODULE,
tgt,
params.norm1_weight,
params.norm1_bias,
1e-5,
)?;
let self_attn_out = multi_head_attention_cross::<T, B, H, SelfM>(
&normed1,
&normed1,
&normed1,
params.self_attn,
None,
)?;
let dropped1 = apply_dropout(
&self_attn_out,
params.self_attn_residual_dropout_p,
params.self_attn_residual_training,
)?;
let x = coeus_autograd::add(tgt, &dropped1)?;
let normed2 = super::normalization::layer_norm_three_dimensional(
MODULE,
&x,
params.norm2_weight,
params.norm2_bias,
1e-5,
)?;
let cross_attn_out = multi_head_attention_cross::<T, B, H, CrossM>(
&normed2,
memory,
memory,
params.cross_attn,
None,
)?;
let dropped2 = apply_dropout(
&cross_attn_out,
params.cross_attn_residual_dropout_p,
params.cross_attn_residual_training,
)?;
let x = coeus_autograd::add(&x, &dropped2)?;
let normed3 = super::normalization::layer_norm_three_dimensional(
MODULE,
&x,
params.norm3_weight,
params.norm3_bias,
1e-5,
)?;
let ff_out = feed_forward_with_training(
&normed3,
params.ffn_w1,
params.ffn_b1,
params.ffn_w2,
params.ffn_b2,
params.ffn_hidden_dropout_p,
params.ffn_hidden_training,
)?;
let dropped3 = apply_dropout(
&ff_out,
params.ffn_residual_dropout_p,
params.ffn_residual_training,
)?;
Ok(coeus_autograd::add(&x, &dropped3)?)
}
pub struct TransformerDecoderLayer<
T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default = MoiraiBackend,
const H: usize = 8,
SelfM: AttentionMask = CausalMask,
CrossM: AttentionMask = NullMask,
> {
pub norm1: LayerNorm<T, B>,
pub self_attn: MultiHeadAttention<T, B, H, SelfM>,
pub dropout1: Dropout,
pub norm2: LayerNorm<T, B>,
pub cross_attn: MultiHeadAttention<T, B, H, CrossM>,
pub dropout2: Dropout,
pub norm3: LayerNorm<T, B>,
pub ffn: FeedForward<T, B>,
pub dropout3: Dropout,
}
impl<
T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
const H: usize,
SelfM: AttentionMask,
CrossM: AttentionMask,
> TransformerDecoderLayer<T, B, H, SelfM, CrossM>
{
pub fn new(
d_model: usize,
d_ff: usize,
dropout_p: f64,
) -> Result<Self, crate::init::InitializationError<B::Error>>
where
T: coeus_cpu::RealScalar,
B: coeus_ops::RandomInitOps<T>,
{
Ok(Self {
norm1: LayerNorm::new(d_model, 1e-5)?,
self_attn: MultiHeadAttention::new(d_model, true)?,
dropout1: Dropout::new(dropout_p),
norm2: LayerNorm::new(d_model, 1e-5)?,
cross_attn: MultiHeadAttention::new(d_model, true)?,
dropout2: Dropout::new(dropout_p),
norm3: LayerNorm::new(d_model, 1e-5)?,
ffn: FeedForward::new(d_model, d_ff, dropout_p)?,
dropout3: Dropout::new(dropout_p),
})
}
pub fn forward_decoder(
&self,
tgt: &Var<T, B>,
memory: &Var<T, B>,
) -> Result<Var<T, B>, ModuleError<B::Error>> {
transformer_decoder_layer::<T, B, H, SelfM, CrossM>(
tgt,
memory,
TransformerDecoderLayerParams {
norm1_weight: &self.norm1.weight,
norm1_bias: &self.norm1.bias,
self_attn: MhaProjectionParams {
w_q: &self.self_attn.w_q,
b_q: self.self_attn.b_q.as_ref(),
w_k: &self.self_attn.w_k,
b_k: self.self_attn.b_k.as_ref(),
w_v: &self.self_attn.w_v,
b_v: self.self_attn.b_v.as_ref(),
w_o: &self.self_attn.w_o,
b_o: self.self_attn.b_o.as_ref(),
},
norm2_weight: &self.norm2.weight,
norm2_bias: &self.norm2.bias,
cross_attn: MhaProjectionParams {
w_q: &self.cross_attn.w_q,
b_q: self.cross_attn.b_q.as_ref(),
w_k: &self.cross_attn.w_k,
b_k: self.cross_attn.b_k.as_ref(),
w_v: &self.cross_attn.w_v,
b_v: self.cross_attn.b_v.as_ref(),
w_o: &self.cross_attn.w_o,
b_o: self.cross_attn.b_o.as_ref(),
},
norm3_weight: &self.norm3.weight,
norm3_bias: &self.norm3.bias,
ffn_w1: &self.ffn.linear1.weight,
ffn_b1: self.ffn.linear1.bias.as_ref(),
ffn_w2: &self.ffn.linear2.weight,
ffn_b2: self.ffn.linear2.bias.as_ref(),
self_attn_residual_dropout_p: self.dropout1.p,
self_attn_residual_training: self.dropout1.is_training,
cross_attn_residual_dropout_p: self.dropout2.p,
cross_attn_residual_training: self.dropout2.is_training,
ffn_hidden_dropout_p: self.ffn.dropout.p,
ffn_hidden_training: self.ffn.dropout.is_training,
ffn_residual_dropout_p: self.dropout3.p,
ffn_residual_training: self.dropout3.is_training,
},
)
}
}
impl<
T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
const H: usize,
SelfM: AttentionMask,
CrossM: AttentionMask,
> TransformerDecoderLayer<T, B, H, SelfM, CrossM>
{
pub fn parameters(&self) -> Vec<Var<T, B>> {
let mut p = self.norm1.parameters();
p.extend(self.self_attn.parameters());
p.extend(self.norm2.parameters());
p.extend(self.cross_attn.parameters());
p.extend(self.norm3.parameters());
p.extend(self.ffn.parameters());
p
}
pub fn named_parameters(&self) -> Vec<coeus_autograd::Parameter<T, B>> {
let mut parameters = prefixed_parameters("norm1", &self.norm1);
parameters.extend(prefixed_parameters("self_attention", &self.self_attn));
parameters.extend(prefixed_parameters("norm2", &self.norm2));
parameters.extend(prefixed_parameters("cross_attention", &self.cross_attn));
parameters.extend(prefixed_parameters("norm3", &self.norm3));
parameters.extend(prefixed_parameters("feed_forward", &self.ffn));
parameters
}
}
impl<
T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
const H: usize,
SelfM: AttentionMask,
CrossM: AttentionMask,
> Clone for TransformerDecoderLayer<T, B, H, SelfM, CrossM>
where
LayerNorm<T, B>: Clone,
MultiHeadAttention<T, B, H, SelfM>: Clone,
MultiHeadAttention<T, B, H, CrossM>: Clone,
FeedForward<T, B>: Clone,
{
fn clone(&self) -> Self {
Self {
norm1: self.norm1.clone(),
self_attn: self.self_attn.clone(),
dropout1: self.dropout1.clone(),
norm2: self.norm2.clone(),
cross_attn: self.cross_attn.clone(),
dropout2: self.dropout2.clone(),
norm3: self.norm3.clone(),
ffn: self.ffn.clone(),
dropout3: self.dropout3.clone(),
}
}
}