use super::decoder::TransformerDecoder;
use super::encoder::TransformerEncoder;
use crate::module::{prefixed_parameters, Module, ModuleError};
use coeus_autograd::{AttentionMask, CausalMask, NullMask, Var};
use coeus_core::MoiraiBackend;
pub struct Transformer<
T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default = MoiraiBackend,
const H: usize = 8,
const NUM_ENC: usize = 6,
const NUM_DEC: usize = 6,
EncM: AttentionMask = NullMask,
DecSelfM: AttentionMask = CausalMask,
DecCrossM: AttentionMask = NullMask,
> {
pub encoder: TransformerEncoder<T, B, H, NUM_ENC, EncM>,
pub decoder: TransformerDecoder<T, B, H, NUM_DEC, DecSelfM, DecCrossM>,
}
impl<
T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
const H: usize,
const NUM_ENC: usize,
const NUM_DEC: usize,
EncM: AttentionMask,
DecSelfM: AttentionMask,
DecCrossM: AttentionMask,
> Transformer<T, B, H, NUM_ENC, NUM_DEC, EncM, DecSelfM, DecCrossM>
where
TransformerEncoder<T, B, H, NUM_ENC, EncM>: Clone,
TransformerDecoder<T, B, H, NUM_DEC, DecSelfM, DecCrossM>: Clone,
{
pub fn new(
d_model: usize,
d_ff: usize,
dropout_p: f64,
) -> Result<Self, crate::init::InitializationError<B::Error>>
where
T: coeus_cpu::RealScalar,
B: coeus_ops::RandomInitOps<T>,
{
Ok(Self {
encoder: TransformerEncoder::new(d_model, d_ff, dropout_p)?,
decoder: TransformerDecoder::new(d_model, d_ff, dropout_p)?,
})
}
pub fn forward_seq2seq(
&self,
src: &Var<T, B>,
tgt: &Var<T, B>,
) -> Result<Var<T, B>, ModuleError<B::Error>> {
let memory = self.encoder.forward_with_mask(src, None)?;
self.decoder.forward_decoder(tgt, &memory)
}
pub fn forward_seq2seq_with_src_mask(
&self,
src: &Var<T, B>,
tgt: &Var<T, B>,
src_key_padding_mask: Option<&Var<T, B>>,
) -> Result<Var<T, B>, ModuleError<B::Error>> {
let memory = self.encoder.forward_with_mask(src, src_key_padding_mask)?;
self.decoder.forward_decoder(tgt, &memory)
}
}
impl<
T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
const H: usize,
const NUM_ENC: usize,
const NUM_DEC: usize,
EncM: AttentionMask,
DecSelfM: AttentionMask,
DecCrossM: AttentionMask,
> Transformer<T, B, H, NUM_ENC, NUM_DEC, EncM, DecSelfM, DecCrossM>
{
pub fn parameters(&self) -> Vec<Var<T, B>> {
let mut p = self.encoder.parameters();
p.extend(self.decoder.parameters());
p
}
pub fn named_parameters(&self) -> Vec<coeus_autograd::Parameter<T, B>> {
let mut parameters = prefixed_parameters("encoder", &self.encoder);
parameters.extend(
self.decoder
.named_parameters()
.into_iter()
.map(|parameter| parameter.with_prefix("decoder")),
);
parameters
}
}
impl<
T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
const H: usize,
const NUM_ENC: usize,
const NUM_DEC: usize,
EncM: AttentionMask,
DecSelfM: AttentionMask,
DecCrossM: AttentionMask,
> Clone for Transformer<T, B, H, NUM_ENC, NUM_DEC, EncM, DecSelfM, DecCrossM>
where
TransformerEncoder<T, B, H, NUM_ENC, EncM>: Clone,
TransformerDecoder<T, B, H, NUM_DEC, DecSelfM, DecCrossM>: Clone,
{
fn clone(&self) -> Self {
Self {
encoder: self.encoder.clone(),
decoder: self.decoder.clone(),
}
}
}