use super::decoder_layer::TransformerDecoderLayer;
use crate::module::ModuleError;
use coeus_autograd::{AttentionMask, CausalMask, NullMask, Var};
use coeus_core::MoiraiBackend;
pub struct TransformerDecoder<
T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default = MoiraiBackend,
const H: usize = 8,
const N: usize = 6,
SelfM: AttentionMask = CausalMask,
CrossM: AttentionMask = NullMask,
> {
pub layers: [TransformerDecoderLayer<T, B, H, SelfM, CrossM>; N],
}
impl<
T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
const H: usize,
const N: usize,
SelfM: AttentionMask,
CrossM: AttentionMask,
> TransformerDecoder<T, B, H, N, SelfM, CrossM>
where
TransformerDecoderLayer<T, B, H, SelfM, CrossM>: Clone,
{
pub fn new(
d_model: usize,
d_ff: usize,
dropout_p: f64,
) -> Result<Self, crate::init::InitializationError<B::Error>>
where
T: coeus_cpu::RealScalar,
B: coeus_ops::RandomInitOps<T>,
{
if N == 0 {
return Err(crate::init::InitializationError::InvalidLayerCount { layers: N });
}
let layers = (0..N)
.map(|_| {
TransformerDecoderLayer::<T, B, H, SelfM, CrossM>::new(d_model, d_ff, dropout_p)
})
.collect::<Result<Vec<_>, _>>()?
.try_into()
.map_err(|_| crate::init::InitializationError::InvalidLayerCount { layers: N })?;
Ok(Self { layers })
}
pub fn forward_decoder(
&self,
tgt: &Var<T, B>,
memory: &Var<T, B>,
) -> Result<Var<T, B>, ModuleError<B::Error>> {
let mut x = tgt.clone();
for layer in &self.layers {
x = layer.forward_decoder(&x, memory)?;
}
Ok(x)
}
}
impl<
T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
const H: usize,
const N: usize,
SelfM: AttentionMask,
CrossM: AttentionMask,
> TransformerDecoder<T, B, H, N, SelfM, CrossM>
{
pub fn parameters(&self) -> Vec<Var<T, B>> {
self.layers.iter().flat_map(|l| l.parameters()).collect()
}
pub fn named_parameters(&self) -> Vec<coeus_autograd::Parameter<T, B>> {
self.layers
.iter()
.enumerate()
.flat_map(|(index, layer)| {
let prefix = format!("layers.{index}");
layer
.named_parameters()
.into_iter()
.map(move |parameter| parameter.with_prefix(&prefix))
})
.collect()
}
}
impl<
T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
const H: usize,
const N: usize,
SelfM: AttentionMask,
CrossM: AttentionMask,
> Clone for TransformerDecoder<T, B, H, N, SelfM, CrossM>
where
TransformerDecoderLayer<T, B, H, SelfM, CrossM>: Clone,
{
fn clone(&self) -> Self {
Self {
layers: self.layers.clone(),
}
}
}