coeus_nn/transformer/
decoder.rs1use super::decoder_layer::TransformerDecoderLayer;
4use crate::module::ModuleError;
5use coeus_autograd::{AttentionMask, CausalMask, NullMask, Var};
6use coeus_core::MoiraiBackend;
7
8pub struct TransformerDecoder<
16 T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
17 B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default = MoiraiBackend,
18 const H: usize = 8,
19 const N: usize = 6,
20 SelfM: AttentionMask = CausalMask,
21 CrossM: AttentionMask = NullMask,
22> {
23 pub layers: [TransformerDecoderLayer<T, B, H, SelfM, CrossM>; N],
25}
26
27impl<
28 T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
29 B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
30 const H: usize,
31 const N: usize,
32 SelfM: AttentionMask,
33 CrossM: AttentionMask,
34 > TransformerDecoder<T, B, H, N, SelfM, CrossM>
35where
36 TransformerDecoderLayer<T, B, H, SelfM, CrossM>: Clone,
37{
38 pub fn new(
44 d_model: usize,
45 d_ff: usize,
46 dropout_p: f64,
47 ) -> Result<Self, crate::init::InitializationError<B::Error>>
48 where
49 T: coeus_cpu::RealScalar,
50 B: coeus_ops::RandomInitOps<T>,
51 {
52 if N == 0 {
53 return Err(crate::init::InitializationError::InvalidLayerCount { layers: N });
54 }
55 let layers = (0..N)
56 .map(|_| {
57 TransformerDecoderLayer::<T, B, H, SelfM, CrossM>::new(d_model, d_ff, dropout_p)
58 })
59 .collect::<Result<Vec<_>, _>>()?
60 .try_into()
61 .map_err(|_| crate::init::InitializationError::InvalidLayerCount { layers: N })?;
62 Ok(Self { layers })
63 }
64
65 pub fn forward_decoder(
69 &self,
70 tgt: &Var<T, B>,
71 memory: &Var<T, B>,
72 ) -> Result<Var<T, B>, ModuleError<B::Error>> {
73 let mut x = tgt.clone();
74 for layer in &self.layers {
75 x = layer.forward_decoder(&x, memory)?;
76 }
77 Ok(x)
78 }
79}
80
81impl<
82 T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
83 B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
84 const H: usize,
85 const N: usize,
86 SelfM: AttentionMask,
87 CrossM: AttentionMask,
88 > TransformerDecoder<T, B, H, N, SelfM, CrossM>
89{
90 pub fn parameters(&self) -> Vec<Var<T, B>> {
92 self.layers.iter().flat_map(|l| l.parameters()).collect()
93 }
94
95 pub fn named_parameters(&self) -> Vec<coeus_autograd::Parameter<T, B>> {
97 self.layers
98 .iter()
99 .enumerate()
100 .flat_map(|(index, layer)| {
101 let prefix = format!("layers.{index}");
102 layer
103 .named_parameters()
104 .into_iter()
105 .map(move |parameter| parameter.with_prefix(&prefix))
106 })
107 .collect()
108 }
109}
110
111impl<
113 T: coeus_core::FloatElement + coeus_ops::AttentionScalar + coeus_cpu::RealScalar,
114 B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
115 const H: usize,
116 const N: usize,
117 SelfM: AttentionMask,
118 CrossM: AttentionMask,
119 > Clone for TransformerDecoder<T, B, H, N, SelfM, CrossM>
120where
121 TransformerDecoderLayer<T, B, H, SelfM, CrossM>: Clone,
122{
123 fn clone(&self) -> Self {
124 Self {
125 layers: self.layers.clone(),
126 }
127 }
128}