Struct dfdx::nn::modules::Transformer
source · pub struct Transformer<const MODEL_DIM: usize, const NUM_HEADS: usize, const NUM_ENCODER_LAYERS: usize, const NUM_DECODER_LAYERS: usize, const FF_DIM: usize, E: Dtype, D: DeviceStorage> {
pub encoder: TransformerEncoder<MODEL_DIM, NUM_HEADS, FF_DIM, NUM_ENCODER_LAYERS, E, D>,
pub decoder: TransformerDecoder<MODEL_DIM, NUM_HEADS, FF_DIM, NUM_DECODER_LAYERS, E, D>,
}
Expand description
Transformer architecture as described in Attention is all you need.
This is comprised of a TransformerEncoder and a TransformerDecoder.
Generics:
MODEL_DIM
: Size of the input features to the encoder/decoder.NUM_HEADS
: Number of heads for MultiHeadAttention.NUM_ENCODER_LAYERS
: Number of TransformerEncoderBlock to useNUM_DECODER_LAYERS
: Number of TransformerDecoderBlock to useFF_DIM
: Feedforward hidden dimension for both encoder/decoder
Pytorch equivalent:
torch.nn.Transformer(
d_model=MODEL_DIM,
nhead=NUM_HEADS,
num_encoder_layers=NUM_ENCODER_LAYERS,
num_decoder_layers=NUM_DECODER_LAYERS,
dim_feedforward=FF_DIM,
batch_first=True,
)
Fields§
§encoder: TransformerEncoder<MODEL_DIM, NUM_HEADS, FF_DIM, NUM_ENCODER_LAYERS, E, D>
§decoder: TransformerDecoder<MODEL_DIM, NUM_HEADS, FF_DIM, NUM_DECODER_LAYERS, E, D>
Trait Implementations§
source§impl<const MODEL_DIM: usize, const NUM_HEADS: usize, const NUM_ENCODER_LAYERS: usize, const NUM_DECODER_LAYERS: usize, const FF_DIM: usize, E: Clone + Dtype, D: Clone + DeviceStorage> Clone for Transformer<MODEL_DIM, NUM_HEADS, NUM_ENCODER_LAYERS, NUM_DECODER_LAYERS, FF_DIM, E, D>
impl<const MODEL_DIM: usize, const NUM_HEADS: usize, const NUM_ENCODER_LAYERS: usize, const NUM_DECODER_LAYERS: usize, const FF_DIM: usize, E: Clone + Dtype, D: Clone + DeviceStorage> Clone for Transformer<MODEL_DIM, NUM_HEADS, NUM_ENCODER_LAYERS, NUM_DECODER_LAYERS, FF_DIM, E, D>
source§fn clone(
&self
) -> Transformer<MODEL_DIM, NUM_HEADS, NUM_ENCODER_LAYERS, NUM_DECODER_LAYERS, FF_DIM, E, D>
fn clone( &self ) -> Transformer<MODEL_DIM, NUM_HEADS, NUM_ENCODER_LAYERS, NUM_DECODER_LAYERS, FF_DIM, E, D>
Returns a copy of the value. Read more
1.0.0 · source§fn clone_from(&mut self, source: &Self)
fn clone_from(&mut self, source: &Self)
Performs copy-assignment from
source
. Read moresource§impl<const MODEL_DIM: usize, const NUM_HEADS: usize, const NUM_ENCODER_LAYERS: usize, const NUM_DECODER_LAYERS: usize, const FF_DIM: usize, E: Debug + Dtype, D: Debug + DeviceStorage> Debug for Transformer<MODEL_DIM, NUM_HEADS, NUM_ENCODER_LAYERS, NUM_DECODER_LAYERS, FF_DIM, E, D>
impl<const MODEL_DIM: usize, const NUM_HEADS: usize, const NUM_ENCODER_LAYERS: usize, const NUM_DECODER_LAYERS: usize, const FF_DIM: usize, E: Debug + Dtype, D: Debug + DeviceStorage> Debug for Transformer<MODEL_DIM, NUM_HEADS, NUM_ENCODER_LAYERS, NUM_DECODER_LAYERS, FF_DIM, E, D>
source§impl<const M: usize, const H: usize, const EL: usize, const DL: usize, const F: usize, E: Dtype, D: Device<E>, Src: SplitTape, Tgt: PutTape<Src::Tape>> Module<(Src, Tgt)> for Transformer<M, H, EL, DL, F, E, D>where
TransformerEncoder<M, H, F, EL, E, D>: Module<Src, Output = Src, Error = D::Err>,
TransformerDecoder<M, H, F, DL, E, D>: Module<(<Tgt as PutTape<Src::Tape>>::Output, Src::NoTape), Output = <Tgt as PutTape<Src::Tape>>::Output, Error = D::Err>,
impl<const M: usize, const H: usize, const EL: usize, const DL: usize, const F: usize, E: Dtype, D: Device<E>, Src: SplitTape, Tgt: PutTape<Src::Tape>> Module<(Src, Tgt)> for Transformer<M, H, EL, DL, F, E, D>where TransformerEncoder<M, H, F, EL, E, D>: Module<Src, Output = Src, Error = D::Err>, TransformerDecoder<M, H, F, DL, E, D>: Module<(<Tgt as PutTape<Src::Tape>>::Output, Src::NoTape), Output = <Tgt as PutTape<Src::Tape>>::Output, Error = D::Err>,
source§impl<const M: usize, const H: usize, const A: usize, const B: usize, const F: usize, E, D> TensorCollection<E, D> for Transformer<M, H, A, B, F, E, D>where
E: Dtype + Float + SampleUniform,
D: Device<E>,
impl<const M: usize, const H: usize, const A: usize, const B: usize, const F: usize, E, D> TensorCollection<E, D> for Transformer<M, H, A, B, F, E, D>where E: Dtype + Float + SampleUniform, D: Device<E>,
§type To<E2: Dtype, D2: Device<E2>> = Transformer<M, H, A, B, F, E2, D2>
type To<E2: Dtype, D2: Device<E2>> = Transformer<M, H, A, B, F, E2, D2>
Type alias that specifies the how a module’s type changes when using a different dtype and/or
device.
source§fn iter_tensors<V: ModuleVisitor<Self, E, D>>(
visitor: &mut V
) -> Result<Option<Self::To<V::E2, V::D2>>, V::Err>
fn iter_tensors<V: ModuleVisitor<Self, E, D>>( visitor: &mut V ) -> Result<Option<Self::To<V::E2, V::D2>>, V::Err>
Specifies how to iterate through tensors or modules containted within this module, and how
to contruct this module given values for its fields. Returns
Err(_)
to indicate an error,
Ok(None)
to indicate that there is no error and a module has not been built, and
Ok(Some(_))
contains Self::Output<E2, D2>
source§fn module<F1, F2, Field>(
name: &str,
get_ref: F1,
get_mut: F2
) -> ModuleField<'_, F1, F2, Self, Field>where
F1: FnMut(&Self) -> &Field,
F2: FnMut(&mut Self) -> &mut Field,
Field: TensorCollection<E, D>,
fn module<F1, F2, Field>( name: &str, get_ref: F1, get_mut: F2 ) -> ModuleField<'_, F1, F2, Self, Field>where F1: FnMut(&Self) -> &Field, F2: FnMut(&mut Self) -> &mut Field, Field: TensorCollection<E, D>,
Creates a ModuleFields that represents a field that may contain one or more tensors. Read more
source§fn tensor<F1, F2, S>(
name: &str,
get_ref: F1,
get_mut: F2,
options: TensorOptions<S, E, D>
) -> TensorField<'_, F1, F2, Self, S, E, D>where
F1: FnMut(&Self) -> &Tensor<S, E, D>,
F2: FnMut(&mut Self) -> &mut Tensor<S, E, D>,
S: Shape,
fn tensor<F1, F2, S>( name: &str, get_ref: F1, get_mut: F2, options: TensorOptions<S, E, D> ) -> TensorField<'_, F1, F2, Self, S, E, D>where F1: FnMut(&Self) -> &Tensor<S, E, D>, F2: FnMut(&mut Self) -> &mut Tensor<S, E, D>, S: Shape,
Creates a ModuleFields that represents a tensor field. Read more