Struct TransformerEncoderBlock 

pub struct TransformerEncoderBlock {
    pub attention: MultiHeadSelfAttention,
    pub ln1: LayerNorm,
    pub ffn: FeedForward,
    pub ln2: LayerNorm,
    pub dropout: Dropout,
    pub pre_norm: bool,
}

Transformer encoder block (BERT-style).

Fields

attention: MultiHeadSelfAttention

Self-attention layer

ln1: LayerNorm

First layer norm (pre-attention or post-attention)

ffn: FeedForward

Feed-forward network

ln2: LayerNorm

Second layer norm (pre-FFN or post-FFN)

dropout: Dropout

Residual dropout

pre_norm: bool

Whether to use pre-norm (like GPT) or post-norm (like original BERT)

Implementations

impl TransformerEncoderBlock

pub fn new( hidden_size: usize, num_heads: usize, intermediate_size: usize, dropout: f32, layer_norm_eps: f32, activation: &str, pre_norm: bool, ) -> Self

Creates a new transformer encoder block.

pub fn forward_with_mask( &self, hidden_states: &Variable, attention_mask: Option<&Tensor<f32>>, ) -> Variable

Forward pass with optional attention mask.

Trait Implementations

impl Debug for TransformerEncoderBlock

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Module for TransformerEncoderBlock

fn forward(&self, input: &Variable) -> Variable

Performs the forward pass.

fn parameters(&self) -> Vec<Parameter>

Returns all parameters of this module.

fn train(&mut self)

Sets the module to training mode.

fn eval(&mut self)

Sets the module to evaluation mode.

fn named_parameters(&self) -> HashMap<String, Parameter>

Returns named parameters of this module.

fn num_parameters(&self) -> usize

Returns the number of trainable parameters.

fn set_training(&mut self, _training: bool)

Sets the training mode.

fn is_training(&self) -> bool

Returns whether the module is in training mode.

fn zero_grad(&self)

Zeros all gradients of parameters.

fn name(&self) -> &'static str

Returns the module name for debugging.