Struct BertConfig

pub struct BertConfig {
    pub vocab_size: usize,
    pub max_position_embeddings: usize,
    pub hidden_size: usize,
    pub num_hidden_layers: usize,
    pub num_attention_heads: usize,
    pub intermediate_size: usize,
    pub hidden_act: String,
    pub hidden_dropout_prob: f64,
    pub attention_probs_dropout_prob: f64,
    pub type_vocab_size: usize,
    pub layer_norm_eps: f64,
    pub initializer_range: f64,
}

Configuration for a BERT model

Fields

vocab_size: usize

Vocabulary size

max_position_embeddings: usize

Maximum sequence length

hidden_size: usize

Hidden size

num_hidden_layers: usize

Number of hidden layers

num_attention_heads: usize

Number of attention heads

intermediate_size: usize

Intermediate size in feed-forward networks

hidden_act: String

Hidden activation function

hidden_dropout_prob: f64

Hidden dropout probability

attention_probs_dropout_prob: f64

Attention dropout probability

type_vocab_size: usize

Type vocabulary size (usually 2 for sentence pair tasks)

layer_norm_eps: f64

Layer norm epsilon

initializer_range: f64

Initializer range

Implementations

impl BertConfig

pub fn bert_base_uncased() -> Self

Create a BERT-Base configuration

pub fn bert_large_uncased() -> Self

Create a BERT-Large configuration

pub fn custom( vocab_size: usize, hidden_size: usize, num_hidden_layers: usize, num_attention_heads: usize, ) -> Self

Create a custom BERT configuration

Examples found in repository

examples/bert_example.rs (lines 11-16)

fn main() -> Result<(), Box<dyn std::error::Error>> {
    println!("BERT Model Example");

    // Create a small BERT model for demonstration
    println!("Creating a small BERT model...");

    let config = BertConfig::custom(
        10000, // vocab_size
        128,   // hidden_size
        2,     // num_hidden_layers
        2,     // num_attention_heads
    );

    let model = BertModel::<f32>::new(config)?;

    // Create dummy input (batch_size=2, seq_len=16)
    // Input tensor contains token IDs
    let input = Array::from_shape_fn(
        IxDyn(&[2, 16]),
        |_| rand::random::<f32>() * 100.0, // Random token IDs between 0 and 100
    );

    println!("Input shape: {:?}", input.shape());

    // Get sequence output (hidden states)
    let sequence_output = model.forward(&input)?;

    println!("Sequence output shape: {:?}", sequence_output.shape());

    // Get pooled output (for classification tasks)
    let pooled_output = model.get_pooled_output(&input)?;

    println!("Pooled output shape: {:?}", pooled_output.shape());

    // Let's create a BERT-Base model
    println!("\nCreating a BERT-Base model...");

    let bert_base = BertModel::<f32>::bert_base_uncased()?;

    // Create dummy input for a longer sequence
    let base_input = Array::from_shape_fn(
        IxDyn(&[1, 64]),
        |_| rand::random::<f32>() * 1000.0, // Random token IDs
    );

    println!("BERT-Base input shape: {:?}", base_input.shape());

    // Forward pass to get pooled output
    let base_pooled_output = bert_base.get_pooled_output(&base_input)?;

    println!(
        "BERT-Base pooled output shape: {:?}",
        base_pooled_output.shape()
    );
    println!(
        "BERT-Base hidden dimension: {}",
        base_pooled_output.shape()[1]
    );

    println!("\nBERT example completed successfully!");

    Ok(())
}

Trait Implementations

impl Clone for BertConfig

fn clone(&self) -> BertConfig

Returns a duplicate of the value.

const fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for BertConfig

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.