Struct BertTokenizer 

pub struct BertTokenizer { /* private fields */ }

Wrapper around HuggingFace tokenizer configured for BERT-style encoding.

Implementations

impl BertTokenizer

pub fn from_dir(dir: &Path, max_length: usize) -> Result<Self>

Load tokenizer from a model directory containing:

• tokenizer.json
• config.json
• special_tokens_map.json
• tokenizer_config.json

Examples found in repository

examples/semantic_check.rs (line 26)

fn main() -> anyhow::Result<()> {
    let dir = std::env::args()
        .nth(1)
        .expect("usage: semantic_check <model_dir>");
    let dir = Path::new(&dir);
    let pooling = if dir.to_string_lossy().to_lowercase().contains("bge") {
        Pooling::Cls
    } else {
        Pooling::Mean
    };

    let tok = BertTokenizer::from_dir(dir, 64)?;
    let mut model = RlxBertModel::load(
        &dir.join("config.json"),
        dir.join("model.safetensors").to_str().unwrap(),
    )?;

    let texts = [
        "A man is playing a guitar.",                  // 0
        "Someone is strumming an acoustic guitar.",    // 1  ~ 0
        "It is freezing outside and snow is falling.", // 2
        "The weather is cold and it is snowing.",      // 3  ~ 2
    ];
    let vecs = embed_with_rlx(&mut model, &tok, &texts, pooling)?;
    println!("dim={} pooling={:?}", vecs[0].len(), pooling);

    println!("\ncosine matrix:");
    print!("      ");
    for j in 0..texts.len() {
        print!("  s{j:<5}");
    }
    println!();
    for i in 0..texts.len() {
        print!("s{i}  ");
        for j in 0..texts.len() {
            print!("  {:.3} ", cosine(&vecs[i], &vecs[j]));
        }
        println!();
    }

    let sim_related = (cosine(&vecs[0], &vecs[1]) + cosine(&vecs[2], &vecs[3])) / 2.0;
    let sim_unrelated = (cosine(&vecs[0], &vecs[2])
        + cosine(&vecs[0], &vecs[3])
        + cosine(&vecs[1], &vecs[2])
        + cosine(&vecs[1], &vecs[3]))
        / 4.0;
    println!("\nmean related  (0-1, 2-3): {sim_related:.3}");
    println!("mean unrelated (cross)  : {sim_unrelated:.3}");
    if sim_related > sim_unrelated + 0.05 {
        println!("PASS: related pairs are clearly more similar than unrelated pairs.");
    } else {
        println!("FAIL: semantic ordering not observed.");
        std::process::exit(1);
    }
    Ok(())
}

pub fn from_bytes( tokenizer_json: &[u8], config_json: &[u8], special_tokens_map_json: &[u8], tokenizer_config_json: &[u8], max_length: usize, ) -> Result<Self>

Load tokenizer from raw file bytes.

pub fn encode_batch(&self, texts: &[&str]) -> Result<TokenizedBatch>

Tokenize a batch of texts.

Returns input_ids, attention_mask, and token_type_ids for each text, all padded to the same length (longest in batch).