Struct tokenizers::models::bpe::BpeTrainer

pub struct BpeTrainer { /* fields omitted */ }

In charge of training a BPE model from a mapping of words to word counts.

Examples

use std::collections::HashMap;
use tokenizers::tokenizer::Trainer;
use tokenizers::models::bpe::BpeTrainer;

let word_counts: HashMap<String, u32> = [
    (String::from("Hello"), 1),
    (String::from("World"), 1),
].iter().cloned().collect();
let trainer = BpeTrainer::default();
let (model, special_tokens) = trainer.train(word_counts).unwrap();

Methods

impl BpeTrainer

pub fn new(min_frequency: u32, vocab_size: usize) -> Self

pub fn builder() -> BpeTrainerBuilder

pub fn train(
    &self,
    word_counts: HashMap<String, u32>
) -> Result<(BPE, Vec<AddedToken>)>

Trait Implementations

impl Default for BpeTrainer

fn default() -> Self

Returns the "default value" for a type.

impl Trainer for BpeTrainer

fn train(
    &self,
    word_counts: HashMap<String, u32>
) -> Result<(Box<dyn Model>, Vec<AddedToken>)>

Train a BPE model

fn process_tokens(&self, words: &mut HashMap<String, u32>, tokens: Vec<String>)

Process a bunch of tokens, counting them

fn should_show_progress(&self) -> bool

Whether we should show progress