Struct tokenizers::models::bpe::BPE

pub struct BPE { /* fields omitted */ }

A Byte Pair Encoding model.

Methods

impl BPE

pub fn builder() -> BpeBuilder

Initialize a BpeBuilder.

pub fn new(
    vocab: HashMap<String, u32>,
    merges: HashMap<(u32, u32), (u32, u32)>
) -> Self

Create a new BPE model with the given vocab and merges.

pub fn from_files(vocab: &str, merges: &str) -> BpeBuilder

Initialize a BpeBuilder model from vocab and merges files

pub fn read_files(
    vocab: &str,
    merges: &str
) -> Result<(HashMap<String, u32>, HashMap<(u32, u32), (u32, u32)>)>

Read the given files to extract the vocab and merges

pub fn clear_cache(&self)

Reset the cache.

pub fn get_vocab(&self) -> &HashMap<String, u32>

pub fn get_unk_token(&self) -> &Option<String>

pub fn get_continuing_subword_prefix(&self) -> &Option<String>

Trait Implementations

impl Clone for BPE

fn clone(&self) -> Self

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Default for BPE

fn default() -> Self

Returns the "default value" for a type.

impl Model for BPE

fn get_vocab(&self) -> &HashMap<String, u32>

fn get_vocab_size(&self) -> usize

fn tokenize(&self, sentence: Vec<(String, Offsets)>) -> Result<Vec<Token>>

fn token_to_id(&self, token: &str) -> Option<u32>

fn id_to_token(&self, id: u32) -> Option<String>

fn save(&self, folder: &Path, name: Option<&str>) -> Result<Vec<PathBuf>>