Struct rust_tokenizers::vocab::BaseVocab

pub struct BaseVocab {
    pub values: HashMap<String, i64>,
    pub indices: HashMap<i64, String>,
    pub special_token_map: SpecialTokenMap,
    pub special_values: HashMap<String, i64>,
    pub special_indices: HashMap<i64, String>,
}

BaseVocab

Base vocabulary with [UNK] unknown token used as a pre-tokenization step for BERT-class tokenizers. Expects a flat text vocabulary when created from file.

Fields

values: HashMap<String, i64>

A mapping of tokens as string to indices (i.e. the encoder base)

indices: HashMap<i64, String>

A mapping of token ids to strings (i.e. the decoder base)

special_token_map: SpecialTokenMap

Special tokens used by the vocabulary

special_values: HashMap<String, i64>

A mapping of special value tokens as strings to IDs (i.e. the encoder base for special values), special values typically include things like BOS/EOS markers, class markers, mask markers and padding markers

special_indices: HashMap<i64, String>

A mapping of special value tokens as IDs to strings (i.e. the decoder base for special values)

Trait Implementations

impl Clone for BaseVocab

fn clone(&self) -> BaseVocab

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for BaseVocab

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Vocab for BaseVocab

fn get_unknown_value(&self) -> &str

Returns the unknown value on an instance

fn values(&self) -> &HashMap<String, i64>

Return the map of token strings to IDs

fn indices(&self) -> &HashMap<i64, String>

Return the map of token IDs to strings

fn special_values(&self) -> &HashMap<String, i64>

Return the map of token strings to IDs

fn special_indices(&self) -> &HashMap<i64, String>

Return the map of token IDs to strings for special values

fn values_mut(&mut self) -> &mut HashMap<String, i64>

Return a mutable reference to the map of token strings to IDs

fn indices_mut(&mut self) -> &mut HashMap<i64, String>

Return a mutable reference to the map of token IDs to strings

fn special_values_mut(&mut self) -> &mut HashMap<String, i64>

Return a mutable reference to the map of token strings to IDs

fn special_indices_mut(&mut self) -> &mut HashMap<i64, String>

Return a mutable reference to the map of token IDs to strings for special values

fn from_file<P: AsRef<Path>>(path: P) -> Result<BaseVocab, TokenizerError>

Read a vocabulary from file

fn from_file_with_special_token_mapping<P: AsRef<Path>, S: AsRef<Path>>( path: P, special_token_mapping_path: S ) -> Result<Self, TokenizerError>

Read a vocabulary from file with special token mapping

fn from_values_and_special_token_map( values: HashMap<String, i64>, special_token_map: SpecialTokenMap ) -> Result<Self, TokenizerError>where Self: Sized,

fn token_to_id(&self, token: &str) -> i64

Converts a token to an id.

fn id_to_token(&self, id: &i64) -> String

Converts an id to a token.

fn _token_to_id( &self, token: &str, values: &HashMap<String, i64>, special_values: &HashMap<String, i64>, unknown_value: &str ) -> i64

Converts a token to an id, provided a HashMap of values, a HashMap of special values and the unknown value token string representation. This is not meant to be directly used, the method token_to_id offers a more convenient interface for most vocabularies, but needs to be implemented by the specific vocabulary.

fn _id_to_token( &self, id: &i64, indices: &HashMap<i64, String>, special_indices: &HashMap<i64, String>, unknown_value: &str ) -> String

Converts an id to a token, provided a HashMap of values, a HashMap of special values and the unknown value token string representation. This is not meant to be directly used, the method id_to_token offers a more convenient interface for most vocabularies, but needs to be implemented by the specific vocabulary.

fn convert_tokens_to_ids(&self, tokens: &[&str]) -> Vec<i64>

Converts a list of tokens to a list of indices.

fn add_extra_ids(&mut self, num_extra_ids: i64)

Add extra token ids to the vocab

fn add_tokens(&mut self, tokens: &[&str])

Add arbitrary tokens to the vocabulary.