Struct rust_tokenizers::vocab::MarianVocab

pub struct MarianVocab {
    pub values: HashMap<String, i64>,
    pub indices: HashMap<i64, String>,
    pub unknown_value: &'static str,
    pub special_values: HashMap<String, i64>,
    pub special_indices: HashMap<i64, String>,
}

Marian Vocab

Vocabulary for Marian tokenizer. Contains the following special values:

• PAD token
• EOS token

Expects a JSON-format vocabulary when created from file.

Fields

values: HashMap<String, i64>

A mapping of tokens as string to indices (i.e. the encoder base)

indices: HashMap<i64, String>

A mapping of token ids to strings (i.e. the decoder base)

unknown_value: &'static str

The string to use for unknown (out of vocabulary) tokens

special_values: HashMap<String, i64>

A mapping of special value tokens as strings to IDs (i.e. the encoder base for special values), special values typically include things like BOS/EOS markers, class markers, mask markers and padding markers

special_indices: HashMap<i64, String>

A mapping of special value tokens as IDs to strings (i.e. the decoder base for special values)

Implementations

impl MarianVocab

pub fn pad_value() -> &'static str

Returns the PAD token for Marian (<pad>)

pub fn eos_value() -> &'static str

Returns the EOS token for Marian (</s>)

Trait Implementations

impl Clone for MarianVocab

fn clone(&self) -> MarianVocab

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for MarianVocab

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl MultiThreadedTokenizer<MarianVocab> for MarianTokenizer

fn vocab(&self) -> &T

returns a reference to the tokenizer vocabulary

fn tokenize_list_with_offsets<S, ST>(
    &self,
    text_list: S
) -> Vec<TokensWithOffsets> where
    S: AsRef<[ST]>,
    ST: AsRef<str> + Sync, 

Tokenize a list of strings (with multithreading), where each corresponds to for example a sentence, returns a vector of TokensWithOffsets containing the tokens and their offset information. This calls tokenize_with_offsets on the list provided.

fn tokenize_list<S, ST>(&self, text_list: S) -> Vec<Vec<String>> where
    S: AsRef<[ST]>,
    ST: AsRef<str> + Sync, 

Multithreaded tokenization of a list of strings, returning tokens with offset information

fn encode_list<S, ST>(
    &self,
    text_list: S,
    max_len: usize,
    truncation_strategy: &TruncationStrategy,
    stride: usize
) -> Vec<TokenizedInput> where
    S: AsRef<[ST]>,
    ST: AsRef<str> + Sync, 

Multithreaded encoding of a sequence of string-like texts (tokenization followed by encoding). Not that in contrast with encode optional second text, each text provided is encoded independently.

fn encode_pair_list<S, ST>(
    &self,
    text_list: S,
    max_len: usize,
    truncation_strategy: &TruncationStrategy,
    stride: usize
) -> Vec<TokenizedInput> where
    S: AsRef<[(ST, ST)]>,
    ST: AsRef<str> + Sync, 

Multithreaded ncoding of a sequence of string-like text pairs (tokenization followed by encoding). This combines with encode with the list processing of encode_list.

fn decode_list(
    &self,
    token_ids_list: Vec<Vec<i64>>,
    skip_special_tokens: bool,
    clean_up_tokenization_spaces: bool
) -> Vec<String>

Multithreaded conversion a list of sequence of ids (integer) into a string, using the tokenizer and vocabulary with options to remove special tokens and clean up tokenization spaces. This calls decode for each provided sequence of ids

impl Tokenizer<MarianVocab> for MarianTokenizer

fn vocab(&self) -> &MarianVocab

returns a reference to the tokenizer vocabulary

fn tokenize_to_tokens(&self, text: TokenRef<'_>) -> Vec<Token>

Tokenize a TokenRef, returning a sequence of tokens

fn convert_tokens_to_string(&self, tokens: Vec<String>) -> String

Converts a sequence of strings into a single string. This will clean-up artifacts from tokenization (for example sub ##word) and generate a single output string

fn build_input_with_special_tokens(
    &self,
    tokens_ids_with_offsets_1: TokenIdsWithOffsets,
    tokens_ids_with_offsets_2: Option<TokenIdsWithOffsets>
) -> TokenIdsWithSpecialTokens

Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens.

fn tokenize<S: AsRef<str>>(&self, text: S) -> Vec<String>

Tokenize a string, returns a vector of tokens as strings. Use tokenize_with_offsets or tokenize_to_tokens to return offset information.

fn tokenize_with_offsets<S: AsRef<str>>(&self, text: S) -> TokensWithOffsets

Tokenize a string, returning tokens with offset information

fn tokenize_list<S, ST>(&self, text_list: S) -> Vec<Vec<String>> where
    S: AsRef<[ST]>,
    ST: AsRef<str>, 

Tokenize a list of strings, returning tokens with offset information

fn tokenize_list_with_offsets<S, ST>(
    &self,
    text_list: S
) -> Vec<TokensWithOffsets> where
    S: AsRef<[ST]>,
    ST: AsRef<str>, 

Tokenize a list of strings, where each corresponds to for example a sentence, returns a vector of TokensWithOffsets containing the tokens and their offset information. This calls tokenize_with_offsets on the list provided.

fn convert_tokens_to_ids<S, ST>(&self, tokens: S) -> Vec<i64> where
    S: AsRef<[ST]>,
    ST: AsRef<str>, 

Convert a slice of string-like to a vector ot token indices

fn encode<S: AsRef<str>>(
    &self,
    text_1: S,
    text_2: Option<S>,
    max_len: usize,
    truncation_strategy: &TruncationStrategy,
    stride: usize
) -> TokenizedInput

Encode a string-like (tokenization followed by encoding)

fn encode_list<S, ST>(
    &self,
    text_list: S,
    max_len: usize,
    truncation_strategy: &TruncationStrategy,
    stride: usize
) -> Vec<TokenizedInput> where
    S: AsRef<[ST]>,
    ST: AsRef<str>, 

Encode a sequence of string-like texts (tokenization followed by encoding). Not that in contrast with encode optional second text, each text provided is encoded independently.

fn encode_pair_list<S, ST>(
    &self,
    text_list: S,
    max_len: usize,
    truncation_strategy: &TruncationStrategy,
    stride: usize
) -> Vec<TokenizedInput> where
    S: AsRef<[(ST, ST)]>,
    ST: AsRef<str>, 

Encode a sequence of string-like text pairs (tokenization followed by encoding). This combines with encode with the list processing of encode_list.

fn decode_to_vec(
    &self,
    token_ids: Vec<i64>,
    skip_special_tokens: bool
) -> Vec<String>

Decode a sequence of token indices to a sequence of Strings, optionally skipping special indices

fn decode(
    &self,
    token_ids: Vec<i64>,
    skip_special_tokens: bool,
    clean_up_tokenization_spaces: bool
) -> String

Converts a sequence of ids (integer) into a string, using the tokenizer and vocabulary with options to remove special tokens and clean up tokenization spaces.

fn clean_up_tokenization(&self, input_string: String) -> String

Cleans-up tokenization artifacts (for example whitespace before punctuation)

fn decode_list(
    &self,
    token_ids_list: Vec<Vec<i64>>,
    skip_special_tokens: bool,
    clean_up_tokenization_spaces: bool
) -> Vec<String>

Converts a list of sequence of ids (integer) into a string, using the tokenizer and vocabulary with options to remove special tokens and clean up tokenization spaces. This calls decode for each provided sequence of ids

impl Vocab for MarianVocab

fn unknown_value() -> &'static str

Associative function returning the unknown value for the vocabulary

fn get_unknown_value(&self) -> &'static str

Returns the unknown value on an instance

fn values(&self) -> &HashMap<String, i64>

Return the map of token strings to IDs

fn indices(&self) -> &HashMap<i64, String>

Return the map of token IDs to strings

fn special_values(&self) -> &HashMap<String, i64>

Return the map of token strings to IDs

fn special_indices(&self) -> &HashMap<i64, String>

Return the map of token IDs to strings for special values

fn from_file(path: &str) -> Result<MarianVocab, TokenizerError>

Read a vocabulary from file

fn token_to_id(&self, token: &str) -> i64

Converts a token to an id.

fn id_to_token(&self, id: &i64) -> String

Converts an id to a token.

fn read_vocab_file(path: &str) -> Result<HashMap<String, i64>, TokenizerError>

Read a Bert-style vocab.txt file (single column, one token per line) The from_file method should be preferred, and needs to be implemented by the specific vocabularies

fn _token_to_id(
    &self,
    token: &str,
    values: &HashMap<String, i64>,
    special_values: &HashMap<String, i64>,
    unknown_value: &str
) -> i64

Converts a token to an id, provided a HashMap of values, a HashMap of special values and the unknown value token string representation. This is not meant to be directly used, the method token_to_id offers a more convenient interface for most vocabularies, but needs to be implemented by the specific vocabulary.

fn _id_to_token(
    &self,
    id: &i64,
    indices: &HashMap<i64, String>,
    special_indices: &HashMap<i64, String>,
    unknown_value: &str
) -> String

Converts an id to a token, provided a HashMap of values, a HashMap of special values and the unknown value token string representation. This is not meant to be directly used, the method id_to_token offers a more convenient interface for most vocabularies, but needs to be implemented by the specific vocabulary.

fn _register_as_special_value(
    token: &str,
    values: &HashMap<String, i64>,
    special_values: &mut HashMap<String, i64>
) -> Result<(), TokenizerError>

Register a token as a special value

fn convert_tokens_to_ids(&self, tokens: &[&str]) -> Vec<i64>

Converts a list of tokens to a list of indices.