Struct rust_tokenizers::tokenizer::OpenAiGptTokenizer

pub struct OpenAiGptTokenizer { /* fields omitted */ }

GPT tokenizer

GPT tokenizer performing:

• BaseTokenizer tokenization (see BaseTokenizer for more details)
• BPE tokenization

Implementations

impl OpenAiGptTokenizer

pub fn from_file(
    vocab_path: &str,
    merges_path: &str,
    lower_case: bool
) -> Result<OpenAiGptTokenizer, TokenizerError>

Create a new instance of a OpenAiGptTokenizer Expects a vocabulary flat file and merges file as an input.

Parameters

• vocab_path (&str): path to the vocabulary file
• merges_path (&str): path to the merges file (use as part of the BPE encoding process)
• lower_case (bool): flag indicating if the text should be lower-cased as part of the tokenization

Example

use rust_tokenizers::tokenizer::{OpenAiGptTokenizer, Tokenizer};
let lower_case = false;
let tokenizer =
    OpenAiGptTokenizer::from_file("path/to/vocab/file", "path/to/merges/file", lower_case)
        .unwrap();

pub fn from_existing_vocab_and_merges(
    vocab: OpenAiGptVocab,
    merges: BpePairVocab,
    lower_case: bool
) -> OpenAiGptTokenizer

Create a new instance of a OpenAiGptTokenizer from an existing vocabulary and merges

Parameters

• vocab (Gpt2Vocab): GPT-like vocabulary
• merges (BpePairVocab): BPE pairs vocabulary
• lower_case (bool): flag indicating if the text should be lower-cased as part of the tokenization

Example

use rust_tokenizers::tokenizer::{OpenAiGptTokenizer, Tokenizer};
use rust_tokenizers::vocab::{BpePairVocab, OpenAiGptVocab, Vocab};
let lower_case = false;
let vocab = OpenAiGptVocab::from_file("path/to/vocab/file").unwrap();
let merges = BpePairVocab::from_file("path/to/merges/file").unwrap();

let tokenizer = OpenAiGptTokenizer::from_existing_vocab_and_merges(vocab, merges, lower_case);

Trait Implementations

impl MultiThreadedTokenizer<OpenAiGptVocab> for OpenAiGptTokenizer

fn vocab(&self) -> &T

returns a reference to the tokenizer vocabulary

fn tokenize_list_with_offsets<S, ST>(
    &self,
    text_list: S
) -> Vec<TokensWithOffsets> where
    S: AsRef<[ST]>,
    ST: AsRef<str> + Sync, 

Tokenize a list of strings (with multithreading), where each corresponds to for example a sentence, returns a vector of TokensWithOffsets containing the tokens and their offset information. This calls tokenize_with_offsets on the list provided.

fn tokenize_list<S, ST>(&self, text_list: S) -> Vec<Vec<String>> where
    S: AsRef<[ST]>,
    ST: AsRef<str> + Sync, 

Multithreaded tokenization of a list of strings, returning tokens with offset information

fn encode_list<S, ST>(
    &self,
    text_list: S,
    max_len: usize,
    truncation_strategy: &TruncationStrategy,
    stride: usize
) -> Vec<TokenizedInput> where
    S: AsRef<[ST]>,
    ST: AsRef<str> + Sync, 

Multithreaded encoding of a sequence of string-like texts (tokenization followed by encoding). Not that in contrast with encode optional second text, each text provided is encoded independently.

fn encode_pair_list<S, ST>(
    &self,
    text_list: S,
    max_len: usize,
    truncation_strategy: &TruncationStrategy,
    stride: usize
) -> Vec<TokenizedInput> where
    S: AsRef<[(ST, ST)]>,
    ST: AsRef<str> + Sync, 

Multithreaded ncoding of a sequence of string-like text pairs (tokenization followed by encoding). This combines with encode with the list processing of encode_list.

fn decode_list(
    &self,
    token_ids_list: Vec<Vec<i64>>,
    skip_special_tokens: bool,
    clean_up_tokenization_spaces: bool
) -> Vec<String>

Multithreaded conversion a list of sequence of ids (integer) into a string, using the tokenizer and vocabulary with options to remove special tokens and clean up tokenization spaces. This calls decode for each provided sequence of ids

impl Tokenizer<OpenAiGptVocab> for OpenAiGptTokenizer

fn vocab(&self) -> &OpenAiGptVocab

returns a reference to the tokenizer vocabulary

fn tokenize_to_tokens(&self, initial_token: TokenRef<'_>) -> Vec<Token>

Tokenize a TokenRef, returning a sequence of tokens

fn convert_tokens_to_string(&self, tokens: Vec<String>) -> String

Converts a sequence of strings into a single string. This will clean-up artifacts from tokenization (for example sub ##word) and generate a single output string

fn tokenize<S: AsRef<str>>(&self, text: S) -> Vec<String>

Tokenize a string, returns a vector of tokens as strings. Use tokenize_with_offsets or tokenize_to_tokens to return offset information.

fn tokenize_with_offsets<S: AsRef<str>>(&self, text: S) -> TokensWithOffsets

Tokenize a string, returning tokens with offset information

fn tokenize_list<S, ST>(&self, text_list: S) -> Vec<Vec<String>> where
    S: AsRef<[ST]>,
    ST: AsRef<str>, 

Tokenize a list of strings, returning tokens with offset information

fn tokenize_list_with_offsets<S, ST>(
    &self,
    text_list: S
) -> Vec<TokensWithOffsets> where
    S: AsRef<[ST]>,
    ST: AsRef<str>, 

Tokenize a list of strings, where each corresponds to for example a sentence, returns a vector of TokensWithOffsets containing the tokens and their offset information. This calls tokenize_with_offsets on the list provided.

fn convert_tokens_to_ids<S, ST>(&self, tokens: S) -> Vec<i64> where
    S: AsRef<[ST]>,
    ST: AsRef<str>, 

Convert a slice of string-like to a vector ot token indices

fn encode<S: AsRef<str>>(
    &self,
    text_1: S,
    text_2: Option<S>,
    max_len: usize,
    truncation_strategy: &TruncationStrategy,
    stride: usize
) -> TokenizedInput

Encode a string-like (tokenization followed by encoding)

fn encode_list<S, ST>(
    &self,
    text_list: S,
    max_len: usize,
    truncation_strategy: &TruncationStrategy,
    stride: usize
) -> Vec<TokenizedInput> where
    S: AsRef<[ST]>,
    ST: AsRef<str>, 

Encode a sequence of string-like texts (tokenization followed by encoding). Not that in contrast with encode optional second text, each text provided is encoded independently.

fn encode_pair_list<S, ST>(
    &self,
    text_list: S,
    max_len: usize,
    truncation_strategy: &TruncationStrategy,
    stride: usize
) -> Vec<TokenizedInput> where
    S: AsRef<[(ST, ST)]>,
    ST: AsRef<str>, 

Encode a sequence of string-like text pairs (tokenization followed by encoding). This combines with encode with the list processing of encode_list.

fn decode_to_vec(
    &self,
    token_ids: Vec<i64>,
    skip_special_tokens: bool
) -> Vec<String>

Decode a sequence of token indices to a sequence of Strings, optionally skipping special indices

fn decode(
    &self,
    token_ids: Vec<i64>,
    skip_special_tokens: bool,
    clean_up_tokenization_spaces: bool
) -> String

Converts a sequence of ids (integer) into a string, using the tokenizer and vocabulary with options to remove special tokens and clean up tokenization spaces.

fn clean_up_tokenization(&self, input_string: String) -> String

Cleans-up tokenization artifacts (for example whitespace before punctuation)

fn decode_list(
    &self,
    token_ids_list: Vec<Vec<i64>>,
    skip_special_tokens: bool,
    clean_up_tokenization_spaces: bool
) -> Vec<String>

Converts a list of sequence of ids (integer) into a string, using the tokenizer and vocabulary with options to remove special tokens and clean up tokenization spaces. This calls decode for each provided sequence of ids

fn build_input_with_special_tokens(
    &self,
    tokens_ids_with_offsets_1: TokenIdsWithOffsets,
    tokens_ids_with_offsets_2: Option<TokenIdsWithOffsets>
) -> TokenIdsWithSpecialTokens

Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens.