Struct CoreBPE

Source

pub struct CoreBPE { /* private fields */ }

Implementations§

Source §

impl CoreBPE

Rust API

Source

pub fn new( encoder: HashMap<Vec<u8>, Rank>, special_tokens_encoder: HashMap<String, Rank>, pattern: &str, ) -> Result<Self>

Source

pub fn encode_ordinary_as<T: FromRank>(&self, text: &str) -> Vec<T>

Like encode_ordinary, but converts each token from Rank (u32) into T.

This is useful when you need tokens in a different integer type (e.g. usize for indexing, or u64 for ML frameworks).

§Examples

use tiktoken_rs::cl100k_base;

let bpe = cl100k_base().unwrap();
let tokens: Vec<usize> = bpe.encode_ordinary_as("hello world");

Source

pub fn encode_with_special_tokens_as<T: FromRank>(&self, text: &str) -> Vec<T>

Like encode_with_special_tokens, but converts each token from Rank (u32) into T.

§Examples

use tiktoken_rs::cl100k_base;

let bpe = cl100k_base().unwrap();
let tokens: Vec<u64> = bpe.encode_with_special_tokens_as("hello <|endoftext|>");

Source

pub fn encode_as<T: FromRank>( &self, text: &str, allowed_special: &HashSet<&str>, ) -> Result<(Vec<T>, usize)>

Like encode, but converts each token from Rank (u32) into T.

Returns the same error as encode if tokenization fails.

Source

pub fn count_ordinary(&self, text: &str) -> usize

Returns the number of tokens that encode_ordinary would produce, without returning the token list itself.

Equivalent to self.encode_ordinary(text).len().

Source

pub fn count( &self, text: &str, allowed_special: &HashSet<&str>, ) -> Result<usize>

Returns the number of tokens that encode would produce for the given allowed_special set, without returning the token list itself.

Equivalent to self.encode(text, allowed_special)?.0.len().

Source

pub fn count_with_special_tokens(&self, text: &str) -> usize

Returns the number of tokens that encode_with_special_tokens would produce, without returning the token list itself.

Equivalent to self.encode_with_special_tokens(text).len().

Source

pub fn decode(&self, tokens: &[Rank]) -> Result<String>

Decode a vector of tokens into a valid UTF-8 String

If unicode validation is not wanted, see _decode_native.

Source

pub fn _decode_native_and_split( &self, tokens: Vec<Rank>, ) -> impl Iterator<Item = Vec<u8>> + '_

Source

pub fn split_by_token<'a>( &'a self, text: &'a str, use_special_tokens: bool, ) -> Result<Vec<String>>

Tokenize a string and return the decoded tokens using the correct BPE model.

This method takes a string, encodes it using the BPE model, and decodes the encoded tokens into a vector of strings. It can be used to tokenize a string and return the decoded tokens using the correct BPE model.

§Examples

    use tiktoken_rs::cl100k_base;
    let bpe = cl100k_base().unwrap();
    let tokenized: Result<Vec<_>, _> = bpe
        .split_by_token("This is a test         with a lot of spaces", true);
    let tokenized = tokenized.unwrap();
    assert_eq!(
        tokenized,
        vec!["This", " is", " a", " test", "        ", " with", " a", " lot", " of", " spaces"]
    );

§Arguments

text: A string slice containing the text to be tokenized.
use_special_tokens: A boolean indicating whether to use the special tokens in the BPE model.

§Returns

Result<Vec<String>>: A Result containing a vector of decoded tokens as strings, or an error if the string cannot be converted into a valid UTF-8 string.