Enum Encoding 

pub enum Encoding {
    Cl100kBase,
    O200kBase,
    P50kBase,
}

Supported tokenizer encodings.

Variants

Cl100kBase

cl100k_base - GPT-4, GPT-3.5-turbo, text-embedding-ada-002

O200kBase

o200k_base - GPT-4o, GPT-4.1, o1, o3

P50kBase

p50k_base - Codex, text-davinci-002/003

Implementations

impl Encoding

pub fn as_str(&self) -> &'static str

Get the encoding name as a string.

Trait Implementations

impl Clone for Encoding

fn clone(&self) -> Encoding

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Copy for Encoding

impl Debug for Encoding

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for Encoding

fn default() -> Encoding

Returns the “default value” for a type.

impl Eq for Encoding

impl FromStr for Encoding

type Err = ()

The associated error which can be returned from parsing.

fn from_str(s: &str) -> Result<Self, Self::Err>

Parses a string s to return a value of this type.

impl PartialEq for Encoding

fn eq(&self, other: &Encoding) -> bool

Equality operator ==.

fn ne(&self, other: &Rhs) -> bool

Inequality operator !=.

impl StructuralPartialEq for Encoding