Struct Encoding 

pub struct Encoding { /* private fields */ }

WASM wrapper around a tiktoken encoding instance.

Created via get_encoding or encoding_for_model. Call .free() when done to release WASM memory.

Implementations

impl Encoding

pub fn encode(&self, text: &str) -> Vec<u32>

Encode text into token ids (returns Uint32Array in JS).

Special tokens like <|endoftext|> are treated as ordinary text. Use encodeWithSpecialTokens() to recognize them.

pub fn encode_with_special_tokens(&self, text: &str) -> Vec<u32>

Encode text into token ids, recognizing special tokens.

Special tokens (e.g. <|endoftext|>) are encoded as their designated ids instead of being split into sub-word pieces.

pub fn decode(&self, tokens: &[u32]) -> String

Decode token ids back to a UTF-8 string.

Uses lossy UTF-8 conversion — invalid byte sequences are replaced with U+FFFD.

pub fn count(&self, text: &str) -> usize

Count tokens without building the full token id array.

Faster than encode(text).length for cases where you only need the count.

pub fn count_with_special_tokens(&self, text: &str) -> usize

Count tokens, recognizing special tokens.

Like count() but special tokens (e.g. <|endoftext|>) are counted as single tokens instead of being split into sub-word pieces.

pub fn vocab_size(&self) -> usize

Get the number of regular (non-special) tokens in the vocabulary.

pub fn num_special_tokens(&self) -> usize

Get the number of special tokens in the vocabulary.

pub fn name(&self) -> String

Get the encoding name (e.g. "cl100k_base").

Trait Implementations

impl From<Encoding> for JsValue

fn from(value: Encoding) -> Self

Converts to this type from the input type.

impl FromWasmAbi for Encoding

type Abi = WasmPtr<WasmRefCell<Encoding>>

The Wasm ABI type that this converts from when coming back out from the ABI boundary.

unsafe fn from_abi(js: Self::Abi) -> Self

Recover a Self from Self::Abi.

impl IntoWasmAbi for Encoding

type Abi = WasmPtr<WasmRefCell<Encoding>>

The Wasm ABI type that this converts into when crossing the ABI boundary.

fn into_abi(self) -> Self::Abi

Convert self into Self::Abi so that it can be sent across the wasm ABI boundary.

impl LongRefFromWasmAbi for Encoding

type Abi = WasmPtr<WasmRefCell<Encoding>>

Same as RefFromWasmAbi::Abi

type Anchor = RcRef<Encoding>

Same as RefFromWasmAbi::Anchor

unsafe fn long_ref_from_abi(js: Self::Abi) -> Self::Anchor

Same as RefFromWasmAbi::ref_from_abi

impl OptionFromWasmAbi for Encoding

fn is_none(abi: &Self::Abi) -> bool

Tests whether the argument is a “none” instance. If so it will be deserialized as None, and otherwise it will be passed to FromWasmAbi.

impl OptionIntoWasmAbi for Encoding

fn none() -> Self::Abi

Returns an ABI instance indicating “none”, which JS will interpret as the None branch of this option.

impl RefFromWasmAbi for Encoding

type Abi = WasmPtr<WasmRefCell<Encoding>>

The Wasm ABI type references to Self are recovered from.

type Anchor = RcRef<Encoding>

The type that holds the reference to Self for the duration of the invocation of the function that has an &Self parameter. This is required to ensure that the lifetimes don’t persist beyond one function call, and so that they remain anonymous.

unsafe fn ref_from_abi(js: Self::Abi) -> Self::Anchor

Recover a Self::Anchor from Self::Abi.

impl RefMutFromWasmAbi for Encoding

type Abi = WasmPtr<WasmRefCell<Encoding>>

Same as RefFromWasmAbi::Abi

type Anchor = RcRefMut<Encoding>

Same as RefFromWasmAbi::Anchor

unsafe fn ref_mut_from_abi(js: Self::Abi) -> Self::Anchor

Same as RefFromWasmAbi::ref_from_abi

impl SupportsConstructor for Encoding

impl SupportsInstanceProperty for Encoding

impl SupportsStaticProperty for Encoding

impl TryFromJsValue for Encoding

fn try_from_js_value(value: JsValue) -> Result<Self, JsValue>

Performs the conversion.

fn try_from_js_value_ref(value: &JsValue) -> Option<Self>

Performs the conversion.

impl VectorFromWasmAbi for Encoding

type Abi = <Box<[JsValue]> as FromWasmAbi>::Abi

unsafe fn vector_from_abi(js: Self::Abi) -> Box<[Encoding]>

impl VectorIntoWasmAbi for Encoding

type Abi = <Box<[JsValue]> as IntoWasmAbi>::Abi

fn vector_into_abi(vector: Box<[Encoding]>) -> Self::Abi

impl WasmDescribe for Encoding

fn describe()

impl WasmDescribeVector for Encoding

fn describe_vector()