Struct TokenNativeCodec 

pub struct TokenNativeCodec { /* private fields */ }

Token-native compression codec

Compresses text by converting to token IDs and encoding with VarInt.

Implementations

impl TokenNativeCodec

pub fn new(encoding: Encoding) -> Self

Create a new token-native codec with the specified encoding

pub fn cl100k() -> Self

Create codec with cl100k_base (canonical/default)

pub fn o200k() -> Self

Create codec with o200k_base

pub fn encoding(&self) -> Encoding

Get the encoding used by this codec

pub fn compress(&self, text: &str) -> Result<CompressionResult>

Compress text to token-native wire format

pub fn decompress(&self, wire: &str) -> Result<String>

Decompress from token-native wire format

pub fn compress_raw(&self, text: &str) -> Vec<u8>

Compress and return raw bytes (no wire format prefix)

pub fn decompress_raw(&self, bytes: &[u8]) -> Result<String>

Decompress from raw bytes

pub fn compress_binary(&self, text: &str) -> Vec<u8>

Compress to binary wire format (tokenizer ID + raw bytes)

Binary format: <tokenizer_byte><varint_tokens>

• Byte 0: Tokenizer ID (0=cl100k, 1=o200k, 2=llama)
• Bytes 1+: VarInt-encoded token IDs

Use this for binary-safe channels (WebSocket binary, QUIC, etc.) to achieve maximum compression (~50% of original).

pub fn decompress_binary(bytes: &[u8]) -> Result<String>

Decompress from binary wire format

Trait Implementations

impl Clone for TokenNativeCodec

fn clone(&self) -> TokenNativeCodec

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for TokenNativeCodec

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for TokenNativeCodec

fn default() -> Self

Returns the “default value” for a type.

impl Copy for TokenNativeCodec