Trait Encoder 

pub trait Encoder: Send + Sync {
    // Required methods
    fn encode(&self, input: &str) -> Result<Encoding>;
    fn encode_batch(&self, inputs: &[&str]) -> Result<Vec<Encoding>>;
}

Required Methods

fn encode(&self, input: &str) -> Result<Encoding>

fn encode_batch(&self, inputs: &[&str]) -> Result<Vec<Encoding>>

Dyn Compatibility

This trait is dyn compatible.

In older versions of Rust, dyn compatibility was called "object safety".

Implementors

impl Encoder for CachedTokenizer

impl Encoder for FastTokenizer

impl Encoder for HuggingFaceTokenizer

impl Encoder for TikTokenTokenizer