Trait Encoder 

pub trait Encoder: Send + Sync {
    // Required methods
    fn encode(&self, input: &str, add_special_tokens: bool) -> Result<Encoding>;
    fn encode_batch(
        &self,
        inputs: &[&str],
        add_special_tokens: bool,
    ) -> Result<Vec<Encoding>>;
}

Core encoding trait - separate from decoding for modularity

Required Methods

fn encode(&self, input: &str, add_special_tokens: bool) -> Result<Encoding>

fn encode_batch( &self, inputs: &[&str], add_special_tokens: bool, ) -> Result<Vec<Encoding>>

Implementors

impl Encoder for CachedTokenizer

impl Encoder for HuggingFaceTokenizer

impl Encoder for MockTokenizer

impl Encoder for TiktokenTokenizer