Skip to main content

AsyncTokenizer

ferrum_interfaces::tokenizer

Trait AsyncTokenizer 

Source 
pub trait AsyncTokenizer: Tokenizer {
    // Required methods
    fn encode_async<'life0, 'life1, 'async_trait>(
        &'life0 self,
        text: &'life1 str,
        add_special: bool,
    ) -> Pin<Box<dyn Future<Output = Result<Vec<TokenId>>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait;
    fn decode_async<'life0, 'life1, 'async_trait>(
        &'life0 self,
        tokens: &'life1 [TokenId],
        skip_special: bool,
    ) -> Pin<Box<dyn Future<Output = Result<String>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait;
    fn encode_batch<'life0, 'life1, 'life2, 'async_trait>(
        &'life0 self,
        texts: &'life1 [&'life2 str],
        add_special: bool,
    ) -> Pin<Box<dyn Future<Output = Result<Vec<Vec<TokenId>>>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait,
             'life2: 'async_trait;
    fn decode_batch<'life0, 'life1, 'life2, 'async_trait>(
        &'life0 self,
        token_sequences: &'life1 [&'life2 [TokenId]],
        skip_special: bool,
    ) -> Pin<Box<dyn Future<Output = Result<Vec<String>>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait,
             'life2: 'async_trait;
}
Expand description

Asynchronous tokenizer operations for I/O-bound tokenization

Required Methods§

Source

fn encode_async<'life0, 'life1, 'async_trait>( &'life0 self, text: &'life1 str, add_special: bool, ) -> Pin<Box<dyn Future<Output = Result<Vec<TokenId>>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Asynchronous encoding (useful for very large texts)

Source

fn decode_async<'life0, 'life1, 'async_trait>( &'life0 self, tokens: &'life1 [TokenId], skip_special: bool, ) -> Pin<Box<dyn Future<Output = Result<String>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Asynchronous decoding

Source

fn encode_batch<'life0, 'life1, 'life2, 'async_trait>( &'life0 self, texts: &'life1 [&'life2 str], add_special: bool, ) -> Pin<Box<dyn Future<Output = Result<Vec<Vec<TokenId>>>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait,

Batch encoding for multiple texts

Source

fn decode_batch<'life0, 'life1, 'life2, 'async_trait>( &'life0 self, token_sequences: &'life1 [&'life2 [TokenId]], skip_special: bool, ) -> Pin<Box<dyn Future<Output = Result<Vec<String>>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait,

Batch decoding for multiple token sequences

Implementors§