Trait IncrementalTokenizer 

pub trait IncrementalTokenizer: Tokenizer {
    type State: Send + Sync;

    // Required methods
    fn create_state(&self) -> Self::State;
    fn decode_incremental_with_state(
        &self,
        state: &mut Self::State,
        token: TokenId,
    ) -> Result<String, FerrumError>;
    fn reset_state(&self, state: &mut Self::State);
    fn get_decoded_text(&self, state: &Self::State) -> String;
}

Incremental tokenizer state for streaming

Required Associated Types

type State: Send + Sync

Tokenizer state for incremental operations

Required Methods

fn create_state(&self) -> Self::State

Create initial state for incremental decoding

fn decode_incremental_with_state( &self, state: &mut Self::State, token: TokenId, ) -> Result<String, FerrumError>

Add token to state and get incremental text

fn reset_state(&self, state: &mut Self::State)

Reset state to initial condition

fn get_decoded_text(&self, state: &Self::State) -> String

Get all decoded text from current state

Dyn Compatibility

This trait is dyn compatible.

In older versions of Rust, dyn compatibility was called "object safety".

Implementations on Foreign Types

impl IncrementalTokenizer for HuggingFaceTokenizer

type State = IncrementalState

fn create_state(&self) -> <HuggingFaceTokenizer as IncrementalTokenizer>::State

fn decode_incremental_with_state( &self, state: &mut <HuggingFaceTokenizer as IncrementalTokenizer>::State, token: TokenId, ) -> Result<String, FerrumError>

fn reset_state( &self, state: &mut <HuggingFaceTokenizer as IncrementalTokenizer>::State, )

fn get_decoded_text( &self, state: &<HuggingFaceTokenizer as IncrementalTokenizer>::State, ) -> String

Implementors