Trait IncrementalTokenizer 

pub trait IncrementalTokenizer: Tokenizer {
    type State: Send + Sync;

    // Required methods
    fn create_state(&self) -> Self::State;
    fn decode_incremental_with_state(
        &self,
        state: &mut Self::State,
        token: TokenId,
    ) -> Result<String>;
    fn reset_state(&self, state: &mut Self::State);
    fn get_decoded_text(&self, state: &Self::State) -> String;
}

Incremental tokenizer state for streaming

Required Associated Types

type State: Send + Sync

Tokenizer state for incremental operations

Required Methods

fn create_state(&self) -> Self::State

Create initial state for incremental decoding

fn decode_incremental_with_state( &self, state: &mut Self::State, token: TokenId, ) -> Result<String>

Add token to state and get incremental text

fn reset_state(&self, state: &mut Self::State)

Reset state to initial condition

fn get_decoded_text(&self, state: &Self::State) -> String

Get all decoded text from current state

Implementors