Struct StreamingTokenizer 

pub struct StreamingTokenizer<T: Tokenizer> { /* private fields */ }

Streaming tokenizer for processing large texts efficiently

Implementations

impl<T: Tokenizer> StreamingTokenizer<T>

pub fn new(tokenizer: T) -> Self

Create a new streaming tokenizer

pub fn with_buffer_size(self, buffer_size: usize) -> Self

Set the buffer size for reading from stream

pub fn with_overlap_size(self, overlap_size: usize) -> Self

Set the overlap size between chunks

pub fn with_max_chunk_length(self, max_length: usize) -> Self

Set maximum chunk length for tokenization

pub fn process_stream<R: Read>(&self, reader: R) -> Result<Vec<TokenizedInput>>

Process a stream of text and return tokenized chunks

pub fn process_text(&self, text: &str) -> Result<Vec<TokenizedInput>>

Process text from a string in streaming fashion

pub fn process_lines<I>(&self, lines: I) -> Result<Vec<TokenizedInput>>

where I: Iterator<Item = String>,

Process an iterator of text lines

pub fn tokenizer(&self) -> &T

Get the underlying tokenizer

pub fn buffer_size(&self) -> usize

Get buffer size

pub fn overlap_size(&self) -> usize

Get overlap size

pub fn max_chunk_length(&self) -> Option<usize>

Get max chunk length