Struct StreamingTextProcessor 

pub struct StreamingTextProcessor<T: Tokenizer> { /* private fields */ }

Streaming text processor for handling arbitrarily large files

Implementations

impl<T: Tokenizer> StreamingTextProcessor<T>

pub fn new(tokenizer: T) -> Self

Create a new streaming processor

pub fn with_buffer_size(self, size: usize) -> Self

Set custom buffer size

pub fn process_lines<P, F, R>(&self, path: P, processor: F) -> Result<Vec<R>>

where P: AsRef<Path>, F: FnMut(&str, usize) -> Result<Option<R>>,

Process a file line by line

pub fn process_reader_lines<R: BufRead, F, U>( &self, reader: R, processor: F, ) -> Result<Vec<U>>

where F: FnMut(&str, usize) -> Result<Option<U>>,

Process lines from any reader

pub fn build_vocabulary_streaming<P: AsRef<Path>>( &self, path: P, min_count: usize, ) -> Result<Vocabulary>

Build vocabulary from a streaming corpus

impl StreamingTextProcessor<WordTokenizer>

pub fn with_default_tokenizer() -> Self

Create a streaming processor with default word tokenizer