Struct StreamingTextProcessor 

pub struct StreamingTextProcessor { /* private fields */ }

Memory-efficient streaming text processor

Processes large text datasets in chunks to minimize memory usage while maintaining feature extraction quality.

Implementations

impl StreamingTextProcessor

pub fn new() -> Self

Create a new streaming text processor

pub fn chunk_size(self, chunk_size: usize) -> Self

Set chunk size in characters

pub fn overlap_size(self, overlap_size: usize) -> Self

Set overlap size between chunks

pub fn min_chunk_words(self, min_words: usize) -> Self

Set minimum words per chunk

pub fn stream_process_with_count_vectorizer( &self, large_text: &str, vectorizer: &mut CountVectorizer, ) -> SklResult<Array2<Float>>

Process large text with a streaming vectorizer

pub fn stream_process_with_tfidf( &self, large_text: &str, vectorizer: &mut TfidfVectorizer, ) -> SklResult<Array2<Float>>

Process large text with streaming TF-IDF

pub fn extract_streaming_stats( &self, large_text: &str, ) -> SklResult<Array1<Float>>

Extract memory-efficient statistical features

Trait Implementations

impl Clone for StreamingTextProcessor

fn clone(&self) -> Self

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for StreamingTextProcessor

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for StreamingTextProcessor

fn default() -> Self

Returns the “default value” for a type.