Struct ParallelCorpusProcessor 

pub struct ParallelCorpusProcessor { /* private fields */ }

Parallel corpus processor for handling larger datasets

Implementations

impl ParallelCorpusProcessor

pub fn new(_batchsize: usize) -> Self

Create a new parallel corpus processor

pub fn with_threads(self, numthreads: usize) -> Self

Set the number of threads

pub fn with_max_memory(self, maxmemory: usize) -> Self

Set the maximum memory usage

pub fn process<F, R>(&self, corpus: &[&str], processor: F) -> Result<Vec<R>>

where F: Fn(&[&str]) -> Result<Vec<R>> + Send + Sync, R: Send,

Process a corpus in parallel with a thread pool

pub fn process_with_progress<F, R>( &self, corpus: &[&str], processor: F, progress_callback: impl Fn(usize, usize) + Send + Sync, ) -> Result<Vec<R>>

where F: Fn(&[&str]) -> Result<Vec<R>> + Send + Sync, R: Send,

Process a corpus in parallel with progress tracking