Struct MemoryMappedCorpus 

pub struct MemoryMappedCorpus { /* private fields */ }

Memory-mapped corpus for efficient large file processing

Implementations

impl MemoryMappedCorpus

pub fn from_file<P: AsRef<Path>>(path: P) -> Result<Self>

Create a new memory-mapped corpus from a file

pub fn num_documents(&self) -> usize

Get the number of documents (lines) in the corpus

pub fn get_document(&self, index: usize) -> Result<&str>

Get a specific document by index

pub fn iter(&self) -> CorpusIterator<'_>

Iterate over all documents

pub fn parallel_process<F, R>( &self, chunksize: usize, processor: F, ) -> Result<Vec<R>>

where F: Fn(&[&str]) -> Result<R> + Send + Sync, R: Send,

Process documents in parallel chunks