Struct ChunkedDataProcessor 

pub struct ChunkedDataProcessor { /* private fields */ }

Memory-efficient chunked data processor

Processes large datasets in chunks to avoid memory overflow.

Example

use torsh_cluster::utils::memory_efficient::ChunkedDataProcessor;
use torsh_tensor::Tensor;

let large_data = Tensor::from_vec(
    (0..10000).map(|i| i as f32).collect(),
    &[1000, 10]
)?;

let processor = ChunkedDataProcessor::new(100); // Process 100 samples at a time

let mut sum = 0.0;
processor.process(&large_data, |chunk| {
    // Process each chunk
    let chunk_sum: f32 = chunk.iter().sum();
    sum += chunk_sum;
    Ok(())
})?;

Implementations

impl ChunkedDataProcessor

pub fn new(chunk_size: usize) -> Self

Create a new chunked data processor

pub fn parallel(self, parallel: bool) -> Self

Set whether to use parallel processing

pub fn process<F>(&self, data: &Tensor, f: F) -> ClusterResult<()>

where F: FnMut(ArrayView2<'_, f32>) -> ClusterResult<()>,

Process data in chunks

Applies the given function to each chunk of data sequentially.

pub fn process_parallel<F, R>( &self, data: &Tensor, f: F, ) -> ClusterResult<Vec<R>>

where F: Fn(ArrayView2<'_, f32>) -> ClusterResult<R> + Send + Sync, R: Send,

Process data in parallel chunks

Applies the given function to each chunk in parallel. Results from all chunks are collected and returned.

pub fn optimal_chunk_size( n_samples: usize, n_features: usize, available_memory_mb: usize, ) -> usize

Calculate optimal chunk size based on available memory and data dimensions