Struct DistributedProcessor 

pub struct DistributedProcessor { /* private fields */ }

Distributed dataset processor

Implementations

impl DistributedProcessor

pub fn new(config: DistributedConfig) -> Result<Self>

Create a new distributed processor

pub fn default_config() -> Result<Self>

Create with default configuration

pub fn process_dataset_parallel<F, R>( &self, dataset: &Dataset, processor: F, ) -> Result<Vec<R>>

where F: Fn(&Dataset) -> Result<R> + Send + Sync + Clone + 'static, R: Send + 'static,

Process a large dataset in parallel chunks

pub fn map_reduce_dataset<M, R, C>( &self, dataset: &Dataset, mapper: M, reducer: R, ) -> Result<C>

where M: Fn(&Dataset) -> Result<Vec<C>> + Send + Sync + Clone + 'static, R: Fn(Vec<C>) -> Result<C> + Send + Sync + 'static, C: Send + 'static,

Distribute dataset across multiple workers with MapReduce pattern

pub fn split_dataset_into_chunks( &self, dataset: &Dataset, ) -> Result<Vec<Dataset>>

Split a dataset into balanced chunks for distribution

pub fn distributed_sample( &self, dataset: &Dataset, n_samples: usize, random_state: Option<u64>, ) -> Result<Dataset>

Distributed random sampling across workers

pub fn distributed_k_fold( &self, dataset: &Dataset, k: usize, shuffle: bool, random_state: Option<u64>, ) -> Result<Vec<(Dataset, Dataset)>>

Distributed cross-validation split

pub fn distributed_stratified_sample( &self, dataset: &Dataset, n_samples: usize, random_state: Option<u64>, ) -> Result<Dataset>

Distributed stratified sampling

pub fn distributed_scale( &self, dataset: &Dataset, method: ScalingMethod, ) -> Result<(Dataset, ScalingParameters)>

Parallel feature scaling across workers