Struct ParallelPipeline 

pub struct ParallelPipeline { /* private fields */ }

Parallel preprocessing pipeline

Implementations

impl ParallelPipeline

pub fn new(config: ParallelConfig, preprocess_fn: PreprocessFn) -> Self

Create a new parallel preprocessing pipeline

Arguments

• config - Pipeline configuration
• preprocess_fn - Function to apply to each data chunk

Returns

• ParallelPipeline - The pipeline instance

pub fn submit( &mut self, data: Array2<f64>, target: Option<Array1<f64>>, ) -> Result<usize>

Submit data for processing

Arguments

• data - Input data array
• target - Optional target values

Returns

• Ok(usize) - ID of the submitted item
• Err(DatasetsError) - If submission fails

pub fn submit_dataset(&mut self, dataset: &Dataset) -> Result<usize>

Submit a dataset for processing

pub fn submit_chunk(&mut self, chunk: &DataChunk) -> Result<usize>

Submit a data chunk for processing

pub fn receive(&mut self) -> Result<Option<Dataset>>

Receive a processed result

Returns

• Ok(Some(Dataset)) - Processed dataset
• Ok(None) - No more results (all workers finished)
• Err(DatasetsError) - If receive fails

pub fn try_receive(&mut self) -> Result<Option<Dataset>>

Try to receive a result without blocking

pub fn process_batch(&mut self, datasets: &[Dataset]) -> Result<Vec<Dataset>>

Process a batch of datasets

pub fn items_processed(&self) -> usize

Get number of items processed

pub fn stop(&mut self)

Stop the pipeline gracefully

pub fn join(self) -> Result<()>

Wait for all workers to finish

Trait Implementations

impl Drop for ParallelPipeline

fn drop(&mut self)

Executes the destructor for this type.