Struct ParallelPipeline 

pub struct ParallelPipeline { /* private fields */ }

High-performance parallel data pipeline

Implementations

impl ParallelPipeline

pub fn new(capacity: usize) -> Self

Create a new parallel pipeline

pub fn with_max_concurrency(self, max: usize) -> Self

Set maximum concurrency

pub fn with_ttl(self, ttl: Duration) -> Self

Set cache TTL

pub fn with_loader<F>(self, loader: F) -> Self

where F: Fn(String) -> DataSource + Send + Sync + 'static,

Set custom data loader

pub async fn process(&self, source_id: &str) -> Result<DataSource, DataError>

Process a single source

pub async fn process_parallel( &self, source_ids: Vec<String>, ) -> Vec<Result<DataSource, DataError>>

Process multiple sources in parallel

pub fn metrics(&self) -> &DataMatchingMetrics

Get metrics

pub fn clear_cache(&self)

Clear cache