Struct ParallelProcessor 

pub struct ParallelProcessor { /* private fields */ }

Parallel stream processor

Implementations

impl ParallelProcessor

pub fn new( config: StreamConfig, processor: Arc<dyn StreamProcessor>, state_manager: Arc<StateManager>, ) -> Self

Create a new parallel processor

pub fn with_pipeline_id( config: StreamConfig, processor: Arc<dyn StreamProcessor>, state_manager: Arc<StateManager>, pipeline_id: String, ) -> Self

Create a new parallel processor with a specific pipeline ID

pub fn state_manager(&self) -> &Arc<StateManager>

Get the state manager for external access

pub async fn save_checkpoint(&self) -> Result<()>

Save checkpoint using the state manager

pub async fn restore_checkpoint(&self) -> Result<()>

Restore from checkpoint

pub async fn process_stream<S>(&self, stream: S) -> Result<Vec<StreamItem>>

where S: Stream<Item = Result<StreamItem>> + Unpin + Send,

Process a stream in parallel

pub async fn process_batch( &self, items: Vec<StreamItem>, ) -> Result<Vec<StreamItem>>

Process a batch of items