Skip to main content

mytheclipse_queue/
pipeline.rs

1//! Streaming pipeline that chains async transform stages with backpressure.
2//!
3//! Each stage processes items from the previous stage via a bounded channel,
4//! providing natural backpressure between stages.
5
6use std::marker::PhantomData;
7use std::sync::Arc;
8
9use async_trait::async_trait;
10use tokio::sync::mpsc;
11use tokio::task::JoinHandle;
12
13/// A single transform in the pipeline.
14#[async_trait]
15pub trait Stage<I: Send + 'static, O: Send + 'static>: Send + Sync {
16    async fn process(&self, input: I) -> Result<O, StageError>;
17}
18
19/// Errors from pipeline stages.
20#[derive(Debug)]
21pub enum StageError {
22    Processing(String),
23    ChannelClosed,
24}
25
26impl std::fmt::Display for StageError {
27    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
28        match self {
29            StageError::Processing(s) => write!(f, "stage error: {s}"),
30            StageError::ChannelClosed => write!(f, "channel closed"),
31        }
32    }
33}
34
35impl std::error::Error for StageError {}
36
37/// Runs a single stage as a background task, consuming from input and
38/// forwarding results to output.
39pub struct StageRunner<S, I, O>
40where
41    S: Stage<I, O>,
42    I: Send + 'static,
43    O: Send + 'static,
44{
45    stage: Arc<S>,
46    _phantom: PhantomData<(I, O)>,
47}
48
49impl<S, I, O> StageRunner<S, I, O>
50where
51    S: Stage<I, O> + 'static,
52    I: Send + 'static,
53    O: Send + 'static,
54{
55    /// Creates a runner for a single stage with the given channel capacity.
56    pub fn new(stage: S) -> Self {
57        Self {
58            stage: Arc::new(stage),
59            _phantom: PhantomData,
60        }
61    }
62
63    /// Consumes items from `input`, applies the stage, sends to `output`.
64    /// Completes when the input stream ends.
65    pub fn run(
66        self,
67        input: mpsc::Receiver<I>,
68        output: mpsc::Sender<O>,
69    ) -> JoinHandle<Result<(), StageError>>
70    where
71        S: 'static,
72    {
73        let stage = self.stage;
74        tokio::spawn(async move {
75            let mut input = input;
76            loop {
77                match input.recv().await {
78                    Some(item) => match stage.process(item).await {
79                        Ok(out) => {
80                            if output.send(out).await.is_err() {
81                                return Err(StageError::ChannelClosed);
82                            }
83                        }
84                        Err(e) => return Err(e),
85                    },
86                    None => return Ok(()),
87                }
88            }
89        })
90    }
91}
92
93impl<S, I, O> Default for StageRunner<S, I, O>
94where
95    S: Stage<I, O> + Default + 'static,
96    I: Send + 'static,
97    O: Send + 'static,
98{
99    fn default() -> Self {
100        Self::new(S::default())
101    }
102}
103
104#[cfg(test)]
105mod tests {
106    use super::*;
107
108    struct DoubleStage;
109
110    #[async_trait]
111    impl Stage<u32, u32> for DoubleStage {
112        async fn process(&self, input: u32) -> Result<u32, StageError> {
113            Ok(input * 2)
114        }
115    }
116
117    #[tokio::test]
118    async fn stage_runner_doubles_values() {
119        let (tx, rx_in) = mpsc::channel::<u32>(16);
120        let (tx_out, mut rx_out) = mpsc::channel::<u32>(16);
121
122        let runner = StageRunner::new(DoubleStage);
123        let handle = runner.run(rx_in, tx_out);
124
125        tx.send(5).await.unwrap();
126        tx.send(7).await.unwrap();
127        drop(tx);
128
129        assert_eq!(rx_out.recv().await, Some(10));
130        assert_eq!(rx_out.recv().await, Some(14));
131        assert_eq!(rx_out.recv().await, None);
132
133        assert!(handle.await.unwrap().is_ok());
134    }
135}