Skip to main content

mytheclipse_queue/
pipeline.rs

1//! Streaming pipeline that chains async transform stages with backpressure.
2//!
3//! Each stage processes items from the previous stage via a bounded channel,
4//! providing natural backpressure between stages.
5
6use std::marker::PhantomData;
7use std::sync::Arc;
8
9use async_trait::async_trait;
10use tokio::sync::mpsc;
11use tokio::task::JoinHandle;
12
13/// A single transform in the pipeline.
14#[async_trait]
15pub trait Stage<I: Send + 'static, O: Send + 'static>: Send + Sync {
16    /// Transforms one input into one output, or fails the stage.
17    async fn process(&self, input: I) -> Result<O, StageError>;
18}
19
20/// Errors from pipeline stages.
21#[derive(Debug)]
22pub enum StageError {
23    /// The stage itself failed, with a message from the implementation.
24    Processing(String),
25    /// The pipeline shut down before this item could be processed.
26    ChannelClosed,
27}
28
29impl std::fmt::Display for StageError {
30    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
31        match self {
32            StageError::Processing(s) => write!(f, "stage error: {s}"),
33            StageError::ChannelClosed => write!(f, "channel closed"),
34        }
35    }
36}
37
38impl std::error::Error for StageError {}
39
40/// Runs a single stage as a background task, consuming from input and
41/// forwarding results to output.
42pub struct StageRunner<S, I, O>
43where
44    S: Stage<I, O>,
45    I: Send + 'static,
46    O: Send + 'static,
47{
48    stage: Arc<S>,
49    _phantom: PhantomData<(I, O)>,
50}
51
52impl<S, I, O> StageRunner<S, I, O>
53where
54    S: Stage<I, O> + 'static,
55    I: Send + 'static,
56    O: Send + 'static,
57{
58    /// Creates a runner for a single stage with the given channel capacity.
59    pub fn new(stage: S) -> Self {
60        Self {
61            stage: Arc::new(stage),
62            _phantom: PhantomData,
63        }
64    }
65
66    /// Consumes items from `input`, applies the stage, sends to `output`.
67    /// Completes when the input stream ends.
68    pub fn run(
69        self,
70        input: mpsc::Receiver<I>,
71        output: mpsc::Sender<O>,
72    ) -> JoinHandle<Result<(), StageError>>
73    where
74        S: 'static,
75    {
76        let stage = self.stage;
77        tokio::spawn(async move {
78            let mut input = input;
79            loop {
80                match input.recv().await {
81                    Some(item) => match stage.process(item).await {
82                        Ok(out) => {
83                            if output.send(out).await.is_err() {
84                                return Err(StageError::ChannelClosed);
85                            }
86                        }
87                        Err(e) => return Err(e),
88                    },
89                    None => return Ok(()),
90                }
91            }
92        })
93    }
94}
95
96impl<S, I, O> Default for StageRunner<S, I, O>
97where
98    S: Stage<I, O> + Default + 'static,
99    I: Send + 'static,
100    O: Send + 'static,
101{
102    fn default() -> Self {
103        Self::new(S::default())
104    }
105}
106
107#[cfg(test)]
108mod tests {
109    use super::*;
110
111    struct DoubleStage;
112
113    #[async_trait]
114    impl Stage<u32, u32> for DoubleStage {
115        async fn process(&self, input: u32) -> Result<u32, StageError> {
116            Ok(input * 2)
117        }
118    }
119
120    #[tokio::test]
121    async fn stage_runner_doubles_values() {
122        let (tx, rx_in) = mpsc::channel::<u32>(16);
123        let (tx_out, mut rx_out) = mpsc::channel::<u32>(16);
124
125        let runner = StageRunner::new(DoubleStage);
126        let handle = runner.run(rx_in, tx_out);
127
128        tx.send(5).await.unwrap();
129        tx.send(7).await.unwrap();
130        drop(tx);
131
132        assert_eq!(rx_out.recv().await, Some(10));
133        assert_eq!(rx_out.recv().await, Some(14));
134        assert_eq!(rx_out.recv().await, None);
135
136        assert!(handle.await.unwrap().is_ok());
137    }
138}