Skip to main content

mytheclipse_queue/
worker.rs

1//! Worker pool for processing queued jobs with retry, backoff, and graceful shutdown.
2
3use std::pin::Pin;
4use std::sync::Arc;
5use std::time::Duration;
6
7use tokio::sync::Semaphore;
8
9use crate::error::JobError;
10use crate::job::Job;
11use crate::traits::Queue;
12
13/// A future returned by a job handler.
14pub type JobFuture = Pin<Box<dyn std::future::Future<Output = Result<(), JobError>> + Send>>;
15
16/// A handler for processing a single job.
17pub trait JobHandler: Send + Sync {
18    fn handle(&self, job: Job) -> JobFuture;
19}
20
21impl<F, Fut> JobHandler for F
22where
23    F: Fn(Job) -> Fut + Send + Sync,
24    Fut: std::future::Future<Output = Result<(), JobError>> + Send + 'static,
25{
26    fn handle(&self, job: Job) -> JobFuture {
27        Box::pin((self)(job))
28    }
29}
30
31/// Configuration for the worker pool.
32#[derive(Debug, Clone)]
33pub struct WorkerConfig {
34    /// Maximum concurrent job handlers.
35    pub concurrency: usize,
36    /// Maximum number of retry attempts (0 = no retries).
37    pub max_retries: u32,
38    /// Base delay for exponential backoff between retries.
39    pub retry_base_delay: Duration,
40    /// Maximum delay cap for retry backoff.
41    pub retry_max_delay: Duration,
42    /// Backoff multiplier.
43    pub retry_factor: f64,
44    /// Visibility timeout for in-progress jobs.
45    pub visibility_timeout: Duration,
46    /// Polling interval when a queue is empty.
47    pub poll_interval: Duration,
48}
49
50impl Default for WorkerConfig {
51    fn default() -> Self {
52        Self {
53            concurrency: 4,
54            max_retries: 3,
55            retry_base_delay: Duration::from_millis(500),
56            retry_max_delay: Duration::from_secs(10),
57            retry_factor: 2.0,
58            visibility_timeout: Duration::from_secs(30),
59            poll_interval: Duration::from_millis(100),
60        }
61    }
62}
63
64/// A pool of workers consuming jobs from a `Queue`.
65pub struct WorkerPool<Q: Queue + 'static> {
66    queue: Arc<Q>,
67    config: WorkerConfig,
68    semaphore: Arc<Semaphore>,
69}
70
71impl<Q: Queue + 'static> WorkerPool<Q> {
72    /// Creates a new worker pool with the given concurrency.
73    pub fn new(queue: Q, concurrency: usize) -> Self {
74        Self::with_config(
75            queue,
76            WorkerConfig {
77                concurrency,
78                ..Default::default()
79            },
80        )
81    }
82
83    /// Creates a new worker pool with explicit configuration.
84    pub fn with_config(queue: Q, config: WorkerConfig) -> Self {
85        let sem = Arc::new(Semaphore::new(config.concurrency.max(1)));
86        Self {
87            queue: Arc::new(queue),
88            config,
89            semaphore: sem,
90        }
91    }
92
93    /// Starts `concurrency` workers consuming from `topic`.
94    pub fn start<H>(&self, topic: &str, handler: H)
95    where
96        H: JobHandler + 'static,
97    {
98        let queue = Arc::clone(&self.queue);
99        let config = self.config.clone();
100        let semaphore = Arc::clone(&self.semaphore);
101        let handler: Arc<dyn JobHandler> = Arc::new(handler);
102        let topic_owned = topic.to_string();
103
104        for _ in 0..config.concurrency {
105            let q = Arc::clone(&queue);
106            let sem = Arc::clone(&semaphore);
107            let h = Arc::clone(&handler);
108            let cfg = config.clone();
109            let topic_inner = topic_owned.clone();
110
111            tokio::spawn(async move {
112                loop {
113                    // Take a concurrency slot *before* pulling work, then hand
114                    // it to the job task: the permit must live as long as the
115                    // handler runs, otherwise the cap never binds.
116                    let permit = match sem.clone().acquire_owned().await {
117                        Ok(permit) => permit,
118                        Err(_) => break, // pool shutting down
119                    };
120                    match q.dequeue(&topic_inner, cfg.poll_interval).await {
121                        Ok(Some(job)) => {
122                            let q2 = Arc::clone(&q);
123                            let h2 = Arc::clone(&h);
124                            let cfg2 = cfg.clone();
125                            let t2 = topic_inner.clone();
126                            let j2 = job.clone();
127
128                            tokio::spawn(async move {
129                                let _permit = permit;
130                                let fut = h2.handle(j2.clone());
131                                match fut.await {
132                                    Ok(()) => {
133                                        let _ = q2.ack(&j2).await;
134                                    }
135                                    Err(e) => {
136                                        if j2.attempt < cfg2.max_retries {
137                                            let delay = retry_delay(&cfg2, j2.attempt);
138                                            tracing::warn!(
139                                                job = %j2.id,
140                                                attempt = j2.attempt + 1,
141                                                max_retries = cfg2.max_retries,
142                                                delay_ms = delay.as_millis() as u64,
143                                                error = %e,
144                                                "job failed; retrying after backoff"
145                                            );
146                                            tokio::time::sleep(delay).await;
147                                            let _ = q2.nack(&j2, true).await;
148                                        } else {
149                                            let _ = q2.dlq_move(&t2, j2.clone()).await;
150                                            let _ = q2.nack(&j2, false).await;
151                                        }
152                                    }
153                                }
154                            });
155                        }
156                        Ok(None) => {}
157                        Err(e) => {
158                            tracing::error!("queue error: {e}");
159                            tokio::time::sleep(cfg.poll_interval).await;
160                        }
161                    }
162                }
163            });
164        }
165    }
166}
167
168/// Computes the (capped) exponential backoff delay.
169pub fn retry_delay(config: &WorkerConfig, attempt: u32) -> Duration {
170    let exponent = attempt as f64;
171    let computed =
172        config.retry_base_delay.as_millis() as f64 * config.retry_factor.powf(exponent.max(0.0));
173    let capped = computed.min(config.retry_max_delay.as_millis() as f64);
174    Duration::from_millis(capped as u64)
175}
176
177#[cfg(test)]
178mod tests {
179    use super::*;
180
181    #[test]
182    fn retry_delay_bounded() {
183        let config = WorkerConfig::default();
184        let d = retry_delay(&config, 0);
185        assert!(d <= config.retry_max_delay);
186    }
187
188    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
189    async fn worker_respects_configured_concurrency() {
190        use crate::in_memory::InMemoryQueue;
191        use std::sync::atomic::{AtomicUsize, Ordering};
192
193        let queue = InMemoryQueue::new();
194        for i in 0..6u8 {
195            queue.enqueue("t", vec![i]).await.unwrap();
196        }
197
198        let in_flight = Arc::new(AtomicUsize::new(0));
199        let peak = Arc::new(AtomicUsize::new(0));
200        let pool = WorkerPool::new(queue, 2);
201
202        let (live, high) = (Arc::clone(&in_flight), Arc::clone(&peak));
203        pool.start("t", move |_job: Job| {
204            let live = Arc::clone(&live);
205            let high = Arc::clone(&high);
206            async move {
207                let now = live.fetch_add(1, Ordering::SeqCst) + 1;
208                high.fetch_max(now, Ordering::SeqCst);
209                tokio::time::sleep(Duration::from_millis(20)).await;
210                live.fetch_sub(1, Ordering::SeqCst);
211                Ok(())
212            }
213        });
214
215        tokio::time::sleep(Duration::from_millis(400)).await;
216        let peak = peak.load(Ordering::SeqCst);
217        assert!(peak <= 2, "handlers ran {peak} at a time, cap was 2");
218        assert_eq!(in_flight.load(Ordering::SeqCst), 0);
219    }
220}