Skip to main content

mytheclipse_queue/
worker.rs

1//! Worker pool for processing queued jobs with retry, backoff, and graceful shutdown.
2
3use std::pin::Pin;
4use std::sync::Arc;
5use std::time::Duration;
6
7use tokio::sync::Semaphore;
8
9use crate::error::JobError;
10use crate::host;
11use crate::job::Job;
12use crate::traits::Queue;
13
14/// A future returned by a job handler.
15pub type JobFuture = Pin<Box<dyn std::future::Future<Output = Result<(), JobError>> + Send>>;
16
17/// A handler for processing a single job.
18pub trait JobHandler: Send + Sync {
19    fn handle(&self, job: Job) -> JobFuture;
20}
21
22impl<F, Fut> JobHandler for F
23where
24    F: Fn(Job) -> Fut + Send + Sync,
25    Fut: std::future::Future<Output = Result<(), JobError>> + Send + 'static,
26{
27    fn handle(&self, job: Job) -> JobFuture {
28        Box::pin((self)(job))
29    }
30}
31
32/// Configuration for the worker pool.
33#[derive(Debug, Clone)]
34pub struct WorkerConfig {
35    /// Maximum concurrent job handlers (default: one per logical CPU).
36    pub concurrency: usize,
37    /// Maximum number of retry attempts (0 = no retries).
38    pub max_retries: u32,
39    /// Base delay for exponential backoff between retries.
40    pub retry_base_delay: Duration,
41    /// Maximum delay cap for retry backoff.
42    pub retry_max_delay: Duration,
43    /// Backoff multiplier.
44    pub retry_factor: f64,
45    /// Visibility timeout for in-progress jobs.
46    pub visibility_timeout: Duration,
47    /// Polling interval when a queue is empty.
48    pub poll_interval: Duration,
49}
50
51impl Default for WorkerConfig {
52    fn default() -> Self {
53        Self {
54            concurrency: host::available_parallelism(),
55            max_retries: 3,
56            retry_base_delay: Duration::from_millis(500),
57            retry_max_delay: Duration::from_secs(10),
58            retry_factor: 2.0,
59            visibility_timeout: Duration::from_secs(30),
60            poll_interval: Duration::from_millis(100),
61        }
62    }
63}
64
65/// A pool of workers consuming jobs from a `Queue`.
66pub struct WorkerPool<Q: Queue + 'static> {
67    queue: Arc<Q>,
68    config: WorkerConfig,
69    semaphore: Arc<Semaphore>,
70}
71
72impl<Q: Queue + 'static> WorkerPool<Q> {
73    /// Creates a new worker pool with the given concurrency.
74    pub fn new(queue: Q, concurrency: usize) -> Self {
75        Self::with_config(
76            queue,
77            WorkerConfig {
78                concurrency,
79                ..Default::default()
80            },
81        )
82    }
83
84    /// Creates a new worker pool with explicit configuration.
85    pub fn with_config(queue: Q, config: WorkerConfig) -> Self {
86        let sem = Arc::new(Semaphore::new(config.concurrency.max(1)));
87        Self {
88            queue: Arc::new(queue),
89            config,
90            semaphore: sem,
91        }
92    }
93
94    /// Starts `concurrency` workers consuming from `topic`.
95    pub fn start<H>(&self, topic: &str, handler: H)
96    where
97        H: JobHandler + 'static,
98    {
99        let queue = Arc::clone(&self.queue);
100        let config = self.config.clone();
101        let semaphore = Arc::clone(&self.semaphore);
102        let handler: Arc<dyn JobHandler> = Arc::new(handler);
103        let topic_owned = topic.to_string();
104
105        for _ in 0..config.concurrency {
106            let q = Arc::clone(&queue);
107            let sem = Arc::clone(&semaphore);
108            let h = Arc::clone(&handler);
109            let cfg = config.clone();
110            let topic_inner = topic_owned.clone();
111
112            tokio::spawn(async move {
113                loop {
114                    // Take a concurrency slot *before* pulling work, then hand
115                    // it to the job task: the permit must live as long as the
116                    // handler runs, otherwise the cap never binds.
117                    let permit = match sem.clone().acquire_owned().await {
118                        Ok(permit) => permit,
119                        Err(_) => break, // pool shutting down
120                    };
121                    match q.dequeue(&topic_inner, cfg.poll_interval).await {
122                        Ok(Some(job)) => {
123                            let q2 = Arc::clone(&q);
124                            let h2 = Arc::clone(&h);
125                            let cfg2 = cfg.clone();
126                            let t2 = topic_inner.clone();
127                            let j2 = job.clone();
128
129                            tokio::spawn(async move {
130                                let _permit = permit;
131                                let fut = h2.handle(j2.clone());
132                                match fut.await {
133                                    Ok(()) => {
134                                        let _ = q2.ack(&j2).await;
135                                    }
136                                    Err(e) => {
137                                        if j2.attempt < cfg2.max_retries {
138                                            let delay = retry_delay(&cfg2, j2.attempt);
139                                            tracing::warn!(
140                                                job = %j2.id,
141                                                attempt = j2.attempt + 1,
142                                                max_retries = cfg2.max_retries,
143                                                delay_ms = delay.as_millis() as u64,
144                                                error = %e,
145                                                "job failed; retrying after backoff"
146                                            );
147                                            tokio::time::sleep(delay).await;
148                                            let _ = q2.nack(&j2, true).await;
149                                        } else {
150                                            let _ = q2.dlq_move(&t2, j2.clone()).await;
151                                            let _ = q2.nack(&j2, false).await;
152                                        }
153                                    }
154                                }
155                            });
156                        }
157                        Ok(None) => {}
158                        Err(e) => {
159                            tracing::error!("queue error: {e}");
160                            tokio::time::sleep(cfg.poll_interval).await;
161                        }
162                    }
163                }
164            });
165        }
166    }
167}
168
169/// Computes the (capped) exponential backoff delay.
170pub fn retry_delay(config: &WorkerConfig, attempt: u32) -> Duration {
171    let exponent = attempt as f64;
172    let computed =
173        config.retry_base_delay.as_millis() as f64 * config.retry_factor.powf(exponent.max(0.0));
174    let capped = computed.min(config.retry_max_delay.as_millis() as f64);
175    Duration::from_millis(capped as u64)
176}
177
178#[cfg(test)]
179mod tests {
180    use super::*;
181
182    #[test]
183    fn retry_delay_bounded() {
184        let config = WorkerConfig::default();
185        let d = retry_delay(&config, 0);
186        assert!(d <= config.retry_max_delay);
187    }
188
189    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
190    async fn worker_respects_configured_concurrency() {
191        use crate::in_memory::InMemoryQueue;
192        use std::sync::atomic::{AtomicUsize, Ordering};
193
194        let queue = InMemoryQueue::new();
195        for i in 0..6u8 {
196            queue.enqueue("t", vec![i]).await.unwrap();
197        }
198
199        let in_flight = Arc::new(AtomicUsize::new(0));
200        let peak = Arc::new(AtomicUsize::new(0));
201        let pool = WorkerPool::new(queue, 2);
202
203        let (live, high) = (Arc::clone(&in_flight), Arc::clone(&peak));
204        pool.start("t", move |_job: Job| {
205            let live = Arc::clone(&live);
206            let high = Arc::clone(&high);
207            async move {
208                let now = live.fetch_add(1, Ordering::SeqCst) + 1;
209                high.fetch_max(now, Ordering::SeqCst);
210                tokio::time::sleep(Duration::from_millis(20)).await;
211                live.fetch_sub(1, Ordering::SeqCst);
212                Ok(())
213            }
214        });
215
216        tokio::time::sleep(Duration::from_millis(400)).await;
217        let peak = peak.load(Ordering::SeqCst);
218        assert!(peak <= 2, "handlers ran {peak} at a time, cap was 2");
219        assert_eq!(in_flight.load(Ordering::SeqCst), 0);
220    }
221}