Skip to main content

mytheclipse_queue/
worker.rs

1//! Worker pool for processing queued jobs with retry, backoff, and graceful shutdown.
2
3use std::pin::Pin;
4use std::sync::Arc;
5use std::time::Duration;
6
7use tokio::sync::Semaphore;
8
9use crate::error::JobError;
10use crate::host;
11use crate::job::Job;
12use crate::traits::Queue;
13
14/// A future returned by a job handler.
15pub type JobFuture = Pin<Box<dyn std::future::Future<Output = Result<(), JobError>> + Send>>;
16
17/// A handler for processing a single job.
18pub trait JobHandler: Send + Sync {
19    /// Handles one job, returning a future the pool awaits before acking.
20    fn handle(&self, job: Job) -> JobFuture;
21}
22
23impl<F, Fut> JobHandler for F
24where
25    F: Fn(Job) -> Fut + Send + Sync,
26    Fut: std::future::Future<Output = Result<(), JobError>> + Send + 'static,
27{
28    fn handle(&self, job: Job) -> JobFuture {
29        Box::pin((self)(job))
30    }
31}
32
33/// Configuration for the worker pool.
34#[derive(Debug, Clone)]
35pub struct WorkerConfig {
36    /// Maximum concurrent job handlers (default: one per logical CPU).
37    pub concurrency: usize,
38    /// Maximum number of retry attempts (0 = no retries).
39    pub max_retries: u32,
40    /// Base delay for exponential backoff between retries.
41    pub retry_base_delay: Duration,
42    /// Maximum delay cap for retry backoff.
43    pub retry_max_delay: Duration,
44    /// Backoff multiplier.
45    pub retry_factor: f64,
46    /// Visibility timeout for in-progress jobs.
47    pub visibility_timeout: Duration,
48    /// Polling interval when a queue is empty.
49    pub poll_interval: Duration,
50}
51
52impl Default for WorkerConfig {
53    fn default() -> Self {
54        Self {
55            concurrency: host::available_parallelism(),
56            max_retries: 3,
57            retry_base_delay: Duration::from_millis(500),
58            retry_max_delay: Duration::from_secs(10),
59            retry_factor: 2.0,
60            visibility_timeout: Duration::from_secs(30),
61            poll_interval: Duration::from_millis(100),
62        }
63    }
64}
65
66/// A pool of workers consuming jobs from a `Queue`.
67pub struct WorkerPool<Q: Queue + 'static> {
68    queue: Arc<Q>,
69    config: WorkerConfig,
70    semaphore: Arc<Semaphore>,
71}
72
73impl<Q: Queue + 'static> WorkerPool<Q> {
74    /// Creates a new worker pool with the given concurrency.
75    pub fn new(queue: Q, concurrency: usize) -> Self {
76        Self::with_config(
77            queue,
78            WorkerConfig {
79                concurrency,
80                ..Default::default()
81            },
82        )
83    }
84
85    /// Creates a new worker pool with explicit configuration.
86    pub fn with_config(queue: Q, config: WorkerConfig) -> Self {
87        let sem = Arc::new(Semaphore::new(config.concurrency.max(1)));
88        Self {
89            queue: Arc::new(queue),
90            config,
91            semaphore: sem,
92        }
93    }
94
95    /// Starts `concurrency` workers consuming from `topic`.
96    pub fn start<H>(&self, topic: &str, handler: H)
97    where
98        H: JobHandler + 'static,
99    {
100        let queue = Arc::clone(&self.queue);
101        let config = self.config.clone();
102        let semaphore = Arc::clone(&self.semaphore);
103        let handler: Arc<dyn JobHandler> = Arc::new(handler);
104        let topic_owned = topic.to_string();
105
106        for _ in 0..config.concurrency {
107            let q = Arc::clone(&queue);
108            let sem = Arc::clone(&semaphore);
109            let h = Arc::clone(&handler);
110            let cfg = config.clone();
111            let topic_inner = topic_owned.clone();
112
113            tokio::spawn(async move {
114                loop {
115                    // Take a concurrency slot *before* pulling work, then hand
116                    // it to the job task: the permit must live as long as the
117                    // handler runs, otherwise the cap never binds.
118                    let permit = match sem.clone().acquire_owned().await {
119                        Ok(permit) => permit,
120                        Err(_) => break, // pool shutting down
121                    };
122                    match q.dequeue(&topic_inner, cfg.poll_interval).await {
123                        Ok(Some(job)) => {
124                            let q2 = Arc::clone(&q);
125                            let h2 = Arc::clone(&h);
126                            let cfg2 = cfg.clone();
127                            let t2 = topic_inner.clone();
128                            let j2 = job.clone();
129
130                            tokio::spawn(async move {
131                                let _permit = permit;
132                                let fut = h2.handle(j2.clone());
133                                match fut.await {
134                                    Ok(()) => {
135                                        let _ = q2.ack(&j2).await;
136                                    }
137                                    Err(e) => {
138                                        if j2.attempt < cfg2.max_retries {
139                                            let delay = retry_delay(&cfg2, j2.attempt);
140                                            tracing::warn!(
141                                                job = %j2.id,
142                                                attempt = j2.attempt + 1,
143                                                max_retries = cfg2.max_retries,
144                                                delay_ms = delay.as_millis() as u64,
145                                                error = %e,
146                                                "job failed; retrying after backoff"
147                                            );
148                                            tokio::time::sleep(delay).await;
149                                            let _ = q2.nack(&j2, true).await;
150                                        } else {
151                                            let _ = q2.dlq_move(&t2, j2.clone()).await;
152                                            let _ = q2.nack(&j2, false).await;
153                                        }
154                                    }
155                                }
156                            });
157                        }
158                        Ok(None) => {}
159                        Err(e) => {
160                            tracing::error!("queue error: {e}");
161                            tokio::time::sleep(cfg.poll_interval).await;
162                        }
163                    }
164                }
165            });
166        }
167    }
168}
169
170/// Computes the (capped) exponential backoff delay.
171pub fn retry_delay(config: &WorkerConfig, attempt: u32) -> Duration {
172    let exponent = attempt as f64;
173    let computed =
174        config.retry_base_delay.as_millis() as f64 * config.retry_factor.powf(exponent.max(0.0));
175    let capped = computed.min(config.retry_max_delay.as_millis() as f64);
176    Duration::from_millis(capped as u64)
177}
178
179#[cfg(test)]
180mod tests {
181    use super::*;
182
183    #[test]
184    fn retry_delay_bounded() {
185        let config = WorkerConfig::default();
186        let d = retry_delay(&config, 0);
187        assert!(d <= config.retry_max_delay);
188    }
189
190    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
191    async fn worker_respects_configured_concurrency() {
192        use crate::in_memory::InMemoryQueue;
193        use std::sync::atomic::{AtomicUsize, Ordering};
194
195        let queue = InMemoryQueue::new();
196        for i in 0..6u8 {
197            queue.enqueue("t", vec![i]).await.unwrap();
198        }
199
200        let in_flight = Arc::new(AtomicUsize::new(0));
201        let peak = Arc::new(AtomicUsize::new(0));
202        let pool = WorkerPool::new(queue, 2);
203
204        let (live, high) = (Arc::clone(&in_flight), Arc::clone(&peak));
205        pool.start("t", move |_job: Job| {
206            let live = Arc::clone(&live);
207            let high = Arc::clone(&high);
208            async move {
209                let now = live.fetch_add(1, Ordering::SeqCst) + 1;
210                high.fetch_max(now, Ordering::SeqCst);
211                tokio::time::sleep(Duration::from_millis(20)).await;
212                live.fetch_sub(1, Ordering::SeqCst);
213                Ok(())
214            }
215        });
216
217        tokio::time::sleep(Duration::from_millis(400)).await;
218        let peak = peak.load(Ordering::SeqCst);
219        assert!(peak <= 2, "handlers ran {peak} at a time, cap was 2");
220        assert_eq!(in_flight.load(Ordering::SeqCst), 0);
221    }
222}