mytheclipse_queue/
worker.rs1use std::pin::Pin;
4use std::sync::Arc;
5use std::time::Duration;
6
7use tokio::sync::Semaphore;
8
9use crate::error::JobError;
10use crate::host;
11use crate::job::Job;
12use crate::traits::Queue;
13
14pub type JobFuture = Pin<Box<dyn std::future::Future<Output = Result<(), JobError>> + Send>>;
16
17pub trait JobHandler: Send + Sync {
19 fn handle(&self, job: Job) -> JobFuture;
20}
21
22impl<F, Fut> JobHandler for F
23where
24 F: Fn(Job) -> Fut + Send + Sync,
25 Fut: std::future::Future<Output = Result<(), JobError>> + Send + 'static,
26{
27 fn handle(&self, job: Job) -> JobFuture {
28 Box::pin((self)(job))
29 }
30}
31
32#[derive(Debug, Clone)]
34pub struct WorkerConfig {
35 pub concurrency: usize,
37 pub max_retries: u32,
39 pub retry_base_delay: Duration,
41 pub retry_max_delay: Duration,
43 pub retry_factor: f64,
45 pub visibility_timeout: Duration,
47 pub poll_interval: Duration,
49}
50
51impl Default for WorkerConfig {
52 fn default() -> Self {
53 Self {
54 concurrency: host::available_parallelism(),
55 max_retries: 3,
56 retry_base_delay: Duration::from_millis(500),
57 retry_max_delay: Duration::from_secs(10),
58 retry_factor: 2.0,
59 visibility_timeout: Duration::from_secs(30),
60 poll_interval: Duration::from_millis(100),
61 }
62 }
63}
64
65pub struct WorkerPool<Q: Queue + 'static> {
67 queue: Arc<Q>,
68 config: WorkerConfig,
69 semaphore: Arc<Semaphore>,
70}
71
72impl<Q: Queue + 'static> WorkerPool<Q> {
73 pub fn new(queue: Q, concurrency: usize) -> Self {
75 Self::with_config(
76 queue,
77 WorkerConfig {
78 concurrency,
79 ..Default::default()
80 },
81 )
82 }
83
84 pub fn with_config(queue: Q, config: WorkerConfig) -> Self {
86 let sem = Arc::new(Semaphore::new(config.concurrency.max(1)));
87 Self {
88 queue: Arc::new(queue),
89 config,
90 semaphore: sem,
91 }
92 }
93
94 pub fn start<H>(&self, topic: &str, handler: H)
96 where
97 H: JobHandler + 'static,
98 {
99 let queue = Arc::clone(&self.queue);
100 let config = self.config.clone();
101 let semaphore = Arc::clone(&self.semaphore);
102 let handler: Arc<dyn JobHandler> = Arc::new(handler);
103 let topic_owned = topic.to_string();
104
105 for _ in 0..config.concurrency {
106 let q = Arc::clone(&queue);
107 let sem = Arc::clone(&semaphore);
108 let h = Arc::clone(&handler);
109 let cfg = config.clone();
110 let topic_inner = topic_owned.clone();
111
112 tokio::spawn(async move {
113 loop {
114 let permit = match sem.clone().acquire_owned().await {
118 Ok(permit) => permit,
119 Err(_) => break, };
121 match q.dequeue(&topic_inner, cfg.poll_interval).await {
122 Ok(Some(job)) => {
123 let q2 = Arc::clone(&q);
124 let h2 = Arc::clone(&h);
125 let cfg2 = cfg.clone();
126 let t2 = topic_inner.clone();
127 let j2 = job.clone();
128
129 tokio::spawn(async move {
130 let _permit = permit;
131 let fut = h2.handle(j2.clone());
132 match fut.await {
133 Ok(()) => {
134 let _ = q2.ack(&j2).await;
135 }
136 Err(e) => {
137 if j2.attempt < cfg2.max_retries {
138 let delay = retry_delay(&cfg2, j2.attempt);
139 tracing::warn!(
140 job = %j2.id,
141 attempt = j2.attempt + 1,
142 max_retries = cfg2.max_retries,
143 delay_ms = delay.as_millis() as u64,
144 error = %e,
145 "job failed; retrying after backoff"
146 );
147 tokio::time::sleep(delay).await;
148 let _ = q2.nack(&j2, true).await;
149 } else {
150 let _ = q2.dlq_move(&t2, j2.clone()).await;
151 let _ = q2.nack(&j2, false).await;
152 }
153 }
154 }
155 });
156 }
157 Ok(None) => {}
158 Err(e) => {
159 tracing::error!("queue error: {e}");
160 tokio::time::sleep(cfg.poll_interval).await;
161 }
162 }
163 }
164 });
165 }
166 }
167}
168
169pub fn retry_delay(config: &WorkerConfig, attempt: u32) -> Duration {
171 let exponent = attempt as f64;
172 let computed =
173 config.retry_base_delay.as_millis() as f64 * config.retry_factor.powf(exponent.max(0.0));
174 let capped = computed.min(config.retry_max_delay.as_millis() as f64);
175 Duration::from_millis(capped as u64)
176}
177
178#[cfg(test)]
179mod tests {
180 use super::*;
181
182 #[test]
183 fn retry_delay_bounded() {
184 let config = WorkerConfig::default();
185 let d = retry_delay(&config, 0);
186 assert!(d <= config.retry_max_delay);
187 }
188
189 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
190 async fn worker_respects_configured_concurrency() {
191 use crate::in_memory::InMemoryQueue;
192 use std::sync::atomic::{AtomicUsize, Ordering};
193
194 let queue = InMemoryQueue::new();
195 for i in 0..6u8 {
196 queue.enqueue("t", vec![i]).await.unwrap();
197 }
198
199 let in_flight = Arc::new(AtomicUsize::new(0));
200 let peak = Arc::new(AtomicUsize::new(0));
201 let pool = WorkerPool::new(queue, 2);
202
203 let (live, high) = (Arc::clone(&in_flight), Arc::clone(&peak));
204 pool.start("t", move |_job: Job| {
205 let live = Arc::clone(&live);
206 let high = Arc::clone(&high);
207 async move {
208 let now = live.fetch_add(1, Ordering::SeqCst) + 1;
209 high.fetch_max(now, Ordering::SeqCst);
210 tokio::time::sleep(Duration::from_millis(20)).await;
211 live.fetch_sub(1, Ordering::SeqCst);
212 Ok(())
213 }
214 });
215
216 tokio::time::sleep(Duration::from_millis(400)).await;
217 let peak = peak.load(Ordering::SeqCst);
218 assert!(peak <= 2, "handlers ran {peak} at a time, cap was 2");
219 assert_eq!(in_flight.load(Ordering::SeqCst), 0);
220 }
221}