mytheclipse_queue/
worker.rs1use std::pin::Pin;
4use std::sync::Arc;
5use std::time::Duration;
6
7use tokio::sync::Semaphore;
8
9use crate::error::JobError;
10use crate::job::Job;
11use crate::traits::Queue;
12
13pub type JobFuture = Pin<Box<dyn std::future::Future<Output = Result<(), JobError>> + Send>>;
15
16pub trait JobHandler: Send + Sync {
18 fn handle(&self, job: Job) -> JobFuture;
19}
20
21impl<F, Fut> JobHandler for F
22where
23 F: Fn(Job) -> Fut + Send + Sync,
24 Fut: std::future::Future<Output = Result<(), JobError>> + Send + 'static,
25{
26 fn handle(&self, job: Job) -> JobFuture {
27 Box::pin((self)(job))
28 }
29}
30
31#[derive(Debug, Clone)]
33pub struct WorkerConfig {
34 pub concurrency: usize,
36 pub max_retries: u32,
38 pub retry_base_delay: Duration,
40 pub retry_max_delay: Duration,
42 pub retry_factor: f64,
44 pub visibility_timeout: Duration,
46 pub poll_interval: Duration,
48}
49
50impl Default for WorkerConfig {
51 fn default() -> Self {
52 Self {
53 concurrency: 4,
54 max_retries: 3,
55 retry_base_delay: Duration::from_millis(500),
56 retry_max_delay: Duration::from_secs(10),
57 retry_factor: 2.0,
58 visibility_timeout: Duration::from_secs(30),
59 poll_interval: Duration::from_millis(100),
60 }
61 }
62}
63
64pub struct WorkerPool<Q: Queue + 'static> {
66 queue: Arc<Q>,
67 config: WorkerConfig,
68 semaphore: Arc<Semaphore>,
69}
70
71impl<Q: Queue + 'static> WorkerPool<Q> {
72 pub fn new(queue: Q, concurrency: usize) -> Self {
74 Self::with_config(
75 queue,
76 WorkerConfig {
77 concurrency,
78 ..Default::default()
79 },
80 )
81 }
82
83 pub fn with_config(queue: Q, config: WorkerConfig) -> Self {
85 let sem = Arc::new(Semaphore::new(config.concurrency.max(1)));
86 Self {
87 queue: Arc::new(queue),
88 config,
89 semaphore: sem,
90 }
91 }
92
93 pub fn start<H>(&self, topic: &str, handler: H)
95 where
96 H: JobHandler + 'static,
97 {
98 let queue = Arc::clone(&self.queue);
99 let config = self.config.clone();
100 let semaphore = Arc::clone(&self.semaphore);
101 let handler: Arc<dyn JobHandler> = Arc::new(handler);
102 let topic_owned = topic.to_string();
103
104 for _ in 0..config.concurrency {
105 let q = Arc::clone(&queue);
106 let sem = Arc::clone(&semaphore);
107 let h = Arc::clone(&handler);
108 let cfg = config.clone();
109 let topic_inner = topic_owned.clone();
110
111 tokio::spawn(async move {
112 loop {
113 let permit = match sem.clone().acquire_owned().await {
117 Ok(permit) => permit,
118 Err(_) => break, };
120 match q.dequeue(&topic_inner, cfg.poll_interval).await {
121 Ok(Some(job)) => {
122 let q2 = Arc::clone(&q);
123 let h2 = Arc::clone(&h);
124 let cfg2 = cfg.clone();
125 let t2 = topic_inner.clone();
126 let j2 = job.clone();
127
128 tokio::spawn(async move {
129 let _permit = permit;
130 let fut = h2.handle(j2.clone());
131 match fut.await {
132 Ok(()) => {
133 let _ = q2.ack(&j2).await;
134 }
135 Err(e) => {
136 if j2.attempt < cfg2.max_retries {
137 let delay = retry_delay(&cfg2, j2.attempt);
138 tracing::warn!(
139 job = %j2.id,
140 attempt = j2.attempt + 1,
141 max_retries = cfg2.max_retries,
142 delay_ms = delay.as_millis() as u64,
143 error = %e,
144 "job failed; retrying after backoff"
145 );
146 tokio::time::sleep(delay).await;
147 let _ = q2.nack(&j2, true).await;
148 } else {
149 let _ = q2.dlq_move(&t2, j2.clone()).await;
150 let _ = q2.nack(&j2, false).await;
151 }
152 }
153 }
154 });
155 }
156 Ok(None) => {}
157 Err(e) => {
158 tracing::error!("queue error: {e}");
159 tokio::time::sleep(cfg.poll_interval).await;
160 }
161 }
162 }
163 });
164 }
165 }
166}
167
168pub fn retry_delay(config: &WorkerConfig, attempt: u32) -> Duration {
170 let exponent = attempt as f64;
171 let computed =
172 config.retry_base_delay.as_millis() as f64 * config.retry_factor.powf(exponent.max(0.0));
173 let capped = computed.min(config.retry_max_delay.as_millis() as f64);
174 Duration::from_millis(capped as u64)
175}
176
177#[cfg(test)]
178mod tests {
179 use super::*;
180
181 #[test]
182 fn retry_delay_bounded() {
183 let config = WorkerConfig::default();
184 let d = retry_delay(&config, 0);
185 assert!(d <= config.retry_max_delay);
186 }
187
188 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
189 async fn worker_respects_configured_concurrency() {
190 use crate::in_memory::InMemoryQueue;
191 use std::sync::atomic::{AtomicUsize, Ordering};
192
193 let queue = InMemoryQueue::new();
194 for i in 0..6u8 {
195 queue.enqueue("t", vec![i]).await.unwrap();
196 }
197
198 let in_flight = Arc::new(AtomicUsize::new(0));
199 let peak = Arc::new(AtomicUsize::new(0));
200 let pool = WorkerPool::new(queue, 2);
201
202 let (live, high) = (Arc::clone(&in_flight), Arc::clone(&peak));
203 pool.start("t", move |_job: Job| {
204 let live = Arc::clone(&live);
205 let high = Arc::clone(&high);
206 async move {
207 let now = live.fetch_add(1, Ordering::SeqCst) + 1;
208 high.fetch_max(now, Ordering::SeqCst);
209 tokio::time::sleep(Duration::from_millis(20)).await;
210 live.fetch_sub(1, Ordering::SeqCst);
211 Ok(())
212 }
213 });
214
215 tokio::time::sleep(Duration::from_millis(400)).await;
216 let peak = peak.load(Ordering::SeqCst);
217 assert!(peak <= 2, "handlers ran {peak} at a time, cap was 2");
218 assert_eq!(in_flight.load(Ordering::SeqCst), 0);
219 }
220}