mytheclipse_queue/
worker.rs1use std::pin::Pin;
4use std::sync::Arc;
5use std::time::Duration;
6
7use tokio::sync::Semaphore;
8
9use crate::error::JobError;
10use crate::host;
11use crate::job::Job;
12use crate::traits::Queue;
13
14pub type JobFuture = Pin<Box<dyn std::future::Future<Output = Result<(), JobError>> + Send>>;
16
17pub trait JobHandler: Send + Sync {
19 fn handle(&self, job: Job) -> JobFuture;
21}
22
23impl<F, Fut> JobHandler for F
24where
25 F: Fn(Job) -> Fut + Send + Sync,
26 Fut: std::future::Future<Output = Result<(), JobError>> + Send + 'static,
27{
28 fn handle(&self, job: Job) -> JobFuture {
29 Box::pin((self)(job))
30 }
31}
32
33#[derive(Debug, Clone)]
35pub struct WorkerConfig {
36 pub concurrency: usize,
38 pub max_retries: u32,
40 pub retry_base_delay: Duration,
42 pub retry_max_delay: Duration,
44 pub retry_factor: f64,
46 pub visibility_timeout: Duration,
48 pub poll_interval: Duration,
50}
51
52impl Default for WorkerConfig {
53 fn default() -> Self {
54 Self {
55 concurrency: host::available_parallelism(),
56 max_retries: 3,
57 retry_base_delay: Duration::from_millis(500),
58 retry_max_delay: Duration::from_secs(10),
59 retry_factor: 2.0,
60 visibility_timeout: Duration::from_secs(30),
61 poll_interval: Duration::from_millis(100),
62 }
63 }
64}
65
66pub struct WorkerPool<Q: Queue + 'static> {
68 queue: Arc<Q>,
69 config: WorkerConfig,
70 semaphore: Arc<Semaphore>,
71}
72
73impl<Q: Queue + 'static> WorkerPool<Q> {
74 pub fn new(queue: Q, concurrency: usize) -> Self {
76 Self::with_config(
77 queue,
78 WorkerConfig {
79 concurrency,
80 ..Default::default()
81 },
82 )
83 }
84
85 pub fn with_config(queue: Q, config: WorkerConfig) -> Self {
87 let sem = Arc::new(Semaphore::new(config.concurrency.max(1)));
88 Self {
89 queue: Arc::new(queue),
90 config,
91 semaphore: sem,
92 }
93 }
94
95 pub fn start<H>(&self, topic: &str, handler: H)
97 where
98 H: JobHandler + 'static,
99 {
100 let queue = Arc::clone(&self.queue);
101 let config = self.config.clone();
102 let semaphore = Arc::clone(&self.semaphore);
103 let handler: Arc<dyn JobHandler> = Arc::new(handler);
104 let topic_owned = topic.to_string();
105
106 for _ in 0..config.concurrency {
107 let q = Arc::clone(&queue);
108 let sem = Arc::clone(&semaphore);
109 let h = Arc::clone(&handler);
110 let cfg = config.clone();
111 let topic_inner = topic_owned.clone();
112
113 tokio::spawn(async move {
114 loop {
115 let permit = match sem.clone().acquire_owned().await {
119 Ok(permit) => permit,
120 Err(_) => break, };
122 match q.dequeue(&topic_inner, cfg.poll_interval).await {
123 Ok(Some(job)) => {
124 let q2 = Arc::clone(&q);
125 let h2 = Arc::clone(&h);
126 let cfg2 = cfg.clone();
127 let t2 = topic_inner.clone();
128 let j2 = job.clone();
129
130 tokio::spawn(async move {
131 let _permit = permit;
132 let fut = h2.handle(j2.clone());
133 match fut.await {
134 Ok(()) => {
135 let _ = q2.ack(&j2).await;
136 }
137 Err(e) => {
138 if j2.attempt < cfg2.max_retries {
139 let delay = retry_delay(&cfg2, j2.attempt);
140 tracing::warn!(
141 job = %j2.id,
142 attempt = j2.attempt + 1,
143 max_retries = cfg2.max_retries,
144 delay_ms = delay.as_millis() as u64,
145 error = %e,
146 "job failed; retrying after backoff"
147 );
148 tokio::time::sleep(delay).await;
149 let _ = q2.nack(&j2, true).await;
150 } else {
151 let _ = q2.dlq_move(&t2, j2.clone()).await;
152 let _ = q2.nack(&j2, false).await;
153 }
154 }
155 }
156 });
157 }
158 Ok(None) => {}
159 Err(e) => {
160 tracing::error!("queue error: {e}");
161 tokio::time::sleep(cfg.poll_interval).await;
162 }
163 }
164 }
165 });
166 }
167 }
168}
169
170pub fn retry_delay(config: &WorkerConfig, attempt: u32) -> Duration {
172 let exponent = attempt as f64;
173 let computed =
174 config.retry_base_delay.as_millis() as f64 * config.retry_factor.powf(exponent.max(0.0));
175 let capped = computed.min(config.retry_max_delay.as_millis() as f64);
176 Duration::from_millis(capped as u64)
177}
178
179#[cfg(test)]
180mod tests {
181 use super::*;
182
183 #[test]
184 fn retry_delay_bounded() {
185 let config = WorkerConfig::default();
186 let d = retry_delay(&config, 0);
187 assert!(d <= config.retry_max_delay);
188 }
189
190 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
191 async fn worker_respects_configured_concurrency() {
192 use crate::in_memory::InMemoryQueue;
193 use std::sync::atomic::{AtomicUsize, Ordering};
194
195 let queue = InMemoryQueue::new();
196 for i in 0..6u8 {
197 queue.enqueue("t", vec![i]).await.unwrap();
198 }
199
200 let in_flight = Arc::new(AtomicUsize::new(0));
201 let peak = Arc::new(AtomicUsize::new(0));
202 let pool = WorkerPool::new(queue, 2);
203
204 let (live, high) = (Arc::clone(&in_flight), Arc::clone(&peak));
205 pool.start("t", move |_job: Job| {
206 let live = Arc::clone(&live);
207 let high = Arc::clone(&high);
208 async move {
209 let now = live.fetch_add(1, Ordering::SeqCst) + 1;
210 high.fetch_max(now, Ordering::SeqCst);
211 tokio::time::sleep(Duration::from_millis(20)).await;
212 live.fetch_sub(1, Ordering::SeqCst);
213 Ok(())
214 }
215 });
216
217 tokio::time::sleep(Duration::from_millis(400)).await;
218 let peak = peak.load(Ordering::SeqCst);
219 assert!(peak <= 2, "handlers ran {peak} at a time, cap was 2");
220 assert_eq!(in_flight.load(Ordering::SeqCst), 0);
221 }
222}