1use std::time::Duration;
42use chrono::{DateTime, Utc};
43use async_trait::async_trait;
44use redis::{
45 AsyncCommands, Client,
46 aio::ConnectionManager,
47};
48use uuid::Uuid;
49
50use asyncq::{
51 backend::Backend,
52 error::{Error, Result},
53 job::{JobId, JobRecord, QueueStats},
54};
55
56fn key_pending(queue: &str) -> String { format!("asyncq:q:{queue}:pending") }
59fn key_delayed(queue: &str) -> String { format!("asyncq:q:{queue}:delayed") }
60fn key_dead(queue: &str) -> String { format!("asyncq:q:{queue}:dead") }
61fn key_completed(queue: &str) -> String { format!("asyncq:q:{queue}:completed") }
62fn key_job(id: JobId) -> String { format!("asyncq:jobs:{id}") }
63const KEY_PROCESSING: &str = "asyncq:processing";
64
65#[derive(Clone)]
72pub struct RedisBackend {
73 conn: ConnectionManager,
74}
75
76impl RedisBackend {
77 pub async fn new(url: &str) -> Result<Self> {
80 let client = Client::open(url)
81 .map_err(|e| Error::Backend(e.to_string()))?;
82 let conn = ConnectionManager::new(client)
83 .await
84 .map_err(|e| Error::Backend(e.to_string()))?;
85 Ok(Self { conn })
86 }
87
88 async fn save_job(&self, record: &JobRecord) -> Result<()> {
91 let mut conn = self.conn.clone();
92 let key = key_job(record.id);
93 let payload_b64 = base64_encode(&record.payload);
94 redis::cmd("HSET")
95 .arg(&key)
96 .arg("id") .arg(record.id.to_string())
97 .arg("kind") .arg(&record.kind)
98 .arg("queue") .arg(&record.queue)
99 .arg("payload") .arg(&payload_b64)
100 .arg("attempt") .arg(record.attempt)
101 .arg("max_attempts") .arg(record.max_attempts)
102 .arg("scheduled_at") .arg(record.scheduled_at.timestamp())
103 .arg("created_at") .arg(record.created_at.timestamp())
104 .arg("last_error") .arg(record.last_error.as_deref().unwrap_or(""))
105 .query_async::<()>(&mut conn)
106 .await
107 .map_err(|e| Error::Backend(e.to_string()))
108 }
109
110 async fn load_job(&self, id: JobId) -> Result<Option<JobRecord>> {
111 let mut conn = self.conn.clone();
112 let key = key_job(id);
113 let fields: Vec<Option<String>> = redis::cmd("HMGET")
114 .arg(&key)
115 .arg("id").arg("kind").arg("queue").arg("payload")
116 .arg("attempt").arg("max_attempts").arg("scheduled_at")
117 .arg("created_at").arg("last_error")
118 .query_async(&mut conn)
119 .await
120 .map_err(|e| Error::Backend(e.to_string()))?;
121
122 let get = |i: usize| fields[i].as_deref().unwrap_or("").to_owned();
123 if get(0).is_empty() { return Ok(None); }
124
125 let payload = base64_decode(&get(3))
126 .map_err(|e| Error::Backend(format!("payload decode: {e}")))?;
127 let scheduled_at = DateTime::from_timestamp(get(6).parse::<i64>().unwrap_or(0), 0)
128 .unwrap_or_else(Utc::now);
129 let created_at = DateTime::from_timestamp(get(7).parse::<i64>().unwrap_or(0), 0)
130 .unwrap_or_else(Utc::now);
131 let last_error = if get(8).is_empty() { None } else { Some(get(8)) };
132
133 Ok(Some(JobRecord {
134 id,
135 kind: get(1),
136 queue: get(2),
137 payload,
138 attempt: get(4).parse().unwrap_or(0),
139 max_attempts: get(5).parse().unwrap_or(10),
140 scheduled_at,
141 created_at,
142 last_error,
143 }))
144 }
145
146 async fn promote_delayed(&self, queues: &[&str]) -> Result<()> {
148 let now = Utc::now().timestamp();
149 let mut conn = self.conn.clone();
150 for &queue in queues {
151 let delayed_key = key_delayed(queue);
152 let pending_key = key_pending(queue);
153 let ids: Vec<String> = conn
155 .zrangebyscore(&delayed_key, "-inf", now)
156 .await
157 .map_err(|e| Error::Backend(e.to_string()))?;
158 for id in ids {
159 let removed: i64 = conn
161 .zrem(&delayed_key, &id)
162 .await
163 .map_err(|e| Error::Backend(e.to_string()))?;
164 if removed > 0 {
165 conn.lpush::<_, _, ()>(&pending_key, &id)
166 .await
167 .map_err(|e| Error::Backend(e.to_string()))?;
168 }
169 }
170 }
171 Ok(())
172 }
173}
174
175#[async_trait]
176impl Backend for RedisBackend {
177 async fn enqueue(&self, record: JobRecord) -> Result<JobId> {
178 let id = record.id;
179 self.save_job(&record).await?;
180 let mut conn = self.conn.clone();
181 let now = Utc::now().timestamp();
182
183 if record.scheduled_at.timestamp() > now {
184 conn.zadd::<_, _, _, ()>(
186 key_delayed(&record.queue),
187 id.to_string(),
188 record.scheduled_at.timestamp(),
189 )
190 .await
191 .map_err(|e| Error::Backend(e.to_string()))?;
192 } else {
193 conn.lpush::<_, _, ()>(key_pending(&record.queue), id.to_string())
195 .await
196 .map_err(|e| Error::Backend(e.to_string()))?;
197 }
198 Ok(id)
199 }
200
201 async fn claim(&self, queues: &[&str], timeout: Duration) -> Result<Option<JobRecord>> {
202 self.promote_delayed(queues).await?;
204
205 let pending_keys: Vec<String> = queues.iter().map(|q| key_pending(q)).collect();
206 let mut conn = self.conn.clone();
207
208 let result: Option<(String, String)> = if timeout.is_zero() {
211 let mut found = None;
212 for key in &pending_keys {
213 let val: Option<String> = conn.rpop(key, None)
214 .await
215 .map_err(|e| Error::Backend(e.to_string()))?;
216 if let Some(id_str) = val {
217 found = Some((key.clone(), id_str));
218 break;
219 }
220 }
221 found
222 } else {
223 let timeout_secs = timeout.as_secs_f64().max(0.001);
225 redis::cmd("BRPOP")
226 .arg(&pending_keys)
227 .arg(timeout_secs)
228 .query_async(&mut conn)
229 .await
230 .map_err(|e| Error::Backend(e.to_string()))?
231 };
232
233 let Some((_key, id_str)) = result else {
234 return Ok(None);
235 };
236
237 let id = Uuid::parse_str(&id_str)
238 .map_err(|e| Error::Backend(format!("invalid job id: {e}")))?;
239
240 let mut record = match self.load_job(id).await? {
241 Some(r) => r,
242 None => return Ok(None), };
244
245 record.attempt += 1;
246
247 let now = Utc::now().timestamp();
249 let mut conn2 = self.conn.clone();
250 redis::cmd("HSET")
251 .arg(key_job(id))
252 .arg("attempt")
253 .arg(record.attempt)
254 .query_async::<()>(&mut conn2)
255 .await
256 .map_err(|e| Error::Backend(e.to_string()))?;
257
258 conn2.zadd::<_, _, _, ()>(KEY_PROCESSING, id.to_string(), now)
259 .await
260 .map_err(|e| Error::Backend(e.to_string()))?;
261
262 Ok(Some(record))
263 }
264
265 async fn ack(&self, id: JobId) -> Result<()> {
266 let mut conn = self.conn.clone();
267 let queue: Option<String> = conn.hget(key_job(id), "queue")
269 .await
270 .map_err(|e| Error::Backend(e.to_string()))?;
271 conn.zrem::<_, _, ()>(KEY_PROCESSING, id.to_string())
273 .await
274 .map_err(|e| Error::Backend(e.to_string()))?;
275 conn.del::<_, ()>(key_job(id))
276 .await
277 .map_err(|e| Error::Backend(e.to_string()))?;
278 if let Some(q) = queue {
280 conn.incr::<_, _, ()>(key_completed(&q), 1_u64)
281 .await
282 .map_err(|e| Error::Backend(e.to_string()))?;
283 }
284 Ok(())
285 }
286
287 async fn nack(&self, id: JobId, error: &str, retry_at: Option<DateTime<Utc>>) -> Result<()> {
288 let mut conn = self.conn.clone();
289 conn.zrem::<_, _, ()>(KEY_PROCESSING, id.to_string())
291 .await
292 .map_err(|e| Error::Backend(e.to_string()))?;
293
294 redis::cmd("HSET")
296 .arg(key_job(id))
297 .arg("last_error")
298 .arg(error)
299 .query_async::<()>(&mut conn)
300 .await
301 .map_err(|e| Error::Backend(e.to_string()))?;
302
303 match retry_at {
304 Some(at) => {
305 let record = self.load_job(id).await?
307 .ok_or_else(|| Error::NotFound(id.to_string()))?;
308 redis::cmd("HSET")
309 .arg(key_job(id))
310 .arg("scheduled_at")
311 .arg(at.timestamp())
312 .query_async::<()>(&mut conn)
313 .await
314 .map_err(|e| Error::Backend(e.to_string()))?;
315 conn.zadd::<_, _, _, ()>(key_delayed(&record.queue), id.to_string(), at.timestamp())
317 .await
318 .map_err(|e| Error::Backend(e.to_string()))?;
319 }
320 None => {
321 let record = self.load_job(id).await?
323 .ok_or_else(|| Error::NotFound(id.to_string()))?;
324 let now = Utc::now().timestamp();
325 conn.zadd::<_, _, _, ()>(key_dead(&record.queue), id.to_string(), now)
326 .await
327 .map_err(|e| Error::Backend(e.to_string()))?;
328 }
329 }
330 Ok(())
331 }
332
333 async fn heartbeat(&self, id: JobId) -> Result<()> {
334 let mut conn = self.conn.clone();
335 let now = Utc::now().timestamp();
336 conn.zadd::<_, _, _, ()>(KEY_PROCESSING, id.to_string(), now)
337 .await
338 .map_err(|e| Error::Backend(e.to_string()))?;
339 Ok(())
340 }
341
342 async fn reap_stuck(&self, older_than: Duration) -> Result<u64> {
343 let cutoff = Utc::now().timestamp() - older_than.as_secs() as i64;
344 let mut conn = self.conn.clone();
345 let stuck_ids: Vec<String> = conn
347 .zrangebyscore(KEY_PROCESSING, "-inf", cutoff)
348 .await
349 .map_err(|e| Error::Backend(e.to_string()))?;
350
351 let count = stuck_ids.len() as u64;
352 for id_str in stuck_ids {
353 let id = match Uuid::parse_str(&id_str) {
354 Ok(id) => id,
355 Err(_) => continue,
356 };
357 conn.zrem::<_, _, ()>(KEY_PROCESSING, &id_str)
359 .await
360 .map_err(|e| Error::Backend(e.to_string()))?;
361
362 if let Some(mut record) = self.load_job(id).await? {
364 if record.attempt > 0 { record.attempt -= 1; }
365 let now_ts = Utc::now().timestamp();
366 redis::cmd("HSET")
367 .arg(key_job(id))
368 .arg("attempt").arg(record.attempt)
369 .arg("scheduled_at").arg(now_ts)
370 .query_async::<()>(&mut conn)
371 .await
372 .map_err(|e| Error::Backend(e.to_string()))?;
373 conn.lpush::<_, _, ()>(key_pending(&record.queue), &id_str)
374 .await
375 .map_err(|e| Error::Backend(e.to_string()))?;
376 }
377 }
378 Ok(count)
379 }
380
381 async fn stats(&self, queue: &str) -> Result<QueueStats> {
382 let mut conn = self.conn.clone();
383 let pending: u64 = conn.llen(key_pending(queue)).await
384 .map_err(|e| Error::Backend(e.to_string()))?;
385 let delayed: u64 = conn.zcard(key_delayed(queue)).await
386 .map_err(|e| Error::Backend(e.to_string()))?;
387 let dead: u64 = conn.zcard(key_dead(queue)).await
388 .map_err(|e| Error::Backend(e.to_string()))?;
389 let running: u64 = conn.zcard(KEY_PROCESSING).await
390 .map_err(|e| Error::Backend(e.to_string()))?;
391 let completed: u64 = conn
393 .get::<_, Option<u64>>(key_completed(queue))
394 .await
395 .map_err(|e| Error::Backend(e.to_string()))?
396 .unwrap_or(0);
397
398 Ok(QueueStats {
399 queue: queue.to_owned(),
400 pending: pending + delayed,
401 running,
402 completed,
403 failed: 0,
404 dead,
405 })
406 }
407
408 async fn dead_jobs(&self, queue: &str, limit: i64, offset: i64) -> Result<Vec<JobRecord>> {
409 let mut conn = self.conn.clone();
410 let ids: Vec<String> = conn
411 .zrevrange(key_dead(queue), offset as isize, (offset + limit - 1) as isize)
412 .await
413 .map_err(|e| Error::Backend(e.to_string()))?;
414
415 let mut records = Vec::new();
416 for id_str in ids {
417 if let Ok(id) = Uuid::parse_str(&id_str) {
418 if let Some(record) = self.load_job(id).await? {
419 records.push(record);
420 }
421 }
422 }
423 Ok(records)
424 }
425
426 async fn retry_dead(&self, id: JobId) -> Result<()> {
427 let record = self.load_job(id).await?
428 .ok_or_else(|| Error::NotFound(id.to_string()))?;
429 let mut conn = self.conn.clone();
430 let removed: i64 = conn.zrem(key_dead(&record.queue), id.to_string())
432 .await
433 .map_err(|e| Error::Backend(e.to_string()))?;
434 if removed == 0 {
435 return Err(Error::NotFound(id.to_string()));
436 }
437 let now_ts = Utc::now().timestamp();
439 redis::cmd("HSET")
440 .arg(key_job(id))
441 .arg("scheduled_at").arg(now_ts)
442 .arg("last_error").arg("")
443 .query_async::<()>(&mut conn)
444 .await
445 .map_err(|e| Error::Backend(e.to_string()))?;
446 conn.lpush::<_, _, ()>(key_pending(&record.queue), id.to_string())
447 .await
448 .map_err(|e| Error::Backend(e.to_string()))?;
449 Ok(())
450 }
451
452 async fn retry_all_dead(&self, queue: &str) -> Result<u64> {
453 let dead_key = key_dead(queue);
454 let mut conn = self.conn.clone();
455 let ids: Vec<String> = conn
456 .zrange(&dead_key, 0isize, -1isize)
457 .await
458 .map_err(|e| Error::Backend(e.to_string()))?;
459
460 let count = ids.len() as u64;
461 for id_str in ids {
462 if let Ok(id) = Uuid::parse_str(&id_str) {
463 let _ = self.retry_dead(id).await;
464 }
465 }
466 Ok(count)
467 }
468}
469
470fn base64_encode(data: &[u8]) -> String {
473 use std::fmt::Write;
474 const TABLE: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
475 let mut out = String::new();
476 for chunk in data.chunks(3) {
477 let b0 = chunk[0] as u32;
478 let b1 = if chunk.len() > 1 { chunk[1] as u32 } else { 0 };
479 let b2 = if chunk.len() > 2 { chunk[2] as u32 } else { 0 };
480 let combined = (b0 << 16) | (b1 << 8) | b2;
481 let _ = write!(out, "{}", TABLE[((combined >> 18) & 63) as usize] as char);
482 let _ = write!(out, "{}", TABLE[((combined >> 12) & 63) as usize] as char);
483 let _ = write!(out, "{}", if chunk.len() > 1 { TABLE[((combined >> 6) & 63) as usize] as char } else { '=' });
484 let _ = write!(out, "{}", if chunk.len() > 2 { TABLE[(combined & 63) as usize] as char } else { '=' });
485 }
486 out
487}
488
489fn base64_decode(s: &str) -> std::result::Result<Vec<u8>, &'static str> {
490 let s = s.trim_end_matches('=');
491 let mut out = Vec::new();
492 let decode_char = |c: char| -> Option<u32> {
493 match c {
494 'A'..='Z' => Some(c as u32 - 'A' as u32),
495 'a'..='z' => Some(c as u32 - 'a' as u32 + 26),
496 '0'..='9' => Some(c as u32 - '0' as u32 + 52),
497 '+' => Some(62), '/' => Some(63), _ => None,
498 }
499 };
500 let chars: Vec<char> = s.chars().collect();
501 for chunk in chars.chunks(4) {
502 let b: Vec<u32> = chunk.iter().filter_map(|&c| decode_char(c)).collect();
503 if b.is_empty() { continue; }
504 let combined = b[0] << 18
505 | b.get(1).copied().unwrap_or(0) << 12
506 | b.get(2).copied().unwrap_or(0) << 6
507 | b.get(3).copied().unwrap_or(0);
508 out.push(((combined >> 16) & 0xFF) as u8);
509 if b.len() > 2 { out.push(((combined >> 8) & 0xFF) as u8); }
510 if b.len() > 3 { out.push((combined & 0xFF) as u8); }
511 }
512 Ok(out)
513}