1use super::history::{FinishedKind, JobHistory};
103use super::{
104 ApiResponse, FilterParams, PaginatedResponse, PaginationMeta, PaginationParams, SortParams,
105 with_filters, with_pagination, with_sort,
106};
107use super::{error_reply, json_reply, with_shared};
108use hammerwork::queue::DatabaseQueue;
109use serde::{Deserialize, Serialize};
110use std::sync::Arc;
111use warp::http::StatusCode;
112use warp::{Filter, Reply};
113
114#[derive(Debug, Serialize, Clone)]
116pub struct QueueInfo {
117 pub name: String,
118 pub pending_count: u64,
119 pub running_count: u64,
120 pub completed_count: u64,
121 pub failed_count: u64,
122 pub dead_count: u64,
123 pub avg_processing_time_ms: f64,
124 pub throughput_per_minute: f64,
125 pub error_rate: f64,
126 pub last_job_at: Option<chrono::DateTime<chrono::Utc>>,
127 pub oldest_pending_job: Option<chrono::DateTime<chrono::Utc>>,
128 pub is_paused: bool,
129 pub paused_at: Option<chrono::DateTime<chrono::Utc>>,
130 pub paused_by: Option<String>,
131}
132
133#[derive(Debug, Serialize)]
135pub struct DetailedQueueStats {
136 pub queue_info: QueueInfo,
137 pub priority_breakdown: std::collections::HashMap<String, u64>,
138 pub status_breakdown: std::collections::HashMap<String, u64>,
139 pub hourly_throughput: Vec<HourlyThroughput>,
140 pub recent_errors: Vec<RecentError>,
141}
142
143#[derive(Debug, Serialize)]
145pub struct HourlyThroughput {
146 pub hour: chrono::DateTime<chrono::Utc>,
147 pub completed: u64,
148 pub failed: u64,
149}
150
151#[derive(Debug, Serialize)]
153pub struct RecentError {
154 pub job_id: String,
155 pub error_message: String,
156 pub occurred_at: chrono::DateTime<chrono::Utc>,
157 pub attempts: i32,
158}
159
160#[derive(Debug, Deserialize)]
162pub struct QueueActionRequest {
163 pub action: String, pub confirm: Option<bool>,
165}
166
167pub fn routes<T>(
169 queue: Arc<T>,
170) -> impl Filter<Extract = impl Reply, Error = warp::Rejection> + Clone
171where
172 T: JobHistory + 'static,
173{
174 let queue_filter = with_shared(queue);
175
176 let list_queues = warp::path("queues")
177 .and(warp::path::end())
178 .and(warp::get())
179 .and(queue_filter.clone())
180 .and(with_pagination())
181 .and(with_filters())
182 .and(with_sort())
183 .and_then(list_queues_handler);
184
185 let get_queue = warp::path("queues")
186 .and(warp::path::param::<String>())
187 .and(warp::path::end())
188 .and(warp::get())
189 .and(queue_filter.clone())
190 .and_then(get_queue_handler);
191
192 let queue_action = warp::path("queues")
193 .and(warp::path::param::<String>())
194 .and(warp::path("actions"))
195 .and(warp::path::end())
196 .and(warp::post())
197 .and(queue_filter.clone())
198 .and(crate::security::json_body())
199 .and_then(queue_action_handler);
200
201 let queue_jobs = warp::path("queues")
202 .and(warp::path::param::<String>())
203 .and(warp::path("jobs"))
204 .and(warp::path::end())
205 .and(warp::get())
206 .and(queue_filter)
207 .and(with_pagination())
208 .and(with_filters())
209 .and(with_sort())
210 .and_then(queue_jobs_handler);
211
212 list_queues.or(get_queue).or(queue_action).or(queue_jobs)
213}
214
215async fn list_queues_handler<T>(
217 queue: Arc<T>,
218 pagination: PaginationParams,
219 _filters: FilterParams,
220 _sort: SortParams,
221) -> Result<impl Reply, warp::Rejection>
222where
223 T: DatabaseQueue + Send + Sync,
224{
225 match queue.get_all_queue_stats().await {
227 Ok(all_stats) => {
228 let mut queue_infos: Vec<QueueInfo> = Vec::new();
229
230 for stats in all_stats {
231 let pause_info = try_api!(
233 queue.get_queue_pause_info(&stats.queue_name).await,
234 "Failed to get queue pause info"
235 );
236
237 let queue_info = QueueInfo {
238 name: stats.queue_name.clone(),
239 pending_count: stats.pending_count,
240 running_count: stats.running_count,
241 completed_count: stats.completed_count,
242 failed_count: super::failed_job_count(&stats),
243 dead_count: stats.dead_count,
244 avg_processing_time_ms: stats.statistics.avg_processing_time_ms,
245 throughput_per_minute: stats.statistics.throughput_per_minute,
246 error_rate: stats.statistics.error_rate,
247 last_job_at: try_api!(
248 get_last_job_time(&queue, &stats.queue_name).await,
249 "Failed to get last job time"
250 ),
251 oldest_pending_job: try_api!(
252 get_oldest_pending_job(&queue, &stats.queue_name).await,
253 "Failed to get oldest pending job"
254 ),
255 is_paused: pause_info.is_some(),
256 paused_at: pause_info.as_ref().map(|p| p.paused_at),
257 paused_by: pause_info.as_ref().and_then(|p| p.paused_by.clone()),
258 };
259 queue_infos.push(queue_info);
260 }
261
262 let total = queue_infos.len() as u64;
264 let offset = pagination.get_offset() as usize;
265 let limit = pagination.get_limit() as usize;
266
267 let items = if offset < queue_infos.len() {
268 let end = (offset + limit).min(queue_infos.len());
269 queue_infos[offset..end].to_vec()
270 } else {
271 Vec::new()
272 };
273
274 let response = PaginatedResponse {
275 items,
276 pagination: PaginationMeta::new(&pagination, total),
277 };
278
279 Ok(json_reply(&ApiResponse::success(response)))
280 }
281 Err(e) => Ok(error_reply(
282 StatusCode::INTERNAL_SERVER_ERROR,
283 format!("Failed to get queue statistics: {}", e),
284 )),
285 }
286}
287
288fn decode_queue_name(raw: &str) -> Result<String, ()> {
291 percent_encoding::percent_decode_str(raw)
292 .decode_utf8()
293 .map(|name| name.into_owned())
294 .map_err(|_| ())
295}
296
297fn invalid_queue_name() -> warp::reply::Response {
299 error_reply(
300 StatusCode::BAD_REQUEST,
301 "Queue name in the path is not valid UTF-8",
302 )
303}
304
305async fn get_queue_handler<T>(
307 queue_name: String,
308 queue: Arc<T>,
309) -> Result<impl Reply, warp::Rejection>
310where
311 T: JobHistory,
312{
313 let queue_name = match decode_queue_name(&queue_name) {
314 Ok(name) => name,
315 Err(()) => return Ok(invalid_queue_name()),
316 };
317 match queue.get_all_queue_stats().await {
318 Ok(all_stats) => {
319 if let Some(stats) = all_stats.into_iter().find(|s| s.queue_name == queue_name) {
320 let priority_breakdown = try_api!(
322 get_priority_breakdown(&queue, &queue_name).await,
323 "Failed to get priority breakdown"
324 );
325 let status_breakdown = try_api!(
326 get_status_breakdown(&queue, &queue_name).await,
327 "Failed to get status breakdown"
328 );
329 let hourly_throughput = try_api!(
330 get_hourly_throughput(&*queue, &queue_name).await,
331 "Failed to get hourly throughput"
332 );
333 let recent_errors = try_api!(
334 get_recent_errors(&queue, &queue_name).await,
335 "Failed to get recent errors"
336 );
337
338 let pause_info = try_api!(
340 queue.get_queue_pause_info(&queue_name).await,
341 "Failed to get queue pause info"
342 );
343
344 let queue_info = QueueInfo {
345 name: stats.queue_name.clone(),
346 pending_count: stats.pending_count,
347 running_count: stats.running_count,
348 completed_count: stats.completed_count,
349 failed_count: super::failed_job_count(&stats),
350 dead_count: stats.dead_count,
351 avg_processing_time_ms: stats.statistics.avg_processing_time_ms,
352 throughput_per_minute: stats.statistics.throughput_per_minute,
353 error_rate: stats.statistics.error_rate,
354 last_job_at: try_api!(
355 get_last_job_time(&queue, &stats.queue_name).await,
356 "Failed to get last job time"
357 ),
358 oldest_pending_job: try_api!(
359 get_oldest_pending_job(&queue, &stats.queue_name).await,
360 "Failed to get oldest pending job"
361 ),
362 is_paused: pause_info.is_some(),
363 paused_at: pause_info.as_ref().map(|p| p.paused_at),
364 paused_by: pause_info.as_ref().and_then(|p| p.paused_by.clone()),
365 };
366
367 let detailed_stats = DetailedQueueStats {
368 queue_info,
369 priority_breakdown,
370 status_breakdown,
371 hourly_throughput,
372 recent_errors,
373 };
374
375 Ok(json_reply(&ApiResponse::success(detailed_stats)))
376 } else {
377 Ok(error_reply(
378 StatusCode::NOT_FOUND,
379 format!("Queue '{}' not found", queue_name),
380 ))
381 }
382 }
383 Err(e) => Ok(error_reply(
384 StatusCode::INTERNAL_SERVER_ERROR,
385 format!("Failed to get queue statistics: {}", e),
386 )),
387 }
388}
389
390async fn queue_action_handler<T>(
392 queue_name: String,
393 queue: Arc<T>,
394 action_request: QueueActionRequest,
395) -> Result<impl Reply, warp::Rejection>
396where
397 T: JobHistory,
398{
399 let queue_name = match decode_queue_name(&queue_name) {
400 Ok(name) => name,
401 Err(()) => return Ok(invalid_queue_name()),
402 };
403 match action_request.action.as_str() {
404 "clear_dead" => {
405 let older_than = chrono::Utc::now() - chrono::Duration::days(7); match queue
407 .delete_jobs(Some(&queue_name), FinishedKind::Dead, Some(older_than))
408 .await
409 {
410 Ok(count) => {
411 let response = ApiResponse::success(serde_json::json!({
412 "message": format!(
413 "Cleared {} dead jobs older than 7 days from queue '{}'",
414 count, queue_name
415 ),
416 "queue": queue_name,
417 "count": count
418 }));
419 Ok(json_reply(&response))
420 }
421 Err(e) => Ok(error_reply(
422 StatusCode::INTERNAL_SERVER_ERROR,
423 format!("Failed to clear dead jobs: {}", e),
424 )),
425 }
426 }
427 "clear_completed" => match queue
428 .delete_jobs(Some(&queue_name), FinishedKind::Completed, None)
429 .await
430 {
431 Ok(count) => {
432 let response = ApiResponse::success(serde_json::json!({
433 "message": format!("Cleared {} completed jobs from queue '{}'", count, queue_name),
434 "queue": queue_name,
435 "cleared_count": count
436 }));
437 Ok(json_reply(&response))
438 }
439 Err(e) => Ok(error_reply(
440 StatusCode::INTERNAL_SERVER_ERROR,
441 format!("Failed to clear completed jobs: {}", e),
442 )),
443 },
444 "pause" => match queue.pause_queue(&queue_name, Some("web-ui")).await {
445 Ok(()) => {
446 let response = ApiResponse::success(serde_json::json!({
447 "message": format!("Queue '{}' has been paused", queue_name),
448 "queue": queue_name,
449 "action": "pause"
450 }));
451 Ok(json_reply(&response))
452 }
453 Err(e) => Ok(error_reply(
454 StatusCode::INTERNAL_SERVER_ERROR,
455 format!("Failed to pause queue: {}", e),
456 )),
457 },
458 "resume" => match queue.resume_queue(&queue_name, Some("web-ui")).await {
459 Ok(()) => {
460 let response = ApiResponse::success(serde_json::json!({
461 "message": format!("Queue '{}' has been resumed", queue_name),
462 "queue": queue_name,
463 "action": "resume"
464 }));
465 Ok(json_reply(&response))
466 }
467 Err(e) => Ok(error_reply(
468 StatusCode::INTERNAL_SERVER_ERROR,
469 format!("Failed to resume queue: {}", e),
470 )),
471 },
472 _ => Ok(error_reply(
473 StatusCode::BAD_REQUEST,
474 format!("Unknown action: {}", action_request.action),
475 )),
476 }
477}
478
479async fn queue_jobs_handler<T>(
481 queue_name: String,
482 queue: Arc<T>,
483 pagination: PaginationParams,
484 mut filters: FilterParams,
485 sort: SortParams,
486) -> Result<impl Reply, warp::Rejection>
487where
488 T: JobHistory,
489{
490 let queue_name = match decode_queue_name(&queue_name) {
491 Ok(name) => name,
492 Err(()) => return Ok(invalid_queue_name()),
493 };
494 filters.queue = Some(queue_name);
495 Ok(
496 super::jobs::list_jobs_handler(queue, pagination, filters, sort)
497 .await?
498 .into_response(),
499 )
500}
501
502async fn get_last_job_time<T>(
504 queue: &Arc<T>,
505 queue_name: &str,
506) -> hammerwork::Result<Option<chrono::DateTime<chrono::Utc>>>
507where
508 T: DatabaseQueue + Send + Sync,
509{
510 let mut latest_time: Option<chrono::DateTime<chrono::Utc>> = None;
512
513 let ready_jobs = queue.get_ready_jobs(queue_name, 10).await?;
515 for job in ready_jobs {
516 if let Some(time) = job.completed_at.or(job.started_at).or(Some(job.created_at)) {
517 latest_time = match latest_time {
518 Some(current) if time > current => Some(time),
519 None => Some(time),
520 _ => latest_time,
521 };
522 }
523 }
524
525 let dead_jobs = queue
527 .get_dead_jobs_by_queue(queue_name, Some(10), Some(0))
528 .await?;
529 for job in dead_jobs {
530 if let Some(time) = job
531 .failed_at
532 .or(job.completed_at)
533 .or(job.started_at)
534 .or(Some(job.created_at))
535 {
536 latest_time = match latest_time {
537 Some(current) if time > current => Some(time),
538 None => Some(time),
539 _ => latest_time,
540 };
541 }
542 }
543
544 Ok(latest_time)
545}
546
547async fn get_oldest_pending_job<T>(
549 queue: &Arc<T>,
550 queue_name: &str,
551) -> hammerwork::Result<Option<chrono::DateTime<chrono::Utc>>>
552where
553 T: DatabaseQueue + Send + Sync,
554{
555 let ready_jobs = queue.get_ready_jobs(queue_name, 100).await?;
557 Ok(ready_jobs
558 .iter()
559 .filter(|job| matches!(job.status, hammerwork::job::JobStatus::Pending))
560 .map(|job| job.created_at)
561 .min())
562}
563
564async fn get_priority_breakdown<T>(
566 queue: &Arc<T>,
567 queue_name: &str,
568) -> hammerwork::Result<std::collections::HashMap<String, u64>>
569where
570 T: DatabaseQueue + Send + Sync,
571{
572 let priority_stats = queue.get_priority_stats(queue_name).await?;
573 let mut breakdown = std::collections::HashMap::new();
574 for (priority, count) in priority_stats.job_counts {
575 breakdown.insert(priority.to_string(), count);
576 }
577 Ok(breakdown)
578}
579
580async fn get_status_breakdown<T>(
582 queue: &Arc<T>,
583 queue_name: &str,
584) -> hammerwork::Result<std::collections::HashMap<String, u64>>
585where
586 T: DatabaseQueue + Send + Sync,
587{
588 let counts = queue.get_job_counts_by_status(queue_name).await?;
590 Ok(counts.into_iter().collect())
591}
592
593async fn get_hourly_throughput<T>(
596 queue: &T,
597 queue_name: &str,
598) -> hammerwork::Result<Vec<HourlyThroughput>>
599where
600 T: JobHistory,
601{
602 let now = chrono::Utc::now();
603 let start = super::history::hour_floor(now) - chrono::Duration::hours(23);
604 let buckets = queue.hourly_activity(Some(queue_name), start, now).await?;
605 Ok(buckets
606 .into_iter()
607 .map(|b| HourlyThroughput {
608 hour: b.hour,
609 completed: b.completed,
610 failed: b.failed,
611 })
612 .collect())
613}
614
615async fn get_recent_errors<T>(
617 queue: &Arc<T>,
618 queue_name: &str,
619) -> hammerwork::Result<Vec<RecentError>>
620where
621 T: DatabaseQueue + Send + Sync,
622{
623 let dead_jobs = queue
625 .get_dead_jobs_by_queue(queue_name, Some(20), Some(0))
626 .await?;
627 Ok(dead_jobs
628 .into_iter()
629 .filter_map(|job| {
630 job.error_message.map(|error_msg| RecentError {
631 job_id: job.id.to_string(),
632 error_message: error_msg,
633 occurred_at: job.failed_at.unwrap_or(job.created_at),
634 attempts: job.attempts,
635 })
636 })
637 .collect())
638}
639
640#[cfg(test)]
641mod tests {
642 use super::*;
643 use crate::api::test_support::{body_json, unreachable_queue};
644
645 #[tokio::test]
646 async fn test_list_queues_returns_500_when_database_is_down() {
647 let response = list_queues_handler(
648 unreachable_queue(),
649 PaginationParams::default(),
650 serde_json::from_value(serde_json::json!({})).unwrap(),
651 SortParams {
652 sort_by: None,
653 sort_order: None,
654 },
655 )
656 .await
657 .unwrap()
658 .into_response();
659 let (status, body) = body_json(response).await;
660 assert_eq!(status, 500);
661 assert_eq!(body["success"], false);
662 }
663
664 #[tokio::test]
665 async fn test_clear_actions_report_database_failures_not_success() {
666 for action in ["clear_completed", "clear_dead"] {
667 let response = queue_action_handler(
668 "q".to_string(),
669 unreachable_queue(),
670 QueueActionRequest {
671 action: action.to_string(),
672 confirm: Some(true),
673 },
674 )
675 .await
676 .unwrap()
677 .into_response();
678 let (status, body) = body_json(response).await;
679 assert_eq!(status, 500, "{action}");
680 assert_eq!(body["success"], false);
681 }
682 }
683
684 #[tokio::test]
685 async fn test_queue_jobs_delegates_to_job_listing() {
686 let response = queue_jobs_handler(
687 "q".to_string(),
688 unreachable_queue(),
689 PaginationParams::default(),
690 serde_json::from_value(serde_json::json!({})).unwrap(),
691 SortParams {
692 sort_by: None,
693 sort_order: None,
694 },
695 )
696 .await
697 .unwrap()
698 .into_response();
699 let (status, body) = body_json(response).await;
700 assert_eq!(status, 500);
702 assert!(body.get("data").is_none_or(|d| d.is_null()));
703 }
704
705 #[tokio::test]
707 async fn test_queue_routes_percent_decode_the_queue_name() {
708 let queue = Arc::new(hammerwork::queue::test::TestQueue::new());
709 let filter = routes(queue.clone());
710
711 let reply = warp::test::request()
712 .method("POST")
713 .path("/queues/email%20jobs/actions")
714 .header("content-type", "application/json")
715 .body(r#"{"action": "pause"}"#)
716 .reply(&filter)
717 .await;
718 assert_eq!(reply.status(), 200, "{:?}", reply.body());
719 let body: serde_json::Value = serde_json::from_slice(reply.body()).unwrap();
720 assert_eq!(body["data"]["queue"], "email jobs");
721 assert!(queue.is_queue_paused("email jobs").await.unwrap());
722 assert!(!queue.is_queue_paused("email%20jobs").await.unwrap());
723
724 let reply = warp::test::request()
726 .method("POST")
727 .path("/queues/100%2525/actions")
728 .header("content-type", "application/json")
729 .body(r#"{"action": "pause"}"#)
730 .reply(&filter)
731 .await;
732 let body: serde_json::Value = serde_json::from_slice(reply.body()).unwrap();
733 assert_eq!(body["data"]["queue"], "100%25");
734 }
735
736 #[tokio::test]
737 async fn test_queue_routes_reject_invalid_utf8_names() {
738 let queue = Arc::new(hammerwork::queue::test::TestQueue::new());
739 let filter = routes(queue);
740 let reply = warp::test::request()
741 .method("POST")
742 .path("/queues/%FF/actions")
743 .header("content-type", "application/json")
744 .body(r#"{"action": "pause"}"#)
745 .reply(&filter)
746 .await;
747 assert_eq!(reply.status(), 400);
748 let reply = warp::test::request()
749 .path("/queues/%FF")
750 .reply(&filter)
751 .await;
752 assert_eq!(reply.status(), 400);
753 let reply = warp::test::request()
754 .path("/queues/%FF/jobs")
755 .reply(&filter)
756 .await;
757 assert_eq!(reply.status(), 400);
758 }
759
760 #[test]
761 fn test_queue_action_request_deserialization() {
762 let json = r#"{"action": "clear_dead", "confirm": true}"#;
763 let request: QueueActionRequest = serde_json::from_str(json).unwrap();
764 assert_eq!(request.action, "clear_dead");
765 assert_eq!(request.confirm, Some(true));
766 }
767
768 #[test]
769 fn test_queue_info_serialization() {
770 let queue_info = QueueInfo {
771 name: "test_queue".to_string(),
772 pending_count: 42,
773 running_count: 3,
774 completed_count: 1000,
775 failed_count: 5,
776 dead_count: 2,
777 avg_processing_time_ms: 150.5,
778 throughput_per_minute: 25.0,
779 error_rate: 0.05,
780 last_job_at: None,
781 oldest_pending_job: None,
782 is_paused: false,
783 paused_at: None,
784 paused_by: None,
785 };
786
787 let json = serde_json::to_string(&queue_info).unwrap();
788 assert!(json.contains("test_queue"));
789 assert!(json.contains("42"));
790 assert!(json.contains("is_paused"));
791 }
792}