Skip to main content

cargo_hammerwork/commands/
monitor.rs

1use anyhow::Result;
2use clap::Subcommand;
3use sqlx::Row;
4use std::time::Duration;
5use tokio::time::interval;
6
7use crate::config::Config;
8use crate::utils::database::DatabasePool;
9
10#[derive(Subcommand)]
11pub enum MonitorCommand {
12    #[command(about = "Real-time monitoring dashboard")]
13    Dashboard {
14        #[arg(short = 'u', long, help = "Database connection URL")]
15        database_url: Option<String>,
16        #[arg(
17            short = 'i',
18            long,
19            default_value = "5",
20            help = "Refresh interval in seconds"
21        )]
22        refresh: u64,
23        #[arg(short = 'n', long, help = "Specific queue to monitor")]
24        queue: Option<String>,
25    },
26    #[command(about = "Check system health")]
27    Health {
28        #[arg(short = 'u', long, help = "Database connection URL")]
29        database_url: Option<String>,
30        #[arg(long, help = "Output format (table, json)")]
31        format: Option<String>,
32    },
33    #[command(about = "Show performance metrics")]
34    Metrics {
35        #[arg(short = 'u', long, help = "Database connection URL")]
36        database_url: Option<String>,
37        #[arg(short = 't', long, help = "Time period (1h, 24h, 7d)")]
38        period: Option<String>,
39        #[arg(short = 'n', long, help = "Specific queue to analyze")]
40        queue: Option<String>,
41    },
42    #[command(about = "Tail job logs (placeholder)")]
43    Logs {
44        #[arg(short = 'u', long, help = "Database connection URL")]
45        database_url: Option<String>,
46        #[arg(short = 'l', long, help = "Number of recent log entries")]
47        lines: Option<u32>,
48        #[arg(short = 'n', long, help = "Filter by queue")]
49        queue: Option<String>,
50        #[arg(long, help = "Follow logs in real-time")]
51        follow: bool,
52    },
53}
54
55impl MonitorCommand {
56    pub async fn execute(&self, config: &Config) -> Result<()> {
57        let db_url = self.get_database_url(config)?;
58        let pool = DatabasePool::connect(&db_url, config.get_connection_pool_size()).await?;
59
60        match self {
61            MonitorCommand::Dashboard { refresh, queue, .. } => {
62                run_dashboard(pool, *refresh, queue.clone()).await?;
63            }
64            MonitorCommand::Health { format, .. } => {
65                check_health(pool, format.clone()).await?;
66            }
67            MonitorCommand::Metrics { period, queue, .. } => {
68                show_metrics(pool, period.clone(), queue.clone()).await?;
69            }
70            MonitorCommand::Logs {
71                lines,
72                queue,
73                follow,
74                ..
75            } => {
76                show_logs(pool, *lines, queue.clone(), *follow).await?;
77            }
78        }
79        Ok(())
80    }
81
82    fn get_database_url(&self, config: &Config) -> Result<String> {
83        let url_option = match self {
84            MonitorCommand::Dashboard { database_url, .. } => database_url,
85            MonitorCommand::Health { database_url, .. } => database_url,
86            MonitorCommand::Metrics { database_url, .. } => database_url,
87            MonitorCommand::Logs { database_url, .. } => database_url,
88        };
89
90        url_option
91            .as_ref()
92            .map(|s| s.as_str())
93            .or(config.get_database_url())
94            .map(|s| s.to_string())
95            .ok_or_else(|| anyhow::anyhow!("Database URL is required"))
96    }
97}
98
99async fn run_dashboard(pool: DatabasePool, refresh_secs: u64, queue: Option<String>) -> Result<()> {
100    println!("šŸ“Š Hammerwork Dashboard");
101    println!("Press Ctrl+C to exit\n");
102
103    let mut interval = interval(Duration::from_secs(refresh_secs));
104
105    loop {
106        tokio::select! {
107            _ = interval.tick() => {
108                // Clear screen
109                print!("\x1B[2J\x1B[1;1H");
110
111                println!("šŸ“Š Hammerwork Dashboard - {}", chrono::Utc::now().format("%Y-%m-%d %H:%M:%S UTC"));
112                if let Some(q) = &queue {
113                    println!("šŸŽÆ Queue: {}", q);
114                }
115                println!("šŸ”„ Refresh: {}s\n", refresh_secs);
116
117                if let Err(e) = display_dashboard_data(&pool, &queue).await {
118                    println!("āŒ Error updating dashboard: {}", e);
119                }
120
121                println!("\nPress Ctrl+C to exit");
122            }
123            _ = tokio::signal::ctrl_c() => {
124                println!("\nšŸ‘‹ Dashboard stopped");
125                break;
126            }
127        }
128    }
129
130    Ok(())
131}
132
133async fn display_dashboard_data(pool: &DatabasePool, queue: &Option<String>) -> Result<()> {
134    // Quick stats
135    let mut stats_table = comfy_table::Table::new();
136    stats_table.set_header(vec!["Status", "Count"]);
137
138    let queue_filter = if let Some(q) = queue {
139        format!(" WHERE queue_name = '{}'", q)
140    } else {
141        String::new()
142    };
143
144    match pool {
145        DatabasePool::Postgres(pg_pool) => {
146            let query = format!(
147                "SELECT status, COUNT(*) as count FROM hammerwork_jobs{} GROUP BY status ORDER BY status",
148                queue_filter
149            );
150            let rows = sqlx::query(&query).fetch_all(pg_pool).await?;
151
152            for row in rows {
153                let status: String = row.try_get("status")?;
154                let count: i64 = row.try_get("count")?;
155
156                let status_with_icon = match status.as_str() {
157                    "pending" => format!("🟔 {}", status),
158                    "running" => format!("šŸ”µ {}", status),
159                    "completed" => format!("🟢 {}", status),
160                    "failed" => format!("šŸ”“ {}", status),
161                    "dead" => format!("šŸ’€ {}", status),
162                    _ => status,
163                };
164
165                stats_table.add_row(vec![status_with_icon, count.to_string()]);
166            }
167        }
168        DatabasePool::MySQL(mysql_pool) => {
169            let query = format!(
170                "SELECT status, COUNT(*) as count FROM hammerwork_jobs{} GROUP BY status ORDER BY status",
171                queue_filter
172            );
173            let rows = sqlx::query(&query).fetch_all(mysql_pool).await?;
174
175            for row in rows {
176                let status: String = row.try_get("status")?;
177                let count: i64 = row.try_get("count")?;
178
179                let status_with_icon = match status.as_str() {
180                    "pending" => format!("🟔 {}", status),
181                    "running" => format!("šŸ”µ {}", status),
182                    "completed" => format!("🟢 {}", status),
183                    "failed" => format!("šŸ”“ {}", status),
184                    "dead" => format!("šŸ’€ {}", status),
185                    _ => status,
186                };
187
188                stats_table.add_row(vec![status_with_icon, count.to_string()]);
189            }
190        }
191    }
192
193    println!("šŸ“ˆ Job Status Overview");
194    println!("{}", stats_table);
195
196    // Recent activity (last 10 jobs)
197    let recent_query = format!(
198        "SELECT id, queue_name, status, priority, created_at FROM hammerwork_jobs{} ORDER BY created_at DESC LIMIT 10",
199        queue_filter
200    );
201
202    let mut recent_table = comfy_table::Table::new();
203    recent_table.set_header(vec!["ID", "Queue", "Status", "Priority", "Created"]);
204
205    match pool {
206        DatabasePool::Postgres(pg_pool) => {
207            let rows = sqlx::query(&recent_query).fetch_all(pg_pool).await?;
208            for row in rows {
209                let id: uuid::Uuid = row.try_get("id")?;
210                let queue_name: String = row.try_get("queue_name")?;
211                let status: String = row.try_get("status")?;
212                let priority: String = row.try_get("priority")?;
213                let created_at: chrono::DateTime<chrono::Utc> = row.try_get("created_at")?;
214
215                recent_table.add_row(vec![
216                    &id.to_string()[..8],
217                    &queue_name,
218                    &status,
219                    &priority,
220                    &created_at.format("%H:%M:%S").to_string(),
221                ]);
222            }
223        }
224        DatabasePool::MySQL(mysql_pool) => {
225            let rows = sqlx::query(&recent_query).fetch_all(mysql_pool).await?;
226            for row in rows {
227                let id: String = row.try_get("id")?;
228                let queue_name: String = row.try_get("queue_name")?;
229                let status: String = row.try_get("status")?;
230                let priority: String = row.try_get("priority")?;
231                let created_at: chrono::DateTime<chrono::Utc> = row.try_get("created_at")?;
232
233                recent_table.add_row(vec![
234                    &id[..8],
235                    &queue_name,
236                    &status,
237                    &priority,
238                    &created_at.format("%H:%M:%S").to_string(),
239                ]);
240            }
241        }
242    }
243
244    println!("\nšŸ• Recent Activity");
245    println!("{}", recent_table);
246
247    Ok(())
248}
249
250async fn check_health(pool: DatabasePool, format: Option<String>) -> Result<()> {
251    let mut health_data: Vec<(&str, &str, String)> = Vec::new();
252
253    // Check database connectivity
254    let db_status: (&str, &str, String) = match pool {
255        DatabasePool::Postgres(ref pg_pool) => {
256            match sqlx::query("SELECT 1").fetch_one(pg_pool).await {
257                Ok(_) => ("🟢", "Database", "Connected".to_string()),
258                Err(_) => ("šŸ”“", "Database", "Connection Failed".to_string()),
259            }
260        }
261        DatabasePool::MySQL(ref mysql_pool) => {
262            match sqlx::query("SELECT 1").fetch_one(mysql_pool).await {
263                Ok(_) => ("🟢", "Database", "Connected".to_string()),
264                Err(_) => ("šŸ”“", "Database", "Connection Failed".to_string()),
265            }
266        }
267    };
268    health_data.push(db_status);
269
270    // Check for stuck jobs (running > 1 hour)
271    let stuck_count = match pool {
272        DatabasePool::Postgres(ref pg_pool) => {
273            let result = sqlx::query(
274                "SELECT COUNT(*) as count FROM hammerwork_jobs 
275                 WHERE status = 'running' AND started_at < NOW() - INTERVAL '1 hour'",
276            )
277            .fetch_one(pg_pool)
278            .await?;
279            result.try_get::<i64, _>("count").unwrap_or(0)
280        }
281        DatabasePool::MySQL(ref mysql_pool) => {
282            let result = sqlx::query(
283                "SELECT COUNT(*) as count FROM hammerwork_jobs 
284                 WHERE status = 'running' AND started_at < DATE_SUB(NOW(), INTERVAL 1 HOUR)",
285            )
286            .fetch_one(mysql_pool)
287            .await?;
288            result.try_get::<i64, _>("count").unwrap_or(0)
289        }
290    };
291
292    let stuck_status = if stuck_count == 0 {
293        ("🟢", "Stuck Jobs", "None".to_string())
294    } else if stuck_count < 5 {
295        ("🟔", "Stuck Jobs", format!("{} jobs", stuck_count))
296    } else {
297        ("šŸ”“", "Stuck Jobs", format!("{} jobs", stuck_count))
298    };
299    health_data.push(stuck_status);
300
301    // Check failure rate in last hour
302    let (total_recent, failed_recent) = match pool {
303        DatabasePool::Postgres(ref pg_pool) => {
304            let total_result = sqlx::query(
305                "SELECT COUNT(*) as count FROM hammerwork_jobs 
306                 WHERE created_at > NOW() - INTERVAL '1 hour'",
307            )
308            .fetch_one(pg_pool)
309            .await?;
310
311            let failed_result = sqlx::query(
312                "SELECT COUNT(*) as count FROM hammerwork_jobs 
313                 WHERE status = 'failed' AND failed_at > NOW() - INTERVAL '1 hour'",
314            )
315            .fetch_one(pg_pool)
316            .await?;
317
318            (
319                total_result.try_get::<i64, _>("count").unwrap_or(0),
320                failed_result.try_get::<i64, _>("count").unwrap_or(0),
321            )
322        }
323        DatabasePool::MySQL(ref mysql_pool) => {
324            let total_result = sqlx::query(
325                "SELECT COUNT(*) as count FROM hammerwork_jobs 
326                 WHERE created_at > DATE_SUB(NOW(), INTERVAL 1 HOUR)",
327            )
328            .fetch_one(mysql_pool)
329            .await?;
330
331            let failed_result = sqlx::query(
332                "SELECT COUNT(*) as count FROM hammerwork_jobs 
333                 WHERE status = 'failed' AND failed_at > DATE_SUB(NOW(), INTERVAL 1 HOUR)",
334            )
335            .fetch_one(mysql_pool)
336            .await?;
337
338            (
339                total_result.try_get::<i64, _>("count").unwrap_or(0),
340                failed_result.try_get::<i64, _>("count").unwrap_or(0),
341            )
342        }
343    };
344
345    let failure_rate = if total_recent > 0 {
346        (failed_recent as f64 / total_recent as f64) * 100.0
347    } else {
348        0.0
349    };
350
351    let failure_status = if failure_rate < 5.0 {
352        ("🟢", "Failure Rate", format!("{:.1}%", failure_rate))
353    } else if failure_rate < 15.0 {
354        ("🟔", "Failure Rate", format!("{:.1}%", failure_rate))
355    } else {
356        ("šŸ”“", "Failure Rate", format!("{:.1}%", failure_rate))
357    };
358    health_data.push(failure_status);
359
360    // Output results
361    match format.as_deref() {
362        Some("json") => {
363            let json_health = serde_json::json!({
364                "timestamp": chrono::Utc::now().to_rfc3339(),
365                "checks": health_data.iter().map(|(status, check, value)| {
366                    serde_json::json!({
367                        "check": check,
368                        "status": if status.contains("🟢") { "healthy" } else if status.contains("🟔") { "warning" } else { "critical" },
369                        "value": value
370                    })
371                }).collect::<Vec<_>>()
372            });
373            println!("{}", serde_json::to_string_pretty(&json_health)?);
374        }
375        _ => {
376            let mut table = comfy_table::Table::new();
377            table.set_header(vec!["Status", "Check", "Value"]);
378
379            for (status, check, value) in health_data {
380                table.add_row(vec![status, check, &value]);
381            }
382
383            println!("šŸ„ System Health Check");
384            println!("{}", table);
385        }
386    }
387
388    Ok(())
389}
390
391async fn show_metrics(
392    pool: DatabasePool,
393    period: Option<String>,
394    queue: Option<String>,
395) -> Result<()> {
396    let period_str = period.as_deref().unwrap_or("24h");
397    let (pg_interval, mysql_interval) = match period_str {
398        "1h" => ("1 hour", "1 HOUR"),
399        "24h" => ("24 hours", "24 HOUR"),
400        "7d" => ("7 days", "7 DAY"),
401        _ => ("24 hours", "24 HOUR"),
402    };
403
404    println!("šŸ“Š Performance Metrics ({})", period_str);
405    if let Some(q) = &queue {
406        println!("šŸŽÆ Queue: {}", q);
407    }
408    println!("═══════════════════════════════");
409
410    let queue_filter = if let Some(q) = queue {
411        format!(" AND queue_name = '{}'", q)
412    } else {
413        String::new()
414    };
415
416    match pool {
417        DatabasePool::Postgres(pg_pool) => {
418            // Throughput metrics
419            let throughput_result = sqlx::query(&format!(
420                "SELECT 
421                    COUNT(*) as total_jobs,
422                    COUNT(CASE WHEN status = 'completed' THEN 1 END) as completed_jobs,
423                    COUNT(CASE WHEN status = 'failed' THEN 1 END) as failed_jobs
424                 FROM hammerwork_jobs 
425                 WHERE created_at > NOW() - INTERVAL '{}'{}",
426                pg_interval, queue_filter
427            ))
428            .fetch_one(&pg_pool)
429            .await?;
430
431            let total: i64 = throughput_result.try_get("total_jobs")?;
432            let completed: i64 = throughput_result.try_get("completed_jobs")?;
433            let failed: i64 = throughput_result.try_get("failed_jobs")?;
434
435            println!("šŸ“ˆ Throughput:");
436            println!("   Total Jobs: {}", total);
437            println!(
438                "   Completed: {} ({:.1}%)",
439                completed,
440                if total > 0 {
441                    (completed as f64 / total as f64) * 100.0
442                } else {
443                    0.0
444                }
445            );
446            println!(
447                "   Failed: {} ({:.1}%)",
448                failed,
449                if total > 0 {
450                    (failed as f64 / total as f64) * 100.0
451                } else {
452                    0.0
453                }
454            );
455
456            // Average processing time for completed jobs
457            let avg_time_result = sqlx::query(&format!(
458                "SELECT AVG(EXTRACT(EPOCH FROM (completed_at - started_at))) as avg_duration
459                 FROM hammerwork_jobs 
460                 WHERE status = 'completed' AND completed_at > NOW() - INTERVAL '{}'{}",
461                pg_interval, queue_filter
462            ))
463            .fetch_one(&pg_pool)
464            .await?;
465
466            if let Ok(Some(avg_duration)) =
467                avg_time_result.try_get::<Option<f64>, _>("avg_duration")
468            {
469                println!("   Avg Processing Time: {:.1}s", avg_duration);
470            }
471        }
472        DatabasePool::MySQL(mysql_pool) => {
473            let throughput_result = sqlx::query(&format!(
474                "SELECT 
475                    COUNT(*) as total_jobs,
476                    COUNT(CASE WHEN status = 'completed' THEN 1 END) as completed_jobs,
477                    COUNT(CASE WHEN status = 'failed' THEN 1 END) as failed_jobs
478                 FROM hammerwork_jobs 
479                 WHERE created_at > DATE_SUB(NOW(), INTERVAL {}){}",
480                mysql_interval, queue_filter
481            ))
482            .fetch_one(&mysql_pool)
483            .await?;
484
485            let total: i64 = throughput_result.try_get("total_jobs")?;
486            let completed: i64 = throughput_result.try_get("completed_jobs")?;
487            let failed: i64 = throughput_result.try_get("failed_jobs")?;
488
489            println!("šŸ“ˆ Throughput:");
490            println!("   Total Jobs: {}", total);
491            println!(
492                "   Completed: {} ({:.1}%)",
493                completed,
494                if total > 0 {
495                    (completed as f64 / total as f64) * 100.0
496                } else {
497                    0.0
498                }
499            );
500            println!(
501                "   Failed: {} ({:.1}%)",
502                failed,
503                if total > 0 {
504                    (failed as f64 / total as f64) * 100.0
505                } else {
506                    0.0
507                }
508            );
509
510            // Average processing time for completed jobs
511            let avg_time_result = sqlx::query(&format!(
512                "SELECT AVG(TIMESTAMPDIFF(SECOND, started_at, completed_at)) as avg_duration
513                 FROM hammerwork_jobs 
514                 WHERE status = 'completed' AND completed_at > DATE_SUB(NOW(), INTERVAL {}){}",
515                mysql_interval, queue_filter
516            ))
517            .fetch_one(&mysql_pool)
518            .await?;
519
520            if let Ok(Some(avg_duration)) =
521                avg_time_result.try_get::<Option<f64>, _>("avg_duration")
522            {
523                println!("   Avg Processing Time: {:.1}s", avg_duration);
524            }
525        }
526    }
527
528    Ok(())
529}
530
531async fn show_logs(
532    _pool: DatabasePool,
533    lines: Option<u32>,
534    queue: Option<String>,
535    follow: bool,
536) -> Result<()> {
537    let lines_count = lines.unwrap_or(50);
538
539    println!("šŸ“œ Job Logs (Placeholder)");
540    if let Some(q) = &queue {
541        println!("šŸŽÆ Queue: {}", q);
542    }
543    println!("šŸ“„ Lines: {}", lines_count);
544    if follow {
545        println!("šŸ‘ļø  Following logs...");
546    }
547    println!("═══════════════════════════");
548
549    // In a real implementation, this would:
550    // 1. Query job events/logs from a logs table
551    // 2. Show recent job state changes
552    // 3. Display error messages and stack traces
553    // 4. Support real-time tailing with --follow
554
555    // Mock log entries
556    let mock_logs = vec![
557        (
558            "2024-06-28 10:30:15",
559            "INFO",
560            "emails",
561            "Job 12345678 started processing",
562        ),
563        (
564            "2024-06-28 10:30:14",
565            "INFO",
566            "emails",
567            "Job 87654321 completed successfully",
568        ),
569        (
570            "2024-06-28 10:30:12",
571            "ERROR",
572            "reports",
573            "Job 11111111 failed: Connection timeout",
574        ),
575        (
576            "2024-06-28 10:30:10",
577            "INFO",
578            "notifications",
579            "Job 22222222 enqueued",
580        ),
581        (
582            "2024-06-28 10:30:08",
583            "WARN",
584            "emails",
585            "Job 33333333 retry attempt 2/3",
586        ),
587    ];
588
589    for (timestamp, level, log_queue, message) in mock_logs.iter().take(lines_count as usize) {
590        if let Some(filter_queue) = &queue {
591            if log_queue != filter_queue {
592                continue;
593            }
594        }
595
596        let level_icon = match *level {
597            "INFO" => "ā„¹ļø",
598            "WARN" => "āš ļø",
599            "ERROR" => "āŒ",
600            _ => "šŸ“",
601        };
602
603        println!(
604            "{} {} [{}] {}: {}",
605            timestamp, level_icon, level, log_queue, message
606        );
607    }
608
609    if follow {
610        println!("\nšŸ‘ļø  Following logs (Ctrl+C to stop)...");
611        println!("šŸ’” This is a placeholder. Real implementation would stream live logs.");
612
613        // Simulate following logs
614        tokio::select! {
615            _ = tokio::time::sleep(tokio::time::Duration::from_secs(5)) => {
616                println!("ā° Demo timeout - stopping log follow");
617            }
618            _ = tokio::signal::ctrl_c() => {
619                println!("\nšŸ‘‹ Log following stopped");
620            }
621        }
622    }
623
624    Ok(())
625}