cargo_hammerwork/commands/
maintenance.rs1use anyhow::Result;
2use clap::Subcommand;
3use tracing::info;
4
5use crate::config::Config;
6use crate::utils::database::DatabasePool;
7use crate::utils::db_helpers::*;
8
9#[derive(Subcommand)]
10pub enum MaintenanceCommand {
11 #[command(about = "Clean up old completed and failed jobs")]
12 Vacuum {
13 #[arg(short = 'u', long, help = "Database connection URL")]
14 database_url: Option<String>,
15 #[arg(long, default_value = "30", help = "Days to keep completed jobs")]
16 keep_completed_days: u32,
17 #[arg(long, default_value = "7", help = "Days to keep failed jobs")]
18 keep_failed_days: u32,
19 #[arg(long, help = "Confirm the vacuum operation")]
20 confirm: bool,
21 #[arg(long, help = "Dry run - show what would be deleted")]
22 dry_run: bool,
23 },
24 #[command(about = "Clean up dead/stale jobs")]
25 DeadJobs {
26 #[arg(short = 'u', long, help = "Database connection URL")]
27 database_url: Option<String>,
28 #[arg(
29 long,
30 default_value = "24",
31 help = "Hours without update to consider dead"
32 )]
33 stale_hours: u32,
34 #[arg(long, help = "Confirm the cleanup operation")]
35 confirm: bool,
36 #[arg(long, help = "Dry run - show what would be cleaned")]
37 dry_run: bool,
38 },
39 #[command(about = "Rebuild database indexes for optimal performance")]
40 Reindex {
41 #[arg(short = 'u', long, help = "Database connection URL")]
42 database_url: Option<String>,
43 #[arg(long, help = "Confirm the reindex operation")]
44 confirm: bool,
45 },
46 #[command(about = "Update database statistics for query optimization")]
47 Analyze {
48 #[arg(short = 'u', long, help = "Database connection URL")]
49 database_url: Option<String>,
50 },
51 #[command(about = "Check database integrity and job consistency")]
52 Check {
53 #[arg(short = 'u', long, help = "Database connection URL")]
54 database_url: Option<String>,
55 #[arg(long, help = "Fix minor issues automatically")]
56 fix: bool,
57 },
58}
59
60impl MaintenanceCommand {
61 pub async fn execute(&self, config: &Config) -> Result<()> {
62 let db_url = self.get_database_url(config)?;
63 let pool = DatabasePool::connect(&db_url, config.get_connection_pool_size()).await?;
64
65 match self {
66 MaintenanceCommand::Vacuum {
67 keep_completed_days,
68 keep_failed_days,
69 confirm,
70 dry_run,
71 ..
72 } => {
73 vacuum_jobs(
74 pool,
75 *keep_completed_days,
76 *keep_failed_days,
77 *confirm,
78 *dry_run,
79 )
80 .await?;
81 }
82 MaintenanceCommand::DeadJobs {
83 stale_hours,
84 confirm,
85 dry_run,
86 ..
87 } => {
88 cleanup_dead_jobs(pool, *stale_hours, *confirm, *dry_run).await?;
89 }
90 MaintenanceCommand::Reindex { confirm, .. } => {
91 reindex_database(pool, *confirm).await?;
92 }
93 MaintenanceCommand::Analyze { .. } => {
94 analyze_database(pool).await?;
95 }
96 MaintenanceCommand::Check { fix, .. } => {
97 check_database(pool, *fix).await?;
98 }
99 }
100 Ok(())
101 }
102
103 fn get_database_url(&self, config: &Config) -> Result<String> {
104 let url = match self {
105 MaintenanceCommand::Vacuum { database_url, .. } => database_url,
106 MaintenanceCommand::DeadJobs { database_url, .. } => database_url,
107 MaintenanceCommand::Reindex { database_url, .. } => database_url,
108 MaintenanceCommand::Analyze { database_url, .. } => database_url,
109 MaintenanceCommand::Check { database_url, .. } => database_url,
110 };
111
112 url.as_ref()
113 .map(|s| s.as_str())
114 .or(config.get_database_url())
115 .ok_or_else(|| anyhow::anyhow!("Database URL is required"))
116 .map(|s| s.to_string())
117 }
118}
119
120async fn vacuum_jobs(
121 pool: DatabasePool,
122 keep_completed_days: u32,
123 keep_failed_days: u32,
124 confirm: bool,
125 dry_run: bool,
126) -> Result<()> {
127 if !confirm && !dry_run {
128 println!(
129 "โ ๏ธ This will permanently delete old jobs. Use --confirm to proceed or --dry-run to preview."
130 );
131 return Ok(());
132 }
133
134 let completed_cutoff = chrono::Utc::now() - chrono::Duration::days(keep_completed_days as i64);
135 let failed_cutoff = chrono::Utc::now() - chrono::Duration::days(keep_failed_days as i64);
136
137 let completed_query = format!(
139 "SELECT COUNT(*) as count FROM hammerwork_jobs WHERE status = 'Completed' AND completed_at < '{}'",
140 completed_cutoff.format("%Y-%m-%d %H:%M:%S")
141 );
142 let failed_query = format!(
143 "SELECT COUNT(*) as count FROM hammerwork_jobs WHERE status = 'Failed' AND failed_at < '{}'",
144 failed_cutoff.format("%Y-%m-%d %H:%M:%S")
145 );
146
147 let completed_count = execute_count_query(&pool, &completed_query).await?;
148 let failed_count = execute_count_query(&pool, &failed_query).await?;
149
150 println!("๐งน Vacuum Analysis");
151 println!("โโโโโโโโโโโโโโโโโโโโ");
152 println!(
153 "Completed jobs older than {} days: {}",
154 keep_completed_days, completed_count
155 );
156 println!(
157 "Failed jobs older than {} days: {}",
158 keep_failed_days, failed_count
159 );
160 println!("Total jobs to delete: {}", completed_count + failed_count);
161
162 if dry_run {
163 println!("\n๐ก This was a dry run. Use --confirm to actually delete these jobs.");
164 return Ok(());
165 }
166
167 if completed_count == 0 && failed_count == 0 {
168 println!("โจ No old jobs found. Database is clean!");
169 return Ok(());
170 }
171
172 info!("Starting vacuum operation");
173
174 if completed_count > 0 {
176 let delete_completed_query = format!(
177 "DELETE FROM hammerwork_jobs WHERE status = 'Completed' AND completed_at < '{}'",
178 completed_cutoff.format("%Y-%m-%d %H:%M:%S")
179 );
180 execute_update_query(&pool, &delete_completed_query).await?;
181 info!("Deleted {} completed jobs", completed_count);
182 }
183
184 if failed_count > 0 {
186 let delete_failed_query = format!(
187 "DELETE FROM hammerwork_jobs WHERE status = 'Failed' AND failed_at < '{}'",
188 failed_cutoff.format("%Y-%m-%d %H:%M:%S")
189 );
190 execute_update_query(&pool, &delete_failed_query).await?;
191 info!("Deleted {} failed jobs", failed_count);
192 }
193
194 println!("โ
Vacuum completed successfully");
195 println!(" Deleted {} completed jobs", completed_count);
196 println!(" Deleted {} failed jobs", failed_count);
197
198 Ok(())
199}
200
201async fn cleanup_dead_jobs(
202 pool: DatabasePool,
203 stale_hours: u32,
204 confirm: bool,
205 dry_run: bool,
206) -> Result<()> {
207 if !confirm && !dry_run {
208 println!(
209 "โ ๏ธ This will mark stale jobs as dead. Use --confirm to proceed or --dry-run to preview."
210 );
211 return Ok(());
212 }
213
214 let stale_cutoff = chrono::Utc::now() - chrono::Duration::hours(stale_hours as i64);
215
216 let query = format!(
218 "SELECT COUNT(*) as count FROM hammerwork_jobs WHERE status = 'Running' AND started_at < '{}'",
219 stale_cutoff.format("%Y-%m-%d %H:%M:%S")
220 );
221 let stale_count = execute_count_query(&pool, &query).await?;
222
223 println!("๐ Dead Jobs Analysis");
224 println!("โโโโโโโโโโโโโโโโโโโโโโโ");
225 println!(
226 "Stale running jobs (no update for {} hours): {}",
227 stale_hours, stale_count
228 );
229
230 if dry_run {
231 println!("\n๐ก This was a dry run. Use --confirm to actually mark these jobs as dead.");
232 return Ok(());
233 }
234
235 if stale_count == 0 {
236 println!("โจ No stale jobs found!");
237 return Ok(());
238 }
239
240 info!("Marking {} stale jobs as dead", stale_count);
241
242 let update_query = format!(
244 "UPDATE hammerwork_jobs SET status = 'Dead', error_message = 'Job marked as dead due to inactivity' WHERE status = 'Running' AND started_at < '{}'",
245 stale_cutoff.format("%Y-%m-%d %H:%M:%S")
246 );
247 execute_update_query(&pool, &update_query).await?;
248
249 println!("โ
Dead jobs cleanup completed");
250 println!(" Marked {} jobs as dead", stale_count);
251
252 Ok(())
253}
254
255async fn reindex_database(pool: DatabasePool, confirm: bool) -> Result<()> {
256 if !confirm {
257 println!("โ ๏ธ This will rebuild database indexes. Use --confirm to proceed.");
258 return Ok(());
259 }
260
261 info!("Starting database reindex operation");
262
263 match &pool {
264 DatabasePool::Postgres(pg_pool) => {
265 println!("๐ Rebuilding PostgreSQL indexes...");
267
268 sqlx::query("REINDEX INDEX idx_hammerwork_queue_priority")
269 .execute(pg_pool)
270 .await?;
271 sqlx::query("REINDEX INDEX idx_hammerwork_status")
272 .execute(pg_pool)
273 .await?;
274 sqlx::query("REINDEX INDEX idx_hammerwork_scheduled")
275 .execute(pg_pool)
276 .await?;
277 sqlx::query("REINDEX INDEX idx_hammerwork_cron")
278 .execute(pg_pool)
279 .await?;
280 sqlx::query("REINDEX INDEX idx_hammerwork_batch")
281 .execute(pg_pool)
282 .await?;
283 sqlx::query("REINDEX INDEX idx_hammerwork_failed_at")
284 .execute(pg_pool)
285 .await?;
286 sqlx::query("REINDEX INDEX idx_hammerwork_completed_at")
287 .execute(pg_pool)
288 .await?;
289
290 println!("โ
PostgreSQL indexes rebuilt");
291 }
292 DatabasePool::MySQL(mysql_pool) => {
293 println!("๐ Optimizing MySQL tables...");
295
296 sqlx::query("OPTIMIZE TABLE hammerwork_jobs")
297 .execute(mysql_pool)
298 .await?;
299
300 println!("โ
MySQL tables optimized");
301 }
302 }
303
304 info!("Database reindex completed successfully");
305 Ok(())
306}
307
308async fn analyze_database(pool: DatabasePool) -> Result<()> {
309 info!("Analyzing database statistics");
310
311 match &pool {
312 DatabasePool::Postgres(pg_pool) => {
313 println!("๐ Updating PostgreSQL statistics...");
314 sqlx::query("ANALYZE hammerwork_jobs")
315 .execute(pg_pool)
316 .await?;
317 println!("โ
PostgreSQL statistics updated");
318 }
319 DatabasePool::MySQL(mysql_pool) => {
320 println!("๐ Updating MySQL statistics...");
321 sqlx::query("ANALYZE TABLE hammerwork_jobs")
322 .execute(mysql_pool)
323 .await?;
324 println!("โ
MySQL statistics updated");
325 }
326 }
327
328 Ok(())
329}
330
331async fn check_database(pool: DatabasePool, fix: bool) -> Result<()> {
332 info!("Checking database integrity");
333
334 println!("๐ Database Integrity Check");
335 println!("โโโโโโโโโโโโโโโโโโโโโโโโโโโ");
336
337 let orphaned_query = "SELECT COUNT(*) as count FROM hammerwork_jobs WHERE status = 'Running' AND started_at IS NULL";
339 let orphaned_count = execute_count_query(&pool, orphaned_query).await?;
340
341 println!("Orphaned running jobs (no start time): {}", orphaned_count);
342
343 if fix && orphaned_count > 0 {
344 let fix_query = "UPDATE hammerwork_jobs SET status = 'Pending' WHERE status = 'Running' AND started_at IS NULL";
345 execute_update_query(&pool, fix_query).await?;
346 println!("โ
Fixed {} orphaned jobs", orphaned_count);
347 }
348
349 let invalid_priority_query = "SELECT COUNT(*) as count FROM hammerwork_jobs WHERE priority NOT IN ('background', 'low', 'normal', 'high', 'critical')";
351 let invalid_priority_count = execute_count_query(&pool, invalid_priority_query).await?;
352
353 println!("Jobs with invalid priority: {}", invalid_priority_count);
354
355 if fix && invalid_priority_count > 0 {
356 let fix_priority_query = "UPDATE hammerwork_jobs SET priority = 'normal' WHERE priority NOT IN ('background', 'low', 'normal', 'high', 'critical')";
357 execute_update_query(&pool, fix_priority_query).await?;
358 println!(
359 "โ
Fixed {} jobs with invalid priority",
360 invalid_priority_count
361 );
362 }
363
364 let negative_attempts_query =
366 "SELECT COUNT(*) as count FROM hammerwork_jobs WHERE attempts < 0";
367 let negative_attempts_count = execute_count_query(&pool, negative_attempts_query).await?;
368
369 println!("Jobs with negative attempts: {}", negative_attempts_count);
370
371 if fix && negative_attempts_count > 0 {
372 let fix_attempts_query = "UPDATE hammerwork_jobs SET attempts = 0 WHERE attempts < 0";
373 execute_update_query(&pool, fix_attempts_query).await?;
374 println!(
375 "โ
Fixed {} jobs with negative attempts",
376 negative_attempts_count
377 );
378 }
379
380 let total_issues = orphaned_count + invalid_priority_count + negative_attempts_count;
381 if total_issues == 0 {
382 println!("\nโจ Database integrity check passed! No issues found.");
383 } else if fix {
384 println!("\nโ
Database check completed with fixes applied");
385 } else {
386 println!(
387 "\nโ ๏ธ Found {} issues. Use --fix to automatically repair them.",
388 total_issues
389 );
390 }
391
392 Ok(())
393}