Skip to main content

acton_service/agents/
background_worker.rs

1//! Background Task Management Agent
2//!
3//! This module provides a managed alternative to ad-hoc `tokio::spawn` calls.
4//! The `BackgroundWorker` agent offers:
5//!
6//! - **Named task tracking**: Each task has a unique ID for monitoring
7//! - **Graceful shutdown**: Tasks are cancelled and awaited during agent shutdown
8//! - **Status monitoring**: Query task status via message passing
9//! - **Cancellation support**: Individual tasks can be cancelled on demand
10//! - **Concurrency limiting**: Optional semaphore-based backpressure
11//! - **Periodic cleanup**: Automatic removal of finished tasks
12//!
13//! # Architecture
14//!
15//! The agent owns the task registry outright: it lives in
16//! [`BackgroundWorkerState`] as a plain `HashMap`, reached only from handlers,
17//! which the runtime already serializes. Nothing here is behind a lock, because
18//! nothing else writes it.
19//!
20//! Spawned work reports its outcome back as a [`TaskCompleted`] message rather
21//! than writing a shared status cell. Registration is sent before the task is
22//! spawned, and mailboxes are FIFO, so the agent can never see a completion for
23//! a task it has not yet registered.
24//!
25//! The one shared handle is a [`TaskTracker`], which exists so `before_stop` can
26//! await in-flight work. That is a task registry, not decision state: the agent
27//! never branches on it, and draining cannot be done through the mailbox because
28//! the message loop is not running while `before_stop` is.
29//!
30//! # Example
31//!
32//! ```rust,ignore
33//! use acton_service::agents::prelude::*;
34//!
35//! let mut runtime = ActonApp::launch();
36//! let config = BackgroundWorkerConfig { enabled: true, ..Default::default() };
37//! let worker = BackgroundWorker::spawn(&mut runtime, &config).await?;
38//!
39//! // Submit a background task
40//! worker.submit("my-task", || async move {
41//!     // Do background work
42//!     tokio::time::sleep(Duration::from_secs(10)).await;
43//!     Ok(())
44//! }).await;
45//!
46//! // Check task status
47//! let status = worker.task_status("my-task").await?;
48//!
49//! // Cancel a specific task and wait for it to stop
50//! worker.cancel("my-task").await?;
51//!
52//! // Graceful shutdown cancels all remaining tasks
53//! runtime.shutdown_all().await?;
54//! ```
55
56use std::collections::HashMap;
57use std::future::Future;
58use std::sync::Arc;
59use std::time::Duration;
60
61use acton_reactive::prelude::{Reply, *};
62use serde::{Deserialize, Serialize};
63use tokio::sync::Semaphore;
64use tokio_util::sync::CancellationToken;
65use tokio_util::task::TaskTracker;
66
67use super::messages::{
68    CancelTask, CleanupFinishedTasks, GetAllTaskStatuses, GetTaskStatus, RegisterTask,
69    TaskCompleted, TaskStatusResponse, WaitForTask,
70};
71
72fn default_task_shutdown_timeout_secs() -> u64 {
73    5
74}
75
76/// Configuration for the background worker agent
77#[derive(Debug, Clone, Serialize, Deserialize)]
78pub struct BackgroundWorkerConfig {
79    /// Whether the background worker is enabled
80    #[serde(default)]
81    pub enabled: bool,
82    /// Maximum number of concurrent tasks (0 = unlimited)
83    #[serde(default)]
84    pub max_concurrent_tasks: usize,
85    /// Timeout in seconds for individual task shutdown during cancellation
86    #[serde(default = "default_task_shutdown_timeout_secs")]
87    pub task_shutdown_timeout_secs: u64,
88    /// Interval in seconds for automatic cleanup of finished tasks (0 = disabled)
89    #[serde(default)]
90    pub cleanup_interval_secs: u64,
91}
92
93impl Default for BackgroundWorkerConfig {
94    fn default() -> Self {
95        Self {
96            enabled: false,
97            max_concurrent_tasks: 0,
98            task_shutdown_timeout_secs: 5,
99            cleanup_interval_secs: 0,
100        }
101    }
102}
103
104/// Status of a background task
105#[derive(Clone, Debug, Default, PartialEq, Eq)]
106pub enum TaskStatus {
107    /// Task is queued but not yet started
108    #[default]
109    Pending,
110    /// Task is currently running
111    Running,
112    /// Task completed successfully
113    Completed,
114    /// Task failed with an error
115    Failed(String),
116    /// Task was cancelled
117    Cancelled,
118}
119
120impl TaskStatus {
121    /// Whether the task has reached a state it will not leave.
122    #[must_use]
123    pub const fn is_terminal(&self) -> bool {
124        matches!(self, Self::Completed | Self::Failed(_) | Self::Cancelled)
125    }
126}
127
128/// One tracked task, owned exclusively by the agent.
129///
130/// Plain fields rather than `Arc<Mutex<..>>`: the agent's message loop is the
131/// mutual exclusion, so a lock here would guard against a writer that does not
132/// exist.
133#[derive(Debug)]
134struct TaskRecord {
135    /// Cancels this task specifically
136    cancellation_token: CancellationToken,
137    /// Current status, updated only by the `TaskCompleted` handler
138    status: TaskStatus,
139}
140
141/// State for the background worker agent
142#[derive(Debug, Default)]
143pub struct BackgroundWorkerState {
144    /// Every task the agent knows about, keyed by task ID
145    tasks: HashMap<String, TaskRecord>,
146    /// Callers blocked in `WaitForTask`, keyed by the task they are waiting on
147    waiters: HashMap<String, Vec<OutboundEnvelope>>,
148    /// Root cancellation token; cancelling it cancels every task's child token
149    root_token: Option<CancellationToken>,
150    /// Tracks spawned tasks so shutdown can await them
151    tracker: Option<TaskTracker>,
152    /// How long shutdown waits for in-flight tasks
153    shutdown_timeout: Duration,
154}
155
156/// Service wrapper for the background worker agent
157///
158/// Provides a clean API for submitting and managing background tasks. Every
159/// query goes through the agent, which is the only owner of task state.
160#[derive(Clone)]
161pub struct BackgroundWorker {
162    /// Handle for sending messages to the agent
163    agent_handle: ActorHandle,
164    /// Root cancellation token for creating child tokens
165    root_token: CancellationToken,
166    /// Optional semaphore for concurrency limiting
167    semaphore: Option<Arc<Semaphore>>,
168    /// Tracks spawned tasks so shutdown can await them
169    tracker: TaskTracker,
170    /// Timeout for individual task shutdown
171    shutdown_timeout: Duration,
172}
173
174impl BackgroundWorker {
175    /// Spawn a new background worker agent
176    ///
177    /// The worker will manage background tasks with graceful shutdown support.
178    /// Configuration controls concurrency limits, shutdown timeouts, and
179    /// periodic cleanup behavior.
180    pub async fn spawn(
181        runtime: &mut ActorRuntime,
182        config: &BackgroundWorkerConfig,
183    ) -> anyhow::Result<Self> {
184        let root_token = CancellationToken::new();
185        let tracker = TaskTracker::new();
186        let shutdown_timeout = Duration::from_secs(config.task_shutdown_timeout_secs);
187
188        let semaphore = if config.max_concurrent_tasks > 0 {
189            Some(Arc::new(Semaphore::new(config.max_concurrent_tasks)))
190        } else {
191            None
192        };
193
194        let mut agent = runtime.new_actor::<BackgroundWorkerState>();
195
196        agent.model.root_token = Some(root_token.clone());
197        agent.model.tracker = Some(tracker.clone());
198        agent.model.shutdown_timeout = shutdown_timeout;
199
200        // A task becomes visible to the agent here, before it is spawned, so a
201        // completion can never arrive for an unknown ID.
202        agent.mutate_on::<RegisterTask>(|agent, envelope| {
203            let msg = envelope.message();
204            agent.model.tasks.insert(
205                msg.task_id.clone(),
206                TaskRecord {
207                    cancellation_token: msg.cancellation_token.clone(),
208                    status: TaskStatus::Running,
209                },
210            );
211            tracing::info!(task_id = %msg.task_id, "Background task submitted");
212            Reply::ready()
213        });
214
215        // The spawned task reports its own outcome; the agent is the only
216        // writer of status.
217        agent.mutate_on::<TaskCompleted>(|agent, envelope| {
218            let msg = envelope.message();
219            if let Some(record) = agent.model.tasks.get_mut(&msg.task_id) {
220                record.status = msg.status.clone();
221            }
222
223            tracing::debug!(
224                task_id = %msg.task_id,
225                status = ?msg.status,
226                "Background task reached a terminal state"
227            );
228
229            // Release anyone blocked on this task finishing.
230            let waiters = agent.model.waiters.remove(&msg.task_id).unwrap_or_default();
231            let response = TaskStatusResponse {
232                task_id: msg.task_id.clone(),
233                status: msg.status.clone(),
234            };
235
236            Reply::pending(async move {
237                for waiter in waiters {
238                    waiter.send(Some(response.clone())).await;
239                }
240            })
241        });
242
243        // Cancellation is a request, not a wait: the task notices its token and
244        // reports back through `TaskCompleted`. Callers who want to wait use
245        // `WaitForTask`, which is what `BackgroundWorker::cancel` does.
246        agent.mutate_on::<CancelTask>(|agent, envelope| {
247            let task_id = &envelope.message().task_id;
248            if let Some(record) = agent.model.tasks.get(task_id) {
249                record.cancellation_token.cancel();
250                tracing::info!(task_id = %task_id, "Task cancellation requested");
251            } else {
252                tracing::warn!(task_id = %task_id, "Task not found for cancellation");
253            }
254            Reply::ready()
255        });
256
257        // Answers now if the task is already finished or unknown, otherwise
258        // parks the reply envelope until `TaskCompleted` arrives. Holding the
259        // envelope is what lets a caller await completion without polling.
260        agent.mutate_on::<WaitForTask>(|agent, envelope| {
261            let task_id = envelope.message().task_id.clone();
262            let reply = envelope.reply_envelope();
263
264            let settled = match agent.model.tasks.get(&task_id) {
265                None => Some(None),
266                Some(record) if record.status.is_terminal() => Some(Some(TaskStatusResponse {
267                    task_id: task_id.clone(),
268                    status: record.status.clone(),
269                })),
270                Some(_) => None,
271            };
272
273            match settled {
274                Some(response) => Reply::pending(async move {
275                    reply.send(response).await;
276                }),
277                None => {
278                    agent.model.waiters.entry(task_id).or_default().push(reply);
279                    Reply::ready()
280                }
281            }
282        });
283
284        agent.act_on::<GetTaskStatus>(|agent, envelope| {
285            let task_id = envelope.message().task_id.clone();
286            let reply = envelope.reply_envelope();
287
288            let response = agent
289                .model
290                .tasks
291                .get(&task_id)
292                .map(|record| TaskStatusResponse {
293                    task_id,
294                    status: record.status.clone(),
295                });
296
297            Reply::pending(async move {
298                reply.send(response).await;
299            })
300        });
301
302        agent.act_on::<GetAllTaskStatuses>(|agent, envelope| {
303            let reply = envelope.reply_envelope();
304            let statuses: Vec<TaskStatusResponse> = agent
305                .model
306                .tasks
307                .iter()
308                .map(|(task_id, record)| TaskStatusResponse {
309                    task_id: task_id.clone(),
310                    status: record.status.clone(),
311                })
312                .collect();
313
314            Reply::pending(async move {
315                reply.send(statuses).await;
316            })
317        });
318
319        // Drop finished tasks so the registry does not grow without bound.
320        agent.mutate_on::<CleanupFinishedTasks>(|agent, envelope| {
321            let before = agent.model.tasks.len();
322            agent
323                .model
324                .tasks
325                .retain(|_, record| !record.status.is_terminal());
326            let removed = before - agent.model.tasks.len();
327
328            let reply = envelope.reply_envelope();
329            Reply::pending(async move {
330                reply.send(removed).await;
331            })
332        });
333
334        // Graceful shutdown - cancel all tasks, then wait for them
335        agent.before_stop(|agent| {
336            let root_token = agent.model.root_token.clone();
337            let tracker = agent.model.tracker.clone();
338            let timeout = agent.model.shutdown_timeout;
339            let task_count = agent.model.tasks.len();
340
341            Reply::pending(async move {
342                let Some(tracker) = tracker else {
343                    return;
344                };
345
346                if task_count == 0 && tracker.is_empty() {
347                    tracing::info!("BackgroundWorker stopping with no active tasks");
348                    return;
349                }
350
351                tracing::info!(
352                    task_count,
353                    "BackgroundWorker stopping, cancelling all tasks..."
354                );
355
356                if let Some(token) = root_token {
357                    token.cancel();
358                }
359
360                // Closing refuses new registrations so `wait` can terminate.
361                tracker.close();
362                if tokio::time::timeout(timeout, tracker.wait()).await.is_err() {
363                    tracing::warn!(
364                        remaining = tracker.len(),
365                        "Background tasks did not stop within the shutdown timeout"
366                    );
367                } else {
368                    tracing::info!("All background tasks stopped");
369                }
370            })
371        });
372
373        agent.after_start(|_agent| {
374            tracing::info!("BackgroundWorker agent started");
375            Reply::ready()
376        });
377
378        let handle = agent.start().await;
379
380        let worker = Self {
381            agent_handle: handle,
382            root_token,
383            semaphore,
384            tracker,
385            shutdown_timeout,
386        };
387
388        // Spawn periodic cleanup task if configured
389        if config.cleanup_interval_secs > 0 {
390            let cleanup_handle = worker.agent_handle.clone();
391            let cleanup_token = worker.root_token.child_token();
392            let interval = Duration::from_secs(config.cleanup_interval_secs);
393            tokio::spawn(async move {
394                let mut ticker = tokio::time::interval(interval);
395                ticker.tick().await;
396                loop {
397                    tokio::select! {
398                        biased;
399                        () = cleanup_token.cancelled() => break,
400                        _ = ticker.tick() => {
401                            cleanup_handle.send(CleanupFinishedTasks).await;
402                            tracing::debug!("Periodic background task cleanup requested");
403                        }
404                    }
405                }
406            });
407        }
408
409        Ok(worker)
410    }
411
412    /// Submit a new background task
413    ///
414    /// The task is spawned and tracked by the agent. If a concurrency limit is
415    /// configured, this method awaits a free slot, providing backpressure to
416    /// callers.
417    ///
418    /// # Arguments
419    ///
420    /// * `task_id` - Unique identifier for the task
421    /// * `work` - Closure producing the future that performs the work
422    ///
423    /// # Example
424    ///
425    /// ```rust,ignore
426    /// worker.submit("cleanup-job", || async {
427    ///     do_cleanup().await?;
428    ///     Ok(())
429    /// }).await;
430    /// ```
431    pub async fn submit<F, Fut>(&self, task_id: impl Into<String>, work: F)
432    where
433        F: FnOnce() -> Fut + Send + 'static,
434        Fut: Future<Output = anyhow::Result<()>> + Send + 'static,
435    {
436        let task_id = task_id.into();
437
438        // Acquire concurrency permit before spawning (provides backpressure)
439        let permit = if let Some(ref sem) = self.semaphore {
440            match sem.clone().acquire_owned().await {
441                Ok(permit) => Some(permit),
442                Err(_) => {
443                    tracing::warn!(task_id = %task_id, "Semaphore closed, task not submitted");
444                    return;
445                }
446            }
447        } else {
448            None
449        };
450
451        let cancel_token = self.root_token.child_token();
452
453        // Registered before the task exists. Mailboxes are FIFO, so the agent
454        // processes this ahead of the completion the task will send.
455        self.agent_handle
456            .send(RegisterTask {
457                task_id: task_id.clone(),
458                cancellation_token: cancel_token.clone(),
459            })
460            .await;
461
462        let agent_handle = self.agent_handle.clone();
463        self.tracker.spawn(async move {
464            let _permit = permit;
465
466            let status = tokio::select! {
467                biased;
468
469                () = cancel_token.cancelled() => {
470                    tracing::debug!(task_id = %task_id, "Task cancelled");
471                    TaskStatus::Cancelled
472                }
473                result = work() => match result {
474                    Ok(()) => {
475                        tracing::debug!(task_id = %task_id, "Task completed successfully");
476                        TaskStatus::Completed
477                    }
478                    Err(e) => {
479                        tracing::warn!(task_id = %task_id, error = %e, "Task failed");
480                        TaskStatus::Failed(e.to_string())
481                    }
482                },
483            };
484
485            agent_handle.send(TaskCompleted { task_id, status }).await;
486        });
487    }
488
489    /// Cancel a task and wait for it to stop
490    ///
491    /// Returns the task's final status, or `None` if no such task is tracked.
492    /// Waiting is bounded by the configured shutdown timeout.
493    ///
494    /// # Errors
495    ///
496    /// Returns [`AskError`] if the agent cannot be reached or does not answer
497    /// within the timeout.
498    pub async fn cancel(&self, task_id: impl Into<String>) -> Result<Option<TaskStatus>, AskError> {
499        let task_id = task_id.into();
500        self.agent_handle
501            .send(CancelTask {
502                task_id: task_id.clone(),
503            })
504            .await;
505
506        let response = self
507            .agent_handle
508            .ask_with_timeout(WaitForTask { task_id }, self.shutdown_timeout)
509            .await?;
510
511        Ok(response.map(|r| r.status))
512    }
513
514    /// Wait for a task to reach a terminal state, returning its final status
515    ///
516    /// Returns `None` if no such task is tracked. Answers immediately for a
517    /// task that has already finished.
518    ///
519    /// # Errors
520    ///
521    /// Returns [`AskError`] if the agent cannot be reached or does not answer.
522    pub async fn wait_for_task(
523        &self,
524        task_id: impl Into<String>,
525    ) -> Result<Option<TaskStatus>, AskError> {
526        let response = self
527            .agent_handle
528            .ask(WaitForTask {
529                task_id: task_id.into(),
530            })
531            .await?;
532        Ok(response.map(|r| r.status))
533    }
534
535    /// Get the status of a specific task, or `None` if it is not tracked
536    ///
537    /// # Errors
538    ///
539    /// Returns [`AskError`] if the agent cannot be reached or does not answer.
540    pub async fn task_status(
541        &self,
542        task_id: impl Into<String>,
543    ) -> Result<Option<TaskStatus>, AskError> {
544        let response = self
545            .agent_handle
546            .ask(GetTaskStatus {
547                task_id: task_id.into(),
548            })
549            .await?;
550        Ok(response.map(|r| r.status))
551    }
552
553    /// Get the status of every tracked task
554    ///
555    /// # Errors
556    ///
557    /// Returns [`AskError`] if the agent cannot be reached or does not answer.
558    pub async fn all_task_statuses(&self) -> Result<Vec<TaskStatusResponse>, AskError> {
559        self.agent_handle.ask(GetAllTaskStatuses).await
560    }
561
562    /// Get the count of tracked tasks
563    ///
564    /// # Errors
565    ///
566    /// Returns [`AskError`] if the agent cannot be reached or does not answer.
567    pub async fn task_count(&self) -> Result<usize, AskError> {
568        Ok(self.all_task_statuses().await?.len())
569    }
570
571    /// Get the count of tasks that have not yet reached a terminal state
572    ///
573    /// # Errors
574    ///
575    /// Returns [`AskError`] if the agent cannot be reached or does not answer.
576    pub async fn running_task_count(&self) -> Result<usize, AskError> {
577        Ok(self
578            .all_task_statuses()
579            .await?
580            .into_iter()
581            .filter(|r| r.status == TaskStatus::Running)
582            .count())
583    }
584
585    /// Check if a task is tracked
586    ///
587    /// # Errors
588    ///
589    /// Returns [`AskError`] if the agent cannot be reached or does not answer.
590    pub async fn has_task(&self, task_id: impl Into<String>) -> Result<bool, AskError> {
591        Ok(self.task_status(task_id).await?.is_some())
592    }
593
594    /// Remove completed, failed and cancelled tasks from tracking
595    ///
596    /// Returns how many records were dropped.
597    ///
598    /// # Errors
599    ///
600    /// Returns [`AskError`] if the agent cannot be reached or does not answer.
601    pub async fn cleanup_finished_tasks(&self) -> Result<usize, AskError> {
602        self.agent_handle.ask(CleanupFinishedTasks).await
603    }
604
605    /// Get the agent handle for direct message sending
606    #[must_use]
607    pub fn handle(&self) -> &ActorHandle {
608        &self.agent_handle
609    }
610
611    /// Get the configured shutdown timeout
612    #[must_use]
613    pub fn shutdown_timeout(&self) -> Duration {
614        self.shutdown_timeout
615    }
616}
617
618#[cfg(test)]
619mod tests {
620    use super::*;
621
622    #[test]
623    fn test_config_defaults() {
624        let config = BackgroundWorkerConfig::default();
625        assert!(!config.enabled);
626        assert_eq!(config.max_concurrent_tasks, 0);
627        assert_eq!(config.task_shutdown_timeout_secs, 5);
628        assert_eq!(config.cleanup_interval_secs, 0);
629    }
630
631    #[test]
632    fn test_config_serde_empty_object() {
633        let config: BackgroundWorkerConfig = serde_json::from_str("{}").unwrap();
634        assert!(!config.enabled);
635        assert_eq!(config.max_concurrent_tasks, 0);
636        assert_eq!(config.task_shutdown_timeout_secs, 5);
637        assert_eq!(config.cleanup_interval_secs, 0);
638    }
639
640    #[test]
641    fn test_config_serde_partial() {
642        let config: BackgroundWorkerConfig =
643            serde_json::from_str(r#"{"enabled": true, "max_concurrent_tasks": 10}"#).unwrap();
644        assert!(config.enabled);
645        assert_eq!(config.max_concurrent_tasks, 10);
646        assert_eq!(config.task_shutdown_timeout_secs, 5);
647        assert_eq!(config.cleanup_interval_secs, 0);
648    }
649
650    #[test]
651    fn terminal_statuses_are_classified() {
652        assert!(TaskStatus::Completed.is_terminal());
653        assert!(TaskStatus::Cancelled.is_terminal());
654        assert!(TaskStatus::Failed("boom".into()).is_terminal());
655        assert!(!TaskStatus::Running.is_terminal());
656        assert!(!TaskStatus::Pending.is_terminal());
657    }
658
659    async fn worker_with(config: BackgroundWorkerConfig) -> (ActorRuntime, BackgroundWorker) {
660        let mut runtime = ActonApp::launch_async().await;
661        let worker = BackgroundWorker::spawn(&mut runtime, &config)
662            .await
663            .unwrap();
664        (runtime, worker)
665    }
666
667    #[tokio::test]
668    async fn completed_task_reports_its_own_outcome() {
669        let (mut runtime, worker) = worker_with(BackgroundWorkerConfig::default()).await;
670
671        worker.submit("job", || async { Ok(()) }).await;
672
673        // wait_for_task is the barrier: the agent answers it only once the
674        // spawned task has reported completion.
675        let status = worker.wait_for_task("job").await.unwrap();
676        assert_eq!(status, Some(TaskStatus::Completed));
677
678        runtime.shutdown_all().await.unwrap();
679    }
680
681    #[tokio::test]
682    async fn failing_task_records_its_error() {
683        let (mut runtime, worker) = worker_with(BackgroundWorkerConfig::default()).await;
684
685        worker
686            .submit("doomed", || async { Err(anyhow::anyhow!("kaboom")) })
687            .await;
688
689        let status = worker.wait_for_task("doomed").await.unwrap();
690        assert_eq!(status, Some(TaskStatus::Failed("kaboom".into())));
691
692        runtime.shutdown_all().await.unwrap();
693    }
694
695    #[tokio::test]
696    async fn unknown_task_is_reported_as_absent() {
697        let (mut runtime, worker) = worker_with(BackgroundWorkerConfig::default()).await;
698
699        assert_eq!(worker.task_status("nope").await.unwrap(), None);
700        assert!(!worker.has_task("nope").await.unwrap());
701
702        runtime.shutdown_all().await.unwrap();
703    }
704
705    #[tokio::test]
706    async fn cancel_stops_a_running_task_and_waits_for_it() {
707        let (mut runtime, worker) = worker_with(BackgroundWorkerConfig::default()).await;
708
709        // A task that never finishes on its own, so only cancellation can end it.
710        worker
711            .submit("forever", || async {
712                std::future::pending::<()>().await;
713                Ok(())
714            })
715            .await;
716
717        let status = worker.cancel("forever").await.unwrap();
718        assert_eq!(
719            status,
720            Some(TaskStatus::Cancelled),
721            "cancel should not return until the task has actually stopped"
722        );
723
724        runtime.shutdown_all().await.unwrap();
725    }
726
727    #[tokio::test]
728    async fn cancelling_an_unknown_task_reports_absence() {
729        let (mut runtime, worker) = worker_with(BackgroundWorkerConfig::default()).await;
730
731        assert_eq!(worker.cancel("ghost").await.unwrap(), None);
732
733        runtime.shutdown_all().await.unwrap();
734    }
735
736    #[tokio::test]
737    async fn cleanup_drops_only_finished_tasks() {
738        let (mut runtime, worker) = worker_with(BackgroundWorkerConfig::default()).await;
739
740        for i in 0..3 {
741            worker
742                .submit(format!("done-{i}"), || async { Ok(()) })
743                .await;
744        }
745        worker
746            .submit("still-running", || async {
747                std::future::pending::<()>().await;
748                Ok(())
749            })
750            .await;
751
752        for i in 0..3 {
753            worker.wait_for_task(format!("done-{i}")).await.unwrap();
754        }
755
756        assert_eq!(worker.task_count().await.unwrap(), 4);
757        assert_eq!(worker.cleanup_finished_tasks().await.unwrap(), 3);
758        assert_eq!(worker.task_count().await.unwrap(), 1);
759        assert!(
760            worker.has_task("still-running").await.unwrap(),
761            "a running task must survive cleanup"
762        );
763
764        runtime.shutdown_all().await.unwrap();
765    }
766
767    #[tokio::test]
768    async fn semaphore_caps_concurrent_tasks() {
769        let (mut runtime, worker) = worker_with(BackgroundWorkerConfig {
770            enabled: true,
771            max_concurrent_tasks: 2,
772            task_shutdown_timeout_secs: 5,
773            cleanup_interval_secs: 0,
774        })
775        .await;
776
777        let (release_tx, release_rx) = tokio::sync::watch::channel(false);
778        let running = Arc::new(std::sync::atomic::AtomicUsize::new(0));
779        let max_observed = Arc::new(std::sync::atomic::AtomicUsize::new(0));
780        let started = Arc::new(tokio::sync::Semaphore::new(0));
781
782        // submit() blocks on the permit, so the submits must run concurrently
783        // or the third would wait forever on a permit held by the first two.
784        for i in 0..4 {
785            let rx = release_rx.clone();
786            let running = running.clone();
787            let max_observed = max_observed.clone();
788            let started = started.clone();
789            let worker = worker.clone();
790            tokio::spawn(async move {
791                worker
792                    .submit(format!("task-{i}"), move || async move {
793                        let current = running.fetch_add(1, std::sync::atomic::Ordering::SeqCst) + 1;
794                        max_observed.fetch_max(current, std::sync::atomic::Ordering::SeqCst);
795                        started.add_permits(1);
796
797                        let mut rx = rx;
798                        let _ = rx.wait_for(|v| *v).await;
799
800                        running.fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
801                        Ok(())
802                    })
803                    .await;
804            });
805        }
806
807        // Wait until the cap's worth of tasks are actually inside their bodies.
808        // This is the barrier that makes the assertion meaningful rather than
809        // merely early: two are running and the rest cannot start.
810        let _ = started.acquire_many(2).await.unwrap();
811        assert_eq!(
812            max_observed.load(std::sync::atomic::Ordering::SeqCst),
813            2,
814            "at most max_concurrent_tasks should run at once"
815        );
816
817        release_tx.send(true).unwrap();
818        for i in 0..4 {
819            worker.wait_for_task(format!("task-{i}")).await.unwrap();
820        }
821        assert_eq!(
822            max_observed.load(std::sync::atomic::Ordering::SeqCst),
823            2,
824            "the cap should hold for the whole run, not just at the start"
825        );
826
827        runtime.shutdown_all().await.unwrap();
828    }
829
830    #[tokio::test]
831    async fn test_configurable_shutdown_timeout() {
832        let (mut runtime, worker) = worker_with(BackgroundWorkerConfig {
833            enabled: true,
834            max_concurrent_tasks: 0,
835            task_shutdown_timeout_secs: 10,
836            cleanup_interval_secs: 0,
837        })
838        .await;
839
840        assert_eq!(worker.shutdown_timeout(), Duration::from_secs(10));
841
842        runtime.shutdown_all().await.unwrap();
843    }
844
845    #[tokio::test]
846    async fn shutdown_cancels_tasks_that_are_still_running() {
847        let (mut runtime, worker) = worker_with(BackgroundWorkerConfig {
848            enabled: true,
849            max_concurrent_tasks: 0,
850            task_shutdown_timeout_secs: 5,
851            cleanup_interval_secs: 0,
852        })
853        .await;
854
855        let observed_cancel = Arc::new(std::sync::atomic::AtomicBool::new(false));
856        let flag = observed_cancel.clone();
857
858        worker
859            .submit("long-runner", move || async move {
860                std::future::pending::<()>().await;
861                flag.store(true, std::sync::atomic::Ordering::SeqCst);
862                Ok(())
863            })
864            .await;
865
866        // shutdown_all runs before_stop, which cancels the root token and then
867        // waits on the tracker. If cancellation did not reach the task this
868        // would block until the 5s timeout instead of returning promptly.
869        runtime.shutdown_all().await.unwrap();
870
871        assert!(
872            !observed_cancel.load(std::sync::atomic::Ordering::SeqCst),
873            "the task body should have been cancelled, not run to completion"
874        );
875    }
876}