1use std::collections::HashMap;
57use std::future::Future;
58use std::sync::Arc;
59use std::time::Duration;
60
61use acton_reactive::prelude::{Reply, *};
62use serde::{Deserialize, Serialize};
63use tokio::sync::Semaphore;
64use tokio_util::sync::CancellationToken;
65use tokio_util::task::TaskTracker;
66
67use super::messages::{
68 CancelTask, CleanupFinishedTasks, GetAllTaskStatuses, GetTaskStatus, RegisterTask,
69 TaskCompleted, TaskStatusResponse, WaitForTask,
70};
71
72fn default_task_shutdown_timeout_secs() -> u64 {
73 5
74}
75
76#[derive(Debug, Clone, Serialize, Deserialize)]
78pub struct BackgroundWorkerConfig {
79 #[serde(default)]
81 pub enabled: bool,
82 #[serde(default)]
84 pub max_concurrent_tasks: usize,
85 #[serde(default = "default_task_shutdown_timeout_secs")]
87 pub task_shutdown_timeout_secs: u64,
88 #[serde(default)]
90 pub cleanup_interval_secs: u64,
91}
92
93impl Default for BackgroundWorkerConfig {
94 fn default() -> Self {
95 Self {
96 enabled: false,
97 max_concurrent_tasks: 0,
98 task_shutdown_timeout_secs: 5,
99 cleanup_interval_secs: 0,
100 }
101 }
102}
103
104#[derive(Clone, Debug, Default, PartialEq, Eq)]
106pub enum TaskStatus {
107 #[default]
109 Pending,
110 Running,
112 Completed,
114 Failed(String),
116 Cancelled,
118}
119
120impl TaskStatus {
121 #[must_use]
123 pub const fn is_terminal(&self) -> bool {
124 matches!(self, Self::Completed | Self::Failed(_) | Self::Cancelled)
125 }
126}
127
128#[derive(Debug)]
134struct TaskRecord {
135 cancellation_token: CancellationToken,
137 status: TaskStatus,
139}
140
141#[derive(Debug, Default)]
143pub struct BackgroundWorkerState {
144 tasks: HashMap<String, TaskRecord>,
146 waiters: HashMap<String, Vec<OutboundEnvelope>>,
148 root_token: Option<CancellationToken>,
150 tracker: Option<TaskTracker>,
152 shutdown_timeout: Duration,
154}
155
156#[derive(Clone)]
161pub struct BackgroundWorker {
162 agent_handle: ActorHandle,
164 root_token: CancellationToken,
166 semaphore: Option<Arc<Semaphore>>,
168 tracker: TaskTracker,
170 shutdown_timeout: Duration,
172}
173
174impl BackgroundWorker {
175 pub async fn spawn(
181 runtime: &mut ActorRuntime,
182 config: &BackgroundWorkerConfig,
183 ) -> anyhow::Result<Self> {
184 let root_token = CancellationToken::new();
185 let tracker = TaskTracker::new();
186 let shutdown_timeout = Duration::from_secs(config.task_shutdown_timeout_secs);
187
188 let semaphore = if config.max_concurrent_tasks > 0 {
189 Some(Arc::new(Semaphore::new(config.max_concurrent_tasks)))
190 } else {
191 None
192 };
193
194 let mut agent = runtime.new_actor::<BackgroundWorkerState>();
195
196 agent.model.root_token = Some(root_token.clone());
197 agent.model.tracker = Some(tracker.clone());
198 agent.model.shutdown_timeout = shutdown_timeout;
199
200 agent.mutate_on::<RegisterTask>(|agent, envelope| {
203 let msg = envelope.message();
204 agent.model.tasks.insert(
205 msg.task_id.clone(),
206 TaskRecord {
207 cancellation_token: msg.cancellation_token.clone(),
208 status: TaskStatus::Running,
209 },
210 );
211 tracing::info!(task_id = %msg.task_id, "Background task submitted");
212 Reply::ready()
213 });
214
215 agent.mutate_on::<TaskCompleted>(|agent, envelope| {
218 let msg = envelope.message();
219 if let Some(record) = agent.model.tasks.get_mut(&msg.task_id) {
220 record.status = msg.status.clone();
221 }
222
223 tracing::debug!(
224 task_id = %msg.task_id,
225 status = ?msg.status,
226 "Background task reached a terminal state"
227 );
228
229 let waiters = agent.model.waiters.remove(&msg.task_id).unwrap_or_default();
231 let response = TaskStatusResponse {
232 task_id: msg.task_id.clone(),
233 status: msg.status.clone(),
234 };
235
236 Reply::pending(async move {
237 for waiter in waiters {
238 waiter.send(Some(response.clone())).await;
239 }
240 })
241 });
242
243 agent.mutate_on::<CancelTask>(|agent, envelope| {
247 let task_id = &envelope.message().task_id;
248 if let Some(record) = agent.model.tasks.get(task_id) {
249 record.cancellation_token.cancel();
250 tracing::info!(task_id = %task_id, "Task cancellation requested");
251 } else {
252 tracing::warn!(task_id = %task_id, "Task not found for cancellation");
253 }
254 Reply::ready()
255 });
256
257 agent.mutate_on::<WaitForTask>(|agent, envelope| {
261 let task_id = envelope.message().task_id.clone();
262 let reply = envelope.reply_envelope();
263
264 let settled = match agent.model.tasks.get(&task_id) {
265 None => Some(None),
266 Some(record) if record.status.is_terminal() => Some(Some(TaskStatusResponse {
267 task_id: task_id.clone(),
268 status: record.status.clone(),
269 })),
270 Some(_) => None,
271 };
272
273 match settled {
274 Some(response) => Reply::pending(async move {
275 reply.send(response).await;
276 }),
277 None => {
278 agent.model.waiters.entry(task_id).or_default().push(reply);
279 Reply::ready()
280 }
281 }
282 });
283
284 agent.act_on::<GetTaskStatus>(|agent, envelope| {
285 let task_id = envelope.message().task_id.clone();
286 let reply = envelope.reply_envelope();
287
288 let response = agent
289 .model
290 .tasks
291 .get(&task_id)
292 .map(|record| TaskStatusResponse {
293 task_id,
294 status: record.status.clone(),
295 });
296
297 Reply::pending(async move {
298 reply.send(response).await;
299 })
300 });
301
302 agent.act_on::<GetAllTaskStatuses>(|agent, envelope| {
303 let reply = envelope.reply_envelope();
304 let statuses: Vec<TaskStatusResponse> = agent
305 .model
306 .tasks
307 .iter()
308 .map(|(task_id, record)| TaskStatusResponse {
309 task_id: task_id.clone(),
310 status: record.status.clone(),
311 })
312 .collect();
313
314 Reply::pending(async move {
315 reply.send(statuses).await;
316 })
317 });
318
319 agent.mutate_on::<CleanupFinishedTasks>(|agent, envelope| {
321 let before = agent.model.tasks.len();
322 agent
323 .model
324 .tasks
325 .retain(|_, record| !record.status.is_terminal());
326 let removed = before - agent.model.tasks.len();
327
328 let reply = envelope.reply_envelope();
329 Reply::pending(async move {
330 reply.send(removed).await;
331 })
332 });
333
334 agent.before_stop(|agent| {
336 let root_token = agent.model.root_token.clone();
337 let tracker = agent.model.tracker.clone();
338 let timeout = agent.model.shutdown_timeout;
339 let task_count = agent.model.tasks.len();
340
341 Reply::pending(async move {
342 let Some(tracker) = tracker else {
343 return;
344 };
345
346 if task_count == 0 && tracker.is_empty() {
347 tracing::info!("BackgroundWorker stopping with no active tasks");
348 return;
349 }
350
351 tracing::info!(
352 task_count,
353 "BackgroundWorker stopping, cancelling all tasks..."
354 );
355
356 if let Some(token) = root_token {
357 token.cancel();
358 }
359
360 tracker.close();
362 if tokio::time::timeout(timeout, tracker.wait()).await.is_err() {
363 tracing::warn!(
364 remaining = tracker.len(),
365 "Background tasks did not stop within the shutdown timeout"
366 );
367 } else {
368 tracing::info!("All background tasks stopped");
369 }
370 })
371 });
372
373 agent.after_start(|_agent| {
374 tracing::info!("BackgroundWorker agent started");
375 Reply::ready()
376 });
377
378 let handle = agent.start().await;
379
380 let worker = Self {
381 agent_handle: handle,
382 root_token,
383 semaphore,
384 tracker,
385 shutdown_timeout,
386 };
387
388 if config.cleanup_interval_secs > 0 {
390 let cleanup_handle = worker.agent_handle.clone();
391 let cleanup_token = worker.root_token.child_token();
392 let interval = Duration::from_secs(config.cleanup_interval_secs);
393 tokio::spawn(async move {
394 let mut ticker = tokio::time::interval(interval);
395 ticker.tick().await;
396 loop {
397 tokio::select! {
398 biased;
399 () = cleanup_token.cancelled() => break,
400 _ = ticker.tick() => {
401 cleanup_handle.send(CleanupFinishedTasks).await;
402 tracing::debug!("Periodic background task cleanup requested");
403 }
404 }
405 }
406 });
407 }
408
409 Ok(worker)
410 }
411
412 pub async fn submit<F, Fut>(&self, task_id: impl Into<String>, work: F)
432 where
433 F: FnOnce() -> Fut + Send + 'static,
434 Fut: Future<Output = anyhow::Result<()>> + Send + 'static,
435 {
436 let task_id = task_id.into();
437
438 let permit = if let Some(ref sem) = self.semaphore {
440 match sem.clone().acquire_owned().await {
441 Ok(permit) => Some(permit),
442 Err(_) => {
443 tracing::warn!(task_id = %task_id, "Semaphore closed, task not submitted");
444 return;
445 }
446 }
447 } else {
448 None
449 };
450
451 let cancel_token = self.root_token.child_token();
452
453 self.agent_handle
456 .send(RegisterTask {
457 task_id: task_id.clone(),
458 cancellation_token: cancel_token.clone(),
459 })
460 .await;
461
462 let agent_handle = self.agent_handle.clone();
463 self.tracker.spawn(async move {
464 let _permit = permit;
465
466 let status = tokio::select! {
467 biased;
468
469 () = cancel_token.cancelled() => {
470 tracing::debug!(task_id = %task_id, "Task cancelled");
471 TaskStatus::Cancelled
472 }
473 result = work() => match result {
474 Ok(()) => {
475 tracing::debug!(task_id = %task_id, "Task completed successfully");
476 TaskStatus::Completed
477 }
478 Err(e) => {
479 tracing::warn!(task_id = %task_id, error = %e, "Task failed");
480 TaskStatus::Failed(e.to_string())
481 }
482 },
483 };
484
485 agent_handle.send(TaskCompleted { task_id, status }).await;
486 });
487 }
488
489 pub async fn cancel(&self, task_id: impl Into<String>) -> Result<Option<TaskStatus>, AskError> {
499 let task_id = task_id.into();
500 self.agent_handle
501 .send(CancelTask {
502 task_id: task_id.clone(),
503 })
504 .await;
505
506 let response = self
507 .agent_handle
508 .ask_with_timeout(WaitForTask { task_id }, self.shutdown_timeout)
509 .await?;
510
511 Ok(response.map(|r| r.status))
512 }
513
514 pub async fn wait_for_task(
523 &self,
524 task_id: impl Into<String>,
525 ) -> Result<Option<TaskStatus>, AskError> {
526 let response = self
527 .agent_handle
528 .ask(WaitForTask {
529 task_id: task_id.into(),
530 })
531 .await?;
532 Ok(response.map(|r| r.status))
533 }
534
535 pub async fn task_status(
541 &self,
542 task_id: impl Into<String>,
543 ) -> Result<Option<TaskStatus>, AskError> {
544 let response = self
545 .agent_handle
546 .ask(GetTaskStatus {
547 task_id: task_id.into(),
548 })
549 .await?;
550 Ok(response.map(|r| r.status))
551 }
552
553 pub async fn all_task_statuses(&self) -> Result<Vec<TaskStatusResponse>, AskError> {
559 self.agent_handle.ask(GetAllTaskStatuses).await
560 }
561
562 pub async fn task_count(&self) -> Result<usize, AskError> {
568 Ok(self.all_task_statuses().await?.len())
569 }
570
571 pub async fn running_task_count(&self) -> Result<usize, AskError> {
577 Ok(self
578 .all_task_statuses()
579 .await?
580 .into_iter()
581 .filter(|r| r.status == TaskStatus::Running)
582 .count())
583 }
584
585 pub async fn has_task(&self, task_id: impl Into<String>) -> Result<bool, AskError> {
591 Ok(self.task_status(task_id).await?.is_some())
592 }
593
594 pub async fn cleanup_finished_tasks(&self) -> Result<usize, AskError> {
602 self.agent_handle.ask(CleanupFinishedTasks).await
603 }
604
605 #[must_use]
607 pub fn handle(&self) -> &ActorHandle {
608 &self.agent_handle
609 }
610
611 #[must_use]
613 pub fn shutdown_timeout(&self) -> Duration {
614 self.shutdown_timeout
615 }
616}
617
618#[cfg(test)]
619mod tests {
620 use super::*;
621
622 #[test]
623 fn test_config_defaults() {
624 let config = BackgroundWorkerConfig::default();
625 assert!(!config.enabled);
626 assert_eq!(config.max_concurrent_tasks, 0);
627 assert_eq!(config.task_shutdown_timeout_secs, 5);
628 assert_eq!(config.cleanup_interval_secs, 0);
629 }
630
631 #[test]
632 fn test_config_serde_empty_object() {
633 let config: BackgroundWorkerConfig = serde_json::from_str("{}").unwrap();
634 assert!(!config.enabled);
635 assert_eq!(config.max_concurrent_tasks, 0);
636 assert_eq!(config.task_shutdown_timeout_secs, 5);
637 assert_eq!(config.cleanup_interval_secs, 0);
638 }
639
640 #[test]
641 fn test_config_serde_partial() {
642 let config: BackgroundWorkerConfig =
643 serde_json::from_str(r#"{"enabled": true, "max_concurrent_tasks": 10}"#).unwrap();
644 assert!(config.enabled);
645 assert_eq!(config.max_concurrent_tasks, 10);
646 assert_eq!(config.task_shutdown_timeout_secs, 5);
647 assert_eq!(config.cleanup_interval_secs, 0);
648 }
649
650 #[test]
651 fn terminal_statuses_are_classified() {
652 assert!(TaskStatus::Completed.is_terminal());
653 assert!(TaskStatus::Cancelled.is_terminal());
654 assert!(TaskStatus::Failed("boom".into()).is_terminal());
655 assert!(!TaskStatus::Running.is_terminal());
656 assert!(!TaskStatus::Pending.is_terminal());
657 }
658
659 async fn worker_with(config: BackgroundWorkerConfig) -> (ActorRuntime, BackgroundWorker) {
660 let mut runtime = ActonApp::launch_async().await;
661 let worker = BackgroundWorker::spawn(&mut runtime, &config)
662 .await
663 .unwrap();
664 (runtime, worker)
665 }
666
667 #[tokio::test]
668 async fn completed_task_reports_its_own_outcome() {
669 let (mut runtime, worker) = worker_with(BackgroundWorkerConfig::default()).await;
670
671 worker.submit("job", || async { Ok(()) }).await;
672
673 let status = worker.wait_for_task("job").await.unwrap();
676 assert_eq!(status, Some(TaskStatus::Completed));
677
678 runtime.shutdown_all().await.unwrap();
679 }
680
681 #[tokio::test]
682 async fn failing_task_records_its_error() {
683 let (mut runtime, worker) = worker_with(BackgroundWorkerConfig::default()).await;
684
685 worker
686 .submit("doomed", || async { Err(anyhow::anyhow!("kaboom")) })
687 .await;
688
689 let status = worker.wait_for_task("doomed").await.unwrap();
690 assert_eq!(status, Some(TaskStatus::Failed("kaboom".into())));
691
692 runtime.shutdown_all().await.unwrap();
693 }
694
695 #[tokio::test]
696 async fn unknown_task_is_reported_as_absent() {
697 let (mut runtime, worker) = worker_with(BackgroundWorkerConfig::default()).await;
698
699 assert_eq!(worker.task_status("nope").await.unwrap(), None);
700 assert!(!worker.has_task("nope").await.unwrap());
701
702 runtime.shutdown_all().await.unwrap();
703 }
704
705 #[tokio::test]
706 async fn cancel_stops_a_running_task_and_waits_for_it() {
707 let (mut runtime, worker) = worker_with(BackgroundWorkerConfig::default()).await;
708
709 worker
711 .submit("forever", || async {
712 std::future::pending::<()>().await;
713 Ok(())
714 })
715 .await;
716
717 let status = worker.cancel("forever").await.unwrap();
718 assert_eq!(
719 status,
720 Some(TaskStatus::Cancelled),
721 "cancel should not return until the task has actually stopped"
722 );
723
724 runtime.shutdown_all().await.unwrap();
725 }
726
727 #[tokio::test]
728 async fn cancelling_an_unknown_task_reports_absence() {
729 let (mut runtime, worker) = worker_with(BackgroundWorkerConfig::default()).await;
730
731 assert_eq!(worker.cancel("ghost").await.unwrap(), None);
732
733 runtime.shutdown_all().await.unwrap();
734 }
735
736 #[tokio::test]
737 async fn cleanup_drops_only_finished_tasks() {
738 let (mut runtime, worker) = worker_with(BackgroundWorkerConfig::default()).await;
739
740 for i in 0..3 {
741 worker
742 .submit(format!("done-{i}"), || async { Ok(()) })
743 .await;
744 }
745 worker
746 .submit("still-running", || async {
747 std::future::pending::<()>().await;
748 Ok(())
749 })
750 .await;
751
752 for i in 0..3 {
753 worker.wait_for_task(format!("done-{i}")).await.unwrap();
754 }
755
756 assert_eq!(worker.task_count().await.unwrap(), 4);
757 assert_eq!(worker.cleanup_finished_tasks().await.unwrap(), 3);
758 assert_eq!(worker.task_count().await.unwrap(), 1);
759 assert!(
760 worker.has_task("still-running").await.unwrap(),
761 "a running task must survive cleanup"
762 );
763
764 runtime.shutdown_all().await.unwrap();
765 }
766
767 #[tokio::test]
768 async fn semaphore_caps_concurrent_tasks() {
769 let (mut runtime, worker) = worker_with(BackgroundWorkerConfig {
770 enabled: true,
771 max_concurrent_tasks: 2,
772 task_shutdown_timeout_secs: 5,
773 cleanup_interval_secs: 0,
774 })
775 .await;
776
777 let (release_tx, release_rx) = tokio::sync::watch::channel(false);
778 let running = Arc::new(std::sync::atomic::AtomicUsize::new(0));
779 let max_observed = Arc::new(std::sync::atomic::AtomicUsize::new(0));
780 let started = Arc::new(tokio::sync::Semaphore::new(0));
781
782 for i in 0..4 {
785 let rx = release_rx.clone();
786 let running = running.clone();
787 let max_observed = max_observed.clone();
788 let started = started.clone();
789 let worker = worker.clone();
790 tokio::spawn(async move {
791 worker
792 .submit(format!("task-{i}"), move || async move {
793 let current = running.fetch_add(1, std::sync::atomic::Ordering::SeqCst) + 1;
794 max_observed.fetch_max(current, std::sync::atomic::Ordering::SeqCst);
795 started.add_permits(1);
796
797 let mut rx = rx;
798 let _ = rx.wait_for(|v| *v).await;
799
800 running.fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
801 Ok(())
802 })
803 .await;
804 });
805 }
806
807 let _ = started.acquire_many(2).await.unwrap();
811 assert_eq!(
812 max_observed.load(std::sync::atomic::Ordering::SeqCst),
813 2,
814 "at most max_concurrent_tasks should run at once"
815 );
816
817 release_tx.send(true).unwrap();
818 for i in 0..4 {
819 worker.wait_for_task(format!("task-{i}")).await.unwrap();
820 }
821 assert_eq!(
822 max_observed.load(std::sync::atomic::Ordering::SeqCst),
823 2,
824 "the cap should hold for the whole run, not just at the start"
825 );
826
827 runtime.shutdown_all().await.unwrap();
828 }
829
830 #[tokio::test]
831 async fn test_configurable_shutdown_timeout() {
832 let (mut runtime, worker) = worker_with(BackgroundWorkerConfig {
833 enabled: true,
834 max_concurrent_tasks: 0,
835 task_shutdown_timeout_secs: 10,
836 cleanup_interval_secs: 0,
837 })
838 .await;
839
840 assert_eq!(worker.shutdown_timeout(), Duration::from_secs(10));
841
842 runtime.shutdown_all().await.unwrap();
843 }
844
845 #[tokio::test]
846 async fn shutdown_cancels_tasks_that_are_still_running() {
847 let (mut runtime, worker) = worker_with(BackgroundWorkerConfig {
848 enabled: true,
849 max_concurrent_tasks: 0,
850 task_shutdown_timeout_secs: 5,
851 cleanup_interval_secs: 0,
852 })
853 .await;
854
855 let observed_cancel = Arc::new(std::sync::atomic::AtomicBool::new(false));
856 let flag = observed_cancel.clone();
857
858 worker
859 .submit("long-runner", move || async move {
860 std::future::pending::<()>().await;
861 flag.store(true, std::sync::atomic::Ordering::SeqCst);
862 Ok(())
863 })
864 .await;
865
866 runtime.shutdown_all().await.unwrap();
870
871 assert!(
872 !observed_cancel.load(std::sync::atomic::Ordering::SeqCst),
873 "the task body should have been cancelled, not run to completion"
874 );
875 }
876}