1use std::collections::HashSet;
5use std::panic::AssertUnwindSafe;
6use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
7use std::sync::Arc;
8use std::time::{Duration, Instant};
9
10use futures::FutureExt as _;
11use tokio::sync::Semaphore;
12use tracing::{debug, error, info, warn};
13
14use crate::client::TaskClient;
15use crate::configuration::{resolve_worker_config, WorkerConfig};
16use crate::error::Result;
17use crate::events::{
18 exception_label, EventDispatcher, PollCompleted, PollFailure, PollSkippedPaused, PollStarted,
19 TaskExecutionCompleted, TaskExecutionFailure, TaskExecutionStarted, TaskUpdateCompleted,
20 TaskUpdateFailure, ThreadUncaughtException,
21};
22use crate::models::Task;
23
24use super::{Worker, WorkerOutput};
25
26enum TaskOutcome {
32 Ok,
33 Err(crate::error::ConductorError),
34 Panic(String),
35}
36
37pub struct TaskRunner {
39 worker: Arc<dyn Worker>,
40 task_client: TaskClient,
41 config: Arc<WorkerConfig>,
42 event_dispatcher: EventDispatcher,
43
44 running: Arc<AtomicBool>,
46 paused: Arc<AtomicBool>,
47
48 consecutive_empty_polls: Arc<AtomicU64>,
50 last_poll_time: Arc<parking_lot::Mutex<Instant>>,
51
52 semaphore: Arc<Semaphore>,
54 active_task_count: Arc<AtomicUsize>,
56 running_tasks: Arc<parking_lot::Mutex<HashSet<String>>>,
58 spawned_task_count: Arc<AtomicUsize>,
60 poll_attempts: Arc<AtomicU64>,
66}
67
68impl TaskRunner {
69 pub fn new(
71 worker: Arc<dyn Worker>,
72 task_client: TaskClient,
73 event_dispatcher: EventDispatcher,
74 ) -> Self {
75 let defaults = WorkerConfig {
77 task_definition_name: worker.task_definition_name().to_owned(),
78 poll_interval: Duration::from_millis(worker.poll_interval_millis()),
79 domain: worker.domain().map(std::borrow::ToOwned::to_owned),
80 worker_id: worker.identity(),
81 thread_count: worker.thread_count(),
82 ..Default::default()
83 };
84
85 let config = resolve_worker_config(worker.task_definition_name(), defaults);
86
87 info!(
88 task_type = %config.task_definition_name,
89 worker_id = %config.worker_id,
90 thread_count = config.thread_count,
91 poll_interval_ms = config.poll_interval.as_millis(),
92 domain = ?config.domain,
93 paused = config.paused,
94 "Task runner initialized"
95 );
96
97 let paused = config.paused;
98 let thread_count = config.thread_count;
99
100 Self {
101 worker,
102 task_client,
103 config: Arc::new(config),
104 event_dispatcher,
105 running: Arc::new(AtomicBool::new(false)),
106 paused: Arc::new(AtomicBool::new(paused)),
107 consecutive_empty_polls: Arc::new(AtomicU64::new(0)),
108 last_poll_time: Arc::new(parking_lot::Mutex::new(Instant::now())),
109 semaphore: Arc::new(Semaphore::new(thread_count)),
110 active_task_count: Arc::new(AtomicUsize::new(0)),
111 running_tasks: Arc::new(parking_lot::Mutex::new(HashSet::new())),
112 spawned_task_count: Arc::new(AtomicUsize::new(0)),
113 poll_attempts: Arc::new(AtomicU64::new(0)),
114 }
115 }
116
117 #[must_use]
119 pub fn task_type(&self) -> &str {
120 &self.config.task_definition_name
121 }
122
123 #[must_use]
125 pub fn config(&self) -> &WorkerConfig {
126 &self.config
127 }
128
129 #[must_use]
131 pub fn active_task_count(&self) -> usize {
132 self.active_task_count.load(Ordering::SeqCst)
133 }
134
135 #[must_use]
137 pub fn spawned_task_count(&self) -> usize {
138 self.spawned_task_count.load(Ordering::SeqCst)
139 }
140
141 #[must_use]
145 pub fn poll_attempt_count(&self) -> u64 {
146 self.poll_attempts.load(Ordering::SeqCst)
147 }
148
149 #[must_use]
151 pub fn is_running(&self) -> bool {
152 self.running.load(Ordering::SeqCst)
153 }
154
155 #[must_use]
157 pub fn is_paused(&self) -> bool {
158 self.paused.load(Ordering::SeqCst)
159 }
160
161 pub fn pause(&self) {
163 self.paused.store(true, Ordering::SeqCst);
164 info!(task_type = %self.config.task_definition_name, "Task runner paused");
165 }
166
167 pub fn resume(&self) {
169 self.paused.store(false, Ordering::SeqCst);
170 info!(task_type = %self.config.task_definition_name, "Task runner resumed");
171 }
172
173 pub fn stop(&self) {
175 self.running.store(false, Ordering::SeqCst);
176 info!(task_type = %self.config.task_definition_name, "Task runner stopped");
177 }
178
179 pub async fn run(&self) {
181 self.running.store(true, Ordering::SeqCst);
182
183 info!(
184 task_type = %self.config.task_definition_name,
185 "Starting task runner polling loop"
186 );
187
188 while self.running.load(Ordering::SeqCst) {
189 if let Err(e) = self.run_once().await {
190 error!(
191 task_type = %self.config.task_definition_name,
192 error = %e,
193 "Error in polling loop"
194 );
195 }
196 }
197
198 self.wait_for_tasks_to_complete().await;
200
201 info!(
202 task_type = %self.config.task_definition_name,
203 "Task runner polling loop ended"
204 );
205 }
206
207 async fn wait_for_tasks_to_complete(&self) {
209 let shutdown_timeout = Duration::from_secs(30);
210 let start = Instant::now();
211
212 while self.spawned_task_count.load(Ordering::SeqCst) > 0 {
213 if start.elapsed() > shutdown_timeout {
214 let remaining = self.spawned_task_count.load(Ordering::SeqCst);
215 warn!(
216 task_type = %self.config.task_definition_name,
217 remaining_tasks = remaining,
218 "Shutdown timeout reached, {} tasks still in flight",
219 remaining
220 );
221 break;
222 }
223 tokio::time::sleep(Duration::from_millis(100)).await;
224 }
225 }
226
227 async fn run_once(&self) -> Result<()> {
229 if self.paused.load(Ordering::SeqCst) {
231 self.event_dispatcher
232 .publish_poll_skipped_paused(&PollSkippedPaused::new(
233 &self.config.task_definition_name,
234 &self.config.worker_id,
235 ));
236 tokio::time::sleep(Duration::from_millis(100)).await;
237 return Ok(());
238 }
239
240 let active_count = self.active_task_count.load(Ordering::SeqCst);
243 let available_slots = self.config.thread_count.saturating_sub(active_count);
244
245 if available_slots == 0 {
246 tokio::time::sleep(Duration::from_millis(1)).await;
248 return Ok(());
249 }
250
251 let empty_polls = self.consecutive_empty_polls.load(Ordering::SeqCst);
253 if empty_polls > 0 {
254 let backoff = Duration::from_millis(1 << empty_polls.min(10));
255 let backoff = backoff.min(self.config.poll_interval);
256
257 let elapsed = self.last_poll_time.lock().elapsed();
258 if elapsed < backoff {
259 tokio::time::sleep(backoff.checked_sub(elapsed).unwrap_or_default()).await;
260 }
261 }
262
263 let poll_start = Instant::now();
265
266 self.event_dispatcher
268 .publish_poll_started(&PollStarted::new(
269 &self.config.task_definition_name,
270 &self.config.worker_id,
271 available_slots,
272 ));
273
274 let poll_result = self
275 .task_client
276 .batch_poll(
277 &self.config.task_definition_name,
278 Some(&self.config.worker_id),
279 self.config.domain.as_deref(),
280 available_slots,
281 self.config.poll_timeout,
282 )
283 .await;
284
285 let poll_duration = poll_start.elapsed();
286 *self.last_poll_time.lock() = Instant::now();
287 self.poll_attempts.fetch_add(1, Ordering::SeqCst);
288
289 match poll_result {
290 Ok(tasks) => {
291 self.event_dispatcher
293 .publish_poll_completed(&PollCompleted::new(
294 &self.config.task_definition_name,
295 &self.config.worker_id,
296 poll_duration,
297 tasks.len(),
298 ));
299
300 if tasks.is_empty() {
301 self.consecutive_empty_polls.fetch_add(1, Ordering::SeqCst);
302 } else {
303 self.consecutive_empty_polls.store(0, Ordering::SeqCst);
304
305 for task in tasks {
307 self.spawn_task_execution(task);
308 }
309 }
310 }
311 Err(e) => {
312 let exception = exception_label(&e);
313
314 self.event_dispatcher
315 .publish_poll_failure(&PollFailure::new(
316 &self.config.task_definition_name,
317 &self.config.worker_id,
318 poll_duration,
319 e.to_string(),
320 exception,
321 ));
322
323 self.consecutive_empty_polls.fetch_add(1, Ordering::SeqCst);
324 }
325 }
326
327 Ok(())
328 }
329
330 fn spawn_task_execution(&self, task: Task) {
343 let task_id = task.task_id.clone();
344
345 self.spawned_task_count.fetch_add(1, Ordering::SeqCst);
347
348 let worker = Arc::clone(&self.worker);
349 let task_client = self.task_client.clone();
350 let event_dispatcher = self.event_dispatcher.clone();
351 let config = Arc::clone(&self.config);
352 let semaphore = Arc::clone(&self.semaphore);
353 let active_task_count = Arc::clone(&self.active_task_count);
354 let running_tasks = Arc::clone(&self.running_tasks);
355 let spawned_task_count = Arc::clone(&self.spawned_task_count);
356
357 let task_type = self.config.task_definition_name.clone();
358
359 tokio::spawn(async move {
360 let Ok(_permit) = semaphore.acquire().await else {
362 error!(task_id = %task_id, "Semaphore closed, dropping task");
364 spawned_task_count.fetch_sub(1, Ordering::SeqCst);
365 return;
366 };
367
368 active_task_count.fetch_add(1, Ordering::SeqCst);
371 running_tasks.lock().insert(task_id.clone());
372
373 let outcome =
376 Self::execute_catching_panic(worker, &task_client, &event_dispatcher, config, task)
377 .await;
378
379 running_tasks.lock().remove(&task_id);
381 active_task_count.fetch_sub(1, Ordering::SeqCst);
382 spawned_task_count.fetch_sub(1, Ordering::SeqCst);
383
384 match outcome {
385 TaskOutcome::Ok => {}
386 TaskOutcome::Err(e) => {
387 error!(
388 task_id = %task_id,
389 error = %e,
390 "Task execution failed"
391 );
392 }
393 TaskOutcome::Panic(panic_msg) => {
394 error!(
395 task_id = %task_id,
396 task_type = %task_type,
397 panic_message = %panic_msg,
398 "Uncaught panic in worker task"
399 );
400 event_dispatcher.publish_thread_uncaught_exception(
401 &ThreadUncaughtException::new(&task_type, "Panic"),
402 );
403 }
404 }
405 });
406 }
407
408 async fn execute_catching_panic(
415 worker: Arc<dyn Worker>,
416 task_client: &TaskClient,
417 event_dispatcher: &EventDispatcher,
418 config: Arc<WorkerConfig>,
419 task: Task,
420 ) -> TaskOutcome {
421 match AssertUnwindSafe(Self::execute_and_update_task(
422 &worker,
423 task_client,
424 event_dispatcher,
425 &config,
426 task,
427 ))
428 .catch_unwind()
429 .await
430 {
431 Ok(Ok(())) => TaskOutcome::Ok,
432 Ok(Err(e)) => TaskOutcome::Err(e),
433 Err(panic_payload) => {
434 let msg = panic_payload
435 .downcast_ref::<String>()
436 .map(std::string::String::as_str)
437 .or_else(|| panic_payload.downcast_ref::<&str>().copied())
438 .unwrap_or("<non-string panic>");
439 TaskOutcome::Panic(msg.to_owned())
440 }
441 }
442 }
443
444 async fn execute_and_update_task(
449 worker: &Arc<dyn Worker>,
450 task_client: &TaskClient,
451 event_dispatcher: &EventDispatcher,
452 config: &Arc<WorkerConfig>,
453 task: Task,
454 ) -> Result<()> {
455 let task = Arc::new(task);
457
458 let task_id = &task.task_id;
459 let task_type = &task.task_type;
460 let workflow_id = &task.workflow_instance_id;
461
462 debug!(
463 task_id = %task_id,
464 task_type = %task_type,
465 "Executing task"
466 );
467
468 event_dispatcher.publish_task_execution_started(&TaskExecutionStarted::new(
470 task_type,
471 task_id,
472 workflow_id,
473 &config.worker_id,
474 ));
475
476 let exec_start = Instant::now();
477
478 let heartbeat_guard = Self::maybe_spawn_lease_heartbeat(task_client, &task, config)
485 .map(tokio_util::task::AbortOnDropHandle::new);
486
487 let exec_result = worker.execute(&task).await;
489 let exec_duration = exec_start.elapsed();
490
491 drop(heartbeat_guard);
492
493 let task_result = match exec_result {
495 Ok(output) => {
496 let output_size = match &output {
497 WorkerOutput::Completed(data) => {
498 serde_json::to_string(data).map(|s| s.len()).ok()
499 }
500 _ => None,
501 };
502
503 event_dispatcher.publish_task_execution_completed(&TaskExecutionCompleted::new(
505 task_type,
506 task_id,
507 workflow_id,
508 &config.worker_id,
509 exec_duration,
510 output_size,
511 ));
512
513 output.into_task_result(&task, &config.worker_id)
514 }
515 Err(e) => {
516 let error_msg = e.to_string();
517 let exception = exception_label(&e);
518 let is_retryable = e.is_retryable();
519
520 event_dispatcher.publish_task_execution_failure(&TaskExecutionFailure::new(
521 task_type,
522 task_id,
523 workflow_id,
524 &config.worker_id,
525 exec_duration,
526 &error_msg,
527 exception,
528 is_retryable,
529 ));
530
531 WorkerOutput::Failed(error_msg).into_task_result(&task, &config.worker_id)
532 }
533 };
534
535 let update_start = Instant::now();
537 match task_client.update_task_with_retry(&task_result, 4).await {
538 Ok(_) => {
539 let update_duration = update_start.elapsed();
540 debug!(task_id = %task_id, "Task updated successfully");
541
542 event_dispatcher.publish_task_update_completed(&TaskUpdateCompleted::new(
543 task_type,
544 task_id,
545 workflow_id,
546 &config.worker_id,
547 update_duration,
548 ));
549 }
550 Err(e) => {
551 let update_duration = update_start.elapsed();
552 error!(task_id = %task_id, error = %e, "Failed to update task after retries");
553 let exception = exception_label(&e);
554
555 event_dispatcher.publish_task_update_failure(&TaskUpdateFailure::new(
556 task_type,
557 task_id,
558 workflow_id,
559 &config.worker_id,
560 update_duration,
561 e.to_string(),
562 exception,
563 4,
564 ));
565 }
566 }
567
568 Ok(())
569 }
570
571 fn maybe_spawn_lease_heartbeat(
575 task_client: &TaskClient,
576 task: &Arc<Task>,
577 config: &Arc<WorkerConfig>,
578 ) -> Option<tokio::task::JoinHandle<()>> {
579 if !config.lease_extend_enabled {
580 return None;
581 }
582 if task.response_timeout_seconds <= 0 {
583 return None;
584 }
585 let interval_secs = task.response_timeout_seconds as f64 * config.lease_extend_threshold;
586 if interval_secs < 1.0 {
588 return None;
589 }
590
591 let task_client = task_client.clone();
592 let task_id = task.task_id.clone();
593 let workflow_instance_id = task.workflow_instance_id.clone();
594 let interval = Duration::from_secs_f64(interval_secs);
595
596 Some(tokio::spawn(Self::send_lease_heartbeats(
597 task_client,
598 task_id,
599 workflow_instance_id,
600 interval,
601 )))
602 }
603
604 #[expect(clippy::infinite_loop)]
608 async fn send_lease_heartbeats(
609 task_client: TaskClient,
610 task_id: String,
611 workflow_instance_id: String,
612 interval: Duration,
613 ) {
614 const LEASE_EXTEND_RETRY_COUNT: u32 = 3;
619
620 let mut ticker = tokio::time::interval_at(tokio::time::Instant::now() + interval, interval);
621 loop {
622 ticker.tick().await;
623 let heartbeat = crate::models::TaskResult {
624 task_id: task_id.clone(),
625 workflow_instance_id: workflow_instance_id.clone(),
626 status: crate::models::TaskResultStatus::InProgress,
627 extend_lease: true,
628 ..Default::default()
629 };
630
631 for attempt in 0..LEASE_EXTEND_RETRY_COUNT {
632 match task_client.update_task(&heartbeat).await {
633 Ok(_) => {
634 debug!(task_id = %task_id, "Extended lease");
635 break;
636 }
637 Err(e) if attempt + 1 < LEASE_EXTEND_RETRY_COUNT => {
638 warn!(task_id = %task_id, error = %e, attempt, "Lease heartbeat failed, retrying");
639 tokio::time::sleep(Duration::from_millis(500 * u64::from(attempt + 2)))
640 .await;
641 }
642 Err(e) => {
643 error!(task_id = %task_id, error = %e, "Failed to extend lease after {LEASE_EXTEND_RETRY_COUNT} attempts");
644 }
645 }
646 }
647 }
648 }
649}
650
651#[cfg(test)]
652mod tests {
653 use super::*;
654 use crate::configuration::Configuration;
655 use crate::http::ApiClient;
656 use async_trait::async_trait;
657
658 struct TestWorker;
659
660 #[async_trait]
661 impl Worker for TestWorker {
662 fn task_definition_name(&self) -> &'static str {
663 "test_task"
664 }
665
666 async fn execute(&self, task: &Task) -> Result<WorkerOutput> {
667 let name = task
668 .get_input_string("name")
669 .unwrap_or_else(|| "World".to_owned());
670 Ok(WorkerOutput::completed_with_result(format!(
671 "Hello, {name}!"
672 )))
673 }
674
675 fn thread_count(&self) -> usize {
676 5
677 }
678 }
679
680 #[test]
681 fn test_task_runner_config() {
682 let config = Configuration::new("http://localhost:8080/api");
683 let api = ApiClient::new(config).unwrap();
684 let task_client = TaskClient::new(api);
685 let worker = Arc::new(TestWorker);
686 let dispatcher = EventDispatcher::new();
687
688 let runner = TaskRunner::new(worker, task_client, dispatcher);
689
690 assert_eq!(runner.task_type(), "test_task");
691 assert_eq!(runner.config().thread_count, 5);
692 assert_eq!(runner.poll_attempt_count(), 0);
693 assert!(!runner.is_running());
694 }
695
696 fn test_task_client() -> TaskClient {
697 let config = Configuration::new("http://localhost:8080/api");
698 TaskClient::new(ApiClient::new(config).unwrap())
699 }
700
701 #[test]
702 fn test_lease_heartbeat_not_spawned_when_disabled() {
703 let task_client = test_task_client();
704 let task = Arc::new(Task {
705 response_timeout_seconds: 30,
706 ..Default::default()
707 });
708 let config = Arc::new(WorkerConfig::new("t").with_lease_extend_enabled(false));
709
710 let handle = TaskRunner::maybe_spawn_lease_heartbeat(&task_client, &task, &config);
711 assert!(handle.is_none());
712 }
713
714 #[test]
715 fn test_lease_heartbeat_not_spawned_without_response_timeout() {
716 let task_client = test_task_client();
717 let task = Arc::new(Task {
718 response_timeout_seconds: 0,
719 ..Default::default()
720 });
721 let config = Arc::new(WorkerConfig::new("t").with_lease_extend_enabled(true));
722
723 let handle = TaskRunner::maybe_spawn_lease_heartbeat(&task_client, &task, &config);
724 assert!(handle.is_none());
725 }
726
727 #[test]
728 fn test_lease_heartbeat_not_spawned_when_interval_too_short() {
729 let task_client = test_task_client();
730 let task = Arc::new(Task {
732 response_timeout_seconds: 1,
733 ..Default::default()
734 });
735 let config = Arc::new(WorkerConfig::new("t").with_lease_extend_enabled(true));
736
737 let handle = TaskRunner::maybe_spawn_lease_heartbeat(&task_client, &task, &config);
738 assert!(handle.is_none());
739 }
740
741 #[tokio::test]
742 async fn test_lease_heartbeat_spawned_when_enabled_and_worthwhile() {
743 let task_client = test_task_client();
744 let task = Arc::new(Task {
745 response_timeout_seconds: 30,
746 ..Default::default()
747 });
748 let config = Arc::new(
749 WorkerConfig::new("t")
750 .with_lease_extend_enabled(true)
751 .with_lease_extend_threshold(0.8),
752 );
753
754 let handle = TaskRunner::maybe_spawn_lease_heartbeat(&task_client, &task, &config);
755 assert!(handle.is_some());
756 handle.unwrap().abort();
759 }
760}