Skip to main content

conductor/worker/
task_runner.rs

1// Copyright {{.Year}} Conductor OSS
2// Licensed under the Apache License, Version 2.0. See LICENSE in the project root for license information.
3
4use std::collections::HashSet;
5use std::panic::AssertUnwindSafe;
6use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
7use std::sync::Arc;
8use std::time::{Duration, Instant};
9
10use futures::FutureExt as _;
11use tokio::sync::Semaphore;
12use tracing::{debug, error, info, warn};
13
14use crate::client::TaskClient;
15use crate::configuration::{resolve_worker_config, WorkerConfig};
16use crate::error::Result;
17use crate::events::{
18    exception_label, EventDispatcher, PollCompleted, PollFailure, PollSkippedPaused, PollStarted,
19    TaskExecutionCompleted, TaskExecutionFailure, TaskExecutionStarted, TaskUpdateCompleted,
20    TaskUpdateFailure, ThreadUncaughtException,
21};
22use crate::models::Task;
23
24use super::{Worker, WorkerOutput};
25
26/// Result of a task execution attempt, including panics.
27///
28/// Returned by [`TaskRunner::execute_catching_panic`] so the caller can
29/// handle success, regular errors, and panics without accessing any
30/// state that was inside the `AssertUnwindSafe` boundary.
31enum TaskOutcome {
32    Ok,
33    Err(crate::error::ConductorError),
34    Panic(String),
35}
36
37/// Task runner for a single worker type.
38pub struct TaskRunner {
39    worker: Arc<dyn Worker>,
40    task_client: TaskClient,
41    config: Arc<WorkerConfig>,
42    event_dispatcher: EventDispatcher,
43
44    // Control state
45    running: Arc<AtomicBool>,
46    paused: Arc<AtomicBool>,
47
48    // Polling state
49    consecutive_empty_polls: Arc<AtomicU64>,
50    last_poll_time: Arc<parking_lot::Mutex<Instant>>,
51
52    // Concurrency control - use atomic counter instead of HashSet for better performance
53    semaphore: Arc<Semaphore>,
54    /// Count of tasks currently being executed (after semaphore acquired).
55    active_task_count: Arc<AtomicUsize>,
56    /// Set of task IDs currently in flight (for debugging/monitoring).
57    running_tasks: Arc<parking_lot::Mutex<HashSet<String>>>,
58    /// Count of spawned tasks (including those waiting for semaphore).
59    spawned_task_count: Arc<AtomicUsize>,
60    /// Count of real poll attempts (successful or failed) made against the server so far. Used
61    /// by [`super::TaskHandler::verify_workers_started`] to tell "the polling loop hasn't
62    /// attempted a single real poll yet" apart from "the loop started but is legitimately
63    /// waiting" -- [`TaskRunner::is_running`] alone can't distinguish those, since it flips
64    /// `true` before the first poll ever happens.
65    poll_attempts: Arc<AtomicU64>,
66}
67
68impl TaskRunner {
69    /// Create a new task runner.
70    pub fn new(
71        worker: Arc<dyn Worker>,
72        task_client: TaskClient,
73        event_dispatcher: EventDispatcher,
74    ) -> Self {
75        // Resolve configuration from environment
76        let defaults = WorkerConfig {
77            task_definition_name: worker.task_definition_name().to_owned(),
78            poll_interval: Duration::from_millis(worker.poll_interval_millis()),
79            domain: worker.domain().map(std::borrow::ToOwned::to_owned),
80            worker_id: worker.identity(),
81            thread_count: worker.thread_count(),
82            ..Default::default()
83        };
84
85        let config = resolve_worker_config(worker.task_definition_name(), defaults);
86
87        info!(
88            task_type = %config.task_definition_name,
89            worker_id = %config.worker_id,
90            thread_count = config.thread_count,
91            poll_interval_ms = config.poll_interval.as_millis(),
92            domain = ?config.domain,
93            paused = config.paused,
94            "Task runner initialized"
95        );
96
97        let paused = config.paused;
98        let thread_count = config.thread_count;
99
100        Self {
101            worker,
102            task_client,
103            config: Arc::new(config),
104            event_dispatcher,
105            running: Arc::new(AtomicBool::new(false)),
106            paused: Arc::new(AtomicBool::new(paused)),
107            consecutive_empty_polls: Arc::new(AtomicU64::new(0)),
108            last_poll_time: Arc::new(parking_lot::Mutex::new(Instant::now())),
109            semaphore: Arc::new(Semaphore::new(thread_count)),
110            active_task_count: Arc::new(AtomicUsize::new(0)),
111            running_tasks: Arc::new(parking_lot::Mutex::new(HashSet::new())),
112            spawned_task_count: Arc::new(AtomicUsize::new(0)),
113            poll_attempts: Arc::new(AtomicU64::new(0)),
114        }
115    }
116
117    /// Get the task type this runner handles.
118    #[must_use]
119    pub fn task_type(&self) -> &str {
120        &self.config.task_definition_name
121    }
122
123    /// Get the worker configuration.
124    #[must_use]
125    pub fn config(&self) -> &WorkerConfig {
126        &self.config
127    }
128
129    /// Get the number of currently active tasks (executing, not waiting for semaphore).
130    #[must_use]
131    pub fn active_task_count(&self) -> usize {
132        self.active_task_count.load(Ordering::SeqCst)
133    }
134
135    /// Get the number of spawned tasks (including those waiting for semaphore).
136    #[must_use]
137    pub fn spawned_task_count(&self) -> usize {
138        self.spawned_task_count.load(Ordering::SeqCst)
139    }
140
141    /// Get the number of real poll attempts (successful or failed) made against the server so
142    /// far. Zero until the first one completes -- see the field's doc comment for why this
143    /// exists separately from [`TaskRunner::is_running`].
144    #[must_use]
145    pub fn poll_attempt_count(&self) -> u64 {
146        self.poll_attempts.load(Ordering::SeqCst)
147    }
148
149    /// Check if the runner is running.
150    #[must_use]
151    pub fn is_running(&self) -> bool {
152        self.running.load(Ordering::SeqCst)
153    }
154
155    /// Check if the runner is paused.
156    #[must_use]
157    pub fn is_paused(&self) -> bool {
158        self.paused.load(Ordering::SeqCst)
159    }
160
161    /// Pause the runner.
162    pub fn pause(&self) {
163        self.paused.store(true, Ordering::SeqCst);
164        info!(task_type = %self.config.task_definition_name, "Task runner paused");
165    }
166
167    /// Resume the runner.
168    pub fn resume(&self) {
169        self.paused.store(false, Ordering::SeqCst);
170        info!(task_type = %self.config.task_definition_name, "Task runner resumed");
171    }
172
173    /// Stop the runner.
174    pub fn stop(&self) {
175        self.running.store(false, Ordering::SeqCst);
176        info!(task_type = %self.config.task_definition_name, "Task runner stopped");
177    }
178
179    /// Run the polling loop.
180    pub async fn run(&self) {
181        self.running.store(true, Ordering::SeqCst);
182
183        info!(
184            task_type = %self.config.task_definition_name,
185            "Starting task runner polling loop"
186        );
187
188        while self.running.load(Ordering::SeqCst) {
189            if let Err(e) = self.run_once().await {
190                error!(
191                    task_type = %self.config.task_definition_name,
192                    error = %e,
193                    "Error in polling loop"
194                );
195            }
196        }
197
198        // Wait for in-flight tasks to complete (graceful shutdown)
199        self.wait_for_tasks_to_complete().await;
200
201        info!(
202            task_type = %self.config.task_definition_name,
203            "Task runner polling loop ended"
204        );
205    }
206
207    /// Wait for all spawned tasks to complete (used during shutdown).
208    async fn wait_for_tasks_to_complete(&self) {
209        let shutdown_timeout = Duration::from_secs(30);
210        let start = Instant::now();
211
212        while self.spawned_task_count.load(Ordering::SeqCst) > 0 {
213            if start.elapsed() > shutdown_timeout {
214                let remaining = self.spawned_task_count.load(Ordering::SeqCst);
215                warn!(
216                    task_type = %self.config.task_definition_name,
217                    remaining_tasks = remaining,
218                    "Shutdown timeout reached, {} tasks still in flight",
219                    remaining
220                );
221                break;
222            }
223            tokio::time::sleep(Duration::from_millis(100)).await;
224        }
225    }
226
227    /// Run one iteration of the polling loop.
228    async fn run_once(&self) -> Result<()> {
229        // Check if paused
230        if self.paused.load(Ordering::SeqCst) {
231            self.event_dispatcher
232                .publish_poll_skipped_paused(&PollSkippedPaused::new(
233                    &self.config.task_definition_name,
234                    &self.config.worker_id,
235                ));
236            tokio::time::sleep(Duration::from_millis(100)).await;
237            return Ok(());
238        }
239
240        // Calculate available capacity based on ACTIVE tasks (those that have acquired semaphore)
241        // This is more accurate than tracking spawned tasks since semaphore controls actual concurrency
242        let active_count = self.active_task_count.load(Ordering::SeqCst);
243        let available_slots = self.config.thread_count.saturating_sub(active_count);
244
245        if available_slots == 0 {
246            // At capacity, wait briefly
247            tokio::time::sleep(Duration::from_millis(1)).await;
248            return Ok(());
249        }
250
251        // Apply adaptive backoff
252        let empty_polls = self.consecutive_empty_polls.load(Ordering::SeqCst);
253        if empty_polls > 0 {
254            let backoff = Duration::from_millis(1 << empty_polls.min(10));
255            let backoff = backoff.min(self.config.poll_interval);
256
257            let elapsed = self.last_poll_time.lock().elapsed();
258            if elapsed < backoff {
259                tokio::time::sleep(backoff.checked_sub(elapsed).unwrap_or_default()).await;
260            }
261        }
262
263        // Poll for tasks
264        let poll_start = Instant::now();
265
266        // Publish poll started event
267        self.event_dispatcher
268            .publish_poll_started(&PollStarted::new(
269                &self.config.task_definition_name,
270                &self.config.worker_id,
271                available_slots,
272            ));
273
274        let poll_result = self
275            .task_client
276            .batch_poll(
277                &self.config.task_definition_name,
278                Some(&self.config.worker_id),
279                self.config.domain.as_deref(),
280                available_slots,
281                self.config.poll_timeout,
282            )
283            .await;
284
285        let poll_duration = poll_start.elapsed();
286        *self.last_poll_time.lock() = Instant::now();
287        self.poll_attempts.fetch_add(1, Ordering::SeqCst);
288
289        match poll_result {
290            Ok(tasks) => {
291                // Publish poll completed event
292                self.event_dispatcher
293                    .publish_poll_completed(&PollCompleted::new(
294                        &self.config.task_definition_name,
295                        &self.config.worker_id,
296                        poll_duration,
297                        tasks.len(),
298                    ));
299
300                if tasks.is_empty() {
301                    self.consecutive_empty_polls.fetch_add(1, Ordering::SeqCst);
302                } else {
303                    self.consecutive_empty_polls.store(0, Ordering::SeqCst);
304
305                    // Process tasks
306                    for task in tasks {
307                        self.spawn_task_execution(task);
308                    }
309                }
310            }
311            Err(e) => {
312                let exception = exception_label(&e);
313
314                self.event_dispatcher
315                    .publish_poll_failure(&PollFailure::new(
316                        &self.config.task_definition_name,
317                        &self.config.worker_id,
318                        poll_duration,
319                        e.to_string(),
320                        exception,
321                    ));
322
323                self.consecutive_empty_polls.fetch_add(1, Ordering::SeqCst);
324            }
325        }
326
327        Ok(())
328    }
329
330    /// Spawn task execution in background.
331    ///
332    /// This method correctly handles the semaphore acquisition order to avoid
333    /// race conditions in capacity calculation:
334    /// 1. Increment spawned count (for shutdown tracking)
335    /// 2. Spawn task
336    /// 3. Acquire semaphore (wait if at capacity)
337    /// 4. Increment active count (now executing)
338    /// 5. Track task ID in `running_tasks`
339    /// 6. Execute task
340    /// 7. Decrement active count and remove from `running_tasks`
341    /// 8. Decrement spawned count
342    fn spawn_task_execution(&self, task: Task) {
343        let task_id = task.task_id.clone();
344
345        // Increment spawned task count for shutdown tracking
346        self.spawned_task_count.fetch_add(1, Ordering::SeqCst);
347
348        let worker = Arc::clone(&self.worker);
349        let task_client = self.task_client.clone();
350        let event_dispatcher = self.event_dispatcher.clone();
351        let config = Arc::clone(&self.config);
352        let semaphore = Arc::clone(&self.semaphore);
353        let active_task_count = Arc::clone(&self.active_task_count);
354        let running_tasks = Arc::clone(&self.running_tasks);
355        let spawned_task_count = Arc::clone(&self.spawned_task_count);
356
357        let task_type = self.config.task_definition_name.clone();
358
359        tokio::spawn(async move {
360            // Acquire semaphore permit FIRST - this is the actual concurrency control
361            let Ok(_permit) = semaphore.acquire().await else {
362                // Semaphore was closed (shouldn't happen in normal operation)
363                error!(task_id = %task_id, "Semaphore closed, dropping task");
364                spawned_task_count.fetch_sub(1, Ordering::SeqCst);
365                return;
366            };
367
368            // NOW increment active count and track the task
369            // This ensures capacity calculation is accurate
370            active_task_count.fetch_add(1, Ordering::SeqCst);
371            running_tasks.lock().insert(task_id.clone());
372
373            // `worker`, `config`, and `task` are moved into the panic-catching
374            // boundary and cannot be accessed in the cleanup code below.
375            let outcome =
376                Self::execute_catching_panic(worker, &task_client, &event_dispatcher, config, task)
377                    .await;
378
379            // Cleanup: only atomics, locks, and event_dispatcher are accessible
380            running_tasks.lock().remove(&task_id);
381            active_task_count.fetch_sub(1, Ordering::SeqCst);
382            spawned_task_count.fetch_sub(1, Ordering::SeqCst);
383
384            match outcome {
385                TaskOutcome::Ok => {}
386                TaskOutcome::Err(e) => {
387                    error!(
388                        task_id = %task_id,
389                        error = %e,
390                        "Task execution failed"
391                    );
392                }
393                TaskOutcome::Panic(panic_msg) => {
394                    error!(
395                        task_id = %task_id,
396                        task_type = %task_type,
397                        panic_message = %panic_msg,
398                        "Uncaught panic in worker task"
399                    );
400                    event_dispatcher.publish_thread_uncaught_exception(
401                        &ThreadUncaughtException::new(&task_type, "Panic"),
402                    );
403                }
404            }
405        });
406    }
407
408    /// Execute a task inside a panic-catching boundary.
409    ///
410    /// `worker`, `config`, and `task` are consumed so that the caller
411    /// cannot access them after a potential panic — only the returned
412    /// [`TaskOutcome`] carries the information needed for logging and
413    /// event publishing.
414    async fn execute_catching_panic(
415        worker: Arc<dyn Worker>,
416        task_client: &TaskClient,
417        event_dispatcher: &EventDispatcher,
418        config: Arc<WorkerConfig>,
419        task: Task,
420    ) -> TaskOutcome {
421        match AssertUnwindSafe(Self::execute_and_update_task(
422            &worker,
423            task_client,
424            event_dispatcher,
425            &config,
426            task,
427        ))
428        .catch_unwind()
429        .await
430        {
431            Ok(Ok(())) => TaskOutcome::Ok,
432            Ok(Err(e)) => TaskOutcome::Err(e),
433            Err(panic_payload) => {
434                let msg = panic_payload
435                    .downcast_ref::<String>()
436                    .map(std::string::String::as_str)
437                    .or_else(|| panic_payload.downcast_ref::<&str>().copied())
438                    .unwrap_or("<non-string panic>");
439                TaskOutcome::Panic(msg.to_owned())
440            }
441        }
442    }
443
444    /// Execute a task and update the result.
445    ///
446    /// Takes ownership of the Task to wrap it in Arc, avoiding clones
447    /// when passing to workers.
448    async fn execute_and_update_task(
449        worker: &Arc<dyn Worker>,
450        task_client: &TaskClient,
451        event_dispatcher: &EventDispatcher,
452        config: &Arc<WorkerConfig>,
453        task: Task,
454    ) -> Result<()> {
455        // Wrap task in Arc once - this is the only allocation
456        let task = Arc::new(task);
457
458        let task_id = &task.task_id;
459        let task_type = &task.task_type;
460        let workflow_id = &task.workflow_instance_id;
461
462        debug!(
463            task_id = %task_id,
464            task_type = %task_type,
465            "Executing task"
466        );
467
468        // Publish execution started event
469        event_dispatcher.publish_task_execution_started(&TaskExecutionStarted::new(
470            task_type,
471            task_id,
472            workflow_id,
473            &config.worker_id,
474        ));
475
476        let exec_start = Instant::now();
477
478        // Start a lease-extension heartbeat alongside execution, if configured -- a per-task
479        // spawned tokio task (cheap here, unlike an OS thread) rather than a shared
480        // background-thread manager.
481        // The guard aborts the heartbeat when it goes out of scope, including by unwinding if
482        // the worker future panics -- a bare `JoinHandle` only detaches on drop, which would
483        // leave the heartbeat extending the lease indefinitely.
484        let heartbeat_guard = Self::maybe_spawn_lease_heartbeat(task_client, &task, config)
485            .map(tokio_util::task::AbortOnDropHandle::new);
486
487        // Execute the worker - pass reference to avoid clone in worker trait
488        let exec_result = worker.execute(&task).await;
489        let exec_duration = exec_start.elapsed();
490
491        drop(heartbeat_guard);
492
493        // Convert result to TaskResult
494        let task_result = match exec_result {
495            Ok(output) => {
496                let output_size = match &output {
497                    WorkerOutput::Completed(data) => {
498                        serde_json::to_string(data).map(|s| s.len()).ok()
499                    }
500                    _ => None,
501                };
502
503                // Publish execution completed event
504                event_dispatcher.publish_task_execution_completed(&TaskExecutionCompleted::new(
505                    task_type,
506                    task_id,
507                    workflow_id,
508                    &config.worker_id,
509                    exec_duration,
510                    output_size,
511                ));
512
513                output.into_task_result(&task, &config.worker_id)
514            }
515            Err(e) => {
516                let error_msg = e.to_string();
517                let exception = exception_label(&e);
518                let is_retryable = e.is_retryable();
519
520                event_dispatcher.publish_task_execution_failure(&TaskExecutionFailure::new(
521                    task_type,
522                    task_id,
523                    workflow_id,
524                    &config.worker_id,
525                    exec_duration,
526                    &error_msg,
527                    exception,
528                    is_retryable,
529                ));
530
531                WorkerOutput::Failed(error_msg).into_task_result(&task, &config.worker_id)
532            }
533        };
534
535        // Update task with retry
536        let update_start = Instant::now();
537        match task_client.update_task_with_retry(&task_result, 4).await {
538            Ok(_) => {
539                let update_duration = update_start.elapsed();
540                debug!(task_id = %task_id, "Task updated successfully");
541
542                event_dispatcher.publish_task_update_completed(&TaskUpdateCompleted::new(
543                    task_type,
544                    task_id,
545                    workflow_id,
546                    &config.worker_id,
547                    update_duration,
548                ));
549            }
550            Err(e) => {
551                let update_duration = update_start.elapsed();
552                error!(task_id = %task_id, error = %e, "Failed to update task after retries");
553                let exception = exception_label(&e);
554
555                event_dispatcher.publish_task_update_failure(&TaskUpdateFailure::new(
556                    task_type,
557                    task_id,
558                    workflow_id,
559                    &config.worker_id,
560                    update_duration,
561                    e.to_string(),
562                    exception,
563                    4,
564                ));
565            }
566        }
567
568        Ok(())
569    }
570
571    /// Start a background heartbeat loop for `task`, if lease extension is enabled and the
572    /// task's `response_timeout_seconds` makes it worthwhile. Returns `None` (spawning nothing)
573    /// when disabled.
574    fn maybe_spawn_lease_heartbeat(
575        task_client: &TaskClient,
576        task: &Arc<Task>,
577        config: &Arc<WorkerConfig>,
578    ) -> Option<tokio::task::JoinHandle<()>> {
579        if !config.lease_extend_enabled {
580            return None;
581        }
582        if task.response_timeout_seconds <= 0 {
583            return None;
584        }
585        let interval_secs = task.response_timeout_seconds as f64 * config.lease_extend_threshold;
586        // An interval under a second isn't worth scheduling a repeating heartbeat for.
587        if interval_secs < 1.0 {
588            return None;
589        }
590
591        let task_client = task_client.clone();
592        let task_id = task.task_id.clone();
593        let workflow_instance_id = task.workflow_instance_id.clone();
594        let interval = Duration::from_secs_f64(interval_secs);
595
596        Some(tokio::spawn(Self::send_lease_heartbeats(
597            task_client,
598            task_id,
599            workflow_instance_id,
600            interval,
601        )))
602    }
603
604    /// Send a lease-extension heartbeat every `interval`, starting `interval` after this is
605    /// spawned (not immediately). Runs until the caller aborts the returned `JoinHandle` (when
606    /// the task finishes), which is the only way this loop ends.
607    #[expect(clippy::infinite_loop)]
608    async fn send_lease_heartbeats(
609        task_client: TaskClient,
610        task_id: String,
611        workflow_instance_id: String,
612        interval: Duration,
613    ) {
614        // A short, fixed retry count with fast backoff -- deliberately not
615        // `TaskClient::update_task_with_retry`'s 10/20/30s schedule, which is sized for
616        // terminal completion updates, not a fast-repeating keep-alive that will just get
617        // another chance at the next tick anyway.
618        const LEASE_EXTEND_RETRY_COUNT: u32 = 3;
619
620        let mut ticker = tokio::time::interval_at(tokio::time::Instant::now() + interval, interval);
621        loop {
622            ticker.tick().await;
623            let heartbeat = crate::models::TaskResult {
624                task_id: task_id.clone(),
625                workflow_instance_id: workflow_instance_id.clone(),
626                status: crate::models::TaskResultStatus::InProgress,
627                extend_lease: true,
628                ..Default::default()
629            };
630
631            for attempt in 0..LEASE_EXTEND_RETRY_COUNT {
632                match task_client.update_task(&heartbeat).await {
633                    Ok(_) => {
634                        debug!(task_id = %task_id, "Extended lease");
635                        break;
636                    }
637                    Err(e) if attempt + 1 < LEASE_EXTEND_RETRY_COUNT => {
638                        warn!(task_id = %task_id, error = %e, attempt, "Lease heartbeat failed, retrying");
639                        tokio::time::sleep(Duration::from_millis(500 * u64::from(attempt + 2)))
640                            .await;
641                    }
642                    Err(e) => {
643                        error!(task_id = %task_id, error = %e, "Failed to extend lease after {LEASE_EXTEND_RETRY_COUNT} attempts");
644                    }
645                }
646            }
647        }
648    }
649}
650
651#[cfg(test)]
652mod tests {
653    use super::*;
654    use crate::configuration::Configuration;
655    use crate::http::ApiClient;
656    use async_trait::async_trait;
657
658    struct TestWorker;
659
660    #[async_trait]
661    impl Worker for TestWorker {
662        fn task_definition_name(&self) -> &'static str {
663            "test_task"
664        }
665
666        async fn execute(&self, task: &Task) -> Result<WorkerOutput> {
667            let name = task
668                .get_input_string("name")
669                .unwrap_or_else(|| "World".to_owned());
670            Ok(WorkerOutput::completed_with_result(format!(
671                "Hello, {name}!"
672            )))
673        }
674
675        fn thread_count(&self) -> usize {
676            5
677        }
678    }
679
680    #[test]
681    fn test_task_runner_config() {
682        let config = Configuration::new("http://localhost:8080/api");
683        let api = ApiClient::new(config).unwrap();
684        let task_client = TaskClient::new(api);
685        let worker = Arc::new(TestWorker);
686        let dispatcher = EventDispatcher::new();
687
688        let runner = TaskRunner::new(worker, task_client, dispatcher);
689
690        assert_eq!(runner.task_type(), "test_task");
691        assert_eq!(runner.config().thread_count, 5);
692        assert_eq!(runner.poll_attempt_count(), 0);
693        assert!(!runner.is_running());
694    }
695
696    fn test_task_client() -> TaskClient {
697        let config = Configuration::new("http://localhost:8080/api");
698        TaskClient::new(ApiClient::new(config).unwrap())
699    }
700
701    #[test]
702    fn test_lease_heartbeat_not_spawned_when_disabled() {
703        let task_client = test_task_client();
704        let task = Arc::new(Task {
705            response_timeout_seconds: 30,
706            ..Default::default()
707        });
708        let config = Arc::new(WorkerConfig::new("t").with_lease_extend_enabled(false));
709
710        let handle = TaskRunner::maybe_spawn_lease_heartbeat(&task_client, &task, &config);
711        assert!(handle.is_none());
712    }
713
714    #[test]
715    fn test_lease_heartbeat_not_spawned_without_response_timeout() {
716        let task_client = test_task_client();
717        let task = Arc::new(Task {
718            response_timeout_seconds: 0,
719            ..Default::default()
720        });
721        let config = Arc::new(WorkerConfig::new("t").with_lease_extend_enabled(true));
722
723        let handle = TaskRunner::maybe_spawn_lease_heartbeat(&task_client, &task, &config);
724        assert!(handle.is_none());
725    }
726
727    #[test]
728    fn test_lease_heartbeat_not_spawned_when_interval_too_short() {
729        let task_client = test_task_client();
730        // 1s timeout * 0.8 threshold = 0.8s, under the 1-second skip threshold.
731        let task = Arc::new(Task {
732            response_timeout_seconds: 1,
733            ..Default::default()
734        });
735        let config = Arc::new(WorkerConfig::new("t").with_lease_extend_enabled(true));
736
737        let handle = TaskRunner::maybe_spawn_lease_heartbeat(&task_client, &task, &config);
738        assert!(handle.is_none());
739    }
740
741    #[tokio::test]
742    async fn test_lease_heartbeat_spawned_when_enabled_and_worthwhile() {
743        let task_client = test_task_client();
744        let task = Arc::new(Task {
745            response_timeout_seconds: 30,
746            ..Default::default()
747        });
748        let config = Arc::new(
749            WorkerConfig::new("t")
750                .with_lease_extend_enabled(true)
751                .with_lease_extend_threshold(0.8),
752        );
753
754        let handle = TaskRunner::maybe_spawn_lease_heartbeat(&task_client, &task, &config);
755        assert!(handle.is_some());
756        // Abort immediately -- this test only checks that a heartbeat loop gets scheduled at
757        // all, not its actual network behavior (which needs a live/mock server and real time).
758        handle.unwrap().abort();
759    }
760}