Skip to main content

conductor/worker/
task_handler.rs

1// Copyright {{.Year}} Conductor OSS
2// Licensed under the Apache License, Version 2.0. See LICENSE in the project root for license information.
3
4use std::sync::Arc;
5use std::time::{Duration, Instant};
6use tokio::task::JoinHandle;
7use tracing::{debug, error, info, warn};
8
9use crate::client::{ConductorClient, MetadataClient, SchemaClient, TaskClient};
10use crate::configuration::{resolve_worker_config, Configuration, WorkerConfig};
11use crate::error::{ConductorError, Result};
12use crate::events::{EventDispatcher, TaskRunnerEventsListener};
13use crate::http::ApiClient;
14use crate::metrics::{MetricsCollector, MetricsSettings};
15use crate::models::{SchemaDef, TaskDef};
16
17use super::{TaskRunner, Worker};
18
19/// Task handler for managing multiple workers.
20pub struct TaskHandler {
21    #[expect(dead_code)] // Stored for potential future use (e.g., reconfiguration)
22    config: Configuration,
23    api_client: ApiClient,
24    event_dispatcher: EventDispatcher,
25    workers: Vec<Arc<dyn Worker>>,
26    runners: Vec<Arc<TaskRunner>>,
27    handles: Vec<JoinHandle<()>>,
28    metrics_collector: Option<Arc<MetricsCollector>>,
29    metrics_handle: Option<JoinHandle<()>>,
30}
31
32impl TaskHandler {
33    /// Create a new task handler with the given configuration.
34    ///
35    /// # Errors
36    ///
37    /// Returns [`crate::error::ConductorError::Http`] if the underlying `reqwest` client fails to build (see [`ApiClient::new`]).
38    pub fn new(config: Configuration) -> Result<Self> {
39        let api_client = ApiClient::new(config.clone())?;
40        let event_dispatcher = EventDispatcher::new();
41
42        Ok(Self {
43            config,
44            api_client,
45            event_dispatcher,
46            workers: Vec::new(),
47            runners: Vec::new(),
48            handles: Vec::new(),
49            metrics_collector: None,
50            metrics_handle: None,
51        })
52    }
53
54    /// Create a builder for more flexible configuration.
55    #[must_use]
56    pub fn builder(config: Configuration) -> TaskHandlerBuilder {
57        TaskHandlerBuilder::new(config)
58    }
59
60    /// Add a worker to the handler.
61    ///
62    /// Registration is always explicit -- there is no scan-a-package auto-discovery here.
63    /// Rust's callers already write out every `TaskDef`/`AgentDef`/`ToolDef` by hand at a
64    /// well-typed call site, so scanning for them would only add indirection, not remove real
65    /// boilerplate. A crate-level `inventory` dependency was carried for a while in anticipation
66    /// of a `#[worker]`-style attribute macro doing `inventory::submit!`-based collection
67    /// instead, but was removed unused: besides never having a caller, that approach has a real
68    /// sharp edge `inventory` itself documents -- registrations only get linked in if the
69    /// registering crate is a *direct* dependency of the final binary, which link-time
70    /// dead-code elimination can silently defeat depending on build settings.
71    /// `add_worker`/`add_workers` (and [`TaskHandlerBuilder::worker`]/
72    /// [`TaskHandlerBuilder::workers`]) staying explicit sidesteps that class of bug entirely.
73    pub fn add_worker(&mut self, worker: impl Worker + 'static) {
74        self.workers.push(Arc::new(worker));
75    }
76
77    /// Add multiple workers.
78    pub fn add_workers(&mut self, workers: impl IntoIterator<Item = Arc<dyn Worker>>) {
79        self.workers.extend(workers);
80    }
81
82    /// Register an event listener.
83    pub fn add_event_listener(&self, listener: Arc<dyn TaskRunnerEventsListener>) {
84        self.event_dispatcher.register(listener);
85    }
86
87    /// Enable metrics collection and (optionally) the HTTP scrape endpoint.
88    ///
89    /// Creates a [`MetricsCollector`] with the given settings and wires it
90    /// as both the [`TaskRunnerEventsListener`] (for task/workflow counters
91    /// and histograms) and the
92    /// [`HttpMetricsObserver`](crate::http::HttpMetricsObserver) (for
93    /// `http_api_client_request_seconds`).
94    ///
95    /// Must be called **before** [`start`](Self::start). Clients vended
96    /// after this call (via [`conductor_client`](Self::conductor_client),
97    /// [`task_client`](Self::task_client), etc.) will share the observer.
98    pub fn enable_metrics(&mut self, settings: MetricsSettings) {
99        let collector = Arc::new(MetricsCollector::new(settings));
100        self.event_dispatcher
101            .register(Arc::clone(&collector) as Arc<dyn TaskRunnerEventsListener>);
102        self.api_client
103            .set_http_observer(Arc::clone(&collector) as Arc<dyn crate::http::HttpMetricsObserver>);
104        self.metrics_collector = Some(collector);
105    }
106
107    /// Get the metrics collector.
108    #[must_use]
109    pub fn metrics_collector(&self) -> Option<&Arc<MetricsCollector>> {
110        self.metrics_collector.as_ref()
111    }
112
113    /// Get the event dispatcher.
114    #[must_use]
115    pub fn event_dispatcher(&self) -> &EventDispatcher {
116        &self.event_dispatcher
117    }
118
119    /// Get a task client.
120    #[must_use]
121    pub fn task_client(&self) -> TaskClient {
122        TaskClient::new(self.api_client.clone())
123    }
124
125    /// Get a metadata client.
126    #[must_use]
127    pub fn metadata_client(&self) -> MetadataClient {
128        MetadataClient::new(self.api_client.clone())
129    }
130
131    /// Get the Conductor client wired to this handler's event dispatcher.
132    ///
133    /// Sharing the dispatcher means `WorkflowStarted` /
134    /// `WorkflowStartFailure` events emitted by the returned client's
135    /// `WorkflowClient` flow into the same `MetricsCollector` that
136    /// [`enable_metrics`](Self::enable_metrics) installed.
137    #[must_use]
138    pub fn conductor_client(&self) -> ConductorClient {
139        ConductorClient::from_api_client(self.api_client.clone())
140            .with_event_dispatcher(self.event_dispatcher.clone())
141    }
142
143    /// Get a schema client.
144    #[must_use]
145    pub fn schema_client(&self) -> SchemaClient {
146        SchemaClient::new(self.api_client.clone())
147    }
148
149    /// Start all workers.
150    ///
151    /// # Errors
152    ///
153    /// Returns [`crate::error::ConductorError::Worker`] if no workers are registered.
154    pub async fn start(&mut self) -> Result<()> {
155        if self.workers.is_empty() {
156            return Err(ConductorError::worker("No workers registered"));
157        }
158
159        info!(worker_count = self.workers.len(), "Starting task handler");
160
161        // Start metrics HTTP server if configured
162        if let Some(ref collector) = self.metrics_collector {
163            self.metrics_handle = collector.start_http_server().await;
164        }
165
166        // Register task definitions if configured
167        let metadata_client = self.metadata_client();
168        for worker in &self.workers {
169            // Resolve worker configuration
170            let config = self.resolve_worker_config(worker.as_ref());
171
172            // Register task definition if configured
173            if config.register_task_def {
174                if let Err(e) = self
175                    .register_task_definition(&metadata_client, worker.as_ref(), &config)
176                    .await
177                {
178                    warn!(
179                        task_type = %worker.task_definition_name(),
180                        error = %e,
181                        "Failed to register task definition (worker will still start)"
182                    );
183                }
184            }
185
186            debug_log_worker(worker.as_ref(), &config);
187        }
188
189        // Create and start task runners
190        let task_client = self.task_client();
191
192        for worker in &self.workers {
193            let runner = Arc::new(TaskRunner::new(
194                Arc::clone(worker),
195                task_client.clone(),
196                self.event_dispatcher.clone(),
197            ));
198
199            self.runners.push(Arc::clone(&runner));
200
201            let handle = tokio::spawn(async move {
202                runner.run().await;
203            });
204
205            self.handles.push(handle);
206        }
207
208        info!(
209            runner_count = self.runners.len(),
210            "Task handler started successfully"
211        );
212
213        Ok(())
214    }
215
216    /// Verify that every registered worker's polling task actually started and completed at
217    /// least one real poll attempt against the server, within `timeout`. Call this right after
218    /// [`TaskHandler::start`].
219    ///
220    /// This crate's workers are `tokio::spawn`ed futures in the same process, where spawning
221    /// itself essentially never fails silently -- so the real risk isn't "did the task get
222    /// scheduled," it's "did this worker's task actually reach and complete its first poll,"
223    /// which catches an early panic during setup (before or shortly into [`TaskRunner::run`])
224    /// or a spawned task that's starved and never reaches the network call.
225    /// [`TaskRunner::poll_attempt_count`] is the signal used to tell that apart from
226    /// [`TaskRunner::is_running`] merely being `true`, which happens before the first poll is
227    /// even attempted.
228    ///
229    /// No-op (returns `Ok(())` immediately) if [`TaskHandler::start`] hasn't been called yet, or
230    /// registered no workers -- there's nothing to verify.
231    ///
232    /// # Errors
233    ///
234    /// Returns [`crate::error::ConductorError::Worker`] naming every task type that either
235    /// exited before its first poll (its spawned task ended, e.g. via an early panic) or simply
236    /// hadn't completed one within `timeout`.
237    pub async fn verify_workers_started(&self, timeout: Duration) -> Result<()> {
238        if self.runners.is_empty() {
239            return Ok(());
240        }
241
242        let deadline = Instant::now() + timeout;
243        let poll_interval = Duration::from_millis(50);
244
245        loop {
246            let not_ready: Vec<String> = self
247                .runners
248                .iter()
249                .zip(&self.handles)
250                .filter_map(|(runner, handle)| {
251                    if handle.is_finished() {
252                        Some(format!(
253                            "{} (task exited before its first poll)",
254                            runner.task_type()
255                        ))
256                    } else if runner.poll_attempt_count() == 0 {
257                        Some(runner.task_type().to_owned())
258                    } else {
259                        None
260                    }
261                })
262                .collect();
263
264            if not_ready.is_empty() {
265                return Ok(());
266            }
267            if Instant::now() >= deadline {
268                return Err(ConductorError::worker(format!(
269                    "Worker startup verification failed: worker(s) never completed a first \
270                     poll within {timeout:?}: [{}]. This usually means a worker panicked during \
271                     setup, or the async runtime is starved. Check logs and retry start().",
272                    not_ready.join(", "),
273                )));
274            }
275            tokio::time::sleep(poll_interval).await;
276        }
277    }
278
279    /// Stop all workers gracefully.
280    ///
281    /// This method:
282    /// 1. Signals all runners to stop polling
283    /// 2. Waits for in-flight tasks to complete (up to 30 seconds)
284    /// 3. Stops the metrics server
285    ///
286    /// # Errors
287    ///
288    /// Currently always returns `Ok(())`; the `Result` is reserved for future shutdown failures (e.g. a worker task panic is logged, not propagated).
289    pub async fn stop(&mut self) -> Result<()> {
290        info!("Stopping task handler");
291
292        // Signal all runners to stop
293        for runner in &self.runners {
294            runner.stop();
295        }
296
297        // Wait for all runner handles to complete (with timeout)
298        // The runners themselves wait for their spawned tasks
299        let shutdown_timeout = std::time::Duration::from_secs(30);
300        let start = std::time::Instant::now();
301
302        // Take handles using futures::future::join_all for cleaner handling
303        let handles: Vec<_> = std::mem::take(&mut self.handles);
304
305        if !handles.is_empty() {
306            let wait_result =
307                tokio::time::timeout(shutdown_timeout, futures::future::join_all(handles)).await;
308
309            match wait_result {
310                Ok(results) => {
311                    for result in results {
312                        if let Err(e) = result {
313                            error!(error = %e, "Worker task panicked during shutdown");
314                        }
315                    }
316                    debug!(
317                        elapsed_ms = start.elapsed().as_millis(),
318                        "All workers stopped gracefully"
319                    );
320                }
321                Err(_) => {
322                    warn!(
323                        timeout_secs = shutdown_timeout.as_secs(),
324                        "Timeout waiting for workers to stop, some tasks may still be running"
325                    );
326                }
327            }
328        }
329
330        // Stop metrics server
331        if let Some(handle) = self.metrics_handle.take() {
332            handle.abort();
333        }
334
335        info!("Task handler stopped");
336        Ok(())
337    }
338
339    /// Wait for all workers to complete.
340    ///
341    /// # Errors
342    ///
343    /// Currently always returns `Ok(())`; a worker task panic is logged, not propagated.
344    pub async fn join(&mut self) -> Result<()> {
345        for handle in self.handles.drain(..) {
346            if let Err(e) = handle.await {
347                error!(error = %e, "Worker task panicked");
348            }
349        }
350        Ok(())
351    }
352
353    /// Get the number of registered workers.
354    #[must_use]
355    pub fn worker_count(&self) -> usize {
356        self.workers.len()
357    }
358
359    /// Check if all workers are running.
360    #[must_use]
361    pub fn is_running(&self) -> bool {
362        !self.runners.is_empty() && self.runners.iter().all(|r| r.is_running())
363    }
364
365    /// Pause a specific worker by task type.
366    pub fn pause_worker(&self, task_type: &str) {
367        for runner in &self.runners {
368            if runner.task_type() == task_type {
369                runner.pause();
370            }
371        }
372    }
373
374    /// Resume a specific worker by task type.
375    pub fn resume_worker(&self, task_type: &str) {
376        for runner in &self.runners {
377            if runner.task_type() == task_type {
378                runner.resume();
379            }
380        }
381    }
382
383    /// Pause all workers.
384    pub fn pause_all(&self) {
385        for runner in &self.runners {
386            runner.pause();
387        }
388    }
389
390    /// Resume all workers.
391    pub fn resume_all(&self) {
392        for runner in &self.runners {
393            runner.resume();
394        }
395    }
396
397    /// Resolve worker configuration from environment variables.
398    fn resolve_worker_config(&self, worker: &dyn Worker) -> WorkerConfig {
399        let defaults = WorkerConfig {
400            task_definition_name: worker.task_definition_name().to_owned(),
401            poll_interval: std::time::Duration::from_millis(worker.poll_interval_millis()),
402            domain: worker.domain().map(std::borrow::ToOwned::to_owned),
403            worker_id: worker.identity(),
404            thread_count: worker.thread_count(),
405            ..Default::default()
406        };
407
408        resolve_worker_config(worker.task_definition_name(), defaults)
409    }
410
411    /// Register a task definition for a worker.
412    async fn register_task_definition(
413        &self,
414        metadata_client: &MetadataClient,
415        worker: &dyn Worker,
416        config: &WorkerConfig,
417    ) -> Result<()> {
418        let task_name = worker.task_definition_name();
419
420        // Check if task definition exists
421        let exists = metadata_client.task_def_exists(task_name).await?;
422
423        if exists && !config.overwrite_task_def {
424            debug!(
425                task_type = %task_name,
426                "Task definition already exists, skipping registration (overwrite_task_def=false)"
427            );
428            return Ok(());
429        }
430
431        // Create task definition from worker
432        let task_def = TaskDef::new(task_name)
433            .with_description("Task registered by Rust SDK worker")
434            .with_runtime_metadata(worker.declared_credentials());
435
436        if exists {
437            info!(
438                task_type = %task_name,
439                "Updating existing task definition"
440            );
441            metadata_client.update_task_def(&task_def).await?;
442        } else {
443            info!(
444                task_type = %task_name,
445                "Registering new task definition"
446            );
447            metadata_client.register_task_def(&task_def).await?;
448        }
449
450        // Register input/output schemas if provided
451        self.register_worker_schemas(worker, task_name).await?;
452
453        Ok(())
454    }
455
456    /// Register input and output schemas for a worker.
457    async fn register_worker_schemas(&self, worker: &dyn Worker, task_name: &str) -> Result<()> {
458        let schema_client = self.schema_client();
459
460        // Register input schema if provided
461        if let Some(input_schema) = worker.input_schema() {
462            let schema_def = SchemaDef::new(format!("{task_name}_input"), 1, input_schema);
463
464            match schema_client.register_schema(&schema_def).await {
465                Ok(()) => {
466                    info!(
467                        task_type = %task_name,
468                        schema_name = %schema_def.name,
469                        "Registered input schema"
470                    );
471                }
472                Err(e) => {
473                    warn!(
474                        task_type = %task_name,
475                        error = %e,
476                        "Failed to register input schema (continuing anyway)"
477                    );
478                }
479            }
480        }
481
482        // Register output schema if provided
483        if let Some(output_schema) = worker.output_schema() {
484            let schema_def = SchemaDef::new(format!("{task_name}_output"), 1, output_schema);
485
486            match schema_client.register_schema(&schema_def).await {
487                Ok(()) => {
488                    info!(
489                        task_type = %task_name,
490                        schema_name = %schema_def.name,
491                        "Registered output schema"
492                    );
493                }
494                Err(e) => {
495                    warn!(
496                        task_type = %task_name,
497                        error = %e,
498                        "Failed to register output schema (continuing anyway)"
499                    );
500                }
501            }
502        }
503
504        Ok(())
505    }
506}
507
508fn debug_log_worker(worker: &dyn Worker, config: &WorkerConfig) {
509    info!(
510        task_type = %worker.task_definition_name(),
511        worker_id = %config.worker_id,
512        thread_count = config.thread_count,
513        poll_interval_ms = config.poll_interval.as_millis(),
514        domain = ?config.domain,
515        paused = config.paused,
516        register_task_def = config.register_task_def,
517        "Conductor Worker registered"
518    );
519}
520
521/// Builder for `TaskHandler`.
522pub struct TaskHandlerBuilder {
523    config: Configuration,
524    workers: Vec<Arc<dyn Worker>>,
525    event_listeners: Vec<Arc<dyn TaskRunnerEventsListener>>,
526    metrics_settings: Option<MetricsSettings>,
527}
528
529impl TaskHandlerBuilder {
530    /// Create a new builder.
531    #[must_use]
532    pub fn new(config: Configuration) -> Self {
533        Self {
534            config,
535            workers: Vec::new(),
536            event_listeners: Vec::new(),
537            metrics_settings: None,
538        }
539    }
540
541    /// Add a worker.
542    #[must_use]
543    pub fn worker(mut self, worker: impl Worker + 'static) -> Self {
544        self.workers.push(Arc::new(worker));
545        self
546    }
547
548    /// Add multiple workers.
549    #[must_use]
550    pub fn workers(mut self, workers: impl IntoIterator<Item = Arc<dyn Worker>>) -> Self {
551        self.workers.extend(workers);
552        self
553    }
554
555    /// Add an event listener.
556    #[must_use]
557    pub fn event_listener(mut self, listener: impl TaskRunnerEventsListener + 'static) -> Self {
558        self.event_listeners.push(Arc::new(listener));
559        self
560    }
561
562    /// Enable metrics with the given settings.
563    #[must_use]
564    pub fn metrics(mut self, settings: MetricsSettings) -> Self {
565        self.metrics_settings = Some(settings);
566        self
567    }
568
569    /// Build the task handler.
570    ///
571    /// # Errors
572    ///
573    /// Returns [`crate::error::ConductorError::Http`] if the underlying `reqwest` client fails to build (see [`TaskHandler::new`]).
574    pub fn build(self) -> Result<TaskHandler> {
575        let mut handler = TaskHandler::new(self.config)?;
576
577        handler.add_workers(self.workers);
578
579        for listener in self.event_listeners {
580            handler.add_event_listener(listener);
581        }
582
583        if let Some(settings) = self.metrics_settings {
584            handler.enable_metrics(settings);
585        }
586
587        Ok(handler)
588    }
589}
590
591#[cfg(test)]
592mod tests {
593    use super::*;
594    use crate::worker::{Worker, WorkerOutput};
595    use async_trait::async_trait;
596
597    struct TestWorker {
598        name: String,
599    }
600
601    #[async_trait]
602    impl Worker for TestWorker {
603        fn task_definition_name(&self) -> &str {
604            &self.name
605        }
606
607        async fn execute(&self, _task: &crate::models::Task) -> Result<WorkerOutput> {
608            Ok(WorkerOutput::completed_with_result("test"))
609        }
610    }
611
612    #[test]
613    fn test_task_handler_builder() {
614        let config = Configuration::new("http://localhost:8080/api");
615
616        let handler = TaskHandlerBuilder::new(config)
617            .worker(TestWorker {
618                name: "task1".to_owned(),
619            })
620            .worker(TestWorker {
621                name: "task2".to_owned(),
622            })
623            .build();
624
625        assert!(handler.is_ok());
626        let handler = handler.unwrap();
627        assert_eq!(handler.worker_count(), 2);
628    }
629
630    #[tokio::test]
631    async fn test_verify_workers_started_is_a_no_op_before_start() {
632        let config = Configuration::new("http://localhost:8080/api");
633        let mut handler = TaskHandler::new(config).unwrap();
634        handler.add_worker(TestWorker {
635            name: "never_started".to_owned(),
636        });
637
638        // start() was never called, so there are no runners/handles yet -- nothing to verify.
639        handler
640            .verify_workers_started(Duration::ZERO)
641            .await
642            .expect("no-op before start() must not error");
643    }
644
645    #[tokio::test]
646    async fn test_verify_workers_started_succeeds_after_a_real_poll() {
647        let mock_server = wiremock::MockServer::start().await;
648
649        wiremock::Mock::given(wiremock::matchers::method("GET"))
650            .and(wiremock::matchers::path(
651                "/api/tasks/poll/batch/verify_started_ok",
652            ))
653            .respond_with(wiremock::ResponseTemplate::new(200).set_body_json(serde_json::json!([])))
654            .mount(&mock_server)
655            .await;
656
657        let config = Configuration::new(format!("{}/api", mock_server.uri()));
658        let mut handler = TaskHandler::new(config).unwrap();
659        handler.add_worker(TestWorker {
660            name: "verify_started_ok".to_owned(),
661        });
662        handler.start().await.unwrap();
663
664        handler
665            .verify_workers_started(Duration::from_secs(2))
666            .await
667            .expect("worker should complete a real poll well within 2s");
668
669        // Not calling handler.stop() -- see test_resume_registers_workers_... in
670        // agents::runtime for why: the #[tokio::test] runtime aborts the spawned poller when
671        // this function returns, which is faster and just as clean here.
672    }
673
674    #[tokio::test]
675    async fn test_verify_workers_started_times_out_when_poll_never_completes() {
676        let mock_server = wiremock::MockServer::start().await;
677
678        // A response slower than the timeout below simulates a worker whose task is legitimately
679        // still running but hasn't reached (completed) its first real poll yet.
680        wiremock::Mock::given(wiremock::matchers::method("GET"))
681            .and(wiremock::matchers::path(
682                "/api/tasks/poll/batch/verify_started_stuck",
683            ))
684            .respond_with(
685                wiremock::ResponseTemplate::new(200)
686                    .set_body_json(serde_json::json!([]))
687                    .set_delay(Duration::from_secs(5)),
688            )
689            .mount(&mock_server)
690            .await;
691
692        let config = Configuration::new(format!("{}/api", mock_server.uri()));
693        let mut handler = TaskHandler::new(config).unwrap();
694        handler.add_worker(TestWorker {
695            name: "verify_started_stuck".to_owned(),
696        });
697        handler.start().await.unwrap();
698
699        let err = handler
700            .verify_workers_started(Duration::from_millis(200))
701            .await
702            .expect_err("poll response is delayed well past the 200ms timeout");
703        assert!(err.to_string().contains("verify_started_stuck"));
704    }
705}