1use std::sync::Arc;
5use std::time::{Duration, Instant};
6use tokio::task::JoinHandle;
7use tracing::{debug, error, info, warn};
8
9use crate::client::{ConductorClient, MetadataClient, SchemaClient, TaskClient};
10use crate::configuration::{resolve_worker_config, Configuration, WorkerConfig};
11use crate::error::{ConductorError, Result};
12use crate::events::{EventDispatcher, TaskRunnerEventsListener};
13use crate::http::ApiClient;
14use crate::metrics::{MetricsCollector, MetricsSettings};
15use crate::models::{SchemaDef, TaskDef};
16
17use super::{TaskRunner, Worker};
18
19pub struct TaskHandler {
21 #[expect(dead_code)] config: Configuration,
23 api_client: ApiClient,
24 event_dispatcher: EventDispatcher,
25 workers: Vec<Arc<dyn Worker>>,
26 runners: Vec<Arc<TaskRunner>>,
27 handles: Vec<JoinHandle<()>>,
28 metrics_collector: Option<Arc<MetricsCollector>>,
29 metrics_handle: Option<JoinHandle<()>>,
30}
31
32impl TaskHandler {
33 pub fn new(config: Configuration) -> Result<Self> {
39 let api_client = ApiClient::new(config.clone())?;
40 let event_dispatcher = EventDispatcher::new();
41
42 Ok(Self {
43 config,
44 api_client,
45 event_dispatcher,
46 workers: Vec::new(),
47 runners: Vec::new(),
48 handles: Vec::new(),
49 metrics_collector: None,
50 metrics_handle: None,
51 })
52 }
53
54 #[must_use]
56 pub fn builder(config: Configuration) -> TaskHandlerBuilder {
57 TaskHandlerBuilder::new(config)
58 }
59
60 pub fn add_worker(&mut self, worker: impl Worker + 'static) {
74 self.workers.push(Arc::new(worker));
75 }
76
77 pub fn add_workers(&mut self, workers: impl IntoIterator<Item = Arc<dyn Worker>>) {
79 self.workers.extend(workers);
80 }
81
82 pub fn add_event_listener(&self, listener: Arc<dyn TaskRunnerEventsListener>) {
84 self.event_dispatcher.register(listener);
85 }
86
87 pub fn enable_metrics(&mut self, settings: MetricsSettings) {
99 let collector = Arc::new(MetricsCollector::new(settings));
100 self.event_dispatcher
101 .register(Arc::clone(&collector) as Arc<dyn TaskRunnerEventsListener>);
102 self.api_client
103 .set_http_observer(Arc::clone(&collector) as Arc<dyn crate::http::HttpMetricsObserver>);
104 self.metrics_collector = Some(collector);
105 }
106
107 #[must_use]
109 pub fn metrics_collector(&self) -> Option<&Arc<MetricsCollector>> {
110 self.metrics_collector.as_ref()
111 }
112
113 #[must_use]
115 pub fn event_dispatcher(&self) -> &EventDispatcher {
116 &self.event_dispatcher
117 }
118
119 #[must_use]
121 pub fn task_client(&self) -> TaskClient {
122 TaskClient::new(self.api_client.clone())
123 }
124
125 #[must_use]
127 pub fn metadata_client(&self) -> MetadataClient {
128 MetadataClient::new(self.api_client.clone())
129 }
130
131 #[must_use]
138 pub fn conductor_client(&self) -> ConductorClient {
139 ConductorClient::from_api_client(self.api_client.clone())
140 .with_event_dispatcher(self.event_dispatcher.clone())
141 }
142
143 #[must_use]
145 pub fn schema_client(&self) -> SchemaClient {
146 SchemaClient::new(self.api_client.clone())
147 }
148
149 pub async fn start(&mut self) -> Result<()> {
155 if self.workers.is_empty() {
156 return Err(ConductorError::worker("No workers registered"));
157 }
158
159 info!(worker_count = self.workers.len(), "Starting task handler");
160
161 if let Some(ref collector) = self.metrics_collector {
163 self.metrics_handle = collector.start_http_server().await;
164 }
165
166 let metadata_client = self.metadata_client();
168 for worker in &self.workers {
169 let config = self.resolve_worker_config(worker.as_ref());
171
172 if config.register_task_def {
174 if let Err(e) = self
175 .register_task_definition(&metadata_client, worker.as_ref(), &config)
176 .await
177 {
178 warn!(
179 task_type = %worker.task_definition_name(),
180 error = %e,
181 "Failed to register task definition (worker will still start)"
182 );
183 }
184 }
185
186 debug_log_worker(worker.as_ref(), &config);
187 }
188
189 let task_client = self.task_client();
191
192 for worker in &self.workers {
193 let runner = Arc::new(TaskRunner::new(
194 Arc::clone(worker),
195 task_client.clone(),
196 self.event_dispatcher.clone(),
197 ));
198
199 self.runners.push(Arc::clone(&runner));
200
201 let handle = tokio::spawn(async move {
202 runner.run().await;
203 });
204
205 self.handles.push(handle);
206 }
207
208 info!(
209 runner_count = self.runners.len(),
210 "Task handler started successfully"
211 );
212
213 Ok(())
214 }
215
216 pub async fn verify_workers_started(&self, timeout: Duration) -> Result<()> {
238 if self.runners.is_empty() {
239 return Ok(());
240 }
241
242 let deadline = Instant::now() + timeout;
243 let poll_interval = Duration::from_millis(50);
244
245 loop {
246 let not_ready: Vec<String> = self
247 .runners
248 .iter()
249 .zip(&self.handles)
250 .filter_map(|(runner, handle)| {
251 if handle.is_finished() {
252 Some(format!(
253 "{} (task exited before its first poll)",
254 runner.task_type()
255 ))
256 } else if runner.poll_attempt_count() == 0 {
257 Some(runner.task_type().to_owned())
258 } else {
259 None
260 }
261 })
262 .collect();
263
264 if not_ready.is_empty() {
265 return Ok(());
266 }
267 if Instant::now() >= deadline {
268 return Err(ConductorError::worker(format!(
269 "Worker startup verification failed: worker(s) never completed a first \
270 poll within {timeout:?}: [{}]. This usually means a worker panicked during \
271 setup, or the async runtime is starved. Check logs and retry start().",
272 not_ready.join(", "),
273 )));
274 }
275 tokio::time::sleep(poll_interval).await;
276 }
277 }
278
279 pub async fn stop(&mut self) -> Result<()> {
290 info!("Stopping task handler");
291
292 for runner in &self.runners {
294 runner.stop();
295 }
296
297 let shutdown_timeout = std::time::Duration::from_secs(30);
300 let start = std::time::Instant::now();
301
302 let handles: Vec<_> = std::mem::take(&mut self.handles);
304
305 if !handles.is_empty() {
306 let wait_result =
307 tokio::time::timeout(shutdown_timeout, futures::future::join_all(handles)).await;
308
309 match wait_result {
310 Ok(results) => {
311 for result in results {
312 if let Err(e) = result {
313 error!(error = %e, "Worker task panicked during shutdown");
314 }
315 }
316 debug!(
317 elapsed_ms = start.elapsed().as_millis(),
318 "All workers stopped gracefully"
319 );
320 }
321 Err(_) => {
322 warn!(
323 timeout_secs = shutdown_timeout.as_secs(),
324 "Timeout waiting for workers to stop, some tasks may still be running"
325 );
326 }
327 }
328 }
329
330 if let Some(handle) = self.metrics_handle.take() {
332 handle.abort();
333 }
334
335 info!("Task handler stopped");
336 Ok(())
337 }
338
339 pub async fn join(&mut self) -> Result<()> {
345 for handle in self.handles.drain(..) {
346 if let Err(e) = handle.await {
347 error!(error = %e, "Worker task panicked");
348 }
349 }
350 Ok(())
351 }
352
353 #[must_use]
355 pub fn worker_count(&self) -> usize {
356 self.workers.len()
357 }
358
359 #[must_use]
361 pub fn is_running(&self) -> bool {
362 !self.runners.is_empty() && self.runners.iter().all(|r| r.is_running())
363 }
364
365 pub fn pause_worker(&self, task_type: &str) {
367 for runner in &self.runners {
368 if runner.task_type() == task_type {
369 runner.pause();
370 }
371 }
372 }
373
374 pub fn resume_worker(&self, task_type: &str) {
376 for runner in &self.runners {
377 if runner.task_type() == task_type {
378 runner.resume();
379 }
380 }
381 }
382
383 pub fn pause_all(&self) {
385 for runner in &self.runners {
386 runner.pause();
387 }
388 }
389
390 pub fn resume_all(&self) {
392 for runner in &self.runners {
393 runner.resume();
394 }
395 }
396
397 fn resolve_worker_config(&self, worker: &dyn Worker) -> WorkerConfig {
399 let defaults = WorkerConfig {
400 task_definition_name: worker.task_definition_name().to_owned(),
401 poll_interval: std::time::Duration::from_millis(worker.poll_interval_millis()),
402 domain: worker.domain().map(std::borrow::ToOwned::to_owned),
403 worker_id: worker.identity(),
404 thread_count: worker.thread_count(),
405 ..Default::default()
406 };
407
408 resolve_worker_config(worker.task_definition_name(), defaults)
409 }
410
411 async fn register_task_definition(
413 &self,
414 metadata_client: &MetadataClient,
415 worker: &dyn Worker,
416 config: &WorkerConfig,
417 ) -> Result<()> {
418 let task_name = worker.task_definition_name();
419
420 let exists = metadata_client.task_def_exists(task_name).await?;
422
423 if exists && !config.overwrite_task_def {
424 debug!(
425 task_type = %task_name,
426 "Task definition already exists, skipping registration (overwrite_task_def=false)"
427 );
428 return Ok(());
429 }
430
431 let task_def = TaskDef::new(task_name)
433 .with_description("Task registered by Rust SDK worker")
434 .with_runtime_metadata(worker.declared_credentials());
435
436 if exists {
437 info!(
438 task_type = %task_name,
439 "Updating existing task definition"
440 );
441 metadata_client.update_task_def(&task_def).await?;
442 } else {
443 info!(
444 task_type = %task_name,
445 "Registering new task definition"
446 );
447 metadata_client.register_task_def(&task_def).await?;
448 }
449
450 self.register_worker_schemas(worker, task_name).await?;
452
453 Ok(())
454 }
455
456 async fn register_worker_schemas(&self, worker: &dyn Worker, task_name: &str) -> Result<()> {
458 let schema_client = self.schema_client();
459
460 if let Some(input_schema) = worker.input_schema() {
462 let schema_def = SchemaDef::new(format!("{task_name}_input"), 1, input_schema);
463
464 match schema_client.register_schema(&schema_def).await {
465 Ok(()) => {
466 info!(
467 task_type = %task_name,
468 schema_name = %schema_def.name,
469 "Registered input schema"
470 );
471 }
472 Err(e) => {
473 warn!(
474 task_type = %task_name,
475 error = %e,
476 "Failed to register input schema (continuing anyway)"
477 );
478 }
479 }
480 }
481
482 if let Some(output_schema) = worker.output_schema() {
484 let schema_def = SchemaDef::new(format!("{task_name}_output"), 1, output_schema);
485
486 match schema_client.register_schema(&schema_def).await {
487 Ok(()) => {
488 info!(
489 task_type = %task_name,
490 schema_name = %schema_def.name,
491 "Registered output schema"
492 );
493 }
494 Err(e) => {
495 warn!(
496 task_type = %task_name,
497 error = %e,
498 "Failed to register output schema (continuing anyway)"
499 );
500 }
501 }
502 }
503
504 Ok(())
505 }
506}
507
508fn debug_log_worker(worker: &dyn Worker, config: &WorkerConfig) {
509 info!(
510 task_type = %worker.task_definition_name(),
511 worker_id = %config.worker_id,
512 thread_count = config.thread_count,
513 poll_interval_ms = config.poll_interval.as_millis(),
514 domain = ?config.domain,
515 paused = config.paused,
516 register_task_def = config.register_task_def,
517 "Conductor Worker registered"
518 );
519}
520
521pub struct TaskHandlerBuilder {
523 config: Configuration,
524 workers: Vec<Arc<dyn Worker>>,
525 event_listeners: Vec<Arc<dyn TaskRunnerEventsListener>>,
526 metrics_settings: Option<MetricsSettings>,
527}
528
529impl TaskHandlerBuilder {
530 #[must_use]
532 pub fn new(config: Configuration) -> Self {
533 Self {
534 config,
535 workers: Vec::new(),
536 event_listeners: Vec::new(),
537 metrics_settings: None,
538 }
539 }
540
541 #[must_use]
543 pub fn worker(mut self, worker: impl Worker + 'static) -> Self {
544 self.workers.push(Arc::new(worker));
545 self
546 }
547
548 #[must_use]
550 pub fn workers(mut self, workers: impl IntoIterator<Item = Arc<dyn Worker>>) -> Self {
551 self.workers.extend(workers);
552 self
553 }
554
555 #[must_use]
557 pub fn event_listener(mut self, listener: impl TaskRunnerEventsListener + 'static) -> Self {
558 self.event_listeners.push(Arc::new(listener));
559 self
560 }
561
562 #[must_use]
564 pub fn metrics(mut self, settings: MetricsSettings) -> Self {
565 self.metrics_settings = Some(settings);
566 self
567 }
568
569 pub fn build(self) -> Result<TaskHandler> {
575 let mut handler = TaskHandler::new(self.config)?;
576
577 handler.add_workers(self.workers);
578
579 for listener in self.event_listeners {
580 handler.add_event_listener(listener);
581 }
582
583 if let Some(settings) = self.metrics_settings {
584 handler.enable_metrics(settings);
585 }
586
587 Ok(handler)
588 }
589}
590
591#[cfg(test)]
592mod tests {
593 use super::*;
594 use crate::worker::{Worker, WorkerOutput};
595 use async_trait::async_trait;
596
597 struct TestWorker {
598 name: String,
599 }
600
601 #[async_trait]
602 impl Worker for TestWorker {
603 fn task_definition_name(&self) -> &str {
604 &self.name
605 }
606
607 async fn execute(&self, _task: &crate::models::Task) -> Result<WorkerOutput> {
608 Ok(WorkerOutput::completed_with_result("test"))
609 }
610 }
611
612 #[test]
613 fn test_task_handler_builder() {
614 let config = Configuration::new("http://localhost:8080/api");
615
616 let handler = TaskHandlerBuilder::new(config)
617 .worker(TestWorker {
618 name: "task1".to_owned(),
619 })
620 .worker(TestWorker {
621 name: "task2".to_owned(),
622 })
623 .build();
624
625 assert!(handler.is_ok());
626 let handler = handler.unwrap();
627 assert_eq!(handler.worker_count(), 2);
628 }
629
630 #[tokio::test]
631 async fn test_verify_workers_started_is_a_no_op_before_start() {
632 let config = Configuration::new("http://localhost:8080/api");
633 let mut handler = TaskHandler::new(config).unwrap();
634 handler.add_worker(TestWorker {
635 name: "never_started".to_owned(),
636 });
637
638 handler
640 .verify_workers_started(Duration::ZERO)
641 .await
642 .expect("no-op before start() must not error");
643 }
644
645 #[tokio::test]
646 async fn test_verify_workers_started_succeeds_after_a_real_poll() {
647 let mock_server = wiremock::MockServer::start().await;
648
649 wiremock::Mock::given(wiremock::matchers::method("GET"))
650 .and(wiremock::matchers::path(
651 "/api/tasks/poll/batch/verify_started_ok",
652 ))
653 .respond_with(wiremock::ResponseTemplate::new(200).set_body_json(serde_json::json!([])))
654 .mount(&mock_server)
655 .await;
656
657 let config = Configuration::new(format!("{}/api", mock_server.uri()));
658 let mut handler = TaskHandler::new(config).unwrap();
659 handler.add_worker(TestWorker {
660 name: "verify_started_ok".to_owned(),
661 });
662 handler.start().await.unwrap();
663
664 handler
665 .verify_workers_started(Duration::from_secs(2))
666 .await
667 .expect("worker should complete a real poll well within 2s");
668
669 }
673
674 #[tokio::test]
675 async fn test_verify_workers_started_times_out_when_poll_never_completes() {
676 let mock_server = wiremock::MockServer::start().await;
677
678 wiremock::Mock::given(wiremock::matchers::method("GET"))
681 .and(wiremock::matchers::path(
682 "/api/tasks/poll/batch/verify_started_stuck",
683 ))
684 .respond_with(
685 wiremock::ResponseTemplate::new(200)
686 .set_body_json(serde_json::json!([]))
687 .set_delay(Duration::from_secs(5)),
688 )
689 .mount(&mock_server)
690 .await;
691
692 let config = Configuration::new(format!("{}/api", mock_server.uri()));
693 let mut handler = TaskHandler::new(config).unwrap();
694 handler.add_worker(TestWorker {
695 name: "verify_started_stuck".to_owned(),
696 });
697 handler.start().await.unwrap();
698
699 let err = handler
700 .verify_workers_started(Duration::from_millis(200))
701 .await
702 .expect_err("poll response is delayed well past the 200ms timeout");
703 assert!(err.to_string().contains("verify_started_stuck"));
704 }
705}