1#![doc = include_str!("../README.md")]
2
3use std::{
4 any::{Any, TypeId},
5 collections::{BTreeMap, HashMap},
6 future::Future,
7 pin::Pin,
8 sync::{
9 atomic::{AtomicBool, Ordering},
10 Arc, Mutex, OnceLock,
11 },
12 task::{Context as TaskContext, Poll},
13 time::{Duration, Instant, SystemTime, UNIX_EPOCH},
14};
15
16use apache_avro::{from_avro_datum, from_value, to_avro_datum, to_value, Schema};
17use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
18use futures_util::{future::OptionFuture, task::noop_waker_ref};
19use serde::{de::DeserializeOwned, Deserialize, Serialize};
20pub use serde_json::{json, Value};
21use thiserror::Error;
22
23pub const WORKER_PROTOCOL_VERSION: &str = "1.2";
24pub const CONTROL_PLANE_VERSION: &str = "2";
25pub const DEFAULT_CODEC: &str = "avro";
26pub const JSON_CODEC: &str = "json";
27pub const SDK_VERSION: &str = concat!("durable-workflow-rust/", env!("CARGO_PKG_VERSION"));
28pub const QUERY_TASKS_CAPABILITY: &str = "query_tasks";
30pub const QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
32
33const MAX_LONG_POLL_TIMEOUT_SECONDS: u64 = 60;
34const WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE: &str =
35 "Workflow task waiting for scheduled history.";
36const WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE: &str = "WorkflowTaskWaitingForHistory";
37
38const QUERY_TASK_FINAL_REJECTION_REASONS: &[&str] = &[
39 "lease_expired",
40 "query_task_not_found",
41 "query_task_not_leased",
42 "query_task_timed_out",
43];
44
45const AVRO_PAYLOAD_SCHEMA_JSON: &str = r#"{"type":"record","name":"Payload","namespace":"durable_workflow","fields":[{"name":"json","type":"string"},{"name":"version","type":"int","default":1}]}"#;
46const AVRO_PAYLOAD_VERSION: i32 = 1;
47
48static AVRO_PAYLOAD_SCHEMA: OnceLock<std::result::Result<Schema, String>> = OnceLock::new();
49
50#[derive(Clone, Copy)]
51enum RequestProtocol {
52 ControlPlane,
53 Worker(&'static str),
54}
55
56impl RequestProtocol {
57 fn is_worker(self) -> bool {
58 matches!(self, Self::Worker(_))
59 }
60}
61
62pub type Result<T> = std::result::Result<T, Error>;
63
64#[derive(Debug, Error)]
65pub enum Error {
66 #[error("transport error: {0}")]
67 Transport(#[from] reqwest::Error),
68 #[error("json error: {0}")]
69 Json(#[from] serde_json::Error),
70 #[error("http {status}: {body}")]
71 Http {
72 status: reqwest::StatusCode,
73 body: String,
74 },
75 #[error("codec error: {0}")]
76 Codec(String),
77 #[error(transparent)]
78 QueryFailed(QueryFailure),
79 #[error(transparent)]
80 Protocol(ProtocolFailure),
81 #[error(transparent)]
82 NonDeterministicReplay(ReplayFailure),
83 #[error(transparent)]
84 ChildWorkflowFailed(ChildWorkflowFailure),
85 #[error(transparent)]
86 ActivityFailed(ActivityFailure),
87 #[error(transparent)]
88 WorkflowCommandRejected(WorkflowCommandRejection),
89 #[error(transparent)]
90 WorkflowFailed(WorkflowTerminalOutcome),
91 #[error(transparent)]
92 WorkflowCancelled(WorkflowTerminalOutcome),
93 #[error(transparent)]
94 WorkflowTerminated(WorkflowTerminalOutcome),
95 #[error(transparent)]
96 WorkflowTimedOut(WorkflowTerminalOutcome),
97 #[error(transparent)]
98 ActivityTaskRejected(ActivityTaskRejection),
99 #[error("workflow handler {0:?} is not registered")]
100 WorkflowNotRegistered(String),
101 #[error("activity handler {0:?} is not registered")]
102 ActivityNotRegistered(String),
103 #[error("workflow future yielded without emitting a durable command")]
104 WorkflowYieldedWithoutCommand,
105 #[error("workflow state lock is poisoned")]
106 WorkflowStatePoisoned,
107 #[error("timer duration is too large for the worker protocol")]
108 TimerDurationOverflow,
109 #[error("operation timed out")]
110 Timeout,
111 #[error("worker loop error: {0}")]
112 WorkerLoop(String),
113 #[error("invalid child workflow options: {0}")]
114 InvalidChildWorkflowOptions(String),
115 #[error(transparent)]
116 InvalidActivityOptions(ActivityOptionsError),
117}
118
119#[derive(Clone, Copy, Debug, PartialEq, Eq)]
121pub enum WorkflowCommandKind {
122 Cancel,
123 Terminate,
124}
125
126impl WorkflowCommandKind {
127 fn as_str(self) -> &'static str {
128 match self {
129 Self::Cancel => "cancel",
130 Self::Terminate => "terminate",
131 }
132 }
133}
134
135#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
137pub struct WorkflowCommandOptions {
138 #[serde(skip_serializing_if = "Option::is_none")]
139 pub reason: Option<String>,
140 #[serde(skip_serializing_if = "Option::is_none")]
141 pub request_id: Option<String>,
142}
143
144impl WorkflowCommandOptions {
145 pub fn new() -> Self {
146 Self::default()
147 }
148
149 pub fn reason(mut self, reason: impl Into<String>) -> Self {
150 self.reason = Some(reason.into());
151 self
152 }
153
154 pub fn request_id(mut self, request_id: impl Into<String>) -> Self {
155 self.request_id = Some(request_id.into());
156 self
157 }
158}
159
160#[derive(Clone, Debug, PartialEq)]
162pub struct WorkflowCommandResult {
163 pub command: WorkflowCommandKind,
164 pub workflow_id: String,
165 pub run_id: Option<String>,
166 pub outcome: Option<String>,
167 pub reason: Option<String>,
168 pub command_status: Option<String>,
169 pub raw: Value,
170}
171
172#[derive(Clone, Debug, Error)]
174#[error("workflow {command:?} rejected ({reason}, HTTP {status}): {message}")]
175pub struct WorkflowCommandRejection {
176 pub command: WorkflowCommandKind,
177 pub status: u16,
178 pub reason: String,
179 pub message: String,
180 pub workflow_id: String,
181 pub run_id: Option<String>,
182 pub target_scope: Option<String>,
183 pub body: Value,
184}
185
186#[derive(Clone, Copy, Debug, PartialEq, Eq)]
188pub enum WorkflowTerminalKind {
189 Failed,
190 Cancelled,
191 Terminated,
192 TimedOut,
193}
194
195#[derive(Clone, Debug, Error)]
201#[error("workflow {workflow_id} run {run_id:?} ended as {kind:?} ({reason})")]
202pub struct WorkflowTerminalOutcome {
203 pub kind: WorkflowTerminalKind,
204 pub workflow_id: String,
205 pub run_id: Option<String>,
206 pub reason: String,
207 pub failure_category: Option<String>,
208 pub failure_id: Option<String>,
209 pub exception_type: Option<String>,
210 pub exception_class: Option<String>,
211 pub non_retryable: Option<bool>,
212 pub message: Option<String>,
213 pub exception: Option<Value>,
214 pub raw: Value,
215}
216
217#[derive(Clone, Debug, Error)]
219#[error("activity task {operation} rejected ({reason}, HTTP {status})")]
220pub struct ActivityTaskRejection {
221 pub operation: String,
222 pub status: u16,
223 pub reason: String,
224 pub task_id: String,
225 pub activity_attempt_id: String,
226 pub cancel_requested: bool,
227 pub can_continue: Option<bool>,
228 pub run_closed_reason: Option<String>,
229 pub body: Value,
230}
231
232#[derive(Clone, Copy, Debug, PartialEq, Eq)]
234pub enum ActivityOptionsErrorKind {
235 EmptyTaskQueue,
236 EmptyRetryPolicy,
237 InvalidMaxAttempts,
238 BackoffWithoutRetryBudget,
239 TooManyBackoffIntervals,
240 InvalidBackoffCoefficient,
241 BackoffGenerationTooLarge,
242 BackoffOverflow,
243 EmptyNonRetryableErrorType,
244 TimeoutNotPositive,
245 TimeoutOverflow,
246 TimeoutOrder,
247}
248
249#[derive(Clone, Debug, Error, PartialEq, Eq)]
251#[error("invalid activity options ({kind:?}, {field:?}): {message}")]
252pub struct ActivityOptionsError {
253 pub kind: ActivityOptionsErrorKind,
254 pub field: Option<&'static str>,
255 pub message: String,
256}
257
258impl ActivityOptionsError {
259 fn new(
260 kind: ActivityOptionsErrorKind,
261 field: Option<&'static str>,
262 message: impl Into<String>,
263 ) -> Self {
264 Self {
265 kind,
266 field,
267 message: message.into(),
268 }
269 }
270}
271
272#[derive(Clone, Copy, Debug, PartialEq, Eq)]
274pub enum ActivityFailureKind {
275 Failed,
276 Cancelled,
277 TimedOut,
278}
279
280#[derive(Clone, Debug, Error)]
285#[error("activity failed ({reason}): {message}")]
286pub struct ActivityFailure {
287 pub kind: ActivityFailureKind,
288 pub reason: String,
289 pub message: String,
290 pub activity_execution_id: Option<String>,
291 pub activity_attempt_id: Option<String>,
292 pub activity_type: Option<String>,
293 pub activity_class: Option<String>,
294 pub attempt_number: Option<u64>,
295 pub failure_id: Option<String>,
296 pub failure_category: Option<String>,
297 pub timeout_kind: Option<String>,
298 pub non_retryable: bool,
299 pub exception_type: Option<String>,
300 pub exception_class: Option<String>,
301 pub code: Option<Value>,
302 pub exception: Option<Value>,
303}
304
305#[derive(Clone, Copy, Debug, PartialEq, Eq)]
307pub enum ChildWorkflowFailureKind {
308 Failed,
309 Cancelled,
310 Terminated,
311}
312
313#[derive(Clone, Debug, Error)]
319#[error("child workflow failed ({reason}): {message}")]
320pub struct ChildWorkflowFailure {
321 pub kind: ChildWorkflowFailureKind,
322 pub reason: String,
323 pub message: String,
324 pub parent_workflow_id: Option<String>,
325 pub parent_workflow_run_id: Option<String>,
326 pub child_workflow_id: Option<String>,
327 pub child_workflow_run_id: Option<String>,
328 pub child_workflow_type: Option<String>,
329 pub failure_id: Option<String>,
330 pub failure_category: Option<String>,
331 pub exception_type: Option<String>,
332 pub exception_class: Option<String>,
333 pub non_retryable: bool,
334 pub code: Option<Value>,
335 pub exception: Option<Value>,
336}
337
338#[derive(Clone, Debug, PartialEq, Eq)]
340pub struct WorkflowIdentity {
341 pub workflow_id: Option<String>,
342 pub run_id: Option<String>,
343}
344
345#[derive(Clone, Debug, PartialEq)]
347pub struct ChildWorkflowResult {
348 pub parent: WorkflowIdentity,
349 pub child: WorkflowIdentity,
350 pub child_workflow_type: Option<String>,
351 pub result: Value,
352}
353
354#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
356pub enum ParentClosePolicy {
357 #[default]
358 Abandon,
359 RequestCancel,
360 Terminate,
361}
362
363impl ParentClosePolicy {
364 fn as_str(self) -> &'static str {
365 match self {
366 Self::Abandon => "abandon",
367 Self::RequestCancel => "request_cancel",
368 Self::Terminate => "terminate",
369 }
370 }
371}
372
373#[derive(Clone, Debug, Default, PartialEq, Eq)]
375pub struct ChildWorkflowRetryPolicy {
376 pub max_attempts: Option<u32>,
377 pub backoff_seconds: Vec<u64>,
378 pub non_retryable_error_types: Vec<String>,
379}
380
381#[derive(Clone, Debug, PartialEq, Eq)]
385pub struct ChildWorkflowOptions {
386 pub task_queue: String,
387 pub parent_close_policy: ParentClosePolicy,
388 pub retry_policy: Option<ChildWorkflowRetryPolicy>,
389 pub execution_timeout_seconds: Option<u64>,
390 pub run_timeout_seconds: Option<u64>,
391}
392
393impl ChildWorkflowOptions {
394 pub fn new(task_queue: impl Into<String>) -> Self {
395 Self {
396 task_queue: task_queue.into(),
397 parent_close_policy: ParentClosePolicy::Abandon,
398 retry_policy: None,
399 execution_timeout_seconds: None,
400 run_timeout_seconds: None,
401 }
402 }
403
404 pub fn parent_close_policy(mut self, policy: ParentClosePolicy) -> Self {
405 self.parent_close_policy = policy;
406 self
407 }
408
409 pub fn retry_policy(mut self, policy: ChildWorkflowRetryPolicy) -> Self {
410 self.retry_policy = Some(policy);
411 self
412 }
413
414 pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
415 self.execution_timeout_seconds = Some(seconds);
416 self
417 }
418
419 pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
420 self.run_timeout_seconds = Some(seconds);
421 self
422 }
423}
424
425#[derive(Clone, Debug, PartialEq, Eq)]
427pub enum ActivityBackoff {
428 Explicit(Vec<Duration>),
431 Exponential {
433 initial_interval: Duration,
434 coefficient: u32,
435 maximum_interval: Option<Duration>,
436 },
437}
438
439#[derive(Clone, Debug, Default, PartialEq, Eq)]
441pub struct ActivityRetryPolicy {
442 pub max_attempts: Option<u32>,
443 pub backoff: Option<ActivityBackoff>,
444 pub non_retryable_error_types: Vec<String>,
445}
446
447impl ActivityRetryPolicy {
448 pub fn new(max_attempts: u32) -> Self {
450 Self {
451 max_attempts: Some(max_attempts),
452 ..Self::default()
453 }
454 }
455
456 pub fn backoff_intervals(mut self, intervals: impl IntoIterator<Item = Duration>) -> Self {
457 self.backoff = Some(ActivityBackoff::Explicit(intervals.into_iter().collect()));
458 self
459 }
460
461 pub fn exponential_backoff(
462 mut self,
463 initial_interval: Duration,
464 coefficient: u32,
465 maximum_interval: Option<Duration>,
466 ) -> Self {
467 self.backoff = Some(ActivityBackoff::Exponential {
468 initial_interval,
469 coefficient,
470 maximum_interval,
471 });
472 self
473 }
474
475 pub fn non_retryable_error_type(mut self, error_type: impl Into<String>) -> Self {
476 self.non_retryable_error_types.push(error_type.into());
477 self
478 }
479
480 pub fn non_retryable_error_types(
481 mut self,
482 error_types: impl IntoIterator<Item = impl Into<String>>,
483 ) -> Self {
484 self.non_retryable_error_types
485 .extend(error_types.into_iter().map(Into::into));
486 self
487 }
488}
489
490#[derive(Clone, Debug, Default, PartialEq, Eq)]
495pub struct ActivityOptions {
496 pub task_queue: Option<String>,
497 pub retry_policy: Option<ActivityRetryPolicy>,
498 pub start_to_close_timeout: Option<Duration>,
499 pub schedule_to_start_timeout: Option<Duration>,
500 pub schedule_to_close_timeout: Option<Duration>,
501 pub heartbeat_timeout: Option<Duration>,
502}
503
504impl ActivityOptions {
505 pub fn new() -> Self {
506 Self::default()
507 }
508
509 pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
510 self.task_queue = Some(task_queue.into());
511 self
512 }
513
514 pub fn retry_policy(mut self, policy: ActivityRetryPolicy) -> Self {
515 self.retry_policy = Some(policy);
516 self
517 }
518
519 pub fn start_to_close_timeout(mut self, timeout: Duration) -> Self {
520 self.start_to_close_timeout = Some(timeout);
521 self
522 }
523
524 pub fn schedule_to_start_timeout(mut self, timeout: Duration) -> Self {
525 self.schedule_to_start_timeout = Some(timeout);
526 self
527 }
528
529 pub fn schedule_to_close_timeout(mut self, timeout: Duration) -> Self {
530 self.schedule_to_close_timeout = Some(timeout);
531 self
532 }
533
534 pub fn heartbeat_timeout(mut self, timeout: Duration) -> Self {
535 self.heartbeat_timeout = Some(timeout);
536 self
537 }
538
539 fn validate(&self) -> std::result::Result<ValidatedActivityOptions, ActivityOptionsError> {
540 if self
541 .task_queue
542 .as_deref()
543 .is_some_and(|queue| queue.trim().is_empty())
544 {
545 return Err(ActivityOptionsError::new(
546 ActivityOptionsErrorKind::EmptyTaskQueue,
547 Some("task_queue"),
548 "task_queue must not be empty",
549 ));
550 }
551
552 for (field, value) in [
553 ("start_to_close_timeout", self.start_to_close_timeout),
554 ("schedule_to_start_timeout", self.schedule_to_start_timeout),
555 ("schedule_to_close_timeout", self.schedule_to_close_timeout),
556 ("heartbeat_timeout", self.heartbeat_timeout),
557 ] {
558 if value.is_some_and(|value| value.is_zero()) {
559 return Err(ActivityOptionsError::new(
560 ActivityOptionsErrorKind::TimeoutNotPositive,
561 Some(field),
562 format!("{field} must be positive"),
563 ));
564 }
565 }
566
567 validate_timeout_order(
568 "heartbeat_timeout",
569 self.heartbeat_timeout,
570 "start_to_close_timeout",
571 self.start_to_close_timeout,
572 )?;
573 validate_timeout_order(
574 "start_to_close_timeout",
575 self.start_to_close_timeout,
576 "schedule_to_close_timeout",
577 self.schedule_to_close_timeout,
578 )?;
579 validate_timeout_order(
580 "schedule_to_start_timeout",
581 self.schedule_to_start_timeout,
582 "schedule_to_close_timeout",
583 self.schedule_to_close_timeout,
584 )?;
585
586 Ok(ValidatedActivityOptions {
587 task_queue: self.task_queue.clone(),
588 retry_policy: self
589 .retry_policy
590 .as_ref()
591 .map(validate_activity_retry_policy)
592 .transpose()?,
593 start_to_close_timeout: timeout_seconds(
594 "start_to_close_timeout",
595 self.start_to_close_timeout,
596 )?,
597 schedule_to_start_timeout: timeout_seconds(
598 "schedule_to_start_timeout",
599 self.schedule_to_start_timeout,
600 )?,
601 schedule_to_close_timeout: timeout_seconds(
602 "schedule_to_close_timeout",
603 self.schedule_to_close_timeout,
604 )?,
605 heartbeat_timeout: timeout_seconds("heartbeat_timeout", self.heartbeat_timeout)?,
606 })
607 }
608}
609
610#[derive(Clone, Debug)]
611struct ValidatedActivityOptions {
612 task_queue: Option<String>,
613 retry_policy: Option<Value>,
614 start_to_close_timeout: Option<u64>,
615 schedule_to_start_timeout: Option<u64>,
616 schedule_to_close_timeout: Option<u64>,
617 heartbeat_timeout: Option<u64>,
618}
619
620fn validate_timeout_order(
621 smaller_name: &'static str,
622 smaller: Option<Duration>,
623 larger_name: &'static str,
624 larger: Option<Duration>,
625) -> std::result::Result<(), ActivityOptionsError> {
626 if matches!((smaller, larger), (Some(smaller), Some(larger)) if smaller > larger) {
627 return Err(ActivityOptionsError::new(
628 ActivityOptionsErrorKind::TimeoutOrder,
629 Some(smaller_name),
630 format!("{smaller_name} must be <= {larger_name}"),
631 ));
632 }
633 Ok(())
634}
635
636fn timeout_seconds(
637 field: &'static str,
638 value: Option<Duration>,
639) -> std::result::Result<Option<u64>, ActivityOptionsError> {
640 value
641 .map(|value| {
642 activity_protocol_seconds(value).ok_or_else(|| {
643 ActivityOptionsError::new(
644 ActivityOptionsErrorKind::TimeoutOverflow,
645 Some(field),
646 format!("{field} is too large for the worker protocol"),
647 )
648 })
649 })
650 .transpose()
651}
652
653fn duration_seconds_ceil(value: Duration) -> Option<u64> {
654 value
655 .as_secs()
656 .checked_add(u64::from(value.subsec_nanos() > 0))
657}
658
659fn activity_protocol_seconds(value: Duration) -> Option<u64> {
660 duration_seconds_ceil(value).filter(|seconds| *seconds <= i64::MAX as u64)
661}
662
663fn validate_activity_retry_policy(
664 policy: &ActivityRetryPolicy,
665) -> std::result::Result<Value, ActivityOptionsError> {
666 if policy.max_attempts.is_none()
667 && policy.backoff.is_none()
668 && policy.non_retryable_error_types.is_empty()
669 {
670 return Err(ActivityOptionsError::new(
671 ActivityOptionsErrorKind::EmptyRetryPolicy,
672 Some("retry_policy"),
673 "retry_policy must configure at least one field",
674 ));
675 }
676 if policy.max_attempts == Some(0) {
677 return Err(ActivityOptionsError::new(
678 ActivityOptionsErrorKind::InvalidMaxAttempts,
679 Some("retry_policy.max_attempts"),
680 "max_attempts must be >= 1",
681 ));
682 }
683 if policy
684 .non_retryable_error_types
685 .iter()
686 .any(|error_type| error_type.trim().is_empty())
687 {
688 return Err(ActivityOptionsError::new(
689 ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
690 Some("retry_policy.non_retryable_error_types"),
691 "non_retryable_error_types must not contain empty values",
692 ));
693 }
694
695 let backoff_seconds = match &policy.backoff {
696 None => None,
697 Some(backoff) => {
698 let max_attempts = policy.max_attempts.ok_or_else(|| {
699 ActivityOptionsError::new(
700 ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
701 Some("retry_policy.backoff"),
702 "backoff requires max_attempts",
703 )
704 })?;
705 let retry_count = max_attempts.saturating_sub(1) as usize;
706 let intervals = match backoff {
707 ActivityBackoff::Explicit(intervals) => {
708 if intervals.len() > retry_count {
709 return Err(ActivityOptionsError::new(
710 ActivityOptionsErrorKind::TooManyBackoffIntervals,
711 Some("retry_policy.backoff"),
712 "backoff interval count must not exceed max_attempts - 1",
713 ));
714 }
715 intervals.clone()
716 }
717 ActivityBackoff::Exponential {
718 initial_interval,
719 coefficient,
720 maximum_interval,
721 } => {
722 if *coefficient < 1 {
723 return Err(ActivityOptionsError::new(
724 ActivityOptionsErrorKind::InvalidBackoffCoefficient,
725 Some("retry_policy.backoff.coefficient"),
726 "backoff coefficient must be >= 1",
727 ));
728 }
729 if retry_count > 10_000 {
730 return Err(ActivityOptionsError::new(
731 ActivityOptionsErrorKind::BackoffGenerationTooLarge,
732 Some("retry_policy.max_attempts"),
733 "generated backoff supports at most 10000 retry intervals",
734 ));
735 }
736 let mut current = *initial_interval;
737 let mut intervals = Vec::with_capacity(retry_count);
738 for _ in 0..retry_count {
739 let interval = maximum_interval
740 .map(|maximum| current.min(maximum))
741 .unwrap_or(current);
742 intervals.push(interval);
743 if maximum_interval.is_some_and(|maximum| interval == maximum) {
744 break;
745 }
746 current = current.checked_mul(*coefficient).ok_or_else(|| {
747 ActivityOptionsError::new(
748 ActivityOptionsErrorKind::BackoffOverflow,
749 Some("retry_policy.backoff"),
750 "generated backoff interval overflowed",
751 )
752 })?;
753 }
754 intervals
755 }
756 };
757 Some(
758 intervals
759 .into_iter()
760 .map(|interval| {
761 activity_protocol_seconds(interval).ok_or_else(|| {
762 ActivityOptionsError::new(
763 ActivityOptionsErrorKind::BackoffOverflow,
764 Some("retry_policy.backoff"),
765 "backoff interval is too large for the worker protocol",
766 )
767 })
768 })
769 .collect::<std::result::Result<Vec<_>, _>>()?,
770 )
771 }
772 };
773
774 let mut encoded = serde_json::Map::new();
775 if let Some(max_attempts) = policy.max_attempts {
776 encoded.insert("max_attempts".to_string(), json!(max_attempts));
777 }
778 if let Some(backoff_seconds) = backoff_seconds {
779 encoded.insert("backoff_seconds".to_string(), json!(backoff_seconds));
780 }
781 if !policy.non_retryable_error_types.is_empty() {
782 let mut canonical_error_types = Vec::new();
783 for error_type in policy
784 .non_retryable_error_types
785 .iter()
786 .map(|error_type| error_type.trim())
787 {
788 if !canonical_error_types.contains(&error_type) {
789 canonical_error_types.push(error_type);
790 }
791 }
792 encoded.insert(
793 "non_retryable_error_types".to_string(),
794 json!(canonical_error_types),
795 );
796 }
797 Ok(Value::Object(encoded))
798}
799
800#[derive(Clone, Debug, Error)]
803#[error("non-deterministic workflow replay ({reason}) at sequence {sequence:?}: {message}")]
804pub struct ReplayFailure {
805 pub reason: String,
806 pub sequence: Option<u64>,
807 pub expected: Option<String>,
808 pub actual: Option<String>,
809 pub message: String,
810}
811
812impl ReplayFailure {
813 fn new(
814 reason: impl Into<String>,
815 sequence: Option<u64>,
816 expected: Option<String>,
817 actual: Option<String>,
818 message: impl Into<String>,
819 ) -> Self {
820 Self {
821 reason: reason.into(),
822 sequence,
823 expected,
824 actual,
825 message: message.into(),
826 }
827 }
828}
829
830#[derive(Clone, Debug, Error)]
832#[error("query failed ({reason}, HTTP {status}): {message}")]
833pub struct QueryFailure {
834 pub status: u16,
835 pub reason: String,
836 pub message: String,
837 pub body: Value,
838}
839
840#[derive(Clone, Debug, Error)]
842#[error("protocol rejected ({reason}, HTTP {status}): {message}")]
843pub struct ProtocolFailure {
844 pub status: u16,
845 pub reason: String,
846 pub message: String,
847 pub supported_version: Option<String>,
848 pub requested_version: Option<String>,
849 pub body: Value,
850}
851
852#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
853pub struct PayloadEnvelope {
854 pub codec: String,
855 pub blob: String,
856}
857
858impl PayloadEnvelope {
859 pub fn avro<T: Serialize>(value: &T) -> Result<Self> {
860 encode_payload(value, DEFAULT_CODEC)
861 }
862
863 pub fn json<T: Serialize>(value: &T) -> Result<Self> {
864 encode_payload(value, JSON_CODEC)
865 }
866}
867
868pub fn encode_payload<T: Serialize>(value: &T, codec: &str) -> Result<PayloadEnvelope> {
869 let value = serde_json::to_value(value)?;
870 let blob = encode_value_blob(&value, codec)?;
871
872 Ok(PayloadEnvelope {
873 codec: codec.to_string(),
874 blob,
875 })
876}
877
878pub fn decode_payload<T: DeserializeOwned>(envelope: &PayloadEnvelope) -> Result<T> {
879 let value = decode_blob(&envelope.blob, &envelope.codec)?;
880 Ok(serde_json::from_value(value)?)
881}
882
883fn encode_value_envelope(value: &Value, codec: &str) -> Result<Value> {
884 Ok(serde_json::to_value(encode_payload(value, codec)?)?)
885}
886
887fn encode_value_blob(value: &Value, codec: &str) -> Result<String> {
888 match codec {
889 JSON_CODEC => Ok(serde_json::to_string(value)?),
890 DEFAULT_CODEC => encode_avro_generic(value),
891 other => Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
892 }
893}
894
895fn decode_wire_value(value: &Value, fallback_codec: &str) -> Result<Value> {
896 if value.is_null() {
897 return Ok(Value::Null);
898 }
899
900 if let Some(object) = value.as_object() {
901 if let (Some(codec), Some(blob)) = (
902 object.get("codec").and_then(Value::as_str),
903 object.get("blob").and_then(Value::as_str),
904 ) {
905 return decode_blob(blob, codec);
906 }
907 }
908
909 if let Some(blob) = value.as_str() {
910 return decode_blob(blob, fallback_codec);
911 }
912
913 Ok(value.clone())
914}
915
916fn decode_blob(blob: &str, codec: &str) -> Result<Value> {
917 match codec {
918 JSON_CODEC => Ok(serde_json::from_str(blob)?),
919 DEFAULT_CODEC => decode_avro_generic(blob),
920 other => Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
921 }
922}
923
924fn encode_avro_generic(value: &Value) -> Result<String> {
925 let json = serde_json::to_string(value)?;
926 let datum = to_value(AvroPayload {
927 json,
928 version: AVRO_PAYLOAD_VERSION,
929 })
930 .map_err(|err| Error::Codec(format!("could not convert avro generic wrapper: {err}")))?;
931 let datum = to_avro_datum(avro_payload_schema()?, datum)
932 .map_err(|err| Error::Codec(format!("could not encode avro generic wrapper: {err}")))?;
933
934 let mut bytes = Vec::with_capacity(datum.len() + 1);
935 bytes.push(0x00);
936 bytes.extend_from_slice(&datum);
937 Ok(BASE64.encode(bytes))
938}
939
940fn decode_avro_generic(blob: &str) -> Result<Value> {
941 let bytes = BASE64
942 .decode(blob)
943 .map_err(|err| Error::Codec(format!("invalid avro base64 payload: {err}")))?;
944
945 if bytes.is_empty() {
946 return Err(Error::Codec("avro payload is empty".to_string()));
947 }
948
949 match bytes[0] {
950 0x00 => {}
951 0x01 => {
952 return Err(Error::Codec(
953 "typed avro payloads require a schema context; v1 supports the generic wrapper"
954 .to_string(),
955 ));
956 }
957 other => {
958 return Err(Error::Codec(format!(
959 "unknown avro payload prefix 0x{other:02x}"
960 )));
961 }
962 }
963
964 let mut datum = &bytes[1..];
965 let datum = from_avro_datum(avro_payload_schema()?, &mut datum, None)
966 .map_err(|err| Error::Codec(format!("could not decode avro generic wrapper: {err}")))?;
967 let payload: AvroPayload = from_value(&datum)
968 .map_err(|err| Error::Codec(format!("invalid avro generic wrapper record: {err}")))?;
969
970 if payload.version != AVRO_PAYLOAD_VERSION {
971 return Err(Error::Codec(format!(
972 "unsupported avro generic wrapper version {}",
973 payload.version
974 )));
975 }
976
977 Ok(serde_json::from_str(&payload.json)?)
978}
979
980#[derive(Debug, Serialize, Deserialize)]
981struct AvroPayload {
982 json: String,
983 version: i32,
984}
985
986fn avro_payload_schema() -> Result<&'static Schema> {
987 match AVRO_PAYLOAD_SCHEMA.get_or_init(|| {
988 Schema::parse_str(AVRO_PAYLOAD_SCHEMA_JSON)
989 .map_err(|err| format!("could not parse avro payload schema: {err}"))
990 }) {
991 Ok(schema) => Ok(schema),
992 Err(message) => Err(Error::Codec(message.clone())),
993 }
994}
995
996#[derive(Clone, Debug)]
997pub struct Client {
998 http: reqwest::Client,
999 base_url: String,
1000 token: Option<String>,
1001 control_token: Option<String>,
1002 worker_token: Option<String>,
1003 namespace: String,
1004}
1005
1006impl Client {
1007 pub fn new(base_url: impl Into<String>) -> Result<Self> {
1008 Self::builder(base_url).build()
1009 }
1010
1011 pub fn builder(base_url: impl Into<String>) -> ClientBuilder {
1012 ClientBuilder {
1013 base_url: base_url.into(),
1014 token: None,
1015 control_token: None,
1016 worker_token: None,
1017 namespace: "default".to_string(),
1018 timeout: Duration::from_secs(60),
1019 }
1020 }
1021
1022 pub async fn health(&self) -> Result<Value> {
1023 self.request_json(
1024 reqwest::Method::GET,
1025 "/health",
1026 RequestProtocol::ControlPlane,
1027 Option::<&Value>::None,
1028 )
1029 .await
1030 }
1031
1032 pub async fn cluster_info(&self) -> Result<Value> {
1033 self.request_json(
1034 reqwest::Method::GET,
1035 "/cluster/info",
1036 RequestProtocol::ControlPlane,
1037 Option::<&Value>::None,
1038 )
1039 .await
1040 }
1041
1042 pub async fn start_workflow<T: Serialize>(
1043 &self,
1044 workflow_type: &str,
1045 task_queue: &str,
1046 workflow_id: &str,
1047 input: T,
1048 ) -> Result<WorkflowHandle> {
1049 let input = serde_json::to_value(input)?;
1050 let input_envelope = encode_value_envelope(&normalize_arguments(input), DEFAULT_CODEC)?;
1051 let body = json!({
1052 "workflow_id": workflow_id,
1053 "workflow_type": workflow_type,
1054 "task_queue": task_queue,
1055 "input": input_envelope,
1056 "execution_timeout_seconds": 3600,
1057 "run_timeout_seconds": 600
1058 });
1059
1060 let data: Value = self
1061 .request_json(
1062 reqwest::Method::POST,
1063 "/workflows",
1064 RequestProtocol::ControlPlane,
1065 Some(&body),
1066 )
1067 .await?;
1068
1069 Ok(WorkflowHandle {
1070 client: self.clone(),
1071 workflow_id: data
1072 .get("workflow_id")
1073 .and_then(Value::as_str)
1074 .unwrap_or(workflow_id)
1075 .to_string(),
1076 run_id: data
1077 .get("run_id")
1078 .and_then(Value::as_str)
1079 .map(str::to_string),
1080 workflow_type: data
1081 .get("workflow_type")
1082 .and_then(Value::as_str)
1083 .unwrap_or(workflow_type)
1084 .to_string(),
1085 })
1086 }
1087
1088 pub async fn signal_workflow<T: Serialize>(
1089 &self,
1090 workflow_id: &str,
1091 signal_name: &str,
1092 input: T,
1093 ) -> Result<Value> {
1094 let input = serde_json::to_value(input)?;
1095 let input_envelope = encode_value_envelope(&normalize_arguments(input), DEFAULT_CODEC)?;
1096 let body = json!({
1097 "input": input_envelope
1098 });
1099 let path = format!("/workflows/{workflow_id}/signal/{signal_name}");
1100 self.request_json(
1101 reqwest::Method::POST,
1102 &path,
1103 RequestProtocol::ControlPlane,
1104 Some(&body),
1105 )
1106 .await
1107 }
1108
1109 pub async fn cancel_workflow(
1111 &self,
1112 workflow_id: &str,
1113 options: WorkflowCommandOptions,
1114 ) -> Result<WorkflowCommandResult> {
1115 self.workflow_command(workflow_id, None, WorkflowCommandKind::Cancel, options)
1116 .await
1117 }
1118
1119 pub async fn cancel_workflow_run(
1121 &self,
1122 workflow_id: &str,
1123 run_id: &str,
1124 options: WorkflowCommandOptions,
1125 ) -> Result<WorkflowCommandResult> {
1126 self.workflow_command(
1127 workflow_id,
1128 Some(run_id),
1129 WorkflowCommandKind::Cancel,
1130 options,
1131 )
1132 .await
1133 }
1134
1135 pub async fn terminate_workflow(
1137 &self,
1138 workflow_id: &str,
1139 options: WorkflowCommandOptions,
1140 ) -> Result<WorkflowCommandResult> {
1141 self.workflow_command(workflow_id, None, WorkflowCommandKind::Terminate, options)
1142 .await
1143 }
1144
1145 pub async fn terminate_workflow_run(
1147 &self,
1148 workflow_id: &str,
1149 run_id: &str,
1150 options: WorkflowCommandOptions,
1151 ) -> Result<WorkflowCommandResult> {
1152 self.workflow_command(
1153 workflow_id,
1154 Some(run_id),
1155 WorkflowCommandKind::Terminate,
1156 options,
1157 )
1158 .await
1159 }
1160
1161 async fn workflow_command(
1162 &self,
1163 workflow_id: &str,
1164 run_id: Option<&str>,
1165 command: WorkflowCommandKind,
1166 options: WorkflowCommandOptions,
1167 ) -> Result<WorkflowCommandResult> {
1168 let path = match run_id {
1169 Some(run_id) => format!(
1170 "/workflows/{workflow_id}/runs/{run_id}/{}",
1171 command.as_str()
1172 ),
1173 None => format!("/workflows/{workflow_id}/{}", command.as_str()),
1174 };
1175 let data = match self
1176 .request_json(
1177 reqwest::Method::POST,
1178 &path,
1179 RequestProtocol::ControlPlane,
1180 Some(&options),
1181 )
1182 .await
1183 {
1184 Ok(data) => data,
1185 Err(Error::Http { status, body }) => {
1186 return Err(Error::WorkflowCommandRejected(workflow_command_rejection(
1187 command,
1188 status,
1189 body,
1190 workflow_id,
1191 run_id,
1192 )));
1193 }
1194 Err(error) => return Err(error),
1195 };
1196
1197 Ok(workflow_command_result(command, data, workflow_id, run_id))
1198 }
1199
1200 pub async fn query_workflow<T: Serialize>(
1206 &self,
1207 workflow_id: &str,
1208 query_name: &str,
1209 input: T,
1210 ) -> Result<Value> {
1211 let input = serde_json::to_value(input)?;
1212 let input_envelope = encode_value_envelope(&normalize_arguments(input), DEFAULT_CODEC)?;
1213 let body = json!({
1214 "input": input_envelope
1215 });
1216 let path = format!("/workflows/{workflow_id}/query/{query_name}");
1217 let response: Value = match self
1218 .request_json(
1219 reqwest::Method::POST,
1220 &path,
1221 RequestProtocol::ControlPlane,
1222 Some(&body),
1223 )
1224 .await
1225 {
1226 Ok(response) => response,
1227 Err(Error::Http { status, body }) => {
1228 return Err(Error::QueryFailed(query_failure(status, body)));
1229 }
1230 Err(error) => return Err(error),
1231 };
1232
1233 if let Some(envelope) = response
1234 .get("result_envelope")
1235 .filter(|envelope| !envelope.is_null())
1236 {
1237 return decode_wire_value(envelope, DEFAULT_CODEC);
1238 }
1239
1240 Ok(response.get("result").cloned().unwrap_or(Value::Null))
1241 }
1242
1243 pub async fn describe_workflow(&self, workflow_id: &str) -> Result<WorkflowDescription> {
1244 let path = format!("/workflows/{workflow_id}");
1245 let mut data: WorkflowDescription = self
1246 .request_json(
1247 reqwest::Method::GET,
1248 &path,
1249 RequestProtocol::ControlPlane,
1250 Option::<&Value>::None,
1251 )
1252 .await?;
1253 data.decode_payloads()?;
1254 Ok(data)
1255 }
1256
1257 pub async fn describe_workflow_run(
1259 &self,
1260 workflow_id: &str,
1261 run_id: &str,
1262 ) -> Result<WorkflowDescription> {
1263 let path = format!("/workflows/{workflow_id}/runs/{run_id}");
1264 let mut data: WorkflowDescription = self
1265 .request_json(
1266 reqwest::Method::GET,
1267 &path,
1268 RequestProtocol::ControlPlane,
1269 Option::<&Value>::None,
1270 )
1271 .await?;
1272 data.decode_payloads()?;
1273 Ok(data)
1274 }
1275
1276 pub async fn register_worker(
1277 &self,
1278 worker_id: &str,
1279 task_queue: &str,
1280 supported_workflow_types: Vec<String>,
1281 supported_activity_types: Vec<String>,
1282 max_concurrent_workflow_tasks: usize,
1283 max_concurrent_activity_tasks: usize,
1284 ) -> Result<RegisterWorkerResponse> {
1285 self.register_worker_with_capabilities(
1286 worker_id,
1287 task_queue,
1288 supported_workflow_types,
1289 supported_activity_types,
1290 max_concurrent_workflow_tasks,
1291 max_concurrent_activity_tasks,
1292 Vec::new(),
1293 )
1294 .await
1295 }
1296
1297 pub async fn register_worker_with_capabilities(
1299 &self,
1300 worker_id: &str,
1301 task_queue: &str,
1302 supported_workflow_types: Vec<String>,
1303 supported_activity_types: Vec<String>,
1304 max_concurrent_workflow_tasks: usize,
1305 max_concurrent_activity_tasks: usize,
1306 capabilities: Vec<String>,
1307 ) -> Result<RegisterWorkerResponse> {
1308 let body = json!({
1309 "worker_id": worker_id,
1310 "task_queue": task_queue,
1311 "runtime": "rust",
1312 "sdk_version": SDK_VERSION,
1313 "supported_workflow_types": supported_workflow_types,
1314 "supported_activity_types": supported_activity_types,
1315 "capabilities": capabilities,
1316 "max_concurrent_workflow_tasks": max_concurrent_workflow_tasks,
1317 "max_concurrent_activity_tasks": max_concurrent_activity_tasks
1318 });
1319
1320 self.request_json(
1321 reqwest::Method::POST,
1322 "/worker/register",
1323 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
1324 Some(&body),
1325 )
1326 .await
1327 }
1328
1329 pub async fn poll_query_task(
1331 &self,
1332 worker_id: &str,
1333 task_queue: &str,
1334 timeout: Duration,
1335 ) -> Result<Option<QueryTask>> {
1336 Ok(self
1337 .poll_query_task_response(worker_id, task_queue, timeout)
1338 .await?
1339 .task)
1340 }
1341
1342 pub async fn poll_query_task_response(
1344 &self,
1345 worker_id: &str,
1346 task_queue: &str,
1347 timeout: Duration,
1348 ) -> Result<PollQueryTaskResponse> {
1349 let timeout_seconds = long_poll_timeout_seconds(timeout);
1350 let body = json!({
1351 "worker_id": worker_id,
1352 "task_queue": task_queue,
1353 "poll_request_id": unique_request_id("rust-query-poll"),
1354 "timeout_seconds": timeout_seconds,
1355 });
1356 worker_poll_response(
1357 self.request_json_with_timeout(
1358 reqwest::Method::POST,
1359 "/worker/query-tasks/poll",
1360 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
1361 Some(&body),
1362 timeout + Duration::from_secs(5),
1363 )
1364 .await,
1365 )
1366 }
1367
1368 pub async fn complete_query_task(
1370 &self,
1371 query_task_id: &str,
1372 lease_owner: &str,
1373 query_task_attempt: u64,
1374 result: Value,
1375 codec: &str,
1376 ) -> Result<Value> {
1377 let result_envelope = encode_value_envelope(&result, codec)?;
1378 self.complete_query_task_with_envelope(
1379 query_task_id,
1380 lease_owner,
1381 query_task_attempt,
1382 result,
1383 result_envelope,
1384 )
1385 .await
1386 }
1387
1388 async fn complete_query_task_with_envelope(
1389 &self,
1390 query_task_id: &str,
1391 lease_owner: &str,
1392 query_task_attempt: u64,
1393 result: Value,
1394 result_envelope: Value,
1395 ) -> Result<Value> {
1396 let body = json!({
1397 "lease_owner": lease_owner,
1398 "query_task_attempt": query_task_attempt,
1399 "result": result,
1400 "result_envelope": result_envelope,
1401 });
1402 let path = format!("/worker/query-tasks/{query_task_id}/complete");
1403 let response = self
1404 .request_json(
1405 reqwest::Method::POST,
1406 &path,
1407 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
1408 Some(&body),
1409 )
1410 .await;
1411 query_task_response(response)
1412 }
1413
1414 pub async fn fail_query_task(
1416 &self,
1417 query_task_id: &str,
1418 lease_owner: &str,
1419 query_task_attempt: u64,
1420 message: impl Into<String>,
1421 reason: impl Into<String>,
1422 failure_type: impl Into<String>,
1423 ) -> Result<Value> {
1424 let body = json!({
1425 "lease_owner": lease_owner,
1426 "query_task_attempt": query_task_attempt,
1427 "failure": {
1428 "message": message.into(),
1429 "reason": reason.into(),
1430 "type": failure_type.into(),
1431 }
1432 });
1433 let path = format!("/worker/query-tasks/{query_task_id}/fail");
1434 let response = self
1435 .request_json(
1436 reqwest::Method::POST,
1437 &path,
1438 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
1439 Some(&body),
1440 )
1441 .await;
1442 query_task_response(response)
1443 }
1444
1445 pub async fn heartbeat_worker(
1446 &self,
1447 worker_id: &str,
1448 workflow_available: usize,
1449 activity_available: usize,
1450 ) -> Result<Value> {
1451 let body = json!({
1452 "worker_id": worker_id,
1453 "task_slots": {
1454 "workflow_available": workflow_available,
1455 "activity_available": activity_available
1456 },
1457 "process_metrics": {
1458 "process_id": std::process::id(),
1459 "process_uptime_seconds": 0
1460 }
1461 });
1462
1463 self.request_json(
1464 reqwest::Method::POST,
1465 "/worker/heartbeat",
1466 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
1467 Some(&body),
1468 )
1469 .await
1470 }
1471
1472 pub async fn poll_workflow_task(
1473 &self,
1474 worker_id: &str,
1475 task_queue: &str,
1476 timeout: Duration,
1477 ) -> Result<Option<WorkflowTask>> {
1478 Ok(self
1479 .poll_workflow_task_response(worker_id, task_queue, timeout)
1480 .await?
1481 .task)
1482 }
1483
1484 pub async fn poll_workflow_task_response(
1485 &self,
1486 worker_id: &str,
1487 task_queue: &str,
1488 timeout: Duration,
1489 ) -> Result<PollWorkflowTaskResponse> {
1490 let body = json!({
1491 "worker_id": worker_id,
1492 "task_queue": task_queue,
1493 "timeout_seconds": long_poll_timeout_seconds(timeout),
1494 });
1495 let mut data: PollWorkflowTaskResponse = worker_poll_response(
1496 self.request_json_with_timeout(
1497 reqwest::Method::POST,
1498 "/worker/workflow-tasks/poll",
1499 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
1500 Some(&body),
1501 timeout + Duration::from_secs(5),
1502 )
1503 .await,
1504 )?;
1505
1506 if let Some(task) = data.task.as_mut() {
1507 self.fetch_remaining_workflow_history(worker_id, task)
1508 .await?;
1509 }
1510
1511 Ok(data)
1512 }
1513
1514 async fn fetch_remaining_workflow_history(
1515 &self,
1516 worker_id: &str,
1517 task: &mut WorkflowTask,
1518 ) -> Result<()> {
1519 let mut next_token = task.next_history_page_token.clone();
1520
1521 while let Some(token) = next_token.take().filter(|token| !token.is_empty()) {
1522 let lease_owner = task
1523 .lease_owner
1524 .clone()
1525 .unwrap_or_else(|| worker_id.to_string());
1526 let page = self
1527 .workflow_task_history_page(
1528 &task.task_id,
1529 &lease_owner,
1530 task.workflow_task_attempt,
1531 &token,
1532 )
1533 .await?;
1534
1535 task.append_history_page(page);
1536
1537 if task.next_history_page_token.as_deref() == Some(token.as_str()) {
1538 return Err(Error::Codec(
1539 "workflow history pagination returned the same page token".to_string(),
1540 ));
1541 }
1542
1543 next_token = task.next_history_page_token.clone();
1544 }
1545
1546 Ok(())
1547 }
1548
1549 async fn workflow_task_history_page(
1550 &self,
1551 task_id: &str,
1552 lease_owner: &str,
1553 workflow_task_attempt: u64,
1554 next_history_page_token: &str,
1555 ) -> Result<WorkflowTaskHistoryPage> {
1556 let body = json!({
1557 "lease_owner": lease_owner,
1558 "workflow_task_attempt": workflow_task_attempt,
1559 "next_history_page_token": next_history_page_token
1560 });
1561 let path = format!("/worker/workflow-tasks/{task_id}/history");
1562
1563 self.request_json(
1564 reqwest::Method::POST,
1565 &path,
1566 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
1567 Some(&body),
1568 )
1569 .await
1570 }
1571
1572 pub async fn complete_workflow_task(
1573 &self,
1574 task_id: &str,
1575 lease_owner: &str,
1576 workflow_task_attempt: u64,
1577 commands: Vec<Value>,
1578 ) -> Result<Value> {
1579 let body = json!({
1580 "lease_owner": lease_owner,
1581 "workflow_task_attempt": workflow_task_attempt,
1582 "commands": commands
1583 });
1584 let path = format!("/worker/workflow-tasks/{task_id}/complete");
1585 self.request_json(
1586 reqwest::Method::POST,
1587 &path,
1588 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
1589 Some(&body),
1590 )
1591 .await
1592 }
1593
1594 pub async fn fail_workflow_task(
1595 &self,
1596 task_id: &str,
1597 lease_owner: &str,
1598 workflow_task_attempt: u64,
1599 message: impl Into<String>,
1600 ) -> Result<Value> {
1601 self.fail_workflow_task_with_type(
1602 task_id,
1603 lease_owner,
1604 workflow_task_attempt,
1605 message,
1606 "RustWorkflowTaskFailure",
1607 )
1608 .await
1609 }
1610
1611 async fn fail_workflow_task_with_type(
1612 &self,
1613 task_id: &str,
1614 lease_owner: &str,
1615 workflow_task_attempt: u64,
1616 message: impl Into<String>,
1617 failure_type: &str,
1618 ) -> Result<Value> {
1619 let body = json!({
1620 "lease_owner": lease_owner,
1621 "workflow_task_attempt": workflow_task_attempt,
1622 "failure": {
1623 "message": message.into(),
1624 "type": failure_type
1625 }
1626 });
1627 let path = format!("/worker/workflow-tasks/{task_id}/fail");
1628 self.request_json(
1629 reqwest::Method::POST,
1630 &path,
1631 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
1632 Some(&body),
1633 )
1634 .await
1635 }
1636
1637 pub async fn poll_activity_task(
1638 &self,
1639 worker_id: &str,
1640 task_queue: &str,
1641 timeout: Duration,
1642 ) -> Result<Option<ActivityTask>> {
1643 Ok(self
1644 .poll_activity_task_response(worker_id, task_queue, timeout)
1645 .await?
1646 .task)
1647 }
1648
1649 pub async fn poll_activity_task_response(
1651 &self,
1652 worker_id: &str,
1653 task_queue: &str,
1654 timeout: Duration,
1655 ) -> Result<PollActivityTaskResponse> {
1656 let body = json!({
1657 "worker_id": worker_id,
1658 "task_queue": task_queue,
1659 "timeout_seconds": long_poll_timeout_seconds(timeout),
1660 });
1661 let data: PollActivityTaskResponse = worker_poll_response(
1662 self.request_json_with_timeout(
1663 reqwest::Method::POST,
1664 "/worker/activity-tasks/poll",
1665 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
1666 Some(&body),
1667 timeout + Duration::from_secs(5),
1668 )
1669 .await,
1670 )?;
1671 Ok(data)
1672 }
1673
1674 pub async fn complete_activity_task(
1675 &self,
1676 task_id: &str,
1677 activity_attempt_id: &str,
1678 lease_owner: &str,
1679 result: Value,
1680 codec: &str,
1681 ) -> Result<Value> {
1682 let result = encode_value_envelope(&result, codec)?;
1683 let body = json!({
1684 "activity_attempt_id": activity_attempt_id,
1685 "lease_owner": lease_owner,
1686 "result": result
1687 });
1688 let path = format!("/worker/activity-tasks/{task_id}/complete");
1689 activity_task_response(
1690 self.request_json(
1691 reqwest::Method::POST,
1692 &path,
1693 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
1694 Some(&body),
1695 )
1696 .await,
1697 "complete",
1698 task_id,
1699 activity_attempt_id,
1700 )
1701 }
1702
1703 pub async fn fail_activity_task(
1704 &self,
1705 task_id: &str,
1706 activity_attempt_id: &str,
1707 lease_owner: &str,
1708 message: impl Into<String>,
1709 non_retryable: bool,
1710 ) -> Result<Value> {
1711 let body = json!({
1712 "activity_attempt_id": activity_attempt_id,
1713 "lease_owner": lease_owner,
1714 "failure": {
1715 "message": message.into(),
1716 "type": "RustActivityFailure",
1717 "non_retryable": non_retryable
1718 }
1719 });
1720 let path = format!("/worker/activity-tasks/{task_id}/fail");
1721 activity_task_response(
1722 self.request_json(
1723 reqwest::Method::POST,
1724 &path,
1725 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
1726 Some(&body),
1727 )
1728 .await,
1729 "fail",
1730 task_id,
1731 activity_attempt_id,
1732 )
1733 }
1734
1735 pub async fn heartbeat_activity_task(
1736 &self,
1737 task_id: &str,
1738 activity_attempt_id: &str,
1739 lease_owner: &str,
1740 details: Value,
1741 ) -> Result<ActivityHeartbeatResponse> {
1742 let body = json!({
1743 "activity_attempt_id": activity_attempt_id,
1744 "lease_owner": lease_owner,
1745 "details": details
1746 });
1747 let path = format!("/worker/activity-tasks/{task_id}/heartbeat");
1748 activity_task_response(
1749 self.request_json(
1750 reqwest::Method::POST,
1751 &path,
1752 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
1753 Some(&body),
1754 )
1755 .await,
1756 "heartbeat",
1757 task_id,
1758 activity_attempt_id,
1759 )
1760 }
1761
1762 async fn request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
1763 &self,
1764 method: reqwest::Method,
1765 path: &str,
1766 protocol: RequestProtocol,
1767 body: Option<&B>,
1768 ) -> Result<T> {
1769 self.request_json_with_timeout(method, path, protocol, body, Duration::from_secs(60))
1770 .await
1771 }
1772
1773 async fn request_json_with_timeout<T: DeserializeOwned, B: Serialize + ?Sized>(
1774 &self,
1775 method: reqwest::Method,
1776 path: &str,
1777 protocol: RequestProtocol,
1778 body: Option<&B>,
1779 timeout: Duration,
1780 ) -> Result<T> {
1781 let mut request = self
1782 .http
1783 .request(method, format!("{}/api{}", self.base_url, path))
1784 .timeout(timeout)
1785 .header(reqwest::header::ACCEPT, "application/json")
1786 .header(reqwest::header::CONTENT_TYPE, "application/json")
1787 .header("X-Namespace", &self.namespace);
1788
1789 match protocol {
1790 RequestProtocol::Worker(version) => {
1791 request = request.header("X-Durable-Workflow-Protocol-Version", version);
1792 }
1793 RequestProtocol::ControlPlane => {
1794 request = request.header(
1795 "X-Durable-Workflow-Control-Plane-Version",
1796 CONTROL_PLANE_VERSION,
1797 );
1798 }
1799 }
1800
1801 if let Some(token) = self.auth_token(protocol.is_worker()) {
1802 request = request.bearer_auth(token);
1803 }
1804
1805 if let Some(body) = body {
1806 request = request.json(body);
1807 }
1808
1809 let response = request.send().await?;
1810 let status = response.status();
1811 let bytes = response.bytes().await?;
1812
1813 if !status.is_success() {
1814 let body = String::from_utf8_lossy(&bytes).to_string();
1815 if let Some(protocol) = protocol_failure(status, &body) {
1816 return Err(Error::Protocol(protocol));
1817 }
1818 return Err(Error::Http { status, body });
1819 }
1820
1821 if bytes.is_empty() {
1822 return Ok(serde_json::from_value(Value::Null)?);
1823 }
1824
1825 Ok(serde_json::from_slice(&bytes)?)
1826 }
1827
1828 fn auth_token(&self, worker: bool) -> Option<&str> {
1829 if worker {
1830 self.worker_token
1831 .as_deref()
1832 .or(self.token.as_deref())
1833 .or(self.control_token.as_deref())
1834 } else {
1835 self.control_token
1836 .as_deref()
1837 .or(self.token.as_deref())
1838 .or(self.worker_token.as_deref())
1839 }
1840 }
1841}
1842
1843fn query_failure(status: reqwest::StatusCode, raw_body: String) -> QueryFailure {
1844 let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
1845 let reason = body
1846 .get("reason")
1847 .and_then(Value::as_str)
1848 .unwrap_or("query_rejected")
1849 .to_string();
1850 let message = body
1851 .get("message")
1852 .or_else(|| body.get("error"))
1853 .and_then(Value::as_str)
1854 .unwrap_or("workflow query was rejected")
1855 .to_string();
1856
1857 QueryFailure {
1858 status: status.as_u16(),
1859 reason,
1860 message,
1861 body,
1862 }
1863}
1864
1865fn workflow_command_result(
1866 command: WorkflowCommandKind,
1867 data: Value,
1868 workflow_id: &str,
1869 run_id: Option<&str>,
1870) -> WorkflowCommandResult {
1871 WorkflowCommandResult {
1872 command,
1873 workflow_id: data
1874 .get("workflow_id")
1875 .and_then(Value::as_str)
1876 .unwrap_or(workflow_id)
1877 .to_string(),
1878 run_id: data
1879 .get("run_id")
1880 .and_then(Value::as_str)
1881 .or(run_id)
1882 .map(str::to_string),
1883 outcome: data
1884 .get("outcome")
1885 .and_then(Value::as_str)
1886 .map(str::to_string),
1887 reason: data
1888 .get("reason")
1889 .and_then(Value::as_str)
1890 .map(str::to_string),
1891 command_status: data
1892 .get("command_status")
1893 .and_then(Value::as_str)
1894 .map(str::to_string),
1895 raw: data,
1896 }
1897}
1898
1899fn workflow_command_rejection(
1900 command: WorkflowCommandKind,
1901 status: reqwest::StatusCode,
1902 raw_body: String,
1903 workflow_id: &str,
1904 run_id: Option<&str>,
1905) -> WorkflowCommandRejection {
1906 let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
1907 WorkflowCommandRejection {
1908 command,
1909 status: status.as_u16(),
1910 reason: body
1911 .get("reason")
1912 .and_then(Value::as_str)
1913 .unwrap_or("workflow_command_rejected")
1914 .to_string(),
1915 message: body
1916 .get("message")
1917 .or_else(|| body.get("error"))
1918 .and_then(Value::as_str)
1919 .unwrap_or("workflow lifecycle command was rejected")
1920 .to_string(),
1921 workflow_id: body
1922 .get("workflow_id")
1923 .and_then(Value::as_str)
1924 .unwrap_or(workflow_id)
1925 .to_string(),
1926 run_id: body
1927 .get("run_id")
1928 .and_then(Value::as_str)
1929 .or(run_id)
1930 .map(str::to_string),
1931 target_scope: body
1932 .get("target_scope")
1933 .and_then(Value::as_str)
1934 .map(str::to_string),
1935 body,
1936 }
1937}
1938
1939fn query_task_response(response: Result<Value>) -> Result<Value> {
1940 match response {
1941 Err(Error::Http { status, body }) => Err(Error::QueryFailed(query_failure(status, body))),
1942 response => response,
1943 }
1944}
1945
1946fn worker_poll_response<T: DeserializeOwned>(response: Result<T>) -> Result<T> {
1947 match response {
1948 Err(Error::Http { status, body })
1949 if status == reqwest::StatusCode::CONFLICT && worker_poll_body_is_stop(&body) =>
1950 {
1951 Ok(serde_json::from_str(&body)?)
1952 }
1953 response => response,
1954 }
1955}
1956
1957fn worker_poll_body_is_stop(body: &str) -> bool {
1958 serde_json::from_str::<Value>(body)
1959 .ok()
1960 .is_some_and(|body| {
1961 worker_poll_is_stop(
1962 body.get("poll_status").and_then(Value::as_str),
1963 body.get("reason").and_then(Value::as_str),
1964 )
1965 })
1966}
1967
1968fn worker_poll_is_stop(poll_status: Option<&str>, reason: Option<&str>) -> bool {
1969 matches!(poll_status, Some("draining" | "stopped"))
1970 || matches!(reason, Some("worker_draining" | "worker_stopped"))
1971}
1972
1973fn query_task_rejection_is_final(error: &Error) -> bool {
1974 matches!(
1975 error,
1976 Error::QueryFailed(failure)
1977 if QUERY_TASK_FINAL_REJECTION_REASONS.contains(&failure.reason.as_str())
1978 )
1979}
1980
1981fn activity_task_response<T>(
1982 response: Result<T>,
1983 operation: &str,
1984 task_id: &str,
1985 activity_attempt_id: &str,
1986) -> Result<T> {
1987 match response {
1988 Err(Error::Http { status, body }) => {
1989 let body = serde_json::from_str(&body).unwrap_or_else(|_| json!({"message": body}));
1990 Err(Error::ActivityTaskRejected(ActivityTaskRejection {
1991 operation: operation.to_string(),
1992 status: status.as_u16(),
1993 reason: body
1994 .get("reason")
1995 .and_then(Value::as_str)
1996 .unwrap_or("activity_task_rejected")
1997 .to_string(),
1998 task_id: body
1999 .get("task_id")
2000 .and_then(Value::as_str)
2001 .unwrap_or(task_id)
2002 .to_string(),
2003 activity_attempt_id: body
2004 .get("activity_attempt_id")
2005 .and_then(Value::as_str)
2006 .unwrap_or(activity_attempt_id)
2007 .to_string(),
2008 cancel_requested: body
2009 .get("cancel_requested")
2010 .and_then(Value::as_bool)
2011 .unwrap_or(false),
2012 can_continue: body.get("can_continue").and_then(Value::as_bool),
2013 run_closed_reason: body
2014 .get("run_closed_reason")
2015 .and_then(Value::as_str)
2016 .map(str::to_string),
2017 body,
2018 }))
2019 }
2020 response => response,
2021 }
2022}
2023
2024fn activity_task_rejection_is_final(error: &Error) -> bool {
2025 matches!(
2026 error,
2027 Error::ActivityTaskRejected(rejection)
2028 if matches!(
2029 rejection.reason.as_str(),
2030 "run_cancelled"
2031 | "run_terminated"
2032 | "attempt_closed"
2033 | "stale_attempt"
2034 | "activity_cancelled"
2035 | "task_cancelled"
2036 | "run_closed"
2037 | "activity_not_running"
2038 | "attempt_not_found"
2039 )
2040 )
2041}
2042
2043fn protocol_failure(status: reqwest::StatusCode, raw_body: &str) -> Option<ProtocolFailure> {
2044 let body: Value = serde_json::from_str(raw_body).ok()?;
2045 let reason = body.get("reason")?.as_str()?;
2046 if !matches!(
2047 reason,
2048 "missing_protocol_version"
2049 | "unsupported_protocol_version"
2050 | "missing_control_plane_version"
2051 | "unsupported_control_plane_version"
2052 ) {
2053 return None;
2054 }
2055
2056 Some(ProtocolFailure {
2057 status: status.as_u16(),
2058 reason: reason.to_string(),
2059 message: body
2060 .get("message")
2061 .or_else(|| body.get("error"))
2062 .and_then(Value::as_str)
2063 .unwrap_or("protocol version rejected")
2064 .to_string(),
2065 supported_version: body
2066 .get("supported_version")
2067 .and_then(Value::as_str)
2068 .map(str::to_string),
2069 requested_version: body
2070 .get("requested_version")
2071 .and_then(Value::as_str)
2072 .map(str::to_string),
2073 body,
2074 })
2075}
2076
2077fn long_poll_timeout_seconds(timeout: Duration) -> u64 {
2078 timeout
2079 .as_secs()
2080 .saturating_add(u64::from(timeout.subsec_nanos() > 0))
2081 .min(MAX_LONG_POLL_TIMEOUT_SECONDS)
2082}
2083
2084fn worker_operation_is_retryable(error: &Error) -> bool {
2085 match error {
2086 Error::Transport(error) => {
2087 error.is_timeout() || error.is_connect() || error.is_request() || error.is_body()
2088 }
2089 Error::Http { status, .. } => {
2090 matches!(
2091 *status,
2092 reqwest::StatusCode::REQUEST_TIMEOUT | reqwest::StatusCode::TOO_MANY_REQUESTS
2093 ) || status.is_server_error()
2094 }
2095 _ => false,
2096 }
2097}
2098
2099fn worker_retry_delay(policy: WorkerRetryPolicy, retry: usize) -> Duration {
2100 let exponent = retry.saturating_sub(1).min(31) as u32;
2101 policy
2102 .initial_backoff
2103 .saturating_mul(1_u32 << exponent)
2104 .min(policy.max_backoff)
2105}
2106
2107#[derive(Debug)]
2108pub struct ClientBuilder {
2109 base_url: String,
2110 token: Option<String>,
2111 control_token: Option<String>,
2112 worker_token: Option<String>,
2113 namespace: String,
2114 timeout: Duration,
2115}
2116
2117impl ClientBuilder {
2118 pub fn token(mut self, token: Option<String>) -> Self {
2119 self.token = token;
2120 self
2121 }
2122
2123 pub fn control_token(mut self, token: Option<String>) -> Self {
2124 self.control_token = token;
2125 self
2126 }
2127
2128 pub fn worker_token(mut self, token: Option<String>) -> Self {
2129 self.worker_token = token;
2130 self
2131 }
2132
2133 pub fn namespace(mut self, namespace: impl Into<String>) -> Self {
2134 self.namespace = namespace.into();
2135 self
2136 }
2137
2138 pub fn timeout(mut self, timeout: Duration) -> Self {
2139 self.timeout = timeout;
2140 self
2141 }
2142
2143 pub fn build(self) -> Result<Client> {
2144 Ok(Client {
2145 http: reqwest::Client::builder().timeout(self.timeout).build()?,
2146 base_url: self.base_url.trim_end_matches('/').to_string(),
2147 token: self.token,
2148 control_token: self.control_token,
2149 worker_token: self.worker_token,
2150 namespace: self.namespace,
2151 })
2152 }
2153}
2154
2155#[derive(Clone, Debug)]
2156pub struct WorkflowHandle {
2157 client: Client,
2158 pub workflow_id: String,
2159 pub run_id: Option<String>,
2160 pub workflow_type: String,
2161}
2162
2163impl WorkflowHandle {
2164 pub async fn describe(&self) -> Result<WorkflowDescription> {
2165 self.client.describe_workflow(&self.workflow_id).await
2166 }
2167
2168 pub async fn signal<T: Serialize>(&self, signal_name: &str, input: T) -> Result<Value> {
2169 self.client
2170 .signal_workflow(&self.workflow_id, signal_name, input)
2171 .await
2172 }
2173
2174 pub async fn cancel(&self, options: WorkflowCommandOptions) -> Result<WorkflowCommandResult> {
2176 self.client
2177 .cancel_workflow(&self.workflow_id, options)
2178 .await
2179 }
2180
2181 pub async fn cancel_selected_run(
2183 &self,
2184 options: WorkflowCommandOptions,
2185 ) -> Result<WorkflowCommandResult> {
2186 let run_id = self.run_id.as_deref().ok_or_else(|| {
2187 Error::Codec("run_id is required for selected-run cancellation".to_string())
2188 })?;
2189 self.client
2190 .cancel_workflow_run(&self.workflow_id, run_id, options)
2191 .await
2192 }
2193
2194 pub async fn terminate(
2196 &self,
2197 options: WorkflowCommandOptions,
2198 ) -> Result<WorkflowCommandResult> {
2199 self.client
2200 .terminate_workflow(&self.workflow_id, options)
2201 .await
2202 }
2203
2204 pub async fn terminate_selected_run(
2206 &self,
2207 options: WorkflowCommandOptions,
2208 ) -> Result<WorkflowCommandResult> {
2209 let run_id = self.run_id.as_deref().ok_or_else(|| {
2210 Error::Codec("run_id is required for selected-run termination".to_string())
2211 })?;
2212 self.client
2213 .terminate_workflow_run(&self.workflow_id, run_id, options)
2214 .await
2215 }
2216
2217 pub async fn query<T: Serialize>(&self, query_name: &str, input: T) -> Result<Value> {
2219 self.client
2220 .query_workflow(&self.workflow_id, query_name, input)
2221 .await
2222 }
2223
2224 pub async fn result(&self, options: WorkflowResultOptions) -> Result<Value> {
2225 let started = Instant::now();
2226
2227 loop {
2228 let description = match self.run_id.as_deref() {
2229 Some(run_id) => {
2230 self.client
2231 .describe_workflow_run(&self.workflow_id, run_id)
2232 .await?
2233 }
2234 None => self.describe().await?,
2235 };
2236 if description.is_completed() {
2237 return Ok(description.output.unwrap_or(Value::Null));
2238 }
2239
2240 if description.is_terminal() {
2241 let outcome = workflow_terminal_outcome(
2242 &description,
2243 &self.workflow_id,
2244 self.run_id.as_deref(),
2245 );
2246 return Err(match outcome.kind {
2247 WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
2248 WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
2249 WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
2250 WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
2251 });
2252 }
2253
2254 if started.elapsed() >= options.timeout {
2255 return Err(Error::WorkflowTimedOut(WorkflowTerminalOutcome {
2256 kind: WorkflowTerminalKind::TimedOut,
2257 workflow_id: description
2258 .workflow_id
2259 .clone()
2260 .unwrap_or_else(|| self.workflow_id.clone()),
2261 run_id: description.run_id.clone().or_else(|| self.run_id.clone()),
2262 reason: "result_wait_timeout".to_string(),
2263 failure_category: Some("client_timeout".to_string()),
2264 failure_id: None,
2265 exception_type: None,
2266 exception_class: None,
2267 non_retryable: None,
2268 message: Some(format!(
2269 "workflow result was not terminal within {:?}",
2270 options.timeout
2271 )),
2272 exception: None,
2273 raw: description.raw_value(),
2274 }));
2275 }
2276
2277 tokio::time::sleep(options.poll_interval).await;
2278 }
2279 }
2280}
2281
2282#[derive(Clone, Copy, Debug)]
2283pub struct WorkflowResultOptions {
2284 pub poll_interval: Duration,
2285 pub timeout: Duration,
2286}
2287
2288impl Default for WorkflowResultOptions {
2289 fn default() -> Self {
2290 Self {
2291 poll_interval: Duration::from_millis(500),
2292 timeout: Duration::from_secs(30),
2293 }
2294 }
2295}
2296
2297#[derive(Clone, Debug, Deserialize)]
2298pub struct WorkflowDescription {
2299 pub workflow_id: Option<String>,
2300 pub run_id: Option<String>,
2301 pub workflow_type: Option<String>,
2302 pub status: Option<String>,
2303 #[serde(default)]
2304 pub closed_reason: Option<String>,
2305 #[serde(default)]
2306 pub error: Option<String>,
2307 #[serde(default)]
2308 pub failure: Option<Value>,
2309 #[serde(default)]
2310 pub exception: Option<Value>,
2311 #[serde(default)]
2312 pub failures: Vec<Value>,
2313 #[serde(default)]
2314 pub output: Option<Value>,
2315 #[serde(default)]
2316 pub output_envelope: Option<Value>,
2317 #[serde(flatten)]
2318 pub raw: HashMap<String, Value>,
2319}
2320
2321impl WorkflowDescription {
2322 pub fn is_completed(&self) -> bool {
2323 matches!(self.status.as_deref(), Some("completed" | "Completed"))
2324 }
2325
2326 pub fn is_terminal(&self) -> bool {
2327 matches!(
2328 self.status.as_deref(),
2329 Some(
2330 "completed"
2331 | "Completed"
2332 | "failed"
2333 | "Failed"
2334 | "cancelled"
2335 | "Cancelled"
2336 | "terminated"
2337 | "Terminated"
2338 | "timed_out"
2339 | "TimedOut",
2340 )
2341 )
2342 }
2343
2344 fn decode_payloads(&mut self) -> Result<()> {
2345 if let Some(envelope) = &self.output_envelope {
2346 self.output = Some(decode_wire_value(envelope, DEFAULT_CODEC)?);
2347 }
2348
2349 Ok(())
2350 }
2351
2352 fn raw_value(&self) -> Value {
2353 let mut data = self.raw.clone();
2354 data.insert(
2355 "workflow_id".to_string(),
2356 self.workflow_id
2357 .clone()
2358 .map(Value::String)
2359 .unwrap_or(Value::Null),
2360 );
2361 data.insert(
2362 "run_id".to_string(),
2363 self.run_id
2364 .clone()
2365 .map(Value::String)
2366 .unwrap_or(Value::Null),
2367 );
2368 data.insert(
2369 "workflow_type".to_string(),
2370 self.workflow_type
2371 .clone()
2372 .map(Value::String)
2373 .unwrap_or(Value::Null),
2374 );
2375 data.insert(
2376 "status".to_string(),
2377 self.status
2378 .clone()
2379 .map(Value::String)
2380 .unwrap_or(Value::Null),
2381 );
2382 data.insert(
2383 "closed_reason".to_string(),
2384 self.closed_reason
2385 .clone()
2386 .map(Value::String)
2387 .unwrap_or(Value::Null),
2388 );
2389 if let Some(failure) = &self.failure {
2390 data.insert("failure".to_string(), failure.clone());
2391 }
2392 if let Some(exception) = &self.exception {
2393 data.insert("exception".to_string(), exception.clone());
2394 }
2395 Value::Object(data.into_iter().collect())
2396 }
2397}
2398
2399fn workflow_terminal_outcome(
2400 description: &WorkflowDescription,
2401 workflow_id: &str,
2402 run_id: Option<&str>,
2403) -> WorkflowTerminalOutcome {
2404 let terminal_kind = description
2405 .closed_reason
2406 .as_deref()
2407 .or(description.status.as_deref())
2408 .unwrap_or("failed")
2409 .to_ascii_lowercase();
2410 let kind = match terminal_kind.as_str() {
2411 "cancelled" | "canceled" => WorkflowTerminalKind::Cancelled,
2412 "terminated" => WorkflowTerminalKind::Terminated,
2413 "timed_out" | "timedout" => WorkflowTerminalKind::TimedOut,
2414 _ => WorkflowTerminalKind::Failed,
2415 };
2416 let default_reason = match kind {
2417 WorkflowTerminalKind::Failed => "workflow_failed",
2418 WorkflowTerminalKind::Cancelled => "cancelled",
2419 WorkflowTerminalKind::Terminated => "terminated",
2420 WorkflowTerminalKind::TimedOut => "timed_out",
2421 };
2422 let failure = description
2423 .failure
2424 .as_ref()
2425 .filter(|value| value.is_object());
2426 let nested_failure = failure
2427 .and_then(|value| value.get("failures"))
2428 .and_then(Value::as_array)
2429 .and_then(|failures| failures.last())
2430 .or_else(|| description.failures.last());
2431 let exception = description
2432 .exception
2433 .clone()
2434 .or_else(|| failure.and_then(|value| value.get("exception")).cloned())
2435 .or_else(|| {
2436 nested_failure
2437 .and_then(|value| value.get("exception_payload"))
2438 .cloned()
2439 });
2440 let string_field = |name: &str| {
2441 failure
2442 .and_then(|value| value.get(name))
2443 .and_then(Value::as_str)
2444 .or_else(|| {
2445 nested_failure
2446 .and_then(|value| value.get(name))
2447 .and_then(Value::as_str)
2448 })
2449 .map(str::to_string)
2450 };
2451 let exception_field = |name: &str| {
2452 exception
2453 .as_ref()
2454 .and_then(|value| value.get(name))
2455 .and_then(Value::as_str)
2456 .map(str::to_string)
2457 };
2458 let message = description
2459 .error
2460 .clone()
2461 .or_else(|| string_field("message"))
2462 .or_else(|| exception_field("message"));
2463 let reason = description
2464 .raw
2465 .get("reason")
2466 .and_then(Value::as_str)
2467 .map(str::to_string)
2468 .or_else(|| {
2469 failure
2470 .and_then(|value| value.get("reason"))
2471 .and_then(Value::as_str)
2472 .map(str::to_string)
2473 })
2474 .or_else(|| description.closed_reason.clone())
2475 .unwrap_or_else(|| default_reason.to_string());
2476 let failure_id = string_field("failure_id").or_else(|| {
2477 nested_failure
2478 .and_then(|value| value.get("id"))
2479 .and_then(Value::as_str)
2480 .map(str::to_string)
2481 });
2482
2483 WorkflowTerminalOutcome {
2484 kind,
2485 workflow_id: description
2486 .workflow_id
2487 .clone()
2488 .unwrap_or_else(|| workflow_id.to_string()),
2489 run_id: description
2490 .run_id
2491 .clone()
2492 .or_else(|| run_id.map(str::to_string)),
2493 reason,
2494 failure_category: string_field("failure_category")
2495 .or_else(|| Some(default_reason.to_string())),
2496 failure_id,
2497 exception_type: string_field("exception_type").or_else(|| exception_field("type")),
2498 exception_class: string_field("exception_class").or_else(|| exception_field("class")),
2499 non_retryable: failure
2500 .and_then(|value| value.get("non_retryable"))
2501 .and_then(Value::as_bool)
2502 .or_else(|| {
2503 nested_failure
2504 .and_then(|value| value.get("non_retryable"))
2505 .and_then(Value::as_bool)
2506 }),
2507 message,
2508 exception,
2509 raw: description.raw_value(),
2510 }
2511}
2512
2513#[derive(Clone, Debug, Deserialize)]
2514pub struct RegisterWorkerResponse {
2515 pub worker_id: String,
2516 pub registered: bool,
2517 #[serde(default)]
2518 pub heartbeat_interval_seconds: Option<u64>,
2519 #[serde(default)]
2520 pub protocol_version: Option<String>,
2521 #[serde(default)]
2522 pub server_capabilities: Option<Value>,
2523}
2524
2525#[derive(Clone, Debug, Deserialize)]
2526pub struct PollWorkflowTaskResponse {
2527 #[serde(default)]
2528 pub task: Option<WorkflowTask>,
2529 #[serde(default)]
2530 pub poll_status: Option<String>,
2531 #[serde(default)]
2532 pub reason: Option<String>,
2533 #[serde(default)]
2534 pub protocol_version: Option<String>,
2535 #[serde(default)]
2536 pub server_capabilities: Option<Value>,
2537}
2538
2539impl PollWorkflowTaskResponse {
2540 pub fn outcome(&self) -> WorkerPollOutcome {
2542 worker_poll_outcome(
2543 self.task.is_some(),
2544 self.poll_status.as_deref(),
2545 self.reason.as_deref(),
2546 )
2547 }
2548}
2549
2550#[derive(Clone, Debug, Deserialize)]
2551pub struct PollActivityTaskResponse {
2552 #[serde(default)]
2553 pub task: Option<ActivityTask>,
2554 #[serde(default)]
2555 pub poll_status: Option<String>,
2556 #[serde(default)]
2557 pub reason: Option<String>,
2558}
2559
2560impl PollActivityTaskResponse {
2561 pub fn outcome(&self) -> WorkerPollOutcome {
2563 worker_poll_outcome(
2564 self.task.is_some(),
2565 self.poll_status.as_deref(),
2566 self.reason.as_deref(),
2567 )
2568 }
2569}
2570
2571#[derive(Clone, Debug, Deserialize)]
2572pub struct PollQueryTaskResponse {
2573 #[serde(default)]
2574 pub task: Option<QueryTask>,
2575 #[serde(default)]
2576 pub poll_status: Option<String>,
2577 #[serde(default)]
2578 pub reason: Option<String>,
2579}
2580
2581impl PollQueryTaskResponse {
2582 pub fn outcome(&self) -> WorkerPollOutcome {
2584 worker_poll_outcome(
2585 self.task.is_some(),
2586 self.poll_status.as_deref(),
2587 self.reason.as_deref(),
2588 )
2589 }
2590}
2591
2592#[derive(Clone, Debug, PartialEq, Eq)]
2594pub enum WorkerPollOutcome {
2595 Task,
2597 Idle {
2599 poll_status: Option<String>,
2600 reason: Option<String>,
2601 },
2602 Stop {
2604 poll_status: Option<String>,
2605 reason: Option<String>,
2606 },
2607}
2608
2609impl WorkerPollOutcome {
2610 pub fn should_stop(&self) -> bool {
2611 matches!(self, Self::Stop { .. })
2612 }
2613}
2614
2615fn worker_poll_outcome(
2616 has_task: bool,
2617 poll_status: Option<&str>,
2618 reason: Option<&str>,
2619) -> WorkerPollOutcome {
2620 if worker_poll_is_stop(poll_status, reason) {
2621 return WorkerPollOutcome::Stop {
2622 poll_status: poll_status.map(str::to_string),
2623 reason: reason.map(str::to_string),
2624 };
2625 }
2626
2627 if has_task {
2628 WorkerPollOutcome::Task
2629 } else {
2630 WorkerPollOutcome::Idle {
2631 poll_status: poll_status.map(str::to_string),
2632 reason: reason.map(str::to_string),
2633 }
2634 }
2635}
2636
2637#[derive(Clone, Debug, Deserialize)]
2639pub struct QueryTask {
2640 pub query_task_id: String,
2641 #[serde(default = "default_workflow_task_attempt")]
2642 pub query_task_attempt: u64,
2643 #[serde(default)]
2644 pub lease_owner: Option<String>,
2645 #[serde(default)]
2646 pub workflow_id: Option<String>,
2647 #[serde(default)]
2648 pub run_id: Option<String>,
2649 pub workflow_type: String,
2650 pub query_name: String,
2651 #[serde(default = "default_payload_codec")]
2652 pub payload_codec: String,
2653 #[serde(default)]
2654 pub workflow_arguments: Option<Value>,
2655 #[serde(default)]
2656 pub query_arguments: Option<Value>,
2657 #[serde(default)]
2658 pub history_events: Vec<HistoryEvent>,
2659 #[serde(default)]
2660 pub history_export: Option<Value>,
2661 #[serde(default)]
2662 pub run_status: Option<String>,
2663}
2664
2665#[derive(Clone, Debug, Deserialize)]
2666pub struct WorkflowTask {
2667 pub task_id: String,
2668 #[serde(default)]
2669 pub workflow_id: Option<String>,
2670 #[serde(default)]
2671 pub run_id: Option<String>,
2672 pub workflow_type: String,
2673 #[serde(default = "default_payload_codec")]
2674 pub payload_codec: String,
2675 #[serde(default)]
2676 pub arguments: Option<Value>,
2677 #[serde(default)]
2678 pub history_events: Vec<HistoryEvent>,
2679 #[serde(default)]
2680 pub total_history_events: Option<u64>,
2681 #[serde(default)]
2682 pub next_history_page_token: Option<String>,
2683 #[serde(default = "default_workflow_task_attempt")]
2684 pub workflow_task_attempt: u64,
2685 #[serde(default)]
2686 pub workflow_signal_id: Option<String>,
2687 #[serde(default)]
2688 pub signal_name: Option<String>,
2689 #[serde(default)]
2690 pub signal_arguments: Option<Value>,
2691 #[serde(default)]
2692 pub lease_owner: Option<String>,
2693}
2694
2695impl WorkflowTask {
2696 fn append_history_page(&mut self, page: WorkflowTaskHistoryPage) {
2697 self.history_events.extend(page.history_events);
2698
2699 if page.total_history_events.is_some() {
2700 self.total_history_events = page.total_history_events;
2701 }
2702
2703 self.next_history_page_token = page
2704 .next_history_page_token
2705 .filter(|token| !token.is_empty());
2706 }
2707}
2708
2709#[derive(Clone, Debug, Deserialize)]
2710struct WorkflowTaskHistoryPage {
2711 #[serde(default)]
2712 history_events: Vec<HistoryEvent>,
2713 #[serde(default)]
2714 total_history_events: Option<u64>,
2715 #[serde(default)]
2716 next_history_page_token: Option<String>,
2717}
2718
2719#[derive(Clone, Debug, Deserialize)]
2720pub struct ActivityTask {
2721 pub task_id: String,
2722 #[serde(default)]
2723 pub activity_attempt_id: Option<String>,
2724 #[serde(default)]
2725 pub attempt_id: Option<String>,
2726 pub activity_type: String,
2727 #[serde(default = "default_payload_codec")]
2728 pub payload_codec: String,
2729 #[serde(default)]
2730 pub arguments: Option<Value>,
2731 #[serde(default = "default_attempt_number")]
2732 pub attempt_number: u64,
2733 #[serde(default)]
2734 pub lease_owner: Option<String>,
2735}
2736
2737#[derive(Clone, Debug, Deserialize)]
2738pub struct HistoryEvent {
2739 #[serde(alias = "type")]
2740 pub event_type: String,
2741 #[serde(default)]
2742 pub payload: Value,
2743 #[serde(flatten)]
2744 pub raw: HashMap<String, Value>,
2745}
2746
2747#[derive(Clone, Debug, PartialEq)]
2749pub struct QuerySignal {
2750 pub id: Option<String>,
2751 pub name: String,
2752 pub arguments: Vec<Value>,
2753 pub workflow_sequence: Option<u64>,
2754}
2755
2756#[derive(Clone, Debug)]
2762pub struct QueryContext {
2763 pub workflow_id: Option<String>,
2764 pub run_id: Option<String>,
2765 pub workflow_type: String,
2766 pub run_status: Option<String>,
2767 workflow_input: Value,
2768 history_events: Arc<Vec<HistoryEvent>>,
2769 signal_events: Arc<Vec<QuerySignal>>,
2770}
2771
2772impl QueryContext {
2773 pub fn workflow_input(&self) -> &Value {
2775 &self.workflow_input
2776 }
2777
2778 pub fn history_events(&self) -> &[HistoryEvent] {
2780 self.history_events.as_slice()
2781 }
2782
2783 pub fn signal_events(&self) -> &[QuerySignal] {
2785 self.signal_events.as_slice()
2786 }
2787
2788 pub fn signals(&self, signal_name: &str) -> Vec<Vec<Value>> {
2790 self.signal_events
2791 .iter()
2792 .filter(|signal| signal.name == signal_name)
2793 .map(|signal| signal.arguments.clone())
2794 .collect()
2795 }
2796}
2797
2798#[derive(Clone, Debug, Deserialize)]
2799pub struct ActivityHeartbeatResponse {
2800 #[serde(default)]
2801 pub cancel_requested: bool,
2802 #[serde(default)]
2803 pub heartbeat_recorded: bool,
2804 #[serde(default)]
2805 pub can_continue: Option<bool>,
2806 #[serde(default)]
2807 pub reason: Option<String>,
2808 #[serde(default)]
2809 pub run_closed_reason: Option<String>,
2810 #[serde(default)]
2811 pub run_closed_at: Option<String>,
2812 #[serde(default)]
2813 pub lease_expires_at: Option<String>,
2814 #[serde(default)]
2815 pub last_heartbeat_at: Option<String>,
2816}
2817
2818impl ActivityHeartbeatResponse {
2819 pub fn should_stop(&self) -> bool {
2821 self.cancel_requested || self.can_continue == Some(false)
2822 }
2823}
2824
2825fn default_payload_codec() -> String {
2826 DEFAULT_CODEC.to_string()
2827}
2828
2829fn default_workflow_task_attempt() -> u64 {
2830 1
2831}
2832
2833fn default_attempt_number() -> u64 {
2834 1
2835}
2836
2837type WorkflowFuture = Pin<Box<dyn Future<Output = Result<Value>> + Send + 'static>>;
2838type WorkflowHandler = Arc<dyn Fn(WorkflowContext, Value) -> WorkflowFuture + Send + Sync>;
2839type ErasedWorkflowState = Arc<dyn Any + Send + Sync>;
2840type WorkflowStateSnapshot = Arc<dyn Fn() -> Result<ErasedWorkflowState> + Send + Sync>;
2841type ReplayedWorkflowHandler =
2842 Arc<dyn Fn(WorkflowContext, Value) -> ReplayedWorkflowInvocation + Send + Sync>;
2843type ActivityFuture = Pin<Box<dyn Future<Output = Result<Value>> + Send + 'static>>;
2844type ActivityHandler = Arc<dyn Fn(ActivityContext, Value) -> ActivityFuture + Send + Sync>;
2845type QueryFuture = Pin<Box<dyn Future<Output = Result<Value>> + Send + 'static>>;
2846type QueryHandler = Arc<dyn Fn(QueryContext, Value) -> QueryFuture + Send + Sync>;
2847type ReplayedQueryHandler = Arc<
2848 dyn Fn(QueryContext, ErasedWorkflowState, Value) -> std::result::Result<QueryFuture, String>
2849 + Send
2850 + Sync,
2851>;
2852type WorkerHeartbeatObserver = Arc<dyn Fn(&WorkerHeartbeatObservation) + Send + Sync>;
2853
2854struct ReplayedWorkflowInvocation {
2855 future: WorkflowFuture,
2856 snapshot: WorkflowStateSnapshot,
2857}
2858
2859#[derive(Clone)]
2860struct RegisteredWorkflow {
2861 execute: WorkflowHandler,
2862 replay: Option<ReplayedWorkflowHandler>,
2863 state_type: Option<TypeId>,
2864}
2865
2866#[derive(Clone)]
2867enum RegisteredQuery {
2868 Snapshot(QueryHandler),
2869 Replayed {
2870 state_type: TypeId,
2871 handler: ReplayedQueryHandler,
2872 },
2873}
2874
2875#[derive(Clone, Debug)]
2876pub struct WorkerHeartbeatObservation {
2877 pub worker_id: String,
2878 pub task_queue: String,
2879 pub acknowledged_at_unix_millis: u64,
2880 pub acknowledgement: Value,
2881}
2882
2883#[derive(Clone, Copy, Debug)]
2890pub struct WorkerRetryPolicy {
2891 pub max_retries: usize,
2893 pub initial_backoff: Duration,
2895 pub max_backoff: Duration,
2897}
2898
2899impl Default for WorkerRetryPolicy {
2900 fn default() -> Self {
2901 Self {
2902 max_retries: 5,
2903 initial_backoff: Duration::from_millis(100),
2904 max_backoff: Duration::from_secs(5),
2905 }
2906 }
2907}
2908
2909#[derive(Clone, Copy, Debug, PartialEq, Eq)]
2910enum ManagedPollOutcome {
2911 Idle,
2912 Handled,
2913 Stop,
2914}
2915
2916#[derive(Clone)]
2917pub struct Worker {
2918 client: Client,
2919 worker_id: String,
2920 task_queue: String,
2921 workflows: HashMap<String, RegisteredWorkflow>,
2922 activities: HashMap<String, ActivityHandler>,
2923 queries: HashMap<String, HashMap<String, RegisteredQuery>>,
2924 max_concurrent_workflow_tasks: usize,
2925 max_concurrent_activity_tasks: usize,
2926 poll_timeout: Duration,
2927 heartbeat_interval: Duration,
2928 retry_policy: WorkerRetryPolicy,
2929 heartbeat_observer: Option<WorkerHeartbeatObserver>,
2930}
2931
2932impl Worker {
2933 pub fn new(client: Client, task_queue: impl Into<String>) -> Self {
2934 Self {
2935 client,
2936 worker_id: default_worker_id(),
2937 task_queue: task_queue.into(),
2938 workflows: HashMap::new(),
2939 activities: HashMap::new(),
2940 queries: HashMap::new(),
2941 max_concurrent_workflow_tasks: 10,
2942 max_concurrent_activity_tasks: 10,
2943 poll_timeout: Duration::from_secs(30),
2944 heartbeat_interval: Duration::from_secs(60),
2945 retry_policy: WorkerRetryPolicy::default(),
2946 heartbeat_observer: None,
2947 }
2948 }
2949
2950 pub fn worker_id(mut self, worker_id: impl Into<String>) -> Self {
2951 self.worker_id = worker_id.into();
2952 self
2953 }
2954
2955 pub fn poll_timeout(mut self, timeout: Duration) -> Self {
2956 self.poll_timeout = timeout;
2957 self
2958 }
2959
2960 pub fn heartbeat_interval(mut self, interval: Duration) -> Self {
2961 self.heartbeat_interval = interval;
2962 self
2963 }
2964
2965 pub fn retry_policy(mut self, policy: WorkerRetryPolicy) -> Self {
2967 self.retry_policy = policy;
2968 self
2969 }
2970
2971 pub fn on_worker_heartbeat<F>(mut self, observer: F) -> Self
2972 where
2973 F: Fn(&WorkerHeartbeatObservation) + Send + Sync + 'static,
2974 {
2975 self.heartbeat_observer = Some(Arc::new(observer));
2976 self
2977 }
2978
2979 pub fn max_concurrent_workflow_tasks(mut self, count: usize) -> Self {
2980 self.max_concurrent_workflow_tasks = count.max(1);
2981 self
2982 }
2983
2984 pub fn max_concurrent_activity_tasks(mut self, count: usize) -> Self {
2985 self.max_concurrent_activity_tasks = count.max(1);
2986 self
2987 }
2988
2989 pub fn register_workflow<F, Fut>(&mut self, workflow_type: impl Into<String>, handler: F)
2990 where
2991 F: Fn(WorkflowContext, Value) -> Fut + Send + Sync + 'static,
2992 Fut: Future<Output = Result<Value>> + Send + 'static,
2993 {
2994 self.workflows.insert(
2995 workflow_type.into(),
2996 RegisteredWorkflow {
2997 execute: Arc::new(move |ctx, input| Box::pin(handler(ctx, input))),
2998 replay: None,
2999 state_type: None,
3000 },
3001 );
3002 }
3003
3004 pub fn register_replayed_workflow<S, Factory, F, Fut>(
3012 &mut self,
3013 workflow_type: impl Into<String>,
3014 state_factory: Factory,
3015 handler: F,
3016 ) where
3017 S: Clone + Send + Sync + 'static,
3018 Factory: Fn() -> S + Send + Sync + 'static,
3019 F: Fn(WorkflowContext, Value, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
3020 Fut: Future<Output = Result<Value>> + Send + 'static,
3021 {
3022 let state_factory = Arc::new(state_factory);
3023 let handler = Arc::new(handler);
3024
3025 let execute_factory = Arc::clone(&state_factory);
3026 let execute_handler = Arc::clone(&handler);
3027 let execute = Arc::new(move |ctx: WorkflowContext, input: Value| {
3028 let state = WorkflowInstance::new(execute_factory());
3029 let future = execute_handler(ctx, input, state);
3030 Box::pin(future) as WorkflowFuture
3031 });
3032
3033 let replay = Arc::new(move |ctx: WorkflowContext, input: Value| {
3034 let state = WorkflowInstance::new(state_factory());
3035 let snapshot_state = state.clone();
3036 let snapshot: WorkflowStateSnapshot =
3037 Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
3038 let future = handler(ctx, input, state);
3039 ReplayedWorkflowInvocation {
3040 future: Box::pin(future),
3041 snapshot,
3042 }
3043 });
3044
3045 self.workflows.insert(
3046 workflow_type.into(),
3047 RegisteredWorkflow {
3048 execute,
3049 replay: Some(replay),
3050 state_type: Some(TypeId::of::<S>()),
3051 },
3052 );
3053 }
3054
3055 pub fn register_activity<F, Fut>(&mut self, activity_type: impl Into<String>, handler: F)
3056 where
3057 F: Fn(ActivityContext, Value) -> Fut + Send + Sync + 'static,
3058 Fut: Future<Output = Result<Value>> + Send + 'static,
3059 {
3060 self.activities.insert(
3061 activity_type.into(),
3062 Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
3063 );
3064 }
3065
3066 pub fn register_query<F, Fut>(
3072 &mut self,
3073 workflow_type: impl Into<String>,
3074 query_name: impl Into<String>,
3075 handler: F,
3076 ) where
3077 F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
3078 Fut: Future<Output = Result<Value>> + Send + 'static,
3079 {
3080 self.queries
3081 .entry(workflow_type.into())
3082 .or_default()
3083 .insert(
3084 query_name.into(),
3085 RegisteredQuery::Snapshot(Arc::new(move |ctx, args| Box::pin(handler(ctx, args)))),
3086 );
3087 }
3088
3089 pub fn register_replayed_query<S, F, Fut>(
3096 &mut self,
3097 workflow_type: impl Into<String>,
3098 query_name: impl Into<String>,
3099 handler: F,
3100 ) where
3101 S: Clone + Send + Sync + 'static,
3102 F: Fn(QueryContext, Arc<S>, Value) -> Fut + Send + Sync + 'static,
3103 Fut: Future<Output = Result<Value>> + Send + 'static,
3104 {
3105 let handler = Arc::new(handler);
3106 let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
3107 let state = state.downcast::<S>().map_err(|_| {
3108 "registered query state type does not match the replayed workflow state".to_string()
3109 })?;
3110 Ok(Box::pin(handler(ctx, state, args)))
3111 });
3112
3113 self.queries
3114 .entry(workflow_type.into())
3115 .or_default()
3116 .insert(
3117 query_name.into(),
3118 RegisteredQuery::Replayed {
3119 state_type: TypeId::of::<S>(),
3120 handler: erased_handler,
3121 },
3122 );
3123 }
3124
3125 pub async fn register(&self) -> Result<RegisterWorkerResponse> {
3126 self.client
3127 .register_worker_with_capabilities(
3128 &self.worker_id,
3129 &self.task_queue,
3130 self.workflows.keys().cloned().collect(),
3131 self.activities.keys().cloned().collect(),
3132 self.max_concurrent_workflow_tasks,
3133 self.max_concurrent_activity_tasks,
3134 (!self.queries.is_empty())
3135 .then(|| QUERY_TASKS_CAPABILITY.to_string())
3136 .into_iter()
3137 .collect(),
3138 )
3139 .await
3140 }
3141
3142 pub async fn run(&self) -> Result<()> {
3148 self.run_until(std::future::pending::<()>()).await
3149 }
3150
3151 pub async fn run_until<F>(&self, shutdown: F) -> Result<()>
3155 where
3156 F: Future<Output = ()>,
3157 {
3158 let registration = self.register().await?;
3159 let mut heartbeat = tokio::time::interval(Duration::from_secs(
3160 registration
3161 .heartbeat_interval_seconds
3162 .unwrap_or(self.heartbeat_interval.as_secs().max(1)),
3163 ));
3164 tokio::pin!(shutdown);
3165 let stop = Arc::new(AtomicBool::new(false));
3166 let mut workflow_poller = (!self.workflows.is_empty()).then(|| {
3170 let worker = self.clone();
3171 let stop = Arc::clone(&stop);
3172 tokio::spawn(async move { worker.poll_workflows_until_stopped(stop).await })
3173 });
3174 let mut activity_poller = (!self.activities.is_empty()).then(|| {
3175 let worker = self.clone();
3176 let stop = Arc::clone(&stop);
3177 tokio::spawn(async move { worker.poll_activities_until_stopped(stop).await })
3178 });
3179 let mut query_poller = (!self.queries.is_empty()).then(|| {
3180 let worker = self.clone();
3181 let stop = Arc::clone(&stop);
3182 tokio::spawn(async move { worker.poll_queries_until_stopped(stop).await })
3183 });
3184
3185 loop {
3186 tokio::select! {
3187 _ = &mut shutdown => {
3188 stop.store(true, Ordering::SeqCst);
3189 break;
3190 }
3191 _ = heartbeat.tick() => {
3192 match self.retry_worker_operation(|| {
3193 self.client.heartbeat_worker(
3194 &self.worker_id,
3195 self.max_concurrent_workflow_tasks,
3196 self.max_concurrent_activity_tasks,
3197 )
3198 }).await
3199 {
3200 Ok(acknowledgement) => {
3201 if let Some(observer) = &self.heartbeat_observer {
3202 observer(&WorkerHeartbeatObservation {
3203 worker_id: self.worker_id.clone(),
3204 task_queue: self.task_queue.clone(),
3205 acknowledged_at_unix_millis: SystemTime::now()
3206 .duration_since(UNIX_EPOCH)
3207 .unwrap_or_default()
3208 .as_millis()
3209 .min(u64::MAX as u128)
3210 as u64,
3211 acknowledgement,
3212 });
3213 }
3214 }
3215 Err(error) => {
3216 stop.store(true, Ordering::SeqCst);
3217 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await?;
3218 return Err(error);
3219 }
3220 }
3221 }
3222 result = OptionFuture::from(workflow_poller.as_mut()), if workflow_poller.is_some() => {
3223 workflow_poller = None;
3224 let stopped_by_server = stop.load(Ordering::SeqCst);
3225 stop.store(true, Ordering::SeqCst);
3226 let poller_result = optional_poller_result("workflow", result);
3227 let join_result =
3228 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
3229 poller_result?;
3230 join_result?;
3231 if stopped_by_server {
3232 return Ok(());
3233 }
3234 return Err(Error::WorkerLoop(
3235 "workflow poller stopped unexpectedly".to_string(),
3236 ));
3237 }
3238 result = OptionFuture::from(activity_poller.as_mut()), if activity_poller.is_some() => {
3239 activity_poller = None;
3240 let stopped_by_server = stop.load(Ordering::SeqCst);
3241 stop.store(true, Ordering::SeqCst);
3242 let poller_result = optional_poller_result("activity", result);
3243 let join_result =
3244 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
3245 poller_result?;
3246 join_result?;
3247 if stopped_by_server {
3248 return Ok(());
3249 }
3250 return Err(Error::WorkerLoop(
3251 "activity poller stopped unexpectedly".to_string(),
3252 ));
3253 }
3254 result = OptionFuture::from(query_poller.as_mut()), if query_poller.is_some() => {
3255 query_poller = None;
3256 let stopped_by_server = stop.load(Ordering::SeqCst);
3257 stop.store(true, Ordering::SeqCst);
3258 let poller_result = optional_poller_result("query", result);
3259 let join_result =
3260 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
3261 poller_result?;
3262 join_result?;
3263 if stopped_by_server {
3264 return Ok(());
3265 }
3266 return Err(Error::WorkerLoop(
3267 "query poller stopped unexpectedly".to_string(),
3268 ));
3269 }
3270 }
3271 }
3272
3273 join_pollers(
3274 workflow_poller.take(),
3275 activity_poller.take(),
3276 query_poller.take(),
3277 )
3278 .await
3279 }
3280
3281 pub async fn run_once(&self) -> Result<usize> {
3282 let mut handled = 0;
3283 match self.poll_workflow_once().await? {
3284 ManagedPollOutcome::Handled => handled += 1,
3285 ManagedPollOutcome::Stop => return Ok(handled),
3286 ManagedPollOutcome::Idle => {}
3287 }
3288 match self.poll_activity_once().await? {
3289 ManagedPollOutcome::Handled => handled += 1,
3290 ManagedPollOutcome::Stop => return Ok(handled),
3291 ManagedPollOutcome::Idle => {}
3292 }
3293 if !self.queries.is_empty() {
3294 match self.poll_query_once().await? {
3295 ManagedPollOutcome::Handled => handled += 1,
3296 ManagedPollOutcome::Stop => return Ok(handled),
3297 ManagedPollOutcome::Idle => {}
3298 }
3299 }
3300 Ok(handled)
3301 }
3302
3303 async fn poll_workflow_once(&self) -> Result<ManagedPollOutcome> {
3304 let response = self
3305 .retry_worker_operation(|| {
3306 self.client.poll_workflow_task_response(
3307 &self.worker_id,
3308 &self.task_queue,
3309 self.poll_timeout,
3310 )
3311 })
3312 .await?;
3313 if response.outcome().should_stop() {
3314 return Ok(ManagedPollOutcome::Stop);
3315 }
3316 let Some(task) = response.task else {
3317 return Ok(ManagedPollOutcome::Idle);
3318 };
3319
3320 let task_id = task.task_id.clone();
3321 let attempt = task.workflow_task_attempt;
3322 let lease_owner = task
3323 .lease_owner
3324 .clone()
3325 .unwrap_or_else(|| self.worker_id.clone());
3326
3327 match self.execute_workflow_task(task) {
3328 Ok(commands) if commands.is_empty() => {
3329 self.client
3335 .fail_workflow_task_with_type(
3336 &task_id,
3337 &lease_owner,
3338 attempt,
3339 WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE,
3340 WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE,
3341 )
3342 .await?;
3343 }
3344 Ok(commands) => {
3345 self.client
3346 .complete_workflow_task(&task_id, &lease_owner, attempt, commands)
3347 .await?;
3348 }
3349 Err(error) => {
3350 self.client
3351 .fail_workflow_task(&task_id, &lease_owner, attempt, error.to_string())
3352 .await?;
3353 }
3354 }
3355
3356 Ok(ManagedPollOutcome::Handled)
3357 }
3358
3359 async fn poll_workflows_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
3360 while !stop.load(Ordering::SeqCst) {
3361 if self.poll_workflow_once().await? == ManagedPollOutcome::Stop {
3362 stop.store(true, Ordering::SeqCst);
3363 break;
3364 }
3365 }
3366
3367 Ok(())
3368 }
3369
3370 async fn poll_activity_once(&self) -> Result<ManagedPollOutcome> {
3371 let response = self
3372 .retry_worker_operation(|| {
3373 self.client.poll_activity_task_response(
3374 &self.worker_id,
3375 &self.task_queue,
3376 self.poll_timeout,
3377 )
3378 })
3379 .await?;
3380 if response.outcome().should_stop() {
3381 return Ok(ManagedPollOutcome::Stop);
3382 }
3383 let Some(task) = response.task else {
3384 return Ok(ManagedPollOutcome::Idle);
3385 };
3386
3387 let task_id = task.task_id.clone();
3388 let attempt_id = task
3389 .activity_attempt_id
3390 .clone()
3391 .or(task.attempt_id.clone())
3392 .unwrap_or_default();
3393 let lease_owner = task
3394 .lease_owner
3395 .clone()
3396 .unwrap_or_else(|| self.worker_id.clone());
3397 let codec = task.payload_codec.clone();
3398 let result = self.execute_activity_task(task).await;
3399 match result {
3400 Ok(value) => {
3401 let completion = self
3402 .client
3403 .complete_activity_task(&task_id, &attempt_id, &lease_owner, value, &codec)
3404 .await;
3405 if let Err(error) = completion {
3406 if !activity_task_rejection_is_final(&error) {
3407 return Err(error);
3408 }
3409 }
3410 }
3411 Err(error) => {
3412 let failure = self
3413 .client
3414 .fail_activity_task(
3415 &task_id,
3416 &attempt_id,
3417 &lease_owner,
3418 error.to_string(),
3419 false,
3420 )
3421 .await;
3422 if let Err(error) = failure {
3423 if !activity_task_rejection_is_final(&error) {
3424 return Err(error);
3425 }
3426 }
3427 }
3428 }
3429
3430 Ok(ManagedPollOutcome::Handled)
3431 }
3432
3433 async fn poll_activities_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
3434 while !stop.load(Ordering::SeqCst) {
3435 if self.poll_activity_once().await? == ManagedPollOutcome::Stop {
3436 stop.store(true, Ordering::SeqCst);
3437 break;
3438 }
3439 }
3440
3441 Ok(())
3442 }
3443
3444 async fn poll_query_once(&self) -> Result<ManagedPollOutcome> {
3445 let response = self
3446 .retry_worker_operation(|| {
3447 self.client.poll_query_task_response(
3448 &self.worker_id,
3449 &self.task_queue,
3450 self.poll_timeout,
3451 )
3452 })
3453 .await?;
3454 if response.outcome().should_stop() {
3455 return Ok(ManagedPollOutcome::Stop);
3456 }
3457 let Some(task) = response.task else {
3458 return Ok(ManagedPollOutcome::Idle);
3459 };
3460
3461 let query_task_id = task.query_task_id.clone();
3462 let attempt = task.query_task_attempt;
3463 let lease_owner = task
3464 .lease_owner
3465 .clone()
3466 .unwrap_or_else(|| self.worker_id.clone());
3467 let codec = task.payload_codec.clone();
3468
3469 match self.execute_query_task(task).await {
3470 Ok(value) => {
3471 let result_envelope = match encode_value_envelope(&value, &codec) {
3472 Ok(result_envelope) => result_envelope,
3473 Err(error) => {
3474 let failure = self
3475 .client
3476 .fail_query_task(
3477 &query_task_id,
3478 &lease_owner,
3479 attempt,
3480 error.to_string(),
3481 "query_result_encode_failed",
3482 "QueryResultEncodeFailed",
3483 )
3484 .await;
3485 if let Err(error) = failure {
3486 if !query_task_rejection_is_final(&error) {
3487 return Err(error);
3488 }
3489 }
3490 return Ok(ManagedPollOutcome::Handled);
3491 }
3492 };
3493
3494 if let Err(error) = self
3495 .client
3496 .complete_query_task_with_envelope(
3497 &query_task_id,
3498 &lease_owner,
3499 attempt,
3500 value,
3501 result_envelope,
3502 )
3503 .await
3504 {
3505 if !query_task_rejection_is_final(&error) {
3506 return Err(error);
3507 }
3508 }
3509 }
3510 Err(failure) => {
3511 let result = self
3512 .client
3513 .fail_query_task(
3514 &query_task_id,
3515 &lease_owner,
3516 attempt,
3517 failure.message,
3518 failure.reason,
3519 failure.failure_type,
3520 )
3521 .await;
3522 if let Err(error) = result {
3523 if !query_task_rejection_is_final(&error) {
3524 return Err(error);
3525 }
3526 }
3527 }
3528 }
3529
3530 Ok(ManagedPollOutcome::Handled)
3531 }
3532
3533 async fn poll_queries_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
3534 while !stop.load(Ordering::SeqCst) {
3535 if self.poll_query_once().await? == ManagedPollOutcome::Stop {
3536 stop.store(true, Ordering::SeqCst);
3537 break;
3538 }
3539 }
3540
3541 Ok(())
3542 }
3543
3544 async fn retry_worker_operation<T, F, Fut>(&self, mut operation: F) -> Result<T>
3545 where
3546 F: FnMut() -> Fut,
3547 Fut: Future<Output = Result<T>>,
3548 {
3549 let mut retries = 0;
3550
3551 loop {
3552 match operation().await {
3553 Err(error)
3554 if worker_operation_is_retryable(&error)
3555 && retries < self.retry_policy.max_retries =>
3556 {
3557 retries += 1;
3558 tokio::time::sleep(worker_retry_delay(self.retry_policy, retries)).await;
3559 }
3560 result => return result,
3561 }
3562 }
3563 }
3564
3565 async fn execute_query_task(
3566 &self,
3567 mut task: QueryTask,
3568 ) -> std::result::Result<Value, QueryTaskExecutionFailure> {
3569 if !matches!(task.payload_codec.as_str(), DEFAULT_CODEC | JSON_CODEC) {
3570 return Err(QueryTaskExecutionFailure::new(
3571 "query_payload_decode_failed",
3572 format!(
3573 "cannot decode query payload with unsupported codec {:?}",
3574 task.payload_codec
3575 ),
3576 "QueryPayloadDecodeFailed",
3577 ));
3578 }
3579
3580 if !self.workflows.contains_key(&task.workflow_type) {
3581 return Err(QueryTaskExecutionFailure::new(
3582 "query_workflow_type_not_registered",
3583 format!("no workflow registered for type {:?}", task.workflow_type),
3584 "WorkflowTypeNotRegistered",
3585 ));
3586 }
3587
3588 let Some(handlers) = self.queries.get(&task.workflow_type) else {
3589 return Err(QueryTaskExecutionFailure::new(
3590 "query_handler_unavailable",
3591 format!(
3592 "query handlers are unavailable for workflow type {:?}",
3593 task.workflow_type
3594 ),
3595 "QueryHandlerUnavailable",
3596 ));
3597 };
3598 let Some(query) = handlers.get(&task.query_name) else {
3599 return Err(QueryTaskExecutionFailure::new(
3600 "rejected_unknown_query",
3601 format!("unknown query {:?}", task.query_name),
3602 "QueryFailed",
3603 ));
3604 };
3605
3606 let args = decode_task_arguments(task.query_arguments.as_ref(), &task.payload_codec)
3607 .map_err(|error| {
3608 QueryTaskExecutionFailure::new(
3609 "query_payload_decode_failed",
3610 format!("cannot decode query arguments: {error}"),
3611 "QueryPayloadDecodeFailed",
3612 )
3613 })?;
3614 let workflow_input =
3615 decode_task_arguments(task.workflow_arguments.as_ref(), &task.payload_codec).map_err(
3616 |error| {
3617 QueryTaskExecutionFailure::new(
3618 "query_workflow_state_unavailable",
3619 format!("cannot decode workflow start input: {error}"),
3620 "QueryWorkflowStateUnavailable",
3621 )
3622 },
3623 )?;
3624 hydrate_query_history_from_export(&mut task).map_err(|error| {
3625 QueryTaskExecutionFailure::new(
3626 "query_workflow_state_unavailable",
3627 format!("cannot restore query history snapshot: {error}"),
3628 "QueryWorkflowStateUnavailable",
3629 )
3630 })?;
3631 enrich_query_history_from_export(&mut task).map_err(|error| {
3632 QueryTaskExecutionFailure::new(
3633 "query_workflow_state_unavailable",
3634 format!("cannot restore compact query history payloads: {error}"),
3635 "QueryWorkflowStateUnavailable",
3636 )
3637 })?;
3638 let signal_events = query_signal_events(&task).map_err(|error| {
3639 QueryTaskExecutionFailure::new(
3640 "query_workflow_state_unavailable",
3641 format!("cannot decode committed workflow signals: {error}"),
3642 "QueryWorkflowStateUnavailable",
3643 )
3644 })?;
3645 let history_events = Arc::new(std::mem::take(&mut task.history_events));
3646 let context = QueryContext {
3647 workflow_id: task.workflow_id,
3648 run_id: task.run_id,
3649 workflow_type: task.workflow_type.clone(),
3650 run_status: task.run_status,
3651 workflow_input,
3652 history_events: Arc::clone(&history_events),
3653 signal_events: Arc::new(signal_events),
3654 };
3655
3656 let future = match query {
3657 RegisteredQuery::Snapshot(handler) => handler(context, args),
3658 RegisteredQuery::Replayed {
3659 state_type,
3660 handler,
3661 } => {
3662 let workflow = self
3663 .workflows
3664 .get(&task.workflow_type)
3665 .expect("workflow registration was checked above");
3666 if workflow.state_type != Some(*state_type) {
3667 return Err(QueryTaskExecutionFailure::new(
3668 "query_workflow_state_unavailable",
3669 "replayed query state type does not match its workflow registration",
3670 "QueryWorkflowStateUnavailable",
3671 ));
3672 }
3673 let replay = workflow.replay.as_ref().ok_or_else(|| {
3674 QueryTaskExecutionFailure::new(
3675 "query_workflow_state_unavailable",
3676 format!(
3677 "workflow type {:?} is not registered for instance-state replay",
3678 task.workflow_type
3679 ),
3680 "QueryWorkflowStateUnavailable",
3681 )
3682 })?;
3683 let workflow_state = Arc::new(Mutex::new(
3684 WorkflowState::new_with_identity(
3685 history_events.as_ref().clone(),
3686 context.workflow_id.clone(),
3687 context.run_id.clone(),
3688 self.task_queue.clone(),
3689 task.payload_codec,
3690 None,
3691 )
3692 .map_err(|error| {
3693 QueryTaskExecutionFailure::new(
3694 "query_workflow_state_unavailable",
3695 format!("workflow replay failed before query: {error}"),
3696 "QueryWorkflowStateUnavailable",
3697 )
3698 })?,
3699 ));
3700 let workflow_context = WorkflowContext {
3701 state: workflow_state,
3702 };
3703 let mut invocation =
3704 replay(workflow_context.clone(), context.workflow_input.clone());
3705 let mut cx = TaskContext::from_waker(noop_waker_ref());
3706 match invocation.future.as_mut().poll(&mut cx) {
3707 Poll::Ready(Ok(_)) => {
3708 workflow_context
3709 .ensure_history_consumed()
3710 .map_err(|error| {
3711 QueryTaskExecutionFailure::new(
3712 "query_workflow_state_unavailable",
3713 format!("workflow replay failed before query: {error}"),
3714 "QueryWorkflowStateUnavailable",
3715 )
3716 })?;
3717 }
3718 Poll::Ready(Err(error)) => {
3719 return Err(QueryTaskExecutionFailure::new(
3720 "query_workflow_state_unavailable",
3721 format!("workflow replay failed before query: {error}"),
3722 "QueryWorkflowStateUnavailable",
3723 ));
3724 }
3725 Poll::Pending => {
3726 let commands = workflow_context.take_commands().map_err(|error| {
3727 QueryTaskExecutionFailure::new(
3728 "query_workflow_state_unavailable",
3729 format!("workflow replay failed before query: {error}"),
3730 "QueryWorkflowStateUnavailable",
3731 )
3732 })?;
3733 if commands.is_empty()
3734 && !workflow_context
3735 .matched_recorded_pending()
3736 .map_err(|error| {
3737 QueryTaskExecutionFailure::new(
3738 "query_workflow_state_unavailable",
3739 format!("workflow replay failed before query: {error}"),
3740 "QueryWorkflowStateUnavailable",
3741 )
3742 })?
3743 {
3744 return Err(QueryTaskExecutionFailure::new(
3745 "query_workflow_state_unavailable",
3746 "workflow replay yielded without a durable command",
3747 "QueryWorkflowStateUnavailable",
3748 ));
3749 }
3750 }
3751 }
3752 let state = (invocation.snapshot)().map_err(|error| {
3753 QueryTaskExecutionFailure::new(
3754 "query_workflow_state_unavailable",
3755 format!("cannot snapshot replayed workflow state: {error}"),
3756 "QueryWorkflowStateUnavailable",
3757 )
3758 })?;
3759 handler(context, state, args).map_err(|message| {
3760 QueryTaskExecutionFailure::new(
3761 "query_workflow_state_unavailable",
3762 message,
3763 "QueryWorkflowStateUnavailable",
3764 )
3765 })?
3766 }
3767 };
3768
3769 future.await.map_err(|error| {
3770 QueryTaskExecutionFailure::new("query_rejected", error.to_string(), "QueryFailed")
3771 })
3772 }
3773
3774 fn execute_workflow_task(&self, task: WorkflowTask) -> Result<Vec<Value>> {
3775 let workflow = self
3776 .workflows
3777 .get(&task.workflow_type)
3778 .ok_or_else(|| Error::WorkflowNotRegistered(task.workflow_type.clone()))?;
3779 let input = decode_task_arguments(task.arguments.as_ref(), &task.payload_codec)?;
3780 let resume_signal = decode_resume_signal(&task)?;
3781 let state = Arc::new(Mutex::new(WorkflowState::new_with_identity(
3782 task.history_events,
3783 task.workflow_id,
3784 task.run_id,
3785 self.task_queue.clone(),
3786 task.payload_codec.clone(),
3787 resume_signal,
3788 )?));
3789 let ctx = WorkflowContext { state };
3790 let mut future = (workflow.execute)(ctx.clone(), input);
3791 let mut cx = TaskContext::from_waker(noop_waker_ref());
3792
3793 match future.as_mut().poll(&mut cx) {
3794 Poll::Ready(Ok(result)) => {
3795 ctx.ensure_history_consumed()?;
3796 let result = encode_value_envelope(&result, &task.payload_codec)?;
3797 Ok(vec![json!({
3798 "type": "complete_workflow",
3799 "result": result
3800 })])
3801 }
3802 Poll::Ready(Err(error @ Error::ChildWorkflowFailed(_))) => {
3803 Ok(vec![workflow_failure_command(&error)])
3804 }
3805 Poll::Ready(Err(error)) => Err(error),
3806 Poll::Pending => {
3807 let commands = ctx.take_commands()?;
3808 if commands.is_empty() && !ctx.matched_recorded_pending()? {
3809 Err(Error::WorkflowYieldedWithoutCommand)
3810 } else {
3811 Ok(commands)
3812 }
3813 }
3814 }
3815 }
3816
3817 async fn execute_activity_task(&self, task: ActivityTask) -> Result<Value> {
3818 let handler = self
3819 .activities
3820 .get(&task.activity_type)
3821 .ok_or_else(|| Error::ActivityNotRegistered(task.activity_type.clone()))?;
3822 let args = decode_task_arguments(task.arguments.as_ref(), &task.payload_codec)?;
3823 let attempt_id = task
3824 .activity_attempt_id
3825 .clone()
3826 .or(task.attempt_id.clone())
3827 .unwrap_or_default();
3828 let lease_owner = task
3829 .lease_owner
3830 .clone()
3831 .unwrap_or_else(|| self.worker_id.clone());
3832 let ctx = ActivityContext {
3833 client: self.client.clone(),
3834 task_id: task.task_id,
3835 activity_attempt_id: attempt_id,
3836 lease_owner,
3837 activity_type: task.activity_type,
3838 attempt_number: task.attempt_number,
3839 task_queue: self.task_queue.clone(),
3840 worker_id: self.worker_id.clone(),
3841 };
3842
3843 handler(ctx, args).await
3844 }
3845}
3846
3847fn poller_result(
3848 kind: &str,
3849 result: std::result::Result<Result<()>, tokio::task::JoinError>,
3850) -> Result<()> {
3851 match result {
3852 Ok(result) => result,
3853 Err(error) => Err(Error::WorkerLoop(format!(
3854 "{kind} poller join error: {error}"
3855 ))),
3856 }
3857}
3858
3859fn optional_poller_result(
3860 kind: &str,
3861 result: Option<std::result::Result<Result<()>, tokio::task::JoinError>>,
3862) -> Result<()> {
3863 match result {
3864 Some(result) => poller_result(kind, result),
3865 None => Ok(()),
3866 }
3867}
3868
3869async fn join_pollers(
3870 workflow_poller: Option<tokio::task::JoinHandle<Result<()>>>,
3871 activity_poller: Option<tokio::task::JoinHandle<Result<()>>>,
3872 query_poller: Option<tokio::task::JoinHandle<Result<()>>>,
3873) -> Result<()> {
3874 let mut first_error = None;
3875
3876 if let Some(handle) = workflow_poller {
3877 if let Err(error) = poller_result("workflow", handle.await) {
3878 first_error.get_or_insert(error);
3879 }
3880 }
3881
3882 if let Some(handle) = activity_poller {
3883 if let Err(error) = poller_result("activity", handle.await) {
3884 first_error.get_or_insert(error);
3885 }
3886 }
3887
3888 if let Some(handle) = query_poller {
3889 if let Err(error) = poller_result("query", handle.await) {
3890 first_error.get_or_insert(error);
3891 }
3892 }
3893
3894 if let Some(error) = first_error {
3895 Err(error)
3896 } else {
3897 Ok(())
3898 }
3899}
3900
3901fn default_worker_id() -> String {
3902 let millis = SystemTime::now()
3903 .duration_since(UNIX_EPOCH)
3904 .unwrap_or_default()
3905 .as_millis();
3906 format!("rust-worker-{}-{millis}", std::process::id())
3907}
3908
3909fn unique_request_id(prefix: &str) -> String {
3910 let nanos = SystemTime::now()
3911 .duration_since(UNIX_EPOCH)
3912 .unwrap_or_default()
3913 .as_nanos();
3914 format!("{prefix}-{}-{nanos}", std::process::id())
3915}
3916
3917#[derive(Debug)]
3918struct QueryTaskExecutionFailure {
3919 reason: String,
3920 message: String,
3921 failure_type: String,
3922}
3923
3924impl QueryTaskExecutionFailure {
3925 fn new(
3926 reason: impl Into<String>,
3927 message: impl Into<String>,
3928 failure_type: impl Into<String>,
3929 ) -> Self {
3930 Self {
3931 reason: reason.into(),
3932 message: message.into(),
3933 failure_type: failure_type.into(),
3934 }
3935 }
3936}
3937
3938#[derive(Clone, Debug)]
3944pub struct WorkflowInstance<S> {
3945 state: Arc<Mutex<S>>,
3946}
3947
3948impl<S> WorkflowInstance<S> {
3949 fn new(state: S) -> Self {
3950 Self {
3951 state: Arc::new(Mutex::new(state)),
3952 }
3953 }
3954
3955 pub fn read<R>(&self, reader: impl FnOnce(&S) -> R) -> Result<R> {
3957 let state = self
3958 .state
3959 .lock()
3960 .map_err(|_| Error::WorkflowStatePoisoned)?;
3961 Ok(reader(&state))
3962 }
3963
3964 pub fn update<R>(&self, transition: impl FnOnce(&mut S) -> R) -> Result<R> {
3966 let mut state = self
3967 .state
3968 .lock()
3969 .map_err(|_| Error::WorkflowStatePoisoned)?;
3970 Ok(transition(&mut state))
3971 }
3972}
3973
3974impl<S: Clone> WorkflowInstance<S> {
3975 fn snapshot(&self) -> Result<S> {
3976 self.read(Clone::clone)
3977 }
3978}
3979
3980#[derive(Clone, Debug)]
3981pub struct WorkflowContext {
3982 state: Arc<Mutex<WorkflowState>>,
3983}
3984
3985impl WorkflowContext {
3986 pub fn workflow_identity(&self) -> Result<WorkflowIdentity> {
3988 let state = self
3989 .state
3990 .lock()
3991 .map_err(|_| Error::WorkflowStatePoisoned)?;
3992 Ok(WorkflowIdentity {
3993 workflow_id: state.workflow_id.clone(),
3994 run_id: state.run_id.clone(),
3995 })
3996 }
3997
3998 pub fn activity<T: Serialize>(
3999 &self,
4000 activity_type: impl Into<String>,
4001 args: T,
4002 ) -> ActivityCall {
4003 self.activity_with_options(activity_type, ActivityOptions::new(), args)
4004 }
4005
4006 pub fn activity_on_queue<T, Q>(
4007 &self,
4008 activity_type: impl Into<String>,
4009 task_queue: Option<Q>,
4010 args: T,
4011 ) -> ActivityCall
4012 where
4013 T: Serialize,
4014 Q: Into<String>,
4015 {
4016 let mut options = ActivityOptions::new();
4017 options.task_queue = task_queue.map(Into::into);
4018 self.activity_with_options(activity_type, options, args)
4019 }
4020
4021 pub fn activity_with_options<T: Serialize>(
4059 &self,
4060 activity_type: impl Into<String>,
4061 options: ActivityOptions,
4062 args: T,
4063 ) -> ActivityCall {
4064 ActivityCall {
4065 ctx: self.clone(),
4066 activity_type: activity_type.into(),
4067 options,
4068 args: Some(serde_json::to_value(args).map_err(Error::from)),
4069 scheduled: false,
4070 }
4071 }
4072
4073 pub fn wait_signal(&self, signal_name: impl Into<String>) -> SignalCall {
4074 SignalCall {
4075 ctx: self.clone(),
4076 signal_name: signal_name.into(),
4077 opened_wait: false,
4078 matched_pending: false,
4079 }
4080 }
4081
4082 pub fn sleep(&self, duration: Duration) -> TimerCall {
4103 let delay_seconds = duration
4104 .as_secs()
4105 .checked_add(u64::from(duration.subsec_nanos() > 0));
4106 TimerCall {
4107 ctx: self.clone(),
4108 delay_seconds,
4109 scheduled: false,
4110 matched_pending: false,
4111 }
4112 }
4113
4114 pub fn start_timer(&self, duration: Duration) -> TimerCall {
4116 self.sleep(duration)
4117 }
4118
4119 pub fn start_child_workflow<T: Serialize>(
4146 &self,
4147 workflow_type: impl Into<String>,
4148 options: ChildWorkflowOptions,
4149 args: T,
4150 ) -> ChildWorkflowCall {
4151 ChildWorkflowCall {
4152 ctx: self.clone(),
4153 workflow_type: workflow_type.into(),
4154 options,
4155 args: Some(serde_json::to_value(args).map_err(Error::from)),
4156 scheduled: false,
4157 matched_pending: false,
4158 }
4159 }
4160
4161 fn take_commands(&self) -> Result<Vec<Value>> {
4162 let mut state = self
4163 .state
4164 .lock()
4165 .map_err(|_| Error::WorkflowStatePoisoned)?;
4166 Ok(std::mem::take(&mut state.commands))
4167 }
4168
4169 fn matched_recorded_pending(&self) -> Result<bool> {
4170 let state = self
4171 .state
4172 .lock()
4173 .map_err(|_| Error::WorkflowStatePoisoned)?;
4174 Ok(state.matched_recorded_pending)
4175 }
4176
4177 fn ensure_history_consumed(&self) -> Result<()> {
4178 let state = self
4179 .state
4180 .lock()
4181 .map_err(|_| Error::WorkflowStatePoisoned)?;
4182 if let Some(command) = state.recorded_commands.get(state.command_cursor) {
4183 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
4184 "recorded_commands_unconsumed",
4185 Some(command.sequence()),
4186 Some(command.shape().to_string()),
4187 Some("workflow completion".to_string()),
4188 "workflow completed before consuming all recorded durable commands",
4189 )));
4190 }
4191 Ok(())
4192 }
4193}
4194
4195#[derive(Debug)]
4196struct WorkflowState {
4197 history: Vec<HistoryEvent>,
4198 workflow_id: Option<String>,
4199 run_id: Option<String>,
4200 task_queue: String,
4201 payload_codec: String,
4202 resume_signal: Option<ResumeSignal>,
4203 recorded_commands: Vec<RecordedCommand>,
4204 command_cursor: usize,
4205 matched_recorded_pending: bool,
4206 signal_cursors: HashMap<String, usize>,
4207 commands: Vec<Value>,
4208}
4209
4210impl WorkflowState {
4211 #[cfg(test)]
4212 fn new(
4213 history: Vec<HistoryEvent>,
4214 task_queue: String,
4215 payload_codec: String,
4216 resume_signal: Option<ResumeSignal>,
4217 ) -> Result<Self> {
4218 Self::new_with_identity(
4219 history,
4220 None,
4221 None,
4222 task_queue,
4223 payload_codec,
4224 resume_signal,
4225 )
4226 }
4227
4228 fn new_with_identity(
4229 history: Vec<HistoryEvent>,
4230 workflow_id: Option<String>,
4231 run_id: Option<String>,
4232 task_queue: String,
4233 payload_codec: String,
4234 resume_signal: Option<ResumeSignal>,
4235 ) -> Result<Self> {
4236 let recorded_commands = recorded_commands(
4237 &history,
4238 &payload_codec,
4239 WorkflowIdentity {
4240 workflow_id: workflow_id.clone(),
4241 run_id: run_id.clone(),
4242 },
4243 )?;
4244 Ok(Self {
4245 history,
4246 workflow_id,
4247 run_id,
4248 task_queue,
4249 payload_codec,
4250 resume_signal,
4251 recorded_commands,
4252 command_cursor: 0,
4253 matched_recorded_pending: false,
4254 signal_cursors: HashMap::new(),
4255 commands: Vec::new(),
4256 })
4257 }
4258}
4259
4260#[derive(Clone, Debug)]
4261enum RecordedCommand {
4262 Activity {
4263 sequence: u64,
4264 activity_type: Option<String>,
4265 options: Option<RecordedActivityOptions>,
4266 outcome: Option<ActivityOutcome>,
4267 },
4268 Timer {
4269 sequence: u64,
4270 delay_seconds: u64,
4271 fired: bool,
4272 },
4273 ChildWorkflow {
4274 sequence: u64,
4275 workflow_type: Option<String>,
4276 outcome: Option<ChildWorkflowOutcome>,
4277 },
4278 SignalWait {
4279 sequence: u64,
4280 signal_name: Option<String>,
4281 },
4282}
4283
4284#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
4285struct RecordedActivityOptions {
4286 task_queue: RecordedSnapshotValue<Option<String>>,
4287 execution_mode: RecordedSnapshotValue<Option<String>>,
4288 retry_policy: ActivityRetrySnapshot,
4289}
4290
4291#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
4292enum RecordedSnapshotValue<T> {
4293 Unknown,
4295 Known(T),
4296}
4297
4298impl<T: PartialEq> RecordedSnapshotValue<T> {
4299 fn matches_current(&self, current: &Self) -> bool {
4300 match self {
4301 Self::Unknown => true,
4302 Self::Known(recorded) => matches!(current, Self::Known(value) if value == recorded),
4303 }
4304 }
4305}
4306
4307#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
4308struct ActivityRetrySnapshot {
4309 snapshot_version: RecordedSnapshotValue<Option<u64>>,
4310 max_attempts: RecordedSnapshotValue<Option<u64>>,
4311 backoff_seconds: RecordedSnapshotValue<Vec<u64>>,
4312 start_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
4313 schedule_to_start_timeout: RecordedSnapshotValue<Option<u64>>,
4314 schedule_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
4315 heartbeat_timeout: RecordedSnapshotValue<Option<u64>>,
4316 non_retryable_error_types: RecordedSnapshotValue<Vec<String>>,
4317}
4318
4319impl ActivityRetrySnapshot {
4320 fn matches_current(&self, current: &Self) -> bool {
4321 self.snapshot_version
4322 .matches_current(¤t.snapshot_version)
4323 && self.max_attempts.matches_current(¤t.max_attempts)
4324 && self
4325 .backoff_seconds
4326 .matches_current(¤t.backoff_seconds)
4327 && self
4328 .start_to_close_timeout
4329 .matches_current(¤t.start_to_close_timeout)
4330 && self
4331 .schedule_to_start_timeout
4332 .matches_current(¤t.schedule_to_start_timeout)
4333 && self
4334 .schedule_to_close_timeout
4335 .matches_current(¤t.schedule_to_close_timeout)
4336 && self
4337 .heartbeat_timeout
4338 .matches_current(¤t.heartbeat_timeout)
4339 && self
4340 .non_retryable_error_types
4341 .matches_current(¤t.non_retryable_error_types)
4342 }
4343}
4344
4345fn recorded_optional_u64(
4346 object: Option<&serde_json::Map<String, Value>>,
4347 field: &str,
4348) -> RecordedSnapshotValue<Option<u64>> {
4349 match object.and_then(|object| object.get(field)) {
4350 None => RecordedSnapshotValue::Unknown,
4351 Some(Value::Null) => RecordedSnapshotValue::Known(None),
4352 Some(value) => RecordedSnapshotValue::Known(value_as_u64(value)),
4353 }
4354}
4355
4356fn recorded_optional_string(
4357 object: &serde_json::Map<String, Value>,
4358 field: &str,
4359) -> RecordedSnapshotValue<Option<String>> {
4360 match object.get(field) {
4361 None => RecordedSnapshotValue::Unknown,
4362 Some(Value::Null) => RecordedSnapshotValue::Known(None),
4363 Some(value) => RecordedSnapshotValue::Known(value.as_str().map(str::to_string)),
4364 }
4365}
4366
4367fn recorded_activity_retry_snapshot(policy: Option<&Value>) -> ActivityRetrySnapshot {
4368 let policy = policy.and_then(Value::as_object);
4369 let backoff_seconds = policy
4370 .and_then(|policy| policy.get("backoff_seconds"))
4371 .and_then(Value::as_array)
4372 .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
4373 .map_or(RecordedSnapshotValue::Unknown, RecordedSnapshotValue::Known);
4374 let mut non_retryable_error_types = Vec::new();
4375 for error_type in policy
4376 .and_then(|policy| policy.get("non_retryable_error_types"))
4377 .and_then(Value::as_array)
4378 .into_iter()
4379 .flatten()
4380 .filter_map(Value::as_str)
4381 .map(str::trim)
4382 .filter(|error_type| !error_type.is_empty())
4383 {
4384 if !non_retryable_error_types
4385 .iter()
4386 .any(|recorded| recorded == error_type)
4387 {
4388 non_retryable_error_types.push(error_type.to_string());
4389 }
4390 }
4391
4392 ActivityRetrySnapshot {
4393 snapshot_version: recorded_optional_u64(policy, "snapshot_version"),
4394 max_attempts: recorded_optional_u64(policy, "max_attempts"),
4395 backoff_seconds,
4396 start_to_close_timeout: recorded_optional_u64(policy, "start_to_close_timeout"),
4397 schedule_to_start_timeout: recorded_optional_u64(policy, "schedule_to_start_timeout"),
4398 schedule_to_close_timeout: recorded_optional_u64(policy, "schedule_to_close_timeout"),
4399 heartbeat_timeout: recorded_optional_u64(policy, "heartbeat_timeout"),
4400 non_retryable_error_types: if policy
4401 .is_some_and(|policy| policy.contains_key("non_retryable_error_types"))
4402 {
4403 RecordedSnapshotValue::Known(non_retryable_error_types)
4404 } else {
4405 RecordedSnapshotValue::Unknown
4406 },
4407 }
4408}
4409
4410fn current_activity_retry_snapshot(options: &ValidatedActivityOptions) -> ActivityRetrySnapshot {
4411 let policy = options.retry_policy.as_ref();
4412 let max_attempts = match policy.and_then(|policy| policy.get("max_attempts")) {
4413 Some(Value::Null) => None,
4414 Some(value) => value_as_u64(value),
4415 None => Some(1),
4416 };
4417 let backoff_seconds = policy
4418 .and_then(|policy| policy.get("backoff_seconds"))
4419 .and_then(Value::as_array)
4420 .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
4421 .unwrap_or_default();
4422 let non_retryable_error_types = policy
4423 .and_then(|policy| policy.get("non_retryable_error_types"))
4424 .and_then(Value::as_array)
4425 .into_iter()
4426 .flatten()
4427 .filter_map(Value::as_str)
4428 .map(str::to_string)
4429 .collect();
4430
4431 ActivityRetrySnapshot {
4432 snapshot_version: RecordedSnapshotValue::Known(Some(1)),
4433 max_attempts: RecordedSnapshotValue::Known(max_attempts),
4434 backoff_seconds: RecordedSnapshotValue::Known(backoff_seconds),
4435 start_to_close_timeout: RecordedSnapshotValue::Known(options.start_to_close_timeout),
4436 schedule_to_start_timeout: RecordedSnapshotValue::Known(options.schedule_to_start_timeout),
4437 schedule_to_close_timeout: RecordedSnapshotValue::Known(options.schedule_to_close_timeout),
4438 heartbeat_timeout: RecordedSnapshotValue::Known(options.heartbeat_timeout),
4439 non_retryable_error_types: RecordedSnapshotValue::Known(non_retryable_error_types),
4440 }
4441}
4442
4443fn activity_options_description(options: &RecordedActivityOptions) -> String {
4444 serde_json::to_string(options).unwrap_or_else(|_| format!("{options:?}"))
4445}
4446
4447impl RecordedCommand {
4448 fn sequence(&self) -> u64 {
4449 match self {
4450 Self::Activity { sequence, .. }
4451 | Self::Timer { sequence, .. }
4452 | Self::ChildWorkflow { sequence, .. }
4453 | Self::SignalWait { sequence, .. } => *sequence,
4454 }
4455 }
4456
4457 fn shape(&self) -> &'static str {
4458 match self {
4459 Self::Activity { .. } => "activity",
4460 Self::Timer { .. } => "timer",
4461 Self::ChildWorkflow { .. } => "child workflow",
4462 Self::SignalWait { .. } => "signal wait",
4463 }
4464 }
4465}
4466
4467#[derive(Clone, Debug)]
4468struct ResumeSignal {
4469 signal_id: Option<String>,
4470 signal_name: String,
4471 arguments: Vec<Value>,
4472}
4473
4474pub struct ActivityCall {
4475 ctx: WorkflowContext,
4476 activity_type: String,
4477 options: ActivityOptions,
4478 args: Option<Result<Value>>,
4479 scheduled: bool,
4480}
4481
4482impl Future for ActivityCall {
4483 type Output = Result<Value>;
4484
4485 fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
4486 let ctx = self.ctx.clone();
4487 let mut state = match ctx.state.lock() {
4488 Ok(state) => state,
4489 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
4490 };
4491
4492 if self.scheduled {
4493 return Poll::Pending;
4494 }
4495
4496 let options = match self.options.validate() {
4497 Ok(options) => options,
4498 Err(error) => {
4499 return Poll::Ready(Err(Error::InvalidActivityOptions(error)));
4500 }
4501 };
4502 let task_queue = options
4503 .task_queue
4504 .clone()
4505 .unwrap_or_else(|| state.task_queue.clone());
4506 let current_recorded_options = RecordedActivityOptions {
4507 task_queue: RecordedSnapshotValue::Known(Some(task_queue.clone())),
4508 execution_mode: RecordedSnapshotValue::Known(None),
4511 retry_policy: current_activity_retry_snapshot(&options),
4512 };
4513
4514 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
4515 let sequence = recorded.sequence();
4516 match recorded {
4517 RecordedCommand::Activity {
4518 activity_type,
4519 options: recorded_options,
4520 outcome,
4521 ..
4522 } => {
4523 if let Some(recorded_type) = activity_type {
4524 if recorded_type != self.activity_type {
4525 return Poll::Ready(Err(Error::NonDeterministicReplay(
4526 ReplayFailure::new(
4527 "recorded_command_detail_mismatch",
4528 Some(sequence),
4529 Some(format!("activity:{recorded_type}")),
4530 Some(format!("activity:{}", self.activity_type)),
4531 "recorded activity type differs from the current workflow command",
4532 ),
4533 )));
4534 }
4535 }
4536 if let Some(recorded_options) = recorded_options {
4537 if !recorded_options
4538 .task_queue
4539 .matches_current(¤t_recorded_options.task_queue)
4540 {
4541 return Poll::Ready(Err(Error::NonDeterministicReplay(
4542 ReplayFailure::new(
4543 "activity_task_queue_mismatch",
4544 Some(sequence),
4545 Some(activity_options_description(&recorded_options)),
4546 Some(activity_options_description(¤t_recorded_options)),
4547 "recorded activity task queue differs from the current workflow command",
4548 ),
4549 )));
4550 }
4551 if !recorded_options
4552 .execution_mode
4553 .matches_current(¤t_recorded_options.execution_mode)
4554 {
4555 return Poll::Ready(Err(Error::NonDeterministicReplay(
4556 ReplayFailure::new(
4557 "activity_execution_mode_mismatch",
4558 Some(sequence),
4559 Some(activity_options_description(&recorded_options)),
4560 Some(activity_options_description(¤t_recorded_options)),
4561 "recorded activity execution mode differs from the current workflow command",
4562 ),
4563 )));
4564 }
4565 if !recorded_options
4566 .retry_policy
4567 .matches_current(¤t_recorded_options.retry_policy)
4568 {
4569 return Poll::Ready(Err(Error::NonDeterministicReplay(
4570 ReplayFailure::new(
4571 "activity_retry_policy_mismatch",
4572 Some(sequence),
4573 Some(activity_options_description(&recorded_options)),
4574 Some(activity_options_description(¤t_recorded_options)),
4575 "recorded activity retry policy differs from the current workflow command",
4576 ),
4577 )));
4578 }
4579 }
4580 state.command_cursor += 1;
4581 if let Some(outcome) = outcome {
4582 return Poll::Ready(outcome.map_err(Error::ActivityFailed));
4583 }
4584 state.matched_recorded_pending = true;
4585 self.scheduled = true;
4586 return Poll::Pending;
4587 }
4588 other => {
4589 return Poll::Ready(Err(command_mismatch(
4590 &other,
4591 format!("activity:{}", self.activity_type),
4592 )));
4593 }
4594 }
4595 }
4596
4597 if !self.scheduled {
4598 let args = match self.args.take().unwrap_or(Ok(Value::Null)) {
4599 Ok(args) => args,
4600 Err(error) => return Poll::Ready(Err(error)),
4601 };
4602 let arguments = normalize_arguments(args);
4603 let envelope = match encode_value_envelope(&arguments, &state.payload_codec) {
4604 Ok(envelope) => envelope,
4605 Err(error) => return Poll::Ready(Err(error)),
4606 };
4607
4608 let mut command = serde_json::Map::from_iter([
4609 ("type".to_string(), json!("schedule_activity")),
4610 (
4611 "activity_type".to_string(),
4612 json!(self.activity_type.clone()),
4613 ),
4614 ("queue".to_string(), json!(task_queue)),
4615 ("arguments".to_string(), envelope),
4616 ]);
4617 for (field, value) in [
4618 ("start_to_close_timeout", options.start_to_close_timeout),
4619 (
4620 "schedule_to_start_timeout",
4621 options.schedule_to_start_timeout,
4622 ),
4623 (
4624 "schedule_to_close_timeout",
4625 options.schedule_to_close_timeout,
4626 ),
4627 ("heartbeat_timeout", options.heartbeat_timeout),
4628 ] {
4629 if let Some(value) = value {
4630 command.insert(field.to_string(), json!(value));
4631 }
4632 }
4633 if let Some(retry_policy) = options.retry_policy {
4634 command.insert("retry_policy".to_string(), retry_policy);
4635 }
4636 state.commands.push(Value::Object(command));
4637 self.scheduled = true;
4638 }
4639
4640 Poll::Pending
4641 }
4642}
4643
4644pub struct TimerCall {
4646 ctx: WorkflowContext,
4647 delay_seconds: Option<u64>,
4648 scheduled: bool,
4649 matched_pending: bool,
4650}
4651
4652impl Future for TimerCall {
4653 type Output = Result<()>;
4654
4655 fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
4656 if self.matched_pending {
4657 return Poll::Pending;
4658 }
4659
4660 let ctx = self.ctx.clone();
4661 let Some(requested_delay) = self.delay_seconds else {
4662 return Poll::Ready(Err(Error::TimerDurationOverflow));
4663 };
4664 let mut state = match ctx.state.lock() {
4665 Ok(state) => state,
4666 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
4667 };
4668
4669 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
4670 match recorded {
4671 RecordedCommand::Timer {
4672 sequence,
4673 delay_seconds,
4674 fired,
4675 ..
4676 } => {
4677 if delay_seconds != requested_delay {
4678 return Poll::Ready(Err(Error::NonDeterministicReplay(
4679 ReplayFailure::new(
4680 "timer_delay_mismatch",
4681 Some(sequence),
4682 Some(format!("timer:{delay_seconds}s")),
4683 Some(format!("timer:{requested_delay}s")),
4684 "recorded timer delay differs from the current workflow command",
4685 ),
4686 )));
4687 }
4688 state.command_cursor += 1;
4689 if fired {
4690 return Poll::Ready(Ok(()));
4691 }
4692 state.matched_recorded_pending = true;
4693 self.scheduled = true;
4694 self.matched_pending = true;
4695 return Poll::Pending;
4696 }
4697 other => return Poll::Ready(Err(command_mismatch(&other, "timer"))),
4698 }
4699 }
4700
4701 if !self.scheduled {
4702 state.commands.push(json!({
4703 "type": "start_timer",
4704 "delay_seconds": requested_delay,
4705 }));
4706 self.scheduled = true;
4707 }
4708
4709 Poll::Pending
4710 }
4711}
4712
4713pub struct ChildWorkflowCall {
4715 ctx: WorkflowContext,
4716 workflow_type: String,
4717 options: ChildWorkflowOptions,
4718 args: Option<Result<Value>>,
4719 scheduled: bool,
4720 matched_pending: bool,
4721}
4722
4723impl Future for ChildWorkflowCall {
4724 type Output = Result<ChildWorkflowResult>;
4725
4726 fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
4727 if self.matched_pending {
4728 return Poll::Pending;
4729 }
4730
4731 let ctx = self.ctx.clone();
4732 let mut state = match ctx.state.lock() {
4733 Ok(state) => state,
4734 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
4735 };
4736
4737 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
4738 let sequence = recorded.sequence();
4739 match recorded {
4740 RecordedCommand::ChildWorkflow {
4741 workflow_type,
4742 outcome,
4743 ..
4744 } => {
4745 if let Some(recorded_type) = workflow_type {
4746 if recorded_type != self.workflow_type {
4747 return Poll::Ready(Err(Error::NonDeterministicReplay(
4748 ReplayFailure::new(
4749 "recorded_command_detail_mismatch",
4750 Some(sequence),
4751 Some(format!("child workflow:{recorded_type}")),
4752 Some(format!("child workflow:{}", self.workflow_type)),
4753 "recorded child workflow type differs from the current workflow command",
4754 ),
4755 )));
4756 }
4757 }
4758 state.command_cursor += 1;
4759 if let Some(outcome) = outcome {
4760 return Poll::Ready(outcome.map_err(Error::ChildWorkflowFailed));
4761 }
4762 state.matched_recorded_pending = true;
4763 self.scheduled = true;
4764 self.matched_pending = true;
4765 return Poll::Pending;
4766 }
4767 other => {
4768 return Poll::Ready(Err(command_mismatch(
4769 &other,
4770 format!("child workflow:{}", self.workflow_type),
4771 )));
4772 }
4773 }
4774 }
4775
4776 if !self.scheduled {
4777 if self.options.task_queue.trim().is_empty() {
4778 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
4779 "task_queue must not be empty".to_string(),
4780 )));
4781 }
4782 for (name, value) in [
4783 (
4784 "execution_timeout_seconds",
4785 self.options.execution_timeout_seconds,
4786 ),
4787 ("run_timeout_seconds", self.options.run_timeout_seconds),
4788 ] {
4789 if value == Some(0) {
4790 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(format!(
4791 "{name} must be at least 1"
4792 ))));
4793 }
4794 }
4795
4796 let args = match self.args.take().unwrap_or(Ok(Value::Null)) {
4797 Ok(args) => args,
4798 Err(error) => return Poll::Ready(Err(error)),
4799 };
4800 let arguments =
4801 match encode_value_envelope(&normalize_arguments(args), &state.payload_codec) {
4802 Ok(arguments) => arguments,
4803 Err(error) => return Poll::Ready(Err(error)),
4804 };
4805 let mut command = json!({
4806 "type": "start_child_workflow",
4807 "workflow_type": self.workflow_type,
4808 "queue": self.options.task_queue,
4809 "parent_close_policy": self.options.parent_close_policy.as_str(),
4810 "arguments": arguments,
4811 });
4812 let object = command
4813 .as_object_mut()
4814 .expect("child workflow command is always an object");
4815 if let Some(policy) = &self.options.retry_policy {
4816 let mut retry_policy = serde_json::Map::new();
4817 if let Some(max_attempts) = policy.max_attempts {
4818 if max_attempts == 0 {
4819 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
4820 "retry_policy.max_attempts must be at least 1".to_string(),
4821 )));
4822 }
4823 retry_policy.insert("max_attempts".to_string(), json!(max_attempts));
4824 }
4825 if !policy.backoff_seconds.is_empty() {
4826 retry_policy
4827 .insert("backoff_seconds".to_string(), json!(policy.backoff_seconds));
4828 }
4829 if !policy.non_retryable_error_types.is_empty() {
4830 retry_policy.insert(
4831 "non_retryable_error_types".to_string(),
4832 json!(policy.non_retryable_error_types),
4833 );
4834 }
4835 if retry_policy.is_empty() {
4836 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
4837 "retry_policy must configure at least one field".to_string(),
4838 )));
4839 }
4840 object.insert("retry_policy".to_string(), Value::Object(retry_policy));
4841 }
4842 if let Some(seconds) = self.options.execution_timeout_seconds {
4843 object.insert("execution_timeout_seconds".to_string(), json!(seconds));
4844 }
4845 if let Some(seconds) = self.options.run_timeout_seconds {
4846 object.insert("run_timeout_seconds".to_string(), json!(seconds));
4847 }
4848 state.commands.push(command);
4849 self.scheduled = true;
4850 }
4851
4852 Poll::Pending
4853 }
4854}
4855
4856fn command_mismatch(recorded: &RecordedCommand, actual: impl Into<String>) -> Error {
4857 Error::NonDeterministicReplay(ReplayFailure::new(
4858 "recorded_command_mismatch",
4859 Some(recorded.sequence()),
4860 Some(recorded.shape().to_string()),
4861 Some(actual.into()),
4862 "current workflow command does not match the recorded durable command sequence",
4863 ))
4864}
4865
4866pub struct SignalCall {
4867 ctx: WorkflowContext,
4868 signal_name: String,
4869 opened_wait: bool,
4870 matched_pending: bool,
4871}
4872
4873impl Future for SignalCall {
4874 type Output = Result<Vec<Value>>;
4875
4876 fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
4877 if self.matched_pending {
4878 return Poll::Pending;
4879 }
4880
4881 let ctx = self.ctx.clone();
4882 let mut state = match ctx.state.lock() {
4883 Ok(state) => state,
4884 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
4885 };
4886
4887 let signals = match signal_values(
4888 &state.history,
4889 &self.signal_name,
4890 &state.payload_codec,
4891 state.resume_signal.as_ref(),
4892 ) {
4893 Ok(signals) => signals,
4894 Err(error) => return Poll::Ready(Err(error)),
4895 };
4896 let cursor = *state.signal_cursors.get(&self.signal_name).unwrap_or(&0);
4897
4898 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
4899 match recorded {
4900 RecordedCommand::SignalWait {
4901 sequence,
4902 signal_name,
4903 } => {
4904 if let Some(recorded_name) = signal_name {
4905 if recorded_name != self.signal_name {
4906 return Poll::Ready(Err(Error::NonDeterministicReplay(
4907 ReplayFailure::new(
4908 "recorded_command_detail_mismatch",
4909 Some(sequence),
4910 Some(format!("signal wait:{recorded_name}")),
4911 Some(format!("signal wait:{}", self.signal_name)),
4912 "recorded signal name differs from the current workflow command",
4913 ),
4914 )));
4915 }
4916 }
4917
4918 state.command_cursor += 1;
4919 if cursor < signals.len() {
4920 state
4921 .signal_cursors
4922 .insert(self.signal_name.clone(), cursor + 1);
4923 return Poll::Ready(Ok(signals[cursor].clone()));
4924 }
4925
4926 state.matched_recorded_pending = true;
4927 self.opened_wait = true;
4928 self.matched_pending = true;
4929 return Poll::Pending;
4930 }
4931 other => {
4932 return Poll::Ready(Err(command_mismatch(
4933 &other,
4934 format!("signal wait:{}", self.signal_name),
4935 )));
4936 }
4937 }
4938 }
4939
4940 if cursor < signals.len() {
4941 state
4942 .signal_cursors
4943 .insert(self.signal_name.clone(), cursor + 1);
4944 return Poll::Ready(Ok(signals[cursor].clone()));
4945 }
4946
4947 if !self.opened_wait {
4948 state.commands.push(json!({
4949 "type": "open_condition_wait",
4950 "condition_key": format!("signal:{}", self.signal_name)
4951 }));
4952 self.opened_wait = true;
4953 }
4954
4955 Poll::Pending
4956 }
4957}
4958
4959#[derive(Clone, Debug)]
4960pub struct ActivityContext {
4961 client: Client,
4962 pub task_id: String,
4963 pub activity_attempt_id: String,
4964 pub lease_owner: String,
4965 pub activity_type: String,
4966 pub attempt_number: u64,
4967 pub task_queue: String,
4968 pub worker_id: String,
4969}
4970
4971impl ActivityContext {
4972 pub async fn heartbeat<T: Serialize>(&self, details: T) -> Result<ActivityHeartbeatResponse> {
4973 self.client
4974 .heartbeat_activity_task(
4975 &self.task_id,
4976 &self.activity_attempt_id,
4977 &self.lease_owner,
4978 serde_json::to_value(details)?,
4979 )
4980 .await
4981 }
4982}
4983
4984fn decode_task_arguments(value: Option<&Value>, codec: &str) -> Result<Value> {
4985 match value {
4986 Some(value) => Ok(normalize_arguments(decode_wire_value(value, codec)?)),
4987 None => Ok(Value::Array(Vec::new())),
4988 }
4989}
4990
4991fn decode_resume_signal(task: &WorkflowTask) -> Result<Option<ResumeSignal>> {
4992 let Some(signal_name) = task
4993 .signal_name
4994 .as_deref()
4995 .filter(|value| !value.is_empty())
4996 else {
4997 return Ok(None);
4998 };
4999 let Some(arguments) = task.signal_arguments.as_ref() else {
5000 return Ok(None);
5001 };
5002
5003 let decoded = normalize_arguments(decode_wire_value(arguments, &task.payload_codec)?);
5004 let Value::Array(arguments) = decoded else {
5005 unreachable!("normalize_arguments always returns an array");
5006 };
5007
5008 Ok(Some(ResumeSignal {
5009 signal_id: task.workflow_signal_id.clone(),
5010 signal_name: signal_name.to_string(),
5011 arguments,
5012 }))
5013}
5014
5015fn normalize_arguments(value: Value) -> Value {
5016 match value {
5017 Value::Null => Value::Array(Vec::new()),
5018 Value::Array(_) => value,
5019 other => Value::Array(vec![other]),
5020 }
5021}
5022
5023fn recorded_commands(
5024 events: &[HistoryEvent],
5025 fallback_codec: &str,
5026 parent: WorkflowIdentity,
5027) -> Result<Vec<RecordedCommand>> {
5028 let mut events_by_sequence: BTreeMap<u64, Vec<&HistoryEvent>> = BTreeMap::new();
5029
5030 for event in events {
5031 let is_activity = matches!(
5032 event.event_type.as_str(),
5033 "ActivityScheduled"
5034 | "ActivityStarted"
5035 | "ActivityHeartbeatRecorded"
5036 | "ActivityRetryScheduled"
5037 | "ActivityCompleted"
5038 | "ActivityFailed"
5039 | "ActivityCancelled"
5040 | "ActivityTimedOut"
5041 );
5042 let is_workflow_timer = matches!(
5043 event.event_type.as_str(),
5044 "TimerScheduled" | "TimerCancelled" | "TimerFired"
5045 ) && !is_internal_timer_event(event);
5046 let is_child_workflow = matches!(
5047 event.event_type.as_str(),
5048 "ChildWorkflowScheduled"
5049 | "ChildRunCompleted"
5050 | "ChildRunFailed"
5051 | "ChildRunCancelled"
5052 | "ChildRunTerminated"
5053 );
5054 let is_signal_wait = is_recorded_signal_wait_event(event);
5055 if !is_activity && !is_workflow_timer && !is_child_workflow && !is_signal_wait {
5056 continue;
5057 }
5058
5059 let sequence = durable_event_sequence(event).ok_or_else(|| {
5060 Error::NonDeterministicReplay(ReplayFailure::new(
5061 "durable_command_sequence_missing",
5062 None,
5063 Some("positive workflow sequence".to_string()),
5064 Some(event.event_type.clone()),
5065 "durable command history event has no workflow sequence",
5066 ))
5067 })?;
5068 if sequence == 0 {
5069 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5070 "durable_command_sequence_invalid",
5071 Some(sequence),
5072 Some("positive workflow sequence".to_string()),
5073 Some(sequence.to_string()),
5074 "durable command history uses an invalid workflow sequence",
5075 )));
5076 }
5077 events_by_sequence.entry(sequence).or_default().push(event);
5078 }
5079
5080 events_by_sequence
5081 .into_iter()
5082 .map(|(sequence, sequence_events)| {
5083 let activity_events: Vec<_> = sequence_events
5084 .iter()
5085 .copied()
5086 .filter(|event| event.event_type.starts_with("Activity"))
5087 .collect();
5088 let timer_events: Vec<_> = sequence_events
5089 .iter()
5090 .copied()
5091 .filter(|event| event.event_type.starts_with("Timer"))
5092 .collect();
5093 let child_events: Vec<_> = sequence_events
5094 .iter()
5095 .copied()
5096 .filter(|event| {
5097 event.event_type == "ChildWorkflowScheduled"
5098 || event.event_type.starts_with("ChildRun")
5099 })
5100 .collect();
5101 let signal_wait_events: Vec<_> = sequence_events
5102 .iter()
5103 .copied()
5104 .filter(|event| is_recorded_signal_wait_event(event))
5105 .collect();
5106
5107 let command_kind_count = usize::from(!activity_events.is_empty())
5108 + usize::from(!timer_events.is_empty())
5109 + usize::from(!child_events.is_empty())
5110 + usize::from(!signal_wait_events.is_empty());
5111 if command_kind_count > 1 {
5112 let actual = [
5113 (!activity_events.is_empty()).then_some("activity"),
5114 (!timer_events.is_empty()).then_some("timer"),
5115 (!child_events.is_empty()).then_some("child workflow"),
5116 (!signal_wait_events.is_empty()).then_some("signal wait"),
5117 ]
5118 .into_iter()
5119 .flatten()
5120 .collect::<Vec<_>>()
5121 .join(" and ");
5122 return Err(invalid_recorded_history(
5123 "durable_command_sequence_collision",
5124 sequence,
5125 "one durable command kind",
5126 &actual,
5127 "one workflow sequence records more than one durable command kind",
5128 ));
5129 }
5130
5131 if !activity_events.is_empty() {
5132 let scheduled_count = activity_events
5133 .iter()
5134 .filter(|event| event.event_type == "ActivityScheduled")
5135 .count();
5136 if scheduled_count > 1 {
5137 return Err(invalid_recorded_history(
5138 "duplicate_activity_schedule",
5139 sequence,
5140 "at most one ActivityScheduled event",
5141 "multiple ActivityScheduled events",
5142 "activity history schedules more than one command at one workflow sequence",
5143 ));
5144 }
5145 let activity_type = activity_events.iter().find_map(|event| {
5146 event
5147 .payload
5148 .get("activity_type")
5149 .or_else(|| event.payload.get("activity_name"))
5150 .and_then(Value::as_str)
5151 .map(str::to_string)
5152 });
5153 if activity_events.iter().filter_map(|event| {
5154 event
5155 .payload
5156 .get("activity_type")
5157 .or_else(|| event.payload.get("activity_name"))
5158 .and_then(Value::as_str)
5159 }).any(|candidate| Some(candidate) != activity_type.as_deref()) {
5160 return Err(invalid_recorded_history(
5161 "activity_identity_mismatch",
5162 sequence,
5163 activity_type.as_deref().unwrap_or("one activity identity"),
5164 "conflicting activity identities",
5165 "activity lifecycle events at one workflow sequence disagree on identity",
5166 ));
5167 }
5168 let terminal: Vec<_> = activity_events
5169 .iter()
5170 .copied()
5171 .filter(|event| {
5172 matches!(
5173 event.event_type.as_str(),
5174 "ActivityCompleted"
5175 | "ActivityFailed"
5176 | "ActivityCancelled"
5177 | "ActivityTimedOut"
5178 )
5179 })
5180 .collect();
5181 if terminal.len() > 1 {
5182 return Err(invalid_recorded_history(
5183 "duplicate_activity_terminal_event",
5184 sequence,
5185 "at most one terminal activity event",
5186 "multiple terminal activity events",
5187 "activity history settles one command more than once",
5188 ));
5189 }
5190 let outcome = terminal
5191 .first()
5192 .map(|event| activity_outcome(event, fallback_codec, activity_type.clone()))
5193 .transpose()?;
5194 let options = activity_events
5195 .iter()
5196 .find(|event| event.event_type == "ActivityScheduled")
5197 .and_then(|event| event.payload.get("activity"))
5198 .and_then(Value::as_object)
5199 .map(|activity| RecordedActivityOptions {
5200 task_queue: recorded_optional_string(activity, "queue"),
5201 execution_mode: recorded_optional_string(activity, "execution_mode"),
5202 retry_policy: recorded_activity_retry_snapshot(
5203 activity.get("retry_policy"),
5204 ),
5205 });
5206 return Ok(RecordedCommand::Activity {
5207 sequence,
5208 activity_type,
5209 options,
5210 outcome,
5211 });
5212 }
5213
5214 if !child_events.is_empty() {
5215 let scheduled: Vec<_> = child_events
5216 .iter()
5217 .copied()
5218 .filter(|event| event.event_type == "ChildWorkflowScheduled")
5219 .collect();
5220 if scheduled.len() != 1 {
5221 return Err(invalid_recorded_history(
5222 "child_workflow_schedule_missing_or_duplicate",
5223 sequence,
5224 "one ChildWorkflowScheduled event",
5225 &format!("{} ChildWorkflowScheduled events", scheduled.len()),
5226 "child workflow replay requires exactly one recorded schedule event",
5227 ));
5228 }
5229 let workflow_type = child_events.iter().find_map(|event| {
5230 event
5231 .payload
5232 .get("child_workflow_type")
5233 .or_else(|| event.payload.get("workflow_type"))
5234 .and_then(Value::as_str)
5235 .filter(|value| !value.is_empty())
5236 .map(str::to_string)
5237 });
5238 if child_events
5239 .iter()
5240 .filter_map(|event| {
5241 event
5242 .payload
5243 .get("child_workflow_type")
5244 .or_else(|| event.payload.get("workflow_type"))
5245 .and_then(Value::as_str)
5246 })
5247 .any(|candidate| Some(candidate) != workflow_type.as_deref())
5248 {
5249 return Err(invalid_recorded_history(
5250 "child_workflow_identity_mismatch",
5251 sequence,
5252 workflow_type
5253 .as_deref()
5254 .unwrap_or("one child workflow type"),
5255 "conflicting child workflow types",
5256 "child workflow lifecycle events at one sequence disagree on type",
5257 ));
5258 }
5259 let mut outcomes = child_workflow_outcomes(
5260 &child_events.iter().map(|event| (*event).clone()).collect::<Vec<_>>(),
5261 fallback_codec,
5262 parent.clone(),
5263 )?;
5264 if outcomes.len() > 1 {
5265 return Err(invalid_recorded_history(
5266 "duplicate_child_workflow_terminal_event",
5267 sequence,
5268 "at most one terminal child event",
5269 "multiple terminal child events",
5270 "child workflow history settles one command more than once",
5271 ));
5272 }
5273 return Ok(RecordedCommand::ChildWorkflow {
5274 sequence,
5275 workflow_type,
5276 outcome: outcomes.pop(),
5277 });
5278 }
5279
5280 if !signal_wait_events.is_empty() {
5281 let opened_count = signal_wait_events
5282 .iter()
5283 .filter(|event| {
5284 matches!(
5285 event.event_type.as_str(),
5286 "SignalWaitOpened" | "ConditionWaitOpened"
5287 )
5288 })
5289 .count();
5290 if opened_count > 1 {
5291 return Err(invalid_recorded_history(
5292 "duplicate_signal_wait_open",
5293 sequence,
5294 "at most one signal wait open event",
5295 "multiple signal wait open events",
5296 "signal history opens more than one durable wait at one workflow sequence",
5297 ));
5298 }
5299
5300 let signal_name = signal_wait_events
5301 .iter()
5302 .find_map(|event| recorded_signal_wait_name(event));
5303 if signal_wait_events
5304 .iter()
5305 .filter_map(|event| recorded_signal_wait_name(event))
5306 .any(|candidate| Some(candidate.as_str()) != signal_name.as_deref())
5307 {
5308 return Err(invalid_recorded_history(
5309 "signal_wait_identity_mismatch",
5310 sequence,
5311 signal_name.as_deref().unwrap_or("one signal name"),
5312 "conflicting signal names",
5313 "signal wait lifecycle events at one workflow sequence disagree on identity",
5314 ));
5315 }
5316 return Ok(RecordedCommand::SignalWait {
5317 sequence,
5318 signal_name,
5319 });
5320 }
5321
5322 let scheduled: Vec<_> = timer_events
5323 .iter()
5324 .copied()
5325 .filter(|event| event.event_type == "TimerScheduled")
5326 .collect();
5327 let fired: Vec<_> = timer_events
5328 .iter()
5329 .copied()
5330 .filter(|event| event.event_type == "TimerFired")
5331 .collect();
5332 if scheduled.len() != 1 {
5333 return Err(invalid_recorded_history(
5334 "timer_schedule_missing_or_duplicate",
5335 sequence,
5336 "one TimerScheduled event",
5337 &format!("{} TimerScheduled events", scheduled.len()),
5338 "timer replay requires exactly one recorded schedule event",
5339 ));
5340 }
5341 if fired.len() > 1 {
5342 return Err(invalid_recorded_history(
5343 "duplicate_timer_fire",
5344 sequence,
5345 "at most one TimerFired event",
5346 "multiple TimerFired events",
5347 "timer history contains more than one fire event for a workflow sequence",
5348 ));
5349 }
5350
5351 let scheduled = scheduled[0];
5352 let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
5353 let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
5354 if let Some(fired) = fired.first() {
5355 let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
5356 if fired_timer_id != timer_id {
5357 return Err(invalid_recorded_history(
5358 "timer_identity_mismatch",
5359 sequence,
5360 &timer_id,
5361 &fired_timer_id,
5362 "TimerFired does not correspond to the recorded TimerScheduled event",
5363 ));
5364 }
5365 let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
5366 if fired_delay != delay_seconds {
5367 return Err(invalid_recorded_history(
5368 "timer_history_delay_mismatch",
5369 sequence,
5370 &delay_seconds.to_string(),
5371 &fired_delay.to_string(),
5372 "TimerScheduled and TimerFired record different delays",
5373 ));
5374 }
5375 }
5376
5377 Ok(RecordedCommand::Timer {
5378 sequence,
5379 delay_seconds,
5380 fired: !fired.is_empty(),
5381 })
5382 })
5383 .collect()
5384}
5385
5386fn durable_event_sequence(event: &HistoryEvent) -> Option<u64> {
5387 event
5388 .payload
5389 .get("sequence")
5390 .or_else(|| event.payload.get("workflow_sequence"))
5391 .or_else(|| event.raw.get("sequence"))
5392 .or_else(|| event.raw.get("workflow_sequence"))
5393 .and_then(value_as_u64)
5394}
5395
5396fn is_internal_timer_event(event: &HistoryEvent) -> bool {
5397 matches!(
5398 event
5399 .payload
5400 .get("timer_kind")
5401 .or_else(|| event.raw.get("timer_kind"))
5402 .and_then(Value::as_str),
5403 Some("condition_timeout" | "signal_timeout")
5404 )
5405}
5406
5407fn recorded_signal_wait_name(event: &HistoryEvent) -> Option<String> {
5408 match event.event_type.as_str() {
5409 "SignalWaitOpened" | "SignalApplied" => event
5410 .payload
5411 .get("signal_name")
5412 .or_else(|| event.raw.get("signal_name"))
5413 .and_then(Value::as_str)
5414 .filter(|value| !value.is_empty())
5415 .map(str::to_string),
5416 "ConditionWaitOpened" | "ConditionWaitSatisfied" | "ConditionWaitTimedOut" => event
5417 .payload
5418 .get("condition_key")
5419 .or_else(|| event.raw.get("condition_key"))
5420 .and_then(Value::as_str)
5421 .and_then(|key| key.strip_prefix("signal:"))
5422 .filter(|value| !value.is_empty())
5423 .map(str::to_string),
5424 _ => None,
5425 }
5426}
5427
5428fn is_recorded_signal_wait_event(event: &HistoryEvent) -> bool {
5429 match event.event_type.as_str() {
5430 "SignalWaitOpened" | "SignalApplied" => true,
5431 "ConditionWaitOpened" | "ConditionWaitSatisfied" | "ConditionWaitTimedOut" => event
5432 .payload
5433 .get("condition_key")
5434 .or_else(|| event.raw.get("condition_key"))
5435 .and_then(Value::as_str)
5436 .is_some_and(|key| key.starts_with("signal:")),
5437 _ => false,
5438 }
5439}
5440
5441fn required_history_string(event: &HistoryEvent, field: &str, sequence: u64) -> Result<String> {
5442 event
5443 .payload
5444 .get(field)
5445 .and_then(Value::as_str)
5446 .filter(|value| !value.is_empty())
5447 .map(str::to_string)
5448 .ok_or_else(|| {
5449 invalid_recorded_history(
5450 "timer_history_field_missing",
5451 sequence,
5452 field,
5453 &event.event_type,
5454 "timer history is missing a required identity field",
5455 )
5456 })
5457}
5458
5459fn required_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<u64> {
5460 event
5461 .payload
5462 .get(field)
5463 .and_then(value_as_u64)
5464 .ok_or_else(|| {
5465 invalid_recorded_history(
5466 "timer_history_field_missing",
5467 sequence,
5468 field,
5469 &event.event_type,
5470 "timer history is missing a required numeric field",
5471 )
5472 })
5473}
5474
5475fn invalid_recorded_history(
5476 reason: &str,
5477 sequence: u64,
5478 expected: &str,
5479 actual: &str,
5480 message: &str,
5481) -> Error {
5482 Error::NonDeterministicReplay(ReplayFailure::new(
5483 reason,
5484 Some(sequence),
5485 Some(expected.to_string()),
5486 Some(actual.to_string()),
5487 message,
5488 ))
5489}
5490
5491type ActivityOutcome = std::result::Result<Value, ActivityFailure>;
5492
5493fn activity_outcome(
5494 event: &HistoryEvent,
5495 fallback_codec: &str,
5496 recorded_activity_type: Option<String>,
5497) -> Result<ActivityOutcome> {
5498 if event.event_type == "ActivityCompleted" {
5499 let codec = event
5500 .payload
5501 .get("payload_codec")
5502 .and_then(Value::as_str)
5503 .unwrap_or(fallback_codec);
5504 return Ok(Ok(decode_wire_value(
5505 event.payload.get("result").unwrap_or(&Value::Null),
5506 codec,
5507 )?));
5508 }
5509
5510 let payload = &event.payload;
5511 let (kind, fallback_reason, fallback_message) = match event.event_type.as_str() {
5512 "ActivityFailed" => (ActivityFailureKind::Failed, "activity", "activity failed"),
5513 "ActivityCancelled" => (
5514 ActivityFailureKind::Cancelled,
5515 "cancelled",
5516 "activity was cancelled",
5517 ),
5518 "ActivityTimedOut" => (
5519 ActivityFailureKind::TimedOut,
5520 "timeout",
5521 "activity timed out",
5522 ),
5523 _ => unreachable!("activity_outcome is called only for terminal activity events"),
5524 };
5525 let exception = payload
5526 .get("exception")
5527 .filter(|value| !value.is_null())
5528 .cloned();
5529 let failure_category = payload_string(payload, "failure_category");
5530 let timeout_kind = payload_string(payload, "timeout_kind");
5531 let reason = payload_string(payload, "reason").unwrap_or_else(|| match kind {
5532 ActivityFailureKind::Failed => failure_category
5533 .clone()
5534 .unwrap_or_else(|| fallback_reason.to_string()),
5535 ActivityFailureKind::Cancelled => fallback_reason.to_string(),
5536 ActivityFailureKind::TimedOut => timeout_kind
5537 .clone()
5538 .unwrap_or_else(|| fallback_reason.to_string()),
5539 });
5540 let message = payload_string(payload, "message")
5541 .or_else(|| {
5542 exception
5543 .as_ref()
5544 .and_then(|value| payload_string(value, "message"))
5545 })
5546 .unwrap_or_else(|| fallback_message.to_string());
5547
5548 Ok(Err(ActivityFailure {
5549 kind,
5550 reason,
5551 message,
5552 activity_execution_id: payload_string(payload, "activity_execution_id"),
5553 activity_attempt_id: payload_string(payload, "activity_attempt_id"),
5554 activity_type: payload_string(payload, "activity_type")
5555 .or_else(|| payload_string(payload, "activity_name"))
5556 .or(recorded_activity_type),
5557 activity_class: payload_string(payload, "activity_class"),
5558 attempt_number: payload.get("attempt_number").and_then(value_as_u64),
5559 failure_id: payload_string(payload, "failure_id"),
5560 failure_category,
5561 timeout_kind,
5562 non_retryable: payload
5563 .get("non_retryable")
5564 .and_then(Value::as_bool)
5565 .unwrap_or(false),
5566 exception_type: payload_string(payload, "exception_type").or_else(|| {
5567 exception
5568 .as_ref()
5569 .and_then(|value| payload_string(value, "type"))
5570 }),
5571 exception_class: payload_string(payload, "exception_class").or_else(|| {
5572 exception
5573 .as_ref()
5574 .and_then(|value| payload_string(value, "class"))
5575 }),
5576 code: payload
5577 .get("code")
5578 .filter(|value| !value.is_null())
5579 .cloned(),
5580 exception,
5581 }))
5582}
5583
5584type ChildWorkflowOutcome = std::result::Result<ChildWorkflowResult, ChildWorkflowFailure>;
5585
5586fn child_workflow_outcomes(
5587 events: &[HistoryEvent],
5588 fallback_codec: &str,
5589 parent: WorkflowIdentity,
5590) -> Result<Vec<ChildWorkflowOutcome>> {
5591 let mut outcomes = Vec::new();
5592
5593 for event in events {
5594 let kind = match event.event_type.as_str() {
5595 "ChildRunCompleted" => None,
5596 "ChildRunFailed" => Some((
5597 ChildWorkflowFailureKind::Failed,
5598 "child_workflow",
5599 "child workflow failed",
5600 )),
5601 "ChildRunCancelled" => Some((
5602 ChildWorkflowFailureKind::Cancelled,
5603 "cancelled",
5604 "child workflow was cancelled",
5605 )),
5606 "ChildRunTerminated" => Some((
5607 ChildWorkflowFailureKind::Terminated,
5608 "terminated",
5609 "child workflow was terminated",
5610 )),
5611 _ => continue,
5612 };
5613 let payload = &event.payload;
5614 let child_workflow_id = payload_string(payload, "child_workflow_instance_id");
5615 let child_workflow_run_id = payload_string(payload, "child_workflow_run_id");
5616 let child_workflow_type = payload_string(payload, "child_workflow_type");
5617
5618 if let Some((kind, reason, fallback_message)) = kind {
5619 let exception = payload
5620 .get("exception")
5621 .filter(|value| !value.is_null())
5622 .cloned();
5623 let message = payload_string(payload, "message")
5624 .or_else(|| {
5625 exception
5626 .as_ref()
5627 .and_then(|value| payload_string(value, "message"))
5628 })
5629 .unwrap_or_else(|| fallback_message.to_string());
5630 let exception_type = payload_string(payload, "exception_type").or_else(|| {
5631 exception
5632 .as_ref()
5633 .and_then(|value| payload_string(value, "type"))
5634 });
5635 let exception_class = payload_string(payload, "exception_class").or_else(|| {
5636 exception
5637 .as_ref()
5638 .and_then(|value| payload_string(value, "class"))
5639 });
5640 outcomes.push(Err(ChildWorkflowFailure {
5641 kind,
5642 reason: reason.to_string(),
5643 message,
5644 parent_workflow_id: parent.workflow_id.clone(),
5645 parent_workflow_run_id: parent.run_id.clone(),
5646 child_workflow_id,
5647 child_workflow_run_id,
5648 child_workflow_type,
5649 failure_id: payload_string(payload, "failure_id"),
5650 failure_category: payload_string(payload, "failure_category"),
5651 exception_type,
5652 exception_class,
5653 non_retryable: payload
5654 .get("non_retryable")
5655 .and_then(Value::as_bool)
5656 .unwrap_or(false),
5657 code: payload
5658 .get("code")
5659 .filter(|value| !value.is_null())
5660 .cloned(),
5661 exception,
5662 }));
5663 continue;
5664 }
5665
5666 let codec = payload
5667 .get("payload_codec")
5668 .and_then(Value::as_str)
5669 .unwrap_or(fallback_codec);
5670 let result = payload
5671 .get("result")
5672 .or_else(|| payload.get("output"))
5673 .unwrap_or(&Value::Null);
5674 outcomes.push(Ok(ChildWorkflowResult {
5675 parent: parent.clone(),
5676 child: WorkflowIdentity {
5677 workflow_id: child_workflow_id,
5678 run_id: child_workflow_run_id,
5679 },
5680 child_workflow_type,
5681 result: decode_wire_value(result, codec)?,
5682 }));
5683 }
5684
5685 Ok(outcomes)
5686}
5687
5688fn payload_string(payload: &Value, key: &str) -> Option<String> {
5689 payload
5690 .get(key)
5691 .and_then(Value::as_str)
5692 .filter(|value| !value.is_empty())
5693 .map(str::to_string)
5694}
5695
5696fn workflow_failure_command(error: &Error) -> Value {
5697 let (exception_type, exception_class, properties) = match error {
5698 Error::ActivityFailed(failure) => (
5699 match failure.kind {
5700 ActivityFailureKind::Failed => "ActivityFailed",
5701 ActivityFailureKind::Cancelled => "ActivityCancelled",
5702 ActivityFailureKind::TimedOut => "ActivityTimedOut",
5703 },
5704 "durable_workflow::ActivityFailure",
5705 json!({
5706 "reason": failure.reason,
5707 "activity_execution_id": failure.activity_execution_id,
5708 "activity_attempt_id": failure.activity_attempt_id,
5709 "activity_type": failure.activity_type,
5710 "activity_class": failure.activity_class,
5711 "attempt_number": failure.attempt_number,
5712 "failure_id": failure.failure_id,
5713 "failure_category": failure.failure_category,
5714 "timeout_kind": failure.timeout_kind,
5715 "activity_non_retryable": failure.non_retryable,
5716 "activity_exception_type": failure.exception_type,
5717 "activity_exception_class": failure.exception_class,
5718 "activity_code": failure.code,
5719 "activity_exception": failure.exception,
5720 }),
5721 ),
5722 Error::ChildWorkflowFailed(failure) => (
5723 match failure.kind {
5724 ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
5725 ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
5726 ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
5727 },
5728 "durable_workflow::ChildWorkflowFailure",
5729 json!({
5730 "reason": failure.reason,
5731 "parent_workflow_id": failure.parent_workflow_id,
5732 "parent_workflow_run_id": failure.parent_workflow_run_id,
5733 "child_workflow_id": failure.child_workflow_id,
5734 "child_workflow_run_id": failure.child_workflow_run_id,
5735 "child_workflow_type": failure.child_workflow_type,
5736 "failure_id": failure.failure_id,
5737 "failure_category": failure.failure_category,
5738 "child_exception_type": failure.exception_type,
5739 "child_exception_class": failure.exception_class,
5740 "child_non_retryable": failure.non_retryable,
5741 "child_code": failure.code,
5742 "child_exception": failure.exception,
5743 }),
5744 ),
5745 Error::NonDeterministicReplay(_) => (
5746 "NonDeterministicReplay",
5747 "durable_workflow::Error",
5748 Value::Null,
5749 ),
5750 _ => ("RustWorkflowError", "durable_workflow::Error", Value::Null),
5751 };
5752 let non_retryable = match error {
5753 Error::ActivityFailed(failure) => failure.non_retryable,
5754 Error::ChildWorkflowFailed(failure) => failure.non_retryable,
5755 Error::NonDeterministicReplay(_) => true,
5756 _ => false,
5757 };
5758
5759 json!({
5760 "type": "fail_workflow",
5761 "message": error.to_string(),
5762 "exception_type": exception_type,
5763 "exception_class": exception_class,
5764 "non_retryable": non_retryable,
5765 "exception": {
5766 "type": exception_type,
5767 "class": exception_class,
5768 "message": error.to_string(),
5769 "properties": properties,
5770 }
5771 })
5772}
5773
5774fn signal_values(
5775 events: &[HistoryEvent],
5776 signal_name: &str,
5777 fallback_codec: &str,
5778 resume_signal: Option<&ResumeSignal>,
5779) -> Result<Vec<Vec<Value>>> {
5780 let mut signals = Vec::new();
5781
5782 for event in events {
5783 if event.event_type != "SignalApplied" && event.event_type != "SignalReceived" {
5784 continue;
5785 }
5786
5787 if event.payload.get("signal_name").and_then(Value::as_str) != Some(signal_name) {
5788 continue;
5789 }
5790
5791 let codec = event
5792 .payload
5793 .get("payload_codec")
5794 .and_then(Value::as_str)
5795 .unwrap_or(fallback_codec);
5796 let raw = event
5797 .payload
5798 .get("value")
5799 .or_else(|| event.payload.get("input"))
5800 .or_else(|| event.payload.get("arguments"));
5801 let decoded = match raw.filter(|value| !value.is_null()) {
5802 Some(value) => decode_wire_value(value, codec)?,
5803 None => resume_signal
5804 .filter(|signal| resume_signal_matches_event(signal, event, signal_name))
5805 .map(|signal| Value::Array(signal.arguments.clone()))
5806 .unwrap_or_else(|| Value::Array(Vec::new())),
5807 };
5808 let args = match normalize_arguments(decoded) {
5809 Value::Array(values) => values,
5810 _ => unreachable!("normalize_arguments always returns an array"),
5811 };
5812 signals.push(args);
5813 }
5814
5815 Ok(signals)
5816}
5817
5818fn hydrate_query_history_from_export(task: &mut QueryTask) -> Result<()> {
5819 let Some(export_events) = task
5820 .history_export
5821 .as_ref()
5822 .and_then(|export| export.get("history_events"))
5823 .and_then(Value::as_array)
5824 else {
5825 return Ok(());
5826 };
5827
5828 if export_events.len() > task.history_events.len() {
5829 task.history_events = serde_json::from_value(Value::Array(export_events.clone()))?;
5830 }
5831
5832 Ok(())
5833}
5834
5835fn enrich_query_history_from_export(task: &mut QueryTask) -> Result<()> {
5836 let Some(export) = task.history_export.as_ref() else {
5837 return Ok(());
5838 };
5839 let signals = export
5840 .get("signals")
5841 .and_then(Value::as_array)
5842 .cloned()
5843 .unwrap_or_default();
5844 let activities = export
5845 .get("activities")
5846 .and_then(Value::as_array)
5847 .cloned()
5848 .unwrap_or_default();
5849 let export_codec = export
5850 .get("payloads")
5851 .and_then(|payloads| payloads.get("codec"))
5852 .and_then(Value::as_str)
5853 .unwrap_or(&task.payload_codec)
5854 .to_string();
5855 let mut signal_name_offsets: HashMap<String, usize> = HashMap::new();
5856
5857 for event in &mut task.history_events {
5858 if event.event_type == "ActivityCompleted" {
5859 let sequence = event
5860 .payload
5861 .get("sequence")
5862 .or_else(|| event.payload.get("workflow_sequence"))
5863 .and_then(value_as_u64);
5864 let Some(activity) = sequence.and_then(|sequence| {
5865 activities.iter().find(|activity| {
5866 activity.get("sequence").and_then(value_as_u64) == Some(sequence)
5867 })
5868 }) else {
5869 continue;
5870 };
5871 let Some(payload) = event.payload.as_object_mut() else {
5872 continue;
5873 };
5874 if missing_payload(payload.get("result")) {
5875 if let Some(result) = activity
5876 .get("result")
5877 .filter(|value| !missing_payload(Some(value)))
5878 {
5879 payload.insert("result".to_string(), result.clone());
5880 }
5881 }
5882 for field in ["payload_codec", "activity_type"] {
5883 if payload
5884 .get(field)
5885 .and_then(Value::as_str)
5886 .unwrap_or_default()
5887 .is_empty()
5888 {
5889 if let Some(value) = activity.get(field) {
5890 payload.insert(field.to_string(), value.clone());
5891 }
5892 }
5893 }
5894 continue;
5895 }
5896
5897 if event.event_type != "SignalReceived" && event.event_type != "SignalApplied" {
5898 continue;
5899 }
5900 let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
5901 let command_id = event
5902 .payload
5903 .get("workflow_command_id")
5904 .or_else(|| event.raw.get("workflow_command_id"))
5905 .and_then(Value::as_str);
5906 let signal_name = event
5907 .payload
5908 .get("signal_name")
5909 .and_then(Value::as_str)
5910 .unwrap_or_default()
5911 .to_string();
5912 let matched = signals
5913 .iter()
5914 .find(|signal| {
5915 signal_id.is_some() && signal.get("id").and_then(Value::as_str) == signal_id
5916 })
5917 .or_else(|| {
5918 signals.iter().find(|signal| {
5919 command_id.is_some()
5920 && signal.get("command_id").and_then(Value::as_str) == command_id
5921 })
5922 })
5923 .or_else(|| {
5924 let offset = signal_name_offsets.entry(signal_name.clone()).or_default();
5925 let signal = signals
5926 .iter()
5927 .filter(|signal| {
5928 signal.get("name").and_then(Value::as_str) == Some(signal_name.as_str())
5929 })
5930 .nth(*offset);
5931 if signal.is_some() {
5932 *offset += 1;
5933 }
5934 signal
5935 });
5936 let Some(signal) = matched else {
5937 continue;
5938 };
5939 let signal_codec = signal
5940 .get("payload_codec")
5941 .and_then(Value::as_str)
5942 .unwrap_or(&export_codec);
5943 let Some(payload) = event.payload.as_object_mut() else {
5944 continue;
5945 };
5946 if missing_payload(payload.get("arguments")) {
5947 if let Some(arguments) = signal
5948 .get("arguments")
5949 .filter(|value| !missing_payload(Some(value)))
5950 {
5951 let envelope = match arguments {
5952 Value::String(blob) => json!({"codec": signal_codec, "blob": blob}),
5953 other => other.clone(),
5954 };
5955 payload.insert("arguments".to_string(), envelope);
5956 }
5957 }
5958 if payload
5959 .get("payload_codec")
5960 .and_then(Value::as_str)
5961 .unwrap_or_default()
5962 .is_empty()
5963 {
5964 payload.insert("payload_codec".to_string(), json!(signal_codec));
5965 }
5966 }
5967
5968 Ok(())
5969}
5970
5971fn missing_payload(value: Option<&Value>) -> bool {
5972 match value {
5973 None | Some(Value::Null) => true,
5974 Some(Value::String(value)) => value.is_empty(),
5975 Some(_) => false,
5976 }
5977}
5978
5979fn query_signal_events(task: &QueryTask) -> Result<Vec<QuerySignal>> {
5980 let export_signals = task
5981 .history_export
5982 .as_ref()
5983 .and_then(|export| export.get("signals"))
5984 .and_then(Value::as_array)
5985 .cloned()
5986 .unwrap_or_default();
5987 let export_codec = task
5988 .history_export
5989 .as_ref()
5990 .and_then(|export| export.get("payloads"))
5991 .and_then(|payloads| payloads.get("codec"))
5992 .and_then(Value::as_str)
5993 .unwrap_or(&task.payload_codec);
5994 let mut name_offsets: HashMap<String, usize> = HashMap::new();
5995 let mut signals = Vec::new();
5996
5997 for event in &task.history_events {
5998 if event.event_type != "SignalApplied" && event.event_type != "SignalReceived" {
5999 continue;
6000 }
6001
6002 let name = event
6003 .payload
6004 .get("signal_name")
6005 .and_then(Value::as_str)
6006 .unwrap_or_default();
6007 if name.is_empty() {
6008 continue;
6009 }
6010 let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
6011 let command_id = event
6012 .payload
6013 .get("workflow_command_id")
6014 .or_else(|| event.raw.get("workflow_command_id"))
6015 .and_then(Value::as_str);
6016 let matched_export = export_signals
6017 .iter()
6018 .find(|candidate| {
6019 signal_id.is_some() && candidate.get("id").and_then(Value::as_str) == signal_id
6020 })
6021 .or_else(|| {
6022 export_signals.iter().find(|candidate| {
6023 command_id.is_some()
6024 && candidate.get("command_id").and_then(Value::as_str) == command_id
6025 })
6026 })
6027 .or_else(|| {
6028 let offset = name_offsets.entry(name.to_string()).or_default();
6029 let candidate = export_signals
6030 .iter()
6031 .filter(|candidate| candidate.get("name").and_then(Value::as_str) == Some(name))
6032 .nth(*offset);
6033 if candidate.is_some() {
6034 *offset += 1;
6035 }
6036 candidate
6037 });
6038 let codec = event
6039 .payload
6040 .get("payload_codec")
6041 .and_then(Value::as_str)
6042 .or_else(|| {
6043 matched_export
6044 .and_then(|signal| signal.get("payload_codec"))
6045 .and_then(Value::as_str)
6046 })
6047 .unwrap_or(export_codec);
6048 let raw_arguments = event
6049 .payload
6050 .get("value")
6051 .or_else(|| event.payload.get("input"))
6052 .or_else(|| event.payload.get("arguments"))
6053 .filter(|value| !value.is_null())
6054 .or_else(|| matched_export.and_then(|signal| signal.get("arguments")));
6055 let arguments = decode_query_signal_arguments(raw_arguments, codec)?;
6056 let workflow_sequence = event
6057 .payload
6058 .get("workflow_sequence")
6059 .and_then(value_as_u64)
6060 .or_else(|| {
6061 matched_export
6062 .and_then(|signal| signal.get("workflow_sequence"))
6063 .and_then(value_as_u64)
6064 });
6065
6066 signals.push(QuerySignal {
6067 id: signal_id.map(str::to_string).or_else(|| {
6068 matched_export
6069 .and_then(|signal| signal.get("id"))
6070 .and_then(Value::as_str)
6071 .map(str::to_string)
6072 }),
6073 name: name.to_string(),
6074 arguments,
6075 workflow_sequence,
6076 });
6077 }
6078
6079 if signals.is_empty() {
6080 for signal in export_signals {
6081 if signal.get("status").and_then(Value::as_str) == Some("rejected") {
6082 continue;
6083 }
6084 let Some(name) = signal.get("name").and_then(Value::as_str) else {
6085 continue;
6086 };
6087 let codec = signal
6088 .get("payload_codec")
6089 .and_then(Value::as_str)
6090 .unwrap_or(export_codec);
6091 let arguments = decode_query_signal_arguments(signal.get("arguments"), codec)?;
6092 signals.push(QuerySignal {
6093 id: signal.get("id").and_then(Value::as_str).map(str::to_string),
6094 name: name.to_string(),
6095 arguments,
6096 workflow_sequence: signal.get("workflow_sequence").and_then(value_as_u64),
6097 });
6098 }
6099 signals.sort_by_key(|signal| signal.workflow_sequence.unwrap_or(u64::MAX));
6100 }
6101
6102 Ok(signals)
6103}
6104
6105fn decode_query_signal_arguments(raw: Option<&Value>, codec: &str) -> Result<Vec<Value>> {
6106 let decoded = match raw.filter(|value| !value.is_null()) {
6107 Some(value) => decode_wire_value(value, codec)?,
6108 None => Value::Array(Vec::new()),
6109 };
6110 let Value::Array(arguments) = normalize_arguments(decoded) else {
6111 unreachable!("normalize_arguments always returns an array");
6112 };
6113 Ok(arguments)
6114}
6115
6116fn value_as_u64(value: &Value) -> Option<u64> {
6117 value
6118 .as_u64()
6119 .or_else(|| value.as_str().and_then(|value| value.parse().ok()))
6120}
6121
6122fn resume_signal_matches_event(
6123 resume_signal: &ResumeSignal,
6124 event: &HistoryEvent,
6125 signal_name: &str,
6126) -> bool {
6127 if resume_signal.signal_name != signal_name {
6128 return false;
6129 }
6130
6131 match (
6132 resume_signal.signal_id.as_deref(),
6133 event.payload.get("signal_id").and_then(Value::as_str),
6134 ) {
6135 (Some(resume_id), Some(event_id)) => resume_id == event_id,
6136 _ => true,
6137 }
6138}
6139
6140#[cfg(test)]
6141mod tests {
6142 use super::*;
6143 use std::{
6144 io::{Read, Write},
6145 net::{SocketAddr, TcpListener, TcpStream},
6146 thread,
6147 };
6148
6149 #[derive(Clone, Debug, Default, PartialEq)]
6150 struct ReplayCounterState {
6151 loaded: Option<String>,
6152 count: i64,
6153 finished: bool,
6154 }
6155
6156 fn replay_counter_worker() -> Worker {
6157 let client = Client::new("http://127.0.0.1:8080").expect("client");
6158 let mut worker = Worker::new(client, "rust-workers");
6159 worker.register_replayed_workflow(
6160 "replay-counter",
6161 ReplayCounterState::default,
6162 |ctx, _input, state| async move {
6163 let loaded = ctx.activity("load-counter", json!([])).await?;
6164 state.update(|current| {
6165 current.loaded = loaded.as_str().map(str::to_string);
6166 })?;
6167 for _ in 0..2 {
6168 let signal = ctx.wait_signal("increment").await?;
6169 let amount = signal.first().and_then(Value::as_i64).unwrap_or_default();
6170 state.update(|current| current.count += amount)?;
6171 }
6172 state.update(|current| current.finished = true)?;
6173 state.read(|current| Ok(json!(current.count)))?
6174 },
6175 );
6176 worker.register_replayed_query::<ReplayCounterState, _, _>(
6177 "replay-counter",
6178 "current",
6179 |_ctx, state, _args| async move {
6180 Ok(json!({
6181 "loaded": state.loaded,
6182 "count": state.count,
6183 "finished": state.finished,
6184 }))
6185 },
6186 );
6187 worker.register_replayed_query::<ReplayCounterState, _, _>(
6188 "replay-counter",
6189 "detached-mutation",
6190 |_ctx, state, _args| async move {
6191 let mut detached = (*state).clone();
6192 detached.count = 999;
6193 Ok(json!(detached.count))
6194 },
6195 );
6196 worker.register_replayed_query::<ReplayCounterState, _, _>(
6197 "replay-counter",
6198 "failed-mutation",
6199 |_ctx, state, _args| async move {
6200 let mut detached = (*state).clone();
6201 detached.count = 999;
6202 Err(Error::WorkerLoop("query refused".to_string()))
6203 },
6204 );
6205 worker
6206 }
6207
6208 fn replay_counter_query(
6209 query_name: &str,
6210 history_events: Value,
6211 run_status: &str,
6212 ) -> QueryTask {
6213 serde_json::from_value(json!({
6214 "query_task_id": format!("query-{query_name}"),
6215 "workflow_type": "replay-counter",
6216 "query_name": query_name,
6217 "payload_codec": "json",
6218 "workflow_arguments": {"codec": "json", "blob": "[]"},
6219 "query_arguments": {"codec": "json", "blob": "[]"},
6220 "history_events": history_events,
6221 "run_status": run_status,
6222 }))
6223 .expect("query task")
6224 }
6225
6226 fn workflow_context(history: Vec<HistoryEvent>) -> WorkflowContext {
6227 WorkflowContext {
6228 state: Arc::new(Mutex::new(
6229 WorkflowState::new_with_identity(
6230 history,
6231 None,
6232 None,
6233 "rust-workers".to_string(),
6234 JSON_CODEC.to_string(),
6235 None,
6236 )
6237 .expect("valid workflow history"),
6238 )),
6239 }
6240 }
6241
6242 fn history_event(event_type: &str, payload: Value) -> HistoryEvent {
6243 HistoryEvent {
6244 event_type: event_type.to_string(),
6245 payload,
6246 raw: HashMap::new(),
6247 }
6248 }
6249
6250 fn completed_retry_activity_history() -> Vec<HistoryEvent> {
6251 vec![
6252 history_event(
6253 "ActivityScheduled",
6254 json!({
6255 "sequence": 1,
6256 "activity_type": "flaky",
6257 "activity_execution_id": "act-1",
6258 "activity": {
6259 "id": "act-1",
6260 "sequence": 1,
6261 "type": "flaky",
6262 "queue": "critical-activities",
6263 "execution_mode": null,
6264 "retry_policy": {
6265 "snapshot_version": 1,
6266 "max_attempts": 3,
6267 "backoff_seconds": [2, 4],
6268 "start_to_close_timeout": 30,
6269 "schedule_to_start_timeout": 5,
6270 "schedule_to_close_timeout": 90,
6271 "heartbeat_timeout": 10,
6272 "non_retryable_error_types": ["PermanentError"]
6273 }
6274 }
6275 }),
6276 ),
6277 history_event(
6278 "ActivityStarted",
6279 json!({
6280 "sequence": 1,
6281 "activity_type": "flaky",
6282 "activity_execution_id": "act-1",
6283 "activity_attempt_id": "attempt-1",
6284 "attempt_number": 1
6285 }),
6286 ),
6287 history_event(
6288 "ActivityRetryScheduled",
6289 json!({
6290 "sequence": 1,
6291 "activity_type": "flaky",
6292 "activity_execution_id": "act-1",
6293 "activity_attempt_id": "attempt-1",
6294 "attempt_number": 1,
6295 "retry_after_attempt": 1,
6296 "retry_backoff_seconds": 2,
6297 "failure_category": "activity",
6298 "exception_type": "TransientError"
6299 }),
6300 ),
6301 history_event(
6302 "ActivityStarted",
6303 json!({
6304 "sequence": 1,
6305 "activity_type": "flaky",
6306 "activity_execution_id": "act-1",
6307 "activity_attempt_id": "attempt-2",
6308 "attempt_number": 2
6309 }),
6310 ),
6311 history_event(
6312 "ActivityCompleted",
6313 json!({
6314 "sequence": 1,
6315 "activity_type": "flaky",
6316 "activity_execution_id": "act-1",
6317 "activity_attempt_id": "attempt-2",
6318 "attempt_number": 2,
6319 "payload_codec": "json",
6320 "result": {"codec": "json", "blob": "{\"status\":\"recovered\"}"}
6321 }),
6322 ),
6323 ]
6324 }
6325
6326 fn retry_activity_options() -> ActivityOptions {
6327 ActivityOptions::new()
6328 .task_queue("critical-activities")
6329 .retry_policy(
6330 ActivityRetryPolicy::new(3)
6331 .backoff_intervals([Duration::from_secs(2), Duration::from_secs(4)])
6332 .non_retryable_error_type("PermanentError"),
6333 )
6334 .start_to_close_timeout(Duration::from_secs(30))
6335 .schedule_to_start_timeout(Duration::from_secs(5))
6336 .schedule_to_close_timeout(Duration::from_secs(90))
6337 .heartbeat_timeout(Duration::from_secs(10))
6338 }
6339
6340 #[test]
6341 fn avro_generic_wrapper_round_trips_json_values() {
6342 let value = json!({"greeting": "hello", "count": 3, "ok": true});
6343 let envelope = PayloadEnvelope::avro(&value).expect("encode");
6344 assert_eq!(envelope.codec, DEFAULT_CODEC);
6345 assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
6346 }
6347
6348 #[test]
6349 fn json_codec_remains_plain_json() {
6350 let value = json!({"greeting": "hello", "count": 3, "ok": true});
6351 let envelope = PayloadEnvelope::json(&value).expect("encode");
6352
6353 assert_eq!(envelope.codec, JSON_CODEC);
6354 assert_eq!(envelope.blob, serde_json::to_string(&value).expect("json"));
6355 assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
6356 }
6357
6358 #[test]
6359 fn typed_avro_payload_without_schema_context_keeps_diagnostic() {
6360 let envelope = PayloadEnvelope {
6361 codec: DEFAULT_CODEC.to_string(),
6362 blob: BASE64.encode([0x01]),
6363 };
6364
6365 let error = decode_payload::<Value>(&envelope).expect_err("typed payload must fail");
6366 assert_eq!(
6367 error.to_string(),
6368 "codec error: typed avro payloads require a schema context; v1 supports the generic wrapper"
6369 );
6370 }
6371
6372 #[test]
6373 fn workflow_context_schedules_activity_until_completion_is_in_history() {
6374 let ctx = WorkflowContext {
6375 state: Arc::new(Mutex::new(
6376 WorkflowState::new_with_identity(
6377 Vec::new(),
6378 Some("wf-parent".to_string()),
6379 Some("run-parent".to_string()),
6380 "rust-workers".to_string(),
6381 DEFAULT_CODEC.to_string(),
6382 None,
6383 )
6384 .expect("workflow state"),
6385 )),
6386 };
6387
6388 let mut call = Box::pin(ctx.activity("hello.activity", json!(["Ada"])));
6389 let mut task_context = TaskContext::from_waker(noop_waker_ref());
6390 assert!(matches!(
6391 call.as_mut().poll(&mut task_context),
6392 Poll::Pending
6393 ));
6394
6395 let commands = ctx.take_commands().expect("commands");
6396 assert_eq!(commands[0]["type"], "schedule_activity");
6397 assert_eq!(commands[0]["activity_type"], "hello.activity");
6398 }
6399
6400 #[test]
6401 fn activity_options_encode_retry_policy_queue_and_every_timeout() {
6402 let ctx = workflow_context(Vec::new());
6403 let options = ActivityOptions::new()
6404 .task_queue("payments")
6405 .retry_policy(
6406 ActivityRetryPolicy::new(4)
6407 .exponential_backoff(Duration::from_secs(1), 3, Some(Duration::from_secs(10)))
6408 .non_retryable_error_type("ValidationError"),
6409 )
6410 .start_to_close_timeout(Duration::from_secs(120))
6411 .schedule_to_start_timeout(Duration::from_secs(10))
6412 .schedule_to_close_timeout(Duration::from_secs(300))
6413 .heartbeat_timeout(Duration::from_secs(15));
6414 let mut call = Box::pin(ctx.activity_with_options(
6415 "charge-card",
6416 options,
6417 json!([{"order_id": "o-1"}]),
6418 ));
6419 let mut task_context = TaskContext::from_waker(noop_waker_ref());
6420
6421 assert!(matches!(
6422 call.as_mut().poll(&mut task_context),
6423 Poll::Pending
6424 ));
6425 assert!(matches!(
6426 call.as_mut().poll(&mut task_context),
6427 Poll::Pending
6428 ));
6429
6430 let commands = ctx.take_commands().expect("activity command");
6431 assert_eq!(commands.len(), 1, "one future emits one logical schedule");
6432 assert_eq!(commands[0]["queue"], "payments");
6433 assert_eq!(
6434 commands[0]["retry_policy"],
6435 json!({
6436 "max_attempts": 4,
6437 "backoff_seconds": [1, 3, 9],
6438 "non_retryable_error_types": ["ValidationError"],
6439 })
6440 );
6441 assert_eq!(commands[0]["start_to_close_timeout"], 120);
6442 assert_eq!(commands[0]["schedule_to_start_timeout"], 10);
6443 assert_eq!(commands[0]["schedule_to_close_timeout"], 300);
6444 assert_eq!(commands[0]["heartbeat_timeout"], 15);
6445 }
6446
6447 #[test]
6448 fn activity_options_encode_explicit_and_rounded_backoff_intervals() {
6449 let ctx = workflow_context(Vec::new());
6450 let options = ActivityOptions::new().retry_policy(
6451 ActivityRetryPolicy::new(3)
6452 .backoff_intervals([Duration::from_millis(1), Duration::from_millis(1_001)]),
6453 );
6454 let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
6455 let mut task_context = TaskContext::from_waker(noop_waker_ref());
6456
6457 assert!(matches!(
6458 call.as_mut().poll(&mut task_context),
6459 Poll::Pending
6460 ));
6461 assert_eq!(
6462 ctx.take_commands().expect("command")[0]["retry_policy"]["backoff_seconds"],
6463 json!([1, 2])
6464 );
6465 }
6466
6467 #[test]
6468 fn invalid_activity_options_return_typed_errors_before_emitting_commands() {
6469 let cases = [
6470 (
6471 ActivityOptions::new().task_queue(" "),
6472 ActivityOptionsErrorKind::EmptyTaskQueue,
6473 ),
6474 (
6475 ActivityOptions::new().retry_policy(ActivityRetryPolicy::default()),
6476 ActivityOptionsErrorKind::EmptyRetryPolicy,
6477 ),
6478 (
6479 ActivityOptions::new().retry_policy(ActivityRetryPolicy::new(0)),
6480 ActivityOptionsErrorKind::InvalidMaxAttempts,
6481 ),
6482 (
6483 ActivityOptions::new().retry_policy(ActivityRetryPolicy {
6484 max_attempts: None,
6485 backoff: Some(ActivityBackoff::Explicit(vec![Duration::from_secs(1)])),
6486 non_retryable_error_types: Vec::new(),
6487 }),
6488 ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
6489 ),
6490 (
6491 ActivityOptions::new().retry_policy(
6492 ActivityRetryPolicy::new(2)
6493 .backoff_intervals([Duration::from_secs(1), Duration::from_secs(2)]),
6494 ),
6495 ActivityOptionsErrorKind::TooManyBackoffIntervals,
6496 ),
6497 (
6498 ActivityOptions::new().retry_policy(
6499 ActivityRetryPolicy::new(2).exponential_backoff(
6500 Duration::from_secs(1),
6501 0,
6502 None,
6503 ),
6504 ),
6505 ActivityOptionsErrorKind::InvalidBackoffCoefficient,
6506 ),
6507 (
6508 ActivityOptions::new()
6509 .retry_policy(ActivityRetryPolicy::new(2).non_retryable_error_type(" ")),
6510 ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
6511 ),
6512 (
6513 ActivityOptions::new().retry_policy(
6514 ActivityRetryPolicy::new(10_002).exponential_backoff(
6515 Duration::from_secs(1),
6516 1,
6517 None,
6518 ),
6519 ),
6520 ActivityOptionsErrorKind::BackoffGenerationTooLarge,
6521 ),
6522 (
6523 ActivityOptions::new().retry_policy(
6524 ActivityRetryPolicy::new(2)
6525 .backoff_intervals([Duration::from_secs(i64::MAX as u64 + 1)]),
6526 ),
6527 ActivityOptionsErrorKind::BackoffOverflow,
6528 ),
6529 ];
6530
6531 for (options, expected_kind) in cases {
6532 let ctx = workflow_context(Vec::new());
6533 let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
6534 let mut task_context = TaskContext::from_waker(noop_waker_ref());
6535 let Poll::Ready(Err(Error::InvalidActivityOptions(error))) =
6536 call.as_mut().poll(&mut task_context)
6537 else {
6538 panic!("expected typed activity validation error");
6539 };
6540 assert_eq!(error.kind, expected_kind);
6541 assert!(ctx.take_commands().expect("commands").is_empty());
6542 }
6543 }
6544
6545 #[test]
6546 fn activity_options_validate_positive_and_ordered_timeouts() {
6547 let zero_timeout_cases = [
6548 ActivityOptions::new().start_to_close_timeout(Duration::ZERO),
6549 ActivityOptions::new().schedule_to_start_timeout(Duration::ZERO),
6550 ActivityOptions::new().schedule_to_close_timeout(Duration::ZERO),
6551 ActivityOptions::new().heartbeat_timeout(Duration::ZERO),
6552 ];
6553 for options in zero_timeout_cases {
6554 assert_eq!(
6555 options.validate().expect_err("zero timeout").kind,
6556 ActivityOptionsErrorKind::TimeoutNotPositive
6557 );
6558 }
6559
6560 let ordering_cases = [
6561 ActivityOptions::new()
6562 .heartbeat_timeout(Duration::from_secs(11))
6563 .start_to_close_timeout(Duration::from_secs(10)),
6564 ActivityOptions::new()
6565 .start_to_close_timeout(Duration::from_secs(31))
6566 .schedule_to_close_timeout(Duration::from_secs(30)),
6567 ActivityOptions::new()
6568 .schedule_to_start_timeout(Duration::from_secs(31))
6569 .schedule_to_close_timeout(Duration::from_secs(30)),
6570 ];
6571 for options in ordering_cases {
6572 assert_eq!(
6573 options.validate().expect_err("timeout order").kind,
6574 ActivityOptionsErrorKind::TimeoutOrder
6575 );
6576 }
6577
6578 assert_eq!(
6579 ActivityOptions::new()
6580 .start_to_close_timeout(Duration::from_secs(i64::MAX as u64 + 1))
6581 .validate()
6582 .expect_err("protocol integer overflow")
6583 .kind,
6584 ActivityOptionsErrorKind::TimeoutOverflow
6585 );
6586 }
6587
6588 #[test]
6589 fn replayed_activity_retry_history_completes_without_duplicate_schedule() {
6590 let ctx = workflow_context(completed_retry_activity_history());
6591 let mut call =
6592 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
6593 let mut task_context = TaskContext::from_waker(noop_waker_ref());
6594
6595 assert!(matches!(
6596 call.as_mut().poll(&mut task_context),
6597 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
6598 ));
6599 assert!(ctx.take_commands().expect("commands").is_empty());
6600 ctx.ensure_history_consumed().expect("history consumed");
6601 }
6602
6603 #[test]
6604 fn duplicate_non_retryable_types_use_one_command_and_replay_representation() {
6605 let mut options = retry_activity_options();
6606 options
6607 .retry_policy
6608 .as_mut()
6609 .expect("retry policy")
6610 .non_retryable_error_types
6611 .extend([" PermanentError ".to_string(), "PermanentError".to_string()]);
6612
6613 let new_ctx = workflow_context(Vec::new());
6614 let mut new_call =
6615 Box::pin(new_ctx.activity_with_options("flaky", options.clone(), json!([])));
6616 let mut task_context = TaskContext::from_waker(noop_waker_ref());
6617 assert!(matches!(
6618 new_call.as_mut().poll(&mut task_context),
6619 Poll::Pending
6620 ));
6621 let commands = new_ctx.take_commands().expect("commands");
6622 assert_eq!(commands.len(), 1);
6623 assert_eq!(
6624 commands[0]["retry_policy"]["non_retryable_error_types"],
6625 json!(["PermanentError"])
6626 );
6627
6628 let replay_ctx = workflow_context(completed_retry_activity_history());
6629 let mut replay_call =
6630 Box::pin(replay_ctx.activity_with_options("flaky", options, json!([])));
6631 assert!(matches!(
6632 replay_call.as_mut().poll(&mut task_context),
6633 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
6634 ));
6635 assert!(replay_ctx.take_commands().expect("commands").is_empty());
6636 replay_ctx
6637 .ensure_history_consumed()
6638 .expect("history consumed");
6639 }
6640
6641 #[test]
6642 fn replayed_intermediate_retry_remains_pending_across_restarts() {
6643 let history = completed_retry_activity_history()
6644 .into_iter()
6645 .take(3)
6646 .collect::<Vec<_>>();
6647
6648 for _restart in 0..2 {
6649 let ctx = workflow_context(history.clone());
6650 let mut call =
6651 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
6652 let mut task_context = TaskContext::from_waker(noop_waker_ref());
6653 assert!(matches!(
6654 call.as_mut().poll(&mut task_context),
6655 Poll::Pending
6656 ));
6657 assert!(ctx.take_commands().expect("commands").is_empty());
6658 }
6659 }
6660
6661 #[test]
6662 fn replayed_activity_rejects_changed_queue_retry_and_every_timeout_field() {
6663 let mut changed_queue = retry_activity_options();
6664 changed_queue.task_queue = Some("different-queue".to_string());
6665
6666 let mut changed_max_attempts = retry_activity_options();
6667 let retry_policy = changed_max_attempts
6668 .retry_policy
6669 .as_mut()
6670 .expect("retry policy");
6671 retry_policy.max_attempts = Some(4);
6672
6673 let mut changed_backoff = retry_activity_options();
6674 let retry_policy = changed_backoff.retry_policy.as_mut().expect("retry policy");
6675 retry_policy.backoff = Some(ActivityBackoff::Explicit(vec![
6676 Duration::from_secs(3),
6677 Duration::from_secs(4),
6678 ]));
6679
6680 let mut changed_non_retryable_types = retry_activity_options();
6681 let retry_policy = changed_non_retryable_types
6682 .retry_policy
6683 .as_mut()
6684 .expect("retry policy");
6685 retry_policy.non_retryable_error_types = vec!["AnotherPermanentError".to_string()];
6686
6687 let mut changed_start_to_close = retry_activity_options();
6688 changed_start_to_close.start_to_close_timeout = Some(Duration::from_secs(31));
6689 let mut changed_schedule_to_start = retry_activity_options();
6690 changed_schedule_to_start.schedule_to_start_timeout = Some(Duration::from_secs(6));
6691 let mut changed_schedule_to_close = retry_activity_options();
6692 changed_schedule_to_close.schedule_to_close_timeout = Some(Duration::from_secs(91));
6693 let mut changed_heartbeat = retry_activity_options();
6694 changed_heartbeat.heartbeat_timeout = Some(Duration::from_secs(11));
6695
6696 let cases = [
6697 (changed_queue, "activity_task_queue_mismatch"),
6698 (changed_max_attempts, "activity_retry_policy_mismatch"),
6699 (changed_backoff, "activity_retry_policy_mismatch"),
6700 (
6701 changed_non_retryable_types,
6702 "activity_retry_policy_mismatch",
6703 ),
6704 (changed_start_to_close, "activity_retry_policy_mismatch"),
6705 (changed_schedule_to_start, "activity_retry_policy_mismatch"),
6706 (changed_schedule_to_close, "activity_retry_policy_mismatch"),
6707 (changed_heartbeat, "activity_retry_policy_mismatch"),
6708 ];
6709
6710 for (options, expected_reason) in cases {
6711 let ctx = workflow_context(completed_retry_activity_history());
6712 let mut call = Box::pin(ctx.activity_with_options("flaky", options, json!([])));
6713 let mut task_context = TaskContext::from_waker(noop_waker_ref());
6714 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
6715 call.as_mut().poll(&mut task_context)
6716 else {
6717 panic!("changed activity options must fail replay");
6718 };
6719 assert_eq!(failure.reason, expected_reason);
6720 assert_eq!(failure.sequence, Some(1));
6721 assert!(ctx.take_commands().expect("commands").is_empty());
6722 }
6723 }
6724
6725 #[test]
6726 fn replayed_activity_rejects_changed_execution_mode_and_snapshot_version() {
6727 let cases = [
6728 (
6729 "execution_mode",
6730 json!("local"),
6731 "activity_execution_mode_mismatch",
6732 ),
6733 (
6734 "snapshot_version",
6735 json!(2),
6736 "activity_retry_policy_mismatch",
6737 ),
6738 ];
6739
6740 for (field, value, expected_reason) in cases {
6741 let mut history = completed_retry_activity_history();
6742 let activity = history[0].payload["activity"]
6743 .as_object_mut()
6744 .expect("activity snapshot");
6745 if field == "execution_mode" {
6746 activity.insert(field.to_string(), value);
6747 } else {
6748 activity["retry_policy"]
6749 .as_object_mut()
6750 .expect("retry snapshot")
6751 .insert(field.to_string(), value);
6752 }
6753
6754 let ctx = workflow_context(history);
6755 let mut call =
6756 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
6757 let mut task_context = TaskContext::from_waker(noop_waker_ref());
6758 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
6759 call.as_mut().poll(&mut task_context)
6760 else {
6761 panic!("changed {field} must fail replay");
6762 };
6763 assert_eq!(failure.reason, expected_reason);
6764 assert_eq!(failure.sequence, Some(1));
6765 assert!(ctx.take_commands().expect("commands").is_empty());
6766 }
6767 }
6768
6769 #[test]
6770 fn replayed_legacy_activity_treats_missing_option_snapshot_as_unknown() {
6771 let mut history = completed_retry_activity_history();
6772 let activity = history[0].payload["activity"]
6773 .as_object_mut()
6774 .expect("activity snapshot");
6775 activity.remove("execution_mode");
6776 activity.remove("retry_policy");
6777
6778 let mut current = retry_activity_options();
6779 current.start_to_close_timeout = Some(Duration::from_secs(45));
6780 current.schedule_to_start_timeout = Some(Duration::from_secs(8));
6781 current.schedule_to_close_timeout = Some(Duration::from_secs(120));
6782 current.heartbeat_timeout = Some(Duration::from_secs(12));
6783
6784 let ctx = workflow_context(history);
6785 let mut call = Box::pin(ctx.activity_with_options("flaky", current, json!([])));
6786 let mut task_context = TaskContext::from_waker(noop_waker_ref());
6787 assert!(matches!(
6788 call.as_mut().poll(&mut task_context),
6789 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
6790 ));
6791 assert!(ctx.take_commands().expect("commands").is_empty());
6792 ctx.ensure_history_consumed().expect("history consumed");
6793 }
6794
6795 #[test]
6796 fn terminal_activity_failed_after_start_returns_typed_failure() {
6797 let history = vec![
6798 history_event(
6799 "ActivityScheduled",
6800 json!({
6801 "sequence": 1,
6802 "activity_type": "flaky",
6803 "activity_execution_id": "act-terminal",
6804 "activity": {
6805 "id": "act-terminal",
6806 "sequence": 1,
6807 "type": "flaky",
6808 "queue": "critical-activities",
6809 "retry_policy": {
6810 "snapshot_version": 1,
6811 "max_attempts": 3,
6812 "backoff_seconds": [2, 4],
6813 "non_retryable_error_types": ["PermanentError"]
6814 }
6815 }
6816 }),
6817 ),
6818 history_event(
6819 "ActivityStarted",
6820 json!({
6821 "sequence": 1,
6822 "activity_type": "flaky",
6823 "activity_execution_id": "act-terminal",
6824 "activity_attempt_id": "attempt-1",
6825 "attempt_number": 1
6826 }),
6827 ),
6828 history_event(
6829 "ActivityFailed",
6830 json!({
6831 "sequence": 1,
6832 "activity_type": "flaky",
6833 "activity_execution_id": "act-terminal",
6834 "activity_attempt_id": "attempt-1",
6835 "attempt_number": 1,
6836 "failure_id": "failure-terminal",
6837 "failure_category": "activity",
6838 "exception_type": "PermanentError",
6839 "message": "cannot retry",
6840 "non_retryable": true
6841 }),
6842 ),
6843 ];
6844 let ctx = workflow_context(history);
6845 let mut call =
6846 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
6847 let mut task_context = TaskContext::from_waker(noop_waker_ref());
6848
6849 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
6850 call.as_mut().poll(&mut task_context)
6851 else {
6852 panic!("terminal ActivityFailed must settle the activity future");
6853 };
6854 assert_eq!(failure.kind, ActivityFailureKind::Failed);
6855 assert_eq!(
6856 failure.activity_execution_id.as_deref(),
6857 Some("act-terminal")
6858 );
6859 assert_eq!(failure.exception_type.as_deref(), Some("PermanentError"));
6860 assert!(failure.non_retryable);
6861 assert!(ctx.take_commands().expect("commands").is_empty());
6862 ctx.ensure_history_consumed().expect("history consumed");
6863 }
6864
6865 #[test]
6866 fn activity_terminal_events_return_machine_readable_failures() {
6867 let cases = [
6868 (
6869 "ActivityFailed",
6870 json!({
6871 "sequence": 1,
6872 "activity_type": "charge-card",
6873 "activity_execution_id": "act-1",
6874 "activity_attempt_id": "attempt-2",
6875 "attempt_number": 2,
6876 "failure_id": "failure-1",
6877 "failure_category": "activity",
6878 "exception_type": "PaymentDeclined",
6879 "exception_class": "payments.PaymentDeclined",
6880 "message": "card declined",
6881 "non_retryable": true
6882 }),
6883 ActivityFailureKind::Failed,
6884 "activity",
6885 ),
6886 (
6887 "ActivityCancelled",
6888 json!({
6889 "sequence": 1,
6890 "activity_type": "charge-card",
6891 "activity_execution_id": "act-1",
6892 "activity_attempt_id": "attempt-1"
6893 }),
6894 ActivityFailureKind::Cancelled,
6895 "cancelled",
6896 ),
6897 ];
6898
6899 for (event_type, payload, expected_kind, expected_reason) in cases {
6900 let ctx = workflow_context(vec![history_event(event_type, payload)]);
6901 let mut call = Box::pin(ctx.activity("charge-card", json!([])));
6902 let mut task_context = TaskContext::from_waker(noop_waker_ref());
6903 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
6904 call.as_mut().poll(&mut task_context)
6905 else {
6906 panic!("expected terminal activity failure");
6907 };
6908 assert_eq!(failure.kind, expected_kind);
6909 assert_eq!(failure.reason, expected_reason);
6910 assert_eq!(failure.activity_execution_id.as_deref(), Some("act-1"));
6911 assert_eq!(failure.activity_type.as_deref(), Some("charge-card"));
6912 }
6913 }
6914
6915 #[test]
6916 fn every_activity_timeout_class_is_typed() {
6917 for timeout_kind in [
6918 "start_to_close",
6919 "schedule_to_start",
6920 "schedule_to_close",
6921 "heartbeat",
6922 ] {
6923 let ctx = workflow_context(vec![history_event(
6924 "ActivityTimedOut",
6925 json!({
6926 "sequence": 1,
6927 "activity_type": "slow",
6928 "activity_execution_id": "act-timeout",
6929 "activity_attempt_id": "attempt-timeout",
6930 "failure_category": "timeout",
6931 "timeout_kind": timeout_kind,
6932 "message": "deadline expired"
6933 }),
6934 )]);
6935 let mut call = Box::pin(ctx.activity("slow", json!([])));
6936 let mut task_context = TaskContext::from_waker(noop_waker_ref());
6937 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
6938 call.as_mut().poll(&mut task_context)
6939 else {
6940 panic!("expected timeout failure");
6941 };
6942 assert_eq!(failure.kind, ActivityFailureKind::TimedOut);
6943 assert_eq!(failure.reason, timeout_kind);
6944 assert_eq!(failure.timeout_kind.as_deref(), Some(timeout_kind));
6945 assert_eq!(failure.failure_category.as_deref(), Some("timeout"));
6946 }
6947 }
6948
6949 #[test]
6950 fn workflow_sleep_emits_one_durable_timer_and_rounds_up() {
6951 let ctx = workflow_context(Vec::new());
6952 let mut sleep = Box::pin(ctx.sleep(Duration::from_millis(1_001)));
6953 let mut task_context = TaskContext::from_waker(noop_waker_ref());
6954
6955 assert!(matches!(
6956 sleep.as_mut().poll(&mut task_context),
6957 Poll::Pending
6958 ));
6959 assert!(matches!(
6960 sleep.as_mut().poll(&mut task_context),
6961 Poll::Pending
6962 ));
6963
6964 let commands = ctx.take_commands().expect("timer command");
6965 assert_eq!(
6966 commands,
6967 vec![json!({
6968 "type": "start_timer",
6969 "delay_seconds": 2,
6970 })]
6971 );
6972 }
6973
6974 #[test]
6975 fn workflow_sleep_replays_matching_schedule_and_fire_without_a_command() {
6976 let history = vec![
6977 history_event(
6978 "TimerScheduled",
6979 json!({
6980 "sequence": 1,
6981 "timer_id": "timer-1",
6982 "delay_seconds": 5,
6983 "fire_at": "2026-07-11T12:00:05Z",
6984 }),
6985 ),
6986 history_event(
6987 "TimerFired",
6988 json!({
6989 "sequence": 1,
6990 "timer_id": "timer-1",
6991 "delay_seconds": 5,
6992 "fire_at": "2026-07-11T12:00:05Z",
6993 "fired_at": "2026-07-11T12:00:05Z",
6994 }),
6995 ),
6996 ];
6997
6998 for _restart in 0..2 {
6999 let ctx = workflow_context(history.clone());
7000 let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(5)));
7001 let mut task_context = TaskContext::from_waker(noop_waker_ref());
7002 assert!(matches!(
7003 sleep.as_mut().poll(&mut task_context),
7004 Poll::Ready(Ok(()))
7005 ));
7006 assert!(ctx.take_commands().expect("commands").is_empty());
7007 ctx.ensure_history_consumed().expect("history consumed");
7008 }
7009 }
7010
7011 #[test]
7012 fn workflow_sleep_rejects_changed_delay_during_replay() {
7013 let ctx = workflow_context(vec![
7014 history_event(
7015 "TimerScheduled",
7016 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7017 ),
7018 history_event(
7019 "TimerFired",
7020 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7021 ),
7022 ]);
7023 let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(500)));
7024 let mut task_context = TaskContext::from_waker(noop_waker_ref());
7025
7026 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
7027 sleep.as_mut().poll(&mut task_context)
7028 else {
7029 panic!("changed timer delay must be rejected");
7030 };
7031 assert_eq!(failure.reason, "timer_delay_mismatch");
7032 assert_eq!(failure.sequence, Some(1));
7033 }
7034
7035 #[test]
7036 fn workflow_history_rejects_unpaired_or_mismatched_timer_events() {
7037 let lone_fire = WorkflowState::new(
7038 vec![history_event(
7039 "TimerFired",
7040 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7041 )],
7042 "rust-workers".to_string(),
7043 JSON_CODEC.to_string(),
7044 None,
7045 )
7046 .expect_err("TimerFired requires TimerScheduled");
7047 assert!(matches!(
7048 lone_fire,
7049 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
7050 if reason == "timer_schedule_missing_or_duplicate"
7051 ));
7052
7053 let wrong_identity = WorkflowState::new(
7054 vec![
7055 history_event(
7056 "TimerScheduled",
7057 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7058 ),
7059 history_event(
7060 "TimerFired",
7061 json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
7062 ),
7063 ],
7064 "rust-workers".to_string(),
7065 JSON_CODEC.to_string(),
7066 None,
7067 )
7068 .expect_err("fire must match scheduled timer identity");
7069 assert!(matches!(
7070 wrong_identity,
7071 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
7072 if reason == "timer_identity_mismatch"
7073 ));
7074
7075 let duplicate_fire = WorkflowState::new(
7076 vec![
7077 history_event(
7078 "TimerScheduled",
7079 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7080 ),
7081 history_event(
7082 "TimerFired",
7083 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7084 ),
7085 history_event(
7086 "TimerFired",
7087 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7088 ),
7089 ],
7090 "rust-workers".to_string(),
7091 JSON_CODEC.to_string(),
7092 None,
7093 )
7094 .expect_err("a durable timer cannot fire twice");
7095 assert!(matches!(
7096 duplicate_fire,
7097 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
7098 if reason == "duplicate_timer_fire"
7099 ));
7100
7101 let wrong_fired_delay = WorkflowState::new(
7102 vec![
7103 history_event(
7104 "TimerScheduled",
7105 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7106 ),
7107 history_event(
7108 "TimerFired",
7109 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 6}),
7110 ),
7111 ],
7112 "rust-workers".to_string(),
7113 JSON_CODEC.to_string(),
7114 None,
7115 )
7116 .expect_err("timer schedule and fire delays must agree");
7117 assert!(matches!(
7118 wrong_fired_delay,
7119 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
7120 if reason == "timer_history_delay_mismatch"
7121 ));
7122 }
7123
7124 #[test]
7125 fn replay_rejects_activity_moved_before_recorded_timer() {
7126 let ctx = workflow_context(vec![
7127 history_event(
7128 "TimerScheduled",
7129 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7130 ),
7131 history_event(
7132 "TimerFired",
7133 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7134 ),
7135 history_event(
7136 "ActivityCompleted",
7137 json!({
7138 "sequence": 2,
7139 "activity_type": "after-timer",
7140 "payload_codec": "json",
7141 "result": {"codec": "json", "blob": "\"done\""},
7142 }),
7143 ),
7144 ]);
7145 let mut activity = Box::pin(ctx.activity("after-timer", json!([])));
7146 let mut task_context = TaskContext::from_waker(noop_waker_ref());
7147
7148 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
7149 activity.as_mut().poll(&mut task_context)
7150 else {
7151 panic!("reordered durable command must be rejected");
7152 };
7153 assert_eq!(failure.reason, "recorded_command_mismatch");
7154 assert_eq!(failure.sequence, Some(1));
7155 assert_eq!(failure.expected.as_deref(), Some("timer"));
7156 assert_eq!(failure.actual.as_deref(), Some("activity:after-timer"));
7157 }
7158
7159 #[test]
7160 fn replay_orders_signal_waits_and_timers_in_one_command_stream() {
7161 let signal_then_timer = vec![
7162 history_event(
7163 "ConditionWaitOpened",
7164 json!({"sequence": 1, "condition_key": "signal:go"}),
7165 ),
7166 history_event(
7167 "SignalReceived",
7168 json!({
7169 "signal_name": "go",
7170 "arguments": ["now"],
7171 }),
7172 ),
7173 history_event(
7174 "TimerScheduled",
7175 json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
7176 ),
7177 history_event(
7178 "TimerFired",
7179 json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
7180 ),
7181 ];
7182
7183 let ctx = workflow_context(signal_then_timer.clone());
7184 let mut signal = Box::pin(ctx.wait_signal("go"));
7185 let mut task_context = TaskContext::from_waker(noop_waker_ref());
7186 assert!(matches!(
7187 signal.as_mut().poll(&mut task_context),
7188 Poll::Ready(Ok(arguments)) if arguments == vec![json!("now")]
7189 ));
7190 let mut timer = Box::pin(ctx.sleep(Duration::from_secs(5)));
7191 assert!(matches!(
7192 timer.as_mut().poll(&mut task_context),
7193 Poll::Ready(Ok(()))
7194 ));
7195 ctx.ensure_history_consumed()
7196 .expect("signal and timer history consumed in order");
7197
7198 let reordered = workflow_context(signal_then_timer);
7199 let mut timer_first = Box::pin(reordered.sleep(Duration::from_secs(5)));
7200 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
7201 timer_first.as_mut().poll(&mut task_context)
7202 else {
7203 panic!("timer cannot consume signal-wait-first history");
7204 };
7205 assert_eq!(failure.reason, "recorded_command_mismatch");
7206 assert_eq!(failure.sequence, Some(1));
7207 assert_eq!(failure.expected.as_deref(), Some("signal wait"));
7208
7209 let timer_then_signal = vec![
7210 history_event(
7211 "TimerScheduled",
7212 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7213 ),
7214 history_event(
7215 "TimerFired",
7216 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7217 ),
7218 history_event(
7219 "ConditionWaitOpened",
7220 json!({"sequence": 2, "condition_key": "signal:go"}),
7221 ),
7222 history_event(
7223 "SignalReceived",
7224 json!({"signal_name": "go", "arguments": []}),
7225 ),
7226 ];
7227 let reordered = workflow_context(timer_then_signal);
7228 let mut signal_first = Box::pin(reordered.wait_signal("go"));
7229 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
7230 signal_first.as_mut().poll(&mut task_context)
7231 else {
7232 panic!("signal wait cannot consume timer-first history");
7233 };
7234 assert_eq!(failure.reason, "recorded_command_mismatch");
7235 assert_eq!(failure.sequence, Some(1));
7236 assert_eq!(failure.expected.as_deref(), Some("timer"));
7237 }
7238
7239 #[test]
7240 fn workflow_history_rejects_duplicate_or_colliding_command_sequences() {
7241 let duplicate_timer = WorkflowState::new(
7242 vec![
7243 history_event(
7244 "TimerScheduled",
7245 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7246 ),
7247 history_event(
7248 "TimerScheduled",
7249 json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
7250 ),
7251 ],
7252 "rust-workers".to_string(),
7253 JSON_CODEC.to_string(),
7254 None,
7255 )
7256 .expect_err("one workflow sequence cannot schedule two timers");
7257 assert!(matches!(
7258 duplicate_timer,
7259 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
7260 if reason == "timer_schedule_missing_or_duplicate"
7261 ));
7262
7263 let colliding_kinds = WorkflowState::new(
7264 vec![
7265 history_event(
7266 "TimerScheduled",
7267 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7268 ),
7269 history_event(
7270 "ActivityCompleted",
7271 json!({"sequence": 1, "activity_type": "same-sequence"}),
7272 ),
7273 ],
7274 "rust-workers".to_string(),
7275 JSON_CODEC.to_string(),
7276 None,
7277 )
7278 .expect_err("one workflow sequence cannot identify two command kinds");
7279 assert!(matches!(
7280 colliding_kinds,
7281 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
7282 if reason == "durable_command_sequence_collision"
7283 ));
7284
7285 let duplicate_signal_wait = WorkflowState::new(
7286 vec![
7287 history_event(
7288 "SignalWaitOpened",
7289 json!({"sequence": 1, "signal_name": "go"}),
7290 ),
7291 history_event(
7292 "SignalWaitOpened",
7293 json!({"sequence": 1, "signal_name": "go"}),
7294 ),
7295 ],
7296 "rust-workers".to_string(),
7297 JSON_CODEC.to_string(),
7298 None,
7299 )
7300 .expect_err("one workflow sequence cannot open two signal waits");
7301 assert!(matches!(
7302 duplicate_signal_wait,
7303 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
7304 if reason == "duplicate_signal_wait_open"
7305 ));
7306 }
7307
7308 #[test]
7309 fn workflow_sleep_rejects_unrepresentable_rounded_duration() {
7310 let ctx = workflow_context(Vec::new());
7311 let mut sleep = Box::pin(ctx.start_timer(Duration::new(u64::MAX, 1)));
7312 let mut task_context = TaskContext::from_waker(noop_waker_ref());
7313 assert!(matches!(
7314 sleep.as_mut().poll(&mut task_context),
7315 Poll::Ready(Err(Error::TimerDurationOverflow))
7316 ));
7317 assert!(ctx.take_commands().expect("commands").is_empty());
7318 }
7319
7320 #[test]
7321 fn workflow_task_replay_completes_without_rescheduling_recorded_commands() {
7322 let client = Client::new("http://127.0.0.1:8080").expect("client");
7323 let mut worker = Worker::new(client, "rust-workers");
7324 worker.register_workflow("rust.timer", |ctx, _input| async move {
7325 ctx.sleep(Duration::from_secs(5)).await?;
7326 ctx.activity("after-timer", json!([])).await
7327 });
7328
7329 let task = |history_events| WorkflowTask {
7330 task_id: "wft-rust-timer-1".to_string(),
7331 workflow_id: Some("wf-rust-timer".to_string()),
7332 run_id: Some("run-rust-timer".to_string()),
7333 workflow_type: "rust.timer".to_string(),
7334 payload_codec: JSON_CODEC.to_string(),
7335 arguments: Some(json!({"codec": "json", "blob": "[]"})),
7336 history_events,
7337 total_history_events: None,
7338 next_history_page_token: None,
7339 workflow_task_attempt: 1,
7340 workflow_signal_id: None,
7341 signal_name: None,
7342 signal_arguments: None,
7343 lease_owner: Some("rust-worker".to_string()),
7344 };
7345
7346 let initial = worker
7347 .execute_workflow_task(task(Vec::new()))
7348 .expect("initial timer task");
7349 assert_eq!(
7350 initial,
7351 vec![json!({"type": "start_timer", "delay_seconds": 5})]
7352 );
7353
7354 let replayed = worker
7355 .execute_workflow_task(task(vec![
7356 history_event(
7357 "TimerScheduled",
7358 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7359 ),
7360 history_event(
7361 "TimerFired",
7362 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7363 ),
7364 history_event(
7365 "ActivityCompleted",
7366 json!({
7367 "sequence": 2,
7368 "activity_type": "after-timer",
7369 "payload_codec": "json",
7370 "result": {"codec": "json", "blob": "\"done\""},
7371 }),
7372 ),
7373 ]))
7374 .expect("replayed workflow task");
7375 assert_eq!(replayed.len(), 1);
7376 assert_eq!(replayed[0]["type"], "complete_workflow");
7377 assert_eq!(
7378 decode_wire_value(&replayed[0]["result"], JSON_CODEC).expect("result"),
7379 json!("done")
7380 );
7381 }
7382
7383 #[test]
7384 fn workflow_task_replay_keeps_recorded_unfired_timer_pending_without_rescheduling() {
7385 let client = Client::new("http://127.0.0.1:8080").expect("client");
7386 let mut worker = Worker::new(client, "rust-workers");
7387 worker.register_workflow("rust.timer.pending", |ctx, _input| async move {
7388 ctx.sleep(Duration::from_secs(5)).await?;
7389 Ok(json!({"status": "timer fired"}))
7390 });
7391
7392 let task = WorkflowTask {
7393 task_id: "wft-rust-timer-pending".to_string(),
7394 workflow_id: Some("wf-rust-timer".to_string()),
7395 run_id: Some("run-rust-timer".to_string()),
7396 workflow_type: "rust.timer.pending".to_string(),
7397 payload_codec: JSON_CODEC.to_string(),
7398 arguments: Some(json!({"codec": "json", "blob": "[]"})),
7399 history_events: vec![history_event(
7400 "TimerScheduled",
7401 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7402 )],
7403 total_history_events: Some(1),
7404 next_history_page_token: None,
7405 workflow_task_attempt: 1,
7406 workflow_signal_id: None,
7407 signal_name: None,
7408 signal_arguments: None,
7409 lease_owner: Some("rust-worker".to_string()),
7410 };
7411
7412 for _redelivery_or_restart in 0..2 {
7413 let commands = worker
7414 .execute_workflow_task(task.clone())
7415 .expect("recorded timer remains pending");
7416 assert!(
7417 commands.is_empty(),
7418 "recorded timer must not be rescheduled"
7419 );
7420 }
7421 }
7422
7423 #[test]
7424 fn workflow_task_rejects_recorded_command_removed_from_workflow_code() {
7425 let client = Client::new("http://127.0.0.1:8080").expect("client");
7426 let mut worker = Worker::new(client, "rust-workers");
7427 worker.register_workflow("rust.timer.removed", |_ctx, _input| async move {
7428 Ok(json!({"status": "completed"}))
7429 });
7430 let task = WorkflowTask {
7431 task_id: "wft-rust-timer-removed".to_string(),
7432 workflow_id: Some("wf-rust-timer".to_string()),
7433 run_id: Some("run-rust-timer".to_string()),
7434 workflow_type: "rust.timer.removed".to_string(),
7435 payload_codec: JSON_CODEC.to_string(),
7436 arguments: Some(json!({"codec": "json", "blob": "[]"})),
7437 history_events: vec![
7438 history_event(
7439 "TimerScheduled",
7440 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7441 ),
7442 history_event(
7443 "TimerFired",
7444 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
7445 ),
7446 ],
7447 total_history_events: Some(2),
7448 next_history_page_token: None,
7449 workflow_task_attempt: 1,
7450 workflow_signal_id: None,
7451 signal_name: None,
7452 signal_arguments: None,
7453 lease_owner: Some("rust-worker".to_string()),
7454 };
7455
7456 let Error::NonDeterministicReplay(failure) = worker
7457 .execute_workflow_task(task)
7458 .expect_err("removed timer must fail replay")
7459 else {
7460 panic!("expected typed replay failure");
7461 };
7462 assert_eq!(failure.reason, "recorded_commands_unconsumed");
7463 assert_eq!(failure.sequence, Some(1));
7464 }
7465
7466 #[test]
7467 fn workflow_context_emits_explicit_child_workflow_contract() {
7468 let ctx = WorkflowContext {
7469 state: Arc::new(Mutex::new(
7470 WorkflowState::new_with_identity(
7471 Vec::new(),
7472 Some("wf-parent".to_string()),
7473 Some("run-parent".to_string()),
7474 "parent-workers".to_string(),
7475 JSON_CODEC.to_string(),
7476 None,
7477 )
7478 .expect("workflow state"),
7479 )),
7480 };
7481 let options = ChildWorkflowOptions::new("python-workers")
7482 .parent_close_policy(ParentClosePolicy::RequestCancel)
7483 .retry_policy(ChildWorkflowRetryPolicy {
7484 max_attempts: Some(3),
7485 backoff_seconds: vec![1, 5],
7486 non_retryable_error_types: vec!["ValidationError".to_string()],
7487 })
7488 .execution_timeout_seconds(600)
7489 .run_timeout_seconds(120);
7490 let mut call = Box::pin(ctx.start_child_workflow(
7491 "python.fulfil-order",
7492 options,
7493 json!([{"order_id": "order-42"}]),
7494 ));
7495 let mut task_context = TaskContext::from_waker(noop_waker_ref());
7496
7497 assert!(matches!(
7498 call.as_mut().poll(&mut task_context),
7499 Poll::Pending
7500 ));
7501 let commands = ctx.take_commands().expect("commands");
7502 assert_eq!(commands.len(), 1);
7503 let command = &commands[0];
7504 assert_eq!(command["type"], "start_child_workflow");
7505 assert_eq!(command["workflow_type"], "python.fulfil-order");
7506 assert_eq!(command["queue"], "python-workers");
7507 assert_eq!(command["parent_close_policy"], "request_cancel");
7508 assert_eq!(command["retry_policy"]["max_attempts"], 3);
7509 assert_eq!(command["execution_timeout_seconds"], 600);
7510 assert_eq!(command["run_timeout_seconds"], 120);
7511 assert_eq!(
7512 decode_wire_value(&command["arguments"], JSON_CODEC).expect("child args"),
7513 json!([{"order_id": "order-42"}])
7514 );
7515 }
7516
7517 fn child_parent_worker() -> Worker {
7518 let client = Client::new("http://127.0.0.1:8080").expect("client");
7519 let mut worker = Worker::new(client, "rust-parent-workers");
7520 worker.register_workflow("rust.parent", |ctx, _input| async move {
7521 let child = ctx
7522 .start_child_workflow(
7523 "python.child",
7524 ChildWorkflowOptions::new("python-child-workers")
7525 .parent_close_policy(ParentClosePolicy::Terminate),
7526 json!([{"codec_probe": [1, true, "rust"]}]),
7527 )
7528 .await?;
7529 Ok(json!({
7530 "parent_workflow_id": child.parent.workflow_id,
7531 "parent_run_id": child.parent.run_id,
7532 "child_workflow_id": child.child.workflow_id,
7533 "child_run_id": child.child.run_id,
7534 "child_workflow_type": child.child_workflow_type,
7535 "result": child.result,
7536 }))
7537 });
7538 worker
7539 }
7540
7541 fn child_parent_task(event_type: &str, payload: Value) -> WorkflowTask {
7542 WorkflowTask {
7543 task_id: "wft-child-parent".to_string(),
7544 workflow_id: Some("wf-parent".to_string()),
7545 run_id: Some("run-parent".to_string()),
7546 workflow_type: "rust.parent".to_string(),
7547 payload_codec: JSON_CODEC.to_string(),
7548 arguments: Some(encode_value_envelope(&json!([]), JSON_CODEC).expect("input")),
7549 history_events: vec![
7550 HistoryEvent {
7551 event_type: "ChildWorkflowScheduled".to_string(),
7552 payload: json!({
7553 "sequence": 1,
7554 "child_call_id": "call-child",
7555 "child_workflow_instance_id": "wf-child",
7556 "child_workflow_run_id": "run-child",
7557 "child_workflow_type": "python.child",
7558 }),
7559 raw: HashMap::new(),
7560 },
7561 HistoryEvent {
7562 event_type: event_type.to_string(),
7563 payload,
7564 raw: HashMap::new(),
7565 },
7566 ],
7567 total_history_events: Some(2),
7568 next_history_page_token: None,
7569 workflow_task_attempt: 1,
7570 workflow_signal_id: None,
7571 signal_name: None,
7572 signal_arguments: None,
7573 lease_owner: Some("rust-worker".to_string()),
7574 }
7575 }
7576
7577 #[test]
7578 fn committed_child_result_replays_without_starting_a_duplicate() {
7579 let worker = child_parent_worker();
7580 let task = child_parent_task(
7581 "ChildRunCompleted",
7582 json!({
7583 "sequence": 1,
7584 "child_call_id": "call-child",
7585 "child_workflow_instance_id": "wf-child",
7586 "child_workflow_run_id": "run-child",
7587 "child_workflow_type": "python.child",
7588 "payload_codec": "json",
7589 "result": {"codec": "json", "blob": "{\"from\":\"python\",\"ok\":true}"},
7590 }),
7591 );
7592
7593 for _restart in 0..2 {
7594 let commands = worker
7595 .execute_workflow_task(task.clone())
7596 .expect("replayed parent task");
7597 assert_eq!(commands.len(), 1);
7598 assert_eq!(commands[0]["type"], "complete_workflow");
7599 assert!(!commands
7600 .iter()
7601 .any(|command| command["type"] == "start_child_workflow"));
7602 let output =
7603 decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("parent output");
7604 assert_eq!(output["parent_workflow_id"], "wf-parent");
7605 assert_eq!(output["parent_run_id"], "run-parent");
7606 assert_eq!(output["child_workflow_id"], "wf-child");
7607 assert_eq!(output["child_run_id"], "run-child");
7608 assert_eq!(output["result"], json!({"from": "python", "ok": true}));
7609 }
7610 }
7611
7612 #[test]
7613 fn pending_child_replays_after_restart_without_starting_a_duplicate() {
7614 let worker = child_parent_worker();
7615 let mut task = child_parent_task("unused", Value::Null);
7616 task.history_events.truncate(1);
7617 task.total_history_events = Some(1);
7618
7619 for _redelivery_or_restart in 0..2 {
7620 let commands = worker
7621 .execute_workflow_task(task.clone())
7622 .expect("recorded child remains pending");
7623 assert!(
7624 commands.is_empty(),
7625 "recorded pending child must not be started again"
7626 );
7627 }
7628 }
7629
7630 #[test]
7631 fn child_cancellation_becomes_stable_parent_failure_command() {
7632 let worker = child_parent_worker();
7633 let task = child_parent_task(
7634 "ChildRunCancelled",
7635 json!({
7636 "sequence": 1,
7637 "child_workflow_instance_id": "wf-child",
7638 "child_workflow_run_id": "run-child",
7639 "child_workflow_type": "python.child",
7640 "failure_id": "failure-child",
7641 "failure_category": "cancelled",
7642 "message": "cancelled by parent-close policy",
7643 }),
7644 );
7645
7646 let commands = worker
7647 .execute_workflow_task(task)
7648 .expect("parent settlement");
7649 assert_eq!(commands.len(), 1);
7650 assert_eq!(commands[0]["type"], "fail_workflow");
7651 assert_eq!(commands[0]["exception_type"], "ChildWorkflowCancelled");
7652 assert_eq!(
7653 commands[0]["exception"]["properties"]["reason"],
7654 "cancelled"
7655 );
7656 assert_eq!(
7657 commands[0]["exception"]["properties"]["child_workflow_run_id"],
7658 "run-child"
7659 );
7660 }
7661
7662 #[test]
7663 fn workflow_can_handle_typed_child_failure() {
7664 let client = Client::new("http://127.0.0.1:8080").expect("client");
7665 let mut worker = Worker::new(client, "rust-parent-workers");
7666 worker.register_workflow("rust.handled-parent", |ctx, _input| async move {
7667 match ctx
7668 .start_child_workflow(
7669 "python.child",
7670 ChildWorkflowOptions::new("python-child-workers"),
7671 json!([]),
7672 )
7673 .await
7674 {
7675 Err(Error::ChildWorkflowFailed(failure)) => Ok(json!({
7676 "reason": failure.reason,
7677 "failure_id": failure.failure_id,
7678 "exception_class": failure.exception_class,
7679 "child_run_id": failure.child_workflow_run_id,
7680 })),
7681 Err(error) => Err(error),
7682 Ok(_) => Err(Error::WorkerLoop(
7683 "child unexpectedly succeeded".to_string(),
7684 )),
7685 }
7686 });
7687 let mut task = child_parent_task(
7688 "ChildRunFailed",
7689 json!({
7690 "sequence": 1,
7691 "child_workflow_instance_id": "wf-child",
7692 "child_workflow_run_id": "run-child",
7693 "child_workflow_type": "python.child",
7694 "failure_id": "failure-child",
7695 "failure_category": "child_workflow",
7696 "message": "payment rejected",
7697 "exception": {
7698 "type": "PaymentRejected",
7699 "class": "payments.PaymentRejected",
7700 "message": "payment rejected"
7701 }
7702 }),
7703 );
7704 task.workflow_type = "rust.handled-parent".to_string();
7705
7706 let commands = worker.execute_workflow_task(task).expect("handled failure");
7707 assert_eq!(commands[0]["type"], "complete_workflow");
7708 let output = decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("parent output");
7709 assert_eq!(output["reason"], "child_workflow");
7710 assert_eq!(output["failure_id"], "failure-child");
7711 assert_eq!(output["exception_class"], "payments.PaymentRejected");
7712 assert_eq!(output["child_run_id"], "run-child");
7713 }
7714
7715 #[test]
7716 fn rust_hello_world_uses_signal_arguments_from_resume_payload() {
7717 let client = Client::new("http://127.0.0.1:8080").expect("client");
7718 let mut worker = Worker::new(client, "rust-workers");
7719
7720 worker.register_workflow("rust.hello_workflow", |ctx, _input| async move {
7721 let signal = ctx.wait_signal("start").await?;
7722 let name = signal
7723 .first()
7724 .and_then(|value| value.as_str())
7725 .unwrap_or("world");
7726 let greeting = ctx.activity("rust.hello_activity", json!([name])).await?;
7727 Ok(json!({
7728 "greeting": greeting,
7729 "language": "rust"
7730 }))
7731 });
7732
7733 let signal_arguments =
7734 encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC).expect("signal arguments");
7735 let task = WorkflowTask {
7736 task_id: "wft-rust-signal-1".to_string(),
7737 workflow_id: Some("wf-rust-hello".to_string()),
7738 run_id: Some("run-rust-hello".to_string()),
7739 workflow_type: "rust.hello_workflow".to_string(),
7740 payload_codec: DEFAULT_CODEC.to_string(),
7741 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
7742 history_events: vec![HistoryEvent {
7743 event_type: "SignalReceived".to_string(),
7744 payload: json!({
7745 "signal_id": "sig-rust-1",
7746 "signal_name": "start"
7747 }),
7748 raw: HashMap::new(),
7749 }],
7750 total_history_events: Some(1),
7751 next_history_page_token: None,
7752 workflow_task_attempt: 1,
7753 workflow_signal_id: Some("sig-rust-1".to_string()),
7754 signal_name: Some("start".to_string()),
7755 signal_arguments: Some(signal_arguments),
7756 lease_owner: Some("rust-worker".to_string()),
7757 };
7758
7759 let commands = worker.execute_workflow_task(task).expect("workflow task");
7760
7761 assert_eq!(commands.len(), 1);
7762 assert_eq!(commands[0]["type"], "schedule_activity");
7763 assert_eq!(commands[0]["activity_type"], "rust.hello_activity");
7764 assert_eq!(
7765 decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC).expect("activity args"),
7766 json!(["Rust"])
7767 );
7768 }
7769
7770 #[test]
7771 fn workflow_task_appends_paginated_history_events() {
7772 let mut task = WorkflowTask {
7773 task_id: "wft-rust-pages-1".to_string(),
7774 workflow_id: Some("wf-rust-pages".to_string()),
7775 run_id: Some("run-rust-pages".to_string()),
7776 workflow_type: "rust.hello_workflow".to_string(),
7777 payload_codec: DEFAULT_CODEC.to_string(),
7778 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
7779 history_events: vec![HistoryEvent {
7780 event_type: "WorkflowStarted".to_string(),
7781 payload: json!({}),
7782 raw: HashMap::new(),
7783 }],
7784 total_history_events: Some(3),
7785 next_history_page_token: Some("MQ==".to_string()),
7786 workflow_task_attempt: 1,
7787 workflow_signal_id: None,
7788 signal_name: None,
7789 signal_arguments: None,
7790 lease_owner: Some("rust-worker".to_string()),
7791 };
7792
7793 task.append_history_page(WorkflowTaskHistoryPage {
7794 history_events: vec![
7795 HistoryEvent {
7796 event_type: "SignalReceived".to_string(),
7797 payload: json!({
7798 "signal_id": "sig-rust-1",
7799 "signal_name": "start",
7800 "arguments": encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC)
7801 .expect("signal arguments")
7802 }),
7803 raw: HashMap::new(),
7804 },
7805 HistoryEvent {
7806 event_type: "MarkerRecorded".to_string(),
7807 payload: json!({"sequence": 3}),
7808 raw: HashMap::new(),
7809 },
7810 ],
7811 total_history_events: Some(3),
7812 next_history_page_token: None,
7813 });
7814
7815 assert_eq!(task.history_events.len(), 3);
7816 assert_eq!(task.total_history_events, Some(3));
7817 assert_eq!(task.next_history_page_token, None);
7818
7819 let signals =
7820 signal_values(&task.history_events, "start", DEFAULT_CODEC, None).expect("signals");
7821 assert_eq!(signals, vec![vec![json!("Rust")]]);
7822 }
7823
7824 #[tokio::test]
7825 async fn query_handler_reads_ordered_cross_codec_signals_without_commands() {
7826 let client = Client::new("http://127.0.0.1:8080").expect("client");
7827 let mut worker = Worker::new(client, "rust-workers");
7828 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
7829 worker.register_query("counter", "current", |ctx, _args| async move {
7830 let mut count = 0_i64;
7831 for signal in ctx.signal_events() {
7832 let value = signal
7833 .arguments
7834 .first()
7835 .and_then(Value::as_i64)
7836 .unwrap_or_default();
7837 match signal.name.as_str() {
7838 "increment" => count += value,
7839 "set" => count = value,
7840 _ => {}
7841 }
7842 }
7843 Ok(json!(count))
7844 });
7845
7846 let task = QueryTask {
7847 query_task_id: "query-rust-counter".to_string(),
7848 query_task_attempt: 1,
7849 lease_owner: Some("rust-worker".to_string()),
7850 workflow_id: Some("counter-1".to_string()),
7851 run_id: Some("run-counter-1".to_string()),
7852 workflow_type: "counter".to_string(),
7853 query_name: "current".to_string(),
7854 payload_codec: DEFAULT_CODEC.to_string(),
7855 workflow_arguments: Some(
7856 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
7857 ),
7858 query_arguments: Some(
7859 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("query arguments"),
7860 ),
7861 history_events: vec![
7862 HistoryEvent {
7863 event_type: "SignalReceived".to_string(),
7864 payload: json!({
7865 "signal_id": "php-signal-1",
7866 "signal_name": "increment",
7867 "workflow_sequence": 1,
7868 "payload_codec": DEFAULT_CODEC,
7869 "arguments": encode_value_envelope(&json!([3]), DEFAULT_CODEC).expect("php avro signal")
7870 }),
7871 raw: HashMap::new(),
7872 },
7873 HistoryEvent {
7874 event_type: "SignalReceived".to_string(),
7875 payload: json!({
7876 "signal_id": "python-signal-2",
7877 "signal_name": "increment",
7878 "workflow_sequence": 2,
7879 "payload_codec": JSON_CODEC,
7880 "arguments": encode_value_envelope(&json!([5]), JSON_CODEC).expect("python json signal")
7881 }),
7882 raw: HashMap::new(),
7883 },
7884 HistoryEvent {
7885 event_type: "SignalReceived".to_string(),
7886 payload: json!({
7887 "signal_id": "rust-signal-3",
7888 "signal_name": "set",
7889 "workflow_sequence": 3,
7890 "payload_codec": DEFAULT_CODEC,
7891 "arguments": encode_value_envelope(&json!([0]), DEFAULT_CODEC).expect("rust avro signal")
7892 }),
7893 raw: HashMap::new(),
7894 },
7895 ],
7896 history_export: None,
7897 run_status: Some("completed".to_string()),
7898 };
7899
7900 let result = worker.execute_query_task(task).await.expect("query result");
7901 assert_eq!(result, json!(0));
7902 }
7903
7904 #[tokio::test]
7905 async fn replayed_queries_read_running_completed_and_cold_restarted_instance_state() {
7906 let worker = replay_counter_worker();
7907 let running_history = json!([
7908 {
7909 "type": "ActivityCompleted",
7910 "payload": {
7911 "sequence": 1,
7912 "activity_type": "load-counter",
7913 "payload_codec": "json",
7914 "result": {"codec": "json", "blob": "\"loaded\""}
7915 }
7916 },
7917 {
7918 "type": "SignalReceived",
7919 "payload": {
7920 "signal_id": "signal-3",
7921 "signal_name": "increment",
7922 "payload_codec": "json",
7923 "arguments": {"codec": "json", "blob": "[3]"}
7924 }
7925 }
7926 ]);
7927
7928 let running = worker
7929 .execute_query_task(replay_counter_query(
7930 "current",
7931 running_history.clone(),
7932 "running",
7933 ))
7934 .await
7935 .expect("running replay query");
7936 assert_eq!(
7937 running,
7938 json!({"loaded": "loaded", "count": 3, "finished": false})
7939 );
7940
7941 let detached = worker
7942 .execute_query_task(replay_counter_query(
7943 "detached-mutation",
7944 running_history.clone(),
7945 "running",
7946 ))
7947 .await
7948 .expect("query mutates only its detached state clone");
7949 assert_eq!(detached, json!(999));
7950 let failed = worker
7951 .execute_query_task(replay_counter_query(
7952 "failed-mutation",
7953 running_history.clone(),
7954 "running",
7955 ))
7956 .await
7957 .expect_err("failed query");
7958 assert_eq!(failed.reason, "query_rejected");
7959 let unchanged = worker
7960 .execute_query_task(replay_counter_query("current", running_history, "running"))
7961 .await
7962 .expect("later query reconstructs unchanged state");
7963 assert_eq!(unchanged, running);
7964
7965 let restarted_worker = replay_counter_worker();
7966 let restarted_task: QueryTask = serde_json::from_value(json!({
7967 "query_task_id": "query-after-restart",
7968 "workflow_id": "counter-1",
7969 "run_id": "run-counter-1",
7970 "workflow_type": "replay-counter",
7971 "query_name": "current",
7972 "payload_codec": "json",
7973 "workflow_arguments": {"codec": "json", "blob": "[]"},
7974 "query_arguments": {"codec": "json", "blob": "[]"},
7975 "history_events": [],
7976 "history_export": {
7977 "payloads": {"codec": "json"},
7978 "history_events": [
7979 {
7980 "type": "ActivityCompleted",
7981 "payload": {
7982 "sequence": 1,
7983 "activity_type": "load-counter",
7984 "payload_codec": "json",
7985 "result": null
7986 }
7987 },
7988 {
7989 "type": "SignalReceived",
7990 "payload": {"signal_id": "signal-3", "signal_name": "increment"}
7991 },
7992 {
7993 "type": "SignalReceived",
7994 "payload": {"signal_id": "signal-5", "signal_name": "increment"}
7995 }
7996 ],
7997 "activities": [{
7998 "sequence": 1,
7999 "activity_type": "load-counter",
8000 "payload_codec": "json",
8001 "result": {"codec": "json", "blob": "\"loaded\""}
8002 }],
8003 "signals": [
8004 {
8005 "id": "signal-3",
8006 "name": "increment",
8007 "payload_codec": "json",
8008 "arguments": "[3]"
8009 },
8010 {
8011 "id": "signal-5",
8012 "name": "increment",
8013 "payload_codec": "json",
8014 "arguments": "[5]"
8015 }
8016 ]
8017 },
8018 "run_status": "completed"
8019 }))
8020 .expect("cold replay query task");
8021 let completed = restarted_worker
8022 .execute_query_task(restarted_task)
8023 .await
8024 .expect("completed cold replay query");
8025 assert_eq!(
8026 completed,
8027 json!({"loaded": "loaded", "count": 8, "finished": true})
8028 );
8029 }
8030
8031 #[tokio::test]
8032 async fn replayed_query_replay_failures_are_machine_readable() {
8033 let worker = replay_counter_worker();
8034 let task = replay_counter_query(
8035 "current",
8036 json!([{
8037 "type": "ActivityCompleted",
8038 "payload": {
8039 "sequence": 1,
8040 "payload_codec": "json",
8041 "result": {"codec": "json", "blob": "{"}
8042 }
8043 }]),
8044 "running",
8045 );
8046 let failure = worker
8047 .execute_query_task(task)
8048 .await
8049 .expect_err("invalid replay history payload");
8050 assert_eq!(failure.reason, "query_workflow_state_unavailable");
8051 assert_eq!(failure.failure_type, "QueryWorkflowStateUnavailable");
8052 }
8053
8054 #[tokio::test]
8055 async fn query_task_restores_compact_history_from_export() {
8056 let client = Client::new("http://127.0.0.1:8080").expect("client");
8057 let mut worker = Worker::new(client, "rust-workers");
8058 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
8059 worker.register_query("counter", "current", |ctx, _args| async move {
8060 Ok(json!(ctx.signals("increment")[0][0]))
8061 });
8062 let task: QueryTask = serde_json::from_value(json!({
8063 "query_task_id": "query-export",
8064 "workflow_type": "counter",
8065 "query_name": "current",
8066 "payload_codec": "json",
8067 "workflow_arguments": {"codec": "json", "blob": "[]"},
8068 "query_arguments": {"codec": "json", "blob": "[]"},
8069 "history_events": [],
8070 "history_export": {
8071 "payloads": {"codec": "json"},
8072 "history_events": [{
8073 "type": "SignalReceived",
8074 "payload": {"signal_id": "signal-export", "signal_name": "increment"}
8075 }],
8076 "signals": [{
8077 "id": "signal-export",
8078 "name": "increment",
8079 "status": "applied",
8080 "workflow_sequence": 1,
8081 "payload_codec": "json",
8082 "arguments": "[9]"
8083 }]
8084 }
8085 }))
8086 .expect("query task");
8087
8088 let result = worker.execute_query_task(task).await.expect("query result");
8089 assert_eq!(result, json!(9));
8090 }
8091
8092 #[tokio::test]
8093 async fn query_task_failures_have_stable_reasons() {
8094 let client = Client::new("http://127.0.0.1:8080").expect("client");
8095 let mut worker = Worker::new(client, "rust-workers");
8096 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
8097 worker.register_query(
8098 "counter",
8099 "current",
8100 |_ctx, _args| async move { Ok(json!(0)) },
8101 );
8102
8103 let base_task = QueryTask {
8104 query_task_id: "query-errors".to_string(),
8105 query_task_attempt: 1,
8106 lease_owner: None,
8107 workflow_id: Some("counter-errors".to_string()),
8108 run_id: Some("run-errors".to_string()),
8109 workflow_type: "counter".to_string(),
8110 query_name: "missing".to_string(),
8111 payload_codec: JSON_CODEC.to_string(),
8112 workflow_arguments: Some(json!({"codec": "json", "blob": "[]"})),
8113 query_arguments: Some(json!({"codec": "json", "blob": "[]"})),
8114 history_events: Vec::new(),
8115 history_export: None,
8116 run_status: Some("running".to_string()),
8117 };
8118
8119 let unknown = worker
8120 .execute_query_task(base_task.clone())
8121 .await
8122 .expect_err("unknown query");
8123 assert_eq!(unknown.reason, "rejected_unknown_query");
8124
8125 let mut malformed = base_task;
8126 malformed.query_name = "current".to_string();
8127 malformed.query_arguments = Some(json!({"codec": "json", "blob": "{"}));
8128 let malformed = worker
8129 .execute_query_task(malformed)
8130 .await
8131 .expect_err("malformed payload");
8132 assert_eq!(malformed.reason, "query_payload_decode_failed");
8133
8134 let client = Client::new("http://127.0.0.1:8080").expect("client");
8135 let mut unavailable_worker = Worker::new(client, "rust-workers");
8136 unavailable_worker
8137 .register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
8138 let unavailable_task: QueryTask = serde_json::from_value(json!({
8139 "query_task_id": "query-unavailable",
8140 "workflow_type": "counter",
8141 "query_name": "current",
8142 "payload_codec": "json",
8143 "workflow_arguments": {"codec": "json", "blob": "[]"},
8144 "query_arguments": {"codec": "json", "blob": "[]"}
8145 }))
8146 .expect("query task");
8147 let unavailable = unavailable_worker
8148 .execute_query_task(unavailable_task)
8149 .await
8150 .expect_err("query handler unavailable");
8151 assert_eq!(unavailable.reason, "query_handler_unavailable");
8152 }
8153
8154 #[tokio::test]
8155 async fn client_query_decodes_result_and_typed_failure() {
8156 let server = MockWorkerServer::start();
8157 let client = Client::builder(server.base_url())
8158 .timeout(Duration::from_secs(2))
8159 .build()
8160 .expect("client");
8161
8162 let result = client
8163 .query_workflow("counter-1", "current", json!([]))
8164 .await
8165 .expect("query result");
8166 assert_eq!(result, json!({"count": 8}));
8167
8168 let error = client
8169 .query_workflow("counter-1", "missing", json!([]))
8170 .await
8171 .expect_err("unknown query");
8172 let Error::QueryFailed(failure) = error else {
8173 panic!("expected typed query failure");
8174 };
8175 assert_eq!(failure.status, 404);
8176 assert_eq!(failure.reason, "rejected_unknown_query");
8177 }
8178
8179 #[tokio::test]
8180 async fn lifecycle_commands_support_instance_and_selected_run_targets() {
8181 let server = MockWorkerServer::start();
8182 let client = Client::builder(server.base_url())
8183 .timeout(Duration::from_secs(2))
8184 .build()
8185 .expect("client");
8186
8187 let options = WorkflowCommandOptions::new()
8188 .reason("cleanup requested")
8189 .request_id("cancel-17");
8190 let cancelled = client
8191 .cancel_workflow("wf-lifecycle", options)
8192 .await
8193 .expect("instance cancellation");
8194 assert_eq!(cancelled.command, WorkflowCommandKind::Cancel);
8195 assert_eq!(cancelled.run_id.as_deref(), Some("run-current"));
8196 assert_eq!(cancelled.outcome.as_deref(), Some("cancelled"));
8197 assert_eq!(
8198 server.request_body("/api/workflows/wf-lifecycle/cancel"),
8199 json!({"reason":"cleanup requested","request_id":"cancel-17"})
8200 );
8201
8202 let terminated = client
8203 .terminate_workflow(
8204 "wf-lifecycle",
8205 WorkflowCommandOptions::new().reason("forced stop"),
8206 )
8207 .await
8208 .expect("instance termination");
8209 assert_eq!(terminated.command, WorkflowCommandKind::Terminate);
8210 assert_eq!(terminated.outcome.as_deref(), Some("terminated"));
8211
8212 client
8213 .cancel_workflow_run(
8214 "wf-lifecycle",
8215 "run-current",
8216 WorkflowCommandOptions::default(),
8217 )
8218 .await
8219 .expect("selected run cancellation");
8220 client
8221 .terminate_workflow_run(
8222 "wf-lifecycle",
8223 "run-current",
8224 WorkflowCommandOptions::default(),
8225 )
8226 .await
8227 .expect("selected run termination");
8228
8229 for (command, error) in [
8230 (
8231 WorkflowCommandKind::Cancel,
8232 client
8233 .cancel_workflow_run(
8234 "wf-lifecycle",
8235 "run-stale",
8236 WorkflowCommandOptions::default(),
8237 )
8238 .await
8239 .expect_err("stale cancellation must be rejected"),
8240 ),
8241 (
8242 WorkflowCommandKind::Terminate,
8243 client
8244 .terminate_workflow_run(
8245 "wf-lifecycle",
8246 "run-stale",
8247 WorkflowCommandOptions::default(),
8248 )
8249 .await
8250 .expect_err("stale termination must be rejected"),
8251 ),
8252 ] {
8253 let Error::WorkflowCommandRejected(rejection) = error else {
8254 panic!("expected typed command rejection");
8255 };
8256 assert_eq!(rejection.command, command);
8257 assert_eq!(rejection.status, 409);
8258 assert_eq!(rejection.reason, "historical_run_command_rejected");
8259 assert_eq!(rejection.run_id.as_deref(), Some("run-stale"));
8260 assert_eq!(rejection.target_scope.as_deref(), Some("run"));
8261 }
8262 }
8263
8264 #[tokio::test]
8265 async fn workflow_result_returns_each_typed_terminal_outcome() {
8266 let server = MockWorkerServer::start();
8267 let client = Client::builder(server.base_url())
8268 .timeout(Duration::from_secs(2))
8269 .build()
8270 .expect("client");
8271 let options = WorkflowResultOptions {
8272 poll_interval: Duration::ZERO,
8273 timeout: Duration::from_secs(1),
8274 };
8275
8276 let failed = WorkflowHandle {
8277 client: client.clone(),
8278 workflow_id: "wf-failed".to_string(),
8279 run_id: Some("run-failed".to_string()),
8280 workflow_type: "failure".to_string(),
8281 }
8282 .result(options)
8283 .await
8284 .expect_err("failed outcome");
8285 let Error::WorkflowFailed(failure) = failed else {
8286 panic!("expected WorkflowFailed");
8287 };
8288 assert_eq!(failure.workflow_id, "wf-failed");
8289 assert_eq!(failure.run_id.as_deref(), Some("run-failed"));
8290 assert_eq!(failure.failure_id.as_deref(), Some("failure-17"));
8291 assert_eq!(failure.failure_category.as_deref(), Some("application"));
8292 assert_eq!(failure.exception_type.as_deref(), Some("PaymentError"));
8293 assert_eq!(
8294 failure.exception_class.as_deref(),
8295 Some("billing::PaymentError")
8296 );
8297 assert_eq!(failure.non_retryable, Some(true));
8298
8299 for (workflow_id, expected_kind, expected_reason) in [
8300 (
8301 "wf-cancelled",
8302 WorkflowTerminalKind::Cancelled,
8303 "cleanup requested",
8304 ),
8305 (
8306 "wf-terminated",
8307 WorkflowTerminalKind::Terminated,
8308 "forced stop",
8309 ),
8310 (
8311 "wf-timed-out",
8312 WorkflowTerminalKind::TimedOut,
8313 "run_timeout",
8314 ),
8315 ] {
8316 let error = WorkflowHandle {
8317 client: client.clone(),
8318 workflow_id: workflow_id.to_string(),
8319 run_id: None,
8320 workflow_type: "terminal".to_string(),
8321 }
8322 .result(options)
8323 .await
8324 .expect_err("typed terminal outcome");
8325 let outcome = match error {
8326 Error::WorkflowCancelled(outcome) => outcome,
8327 Error::WorkflowTerminated(outcome) => outcome,
8328 Error::WorkflowTimedOut(outcome) => outcome,
8329 other => panic!("unexpected terminal error: {other}"),
8330 };
8331 assert_eq!(outcome.kind, expected_kind);
8332 assert_eq!(outcome.workflow_id, workflow_id);
8333 assert_eq!(outcome.reason, expected_reason);
8334 }
8335
8336 let wait_timeout = WorkflowHandle {
8337 client,
8338 workflow_id: "wf-waiting".to_string(),
8339 run_id: Some("run-waiting".to_string()),
8340 workflow_type: "waiting".to_string(),
8341 }
8342 .result(WorkflowResultOptions {
8343 poll_interval: Duration::ZERO,
8344 timeout: Duration::ZERO,
8345 })
8346 .await
8347 .expect_err("client wait timeout");
8348 let Error::WorkflowTimedOut(timeout) = wait_timeout else {
8349 panic!("expected typed client timeout");
8350 };
8351 assert_eq!(timeout.reason, "result_wait_timeout");
8352 assert_eq!(timeout.failure_category.as_deref(), Some("client_timeout"));
8353 assert_eq!(timeout.run_id.as_deref(), Some("run-waiting"));
8354 }
8355
8356 #[tokio::test]
8357 async fn workflow_result_awaits_the_handle_selected_run() {
8358 let server = MockWorkerServer::start();
8359 let client = Client::builder(server.base_url())
8360 .timeout(Duration::from_secs(2))
8361 .build()
8362 .expect("client");
8363
8364 let error = WorkflowHandle {
8365 client,
8366 workflow_id: "wf-selected".to_string(),
8367 run_id: Some("run-selected".to_string()),
8368 workflow_type: "selected".to_string(),
8369 }
8370 .result(WorkflowResultOptions {
8371 poll_interval: Duration::ZERO,
8372 timeout: Duration::from_secs(1),
8373 })
8374 .await
8375 .expect_err("the selected run is cancelled even though the current run completed");
8376
8377 let Error::WorkflowCancelled(outcome) = error else {
8378 panic!("expected selected run cancellation");
8379 };
8380 assert_eq!(outcome.run_id.as_deref(), Some("run-selected"));
8381 assert_eq!(outcome.reason, "selected run cancelled");
8382 assert_eq!(
8383 server.request_count("/api/workflows/wf-selected/runs/run-selected"),
8384 1
8385 );
8386 assert_eq!(server.request_count("/api/workflows/wf-selected"), 0);
8387 }
8388
8389 #[tokio::test]
8390 async fn poll_responses_decode_http_conflict_drain_as_a_stable_stop() {
8391 let server = MockWorkerServer::draining_polls();
8392 let client = Client::builder(server.base_url())
8393 .timeout(Duration::from_secs(2))
8394 .build()
8395 .expect("client");
8396
8397 let workflow = client
8398 .poll_workflow_task_response("draining-worker", "rust-workers", Duration::ZERO)
8399 .await
8400 .expect("workflow drain response");
8401 let activity = client
8402 .poll_activity_task_response("draining-worker", "rust-workers", Duration::ZERO)
8403 .await
8404 .expect("activity drain response");
8405 let query = client
8406 .poll_query_task_response("draining-worker", "rust-workers", Duration::ZERO)
8407 .await
8408 .expect("query drain response");
8409
8410 for outcome in [workflow.outcome(), activity.outcome(), query.outcome()] {
8411 assert_eq!(
8412 outcome,
8413 WorkerPollOutcome::Stop {
8414 poll_status: Some("draining".to_string()),
8415 reason: Some("worker_draining".to_string()),
8416 }
8417 );
8418 }
8419
8420 assert!(client
8421 .poll_workflow_task("draining-worker", "rust-workers", Duration::ZERO)
8422 .await
8423 .expect("compatibility poll")
8424 .is_none());
8425 }
8426
8427 #[tokio::test]
8428 async fn managed_worker_honors_drain_stop_for_every_task_family() {
8429 let server = MockWorkerServer::draining_polls();
8430 let client = Client::builder(server.base_url())
8431 .timeout(Duration::from_secs(2))
8432 .build()
8433 .expect("client");
8434
8435 let mut workflow_worker = Worker::new(client.clone(), "rust-workers")
8436 .worker_id("draining-workflow-worker")
8437 .poll_timeout(Duration::ZERO);
8438 workflow_worker.register_workflow("counter", |_ctx, _args| async { Ok(Value::Null) });
8439 workflow_worker
8440 .run()
8441 .await
8442 .expect("workflow drain is a clean stop");
8443
8444 let mut activity_worker = Worker::new(client.clone(), "rust-workers")
8445 .worker_id("draining-activity-worker")
8446 .poll_timeout(Duration::ZERO);
8447 activity_worker.register_activity("write", |_ctx, _args| async { Ok(Value::Null) });
8448 activity_worker
8449 .run()
8450 .await
8451 .expect("activity drain is a clean stop");
8452
8453 let mut query_worker = Worker::new(client, "rust-workers")
8454 .worker_id("draining-query-worker")
8455 .poll_timeout(Duration::ZERO);
8456 query_worker.register_query("counter", "current", |_ctx, _args| async {
8457 Ok(Value::Null)
8458 });
8459 query_worker
8460 .run()
8461 .await
8462 .expect("query drain is a clean stop");
8463 }
8464
8465 #[tokio::test]
8466 async fn activity_cancellation_and_late_completion_remain_machine_readable() {
8467 let server = MockWorkerServer::start();
8468 let client = Client::builder(server.base_url())
8469 .timeout(Duration::from_secs(2))
8470 .build()
8471 .expect("client");
8472
8473 let heartbeat = client
8474 .heartbeat_activity_task(
8475 "activity-cancel",
8476 "attempt-cancel",
8477 "rust-worker",
8478 json!({"stage":"cleanup"}),
8479 )
8480 .await
8481 .expect("cancellation heartbeat");
8482 assert!(heartbeat.cancel_requested);
8483 assert!(heartbeat.should_stop());
8484 assert_eq!(heartbeat.reason.as_deref(), Some("run_cancelled"));
8485 assert_eq!(heartbeat.run_closed_reason.as_deref(), Some("cancelled"));
8486
8487 let error = client
8488 .complete_activity_task(
8489 "activity-cancel",
8490 "attempt-cancel",
8491 "rust-worker",
8492 json!({"late":true}),
8493 JSON_CODEC,
8494 )
8495 .await
8496 .expect_err("late completion must be refused");
8497 assert!(activity_task_rejection_is_final(&error));
8498 let Error::ActivityTaskRejected(rejection) = error else {
8499 panic!("expected typed activity rejection");
8500 };
8501 assert_eq!(rejection.status, 409);
8502 assert_eq!(rejection.reason, "run_cancelled");
8503 assert!(rejection.cancel_requested);
8504 assert_eq!(rejection.can_continue, Some(false));
8505 }
8506
8507 #[tokio::test]
8508 async fn managed_worker_survives_late_completion_and_restart_during_cancellation() {
8509 let server = MockWorkerServer::cancelled_activity();
8510 let client = Client::builder(server.base_url())
8511 .timeout(Duration::from_secs(2))
8512 .build()
8513 .expect("client");
8514 let cancellation_observed = Arc::new(AtomicBool::new(false));
8515 let observed = Arc::clone(&cancellation_observed);
8516 let mut worker = Worker::new(client.clone(), "rust-workers")
8517 .worker_id("rust-cancel-worker")
8518 .poll_timeout(Duration::from_millis(10));
8519 worker.register_activity("cancel-aware", move |ctx, _args| {
8520 let observed = Arc::clone(&observed);
8521 async move {
8522 let heartbeat = ctx.heartbeat(json!({"stage":"running"})).await?;
8523 observed.store(heartbeat.should_stop(), Ordering::SeqCst);
8524 Ok(json!({"late":"completion"}))
8525 }
8526 });
8527
8528 assert_eq!(
8529 worker.run_once().await.expect("cancelled attempt handled"),
8530 1
8531 );
8532 assert!(cancellation_observed.load(Ordering::SeqCst));
8533 assert_eq!(
8534 server.request_count("/api/worker/activity-tasks/activity-cancel/complete"),
8535 1
8536 );
8537
8538 let mut restarted = Worker::new(client, "rust-workers")
8539 .worker_id("rust-cancel-worker-restarted")
8540 .poll_timeout(Duration::from_millis(10));
8541 restarted.register_activity("cancel-aware", |_ctx, _args| async move { Ok(Value::Null) });
8542 assert_eq!(
8543 restarted
8544 .run_once()
8545 .await
8546 .expect("replacement worker continues polling"),
8547 0
8548 );
8549 }
8550
8551 #[tokio::test]
8552 async fn baseline_worker_endpoints_send_the_baseline_protocol() {
8553 let server = MockWorkerServer::start();
8554 let client = Client::builder(server.base_url())
8555 .timeout(Duration::from_secs(2))
8556 .build()
8557 .expect("client");
8558
8559 client
8560 .register_worker("capture-worker", "capture", vec![], vec![], 1, 1)
8561 .await
8562 .expect("register");
8563 client
8564 .heartbeat_worker("capture-worker", 1, 1)
8565 .await
8566 .expect("heartbeat");
8567 client
8568 .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
8569 .await
8570 .expect("workflow poll");
8571 client
8572 .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
8573 .await
8574 .expect("activity poll");
8575
8576 for path in [
8577 "/api/worker/register",
8578 "/api/worker/heartbeat",
8579 "/api/worker/workflow-tasks/poll",
8580 "/api/worker/activity-tasks/poll",
8581 ] {
8582 assert_eq!(
8583 server.worker_protocol_for(path).as_deref(),
8584 Some(WORKER_PROTOCOL_VERSION),
8585 "unexpected protocol for {path}"
8586 );
8587 }
8588
8589 assert_eq!(
8590 server.request_body("/api/worker/workflow-tasks/poll")["timeout_seconds"],
8591 1
8592 );
8593 assert_eq!(
8594 server.request_body("/api/worker/activity-tasks/poll")["timeout_seconds"],
8595 1
8596 );
8597 }
8598
8599 #[tokio::test]
8600 async fn query_task_endpoints_send_the_query_feature_protocol() {
8601 let server = MockWorkerServer::start();
8602 let client = Client::builder(server.base_url())
8603 .timeout(Duration::from_secs(2))
8604 .build()
8605 .expect("client");
8606
8607 client
8608 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
8609 .await
8610 .expect("query poll");
8611 client
8612 .complete_query_task("query-capture", "capture-worker", 1, json!(8), JSON_CODEC)
8613 .await
8614 .expect("query complete");
8615 client
8616 .fail_query_task(
8617 "query-capture",
8618 "capture-worker",
8619 1,
8620 "failed",
8621 "query_rejected",
8622 "QueryFailed",
8623 )
8624 .await
8625 .expect("query fail");
8626
8627 for path in [
8628 "/api/worker/query-tasks/poll",
8629 "/api/worker/query-tasks/query-capture/complete",
8630 "/api/worker/query-tasks/query-capture/fail",
8631 ] {
8632 assert_eq!(
8633 server.worker_protocol_for(path).as_deref(),
8634 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
8635 "unexpected protocol for {path}"
8636 );
8637 }
8638
8639 assert_eq!(
8640 server.request_body("/api/worker/query-tasks/poll")["timeout_seconds"],
8641 1
8642 );
8643 }
8644
8645 #[tokio::test]
8646 async fn query_protocol_rejection_from_older_server_is_typed() {
8647 let server = MockWorkerServer::reject_query_protocol();
8648 let client = Client::builder(server.base_url())
8649 .timeout(Duration::from_secs(2))
8650 .build()
8651 .expect("client");
8652
8653 let error = client
8654 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
8655 .await
8656 .expect_err("server below query protocol floor must reject");
8657 let Error::Protocol(failure) = error else {
8658 panic!("expected typed protocol failure");
8659 };
8660
8661 assert_eq!(failure.status, 400);
8662 assert_eq!(failure.reason, "unsupported_protocol_version");
8663 assert_eq!(failure.supported_version.as_deref(), Some("1.7"));
8664 assert_eq!(
8665 failure.requested_version.as_deref(),
8666 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
8667 );
8668 assert_eq!(
8669 server
8670 .worker_protocol_for("/api/worker/query-tasks/poll")
8671 .as_deref(),
8672 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
8673 );
8674 }
8675
8676 #[tokio::test]
8677 async fn run_once_without_query_handlers_keeps_pre_query_server_compatibility() {
8678 let server = MockWorkerServer::reject_query_protocol();
8679 let client = Client::builder(server.base_url())
8680 .timeout(Duration::from_secs(2))
8681 .build()
8682 .expect("client");
8683 let mut worker = Worker::new(client, "rust-workers")
8684 .worker_id("baseline-worker")
8685 .poll_timeout(Duration::from_millis(10));
8686
8687 worker.register_workflow("baseline.workflow", |_ctx, _input| async move {
8688 Ok(Value::Null)
8689 });
8690
8691 assert_eq!(worker.run_once().await.expect("baseline run once"), 0);
8692 assert_eq!(
8693 server
8694 .worker_protocol_for("/api/worker/workflow-tasks/poll")
8695 .as_deref(),
8696 Some(WORKER_PROTOCOL_VERSION)
8697 );
8698 assert_eq!(
8699 server.worker_protocol_for("/api/worker/query-tasks/poll"),
8700 None,
8701 "a worker without query handlers must not use the query-task endpoint"
8702 );
8703 }
8704
8705 #[tokio::test]
8706 async fn completion_time_query_rejection_is_typed_without_stopping_worker() {
8707 let server = MockWorkerServer::reject_query_completion();
8708 let client = Client::builder(server.base_url())
8709 .timeout(Duration::from_secs(2))
8710 .build()
8711 .expect("client");
8712
8713 let error = client
8714 .complete_query_task("query-late", "late-worker", 1, json!(8), JSON_CODEC)
8715 .await
8716 .expect_err("expired completion must be rejected");
8717 let Error::QueryFailed(failure) = error else {
8718 panic!("expected typed query failure");
8719 };
8720 assert_eq!(failure.status, 409);
8721 assert_eq!(failure.reason, "query_task_timed_out");
8722
8723 let mut worker = Worker::new(client, "rust-workers")
8724 .worker_id("late-worker")
8725 .poll_timeout(Duration::from_millis(10));
8726 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
8727 worker.register_query(
8728 "counter",
8729 "current",
8730 |_ctx, _args| async move { Ok(json!(8)) },
8731 );
8732
8733 assert_eq!(worker.run_once().await.expect("late task is handled"), 1);
8734 assert_eq!(
8735 worker
8736 .run_once()
8737 .await
8738 .expect("worker continues after late completion"),
8739 0
8740 );
8741 assert_eq!(
8742 server.request_count("/api/worker/query-tasks/query-late/complete"),
8743 2
8744 );
8745 assert_eq!(
8746 server.request_count("/api/worker/query-tasks/query-late/fail"),
8747 0,
8748 "a server completion rejection must not be reported as an encoding failure"
8749 );
8750 }
8751
8752 #[tokio::test]
8753 async fn activity_only_worker_can_shutdown_without_workflow_poller() {
8754 let server = MockWorkerServer::start();
8755 let client = Client::builder(server.base_url())
8756 .timeout(Duration::from_secs(2))
8757 .build()
8758 .expect("client");
8759 let mut worker = Worker::new(client, "rust-workers")
8760 .worker_id("activity-only-worker")
8761 .poll_timeout(Duration::from_millis(10));
8762
8763 worker.register_activity(
8764 "activity.only",
8765 |_ctx, _args| async move { Ok(Value::Null) },
8766 );
8767
8768 worker.run_until(async {}).await.expect("run worker");
8769 }
8770
8771 #[tokio::test]
8772 async fn workflow_only_worker_can_shutdown_without_activity_poller() {
8773 let server = MockWorkerServer::start();
8774 let client = Client::builder(server.base_url())
8775 .timeout(Duration::from_secs(2))
8776 .build()
8777 .expect("client");
8778 let mut worker = Worker::new(client, "rust-workers")
8779 .worker_id("workflow-only-worker")
8780 .poll_timeout(Duration::from_millis(10));
8781
8782 worker.register_workflow(
8783 "workflow.only",
8784 |_ctx, _input| async move { Ok(Value::Null) },
8785 );
8786
8787 worker.run_until(async {}).await.expect("run worker");
8788 }
8789
8790 #[tokio::test]
8791 async fn worker_heartbeat_observer_receives_server_acknowledgements() {
8792 let server = MockWorkerServer::start();
8793 let client = Client::builder(server.base_url())
8794 .timeout(Duration::from_secs(2))
8795 .build()
8796 .expect("client");
8797 let observations = Arc::new(Mutex::new(Vec::new()));
8798 let observed = Arc::clone(&observations);
8799 let mut worker = Worker::new(client, "rust-workers")
8800 .worker_id("observed-heartbeat-worker")
8801 .poll_timeout(Duration::from_millis(10))
8802 .on_worker_heartbeat(move |observation| {
8803 observed
8804 .lock()
8805 .expect("heartbeat observations")
8806 .push(observation.clone());
8807 });
8808
8809 worker.register_workflow("workflow.observed", |_ctx, _input| async move {
8810 Ok(Value::Null)
8811 });
8812 worker
8813 .run_until(tokio::time::sleep(Duration::from_millis(20)))
8814 .await
8815 .expect("run worker");
8816
8817 let observations = observations.lock().expect("heartbeat observations");
8818 let first = observations.first().expect("heartbeat acknowledgement");
8819 assert_eq!(first.worker_id, "observed-heartbeat-worker");
8820 assert_eq!(first.task_queue, "rust-workers");
8821 assert!(first.acknowledged_at_unix_millis > 0);
8822 assert_eq!(first.acknowledgement, json!({}));
8823 }
8824
8825 #[tokio::test]
8826 async fn query_enabled_worker_stays_live_when_signal_replay_emits_no_commands() {
8827 let server = MockWorkerServer::waiting_query_worker();
8828 let client = Client::builder(server.base_url())
8829 .timeout(Duration::from_secs(2))
8830 .build()
8831 .expect("client");
8832 let observations = Arc::new(Mutex::new(Vec::new()));
8833 let observed = Arc::clone(&observations);
8834 let mut worker = Worker::new(client, "rust-snapshot-workers")
8835 .worker_id("rust-snapshot-worker")
8836 .poll_timeout(Duration::from_millis(10))
8837 .on_worker_heartbeat(move |observation| {
8838 observed
8839 .lock()
8840 .expect("heartbeat observations")
8841 .push(observation.clone());
8842 });
8843
8844 worker.register_workflow("snapshot", |ctx, _input| async move {
8845 ctx.wait_signal("finish").await?;
8846 Ok(json!({"status": "finished"}))
8847 });
8848 worker.register_query("snapshot", "current", |ctx, _args| async move {
8849 let current = ctx
8850 .signals("increment")
8851 .iter()
8852 .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
8853 .sum::<i64>();
8854 Ok(json!(current))
8855 });
8856
8857 worker
8858 .run_until(tokio::time::sleep(Duration::from_millis(3_200)))
8859 .await
8860 .expect("pending workflow and query poller must remain live until shutdown");
8861
8862 assert!(
8863 observations.lock().expect("heartbeat observations").len() >= 4,
8864 "the immediate heartbeat and at least three advertised one-second intervals must be acknowledged"
8865 );
8866 assert!(
8867 server.request_count("/api/worker/workflow-tasks/poll") >= 3,
8868 "workflow polling must continue after empty replay acknowledgements"
8869 );
8870 assert!(
8871 server.request_count("/api/worker/query-tasks/poll") >= 2,
8872 "query polling must continue after serving the current query"
8873 );
8874 assert_eq!(
8875 server.request_body("/api/worker/register")["capabilities"],
8876 json!([QUERY_TASKS_CAPABILITY])
8877 );
8878
8879 for task_id in ["snapshot-wait-3", "snapshot-wait-5"] {
8880 let fail_path = format!("/api/worker/workflow-tasks/{task_id}/fail");
8881 let completion_path = format!("/api/worker/workflow-tasks/{task_id}/complete");
8882 let failure = server.request_body(&fail_path);
8883 assert_eq!(
8884 failure["failure"]["type"],
8885 WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE
8886 );
8887 assert_eq!(server.request_count(&completion_path), 0);
8888 }
8889
8890 let query_completion =
8891 server.request_body("/api/worker/query-tasks/snapshot-current/complete");
8892 assert_eq!(query_completion["result"], json!(8));
8893 }
8894
8895 #[tokio::test]
8896 async fn worker_retries_poll_and_heartbeat_transport_failures_independently() {
8897 let server = MockWorkerServer::transient_worker_failures();
8898 let client = Client::builder(server.base_url())
8899 .timeout(Duration::from_secs(2))
8900 .build()
8901 .expect("client");
8902 let mut worker = Worker::new(client, "rust-workers")
8903 .worker_id("retry-worker")
8904 .poll_timeout(Duration::from_millis(10))
8905 .retry_policy(WorkerRetryPolicy {
8906 max_retries: 2,
8907 initial_backoff: Duration::from_millis(1),
8908 max_backoff: Duration::from_millis(1),
8909 });
8910 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
8911 worker.register_activity(
8912 "counter.activity",
8913 |_ctx, _input| async move { Ok(Value::Null) },
8914 );
8915 worker.register_query(
8916 "counter",
8917 "current",
8918 |_ctx, _args| async move { Ok(json!(8)) },
8919 );
8920
8921 worker
8922 .run_until(tokio::time::sleep(Duration::from_millis(75)))
8923 .await
8924 .expect("transient failures must not stop the worker");
8925
8926 for path in [
8927 "/api/worker/heartbeat",
8928 "/api/worker/workflow-tasks/poll",
8929 "/api/worker/activity-tasks/poll",
8930 "/api/worker/query-tasks/poll",
8931 ] {
8932 assert!(
8933 server.request_count(path) >= 2,
8934 "{path} must continue after its transient failure"
8935 );
8936 }
8937 }
8938
8939 #[tokio::test]
8940 async fn worker_bounds_transport_retries() {
8941 let server = MockWorkerServer::unavailable_polls();
8942 let client = Client::builder(server.base_url())
8943 .timeout(Duration::from_secs(2))
8944 .build()
8945 .expect("client");
8946 let mut worker = Worker::new(client, "rust-workers")
8947 .worker_id("bounded-retry-worker")
8948 .poll_timeout(Duration::from_millis(10))
8949 .retry_policy(WorkerRetryPolicy {
8950 max_retries: 2,
8951 initial_backoff: Duration::from_millis(1),
8952 max_backoff: Duration::from_millis(1),
8953 });
8954 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
8955
8956 let error = worker.run().await.expect_err("retry bound must terminate");
8957 assert!(matches!(error, Error::Transport(_)));
8958 assert_eq!(
8959 server.request_count("/api/worker/workflow-tasks/poll"),
8960 3,
8961 "one initial request plus two retries"
8962 );
8963 }
8964
8965 #[tokio::test]
8966 async fn worker_does_not_retry_authentication_failures() {
8967 let server = MockWorkerServer::unauthorized_polls();
8968 let client = Client::builder(server.base_url())
8969 .timeout(Duration::from_secs(2))
8970 .build()
8971 .expect("client");
8972 let mut worker = Worker::new(client, "rust-workers")
8973 .worker_id("unauthorized-worker")
8974 .poll_timeout(Duration::from_millis(10));
8975 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
8976
8977 let error = worker
8978 .run()
8979 .await
8980 .expect_err("authentication must terminate");
8981 let Error::Http { status, body } = error else {
8982 panic!("expected stable HTTP authentication error");
8983 };
8984 assert_eq!(status, reqwest::StatusCode::UNAUTHORIZED);
8985 assert!(body.contains("authentication_failed"));
8986 assert_eq!(
8987 server.request_count("/api/worker/workflow-tasks/poll"),
8988 1,
8989 "authentication failures must not be retried"
8990 );
8991 }
8992
8993 #[derive(Clone, Debug)]
8994 struct CapturedRequest {
8995 path: String,
8996 worker_protocol: Option<String>,
8997 body: String,
8998 }
8999
9000 struct MockWorkerServer {
9001 addr: SocketAddr,
9002 stop: Arc<AtomicBool>,
9003 requests: Arc<Mutex<Vec<CapturedRequest>>>,
9004 thread: Option<thread::JoinHandle<()>>,
9005 }
9006
9007 #[derive(Clone, Copy, Default)]
9008 struct MockWorkerBehavior {
9009 reject_query_protocol: bool,
9010 reject_query_completion: bool,
9011 waiting_query_worker: bool,
9012 poll_failures_per_path: usize,
9013 heartbeat_failures: usize,
9014 unauthorized_polls: bool,
9015 cancelled_activity: bool,
9016 draining_polls: bool,
9017 }
9018
9019 impl MockWorkerServer {
9020 fn start() -> Self {
9021 Self::start_with_behavior(MockWorkerBehavior::default())
9022 }
9023
9024 fn reject_query_protocol() -> Self {
9025 Self::start_with_behavior(MockWorkerBehavior {
9026 reject_query_protocol: true,
9027 ..MockWorkerBehavior::default()
9028 })
9029 }
9030
9031 fn reject_query_completion() -> Self {
9032 Self::start_with_behavior(MockWorkerBehavior {
9033 reject_query_completion: true,
9034 ..MockWorkerBehavior::default()
9035 })
9036 }
9037
9038 fn waiting_query_worker() -> Self {
9039 Self::start_with_behavior(MockWorkerBehavior {
9040 waiting_query_worker: true,
9041 ..MockWorkerBehavior::default()
9042 })
9043 }
9044
9045 fn transient_worker_failures() -> Self {
9046 Self::start_with_behavior(MockWorkerBehavior {
9047 poll_failures_per_path: 1,
9048 heartbeat_failures: 1,
9049 ..MockWorkerBehavior::default()
9050 })
9051 }
9052
9053 fn unavailable_polls() -> Self {
9054 Self::start_with_behavior(MockWorkerBehavior {
9055 poll_failures_per_path: usize::MAX,
9056 ..MockWorkerBehavior::default()
9057 })
9058 }
9059
9060 fn unauthorized_polls() -> Self {
9061 Self::start_with_behavior(MockWorkerBehavior {
9062 unauthorized_polls: true,
9063 ..MockWorkerBehavior::default()
9064 })
9065 }
9066
9067 fn cancelled_activity() -> Self {
9068 Self::start_with_behavior(MockWorkerBehavior {
9069 cancelled_activity: true,
9070 ..MockWorkerBehavior::default()
9071 })
9072 }
9073
9074 fn draining_polls() -> Self {
9075 Self::start_with_behavior(MockWorkerBehavior {
9076 draining_polls: true,
9077 ..MockWorkerBehavior::default()
9078 })
9079 }
9080
9081 fn start_with_behavior(behavior: MockWorkerBehavior) -> Self {
9082 let listener = TcpListener::bind("127.0.0.1:0").expect("bind mock server");
9083 listener
9084 .set_nonblocking(true)
9085 .expect("configure mock listener");
9086 let addr = listener.local_addr().expect("mock server address");
9087 let stop = Arc::new(AtomicBool::new(false));
9088 let server_stop = Arc::clone(&stop);
9089 let requests = Arc::new(Mutex::new(Vec::new()));
9090 let server_requests = Arc::clone(&requests);
9091 let thread = thread::spawn(move || {
9092 while !server_stop.load(Ordering::SeqCst) {
9093 match listener.accept() {
9094 Ok((mut stream, _)) => {
9095 handle_mock_worker_request(&mut stream, &server_requests, behavior)
9096 }
9097 Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
9098 thread::sleep(Duration::from_millis(5));
9099 }
9100 Err(_) => break,
9101 }
9102 }
9103 });
9104
9105 Self {
9106 addr,
9107 stop,
9108 requests,
9109 thread: Some(thread),
9110 }
9111 }
9112
9113 fn base_url(&self) -> String {
9114 format!("http://{}", self.addr)
9115 }
9116
9117 fn worker_protocol_for(&self, path: &str) -> Option<String> {
9118 self.requests
9119 .lock()
9120 .expect("captured requests")
9121 .iter()
9122 .find(|request| request.path == path)
9123 .and_then(|request| request.worker_protocol.clone())
9124 }
9125
9126 fn request_count(&self, path: &str) -> usize {
9127 self.requests
9128 .lock()
9129 .expect("captured requests")
9130 .iter()
9131 .filter(|request| request.path == path)
9132 .count()
9133 }
9134
9135 fn request_body(&self, path: &str) -> Value {
9136 let requests = self.requests.lock().expect("captured requests");
9137 let body = &requests
9138 .iter()
9139 .find(|request| request.path == path)
9140 .unwrap_or_else(|| panic!("missing request for {path}"))
9141 .body;
9142 serde_json::from_str(body).unwrap_or_else(|error| {
9143 panic!("invalid JSON request body for {path}: {error}: {body:?}")
9144 })
9145 }
9146 }
9147
9148 impl Drop for MockWorkerServer {
9149 fn drop(&mut self) {
9150 self.stop.store(true, Ordering::SeqCst);
9151 let _ = TcpStream::connect(self.addr);
9152
9153 if let Some(thread) = self.thread.take() {
9154 thread.join().expect("join mock server");
9155 }
9156 }
9157 }
9158
9159 fn handle_mock_worker_request(
9160 stream: &mut TcpStream,
9161 requests: &Arc<Mutex<Vec<CapturedRequest>>>,
9162 behavior: MockWorkerBehavior,
9163 ) {
9164 let _ = stream.set_read_timeout(Some(Duration::from_millis(200)));
9165 let mut buffer = [0_u8; 8192];
9166 let mut request = Vec::new();
9167
9168 loop {
9169 match stream.read(&mut buffer) {
9170 Ok(0) => break,
9171 Ok(read) => {
9172 request.extend_from_slice(&buffer[..read]);
9173 if mock_request_is_complete(&request) {
9174 break;
9175 }
9176 }
9177 Err(error)
9178 if matches!(
9179 error.kind(),
9180 std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
9181 ) =>
9182 {
9183 break;
9184 }
9185 Err(_) => return,
9186 }
9187 }
9188
9189 let request = String::from_utf8_lossy(&request);
9190 let body = request
9191 .split_once("\r\n\r\n")
9192 .map(|(_, body)| body)
9193 .unwrap_or_default();
9194 let path = request
9195 .lines()
9196 .next()
9197 .and_then(|line| line.split_whitespace().nth(1))
9198 .unwrap_or_default();
9199 let worker_protocol = request.lines().find_map(|line| {
9200 let (name, value) = line.split_once(':')?;
9201 name.eq_ignore_ascii_case("X-Durable-Workflow-Protocol-Version")
9202 .then(|| value.trim().to_string())
9203 });
9204 let request_number = {
9205 let mut requests = requests.lock().expect("captured requests");
9206 requests.push(CapturedRequest {
9207 path: path.to_string(),
9208 worker_protocol: worker_protocol.clone(),
9209 body: body.to_string(),
9210 });
9211 requests
9212 .iter()
9213 .filter(|request| request.path == path)
9214 .count()
9215 };
9216
9217 let is_poll = matches!(
9218 path,
9219 "/api/worker/workflow-tasks/poll"
9220 | "/api/worker/activity-tasks/poll"
9221 | "/api/worker/query-tasks/poll"
9222 );
9223 if is_poll && request_number <= behavior.poll_failures_per_path {
9224 return;
9225 }
9226 if path == "/api/worker/heartbeat" && request_number <= behavior.heartbeat_failures {
9227 return;
9228 }
9229 if behavior.unauthorized_polls && is_poll {
9230 write_mock_response(
9231 stream,
9232 "401 Unauthorized",
9233 r#"{"reason":"authentication_failed","message":"invalid worker token"}"#,
9234 );
9235 return;
9236 }
9237 if behavior.draining_polls && is_poll {
9238 write_mock_response(
9239 stream,
9240 "409 Conflict",
9241 r#"{"task":null,"poll_status":"draining","reason":"worker_draining","worker_status":"draining","drain_intent":"draining"}"#,
9242 );
9243 return;
9244 }
9245
9246 if behavior.reject_query_protocol && path.starts_with("/api/worker/query-tasks/") {
9247 let requested_version = worker_protocol.as_deref().unwrap_or("missing");
9248 let body = format!(
9249 r#"{{"reason":"unsupported_protocol_version","message":"Query tasks require worker protocol 1.8 or newer.","supported_version":"1.7","requested_version":"{requested_version}"}}"#
9250 );
9251 write_mock_response(stream, "400 Bad Request", &body);
9252 return;
9253 }
9254
9255 if behavior.reject_query_completion && path == "/api/worker/query-tasks/query-late/complete"
9256 {
9257 write_mock_response(
9258 stream,
9259 "409 Conflict",
9260 r#"{"reason":"query_task_timed_out","message":"query task timed out before completion"}"#,
9261 );
9262 return;
9263 }
9264
9265 if behavior.waiting_query_worker {
9266 if path == "/api/worker/workflow-tasks/poll" && request_number <= 2 {
9267 let amounts = if request_number == 1 {
9268 vec![3]
9269 } else {
9270 vec![3, 5]
9271 };
9272 let task_id = if request_number == 1 {
9273 "snapshot-wait-3"
9274 } else {
9275 "snapshot-wait-5"
9276 };
9277 let history_events = std::iter::once(json!({
9278 "event_type": "SignalWaitOpened",
9279 "payload": {"sequence": 1, "signal_name": "finish"}
9280 }))
9281 .chain(amounts.iter().enumerate().map(|(index, amount)| {
9282 json!({
9283 "event_type": "SignalReceived",
9284 "payload": {
9285 "signal_id": format!("increment-{amount}"),
9286 "signal_name": "increment",
9287 "workflow_sequence": index + 2,
9288 "payload_codec": DEFAULT_CODEC,
9289 "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
9290 .expect("Avro signal envelope")
9291 }
9292 })
9293 }))
9294 .collect::<Vec<_>>();
9295 let body = json!({
9296 "task": {
9297 "task_id": task_id,
9298 "workflow_id": "snapshot-1",
9299 "run_id": "snapshot-run-1",
9300 "workflow_type": "snapshot",
9301 "payload_codec": DEFAULT_CODEC,
9302 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
9303 .expect("Avro workflow arguments"),
9304 "history_events": history_events,
9305 "workflow_task_attempt": 1,
9306 "workflow_signal_id": format!("increment-{}", amounts.last().expect("amount")),
9307 "signal_name": "increment",
9308 "signal_arguments": encode_value_envelope(
9309 &json!([amounts.last().expect("amount")]),
9310 DEFAULT_CODEC,
9311 )
9312 .expect("Avro resume signal"),
9313 "lease_owner": "rust-snapshot-worker"
9314 }
9315 })
9316 .to_string();
9317 write_mock_response(stream, "200 OK", &body);
9318 return;
9319 }
9320
9321 if path == "/api/worker/query-tasks/poll" && request_number == 1 {
9322 let history_events = [3, 5]
9323 .into_iter()
9324 .enumerate()
9325 .map(|(index, amount)| {
9326 json!({
9327 "event_type": "SignalReceived",
9328 "payload": {
9329 "signal_id": format!("increment-{amount}"),
9330 "signal_name": "increment",
9331 "workflow_sequence": index + 2,
9332 "payload_codec": DEFAULT_CODEC,
9333 "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
9334 .expect("Avro query signal envelope")
9335 }
9336 })
9337 })
9338 .collect::<Vec<_>>();
9339 let body = json!({
9340 "task": {
9341 "query_task_id": "snapshot-current",
9342 "query_task_attempt": 1,
9343 "lease_owner": "rust-snapshot-worker",
9344 "workflow_id": "snapshot-1",
9345 "run_id": "snapshot-run-1",
9346 "workflow_type": "snapshot",
9347 "query_name": "current",
9348 "payload_codec": DEFAULT_CODEC,
9349 "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
9350 .expect("Avro workflow arguments"),
9351 "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
9352 .expect("Avro query arguments"),
9353 "history_events": history_events,
9354 "run_status": "waiting"
9355 }
9356 })
9357 .to_string();
9358 write_mock_response(stream, "200 OK", &body);
9359 return;
9360 }
9361
9362 if path == "/api/worker/workflow-tasks/snapshot-wait-3/fail"
9363 || path == "/api/worker/workflow-tasks/snapshot-wait-5/fail"
9364 {
9365 write_mock_response(
9366 stream,
9367 "200 OK",
9368 r#"{"outcome":"waiting_for_history","recorded":true}"#,
9369 );
9370 return;
9371 }
9372
9373 if path == "/api/worker/query-tasks/snapshot-current/complete" {
9374 write_mock_response(stream, "200 OK", r#"{"outcome":"completed"}"#);
9375 return;
9376 }
9377 }
9378
9379 let (status, body) = match path {
9380 "/api/worker/register" if behavior.waiting_query_worker => (
9381 "200 OK",
9382 r#"{"worker_id":"rust-snapshot-worker","registered":true,"heartbeat_interval_seconds":1}"#,
9383 ),
9384 "/api/worker/register" => (
9385 "200 OK",
9386 r#"{"worker_id":"mock-worker","registered":true,"heartbeat_interval_seconds":3600}"#,
9387 ),
9388 "/api/worker/heartbeat" => ("200 OK", "{}"),
9389 "/api/worker/activity-tasks/poll"
9390 if behavior.cancelled_activity && request_number == 1 =>
9391 {
9392 (
9393 "200 OK",
9394 r#"{"task":{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","activity_type":"cancel-aware","payload_codec":"json","arguments":{"codec":"json","blob":"[]"},"attempt_number":1,"lease_owner":"rust-cancel-worker"}}"#,
9395 )
9396 }
9397 "/api/worker/activity-tasks/poll" | "/api/worker/workflow-tasks/poll" => {
9398 ("200 OK", r#"{"task":null}"#)
9399 }
9400 "/api/worker/query-tasks/poll"
9401 if behavior.reject_query_completion && request_number == 1 =>
9402 {
9403 (
9404 "200 OK",
9405 r#"{"task":{"query_task_id":"query-late","query_task_attempt":1,"lease_owner":"late-worker","workflow_id":"counter-late","run_id":"run-late","workflow_type":"counter","query_name":"current","payload_codec":"json","workflow_arguments":{"codec":"json","blob":"[]"},"query_arguments":{"codec":"json","blob":"[]"},"history_events":[],"run_status":"running"}}"#,
9406 )
9407 }
9408 "/api/worker/query-tasks/poll" => ("200 OK", r#"{"task":null}"#),
9409 "/api/worker/query-tasks/query-capture/complete"
9410 | "/api/worker/query-tasks/query-capture/fail" => ("200 OK", "{}"),
9411 "/api/worker/activity-tasks/activity-cancel/heartbeat" => (
9412 "200 OK",
9413 r#"{"activity_attempt_id":"attempt-cancel","cancel_requested":true,"can_continue":false,"reason":"run_cancelled","run_closed_reason":"cancelled","heartbeat_recorded":false}"#,
9414 ),
9415 "/api/worker/activity-tasks/activity-cancel/complete" => (
9416 "409 Conflict",
9417 r#"{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","reason":"run_cancelled","cancel_requested":true,"can_continue":false,"run_closed_reason":"cancelled"}"#,
9418 ),
9419 "/api/workflows/counter-1/query/current" => (
9420 "200 OK",
9421 r#"{"workflow_id":"counter-1","query_name":"current","result":{"count":8},"result_envelope":{"codec":"json","blob":"{\"count\":8}"}}"#,
9422 ),
9423 "/api/workflows/counter-1/query/missing" => (
9424 "404 Not Found",
9425 r#"{"workflow_id":"counter-1","query_name":"missing","reason":"rejected_unknown_query","message":"unknown query"}"#,
9426 ),
9427 "/api/workflows/wf-lifecycle/cancel" => (
9428 "200 OK",
9429 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","reason":"cleanup requested","command_status":"accepted"}"#,
9430 ),
9431 "/api/workflows/wf-lifecycle/terminate" => (
9432 "200 OK",
9433 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","reason":"forced stop","command_status":"accepted"}"#,
9434 ),
9435 "/api/workflows/wf-lifecycle/runs/run-current/cancel" => (
9436 "200 OK",
9437 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","command_status":"accepted"}"#,
9438 ),
9439 "/api/workflows/wf-lifecycle/runs/run-current/terminate" => (
9440 "200 OK",
9441 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","command_status":"accepted"}"#,
9442 ),
9443 "/api/workflows/wf-lifecycle/runs/run-stale/cancel"
9444 | "/api/workflows/wf-lifecycle/runs/run-stale/terminate" => (
9445 "409 Conflict",
9446 r#"{"workflow_id":"wf-lifecycle","run_id":"run-stale","reason":"historical_run_command_rejected","target_scope":"run","message":"Commands cannot target historical runs."}"#,
9447 ),
9448 "/api/workflows/wf-failed" | "/api/workflows/wf-failed/runs/run-failed" => (
9449 "200 OK",
9450 r#"{"workflow_id":"wf-failed","run_id":"run-failed","status":"failed","closed_reason":"failed","error":"payment failed","failure":{"message":"payment failed","failure_category":"application","exception_type":"PaymentError","exception_class":"billing::PaymentError","non_retryable":true,"exception":{"type":"PaymentError","class":"billing::PaymentError","message":"payment failed"},"failures":[{"id":"failure-17","failure_category":"application"}]}}"#,
9451 ),
9452 "/api/workflows/wf-cancelled" => (
9453 "200 OK",
9454 r#"{"workflow_id":"wf-cancelled","run_id":"run-cancelled","status":"cancelled","closed_reason":"cancelled","reason":"cleanup requested"}"#,
9455 ),
9456 "/api/workflows/wf-terminated" => (
9457 "200 OK",
9458 r#"{"workflow_id":"wf-terminated","run_id":"run-terminated","status":"terminated","closed_reason":"terminated","reason":"forced stop"}"#,
9459 ),
9460 "/api/workflows/wf-timed-out" => (
9461 "200 OK",
9462 r#"{"workflow_id":"wf-timed-out","run_id":"run-timed-out","status":"failed","closed_reason":"timed_out","reason":"run_timeout"}"#,
9463 ),
9464 "/api/workflows/wf-waiting" | "/api/workflows/wf-waiting/runs/run-waiting" => (
9465 "200 OK",
9466 r#"{"workflow_id":"wf-waiting","run_id":"run-waiting","status":"waiting"}"#,
9467 ),
9468 "/api/workflows/wf-selected" => (
9469 "200 OK",
9470 r#"{"workflow_id":"wf-selected","run_id":"run-current","status":"completed","output":"current run output"}"#,
9471 ),
9472 "/api/workflows/wf-selected/runs/run-selected" => (
9473 "200 OK",
9474 r#"{"workflow_id":"wf-selected","run_id":"run-selected","status":"cancelled","closed_reason":"cancelled","reason":"selected run cancelled"}"#,
9475 ),
9476 _ => ("404 Not Found", r#"{"message":"not found"}"#),
9477 };
9478 write_mock_response(stream, status, body);
9479 }
9480
9481 fn mock_request_is_complete(request: &[u8]) -> bool {
9482 let Some(header_end) = request
9483 .windows(4)
9484 .position(|window| window == b"\r\n\r\n")
9485 .map(|position| position + 4)
9486 else {
9487 return false;
9488 };
9489 let headers = String::from_utf8_lossy(&request[..header_end]);
9490 let content_length = headers.lines().find_map(|line| {
9491 let (name, value) = line.split_once(':')?;
9492 name.eq_ignore_ascii_case("content-length")
9493 .then(|| value.trim().parse::<usize>().ok())
9494 .flatten()
9495 });
9496
9497 request.len() >= header_end + content_length.unwrap_or(0)
9498 }
9499
9500 fn write_mock_response(stream: &mut TcpStream, status: &str, body: &str) {
9501 let response = format!(
9502 "HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
9503 body.len()
9504 );
9505
9506 let _ = stream.write_all(response.as_bytes());
9507 let _ = stream.flush();
9508 }
9509}