1#![doc = include_str!("../README.md")]
2
3use std::{
4 any::{Any, TypeId},
5 collections::{BTreeMap, HashMap},
6 future::Future,
7 io::{self, Read},
8 pin::Pin,
9 sync::{
10 atomic::{AtomicBool, Ordering},
11 Arc, Mutex, OnceLock,
12 },
13 task::{Context as TaskContext, Poll},
14 time::{Duration, Instant, SystemTime, UNIX_EPOCH},
15};
16
17use apache_avro::{from_avro_datum, to_avro_datum, types::Value as AvroDatum, Schema};
18use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
19use futures_util::{future::OptionFuture, task::noop_waker_ref};
20use serde::{
21 de::DeserializeOwned,
22 ser::{SerializeMap, SerializeSeq},
23 Deserialize, Serialize, Serializer,
24};
25pub use serde_json::{json, Value};
26use thiserror::Error;
27pub use uuid::Uuid;
28
29pub const WORKER_PROTOCOL_VERSION: &str = "1.2";
30pub const CONTROL_PLANE_VERSION: &str = "2";
31pub const DEFAULT_CODEC: &str = "avro";
32pub const JSON_CODEC: &str = "json";
33pub const SDK_VERSION: &str = concat!("durable-workflow-rust/", env!("CARGO_PKG_VERSION"));
34pub const QUERY_TASKS_CAPABILITY: &str = "query_tasks";
36pub const WORKFLOW_UPDATES_CAPABILITY: &str = "workflow_updates";
38pub const QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
40
41const MAX_LONG_POLL_TIMEOUT_SECONDS: u64 = 60;
42const WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE: &str =
43 "Workflow task waiting for scheduled history.";
44const WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE: &str = "WorkflowTaskWaitingForHistory";
45
46const QUERY_TASK_FINAL_REJECTION_REASONS: &[&str] = &[
47 "lease_expired",
48 "query_task_not_found",
49 "query_task_not_leased",
50 "query_task_timed_out",
51];
52
53pub const AVRO_VALUE_SCHEMA_JSON: &str =
55 include_str!("../schema/durable_workflow.protocol.Value.v1.avsc");
56pub const AVRO_VALUE_SCHEMA_FINGERPRINT_HEX: &str = "e2a33dff55802237";
57pub const AVRO_VALUE_SCHEMA_FINGERPRINT: [u8; 8] = [0xe2, 0xa3, 0x3d, 0xff, 0x55, 0x80, 0x22, 0x37];
58const AVRO_SINGLE_OBJECT_MAGIC: [u8; 2] = [0xc3, 0x01];
59
60static AVRO_VALUE_SCHEMA: OnceLock<std::result::Result<Schema, String>> = OnceLock::new();
61
62#[derive(Clone, Copy)]
63enum RequestProtocol {
64 ControlPlane,
65 Worker(&'static str),
66}
67
68impl RequestProtocol {
69 fn is_worker(self) -> bool {
70 matches!(self, Self::Worker(_))
71 }
72}
73
74pub type Result<T> = std::result::Result<T, Error>;
75
76#[derive(Debug, Error)]
77pub enum Error {
78 #[error("transport error: {0}")]
79 Transport(#[from] reqwest::Error),
80 #[error("json error: {0}")]
81 Json(#[from] serde_json::Error),
82 #[error("http {status}: {body}")]
83 Http {
84 status: reqwest::StatusCode,
85 body: String,
86 },
87 #[error("codec error: {0}")]
88 Codec(String),
89 #[error(transparent)]
90 QueryFailed(QueryFailure),
91 #[error(transparent)]
92 Protocol(ProtocolFailure),
93 #[error(transparent)]
94 NonDeterministicReplay(ReplayFailure),
95 #[error(transparent)]
96 ChildWorkflowFailed(ChildWorkflowFailure),
97 #[error(transparent)]
98 ActivityFailed(ActivityFailure),
99 #[error(transparent)]
100 WorkflowCommandRejected(WorkflowCommandRejection),
101 #[error(transparent)]
102 WorkflowFailed(WorkflowTerminalOutcome),
103 #[error(transparent)]
104 WorkflowCancelled(WorkflowTerminalOutcome),
105 #[error(transparent)]
106 WorkflowTerminated(WorkflowTerminalOutcome),
107 #[error(transparent)]
108 WorkflowTimedOut(WorkflowTerminalOutcome),
109 #[error(transparent)]
110 ActivityTaskRejected(ActivityTaskRejection),
111 #[error("workflow handler {0:?} is not registered")]
112 WorkflowNotRegistered(String),
113 #[error("activity handler {0:?} is not registered")]
114 ActivityNotRegistered(String),
115 #[error("workflow future yielded without emitting a durable command")]
116 WorkflowYieldedWithoutCommand,
117 #[error("workflow state lock is poisoned")]
118 WorkflowStatePoisoned,
119 #[error("timer duration is too large for the worker protocol")]
120 TimerDurationOverflow,
121 #[error("operation timed out")]
122 Timeout,
123 #[error("worker loop error: {0}")]
124 WorkerLoop(String),
125 #[error("invalid child workflow options: {0}")]
126 InvalidChildWorkflowOptions(String),
127 #[error(transparent)]
128 InvalidActivityOptions(ActivityOptionsError),
129 #[error(transparent)]
130 InvalidContinueAsNewOptions(#[from] ContinueAsNewOptionsError),
131 #[doc(hidden)]
132 #[error("workflow requested continue as new")]
133 ContinueAsNew(ContinueAsNewRequest),
134}
135
136#[derive(Clone, Copy, Debug, PartialEq, Eq)]
138pub enum WorkflowCommandKind {
139 Cancel,
140 Terminate,
141}
142
143impl WorkflowCommandKind {
144 fn as_str(self) -> &'static str {
145 match self {
146 Self::Cancel => "cancel",
147 Self::Terminate => "terminate",
148 }
149 }
150}
151
152#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
154pub struct WorkflowCommandOptions {
155 #[serde(skip_serializing_if = "Option::is_none")]
156 pub reason: Option<String>,
157 #[serde(skip_serializing_if = "Option::is_none")]
158 pub request_id: Option<String>,
159}
160
161#[derive(Clone, Debug, PartialEq, Eq)]
168pub struct WorkflowStartOptions {
169 pub execution_timeout_seconds: u64,
170 pub run_timeout_seconds: u64,
171}
172
173impl Default for WorkflowStartOptions {
174 fn default() -> Self {
175 Self {
176 execution_timeout_seconds: 3600,
177 run_timeout_seconds: 600,
178 }
179 }
180}
181
182impl WorkflowStartOptions {
183 pub fn new() -> Self {
184 Self::default()
185 }
186
187 pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
188 self.execution_timeout_seconds = seconds;
189 self
190 }
191
192 pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
193 self.run_timeout_seconds = seconds;
194 self
195 }
196
197 fn validate(&self) -> Result<()> {
198 if self.execution_timeout_seconds == 0 {
199 return Err(Error::Codec(
200 "execution_timeout_seconds must be at least 1".to_string(),
201 ));
202 }
203 if self.run_timeout_seconds == 0 {
204 return Err(Error::Codec(
205 "run_timeout_seconds must be at least 1".to_string(),
206 ));
207 }
208 if self.run_timeout_seconds > self.execution_timeout_seconds {
209 return Err(Error::Codec(
210 "run_timeout_seconds cannot exceed execution_timeout_seconds".to_string(),
211 ));
212 }
213
214 Ok(())
215 }
216}
217
218#[derive(Clone, Debug, Default, PartialEq, Eq)]
223pub struct ContinueAsNewOptions {
224 pub workflow_type: Option<String>,
225 pub task_queue: Option<String>,
226}
227
228impl ContinueAsNewOptions {
229 pub fn new() -> Self {
230 Self::default()
231 }
232
233 pub fn workflow_type(mut self, workflow_type: impl Into<String>) -> Self {
234 self.workflow_type = Some(workflow_type.into());
235 self
236 }
237
238 pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
239 self.task_queue = Some(task_queue.into());
240 self
241 }
242
243 fn validate(&self) -> std::result::Result<(), ContinueAsNewOptionsError> {
244 for (field, value) in [
245 ("workflow_type", self.workflow_type.as_deref()),
246 ("task_queue", self.task_queue.as_deref()),
247 ] {
248 if value.is_some_and(|value| value.trim().is_empty()) {
249 return Err(ContinueAsNewOptionsError {
250 field,
251 message: format!("{field} must not be empty"),
252 });
253 }
254 }
255 Ok(())
256 }
257}
258
259#[derive(Clone, Debug, Error, PartialEq, Eq)]
261#[error("invalid continue-as-new option {field}: {message}")]
262pub struct ContinueAsNewOptionsError {
263 pub field: &'static str,
264 pub message: String,
265}
266
267#[derive(Clone, Debug, Default, PartialEq, Eq)]
269pub struct WorkflowHistoryBudget {
270 pub event_count: u64,
271 pub size_bytes: Option<u64>,
272 pub continue_as_new_recommended: bool,
273 pub pressure: Option<String>,
274}
275
276#[doc(hidden)]
277#[derive(Clone, Debug)]
278pub struct ContinueAsNewRequest {
279 arguments: AvroValue,
280 options: ContinueAsNewOptions,
281}
282
283impl WorkflowCommandOptions {
284 pub fn new() -> Self {
285 Self::default()
286 }
287
288 pub fn reason(mut self, reason: impl Into<String>) -> Self {
289 self.reason = Some(reason.into());
290 self
291 }
292
293 pub fn request_id(mut self, request_id: impl Into<String>) -> Self {
294 self.request_id = Some(request_id.into());
295 self
296 }
297}
298
299#[derive(Clone, Debug, PartialEq)]
301pub struct WorkflowCommandResult {
302 pub command: WorkflowCommandKind,
303 pub workflow_id: String,
304 pub run_id: Option<String>,
305 pub outcome: Option<String>,
306 pub reason: Option<String>,
307 pub command_status: Option<String>,
308 pub raw: Value,
309}
310
311#[derive(Clone, Debug, Error)]
313#[error("workflow {command:?} rejected ({reason}, HTTP {status}): {message}")]
314pub struct WorkflowCommandRejection {
315 pub command: WorkflowCommandKind,
316 pub status: u16,
317 pub reason: String,
318 pub message: String,
319 pub workflow_id: String,
320 pub run_id: Option<String>,
321 pub target_scope: Option<String>,
322 pub body: Value,
323}
324
325#[derive(Clone, Copy, Debug, PartialEq, Eq)]
327pub enum WorkflowTerminalKind {
328 Failed,
329 Cancelled,
330 Terminated,
331 TimedOut,
332}
333
334#[derive(Clone, Debug, Error)]
340#[error("workflow {workflow_id} run {run_id:?} ended as {kind:?} ({reason})")]
341pub struct WorkflowTerminalOutcome {
342 pub kind: WorkflowTerminalKind,
343 pub workflow_id: String,
344 pub run_id: Option<String>,
345 pub reason: String,
346 pub failure_category: Option<String>,
347 pub failure_id: Option<String>,
348 pub exception_type: Option<String>,
349 pub exception_class: Option<String>,
350 pub non_retryable: Option<bool>,
351 pub message: Option<String>,
352 pub exception: Option<Value>,
353 pub raw: Value,
354}
355
356#[derive(Clone, Debug, Error)]
358#[error("activity task {operation} rejected ({reason}, HTTP {status})")]
359pub struct ActivityTaskRejection {
360 pub operation: String,
361 pub status: u16,
362 pub reason: String,
363 pub task_id: String,
364 pub activity_attempt_id: String,
365 pub cancel_requested: bool,
366 pub can_continue: Option<bool>,
367 pub run_closed_reason: Option<String>,
368 pub body: Value,
369}
370
371#[derive(Clone, Copy, Debug, PartialEq, Eq)]
373pub enum ActivityOptionsErrorKind {
374 EmptyTaskQueue,
375 EmptyRetryPolicy,
376 InvalidMaxAttempts,
377 BackoffWithoutRetryBudget,
378 TooManyBackoffIntervals,
379 InvalidBackoffCoefficient,
380 BackoffGenerationTooLarge,
381 BackoffOverflow,
382 EmptyNonRetryableErrorType,
383 TimeoutNotPositive,
384 TimeoutOverflow,
385 TimeoutOrder,
386}
387
388#[derive(Clone, Debug, Error, PartialEq, Eq)]
390#[error("invalid activity options ({kind:?}, {field:?}): {message}")]
391pub struct ActivityOptionsError {
392 pub kind: ActivityOptionsErrorKind,
393 pub field: Option<&'static str>,
394 pub message: String,
395}
396
397impl ActivityOptionsError {
398 fn new(
399 kind: ActivityOptionsErrorKind,
400 field: Option<&'static str>,
401 message: impl Into<String>,
402 ) -> Self {
403 Self {
404 kind,
405 field,
406 message: message.into(),
407 }
408 }
409}
410
411#[derive(Clone, Copy, Debug, PartialEq, Eq)]
413pub enum ActivityFailureKind {
414 Failed,
415 Cancelled,
416 TimedOut,
417}
418
419#[derive(Clone, Debug, Error)]
424#[error("activity failed ({reason}): {message}")]
425pub struct ActivityFailure {
426 pub kind: ActivityFailureKind,
427 pub reason: String,
428 pub message: String,
429 pub activity_execution_id: Option<String>,
430 pub activity_attempt_id: Option<String>,
431 pub activity_type: Option<String>,
432 pub activity_class: Option<String>,
433 pub attempt_number: Option<u64>,
434 pub failure_id: Option<String>,
435 pub failure_category: Option<String>,
436 pub timeout_kind: Option<String>,
437 pub non_retryable: bool,
438 pub exception_type: Option<String>,
439 pub exception_class: Option<String>,
440 pub code: Option<Value>,
441 pub exception: Option<Value>,
442}
443
444#[derive(Clone, Copy, Debug, PartialEq, Eq)]
446pub enum ChildWorkflowFailureKind {
447 Failed,
448 Cancelled,
449 Terminated,
450}
451
452#[derive(Clone, Debug, Error)]
458#[error("child workflow failed ({reason}): {message}")]
459pub struct ChildWorkflowFailure {
460 pub kind: ChildWorkflowFailureKind,
461 pub reason: String,
462 pub message: String,
463 pub parent_workflow_id: Option<String>,
464 pub parent_workflow_run_id: Option<String>,
465 pub child_workflow_id: Option<String>,
466 pub child_workflow_run_id: Option<String>,
467 pub child_workflow_type: Option<String>,
468 pub failure_id: Option<String>,
469 pub failure_category: Option<String>,
470 pub exception_type: Option<String>,
471 pub exception_class: Option<String>,
472 pub non_retryable: bool,
473 pub code: Option<Value>,
474 pub exception: Option<Value>,
475}
476
477#[derive(Clone, Debug, PartialEq, Eq)]
479pub struct WorkflowIdentity {
480 pub workflow_id: Option<String>,
481 pub run_id: Option<String>,
482}
483
484#[derive(Clone, Debug, PartialEq)]
486pub struct ChildWorkflowResult {
487 pub parent: WorkflowIdentity,
488 pub child: WorkflowIdentity,
489 pub child_workflow_type: Option<String>,
490 pub result: Value,
491}
492
493#[derive(Clone, Debug, PartialEq)]
495pub struct ChildWorkflowAvroResult {
496 pub parent: WorkflowIdentity,
497 pub child: WorkflowIdentity,
498 pub child_workflow_type: Option<String>,
499 pub result: AvroValue,
500}
501
502#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
504pub enum ParentClosePolicy {
505 #[default]
506 Abandon,
507 RequestCancel,
508 Terminate,
509}
510
511impl ParentClosePolicy {
512 fn as_str(self) -> &'static str {
513 match self {
514 Self::Abandon => "abandon",
515 Self::RequestCancel => "request_cancel",
516 Self::Terminate => "terminate",
517 }
518 }
519}
520
521#[derive(Clone, Debug, Default, PartialEq, Eq)]
523pub struct ChildWorkflowRetryPolicy {
524 pub max_attempts: Option<u32>,
525 pub backoff_seconds: Vec<u64>,
526 pub non_retryable_error_types: Vec<String>,
527}
528
529#[derive(Clone, Debug, PartialEq, Eq)]
533pub struct ChildWorkflowOptions {
534 pub task_queue: String,
535 pub parent_close_policy: ParentClosePolicy,
536 pub retry_policy: Option<ChildWorkflowRetryPolicy>,
537 pub execution_timeout_seconds: Option<u64>,
538 pub run_timeout_seconds: Option<u64>,
539}
540
541impl ChildWorkflowOptions {
542 pub fn new(task_queue: impl Into<String>) -> Self {
543 Self {
544 task_queue: task_queue.into(),
545 parent_close_policy: ParentClosePolicy::Abandon,
546 retry_policy: None,
547 execution_timeout_seconds: None,
548 run_timeout_seconds: None,
549 }
550 }
551
552 pub fn parent_close_policy(mut self, policy: ParentClosePolicy) -> Self {
553 self.parent_close_policy = policy;
554 self
555 }
556
557 pub fn retry_policy(mut self, policy: ChildWorkflowRetryPolicy) -> Self {
558 self.retry_policy = Some(policy);
559 self
560 }
561
562 pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
563 self.execution_timeout_seconds = Some(seconds);
564 self
565 }
566
567 pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
568 self.run_timeout_seconds = Some(seconds);
569 self
570 }
571}
572
573#[derive(Clone, Debug, PartialEq, Eq)]
575pub enum ActivityBackoff {
576 Explicit(Vec<Duration>),
579 Exponential {
581 initial_interval: Duration,
582 coefficient: u32,
583 maximum_interval: Option<Duration>,
584 },
585}
586
587#[derive(Clone, Debug, Default, PartialEq, Eq)]
589pub struct ActivityRetryPolicy {
590 pub max_attempts: Option<u32>,
591 pub backoff: Option<ActivityBackoff>,
592 pub non_retryable_error_types: Vec<String>,
593}
594
595impl ActivityRetryPolicy {
596 pub fn new(max_attempts: u32) -> Self {
598 Self {
599 max_attempts: Some(max_attempts),
600 ..Self::default()
601 }
602 }
603
604 pub fn backoff_intervals(mut self, intervals: impl IntoIterator<Item = Duration>) -> Self {
605 self.backoff = Some(ActivityBackoff::Explicit(intervals.into_iter().collect()));
606 self
607 }
608
609 pub fn exponential_backoff(
610 mut self,
611 initial_interval: Duration,
612 coefficient: u32,
613 maximum_interval: Option<Duration>,
614 ) -> Self {
615 self.backoff = Some(ActivityBackoff::Exponential {
616 initial_interval,
617 coefficient,
618 maximum_interval,
619 });
620 self
621 }
622
623 pub fn non_retryable_error_type(mut self, error_type: impl Into<String>) -> Self {
624 self.non_retryable_error_types.push(error_type.into());
625 self
626 }
627
628 pub fn non_retryable_error_types(
629 mut self,
630 error_types: impl IntoIterator<Item = impl Into<String>>,
631 ) -> Self {
632 self.non_retryable_error_types
633 .extend(error_types.into_iter().map(Into::into));
634 self
635 }
636}
637
638#[derive(Clone, Debug, Default, PartialEq, Eq)]
643pub struct ActivityOptions {
644 pub task_queue: Option<String>,
645 pub retry_policy: Option<ActivityRetryPolicy>,
646 pub start_to_close_timeout: Option<Duration>,
647 pub schedule_to_start_timeout: Option<Duration>,
648 pub schedule_to_close_timeout: Option<Duration>,
649 pub heartbeat_timeout: Option<Duration>,
650}
651
652impl ActivityOptions {
653 pub fn new() -> Self {
654 Self::default()
655 }
656
657 pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
658 self.task_queue = Some(task_queue.into());
659 self
660 }
661
662 pub fn retry_policy(mut self, policy: ActivityRetryPolicy) -> Self {
663 self.retry_policy = Some(policy);
664 self
665 }
666
667 pub fn start_to_close_timeout(mut self, timeout: Duration) -> Self {
668 self.start_to_close_timeout = Some(timeout);
669 self
670 }
671
672 pub fn schedule_to_start_timeout(mut self, timeout: Duration) -> Self {
673 self.schedule_to_start_timeout = Some(timeout);
674 self
675 }
676
677 pub fn schedule_to_close_timeout(mut self, timeout: Duration) -> Self {
678 self.schedule_to_close_timeout = Some(timeout);
679 self
680 }
681
682 pub fn heartbeat_timeout(mut self, timeout: Duration) -> Self {
683 self.heartbeat_timeout = Some(timeout);
684 self
685 }
686
687 fn validate(&self) -> std::result::Result<ValidatedActivityOptions, ActivityOptionsError> {
688 if self
689 .task_queue
690 .as_deref()
691 .is_some_and(|queue| queue.trim().is_empty())
692 {
693 return Err(ActivityOptionsError::new(
694 ActivityOptionsErrorKind::EmptyTaskQueue,
695 Some("task_queue"),
696 "task_queue must not be empty",
697 ));
698 }
699
700 for (field, value) in [
701 ("start_to_close_timeout", self.start_to_close_timeout),
702 ("schedule_to_start_timeout", self.schedule_to_start_timeout),
703 ("schedule_to_close_timeout", self.schedule_to_close_timeout),
704 ("heartbeat_timeout", self.heartbeat_timeout),
705 ] {
706 if value.is_some_and(|value| value.is_zero()) {
707 return Err(ActivityOptionsError::new(
708 ActivityOptionsErrorKind::TimeoutNotPositive,
709 Some(field),
710 format!("{field} must be positive"),
711 ));
712 }
713 }
714
715 validate_timeout_order(
716 "heartbeat_timeout",
717 self.heartbeat_timeout,
718 "start_to_close_timeout",
719 self.start_to_close_timeout,
720 )?;
721 validate_timeout_order(
722 "start_to_close_timeout",
723 self.start_to_close_timeout,
724 "schedule_to_close_timeout",
725 self.schedule_to_close_timeout,
726 )?;
727 validate_timeout_order(
728 "schedule_to_start_timeout",
729 self.schedule_to_start_timeout,
730 "schedule_to_close_timeout",
731 self.schedule_to_close_timeout,
732 )?;
733
734 Ok(ValidatedActivityOptions {
735 task_queue: self.task_queue.clone(),
736 retry_policy: self
737 .retry_policy
738 .as_ref()
739 .map(validate_activity_retry_policy)
740 .transpose()?,
741 start_to_close_timeout: timeout_seconds(
742 "start_to_close_timeout",
743 self.start_to_close_timeout,
744 )?,
745 schedule_to_start_timeout: timeout_seconds(
746 "schedule_to_start_timeout",
747 self.schedule_to_start_timeout,
748 )?,
749 schedule_to_close_timeout: timeout_seconds(
750 "schedule_to_close_timeout",
751 self.schedule_to_close_timeout,
752 )?,
753 heartbeat_timeout: timeout_seconds("heartbeat_timeout", self.heartbeat_timeout)?,
754 })
755 }
756}
757
758#[derive(Clone, Debug)]
759struct ValidatedActivityOptions {
760 task_queue: Option<String>,
761 retry_policy: Option<Value>,
762 start_to_close_timeout: Option<u64>,
763 schedule_to_start_timeout: Option<u64>,
764 schedule_to_close_timeout: Option<u64>,
765 heartbeat_timeout: Option<u64>,
766}
767
768fn validate_timeout_order(
769 smaller_name: &'static str,
770 smaller: Option<Duration>,
771 larger_name: &'static str,
772 larger: Option<Duration>,
773) -> std::result::Result<(), ActivityOptionsError> {
774 if matches!((smaller, larger), (Some(smaller), Some(larger)) if smaller > larger) {
775 return Err(ActivityOptionsError::new(
776 ActivityOptionsErrorKind::TimeoutOrder,
777 Some(smaller_name),
778 format!("{smaller_name} must be <= {larger_name}"),
779 ));
780 }
781 Ok(())
782}
783
784fn timeout_seconds(
785 field: &'static str,
786 value: Option<Duration>,
787) -> std::result::Result<Option<u64>, ActivityOptionsError> {
788 value
789 .map(|value| {
790 activity_protocol_seconds(value).ok_or_else(|| {
791 ActivityOptionsError::new(
792 ActivityOptionsErrorKind::TimeoutOverflow,
793 Some(field),
794 format!("{field} is too large for the worker protocol"),
795 )
796 })
797 })
798 .transpose()
799}
800
801fn duration_seconds_ceil(value: Duration) -> Option<u64> {
802 value
803 .as_secs()
804 .checked_add(u64::from(value.subsec_nanos() > 0))
805}
806
807fn activity_protocol_seconds(value: Duration) -> Option<u64> {
808 duration_seconds_ceil(value).filter(|seconds| *seconds <= i64::MAX as u64)
809}
810
811fn validate_activity_retry_policy(
812 policy: &ActivityRetryPolicy,
813) -> std::result::Result<Value, ActivityOptionsError> {
814 if policy.max_attempts.is_none()
815 && policy.backoff.is_none()
816 && policy.non_retryable_error_types.is_empty()
817 {
818 return Err(ActivityOptionsError::new(
819 ActivityOptionsErrorKind::EmptyRetryPolicy,
820 Some("retry_policy"),
821 "retry_policy must configure at least one field",
822 ));
823 }
824 if policy.max_attempts == Some(0) {
825 return Err(ActivityOptionsError::new(
826 ActivityOptionsErrorKind::InvalidMaxAttempts,
827 Some("retry_policy.max_attempts"),
828 "max_attempts must be >= 1",
829 ));
830 }
831 if policy
832 .non_retryable_error_types
833 .iter()
834 .any(|error_type| error_type.trim().is_empty())
835 {
836 return Err(ActivityOptionsError::new(
837 ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
838 Some("retry_policy.non_retryable_error_types"),
839 "non_retryable_error_types must not contain empty values",
840 ));
841 }
842
843 let backoff_seconds = match &policy.backoff {
844 None => None,
845 Some(backoff) => {
846 let max_attempts = policy.max_attempts.ok_or_else(|| {
847 ActivityOptionsError::new(
848 ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
849 Some("retry_policy.backoff"),
850 "backoff requires max_attempts",
851 )
852 })?;
853 let retry_count = max_attempts.saturating_sub(1) as usize;
854 let intervals = match backoff {
855 ActivityBackoff::Explicit(intervals) => {
856 if intervals.len() > retry_count {
857 return Err(ActivityOptionsError::new(
858 ActivityOptionsErrorKind::TooManyBackoffIntervals,
859 Some("retry_policy.backoff"),
860 "backoff interval count must not exceed max_attempts - 1",
861 ));
862 }
863 intervals.clone()
864 }
865 ActivityBackoff::Exponential {
866 initial_interval,
867 coefficient,
868 maximum_interval,
869 } => {
870 if *coefficient < 1 {
871 return Err(ActivityOptionsError::new(
872 ActivityOptionsErrorKind::InvalidBackoffCoefficient,
873 Some("retry_policy.backoff.coefficient"),
874 "backoff coefficient must be >= 1",
875 ));
876 }
877 if retry_count > 10_000 {
878 return Err(ActivityOptionsError::new(
879 ActivityOptionsErrorKind::BackoffGenerationTooLarge,
880 Some("retry_policy.max_attempts"),
881 "generated backoff supports at most 10000 retry intervals",
882 ));
883 }
884 let mut current = *initial_interval;
885 let mut intervals = Vec::with_capacity(retry_count);
886 for _ in 0..retry_count {
887 let interval = maximum_interval
888 .map(|maximum| current.min(maximum))
889 .unwrap_or(current);
890 intervals.push(interval);
891 if maximum_interval.is_some_and(|maximum| interval == maximum) {
892 break;
893 }
894 current = current.checked_mul(*coefficient).ok_or_else(|| {
895 ActivityOptionsError::new(
896 ActivityOptionsErrorKind::BackoffOverflow,
897 Some("retry_policy.backoff"),
898 "generated backoff interval overflowed",
899 )
900 })?;
901 }
902 intervals
903 }
904 };
905 Some(
906 intervals
907 .into_iter()
908 .map(|interval| {
909 activity_protocol_seconds(interval).ok_or_else(|| {
910 ActivityOptionsError::new(
911 ActivityOptionsErrorKind::BackoffOverflow,
912 Some("retry_policy.backoff"),
913 "backoff interval is too large for the worker protocol",
914 )
915 })
916 })
917 .collect::<std::result::Result<Vec<_>, _>>()?,
918 )
919 }
920 };
921
922 let mut encoded = serde_json::Map::new();
923 if let Some(max_attempts) = policy.max_attempts {
924 encoded.insert("max_attempts".to_string(), json!(max_attempts));
925 }
926 if let Some(backoff_seconds) = backoff_seconds {
927 encoded.insert("backoff_seconds".to_string(), json!(backoff_seconds));
928 }
929 if !policy.non_retryable_error_types.is_empty() {
930 let mut canonical_error_types = Vec::new();
931 for error_type in policy
932 .non_retryable_error_types
933 .iter()
934 .map(|error_type| error_type.trim())
935 {
936 if !canonical_error_types.contains(&error_type) {
937 canonical_error_types.push(error_type);
938 }
939 }
940 encoded.insert(
941 "non_retryable_error_types".to_string(),
942 json!(canonical_error_types),
943 );
944 }
945 Ok(Value::Object(encoded))
946}
947
948#[derive(Clone, Debug, Error)]
951#[error("non-deterministic workflow replay ({reason}) at sequence {sequence:?}: {message}")]
952pub struct ReplayFailure {
953 pub reason: String,
954 pub sequence: Option<u64>,
955 pub expected: Option<String>,
956 pub actual: Option<String>,
957 pub message: String,
958}
959
960impl ReplayFailure {
961 fn new(
962 reason: impl Into<String>,
963 sequence: Option<u64>,
964 expected: Option<String>,
965 actual: Option<String>,
966 message: impl Into<String>,
967 ) -> Self {
968 Self {
969 reason: reason.into(),
970 sequence,
971 expected,
972 actual,
973 message: message.into(),
974 }
975 }
976}
977
978#[derive(Clone, Debug, Error)]
980#[error("query failed ({reason}, HTTP {status}): {message}")]
981pub struct QueryFailure {
982 pub status: u16,
983 pub reason: String,
984 pub message: String,
985 pub body: Value,
986}
987
988#[derive(Clone, Debug, Error)]
990#[error("protocol rejected ({reason}, HTTP {status}): {message}")]
991pub struct ProtocolFailure {
992 pub status: u16,
993 pub reason: String,
994 pub message: String,
995 pub supported_version: Option<String>,
996 pub requested_version: Option<String>,
997 pub body: Value,
998}
999
1000#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
1001pub struct PayloadEnvelope {
1002 pub codec: String,
1003 pub blob: String,
1004}
1005
1006impl PayloadEnvelope {
1007 pub fn avro<T: Serialize>(value: &T) -> Result<Self> {
1008 encode_payload(value, DEFAULT_CODEC)
1009 }
1010
1011 pub fn json<T: Serialize>(value: &T) -> Result<Self> {
1012 encode_payload(value, JSON_CODEC)
1013 }
1014
1015 pub fn avro_value(value: &AvroValue) -> Result<Self> {
1018 encode_avro_value(value)
1019 }
1020}
1021
1022#[derive(Clone, Debug, PartialEq)]
1024pub enum AvroValue {
1025 Null,
1026 Boolean(bool),
1027 Long(i64),
1028 Double(f64),
1029 Bytes(Vec<u8>),
1030 String(String),
1031 Array(Vec<AvroValue>),
1032 Map(BTreeMap<String, AvroValue>),
1033}
1034
1035impl AvroValue {
1036 fn from_serialize<T: Serialize>(value: &T) -> Result<Self> {
1037 Self::from_serde_value(
1038 serde_value::to_value(value).map_err(|error| {
1039 Error::Codec(format!("could not adapt value for Avro: {error}"))
1040 })?,
1041 )
1042 }
1043
1044 fn from_serde_value(value: serde_value::Value) -> Result<Self> {
1045 use serde_value::Value as SerdeValue;
1046
1047 match value {
1048 SerdeValue::Unit => Ok(Self::Null),
1049 SerdeValue::Bool(value) => Ok(Self::Boolean(value)),
1050 SerdeValue::I8(value) => Ok(Self::Long(i64::from(value))),
1051 SerdeValue::I16(value) => Ok(Self::Long(i64::from(value))),
1052 SerdeValue::I32(value) => Ok(Self::Long(i64::from(value))),
1053 SerdeValue::I64(value) => Ok(Self::Long(value)),
1054 SerdeValue::U8(value) => Ok(Self::Long(i64::from(value))),
1055 SerdeValue::U16(value) => Ok(Self::Long(i64::from(value))),
1056 SerdeValue::U32(value) => Ok(Self::Long(i64::from(value))),
1057 SerdeValue::U64(value) => i64::try_from(value).map(Self::Long).map_err(|_| {
1058 Error::Codec(
1059 "integer_overflow: Avro Value long must be within signed 64-bit range"
1060 .to_string(),
1061 )
1062 }),
1063 SerdeValue::F32(value) => Self::finite_double(f64::from(value)),
1064 SerdeValue::F64(value) => Self::finite_double(value),
1065 SerdeValue::Char(value) => Ok(Self::String(value.to_string())),
1066 SerdeValue::String(value) => Ok(Self::String(value)),
1067 SerdeValue::Bytes(value) => Ok(Self::Bytes(value)),
1068 SerdeValue::Option(None) => Ok(Self::Null),
1069 SerdeValue::Option(Some(value)) | SerdeValue::Newtype(value) => {
1070 Self::from_serde_value(*value)
1071 }
1072 SerdeValue::Seq(values) => values
1073 .into_iter()
1074 .map(Self::from_serde_value)
1075 .collect::<Result<Vec<_>>>()
1076 .map(Self::Array),
1077 SerdeValue::Map(values) => values
1078 .into_iter()
1079 .map(|(key, value)| {
1080 let SerdeValue::String(key) = key else {
1081 return Err(Error::Codec(
1082 "invalid_map_key: Avro Value map keys must be strings".to_string(),
1083 ));
1084 };
1085
1086 Ok((key, Self::from_serde_value(value)?))
1087 })
1088 .collect::<Result<BTreeMap<_, _>>>()
1089 .map(Self::Map),
1090 }
1091 }
1092
1093 fn finite_double(value: f64) -> Result<Self> {
1094 if !value.is_finite() {
1095 return Err(Error::Codec(
1096 "non_finite_float: Avro Value doubles must be finite".to_string(),
1097 ));
1098 }
1099
1100 Ok(Self::Double(value))
1101 }
1102
1103 fn into_json(self) -> Result<Value> {
1104 match self {
1105 Self::Null => Ok(Value::Null),
1106 Self::Boolean(value) => Ok(Value::Bool(value)),
1107 Self::Long(value) => Ok(Value::Number(value.into())),
1108 Self::Double(value) => serde_json::Number::from_f64(value)
1109 .map(Value::Number)
1110 .ok_or_else(|| {
1111 Error::Codec(
1112 "non_finite_float: decoded Avro Value double is not finite".to_string(),
1113 )
1114 }),
1115 Self::Bytes(value) => Ok(json!({
1116 "$type": "bytes",
1117 "base64": BASE64.encode(value),
1118 })),
1119 Self::String(value) => Ok(Value::String(value)),
1120 Self::Array(values) => values
1121 .into_iter()
1122 .map(Self::into_json)
1123 .collect::<Result<Vec<_>>>()
1124 .map(Value::Array),
1125 Self::Map(values) => values
1126 .into_iter()
1127 .map(|(key, value)| Ok((key, value.into_json()?)))
1128 .collect::<Result<serde_json::Map<_, _>>>()
1129 .map(Value::Object),
1130 }
1131 }
1132
1133 fn into_serde_value(self) -> serde_value::Value {
1134 use serde_value::Value as SerdeValue;
1135
1136 match self {
1137 Self::Null => SerdeValue::Unit,
1138 Self::Boolean(value) => SerdeValue::Bool(value),
1139 Self::Long(value) => SerdeValue::I64(value),
1140 Self::Double(value) => SerdeValue::F64(value),
1141 Self::Bytes(value) => SerdeValue::Bytes(value),
1142 Self::String(value) => SerdeValue::String(value),
1143 Self::Array(values) => {
1144 SerdeValue::Seq(values.into_iter().map(Self::into_serde_value).collect())
1145 }
1146 Self::Map(values) => SerdeValue::Map(
1147 values
1148 .into_iter()
1149 .map(|(key, value)| (SerdeValue::String(key), value.into_serde_value()))
1150 .collect(),
1151 ),
1152 }
1153 }
1154
1155 pub fn deserialize<T: DeserializeOwned>(self) -> Result<T> {
1156 self.into_serde_value().deserialize_into().map_err(|error| {
1157 Error::Codec(format!(
1158 "avro_value_type_mismatch: could not adapt decoded value: {error}"
1159 ))
1160 })
1161 }
1162}
1163
1164impl Serialize for AvroValue {
1165 fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
1166 where
1167 S: Serializer,
1168 {
1169 match self {
1170 Self::Null => serializer.serialize_unit(),
1171 Self::Boolean(value) => serializer.serialize_bool(*value),
1172 Self::Long(value) => serializer.serialize_i64(*value),
1173 Self::Double(value) => serializer.serialize_f64(*value),
1174 Self::Bytes(value) => serializer.serialize_bytes(value),
1175 Self::String(value) => serializer.serialize_str(value),
1176 Self::Array(values) => {
1177 let mut sequence = serializer.serialize_seq(Some(values.len()))?;
1178 for value in values {
1179 sequence.serialize_element(value)?;
1180 }
1181 sequence.end()
1182 }
1183 Self::Map(values) => {
1184 let mut map = serializer.serialize_map(Some(values.len()))?;
1185 for (key, value) in values {
1186 map.serialize_entry(key, value)?;
1187 }
1188 map.end()
1189 }
1190 }
1191 }
1192}
1193
1194pub fn encode_avro_value(value: &AvroValue) -> Result<PayloadEnvelope> {
1195 let datum = avro_value_to_datum(value)?;
1196 let datum = to_avro_datum(avro_value_schema()?, datum)
1197 .map_err(|err| Error::Codec(format!("avro_value_encode_failed: {err}")))?;
1198 let mut bytes = Vec::with_capacity(datum.len() + 10);
1199 bytes.extend_from_slice(&AVRO_SINGLE_OBJECT_MAGIC);
1200 bytes.extend_from_slice(&AVRO_VALUE_SCHEMA_FINGERPRINT);
1201 bytes.extend_from_slice(&datum);
1202 Ok(PayloadEnvelope {
1203 codec: DEFAULT_CODEC.to_string(),
1204 blob: BASE64.encode(bytes),
1205 })
1206}
1207
1208pub fn decode_avro_value(envelope: &PayloadEnvelope) -> Result<AvroValue> {
1209 if envelope.codec != DEFAULT_CODEC {
1210 return Err(Error::Codec(format!(
1211 "unsupported payload codec {:?}",
1212 envelope.codec
1213 )));
1214 }
1215 decode_avro_value_blob(&envelope.blob)
1216}
1217
1218pub fn encode_payload<T: Serialize>(value: &T, codec: &str) -> Result<PayloadEnvelope> {
1219 let blob = match codec {
1220 JSON_CODEC => serde_json::to_string(value)?,
1221 DEFAULT_CODEC => encode_avro_value(&AvroValue::from_serialize(value)?)?.blob,
1222 other => return Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1223 };
1224
1225 Ok(PayloadEnvelope {
1226 codec: codec.to_string(),
1227 blob,
1228 })
1229}
1230
1231pub fn decode_payload<T: DeserializeOwned>(envelope: &PayloadEnvelope) -> Result<T> {
1232 match envelope.codec.as_str() {
1233 JSON_CODEC => Ok(serde_json::from_str(&envelope.blob)?),
1234 DEFAULT_CODEC => decode_avro_value(envelope)?.deserialize(),
1235 other => Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1236 }
1237}
1238
1239#[cfg(test)]
1240fn encode_value_envelope(value: &Value, codec: &str) -> Result<Value> {
1241 Ok(serde_json::to_value(encode_payload(value, codec)?)?)
1242}
1243
1244fn decode_wire_value(value: &Value, fallback_codec: &str) -> Result<Value> {
1245 if value.is_null() {
1246 return Ok(Value::Null);
1247 }
1248
1249 if let Some(object) = value.as_object() {
1250 if let (Some(codec), Some(blob)) = (
1251 object.get("codec").and_then(Value::as_str),
1252 object.get("blob").and_then(Value::as_str),
1253 ) {
1254 return decode_blob(blob, codec);
1255 }
1256 }
1257
1258 if let Some(blob) = value.as_str() {
1259 return decode_blob(blob, fallback_codec);
1260 }
1261
1262 Ok(value.clone())
1263}
1264
1265fn encode_typed_envelope(value: &AvroValue, codec: &str) -> Result<Value> {
1266 let envelope = match codec {
1267 DEFAULT_CODEC => encode_avro_value(value)?,
1268 JSON_CODEC => PayloadEnvelope {
1269 codec: JSON_CODEC.to_string(),
1270 blob: serde_json::to_string(&value.clone().into_json()?)?,
1271 },
1272 other => return Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1273 };
1274 Ok(serde_json::to_value(envelope)?)
1275}
1276
1277fn decode_wire_avro_value(value: &Value, fallback_codec: &str) -> Result<AvroValue> {
1278 if value.is_null() {
1279 return Ok(AvroValue::Null);
1280 }
1281
1282 if let Some(object) = value.as_object() {
1283 if let (Some(codec), Some(blob)) = (
1284 object.get("codec").and_then(Value::as_str),
1285 object.get("blob").and_then(Value::as_str),
1286 ) {
1287 return match codec {
1288 DEFAULT_CODEC => decode_avro_value_blob(blob),
1289 JSON_CODEC => AvroValue::from_serialize(&serde_json::from_str::<Value>(blob)?),
1290 other => Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1291 };
1292 }
1293 }
1294
1295 if let Some(blob) = value.as_str() {
1296 return match fallback_codec {
1297 DEFAULT_CODEC => decode_avro_value_blob(blob),
1298 JSON_CODEC => AvroValue::from_serialize(&serde_json::from_str::<Value>(blob)?),
1299 other => Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1300 };
1301 }
1302
1303 AvroValue::from_serialize(value)
1304}
1305
1306fn normalize_avro_arguments(value: AvroValue) -> AvroValue {
1307 match value {
1308 AvroValue::Null => AvroValue::Array(Vec::new()),
1309 AvroValue::Array(_) => value,
1310 other => AvroValue::Array(vec![other]),
1311 }
1312}
1313
1314fn decode_blob(blob: &str, codec: &str) -> Result<Value> {
1315 match codec {
1316 JSON_CODEC => Ok(serde_json::from_str(blob)?),
1317 DEFAULT_CODEC => decode_avro_value_blob(blob)?.into_json(),
1318 other => Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1319 }
1320}
1321
1322fn decode_avro_value_blob(blob: &str) -> Result<AvroValue> {
1323 let bytes = BASE64.decode(blob).map_err(|err| {
1324 Error::Codec(format!(
1325 "invalid_payload_framing: expected strict base64 Avro single-object bytes: {err}"
1326 ))
1327 })?;
1328
1329 if bytes.len() < 10 || bytes[..2] != AVRO_SINGLE_OBJECT_MAGIC {
1330 return Err(Error::Codec(
1331 "invalid_payload_framing: expected Avro single-object magic c301".to_string(),
1332 ));
1333 }
1334
1335 let fingerprint: [u8; 8] = bytes[2..10]
1336 .try_into()
1337 .map_err(|_| Error::Codec("invalid Avro fingerprint length".to_string()))?;
1338 if fingerprint != AVRO_VALUE_SCHEMA_FINGERPRINT {
1339 return Err(Error::Codec(format!(
1340 "unsupported_payload_schema: unknown CRC-64-AVRO fingerprint {}",
1341 fingerprint
1342 .iter()
1343 .map(|byte| format!("{byte:02x}"))
1344 .collect::<String>()
1345 )));
1346 }
1347
1348 let mut datum_reader = StrictAvroDatumReader::new(&bytes[10..]);
1349 let datum = from_avro_datum(avro_value_schema()?, &mut datum_reader, None);
1353 if datum_reader.truncated {
1354 return Err(Error::Codec(
1355 "invalid_payload_framing: truncated Avro Value datum".to_string(),
1356 ));
1357 }
1358 let datum = datum.map_err(|err| {
1359 Error::Codec(format!(
1360 "invalid_payload_framing: malformed Avro Value datum: {err}"
1361 ))
1362 })?;
1363 if datum_reader.remaining() != 0 {
1364 return Err(Error::Codec(format!(
1365 "invalid_payload_framing: {} trailing bytes after Avro Value datum",
1366 datum_reader.remaining()
1367 )));
1368 }
1369 avro_value_from_datum(datum)
1370}
1371
1372struct StrictAvroDatumReader<'a> {
1373 bytes: &'a [u8],
1374 offset: usize,
1375 truncated: bool,
1376}
1377
1378impl<'a> StrictAvroDatumReader<'a> {
1379 fn new(bytes: &'a [u8]) -> Self {
1380 Self {
1381 bytes,
1382 offset: 0,
1383 truncated: false,
1384 }
1385 }
1386
1387 fn remaining(&self) -> usize {
1388 self.bytes.len() - self.offset
1389 }
1390}
1391
1392impl Read for StrictAvroDatumReader<'_> {
1393 fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
1394 let count = buffer.len().min(self.remaining());
1395 buffer[..count].copy_from_slice(&self.bytes[self.offset..self.offset + count]);
1396 self.offset += count;
1397 if count < buffer.len() {
1398 self.truncated = true;
1399 }
1400
1401 Ok(count)
1402 }
1403}
1404
1405fn avro_value_to_datum(value: &AvroValue) -> Result<AvroDatum> {
1406 let branch = match value {
1407 AvroValue::Null => AvroDatum::Union(0, Box::new(AvroDatum::Null)),
1408 AvroValue::Boolean(value) => AvroDatum::Union(
1409 1,
1410 Box::new(AvroDatum::Record(vec![(
1411 "boolean".to_string(),
1412 AvroDatum::Boolean(*value),
1413 )])),
1414 ),
1415 AvroValue::Long(value) => AvroDatum::Union(
1416 2,
1417 Box::new(AvroDatum::Record(vec![(
1418 "long".to_string(),
1419 AvroDatum::Long(*value),
1420 )])),
1421 ),
1422 AvroValue::Double(value) => {
1423 if !value.is_finite() {
1424 return Err(Error::Codec(
1425 "non_finite_float: Avro Value doubles must be finite".to_string(),
1426 ));
1427 }
1428 AvroDatum::Union(
1429 3,
1430 Box::new(AvroDatum::Record(vec![(
1431 "double".to_string(),
1432 AvroDatum::Double(*value),
1433 )])),
1434 )
1435 }
1436 AvroValue::Bytes(value) => AvroDatum::Union(
1437 4,
1438 Box::new(AvroDatum::Record(vec![(
1439 "bytes".to_string(),
1440 AvroDatum::Bytes(value.clone()),
1441 )])),
1442 ),
1443 AvroValue::String(value) => AvroDatum::Union(
1444 5,
1445 Box::new(AvroDatum::Record(vec![(
1446 "string".to_string(),
1447 AvroDatum::String(value.clone()),
1448 )])),
1449 ),
1450 AvroValue::Array(values) => AvroDatum::Union(
1451 6,
1452 Box::new(AvroDatum::Record(vec![(
1453 "items".to_string(),
1454 AvroDatum::Array(
1455 values
1456 .iter()
1457 .map(avro_value_to_datum)
1458 .collect::<Result<Vec<_>>>()?,
1459 ),
1460 )])),
1461 ),
1462 AvroValue::Map(values) => AvroDatum::Union(
1463 7,
1464 Box::new(AvroDatum::Record(vec![(
1465 "entries".to_string(),
1466 AvroDatum::Map(
1467 values
1468 .iter()
1469 .map(|(key, value)| Ok((key.clone(), avro_value_to_datum(value)?)))
1470 .collect::<Result<HashMap<_, _>>>()?,
1471 ),
1472 )])),
1473 ),
1474 };
1475 Ok(AvroDatum::Record(vec![("value".to_string(), branch)]))
1476}
1477
1478fn avro_value_from_datum(datum: AvroDatum) -> Result<AvroValue> {
1479 let AvroDatum::Record(mut outer) = datum else {
1480 return Err(Error::Codec(
1481 "invalid_payload_framing: datum is not a Value record".to_string(),
1482 ));
1483 };
1484 let (_, branch) = outer
1485 .pop()
1486 .filter(|(name, _)| name == "value")
1487 .ok_or_else(|| Error::Codec("invalid_payload_framing: Value field missing".to_string()))?;
1488 let AvroDatum::Union(_, branch) = branch else {
1489 return Err(Error::Codec(
1490 "invalid_payload_framing: invalid Value union".to_string(),
1491 ));
1492 };
1493 match *branch {
1494 AvroDatum::Null => Ok(AvroValue::Null),
1495 AvroDatum::Record(mut fields) => {
1496 let (name, value) = fields.pop().ok_or_else(|| {
1497 Error::Codec("invalid_payload_framing: empty Value branch".to_string())
1498 })?;
1499 match (name.as_str(), value) {
1500 ("boolean", AvroDatum::Boolean(value)) => Ok(AvroValue::Boolean(value)),
1501 ("long", AvroDatum::Long(value)) => Ok(AvroValue::Long(value)),
1502 ("double", AvroDatum::Double(value)) if value.is_finite() => {
1503 Ok(AvroValue::Double(value))
1504 }
1505 ("bytes", AvroDatum::Bytes(value)) => Ok(AvroValue::Bytes(value)),
1506 ("string", AvroDatum::String(value)) => Ok(AvroValue::String(value)),
1507 ("items", AvroDatum::Array(values)) => values
1508 .into_iter()
1509 .map(avro_value_from_datum)
1510 .collect::<Result<Vec<_>>>()
1511 .map(AvroValue::Array),
1512 ("entries", AvroDatum::Map(values)) => values
1513 .into_iter()
1514 .map(|(key, value)| Ok((key, avro_value_from_datum(value)?)))
1515 .collect::<Result<BTreeMap<_, _>>>()
1516 .map(AvroValue::Map),
1517 _ => Err(Error::Codec(
1518 "invalid_payload_framing: unknown Value branch".to_string(),
1519 )),
1520 }
1521 }
1522 _ => Err(Error::Codec(
1523 "invalid_payload_framing: invalid Value branch".to_string(),
1524 )),
1525 }
1526}
1527
1528fn avro_value_schema() -> Result<&'static Schema> {
1529 match AVRO_VALUE_SCHEMA.get_or_init(|| {
1530 Schema::parse_str(AVRO_VALUE_SCHEMA_JSON)
1531 .map_err(|err| format!("could not parse Avro Value schema: {err}"))
1532 }) {
1533 Ok(schema) => Ok(schema),
1534 Err(message) => Err(Error::Codec(message.clone())),
1535 }
1536}
1537
1538#[derive(Clone, Debug)]
1539pub struct Client {
1540 http: reqwest::Client,
1541 base_url: String,
1542 token: Option<String>,
1543 control_token: Option<String>,
1544 worker_token: Option<String>,
1545 namespace: String,
1546}
1547
1548impl Client {
1549 pub fn new(base_url: impl Into<String>) -> Result<Self> {
1550 Self::builder(base_url).build()
1551 }
1552
1553 pub fn builder(base_url: impl Into<String>) -> ClientBuilder {
1554 ClientBuilder {
1555 base_url: base_url.into(),
1556 token: None,
1557 control_token: None,
1558 worker_token: None,
1559 namespace: "default".to_string(),
1560 timeout: Duration::from_secs(60),
1561 }
1562 }
1563
1564 pub async fn health(&self) -> Result<Value> {
1565 self.request_json(
1566 reqwest::Method::GET,
1567 "/health",
1568 RequestProtocol::ControlPlane,
1569 Option::<&Value>::None,
1570 )
1571 .await
1572 }
1573
1574 pub async fn cluster_info(&self) -> Result<Value> {
1575 self.request_json(
1576 reqwest::Method::GET,
1577 "/cluster/info",
1578 RequestProtocol::ControlPlane,
1579 Option::<&Value>::None,
1580 )
1581 .await
1582 }
1583
1584 pub async fn start_workflow<T: Serialize>(
1585 &self,
1586 workflow_type: &str,
1587 task_queue: &str,
1588 workflow_id: &str,
1589 input: T,
1590 ) -> Result<WorkflowHandle> {
1591 self.start_workflow_with_options(
1592 workflow_type,
1593 task_queue,
1594 workflow_id,
1595 WorkflowStartOptions::default(),
1596 input,
1597 )
1598 .await
1599 }
1600
1601 pub async fn start_workflow_with_options<T: Serialize>(
1604 &self,
1605 workflow_type: &str,
1606 task_queue: &str,
1607 workflow_id: &str,
1608 options: WorkflowStartOptions,
1609 input: T,
1610 ) -> Result<WorkflowHandle> {
1611 options.validate()?;
1612 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1613 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
1614 let body = json!({
1615 "workflow_id": workflow_id,
1616 "workflow_type": workflow_type,
1617 "task_queue": task_queue,
1618 "input": input_envelope,
1619 "execution_timeout_seconds": options.execution_timeout_seconds,
1620 "run_timeout_seconds": options.run_timeout_seconds
1621 });
1622
1623 let data: Value = self
1624 .request_json(
1625 reqwest::Method::POST,
1626 "/workflows",
1627 RequestProtocol::ControlPlane,
1628 Some(&body),
1629 )
1630 .await?;
1631
1632 Ok(WorkflowHandle {
1633 client: self.clone(),
1634 workflow_id: data
1635 .get("workflow_id")
1636 .and_then(Value::as_str)
1637 .unwrap_or(workflow_id)
1638 .to_string(),
1639 run_id: data
1640 .get("run_id")
1641 .and_then(Value::as_str)
1642 .map(str::to_string),
1643 workflow_type: data
1644 .get("workflow_type")
1645 .and_then(Value::as_str)
1646 .unwrap_or(workflow_type)
1647 .to_string(),
1648 })
1649 }
1650
1651 pub async fn signal_workflow<T: Serialize>(
1652 &self,
1653 workflow_id: &str,
1654 signal_name: &str,
1655 input: T,
1656 ) -> Result<Value> {
1657 self.signal_workflow_target(workflow_id, None, signal_name, input)
1658 .await
1659 }
1660
1661 pub async fn signal_workflow_run<T: Serialize>(
1663 &self,
1664 workflow_id: &str,
1665 run_id: &str,
1666 signal_name: &str,
1667 input: T,
1668 ) -> Result<Value> {
1669 self.signal_workflow_target(workflow_id, Some(run_id), signal_name, input)
1670 .await
1671 }
1672
1673 async fn signal_workflow_target<T: Serialize>(
1674 &self,
1675 workflow_id: &str,
1676 run_id: Option<&str>,
1677 signal_name: &str,
1678 input: T,
1679 ) -> Result<Value> {
1680 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1681 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
1682 let body = json!({
1683 "input": input_envelope
1684 });
1685 let path = match run_id {
1686 Some(run_id) => {
1687 format!("/workflows/{workflow_id}/runs/{run_id}/signal/{signal_name}")
1688 }
1689 None => format!("/workflows/{workflow_id}/signal/{signal_name}"),
1690 };
1691 self.request_json(
1692 reqwest::Method::POST,
1693 &path,
1694 RequestProtocol::ControlPlane,
1695 Some(&body),
1696 )
1697 .await
1698 }
1699
1700 pub async fn cancel_workflow(
1702 &self,
1703 workflow_id: &str,
1704 options: WorkflowCommandOptions,
1705 ) -> Result<WorkflowCommandResult> {
1706 self.workflow_command(workflow_id, None, WorkflowCommandKind::Cancel, options)
1707 .await
1708 }
1709
1710 pub async fn cancel_workflow_run(
1712 &self,
1713 workflow_id: &str,
1714 run_id: &str,
1715 options: WorkflowCommandOptions,
1716 ) -> Result<WorkflowCommandResult> {
1717 self.workflow_command(
1718 workflow_id,
1719 Some(run_id),
1720 WorkflowCommandKind::Cancel,
1721 options,
1722 )
1723 .await
1724 }
1725
1726 pub async fn terminate_workflow(
1728 &self,
1729 workflow_id: &str,
1730 options: WorkflowCommandOptions,
1731 ) -> Result<WorkflowCommandResult> {
1732 self.workflow_command(workflow_id, None, WorkflowCommandKind::Terminate, options)
1733 .await
1734 }
1735
1736 pub async fn terminate_workflow_run(
1738 &self,
1739 workflow_id: &str,
1740 run_id: &str,
1741 options: WorkflowCommandOptions,
1742 ) -> Result<WorkflowCommandResult> {
1743 self.workflow_command(
1744 workflow_id,
1745 Some(run_id),
1746 WorkflowCommandKind::Terminate,
1747 options,
1748 )
1749 .await
1750 }
1751
1752 async fn workflow_command(
1753 &self,
1754 workflow_id: &str,
1755 run_id: Option<&str>,
1756 command: WorkflowCommandKind,
1757 options: WorkflowCommandOptions,
1758 ) -> Result<WorkflowCommandResult> {
1759 let path = match run_id {
1760 Some(run_id) => format!(
1761 "/workflows/{workflow_id}/runs/{run_id}/{}",
1762 command.as_str()
1763 ),
1764 None => format!("/workflows/{workflow_id}/{}", command.as_str()),
1765 };
1766 let data = match self
1767 .request_json(
1768 reqwest::Method::POST,
1769 &path,
1770 RequestProtocol::ControlPlane,
1771 Some(&options),
1772 )
1773 .await
1774 {
1775 Ok(data) => data,
1776 Err(Error::Http { status, body }) => {
1777 return Err(Error::WorkflowCommandRejected(workflow_command_rejection(
1778 command,
1779 status,
1780 body,
1781 workflow_id,
1782 run_id,
1783 )));
1784 }
1785 Err(error) => return Err(error),
1786 };
1787
1788 Ok(workflow_command_result(command, data, workflow_id, run_id))
1789 }
1790
1791 pub async fn query_workflow<T: Serialize>(
1797 &self,
1798 workflow_id: &str,
1799 query_name: &str,
1800 input: T,
1801 ) -> Result<Value> {
1802 self.query_workflow_target(workflow_id, None, query_name, input)
1803 .await
1804 }
1805
1806 pub async fn query_workflow_run<T: Serialize>(
1808 &self,
1809 workflow_id: &str,
1810 run_id: &str,
1811 query_name: &str,
1812 input: T,
1813 ) -> Result<Value> {
1814 self.query_workflow_target(workflow_id, Some(run_id), query_name, input)
1815 .await
1816 }
1817
1818 pub async fn query_workflow_avro_value<T: Serialize>(
1820 &self,
1821 workflow_id: &str,
1822 query_name: &str,
1823 input: T,
1824 ) -> Result<AvroValue> {
1825 self.query_workflow_avro_value_target(workflow_id, None, query_name, input)
1826 .await
1827 }
1828
1829 pub async fn query_workflow_run_avro_value<T: Serialize>(
1831 &self,
1832 workflow_id: &str,
1833 run_id: &str,
1834 query_name: &str,
1835 input: T,
1836 ) -> Result<AvroValue> {
1837 self.query_workflow_avro_value_target(workflow_id, Some(run_id), query_name, input)
1838 .await
1839 }
1840
1841 async fn query_workflow_avro_value_target<T: Serialize>(
1842 &self,
1843 workflow_id: &str,
1844 run_id: Option<&str>,
1845 query_name: &str,
1846 input: T,
1847 ) -> Result<AvroValue> {
1848 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1849 let body = json!({"input": encode_typed_envelope(&input, DEFAULT_CODEC)?});
1850 let path = match run_id {
1851 Some(run_id) => {
1852 format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
1853 }
1854 None => format!("/workflows/{workflow_id}/query/{query_name}"),
1855 };
1856 let response: Value = match self
1857 .request_json(
1858 reqwest::Method::POST,
1859 &path,
1860 RequestProtocol::ControlPlane,
1861 Some(&body),
1862 )
1863 .await
1864 {
1865 Ok(response) => response,
1866 Err(Error::Http { status, body }) => {
1867 return Err(Error::QueryFailed(query_failure(status, body)));
1868 }
1869 Err(error) => return Err(error),
1870 };
1871
1872 let envelope = response
1873 .get("result_envelope")
1874 .filter(|envelope| !envelope.is_null())
1875 .ok_or_else(|| {
1876 Error::Codec(
1877 "missing_payload_envelope: typed query result requires result_envelope"
1878 .to_string(),
1879 )
1880 })?;
1881 decode_wire_avro_value(envelope, DEFAULT_CODEC)
1882 }
1883
1884 async fn query_workflow_target<T: Serialize>(
1885 &self,
1886 workflow_id: &str,
1887 run_id: Option<&str>,
1888 query_name: &str,
1889 input: T,
1890 ) -> Result<Value> {
1891 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1892 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
1893 let body = json!({
1894 "input": input_envelope
1895 });
1896 let path = match run_id {
1897 Some(run_id) => {
1898 format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
1899 }
1900 None => format!("/workflows/{workflow_id}/query/{query_name}"),
1901 };
1902 let response: Value = match self
1903 .request_json(
1904 reqwest::Method::POST,
1905 &path,
1906 RequestProtocol::ControlPlane,
1907 Some(&body),
1908 )
1909 .await
1910 {
1911 Ok(response) => response,
1912 Err(Error::Http { status, body }) => {
1913 return Err(Error::QueryFailed(query_failure(status, body)));
1914 }
1915 Err(error) => return Err(error),
1916 };
1917
1918 if let Some(envelope) = response
1919 .get("result_envelope")
1920 .filter(|envelope| !envelope.is_null())
1921 {
1922 return decode_wire_value(envelope, DEFAULT_CODEC);
1923 }
1924
1925 Ok(response.get("result").cloned().unwrap_or(Value::Null))
1926 }
1927
1928 pub async fn update_workflow<T: Serialize>(
1930 &self,
1931 workflow_id: &str,
1932 update_name: &str,
1933 input: T,
1934 request_id: Option<&str>,
1935 ) -> Result<Value> {
1936 let response = self
1937 .update_workflow_response(workflow_id, update_name, input, request_id)
1938 .await?;
1939 if let Some(envelope) = response
1940 .get("result_envelope")
1941 .filter(|envelope| !envelope.is_null())
1942 {
1943 return decode_wire_value(envelope, DEFAULT_CODEC);
1944 }
1945 Ok(response.get("result").cloned().unwrap_or(response))
1946 }
1947
1948 pub async fn update_workflow_avro_value<T: Serialize>(
1950 &self,
1951 workflow_id: &str,
1952 update_name: &str,
1953 input: T,
1954 request_id: Option<&str>,
1955 ) -> Result<AvroValue> {
1956 let response = self
1957 .update_workflow_response(workflow_id, update_name, input, request_id)
1958 .await?;
1959 let envelope = response
1960 .get("result_envelope")
1961 .filter(|envelope| !envelope.is_null())
1962 .ok_or_else(|| {
1963 Error::Codec(
1964 "missing_payload_envelope: typed update result requires result_envelope"
1965 .to_string(),
1966 )
1967 })?;
1968 decode_wire_avro_value(envelope, DEFAULT_CODEC)
1969 }
1970
1971 async fn update_workflow_response<T: Serialize>(
1972 &self,
1973 workflow_id: &str,
1974 update_name: &str,
1975 input: T,
1976 request_id: Option<&str>,
1977 ) -> Result<Value> {
1978 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1979 let mut body = json!({
1980 "input": encode_typed_envelope(&input, DEFAULT_CODEC)?,
1981 "wait_for": "completed",
1982 });
1983 if let Some(request_id) = request_id {
1984 body["request_id"] = json!(request_id);
1985 }
1986 self.request_json(
1987 reqwest::Method::POST,
1988 &format!("/workflows/{workflow_id}/update/{update_name}"),
1989 RequestProtocol::ControlPlane,
1990 Some(&body),
1991 )
1992 .await
1993 }
1994
1995 pub async fn describe_workflow(&self, workflow_id: &str) -> Result<WorkflowDescription> {
1996 let path = format!("/workflows/{workflow_id}");
1997 let mut data: WorkflowDescription = self
1998 .request_json(
1999 reqwest::Method::GET,
2000 &path,
2001 RequestProtocol::ControlPlane,
2002 Option::<&Value>::None,
2003 )
2004 .await?;
2005 data.decode_payloads()?;
2006 Ok(data)
2007 }
2008
2009 pub async fn describe_workflow_run(
2011 &self,
2012 workflow_id: &str,
2013 run_id: &str,
2014 ) -> Result<WorkflowDescription> {
2015 let path = format!("/workflows/{workflow_id}/runs/{run_id}");
2016 let mut data: WorkflowDescription = self
2017 .request_json(
2018 reqwest::Method::GET,
2019 &path,
2020 RequestProtocol::ControlPlane,
2021 Option::<&Value>::None,
2022 )
2023 .await?;
2024 data.decode_payloads()?;
2025 Ok(data)
2026 }
2027
2028 pub async fn register_worker(
2029 &self,
2030 worker_id: &str,
2031 task_queue: &str,
2032 supported_workflow_types: Vec<String>,
2033 supported_activity_types: Vec<String>,
2034 max_concurrent_workflow_tasks: usize,
2035 max_concurrent_activity_tasks: usize,
2036 ) -> Result<RegisterWorkerResponse> {
2037 self.register_worker_with_capabilities(
2038 worker_id,
2039 task_queue,
2040 supported_workflow_types,
2041 supported_activity_types,
2042 max_concurrent_workflow_tasks,
2043 max_concurrent_activity_tasks,
2044 Vec::new(),
2045 )
2046 .await
2047 }
2048
2049 pub async fn register_worker_with_capabilities(
2051 &self,
2052 worker_id: &str,
2053 task_queue: &str,
2054 supported_workflow_types: Vec<String>,
2055 supported_activity_types: Vec<String>,
2056 max_concurrent_workflow_tasks: usize,
2057 max_concurrent_activity_tasks: usize,
2058 capabilities: Vec<String>,
2059 ) -> Result<RegisterWorkerResponse> {
2060 self.register_worker_with_command_contracts(
2061 worker_id,
2062 task_queue,
2063 supported_workflow_types,
2064 supported_activity_types,
2065 max_concurrent_workflow_tasks,
2066 max_concurrent_activity_tasks,
2067 capabilities,
2068 Value::Object(serde_json::Map::new()),
2069 )
2070 .await
2071 }
2072
2073 #[allow(clippy::too_many_arguments)]
2075 pub async fn register_worker_with_command_contracts(
2076 &self,
2077 worker_id: &str,
2078 task_queue: &str,
2079 supported_workflow_types: Vec<String>,
2080 supported_activity_types: Vec<String>,
2081 max_concurrent_workflow_tasks: usize,
2082 max_concurrent_activity_tasks: usize,
2083 capabilities: Vec<String>,
2084 workflow_command_contracts: Value,
2085 ) -> Result<RegisterWorkerResponse> {
2086 let mut body = json!({
2087 "worker_id": worker_id,
2088 "task_queue": task_queue,
2089 "runtime": "rust",
2090 "sdk_version": SDK_VERSION,
2091 "supported_workflow_types": supported_workflow_types,
2092 "supported_activity_types": supported_activity_types,
2093 "capabilities": capabilities,
2094 "max_concurrent_workflow_tasks": max_concurrent_workflow_tasks,
2095 "max_concurrent_activity_tasks": max_concurrent_activity_tasks
2096 });
2097 if workflow_command_contracts
2098 .as_object()
2099 .is_some_and(|contracts| !contracts.is_empty())
2100 {
2101 body["workflow_command_contracts"] = workflow_command_contracts;
2102 }
2103
2104 self.request_json(
2105 reqwest::Method::POST,
2106 "/worker/register",
2107 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2108 Some(&body),
2109 )
2110 .await
2111 }
2112
2113 pub async fn poll_query_task(
2115 &self,
2116 worker_id: &str,
2117 task_queue: &str,
2118 timeout: Duration,
2119 ) -> Result<Option<QueryTask>> {
2120 Ok(self
2121 .poll_query_task_response(worker_id, task_queue, timeout)
2122 .await?
2123 .task)
2124 }
2125
2126 pub async fn poll_query_task_response(
2128 &self,
2129 worker_id: &str,
2130 task_queue: &str,
2131 timeout: Duration,
2132 ) -> Result<PollQueryTaskResponse> {
2133 let poll_request_id = unique_request_id("rust-query-poll");
2134 self.poll_query_task_response_with_request_id(
2135 worker_id,
2136 task_queue,
2137 timeout,
2138 &poll_request_id,
2139 1,
2140 )
2141 .await
2142 }
2143
2144 async fn poll_query_task_response_with_request_id(
2145 &self,
2146 worker_id: &str,
2147 task_queue: &str,
2148 timeout: Duration,
2149 poll_request_id: &str,
2150 transport_retries: usize,
2151 ) -> Result<PollQueryTaskResponse> {
2152 let timeout_seconds = long_poll_timeout_seconds(timeout);
2153 let body = json!({
2154 "worker_id": worker_id,
2155 "task_queue": task_queue,
2156 "poll_request_id": poll_request_id,
2157 "timeout_seconds": timeout_seconds,
2158 });
2159 self.poll_request_json(
2160 "/worker/query-tasks/poll",
2161 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
2162 &body,
2163 timeout + Duration::from_secs(5),
2164 transport_retries,
2165 )
2166 .await
2167 }
2168
2169 pub async fn complete_query_task<T: Serialize>(
2171 &self,
2172 query_task_id: &str,
2173 lease_owner: &str,
2174 query_task_attempt: u64,
2175 result: T,
2176 codec: &str,
2177 ) -> Result<Value> {
2178 let typed_result = AvroValue::from_serialize(&result)?;
2179 let result_envelope = encode_typed_envelope(&typed_result, codec)?;
2180 self.complete_query_task_with_envelope(
2181 query_task_id,
2182 lease_owner,
2183 query_task_attempt,
2184 typed_result.into_json()?,
2185 result_envelope,
2186 )
2187 .await
2188 }
2189
2190 async fn complete_query_task_with_envelope(
2191 &self,
2192 query_task_id: &str,
2193 lease_owner: &str,
2194 query_task_attempt: u64,
2195 result: Value,
2196 result_envelope: Value,
2197 ) -> Result<Value> {
2198 let body = json!({
2199 "lease_owner": lease_owner,
2200 "query_task_attempt": query_task_attempt,
2201 "result": result,
2202 "result_envelope": result_envelope,
2203 });
2204 let path = format!("/worker/query-tasks/{query_task_id}/complete");
2205 let response = self
2206 .request_json(
2207 reqwest::Method::POST,
2208 &path,
2209 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
2210 Some(&body),
2211 )
2212 .await;
2213 query_task_response(response)
2214 }
2215
2216 pub async fn fail_query_task(
2218 &self,
2219 query_task_id: &str,
2220 lease_owner: &str,
2221 query_task_attempt: u64,
2222 message: impl Into<String>,
2223 reason: impl Into<String>,
2224 failure_type: impl Into<String>,
2225 ) -> Result<Value> {
2226 let body = json!({
2227 "lease_owner": lease_owner,
2228 "query_task_attempt": query_task_attempt,
2229 "failure": {
2230 "message": message.into(),
2231 "reason": reason.into(),
2232 "type": failure_type.into(),
2233 }
2234 });
2235 let path = format!("/worker/query-tasks/{query_task_id}/fail");
2236 let response = self
2237 .request_json(
2238 reqwest::Method::POST,
2239 &path,
2240 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
2241 Some(&body),
2242 )
2243 .await;
2244 query_task_response(response)
2245 }
2246
2247 pub async fn heartbeat_worker(
2248 &self,
2249 worker_id: &str,
2250 workflow_available: usize,
2251 activity_available: usize,
2252 ) -> Result<Value> {
2253 let body = json!({
2254 "worker_id": worker_id,
2255 "task_slots": {
2256 "workflow_available": workflow_available,
2257 "activity_available": activity_available
2258 },
2259 "process_metrics": {
2260 "process_id": std::process::id(),
2261 "process_uptime_seconds": 0
2262 }
2263 });
2264
2265 self.request_json(
2266 reqwest::Method::POST,
2267 "/worker/heartbeat",
2268 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2269 Some(&body),
2270 )
2271 .await
2272 }
2273
2274 pub async fn poll_workflow_task(
2275 &self,
2276 worker_id: &str,
2277 task_queue: &str,
2278 timeout: Duration,
2279 ) -> Result<Option<WorkflowTask>> {
2280 Ok(self
2281 .poll_workflow_task_response(worker_id, task_queue, timeout)
2282 .await?
2283 .task)
2284 }
2285
2286 pub async fn poll_workflow_task_response(
2287 &self,
2288 worker_id: &str,
2289 task_queue: &str,
2290 timeout: Duration,
2291 ) -> Result<PollWorkflowTaskResponse> {
2292 let poll_request_id = unique_request_id("rust-workflow-poll");
2293 self.poll_workflow_task_response_with_request_id(
2294 worker_id,
2295 task_queue,
2296 timeout,
2297 &poll_request_id,
2298 1,
2299 )
2300 .await
2301 }
2302
2303 async fn poll_workflow_task_response_with_request_id(
2304 &self,
2305 worker_id: &str,
2306 task_queue: &str,
2307 timeout: Duration,
2308 poll_request_id: &str,
2309 transport_retries: usize,
2310 ) -> Result<PollWorkflowTaskResponse> {
2311 let body = json!({
2312 "worker_id": worker_id,
2313 "task_queue": task_queue,
2314 "poll_request_id": poll_request_id,
2315 "timeout_seconds": long_poll_timeout_seconds(timeout),
2316 });
2317 let mut data: PollWorkflowTaskResponse = self
2318 .poll_request_json(
2319 "/worker/workflow-tasks/poll",
2320 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2321 &body,
2322 timeout + Duration::from_secs(5),
2323 transport_retries,
2324 )
2325 .await?;
2326
2327 if let Some(task) = data.task.as_mut() {
2328 self.fetch_remaining_workflow_history(worker_id, task)
2329 .await?;
2330 }
2331
2332 Ok(data)
2333 }
2334
2335 async fn fetch_remaining_workflow_history(
2336 &self,
2337 worker_id: &str,
2338 task: &mut WorkflowTask,
2339 ) -> Result<()> {
2340 let mut next_token = task.next_history_page_token.clone();
2341
2342 while let Some(token) = next_token.take().filter(|token| !token.is_empty()) {
2343 let lease_owner = task
2344 .lease_owner
2345 .clone()
2346 .unwrap_or_else(|| worker_id.to_string());
2347 let page = self
2348 .workflow_task_history_page(
2349 &task.task_id,
2350 &lease_owner,
2351 task.workflow_task_attempt,
2352 &token,
2353 )
2354 .await?;
2355
2356 task.append_history_page(page);
2357
2358 if task.next_history_page_token.as_deref() == Some(token.as_str()) {
2359 return Err(Error::Codec(
2360 "workflow history pagination returned the same page token".to_string(),
2361 ));
2362 }
2363
2364 next_token = task.next_history_page_token.clone();
2365 }
2366
2367 Ok(())
2368 }
2369
2370 async fn workflow_task_history_page(
2371 &self,
2372 task_id: &str,
2373 lease_owner: &str,
2374 workflow_task_attempt: u64,
2375 next_history_page_token: &str,
2376 ) -> Result<WorkflowTaskHistoryPage> {
2377 let body = json!({
2378 "lease_owner": lease_owner,
2379 "workflow_task_attempt": workflow_task_attempt,
2380 "next_history_page_token": next_history_page_token
2381 });
2382 let path = format!("/worker/workflow-tasks/{task_id}/history");
2383
2384 self.request_json(
2385 reqwest::Method::POST,
2386 &path,
2387 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2388 Some(&body),
2389 )
2390 .await
2391 }
2392
2393 pub async fn complete_workflow_task(
2394 &self,
2395 task_id: &str,
2396 lease_owner: &str,
2397 workflow_task_attempt: u64,
2398 commands: Vec<Value>,
2399 ) -> Result<Value> {
2400 let body = json!({
2401 "lease_owner": lease_owner,
2402 "workflow_task_attempt": workflow_task_attempt,
2403 "commands": commands
2404 });
2405 let path = format!("/worker/workflow-tasks/{task_id}/complete");
2406 self.request_json(
2407 reqwest::Method::POST,
2408 &path,
2409 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2410 Some(&body),
2411 )
2412 .await
2413 }
2414
2415 pub async fn fail_workflow_task(
2416 &self,
2417 task_id: &str,
2418 lease_owner: &str,
2419 workflow_task_attempt: u64,
2420 message: impl Into<String>,
2421 ) -> Result<Value> {
2422 self.fail_workflow_task_with_type(
2423 task_id,
2424 lease_owner,
2425 workflow_task_attempt,
2426 message,
2427 "RustWorkflowTaskFailure",
2428 )
2429 .await
2430 }
2431
2432 async fn fail_workflow_task_with_type(
2433 &self,
2434 task_id: &str,
2435 lease_owner: &str,
2436 workflow_task_attempt: u64,
2437 message: impl Into<String>,
2438 failure_type: &str,
2439 ) -> Result<Value> {
2440 let body = json!({
2441 "lease_owner": lease_owner,
2442 "workflow_task_attempt": workflow_task_attempt,
2443 "failure": {
2444 "message": message.into(),
2445 "type": failure_type
2446 }
2447 });
2448 let path = format!("/worker/workflow-tasks/{task_id}/fail");
2449 self.request_json(
2450 reqwest::Method::POST,
2451 &path,
2452 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2453 Some(&body),
2454 )
2455 .await
2456 }
2457
2458 pub async fn poll_activity_task(
2459 &self,
2460 worker_id: &str,
2461 task_queue: &str,
2462 timeout: Duration,
2463 ) -> Result<Option<ActivityTask>> {
2464 Ok(self
2465 .poll_activity_task_response(worker_id, task_queue, timeout)
2466 .await?
2467 .task)
2468 }
2469
2470 pub async fn poll_activity_task_response(
2472 &self,
2473 worker_id: &str,
2474 task_queue: &str,
2475 timeout: Duration,
2476 ) -> Result<PollActivityTaskResponse> {
2477 let poll_request_id = unique_request_id("rust-activity-poll");
2478 self.poll_activity_task_response_with_request_id(
2479 worker_id,
2480 task_queue,
2481 timeout,
2482 &poll_request_id,
2483 1,
2484 )
2485 .await
2486 }
2487
2488 async fn poll_activity_task_response_with_request_id(
2489 &self,
2490 worker_id: &str,
2491 task_queue: &str,
2492 timeout: Duration,
2493 poll_request_id: &str,
2494 transport_retries: usize,
2495 ) -> Result<PollActivityTaskResponse> {
2496 let body = json!({
2497 "worker_id": worker_id,
2498 "task_queue": task_queue,
2499 "poll_request_id": poll_request_id,
2500 "timeout_seconds": long_poll_timeout_seconds(timeout),
2501 });
2502 let data: PollActivityTaskResponse = self
2503 .poll_request_json(
2504 "/worker/activity-tasks/poll",
2505 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2506 &body,
2507 timeout + Duration::from_secs(5),
2508 transport_retries,
2509 )
2510 .await?;
2511 Ok(data)
2512 }
2513
2514 pub async fn complete_activity_task<T: Serialize>(
2515 &self,
2516 task_id: &str,
2517 activity_attempt_id: &str,
2518 lease_owner: &str,
2519 result: T,
2520 codec: &str,
2521 ) -> Result<Value> {
2522 let result = encode_typed_envelope(&AvroValue::from_serialize(&result)?, codec)?;
2523 let body = json!({
2524 "activity_attempt_id": activity_attempt_id,
2525 "lease_owner": lease_owner,
2526 "result": result
2527 });
2528 let path = format!("/worker/activity-tasks/{task_id}/complete");
2529 activity_task_response(
2530 self.request_json(
2531 reqwest::Method::POST,
2532 &path,
2533 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2534 Some(&body),
2535 )
2536 .await,
2537 "complete",
2538 task_id,
2539 activity_attempt_id,
2540 )
2541 }
2542
2543 pub async fn fail_activity_task(
2544 &self,
2545 task_id: &str,
2546 activity_attempt_id: &str,
2547 lease_owner: &str,
2548 message: impl Into<String>,
2549 non_retryable: bool,
2550 ) -> Result<Value> {
2551 let body = json!({
2552 "activity_attempt_id": activity_attempt_id,
2553 "lease_owner": lease_owner,
2554 "failure": {
2555 "message": message.into(),
2556 "type": "RustActivityFailure",
2557 "non_retryable": non_retryable
2558 }
2559 });
2560 let path = format!("/worker/activity-tasks/{task_id}/fail");
2561 activity_task_response(
2562 self.request_json(
2563 reqwest::Method::POST,
2564 &path,
2565 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2566 Some(&body),
2567 )
2568 .await,
2569 "fail",
2570 task_id,
2571 activity_attempt_id,
2572 )
2573 }
2574
2575 pub async fn heartbeat_activity_task<T: Serialize>(
2576 &self,
2577 task_id: &str,
2578 activity_attempt_id: &str,
2579 lease_owner: &str,
2580 details: T,
2581 ) -> Result<ActivityHeartbeatResponse> {
2582 let details = encode_typed_envelope(&AvroValue::from_serialize(&details)?, DEFAULT_CODEC)?;
2583 let body = json!({
2584 "activity_attempt_id": activity_attempt_id,
2585 "lease_owner": lease_owner,
2586 "details": details
2587 });
2588 let path = format!("/worker/activity-tasks/{task_id}/heartbeat");
2589 activity_task_response(
2590 self.request_json(
2591 reqwest::Method::POST,
2592 &path,
2593 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2594 Some(&body),
2595 )
2596 .await,
2597 "heartbeat",
2598 task_id,
2599 activity_attempt_id,
2600 )
2601 }
2602
2603 async fn request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
2604 &self,
2605 method: reqwest::Method,
2606 path: &str,
2607 protocol: RequestProtocol,
2608 body: Option<&B>,
2609 ) -> Result<T> {
2610 self.request_json_with_timeout(method, path, protocol, body, Duration::from_secs(60))
2611 .await
2612 }
2613
2614 async fn request_json_with_timeout<T: DeserializeOwned, B: Serialize + ?Sized>(
2615 &self,
2616 method: reqwest::Method,
2617 path: &str,
2618 protocol: RequestProtocol,
2619 body: Option<&B>,
2620 timeout: Duration,
2621 ) -> Result<T> {
2622 let mut request = self
2623 .http
2624 .request(method, format!("{}/api{}", self.base_url, path))
2625 .timeout(timeout)
2626 .header(reqwest::header::ACCEPT, "application/json")
2627 .header(reqwest::header::CONTENT_TYPE, "application/json")
2628 .header("X-Namespace", &self.namespace);
2629
2630 match protocol {
2631 RequestProtocol::Worker(version) => {
2632 request = request.header("X-Durable-Workflow-Protocol-Version", version);
2633 }
2634 RequestProtocol::ControlPlane => {
2635 request = request.header(
2636 "X-Durable-Workflow-Control-Plane-Version",
2637 CONTROL_PLANE_VERSION,
2638 );
2639 }
2640 }
2641
2642 if let Some(token) = self.auth_token(protocol.is_worker()) {
2643 request = request.bearer_auth(token);
2644 }
2645
2646 if let Some(body) = body {
2647 request = request.json(body);
2648 }
2649
2650 let response = request.send().await?;
2651 let status = response.status();
2652 let bytes = response.bytes().await?;
2653
2654 if !status.is_success() {
2655 let body = String::from_utf8_lossy(&bytes).to_string();
2656 if let Some(protocol) = protocol_failure(status, &body) {
2657 return Err(Error::Protocol(protocol));
2658 }
2659 return Err(Error::Http { status, body });
2660 }
2661
2662 if bytes.is_empty() {
2663 return Ok(serde_json::from_value(Value::Null)?);
2664 }
2665
2666 Ok(serde_json::from_slice(&bytes)?)
2667 }
2668
2669 async fn poll_request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
2670 &self,
2671 path: &str,
2672 protocol: RequestProtocol,
2673 body: &B,
2674 timeout: Duration,
2675 max_retries: usize,
2676 ) -> Result<T> {
2677 let mut retries = 0;
2678
2679 loop {
2680 let response = self
2681 .request_json_with_timeout(
2682 reqwest::Method::POST,
2683 path,
2684 protocol,
2685 Some(body),
2686 timeout,
2687 )
2688 .await;
2689
2690 match response {
2691 Err(Error::Transport(_)) if retries < max_retries => retries += 1,
2692 response => return worker_poll_response(response),
2693 }
2694 }
2695 }
2696
2697 fn auth_token(&self, worker: bool) -> Option<&str> {
2698 if worker {
2699 self.worker_token
2700 .as_deref()
2701 .or(self.token.as_deref())
2702 .or(self.control_token.as_deref())
2703 } else {
2704 self.control_token
2705 .as_deref()
2706 .or(self.token.as_deref())
2707 .or(self.worker_token.as_deref())
2708 }
2709 }
2710}
2711
2712fn query_failure(status: reqwest::StatusCode, raw_body: String) -> QueryFailure {
2713 let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
2714 let reason = body
2715 .get("reason")
2716 .and_then(Value::as_str)
2717 .unwrap_or("query_rejected")
2718 .to_string();
2719 let message = body
2720 .get("message")
2721 .or_else(|| body.get("error"))
2722 .and_then(Value::as_str)
2723 .unwrap_or("workflow query was rejected")
2724 .to_string();
2725
2726 QueryFailure {
2727 status: status.as_u16(),
2728 reason,
2729 message,
2730 body,
2731 }
2732}
2733
2734fn workflow_command_result(
2735 command: WorkflowCommandKind,
2736 data: Value,
2737 workflow_id: &str,
2738 run_id: Option<&str>,
2739) -> WorkflowCommandResult {
2740 WorkflowCommandResult {
2741 command,
2742 workflow_id: data
2743 .get("workflow_id")
2744 .and_then(Value::as_str)
2745 .unwrap_or(workflow_id)
2746 .to_string(),
2747 run_id: data
2748 .get("run_id")
2749 .and_then(Value::as_str)
2750 .or(run_id)
2751 .map(str::to_string),
2752 outcome: data
2753 .get("outcome")
2754 .and_then(Value::as_str)
2755 .map(str::to_string),
2756 reason: data
2757 .get("reason")
2758 .and_then(Value::as_str)
2759 .map(str::to_string),
2760 command_status: data
2761 .get("command_status")
2762 .and_then(Value::as_str)
2763 .map(str::to_string),
2764 raw: data,
2765 }
2766}
2767
2768fn workflow_command_rejection(
2769 command: WorkflowCommandKind,
2770 status: reqwest::StatusCode,
2771 raw_body: String,
2772 workflow_id: &str,
2773 run_id: Option<&str>,
2774) -> WorkflowCommandRejection {
2775 let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
2776 WorkflowCommandRejection {
2777 command,
2778 status: status.as_u16(),
2779 reason: body
2780 .get("reason")
2781 .and_then(Value::as_str)
2782 .unwrap_or("workflow_command_rejected")
2783 .to_string(),
2784 message: body
2785 .get("message")
2786 .or_else(|| body.get("error"))
2787 .and_then(Value::as_str)
2788 .unwrap_or("workflow lifecycle command was rejected")
2789 .to_string(),
2790 workflow_id: body
2791 .get("workflow_id")
2792 .and_then(Value::as_str)
2793 .unwrap_or(workflow_id)
2794 .to_string(),
2795 run_id: body
2796 .get("run_id")
2797 .and_then(Value::as_str)
2798 .or(run_id)
2799 .map(str::to_string),
2800 target_scope: body
2801 .get("target_scope")
2802 .and_then(Value::as_str)
2803 .map(str::to_string),
2804 body,
2805 }
2806}
2807
2808fn query_task_response(response: Result<Value>) -> Result<Value> {
2809 match response {
2810 Err(Error::Http { status, body }) => Err(Error::QueryFailed(query_failure(status, body))),
2811 response => response,
2812 }
2813}
2814
2815fn worker_poll_response<T: DeserializeOwned>(response: Result<T>) -> Result<T> {
2816 match response {
2817 Err(Error::Http { status, body })
2818 if status == reqwest::StatusCode::CONFLICT && worker_poll_body_is_stop(&body) =>
2819 {
2820 Ok(serde_json::from_str(&body)?)
2821 }
2822 response => response,
2823 }
2824}
2825
2826fn worker_poll_body_is_stop(body: &str) -> bool {
2827 serde_json::from_str::<Value>(body)
2828 .ok()
2829 .is_some_and(|body| {
2830 worker_poll_is_stop(
2831 body.get("poll_status").and_then(Value::as_str),
2832 body.get("reason").and_then(Value::as_str),
2833 )
2834 })
2835}
2836
2837fn worker_poll_is_stop(poll_status: Option<&str>, reason: Option<&str>) -> bool {
2838 matches!(poll_status, Some("draining" | "stopped"))
2839 || matches!(reason, Some("worker_draining" | "worker_stopped"))
2840}
2841
2842fn query_task_rejection_is_final(error: &Error) -> bool {
2843 matches!(
2844 error,
2845 Error::QueryFailed(failure)
2846 if QUERY_TASK_FINAL_REJECTION_REASONS.contains(&failure.reason.as_str())
2847 )
2848}
2849
2850fn activity_task_response<T>(
2851 response: Result<T>,
2852 operation: &str,
2853 task_id: &str,
2854 activity_attempt_id: &str,
2855) -> Result<T> {
2856 match response {
2857 Err(Error::Http { status, body }) => {
2858 let body = serde_json::from_str(&body).unwrap_or_else(|_| json!({"message": body}));
2859 Err(Error::ActivityTaskRejected(ActivityTaskRejection {
2860 operation: operation.to_string(),
2861 status: status.as_u16(),
2862 reason: body
2863 .get("reason")
2864 .and_then(Value::as_str)
2865 .unwrap_or("activity_task_rejected")
2866 .to_string(),
2867 task_id: body
2868 .get("task_id")
2869 .and_then(Value::as_str)
2870 .unwrap_or(task_id)
2871 .to_string(),
2872 activity_attempt_id: body
2873 .get("activity_attempt_id")
2874 .and_then(Value::as_str)
2875 .unwrap_or(activity_attempt_id)
2876 .to_string(),
2877 cancel_requested: body
2878 .get("cancel_requested")
2879 .and_then(Value::as_bool)
2880 .unwrap_or(false),
2881 can_continue: body.get("can_continue").and_then(Value::as_bool),
2882 run_closed_reason: body
2883 .get("run_closed_reason")
2884 .and_then(Value::as_str)
2885 .map(str::to_string),
2886 body,
2887 }))
2888 }
2889 response => response,
2890 }
2891}
2892
2893fn activity_task_rejection_is_final(error: &Error) -> bool {
2894 matches!(
2895 error,
2896 Error::ActivityTaskRejected(rejection)
2897 if matches!(
2898 rejection.reason.as_str(),
2899 "run_cancelled"
2900 | "run_terminated"
2901 | "attempt_closed"
2902 | "stale_attempt"
2903 | "activity_cancelled"
2904 | "task_cancelled"
2905 | "run_closed"
2906 | "activity_not_running"
2907 | "attempt_not_found"
2908 )
2909 )
2910}
2911
2912fn workflow_task_completion_is_terminal_timeout(
2913 error: &Error,
2914 task_id: &str,
2915 workflow_task_attempt: u64,
2916 run_id: Option<&str>,
2917) -> bool {
2918 let Error::Http { status, body } = error else {
2919 return false;
2920 };
2921 if *status != reqwest::StatusCode::CONFLICT {
2922 return false;
2923 }
2924
2925 let Some(run_id) = run_id else {
2926 return false;
2927 };
2928 let Ok(body) = serde_json::from_str::<Value>(body) else {
2929 return false;
2930 };
2931
2932 body.get("recorded").and_then(Value::as_bool) == Some(false)
2933 && body.get("reason").and_then(Value::as_str) == Some("run_timed_out")
2934 && body.get("run_status").and_then(Value::as_str) == Some("failed")
2935 && body.get("run_id").and_then(Value::as_str) == Some(run_id)
2936 && body.get("task_id").and_then(Value::as_str) == Some(task_id)
2937 && body.get("workflow_task_attempt").and_then(Value::as_u64) == Some(workflow_task_attempt)
2938}
2939
2940fn protocol_failure(status: reqwest::StatusCode, raw_body: &str) -> Option<ProtocolFailure> {
2941 let body: Value = serde_json::from_str(raw_body).ok()?;
2942 let reason = body.get("reason")?.as_str()?;
2943 if !matches!(
2944 reason,
2945 "missing_protocol_version"
2946 | "unsupported_protocol_version"
2947 | "missing_control_plane_version"
2948 | "unsupported_control_plane_version"
2949 ) {
2950 return None;
2951 }
2952
2953 Some(ProtocolFailure {
2954 status: status.as_u16(),
2955 reason: reason.to_string(),
2956 message: body
2957 .get("message")
2958 .or_else(|| body.get("error"))
2959 .and_then(Value::as_str)
2960 .unwrap_or("protocol version rejected")
2961 .to_string(),
2962 supported_version: body
2963 .get("supported_version")
2964 .and_then(Value::as_str)
2965 .map(str::to_string),
2966 requested_version: body
2967 .get("requested_version")
2968 .and_then(Value::as_str)
2969 .map(str::to_string),
2970 body,
2971 })
2972}
2973
2974fn long_poll_timeout_seconds(timeout: Duration) -> u64 {
2975 timeout
2976 .as_secs()
2977 .saturating_add(u64::from(timeout.subsec_nanos() > 0))
2978 .min(MAX_LONG_POLL_TIMEOUT_SECONDS)
2979}
2980
2981fn worker_operation_is_retryable(error: &Error) -> bool {
2982 match error {
2983 Error::Transport(error) => {
2984 error.is_timeout() || error.is_connect() || error.is_request() || error.is_body()
2985 }
2986 Error::Http { status, .. } => {
2987 matches!(
2988 *status,
2989 reqwest::StatusCode::REQUEST_TIMEOUT | reqwest::StatusCode::TOO_MANY_REQUESTS
2990 ) || status.is_server_error()
2991 }
2992 _ => false,
2993 }
2994}
2995
2996fn worker_retry_delay(policy: WorkerRetryPolicy, retry: usize) -> Duration {
2997 let exponent = retry.saturating_sub(1).min(31) as u32;
2998 policy
2999 .initial_backoff
3000 .saturating_mul(1_u32 << exponent)
3001 .min(policy.max_backoff)
3002}
3003
3004#[derive(Debug)]
3005pub struct ClientBuilder {
3006 base_url: String,
3007 token: Option<String>,
3008 control_token: Option<String>,
3009 worker_token: Option<String>,
3010 namespace: String,
3011 timeout: Duration,
3012}
3013
3014impl ClientBuilder {
3015 pub fn token(mut self, token: Option<String>) -> Self {
3016 self.token = token;
3017 self
3018 }
3019
3020 pub fn control_token(mut self, token: Option<String>) -> Self {
3021 self.control_token = token;
3022 self
3023 }
3024
3025 pub fn worker_token(mut self, token: Option<String>) -> Self {
3026 self.worker_token = token;
3027 self
3028 }
3029
3030 pub fn namespace(mut self, namespace: impl Into<String>) -> Self {
3031 self.namespace = namespace.into();
3032 self
3033 }
3034
3035 pub fn timeout(mut self, timeout: Duration) -> Self {
3036 self.timeout = timeout;
3037 self
3038 }
3039
3040 pub fn build(self) -> Result<Client> {
3041 Ok(Client {
3042 http: reqwest::Client::builder().timeout(self.timeout).build()?,
3043 base_url: self.base_url.trim_end_matches('/').to_string(),
3044 token: self.token,
3045 control_token: self.control_token,
3046 worker_token: self.worker_token,
3047 namespace: self.namespace,
3048 })
3049 }
3050}
3051
3052#[derive(Clone, Debug)]
3053pub struct WorkflowHandle {
3054 client: Client,
3055 pub workflow_id: String,
3056 pub run_id: Option<String>,
3057 pub workflow_type: String,
3058}
3059
3060impl WorkflowHandle {
3061 pub async fn describe(&self) -> Result<WorkflowDescription> {
3063 self.client.describe_workflow(&self.workflow_id).await
3064 }
3065
3066 pub async fn describe_selected_run(&self) -> Result<WorkflowDescription> {
3068 let run_id = self.run_id.as_deref().ok_or_else(|| {
3069 Error::Codec("run_id is required for selected-run description".to_string())
3070 })?;
3071 self.client
3072 .describe_workflow_run(&self.workflow_id, run_id)
3073 .await
3074 }
3075
3076 pub async fn signal<T: Serialize>(&self, signal_name: &str, input: T) -> Result<Value> {
3077 self.client
3078 .signal_workflow(&self.workflow_id, signal_name, input)
3079 .await
3080 }
3081
3082 pub async fn signal_selected_run<T: Serialize>(
3084 &self,
3085 signal_name: &str,
3086 input: T,
3087 ) -> Result<Value> {
3088 let run_id = self.run_id.as_deref().ok_or_else(|| {
3089 Error::Codec("run_id is required for selected-run signaling".to_string())
3090 })?;
3091 self.client
3092 .signal_workflow_run(&self.workflow_id, run_id, signal_name, input)
3093 .await
3094 }
3095
3096 pub async fn cancel(&self, options: WorkflowCommandOptions) -> Result<WorkflowCommandResult> {
3098 self.client
3099 .cancel_workflow(&self.workflow_id, options)
3100 .await
3101 }
3102
3103 pub async fn cancel_selected_run(
3105 &self,
3106 options: WorkflowCommandOptions,
3107 ) -> Result<WorkflowCommandResult> {
3108 let run_id = self.run_id.as_deref().ok_or_else(|| {
3109 Error::Codec("run_id is required for selected-run cancellation".to_string())
3110 })?;
3111 self.client
3112 .cancel_workflow_run(&self.workflow_id, run_id, options)
3113 .await
3114 }
3115
3116 pub async fn terminate(
3118 &self,
3119 options: WorkflowCommandOptions,
3120 ) -> Result<WorkflowCommandResult> {
3121 self.client
3122 .terminate_workflow(&self.workflow_id, options)
3123 .await
3124 }
3125
3126 pub async fn terminate_selected_run(
3128 &self,
3129 options: WorkflowCommandOptions,
3130 ) -> Result<WorkflowCommandResult> {
3131 let run_id = self.run_id.as_deref().ok_or_else(|| {
3132 Error::Codec("run_id is required for selected-run termination".to_string())
3133 })?;
3134 self.client
3135 .terminate_workflow_run(&self.workflow_id, run_id, options)
3136 .await
3137 }
3138
3139 pub async fn query<T: Serialize>(&self, query_name: &str, input: T) -> Result<Value> {
3141 self.client
3142 .query_workflow(&self.workflow_id, query_name, input)
3143 .await
3144 }
3145
3146 pub async fn query_avro_value<T: Serialize>(
3147 &self,
3148 query_name: &str,
3149 input: T,
3150 ) -> Result<AvroValue> {
3151 self.client
3152 .query_workflow_avro_value(&self.workflow_id, query_name, input)
3153 .await
3154 }
3155
3156 pub async fn update<T: Serialize>(
3157 &self,
3158 update_name: &str,
3159 input: T,
3160 request_id: Option<&str>,
3161 ) -> Result<Value> {
3162 self.client
3163 .update_workflow(&self.workflow_id, update_name, input, request_id)
3164 .await
3165 }
3166
3167 pub async fn update_avro_value<T: Serialize>(
3168 &self,
3169 update_name: &str,
3170 input: T,
3171 request_id: Option<&str>,
3172 ) -> Result<AvroValue> {
3173 self.client
3174 .update_workflow_avro_value(&self.workflow_id, update_name, input, request_id)
3175 .await
3176 }
3177
3178 pub async fn query_selected_run<T: Serialize>(
3180 &self,
3181 query_name: &str,
3182 input: T,
3183 ) -> Result<Value> {
3184 let run_id = self
3185 .run_id
3186 .as_deref()
3187 .ok_or_else(|| Error::Codec("run_id is required for selected-run query".to_string()))?;
3188 self.client
3189 .query_workflow_run(&self.workflow_id, run_id, query_name, input)
3190 .await
3191 }
3192
3193 pub async fn result(&self, options: WorkflowResultOptions) -> Result<Value> {
3195 self.result_target(options, None).await
3196 }
3197
3198 pub async fn result_avro_value(&self, options: WorkflowResultOptions) -> Result<AvroValue> {
3200 self.result_avro_value_target(options, None).await
3201 }
3202
3203 pub async fn result_selected_run(&self, options: WorkflowResultOptions) -> Result<Value> {
3205 let run_id = self.run_id.as_deref().ok_or_else(|| {
3206 Error::Codec("run_id is required for selected-run result".to_string())
3207 })?;
3208 self.result_target(options, Some(run_id)).await
3209 }
3210
3211 pub async fn result_selected_run_avro_value(
3213 &self,
3214 options: WorkflowResultOptions,
3215 ) -> Result<AvroValue> {
3216 let run_id = self.run_id.as_deref().ok_or_else(|| {
3217 Error::Codec("run_id is required for selected-run result".to_string())
3218 })?;
3219 self.result_avro_value_target(options, Some(run_id)).await
3220 }
3221
3222 async fn result_avro_value_target(
3223 &self,
3224 options: WorkflowResultOptions,
3225 selected_run_id: Option<&str>,
3226 ) -> Result<AvroValue> {
3227 let started = Instant::now();
3228
3229 loop {
3230 let description = match selected_run_id {
3231 Some(run_id) => {
3232 self.client
3233 .describe_workflow_run(&self.workflow_id, run_id)
3234 .await?
3235 }
3236 None => self.describe().await?,
3237 };
3238 if description.is_completed() {
3239 return description.output_avro_value.ok_or_else(|| {
3240 Error::Codec(
3241 "missing_payload_envelope: typed workflow result requires output_envelope"
3242 .to_string(),
3243 )
3244 });
3245 }
3246 if description.is_terminal() {
3247 let outcome =
3248 workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
3249 return Err(match outcome.kind {
3250 WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
3251 WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
3252 WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
3253 WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
3254 });
3255 }
3256 if started.elapsed() >= options.timeout {
3257 return Err(Error::Timeout);
3258 }
3259 tokio::time::sleep(options.poll_interval).await;
3260 }
3261 }
3262
3263 async fn result_target(
3264 &self,
3265 options: WorkflowResultOptions,
3266 selected_run_id: Option<&str>,
3267 ) -> Result<Value> {
3268 let started = Instant::now();
3269
3270 loop {
3271 let description = match selected_run_id {
3272 Some(run_id) => {
3273 self.client
3274 .describe_workflow_run(&self.workflow_id, run_id)
3275 .await?
3276 }
3277 None => self.describe().await?,
3278 };
3279 if description.is_completed() {
3280 return Ok(description.output.unwrap_or(Value::Null));
3281 }
3282
3283 if description.is_terminal() {
3284 let outcome =
3285 workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
3286 return Err(match outcome.kind {
3287 WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
3288 WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
3289 WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
3290 WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
3291 });
3292 }
3293
3294 if started.elapsed() >= options.timeout {
3295 return Err(Error::WorkflowTimedOut(WorkflowTerminalOutcome {
3296 kind: WorkflowTerminalKind::TimedOut,
3297 workflow_id: description
3298 .workflow_id
3299 .clone()
3300 .unwrap_or_else(|| self.workflow_id.clone()),
3301 run_id: description
3302 .run_id
3303 .clone()
3304 .or_else(|| selected_run_id.map(str::to_string)),
3305 reason: "result_wait_timeout".to_string(),
3306 failure_category: Some("client_timeout".to_string()),
3307 failure_id: None,
3308 exception_type: None,
3309 exception_class: None,
3310 non_retryable: None,
3311 message: Some(format!(
3312 "workflow result was not terminal within {:?}",
3313 options.timeout
3314 )),
3315 exception: None,
3316 raw: description.raw_value(),
3317 }));
3318 }
3319
3320 tokio::time::sleep(options.poll_interval).await;
3321 }
3322 }
3323}
3324
3325#[derive(Clone, Copy, Debug)]
3326pub struct WorkflowResultOptions {
3327 pub poll_interval: Duration,
3328 pub timeout: Duration,
3329}
3330
3331impl Default for WorkflowResultOptions {
3332 fn default() -> Self {
3333 Self {
3334 poll_interval: Duration::from_millis(500),
3335 timeout: Duration::from_secs(30),
3336 }
3337 }
3338}
3339
3340#[derive(Clone, Debug, Deserialize)]
3341pub struct WorkflowDescription {
3342 pub workflow_id: Option<String>,
3343 pub run_id: Option<String>,
3344 pub workflow_type: Option<String>,
3345 pub status: Option<String>,
3346 #[serde(default)]
3347 pub closed_reason: Option<String>,
3348 #[serde(default)]
3349 pub error: Option<String>,
3350 #[serde(default)]
3351 pub failure: Option<Value>,
3352 #[serde(default)]
3353 pub exception: Option<Value>,
3354 #[serde(default)]
3355 pub failures: Vec<Value>,
3356 #[serde(default)]
3357 pub output: Option<Value>,
3358 #[serde(default)]
3359 pub output_envelope: Option<Value>,
3360 #[serde(skip)]
3361 pub output_avro_value: Option<AvroValue>,
3362 #[serde(flatten)]
3363 pub raw: HashMap<String, Value>,
3364}
3365
3366impl WorkflowDescription {
3367 pub fn is_completed(&self) -> bool {
3368 matches!(self.status.as_deref(), Some("completed" | "Completed"))
3369 }
3370
3371 pub fn is_terminal(&self) -> bool {
3372 matches!(
3373 self.status.as_deref(),
3374 Some(
3375 "completed"
3376 | "Completed"
3377 | "failed"
3378 | "Failed"
3379 | "cancelled"
3380 | "Cancelled"
3381 | "terminated"
3382 | "Terminated"
3383 | "timed_out"
3384 | "TimedOut",
3385 )
3386 )
3387 }
3388
3389 fn decode_payloads(&mut self) -> Result<()> {
3390 if let Some(envelope) = &self.output_envelope {
3391 let value = decode_wire_avro_value(envelope, DEFAULT_CODEC)?;
3392 self.output = Some(value.clone().into_json()?);
3393 self.output_avro_value = Some(value);
3394 }
3395
3396 Ok(())
3397 }
3398
3399 fn raw_value(&self) -> Value {
3400 let mut data = self.raw.clone();
3401 data.insert(
3402 "workflow_id".to_string(),
3403 self.workflow_id
3404 .clone()
3405 .map(Value::String)
3406 .unwrap_or(Value::Null),
3407 );
3408 data.insert(
3409 "run_id".to_string(),
3410 self.run_id
3411 .clone()
3412 .map(Value::String)
3413 .unwrap_or(Value::Null),
3414 );
3415 data.insert(
3416 "workflow_type".to_string(),
3417 self.workflow_type
3418 .clone()
3419 .map(Value::String)
3420 .unwrap_or(Value::Null),
3421 );
3422 data.insert(
3423 "status".to_string(),
3424 self.status
3425 .clone()
3426 .map(Value::String)
3427 .unwrap_or(Value::Null),
3428 );
3429 data.insert(
3430 "closed_reason".to_string(),
3431 self.closed_reason
3432 .clone()
3433 .map(Value::String)
3434 .unwrap_or(Value::Null),
3435 );
3436 if let Some(failure) = &self.failure {
3437 data.insert("failure".to_string(), failure.clone());
3438 }
3439 if let Some(exception) = &self.exception {
3440 data.insert("exception".to_string(), exception.clone());
3441 }
3442 Value::Object(data.into_iter().collect())
3443 }
3444}
3445
3446fn workflow_terminal_outcome(
3447 description: &WorkflowDescription,
3448 workflow_id: &str,
3449 run_id: Option<&str>,
3450) -> WorkflowTerminalOutcome {
3451 let terminal_kind = description
3452 .closed_reason
3453 .as_deref()
3454 .or(description.status.as_deref())
3455 .unwrap_or("failed")
3456 .to_ascii_lowercase();
3457 let kind = match terminal_kind.as_str() {
3458 "cancelled" | "canceled" => WorkflowTerminalKind::Cancelled,
3459 "terminated" => WorkflowTerminalKind::Terminated,
3460 "timed_out" | "timedout" => WorkflowTerminalKind::TimedOut,
3461 _ => WorkflowTerminalKind::Failed,
3462 };
3463 let default_reason = match kind {
3464 WorkflowTerminalKind::Failed => "workflow_failed",
3465 WorkflowTerminalKind::Cancelled => "cancelled",
3466 WorkflowTerminalKind::Terminated => "terminated",
3467 WorkflowTerminalKind::TimedOut => "timed_out",
3468 };
3469 let failure = description
3470 .failure
3471 .as_ref()
3472 .filter(|value| value.is_object());
3473 let nested_failure = failure
3474 .and_then(|value| value.get("failures"))
3475 .and_then(Value::as_array)
3476 .and_then(|failures| failures.last())
3477 .or_else(|| description.failures.last());
3478 let exception = description
3479 .exception
3480 .clone()
3481 .or_else(|| failure.and_then(|value| value.get("exception")).cloned())
3482 .or_else(|| {
3483 nested_failure
3484 .and_then(|value| value.get("exception_payload"))
3485 .cloned()
3486 });
3487 let string_field = |name: &str| {
3488 failure
3489 .and_then(|value| value.get(name))
3490 .and_then(Value::as_str)
3491 .or_else(|| {
3492 nested_failure
3493 .and_then(|value| value.get(name))
3494 .and_then(Value::as_str)
3495 })
3496 .map(str::to_string)
3497 };
3498 let exception_field = |name: &str| {
3499 exception
3500 .as_ref()
3501 .and_then(|value| value.get(name))
3502 .and_then(Value::as_str)
3503 .map(str::to_string)
3504 };
3505 let message = description
3506 .error
3507 .clone()
3508 .or_else(|| string_field("message"))
3509 .or_else(|| exception_field("message"));
3510 let reason = description
3511 .raw
3512 .get("reason")
3513 .and_then(Value::as_str)
3514 .map(str::to_string)
3515 .or_else(|| {
3516 failure
3517 .and_then(|value| value.get("reason"))
3518 .and_then(Value::as_str)
3519 .map(str::to_string)
3520 })
3521 .or_else(|| description.closed_reason.clone())
3522 .unwrap_or_else(|| default_reason.to_string());
3523 let failure_id = string_field("failure_id").or_else(|| {
3524 nested_failure
3525 .and_then(|value| value.get("id"))
3526 .and_then(Value::as_str)
3527 .map(str::to_string)
3528 });
3529
3530 WorkflowTerminalOutcome {
3531 kind,
3532 workflow_id: description
3533 .workflow_id
3534 .clone()
3535 .unwrap_or_else(|| workflow_id.to_string()),
3536 run_id: description
3537 .run_id
3538 .clone()
3539 .or_else(|| run_id.map(str::to_string)),
3540 reason,
3541 failure_category: string_field("failure_category")
3542 .or_else(|| Some(default_reason.to_string())),
3543 failure_id,
3544 exception_type: string_field("exception_type").or_else(|| exception_field("type")),
3545 exception_class: string_field("exception_class").or_else(|| exception_field("class")),
3546 non_retryable: failure
3547 .and_then(|value| value.get("non_retryable"))
3548 .and_then(Value::as_bool)
3549 .or_else(|| {
3550 nested_failure
3551 .and_then(|value| value.get("non_retryable"))
3552 .and_then(Value::as_bool)
3553 }),
3554 message,
3555 exception,
3556 raw: description.raw_value(),
3557 }
3558}
3559
3560#[derive(Clone, Debug, Deserialize)]
3561pub struct RegisterWorkerResponse {
3562 pub worker_id: String,
3563 pub registered: bool,
3564 #[serde(default)]
3565 pub heartbeat_interval_seconds: Option<u64>,
3566 #[serde(default)]
3567 pub protocol_version: Option<String>,
3568 #[serde(default)]
3569 pub server_capabilities: Option<Value>,
3570}
3571
3572#[derive(Clone, Debug, Deserialize)]
3573pub struct PollWorkflowTaskResponse {
3574 #[serde(default)]
3575 pub task: Option<WorkflowTask>,
3576 #[serde(default)]
3577 pub poll_status: Option<String>,
3578 #[serde(default)]
3579 pub reason: Option<String>,
3580 #[serde(default)]
3581 pub protocol_version: Option<String>,
3582 #[serde(default)]
3583 pub server_capabilities: Option<Value>,
3584}
3585
3586impl PollWorkflowTaskResponse {
3587 pub fn outcome(&self) -> WorkerPollOutcome {
3589 worker_poll_outcome(
3590 self.task.is_some(),
3591 self.poll_status.as_deref(),
3592 self.reason.as_deref(),
3593 )
3594 }
3595}
3596
3597#[derive(Clone, Debug, Deserialize)]
3598pub struct PollActivityTaskResponse {
3599 #[serde(default)]
3600 pub task: Option<ActivityTask>,
3601 #[serde(default)]
3602 pub poll_status: Option<String>,
3603 #[serde(default)]
3604 pub reason: Option<String>,
3605}
3606
3607impl PollActivityTaskResponse {
3608 pub fn outcome(&self) -> WorkerPollOutcome {
3610 worker_poll_outcome(
3611 self.task.is_some(),
3612 self.poll_status.as_deref(),
3613 self.reason.as_deref(),
3614 )
3615 }
3616}
3617
3618#[derive(Clone, Debug, Deserialize)]
3619pub struct PollQueryTaskResponse {
3620 #[serde(default)]
3621 pub task: Option<QueryTask>,
3622 #[serde(default)]
3623 pub poll_status: Option<String>,
3624 #[serde(default)]
3625 pub reason: Option<String>,
3626}
3627
3628impl PollQueryTaskResponse {
3629 pub fn outcome(&self) -> WorkerPollOutcome {
3631 worker_poll_outcome(
3632 self.task.is_some(),
3633 self.poll_status.as_deref(),
3634 self.reason.as_deref(),
3635 )
3636 }
3637}
3638
3639#[derive(Clone, Debug, PartialEq, Eq)]
3641pub enum WorkerPollOutcome {
3642 Task,
3644 Idle {
3646 poll_status: Option<String>,
3647 reason: Option<String>,
3648 },
3649 Stop {
3651 poll_status: Option<String>,
3652 reason: Option<String>,
3653 },
3654}
3655
3656impl WorkerPollOutcome {
3657 pub fn should_stop(&self) -> bool {
3658 matches!(self, Self::Stop { .. })
3659 }
3660}
3661
3662fn worker_poll_outcome(
3663 has_task: bool,
3664 poll_status: Option<&str>,
3665 reason: Option<&str>,
3666) -> WorkerPollOutcome {
3667 if worker_poll_is_stop(poll_status, reason) {
3668 return WorkerPollOutcome::Stop {
3669 poll_status: poll_status.map(str::to_string),
3670 reason: reason.map(str::to_string),
3671 };
3672 }
3673
3674 if has_task {
3675 WorkerPollOutcome::Task
3676 } else {
3677 WorkerPollOutcome::Idle {
3678 poll_status: poll_status.map(str::to_string),
3679 reason: reason.map(str::to_string),
3680 }
3681 }
3682}
3683
3684#[derive(Clone, Debug, Deserialize)]
3686pub struct QueryTask {
3687 pub query_task_id: String,
3688 #[serde(default = "default_workflow_task_attempt")]
3689 pub query_task_attempt: u64,
3690 #[serde(default)]
3691 pub lease_owner: Option<String>,
3692 #[serde(default)]
3693 pub workflow_id: Option<String>,
3694 #[serde(default)]
3695 pub run_id: Option<String>,
3696 pub workflow_type: String,
3697 pub query_name: String,
3698 #[serde(default = "default_payload_codec")]
3699 pub payload_codec: String,
3700 #[serde(default)]
3701 pub workflow_arguments: Option<Value>,
3702 #[serde(default)]
3703 pub query_arguments: Option<Value>,
3704 #[serde(default)]
3705 pub history_events: Vec<HistoryEvent>,
3706 #[serde(default)]
3707 pub history_export: Option<Value>,
3708 #[serde(default)]
3709 pub run_status: Option<String>,
3710}
3711
3712#[derive(Clone, Debug, Deserialize)]
3713pub struct WorkflowTask {
3714 pub task_id: String,
3715 #[serde(default)]
3716 pub workflow_id: Option<String>,
3717 #[serde(default)]
3718 pub run_id: Option<String>,
3719 pub workflow_type: String,
3720 #[serde(default = "default_payload_codec")]
3721 pub payload_codec: String,
3722 #[serde(default)]
3723 pub arguments: Option<Value>,
3724 #[serde(default)]
3725 pub history_events: Vec<HistoryEvent>,
3726 #[serde(default)]
3727 pub total_history_events: Option<u64>,
3728 #[serde(default)]
3729 pub history_size_bytes: Option<u64>,
3730 #[serde(default)]
3731 pub continue_as_new_recommended: Option<bool>,
3732 #[serde(default)]
3733 pub history_budget_pressure: Option<String>,
3734 #[serde(default)]
3735 pub next_history_page_token: Option<String>,
3736 #[serde(default = "default_workflow_task_attempt")]
3737 pub workflow_task_attempt: u64,
3738 #[serde(default)]
3739 pub workflow_signal_id: Option<String>,
3740 #[serde(default)]
3741 pub signal_name: Option<String>,
3742 #[serde(default)]
3743 pub signal_arguments: Option<Value>,
3744 #[serde(default)]
3745 pub workflow_update_id: Option<String>,
3746 #[serde(default)]
3747 pub update_name: Option<String>,
3748 #[serde(default)]
3749 pub lease_owner: Option<String>,
3750}
3751
3752impl WorkflowTask {
3753 fn append_history_page(&mut self, page: WorkflowTaskHistoryPage) {
3754 self.history_events.extend(page.history_events);
3755
3756 if page.total_history_events.is_some() {
3757 self.total_history_events = page.total_history_events;
3758 }
3759
3760 self.next_history_page_token = page
3761 .next_history_page_token
3762 .filter(|token| !token.is_empty());
3763 }
3764}
3765
3766#[derive(Clone, Debug, Deserialize)]
3767struct WorkflowTaskHistoryPage {
3768 #[serde(default)]
3769 history_events: Vec<HistoryEvent>,
3770 #[serde(default)]
3771 total_history_events: Option<u64>,
3772 #[serde(default)]
3773 next_history_page_token: Option<String>,
3774}
3775
3776#[derive(Clone, Debug, Deserialize)]
3777pub struct ActivityTask {
3778 pub task_id: String,
3779 #[serde(default)]
3780 pub activity_attempt_id: Option<String>,
3781 #[serde(default)]
3782 pub attempt_id: Option<String>,
3783 pub activity_type: String,
3784 #[serde(default = "default_payload_codec")]
3785 pub payload_codec: String,
3786 #[serde(default)]
3787 pub arguments: Option<Value>,
3788 #[serde(default = "default_attempt_number")]
3789 pub attempt_number: u64,
3790 #[serde(default)]
3791 pub lease_owner: Option<String>,
3792}
3793
3794#[derive(Clone, Debug, Deserialize)]
3795pub struct HistoryEvent {
3796 #[serde(alias = "type")]
3797 pub event_type: String,
3798 #[serde(default)]
3799 pub payload: Value,
3800 #[serde(flatten)]
3801 pub raw: HashMap<String, Value>,
3802}
3803
3804#[derive(Clone, Debug, PartialEq)]
3806pub struct QuerySignal {
3807 pub id: Option<String>,
3808 pub name: String,
3809 pub arguments: Vec<Value>,
3810 avro_arguments: Vec<AvroValue>,
3811 pub workflow_sequence: Option<u64>,
3812}
3813
3814impl QuerySignal {
3815 pub fn arguments_avro_value(&self) -> &[AvroValue] {
3817 &self.avro_arguments
3818 }
3819}
3820
3821#[derive(Clone, Debug)]
3827pub struct QueryContext {
3828 pub workflow_id: Option<String>,
3829 pub run_id: Option<String>,
3830 pub workflow_type: String,
3831 pub run_status: Option<String>,
3832 workflow_input: Value,
3833 workflow_input_avro_value: AvroValue,
3834 history_events: Arc<Vec<HistoryEvent>>,
3835 signal_events: Arc<Vec<QuerySignal>>,
3836}
3837
3838impl QueryContext {
3839 pub fn workflow_input(&self) -> &Value {
3841 &self.workflow_input
3842 }
3843
3844 pub fn workflow_input_avro_value(&self) -> &AvroValue {
3846 &self.workflow_input_avro_value
3847 }
3848
3849 pub fn history_events(&self) -> &[HistoryEvent] {
3851 self.history_events.as_slice()
3852 }
3853
3854 pub fn signal_events(&self) -> &[QuerySignal] {
3856 self.signal_events.as_slice()
3857 }
3858
3859 pub fn signals(&self, signal_name: &str) -> Vec<Vec<Value>> {
3861 self.signal_events
3862 .iter()
3863 .filter(|signal| signal.name == signal_name)
3864 .map(|signal| signal.arguments.clone())
3865 .collect()
3866 }
3867
3868 pub fn signals_avro_value(&self, signal_name: &str) -> Vec<Vec<AvroValue>> {
3870 self.signal_events
3871 .iter()
3872 .filter(|signal| signal.name == signal_name)
3873 .map(|signal| signal.avro_arguments.clone())
3874 .collect()
3875 }
3876}
3877
3878#[derive(Clone, Debug, Deserialize)]
3879pub struct ActivityHeartbeatResponse {
3880 #[serde(default)]
3881 pub cancel_requested: bool,
3882 #[serde(default)]
3883 pub heartbeat_recorded: bool,
3884 #[serde(default)]
3885 pub can_continue: Option<bool>,
3886 #[serde(default)]
3887 pub reason: Option<String>,
3888 #[serde(default)]
3889 pub run_closed_reason: Option<String>,
3890 #[serde(default)]
3891 pub run_closed_at: Option<String>,
3892 #[serde(default)]
3893 pub lease_expires_at: Option<String>,
3894 #[serde(default)]
3895 pub last_heartbeat_at: Option<String>,
3896}
3897
3898impl ActivityHeartbeatResponse {
3899 pub fn should_stop(&self) -> bool {
3901 self.cancel_requested || self.can_continue == Some(false)
3902 }
3903}
3904
3905fn default_payload_codec() -> String {
3906 DEFAULT_CODEC.to_string()
3907}
3908
3909fn default_workflow_task_attempt() -> u64 {
3910 1
3911}
3912
3913fn default_attempt_number() -> u64 {
3914 1
3915}
3916
3917type WorkflowFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
3918type WorkflowHandler = Arc<dyn Fn(WorkflowContext, AvroValue) -> WorkflowFuture + Send + Sync>;
3919type ErasedWorkflowState = Arc<dyn Any + Send + Sync>;
3920type WorkflowStateSnapshot = Arc<dyn Fn() -> Result<ErasedWorkflowState> + Send + Sync>;
3921type ReplayedWorkflowHandler =
3922 Arc<dyn Fn(WorkflowContext, AvroValue) -> ReplayedWorkflowInvocation + Send + Sync>;
3923type ActivityFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
3924type ActivityHandler = Arc<dyn Fn(ActivityContext, AvroValue) -> ActivityFuture + Send + Sync>;
3925type QueryFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
3926type QueryHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
3927type UpdateHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
3928type ReplayedQueryHandler = Arc<
3929 dyn Fn(QueryContext, ErasedWorkflowState, AvroValue) -> std::result::Result<QueryFuture, String>
3930 + Send
3931 + Sync,
3932>;
3933type WorkerHeartbeatObserver = Arc<dyn Fn(&WorkerHeartbeatObservation) + Send + Sync>;
3934
3935struct ReplayedWorkflowInvocation {
3936 future: WorkflowFuture,
3937 snapshot: WorkflowStateSnapshot,
3938}
3939
3940#[derive(Clone)]
3941struct RegisteredWorkflow {
3942 execute: WorkflowHandler,
3943 replay: Option<ReplayedWorkflowHandler>,
3944 state_type: Option<TypeId>,
3945}
3946
3947#[derive(Clone)]
3948enum RegisteredQuery {
3949 Snapshot(QueryHandler),
3950 Replayed {
3951 state_type: TypeId,
3952 handler: ReplayedQueryHandler,
3953 },
3954}
3955
3956#[derive(Clone, Debug)]
3957pub struct WorkerHeartbeatObservation {
3958 pub worker_id: String,
3959 pub task_queue: String,
3960 pub acknowledged_at_unix_millis: u64,
3961 pub acknowledgement: Value,
3962}
3963
3964#[derive(Clone, Copy, Debug)]
3971pub struct WorkerRetryPolicy {
3972 pub max_retries: usize,
3974 pub initial_backoff: Duration,
3976 pub max_backoff: Duration,
3978}
3979
3980impl Default for WorkerRetryPolicy {
3981 fn default() -> Self {
3982 Self {
3983 max_retries: 5,
3984 initial_backoff: Duration::from_millis(100),
3985 max_backoff: Duration::from_secs(5),
3986 }
3987 }
3988}
3989
3990#[derive(Clone, Copy, Debug, PartialEq, Eq)]
3991enum ManagedPollOutcome {
3992 Idle,
3993 Handled,
3994 Stop,
3995}
3996
3997#[derive(Clone)]
3998pub struct Worker {
3999 client: Client,
4000 worker_id: String,
4001 task_queue: String,
4002 workflows: HashMap<String, RegisteredWorkflow>,
4003 activities: HashMap<String, ActivityHandler>,
4004 queries: HashMap<String, HashMap<String, RegisteredQuery>>,
4005 updates: HashMap<String, HashMap<String, UpdateHandler>>,
4006 max_concurrent_workflow_tasks: usize,
4007 max_concurrent_activity_tasks: usize,
4008 poll_timeout: Duration,
4009 heartbeat_interval: Duration,
4010 retry_policy: WorkerRetryPolicy,
4011 heartbeat_observer: Option<WorkerHeartbeatObserver>,
4012}
4013
4014impl Worker {
4015 pub fn new(client: Client, task_queue: impl Into<String>) -> Self {
4016 Self {
4017 client,
4018 worker_id: default_worker_id(),
4019 task_queue: task_queue.into(),
4020 workflows: HashMap::new(),
4021 activities: HashMap::new(),
4022 queries: HashMap::new(),
4023 updates: HashMap::new(),
4024 max_concurrent_workflow_tasks: 10,
4025 max_concurrent_activity_tasks: 10,
4026 poll_timeout: Duration::from_secs(30),
4027 heartbeat_interval: Duration::from_secs(60),
4028 retry_policy: WorkerRetryPolicy::default(),
4029 heartbeat_observer: None,
4030 }
4031 }
4032
4033 pub fn worker_id(mut self, worker_id: impl Into<String>) -> Self {
4034 self.worker_id = worker_id.into();
4035 self
4036 }
4037
4038 pub fn poll_timeout(mut self, timeout: Duration) -> Self {
4039 self.poll_timeout = timeout;
4040 self
4041 }
4042
4043 pub fn heartbeat_interval(mut self, interval: Duration) -> Self {
4044 self.heartbeat_interval = interval;
4045 self
4046 }
4047
4048 pub fn retry_policy(mut self, policy: WorkerRetryPolicy) -> Self {
4050 self.retry_policy = policy;
4051 self
4052 }
4053
4054 pub fn on_worker_heartbeat<F>(mut self, observer: F) -> Self
4055 where
4056 F: Fn(&WorkerHeartbeatObservation) + Send + Sync + 'static,
4057 {
4058 self.heartbeat_observer = Some(Arc::new(observer));
4059 self
4060 }
4061
4062 pub fn max_concurrent_workflow_tasks(mut self, count: usize) -> Self {
4063 self.max_concurrent_workflow_tasks = count.max(1);
4064 self
4065 }
4066
4067 pub fn max_concurrent_activity_tasks(mut self, count: usize) -> Self {
4068 self.max_concurrent_activity_tasks = count.max(1);
4069 self
4070 }
4071
4072 pub fn register_workflow<F, Fut>(&mut self, workflow_type: impl Into<String>, handler: F)
4079 where
4080 F: Fn(WorkflowContext, Value) -> Fut + Send + Sync + 'static,
4081 Fut: Future<Output = Result<Value>> + Send + 'static,
4082 {
4083 let handler = Arc::new(handler);
4084 self.workflows.insert(
4085 workflow_type.into(),
4086 RegisteredWorkflow {
4087 execute: Arc::new(move |ctx, input| {
4088 let handler = Arc::clone(&handler);
4089 Box::pin(async move {
4090 let result = handler(ctx, input.into_json()?).await?;
4091 AvroValue::from_serialize(&result)
4092 })
4093 }),
4094 replay: None,
4095 state_type: None,
4096 },
4097 );
4098 }
4099
4100 pub fn register_workflow_avro_value<F, Fut>(
4102 &mut self,
4103 workflow_type: impl Into<String>,
4104 handler: F,
4105 ) where
4106 F: Fn(WorkflowContext, AvroValue) -> Fut + Send + Sync + 'static,
4107 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4108 {
4109 self.workflows.insert(
4110 workflow_type.into(),
4111 RegisteredWorkflow {
4112 execute: Arc::new(move |ctx, input| Box::pin(handler(ctx, input))),
4113 replay: None,
4114 state_type: None,
4115 },
4116 );
4117 }
4118
4119 pub fn register_replayed_workflow<S, Factory, F, Fut>(
4127 &mut self,
4128 workflow_type: impl Into<String>,
4129 state_factory: Factory,
4130 handler: F,
4131 ) where
4132 S: Clone + Send + Sync + 'static,
4133 Factory: Fn() -> S + Send + Sync + 'static,
4134 F: Fn(WorkflowContext, Value, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
4135 Fut: Future<Output = Result<Value>> + Send + 'static,
4136 {
4137 let state_factory = Arc::new(state_factory);
4138 let handler = Arc::new(handler);
4139
4140 let execute_factory = Arc::clone(&state_factory);
4141 let execute_handler = Arc::clone(&handler);
4142 let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4143 let state = WorkflowInstance::new(execute_factory());
4144 let handler = Arc::clone(&execute_handler);
4145 Box::pin(async move {
4146 let result = handler(ctx, input.into_json()?, state).await?;
4147 AvroValue::from_serialize(&result)
4148 }) as WorkflowFuture
4149 });
4150
4151 let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4152 let state = WorkflowInstance::new(state_factory());
4153 let snapshot_state = state.clone();
4154 let snapshot: WorkflowStateSnapshot =
4155 Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
4156 let replay_handler = Arc::clone(&handler);
4157 let future = async move {
4158 let result = replay_handler(ctx, input.into_json()?, state).await?;
4159 AvroValue::from_serialize(&result)
4160 };
4161 ReplayedWorkflowInvocation {
4162 future: Box::pin(future),
4163 snapshot,
4164 }
4165 });
4166
4167 self.workflows.insert(
4168 workflow_type.into(),
4169 RegisteredWorkflow {
4170 execute,
4171 replay: Some(replay),
4172 state_type: Some(TypeId::of::<S>()),
4173 },
4174 );
4175 }
4176
4177 pub fn register_replayed_workflow_avro_value<S, Factory, F, Fut>(
4179 &mut self,
4180 workflow_type: impl Into<String>,
4181 state_factory: Factory,
4182 handler: F,
4183 ) where
4184 S: Clone + Send + Sync + 'static,
4185 Factory: Fn() -> S + Send + Sync + 'static,
4186 F: Fn(WorkflowContext, AvroValue, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
4187 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4188 {
4189 let state_factory = Arc::new(state_factory);
4190 let handler = Arc::new(handler);
4191
4192 let execute_factory = Arc::clone(&state_factory);
4193 let execute_handler = Arc::clone(&handler);
4194 let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4195 let state = WorkflowInstance::new(execute_factory());
4196 Box::pin(execute_handler(ctx, input, state)) as WorkflowFuture
4197 });
4198
4199 let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4200 let state = WorkflowInstance::new(state_factory());
4201 let snapshot_state = state.clone();
4202 let snapshot: WorkflowStateSnapshot =
4203 Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
4204 ReplayedWorkflowInvocation {
4205 future: Box::pin(handler(ctx, input, state)),
4206 snapshot,
4207 }
4208 });
4209
4210 self.workflows.insert(
4211 workflow_type.into(),
4212 RegisteredWorkflow {
4213 execute,
4214 replay: Some(replay),
4215 state_type: Some(TypeId::of::<S>()),
4216 },
4217 );
4218 }
4219
4220 pub fn register_activity<F, Fut>(&mut self, activity_type: impl Into<String>, handler: F)
4221 where
4222 F: Fn(ActivityContext, Value) -> Fut + Send + Sync + 'static,
4223 Fut: Future<Output = Result<Value>> + Send + 'static,
4224 {
4225 let handler = Arc::new(handler);
4226 self.activities.insert(
4227 activity_type.into(),
4228 Arc::new(move |ctx, args| {
4229 let handler = Arc::clone(&handler);
4230 Box::pin(async move {
4231 let result = handler(ctx, args.into_json()?).await?;
4232 AvroValue::from_serialize(&result)
4233 })
4234 }),
4235 );
4236 }
4237
4238 pub fn register_activity_avro_value<F, Fut>(
4240 &mut self,
4241 activity_type: impl Into<String>,
4242 handler: F,
4243 ) where
4244 F: Fn(ActivityContext, AvroValue) -> Fut + Send + Sync + 'static,
4245 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4246 {
4247 self.activities.insert(
4248 activity_type.into(),
4249 Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
4250 );
4251 }
4252
4253 pub fn register_query<F, Fut>(
4259 &mut self,
4260 workflow_type: impl Into<String>,
4261 query_name: impl Into<String>,
4262 handler: F,
4263 ) where
4264 F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
4265 Fut: Future<Output = Result<Value>> + Send + 'static,
4266 {
4267 let handler = Arc::new(handler);
4268 self.queries
4269 .entry(workflow_type.into())
4270 .or_default()
4271 .insert(
4272 query_name.into(),
4273 RegisteredQuery::Snapshot(Arc::new(move |ctx, args| {
4274 let handler = Arc::clone(&handler);
4275 Box::pin(async move {
4276 let result = handler(ctx, args.into_json()?).await?;
4277 AvroValue::from_serialize(&result)
4278 })
4279 })),
4280 );
4281 }
4282
4283 pub fn register_query_avro_value<F, Fut>(
4285 &mut self,
4286 workflow_type: impl Into<String>,
4287 query_name: impl Into<String>,
4288 handler: F,
4289 ) where
4290 F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
4291 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4292 {
4293 self.queries
4294 .entry(workflow_type.into())
4295 .or_default()
4296 .insert(
4297 query_name.into(),
4298 RegisteredQuery::Snapshot(Arc::new(move |ctx, args| Box::pin(handler(ctx, args)))),
4299 );
4300 }
4301
4302 pub fn register_replayed_query<S, F, Fut>(
4309 &mut self,
4310 workflow_type: impl Into<String>,
4311 query_name: impl Into<String>,
4312 handler: F,
4313 ) where
4314 S: Clone + Send + Sync + 'static,
4315 F: Fn(QueryContext, Arc<S>, Value) -> Fut + Send + Sync + 'static,
4316 Fut: Future<Output = Result<Value>> + Send + 'static,
4317 {
4318 let handler = Arc::new(handler);
4319 let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
4320 let state = state.downcast::<S>().map_err(|_| {
4321 "registered query state type does not match the replayed workflow state".to_string()
4322 })?;
4323 let handler = Arc::clone(&handler);
4324 Ok(Box::pin(async move {
4325 let result = handler(ctx, state, args.into_json()?).await?;
4326 AvroValue::from_serialize(&result)
4327 }))
4328 });
4329
4330 self.queries
4331 .entry(workflow_type.into())
4332 .or_default()
4333 .insert(
4334 query_name.into(),
4335 RegisteredQuery::Replayed {
4336 state_type: TypeId::of::<S>(),
4337 handler: erased_handler,
4338 },
4339 );
4340 }
4341
4342 pub fn register_replayed_query_avro_value<S, F, Fut>(
4344 &mut self,
4345 workflow_type: impl Into<String>,
4346 query_name: impl Into<String>,
4347 handler: F,
4348 ) where
4349 S: Clone + Send + Sync + 'static,
4350 F: Fn(QueryContext, Arc<S>, AvroValue) -> Fut + Send + Sync + 'static,
4351 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4352 {
4353 let handler = Arc::new(handler);
4354 let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
4355 let state = state.downcast::<S>().map_err(|_| {
4356 "registered query state type does not match the replayed workflow state".to_string()
4357 })?;
4358 Ok(Box::pin(handler(ctx, state, args)))
4359 });
4360
4361 self.queries
4362 .entry(workflow_type.into())
4363 .or_default()
4364 .insert(
4365 query_name.into(),
4366 RegisteredQuery::Replayed {
4367 state_type: TypeId::of::<S>(),
4368 handler: erased_handler,
4369 },
4370 );
4371 }
4372
4373 pub fn register_update<F, Fut>(
4375 &mut self,
4376 workflow_type: impl Into<String>,
4377 update_name: impl Into<String>,
4378 handler: F,
4379 ) where
4380 F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
4381 Fut: Future<Output = Result<Value>> + Send + 'static,
4382 {
4383 let handler = Arc::new(handler);
4384 self.updates
4385 .entry(workflow_type.into())
4386 .or_default()
4387 .insert(
4388 update_name.into(),
4389 Arc::new(move |ctx, args| {
4390 let handler = Arc::clone(&handler);
4391 Box::pin(async move {
4392 let result = handler(ctx, args.into_json()?).await?;
4393 AvroValue::from_serialize(&result)
4394 })
4395 }),
4396 );
4397 }
4398
4399 pub fn register_update_avro_value<F, Fut>(
4401 &mut self,
4402 workflow_type: impl Into<String>,
4403 update_name: impl Into<String>,
4404 handler: F,
4405 ) where
4406 F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
4407 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4408 {
4409 self.updates
4410 .entry(workflow_type.into())
4411 .or_default()
4412 .insert(
4413 update_name.into(),
4414 Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
4415 );
4416 }
4417
4418 pub async fn register(&self) -> Result<RegisterWorkerResponse> {
4419 let mut command_contracts = serde_json::Map::new();
4420 for workflow_type in self.workflows.keys() {
4421 let mut queries = self
4422 .queries
4423 .get(workflow_type)
4424 .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
4425 .unwrap_or_default();
4426 queries.sort();
4427 let mut updates = self
4428 .updates
4429 .get(workflow_type)
4430 .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
4431 .unwrap_or_default();
4432 updates.sort();
4433 if !queries.is_empty() || !updates.is_empty() {
4434 command_contracts.insert(
4435 workflow_type.clone(),
4436 json!({
4437 "queries": queries,
4438 "updates": updates,
4439 }),
4440 );
4441 }
4442 }
4443
4444 self.client
4445 .register_worker_with_command_contracts(
4446 &self.worker_id,
4447 &self.task_queue,
4448 self.workflows.keys().cloned().collect(),
4449 self.activities.keys().cloned().collect(),
4450 self.max_concurrent_workflow_tasks,
4451 self.max_concurrent_activity_tasks,
4452 [
4453 (!self.queries.is_empty()).then(|| QUERY_TASKS_CAPABILITY.to_string()),
4454 (!self.updates.is_empty()).then(|| WORKFLOW_UPDATES_CAPABILITY.to_string()),
4455 ]
4456 .into_iter()
4457 .flatten()
4458 .collect(),
4459 Value::Object(command_contracts),
4460 )
4461 .await
4462 }
4463
4464 pub async fn run(&self) -> Result<()> {
4470 self.run_until(std::future::pending::<()>()).await
4471 }
4472
4473 pub async fn run_until<F>(&self, shutdown: F) -> Result<()>
4477 where
4478 F: Future<Output = ()>,
4479 {
4480 let registration = self.register().await?;
4481 let heartbeat_interval = Duration::from_secs(
4482 registration
4483 .heartbeat_interval_seconds
4484 .unwrap_or(self.heartbeat_interval.as_secs().max(1)),
4485 );
4486 let heartbeat = tokio::time::sleep(Duration::ZERO);
4492 tokio::pin!(heartbeat);
4493 tokio::pin!(shutdown);
4494 let stop = Arc::new(AtomicBool::new(false));
4495 let mut workflow_poller = (!self.workflows.is_empty()).then(|| {
4499 let worker = self.clone();
4500 let stop = Arc::clone(&stop);
4501 tokio::spawn(async move { worker.poll_workflows_until_stopped(stop).await })
4502 });
4503 let mut activity_poller = (!self.activities.is_empty()).then(|| {
4504 let worker = self.clone();
4505 let stop = Arc::clone(&stop);
4506 tokio::spawn(async move { worker.poll_activities_until_stopped(stop).await })
4507 });
4508 let mut query_poller = (!self.queries.is_empty()).then(|| {
4509 let worker = self.clone();
4510 let stop = Arc::clone(&stop);
4511 tokio::spawn(async move { worker.poll_queries_until_stopped(stop).await })
4512 });
4513
4514 loop {
4515 tokio::select! {
4516 _ = &mut shutdown => {
4517 stop.store(true, Ordering::SeqCst);
4518 break;
4519 }
4520 _ = &mut heartbeat => {
4521 let result = self.retry_worker_operation(|| {
4522 self.client.heartbeat_worker(
4523 &self.worker_id,
4524 self.max_concurrent_workflow_tasks,
4525 self.max_concurrent_activity_tasks,
4526 )
4527 }).await;
4528 heartbeat
4529 .as_mut()
4530 .reset(tokio::time::Instant::now() + heartbeat_interval);
4531 match result {
4532 Ok(acknowledgement) => {
4533 if let Some(observer) = &self.heartbeat_observer {
4534 observer(&WorkerHeartbeatObservation {
4535 worker_id: self.worker_id.clone(),
4536 task_queue: self.task_queue.clone(),
4537 acknowledged_at_unix_millis: SystemTime::now()
4538 .duration_since(UNIX_EPOCH)
4539 .unwrap_or_default()
4540 .as_millis()
4541 .min(u64::MAX as u128)
4542 as u64,
4543 acknowledgement,
4544 });
4545 }
4546 }
4547 Err(error) => {
4548 stop.store(true, Ordering::SeqCst);
4549 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await?;
4550 return Err(error);
4551 }
4552 }
4553 }
4554 result = OptionFuture::from(workflow_poller.as_mut()), if workflow_poller.is_some() => {
4555 workflow_poller = None;
4556 let stopped_by_server = stop.load(Ordering::SeqCst);
4557 stop.store(true, Ordering::SeqCst);
4558 let poller_result = optional_poller_result("workflow", result);
4559 let join_result =
4560 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
4561 poller_result?;
4562 join_result?;
4563 if stopped_by_server {
4564 return Ok(());
4565 }
4566 return Err(Error::WorkerLoop(
4567 "workflow poller stopped unexpectedly".to_string(),
4568 ));
4569 }
4570 result = OptionFuture::from(activity_poller.as_mut()), if activity_poller.is_some() => {
4571 activity_poller = None;
4572 let stopped_by_server = stop.load(Ordering::SeqCst);
4573 stop.store(true, Ordering::SeqCst);
4574 let poller_result = optional_poller_result("activity", result);
4575 let join_result =
4576 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
4577 poller_result?;
4578 join_result?;
4579 if stopped_by_server {
4580 return Ok(());
4581 }
4582 return Err(Error::WorkerLoop(
4583 "activity poller stopped unexpectedly".to_string(),
4584 ));
4585 }
4586 result = OptionFuture::from(query_poller.as_mut()), if query_poller.is_some() => {
4587 query_poller = None;
4588 let stopped_by_server = stop.load(Ordering::SeqCst);
4589 stop.store(true, Ordering::SeqCst);
4590 let poller_result = optional_poller_result("query", result);
4591 let join_result =
4592 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
4593 poller_result?;
4594 join_result?;
4595 if stopped_by_server {
4596 return Ok(());
4597 }
4598 return Err(Error::WorkerLoop(
4599 "query poller stopped unexpectedly".to_string(),
4600 ));
4601 }
4602 }
4603 }
4604
4605 join_pollers(
4606 workflow_poller.take(),
4607 activity_poller.take(),
4608 query_poller.take(),
4609 )
4610 .await
4611 }
4612
4613 pub async fn run_once(&self) -> Result<usize> {
4627 let mut handled = 0;
4628 match self.poll_workflow_once().await? {
4629 ManagedPollOutcome::Handled => handled += 1,
4630 ManagedPollOutcome::Stop => return Ok(handled),
4631 ManagedPollOutcome::Idle => {}
4632 }
4633 match self.poll_activity_once().await? {
4634 ManagedPollOutcome::Handled => handled += 1,
4635 ManagedPollOutcome::Stop => return Ok(handled),
4636 ManagedPollOutcome::Idle => {}
4637 }
4638 if !self.queries.is_empty() {
4639 match self.poll_query_once().await? {
4640 ManagedPollOutcome::Handled => handled += 1,
4641 ManagedPollOutcome::Stop => return Ok(handled),
4642 ManagedPollOutcome::Idle => {}
4643 }
4644 }
4645 Ok(handled)
4646 }
4647
4648 async fn poll_workflow_once(&self) -> Result<ManagedPollOutcome> {
4649 let poll_request_id = unique_request_id("rust-workflow-poll");
4650 let response = self
4651 .retry_worker_operation(|| {
4652 self.client.poll_workflow_task_response_with_request_id(
4653 &self.worker_id,
4654 &self.task_queue,
4655 self.poll_timeout,
4656 &poll_request_id,
4657 0,
4658 )
4659 })
4660 .await?;
4661 if response.outcome().should_stop() {
4662 return Ok(ManagedPollOutcome::Stop);
4663 }
4664 let Some(task) = response.task else {
4665 return Ok(ManagedPollOutcome::Idle);
4666 };
4667
4668 let task_id = task.task_id.clone();
4669 let attempt = task.workflow_task_attempt;
4670 let run_id = task.run_id.clone();
4671 let lease_owner = task
4672 .lease_owner
4673 .clone()
4674 .unwrap_or_else(|| self.worker_id.clone());
4675
4676 match self.execute_workflow_task(task) {
4677 Ok(commands) if commands.is_empty() => {
4678 self.client
4684 .fail_workflow_task_with_type(
4685 &task_id,
4686 &lease_owner,
4687 attempt,
4688 WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE,
4689 WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE,
4690 )
4691 .await?;
4692 }
4693 Ok(commands) => {
4694 let completion = self
4695 .client
4696 .complete_workflow_task(&task_id, &lease_owner, attempt, commands)
4697 .await;
4698 if let Err(error) = completion {
4699 if !workflow_task_completion_is_terminal_timeout(
4700 &error,
4701 &task_id,
4702 attempt,
4703 run_id.as_deref(),
4704 ) {
4705 return Err(error);
4706 }
4707 }
4708 }
4709 Err(error) => {
4710 self.client
4711 .fail_workflow_task(&task_id, &lease_owner, attempt, error.to_string())
4712 .await?;
4713 }
4714 }
4715
4716 Ok(ManagedPollOutcome::Handled)
4717 }
4718
4719 async fn poll_workflows_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
4720 while !stop.load(Ordering::SeqCst) {
4721 if self.poll_workflow_once().await? == ManagedPollOutcome::Stop {
4722 stop.store(true, Ordering::SeqCst);
4723 break;
4724 }
4725 }
4726
4727 Ok(())
4728 }
4729
4730 async fn poll_activity_once(&self) -> Result<ManagedPollOutcome> {
4731 let poll_request_id = unique_request_id("rust-activity-poll");
4732 let response = self
4733 .retry_worker_operation(|| {
4734 self.client.poll_activity_task_response_with_request_id(
4735 &self.worker_id,
4736 &self.task_queue,
4737 self.poll_timeout,
4738 &poll_request_id,
4739 0,
4740 )
4741 })
4742 .await?;
4743 if response.outcome().should_stop() {
4744 return Ok(ManagedPollOutcome::Stop);
4745 }
4746 let Some(task) = response.task else {
4747 return Ok(ManagedPollOutcome::Idle);
4748 };
4749
4750 let task_id = task.task_id.clone();
4751 let attempt_id = task
4752 .activity_attempt_id
4753 .clone()
4754 .or(task.attempt_id.clone())
4755 .unwrap_or_default();
4756 let lease_owner = task
4757 .lease_owner
4758 .clone()
4759 .unwrap_or_else(|| self.worker_id.clone());
4760 let codec = task.payload_codec.clone();
4761 let result = self.execute_activity_task(task).await;
4762 match result {
4763 Ok(value) => {
4764 let completion = self
4765 .client
4766 .complete_activity_task(&task_id, &attempt_id, &lease_owner, value, &codec)
4767 .await;
4768 if let Err(error) = completion {
4769 if !activity_task_rejection_is_final(&error) {
4770 return Err(error);
4771 }
4772 }
4773 }
4774 Err(error) => {
4775 let failure = self
4776 .client
4777 .fail_activity_task(
4778 &task_id,
4779 &attempt_id,
4780 &lease_owner,
4781 error.to_string(),
4782 false,
4783 )
4784 .await;
4785 if let Err(error) = failure {
4786 if !activity_task_rejection_is_final(&error) {
4787 return Err(error);
4788 }
4789 }
4790 }
4791 }
4792
4793 Ok(ManagedPollOutcome::Handled)
4794 }
4795
4796 async fn poll_activities_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
4797 while !stop.load(Ordering::SeqCst) {
4798 if self.poll_activity_once().await? == ManagedPollOutcome::Stop {
4799 stop.store(true, Ordering::SeqCst);
4800 break;
4801 }
4802 }
4803
4804 Ok(())
4805 }
4806
4807 async fn poll_query_once(&self) -> Result<ManagedPollOutcome> {
4808 let poll_request_id = unique_request_id("rust-query-poll");
4809 let response = self
4810 .retry_worker_operation(|| {
4811 self.client.poll_query_task_response_with_request_id(
4812 &self.worker_id,
4813 &self.task_queue,
4814 self.poll_timeout,
4815 &poll_request_id,
4816 0,
4817 )
4818 })
4819 .await?;
4820 if response.outcome().should_stop() {
4821 return Ok(ManagedPollOutcome::Stop);
4822 }
4823 let Some(task) = response.task else {
4824 return Ok(ManagedPollOutcome::Idle);
4825 };
4826
4827 let query_task_id = task.query_task_id.clone();
4828 let attempt = task.query_task_attempt;
4829 let lease_owner = task
4830 .lease_owner
4831 .clone()
4832 .unwrap_or_else(|| self.worker_id.clone());
4833 let codec = task.payload_codec.clone();
4834
4835 match self.execute_query_task(task).await {
4836 Ok(value) => {
4837 let result_envelope = match encode_typed_envelope(&value, &codec) {
4838 Ok(result_envelope) => result_envelope,
4839 Err(error) => {
4840 let failure = self
4841 .client
4842 .fail_query_task(
4843 &query_task_id,
4844 &lease_owner,
4845 attempt,
4846 error.to_string(),
4847 "query_result_encode_failed",
4848 "QueryResultEncodeFailed",
4849 )
4850 .await;
4851 if let Err(error) = failure {
4852 if !query_task_rejection_is_final(&error) {
4853 return Err(error);
4854 }
4855 }
4856 return Ok(ManagedPollOutcome::Handled);
4857 }
4858 };
4859
4860 if let Err(error) = self
4861 .client
4862 .complete_query_task_with_envelope(
4863 &query_task_id,
4864 &lease_owner,
4865 attempt,
4866 value.clone().into_json()?,
4867 result_envelope,
4868 )
4869 .await
4870 {
4871 if !query_task_rejection_is_final(&error) {
4872 return Err(error);
4873 }
4874 }
4875 }
4876 Err(failure) => {
4877 let result = self
4878 .client
4879 .fail_query_task(
4880 &query_task_id,
4881 &lease_owner,
4882 attempt,
4883 failure.message,
4884 failure.reason,
4885 failure.failure_type,
4886 )
4887 .await;
4888 if let Err(error) = result {
4889 if !query_task_rejection_is_final(&error) {
4890 return Err(error);
4891 }
4892 }
4893 }
4894 }
4895
4896 Ok(ManagedPollOutcome::Handled)
4897 }
4898
4899 async fn poll_queries_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
4900 while !stop.load(Ordering::SeqCst) {
4901 if self.poll_query_once().await? == ManagedPollOutcome::Stop {
4902 stop.store(true, Ordering::SeqCst);
4903 break;
4904 }
4905 }
4906
4907 Ok(())
4908 }
4909
4910 async fn retry_worker_operation<T, F, Fut>(&self, mut operation: F) -> Result<T>
4911 where
4912 F: FnMut() -> Fut,
4913 Fut: Future<Output = Result<T>>,
4914 {
4915 let mut retries = 0;
4916
4917 loop {
4918 match operation().await {
4919 Err(error)
4920 if worker_operation_is_retryable(&error)
4921 && retries < self.retry_policy.max_retries =>
4922 {
4923 retries += 1;
4924 tokio::time::sleep(worker_retry_delay(self.retry_policy, retries)).await;
4925 }
4926 result => return result,
4927 }
4928 }
4929 }
4930
4931 async fn execute_query_task(
4932 &self,
4933 mut task: QueryTask,
4934 ) -> std::result::Result<AvroValue, QueryTaskExecutionFailure> {
4935 if !matches!(task.payload_codec.as_str(), DEFAULT_CODEC | JSON_CODEC) {
4936 return Err(QueryTaskExecutionFailure::new(
4937 "query_payload_decode_failed",
4938 format!(
4939 "cannot decode query payload with unsupported codec {:?}",
4940 task.payload_codec
4941 ),
4942 "QueryPayloadDecodeFailed",
4943 ));
4944 }
4945
4946 if !self.workflows.contains_key(&task.workflow_type) {
4947 return Err(QueryTaskExecutionFailure::new(
4948 "query_workflow_type_not_registered",
4949 format!("no workflow registered for type {:?}", task.workflow_type),
4950 "WorkflowTypeNotRegistered",
4951 ));
4952 }
4953
4954 let Some(handlers) = self.queries.get(&task.workflow_type) else {
4955 return Err(QueryTaskExecutionFailure::new(
4956 "query_handler_unavailable",
4957 format!(
4958 "query handlers are unavailable for workflow type {:?}",
4959 task.workflow_type
4960 ),
4961 "QueryHandlerUnavailable",
4962 ));
4963 };
4964 let Some(query) = handlers.get(&task.query_name) else {
4965 return Err(QueryTaskExecutionFailure::new(
4966 "rejected_unknown_query",
4967 format!("unknown query {:?}", task.query_name),
4968 "QueryFailed",
4969 ));
4970 };
4971
4972 let args = decode_task_avro_arguments(task.query_arguments.as_ref(), &task.payload_codec)
4973 .map_err(|error| {
4974 QueryTaskExecutionFailure::new(
4975 "query_payload_decode_failed",
4976 format!("cannot decode query arguments: {error}"),
4977 "QueryPayloadDecodeFailed",
4978 )
4979 })?;
4980 let workflow_input_typed =
4981 decode_task_avro_arguments(task.workflow_arguments.as_ref(), &task.payload_codec)
4982 .map_err(|error| {
4983 QueryTaskExecutionFailure::new(
4984 "query_workflow_state_unavailable",
4985 format!("cannot decode workflow start input: {error}"),
4986 "QueryWorkflowStateUnavailable",
4987 )
4988 })?;
4989 let workflow_input = workflow_input_typed.clone().into_json().map_err(|error| {
4990 QueryTaskExecutionFailure::new(
4991 "query_workflow_state_unavailable",
4992 format!("cannot project workflow start input: {error}"),
4993 "QueryWorkflowStateUnavailable",
4994 )
4995 })?;
4996 hydrate_query_history_from_export(&mut task).map_err(|error| {
4997 QueryTaskExecutionFailure::new(
4998 "query_workflow_state_unavailable",
4999 format!("cannot restore query history snapshot: {error}"),
5000 "QueryWorkflowStateUnavailable",
5001 )
5002 })?;
5003 enrich_query_history_from_export(&mut task).map_err(|error| {
5004 QueryTaskExecutionFailure::new(
5005 "query_workflow_state_unavailable",
5006 format!("cannot restore compact query history payloads: {error}"),
5007 "QueryWorkflowStateUnavailable",
5008 )
5009 })?;
5010 let signal_events = query_signal_events(&task).map_err(|error| {
5011 QueryTaskExecutionFailure::new(
5012 "query_workflow_state_unavailable",
5013 format!("cannot decode committed workflow signals: {error}"),
5014 "QueryWorkflowStateUnavailable",
5015 )
5016 })?;
5017 let history_events = Arc::new(std::mem::take(&mut task.history_events));
5018 let context = QueryContext {
5019 workflow_id: task.workflow_id,
5020 run_id: task.run_id,
5021 workflow_type: task.workflow_type.clone(),
5022 run_status: task.run_status,
5023 workflow_input,
5024 workflow_input_avro_value: workflow_input_typed.clone(),
5025 history_events: Arc::clone(&history_events),
5026 signal_events: Arc::new(signal_events),
5027 };
5028
5029 let future = match query {
5030 RegisteredQuery::Snapshot(handler) => handler(context, args),
5031 RegisteredQuery::Replayed {
5032 state_type,
5033 handler,
5034 } => {
5035 let workflow = self
5036 .workflows
5037 .get(&task.workflow_type)
5038 .expect("workflow registration was checked above");
5039 if workflow.state_type != Some(*state_type) {
5040 return Err(QueryTaskExecutionFailure::new(
5041 "query_workflow_state_unavailable",
5042 "replayed query state type does not match its workflow registration",
5043 "QueryWorkflowStateUnavailable",
5044 ));
5045 }
5046 let replay = workflow.replay.as_ref().ok_or_else(|| {
5047 QueryTaskExecutionFailure::new(
5048 "query_workflow_state_unavailable",
5049 format!(
5050 "workflow type {:?} is not registered for instance-state replay",
5051 task.workflow_type
5052 ),
5053 "QueryWorkflowStateUnavailable",
5054 )
5055 })?;
5056 let workflow_state = Arc::new(Mutex::new(
5057 WorkflowState::new_with_identity(
5058 history_events.as_ref().clone(),
5059 context.workflow_id.clone(),
5060 context.run_id.clone(),
5061 self.task_queue.clone(),
5062 task.payload_codec,
5063 None,
5064 )
5065 .map_err(|error| {
5066 QueryTaskExecutionFailure::new(
5067 "query_workflow_state_unavailable",
5068 format!("workflow replay failed before query: {error}"),
5069 "QueryWorkflowStateUnavailable",
5070 )
5071 })?,
5072 ));
5073 let workflow_context = WorkflowContext {
5074 state: workflow_state,
5075 };
5076 let mut invocation = replay(workflow_context.clone(), workflow_input_typed.clone());
5077 let mut cx = TaskContext::from_waker(noop_waker_ref());
5078 match invocation.future.as_mut().poll(&mut cx) {
5079 Poll::Ready(Ok(_)) => {
5080 workflow_context
5081 .ensure_history_consumed()
5082 .map_err(|error| {
5083 QueryTaskExecutionFailure::new(
5084 "query_workflow_state_unavailable",
5085 format!("workflow replay failed before query: {error}"),
5086 "QueryWorkflowStateUnavailable",
5087 )
5088 })?;
5089 }
5090 Poll::Ready(Err(error)) => {
5091 return Err(QueryTaskExecutionFailure::new(
5092 "query_workflow_state_unavailable",
5093 format!("workflow replay failed before query: {error}"),
5094 "QueryWorkflowStateUnavailable",
5095 ));
5096 }
5097 Poll::Pending => {
5098 let commands = workflow_context.take_commands().map_err(|error| {
5099 QueryTaskExecutionFailure::new(
5100 "query_workflow_state_unavailable",
5101 format!("workflow replay failed before query: {error}"),
5102 "QueryWorkflowStateUnavailable",
5103 )
5104 })?;
5105 if commands.is_empty()
5106 && !workflow_context
5107 .matched_recorded_pending()
5108 .map_err(|error| {
5109 QueryTaskExecutionFailure::new(
5110 "query_workflow_state_unavailable",
5111 format!("workflow replay failed before query: {error}"),
5112 "QueryWorkflowStateUnavailable",
5113 )
5114 })?
5115 {
5116 return Err(QueryTaskExecutionFailure::new(
5117 "query_workflow_state_unavailable",
5118 "workflow replay yielded without a durable command",
5119 "QueryWorkflowStateUnavailable",
5120 ));
5121 }
5122 }
5123 }
5124 let state = (invocation.snapshot)().map_err(|error| {
5125 QueryTaskExecutionFailure::new(
5126 "query_workflow_state_unavailable",
5127 format!("cannot snapshot replayed workflow state: {error}"),
5128 "QueryWorkflowStateUnavailable",
5129 )
5130 })?;
5131 handler(context, state, args).map_err(|message| {
5132 QueryTaskExecutionFailure::new(
5133 "query_workflow_state_unavailable",
5134 message,
5135 "QueryWorkflowStateUnavailable",
5136 )
5137 })?
5138 }
5139 };
5140
5141 future.await.map_err(|error| {
5142 QueryTaskExecutionFailure::new("query_rejected", error.to_string(), "QueryFailed")
5143 })
5144 }
5145
5146 fn execute_workflow_task(&self, task: WorkflowTask) -> Result<Vec<Value>> {
5147 if let Some(update_id) = task
5148 .workflow_update_id
5149 .as_deref()
5150 .filter(|update_id| !update_id.is_empty())
5151 {
5152 return self.execute_update_task(&task, update_id);
5153 }
5154
5155 let workflow = self
5156 .workflows
5157 .get(&task.workflow_type)
5158 .ok_or_else(|| Error::WorkflowNotRegistered(task.workflow_type.clone()))?;
5159 let input = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
5160 let resume_signal = decode_resume_signal(&task)?;
5161 let history_budget = WorkflowHistoryBudget {
5162 event_count: task
5163 .total_history_events
5164 .unwrap_or_else(|| u64::try_from(task.history_events.len()).unwrap_or(u64::MAX)),
5165 size_bytes: task.history_size_bytes,
5166 continue_as_new_recommended: task.continue_as_new_recommended.unwrap_or(false),
5167 pressure: task.history_budget_pressure.clone(),
5168 };
5169 let mut workflow_state = WorkflowState::new_with_identity(
5170 task.history_events,
5171 task.workflow_id,
5172 task.run_id,
5173 self.task_queue.clone(),
5174 task.payload_codec.clone(),
5175 resume_signal,
5176 )?;
5177 workflow_state.history_budget = history_budget;
5178 let state = Arc::new(Mutex::new(workflow_state));
5179 let ctx = WorkflowContext { state };
5180 let mut future = (workflow.execute)(ctx.clone(), input);
5181 let mut cx = TaskContext::from_waker(noop_waker_ref());
5182
5183 match future.as_mut().poll(&mut cx) {
5184 Poll::Ready(Ok(result)) => {
5185 ctx.ensure_history_consumed()?;
5186 let result = encode_typed_envelope(&result, &task.payload_codec)?;
5187 let mut commands = ctx.take_commands()?;
5188 commands.push(json!({
5189 "type": "complete_workflow",
5190 "result": result
5191 }));
5192 Ok(commands)
5193 }
5194 Poll::Ready(Err(error)) => {
5195 if let Error::ContinueAsNew(request) = error {
5196 let mut commands = ctx.take_commands()?;
5197 if let Some(command) = ctx.continue_as_new_command(request)? {
5198 commands.push(command);
5199 }
5200 ctx.ensure_history_consumed()?;
5201 return Ok(commands);
5202 }
5203 ctx.ensure_history_consumed()?;
5206 if workflow_task_integrity_error(&error) {
5207 return Err(error);
5212 }
5213 let mut commands = ctx.take_commands()?;
5214 commands.push(workflow_failure_command(&error));
5215 Ok(commands)
5216 }
5217 Poll::Pending => {
5218 let commands = ctx.take_commands()?;
5219 if commands.is_empty() && !ctx.matched_recorded_pending()? {
5220 Err(Error::WorkflowYieldedWithoutCommand)
5221 } else {
5222 Ok(commands)
5223 }
5224 }
5225 }
5226 }
5227
5228 fn execute_update_task(&self, task: &WorkflowTask, update_id: &str) -> Result<Vec<Value>> {
5229 if !self.workflows.contains_key(&task.workflow_type) {
5230 return Err(Error::WorkflowNotRegistered(task.workflow_type.clone()));
5231 }
5232
5233 let accepted = task.history_events.iter().rev().find_map(|event| {
5234 (event.event_type == "UpdateAccepted"
5235 && event.payload.get("update_id").and_then(Value::as_str) == Some(update_id))
5236 .then_some(&event.payload)
5237 });
5238 let update_name = accepted
5239 .and_then(|payload| payload.get("update_name"))
5240 .and_then(Value::as_str)
5241 .or(task.update_name.as_deref())
5242 .unwrap_or_default();
5243 let Some(handler) = self
5244 .updates
5245 .get(&task.workflow_type)
5246 .and_then(|handlers| handlers.get(update_name))
5247 else {
5248 return Ok(vec![json!({
5249 "type": "fail_update",
5250 "update_id": update_id,
5251 "message": format!(
5252 "no update handler is registered for {}.{update_name}",
5253 task.workflow_type
5254 ),
5255 "exception_type": "UnknownUpdate",
5256 "non_retryable": true,
5257 })]);
5258 };
5259 let arguments = accepted
5260 .and_then(|payload| payload.get("arguments"))
5261 .or(task.arguments.as_ref());
5262 let arguments = decode_task_avro_arguments(arguments, &task.payload_codec)?;
5263 let context = QueryContext {
5264 workflow_id: task.workflow_id.clone(),
5265 run_id: task.run_id.clone(),
5266 workflow_type: task.workflow_type.clone(),
5267 run_status: Some("running".to_string()),
5268 workflow_input: Value::Null,
5269 workflow_input_avro_value: AvroValue::Null,
5270 history_events: Arc::new(task.history_events.clone()),
5271 signal_events: Arc::new(Vec::new()),
5272 };
5273 let mut future = handler(context, arguments);
5274 let mut cx = TaskContext::from_waker(noop_waker_ref());
5275
5276 match future.as_mut().poll(&mut cx) {
5277 Poll::Ready(Ok(result)) => Ok(vec![json!({
5278 "type": "complete_update",
5279 "update_id": update_id,
5280 "result": encode_typed_envelope(&result, &task.payload_codec)?,
5281 })]),
5282 Poll::Ready(Err(error)) => Ok(vec![json!({
5283 "type": "fail_update",
5284 "update_id": update_id,
5285 "message": error.to_string(),
5286 "exception_type": "UpdateFailed",
5287 "non_retryable": true,
5288 })]),
5289 Poll::Pending => Err(Error::WorkflowYieldedWithoutCommand),
5290 }
5291 }
5292
5293 async fn execute_activity_task(&self, task: ActivityTask) -> Result<AvroValue> {
5294 let handler = self
5295 .activities
5296 .get(&task.activity_type)
5297 .ok_or_else(|| Error::ActivityNotRegistered(task.activity_type.clone()))?;
5298 let args = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
5299 let attempt_id = task
5300 .activity_attempt_id
5301 .clone()
5302 .or(task.attempt_id.clone())
5303 .unwrap_or_default();
5304 let lease_owner = task
5305 .lease_owner
5306 .clone()
5307 .unwrap_or_else(|| self.worker_id.clone());
5308 let ctx = ActivityContext {
5309 client: self.client.clone(),
5310 task_id: task.task_id,
5311 activity_attempt_id: attempt_id,
5312 lease_owner,
5313 activity_type: task.activity_type,
5314 attempt_number: task.attempt_number,
5315 task_queue: self.task_queue.clone(),
5316 worker_id: self.worker_id.clone(),
5317 };
5318
5319 handler(ctx, args).await
5320 }
5321}
5322
5323fn poller_result(
5324 kind: &str,
5325 result: std::result::Result<Result<()>, tokio::task::JoinError>,
5326) -> Result<()> {
5327 match result {
5328 Ok(result) => result,
5329 Err(error) => Err(Error::WorkerLoop(format!(
5330 "{kind} poller join error: {error}"
5331 ))),
5332 }
5333}
5334
5335fn optional_poller_result(
5336 kind: &str,
5337 result: Option<std::result::Result<Result<()>, tokio::task::JoinError>>,
5338) -> Result<()> {
5339 match result {
5340 Some(result) => poller_result(kind, result),
5341 None => Ok(()),
5342 }
5343}
5344
5345async fn join_pollers(
5346 workflow_poller: Option<tokio::task::JoinHandle<Result<()>>>,
5347 activity_poller: Option<tokio::task::JoinHandle<Result<()>>>,
5348 query_poller: Option<tokio::task::JoinHandle<Result<()>>>,
5349) -> Result<()> {
5350 let mut first_error = None;
5351
5352 if let Some(handle) = workflow_poller {
5353 if let Err(error) = poller_result("workflow", handle.await) {
5354 first_error.get_or_insert(error);
5355 }
5356 }
5357
5358 if let Some(handle) = activity_poller {
5359 if let Err(error) = poller_result("activity", handle.await) {
5360 first_error.get_or_insert(error);
5361 }
5362 }
5363
5364 if let Some(handle) = query_poller {
5365 if let Err(error) = poller_result("query", handle.await) {
5366 first_error.get_or_insert(error);
5367 }
5368 }
5369
5370 if let Some(error) = first_error {
5371 Err(error)
5372 } else {
5373 Ok(())
5374 }
5375}
5376
5377fn default_worker_id() -> String {
5378 let millis = SystemTime::now()
5379 .duration_since(UNIX_EPOCH)
5380 .unwrap_or_default()
5381 .as_millis();
5382 format!("rust-worker-{}-{millis}", std::process::id())
5383}
5384
5385fn unique_request_id(prefix: &str) -> String {
5386 let nanos = SystemTime::now()
5387 .duration_since(UNIX_EPOCH)
5388 .unwrap_or_default()
5389 .as_nanos();
5390 format!("{prefix}-{}-{nanos}", std::process::id())
5391}
5392
5393#[derive(Debug)]
5394struct QueryTaskExecutionFailure {
5395 reason: String,
5396 message: String,
5397 failure_type: String,
5398}
5399
5400impl QueryTaskExecutionFailure {
5401 fn new(
5402 reason: impl Into<String>,
5403 message: impl Into<String>,
5404 failure_type: impl Into<String>,
5405 ) -> Self {
5406 Self {
5407 reason: reason.into(),
5408 message: message.into(),
5409 failure_type: failure_type.into(),
5410 }
5411 }
5412}
5413
5414#[derive(Clone, Debug)]
5420pub struct WorkflowInstance<S> {
5421 state: Arc<Mutex<S>>,
5422}
5423
5424impl<S> WorkflowInstance<S> {
5425 fn new(state: S) -> Self {
5426 Self {
5427 state: Arc::new(Mutex::new(state)),
5428 }
5429 }
5430
5431 pub fn read<R>(&self, reader: impl FnOnce(&S) -> R) -> Result<R> {
5433 let state = self
5434 .state
5435 .lock()
5436 .map_err(|_| Error::WorkflowStatePoisoned)?;
5437 Ok(reader(&state))
5438 }
5439
5440 pub fn update<R>(&self, transition: impl FnOnce(&mut S) -> R) -> Result<R> {
5442 let mut state = self
5443 .state
5444 .lock()
5445 .map_err(|_| Error::WorkflowStatePoisoned)?;
5446 Ok(transition(&mut state))
5447 }
5448}
5449
5450impl<S: Clone> WorkflowInstance<S> {
5451 fn snapshot(&self) -> Result<S> {
5452 self.read(Clone::clone)
5453 }
5454}
5455
5456#[derive(Clone, Debug)]
5457pub struct WorkflowContext {
5458 state: Arc<Mutex<WorkflowState>>,
5459}
5460
5461impl WorkflowContext {
5462 pub fn workflow_identity(&self) -> Result<WorkflowIdentity> {
5464 let state = self
5465 .state
5466 .lock()
5467 .map_err(|_| Error::WorkflowStatePoisoned)?;
5468 Ok(WorkflowIdentity {
5469 workflow_id: state.workflow_id.clone(),
5470 run_id: state.run_id.clone(),
5471 })
5472 }
5473
5474 pub fn history_budget(&self) -> Result<WorkflowHistoryBudget> {
5476 let state = self
5477 .state
5478 .lock()
5479 .map_err(|_| Error::WorkflowStatePoisoned)?;
5480 Ok(state.history_budget.clone())
5481 }
5482
5483 pub fn continue_as_new<T: Serialize>(&self, args: T) -> Result<Value> {
5489 self.continue_as_new_with_options(ContinueAsNewOptions::new(), args)
5490 }
5491
5492 pub fn continue_as_new_with_options<T: Serialize>(
5494 &self,
5495 options: ContinueAsNewOptions,
5496 args: T,
5497 ) -> Result<Value> {
5498 options.validate()?;
5499 Err(Error::ContinueAsNew(ContinueAsNewRequest {
5500 arguments: normalize_avro_arguments(AvroValue::from_serialize(&args)?),
5501 options,
5502 }))
5503 }
5504
5505 pub fn activity<T: Serialize>(
5506 &self,
5507 activity_type: impl Into<String>,
5508 args: T,
5509 ) -> ActivityCall {
5510 self.activity_with_options(activity_type, ActivityOptions::new(), args)
5511 }
5512
5513 pub fn activity_on_queue<T, Q>(
5514 &self,
5515 activity_type: impl Into<String>,
5516 task_queue: Option<Q>,
5517 args: T,
5518 ) -> ActivityCall
5519 where
5520 T: Serialize,
5521 Q: Into<String>,
5522 {
5523 let mut options = ActivityOptions::new();
5524 options.task_queue = task_queue.map(Into::into);
5525 self.activity_with_options(activity_type, options, args)
5526 }
5527
5528 pub fn activity_with_options<T: Serialize>(
5566 &self,
5567 activity_type: impl Into<String>,
5568 options: ActivityOptions,
5569 args: T,
5570 ) -> ActivityCall {
5571 ActivityCall {
5572 ctx: self.clone(),
5573 activity_type: activity_type.into(),
5574 options,
5575 args: Some(AvroValue::from_serialize(&args)),
5576 scheduled: false,
5577 }
5578 }
5579
5580 pub async fn activity_avro_value<T: Serialize>(
5581 &self,
5582 activity_type: impl Into<String>,
5583 args: T,
5584 ) -> Result<AvroValue> {
5585 let mut call = self.activity(activity_type, args);
5586 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
5587 }
5588
5589 pub async fn activity_avro_value_with_options<T: Serialize>(
5590 &self,
5591 activity_type: impl Into<String>,
5592 options: ActivityOptions,
5593 args: T,
5594 ) -> Result<AvroValue> {
5595 let mut call = self.activity_with_options(activity_type, options, args);
5596 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
5597 }
5598
5599 pub fn wait_signal(&self, signal_name: impl Into<String>) -> SignalCall {
5600 SignalCall {
5601 ctx: self.clone(),
5602 signal_name: signal_name.into(),
5603 opened_wait: false,
5604 matched_pending: false,
5605 }
5606 }
5607
5608 pub async fn wait_signal_avro_value(
5609 &self,
5610 signal_name: impl Into<String>,
5611 ) -> Result<Vec<AvroValue>> {
5612 let mut call = self.wait_signal(signal_name);
5613 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
5614 }
5615
5616 pub fn sleep(&self, duration: Duration) -> TimerCall {
5637 let delay_seconds = duration
5638 .as_secs()
5639 .checked_add(u64::from(duration.subsec_nanos() > 0));
5640 TimerCall {
5641 ctx: self.clone(),
5642 delay_seconds,
5643 scheduled: false,
5644 matched_pending: false,
5645 }
5646 }
5647
5648 pub fn start_timer(&self, duration: Duration) -> TimerCall {
5650 self.sleep(duration)
5651 }
5652
5653 pub fn side_effect<T, F>(&self, callback: F) -> Result<T>
5660 where
5661 T: Serialize + DeserializeOwned,
5662 F: FnOnce() -> T,
5663 {
5664 {
5665 let mut state = self
5666 .state
5667 .lock()
5668 .map_err(|_| Error::WorkflowStatePoisoned)?;
5669 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
5670 return match recorded {
5671 RecordedCommand::SideEffect { sequence, value } => {
5672 state.command_cursor += 1;
5673 value.deserialize().map_err(|error| {
5674 Error::NonDeterministicReplay(ReplayFailure::new(
5675 "side_effect_type_mismatch",
5676 Some(sequence),
5677 Some(std::any::type_name::<T>().to_string()),
5678 Some(error.to_string()),
5679 "recorded side-effect value is incompatible with the requested Rust type",
5680 ))
5681 })
5682 }
5683 other => Err(command_mismatch(&other, "side effect")),
5684 };
5685 }
5686 }
5687
5688 let value = callback();
5689 let avro_value = AvroValue::from_serialize(&value)?;
5690 let mut state = self
5691 .state
5692 .lock()
5693 .map_err(|_| Error::WorkflowStatePoisoned)?;
5694 let result = encode_typed_envelope(&avro_value, &state.payload_codec)?;
5695 state.commands.push(json!({
5696 "type": "record_side_effect",
5697 "result": result,
5698 }));
5699 Ok(value)
5700 }
5701
5702 pub fn side_effect_avro_value<F>(&self, callback: F) -> Result<AvroValue>
5704 where
5705 F: FnOnce() -> AvroValue,
5706 {
5707 {
5708 let mut state = self
5709 .state
5710 .lock()
5711 .map_err(|_| Error::WorkflowStatePoisoned)?;
5712 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
5713 return match recorded {
5714 RecordedCommand::SideEffect { value, .. } => {
5715 state.command_cursor += 1;
5716 Ok(value)
5717 }
5718 other => Err(command_mismatch(&other, "side effect")),
5719 };
5720 }
5721 }
5722
5723 let value = callback();
5724 let mut state = self
5725 .state
5726 .lock()
5727 .map_err(|_| Error::WorkflowStatePoisoned)?;
5728 let result = encode_typed_envelope(&value, &state.payload_codec)?;
5729 state.commands.push(json!({
5730 "type": "record_side_effect",
5731 "result": result,
5732 }));
5733 Ok(value)
5734 }
5735
5736 pub fn uuid_v4(&self) -> Result<Uuid> {
5738 self.side_effect(Uuid::new_v4)
5739 }
5740
5741 pub fn get_version(
5744 &self,
5745 change_id: impl Into<String>,
5746 min_supported: i32,
5747 max_supported: i32,
5748 ) -> Result<i32> {
5749 let change_id = change_id.into();
5750 if change_id.trim().is_empty() {
5751 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5752 "version_change_id_invalid",
5753 None,
5754 Some("non-empty change ID".to_string()),
5755 Some(change_id),
5756 "version markers require a stable non-empty change ID",
5757 )));
5758 }
5759 if min_supported > max_supported {
5760 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5761 "version_range_invalid",
5762 None,
5763 Some("min_supported <= max_supported".to_string()),
5764 Some(format!("{min_supported}..={max_supported}")),
5765 "version marker supported range is invalid",
5766 )));
5767 }
5768
5769 let mut state = self
5770 .state
5771 .lock()
5772 .map_err(|_| Error::WorkflowStatePoisoned)?;
5773 if let Some((version, sequence)) = state.version_markers.get(&change_id).copied() {
5774 ensure_version_supported(&change_id, version, min_supported, max_supported, sequence)?;
5775 return Ok(version);
5776 }
5777
5778 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
5779 return match recorded {
5780 RecordedCommand::VersionMarker {
5781 sequence,
5782 change_id: recorded_change_id,
5783 version,
5784 ..
5785 } => {
5786 if recorded_change_id != change_id {
5787 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5788 "version_change_id_mismatch",
5789 Some(sequence),
5790 Some(recorded_change_id),
5791 Some(change_id),
5792 "recorded version marker change ID differs from current workflow code",
5793 )));
5794 }
5795 ensure_version_supported(
5796 &change_id,
5797 version,
5798 min_supported,
5799 max_supported,
5800 sequence,
5801 )?;
5802 state.command_cursor += 1;
5803 state.version_markers.insert(change_id, (version, sequence));
5804 Ok(version)
5805 }
5806 other => Err(command_mismatch(
5807 &other,
5808 format!("version marker:{change_id}"),
5809 )),
5810 };
5811 }
5812
5813 let version = max_supported;
5814 state.commands.push(json!({
5815 "type": "record_version_marker",
5816 "change_id": change_id,
5817 "version": version,
5818 "min_supported": min_supported,
5819 "max_supported": max_supported,
5820 }));
5821 state.version_markers.insert(change_id, (version, 0));
5824 Ok(version)
5825 }
5826
5827 pub fn patched(&self, change_id: impl Into<String>) -> Result<bool> {
5829 Ok(self.get_version(change_id, -1, 1)? == 1)
5830 }
5831
5832 pub fn deprecate_patch(&self, change_id: impl Into<String>) -> Result<()> {
5834 self.get_version(change_id, -1, 1).map(|_| ())
5835 }
5836
5837 pub fn start_child_workflow<T: Serialize>(
5864 &self,
5865 workflow_type: impl Into<String>,
5866 options: ChildWorkflowOptions,
5867 args: T,
5868 ) -> ChildWorkflowCall {
5869 ChildWorkflowCall {
5870 ctx: self.clone(),
5871 workflow_type: workflow_type.into(),
5872 options,
5873 args: Some(AvroValue::from_serialize(&args)),
5874 scheduled: false,
5875 matched_pending: false,
5876 }
5877 }
5878
5879 pub async fn start_child_workflow_avro_value<T: Serialize>(
5880 &self,
5881 workflow_type: impl Into<String>,
5882 options: ChildWorkflowOptions,
5883 args: T,
5884 ) -> Result<ChildWorkflowAvroResult> {
5885 let mut call = self.start_child_workflow(workflow_type, options, args);
5886 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
5887 }
5888
5889 fn take_commands(&self) -> Result<Vec<Value>> {
5890 let mut state = self
5891 .state
5892 .lock()
5893 .map_err(|_| Error::WorkflowStatePoisoned)?;
5894 Ok(std::mem::take(&mut state.commands))
5895 }
5896
5897 fn continue_as_new_command(&self, request: ContinueAsNewRequest) -> Result<Option<Value>> {
5898 let mut state = self
5899 .state
5900 .lock()
5901 .map_err(|_| Error::WorkflowStatePoisoned)?;
5902
5903 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
5904 return Err(command_mismatch(&recorded, "continue as new"));
5905 }
5906 if state.recorded_continue_as_new_sequence.is_some() {
5907 state.continue_as_new_consumed = true;
5908 return Ok(None);
5909 }
5910
5911 let arguments = encode_typed_envelope(&request.arguments, &state.payload_codec)?;
5912 let mut command = serde_json::Map::from_iter([
5913 ("type".to_string(), json!("continue_as_new")),
5914 ("arguments".to_string(), arguments),
5915 ("queue".to_string(), json!(state.task_queue.clone())),
5916 ]);
5917 if let Some(workflow_type) = request.options.workflow_type {
5918 command.insert("workflow_type".to_string(), json!(workflow_type));
5919 }
5920 if let Some(task_queue) = request.options.task_queue {
5921 command.insert("queue".to_string(), json!(task_queue));
5922 }
5923 Ok(Some(Value::Object(command)))
5924 }
5925
5926 fn matched_recorded_pending(&self) -> Result<bool> {
5927 let state = self
5928 .state
5929 .lock()
5930 .map_err(|_| Error::WorkflowStatePoisoned)?;
5931 Ok(state.matched_recorded_pending)
5932 }
5933
5934 fn ensure_history_consumed(&self) -> Result<()> {
5935 let state = self
5936 .state
5937 .lock()
5938 .map_err(|_| Error::WorkflowStatePoisoned)?;
5939 if let Some(command) = state.recorded_commands.get(state.command_cursor) {
5940 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5941 "recorded_commands_unconsumed",
5942 Some(command.sequence()),
5943 Some(command.shape().to_string()),
5944 Some("workflow completion".to_string()),
5945 "workflow completed before consuming all recorded durable commands",
5946 )));
5947 }
5948 if let Some(sequence) = state
5949 .recorded_continue_as_new_sequence
5950 .filter(|_| !state.continue_as_new_consumed)
5951 {
5952 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5953 "recorded_continue_as_new_unconsumed",
5954 Some(sequence),
5955 Some("continue as new".to_string()),
5956 Some("workflow completion".to_string()),
5957 "workflow completed without consuming its recorded continue-as-new transition",
5958 )));
5959 }
5960 Ok(())
5961 }
5962}
5963
5964#[derive(Debug)]
5965struct WorkflowState {
5966 workflow_id: Option<String>,
5967 run_id: Option<String>,
5968 task_queue: String,
5969 payload_codec: String,
5970 history_budget: WorkflowHistoryBudget,
5971 resume_signal: Option<ResumeSignal>,
5972 recorded_commands: Vec<RecordedCommand>,
5973 recorded_continue_as_new_sequence: Option<u64>,
5974 continue_as_new_consumed: bool,
5975 command_cursor: usize,
5976 matched_recorded_pending: bool,
5977 version_markers: HashMap<String, (i32, u64)>,
5978 commands: Vec<Value>,
5979}
5980
5981impl WorkflowState {
5982 #[cfg(test)]
5983 fn new(
5984 history: Vec<HistoryEvent>,
5985 task_queue: String,
5986 payload_codec: String,
5987 resume_signal: Option<ResumeSignal>,
5988 ) -> Result<Self> {
5989 Self::new_with_identity(
5990 history,
5991 None,
5992 None,
5993 task_queue,
5994 payload_codec,
5995 resume_signal,
5996 )
5997 }
5998
5999 fn new_with_identity(
6000 history: Vec<HistoryEvent>,
6001 workflow_id: Option<String>,
6002 run_id: Option<String>,
6003 task_queue: String,
6004 payload_codec: String,
6005 resume_signal: Option<ResumeSignal>,
6006 ) -> Result<Self> {
6007 let recorded_commands = recorded_commands(
6008 &history,
6009 &payload_codec,
6010 WorkflowIdentity {
6011 workflow_id: workflow_id.clone(),
6012 run_id: run_id.clone(),
6013 },
6014 )?;
6015 let recorded_continue_as_new = history
6016 .iter()
6017 .filter(|event| event.event_type == "WorkflowContinuedAsNew")
6018 .collect::<Vec<_>>();
6019 if recorded_continue_as_new.len() > 1 {
6020 return Err(invalid_recorded_history(
6021 "duplicate_continue_as_new_transition",
6022 recorded_continue_as_new
6023 .last()
6024 .and_then(|event| durable_event_sequence(event))
6025 .unwrap_or(0),
6026 "one WorkflowContinuedAsNew event",
6027 &format!(
6028 "{} WorkflowContinuedAsNew events",
6029 recorded_continue_as_new.len()
6030 ),
6031 "workflow history records one continue-as-new transition more than once",
6032 ));
6033 }
6034 let recorded_continue_as_new_sequence = recorded_continue_as_new
6035 .first()
6036 .map(|event| {
6037 durable_event_sequence(event).ok_or_else(|| {
6038 Error::NonDeterministicReplay(ReplayFailure::new(
6039 "continue_as_new_sequence_missing",
6040 None,
6041 Some("recorded transition sequence".to_string()),
6042 Some("missing sequence".to_string()),
6043 "WorkflowContinuedAsNew history is missing its recorded sequence",
6044 ))
6045 })
6046 })
6047 .transpose()?;
6048 let event_count = u64::try_from(history.len()).unwrap_or(u64::MAX);
6049 Ok(Self {
6050 workflow_id,
6051 run_id,
6052 task_queue,
6053 payload_codec,
6054 history_budget: WorkflowHistoryBudget {
6055 event_count,
6056 ..WorkflowHistoryBudget::default()
6057 },
6058 resume_signal,
6059 recorded_commands,
6060 recorded_continue_as_new_sequence,
6061 continue_as_new_consumed: false,
6062 command_cursor: 0,
6063 matched_recorded_pending: false,
6064 version_markers: HashMap::new(),
6065 commands: Vec::new(),
6066 })
6067 }
6068}
6069
6070#[derive(Clone, Debug)]
6071enum RecordedCommand {
6072 Activity {
6073 sequence: u64,
6074 activity_type: Option<String>,
6075 options: Option<RecordedActivityOptions>,
6076 outcome: Option<ActivityOutcome>,
6077 },
6078 Timer {
6079 sequence: u64,
6080 delay_seconds: u64,
6081 fired: bool,
6082 },
6083 ChildWorkflow {
6084 sequence: u64,
6085 workflow_type: Option<String>,
6086 outcome: Option<ChildWorkflowOutcome>,
6087 },
6088 SignalWait {
6089 sequence: u64,
6090 signal_name: String,
6091 value: Option<Vec<AvroValue>>,
6092 },
6093 SideEffect {
6094 sequence: u64,
6095 value: AvroValue,
6096 },
6097 VersionMarker {
6098 sequence: u64,
6099 change_id: String,
6100 version: i32,
6101 },
6102}
6103
6104#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
6105struct RecordedActivityOptions {
6106 task_queue: RecordedSnapshotValue<Option<String>>,
6107 execution_mode: RecordedSnapshotValue<Option<String>>,
6108 retry_policy: ActivityRetrySnapshot,
6109}
6110
6111#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
6112enum RecordedSnapshotValue<T> {
6113 Unknown,
6115 Known(T),
6116}
6117
6118impl<T: PartialEq> RecordedSnapshotValue<T> {
6119 fn matches_current(&self, current: &Self) -> bool {
6120 match self {
6121 Self::Unknown => true,
6122 Self::Known(recorded) => matches!(current, Self::Known(value) if value == recorded),
6123 }
6124 }
6125}
6126
6127#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
6128struct ActivityRetrySnapshot {
6129 snapshot_version: RecordedSnapshotValue<Option<u64>>,
6130 max_attempts: RecordedSnapshotValue<Option<u64>>,
6131 backoff_seconds: RecordedSnapshotValue<Vec<u64>>,
6132 start_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
6133 schedule_to_start_timeout: RecordedSnapshotValue<Option<u64>>,
6134 schedule_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
6135 heartbeat_timeout: RecordedSnapshotValue<Option<u64>>,
6136 non_retryable_error_types: RecordedSnapshotValue<Vec<String>>,
6137}
6138
6139impl ActivityRetrySnapshot {
6140 fn matches_current(&self, current: &Self) -> bool {
6141 self.snapshot_version
6142 .matches_current(¤t.snapshot_version)
6143 && self.max_attempts.matches_current(¤t.max_attempts)
6144 && self
6145 .backoff_seconds
6146 .matches_current(¤t.backoff_seconds)
6147 && self
6148 .start_to_close_timeout
6149 .matches_current(¤t.start_to_close_timeout)
6150 && self
6151 .schedule_to_start_timeout
6152 .matches_current(¤t.schedule_to_start_timeout)
6153 && self
6154 .schedule_to_close_timeout
6155 .matches_current(¤t.schedule_to_close_timeout)
6156 && self
6157 .heartbeat_timeout
6158 .matches_current(¤t.heartbeat_timeout)
6159 && self
6160 .non_retryable_error_types
6161 .matches_current(¤t.non_retryable_error_types)
6162 }
6163}
6164
6165fn recorded_optional_u64(
6166 object: Option<&serde_json::Map<String, Value>>,
6167 field: &str,
6168) -> RecordedSnapshotValue<Option<u64>> {
6169 match object.and_then(|object| object.get(field)) {
6170 None => RecordedSnapshotValue::Unknown,
6171 Some(Value::Null) => RecordedSnapshotValue::Known(None),
6172 Some(value) => RecordedSnapshotValue::Known(value_as_u64(value)),
6173 }
6174}
6175
6176fn recorded_optional_string(
6177 object: &serde_json::Map<String, Value>,
6178 field: &str,
6179) -> RecordedSnapshotValue<Option<String>> {
6180 match object.get(field) {
6181 None => RecordedSnapshotValue::Unknown,
6182 Some(Value::Null) => RecordedSnapshotValue::Known(None),
6183 Some(value) => RecordedSnapshotValue::Known(value.as_str().map(str::to_string)),
6184 }
6185}
6186
6187fn recorded_activity_retry_snapshot(policy: Option<&Value>) -> ActivityRetrySnapshot {
6188 let policy = policy.and_then(Value::as_object);
6189 let backoff_seconds = policy
6190 .and_then(|policy| policy.get("backoff_seconds"))
6191 .and_then(Value::as_array)
6192 .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
6193 .map_or(RecordedSnapshotValue::Unknown, RecordedSnapshotValue::Known);
6194 let mut non_retryable_error_types = Vec::new();
6195 for error_type in policy
6196 .and_then(|policy| policy.get("non_retryable_error_types"))
6197 .and_then(Value::as_array)
6198 .into_iter()
6199 .flatten()
6200 .filter_map(Value::as_str)
6201 .map(str::trim)
6202 .filter(|error_type| !error_type.is_empty())
6203 {
6204 if !non_retryable_error_types
6205 .iter()
6206 .any(|recorded| recorded == error_type)
6207 {
6208 non_retryable_error_types.push(error_type.to_string());
6209 }
6210 }
6211
6212 ActivityRetrySnapshot {
6213 snapshot_version: recorded_optional_u64(policy, "snapshot_version"),
6214 max_attempts: recorded_optional_u64(policy, "max_attempts"),
6215 backoff_seconds,
6216 start_to_close_timeout: recorded_optional_u64(policy, "start_to_close_timeout"),
6217 schedule_to_start_timeout: recorded_optional_u64(policy, "schedule_to_start_timeout"),
6218 schedule_to_close_timeout: recorded_optional_u64(policy, "schedule_to_close_timeout"),
6219 heartbeat_timeout: recorded_optional_u64(policy, "heartbeat_timeout"),
6220 non_retryable_error_types: if policy
6221 .is_some_and(|policy| policy.contains_key("non_retryable_error_types"))
6222 {
6223 RecordedSnapshotValue::Known(non_retryable_error_types)
6224 } else {
6225 RecordedSnapshotValue::Unknown
6226 },
6227 }
6228}
6229
6230fn current_activity_retry_snapshot(options: &ValidatedActivityOptions) -> ActivityRetrySnapshot {
6231 let policy = options.retry_policy.as_ref();
6232 let max_attempts = match policy.and_then(|policy| policy.get("max_attempts")) {
6233 Some(Value::Null) => None,
6234 Some(value) => value_as_u64(value),
6235 None => Some(1),
6236 };
6237 let backoff_seconds = policy
6238 .and_then(|policy| policy.get("backoff_seconds"))
6239 .and_then(Value::as_array)
6240 .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
6241 .unwrap_or_default();
6242 let non_retryable_error_types = policy
6243 .and_then(|policy| policy.get("non_retryable_error_types"))
6244 .and_then(Value::as_array)
6245 .into_iter()
6246 .flatten()
6247 .filter_map(Value::as_str)
6248 .map(str::to_string)
6249 .collect();
6250
6251 ActivityRetrySnapshot {
6252 snapshot_version: RecordedSnapshotValue::Known(Some(1)),
6253 max_attempts: RecordedSnapshotValue::Known(max_attempts),
6254 backoff_seconds: RecordedSnapshotValue::Known(backoff_seconds),
6255 start_to_close_timeout: RecordedSnapshotValue::Known(options.start_to_close_timeout),
6256 schedule_to_start_timeout: RecordedSnapshotValue::Known(options.schedule_to_start_timeout),
6257 schedule_to_close_timeout: RecordedSnapshotValue::Known(options.schedule_to_close_timeout),
6258 heartbeat_timeout: RecordedSnapshotValue::Known(options.heartbeat_timeout),
6259 non_retryable_error_types: RecordedSnapshotValue::Known(non_retryable_error_types),
6260 }
6261}
6262
6263fn activity_options_description(options: &RecordedActivityOptions) -> String {
6264 serde_json::to_string(options).unwrap_or_else(|_| format!("{options:?}"))
6265}
6266
6267impl RecordedCommand {
6268 fn sequence(&self) -> u64 {
6269 match self {
6270 Self::Activity { sequence, .. }
6271 | Self::Timer { sequence, .. }
6272 | Self::ChildWorkflow { sequence, .. }
6273 | Self::SignalWait { sequence, .. }
6274 | Self::SideEffect { sequence, .. }
6275 | Self::VersionMarker { sequence, .. } => *sequence,
6276 }
6277 }
6278
6279 fn shape(&self) -> &'static str {
6280 match self {
6281 Self::Activity { .. } => "activity",
6282 Self::Timer { .. } => "timer",
6283 Self::ChildWorkflow { .. } => "child workflow",
6284 Self::SignalWait { .. } => "signal wait",
6285 Self::SideEffect { .. } => "side effect",
6286 Self::VersionMarker { .. } => "version marker",
6287 }
6288 }
6289}
6290
6291fn ensure_version_supported(
6292 change_id: &str,
6293 version: i32,
6294 min_supported: i32,
6295 max_supported: i32,
6296 sequence: u64,
6297) -> Result<()> {
6298 if (min_supported..=max_supported).contains(&version) {
6299 return Ok(());
6300 }
6301 Err(Error::NonDeterministicReplay(ReplayFailure::new(
6302 "version_marker_incompatible_range",
6303 (sequence != 0).then_some(sequence),
6304 Some(format!("{min_supported}..={max_supported}")),
6305 Some(format!("{change_id}:{version}")),
6306 "recorded workflow version is outside the range supported by current code",
6307 )))
6308}
6309
6310#[derive(Clone, Debug)]
6311struct ResumeSignal {
6312 signal_name: String,
6313 arguments: Vec<AvroValue>,
6314}
6315
6316pub struct ActivityCall {
6317 ctx: WorkflowContext,
6318 activity_type: String,
6319 options: ActivityOptions,
6320 args: Option<Result<AvroValue>>,
6321 scheduled: bool,
6322}
6323
6324impl ActivityCall {
6325 fn poll_avro_value(
6326 mut self: Pin<&mut Self>,
6327 _cx: &mut TaskContext<'_>,
6328 ) -> Poll<Result<AvroValue>> {
6329 let ctx = self.ctx.clone();
6330 let mut state = match ctx.state.lock() {
6331 Ok(state) => state,
6332 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6333 };
6334
6335 if self.scheduled {
6336 return Poll::Pending;
6337 }
6338
6339 let options = match self.options.validate() {
6340 Ok(options) => options,
6341 Err(error) => {
6342 return Poll::Ready(Err(Error::InvalidActivityOptions(error)));
6343 }
6344 };
6345 let task_queue = options
6346 .task_queue
6347 .clone()
6348 .unwrap_or_else(|| state.task_queue.clone());
6349 let current_recorded_options = RecordedActivityOptions {
6350 task_queue: RecordedSnapshotValue::Known(Some(task_queue.clone())),
6351 execution_mode: RecordedSnapshotValue::Known(None),
6354 retry_policy: current_activity_retry_snapshot(&options),
6355 };
6356
6357 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6358 let sequence = recorded.sequence();
6359 match recorded {
6360 RecordedCommand::Activity {
6361 activity_type,
6362 options: recorded_options,
6363 outcome,
6364 ..
6365 } => {
6366 if let Some(recorded_type) = activity_type {
6367 if recorded_type != self.activity_type {
6368 return Poll::Ready(Err(Error::NonDeterministicReplay(
6369 ReplayFailure::new(
6370 "recorded_command_detail_mismatch",
6371 Some(sequence),
6372 Some(format!("activity:{recorded_type}")),
6373 Some(format!("activity:{}", self.activity_type)),
6374 "recorded activity type differs from the current workflow command",
6375 ),
6376 )));
6377 }
6378 }
6379 if let Some(recorded_options) = recorded_options {
6380 if !recorded_options
6381 .task_queue
6382 .matches_current(¤t_recorded_options.task_queue)
6383 {
6384 return Poll::Ready(Err(Error::NonDeterministicReplay(
6385 ReplayFailure::new(
6386 "activity_task_queue_mismatch",
6387 Some(sequence),
6388 Some(activity_options_description(&recorded_options)),
6389 Some(activity_options_description(¤t_recorded_options)),
6390 "recorded activity task queue differs from the current workflow command",
6391 ),
6392 )));
6393 }
6394 if !recorded_options
6395 .execution_mode
6396 .matches_current(¤t_recorded_options.execution_mode)
6397 {
6398 return Poll::Ready(Err(Error::NonDeterministicReplay(
6399 ReplayFailure::new(
6400 "activity_execution_mode_mismatch",
6401 Some(sequence),
6402 Some(activity_options_description(&recorded_options)),
6403 Some(activity_options_description(¤t_recorded_options)),
6404 "recorded activity execution mode differs from the current workflow command",
6405 ),
6406 )));
6407 }
6408 if !recorded_options
6409 .retry_policy
6410 .matches_current(¤t_recorded_options.retry_policy)
6411 {
6412 return Poll::Ready(Err(Error::NonDeterministicReplay(
6413 ReplayFailure::new(
6414 "activity_retry_policy_mismatch",
6415 Some(sequence),
6416 Some(activity_options_description(&recorded_options)),
6417 Some(activity_options_description(¤t_recorded_options)),
6418 "recorded activity retry policy differs from the current workflow command",
6419 ),
6420 )));
6421 }
6422 }
6423 state.command_cursor += 1;
6424 if let Some(outcome) = outcome {
6425 return Poll::Ready(outcome.map_err(Error::ActivityFailed));
6426 }
6427 state.matched_recorded_pending = true;
6428 self.scheduled = true;
6429 return Poll::Pending;
6430 }
6431 other => {
6432 return Poll::Ready(Err(command_mismatch(
6433 &other,
6434 format!("activity:{}", self.activity_type),
6435 )));
6436 }
6437 }
6438 }
6439
6440 if !self.scheduled {
6441 let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
6442 Ok(args) => args,
6443 Err(error) => return Poll::Ready(Err(error)),
6444 };
6445 let arguments = normalize_avro_arguments(args);
6446 let envelope = match encode_typed_envelope(&arguments, &state.payload_codec) {
6447 Ok(envelope) => envelope,
6448 Err(error) => return Poll::Ready(Err(error)),
6449 };
6450
6451 let mut command = serde_json::Map::from_iter([
6452 ("type".to_string(), json!("schedule_activity")),
6453 (
6454 "activity_type".to_string(),
6455 json!(self.activity_type.clone()),
6456 ),
6457 ("queue".to_string(), json!(task_queue)),
6458 ("arguments".to_string(), envelope),
6459 ]);
6460 for (field, value) in [
6461 ("start_to_close_timeout", options.start_to_close_timeout),
6462 (
6463 "schedule_to_start_timeout",
6464 options.schedule_to_start_timeout,
6465 ),
6466 (
6467 "schedule_to_close_timeout",
6468 options.schedule_to_close_timeout,
6469 ),
6470 ("heartbeat_timeout", options.heartbeat_timeout),
6471 ] {
6472 if let Some(value) = value {
6473 command.insert(field.to_string(), json!(value));
6474 }
6475 }
6476 if let Some(retry_policy) = options.retry_policy {
6477 command.insert("retry_policy".to_string(), retry_policy);
6478 }
6479 state.commands.push(Value::Object(command));
6480 self.scheduled = true;
6481 }
6482
6483 Poll::Pending
6484 }
6485}
6486
6487impl Future for ActivityCall {
6488 type Output = Result<Value>;
6489
6490 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6491 match self.poll_avro_value(cx) {
6492 Poll::Ready(Ok(value)) => Poll::Ready(value.into_json()),
6493 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
6494 Poll::Pending => Poll::Pending,
6495 }
6496 }
6497}
6498
6499pub struct TimerCall {
6501 ctx: WorkflowContext,
6502 delay_seconds: Option<u64>,
6503 scheduled: bool,
6504 matched_pending: bool,
6505}
6506
6507impl Future for TimerCall {
6508 type Output = Result<()>;
6509
6510 fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6511 if self.matched_pending {
6512 return Poll::Pending;
6513 }
6514
6515 let ctx = self.ctx.clone();
6516 let Some(requested_delay) = self.delay_seconds else {
6517 return Poll::Ready(Err(Error::TimerDurationOverflow));
6518 };
6519 let mut state = match ctx.state.lock() {
6520 Ok(state) => state,
6521 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6522 };
6523
6524 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6525 match recorded {
6526 RecordedCommand::Timer {
6527 sequence,
6528 delay_seconds,
6529 fired,
6530 ..
6531 } => {
6532 if delay_seconds != requested_delay {
6533 return Poll::Ready(Err(Error::NonDeterministicReplay(
6534 ReplayFailure::new(
6535 "timer_delay_mismatch",
6536 Some(sequence),
6537 Some(format!("timer:{delay_seconds}s")),
6538 Some(format!("timer:{requested_delay}s")),
6539 "recorded timer delay differs from the current workflow command",
6540 ),
6541 )));
6542 }
6543 state.command_cursor += 1;
6544 if fired {
6545 return Poll::Ready(Ok(()));
6546 }
6547 state.matched_recorded_pending = true;
6548 self.scheduled = true;
6549 self.matched_pending = true;
6550 return Poll::Pending;
6551 }
6552 other => return Poll::Ready(Err(command_mismatch(&other, "timer"))),
6553 }
6554 }
6555
6556 if !self.scheduled {
6557 state.commands.push(json!({
6558 "type": "start_timer",
6559 "delay_seconds": requested_delay,
6560 }));
6561 self.scheduled = true;
6562 }
6563
6564 Poll::Pending
6565 }
6566}
6567
6568pub struct ChildWorkflowCall {
6570 ctx: WorkflowContext,
6571 workflow_type: String,
6572 options: ChildWorkflowOptions,
6573 args: Option<Result<AvroValue>>,
6574 scheduled: bool,
6575 matched_pending: bool,
6576}
6577
6578impl ChildWorkflowCall {
6579 fn poll_avro_value(
6580 mut self: Pin<&mut Self>,
6581 _cx: &mut TaskContext<'_>,
6582 ) -> Poll<Result<ChildWorkflowAvroResult>> {
6583 if self.matched_pending {
6584 return Poll::Pending;
6585 }
6586
6587 let ctx = self.ctx.clone();
6588 let mut state = match ctx.state.lock() {
6589 Ok(state) => state,
6590 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6591 };
6592
6593 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6594 let sequence = recorded.sequence();
6595 match recorded {
6596 RecordedCommand::ChildWorkflow {
6597 workflow_type,
6598 outcome,
6599 ..
6600 } => {
6601 if let Some(recorded_type) = workflow_type {
6602 if recorded_type != self.workflow_type {
6603 return Poll::Ready(Err(Error::NonDeterministicReplay(
6604 ReplayFailure::new(
6605 "recorded_command_detail_mismatch",
6606 Some(sequence),
6607 Some(format!("child workflow:{recorded_type}")),
6608 Some(format!("child workflow:{}", self.workflow_type)),
6609 "recorded child workflow type differs from the current workflow command",
6610 ),
6611 )));
6612 }
6613 }
6614 state.command_cursor += 1;
6615 if let Some(outcome) = outcome {
6616 return Poll::Ready(outcome.map_err(Error::ChildWorkflowFailed));
6617 }
6618 state.matched_recorded_pending = true;
6619 self.scheduled = true;
6620 self.matched_pending = true;
6621 return Poll::Pending;
6622 }
6623 other => {
6624 return Poll::Ready(Err(command_mismatch(
6625 &other,
6626 format!("child workflow:{}", self.workflow_type),
6627 )));
6628 }
6629 }
6630 }
6631
6632 if !self.scheduled {
6633 if self.options.task_queue.trim().is_empty() {
6634 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
6635 "task_queue must not be empty".to_string(),
6636 )));
6637 }
6638 for (name, value) in [
6639 (
6640 "execution_timeout_seconds",
6641 self.options.execution_timeout_seconds,
6642 ),
6643 ("run_timeout_seconds", self.options.run_timeout_seconds),
6644 ] {
6645 if value == Some(0) {
6646 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(format!(
6647 "{name} must be at least 1"
6648 ))));
6649 }
6650 }
6651
6652 let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
6653 Ok(args) => args,
6654 Err(error) => return Poll::Ready(Err(error)),
6655 };
6656 let arguments = match encode_typed_envelope(
6657 &normalize_avro_arguments(args),
6658 &state.payload_codec,
6659 ) {
6660 Ok(arguments) => arguments,
6661 Err(error) => return Poll::Ready(Err(error)),
6662 };
6663 let mut command = json!({
6664 "type": "start_child_workflow",
6665 "workflow_type": self.workflow_type,
6666 "queue": self.options.task_queue,
6667 "parent_close_policy": self.options.parent_close_policy.as_str(),
6668 "arguments": arguments,
6669 });
6670 let object = command
6671 .as_object_mut()
6672 .expect("child workflow command is always an object");
6673 if let Some(policy) = &self.options.retry_policy {
6674 let mut retry_policy = serde_json::Map::new();
6675 if let Some(max_attempts) = policy.max_attempts {
6676 if max_attempts == 0 {
6677 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
6678 "retry_policy.max_attempts must be at least 1".to_string(),
6679 )));
6680 }
6681 retry_policy.insert("max_attempts".to_string(), json!(max_attempts));
6682 }
6683 if !policy.backoff_seconds.is_empty() {
6684 retry_policy
6685 .insert("backoff_seconds".to_string(), json!(policy.backoff_seconds));
6686 }
6687 if !policy.non_retryable_error_types.is_empty() {
6688 retry_policy.insert(
6689 "non_retryable_error_types".to_string(),
6690 json!(policy.non_retryable_error_types),
6691 );
6692 }
6693 if retry_policy.is_empty() {
6694 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
6695 "retry_policy must configure at least one field".to_string(),
6696 )));
6697 }
6698 object.insert("retry_policy".to_string(), Value::Object(retry_policy));
6699 }
6700 if let Some(seconds) = self.options.execution_timeout_seconds {
6701 object.insert("execution_timeout_seconds".to_string(), json!(seconds));
6702 }
6703 if let Some(seconds) = self.options.run_timeout_seconds {
6704 object.insert("run_timeout_seconds".to_string(), json!(seconds));
6705 }
6706 state.commands.push(command);
6707 self.scheduled = true;
6708 }
6709
6710 Poll::Pending
6711 }
6712}
6713
6714impl Future for ChildWorkflowCall {
6715 type Output = Result<ChildWorkflowResult>;
6716
6717 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6718 match self.poll_avro_value(cx) {
6719 Poll::Ready(Ok(result)) => match result.result.into_json() {
6720 Ok(projected) => Poll::Ready(Ok(ChildWorkflowResult {
6721 parent: result.parent,
6722 child: result.child,
6723 child_workflow_type: result.child_workflow_type,
6724 result: projected,
6725 })),
6726 Err(error) => Poll::Ready(Err(error)),
6727 },
6728 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
6729 Poll::Pending => Poll::Pending,
6730 }
6731 }
6732}
6733
6734fn command_mismatch(recorded: &RecordedCommand, actual: impl Into<String>) -> Error {
6735 Error::NonDeterministicReplay(ReplayFailure::new(
6736 "recorded_command_mismatch",
6737 Some(recorded.sequence()),
6738 Some(recorded.shape().to_string()),
6739 Some(actual.into()),
6740 "current workflow command does not match the recorded durable command sequence",
6741 ))
6742}
6743
6744pub struct SignalCall {
6745 ctx: WorkflowContext,
6746 signal_name: String,
6747 opened_wait: bool,
6748 matched_pending: bool,
6749}
6750
6751impl SignalCall {
6752 fn poll_avro_value(
6753 mut self: Pin<&mut Self>,
6754 _cx: &mut TaskContext<'_>,
6755 ) -> Poll<Result<Vec<AvroValue>>> {
6756 if self.matched_pending {
6757 return Poll::Pending;
6758 }
6759
6760 let ctx = self.ctx.clone();
6761 let mut state = match ctx.state.lock() {
6762 Ok(state) => state,
6763 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6764 };
6765
6766 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6767 match recorded {
6768 RecordedCommand::SignalWait {
6769 sequence,
6770 signal_name,
6771 value,
6772 } => {
6773 if signal_name != self.signal_name {
6774 return Poll::Ready(Err(Error::NonDeterministicReplay(
6775 ReplayFailure::new(
6776 "recorded_command_detail_mismatch",
6777 Some(sequence),
6778 Some(format!("signal wait:{signal_name}")),
6779 Some(format!("signal wait:{}", self.signal_name)),
6780 "recorded signal name differs from the current workflow command",
6781 ),
6782 )));
6783 }
6784
6785 state.command_cursor += 1;
6786 if let Some(value) = value {
6787 return Poll::Ready(Ok(value));
6788 }
6789 if state
6790 .resume_signal
6791 .as_ref()
6792 .is_some_and(|signal| signal.signal_name == self.signal_name)
6793 {
6794 let signal = state
6795 .resume_signal
6796 .take()
6797 .expect("matching resume signal is present");
6798 return Poll::Ready(Ok(signal.arguments));
6799 }
6800
6801 state.matched_recorded_pending = true;
6802 self.opened_wait = true;
6803 self.matched_pending = true;
6804 return Poll::Pending;
6805 }
6806 other => {
6807 return Poll::Ready(Err(command_mismatch(
6808 &other,
6809 format!("signal wait:{}", self.signal_name),
6810 )));
6811 }
6812 }
6813 }
6814
6815 if state
6816 .resume_signal
6817 .as_ref()
6818 .is_some_and(|signal| signal.signal_name == self.signal_name)
6819 {
6820 let signal = state
6821 .resume_signal
6822 .take()
6823 .expect("matching resume signal is present");
6824 return Poll::Ready(Ok(signal.arguments));
6825 }
6826
6827 if !self.opened_wait {
6828 state.commands.push(json!({
6829 "type": "open_signal_wait",
6830 "signal_name": self.signal_name
6831 }));
6832 self.opened_wait = true;
6833 }
6834
6835 Poll::Pending
6836 }
6837}
6838
6839impl Future for SignalCall {
6840 type Output = Result<Vec<Value>>;
6841
6842 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6843 match self.poll_avro_value(cx) {
6844 Poll::Ready(Ok(values)) => Poll::Ready(
6845 values
6846 .into_iter()
6847 .map(AvroValue::into_json)
6848 .collect::<Result<Vec<_>>>(),
6849 ),
6850 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
6851 Poll::Pending => Poll::Pending,
6852 }
6853 }
6854}
6855
6856#[derive(Clone, Debug)]
6857pub struct ActivityContext {
6858 client: Client,
6859 pub task_id: String,
6860 pub activity_attempt_id: String,
6861 pub lease_owner: String,
6862 pub activity_type: String,
6863 pub attempt_number: u64,
6864 pub task_queue: String,
6865 pub worker_id: String,
6866}
6867
6868impl ActivityContext {
6869 pub async fn heartbeat<T: Serialize>(&self, details: T) -> Result<ActivityHeartbeatResponse> {
6870 self.client
6871 .heartbeat_activity_task(
6872 &self.task_id,
6873 &self.activity_attempt_id,
6874 &self.lease_owner,
6875 details,
6876 )
6877 .await
6878 }
6879}
6880
6881fn decode_task_avro_arguments(value: Option<&Value>, codec: &str) -> Result<AvroValue> {
6882 match value {
6883 Some(value) => Ok(normalize_avro_arguments(decode_wire_avro_value(
6884 value, codec,
6885 )?)),
6886 None => Ok(AvroValue::Array(Vec::new())),
6887 }
6888}
6889
6890fn decode_resume_signal(task: &WorkflowTask) -> Result<Option<ResumeSignal>> {
6891 let Some(signal_name) = task
6892 .signal_name
6893 .as_deref()
6894 .filter(|value| !value.is_empty())
6895 else {
6896 return Ok(None);
6897 };
6898 let Some(arguments) = task.signal_arguments.as_ref() else {
6899 return Ok(None);
6900 };
6901
6902 let decoded = normalize_avro_arguments(decode_wire_avro_value(arguments, &task.payload_codec)?);
6903 let AvroValue::Array(arguments) = decoded else {
6904 unreachable!("normalize_avro_arguments always returns an array");
6905 };
6906
6907 Ok(Some(ResumeSignal {
6908 signal_name: signal_name.to_string(),
6909 arguments,
6910 }))
6911}
6912
6913fn recorded_commands(
6914 events: &[HistoryEvent],
6915 fallback_codec: &str,
6916 parent: WorkflowIdentity,
6917) -> Result<Vec<RecordedCommand>> {
6918 let mut events_by_sequence: BTreeMap<u64, Vec<&HistoryEvent>> = BTreeMap::new();
6919 let mut last_new_sequence = None;
6920
6921 for event in events {
6922 let is_activity = matches!(
6923 event.event_type.as_str(),
6924 "ActivityScheduled"
6925 | "ActivityStarted"
6926 | "ActivityHeartbeatRecorded"
6927 | "ActivityRetryScheduled"
6928 | "ActivityCompleted"
6929 | "ActivityFailed"
6930 | "ActivityCancelled"
6931 | "ActivityTimedOut"
6932 );
6933 let is_workflow_timer = matches!(
6934 event.event_type.as_str(),
6935 "TimerScheduled" | "TimerCancelled" | "TimerFired"
6936 ) && !is_internal_timer_event(event);
6937 let is_child_workflow = matches!(
6938 event.event_type.as_str(),
6939 "ChildWorkflowScheduled"
6940 | "ChildRunCompleted"
6941 | "ChildRunFailed"
6942 | "ChildRunCancelled"
6943 | "ChildRunTerminated"
6944 );
6945 let is_signal_wait = is_recorded_signal_wait_event(event);
6946 let is_side_effect = event.event_type == "SideEffectRecorded";
6947 let is_version_marker = event.event_type == "VersionMarkerRecorded";
6948 if !is_activity
6949 && !is_workflow_timer
6950 && !is_child_workflow
6951 && !is_signal_wait
6952 && !is_side_effect
6953 && !is_version_marker
6954 {
6955 continue;
6956 }
6957
6958 let sequence = durable_event_sequence(event).ok_or_else(|| {
6959 Error::NonDeterministicReplay(ReplayFailure::new(
6960 "durable_command_sequence_missing",
6961 None,
6962 Some("positive workflow sequence".to_string()),
6963 Some(event.event_type.clone()),
6964 "durable command history event has no workflow sequence",
6965 ))
6966 })?;
6967 if sequence == 0 {
6968 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
6969 "durable_command_sequence_invalid",
6970 Some(sequence),
6971 Some("positive workflow sequence".to_string()),
6972 Some(sequence.to_string()),
6973 "durable command history uses an invalid workflow sequence",
6974 )));
6975 }
6976 if !events_by_sequence.contains_key(&sequence) {
6977 if let Some(previous) = last_new_sequence {
6978 if sequence < previous {
6979 return Err(invalid_recorded_history(
6980 "durable_command_sequence_mismatch",
6981 sequence,
6982 &format!("workflow sequence greater than {previous}"),
6983 &sequence.to_string(),
6984 "durable commands are not strictly ordered by their recorded workflow sequence",
6985 ));
6986 }
6987 }
6988 last_new_sequence = Some(sequence);
6989 }
6990 events_by_sequence.entry(sequence).or_default().push(event);
6991 }
6992
6993 let commands: Vec<RecordedCommand> = events_by_sequence
6994 .into_iter()
6995 .map(|(sequence, sequence_events)| {
6996 let activity_events: Vec<_> = sequence_events
6997 .iter()
6998 .copied()
6999 .filter(|event| event.event_type.starts_with("Activity"))
7000 .collect();
7001 let timer_events: Vec<_> = sequence_events
7002 .iter()
7003 .copied()
7004 .filter(|event| event.event_type.starts_with("Timer"))
7005 .collect();
7006 let child_events: Vec<_> = sequence_events
7007 .iter()
7008 .copied()
7009 .filter(|event| {
7010 event.event_type == "ChildWorkflowScheduled"
7011 || event.event_type.starts_with("ChildRun")
7012 })
7013 .collect();
7014 let signal_wait_events: Vec<_> = sequence_events
7015 .iter()
7016 .copied()
7017 .filter(|event| is_recorded_signal_wait_event(event))
7018 .collect();
7019 let side_effect_events: Vec<_> = sequence_events
7020 .iter()
7021 .copied()
7022 .filter(|event| event.event_type == "SideEffectRecorded")
7023 .collect();
7024 let version_marker_events: Vec<_> = sequence_events
7025 .iter()
7026 .copied()
7027 .filter(|event| event.event_type == "VersionMarkerRecorded")
7028 .collect();
7029
7030 let command_kind_count = usize::from(!activity_events.is_empty())
7031 + usize::from(!timer_events.is_empty())
7032 + usize::from(!child_events.is_empty())
7033 + usize::from(!signal_wait_events.is_empty())
7034 + usize::from(!side_effect_events.is_empty())
7035 + usize::from(!version_marker_events.is_empty());
7036 if command_kind_count > 1 {
7037 let actual = [
7038 (!activity_events.is_empty()).then_some("activity"),
7039 (!timer_events.is_empty()).then_some("timer"),
7040 (!child_events.is_empty()).then_some("child workflow"),
7041 (!signal_wait_events.is_empty()).then_some("signal wait"),
7042 (!side_effect_events.is_empty()).then_some("side effect"),
7043 (!version_marker_events.is_empty()).then_some("version marker"),
7044 ]
7045 .into_iter()
7046 .flatten()
7047 .collect::<Vec<_>>()
7048 .join(" and ");
7049 return Err(invalid_recorded_history(
7050 "durable_command_sequence_collision",
7051 sequence,
7052 "one durable command kind",
7053 &actual,
7054 "one workflow sequence records more than one durable command kind",
7055 ));
7056 }
7057
7058 if !activity_events.is_empty() {
7059 let scheduled_count = activity_events
7060 .iter()
7061 .filter(|event| event.event_type == "ActivityScheduled")
7062 .count();
7063 if scheduled_count > 1 {
7064 return Err(invalid_recorded_history(
7065 "duplicate_activity_schedule",
7066 sequence,
7067 "at most one ActivityScheduled event",
7068 "multiple ActivityScheduled events",
7069 "activity history schedules more than one command at one workflow sequence",
7070 ));
7071 }
7072 let activity_type = activity_events.iter().find_map(|event| {
7073 event
7074 .payload
7075 .get("activity_type")
7076 .or_else(|| event.payload.get("activity_name"))
7077 .and_then(Value::as_str)
7078 .map(str::to_string)
7079 });
7080 if activity_events.iter().filter_map(|event| {
7081 event
7082 .payload
7083 .get("activity_type")
7084 .or_else(|| event.payload.get("activity_name"))
7085 .and_then(Value::as_str)
7086 }).any(|candidate| Some(candidate) != activity_type.as_deref()) {
7087 return Err(invalid_recorded_history(
7088 "activity_identity_mismatch",
7089 sequence,
7090 activity_type.as_deref().unwrap_or("one activity identity"),
7091 "conflicting activity identities",
7092 "activity lifecycle events at one workflow sequence disagree on identity",
7093 ));
7094 }
7095 let terminal: Vec<_> = activity_events
7096 .iter()
7097 .copied()
7098 .filter(|event| {
7099 matches!(
7100 event.event_type.as_str(),
7101 "ActivityCompleted"
7102 | "ActivityFailed"
7103 | "ActivityCancelled"
7104 | "ActivityTimedOut"
7105 )
7106 })
7107 .collect();
7108 if terminal.len() > 1 {
7109 return Err(invalid_recorded_history(
7110 "duplicate_activity_terminal_event",
7111 sequence,
7112 "at most one terminal activity event",
7113 "multiple terminal activity events",
7114 "activity history settles one command more than once",
7115 ));
7116 }
7117 let outcome = terminal
7118 .first()
7119 .map(|event| activity_outcome(event, fallback_codec, activity_type.clone()))
7120 .transpose()?;
7121 let options = activity_events
7122 .iter()
7123 .find(|event| event.event_type == "ActivityScheduled")
7124 .and_then(|event| event.payload.get("activity"))
7125 .and_then(Value::as_object)
7126 .map(|activity| RecordedActivityOptions {
7127 task_queue: recorded_optional_string(activity, "queue"),
7128 execution_mode: recorded_optional_string(activity, "execution_mode"),
7129 retry_policy: recorded_activity_retry_snapshot(
7130 activity.get("retry_policy"),
7131 ),
7132 });
7133 return Ok(RecordedCommand::Activity {
7134 sequence,
7135 activity_type,
7136 options,
7137 outcome,
7138 });
7139 }
7140
7141 if !child_events.is_empty() {
7142 let scheduled: Vec<_> = child_events
7143 .iter()
7144 .copied()
7145 .filter(|event| event.event_type == "ChildWorkflowScheduled")
7146 .collect();
7147 if scheduled.len() != 1 {
7148 return Err(invalid_recorded_history(
7149 "child_workflow_schedule_missing_or_duplicate",
7150 sequence,
7151 "one ChildWorkflowScheduled event",
7152 &format!("{} ChildWorkflowScheduled events", scheduled.len()),
7153 "child workflow replay requires exactly one recorded schedule event",
7154 ));
7155 }
7156 let workflow_type = child_events.iter().find_map(|event| {
7157 event
7158 .payload
7159 .get("child_workflow_type")
7160 .or_else(|| event.payload.get("workflow_type"))
7161 .and_then(Value::as_str)
7162 .filter(|value| !value.is_empty())
7163 .map(str::to_string)
7164 });
7165 if child_events
7166 .iter()
7167 .filter_map(|event| {
7168 event
7169 .payload
7170 .get("child_workflow_type")
7171 .or_else(|| event.payload.get("workflow_type"))
7172 .and_then(Value::as_str)
7173 })
7174 .any(|candidate| Some(candidate) != workflow_type.as_deref())
7175 {
7176 return Err(invalid_recorded_history(
7177 "child_workflow_identity_mismatch",
7178 sequence,
7179 workflow_type
7180 .as_deref()
7181 .unwrap_or("one child workflow type"),
7182 "conflicting child workflow types",
7183 "child workflow lifecycle events at one sequence disagree on type",
7184 ));
7185 }
7186 let mut outcomes = child_workflow_outcomes(
7187 &child_events.iter().map(|event| (*event).clone()).collect::<Vec<_>>(),
7188 fallback_codec,
7189 parent.clone(),
7190 )?;
7191 if outcomes.len() > 1 {
7192 return Err(invalid_recorded_history(
7193 "duplicate_child_workflow_terminal_event",
7194 sequence,
7195 "at most one terminal child event",
7196 "multiple terminal child events",
7197 "child workflow history settles one command more than once",
7198 ));
7199 }
7200 return Ok(RecordedCommand::ChildWorkflow {
7201 sequence,
7202 workflow_type,
7203 outcome: outcomes.pop(),
7204 });
7205 }
7206
7207 if !signal_wait_events.is_empty() {
7208 let opened: Vec<_> = signal_wait_events
7209 .iter()
7210 .copied()
7211 .filter(|event| event.event_type == "SignalWaitOpened")
7212 .collect();
7213 if opened.len() != 1 {
7214 return Err(invalid_recorded_history(
7215 "signal_wait_open_missing_or_duplicate",
7216 sequence,
7217 "one SignalWaitOpened event",
7218 &format!("{} SignalWaitOpened events", opened.len()),
7219 "signal replay requires exactly one canonical wait-open event",
7220 ));
7221 }
7222
7223 let applied: Vec<_> = signal_wait_events
7224 .iter()
7225 .copied()
7226 .filter(|event| event.event_type == "SignalApplied")
7227 .collect();
7228 if applied.len() > 1 {
7229 return Err(invalid_recorded_history(
7230 "duplicate_signal_wait_apply",
7231 sequence,
7232 "at most one SignalApplied event",
7233 "multiple SignalApplied events",
7234 "signal history applies one durable wait more than once",
7235 ));
7236 }
7237
7238 let signal_names = signal_wait_events
7239 .iter()
7240 .map(|event| required_signal_wait_name(event, sequence))
7241 .collect::<Result<Vec<_>>>()?;
7242 let signal_name = signal_names
7243 .first()
7244 .expect("signal wait events are not empty")
7245 .clone();
7246 if signal_names.iter().any(|candidate| candidate != &signal_name) {
7247 return Err(invalid_recorded_history(
7248 "signal_wait_identity_mismatch",
7249 sequence,
7250 &signal_name,
7251 "conflicting signal names",
7252 "signal wait lifecycle events at one workflow sequence disagree on identity",
7253 ));
7254 }
7255 let value = applied
7256 .first()
7257 .map(|event| decode_signal_event_arguments(event, fallback_codec))
7258 .transpose()?;
7259 return Ok(RecordedCommand::SignalWait {
7260 sequence,
7261 signal_name,
7262 value,
7263 });
7264 }
7265
7266 if !side_effect_events.is_empty() {
7267 if side_effect_events.len() != 1 {
7268 return Err(invalid_recorded_history(
7269 "duplicate_side_effect_record",
7270 sequence,
7271 "one SideEffectRecorded event",
7272 &format!("{} SideEffectRecorded events", side_effect_events.len()),
7273 "side-effect history records one workflow command more than once",
7274 ));
7275 }
7276 let event = side_effect_events[0];
7277 let result = event.payload.get("result").ok_or_else(|| {
7278 invalid_recorded_history(
7279 "side_effect_result_missing",
7280 sequence,
7281 "recorded result payload",
7282 "missing result",
7283 "side-effect history is missing its recorded value",
7284 )
7285 })?;
7286 let has_published_envelope = result.as_str().is_some()
7287 || result.as_object().is_some_and(|envelope| {
7288 envelope.get("codec").and_then(Value::as_str).is_some()
7289 && envelope.get("blob").and_then(Value::as_str).is_some()
7290 });
7291 if !has_published_envelope {
7292 return Err(invalid_recorded_history(
7293 "side_effect_payload_malformed",
7294 sequence,
7295 "payload blob or {codec, blob} envelope",
7296 &result.to_string(),
7297 "side-effect history result does not use a published payload envelope",
7298 ));
7299 }
7300 let codec = event
7301 .payload
7302 .get("payload_codec")
7303 .and_then(Value::as_str)
7304 .unwrap_or(fallback_codec);
7305 let value = decode_wire_avro_value(result, codec).map_err(|error| {
7306 invalid_recorded_history(
7307 "side_effect_payload_incompatible",
7308 sequence,
7309 &format!("valid {codec} payload envelope"),
7310 &error.to_string(),
7311 "side-effect history payload cannot be decoded with its recorded codec",
7312 )
7313 })?;
7314 return Ok(RecordedCommand::SideEffect { sequence, value });
7315 }
7316
7317 if !version_marker_events.is_empty() {
7318 if version_marker_events.len() != 1 {
7319 return Err(invalid_recorded_history(
7320 "duplicate_version_marker_record",
7321 sequence,
7322 "one VersionMarkerRecorded event",
7323 &format!("{} VersionMarkerRecorded events", version_marker_events.len()),
7324 "version-marker history records one workflow command more than once",
7325 ));
7326 }
7327 let payload = &version_marker_events[0].payload;
7328 let change_id = payload
7329 .get("change_id")
7330 .and_then(Value::as_str)
7331 .filter(|value| !value.is_empty())
7332 .map(str::to_string)
7333 .ok_or_else(|| {
7334 invalid_recorded_history(
7335 "version_marker_field_missing",
7336 sequence,
7337 "non-empty change_id",
7338 "missing or invalid change_id",
7339 "version-marker history is missing its stable change ID",
7340 )
7341 })?;
7342 let version = required_version_i32(payload, "version", sequence)?;
7343 let min_supported = required_version_i32(payload, "min_supported", sequence)?;
7344 let max_supported = required_version_i32(payload, "max_supported", sequence)?;
7345 if min_supported > max_supported || version < min_supported || version > max_supported {
7346 return Err(invalid_recorded_history(
7347 "version_marker_history_range_invalid",
7348 sequence,
7349 "min_supported <= version <= max_supported",
7350 &format!("{min_supported} <= {version} <= {max_supported}"),
7351 "recorded version marker contains an internally incompatible range",
7352 ));
7353 }
7354 return Ok(RecordedCommand::VersionMarker {
7355 sequence,
7356 change_id,
7357 version,
7358 });
7359 }
7360
7361 let scheduled: Vec<_> = timer_events
7362 .iter()
7363 .copied()
7364 .filter(|event| event.event_type == "TimerScheduled")
7365 .collect();
7366 let fired: Vec<_> = timer_events
7367 .iter()
7368 .copied()
7369 .filter(|event| event.event_type == "TimerFired")
7370 .collect();
7371 if scheduled.len() != 1 {
7372 return Err(invalid_recorded_history(
7373 "timer_schedule_missing_or_duplicate",
7374 sequence,
7375 "one TimerScheduled event",
7376 &format!("{} TimerScheduled events", scheduled.len()),
7377 "timer replay requires exactly one recorded schedule event",
7378 ));
7379 }
7380 if fired.len() > 1 {
7381 return Err(invalid_recorded_history(
7382 "duplicate_timer_fire",
7383 sequence,
7384 "at most one TimerFired event",
7385 "multiple TimerFired events",
7386 "timer history contains more than one fire event for a workflow sequence",
7387 ));
7388 }
7389
7390 let scheduled = scheduled[0];
7391 let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
7392 let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
7393 if let Some(fired) = fired.first() {
7394 let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
7395 if fired_timer_id != timer_id {
7396 return Err(invalid_recorded_history(
7397 "timer_identity_mismatch",
7398 sequence,
7399 &timer_id,
7400 &fired_timer_id,
7401 "TimerFired does not correspond to the recorded TimerScheduled event",
7402 ));
7403 }
7404 let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
7405 if fired_delay != delay_seconds {
7406 return Err(invalid_recorded_history(
7407 "timer_history_delay_mismatch",
7408 sequence,
7409 &delay_seconds.to_string(),
7410 &fired_delay.to_string(),
7411 "TimerScheduled and TimerFired record different delays",
7412 ));
7413 }
7414 }
7415
7416 Ok(RecordedCommand::Timer {
7417 sequence,
7418 delay_seconds,
7419 fired: !fired.is_empty(),
7420 })
7421 })
7422 .collect::<Result<_>>()?;
7423
7424 let mut marker_sequences = HashMap::new();
7425 for command in &commands {
7426 if let RecordedCommand::VersionMarker {
7427 sequence,
7428 change_id,
7429 ..
7430 } = command
7431 {
7432 if let Some(first_sequence) = marker_sequences.insert(change_id.clone(), *sequence) {
7433 return Err(invalid_recorded_history(
7434 "duplicate_version_marker",
7435 *sequence,
7436 &format!("one marker for change ID {change_id:?}"),
7437 &format!("markers at sequences {first_sequence} and {sequence}"),
7438 "workflow history contains duplicate markers for one stable change ID",
7439 ));
7440 }
7441 }
7442 }
7443
7444 Ok(commands)
7445}
7446
7447fn required_version_i32(payload: &Value, field: &str, sequence: u64) -> Result<i32> {
7448 payload
7449 .get(field)
7450 .and_then(Value::as_i64)
7451 .and_then(|value| i32::try_from(value).ok())
7452 .ok_or_else(|| {
7453 invalid_recorded_history(
7454 "version_marker_field_missing",
7455 sequence,
7456 &format!("integer {field}"),
7457 "missing or out-of-range integer",
7458 "version-marker history is missing a required integer field",
7459 )
7460 })
7461}
7462
7463fn durable_event_sequence(event: &HistoryEvent) -> Option<u64> {
7464 event
7465 .payload
7466 .get("sequence")
7467 .or_else(|| event.payload.get("workflow_sequence"))
7468 .or_else(|| event.raw.get("sequence"))
7469 .or_else(|| event.raw.get("workflow_sequence"))
7470 .and_then(value_as_u64)
7471}
7472
7473fn is_internal_timer_event(event: &HistoryEvent) -> bool {
7474 matches!(
7475 event
7476 .payload
7477 .get("timer_kind")
7478 .or_else(|| event.raw.get("timer_kind"))
7479 .and_then(Value::as_str),
7480 Some("condition_timeout" | "signal_timeout")
7481 )
7482}
7483
7484fn required_signal_wait_name(event: &HistoryEvent, sequence: u64) -> Result<String> {
7485 event
7486 .payload
7487 .get("signal_name")
7488 .or_else(|| event.raw.get("signal_name"))
7489 .and_then(Value::as_str)
7490 .filter(|value| !value.is_empty())
7491 .map(str::to_string)
7492 .ok_or_else(|| {
7493 invalid_recorded_history(
7494 "signal_wait_name_missing",
7495 sequence,
7496 "non-empty signal_name",
7497 &event.event_type,
7498 "canonical signal-wait history is missing its signal identity",
7499 )
7500 })
7501}
7502
7503fn is_recorded_signal_wait_event(event: &HistoryEvent) -> bool {
7504 matches!(
7505 event.event_type.as_str(),
7506 "SignalWaitOpened" | "SignalApplied"
7507 )
7508}
7509
7510fn required_history_string(event: &HistoryEvent, field: &str, sequence: u64) -> Result<String> {
7511 event
7512 .payload
7513 .get(field)
7514 .and_then(Value::as_str)
7515 .filter(|value| !value.is_empty())
7516 .map(str::to_string)
7517 .ok_or_else(|| {
7518 invalid_recorded_history(
7519 "timer_history_field_missing",
7520 sequence,
7521 field,
7522 &event.event_type,
7523 "timer history is missing a required identity field",
7524 )
7525 })
7526}
7527
7528fn required_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<u64> {
7529 event
7530 .payload
7531 .get(field)
7532 .and_then(value_as_u64)
7533 .ok_or_else(|| {
7534 invalid_recorded_history(
7535 "timer_history_field_missing",
7536 sequence,
7537 field,
7538 &event.event_type,
7539 "timer history is missing a required numeric field",
7540 )
7541 })
7542}
7543
7544fn invalid_recorded_history(
7545 reason: &str,
7546 sequence: u64,
7547 expected: &str,
7548 actual: &str,
7549 message: &str,
7550) -> Error {
7551 Error::NonDeterministicReplay(ReplayFailure::new(
7552 reason,
7553 Some(sequence),
7554 Some(expected.to_string()),
7555 Some(actual.to_string()),
7556 message,
7557 ))
7558}
7559
7560type ActivityOutcome = std::result::Result<AvroValue, ActivityFailure>;
7561
7562fn activity_outcome(
7563 event: &HistoryEvent,
7564 fallback_codec: &str,
7565 recorded_activity_type: Option<String>,
7566) -> Result<ActivityOutcome> {
7567 if event.event_type == "ActivityCompleted" {
7568 let codec = event
7569 .payload
7570 .get("payload_codec")
7571 .and_then(Value::as_str)
7572 .unwrap_or(fallback_codec);
7573 return Ok(Ok(decode_wire_avro_value(
7574 event.payload.get("result").unwrap_or(&Value::Null),
7575 codec,
7576 )?));
7577 }
7578
7579 let payload = &event.payload;
7580 let (kind, fallback_reason, fallback_message) = match event.event_type.as_str() {
7581 "ActivityFailed" => (ActivityFailureKind::Failed, "activity", "activity failed"),
7582 "ActivityCancelled" => (
7583 ActivityFailureKind::Cancelled,
7584 "cancelled",
7585 "activity was cancelled",
7586 ),
7587 "ActivityTimedOut" => (
7588 ActivityFailureKind::TimedOut,
7589 "timeout",
7590 "activity timed out",
7591 ),
7592 _ => unreachable!("activity_outcome is called only for terminal activity events"),
7593 };
7594 let exception = payload
7595 .get("exception")
7596 .filter(|value| !value.is_null())
7597 .cloned();
7598 let failure_category = payload_string(payload, "failure_category");
7599 let timeout_kind = payload_string(payload, "timeout_kind");
7600 let reason = payload_string(payload, "reason").unwrap_or_else(|| match kind {
7601 ActivityFailureKind::Failed => failure_category
7602 .clone()
7603 .unwrap_or_else(|| fallback_reason.to_string()),
7604 ActivityFailureKind::Cancelled => fallback_reason.to_string(),
7605 ActivityFailureKind::TimedOut => timeout_kind
7606 .clone()
7607 .unwrap_or_else(|| fallback_reason.to_string()),
7608 });
7609 let message = payload_string(payload, "message")
7610 .or_else(|| {
7611 exception
7612 .as_ref()
7613 .and_then(|value| payload_string(value, "message"))
7614 })
7615 .unwrap_or_else(|| fallback_message.to_string());
7616
7617 Ok(Err(ActivityFailure {
7618 kind,
7619 reason,
7620 message,
7621 activity_execution_id: payload_string(payload, "activity_execution_id"),
7622 activity_attempt_id: payload_string(payload, "activity_attempt_id"),
7623 activity_type: payload_string(payload, "activity_type")
7624 .or_else(|| payload_string(payload, "activity_name"))
7625 .or(recorded_activity_type),
7626 activity_class: payload_string(payload, "activity_class"),
7627 attempt_number: payload.get("attempt_number").and_then(value_as_u64),
7628 failure_id: payload_string(payload, "failure_id"),
7629 failure_category,
7630 timeout_kind,
7631 non_retryable: payload
7632 .get("non_retryable")
7633 .and_then(Value::as_bool)
7634 .unwrap_or(false),
7635 exception_type: payload_string(payload, "exception_type").or_else(|| {
7636 exception
7637 .as_ref()
7638 .and_then(|value| payload_string(value, "type"))
7639 }),
7640 exception_class: payload_string(payload, "exception_class").or_else(|| {
7641 exception
7642 .as_ref()
7643 .and_then(|value| payload_string(value, "class"))
7644 }),
7645 code: payload
7646 .get("code")
7647 .filter(|value| !value.is_null())
7648 .cloned(),
7649 exception,
7650 }))
7651}
7652
7653type ChildWorkflowOutcome = std::result::Result<ChildWorkflowAvroResult, ChildWorkflowFailure>;
7654
7655fn child_workflow_outcomes(
7656 events: &[HistoryEvent],
7657 fallback_codec: &str,
7658 parent: WorkflowIdentity,
7659) -> Result<Vec<ChildWorkflowOutcome>> {
7660 let mut outcomes = Vec::new();
7661
7662 for event in events {
7663 let kind = match event.event_type.as_str() {
7664 "ChildRunCompleted" => None,
7665 "ChildRunFailed" => Some((
7666 ChildWorkflowFailureKind::Failed,
7667 "child_workflow",
7668 "child workflow failed",
7669 )),
7670 "ChildRunCancelled" => Some((
7671 ChildWorkflowFailureKind::Cancelled,
7672 "cancelled",
7673 "child workflow was cancelled",
7674 )),
7675 "ChildRunTerminated" => Some((
7676 ChildWorkflowFailureKind::Terminated,
7677 "terminated",
7678 "child workflow was terminated",
7679 )),
7680 _ => continue,
7681 };
7682 let payload = &event.payload;
7683 let child_workflow_id = payload_string(payload, "child_workflow_instance_id");
7684 let child_workflow_run_id = payload_string(payload, "child_workflow_run_id");
7685 let child_workflow_type = payload_string(payload, "child_workflow_type");
7686
7687 if let Some((kind, reason, fallback_message)) = kind {
7688 let exception = payload
7689 .get("exception")
7690 .filter(|value| !value.is_null())
7691 .cloned();
7692 let message = payload_string(payload, "message")
7693 .or_else(|| {
7694 exception
7695 .as_ref()
7696 .and_then(|value| payload_string(value, "message"))
7697 })
7698 .unwrap_or_else(|| fallback_message.to_string());
7699 let exception_type = payload_string(payload, "exception_type").or_else(|| {
7700 exception
7701 .as_ref()
7702 .and_then(|value| payload_string(value, "type"))
7703 });
7704 let exception_class = payload_string(payload, "exception_class").or_else(|| {
7705 exception
7706 .as_ref()
7707 .and_then(|value| payload_string(value, "class"))
7708 });
7709 outcomes.push(Err(ChildWorkflowFailure {
7710 kind,
7711 reason: reason.to_string(),
7712 message,
7713 parent_workflow_id: parent.workflow_id.clone(),
7714 parent_workflow_run_id: parent.run_id.clone(),
7715 child_workflow_id,
7716 child_workflow_run_id,
7717 child_workflow_type,
7718 failure_id: payload_string(payload, "failure_id"),
7719 failure_category: payload_string(payload, "failure_category"),
7720 exception_type,
7721 exception_class,
7722 non_retryable: payload
7723 .get("non_retryable")
7724 .and_then(Value::as_bool)
7725 .unwrap_or(false),
7726 code: payload
7727 .get("code")
7728 .filter(|value| !value.is_null())
7729 .cloned(),
7730 exception,
7731 }));
7732 continue;
7733 }
7734
7735 let codec = payload
7736 .get("payload_codec")
7737 .and_then(Value::as_str)
7738 .unwrap_or(fallback_codec);
7739 let result = payload
7740 .get("result")
7741 .or_else(|| payload.get("output"))
7742 .unwrap_or(&Value::Null);
7743 outcomes.push(Ok(ChildWorkflowAvroResult {
7744 parent: parent.clone(),
7745 child: WorkflowIdentity {
7746 workflow_id: child_workflow_id,
7747 run_id: child_workflow_run_id,
7748 },
7749 child_workflow_type,
7750 result: decode_wire_avro_value(result, codec)?,
7751 }));
7752 }
7753
7754 Ok(outcomes)
7755}
7756
7757fn payload_string(payload: &Value, key: &str) -> Option<String> {
7758 payload
7759 .get(key)
7760 .and_then(Value::as_str)
7761 .filter(|value| !value.is_empty())
7762 .map(str::to_string)
7763}
7764
7765fn workflow_failure_command(error: &Error) -> Value {
7766 let (exception_type, exception_class, properties) = match error {
7767 Error::ActivityFailed(failure) => (
7768 match failure.kind {
7769 ActivityFailureKind::Failed => "ActivityFailed",
7770 ActivityFailureKind::Cancelled => "ActivityCancelled",
7771 ActivityFailureKind::TimedOut => "ActivityTimedOut",
7772 },
7773 "durable_workflow::ActivityFailure",
7774 json!({
7775 "reason": failure.reason,
7776 "activity_execution_id": failure.activity_execution_id,
7777 "activity_attempt_id": failure.activity_attempt_id,
7778 "activity_type": failure.activity_type,
7779 "activity_class": failure.activity_class,
7780 "attempt_number": failure.attempt_number,
7781 "failure_id": failure.failure_id,
7782 "failure_category": failure.failure_category,
7783 "timeout_kind": failure.timeout_kind,
7784 "activity_non_retryable": failure.non_retryable,
7785 "activity_exception_type": failure.exception_type,
7786 "activity_exception_class": failure.exception_class,
7787 "activity_code": failure.code,
7788 "activity_exception": failure.exception,
7789 }),
7790 ),
7791 Error::ChildWorkflowFailed(failure) => (
7792 match failure.kind {
7793 ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
7794 ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
7795 ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
7796 },
7797 "durable_workflow::ChildWorkflowFailure",
7798 json!({
7799 "reason": failure.reason,
7800 "parent_workflow_id": failure.parent_workflow_id,
7801 "parent_workflow_run_id": failure.parent_workflow_run_id,
7802 "child_workflow_id": failure.child_workflow_id,
7803 "child_workflow_run_id": failure.child_workflow_run_id,
7804 "child_workflow_type": failure.child_workflow_type,
7805 "failure_id": failure.failure_id,
7806 "failure_category": failure.failure_category,
7807 "child_exception_type": failure.exception_type,
7808 "child_exception_class": failure.exception_class,
7809 "child_non_retryable": failure.non_retryable,
7810 "child_code": failure.code,
7811 "child_exception": failure.exception,
7812 }),
7813 ),
7814 Error::NonDeterministicReplay(_) => (
7815 "NonDeterministicReplay",
7816 "durable_workflow::Error",
7817 Value::Null,
7818 ),
7819 _ => ("RustWorkflowError", "durable_workflow::Error", Value::Null),
7820 };
7821 let non_retryable = match error {
7822 Error::ActivityFailed(failure) => failure.non_retryable,
7823 Error::ChildWorkflowFailed(failure) => failure.non_retryable,
7824 Error::NonDeterministicReplay(_) => true,
7825 _ => false,
7826 };
7827
7828 json!({
7829 "type": "fail_workflow",
7830 "message": error.to_string(),
7831 "exception_type": exception_type,
7832 "exception_class": exception_class,
7833 "non_retryable": non_retryable,
7834 "exception": {
7835 "type": exception_type,
7836 "class": exception_class,
7837 "message": error.to_string(),
7838 "properties": properties,
7839 }
7840 })
7841}
7842
7843fn workflow_task_integrity_error(error: &Error) -> bool {
7844 matches!(
7845 error,
7846 Error::NonDeterministicReplay(_) | Error::Protocol(_) | Error::WorkflowStatePoisoned
7847 )
7848}
7849
7850fn decode_signal_event_arguments(
7851 event: &HistoryEvent,
7852 fallback_codec: &str,
7853) -> Result<Vec<AvroValue>> {
7854 let codec = event
7855 .payload
7856 .get("payload_codec")
7857 .and_then(Value::as_str)
7858 .unwrap_or(fallback_codec);
7859 let raw = event
7860 .payload
7861 .get("value")
7862 .or_else(|| event.payload.get("input"))
7863 .or_else(|| event.payload.get("arguments"));
7864 let decoded = match raw.filter(|value| !value.is_null()) {
7865 Some(value) => decode_wire_avro_value(value, codec)?,
7866 None => AvroValue::Array(Vec::new()),
7867 };
7868 let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
7869 unreachable!("normalize_avro_arguments always returns an array");
7870 };
7871 Ok(arguments)
7872}
7873
7874fn hydrate_query_history_from_export(task: &mut QueryTask) -> Result<()> {
7875 let Some(export_events) = task
7876 .history_export
7877 .as_ref()
7878 .and_then(|export| export.get("history_events"))
7879 .and_then(Value::as_array)
7880 else {
7881 return Ok(());
7882 };
7883
7884 if export_events.len() > task.history_events.len() {
7885 task.history_events = serde_json::from_value(Value::Array(export_events.clone()))?;
7886 }
7887
7888 Ok(())
7889}
7890
7891fn enrich_query_history_from_export(task: &mut QueryTask) -> Result<()> {
7892 let Some(export) = task.history_export.as_ref() else {
7893 return Ok(());
7894 };
7895 let signals = export
7896 .get("signals")
7897 .and_then(Value::as_array)
7898 .cloned()
7899 .unwrap_or_default();
7900 let activities = export
7901 .get("activities")
7902 .and_then(Value::as_array)
7903 .cloned()
7904 .unwrap_or_default();
7905 let export_codec = export
7906 .get("payloads")
7907 .and_then(|payloads| payloads.get("codec"))
7908 .and_then(Value::as_str)
7909 .unwrap_or(&task.payload_codec)
7910 .to_string();
7911 let mut signal_name_offsets: HashMap<String, usize> = HashMap::new();
7912
7913 for event in &mut task.history_events {
7914 if event.event_type == "ActivityCompleted" {
7915 let sequence = event
7916 .payload
7917 .get("sequence")
7918 .or_else(|| event.payload.get("workflow_sequence"))
7919 .and_then(value_as_u64);
7920 let Some(activity) = sequence.and_then(|sequence| {
7921 activities.iter().find(|activity| {
7922 activity.get("sequence").and_then(value_as_u64) == Some(sequence)
7923 })
7924 }) else {
7925 continue;
7926 };
7927 let Some(payload) = event.payload.as_object_mut() else {
7928 continue;
7929 };
7930 if missing_payload(payload.get("result")) {
7931 if let Some(result) = activity
7932 .get("result")
7933 .filter(|value| !missing_payload(Some(value)))
7934 {
7935 payload.insert("result".to_string(), result.clone());
7936 }
7937 }
7938 for field in ["payload_codec", "activity_type"] {
7939 if payload
7940 .get(field)
7941 .and_then(Value::as_str)
7942 .unwrap_or_default()
7943 .is_empty()
7944 {
7945 if let Some(value) = activity.get(field) {
7946 payload.insert(field.to_string(), value.clone());
7947 }
7948 }
7949 }
7950 continue;
7951 }
7952
7953 if event.event_type != "SignalReceived" && event.event_type != "SignalApplied" {
7954 continue;
7955 }
7956 let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
7957 let command_id = event
7958 .payload
7959 .get("workflow_command_id")
7960 .or_else(|| event.raw.get("workflow_command_id"))
7961 .and_then(Value::as_str);
7962 let signal_name = event
7963 .payload
7964 .get("signal_name")
7965 .and_then(Value::as_str)
7966 .unwrap_or_default()
7967 .to_string();
7968 let matched = signals
7969 .iter()
7970 .find(|signal| {
7971 signal_id.is_some() && signal.get("id").and_then(Value::as_str) == signal_id
7972 })
7973 .or_else(|| {
7974 signals.iter().find(|signal| {
7975 command_id.is_some()
7976 && signal.get("command_id").and_then(Value::as_str) == command_id
7977 })
7978 })
7979 .or_else(|| {
7980 let offset = signal_name_offsets.entry(signal_name.clone()).or_default();
7981 let signal = signals
7982 .iter()
7983 .filter(|signal| {
7984 signal.get("name").and_then(Value::as_str) == Some(signal_name.as_str())
7985 })
7986 .nth(*offset);
7987 if signal.is_some() {
7988 *offset += 1;
7989 }
7990 signal
7991 });
7992 let Some(signal) = matched else {
7993 continue;
7994 };
7995 let signal_codec = signal
7996 .get("payload_codec")
7997 .and_then(Value::as_str)
7998 .unwrap_or(&export_codec);
7999 let Some(payload) = event.payload.as_object_mut() else {
8000 continue;
8001 };
8002 if missing_payload(payload.get("arguments")) {
8003 if let Some(arguments) = signal
8004 .get("arguments")
8005 .filter(|value| !missing_payload(Some(value)))
8006 {
8007 let envelope = match arguments {
8008 Value::String(blob) => json!({"codec": signal_codec, "blob": blob}),
8009 other => other.clone(),
8010 };
8011 payload.insert("arguments".to_string(), envelope);
8012 }
8013 }
8014 if payload
8015 .get("payload_codec")
8016 .and_then(Value::as_str)
8017 .unwrap_or_default()
8018 .is_empty()
8019 {
8020 payload.insert("payload_codec".to_string(), json!(signal_codec));
8021 }
8022 }
8023
8024 Ok(())
8025}
8026
8027fn missing_payload(value: Option<&Value>) -> bool {
8028 match value {
8029 None | Some(Value::Null) => true,
8030 Some(Value::String(value)) => value.is_empty(),
8031 Some(_) => false,
8032 }
8033}
8034
8035fn query_signal_events(task: &QueryTask) -> Result<Vec<QuerySignal>> {
8036 let export_signals = task
8037 .history_export
8038 .as_ref()
8039 .and_then(|export| export.get("signals"))
8040 .and_then(Value::as_array)
8041 .cloned()
8042 .unwrap_or_default();
8043 let export_codec = task
8044 .history_export
8045 .as_ref()
8046 .and_then(|export| export.get("payloads"))
8047 .and_then(|payloads| payloads.get("codec"))
8048 .and_then(Value::as_str)
8049 .unwrap_or(&task.payload_codec);
8050 let mut name_offsets: HashMap<String, usize> = HashMap::new();
8051 let mut signals = Vec::new();
8052
8053 for event in &task.history_events {
8054 if event.event_type != "SignalApplied" && event.event_type != "SignalReceived" {
8055 continue;
8056 }
8057
8058 let name = event
8059 .payload
8060 .get("signal_name")
8061 .and_then(Value::as_str)
8062 .unwrap_or_default();
8063 if name.is_empty() {
8064 continue;
8065 }
8066 let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
8067 let command_id = event
8068 .payload
8069 .get("workflow_command_id")
8070 .or_else(|| event.raw.get("workflow_command_id"))
8071 .and_then(Value::as_str);
8072 let matched_export = export_signals
8073 .iter()
8074 .find(|candidate| {
8075 signal_id.is_some() && candidate.get("id").and_then(Value::as_str) == signal_id
8076 })
8077 .or_else(|| {
8078 export_signals.iter().find(|candidate| {
8079 command_id.is_some()
8080 && candidate.get("command_id").and_then(Value::as_str) == command_id
8081 })
8082 })
8083 .or_else(|| {
8084 let offset = name_offsets.entry(name.to_string()).or_default();
8085 let candidate = export_signals
8086 .iter()
8087 .filter(|candidate| candidate.get("name").and_then(Value::as_str) == Some(name))
8088 .nth(*offset);
8089 if candidate.is_some() {
8090 *offset += 1;
8091 }
8092 candidate
8093 });
8094 let codec = event
8095 .payload
8096 .get("payload_codec")
8097 .and_then(Value::as_str)
8098 .or_else(|| {
8099 matched_export
8100 .and_then(|signal| signal.get("payload_codec"))
8101 .and_then(Value::as_str)
8102 })
8103 .unwrap_or(export_codec);
8104 let raw_arguments = event
8105 .payload
8106 .get("value")
8107 .or_else(|| event.payload.get("input"))
8108 .or_else(|| event.payload.get("arguments"))
8109 .filter(|value| !value.is_null())
8110 .or_else(|| matched_export.and_then(|signal| signal.get("arguments")));
8111 let (arguments, avro_arguments) = decode_query_signal_arguments(raw_arguments, codec)?;
8112 let workflow_sequence = event
8113 .payload
8114 .get("workflow_sequence")
8115 .and_then(value_as_u64)
8116 .or_else(|| {
8117 matched_export
8118 .and_then(|signal| signal.get("workflow_sequence"))
8119 .and_then(value_as_u64)
8120 });
8121
8122 signals.push(QuerySignal {
8123 id: signal_id.map(str::to_string).or_else(|| {
8124 matched_export
8125 .and_then(|signal| signal.get("id"))
8126 .and_then(Value::as_str)
8127 .map(str::to_string)
8128 }),
8129 name: name.to_string(),
8130 arguments,
8131 avro_arguments,
8132 workflow_sequence,
8133 });
8134 }
8135
8136 if signals.is_empty() {
8137 for signal in export_signals {
8138 if signal.get("status").and_then(Value::as_str) == Some("rejected") {
8139 continue;
8140 }
8141 let Some(name) = signal.get("name").and_then(Value::as_str) else {
8142 continue;
8143 };
8144 let codec = signal
8145 .get("payload_codec")
8146 .and_then(Value::as_str)
8147 .unwrap_or(export_codec);
8148 let (arguments, avro_arguments) =
8149 decode_query_signal_arguments(signal.get("arguments"), codec)?;
8150 signals.push(QuerySignal {
8151 id: signal.get("id").and_then(Value::as_str).map(str::to_string),
8152 name: name.to_string(),
8153 arguments,
8154 avro_arguments,
8155 workflow_sequence: signal.get("workflow_sequence").and_then(value_as_u64),
8156 });
8157 }
8158 signals.sort_by_key(|signal| signal.workflow_sequence.unwrap_or(u64::MAX));
8159 }
8160
8161 Ok(signals)
8162}
8163
8164fn decode_query_signal_arguments(
8165 raw: Option<&Value>,
8166 codec: &str,
8167) -> Result<(Vec<Value>, Vec<AvroValue>)> {
8168 let decoded = match raw.filter(|value| !value.is_null()) {
8169 Some(value) => decode_wire_avro_value(value, codec)?,
8170 None => AvroValue::Array(Vec::new()),
8171 };
8172 let AvroValue::Array(avro_arguments) = normalize_avro_arguments(decoded) else {
8173 unreachable!("normalize_avro_arguments always returns an array");
8174 };
8175 let arguments = avro_arguments
8176 .iter()
8177 .cloned()
8178 .map(AvroValue::into_json)
8179 .collect::<Result<Vec<_>>>()?;
8180 Ok((arguments, avro_arguments))
8181}
8182
8183fn value_as_u64(value: &Value) -> Option<u64> {
8184 value
8185 .as_u64()
8186 .or_else(|| value.as_str().and_then(|value| value.parse().ok()))
8187}
8188
8189#[cfg(test)]
8190mod tests {
8191 use super::*;
8192 use std::{
8193 io::{Read, Write},
8194 net::{SocketAddr, TcpListener, TcpStream},
8195 sync::atomic::AtomicUsize,
8196 thread,
8197 };
8198
8199 fn typed_fidelity_probe() -> AvroValue {
8200 AvroValue::Map(BTreeMap::from([
8201 ("bytes".to_string(), AvroValue::Bytes(vec![0, 0xff])),
8202 ("empty".to_string(), AvroValue::Map(BTreeMap::new())),
8203 (
8204 "numeric".to_string(),
8205 AvroValue::Map(BTreeMap::from([
8206 ("0".to_string(), AvroValue::String("zero".to_string())),
8207 ("1".to_string(), AvroValue::String("one".to_string())),
8208 ])),
8209 ),
8210 (
8211 "nested".to_string(),
8212 AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([(
8213 "enabled".to_string(),
8214 AvroValue::Boolean(true),
8215 )]))]),
8216 ),
8217 (
8218 "projection_collisions".to_string(),
8219 AvroValue::Array(projection_collision_probe()),
8220 ),
8221 ]))
8222 }
8223
8224 fn projection_collision_probe() -> Vec<AvroValue> {
8225 vec![
8226 AvroValue::Map(BTreeMap::from([
8227 ("$type".to_string(), AvroValue::String("bytes".to_string())),
8228 (
8229 "base64".to_string(),
8230 AvroValue::String("ordinary user text".to_string()),
8231 ),
8232 ])),
8233 AvroValue::Map(BTreeMap::from([
8234 ("$type".to_string(), AvroValue::String("map".to_string())),
8235 (
8236 "entries".to_string(),
8237 AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([
8238 ("key".to_string(), AvroValue::String("ordinary".to_string())),
8239 (
8240 "value".to_string(),
8241 AvroValue::String("user map".to_string()),
8242 ),
8243 ]))]),
8244 ),
8245 ])),
8246 ]
8247 }
8248
8249 #[derive(Clone, Debug, Default, PartialEq)]
8250 struct ReplayCounterState {
8251 loaded: Option<String>,
8252 count: i64,
8253 finished: bool,
8254 }
8255
8256 fn replay_counter_worker() -> Worker {
8257 let client = Client::new("http://127.0.0.1:8080").expect("client");
8258 let mut worker = Worker::new(client, "rust-workers");
8259 worker.register_replayed_workflow(
8260 "replay-counter",
8261 ReplayCounterState::default,
8262 |ctx, _input, state| async move {
8263 let loaded = ctx.activity("load-counter", json!([])).await?;
8264 state.update(|current| {
8265 current.loaded = loaded.as_str().map(str::to_string);
8266 })?;
8267 for _ in 0..2 {
8268 let signal = ctx.wait_signal("increment").await?;
8269 let amount = signal.first().and_then(Value::as_i64).unwrap_or_default();
8270 state.update(|current| current.count += amount)?;
8271 }
8272 state.update(|current| current.finished = true)?;
8273 state.read(|current| Ok(json!(current.count)))?
8274 },
8275 );
8276 worker.register_replayed_query::<ReplayCounterState, _, _>(
8277 "replay-counter",
8278 "current",
8279 |_ctx, state, _args| async move {
8280 Ok(json!({
8281 "loaded": state.loaded,
8282 "count": state.count,
8283 "finished": state.finished,
8284 }))
8285 },
8286 );
8287 worker.register_replayed_query::<ReplayCounterState, _, _>(
8288 "replay-counter",
8289 "detached-mutation",
8290 |_ctx, state, _args| async move {
8291 let mut detached = (*state).clone();
8292 detached.count = 999;
8293 Ok(json!(detached.count))
8294 },
8295 );
8296 worker.register_replayed_query::<ReplayCounterState, _, _>(
8297 "replay-counter",
8298 "failed-mutation",
8299 |_ctx, state, _args| async move {
8300 let mut detached = (*state).clone();
8301 detached.count = 999;
8302 Err(Error::WorkerLoop("query refused".to_string()))
8303 },
8304 );
8305 worker
8306 }
8307
8308 fn replay_counter_query(
8309 query_name: &str,
8310 history_events: Value,
8311 run_status: &str,
8312 ) -> QueryTask {
8313 serde_json::from_value(json!({
8314 "query_task_id": format!("query-{query_name}"),
8315 "workflow_type": "replay-counter",
8316 "query_name": query_name,
8317 "payload_codec": "json",
8318 "workflow_arguments": {"codec": "json", "blob": "[]"},
8319 "query_arguments": {"codec": "json", "blob": "[]"},
8320 "history_events": history_events,
8321 "run_status": run_status,
8322 }))
8323 .expect("query task")
8324 }
8325
8326 fn workflow_context(history: Vec<HistoryEvent>) -> WorkflowContext {
8327 workflow_context_with_codec(history, JSON_CODEC)
8328 }
8329
8330 fn workflow_context_with_codec(
8331 history: Vec<HistoryEvent>,
8332 payload_codec: &str,
8333 ) -> WorkflowContext {
8334 WorkflowContext {
8335 state: Arc::new(Mutex::new(
8336 WorkflowState::new_with_identity(
8337 history,
8338 None,
8339 None,
8340 "rust-workers".to_string(),
8341 payload_codec.to_string(),
8342 None,
8343 )
8344 .expect("valid workflow history"),
8345 )),
8346 }
8347 }
8348
8349 fn history_event(event_type: &str, payload: Value) -> HistoryEvent {
8350 HistoryEvent {
8351 event_type: event_type.to_string(),
8352 payload,
8353 raw: HashMap::new(),
8354 }
8355 }
8356
8357 fn workflow_task(
8358 workflow_type: &str,
8359 history_events: Vec<HistoryEvent>,
8360 payload_codec: &str,
8361 ) -> WorkflowTask {
8362 WorkflowTask {
8363 task_id: format!("wft-{workflow_type}"),
8364 workflow_id: Some(format!("wf-{workflow_type}")),
8365 run_id: Some(format!("run-{workflow_type}")),
8366 workflow_type: workflow_type.to_string(),
8367 payload_codec: payload_codec.to_string(),
8368 arguments: Some(
8369 encode_value_envelope(&json!([]), payload_codec).expect("workflow arguments"),
8370 ),
8371 total_history_events: Some(history_events.len() as u64),
8372 history_size_bytes: None,
8373 continue_as_new_recommended: None,
8374 history_budget_pressure: None,
8375 history_events,
8376 next_history_page_token: None,
8377 workflow_task_attempt: 1,
8378 workflow_signal_id: None,
8379 signal_name: None,
8380 signal_arguments: None,
8381 workflow_update_id: None,
8382 update_name: None,
8383 lease_owner: Some("rust-worker".to_string()),
8384 }
8385 }
8386
8387 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
8388 struct SideEffectProbe {
8389 request_id: String,
8390 attempt: u32,
8391 }
8392
8393 #[test]
8394 fn typed_side_effect_runs_callback_once_and_replay_skips_it() {
8395 let calls = AtomicUsize::new(0);
8396 let ctx = workflow_context(Vec::new());
8397 let value = ctx
8398 .side_effect(|| {
8399 calls.fetch_add(1, Ordering::SeqCst);
8400 SideEffectProbe {
8401 request_id: "request-42".to_string(),
8402 attempt: 3,
8403 }
8404 })
8405 .expect("first side effect");
8406 assert_eq!(value.attempt, 3);
8407 assert_eq!(calls.load(Ordering::SeqCst), 1);
8408 let commands = ctx.take_commands().expect("commands");
8409 assert_eq!(commands.len(), 1);
8410 assert_eq!(commands[0]["type"], "record_side_effect");
8411 assert_eq!(
8412 decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("JSON result"),
8413 serde_json::to_value(&value).expect("value")
8414 );
8415
8416 let replay = workflow_context(vec![history_event(
8417 "SideEffectRecorded",
8418 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
8419 )]);
8420 let replayed: SideEffectProbe = replay
8421 .side_effect(|| {
8422 calls.fetch_add(1, Ordering::SeqCst);
8423 panic!("committed side-effect callbacks must not run during replay")
8424 })
8425 .expect("replayed side effect");
8426 assert_eq!(replayed, value);
8427 assert_eq!(calls.load(Ordering::SeqCst), 1);
8428 assert!(replay.take_commands().expect("commands").is_empty());
8429 replay.ensure_history_consumed().expect("history consumed");
8430 }
8431
8432 #[test]
8433 fn side_effect_uses_avro_envelope_and_uuid_is_replay_stable() {
8434 let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
8435 let value = ctx
8436 .side_effect(|| SideEffectProbe {
8437 request_id: "avro-request".to_string(),
8438 attempt: 1,
8439 })
8440 .expect("Avro side effect");
8441 let uuid = ctx.uuid_v4().expect("deterministic UUID");
8442 let commands = ctx.take_commands().expect("commands");
8443 assert_eq!(commands.len(), 2);
8444 assert_eq!(commands[0]["result"]["codec"], DEFAULT_CODEC);
8445 assert_eq!(commands[1]["result"]["codec"], DEFAULT_CODEC);
8446 assert_eq!(
8447 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
8448 serde_json::to_value(&value).expect("value")
8449 );
8450
8451 let replay = workflow_context_with_codec(
8452 vec![
8453 history_event(
8454 "SideEffectRecorded",
8455 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
8456 ),
8457 history_event(
8458 "SideEffectRecorded",
8459 json!({"sequence": 2, "result": commands[1]["result"].clone()}),
8460 ),
8461 ],
8462 DEFAULT_CODEC,
8463 );
8464 let replayed: SideEffectProbe = replay
8465 .side_effect(|| panic!("Avro callback must not run"))
8466 .expect("replayed Avro value");
8467 let replayed_uuid = replay.uuid_v4().expect("replayed UUID");
8468 assert_eq!(replayed, value);
8469 assert_eq!(replayed_uuid, uuid);
8470 assert!(replay.take_commands().expect("commands").is_empty());
8471 }
8472
8473 #[test]
8474 fn typed_side_effect_replay_preserves_bytes_and_maps() {
8475 let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
8476 let value = ctx
8477 .side_effect_avro_value(typed_fidelity_probe)
8478 .expect("typed side effect");
8479 let commands = ctx.take_commands().expect("side-effect command");
8480 assert_eq!(
8481 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
8482 .expect("recorded side effect"),
8483 value
8484 );
8485
8486 let replay = workflow_context_with_codec(
8487 vec![history_event(
8488 "SideEffectRecorded",
8489 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
8490 )],
8491 DEFAULT_CODEC,
8492 );
8493 assert_eq!(
8494 replay
8495 .side_effect_avro_value(|| panic!("replay must not invoke callback"))
8496 .expect("replayed typed side effect"),
8497 value
8498 );
8499 }
8500
8501 #[test]
8502 fn ordered_side_effects_share_the_durable_command_stream() {
8503 let first = encode_value_envelope(&json!("first"), JSON_CODEC).expect("first");
8504 let second = encode_value_envelope(&json!(29), JSON_CODEC).expect("second");
8505 let ctx = workflow_context(vec![
8506 history_event(
8507 "SideEffectRecorded",
8508 json!({"sequence": 1, "result": first}),
8509 ),
8510 history_event(
8511 "SideEffectRecorded",
8512 json!({"sequence": 2, "result": second}),
8513 ),
8514 ]);
8515 let first: String = ctx
8516 .side_effect(|| panic!("first callback must not run"))
8517 .expect("first replay");
8518 let second: i32 = ctx
8519 .side_effect(|| panic!("second callback must not run"))
8520 .expect("second replay");
8521 assert_eq!(first, "first");
8522 assert_eq!(second, 29);
8523 ctx.ensure_history_consumed().expect("ordered history");
8524
8525 let reordered = workflow_context(vec![history_event(
8526 "VersionMarkerRecorded",
8527 json!({
8528 "sequence": 1,
8529 "change_id": "before-side-effect",
8530 "version": 1,
8531 "min_supported": 1,
8532 "max_supported": 1,
8533 }),
8534 )]);
8535 let error = reordered
8536 .side_effect(|| "new".to_string())
8537 .expect_err("command reordering must fail");
8538 assert!(matches!(
8539 error,
8540 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8541 if reason == "recorded_command_mismatch"
8542 ));
8543 }
8544
8545 #[test]
8546 fn version_markers_replay_across_upgrades_and_do_not_duplicate() {
8547 let ctx = workflow_context(Vec::new());
8548 assert_eq!(ctx.get_version("checkout-v2", 1, 2).expect("version"), 2);
8549 assert_eq!(ctx.get_version("checkout-v2", 1, 3).expect("cached"), 2);
8550 assert!(ctx.patched("new-search").expect("patch"));
8551 ctx.deprecate_patch("new-search").expect("deprecate patch");
8552 let commands = ctx.take_commands().expect("commands");
8553 assert_eq!(commands.len(), 2);
8554 assert_eq!(commands[0]["type"], "record_version_marker");
8555 assert_eq!(commands[0]["version"], 2);
8556 assert_eq!(commands[1]["change_id"], "new-search");
8557
8558 let replay = workflow_context(vec![history_event(
8559 "VersionMarkerRecorded",
8560 json!({
8561 "sequence": 1,
8562 "change_id": "checkout-v2",
8563 "version": 2,
8564 "min_supported": 1,
8565 "max_supported": 2,
8566 }),
8567 )]);
8568 assert_eq!(replay.get_version("checkout-v2", 1, 4).expect("upgrade"), 2);
8569 assert_eq!(replay.get_version("checkout-v2", 2, 5).expect("repeat"), 2);
8570 assert!(replay.take_commands().expect("commands").is_empty());
8571 replay.ensure_history_consumed().expect("history consumed");
8572 }
8573
8574 #[test]
8575 fn version_markers_reject_incompatible_or_malformed_history() {
8576 let incompatible = workflow_context(vec![history_event(
8577 "VersionMarkerRecorded",
8578 json!({
8579 "sequence": 1,
8580 "change_id": "checkout-v2",
8581 "version": 1,
8582 "min_supported": 1,
8583 "max_supported": 2,
8584 }),
8585 )]);
8586 let error = incompatible
8587 .get_version("checkout-v2", 2, 3)
8588 .expect_err("old version is unsupported");
8589 assert!(matches!(
8590 error,
8591 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8592 if reason == "version_marker_incompatible_range"
8593 ));
8594
8595 for (history, reason) in [
8596 (
8597 vec![history_event("SideEffectRecorded", json!({"sequence": 1}))],
8598 "side_effect_result_missing",
8599 ),
8600 (
8601 vec![history_event(
8602 "SideEffectRecorded",
8603 json!({
8604 "sequence": 1,
8605 "result": {"codec": "avro", "blob": "not-base64"},
8606 }),
8607 )],
8608 "side_effect_payload_incompatible",
8609 ),
8610 (
8611 vec![history_event(
8612 "SideEffectRecorded",
8613 json!({"sequence": 1, "result": {"unwrapped": true}}),
8614 )],
8615 "side_effect_payload_malformed",
8616 ),
8617 (
8618 vec![history_event(
8619 "VersionMarkerRecorded",
8620 json!({
8621 "sequence": 1,
8622 "change_id": "change",
8623 "version": 1,
8624 "min_supported": 2,
8625 "max_supported": 1,
8626 }),
8627 )],
8628 "version_marker_history_range_invalid",
8629 ),
8630 ] {
8631 let error = WorkflowState::new(
8632 history,
8633 "rust-workers".to_string(),
8634 JSON_CODEC.to_string(),
8635 None,
8636 )
8637 .expect_err("malformed history must fail");
8638 assert!(matches!(
8639 error,
8640 Error::NonDeterministicReplay(ReplayFailure { reason: actual, .. })
8641 if actual == reason
8642 ));
8643 }
8644 }
8645
8646 #[test]
8647 fn duplicate_side_effects_and_version_markers_are_rejected() {
8648 let duplicate_side_effect = WorkflowState::new(
8649 vec![
8650 history_event(
8651 "SideEffectRecorded",
8652 json!({"sequence": 1, "result": {"codec": "json", "blob": "1"}}),
8653 ),
8654 history_event(
8655 "SideEffectRecorded",
8656 json!({"sequence": 1, "result": {"codec": "json", "blob": "2"}}),
8657 ),
8658 ],
8659 "rust-workers".to_string(),
8660 JSON_CODEC.to_string(),
8661 None,
8662 )
8663 .expect_err("duplicate side effect");
8664 assert!(matches!(
8665 duplicate_side_effect,
8666 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8667 if reason == "duplicate_side_effect_record"
8668 ));
8669
8670 let marker = |sequence| {
8671 history_event(
8672 "VersionMarkerRecorded",
8673 json!({
8674 "sequence": sequence,
8675 "change_id": "same-change",
8676 "version": 1,
8677 "min_supported": 1,
8678 "max_supported": 1,
8679 }),
8680 )
8681 };
8682 let duplicate_marker = WorkflowState::new(
8683 vec![marker(1), marker(3)],
8684 "rust-workers".to_string(),
8685 JSON_CODEC.to_string(),
8686 None,
8687 )
8688 .expect_err("duplicate marker");
8689 assert!(matches!(
8690 duplicate_marker,
8691 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8692 if reason == "duplicate_version_marker"
8693 ));
8694 }
8695
8696 #[test]
8697 fn cold_worker_replay_does_not_repeat_committed_side_effects_or_markers() {
8698 fn worker(calls: Arc<AtomicUsize>) -> Worker {
8699 let client = Client::new("http://127.0.0.1:8080").expect("client");
8700 let mut worker = Worker::new(client, "rust-workers");
8701 worker.register_workflow("rust.side-effect-version", move |ctx, _input| {
8702 let calls = Arc::clone(&calls);
8703 async move {
8704 let captured = ctx.side_effect(|| {
8705 calls.fetch_add(1, Ordering::SeqCst);
8706 "captured-once".to_string()
8707 })?;
8708 let version = ctx.get_version("cold-restart", 1, 2)?;
8709 Ok(json!({"captured": captured, "version": version}))
8710 }
8711 });
8712 worker
8713 }
8714
8715 fn task(history_events: Vec<HistoryEvent>) -> WorkflowTask {
8716 WorkflowTask {
8717 task_id: "wft-side-effect-version".to_string(),
8718 workflow_id: Some("wf-side-effect-version".to_string()),
8719 run_id: Some("run-side-effect-version".to_string()),
8720 workflow_type: "rust.side-effect-version".to_string(),
8721 payload_codec: JSON_CODEC.to_string(),
8722 arguments: Some(encode_value_envelope(&json!([]), JSON_CODEC).expect("arguments")),
8723 history_events,
8724 total_history_events: None,
8725 history_size_bytes: None,
8726 continue_as_new_recommended: None,
8727 history_budget_pressure: None,
8728 next_history_page_token: None,
8729 workflow_task_attempt: 1,
8730 workflow_signal_id: None,
8731 signal_name: None,
8732 signal_arguments: None,
8733 workflow_update_id: None,
8734 update_name: None,
8735 lease_owner: Some("rust-worker".to_string()),
8736 }
8737 }
8738
8739 let calls = Arc::new(AtomicUsize::new(0));
8740 let initial = worker(Arc::clone(&calls))
8741 .execute_workflow_task(task(Vec::new()))
8742 .expect("initial execution");
8743 assert_eq!(
8744 initial
8745 .iter()
8746 .map(|command| &command["type"])
8747 .collect::<Vec<_>>(),
8748 vec![
8749 "record_side_effect",
8750 "record_version_marker",
8751 "complete_workflow"
8752 ]
8753 );
8754 assert_eq!(calls.load(Ordering::SeqCst), 1);
8755
8756 let restarted = worker(Arc::clone(&calls));
8757 let replayed = restarted
8758 .execute_workflow_task(task(vec![
8759 history_event(
8760 "SideEffectRecorded",
8761 json!({"sequence": 1, "result": initial[0]["result"].clone()}),
8762 ),
8763 history_event(
8764 "VersionMarkerRecorded",
8765 json!({
8766 "sequence": 2,
8767 "change_id": "cold-restart",
8768 "version": 2,
8769 "min_supported": 1,
8770 "max_supported": 2,
8771 }),
8772 ),
8773 ]))
8774 .expect("cold replay");
8775 assert_eq!(replayed.len(), 1);
8776 assert_eq!(replayed[0]["type"], "complete_workflow");
8777 assert_eq!(calls.load(Ordering::SeqCst), 1);
8778 }
8779
8780 #[test]
8781 fn side_effect_replay_rejects_changed_rust_value_type() {
8782 let result = encode_value_envelope(&json!({"value": 42}), JSON_CODEC).expect("result");
8783 let ctx = workflow_context(vec![history_event(
8784 "SideEffectRecorded",
8785 json!({"sequence": 1, "result": result}),
8786 )]);
8787 let error = ctx
8788 .side_effect::<Vec<String>, _>(|| panic!("callback must not run"))
8789 .expect_err("changed type must fail replay");
8790 assert!(matches!(
8791 error,
8792 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8793 if reason == "side_effect_type_mismatch"
8794 ));
8795 }
8796
8797 fn completed_retry_activity_history() -> Vec<HistoryEvent> {
8798 vec![
8799 history_event(
8800 "ActivityScheduled",
8801 json!({
8802 "sequence": 1,
8803 "activity_type": "flaky",
8804 "activity_execution_id": "act-1",
8805 "activity": {
8806 "id": "act-1",
8807 "sequence": 1,
8808 "type": "flaky",
8809 "queue": "critical-activities",
8810 "execution_mode": null,
8811 "retry_policy": {
8812 "snapshot_version": 1,
8813 "max_attempts": 3,
8814 "backoff_seconds": [2, 4],
8815 "start_to_close_timeout": 30,
8816 "schedule_to_start_timeout": 5,
8817 "schedule_to_close_timeout": 90,
8818 "heartbeat_timeout": 10,
8819 "non_retryable_error_types": ["PermanentError"]
8820 }
8821 }
8822 }),
8823 ),
8824 history_event(
8825 "ActivityStarted",
8826 json!({
8827 "sequence": 1,
8828 "activity_type": "flaky",
8829 "activity_execution_id": "act-1",
8830 "activity_attempt_id": "attempt-1",
8831 "attempt_number": 1
8832 }),
8833 ),
8834 history_event(
8835 "ActivityRetryScheduled",
8836 json!({
8837 "sequence": 1,
8838 "activity_type": "flaky",
8839 "activity_execution_id": "act-1",
8840 "activity_attempt_id": "attempt-1",
8841 "attempt_number": 1,
8842 "retry_after_attempt": 1,
8843 "retry_backoff_seconds": 2,
8844 "failure_category": "activity",
8845 "exception_type": "TransientError"
8846 }),
8847 ),
8848 history_event(
8849 "ActivityStarted",
8850 json!({
8851 "sequence": 1,
8852 "activity_type": "flaky",
8853 "activity_execution_id": "act-1",
8854 "activity_attempt_id": "attempt-2",
8855 "attempt_number": 2
8856 }),
8857 ),
8858 history_event(
8859 "ActivityCompleted",
8860 json!({
8861 "sequence": 1,
8862 "activity_type": "flaky",
8863 "activity_execution_id": "act-1",
8864 "activity_attempt_id": "attempt-2",
8865 "attempt_number": 2,
8866 "payload_codec": "json",
8867 "result": {"codec": "json", "blob": "{\"status\":\"recovered\"}"}
8868 }),
8869 ),
8870 ]
8871 }
8872
8873 fn retry_activity_options() -> ActivityOptions {
8874 ActivityOptions::new()
8875 .task_queue("critical-activities")
8876 .retry_policy(
8877 ActivityRetryPolicy::new(3)
8878 .backoff_intervals([Duration::from_secs(2), Duration::from_secs(4)])
8879 .non_retryable_error_type("PermanentError"),
8880 )
8881 .start_to_close_timeout(Duration::from_secs(30))
8882 .schedule_to_start_timeout(Duration::from_secs(5))
8883 .schedule_to_close_timeout(Duration::from_secs(90))
8884 .heartbeat_timeout(Duration::from_secs(10))
8885 }
8886
8887 #[test]
8888 fn fixed_avro_value_round_trips_json_values() {
8889 let value = json!({"greeting": "hello", "count": 3, "ok": true});
8890 let envelope = PayloadEnvelope::avro(&value).expect("encode");
8891 assert_eq!(envelope.codec, DEFAULT_CODEC);
8892 assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
8893 }
8894
8895 #[tokio::test]
8896 async fn typed_worker_surfaces_preserve_bytes_and_map_list_identity() {
8897 let client = Client::new("http://127.0.0.1:8080").expect("client");
8898 let mut worker = Worker::new(client, "rust-workers");
8899 worker.register_workflow_avro_value("typed.echo", |_ctx, input| async move { Ok(input) });
8900 worker
8901 .register_activity_avro_value("typed.activity", |_ctx, input| async move { Ok(input) });
8902 worker.register_query_avro_value("typed.echo", "inspect", |_ctx, input| async move {
8903 Ok(input)
8904 });
8905 worker.register_update_avro_value("typed.echo", "replace", |_ctx, input| async move {
8906 Ok(input)
8907 });
8908 worker.register_workflow_avro_value("typed.signal", |ctx, _input| async move {
8909 Ok(AvroValue::Array(
8910 ctx.wait_signal_avro_value("changed").await?,
8911 ))
8912 });
8913
8914 let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
8915 let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed envelope");
8916
8917 let mut workflow = workflow_task("typed.echo", Vec::new(), DEFAULT_CODEC);
8918 workflow.arguments = Some(envelope.clone());
8919 let commands = worker
8920 .execute_workflow_task(workflow)
8921 .expect("typed workflow task");
8922 assert_eq!(commands[0]["type"], "complete_workflow");
8923 assert_eq!(
8924 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
8925 .expect("typed workflow result"),
8926 arguments
8927 );
8928
8929 let activity = ActivityTask {
8930 task_id: "activity-typed".to_string(),
8931 activity_attempt_id: Some("attempt-typed".to_string()),
8932 attempt_id: None,
8933 activity_type: "typed.activity".to_string(),
8934 payload_codec: DEFAULT_CODEC.to_string(),
8935 arguments: Some(envelope.clone()),
8936 attempt_number: 1,
8937 lease_owner: Some("rust-worker".to_string()),
8938 };
8939 assert_eq!(
8940 worker
8941 .execute_activity_task(activity)
8942 .await
8943 .expect("typed activity result"),
8944 arguments
8945 );
8946
8947 let query = QueryTask {
8948 query_task_id: "query-typed".to_string(),
8949 query_task_attempt: 1,
8950 lease_owner: Some("rust-worker".to_string()),
8951 workflow_id: Some("typed-1".to_string()),
8952 run_id: Some("run-typed".to_string()),
8953 workflow_type: "typed.echo".to_string(),
8954 query_name: "inspect".to_string(),
8955 payload_codec: DEFAULT_CODEC.to_string(),
8956 workflow_arguments: Some(
8957 encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
8958 .expect("workflow input"),
8959 ),
8960 query_arguments: Some(envelope.clone()),
8961 history_events: Vec::new(),
8962 history_export: None,
8963 run_status: Some("running".to_string()),
8964 };
8965 assert_eq!(
8966 worker
8967 .execute_query_task(query)
8968 .await
8969 .expect("typed query result"),
8970 arguments
8971 );
8972
8973 let mut update = workflow_task(
8974 "typed.echo",
8975 vec![history_event(
8976 "UpdateAccepted",
8977 json!({
8978 "update_id": "update-typed",
8979 "update_name": "replace",
8980 "arguments": envelope.clone(),
8981 }),
8982 )],
8983 DEFAULT_CODEC,
8984 );
8985 update.workflow_update_id = Some("update-typed".to_string());
8986 update.update_name = Some("replace".to_string());
8987 let commands = worker
8988 .execute_workflow_task(update)
8989 .expect("typed update task");
8990 assert_eq!(commands[0]["type"], "complete_update");
8991 assert_eq!(
8992 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
8993 .expect("typed update result"),
8994 arguments
8995 );
8996
8997 let mut signal = workflow_task(
8998 "typed.signal",
8999 vec![history_event(
9000 "SignalReceived",
9001 json!({
9002 "signal_id": "signal-typed",
9003 "signal_name": "changed",
9004 "arguments": envelope.clone(),
9005 }),
9006 )],
9007 DEFAULT_CODEC,
9008 );
9009 signal.workflow_signal_id = Some("signal-typed".to_string());
9010 signal.signal_name = Some("changed".to_string());
9011 signal.signal_arguments = Some(envelope);
9012 let commands = worker
9013 .execute_workflow_task(signal)
9014 .expect("typed signal resume");
9015 assert_eq!(
9016 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
9017 .expect("typed signal result"),
9018 arguments
9019 );
9020 }
9021
9022 #[tokio::test]
9023 async fn typed_helpers_never_parse_json_inspection_projection() {
9024 let collision_values = projection_collision_probe();
9025 let expected = AvroValue::Array(collision_values.clone());
9026 let envelope = encode_typed_envelope(&expected, DEFAULT_CODEC).expect("collision envelope");
9027
9028 let activity_context = workflow_context_with_codec(
9029 vec![history_event(
9030 "ActivityCompleted",
9031 json!({
9032 "sequence": 1,
9033 "activity_type": "collision.activity",
9034 "payload_codec": DEFAULT_CODEC,
9035 "result": envelope.clone(),
9036 }),
9037 )],
9038 DEFAULT_CODEC,
9039 );
9040 assert_eq!(
9041 activity_context
9042 .activity_avro_value("collision.activity", AvroValue::Array(Vec::new()))
9043 .await
9044 .expect("typed activity collision result"),
9045 expected
9046 );
9047
9048 let signal_context = workflow_context_with_codec(
9049 vec![
9050 history_event(
9051 "SignalWaitOpened",
9052 json!({"sequence": 1, "signal_name": "collision"}),
9053 ),
9054 history_event(
9055 "SignalApplied",
9056 json!({
9057 "sequence": 1,
9058 "signal_name": "collision",
9059 "payload_codec": DEFAULT_CODEC,
9060 "value": envelope.clone(),
9061 }),
9062 ),
9063 ],
9064 DEFAULT_CODEC,
9065 );
9066 assert_eq!(
9067 signal_context
9068 .wait_signal_avro_value("collision")
9069 .await
9070 .expect("typed signal collision arguments"),
9071 collision_values
9072 );
9073
9074 let child_context = workflow_context_with_codec(
9075 vec![
9076 history_event(
9077 "ChildWorkflowScheduled",
9078 json!({
9079 "sequence": 1,
9080 "child_workflow_instance_id": "collision-child",
9081 "child_workflow_run_id": "collision-run",
9082 "child_workflow_type": "collision.child",
9083 }),
9084 ),
9085 history_event(
9086 "ChildRunCompleted",
9087 json!({
9088 "sequence": 1,
9089 "child_workflow_instance_id": "collision-child",
9090 "child_workflow_run_id": "collision-run",
9091 "child_workflow_type": "collision.child",
9092 "payload_codec": DEFAULT_CODEC,
9093 "result": envelope,
9094 }),
9095 ),
9096 ],
9097 DEFAULT_CODEC,
9098 );
9099 let child = child_context
9100 .start_child_workflow_avro_value(
9101 "collision.child",
9102 ChildWorkflowOptions::new("collision-workers"),
9103 AvroValue::Array(Vec::new()),
9104 )
9105 .await
9106 .expect("typed child collision result");
9107 assert_eq!(child.result, expected);
9108 }
9109
9110 #[tokio::test]
9111 async fn replayed_typed_query_keeps_lossless_workflow_and_query_inputs() {
9112 let client = Client::new("http://127.0.0.1:8080").expect("client");
9113 let mut worker = Worker::new(client, "rust-workers");
9114 worker.register_replayed_workflow_avro_value(
9115 "typed.replayed",
9116 || (),
9117 |_ctx, input, _state| async move { Ok(input) },
9118 );
9119 worker.register_replayed_query_avro_value::<(), _, _>(
9120 "typed.replayed",
9121 "inspect",
9122 |ctx, _state, args| async move {
9123 let mut signals = ctx.signals_avro_value("collision");
9124 let signal = signals
9125 .pop()
9126 .map(AvroValue::Array)
9127 .unwrap_or_else(|| AvroValue::Array(Vec::new()));
9128 Ok(AvroValue::Array(vec![
9129 ctx.workflow_input_avro_value().clone(),
9130 signal,
9131 args,
9132 ]))
9133 },
9134 );
9135 let arguments = AvroValue::Array(projection_collision_probe());
9136 let signal_arguments =
9137 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed query signal arguments");
9138 let task = QueryTask {
9139 query_task_id: "query-typed-replay".to_string(),
9140 query_task_attempt: 1,
9141 lease_owner: Some("rust-worker".to_string()),
9142 workflow_id: Some("typed-replay".to_string()),
9143 run_id: Some("run-typed-replay".to_string()),
9144 workflow_type: "typed.replayed".to_string(),
9145 query_name: "inspect".to_string(),
9146 payload_codec: DEFAULT_CODEC.to_string(),
9147 workflow_arguments: Some(
9148 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("workflow arguments"),
9149 ),
9150 query_arguments: Some(
9151 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("query arguments"),
9152 ),
9153 history_events: vec![history_event(
9154 "SignalReceived",
9155 json!({
9156 "signal_id": "collision-signal",
9157 "signal_name": "collision",
9158 "workflow_sequence": 1,
9159 "payload_codec": DEFAULT_CODEC,
9160 "arguments": signal_arguments,
9161 }),
9162 )],
9163 history_export: None,
9164 run_status: Some("completed".to_string()),
9165 };
9166
9167 assert_eq!(
9168 worker
9169 .execute_query_task(task)
9170 .await
9171 .expect("typed replay query"),
9172 AvroValue::Array(vec![arguments.clone(), arguments.clone(), arguments])
9173 );
9174 }
9175
9176 #[test]
9177 fn public_avro_adapter_rejects_non_string_map_keys_before_json_conversion() {
9178 let value = BTreeMap::from([(1_i32, "integer key")]);
9179 let error = PayloadEnvelope::avro(&value)
9180 .expect_err("integer map keys must fail")
9181 .to_string();
9182
9183 assert!(error.contains("invalid_map_key"));
9184 }
9185
9186 #[test]
9187 fn json_codec_remains_plain_json() {
9188 let value = json!({"greeting": "hello", "count": 3, "ok": true});
9189 let envelope = PayloadEnvelope::json(&value).expect("encode");
9190
9191 assert_eq!(envelope.codec, JSON_CODEC);
9192 assert_eq!(envelope.blob, serde_json::to_string(&value).expect("json"));
9193 assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
9194 }
9195
9196 #[test]
9197 fn prerelease_avro_payload_without_single_object_frame_is_rejected() {
9198 let envelope = PayloadEnvelope {
9199 codec: DEFAULT_CODEC.to_string(),
9200 blob: BASE64.encode([0x01]),
9201 };
9202
9203 let error = decode_payload::<Value>(&envelope).expect_err("prerelease payload must fail");
9204 assert!(error.to_string().contains("invalid_payload_framing"));
9205 }
9206
9207 #[test]
9208 fn workflow_context_schedules_activity_until_completion_is_in_history() {
9209 let ctx = WorkflowContext {
9210 state: Arc::new(Mutex::new(
9211 WorkflowState::new_with_identity(
9212 Vec::new(),
9213 Some("wf-parent".to_string()),
9214 Some("run-parent".to_string()),
9215 "rust-workers".to_string(),
9216 DEFAULT_CODEC.to_string(),
9217 None,
9218 )
9219 .expect("workflow state"),
9220 )),
9221 };
9222
9223 let mut call = Box::pin(ctx.activity("hello.activity", json!(["Ada"])));
9224 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9225 assert!(matches!(
9226 call.as_mut().poll(&mut task_context),
9227 Poll::Pending
9228 ));
9229
9230 let commands = ctx.take_commands().expect("commands");
9231 assert_eq!(commands[0]["type"], "schedule_activity");
9232 assert_eq!(commands[0]["activity_type"], "hello.activity");
9233 }
9234
9235 #[test]
9236 fn activity_options_encode_retry_policy_queue_and_every_timeout() {
9237 let ctx = workflow_context(Vec::new());
9238 let options = ActivityOptions::new()
9239 .task_queue("payments")
9240 .retry_policy(
9241 ActivityRetryPolicy::new(4)
9242 .exponential_backoff(Duration::from_secs(1), 3, Some(Duration::from_secs(10)))
9243 .non_retryable_error_type("ValidationError"),
9244 )
9245 .start_to_close_timeout(Duration::from_secs(120))
9246 .schedule_to_start_timeout(Duration::from_secs(10))
9247 .schedule_to_close_timeout(Duration::from_secs(300))
9248 .heartbeat_timeout(Duration::from_secs(15));
9249 let mut call = Box::pin(ctx.activity_with_options(
9250 "charge-card",
9251 options,
9252 json!([{"order_id": "o-1"}]),
9253 ));
9254 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9255
9256 assert!(matches!(
9257 call.as_mut().poll(&mut task_context),
9258 Poll::Pending
9259 ));
9260 assert!(matches!(
9261 call.as_mut().poll(&mut task_context),
9262 Poll::Pending
9263 ));
9264
9265 let commands = ctx.take_commands().expect("activity command");
9266 assert_eq!(commands.len(), 1, "one future emits one logical schedule");
9267 assert_eq!(commands[0]["queue"], "payments");
9268 assert_eq!(
9269 commands[0]["retry_policy"],
9270 json!({
9271 "max_attempts": 4,
9272 "backoff_seconds": [1, 3, 9],
9273 "non_retryable_error_types": ["ValidationError"],
9274 })
9275 );
9276 assert_eq!(commands[0]["start_to_close_timeout"], 120);
9277 assert_eq!(commands[0]["schedule_to_start_timeout"], 10);
9278 assert_eq!(commands[0]["schedule_to_close_timeout"], 300);
9279 assert_eq!(commands[0]["heartbeat_timeout"], 15);
9280 }
9281
9282 #[test]
9283 fn activity_options_encode_explicit_and_rounded_backoff_intervals() {
9284 let ctx = workflow_context(Vec::new());
9285 let options = ActivityOptions::new().retry_policy(
9286 ActivityRetryPolicy::new(3)
9287 .backoff_intervals([Duration::from_millis(1), Duration::from_millis(1_001)]),
9288 );
9289 let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
9290 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9291
9292 assert!(matches!(
9293 call.as_mut().poll(&mut task_context),
9294 Poll::Pending
9295 ));
9296 assert_eq!(
9297 ctx.take_commands().expect("command")[0]["retry_policy"]["backoff_seconds"],
9298 json!([1, 2])
9299 );
9300 }
9301
9302 #[test]
9303 fn invalid_activity_options_return_typed_errors_before_emitting_commands() {
9304 let cases = [
9305 (
9306 ActivityOptions::new().task_queue(" "),
9307 ActivityOptionsErrorKind::EmptyTaskQueue,
9308 ),
9309 (
9310 ActivityOptions::new().retry_policy(ActivityRetryPolicy::default()),
9311 ActivityOptionsErrorKind::EmptyRetryPolicy,
9312 ),
9313 (
9314 ActivityOptions::new().retry_policy(ActivityRetryPolicy::new(0)),
9315 ActivityOptionsErrorKind::InvalidMaxAttempts,
9316 ),
9317 (
9318 ActivityOptions::new().retry_policy(ActivityRetryPolicy {
9319 max_attempts: None,
9320 backoff: Some(ActivityBackoff::Explicit(vec![Duration::from_secs(1)])),
9321 non_retryable_error_types: Vec::new(),
9322 }),
9323 ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
9324 ),
9325 (
9326 ActivityOptions::new().retry_policy(
9327 ActivityRetryPolicy::new(2)
9328 .backoff_intervals([Duration::from_secs(1), Duration::from_secs(2)]),
9329 ),
9330 ActivityOptionsErrorKind::TooManyBackoffIntervals,
9331 ),
9332 (
9333 ActivityOptions::new().retry_policy(
9334 ActivityRetryPolicy::new(2).exponential_backoff(
9335 Duration::from_secs(1),
9336 0,
9337 None,
9338 ),
9339 ),
9340 ActivityOptionsErrorKind::InvalidBackoffCoefficient,
9341 ),
9342 (
9343 ActivityOptions::new()
9344 .retry_policy(ActivityRetryPolicy::new(2).non_retryable_error_type(" ")),
9345 ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
9346 ),
9347 (
9348 ActivityOptions::new().retry_policy(
9349 ActivityRetryPolicy::new(10_002).exponential_backoff(
9350 Duration::from_secs(1),
9351 1,
9352 None,
9353 ),
9354 ),
9355 ActivityOptionsErrorKind::BackoffGenerationTooLarge,
9356 ),
9357 (
9358 ActivityOptions::new().retry_policy(
9359 ActivityRetryPolicy::new(2)
9360 .backoff_intervals([Duration::from_secs(i64::MAX as u64 + 1)]),
9361 ),
9362 ActivityOptionsErrorKind::BackoffOverflow,
9363 ),
9364 ];
9365
9366 for (options, expected_kind) in cases {
9367 let ctx = workflow_context(Vec::new());
9368 let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
9369 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9370 let Poll::Ready(Err(Error::InvalidActivityOptions(error))) =
9371 call.as_mut().poll(&mut task_context)
9372 else {
9373 panic!("expected typed activity validation error");
9374 };
9375 assert_eq!(error.kind, expected_kind);
9376 assert!(ctx.take_commands().expect("commands").is_empty());
9377 }
9378 }
9379
9380 #[test]
9381 fn activity_options_validate_positive_and_ordered_timeouts() {
9382 let zero_timeout_cases = [
9383 ActivityOptions::new().start_to_close_timeout(Duration::ZERO),
9384 ActivityOptions::new().schedule_to_start_timeout(Duration::ZERO),
9385 ActivityOptions::new().schedule_to_close_timeout(Duration::ZERO),
9386 ActivityOptions::new().heartbeat_timeout(Duration::ZERO),
9387 ];
9388 for options in zero_timeout_cases {
9389 assert_eq!(
9390 options.validate().expect_err("zero timeout").kind,
9391 ActivityOptionsErrorKind::TimeoutNotPositive
9392 );
9393 }
9394
9395 let ordering_cases = [
9396 ActivityOptions::new()
9397 .heartbeat_timeout(Duration::from_secs(11))
9398 .start_to_close_timeout(Duration::from_secs(10)),
9399 ActivityOptions::new()
9400 .start_to_close_timeout(Duration::from_secs(31))
9401 .schedule_to_close_timeout(Duration::from_secs(30)),
9402 ActivityOptions::new()
9403 .schedule_to_start_timeout(Duration::from_secs(31))
9404 .schedule_to_close_timeout(Duration::from_secs(30)),
9405 ];
9406 for options in ordering_cases {
9407 assert_eq!(
9408 options.validate().expect_err("timeout order").kind,
9409 ActivityOptionsErrorKind::TimeoutOrder
9410 );
9411 }
9412
9413 assert_eq!(
9414 ActivityOptions::new()
9415 .start_to_close_timeout(Duration::from_secs(i64::MAX as u64 + 1))
9416 .validate()
9417 .expect_err("protocol integer overflow")
9418 .kind,
9419 ActivityOptionsErrorKind::TimeoutOverflow
9420 );
9421 }
9422
9423 #[test]
9424 fn replayed_activity_retry_history_completes_without_duplicate_schedule() {
9425 let ctx = workflow_context(completed_retry_activity_history());
9426 let mut call =
9427 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
9428 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9429
9430 assert!(matches!(
9431 call.as_mut().poll(&mut task_context),
9432 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
9433 ));
9434 assert!(ctx.take_commands().expect("commands").is_empty());
9435 ctx.ensure_history_consumed().expect("history consumed");
9436 }
9437
9438 #[test]
9439 fn duplicate_non_retryable_types_use_one_command_and_replay_representation() {
9440 let mut options = retry_activity_options();
9441 options
9442 .retry_policy
9443 .as_mut()
9444 .expect("retry policy")
9445 .non_retryable_error_types
9446 .extend([" PermanentError ".to_string(), "PermanentError".to_string()]);
9447
9448 let new_ctx = workflow_context(Vec::new());
9449 let mut new_call =
9450 Box::pin(new_ctx.activity_with_options("flaky", options.clone(), json!([])));
9451 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9452 assert!(matches!(
9453 new_call.as_mut().poll(&mut task_context),
9454 Poll::Pending
9455 ));
9456 let commands = new_ctx.take_commands().expect("commands");
9457 assert_eq!(commands.len(), 1);
9458 assert_eq!(
9459 commands[0]["retry_policy"]["non_retryable_error_types"],
9460 json!(["PermanentError"])
9461 );
9462
9463 let replay_ctx = workflow_context(completed_retry_activity_history());
9464 let mut replay_call =
9465 Box::pin(replay_ctx.activity_with_options("flaky", options, json!([])));
9466 assert!(matches!(
9467 replay_call.as_mut().poll(&mut task_context),
9468 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
9469 ));
9470 assert!(replay_ctx.take_commands().expect("commands").is_empty());
9471 replay_ctx
9472 .ensure_history_consumed()
9473 .expect("history consumed");
9474 }
9475
9476 #[test]
9477 fn replayed_intermediate_retry_remains_pending_across_restarts() {
9478 let history = completed_retry_activity_history()
9479 .into_iter()
9480 .take(3)
9481 .collect::<Vec<_>>();
9482
9483 for _restart in 0..2 {
9484 let ctx = workflow_context(history.clone());
9485 let mut call =
9486 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
9487 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9488 assert!(matches!(
9489 call.as_mut().poll(&mut task_context),
9490 Poll::Pending
9491 ));
9492 assert!(ctx.take_commands().expect("commands").is_empty());
9493 }
9494 }
9495
9496 #[test]
9497 fn replayed_activity_rejects_changed_queue_retry_and_every_timeout_field() {
9498 let mut changed_queue = retry_activity_options();
9499 changed_queue.task_queue = Some("different-queue".to_string());
9500
9501 let mut changed_max_attempts = retry_activity_options();
9502 let retry_policy = changed_max_attempts
9503 .retry_policy
9504 .as_mut()
9505 .expect("retry policy");
9506 retry_policy.max_attempts = Some(4);
9507
9508 let mut changed_backoff = retry_activity_options();
9509 let retry_policy = changed_backoff.retry_policy.as_mut().expect("retry policy");
9510 retry_policy.backoff = Some(ActivityBackoff::Explicit(vec![
9511 Duration::from_secs(3),
9512 Duration::from_secs(4),
9513 ]));
9514
9515 let mut changed_non_retryable_types = retry_activity_options();
9516 let retry_policy = changed_non_retryable_types
9517 .retry_policy
9518 .as_mut()
9519 .expect("retry policy");
9520 retry_policy.non_retryable_error_types = vec!["AnotherPermanentError".to_string()];
9521
9522 let mut changed_start_to_close = retry_activity_options();
9523 changed_start_to_close.start_to_close_timeout = Some(Duration::from_secs(31));
9524 let mut changed_schedule_to_start = retry_activity_options();
9525 changed_schedule_to_start.schedule_to_start_timeout = Some(Duration::from_secs(6));
9526 let mut changed_schedule_to_close = retry_activity_options();
9527 changed_schedule_to_close.schedule_to_close_timeout = Some(Duration::from_secs(91));
9528 let mut changed_heartbeat = retry_activity_options();
9529 changed_heartbeat.heartbeat_timeout = Some(Duration::from_secs(11));
9530
9531 let cases = [
9532 (changed_queue, "activity_task_queue_mismatch"),
9533 (changed_max_attempts, "activity_retry_policy_mismatch"),
9534 (changed_backoff, "activity_retry_policy_mismatch"),
9535 (
9536 changed_non_retryable_types,
9537 "activity_retry_policy_mismatch",
9538 ),
9539 (changed_start_to_close, "activity_retry_policy_mismatch"),
9540 (changed_schedule_to_start, "activity_retry_policy_mismatch"),
9541 (changed_schedule_to_close, "activity_retry_policy_mismatch"),
9542 (changed_heartbeat, "activity_retry_policy_mismatch"),
9543 ];
9544
9545 for (options, expected_reason) in cases {
9546 let ctx = workflow_context(completed_retry_activity_history());
9547 let mut call = Box::pin(ctx.activity_with_options("flaky", options, json!([])));
9548 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9549 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
9550 call.as_mut().poll(&mut task_context)
9551 else {
9552 panic!("changed activity options must fail replay");
9553 };
9554 assert_eq!(failure.reason, expected_reason);
9555 assert_eq!(failure.sequence, Some(1));
9556 assert!(ctx.take_commands().expect("commands").is_empty());
9557 }
9558 }
9559
9560 #[test]
9561 fn replayed_activity_rejects_changed_execution_mode_and_snapshot_version() {
9562 let cases = [
9563 (
9564 "execution_mode",
9565 json!("local"),
9566 "activity_execution_mode_mismatch",
9567 ),
9568 (
9569 "snapshot_version",
9570 json!(2),
9571 "activity_retry_policy_mismatch",
9572 ),
9573 ];
9574
9575 for (field, value, expected_reason) in cases {
9576 let mut history = completed_retry_activity_history();
9577 let activity = history[0].payload["activity"]
9578 .as_object_mut()
9579 .expect("activity snapshot");
9580 if field == "execution_mode" {
9581 activity.insert(field.to_string(), value);
9582 } else {
9583 activity["retry_policy"]
9584 .as_object_mut()
9585 .expect("retry snapshot")
9586 .insert(field.to_string(), value);
9587 }
9588
9589 let ctx = workflow_context(history);
9590 let mut call =
9591 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
9592 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9593 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
9594 call.as_mut().poll(&mut task_context)
9595 else {
9596 panic!("changed {field} must fail replay");
9597 };
9598 assert_eq!(failure.reason, expected_reason);
9599 assert_eq!(failure.sequence, Some(1));
9600 assert!(ctx.take_commands().expect("commands").is_empty());
9601 }
9602 }
9603
9604 #[test]
9605 fn replayed_legacy_activity_treats_missing_option_snapshot_as_unknown() {
9606 let mut history = completed_retry_activity_history();
9607 let activity = history[0].payload["activity"]
9608 .as_object_mut()
9609 .expect("activity snapshot");
9610 activity.remove("execution_mode");
9611 activity.remove("retry_policy");
9612
9613 let mut current = retry_activity_options();
9614 current.start_to_close_timeout = Some(Duration::from_secs(45));
9615 current.schedule_to_start_timeout = Some(Duration::from_secs(8));
9616 current.schedule_to_close_timeout = Some(Duration::from_secs(120));
9617 current.heartbeat_timeout = Some(Duration::from_secs(12));
9618
9619 let ctx = workflow_context(history);
9620 let mut call = Box::pin(ctx.activity_with_options("flaky", current, json!([])));
9621 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9622 assert!(matches!(
9623 call.as_mut().poll(&mut task_context),
9624 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
9625 ));
9626 assert!(ctx.take_commands().expect("commands").is_empty());
9627 ctx.ensure_history_consumed().expect("history consumed");
9628 }
9629
9630 #[test]
9631 fn terminal_activity_failed_after_start_returns_typed_failure() {
9632 let history = vec![
9633 history_event(
9634 "ActivityScheduled",
9635 json!({
9636 "sequence": 1,
9637 "activity_type": "flaky",
9638 "activity_execution_id": "act-terminal",
9639 "activity": {
9640 "id": "act-terminal",
9641 "sequence": 1,
9642 "type": "flaky",
9643 "queue": "critical-activities",
9644 "retry_policy": {
9645 "snapshot_version": 1,
9646 "max_attempts": 3,
9647 "backoff_seconds": [2, 4],
9648 "non_retryable_error_types": ["PermanentError"]
9649 }
9650 }
9651 }),
9652 ),
9653 history_event(
9654 "ActivityStarted",
9655 json!({
9656 "sequence": 1,
9657 "activity_type": "flaky",
9658 "activity_execution_id": "act-terminal",
9659 "activity_attempt_id": "attempt-1",
9660 "attempt_number": 1
9661 }),
9662 ),
9663 history_event(
9664 "ActivityFailed",
9665 json!({
9666 "sequence": 1,
9667 "activity_type": "flaky",
9668 "activity_execution_id": "act-terminal",
9669 "activity_attempt_id": "attempt-1",
9670 "attempt_number": 1,
9671 "failure_id": "failure-terminal",
9672 "failure_category": "activity",
9673 "exception_type": "PermanentError",
9674 "message": "cannot retry",
9675 "non_retryable": true
9676 }),
9677 ),
9678 ];
9679 let ctx = workflow_context(history);
9680 let mut call =
9681 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
9682 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9683
9684 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
9685 call.as_mut().poll(&mut task_context)
9686 else {
9687 panic!("terminal ActivityFailed must settle the activity future");
9688 };
9689 assert_eq!(failure.kind, ActivityFailureKind::Failed);
9690 assert_eq!(
9691 failure.activity_execution_id.as_deref(),
9692 Some("act-terminal")
9693 );
9694 assert_eq!(failure.exception_type.as_deref(), Some("PermanentError"));
9695 assert!(failure.non_retryable);
9696 assert!(ctx.take_commands().expect("commands").is_empty());
9697 ctx.ensure_history_consumed().expect("history consumed");
9698 }
9699
9700 #[test]
9701 fn activity_terminal_events_return_machine_readable_failures() {
9702 let cases = [
9703 (
9704 "ActivityFailed",
9705 json!({
9706 "sequence": 1,
9707 "activity_type": "charge-card",
9708 "activity_execution_id": "act-1",
9709 "activity_attempt_id": "attempt-2",
9710 "attempt_number": 2,
9711 "failure_id": "failure-1",
9712 "failure_category": "activity",
9713 "exception_type": "PaymentDeclined",
9714 "exception_class": "payments.PaymentDeclined",
9715 "message": "card declined",
9716 "non_retryable": true
9717 }),
9718 ActivityFailureKind::Failed,
9719 "activity",
9720 ),
9721 (
9722 "ActivityCancelled",
9723 json!({
9724 "sequence": 1,
9725 "activity_type": "charge-card",
9726 "activity_execution_id": "act-1",
9727 "activity_attempt_id": "attempt-1"
9728 }),
9729 ActivityFailureKind::Cancelled,
9730 "cancelled",
9731 ),
9732 ];
9733
9734 for (event_type, payload, expected_kind, expected_reason) in cases {
9735 let ctx = workflow_context(vec![history_event(event_type, payload)]);
9736 let mut call = Box::pin(ctx.activity("charge-card", json!([])));
9737 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9738 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
9739 call.as_mut().poll(&mut task_context)
9740 else {
9741 panic!("expected terminal activity failure");
9742 };
9743 assert_eq!(failure.kind, expected_kind);
9744 assert_eq!(failure.reason, expected_reason);
9745 assert_eq!(failure.activity_execution_id.as_deref(), Some("act-1"));
9746 assert_eq!(failure.activity_type.as_deref(), Some("charge-card"));
9747 }
9748 }
9749
9750 #[test]
9751 fn every_activity_timeout_class_is_typed() {
9752 for timeout_kind in [
9753 "start_to_close",
9754 "schedule_to_start",
9755 "schedule_to_close",
9756 "heartbeat",
9757 ] {
9758 let ctx = workflow_context(vec![history_event(
9759 "ActivityTimedOut",
9760 json!({
9761 "sequence": 1,
9762 "activity_type": "slow",
9763 "activity_execution_id": "act-timeout",
9764 "activity_attempt_id": "attempt-timeout",
9765 "failure_category": "timeout",
9766 "timeout_kind": timeout_kind,
9767 "message": "deadline expired"
9768 }),
9769 )]);
9770 let mut call = Box::pin(ctx.activity("slow", json!([])));
9771 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9772 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
9773 call.as_mut().poll(&mut task_context)
9774 else {
9775 panic!("expected timeout failure");
9776 };
9777 assert_eq!(failure.kind, ActivityFailureKind::TimedOut);
9778 assert_eq!(failure.reason, timeout_kind);
9779 assert_eq!(failure.timeout_kind.as_deref(), Some(timeout_kind));
9780 assert_eq!(failure.failure_category.as_deref(), Some("timeout"));
9781 }
9782 }
9783
9784 #[test]
9785 fn workflow_sleep_emits_one_durable_timer_and_rounds_up() {
9786 let ctx = workflow_context(Vec::new());
9787 let mut sleep = Box::pin(ctx.sleep(Duration::from_millis(1_001)));
9788 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9789
9790 assert!(matches!(
9791 sleep.as_mut().poll(&mut task_context),
9792 Poll::Pending
9793 ));
9794 assert!(matches!(
9795 sleep.as_mut().poll(&mut task_context),
9796 Poll::Pending
9797 ));
9798
9799 let commands = ctx.take_commands().expect("timer command");
9800 assert_eq!(
9801 commands,
9802 vec![json!({
9803 "type": "start_timer",
9804 "delay_seconds": 2,
9805 })]
9806 );
9807 }
9808
9809 #[test]
9810 fn workflow_sleep_replays_matching_schedule_and_fire_without_a_command() {
9811 let history = vec![
9812 history_event(
9813 "TimerScheduled",
9814 json!({
9815 "sequence": 1,
9816 "timer_id": "timer-1",
9817 "delay_seconds": 5,
9818 "fire_at": "2026-07-11T12:00:05Z",
9819 }),
9820 ),
9821 history_event(
9822 "TimerFired",
9823 json!({
9824 "sequence": 1,
9825 "timer_id": "timer-1",
9826 "delay_seconds": 5,
9827 "fire_at": "2026-07-11T12:00:05Z",
9828 "fired_at": "2026-07-11T12:00:05Z",
9829 }),
9830 ),
9831 ];
9832
9833 for _restart in 0..2 {
9834 let ctx = workflow_context(history.clone());
9835 let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(5)));
9836 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9837 assert!(matches!(
9838 sleep.as_mut().poll(&mut task_context),
9839 Poll::Ready(Ok(()))
9840 ));
9841 assert!(ctx.take_commands().expect("commands").is_empty());
9842 ctx.ensure_history_consumed().expect("history consumed");
9843 }
9844 }
9845
9846 #[test]
9847 fn workflow_sleep_rejects_changed_delay_during_replay() {
9848 let ctx = workflow_context(vec![
9849 history_event(
9850 "TimerScheduled",
9851 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
9852 ),
9853 history_event(
9854 "TimerFired",
9855 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
9856 ),
9857 ]);
9858 let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(500)));
9859 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9860
9861 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
9862 sleep.as_mut().poll(&mut task_context)
9863 else {
9864 panic!("changed timer delay must be rejected");
9865 };
9866 assert_eq!(failure.reason, "timer_delay_mismatch");
9867 assert_eq!(failure.sequence, Some(1));
9868 }
9869
9870 #[test]
9871 fn workflow_history_rejects_unpaired_or_mismatched_timer_events() {
9872 let lone_fire = WorkflowState::new(
9873 vec![history_event(
9874 "TimerFired",
9875 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
9876 )],
9877 "rust-workers".to_string(),
9878 JSON_CODEC.to_string(),
9879 None,
9880 )
9881 .expect_err("TimerFired requires TimerScheduled");
9882 assert!(matches!(
9883 lone_fire,
9884 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
9885 if reason == "timer_schedule_missing_or_duplicate"
9886 ));
9887
9888 let wrong_identity = WorkflowState::new(
9889 vec![
9890 history_event(
9891 "TimerScheduled",
9892 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
9893 ),
9894 history_event(
9895 "TimerFired",
9896 json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
9897 ),
9898 ],
9899 "rust-workers".to_string(),
9900 JSON_CODEC.to_string(),
9901 None,
9902 )
9903 .expect_err("fire must match scheduled timer identity");
9904 assert!(matches!(
9905 wrong_identity,
9906 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
9907 if reason == "timer_identity_mismatch"
9908 ));
9909
9910 let duplicate_fire = WorkflowState::new(
9911 vec![
9912 history_event(
9913 "TimerScheduled",
9914 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
9915 ),
9916 history_event(
9917 "TimerFired",
9918 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
9919 ),
9920 history_event(
9921 "TimerFired",
9922 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
9923 ),
9924 ],
9925 "rust-workers".to_string(),
9926 JSON_CODEC.to_string(),
9927 None,
9928 )
9929 .expect_err("a durable timer cannot fire twice");
9930 assert!(matches!(
9931 duplicate_fire,
9932 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
9933 if reason == "duplicate_timer_fire"
9934 ));
9935
9936 let wrong_fired_delay = WorkflowState::new(
9937 vec![
9938 history_event(
9939 "TimerScheduled",
9940 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
9941 ),
9942 history_event(
9943 "TimerFired",
9944 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 6}),
9945 ),
9946 ],
9947 "rust-workers".to_string(),
9948 JSON_CODEC.to_string(),
9949 None,
9950 )
9951 .expect_err("timer schedule and fire delays must agree");
9952 assert!(matches!(
9953 wrong_fired_delay,
9954 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
9955 if reason == "timer_history_delay_mismatch"
9956 ));
9957 }
9958
9959 #[test]
9960 fn replay_rejects_activity_moved_before_recorded_timer() {
9961 let ctx = workflow_context(vec![
9962 history_event(
9963 "TimerScheduled",
9964 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
9965 ),
9966 history_event(
9967 "TimerFired",
9968 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
9969 ),
9970 history_event(
9971 "ActivityCompleted",
9972 json!({
9973 "sequence": 2,
9974 "activity_type": "after-timer",
9975 "payload_codec": "json",
9976 "result": {"codec": "json", "blob": "\"done\""},
9977 }),
9978 ),
9979 ]);
9980 let mut activity = Box::pin(ctx.activity("after-timer", json!([])));
9981 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9982
9983 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
9984 activity.as_mut().poll(&mut task_context)
9985 else {
9986 panic!("reordered durable command must be rejected");
9987 };
9988 assert_eq!(failure.reason, "recorded_command_mismatch");
9989 assert_eq!(failure.sequence, Some(1));
9990 assert_eq!(failure.expected.as_deref(), Some("timer"));
9991 assert_eq!(failure.actual.as_deref(), Some("activity:after-timer"));
9992 }
9993
9994 #[test]
9995 fn workflow_context_emits_a_typed_named_signal_wait() {
9996 let ctx = workflow_context(Vec::new());
9997 let mut signal = Box::pin(ctx.wait_signal("finish"));
9998 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9999
10000 assert!(matches!(
10001 signal.as_mut().poll(&mut task_context),
10002 Poll::Pending
10003 ));
10004 assert_eq!(
10005 ctx.take_commands().expect("signal-wait command"),
10006 vec![json!({
10007 "type": "open_signal_wait",
10008 "signal_name": "finish",
10009 })]
10010 );
10011 }
10012
10013 #[test]
10014 fn condition_wait_history_does_not_resolve_a_typed_signal_wait() {
10015 let ctx = workflow_context(vec![
10016 history_event(
10017 "ConditionWaitOpened",
10018 json!({"sequence": 1, "condition_key": "signal:finish"}),
10019 ),
10020 history_event(
10021 "ConditionWaitSatisfied",
10022 json!({"sequence": 1, "condition_key": "signal:finish"}),
10023 ),
10024 history_event(
10025 "SignalReceived",
10026 json!({"signal_name": "finish", "arguments": []}),
10027 ),
10028 ]);
10029 let mut signal = Box::pin(ctx.wait_signal("finish"));
10030 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10031
10032 assert!(matches!(
10033 signal.as_mut().poll(&mut task_context),
10034 Poll::Pending
10035 ));
10036 assert_eq!(
10037 ctx.take_commands().expect("typed signal-wait command"),
10038 vec![json!({
10039 "type": "open_signal_wait",
10040 "signal_name": "finish",
10041 })]
10042 );
10043 }
10044
10045 #[test]
10046 fn replay_orders_signal_waits_and_timers_in_one_command_stream() {
10047 let signal_then_timer = vec![
10048 history_event(
10049 "SignalWaitOpened",
10050 json!({"sequence": 1, "signal_name": "go"}),
10051 ),
10052 history_event(
10053 "SignalApplied",
10054 json!({
10055 "sequence": 1,
10056 "signal_name": "go",
10057 "value": {"codec": "json", "blob": "[\"now\"]"},
10058 }),
10059 ),
10060 history_event(
10061 "TimerScheduled",
10062 json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
10063 ),
10064 history_event(
10065 "TimerFired",
10066 json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
10067 ),
10068 ];
10069
10070 let ctx = workflow_context(signal_then_timer.clone());
10071 let mut signal = Box::pin(ctx.wait_signal("go"));
10072 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10073 assert!(matches!(
10074 signal.as_mut().poll(&mut task_context),
10075 Poll::Ready(Ok(arguments)) if arguments == vec![json!("now")]
10076 ));
10077 let mut timer = Box::pin(ctx.sleep(Duration::from_secs(5)));
10078 assert!(matches!(
10079 timer.as_mut().poll(&mut task_context),
10080 Poll::Ready(Ok(()))
10081 ));
10082 ctx.ensure_history_consumed()
10083 .expect("signal and timer history consumed in order");
10084
10085 let reordered = workflow_context(signal_then_timer);
10086 let mut timer_first = Box::pin(reordered.sleep(Duration::from_secs(5)));
10087 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10088 timer_first.as_mut().poll(&mut task_context)
10089 else {
10090 panic!("timer cannot consume signal-wait-first history");
10091 };
10092 assert_eq!(failure.reason, "recorded_command_mismatch");
10093 assert_eq!(failure.sequence, Some(1));
10094 assert_eq!(failure.expected.as_deref(), Some("signal wait"));
10095
10096 let timer_then_signal = vec![
10097 history_event(
10098 "TimerScheduled",
10099 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10100 ),
10101 history_event(
10102 "TimerFired",
10103 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10104 ),
10105 history_event(
10106 "SignalWaitOpened",
10107 json!({"sequence": 2, "signal_name": "go"}),
10108 ),
10109 history_event(
10110 "SignalApplied",
10111 json!({
10112 "sequence": 2,
10113 "signal_name": "go",
10114 "value": {"codec": "json", "blob": "[]"},
10115 }),
10116 ),
10117 ];
10118 let reordered = workflow_context(timer_then_signal);
10119 let mut signal_first = Box::pin(reordered.wait_signal("go"));
10120 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10121 signal_first.as_mut().poll(&mut task_context)
10122 else {
10123 panic!("signal wait cannot consume timer-first history");
10124 };
10125 assert_eq!(failure.reason, "recorded_command_mismatch");
10126 assert_eq!(failure.sequence, Some(1));
10127 assert_eq!(failure.expected.as_deref(), Some("timer"));
10128 }
10129
10130 #[test]
10131 fn workflow_history_rejects_duplicate_or_colliding_command_sequences() {
10132 let duplicate_timer = WorkflowState::new(
10133 vec![
10134 history_event(
10135 "TimerScheduled",
10136 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10137 ),
10138 history_event(
10139 "TimerScheduled",
10140 json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
10141 ),
10142 ],
10143 "rust-workers".to_string(),
10144 JSON_CODEC.to_string(),
10145 None,
10146 )
10147 .expect_err("one workflow sequence cannot schedule two timers");
10148 assert!(matches!(
10149 duplicate_timer,
10150 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10151 if reason == "timer_schedule_missing_or_duplicate"
10152 ));
10153
10154 let colliding_kinds = WorkflowState::new(
10155 vec![
10156 history_event(
10157 "TimerScheduled",
10158 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10159 ),
10160 history_event(
10161 "ActivityCompleted",
10162 json!({"sequence": 1, "activity_type": "same-sequence"}),
10163 ),
10164 ],
10165 "rust-workers".to_string(),
10166 JSON_CODEC.to_string(),
10167 None,
10168 )
10169 .expect_err("one workflow sequence cannot identify two command kinds");
10170 assert!(matches!(
10171 colliding_kinds,
10172 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10173 if reason == "durable_command_sequence_collision"
10174 ));
10175
10176 let duplicate_signal_wait = WorkflowState::new(
10177 vec![
10178 history_event(
10179 "SignalWaitOpened",
10180 json!({"sequence": 1, "signal_name": "go"}),
10181 ),
10182 history_event(
10183 "SignalWaitOpened",
10184 json!({"sequence": 1, "signal_name": "go"}),
10185 ),
10186 ],
10187 "rust-workers".to_string(),
10188 JSON_CODEC.to_string(),
10189 None,
10190 )
10191 .expect_err("one workflow sequence cannot open two signal waits");
10192 assert!(matches!(
10193 duplicate_signal_wait,
10194 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10195 if reason == "signal_wait_open_missing_or_duplicate"
10196 ));
10197 }
10198
10199 #[test]
10200 fn workflow_history_accepts_a_first_command_after_global_sequence_gaps() {
10201 let result = encode_value_envelope(&json!({"captured": true}), JSON_CODEC)
10202 .expect("side-effect result");
10203 let ctx = workflow_context(vec![history_event(
10204 "SideEffectRecorded",
10205 json!({"sequence": 99, "result": result}),
10206 )]);
10207
10208 let replayed: Value = ctx
10209 .side_effect(|| panic!("recorded side effect must not run"))
10210 .expect("positive global workflow sequence is valid");
10211 assert_eq!(replayed, json!({"captured": true}));
10212 ctx.ensure_history_consumed().expect("history consumed");
10213 }
10214
10215 #[test]
10216 fn workflow_history_rejects_zero_and_descending_command_sequences() {
10217 let result =
10218 encode_value_envelope(&json!("captured"), JSON_CODEC).expect("side-effect result");
10219 let zero = WorkflowState::new(
10220 vec![history_event(
10221 "SideEffectRecorded",
10222 json!({"sequence": 0, "result": result.clone()}),
10223 )],
10224 "rust-workers".to_string(),
10225 JSON_CODEC.to_string(),
10226 None,
10227 )
10228 .expect_err("durable command sequences must be positive");
10229 assert!(matches!(
10230 zero,
10231 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10232 if reason == "durable_command_sequence_invalid"
10233 ));
10234
10235 let descending = WorkflowState::new(
10236 vec![
10237 history_event(
10238 "SideEffectRecorded",
10239 json!({"sequence": 3, "result": result}),
10240 ),
10241 history_event(
10242 "VersionMarkerRecorded",
10243 json!({
10244 "sequence": 2,
10245 "change_id": "descending-marker",
10246 "version": 1,
10247 "min_supported": 1,
10248 "max_supported": 1,
10249 }),
10250 ),
10251 ],
10252 "rust-workers".to_string(),
10253 JSON_CODEC.to_string(),
10254 None,
10255 )
10256 .expect_err("new durable commands must remain strictly ordered");
10257 let Error::NonDeterministicReplay(failure) = descending else {
10258 panic!("expected typed replay failure");
10259 };
10260 assert_eq!(failure.reason, "durable_command_sequence_mismatch");
10261 assert_eq!(failure.sequence, Some(2));
10262 assert_eq!(
10263 failure.expected.as_deref(),
10264 Some("workflow sequence greater than 3")
10265 );
10266 assert_eq!(failure.actual.as_deref(), Some("2"));
10267 }
10268
10269 #[test]
10270 fn workflow_task_replay_completes_after_signals_create_sequence_gaps() {
10271 fn worker() -> Worker {
10272 let client = Client::new("http://127.0.0.1:8080").expect("client");
10273 let mut worker = Worker::new(client, "rust-workers");
10274 worker.register_workflow("rust.finish-after-gaps", |ctx, _input| async move {
10275 ctx.wait_signal("finish").await?;
10276 let marker: String =
10277 ctx.side_effect(|| panic!("recorded side effect must not run"))?;
10278 assert_eq!(marker, "after-finish");
10279 Ok(json!("finished"))
10280 });
10281 worker
10282 }
10283
10284 let marker =
10285 encode_value_envelope(&json!("after-finish"), JSON_CODEC).expect("side-effect result");
10286 let task = workflow_task(
10287 "rust.finish-after-gaps",
10288 vec![
10289 history_event(
10290 "SignalWaitOpened",
10291 json!({"sequence": 1, "signal_name": "finish"}),
10292 ),
10293 history_event(
10294 "SignalReceived",
10295 json!({
10296 "signal_id": "increment-3",
10297 "signal_name": "increment",
10298 "workflow_sequence": 2,
10299 "payload_codec": "json",
10300 "arguments": {"codec": "json", "blob": "[3]"},
10301 }),
10302 ),
10303 history_event(
10304 "SignalReceived",
10305 json!({
10306 "signal_id": "increment-5",
10307 "signal_name": "increment",
10308 "workflow_sequence": 3,
10309 "payload_codec": "json",
10310 "arguments": {"codec": "json", "blob": "[5]"},
10311 }),
10312 ),
10313 history_event(
10314 "SignalReceived",
10315 json!({
10316 "signal_id": "finish",
10317 "signal_name": "finish",
10318 "workflow_sequence": 4,
10319 "payload_codec": "json",
10320 "arguments": {"codec": "json", "blob": "[]"},
10321 }),
10322 ),
10323 history_event(
10324 "SignalApplied",
10325 json!({
10326 "sequence": 1,
10327 "signal_id": "finish",
10328 "signal_name": "finish",
10329 "payload_codec": "json",
10330 "value": {"codec": "json", "blob": "[]"},
10331 }),
10332 ),
10333 history_event(
10334 "SideEffectRecorded",
10335 json!({"sequence": 5, "result": marker}),
10336 ),
10337 ],
10338 JSON_CODEC,
10339 );
10340
10341 for _original_or_cold_worker in 0..2 {
10342 let commands = worker()
10343 .execute_workflow_task(task.clone())
10344 .expect("signal gaps preserve deterministic replay");
10345 assert_eq!(commands.len(), 1, "replay emits only terminal completion");
10346 assert_eq!(commands[0]["type"], "complete_workflow");
10347 assert_eq!(
10348 decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("workflow output"),
10349 json!("finished")
10350 );
10351 }
10352 }
10353
10354 #[test]
10355 fn workflow_sleep_rejects_unrepresentable_rounded_duration() {
10356 let ctx = workflow_context(Vec::new());
10357 let mut sleep = Box::pin(ctx.start_timer(Duration::new(u64::MAX, 1)));
10358 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10359 assert!(matches!(
10360 sleep.as_mut().poll(&mut task_context),
10361 Poll::Ready(Err(Error::TimerDurationOverflow))
10362 ));
10363 assert!(ctx.take_commands().expect("commands").is_empty());
10364 }
10365
10366 #[test]
10367 fn workflow_task_replay_completes_without_rescheduling_recorded_commands() {
10368 let client = Client::new("http://127.0.0.1:8080").expect("client");
10369 let mut worker = Worker::new(client, "rust-workers");
10370 worker.register_workflow("rust.timer", |ctx, _input| async move {
10371 ctx.sleep(Duration::from_secs(5)).await?;
10372 ctx.activity("after-timer", json!([])).await
10373 });
10374
10375 let task = |history_events| WorkflowTask {
10376 task_id: "wft-rust-timer-1".to_string(),
10377 workflow_id: Some("wf-rust-timer".to_string()),
10378 run_id: Some("run-rust-timer".to_string()),
10379 workflow_type: "rust.timer".to_string(),
10380 payload_codec: JSON_CODEC.to_string(),
10381 arguments: Some(json!({"codec": "json", "blob": "[]"})),
10382 history_events,
10383 total_history_events: None,
10384 history_size_bytes: None,
10385 continue_as_new_recommended: None,
10386 history_budget_pressure: None,
10387 next_history_page_token: None,
10388 workflow_task_attempt: 1,
10389 workflow_signal_id: None,
10390 signal_name: None,
10391 signal_arguments: None,
10392 workflow_update_id: None,
10393 update_name: None,
10394 lease_owner: Some("rust-worker".to_string()),
10395 };
10396
10397 let initial = worker
10398 .execute_workflow_task(task(Vec::new()))
10399 .expect("initial timer task");
10400 assert_eq!(
10401 initial,
10402 vec![json!({"type": "start_timer", "delay_seconds": 5})]
10403 );
10404
10405 let replayed = worker
10406 .execute_workflow_task(task(vec![
10407 history_event(
10408 "TimerScheduled",
10409 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10410 ),
10411 history_event(
10412 "TimerFired",
10413 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10414 ),
10415 history_event(
10416 "ActivityCompleted",
10417 json!({
10418 "sequence": 2,
10419 "activity_type": "after-timer",
10420 "payload_codec": "json",
10421 "result": {"codec": "json", "blob": "\"done\""},
10422 }),
10423 ),
10424 ]))
10425 .expect("replayed workflow task");
10426 assert_eq!(replayed.len(), 1);
10427 assert_eq!(replayed[0]["type"], "complete_workflow");
10428 assert_eq!(
10429 decode_wire_value(&replayed[0]["result"], JSON_CODEC).expect("result"),
10430 json!("done")
10431 );
10432 }
10433
10434 #[test]
10435 fn workflow_continue_as_new_emits_arguments_type_and_queue_once() {
10436 let client = Client::new("http://127.0.0.1:8080").expect("client");
10437 let mut worker = Worker::new(client, "rust-workers");
10438 worker.register_workflow("rust.continue", |ctx, _input| async move {
10439 ctx.continue_as_new_with_options(
10440 ContinueAsNewOptions::new()
10441 .workflow_type("rust.next")
10442 .task_queue("next-workers"),
10443 json!([2, {"cursor": "next"}]),
10444 )
10445 });
10446
10447 let commands = worker
10448 .execute_workflow_task(workflow_task("rust.continue", Vec::new(), DEFAULT_CODEC))
10449 .expect("continue-as-new command");
10450
10451 assert_eq!(commands.len(), 1);
10452 assert_eq!(commands[0]["type"], "continue_as_new");
10453 assert_eq!(commands[0]["workflow_type"], "rust.next");
10454 assert_eq!(commands[0]["queue"], "next-workers");
10455 assert_eq!(
10456 decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC)
10457 .expect("continue-as-new arguments"),
10458 json!([2, {"cursor": "next"}])
10459 );
10460 }
10461
10462 #[test]
10463 fn continue_as_new_preserves_typed_arguments() {
10464 let client = Client::new("http://127.0.0.1:8080").expect("client");
10465 let mut worker = Worker::new(client, "rust-workers");
10466 worker.register_workflow_avro_value("rust.typed-continue", |ctx, _input| async move {
10467 ctx.continue_as_new(AvroValue::Array(vec![typed_fidelity_probe()]))?;
10468 unreachable!("continue-as-new returns a control-flow error")
10469 });
10470
10471 let commands = worker
10472 .execute_workflow_task(workflow_task(
10473 "rust.typed-continue",
10474 Vec::new(),
10475 DEFAULT_CODEC,
10476 ))
10477 .expect("typed continue-as-new command");
10478
10479 assert_eq!(commands[0]["type"], "continue_as_new");
10480 assert_eq!(
10481 decode_wire_avro_value(&commands[0]["arguments"], DEFAULT_CODEC)
10482 .expect("typed continue arguments"),
10483 AvroValue::Array(vec![typed_fidelity_probe()])
10484 );
10485 }
10486
10487 #[test]
10488 fn recorded_continue_as_new_is_consumed_without_duplicate_successor_command() {
10489 let client = Client::new("http://127.0.0.1:8080").expect("client");
10490 let mut worker = Worker::new(client, "rust-workers");
10491 worker.register_workflow("rust.continue", |ctx, _input| async move {
10492 ctx.continue_as_new(json!([2]))
10493 });
10494 let task = workflow_task(
10495 "rust.continue",
10496 vec![history_event(
10497 "WorkflowContinuedAsNew",
10498 json!({"sequence": 1, "continued_to_run_id": "run-next"}),
10499 )],
10500 JSON_CODEC,
10501 );
10502
10503 for _worker_restart_or_redelivery in 0..2 {
10504 let commands = worker
10505 .execute_workflow_task(task.clone())
10506 .expect("recorded transition replays");
10507 assert!(
10508 commands.is_empty(),
10509 "replay must not emit another successor"
10510 );
10511 }
10512 }
10513
10514 #[test]
10515 fn continue_as_new_rejects_invalid_overrides_before_emitting_a_command() {
10516 let ctx = workflow_context(Vec::new());
10517 let error = ctx
10518 .continue_as_new_with_options(ContinueAsNewOptions::new().task_queue(" "), json!([1]))
10519 .expect_err("blank queue must be rejected");
10520
10521 let Error::InvalidContinueAsNewOptions(error) = error else {
10522 panic!("expected typed continue-as-new validation error");
10523 };
10524 assert_eq!(error.field, "task_queue");
10525 assert!(ctx.take_commands().expect("commands").is_empty());
10526 }
10527
10528 #[test]
10529 fn workflow_context_exposes_server_history_budget() {
10530 let client = Client::new("http://127.0.0.1:8080").expect("client");
10531 let mut worker = Worker::new(client, "rust-workers");
10532 worker.register_workflow("rust.history-budget", |ctx, _input| async move {
10533 let budget = ctx.history_budget()?;
10534 Ok(json!({
10535 "events": budget.event_count,
10536 "bytes": budget.size_bytes,
10537 "recommended": budget.continue_as_new_recommended,
10538 "pressure": budget.pressure,
10539 }))
10540 });
10541 let task: WorkflowTask = serde_json::from_value(json!({
10542 "task_id": "task-history-budget",
10543 "workflow_type": "rust.history-budget",
10544 "payload_codec": JSON_CODEC,
10545 "history_events": [],
10546 "total_history_events": 480,
10547 "history_size_bytes": 1_048_576,
10548 "continue_as_new_recommended": true,
10549 "history_budget_pressure": "continue_as_new_recommended",
10550 }))
10551 .expect("published workflow task");
10552
10553 let commands = worker
10554 .execute_workflow_task(task)
10555 .expect("history-budget workflow");
10556 let result = decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("result");
10557 assert_eq!(result["events"], 480);
10558 assert_eq!(result["bytes"], 1_048_576);
10559 assert_eq!(result["recommended"], true);
10560 assert_eq!(result["pressure"], "continue_as_new_recommended");
10561 }
10562
10563 #[test]
10564 fn uncaught_workflow_handler_error_emits_terminal_failure_command() {
10565 let client = Client::new("http://127.0.0.1:8080").expect("client");
10566 let mut worker = Worker::new(client, "rust-workers");
10567 worker.register_workflow("rust.failing", |_ctx, _input| async move {
10568 Err(Error::Codec("rust_conformance_failure".to_string()))
10569 });
10570 let task = WorkflowTask {
10571 task_id: "wft-rust-failing-1".to_string(),
10572 workflow_id: Some("wf-rust-failing".to_string()),
10573 run_id: Some("run-rust-failing".to_string()),
10574 workflow_type: "rust.failing".to_string(),
10575 payload_codec: JSON_CODEC.to_string(),
10576 arguments: Some(encode_value_envelope(&json!([]), JSON_CODEC).expect("input")),
10577 history_events: Vec::new(),
10578 total_history_events: Some(0),
10579 history_size_bytes: None,
10580 continue_as_new_recommended: None,
10581 history_budget_pressure: None,
10582 next_history_page_token: None,
10583 workflow_task_attempt: 1,
10584 workflow_signal_id: None,
10585 signal_name: None,
10586 signal_arguments: None,
10587 workflow_update_id: None,
10588 update_name: None,
10589 lease_owner: Some("rust-worker".to_string()),
10590 };
10591
10592 let commands = worker
10593 .execute_workflow_task(task)
10594 .expect("handler failure becomes a workflow command");
10595
10596 assert_eq!(commands.len(), 1);
10597 assert_eq!(commands[0]["type"], "fail_workflow");
10598 assert_eq!(commands[0]["exception_type"], "RustWorkflowError");
10599 assert_eq!(commands[0]["exception_class"], "durable_workflow::Error");
10600 assert_eq!(commands[0]["non_retryable"], false);
10601 assert_eq!(
10602 commands[0]["message"],
10603 "codec error: rust_conformance_failure"
10604 );
10605 assert_eq!(
10606 commands[0]["exception"]["message"],
10607 "codec error: rust_conformance_failure"
10608 );
10609 }
10610
10611 #[test]
10612 fn ordinary_handler_error_preserves_commands_queued_in_the_same_decision() {
10613 let client = Client::new("http://127.0.0.1:8080").expect("client");
10614 let mut worker = Worker::new(client, "rust-workers");
10615 worker.register_workflow("rust.failing-after-side-effect", |ctx, _input| async move {
10616 let _: String = ctx.side_effect(|| "captured".to_string())?;
10617 Err(Error::WorkerLoop("application failure".to_string()))
10618 });
10619
10620 let commands = worker
10621 .execute_workflow_task(workflow_task(
10622 "rust.failing-after-side-effect",
10623 Vec::new(),
10624 JSON_CODEC,
10625 ))
10626 .expect("ordinary failure remains a workflow decision");
10627
10628 assert_eq!(commands.len(), 2);
10629 assert_eq!(commands[0]["type"], "record_side_effect");
10630 assert_eq!(commands[1]["type"], "fail_workflow");
10631 }
10632
10633 #[test]
10634 fn handler_error_cannot_hide_an_unconsumed_committed_side_effect() {
10635 let client = Client::new("http://127.0.0.1:8080").expect("client");
10636 let mut worker = Worker::new(client, "rust-workers");
10637 worker.register_workflow("rust.removed-side-effect", |_ctx, _input| async move {
10638 Err(Error::WorkerLoop("application failure".to_string()))
10639 });
10640 let result =
10641 encode_value_envelope(&json!("committed"), JSON_CODEC).expect("side-effect result");
10642
10643 let error = worker
10644 .execute_workflow_task(workflow_task(
10645 "rust.removed-side-effect",
10646 vec![history_event(
10647 "SideEffectRecorded",
10648 json!({"sequence": 1, "result": result}),
10649 )],
10650 JSON_CODEC,
10651 ))
10652 .expect_err("removed committed history must not become fail_workflow");
10653
10654 let Error::NonDeterministicReplay(failure) = error else {
10655 panic!("expected typed replay failure");
10656 };
10657 assert_eq!(failure.reason, "recorded_commands_unconsumed");
10658 assert_eq!(failure.sequence, Some(1));
10659 assert_eq!(failure.expected.as_deref(), Some("side effect"));
10660 }
10661
10662 #[test]
10663 fn replay_error_discards_side_effect_queued_before_incompatible_marker_check() {
10664 let client = Client::new("http://127.0.0.1:8080").expect("client");
10665 let mut worker = Worker::new(client, "rust-workers");
10666 worker.register_workflow(
10667 "rust.side-effect-before-marker-error",
10668 |ctx, _input| async move {
10669 assert_eq!(ctx.get_version("restart-safe", 1, 1)?, 1);
10670 let _: String = ctx.side_effect(|| "must-not-commit".to_string())?;
10671 ctx.get_version("restart-safe", 2, 2)?;
10672 Ok(Value::Null)
10673 },
10674 );
10675
10676 let error = worker
10677 .execute_workflow_task(workflow_task(
10678 "rust.side-effect-before-marker-error",
10679 vec![history_event(
10680 "VersionMarkerRecorded",
10681 json!({
10682 "sequence": 1,
10683 "change_id": "restart-safe",
10684 "version": 1,
10685 "min_supported": 1,
10686 "max_supported": 1,
10687 }),
10688 )],
10689 JSON_CODEC,
10690 ))
10691 .expect_err("replay error must return no queued workflow commands");
10692
10693 let Error::NonDeterministicReplay(failure) = error else {
10694 panic!("expected typed replay failure");
10695 };
10696 assert_eq!(failure.reason, "version_marker_incompatible_range");
10697 assert_eq!(failure.sequence, Some(1));
10698 }
10699
10700 #[test]
10701 fn workflow_task_replay_keeps_recorded_unfired_timer_pending_without_rescheduling() {
10702 let client = Client::new("http://127.0.0.1:8080").expect("client");
10703 let mut worker = Worker::new(client, "rust-workers");
10704 worker.register_workflow("rust.timer.pending", |ctx, _input| async move {
10705 ctx.sleep(Duration::from_secs(5)).await?;
10706 Ok(json!({"status": "timer fired"}))
10707 });
10708
10709 let task = WorkflowTask {
10710 task_id: "wft-rust-timer-pending".to_string(),
10711 workflow_id: Some("wf-rust-timer".to_string()),
10712 run_id: Some("run-rust-timer".to_string()),
10713 workflow_type: "rust.timer.pending".to_string(),
10714 payload_codec: JSON_CODEC.to_string(),
10715 arguments: Some(json!({"codec": "json", "blob": "[]"})),
10716 history_events: vec![history_event(
10717 "TimerScheduled",
10718 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10719 )],
10720 total_history_events: Some(1),
10721 history_size_bytes: None,
10722 continue_as_new_recommended: None,
10723 history_budget_pressure: None,
10724 next_history_page_token: None,
10725 workflow_task_attempt: 1,
10726 workflow_signal_id: None,
10727 signal_name: None,
10728 signal_arguments: None,
10729 workflow_update_id: None,
10730 update_name: None,
10731 lease_owner: Some("rust-worker".to_string()),
10732 };
10733
10734 for _redelivery_or_restart in 0..2 {
10735 let commands = worker
10736 .execute_workflow_task(task.clone())
10737 .expect("recorded timer remains pending");
10738 assert!(
10739 commands.is_empty(),
10740 "recorded timer must not be rescheduled"
10741 );
10742 }
10743 }
10744
10745 #[test]
10746 fn workflow_task_rejects_recorded_command_removed_from_workflow_code() {
10747 let client = Client::new("http://127.0.0.1:8080").expect("client");
10748 let mut worker = Worker::new(client, "rust-workers");
10749 worker.register_workflow("rust.timer.removed", |_ctx, _input| async move {
10750 Ok(json!({"status": "completed"}))
10751 });
10752 let task = WorkflowTask {
10753 task_id: "wft-rust-timer-removed".to_string(),
10754 workflow_id: Some("wf-rust-timer".to_string()),
10755 run_id: Some("run-rust-timer".to_string()),
10756 workflow_type: "rust.timer.removed".to_string(),
10757 payload_codec: JSON_CODEC.to_string(),
10758 arguments: Some(json!({"codec": "json", "blob": "[]"})),
10759 history_events: vec![
10760 history_event(
10761 "TimerScheduled",
10762 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10763 ),
10764 history_event(
10765 "TimerFired",
10766 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10767 ),
10768 ],
10769 total_history_events: Some(2),
10770 history_size_bytes: None,
10771 continue_as_new_recommended: None,
10772 history_budget_pressure: None,
10773 next_history_page_token: None,
10774 workflow_task_attempt: 1,
10775 workflow_signal_id: None,
10776 signal_name: None,
10777 signal_arguments: None,
10778 workflow_update_id: None,
10779 update_name: None,
10780 lease_owner: Some("rust-worker".to_string()),
10781 };
10782
10783 let Error::NonDeterministicReplay(failure) = worker
10784 .execute_workflow_task(task)
10785 .expect_err("removed timer must fail replay")
10786 else {
10787 panic!("expected typed replay failure");
10788 };
10789 assert_eq!(failure.reason, "recorded_commands_unconsumed");
10790 assert_eq!(failure.sequence, Some(1));
10791 }
10792
10793 #[test]
10794 fn workflow_context_emits_explicit_child_workflow_contract() {
10795 let ctx = WorkflowContext {
10796 state: Arc::new(Mutex::new(
10797 WorkflowState::new_with_identity(
10798 Vec::new(),
10799 Some("wf-parent".to_string()),
10800 Some("run-parent".to_string()),
10801 "parent-workers".to_string(),
10802 JSON_CODEC.to_string(),
10803 None,
10804 )
10805 .expect("workflow state"),
10806 )),
10807 };
10808 let options = ChildWorkflowOptions::new("python-workers")
10809 .parent_close_policy(ParentClosePolicy::RequestCancel)
10810 .retry_policy(ChildWorkflowRetryPolicy {
10811 max_attempts: Some(3),
10812 backoff_seconds: vec![1, 5],
10813 non_retryable_error_types: vec!["ValidationError".to_string()],
10814 })
10815 .execution_timeout_seconds(600)
10816 .run_timeout_seconds(120);
10817 let mut call = Box::pin(ctx.start_child_workflow(
10818 "python.fulfil-order",
10819 options,
10820 json!([{"order_id": "order-42"}]),
10821 ));
10822 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10823
10824 assert!(matches!(
10825 call.as_mut().poll(&mut task_context),
10826 Poll::Pending
10827 ));
10828 let commands = ctx.take_commands().expect("commands");
10829 assert_eq!(commands.len(), 1);
10830 let command = &commands[0];
10831 assert_eq!(command["type"], "start_child_workflow");
10832 assert_eq!(command["workflow_type"], "python.fulfil-order");
10833 assert_eq!(command["queue"], "python-workers");
10834 assert_eq!(command["parent_close_policy"], "request_cancel");
10835 assert_eq!(command["retry_policy"]["max_attempts"], 3);
10836 assert_eq!(command["execution_timeout_seconds"], 600);
10837 assert_eq!(command["run_timeout_seconds"], 120);
10838 assert_eq!(
10839 decode_wire_value(&command["arguments"], JSON_CODEC).expect("child args"),
10840 json!([{"order_id": "order-42"}])
10841 );
10842 }
10843
10844 fn child_parent_worker() -> Worker {
10845 let client = Client::new("http://127.0.0.1:8080").expect("client");
10846 let mut worker = Worker::new(client, "rust-parent-workers");
10847 worker.register_workflow("rust.parent", |ctx, _input| async move {
10848 let child = ctx
10849 .start_child_workflow(
10850 "python.child",
10851 ChildWorkflowOptions::new("python-child-workers")
10852 .parent_close_policy(ParentClosePolicy::Terminate),
10853 json!([{"codec_probe": [1, true, "rust"]}]),
10854 )
10855 .await?;
10856 Ok(json!({
10857 "parent_workflow_id": child.parent.workflow_id,
10858 "parent_run_id": child.parent.run_id,
10859 "child_workflow_id": child.child.workflow_id,
10860 "child_run_id": child.child.run_id,
10861 "child_workflow_type": child.child_workflow_type,
10862 "result": child.result,
10863 }))
10864 });
10865 worker
10866 }
10867
10868 fn child_parent_task(event_type: &str, payload: Value) -> WorkflowTask {
10869 WorkflowTask {
10870 task_id: "wft-child-parent".to_string(),
10871 workflow_id: Some("wf-parent".to_string()),
10872 run_id: Some("run-parent".to_string()),
10873 workflow_type: "rust.parent".to_string(),
10874 payload_codec: JSON_CODEC.to_string(),
10875 arguments: Some(encode_value_envelope(&json!([]), JSON_CODEC).expect("input")),
10876 history_events: vec![
10877 HistoryEvent {
10878 event_type: "ChildWorkflowScheduled".to_string(),
10879 payload: json!({
10880 "sequence": 1,
10881 "child_call_id": "call-child",
10882 "child_workflow_instance_id": "wf-child",
10883 "child_workflow_run_id": "run-child",
10884 "child_workflow_type": "python.child",
10885 }),
10886 raw: HashMap::new(),
10887 },
10888 HistoryEvent {
10889 event_type: event_type.to_string(),
10890 payload,
10891 raw: HashMap::new(),
10892 },
10893 ],
10894 total_history_events: Some(2),
10895 history_size_bytes: None,
10896 continue_as_new_recommended: None,
10897 history_budget_pressure: None,
10898 next_history_page_token: None,
10899 workflow_task_attempt: 1,
10900 workflow_signal_id: None,
10901 signal_name: None,
10902 signal_arguments: None,
10903 workflow_update_id: None,
10904 update_name: None,
10905 lease_owner: Some("rust-worker".to_string()),
10906 }
10907 }
10908
10909 #[test]
10910 fn committed_child_result_replays_without_starting_a_duplicate() {
10911 let worker = child_parent_worker();
10912 let task = child_parent_task(
10913 "ChildRunCompleted",
10914 json!({
10915 "sequence": 1,
10916 "child_call_id": "call-child",
10917 "child_workflow_instance_id": "wf-child",
10918 "child_workflow_run_id": "run-child",
10919 "child_workflow_type": "python.child",
10920 "payload_codec": "json",
10921 "result": {"codec": "json", "blob": "{\"from\":\"python\",\"ok\":true}"},
10922 }),
10923 );
10924
10925 for _restart in 0..2 {
10926 let commands = worker
10927 .execute_workflow_task(task.clone())
10928 .expect("replayed parent task");
10929 assert_eq!(commands.len(), 1);
10930 assert_eq!(commands[0]["type"], "complete_workflow");
10931 assert!(!commands
10932 .iter()
10933 .any(|command| command["type"] == "start_child_workflow"));
10934 let output =
10935 decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("parent output");
10936 assert_eq!(output["parent_workflow_id"], "wf-parent");
10937 assert_eq!(output["parent_run_id"], "run-parent");
10938 assert_eq!(output["child_workflow_id"], "wf-child");
10939 assert_eq!(output["child_run_id"], "run-child");
10940 assert_eq!(output["result"], json!({"from": "python", "ok": true}));
10941 }
10942 }
10943
10944 #[test]
10945 fn typed_child_arguments_and_results_survive_replay() {
10946 let client = Client::new("http://127.0.0.1:8080").expect("client");
10947 let mut worker = Worker::new(client, "rust-parent-workers");
10948 worker.register_workflow_avro_value("rust.typed-parent", |ctx, _input| async move {
10949 let child = ctx
10950 .start_child_workflow_avro_value(
10951 "python.typed-child",
10952 ChildWorkflowOptions::new("python-workers"),
10953 AvroValue::Array(vec![typed_fidelity_probe()]),
10954 )
10955 .await?;
10956 Ok(child.result)
10957 });
10958
10959 let initial = worker
10960 .execute_workflow_task(workflow_task(
10961 "rust.typed-parent",
10962 Vec::new(),
10963 DEFAULT_CODEC,
10964 ))
10965 .expect("typed child start");
10966 assert_eq!(initial[0]["type"], "start_child_workflow");
10967 assert_eq!(
10968 decode_wire_avro_value(&initial[0]["arguments"], DEFAULT_CODEC)
10969 .expect("typed child arguments"),
10970 AvroValue::Array(vec![typed_fidelity_probe()])
10971 );
10972
10973 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
10974 .expect("typed child result");
10975 let task = workflow_task(
10976 "rust.typed-parent",
10977 vec![
10978 history_event(
10979 "ChildWorkflowScheduled",
10980 json!({
10981 "sequence": 1,
10982 "child_call_id": "call-typed",
10983 "child_workflow_instance_id": "wf-child",
10984 "child_workflow_run_id": "run-child",
10985 "child_workflow_type": "python.typed-child",
10986 }),
10987 ),
10988 history_event(
10989 "ChildRunCompleted",
10990 json!({
10991 "sequence": 1,
10992 "child_call_id": "call-typed",
10993 "child_workflow_instance_id": "wf-child",
10994 "child_workflow_run_id": "run-child",
10995 "child_workflow_type": "python.typed-child",
10996 "payload_codec": DEFAULT_CODEC,
10997 "result": result,
10998 }),
10999 ),
11000 ],
11001 DEFAULT_CODEC,
11002 );
11003
11004 let commands = worker
11005 .execute_workflow_task(task)
11006 .expect("typed child replay");
11007 assert_eq!(commands[0]["type"], "complete_workflow");
11008 assert_eq!(
11009 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
11010 .expect("typed parent result"),
11011 typed_fidelity_probe()
11012 );
11013 }
11014
11015 #[test]
11016 fn pending_child_replays_after_restart_without_starting_a_duplicate() {
11017 let worker = child_parent_worker();
11018 let mut task = child_parent_task("unused", Value::Null);
11019 task.history_events.truncate(1);
11020 task.total_history_events = Some(1);
11021
11022 for _redelivery_or_restart in 0..2 {
11023 let commands = worker
11024 .execute_workflow_task(task.clone())
11025 .expect("recorded child remains pending");
11026 assert!(
11027 commands.is_empty(),
11028 "recorded pending child must not be started again"
11029 );
11030 }
11031 }
11032
11033 #[test]
11034 fn child_cancellation_becomes_stable_parent_failure_command() {
11035 let worker = child_parent_worker();
11036 let task = child_parent_task(
11037 "ChildRunCancelled",
11038 json!({
11039 "sequence": 1,
11040 "child_workflow_instance_id": "wf-child",
11041 "child_workflow_run_id": "run-child",
11042 "child_workflow_type": "python.child",
11043 "failure_id": "failure-child",
11044 "failure_category": "cancelled",
11045 "message": "cancelled by parent-close policy",
11046 }),
11047 );
11048
11049 let commands = worker
11050 .execute_workflow_task(task)
11051 .expect("parent settlement");
11052 assert_eq!(commands.len(), 1);
11053 assert_eq!(commands[0]["type"], "fail_workflow");
11054 assert_eq!(commands[0]["exception_type"], "ChildWorkflowCancelled");
11055 assert_eq!(
11056 commands[0]["exception"]["properties"]["reason"],
11057 "cancelled"
11058 );
11059 assert_eq!(
11060 commands[0]["exception"]["properties"]["child_workflow_run_id"],
11061 "run-child"
11062 );
11063 }
11064
11065 #[test]
11066 fn workflow_can_handle_typed_child_failure() {
11067 let client = Client::new("http://127.0.0.1:8080").expect("client");
11068 let mut worker = Worker::new(client, "rust-parent-workers");
11069 worker.register_workflow("rust.handled-parent", |ctx, _input| async move {
11070 match ctx
11071 .start_child_workflow(
11072 "python.child",
11073 ChildWorkflowOptions::new("python-child-workers"),
11074 json!([]),
11075 )
11076 .await
11077 {
11078 Err(Error::ChildWorkflowFailed(failure)) => Ok(json!({
11079 "reason": failure.reason,
11080 "failure_id": failure.failure_id,
11081 "exception_class": failure.exception_class,
11082 "child_run_id": failure.child_workflow_run_id,
11083 })),
11084 Err(error) => Err(error),
11085 Ok(_) => Err(Error::WorkerLoop(
11086 "child unexpectedly succeeded".to_string(),
11087 )),
11088 }
11089 });
11090 let mut task = child_parent_task(
11091 "ChildRunFailed",
11092 json!({
11093 "sequence": 1,
11094 "child_workflow_instance_id": "wf-child",
11095 "child_workflow_run_id": "run-child",
11096 "child_workflow_type": "python.child",
11097 "failure_id": "failure-child",
11098 "failure_category": "child_workflow",
11099 "message": "payment rejected",
11100 "exception": {
11101 "type": "PaymentRejected",
11102 "class": "payments.PaymentRejected",
11103 "message": "payment rejected"
11104 }
11105 }),
11106 );
11107 task.workflow_type = "rust.handled-parent".to_string();
11108
11109 let commands = worker.execute_workflow_task(task).expect("handled failure");
11110 assert_eq!(commands[0]["type"], "complete_workflow");
11111 let output = decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("parent output");
11112 assert_eq!(output["reason"], "child_workflow");
11113 assert_eq!(output["failure_id"], "failure-child");
11114 assert_eq!(output["exception_class"], "payments.PaymentRejected");
11115 assert_eq!(output["child_run_id"], "run-child");
11116 }
11117
11118 #[test]
11119 fn rust_hello_world_uses_signal_arguments_from_resume_payload() {
11120 let client = Client::new("http://127.0.0.1:8080").expect("client");
11121 let mut worker = Worker::new(client, "rust-workers");
11122
11123 worker.register_workflow("rust.hello_workflow", |ctx, _input| async move {
11124 let signal = ctx.wait_signal("start").await?;
11125 let name = signal
11126 .first()
11127 .and_then(|value| value.as_str())
11128 .unwrap_or("world");
11129 let greeting = ctx.activity("rust.hello_activity", json!([name])).await?;
11130 Ok(json!({
11131 "greeting": greeting,
11132 "language": "rust"
11133 }))
11134 });
11135
11136 let signal_arguments =
11137 encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC).expect("signal arguments");
11138 let task = WorkflowTask {
11139 task_id: "wft-rust-signal-1".to_string(),
11140 workflow_id: Some("wf-rust-hello".to_string()),
11141 run_id: Some("run-rust-hello".to_string()),
11142 workflow_type: "rust.hello_workflow".to_string(),
11143 payload_codec: DEFAULT_CODEC.to_string(),
11144 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
11145 history_events: vec![HistoryEvent {
11146 event_type: "SignalReceived".to_string(),
11147 payload: json!({
11148 "signal_id": "sig-rust-1",
11149 "signal_name": "start"
11150 }),
11151 raw: HashMap::new(),
11152 }],
11153 total_history_events: Some(1),
11154 history_size_bytes: None,
11155 continue_as_new_recommended: None,
11156 history_budget_pressure: None,
11157 next_history_page_token: None,
11158 workflow_task_attempt: 1,
11159 workflow_signal_id: Some("sig-rust-1".to_string()),
11160 signal_name: Some("start".to_string()),
11161 signal_arguments: Some(signal_arguments),
11162 workflow_update_id: None,
11163 update_name: None,
11164 lease_owner: Some("rust-worker".to_string()),
11165 };
11166
11167 let commands = worker.execute_workflow_task(task).expect("workflow task");
11168
11169 assert_eq!(commands.len(), 1);
11170 assert_eq!(commands[0]["type"], "schedule_activity");
11171 assert_eq!(commands[0]["activity_type"], "rust.hello_activity");
11172 assert_eq!(
11173 decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC).expect("activity args"),
11174 json!(["Rust"])
11175 );
11176 }
11177
11178 #[test]
11179 fn workflow_task_appends_paginated_history_events() {
11180 let mut task = WorkflowTask {
11181 task_id: "wft-rust-pages-1".to_string(),
11182 workflow_id: Some("wf-rust-pages".to_string()),
11183 run_id: Some("run-rust-pages".to_string()),
11184 workflow_type: "rust.hello_workflow".to_string(),
11185 payload_codec: DEFAULT_CODEC.to_string(),
11186 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
11187 history_events: vec![HistoryEvent {
11188 event_type: "WorkflowStarted".to_string(),
11189 payload: json!({}),
11190 raw: HashMap::new(),
11191 }],
11192 total_history_events: Some(3),
11193 history_size_bytes: None,
11194 continue_as_new_recommended: None,
11195 history_budget_pressure: None,
11196 next_history_page_token: Some("MQ==".to_string()),
11197 workflow_task_attempt: 1,
11198 workflow_signal_id: None,
11199 signal_name: None,
11200 signal_arguments: None,
11201 workflow_update_id: None,
11202 update_name: None,
11203 lease_owner: Some("rust-worker".to_string()),
11204 };
11205
11206 task.append_history_page(WorkflowTaskHistoryPage {
11207 history_events: vec![
11208 HistoryEvent {
11209 event_type: "SignalReceived".to_string(),
11210 payload: json!({
11211 "signal_id": "sig-rust-1",
11212 "signal_name": "start",
11213 "arguments": encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC)
11214 .expect("signal arguments")
11215 }),
11216 raw: HashMap::new(),
11217 },
11218 HistoryEvent {
11219 event_type: "MarkerRecorded".to_string(),
11220 payload: json!({"sequence": 3}),
11221 raw: HashMap::new(),
11222 },
11223 ],
11224 total_history_events: Some(3),
11225 next_history_page_token: None,
11226 });
11227
11228 assert_eq!(task.history_events.len(), 3);
11229 assert_eq!(task.total_history_events, Some(3));
11230 assert_eq!(task.next_history_page_token, None);
11231
11232 let signal = task
11233 .history_events
11234 .iter()
11235 .find(|event| event.event_type == "SignalReceived")
11236 .expect("signal event");
11237 assert_eq!(
11238 decode_signal_event_arguments(signal, DEFAULT_CODEC).expect("signal arguments"),
11239 vec![AvroValue::String("Rust".to_string())]
11240 );
11241 }
11242
11243 #[tokio::test]
11244 async fn query_handler_reads_ordered_cross_codec_signals_without_commands() {
11245 let client = Client::new("http://127.0.0.1:8080").expect("client");
11246 let mut worker = Worker::new(client, "rust-workers");
11247 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
11248 worker.register_query("counter", "current", |ctx, _args| async move {
11249 let mut count = 0_i64;
11250 for signal in ctx.signal_events() {
11251 let value = signal
11252 .arguments
11253 .first()
11254 .and_then(Value::as_i64)
11255 .unwrap_or_default();
11256 match signal.name.as_str() {
11257 "increment" => count += value,
11258 "set" => count = value,
11259 _ => {}
11260 }
11261 }
11262 Ok(json!(count))
11263 });
11264
11265 let task = QueryTask {
11266 query_task_id: "query-rust-counter".to_string(),
11267 query_task_attempt: 1,
11268 lease_owner: Some("rust-worker".to_string()),
11269 workflow_id: Some("counter-1".to_string()),
11270 run_id: Some("run-counter-1".to_string()),
11271 workflow_type: "counter".to_string(),
11272 query_name: "current".to_string(),
11273 payload_codec: DEFAULT_CODEC.to_string(),
11274 workflow_arguments: Some(
11275 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
11276 ),
11277 query_arguments: Some(
11278 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("query arguments"),
11279 ),
11280 history_events: vec![
11281 HistoryEvent {
11282 event_type: "SignalReceived".to_string(),
11283 payload: json!({
11284 "signal_id": "php-signal-1",
11285 "signal_name": "increment",
11286 "workflow_sequence": 1,
11287 "payload_codec": DEFAULT_CODEC,
11288 "arguments": encode_value_envelope(&json!([3]), DEFAULT_CODEC).expect("php avro signal")
11289 }),
11290 raw: HashMap::new(),
11291 },
11292 HistoryEvent {
11293 event_type: "SignalReceived".to_string(),
11294 payload: json!({
11295 "signal_id": "python-signal-2",
11296 "signal_name": "increment",
11297 "workflow_sequence": 2,
11298 "payload_codec": JSON_CODEC,
11299 "arguments": encode_value_envelope(&json!([5]), JSON_CODEC).expect("python json signal")
11300 }),
11301 raw: HashMap::new(),
11302 },
11303 HistoryEvent {
11304 event_type: "SignalReceived".to_string(),
11305 payload: json!({
11306 "signal_id": "rust-signal-3",
11307 "signal_name": "set",
11308 "workflow_sequence": 3,
11309 "payload_codec": DEFAULT_CODEC,
11310 "arguments": encode_value_envelope(&json!([0]), DEFAULT_CODEC).expect("rust avro signal")
11311 }),
11312 raw: HashMap::new(),
11313 },
11314 ],
11315 history_export: None,
11316 run_status: Some("completed".to_string()),
11317 };
11318
11319 let result = worker.execute_query_task(task).await.expect("query result");
11320 assert_eq!(result.into_json().expect("query projection"), json!(0));
11321 }
11322
11323 #[tokio::test]
11324 async fn replayed_queries_read_running_completed_and_cold_restarted_instance_state() {
11325 let worker = replay_counter_worker();
11326 let running_history = json!([
11327 {
11328 "type": "ActivityCompleted",
11329 "payload": {
11330 "sequence": 1,
11331 "activity_type": "load-counter",
11332 "payload_codec": "json",
11333 "result": {"codec": "json", "blob": "\"loaded\""}
11334 }
11335 },
11336 {
11337 "type": "SignalWaitOpened",
11338 "payload": {
11339 "sequence": 3,
11340 "signal_name": "increment"
11341 }
11342 },
11343 {
11344 "type": "SignalReceived",
11345 "payload": {
11346 "signal_id": "signal-3",
11347 "signal_name": "increment",
11348 "workflow_sequence": 2,
11349 "payload_codec": "json",
11350 "arguments": {"codec": "json", "blob": "[3]"}
11351 }
11352 },
11353 {
11354 "type": "SignalApplied",
11355 "payload": {
11356 "sequence": 3,
11357 "signal_id": "signal-3",
11358 "signal_name": "increment",
11359 "payload_codec": "json",
11360 "value": {"codec": "json", "blob": "[3]"}
11361 }
11362 }
11363 ]);
11364
11365 let running = worker
11366 .execute_query_task(replay_counter_query(
11367 "current",
11368 running_history.clone(),
11369 "running",
11370 ))
11371 .await
11372 .expect("running replay query");
11373 assert_eq!(
11374 running.clone().into_json().expect("query projection"),
11375 json!({"loaded": "loaded", "count": 3, "finished": false})
11376 );
11377
11378 let detached = worker
11379 .execute_query_task(replay_counter_query(
11380 "detached-mutation",
11381 running_history.clone(),
11382 "running",
11383 ))
11384 .await
11385 .expect("query mutates only its detached state clone");
11386 assert_eq!(detached.into_json().expect("query projection"), json!(999));
11387 let failed = worker
11388 .execute_query_task(replay_counter_query(
11389 "failed-mutation",
11390 running_history.clone(),
11391 "running",
11392 ))
11393 .await
11394 .expect_err("failed query");
11395 assert_eq!(failed.reason, "query_rejected");
11396 let unchanged = worker
11397 .execute_query_task(replay_counter_query("current", running_history, "running"))
11398 .await
11399 .expect("later query reconstructs unchanged state");
11400 assert_eq!(unchanged, running);
11401
11402 let restarted_worker = replay_counter_worker();
11403 let restarted_task: QueryTask = serde_json::from_value(json!({
11404 "query_task_id": "query-after-restart",
11405 "workflow_id": "counter-1",
11406 "run_id": "run-counter-1",
11407 "workflow_type": "replay-counter",
11408 "query_name": "current",
11409 "payload_codec": "json",
11410 "workflow_arguments": {"codec": "json", "blob": "[]"},
11411 "query_arguments": {"codec": "json", "blob": "[]"},
11412 "history_events": [],
11413 "history_export": {
11414 "payloads": {"codec": "json"},
11415 "history_events": [
11416 {
11417 "type": "ActivityCompleted",
11418 "payload": {
11419 "sequence": 1,
11420 "activity_type": "load-counter",
11421 "payload_codec": "json",
11422 "result": null
11423 }
11424 },
11425 {
11426 "type": "SignalWaitOpened",
11427 "payload": {
11428 "sequence": 3,
11429 "signal_name": "increment"
11430 }
11431 },
11432 {
11433 "type": "SignalReceived",
11434 "payload": {
11435 "signal_id": "signal-3",
11436 "signal_name": "increment",
11437 "workflow_sequence": 2
11438 }
11439 },
11440 {
11441 "type": "SignalApplied",
11442 "payload": {
11443 "sequence": 3,
11444 "signal_id": "signal-3",
11445 "signal_name": "increment"
11446 }
11447 },
11448 {
11449 "type": "SignalWaitOpened",
11450 "payload": {
11451 "sequence": 5,
11452 "signal_name": "increment"
11453 }
11454 },
11455 {
11456 "type": "SignalReceived",
11457 "payload": {
11458 "signal_id": "signal-5",
11459 "signal_name": "increment",
11460 "workflow_sequence": 4
11461 }
11462 },
11463 {
11464 "type": "SignalApplied",
11465 "payload": {
11466 "sequence": 5,
11467 "signal_id": "signal-5",
11468 "signal_name": "increment"
11469 }
11470 }
11471 ],
11472 "activities": [{
11473 "sequence": 1,
11474 "activity_type": "load-counter",
11475 "payload_codec": "json",
11476 "result": {"codec": "json", "blob": "\"loaded\""}
11477 }],
11478 "signals": [
11479 {
11480 "id": "signal-3",
11481 "name": "increment",
11482 "workflow_sequence": 2,
11483 "payload_codec": "json",
11484 "arguments": "[3]"
11485 },
11486 {
11487 "id": "signal-5",
11488 "name": "increment",
11489 "workflow_sequence": 4,
11490 "payload_codec": "json",
11491 "arguments": "[5]"
11492 }
11493 ]
11494 },
11495 "run_status": "completed"
11496 }))
11497 .expect("cold replay query task");
11498 let completed = restarted_worker
11499 .execute_query_task(restarted_task)
11500 .await
11501 .expect("completed cold replay query");
11502 assert_eq!(
11503 completed.into_json().expect("query projection"),
11504 json!({"loaded": "loaded", "count": 8, "finished": true})
11505 );
11506 }
11507
11508 #[tokio::test]
11509 async fn replayed_query_replay_failures_are_machine_readable() {
11510 let worker = replay_counter_worker();
11511 let task = replay_counter_query(
11512 "current",
11513 json!([{
11514 "type": "ActivityCompleted",
11515 "payload": {
11516 "sequence": 1,
11517 "payload_codec": "json",
11518 "result": {"codec": "json", "blob": "{"}
11519 }
11520 }]),
11521 "running",
11522 );
11523 let failure = worker
11524 .execute_query_task(task)
11525 .await
11526 .expect_err("invalid replay history payload");
11527 assert_eq!(failure.reason, "query_workflow_state_unavailable");
11528 assert_eq!(failure.failure_type, "QueryWorkflowStateUnavailable");
11529 }
11530
11531 #[tokio::test]
11532 async fn query_task_restores_compact_history_from_export() {
11533 let client = Client::new("http://127.0.0.1:8080").expect("client");
11534 let mut worker = Worker::new(client, "rust-workers");
11535 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
11536 worker.register_query("counter", "current", |ctx, _args| async move {
11537 Ok(json!(ctx.signals("increment")[0][0]))
11538 });
11539 let task: QueryTask = serde_json::from_value(json!({
11540 "query_task_id": "query-export",
11541 "workflow_type": "counter",
11542 "query_name": "current",
11543 "payload_codec": "json",
11544 "workflow_arguments": {"codec": "json", "blob": "[]"},
11545 "query_arguments": {"codec": "json", "blob": "[]"},
11546 "history_events": [],
11547 "history_export": {
11548 "payloads": {"codec": "json"},
11549 "history_events": [{
11550 "type": "SignalReceived",
11551 "payload": {"signal_id": "signal-export", "signal_name": "increment"}
11552 }],
11553 "signals": [{
11554 "id": "signal-export",
11555 "name": "increment",
11556 "status": "applied",
11557 "workflow_sequence": 1,
11558 "payload_codec": "json",
11559 "arguments": "[9]"
11560 }]
11561 }
11562 }))
11563 .expect("query task");
11564
11565 let result = worker.execute_query_task(task).await.expect("query result");
11566 assert_eq!(result.into_json().expect("query projection"), json!(9));
11567 }
11568
11569 #[tokio::test]
11570 async fn query_task_failures_have_stable_reasons() {
11571 let client = Client::new("http://127.0.0.1:8080").expect("client");
11572 let mut worker = Worker::new(client, "rust-workers");
11573 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
11574 worker.register_query(
11575 "counter",
11576 "current",
11577 |_ctx, _args| async move { Ok(json!(0)) },
11578 );
11579
11580 let base_task = QueryTask {
11581 query_task_id: "query-errors".to_string(),
11582 query_task_attempt: 1,
11583 lease_owner: None,
11584 workflow_id: Some("counter-errors".to_string()),
11585 run_id: Some("run-errors".to_string()),
11586 workflow_type: "counter".to_string(),
11587 query_name: "missing".to_string(),
11588 payload_codec: JSON_CODEC.to_string(),
11589 workflow_arguments: Some(json!({"codec": "json", "blob": "[]"})),
11590 query_arguments: Some(json!({"codec": "json", "blob": "[]"})),
11591 history_events: Vec::new(),
11592 history_export: None,
11593 run_status: Some("running".to_string()),
11594 };
11595
11596 let unknown = worker
11597 .execute_query_task(base_task.clone())
11598 .await
11599 .expect_err("unknown query");
11600 assert_eq!(unknown.reason, "rejected_unknown_query");
11601
11602 let mut malformed = base_task;
11603 malformed.query_name = "current".to_string();
11604 malformed.query_arguments = Some(json!({"codec": "json", "blob": "{"}));
11605 let malformed = worker
11606 .execute_query_task(malformed)
11607 .await
11608 .expect_err("malformed payload");
11609 assert_eq!(malformed.reason, "query_payload_decode_failed");
11610
11611 let client = Client::new("http://127.0.0.1:8080").expect("client");
11612 let mut unavailable_worker = Worker::new(client, "rust-workers");
11613 unavailable_worker
11614 .register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
11615 let unavailable_task: QueryTask = serde_json::from_value(json!({
11616 "query_task_id": "query-unavailable",
11617 "workflow_type": "counter",
11618 "query_name": "current",
11619 "payload_codec": "json",
11620 "workflow_arguments": {"codec": "json", "blob": "[]"},
11621 "query_arguments": {"codec": "json", "blob": "[]"}
11622 }))
11623 .expect("query task");
11624 let unavailable = unavailable_worker
11625 .execute_query_task(unavailable_task)
11626 .await
11627 .expect_err("query handler unavailable");
11628 assert_eq!(unavailable.reason, "query_handler_unavailable");
11629 }
11630
11631 #[tokio::test]
11632 async fn client_query_decodes_result_and_typed_failure() {
11633 let server = MockWorkerServer::start();
11634 let client = Client::builder(server.base_url())
11635 .timeout(Duration::from_secs(2))
11636 .build()
11637 .expect("client");
11638
11639 let result = client
11640 .query_workflow("counter-1", "current", json!([]))
11641 .await
11642 .expect("query result");
11643 assert_eq!(result, json!({"count": 8}));
11644
11645 let error = client
11646 .query_workflow("counter-1", "missing", json!([]))
11647 .await
11648 .expect_err("unknown query");
11649 let Error::QueryFailed(failure) = error else {
11650 panic!("expected typed query failure");
11651 };
11652 assert_eq!(failure.status, 404);
11653 assert_eq!(failure.reason, "rejected_unknown_query");
11654 }
11655
11656 #[tokio::test]
11657 async fn public_client_surfaces_send_and_receive_lossless_avro_values() {
11658 let server = MockWorkerServer::start();
11659 let client = Client::builder(server.base_url())
11660 .timeout(Duration::from_secs(2))
11661 .build()
11662 .expect("client");
11663 let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
11664
11665 client
11666 .start_workflow(
11667 "typed.echo",
11668 "rust-workers",
11669 "typed-start",
11670 arguments.clone(),
11671 )
11672 .await
11673 .expect("typed workflow start");
11674 assert_eq!(
11675 decode_wire_avro_value(
11676 &server.request_body("/api/workflows")["input"],
11677 DEFAULT_CODEC,
11678 )
11679 .expect("typed start input"),
11680 arguments
11681 );
11682
11683 client
11684 .signal_workflow("typed-1", "changed", arguments.clone())
11685 .await
11686 .expect("typed signal");
11687 assert_eq!(
11688 decode_wire_avro_value(
11689 &server.request_body("/api/workflows/typed-1/signal/changed")["input"],
11690 DEFAULT_CODEC,
11691 )
11692 .expect("typed signal input"),
11693 arguments
11694 );
11695
11696 assert_eq!(
11697 client
11698 .query_workflow_avro_value("typed-1", "inspect", arguments.clone())
11699 .await
11700 .expect("typed query"),
11701 typed_fidelity_probe()
11702 );
11703 assert_eq!(
11704 decode_wire_avro_value(
11705 &server.request_body("/api/workflows/typed-1/query/inspect")["input"],
11706 DEFAULT_CODEC,
11707 )
11708 .expect("typed query input"),
11709 arguments
11710 );
11711
11712 assert_eq!(
11713 client
11714 .update_workflow_avro_value(
11715 "typed-1",
11716 "replace",
11717 arguments.clone(),
11718 Some("typed-request"),
11719 )
11720 .await
11721 .expect("typed update"),
11722 typed_fidelity_probe()
11723 );
11724 let update = server.request_body("/api/workflows/typed-1/update/replace");
11725 assert_eq!(update["request_id"], "typed-request");
11726 assert_eq!(
11727 decode_wire_avro_value(&update["input"], DEFAULT_CODEC).expect("typed update input"),
11728 arguments
11729 );
11730
11731 let handle = WorkflowHandle {
11732 client: client.clone(),
11733 workflow_id: "typed-1".to_string(),
11734 run_id: Some("run-typed-1".to_string()),
11735 workflow_type: "typed.echo".to_string(),
11736 };
11737 assert_eq!(
11738 handle
11739 .result_avro_value(WorkflowResultOptions::default())
11740 .await
11741 .expect("typed workflow result"),
11742 typed_fidelity_probe()
11743 );
11744
11745 client
11746 .complete_activity_task(
11747 "activity-typed",
11748 "attempt-typed",
11749 "rust-worker",
11750 typed_fidelity_probe(),
11751 DEFAULT_CODEC,
11752 )
11753 .await
11754 .expect("typed activity completion");
11755 assert_eq!(
11756 decode_wire_avro_value(
11757 &server.request_body("/api/worker/activity-tasks/activity-typed/complete")
11758 ["result"],
11759 DEFAULT_CODEC,
11760 )
11761 .expect("typed activity result"),
11762 typed_fidelity_probe()
11763 );
11764 client
11765 .fail_activity_task(
11766 "activity-typed",
11767 "attempt-typed",
11768 "rust-worker",
11769 "typed failure",
11770 true,
11771 )
11772 .await
11773 .expect("activity failure");
11774 }
11775
11776 #[tokio::test]
11777 async fn lifecycle_commands_support_instance_and_selected_run_targets() {
11778 let server = MockWorkerServer::start();
11779 let client = Client::builder(server.base_url())
11780 .timeout(Duration::from_secs(2))
11781 .build()
11782 .expect("client");
11783
11784 let options = WorkflowCommandOptions::new()
11785 .reason("cleanup requested")
11786 .request_id("cancel-17");
11787 let cancelled = client
11788 .cancel_workflow("wf-lifecycle", options)
11789 .await
11790 .expect("instance cancellation");
11791 assert_eq!(cancelled.command, WorkflowCommandKind::Cancel);
11792 assert_eq!(cancelled.run_id.as_deref(), Some("run-current"));
11793 assert_eq!(cancelled.outcome.as_deref(), Some("cancelled"));
11794 assert_eq!(
11795 server.request_body("/api/workflows/wf-lifecycle/cancel"),
11796 json!({"reason":"cleanup requested","request_id":"cancel-17"})
11797 );
11798
11799 let terminated = client
11800 .terminate_workflow(
11801 "wf-lifecycle",
11802 WorkflowCommandOptions::new().reason("forced stop"),
11803 )
11804 .await
11805 .expect("instance termination");
11806 assert_eq!(terminated.command, WorkflowCommandKind::Terminate);
11807 assert_eq!(terminated.outcome.as_deref(), Some("terminated"));
11808
11809 client
11810 .cancel_workflow_run(
11811 "wf-lifecycle",
11812 "run-current",
11813 WorkflowCommandOptions::default(),
11814 )
11815 .await
11816 .expect("selected run cancellation");
11817 client
11818 .terminate_workflow_run(
11819 "wf-lifecycle",
11820 "run-current",
11821 WorkflowCommandOptions::default(),
11822 )
11823 .await
11824 .expect("selected run termination");
11825
11826 for (command, error) in [
11827 (
11828 WorkflowCommandKind::Cancel,
11829 client
11830 .cancel_workflow_run(
11831 "wf-lifecycle",
11832 "run-stale",
11833 WorkflowCommandOptions::default(),
11834 )
11835 .await
11836 .expect_err("stale cancellation must be rejected"),
11837 ),
11838 (
11839 WorkflowCommandKind::Terminate,
11840 client
11841 .terminate_workflow_run(
11842 "wf-lifecycle",
11843 "run-stale",
11844 WorkflowCommandOptions::default(),
11845 )
11846 .await
11847 .expect_err("stale termination must be rejected"),
11848 ),
11849 ] {
11850 let Error::WorkflowCommandRejected(rejection) = error else {
11851 panic!("expected typed command rejection");
11852 };
11853 assert_eq!(rejection.command, command);
11854 assert_eq!(rejection.status, 409);
11855 assert_eq!(rejection.reason, "historical_run_command_rejected");
11856 assert_eq!(rejection.run_id.as_deref(), Some("run-stale"));
11857 assert_eq!(rejection.target_scope.as_deref(), Some("run"));
11858 }
11859 }
11860
11861 #[tokio::test]
11862 async fn workflow_start_options_send_server_enforced_deadlines() {
11863 let server = MockWorkerServer::start();
11864 let client = Client::builder(server.base_url())
11865 .timeout(Duration::from_secs(2))
11866 .build()
11867 .expect("client");
11868
11869 let handle = client
11870 .start_workflow_with_options(
11871 "rust.timeout",
11872 "rust-timeouts",
11873 "wf-start-options",
11874 WorkflowStartOptions::new()
11875 .execution_timeout_seconds(30)
11876 .run_timeout_seconds(1),
11877 json!([]),
11878 )
11879 .await
11880 .expect("workflow start");
11881
11882 assert_eq!(handle.run_id.as_deref(), Some("run-start-options"));
11883 let body = server.request_body("/api/workflows");
11884 assert_eq!(body["execution_timeout_seconds"], 30);
11885 assert_eq!(body["run_timeout_seconds"], 1);
11886
11887 let invalid = client
11888 .start_workflow_with_options(
11889 "rust.timeout",
11890 "rust-timeouts",
11891 "wf-invalid-options",
11892 WorkflowStartOptions::new()
11893 .execution_timeout_seconds(1)
11894 .run_timeout_seconds(2),
11895 json!([]),
11896 )
11897 .await
11898 .expect_err("invalid deadline ordering");
11899 assert!(invalid
11900 .to_string()
11901 .contains("run_timeout_seconds cannot exceed execution_timeout_seconds"));
11902 }
11903
11904 #[tokio::test]
11905 async fn workflow_result_returns_each_typed_terminal_outcome() {
11906 let server = MockWorkerServer::start();
11907 let client = Client::builder(server.base_url())
11908 .timeout(Duration::from_secs(2))
11909 .build()
11910 .expect("client");
11911 let options = WorkflowResultOptions {
11912 poll_interval: Duration::ZERO,
11913 timeout: Duration::from_secs(1),
11914 };
11915
11916 let failed = WorkflowHandle {
11917 client: client.clone(),
11918 workflow_id: "wf-failed".to_string(),
11919 run_id: Some("run-failed".to_string()),
11920 workflow_type: "failure".to_string(),
11921 }
11922 .result(options)
11923 .await
11924 .expect_err("failed outcome");
11925 let Error::WorkflowFailed(failure) = failed else {
11926 panic!("expected WorkflowFailed");
11927 };
11928 assert_eq!(failure.workflow_id, "wf-failed");
11929 assert_eq!(failure.run_id.as_deref(), Some("run-failed"));
11930 assert_eq!(failure.failure_id.as_deref(), Some("failure-17"));
11931 assert_eq!(failure.failure_category.as_deref(), Some("application"));
11932 assert_eq!(failure.exception_type.as_deref(), Some("PaymentError"));
11933 assert_eq!(
11934 failure.exception_class.as_deref(),
11935 Some("billing::PaymentError")
11936 );
11937 assert_eq!(failure.non_retryable, Some(true));
11938
11939 for (workflow_id, expected_kind, expected_reason) in [
11940 (
11941 "wf-cancelled",
11942 WorkflowTerminalKind::Cancelled,
11943 "cleanup requested",
11944 ),
11945 (
11946 "wf-terminated",
11947 WorkflowTerminalKind::Terminated,
11948 "forced stop",
11949 ),
11950 (
11951 "wf-timed-out",
11952 WorkflowTerminalKind::TimedOut,
11953 "run_timeout",
11954 ),
11955 ] {
11956 let error = WorkflowHandle {
11957 client: client.clone(),
11958 workflow_id: workflow_id.to_string(),
11959 run_id: None,
11960 workflow_type: "terminal".to_string(),
11961 }
11962 .result(options)
11963 .await
11964 .expect_err("typed terminal outcome");
11965 let outcome = match error {
11966 Error::WorkflowCancelled(outcome) => outcome,
11967 Error::WorkflowTerminated(outcome) => outcome,
11968 Error::WorkflowTimedOut(outcome) => outcome,
11969 other => panic!("unexpected terminal error: {other}"),
11970 };
11971 assert_eq!(outcome.kind, expected_kind);
11972 assert_eq!(outcome.workflow_id, workflow_id);
11973 assert_eq!(outcome.reason, expected_reason);
11974 }
11975
11976 let wait_timeout = WorkflowHandle {
11977 client,
11978 workflow_id: "wf-waiting".to_string(),
11979 run_id: Some("run-waiting".to_string()),
11980 workflow_type: "waiting".to_string(),
11981 }
11982 .result(WorkflowResultOptions {
11983 poll_interval: Duration::ZERO,
11984 timeout: Duration::ZERO,
11985 })
11986 .await
11987 .expect_err("client wait timeout");
11988 let Error::WorkflowTimedOut(timeout) = wait_timeout else {
11989 panic!("expected typed client timeout");
11990 };
11991 assert_eq!(timeout.reason, "result_wait_timeout");
11992 assert_eq!(timeout.failure_category.as_deref(), Some("client_timeout"));
11993 assert_eq!(timeout.run_id.as_deref(), Some("run-waiting"));
11994 }
11995
11996 #[tokio::test]
11997 async fn workflow_result_follows_chain_and_selected_result_preserves_history() {
11998 let server = MockWorkerServer::start();
11999 let client = Client::builder(server.base_url())
12000 .timeout(Duration::from_secs(2))
12001 .build()
12002 .expect("client");
12003
12004 let handle = WorkflowHandle {
12005 client,
12006 workflow_id: "wf-selected".to_string(),
12007 run_id: Some("run-selected".to_string()),
12008 workflow_type: "selected".to_string(),
12009 };
12010 let options = WorkflowResultOptions {
12011 poll_interval: Duration::ZERO,
12012 timeout: Duration::from_secs(1),
12013 };
12014
12015 let current = handle
12016 .result(options)
12017 .await
12018 .expect("instance result follows the current run");
12019 assert_eq!(current, json!("current run output"));
12020
12021 let error = handle
12022 .result_selected_run(options)
12023 .await
12024 .expect_err("the selected run is cancelled even though the current run completed");
12025
12026 let Error::WorkflowCancelled(outcome) = error else {
12027 panic!("expected selected run cancellation");
12028 };
12029 assert_eq!(outcome.run_id.as_deref(), Some("run-selected"));
12030 assert_eq!(outcome.reason, "selected run cancelled");
12031 assert_eq!(
12032 server.request_count("/api/workflows/wf-selected/runs/run-selected"),
12033 1
12034 );
12035 assert_eq!(server.request_count("/api/workflows/wf-selected"), 1);
12036 }
12037
12038 #[tokio::test]
12039 async fn poll_responses_decode_http_conflict_drain_as_a_stable_stop() {
12040 let server = MockWorkerServer::draining_polls();
12041 let client = Client::builder(server.base_url())
12042 .timeout(Duration::from_secs(2))
12043 .build()
12044 .expect("client");
12045
12046 let workflow = client
12047 .poll_workflow_task_response("draining-worker", "rust-workers", Duration::ZERO)
12048 .await
12049 .expect("workflow drain response");
12050 let activity = client
12051 .poll_activity_task_response("draining-worker", "rust-workers", Duration::ZERO)
12052 .await
12053 .expect("activity drain response");
12054 let query = client
12055 .poll_query_task_response("draining-worker", "rust-workers", Duration::ZERO)
12056 .await
12057 .expect("query drain response");
12058
12059 for outcome in [workflow.outcome(), activity.outcome(), query.outcome()] {
12060 assert_eq!(
12061 outcome,
12062 WorkerPollOutcome::Stop {
12063 poll_status: Some("draining".to_string()),
12064 reason: Some("worker_draining".to_string()),
12065 }
12066 );
12067 }
12068
12069 assert!(client
12070 .poll_workflow_task("draining-worker", "rust-workers", Duration::ZERO)
12071 .await
12072 .expect("compatibility poll")
12073 .is_none());
12074 }
12075
12076 #[tokio::test]
12077 async fn managed_worker_honors_drain_stop_for_every_task_family() {
12078 let server = MockWorkerServer::draining_polls();
12079 let client = Client::builder(server.base_url())
12080 .timeout(Duration::from_secs(2))
12081 .build()
12082 .expect("client");
12083
12084 let mut workflow_worker = Worker::new(client.clone(), "rust-workers")
12085 .worker_id("draining-workflow-worker")
12086 .poll_timeout(Duration::ZERO);
12087 workflow_worker.register_workflow("counter", |_ctx, _args| async { Ok(Value::Null) });
12088 workflow_worker
12089 .run()
12090 .await
12091 .expect("workflow drain is a clean stop");
12092
12093 let mut activity_worker = Worker::new(client.clone(), "rust-workers")
12094 .worker_id("draining-activity-worker")
12095 .poll_timeout(Duration::ZERO);
12096 activity_worker.register_activity("write", |_ctx, _args| async { Ok(Value::Null) });
12097 activity_worker
12098 .run()
12099 .await
12100 .expect("activity drain is a clean stop");
12101
12102 let mut query_worker = Worker::new(client, "rust-workers")
12103 .worker_id("draining-query-worker")
12104 .poll_timeout(Duration::ZERO);
12105 query_worker.register_query("counter", "current", |_ctx, _args| async {
12106 Ok(Value::Null)
12107 });
12108 query_worker
12109 .run()
12110 .await
12111 .expect("query drain is a clean stop");
12112 }
12113
12114 #[tokio::test]
12115 async fn activity_cancellation_and_late_completion_remain_machine_readable() {
12116 let server = MockWorkerServer::start();
12117 let client = Client::builder(server.base_url())
12118 .timeout(Duration::from_secs(2))
12119 .build()
12120 .expect("client");
12121
12122 let heartbeat = client
12123 .heartbeat_activity_task(
12124 "activity-cancel",
12125 "attempt-cancel",
12126 "rust-worker",
12127 typed_fidelity_probe(),
12128 )
12129 .await
12130 .expect("cancellation heartbeat");
12131 assert!(heartbeat.cancel_requested);
12132 assert!(heartbeat.should_stop());
12133 assert_eq!(heartbeat.reason.as_deref(), Some("run_cancelled"));
12134 assert_eq!(heartbeat.run_closed_reason.as_deref(), Some("cancelled"));
12135 let heartbeat_body =
12136 server.request_body("/api/worker/activity-tasks/activity-cancel/heartbeat");
12137 assert_eq!(heartbeat_body["details"]["codec"], DEFAULT_CODEC);
12138 assert_eq!(
12139 decode_wire_avro_value(&heartbeat_body["details"], DEFAULT_CODEC)
12140 .expect("typed heartbeat details"),
12141 typed_fidelity_probe()
12142 );
12143
12144 let error = client
12145 .complete_activity_task(
12146 "activity-cancel",
12147 "attempt-cancel",
12148 "rust-worker",
12149 json!({"late":true}),
12150 JSON_CODEC,
12151 )
12152 .await
12153 .expect_err("late completion must be refused");
12154 assert!(activity_task_rejection_is_final(&error));
12155 let Error::ActivityTaskRejected(rejection) = error else {
12156 panic!("expected typed activity rejection");
12157 };
12158 assert_eq!(rejection.status, 409);
12159 assert_eq!(rejection.reason, "run_cancelled");
12160 assert!(rejection.cancel_requested);
12161 assert_eq!(rejection.can_continue, Some(false));
12162 }
12163
12164 #[tokio::test]
12165 async fn managed_worker_survives_late_completion_and_restart_during_cancellation() {
12166 let server = MockWorkerServer::cancelled_activity();
12167 let client = Client::builder(server.base_url())
12168 .timeout(Duration::from_secs(2))
12169 .build()
12170 .expect("client");
12171 let cancellation_observed = Arc::new(AtomicBool::new(false));
12172 let observed = Arc::clone(&cancellation_observed);
12173 let mut worker = Worker::new(client.clone(), "rust-workers")
12174 .worker_id("rust-cancel-worker")
12175 .poll_timeout(Duration::from_millis(10));
12176 worker.register_activity("cancel-aware", move |ctx, _args| {
12177 let observed = Arc::clone(&observed);
12178 async move {
12179 let heartbeat = ctx.heartbeat(json!({"stage":"running"})).await?;
12180 observed.store(heartbeat.should_stop(), Ordering::SeqCst);
12181 Ok(json!({"late":"completion"}))
12182 }
12183 });
12184
12185 assert_eq!(
12186 worker.run_once().await.expect("cancelled attempt handled"),
12187 1
12188 );
12189 assert!(cancellation_observed.load(Ordering::SeqCst));
12190 assert_eq!(
12191 server.request_count("/api/worker/activity-tasks/activity-cancel/complete"),
12192 1
12193 );
12194
12195 let mut restarted = Worker::new(client, "rust-workers")
12196 .worker_id("rust-cancel-worker-restarted")
12197 .poll_timeout(Duration::from_millis(10));
12198 restarted.register_activity("cancel-aware", |_ctx, _args| async move { Ok(Value::Null) });
12199 assert_eq!(
12200 restarted
12201 .run_once()
12202 .await
12203 .expect("replacement worker continues polling"),
12204 0
12205 );
12206 }
12207
12208 #[tokio::test]
12209 async fn managed_worker_absorbs_selected_run_terminal_timeout_completion_race() {
12210 let response = r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"outcome":"completed","recorded":false,"run_id":"run-selected-timeout","run_status":"failed","created_task_ids":[],"reason":"run_timed_out"}"#;
12211 let server = MockWorkerServer::workflow_completion("409 Conflict", response);
12212 let client = Client::builder(server.base_url())
12213 .timeout(Duration::from_secs(2))
12214 .build()
12215 .expect("client");
12216
12217 let direct_error = client
12218 .complete_workflow_task(
12219 "workflow-timeout-task",
12220 "timeout-worker",
12221 3,
12222 vec![json!({"type": "complete_workflow", "result": null})],
12223 )
12224 .await
12225 .expect_err("the low-level client preserves the completion rejection");
12226 let Error::Http { status, body } = direct_error else {
12227 panic!("expected the original HTTP completion rejection");
12228 };
12229 assert_eq!(status, reqwest::StatusCode::CONFLICT);
12230 assert_eq!(
12231 serde_json::from_str::<Value>(&body).expect("response body")["reason"],
12232 "run_timed_out"
12233 );
12234
12235 let mut worker = Worker::new(client, "rust-workers")
12236 .worker_id("timeout-worker")
12237 .poll_timeout(Duration::from_millis(10));
12238 worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
12239 Ok(json!({"late": "result"}))
12240 });
12241
12242 assert_eq!(
12243 worker
12244 .run_once()
12245 .await
12246 .expect("authoritative selected-run timeout settles the tick"),
12247 1
12248 );
12249 assert_eq!(
12250 server.request_count("/api/worker/workflow-tasks/workflow-timeout-task/complete"),
12251 2,
12252 "both the direct client proof and managed worker must see the rejection"
12253 );
12254 }
12255
12256 #[tokio::test]
12257 async fn managed_worker_does_not_swallow_nearby_completion_errors() {
12258 for (name, status, response) in [
12259 ("bare conflict", "409 Conflict", r#"{"message":"conflict"}"#),
12260 (
12261 "command was recorded",
12262 "409 Conflict",
12263 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":true,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12264 ),
12265 (
12266 "lease conflict",
12267 "409 Conflict",
12268 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"lease_expired"}"#,
12269 ),
12270 (
12271 "nonterminal run",
12272 "409 Conflict",
12273 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"waiting","reason":"run_timed_out"}"#,
12274 ),
12275 (
12276 "different selected run",
12277 "409 Conflict",
12278 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-reused-workflow-current","run_status":"failed","reason":"run_timed_out"}"#,
12279 ),
12280 (
12281 "different task attempt",
12282 "409 Conflict",
12283 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":4,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12284 ),
12285 (
12286 "authentication failure",
12287 "401 Unauthorized",
12288 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12289 ),
12290 (
12291 "authorization failure",
12292 "403 Forbidden",
12293 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12294 ),
12295 (
12296 "protocol failure",
12297 "400 Bad Request",
12298 r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.2","requested_version":"1.3"}"#,
12299 ),
12300 (
12301 "malformed command",
12302 "422 Unprocessable Entity",
12303 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12304 ),
12305 (
12306 "transient server failure",
12307 "503 Service Unavailable",
12308 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12309 ),
12310 ] {
12311 let server = MockWorkerServer::workflow_completion(status, response);
12312 let client = Client::builder(server.base_url())
12313 .timeout(Duration::from_secs(2))
12314 .build()
12315 .expect("client");
12316 let mut worker = Worker::new(client, "rust-workers")
12317 .worker_id("timeout-worker")
12318 .poll_timeout(Duration::from_millis(10));
12319 worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
12320 Ok(json!({"late": "result"}))
12321 });
12322
12323 let error = worker
12324 .run_once()
12325 .await
12326 .expect_err(&format!("{name} must remain an error"));
12327 assert!(
12328 matches!(error, Error::Http { .. } | Error::Protocol(_)),
12329 "{name} returned an unexpected error variant: {error}"
12330 );
12331 }
12332 }
12333
12334 #[tokio::test]
12335 async fn baseline_worker_endpoints_send_the_baseline_protocol() {
12336 let server = MockWorkerServer::start();
12337 let client = Client::builder(server.base_url())
12338 .timeout(Duration::from_secs(2))
12339 .build()
12340 .expect("client");
12341
12342 client
12343 .register_worker("capture-worker", "capture", vec![], vec![], 1, 1)
12344 .await
12345 .expect("register");
12346 client
12347 .heartbeat_worker("capture-worker", 1, 1)
12348 .await
12349 .expect("heartbeat");
12350 client
12351 .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
12352 .await
12353 .expect("workflow poll");
12354 client
12355 .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
12356 .await
12357 .expect("activity poll");
12358
12359 for path in [
12360 "/api/worker/register",
12361 "/api/worker/heartbeat",
12362 "/api/worker/workflow-tasks/poll",
12363 "/api/worker/activity-tasks/poll",
12364 ] {
12365 assert_eq!(
12366 server.worker_protocol_for(path).as_deref(),
12367 Some(WORKER_PROTOCOL_VERSION),
12368 "unexpected protocol for {path}"
12369 );
12370 }
12371
12372 assert_eq!(
12373 server.request_body("/api/worker/workflow-tasks/poll")["timeout_seconds"],
12374 1
12375 );
12376 assert_eq!(
12377 server.request_body("/api/worker/activity-tasks/poll")["timeout_seconds"],
12378 1
12379 );
12380 assert!(
12381 server.request_body("/api/worker/workflow-tasks/poll")["poll_request_id"]
12382 .as_str()
12383 .is_some_and(|id| id.starts_with("rust-workflow-poll-"))
12384 );
12385 assert!(
12386 server.request_body("/api/worker/activity-tasks/poll")["poll_request_id"]
12387 .as_str()
12388 .is_some_and(|id| id.starts_with("rust-activity-poll-"))
12389 );
12390 }
12391
12392 #[tokio::test]
12393 async fn query_task_endpoints_send_the_query_feature_protocol() {
12394 let server = MockWorkerServer::start();
12395 let client = Client::builder(server.base_url())
12396 .timeout(Duration::from_secs(2))
12397 .build()
12398 .expect("client");
12399
12400 client
12401 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
12402 .await
12403 .expect("query poll");
12404 client
12405 .complete_query_task("query-capture", "capture-worker", 1, json!(8), JSON_CODEC)
12406 .await
12407 .expect("query complete");
12408 client
12409 .fail_query_task(
12410 "query-capture",
12411 "capture-worker",
12412 1,
12413 "failed",
12414 "query_rejected",
12415 "QueryFailed",
12416 )
12417 .await
12418 .expect("query fail");
12419
12420 for path in [
12421 "/api/worker/query-tasks/poll",
12422 "/api/worker/query-tasks/query-capture/complete",
12423 "/api/worker/query-tasks/query-capture/fail",
12424 ] {
12425 assert_eq!(
12426 server.worker_protocol_for(path).as_deref(),
12427 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
12428 "unexpected protocol for {path}"
12429 );
12430 }
12431
12432 assert_eq!(
12433 server.request_body("/api/worker/query-tasks/poll")["timeout_seconds"],
12434 1
12435 );
12436 assert!(
12437 server.request_body("/api/worker/query-tasks/poll")["poll_request_id"]
12438 .as_str()
12439 .is_some_and(|id| id.starts_with("rust-query-poll-"))
12440 );
12441 }
12442
12443 #[tokio::test]
12444 async fn disconnected_client_polls_retry_once_with_the_same_request_id() {
12445 let server = MockWorkerServer::transient_worker_failures();
12446 let client = Client::builder(server.base_url())
12447 .timeout(Duration::from_secs(2))
12448 .build()
12449 .expect("client");
12450
12451 client
12452 .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
12453 .await
12454 .expect("workflow poll retry");
12455 client
12456 .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
12457 .await
12458 .expect("activity poll retry");
12459 client
12460 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
12461 .await
12462 .expect("query poll retry");
12463
12464 for path in [
12465 "/api/worker/workflow-tasks/poll",
12466 "/api/worker/activity-tasks/poll",
12467 "/api/worker/query-tasks/poll",
12468 ] {
12469 let bodies = server.request_bodies(path);
12470 assert_eq!(bodies.len(), 2, "{path} must be retried once");
12471 assert_eq!(
12472 bodies[0]["poll_request_id"], bodies[1]["poll_request_id"],
12473 "{path} must preserve the request binding across retry"
12474 );
12475 }
12476 }
12477
12478 #[tokio::test]
12479 async fn worker_poll_retries_preserve_request_id_across_consecutive_failures() {
12480 let server = MockWorkerServer::consecutive_poll_failures(2);
12481 let client = Client::builder(server.base_url())
12482 .timeout(Duration::from_secs(2))
12483 .build()
12484 .expect("client");
12485 let mut worker = Worker::new(client, "capture")
12486 .worker_id("capture-worker")
12487 .poll_timeout(Duration::from_millis(10))
12488 .retry_policy(WorkerRetryPolicy {
12489 max_retries: 2,
12490 initial_backoff: Duration::from_millis(1),
12491 max_backoff: Duration::from_millis(1),
12492 });
12493 worker.register_workflow(
12494 "capture.workflow",
12495 |_ctx, _input| async move { Ok(Value::Null) },
12496 );
12497 worker.register_activity(
12498 "capture.activity",
12499 |_ctx, _input| async move { Ok(Value::Null) },
12500 );
12501 worker.register_query("capture.workflow", "current", |_ctx, _args| async move {
12502 Ok(Value::Null)
12503 });
12504
12505 assert_eq!(worker.run_once().await.expect("poll retries"), 0);
12506
12507 for path in [
12508 "/api/worker/workflow-tasks/poll",
12509 "/api/worker/activity-tasks/poll",
12510 "/api/worker/query-tasks/poll",
12511 ] {
12512 let bodies = server.request_bodies(path);
12513 assert_eq!(bodies.len(), 3, "{path} must use exactly two retries");
12514 assert!(
12515 bodies
12516 .iter()
12517 .all(|body| body["poll_request_id"] == bodies[0]["poll_request_id"]),
12518 "{path} must preserve one request binding across every retry"
12519 );
12520 }
12521 }
12522
12523 #[tokio::test]
12524 async fn query_protocol_rejection_from_older_server_is_typed() {
12525 let server = MockWorkerServer::reject_query_protocol();
12526 let client = Client::builder(server.base_url())
12527 .timeout(Duration::from_secs(2))
12528 .build()
12529 .expect("client");
12530
12531 let error = client
12532 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
12533 .await
12534 .expect_err("server below query protocol floor must reject");
12535 let Error::Protocol(failure) = error else {
12536 panic!("expected typed protocol failure");
12537 };
12538
12539 assert_eq!(failure.status, 400);
12540 assert_eq!(failure.reason, "unsupported_protocol_version");
12541 assert_eq!(failure.supported_version.as_deref(), Some("1.7"));
12542 assert_eq!(
12543 failure.requested_version.as_deref(),
12544 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
12545 );
12546 assert_eq!(
12547 server
12548 .worker_protocol_for("/api/worker/query-tasks/poll")
12549 .as_deref(),
12550 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
12551 );
12552 }
12553
12554 #[tokio::test]
12555 async fn run_once_without_query_handlers_keeps_pre_query_server_compatibility() {
12556 let server = MockWorkerServer::reject_query_protocol();
12557 let client = Client::builder(server.base_url())
12558 .timeout(Duration::from_secs(2))
12559 .build()
12560 .expect("client");
12561 let mut worker = Worker::new(client, "rust-workers")
12562 .worker_id("baseline-worker")
12563 .poll_timeout(Duration::from_millis(10));
12564
12565 worker.register_workflow("baseline.workflow", |_ctx, _input| async move {
12566 Ok(Value::Null)
12567 });
12568
12569 assert_eq!(worker.run_once().await.expect("baseline run once"), 0);
12570 assert_eq!(
12571 server
12572 .worker_protocol_for("/api/worker/workflow-tasks/poll")
12573 .as_deref(),
12574 Some(WORKER_PROTOCOL_VERSION)
12575 );
12576 assert_eq!(
12577 server.worker_protocol_for("/api/worker/query-tasks/poll"),
12578 None,
12579 "a worker without query handlers must not use the query-task endpoint"
12580 );
12581 }
12582
12583 #[tokio::test]
12584 async fn completion_time_query_rejection_is_typed_without_stopping_worker() {
12585 let server = MockWorkerServer::reject_query_completion();
12586 let client = Client::builder(server.base_url())
12587 .timeout(Duration::from_secs(2))
12588 .build()
12589 .expect("client");
12590
12591 let error = client
12592 .complete_query_task("query-late", "late-worker", 1, json!(8), JSON_CODEC)
12593 .await
12594 .expect_err("expired completion must be rejected");
12595 let Error::QueryFailed(failure) = error else {
12596 panic!("expected typed query failure");
12597 };
12598 assert_eq!(failure.status, 409);
12599 assert_eq!(failure.reason, "query_task_timed_out");
12600
12601 let mut worker = Worker::new(client, "rust-workers")
12602 .worker_id("late-worker")
12603 .poll_timeout(Duration::from_millis(10));
12604 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
12605 worker.register_query(
12606 "counter",
12607 "current",
12608 |_ctx, _args| async move { Ok(json!(8)) },
12609 );
12610
12611 assert_eq!(worker.run_once().await.expect("late task is handled"), 1);
12612 assert_eq!(
12613 worker
12614 .run_once()
12615 .await
12616 .expect("worker continues after late completion"),
12617 0
12618 );
12619 assert_eq!(
12620 server.request_count("/api/worker/query-tasks/query-late/complete"),
12621 2
12622 );
12623 assert_eq!(
12624 server.request_count("/api/worker/query-tasks/query-late/fail"),
12625 0,
12626 "a server completion rejection must not be reported as an encoding failure"
12627 );
12628 }
12629
12630 #[tokio::test]
12631 async fn activity_only_worker_can_shutdown_without_workflow_poller() {
12632 let server = MockWorkerServer::start();
12633 let client = Client::builder(server.base_url())
12634 .timeout(Duration::from_secs(2))
12635 .build()
12636 .expect("client");
12637 let mut worker = Worker::new(client, "rust-workers")
12638 .worker_id("activity-only-worker")
12639 .poll_timeout(Duration::from_millis(10));
12640
12641 worker.register_activity(
12642 "activity.only",
12643 |_ctx, _args| async move { Ok(Value::Null) },
12644 );
12645
12646 worker.run_until(async {}).await.expect("run worker");
12647 }
12648
12649 #[tokio::test]
12650 async fn workflow_only_worker_can_shutdown_without_activity_poller() {
12651 let server = MockWorkerServer::start();
12652 let client = Client::builder(server.base_url())
12653 .timeout(Duration::from_secs(2))
12654 .build()
12655 .expect("client");
12656 let mut worker = Worker::new(client, "rust-workers")
12657 .worker_id("workflow-only-worker")
12658 .poll_timeout(Duration::from_millis(10));
12659
12660 worker.register_workflow(
12661 "workflow.only",
12662 |_ctx, _input| async move { Ok(Value::Null) },
12663 );
12664
12665 worker.run_until(async {}).await.expect("run worker");
12666 }
12667
12668 #[tokio::test]
12669 async fn worker_heartbeat_observer_receives_server_acknowledgements() {
12670 let server = MockWorkerServer::start();
12671 let client = Client::builder(server.base_url())
12672 .timeout(Duration::from_secs(2))
12673 .build()
12674 .expect("client");
12675 let observations = Arc::new(Mutex::new(Vec::new()));
12676 let observed = Arc::clone(&observations);
12677 let mut worker = Worker::new(client, "rust-workers")
12678 .worker_id("observed-heartbeat-worker")
12679 .poll_timeout(Duration::from_millis(10))
12680 .on_worker_heartbeat(move |observation| {
12681 observed
12682 .lock()
12683 .expect("heartbeat observations")
12684 .push(observation.clone());
12685 });
12686
12687 worker.register_workflow("workflow.observed", |_ctx, _input| async move {
12688 Ok(Value::Null)
12689 });
12690 let acknowledged = Arc::clone(&observations);
12691 worker
12692 .run_until(async move {
12693 tokio::time::timeout(Duration::from_secs(2), async move {
12694 loop {
12695 if !acknowledged
12696 .lock()
12697 .expect("heartbeat observations")
12698 .is_empty()
12699 {
12700 break;
12701 }
12702 tokio::time::sleep(Duration::from_millis(1)).await;
12703 }
12704 })
12705 .await
12706 .expect("heartbeat acknowledgement within timeout");
12707 })
12708 .await
12709 .expect("run worker");
12710
12711 let observations = observations.lock().expect("heartbeat observations");
12712 let first = observations.first().expect("heartbeat acknowledgement");
12713 assert_eq!(first.worker_id, "observed-heartbeat-worker");
12714 assert_eq!(first.task_queue, "rust-workers");
12715 assert!(first.acknowledged_at_unix_millis > 0);
12716 assert_eq!(first.acknowledgement, json!({}));
12717 }
12718
12719 #[tokio::test]
12720 async fn delayed_worker_heartbeat_keeps_cadence_and_pollers_live() {
12721 let server = MockWorkerServer::delayed_heartbeat_worker();
12722 let client = Client::builder(server.base_url())
12723 .timeout(Duration::from_secs(3))
12724 .build()
12725 .expect("client");
12726 let observations = Arc::new(Mutex::new(Vec::new()));
12727 let observed = Arc::clone(&observations);
12728 let mut worker = Worker::new(client, "rust-snapshot-workers")
12729 .worker_id("rust-snapshot-worker")
12730 .poll_timeout(Duration::from_millis(10))
12731 .on_worker_heartbeat(move |observation| {
12732 observed
12733 .lock()
12734 .expect("heartbeat observations")
12735 .push(observation.clone());
12736 });
12737
12738 worker.register_workflow("snapshot", |ctx, _input| async move {
12739 ctx.wait_signal("finish").await?;
12740 Ok(json!({"status": "finished"}))
12741 });
12742 worker.register_query("snapshot", "current", |ctx, _args| async move {
12743 Ok(json!(ctx
12744 .signals("increment")
12745 .iter()
12746 .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
12747 .sum::<i64>()))
12748 });
12749 worker.register_activity("cancel-aware", |_ctx, _args| async move {
12750 Ok(json!({"late": "completion"}))
12751 });
12752
12753 worker
12754 .run_until(tokio::time::sleep(Duration::from_millis(3_800)))
12755 .await
12756 .expect("delayed heartbeat must allow a clean worker shutdown");
12757
12758 let observations = observations.lock().expect("heartbeat observations");
12759 assert!(
12760 observations.len() >= 3,
12761 "the immediate heartbeat, delayed acknowledgement, and next cadence heartbeat must complete"
12762 );
12763 assert!(
12764 observations.windows(2).all(|pair| {
12765 pair[1].acknowledged_at_unix_millis
12766 .saturating_sub(pair[0].acknowledged_at_unix_millis)
12767 >= 850
12768 }),
12769 "successful acknowledgements must not catch up faster than the advertised one-second cadence: {observations:?}"
12770 );
12771 drop(observations);
12772
12773 let heartbeat_times = server.request_times("/api/worker/heartbeat");
12774 let delayed_request_at = *heartbeat_times
12775 .get(1)
12776 .expect("intentionally delayed heartbeat request");
12777 let delay_window_start = delayed_request_at + Duration::from_millis(100);
12778 let delay_window_end = delayed_request_at + Duration::from_millis(1_400);
12779 for path in [
12780 "/api/worker/workflow-tasks/poll",
12781 "/api/worker/activity-tasks/poll",
12782 "/api/worker/query-tasks/poll",
12783 ] {
12784 assert!(
12785 server
12786 .request_times(path)
12787 .iter()
12788 .any(|received_at| *received_at >= delay_window_start
12789 && *received_at <= delay_window_end),
12790 "{path} must keep polling while a heartbeat acknowledgement is delayed"
12791 );
12792 }
12793 assert!(
12794 server.request_count("/api/worker/workflow-tasks/snapshot-wait-3/fail") >= 1,
12795 "workflow work must be settled"
12796 );
12797 assert!(
12798 server.request_count("/api/worker/activity-tasks/activity-cancel/complete") >= 1,
12799 "activity work must be settled"
12800 );
12801 assert!(
12802 server.request_count("/api/worker/query-tasks/snapshot-current/complete") >= 1,
12803 "query work must be settled"
12804 );
12805 }
12806
12807 #[tokio::test]
12808 async fn retried_worker_heartbeat_restarts_the_advertised_cadence() {
12809 let server = MockWorkerServer::heartbeat_retry_worker();
12810 let client = Client::builder(server.base_url())
12811 .timeout(Duration::from_secs(2))
12812 .build()
12813 .expect("client");
12814 let observations = Arc::new(Mutex::new(Vec::new()));
12815 let observed = Arc::clone(&observations);
12816 let worker = Worker::new(client, "rust-workers")
12817 .worker_id("heartbeat-retry-worker")
12818 .retry_policy(WorkerRetryPolicy {
12819 max_retries: 1,
12820 initial_backoff: Duration::from_millis(300),
12821 max_backoff: Duration::from_millis(300),
12822 })
12823 .on_worker_heartbeat(move |observation| {
12824 observed
12825 .lock()
12826 .expect("heartbeat observations")
12827 .push(observation.clone());
12828 });
12829
12830 worker
12831 .run_until(tokio::time::sleep(Duration::from_millis(2_700)))
12832 .await
12833 .expect("retryable heartbeat failure must remain bounded and recover");
12834
12835 let observations = observations.lock().expect("heartbeat observations");
12836 assert!(observations.len() >= 3, "heartbeat retry must recover");
12837 assert!(
12838 observations.windows(2).all(|pair| {
12839 pair[1]
12840 .acknowledged_at_unix_millis
12841 .saturating_sub(pair[0].acknowledged_at_unix_millis)
12842 >= 850
12843 }),
12844 "a successful retry must start a fresh advertised cadence: {observations:?}"
12845 );
12846 assert_eq!(
12847 server.request_count("/api/worker/heartbeat"),
12848 observations.len() + 1,
12849 "one retryable failure must add exactly one bounded request"
12850 );
12851 }
12852
12853 #[tokio::test]
12854 async fn query_enabled_worker_ignores_unmatched_signals_then_completes_once() {
12855 let server = MockWorkerServer::waiting_query_worker();
12856 let client = Client::builder(server.base_url())
12857 .timeout(Duration::from_secs(2))
12858 .build()
12859 .expect("client");
12860 let observations = Arc::new(Mutex::new(Vec::new()));
12861 let observed = Arc::clone(&observations);
12862 let mut worker = Worker::new(client, "rust-snapshot-workers")
12863 .worker_id("rust-snapshot-worker")
12864 .poll_timeout(Duration::from_millis(10))
12865 .on_worker_heartbeat(move |observation| {
12866 observed
12867 .lock()
12868 .expect("heartbeat observations")
12869 .push(observation.clone());
12870 });
12871
12872 worker.register_workflow("snapshot", |ctx, _input| async move {
12873 ctx.wait_signal("finish").await?;
12874 Ok(json!({"status": "finished"}))
12875 });
12876 worker.register_query("snapshot", "current", |ctx, _args| async move {
12877 let current = ctx
12878 .signals("increment")
12879 .iter()
12880 .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
12881 .sum::<i64>();
12882 Ok(json!(current))
12883 });
12884 worker.register_update("snapshot", "replace", |_ctx, args| async move { Ok(args) });
12885
12886 worker
12887 .run_until(tokio::time::sleep(Duration::from_millis(3_200)))
12888 .await
12889 .expect("pending workflow and query poller must remain live until shutdown");
12890
12891 assert!(
12892 observations.lock().expect("heartbeat observations").len() >= 4,
12893 "the immediate heartbeat and at least three advertised one-second intervals must be acknowledged"
12894 );
12895 assert!(
12896 server.request_count("/api/worker/workflow-tasks/poll") >= 3,
12897 "workflow polling must continue after empty replay acknowledgements"
12898 );
12899 assert!(
12900 server.request_count("/api/worker/query-tasks/poll") >= 2,
12901 "query polling must continue after serving the current query"
12902 );
12903 assert_eq!(
12904 server.request_body("/api/worker/register")["capabilities"],
12905 json!([QUERY_TASKS_CAPABILITY, WORKFLOW_UPDATES_CAPABILITY])
12906 );
12907 assert_eq!(
12908 server.request_body("/api/worker/register")["workflow_command_contracts"]["snapshot"],
12909 json!({
12910 "queries": ["current"],
12911 "updates": ["replace"],
12912 })
12913 );
12914
12915 let opened = server.request_body("/api/worker/workflow-tasks/snapshot-open/complete");
12916 assert_eq!(
12917 opened["commands"],
12918 json!([{
12919 "type": "open_signal_wait",
12920 "signal_name": "finish",
12921 }])
12922 );
12923
12924 for task_id in ["snapshot-wait-3", "snapshot-wait-5"] {
12925 let fail_path = format!("/api/worker/workflow-tasks/{task_id}/fail");
12926 let completion_path = format!("/api/worker/workflow-tasks/{task_id}/complete");
12927 let failure = server.request_body(&fail_path);
12928 assert_eq!(
12929 failure["failure"]["type"],
12930 WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE
12931 );
12932 assert_eq!(server.request_count(&completion_path), 0);
12933 }
12934
12935 let query_completion =
12936 server.request_body("/api/worker/query-tasks/snapshot-current/complete");
12937 assert_eq!(query_completion["result"], json!(8));
12938
12939 let terminal_path = "/api/worker/workflow-tasks/snapshot-finish/complete";
12940 assert_eq!(
12941 server.request_count(terminal_path),
12942 1,
12943 "the matching signal must settle the workflow exactly once"
12944 );
12945 let terminal = server.request_body(terminal_path);
12946 assert_eq!(terminal["commands"].as_array().map(Vec::len), Some(1));
12947 assert_eq!(terminal["commands"][0]["type"], "complete_workflow");
12948 assert_eq!(
12949 decode_wire_value(&terminal["commands"][0]["result"], DEFAULT_CODEC)
12950 .expect("terminal workflow result"),
12951 json!({"status": "finished"})
12952 );
12953 }
12954
12955 #[tokio::test]
12956 async fn worker_retries_poll_and_heartbeat_transport_failures_independently() {
12957 let server = MockWorkerServer::transient_worker_failures();
12958 let client = Client::builder(server.base_url())
12959 .timeout(Duration::from_secs(2))
12960 .build()
12961 .expect("client");
12962 let mut worker = Worker::new(client, "rust-workers")
12963 .worker_id("retry-worker")
12964 .poll_timeout(Duration::from_millis(10))
12965 .retry_policy(WorkerRetryPolicy {
12966 max_retries: 2,
12967 initial_backoff: Duration::from_millis(1),
12968 max_backoff: Duration::from_millis(1),
12969 });
12970 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
12971 worker.register_activity(
12972 "counter.activity",
12973 |_ctx, _input| async move { Ok(Value::Null) },
12974 );
12975 worker.register_query(
12976 "counter",
12977 "current",
12978 |_ctx, _args| async move { Ok(json!(8)) },
12979 );
12980
12981 worker
12982 .run_until(tokio::time::sleep(Duration::from_millis(75)))
12983 .await
12984 .expect("transient failures must not stop the worker");
12985
12986 for path in [
12987 "/api/worker/heartbeat",
12988 "/api/worker/workflow-tasks/poll",
12989 "/api/worker/activity-tasks/poll",
12990 "/api/worker/query-tasks/poll",
12991 ] {
12992 assert!(
12993 server.request_count(path) >= 2,
12994 "{path} must continue after its transient failure"
12995 );
12996 }
12997 }
12998
12999 #[tokio::test]
13000 async fn worker_bounds_transport_retries() {
13001 let server = MockWorkerServer::unavailable_polls();
13002 let client = Client::builder(server.base_url())
13003 .timeout(Duration::from_secs(2))
13004 .build()
13005 .expect("client");
13006 let mut worker = Worker::new(client, "rust-workers")
13007 .worker_id("bounded-retry-worker")
13008 .poll_timeout(Duration::from_millis(10))
13009 .retry_policy(WorkerRetryPolicy {
13010 max_retries: 2,
13011 initial_backoff: Duration::from_millis(1),
13012 max_backoff: Duration::from_millis(1),
13013 });
13014 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
13015
13016 let error = worker.run().await.expect_err("retry bound must terminate");
13017 assert!(matches!(error, Error::Transport(_)));
13018 assert_eq!(
13019 server.request_count("/api/worker/workflow-tasks/poll"),
13020 3,
13021 "one initial request plus exactly two retries"
13022 );
13023 }
13024
13025 #[tokio::test]
13026 async fn worker_retry_policy_can_disable_poll_retries() {
13027 let server = MockWorkerServer::unavailable_polls();
13028 let client = Client::builder(server.base_url())
13029 .timeout(Duration::from_secs(2))
13030 .build()
13031 .expect("client");
13032 let mut worker = Worker::new(client, "rust-workers")
13033 .worker_id("no-retry-worker")
13034 .poll_timeout(Duration::from_millis(10))
13035 .retry_policy(WorkerRetryPolicy {
13036 max_retries: 0,
13037 initial_backoff: Duration::from_millis(1),
13038 max_backoff: Duration::from_millis(1),
13039 });
13040 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
13041
13042 let error = worker
13043 .run_once()
13044 .await
13045 .expect_err("disabled retries must return the first transport failure");
13046 assert!(matches!(error, Error::Transport(_)));
13047 assert_eq!(
13048 server.request_count("/api/worker/workflow-tasks/poll"),
13049 1,
13050 "max_retries=0 must send only the initial request"
13051 );
13052 }
13053
13054 #[tokio::test]
13055 async fn worker_does_not_retry_authentication_failures() {
13056 let server = MockWorkerServer::unauthorized_polls();
13057 let client = Client::builder(server.base_url())
13058 .timeout(Duration::from_secs(2))
13059 .build()
13060 .expect("client");
13061 let mut worker = Worker::new(client, "rust-workers")
13062 .worker_id("unauthorized-worker")
13063 .poll_timeout(Duration::from_millis(10));
13064 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
13065
13066 let error = worker
13067 .run()
13068 .await
13069 .expect_err("authentication must terminate");
13070 let Error::Http { status, body } = error else {
13071 panic!("expected stable HTTP authentication error");
13072 };
13073 assert_eq!(status, reqwest::StatusCode::UNAUTHORIZED);
13074 assert!(body.contains("authentication_failed"));
13075 assert_eq!(
13076 server.request_count("/api/worker/workflow-tasks/poll"),
13077 1,
13078 "authentication failures must not be retried"
13079 );
13080 }
13081
13082 #[derive(Clone, Debug)]
13083 struct CapturedRequest {
13084 path: String,
13085 worker_protocol: Option<String>,
13086 body: String,
13087 received_at: Instant,
13088 }
13089
13090 struct MockWorkerServer {
13091 addr: SocketAddr,
13092 stop: Arc<AtomicBool>,
13093 requests: Arc<Mutex<Vec<CapturedRequest>>>,
13094 thread: Option<thread::JoinHandle<()>>,
13095 }
13096
13097 #[derive(Clone, Copy, Default)]
13098 struct MockWorkerBehavior {
13099 reject_query_protocol: bool,
13100 reject_query_completion: bool,
13101 waiting_query_worker: bool,
13102 complete_named_signal: bool,
13103 poll_failures_per_path: usize,
13104 heartbeat_failures: usize,
13105 heartbeat_failure_request: Option<usize>,
13106 delayed_heartbeat_request: Option<usize>,
13107 heartbeat_response_delay: Duration,
13108 concurrent_requests: bool,
13109 unauthorized_polls: bool,
13110 cancelled_activity: bool,
13111 draining_polls: bool,
13112 workflow_completion_status: Option<&'static str>,
13113 workflow_completion_body: Option<&'static str>,
13114 }
13115
13116 impl MockWorkerServer {
13117 fn start() -> Self {
13118 Self::start_with_behavior(MockWorkerBehavior::default())
13119 }
13120
13121 fn reject_query_protocol() -> Self {
13122 Self::start_with_behavior(MockWorkerBehavior {
13123 reject_query_protocol: true,
13124 ..MockWorkerBehavior::default()
13125 })
13126 }
13127
13128 fn reject_query_completion() -> Self {
13129 Self::start_with_behavior(MockWorkerBehavior {
13130 reject_query_completion: true,
13131 ..MockWorkerBehavior::default()
13132 })
13133 }
13134
13135 fn waiting_query_worker() -> Self {
13136 Self::start_with_behavior(MockWorkerBehavior {
13137 waiting_query_worker: true,
13138 complete_named_signal: true,
13139 ..MockWorkerBehavior::default()
13140 })
13141 }
13142
13143 fn transient_worker_failures() -> Self {
13144 Self::start_with_behavior(MockWorkerBehavior {
13145 poll_failures_per_path: 1,
13146 heartbeat_failures: 1,
13147 ..MockWorkerBehavior::default()
13148 })
13149 }
13150
13151 fn consecutive_poll_failures(count: usize) -> Self {
13152 Self::start_with_behavior(MockWorkerBehavior {
13153 poll_failures_per_path: count,
13154 ..MockWorkerBehavior::default()
13155 })
13156 }
13157
13158 fn delayed_heartbeat_worker() -> Self {
13159 Self::start_with_behavior(MockWorkerBehavior {
13160 waiting_query_worker: true,
13161 delayed_heartbeat_request: Some(2),
13162 heartbeat_response_delay: Duration::from_millis(1_500),
13163 concurrent_requests: true,
13164 cancelled_activity: true,
13165 ..MockWorkerBehavior::default()
13166 })
13167 }
13168
13169 fn heartbeat_retry_worker() -> Self {
13170 Self::start_with_behavior(MockWorkerBehavior {
13171 waiting_query_worker: true,
13172 heartbeat_failure_request: Some(2),
13173 concurrent_requests: true,
13174 ..MockWorkerBehavior::default()
13175 })
13176 }
13177
13178 fn unavailable_polls() -> Self {
13179 Self::start_with_behavior(MockWorkerBehavior {
13180 poll_failures_per_path: usize::MAX,
13181 ..MockWorkerBehavior::default()
13182 })
13183 }
13184
13185 fn unauthorized_polls() -> Self {
13186 Self::start_with_behavior(MockWorkerBehavior {
13187 unauthorized_polls: true,
13188 ..MockWorkerBehavior::default()
13189 })
13190 }
13191
13192 fn cancelled_activity() -> Self {
13193 Self::start_with_behavior(MockWorkerBehavior {
13194 cancelled_activity: true,
13195 ..MockWorkerBehavior::default()
13196 })
13197 }
13198
13199 fn draining_polls() -> Self {
13200 Self::start_with_behavior(MockWorkerBehavior {
13201 draining_polls: true,
13202 ..MockWorkerBehavior::default()
13203 })
13204 }
13205
13206 fn workflow_completion(status: &'static str, body: &'static str) -> Self {
13207 Self::start_with_behavior(MockWorkerBehavior {
13208 workflow_completion_status: Some(status),
13209 workflow_completion_body: Some(body),
13210 ..MockWorkerBehavior::default()
13211 })
13212 }
13213
13214 fn start_with_behavior(behavior: MockWorkerBehavior) -> Self {
13215 let listener = TcpListener::bind("127.0.0.1:0").expect("bind mock server");
13216 listener
13217 .set_nonblocking(true)
13218 .expect("configure mock listener");
13219 let addr = listener.local_addr().expect("mock server address");
13220 let stop = Arc::new(AtomicBool::new(false));
13221 let server_stop = Arc::clone(&stop);
13222 let requests = Arc::new(Mutex::new(Vec::new()));
13223 let server_requests = Arc::clone(&requests);
13224 let thread = thread::spawn(move || {
13225 let mut request_threads = Vec::new();
13226 while !server_stop.load(Ordering::SeqCst) {
13227 match listener.accept() {
13228 Ok((mut stream, _)) => {
13229 if behavior.concurrent_requests {
13230 let requests = Arc::clone(&server_requests);
13231 request_threads.push(thread::spawn(move || {
13232 handle_mock_worker_request(&mut stream, &requests, behavior)
13233 }));
13234 } else {
13235 handle_mock_worker_request(&mut stream, &server_requests, behavior);
13236 }
13237 }
13238 Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
13239 let mut index = 0;
13240 while index < request_threads.len() {
13241 if request_threads[index].is_finished() {
13242 request_threads
13243 .swap_remove(index)
13244 .join()
13245 .expect("join mock request");
13246 } else {
13247 index += 1;
13248 }
13249 }
13250 thread::sleep(Duration::from_millis(5));
13251 }
13252 Err(_) => break,
13253 }
13254 }
13255 for request_thread in request_threads {
13256 request_thread.join().expect("join mock request");
13257 }
13258 });
13259
13260 Self {
13261 addr,
13262 stop,
13263 requests,
13264 thread: Some(thread),
13265 }
13266 }
13267
13268 fn base_url(&self) -> String {
13269 format!("http://{}", self.addr)
13270 }
13271
13272 fn worker_protocol_for(&self, path: &str) -> Option<String> {
13273 self.requests
13274 .lock()
13275 .expect("captured requests")
13276 .iter()
13277 .find(|request| request.path == path)
13278 .and_then(|request| request.worker_protocol.clone())
13279 }
13280
13281 fn request_count(&self, path: &str) -> usize {
13282 self.requests
13283 .lock()
13284 .expect("captured requests")
13285 .iter()
13286 .filter(|request| request.path == path)
13287 .count()
13288 }
13289
13290 fn request_times(&self, path: &str) -> Vec<Instant> {
13291 self.requests
13292 .lock()
13293 .expect("captured requests")
13294 .iter()
13295 .filter(|request| request.path == path)
13296 .map(|request| request.received_at)
13297 .collect()
13298 }
13299
13300 fn request_body(&self, path: &str) -> Value {
13301 let requests = self.requests.lock().expect("captured requests");
13302 let body = &requests
13303 .iter()
13304 .find(|request| request.path == path)
13305 .unwrap_or_else(|| panic!("missing request for {path}"))
13306 .body;
13307 serde_json::from_str(body).unwrap_or_else(|error| {
13308 panic!("invalid JSON request body for {path}: {error}: {body:?}")
13309 })
13310 }
13311
13312 fn request_bodies(&self, path: &str) -> Vec<Value> {
13313 self.requests
13314 .lock()
13315 .expect("captured requests")
13316 .iter()
13317 .filter(|request| request.path == path)
13318 .map(|request| {
13319 serde_json::from_str(&request.body).unwrap_or_else(|error| {
13320 panic!(
13321 "invalid JSON request body for {path}: {error}: {:?}",
13322 request.body
13323 )
13324 })
13325 })
13326 .collect()
13327 }
13328 }
13329
13330 impl Drop for MockWorkerServer {
13331 fn drop(&mut self) {
13332 self.stop.store(true, Ordering::SeqCst);
13333 let _ = TcpStream::connect(self.addr);
13334
13335 if let Some(thread) = self.thread.take() {
13336 thread.join().expect("join mock server");
13337 }
13338 }
13339 }
13340
13341 fn handle_mock_worker_request(
13342 stream: &mut TcpStream,
13343 requests: &Arc<Mutex<Vec<CapturedRequest>>>,
13344 behavior: MockWorkerBehavior,
13345 ) {
13346 let _ = stream.set_read_timeout(Some(Duration::from_millis(200)));
13347 let mut buffer = [0_u8; 8192];
13348 let mut request = Vec::new();
13349
13350 loop {
13351 match stream.read(&mut buffer) {
13352 Ok(0) => break,
13353 Ok(read) => {
13354 request.extend_from_slice(&buffer[..read]);
13355 if mock_request_is_complete(&request) {
13356 break;
13357 }
13358 }
13359 Err(error)
13360 if matches!(
13361 error.kind(),
13362 std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
13363 ) =>
13364 {
13365 break;
13366 }
13367 Err(_) => return,
13368 }
13369 }
13370
13371 let request = String::from_utf8_lossy(&request);
13372 let body = request
13373 .split_once("\r\n\r\n")
13374 .map(|(_, body)| body)
13375 .unwrap_or_default();
13376 let path = request
13377 .lines()
13378 .next()
13379 .and_then(|line| line.split_whitespace().nth(1))
13380 .unwrap_or_default();
13381 let worker_protocol = request.lines().find_map(|line| {
13382 let (name, value) = line.split_once(':')?;
13383 name.eq_ignore_ascii_case("X-Durable-Workflow-Protocol-Version")
13384 .then(|| value.trim().to_string())
13385 });
13386 let request_number = {
13387 let mut requests = requests.lock().expect("captured requests");
13388 requests.push(CapturedRequest {
13389 path: path.to_string(),
13390 worker_protocol: worker_protocol.clone(),
13391 body: body.to_string(),
13392 received_at: Instant::now(),
13393 });
13394 requests
13395 .iter()
13396 .filter(|request| request.path == path)
13397 .count()
13398 };
13399
13400 let is_poll = matches!(
13401 path,
13402 "/api/worker/workflow-tasks/poll"
13403 | "/api/worker/activity-tasks/poll"
13404 | "/api/worker/query-tasks/poll"
13405 );
13406 if is_poll && request_number <= behavior.poll_failures_per_path {
13407 return;
13408 }
13409 if path == "/api/worker/heartbeat" && request_number <= behavior.heartbeat_failures {
13410 return;
13411 }
13412 if path == "/api/worker/heartbeat"
13413 && behavior.heartbeat_failure_request == Some(request_number)
13414 {
13415 return;
13416 }
13417 if path == "/api/worker/heartbeat"
13418 && behavior.delayed_heartbeat_request == Some(request_number)
13419 {
13420 thread::sleep(behavior.heartbeat_response_delay);
13421 }
13422 if behavior.unauthorized_polls && is_poll {
13423 write_mock_response(
13424 stream,
13425 "401 Unauthorized",
13426 r#"{"reason":"authentication_failed","message":"invalid worker token"}"#,
13427 );
13428 return;
13429 }
13430 if behavior.draining_polls && is_poll {
13431 write_mock_response(
13432 stream,
13433 "409 Conflict",
13434 r#"{"task":null,"poll_status":"draining","reason":"worker_draining","worker_status":"draining","drain_intent":"draining"}"#,
13435 );
13436 return;
13437 }
13438
13439 if behavior.reject_query_protocol && path.starts_with("/api/worker/query-tasks/") {
13440 let requested_version = worker_protocol.as_deref().unwrap_or("missing");
13441 let body = format!(
13442 r#"{{"reason":"unsupported_protocol_version","message":"Query tasks require worker protocol 1.8 or newer.","supported_version":"1.7","requested_version":"{requested_version}"}}"#
13443 );
13444 write_mock_response(stream, "400 Bad Request", &body);
13445 return;
13446 }
13447
13448 if behavior.reject_query_completion && path == "/api/worker/query-tasks/query-late/complete"
13449 {
13450 write_mock_response(
13451 stream,
13452 "409 Conflict",
13453 r#"{"reason":"query_task_timed_out","message":"query task timed out before completion"}"#,
13454 );
13455 return;
13456 }
13457
13458 if behavior.workflow_completion_status.is_some()
13459 && path == "/api/worker/workflow-tasks/poll"
13460 && request_number == 1
13461 {
13462 write_mock_response(
13463 stream,
13464 "200 OK",
13465 r#"{"task":{"task_id":"workflow-timeout-task","workflow_id":"reused-workflow-id","run_id":"run-selected-timeout","workflow_type":"timeout.workflow","payload_codec":"json","arguments":{"codec":"json","blob":"[]"},"history_events":[],"workflow_task_attempt":3,"lease_owner":"timeout-worker"}}"#,
13466 );
13467 return;
13468 }
13469
13470 if path == "/api/worker/workflow-tasks/workflow-timeout-task/complete" {
13471 if let (Some(status), Some(body)) = (
13472 behavior.workflow_completion_status,
13473 behavior.workflow_completion_body,
13474 ) {
13475 write_mock_response(stream, status, body);
13476 return;
13477 }
13478 }
13479
13480 if behavior.waiting_query_worker {
13481 if behavior.complete_named_signal
13482 && path == "/api/worker/workflow-tasks/poll"
13483 && request_number == 1
13484 {
13485 let body = json!({
13486 "task": {
13487 "task_id": "snapshot-open",
13488 "workflow_id": "snapshot-1",
13489 "run_id": "snapshot-run-1",
13490 "workflow_type": "snapshot",
13491 "payload_codec": DEFAULT_CODEC,
13492 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
13493 .expect("Avro workflow arguments"),
13494 "history_events": [],
13495 "workflow_task_attempt": 1,
13496 "lease_owner": "rust-snapshot-worker"
13497 }
13498 })
13499 .to_string();
13500 write_mock_response(stream, "200 OK", &body);
13501 return;
13502 }
13503
13504 let signal_request = request_number - usize::from(behavior.complete_named_signal);
13505 let signal_request_limit = 2 + usize::from(behavior.complete_named_signal);
13506 if path == "/api/worker/workflow-tasks/poll"
13507 && signal_request >= 1
13508 && signal_request <= signal_request_limit
13509 {
13510 let finish = behavior.complete_named_signal && signal_request == 3;
13511 let amounts = if signal_request == 1 {
13512 vec![3]
13513 } else {
13514 vec![3, 5]
13515 };
13516 let task_id = if signal_request == 1 {
13517 "snapshot-wait-3"
13518 } else if finish {
13519 "snapshot-finish"
13520 } else {
13521 "snapshot-wait-5"
13522 };
13523 let mut history_events = std::iter::once(json!({
13524 "event_type": "SignalWaitOpened",
13525 "payload": {"sequence": 1, "signal_name": "finish"}
13526 }))
13527 .chain(amounts.iter().enumerate().map(|(index, amount)| {
13528 json!({
13529 "event_type": "SignalReceived",
13530 "payload": {
13531 "signal_id": format!("increment-{amount}"),
13532 "signal_name": "increment",
13533 "workflow_sequence": index + 2,
13534 "payload_codec": DEFAULT_CODEC,
13535 "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
13536 .expect("Avro signal envelope")
13537 }
13538 })
13539 }))
13540 .collect::<Vec<_>>();
13541 let (resume_id, resume_name, resume_arguments) = if finish {
13542 history_events.push(json!({
13543 "event_type": "SignalReceived",
13544 "payload": {
13545 "signal_id": "finish",
13546 "signal_name": "finish",
13547 "workflow_sequence": 4,
13548 "payload_codec": DEFAULT_CODEC,
13549 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
13550 .expect("Avro finish signal envelope")
13551 }
13552 }));
13553 (
13554 "finish".to_string(),
13555 "finish".to_string(),
13556 encode_value_envelope(&json!([]), DEFAULT_CODEC)
13557 .expect("Avro finish resume signal"),
13558 )
13559 } else {
13560 let amount = amounts.last().expect("amount");
13561 (
13562 format!("increment-{amount}"),
13563 "increment".to_string(),
13564 encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
13565 .expect("Avro increment resume signal"),
13566 )
13567 };
13568 let body = json!({
13569 "task": {
13570 "task_id": task_id,
13571 "workflow_id": "snapshot-1",
13572 "run_id": "snapshot-run-1",
13573 "workflow_type": "snapshot",
13574 "payload_codec": DEFAULT_CODEC,
13575 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
13576 .expect("Avro workflow arguments"),
13577 "history_events": history_events,
13578 "workflow_task_attempt": 1,
13579 "workflow_signal_id": resume_id,
13580 "signal_name": resume_name,
13581 "signal_arguments": resume_arguments,
13582 "lease_owner": "rust-snapshot-worker"
13583 }
13584 })
13585 .to_string();
13586 write_mock_response(stream, "200 OK", &body);
13587 return;
13588 }
13589
13590 if path == "/api/worker/query-tasks/poll" && request_number == 1 {
13591 let history_events = [3, 5]
13592 .into_iter()
13593 .enumerate()
13594 .map(|(index, amount)| {
13595 json!({
13596 "event_type": "SignalReceived",
13597 "payload": {
13598 "signal_id": format!("increment-{amount}"),
13599 "signal_name": "increment",
13600 "workflow_sequence": index + 2,
13601 "payload_codec": DEFAULT_CODEC,
13602 "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
13603 .expect("Avro query signal envelope")
13604 }
13605 })
13606 })
13607 .collect::<Vec<_>>();
13608 let body = json!({
13609 "task": {
13610 "query_task_id": "snapshot-current",
13611 "query_task_attempt": 1,
13612 "lease_owner": "rust-snapshot-worker",
13613 "workflow_id": "snapshot-1",
13614 "run_id": "snapshot-run-1",
13615 "workflow_type": "snapshot",
13616 "query_name": "current",
13617 "payload_codec": DEFAULT_CODEC,
13618 "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
13619 .expect("Avro workflow arguments"),
13620 "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
13621 .expect("Avro query arguments"),
13622 "history_events": history_events,
13623 "run_status": "waiting"
13624 }
13625 })
13626 .to_string();
13627 write_mock_response(stream, "200 OK", &body);
13628 return;
13629 }
13630
13631 if path == "/api/worker/workflow-tasks/snapshot-wait-3/fail"
13632 || path == "/api/worker/workflow-tasks/snapshot-wait-5/fail"
13633 {
13634 write_mock_response(
13635 stream,
13636 "200 OK",
13637 r#"{"outcome":"waiting_for_history","recorded":true}"#,
13638 );
13639 return;
13640 }
13641
13642 if path == "/api/worker/workflow-tasks/snapshot-open/complete" {
13643 write_mock_response(stream, "200 OK", r#"{"outcome":"waiting","recorded":true}"#);
13644 return;
13645 }
13646
13647 if path == "/api/worker/workflow-tasks/snapshot-finish/complete" {
13648 write_mock_response(
13649 stream,
13650 "200 OK",
13651 r#"{"outcome":"completed","run_status":"completed","recorded":true}"#,
13652 );
13653 return;
13654 }
13655
13656 if path == "/api/worker/query-tasks/snapshot-current/complete" {
13657 write_mock_response(stream, "200 OK", r#"{"outcome":"completed"}"#);
13658 return;
13659 }
13660 }
13661
13662 if matches!(
13663 path,
13664 "/api/workflows/typed-1/query/inspect" | "/api/workflows/typed-1/update/replace"
13665 ) {
13666 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
13667 .expect("typed mock result");
13668 let body = json!({
13669 "result": typed_fidelity_probe().into_json().expect("result projection"),
13670 "result_envelope": result,
13671 })
13672 .to_string();
13673 write_mock_response(stream, "200 OK", &body);
13674 return;
13675 }
13676
13677 if path == "/api/workflows/typed-1" {
13678 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
13679 .expect("typed mock result");
13680 let body = json!({
13681 "workflow_id": "typed-1",
13682 "run_id": "run-typed-1",
13683 "workflow_type": "typed.echo",
13684 "status": "completed",
13685 "output": typed_fidelity_probe().into_json().expect("output projection"),
13686 "output_envelope": result,
13687 })
13688 .to_string();
13689 write_mock_response(stream, "200 OK", &body);
13690 return;
13691 }
13692
13693 let (status, body) = match path {
13694 "/api/workflows" => (
13695 "201 Created",
13696 r#"{"workflow_id":"wf-start-options","run_id":"run-start-options","workflow_type":"rust.timeout"}"#,
13697 ),
13698 "/api/worker/register" if behavior.waiting_query_worker => (
13699 "200 OK",
13700 r#"{"worker_id":"rust-snapshot-worker","registered":true,"heartbeat_interval_seconds":1}"#,
13701 ),
13702 "/api/worker/register" => (
13703 "200 OK",
13704 r#"{"worker_id":"mock-worker","registered":true,"heartbeat_interval_seconds":3600}"#,
13705 ),
13706 "/api/worker/heartbeat" => ("200 OK", "{}"),
13707 "/api/worker/activity-tasks/poll"
13708 if behavior.cancelled_activity && request_number == 1 =>
13709 {
13710 (
13711 "200 OK",
13712 r#"{"task":{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","activity_type":"cancel-aware","payload_codec":"json","arguments":{"codec":"json","blob":"[]"},"attempt_number":1,"lease_owner":"rust-cancel-worker"}}"#,
13713 )
13714 }
13715 "/api/worker/activity-tasks/poll" | "/api/worker/workflow-tasks/poll" => {
13716 ("200 OK", r#"{"task":null}"#)
13717 }
13718 "/api/worker/query-tasks/poll"
13719 if behavior.reject_query_completion && request_number == 1 =>
13720 {
13721 (
13722 "200 OK",
13723 r#"{"task":{"query_task_id":"query-late","query_task_attempt":1,"lease_owner":"late-worker","workflow_id":"counter-late","run_id":"run-late","workflow_type":"counter","query_name":"current","payload_codec":"json","workflow_arguments":{"codec":"json","blob":"[]"},"query_arguments":{"codec":"json","blob":"[]"},"history_events":[],"run_status":"running"}}"#,
13724 )
13725 }
13726 "/api/worker/query-tasks/poll" => ("200 OK", r#"{"task":null}"#),
13727 "/api/worker/query-tasks/query-capture/complete"
13728 | "/api/worker/query-tasks/query-capture/fail" => ("200 OK", "{}"),
13729 "/api/worker/activity-tasks/activity-cancel/heartbeat" => (
13730 "200 OK",
13731 r#"{"activity_attempt_id":"attempt-cancel","cancel_requested":true,"can_continue":false,"reason":"run_cancelled","run_closed_reason":"cancelled","heartbeat_recorded":false}"#,
13732 ),
13733 "/api/worker/activity-tasks/activity-cancel/complete" => (
13734 "409 Conflict",
13735 r#"{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","reason":"run_cancelled","cancel_requested":true,"can_continue":false,"run_closed_reason":"cancelled"}"#,
13736 ),
13737 "/api/worker/activity-tasks/activity-typed/complete"
13738 | "/api/worker/activity-tasks/activity-typed/fail"
13739 | "/api/workflows/typed-1/signal/changed" => ("200 OK", "{}"),
13740 "/api/workflows/counter-1/query/current" => (
13741 "200 OK",
13742 r#"{"workflow_id":"counter-1","query_name":"current","result":{"count":8},"result_envelope":{"codec":"json","blob":"{\"count\":8}"}}"#,
13743 ),
13744 "/api/workflows/counter-1/query/missing" => (
13745 "404 Not Found",
13746 r#"{"workflow_id":"counter-1","query_name":"missing","reason":"rejected_unknown_query","message":"unknown query"}"#,
13747 ),
13748 "/api/workflows/wf-lifecycle/cancel" => (
13749 "200 OK",
13750 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","reason":"cleanup requested","command_status":"accepted"}"#,
13751 ),
13752 "/api/workflows/wf-lifecycle/terminate" => (
13753 "200 OK",
13754 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","reason":"forced stop","command_status":"accepted"}"#,
13755 ),
13756 "/api/workflows/wf-lifecycle/runs/run-current/cancel" => (
13757 "200 OK",
13758 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","command_status":"accepted"}"#,
13759 ),
13760 "/api/workflows/wf-lifecycle/runs/run-current/terminate" => (
13761 "200 OK",
13762 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","command_status":"accepted"}"#,
13763 ),
13764 "/api/workflows/wf-lifecycle/runs/run-stale/cancel"
13765 | "/api/workflows/wf-lifecycle/runs/run-stale/terminate" => (
13766 "409 Conflict",
13767 r#"{"workflow_id":"wf-lifecycle","run_id":"run-stale","reason":"historical_run_command_rejected","target_scope":"run","message":"Commands cannot target historical runs."}"#,
13768 ),
13769 "/api/workflows/wf-failed" | "/api/workflows/wf-failed/runs/run-failed" => (
13770 "200 OK",
13771 r#"{"workflow_id":"wf-failed","run_id":"run-failed","status":"failed","closed_reason":"failed","error":"payment failed","failure":{"message":"payment failed","failure_category":"application","exception_type":"PaymentError","exception_class":"billing::PaymentError","non_retryable":true,"exception":{"type":"PaymentError","class":"billing::PaymentError","message":"payment failed"},"failures":[{"id":"failure-17","failure_category":"application"}]}}"#,
13772 ),
13773 "/api/workflows/wf-cancelled" => (
13774 "200 OK",
13775 r#"{"workflow_id":"wf-cancelled","run_id":"run-cancelled","status":"cancelled","closed_reason":"cancelled","reason":"cleanup requested"}"#,
13776 ),
13777 "/api/workflows/wf-terminated" => (
13778 "200 OK",
13779 r#"{"workflow_id":"wf-terminated","run_id":"run-terminated","status":"terminated","closed_reason":"terminated","reason":"forced stop"}"#,
13780 ),
13781 "/api/workflows/wf-timed-out" => (
13782 "200 OK",
13783 r#"{"workflow_id":"wf-timed-out","run_id":"run-timed-out","status":"failed","closed_reason":"timed_out","reason":"run_timeout"}"#,
13784 ),
13785 "/api/workflows/wf-waiting" | "/api/workflows/wf-waiting/runs/run-waiting" => (
13786 "200 OK",
13787 r#"{"workflow_id":"wf-waiting","run_id":"run-waiting","status":"waiting"}"#,
13788 ),
13789 "/api/workflows/wf-selected" => (
13790 "200 OK",
13791 r#"{"workflow_id":"wf-selected","run_id":"run-current","status":"completed","output":"current run output"}"#,
13792 ),
13793 "/api/workflows/wf-selected/runs/run-selected" => (
13794 "200 OK",
13795 r#"{"workflow_id":"wf-selected","run_id":"run-selected","status":"cancelled","closed_reason":"cancelled","reason":"selected run cancelled"}"#,
13796 ),
13797 _ => ("404 Not Found", r#"{"message":"not found"}"#),
13798 };
13799 write_mock_response(stream, status, body);
13800 }
13801
13802 fn mock_request_is_complete(request: &[u8]) -> bool {
13803 let Some(header_end) = request
13804 .windows(4)
13805 .position(|window| window == b"\r\n\r\n")
13806 .map(|position| position + 4)
13807 else {
13808 return false;
13809 };
13810 let headers = String::from_utf8_lossy(&request[..header_end]);
13811 let content_length = headers.lines().find_map(|line| {
13812 let (name, value) = line.split_once(':')?;
13813 name.eq_ignore_ascii_case("content-length")
13814 .then(|| value.trim().parse::<usize>().ok())
13815 .flatten()
13816 });
13817
13818 request.len() >= header_end + content_length.unwrap_or(0)
13819 }
13820
13821 fn write_mock_response(stream: &mut TcpStream, status: &str, body: &str) {
13822 let response = format!(
13823 "HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
13824 body.len()
13825 );
13826
13827 let _ = stream.write_all(response.as_bytes());
13828 let _ = stream.flush();
13829 }
13830}