1#![doc = include_str!("../README.md")]
2
3use std::{
4 any::{Any, TypeId},
5 collections::{BTreeMap, HashMap},
6 future::Future,
7 io::{self, Read},
8 pin::Pin,
9 sync::{
10 atomic::{AtomicBool, Ordering},
11 Arc, Mutex, OnceLock,
12 },
13 task::{Context as TaskContext, Poll},
14 time::{Duration, Instant, SystemTime, UNIX_EPOCH},
15};
16
17use apache_avro::{from_avro_datum, to_avro_datum, types::Value as AvroDatum, Schema};
18use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
19use futures_util::{future::OptionFuture, task::noop_waker_ref};
20use serde::{
21 de::DeserializeOwned,
22 ser::{SerializeMap, SerializeSeq},
23 Deserialize, Serialize, Serializer,
24};
25pub use serde_json::{json, Value};
26use thiserror::Error;
27pub use uuid::Uuid;
28
29pub const WORKER_PROTOCOL_VERSION: &str = "1.2";
30pub const CONTROL_PLANE_VERSION: &str = "2";
31pub const DEFAULT_CODEC: &str = "avro";
32pub const SDK_VERSION: &str = concat!("durable-workflow-rust/", env!("CARGO_PKG_VERSION"));
33pub const QUERY_TASKS_CAPABILITY: &str = "query_tasks";
35pub const WORKFLOW_UPDATES_CAPABILITY: &str = "workflow_updates";
37pub const QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
39
40const MAX_LONG_POLL_TIMEOUT_SECONDS: u64 = 60;
41const WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE: &str =
42 "Workflow task waiting for scheduled history.";
43const WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE: &str = "WorkflowTaskWaitingForHistory";
44
45const QUERY_TASK_FINAL_REJECTION_REASONS: &[&str] = &[
46 "lease_expired",
47 "query_task_not_found",
48 "query_task_not_leased",
49 "query_task_timed_out",
50];
51
52pub const AVRO_VALUE_SCHEMA_JSON: &str =
54 include_str!("../schema/durable_workflow.protocol.Value.v1.avsc");
55pub const AVRO_VALUE_SCHEMA_FINGERPRINT_HEX: &str = "e2a33dff55802237";
56pub const AVRO_VALUE_SCHEMA_FINGERPRINT: [u8; 8] = [0xe2, 0xa3, 0x3d, 0xff, 0x55, 0x80, 0x22, 0x37];
57const AVRO_SINGLE_OBJECT_MAGIC: [u8; 2] = [0xc3, 0x01];
58
59static AVRO_VALUE_SCHEMA: OnceLock<std::result::Result<Schema, String>> = OnceLock::new();
60
61#[derive(Clone, Copy)]
62enum RequestProtocol {
63 ControlPlane,
64 Worker(&'static str),
65}
66
67pub type Result<T> = std::result::Result<T, Error>;
68
69#[derive(Debug, Error)]
70pub enum Error {
71 #[error("transport error: {0}")]
72 Transport(#[from] reqwest::Error),
73 #[error(
74 "invalid Durable Workflow base URL: omit the SDK-owned /api suffix and pass the Server or Cloud runtime base URL; the SDK appends /api automatically"
75 )]
76 InvalidBaseUrl,
77 #[error("json error: {0}")]
78 Json(#[from] serde_json::Error),
79 #[error("http {status}: {body}")]
80 Http {
81 status: reqwest::StatusCode,
82 body: String,
83 },
84 #[error("codec error: {0}")]
85 Codec(String),
86 #[error(transparent)]
87 QueryFailed(QueryFailure),
88 #[error(transparent)]
89 Protocol(ProtocolFailure),
90 #[error(transparent)]
91 NonDeterministicReplay(ReplayFailure),
92 #[error(transparent)]
93 ChildWorkflowFailed(ChildWorkflowFailure),
94 #[error(transparent)]
95 ActivityFailed(ActivityFailure),
96 #[error(transparent)]
97 WorkflowCommandRejected(WorkflowCommandRejection),
98 #[error(transparent)]
99 WorkflowFailed(WorkflowTerminalOutcome),
100 #[error(transparent)]
101 WorkflowCancelled(WorkflowTerminalOutcome),
102 #[error(transparent)]
103 WorkflowTerminated(WorkflowTerminalOutcome),
104 #[error(transparent)]
105 WorkflowTimedOut(WorkflowTerminalOutcome),
106 #[error(transparent)]
107 ActivityTaskRejected(ActivityTaskRejection),
108 #[error("workflow handler {0:?} is not registered")]
109 WorkflowNotRegistered(String),
110 #[error("activity handler {0:?} is not registered")]
111 ActivityNotRegistered(String),
112 #[error("workflow future yielded without emitting a durable command")]
113 WorkflowYieldedWithoutCommand,
114 #[error("workflow state lock is poisoned")]
115 WorkflowStatePoisoned,
116 #[error("timer duration is too large for the worker protocol")]
117 TimerDurationOverflow,
118 #[error("operation timed out")]
119 Timeout,
120 #[error(
121 "missing {role}-plane credentials: configure ClientBuilder::{role}_token or ClientBuilder::token; a {opposite_role}-plane token cannot authorize this request"
122 )]
123 MissingRoleCredentials {
124 role: &'static str,
125 opposite_role: &'static str,
126 },
127 #[error("worker loop error: {0}")]
128 WorkerLoop(String),
129 #[error(
130 "workflow command contract for {workflow_type:?} declares update validators, but this Rust SDK cannot execute synchronous pre-accept update validation"
131 )]
132 UnsupportedUpdateValidators { workflow_type: String },
133 #[error("{primary}; worker deregistration also failed: {deregistration}")]
134 WorkerShutdown {
135 primary: Box<Error>,
136 deregistration: Box<Error>,
137 },
138 #[error("invalid child workflow options: {0}")]
139 InvalidChildWorkflowOptions(String),
140 #[error(transparent)]
141 InvalidActivityOptions(ActivityOptionsError),
142 #[error(transparent)]
143 InvalidContinueAsNewOptions(#[from] ContinueAsNewOptionsError),
144 #[doc(hidden)]
145 #[error("workflow requested continue as new")]
146 ContinueAsNew(ContinueAsNewRequest),
147}
148
149#[derive(Clone, Copy, Debug, PartialEq, Eq)]
151pub enum WorkflowCommandKind {
152 Cancel,
153 Terminate,
154}
155
156impl WorkflowCommandKind {
157 fn as_str(self) -> &'static str {
158 match self {
159 Self::Cancel => "cancel",
160 Self::Terminate => "terminate",
161 }
162 }
163}
164
165#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
167pub struct WorkflowCommandOptions {
168 #[serde(skip_serializing_if = "Option::is_none")]
169 pub reason: Option<String>,
170 #[serde(skip_serializing_if = "Option::is_none")]
171 pub request_id: Option<String>,
172}
173
174#[derive(Clone, Debug, PartialEq, Eq)]
181pub struct WorkflowStartOptions {
182 pub execution_timeout_seconds: u64,
183 pub run_timeout_seconds: u64,
184}
185
186impl Default for WorkflowStartOptions {
187 fn default() -> Self {
188 Self {
189 execution_timeout_seconds: 3600,
190 run_timeout_seconds: 600,
191 }
192 }
193}
194
195impl WorkflowStartOptions {
196 pub fn new() -> Self {
197 Self::default()
198 }
199
200 pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
201 self.execution_timeout_seconds = seconds;
202 self
203 }
204
205 pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
206 self.run_timeout_seconds = seconds;
207 self
208 }
209
210 fn validate(&self) -> Result<()> {
211 if self.execution_timeout_seconds == 0 {
212 return Err(Error::Codec(
213 "execution_timeout_seconds must be at least 1".to_string(),
214 ));
215 }
216 if self.run_timeout_seconds == 0 {
217 return Err(Error::Codec(
218 "run_timeout_seconds must be at least 1".to_string(),
219 ));
220 }
221 if self.run_timeout_seconds > self.execution_timeout_seconds {
222 return Err(Error::Codec(
223 "run_timeout_seconds cannot exceed execution_timeout_seconds".to_string(),
224 ));
225 }
226
227 Ok(())
228 }
229}
230
231#[derive(Clone, Debug, Default, PartialEq, Eq)]
236pub struct ContinueAsNewOptions {
237 pub workflow_type: Option<String>,
238 pub task_queue: Option<String>,
239}
240
241impl ContinueAsNewOptions {
242 pub fn new() -> Self {
243 Self::default()
244 }
245
246 pub fn workflow_type(mut self, workflow_type: impl Into<String>) -> Self {
247 self.workflow_type = Some(workflow_type.into());
248 self
249 }
250
251 pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
252 self.task_queue = Some(task_queue.into());
253 self
254 }
255
256 fn validate(&self) -> std::result::Result<(), ContinueAsNewOptionsError> {
257 for (field, value) in [
258 ("workflow_type", self.workflow_type.as_deref()),
259 ("task_queue", self.task_queue.as_deref()),
260 ] {
261 if value.is_some_and(|value| value.trim().is_empty()) {
262 return Err(ContinueAsNewOptionsError {
263 field,
264 message: format!("{field} must not be empty"),
265 });
266 }
267 }
268 Ok(())
269 }
270}
271
272#[derive(Clone, Debug, Error, PartialEq, Eq)]
274#[error("invalid continue-as-new option {field}: {message}")]
275pub struct ContinueAsNewOptionsError {
276 pub field: &'static str,
277 pub message: String,
278}
279
280#[derive(Clone, Debug, Default, PartialEq, Eq)]
282pub struct WorkflowHistoryBudget {
283 pub event_count: u64,
284 pub size_bytes: Option<u64>,
285 pub continue_as_new_recommended: bool,
286 pub pressure: Option<String>,
287}
288
289#[doc(hidden)]
290#[derive(Clone, Debug)]
291pub struct ContinueAsNewRequest {
292 arguments: AvroValue,
293 options: ContinueAsNewOptions,
294}
295
296impl WorkflowCommandOptions {
297 pub fn new() -> Self {
298 Self::default()
299 }
300
301 pub fn reason(mut self, reason: impl Into<String>) -> Self {
302 self.reason = Some(reason.into());
303 self
304 }
305
306 pub fn request_id(mut self, request_id: impl Into<String>) -> Self {
307 self.request_id = Some(request_id.into());
308 self
309 }
310}
311
312#[derive(Clone, Debug, PartialEq)]
314pub struct WorkflowCommandResult {
315 pub command: WorkflowCommandKind,
316 pub workflow_id: String,
317 pub run_id: Option<String>,
318 pub outcome: Option<String>,
319 pub reason: Option<String>,
320 pub command_status: Option<String>,
321 pub raw: Value,
322}
323
324#[derive(Clone, Debug, Error)]
326#[error("workflow {command:?} rejected ({reason}, HTTP {status}): {message}")]
327pub struct WorkflowCommandRejection {
328 pub command: WorkflowCommandKind,
329 pub status: u16,
330 pub reason: String,
331 pub message: String,
332 pub workflow_id: String,
333 pub run_id: Option<String>,
334 pub target_scope: Option<String>,
335 pub body: Value,
336}
337
338#[derive(Clone, Copy, Debug, PartialEq, Eq)]
340pub enum WorkflowTerminalKind {
341 Failed,
342 Cancelled,
343 Terminated,
344 TimedOut,
345}
346
347#[derive(Clone, Debug, Error)]
353#[error("workflow {workflow_id} run {run_id:?} ended as {kind:?} ({reason})")]
354pub struct WorkflowTerminalOutcome {
355 pub kind: WorkflowTerminalKind,
356 pub workflow_id: String,
357 pub run_id: Option<String>,
358 pub reason: String,
359 pub failure_category: Option<String>,
360 pub failure_id: Option<String>,
361 pub exception_type: Option<String>,
362 pub exception_class: Option<String>,
363 pub non_retryable: Option<bool>,
364 pub message: Option<String>,
365 pub exception: Option<Value>,
366 pub raw: Value,
367}
368
369#[derive(Clone, Debug, Error)]
371#[error("activity task {operation} rejected ({reason}, HTTP {status})")]
372pub struct ActivityTaskRejection {
373 pub operation: String,
374 pub status: u16,
375 pub reason: String,
376 pub task_id: String,
377 pub activity_attempt_id: String,
378 pub cancel_requested: bool,
379 pub can_continue: Option<bool>,
380 pub run_closed_reason: Option<String>,
381 pub body: Value,
382}
383
384#[derive(Clone, Copy, Debug, PartialEq, Eq)]
386pub enum ActivityOptionsErrorKind {
387 EmptyTaskQueue,
388 EmptyRetryPolicy,
389 InvalidMaxAttempts,
390 BackoffWithoutRetryBudget,
391 TooManyBackoffIntervals,
392 InvalidBackoffCoefficient,
393 BackoffGenerationTooLarge,
394 BackoffOverflow,
395 EmptyNonRetryableErrorType,
396 TimeoutNotPositive,
397 TimeoutOverflow,
398 TimeoutOrder,
399}
400
401#[derive(Clone, Debug, Error, PartialEq, Eq)]
403#[error("invalid activity options ({kind:?}, {field:?}): {message}")]
404pub struct ActivityOptionsError {
405 pub kind: ActivityOptionsErrorKind,
406 pub field: Option<&'static str>,
407 pub message: String,
408}
409
410impl ActivityOptionsError {
411 fn new(
412 kind: ActivityOptionsErrorKind,
413 field: Option<&'static str>,
414 message: impl Into<String>,
415 ) -> Self {
416 Self {
417 kind,
418 field,
419 message: message.into(),
420 }
421 }
422}
423
424#[derive(Clone, Copy, Debug, PartialEq, Eq)]
426pub enum ActivityFailureKind {
427 Failed,
428 Cancelled,
429 TimedOut,
430}
431
432#[derive(Clone, Debug, Error)]
437#[error("activity failed ({reason}): {message}")]
438pub struct ActivityFailure {
439 pub kind: ActivityFailureKind,
440 pub reason: String,
441 pub message: String,
442 pub activity_execution_id: Option<String>,
443 pub activity_attempt_id: Option<String>,
444 pub activity_type: Option<String>,
445 pub activity_class: Option<String>,
446 pub attempt_number: Option<u64>,
447 pub failure_id: Option<String>,
448 pub failure_category: Option<String>,
449 pub timeout_kind: Option<String>,
450 pub non_retryable: bool,
451 pub exception_type: Option<String>,
452 pub exception_class: Option<String>,
453 pub code: Option<Value>,
454 pub exception: Option<Value>,
455}
456
457#[derive(Clone, Copy, Debug, PartialEq, Eq)]
459pub enum ChildWorkflowFailureKind {
460 Failed,
461 Cancelled,
462 Terminated,
463}
464
465#[derive(Clone, Debug, Error)]
471#[error("child workflow failed ({reason}): {message}")]
472pub struct ChildWorkflowFailure {
473 pub kind: ChildWorkflowFailureKind,
474 pub reason: String,
475 pub message: String,
476 pub parent_workflow_id: Option<String>,
477 pub parent_workflow_run_id: Option<String>,
478 pub child_workflow_id: Option<String>,
479 pub child_workflow_run_id: Option<String>,
480 pub child_workflow_type: Option<String>,
481 pub failure_id: Option<String>,
482 pub failure_category: Option<String>,
483 pub exception_type: Option<String>,
484 pub exception_class: Option<String>,
485 pub non_retryable: bool,
486 pub code: Option<Value>,
487 pub exception: Option<Value>,
488}
489
490#[derive(Clone, Debug, PartialEq, Eq)]
492pub struct WorkflowIdentity {
493 pub workflow_id: Option<String>,
494 pub run_id: Option<String>,
495}
496
497#[derive(Clone, Debug, PartialEq)]
499pub struct ChildWorkflowResult {
500 pub parent: WorkflowIdentity,
501 pub child: WorkflowIdentity,
502 pub child_workflow_type: Option<String>,
503 pub result: Value,
504}
505
506#[derive(Clone, Debug, PartialEq)]
508pub struct ChildWorkflowAvroResult {
509 pub parent: WorkflowIdentity,
510 pub child: WorkflowIdentity,
511 pub child_workflow_type: Option<String>,
512 pub result: AvroValue,
513}
514
515#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
517pub enum ParentClosePolicy {
518 #[default]
519 Abandon,
520 RequestCancel,
521 Terminate,
522}
523
524impl ParentClosePolicy {
525 fn as_str(self) -> &'static str {
526 match self {
527 Self::Abandon => "abandon",
528 Self::RequestCancel => "request_cancel",
529 Self::Terminate => "terminate",
530 }
531 }
532}
533
534#[derive(Clone, Debug, Default, PartialEq, Eq)]
536pub struct ChildWorkflowRetryPolicy {
537 pub max_attempts: Option<u32>,
538 pub backoff_seconds: Vec<u64>,
539 pub non_retryable_error_types: Vec<String>,
540}
541
542#[derive(Clone, Debug, PartialEq, Eq)]
546pub struct ChildWorkflowOptions {
547 pub task_queue: String,
548 pub parent_close_policy: ParentClosePolicy,
549 pub retry_policy: Option<ChildWorkflowRetryPolicy>,
550 pub execution_timeout_seconds: Option<u64>,
551 pub run_timeout_seconds: Option<u64>,
552}
553
554impl ChildWorkflowOptions {
555 pub fn new(task_queue: impl Into<String>) -> Self {
556 Self {
557 task_queue: task_queue.into(),
558 parent_close_policy: ParentClosePolicy::Abandon,
559 retry_policy: None,
560 execution_timeout_seconds: None,
561 run_timeout_seconds: None,
562 }
563 }
564
565 pub fn parent_close_policy(mut self, policy: ParentClosePolicy) -> Self {
566 self.parent_close_policy = policy;
567 self
568 }
569
570 pub fn retry_policy(mut self, policy: ChildWorkflowRetryPolicy) -> Self {
571 self.retry_policy = Some(policy);
572 self
573 }
574
575 pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
576 self.execution_timeout_seconds = Some(seconds);
577 self
578 }
579
580 pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
581 self.run_timeout_seconds = Some(seconds);
582 self
583 }
584}
585
586#[derive(Clone, Debug, PartialEq, Eq)]
588pub enum ActivityBackoff {
589 Explicit(Vec<Duration>),
592 Exponential {
594 initial_interval: Duration,
595 coefficient: u32,
596 maximum_interval: Option<Duration>,
597 },
598}
599
600#[derive(Clone, Debug, Default, PartialEq, Eq)]
602pub struct ActivityRetryPolicy {
603 pub max_attempts: Option<u32>,
604 pub backoff: Option<ActivityBackoff>,
605 pub non_retryable_error_types: Vec<String>,
606}
607
608impl ActivityRetryPolicy {
609 pub fn new(max_attempts: u32) -> Self {
611 Self {
612 max_attempts: Some(max_attempts),
613 ..Self::default()
614 }
615 }
616
617 pub fn backoff_intervals(mut self, intervals: impl IntoIterator<Item = Duration>) -> Self {
618 self.backoff = Some(ActivityBackoff::Explicit(intervals.into_iter().collect()));
619 self
620 }
621
622 pub fn exponential_backoff(
623 mut self,
624 initial_interval: Duration,
625 coefficient: u32,
626 maximum_interval: Option<Duration>,
627 ) -> Self {
628 self.backoff = Some(ActivityBackoff::Exponential {
629 initial_interval,
630 coefficient,
631 maximum_interval,
632 });
633 self
634 }
635
636 pub fn non_retryable_error_type(mut self, error_type: impl Into<String>) -> Self {
637 self.non_retryable_error_types.push(error_type.into());
638 self
639 }
640
641 pub fn non_retryable_error_types(
642 mut self,
643 error_types: impl IntoIterator<Item = impl Into<String>>,
644 ) -> Self {
645 self.non_retryable_error_types
646 .extend(error_types.into_iter().map(Into::into));
647 self
648 }
649}
650
651#[derive(Clone, Debug, Default, PartialEq, Eq)]
656pub struct ActivityOptions {
657 pub task_queue: Option<String>,
658 pub retry_policy: Option<ActivityRetryPolicy>,
659 pub start_to_close_timeout: Option<Duration>,
660 pub schedule_to_start_timeout: Option<Duration>,
661 pub schedule_to_close_timeout: Option<Duration>,
662 pub heartbeat_timeout: Option<Duration>,
663}
664
665impl ActivityOptions {
666 pub fn new() -> Self {
667 Self::default()
668 }
669
670 pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
671 self.task_queue = Some(task_queue.into());
672 self
673 }
674
675 pub fn retry_policy(mut self, policy: ActivityRetryPolicy) -> Self {
676 self.retry_policy = Some(policy);
677 self
678 }
679
680 pub fn start_to_close_timeout(mut self, timeout: Duration) -> Self {
681 self.start_to_close_timeout = Some(timeout);
682 self
683 }
684
685 pub fn schedule_to_start_timeout(mut self, timeout: Duration) -> Self {
686 self.schedule_to_start_timeout = Some(timeout);
687 self
688 }
689
690 pub fn schedule_to_close_timeout(mut self, timeout: Duration) -> Self {
691 self.schedule_to_close_timeout = Some(timeout);
692 self
693 }
694
695 pub fn heartbeat_timeout(mut self, timeout: Duration) -> Self {
696 self.heartbeat_timeout = Some(timeout);
697 self
698 }
699
700 fn validate(&self) -> std::result::Result<ValidatedActivityOptions, ActivityOptionsError> {
701 if self
702 .task_queue
703 .as_deref()
704 .is_some_and(|queue| queue.trim().is_empty())
705 {
706 return Err(ActivityOptionsError::new(
707 ActivityOptionsErrorKind::EmptyTaskQueue,
708 Some("task_queue"),
709 "task_queue must not be empty",
710 ));
711 }
712
713 for (field, value) in [
714 ("start_to_close_timeout", self.start_to_close_timeout),
715 ("schedule_to_start_timeout", self.schedule_to_start_timeout),
716 ("schedule_to_close_timeout", self.schedule_to_close_timeout),
717 ("heartbeat_timeout", self.heartbeat_timeout),
718 ] {
719 if value.is_some_and(|value| value.is_zero()) {
720 return Err(ActivityOptionsError::new(
721 ActivityOptionsErrorKind::TimeoutNotPositive,
722 Some(field),
723 format!("{field} must be positive"),
724 ));
725 }
726 }
727
728 validate_timeout_order(
729 "heartbeat_timeout",
730 self.heartbeat_timeout,
731 "start_to_close_timeout",
732 self.start_to_close_timeout,
733 )?;
734 validate_timeout_order(
735 "start_to_close_timeout",
736 self.start_to_close_timeout,
737 "schedule_to_close_timeout",
738 self.schedule_to_close_timeout,
739 )?;
740 validate_timeout_order(
741 "schedule_to_start_timeout",
742 self.schedule_to_start_timeout,
743 "schedule_to_close_timeout",
744 self.schedule_to_close_timeout,
745 )?;
746
747 Ok(ValidatedActivityOptions {
748 task_queue: self.task_queue.clone(),
749 retry_policy: self
750 .retry_policy
751 .as_ref()
752 .map(validate_activity_retry_policy)
753 .transpose()?,
754 start_to_close_timeout: timeout_seconds(
755 "start_to_close_timeout",
756 self.start_to_close_timeout,
757 )?,
758 schedule_to_start_timeout: timeout_seconds(
759 "schedule_to_start_timeout",
760 self.schedule_to_start_timeout,
761 )?,
762 schedule_to_close_timeout: timeout_seconds(
763 "schedule_to_close_timeout",
764 self.schedule_to_close_timeout,
765 )?,
766 heartbeat_timeout: timeout_seconds("heartbeat_timeout", self.heartbeat_timeout)?,
767 })
768 }
769}
770
771#[derive(Clone, Debug)]
772struct ValidatedActivityOptions {
773 task_queue: Option<String>,
774 retry_policy: Option<Value>,
775 start_to_close_timeout: Option<u64>,
776 schedule_to_start_timeout: Option<u64>,
777 schedule_to_close_timeout: Option<u64>,
778 heartbeat_timeout: Option<u64>,
779}
780
781fn validate_timeout_order(
782 smaller_name: &'static str,
783 smaller: Option<Duration>,
784 larger_name: &'static str,
785 larger: Option<Duration>,
786) -> std::result::Result<(), ActivityOptionsError> {
787 if matches!((smaller, larger), (Some(smaller), Some(larger)) if smaller > larger) {
788 return Err(ActivityOptionsError::new(
789 ActivityOptionsErrorKind::TimeoutOrder,
790 Some(smaller_name),
791 format!("{smaller_name} must be <= {larger_name}"),
792 ));
793 }
794 Ok(())
795}
796
797fn timeout_seconds(
798 field: &'static str,
799 value: Option<Duration>,
800) -> std::result::Result<Option<u64>, ActivityOptionsError> {
801 value
802 .map(|value| {
803 activity_protocol_seconds(value).ok_or_else(|| {
804 ActivityOptionsError::new(
805 ActivityOptionsErrorKind::TimeoutOverflow,
806 Some(field),
807 format!("{field} is too large for the worker protocol"),
808 )
809 })
810 })
811 .transpose()
812}
813
814fn duration_seconds_ceil(value: Duration) -> Option<u64> {
815 value
816 .as_secs()
817 .checked_add(u64::from(value.subsec_nanos() > 0))
818}
819
820fn activity_protocol_seconds(value: Duration) -> Option<u64> {
821 duration_seconds_ceil(value).filter(|seconds| *seconds <= i64::MAX as u64)
822}
823
824fn validate_activity_retry_policy(
825 policy: &ActivityRetryPolicy,
826) -> std::result::Result<Value, ActivityOptionsError> {
827 if policy.max_attempts.is_none()
828 && policy.backoff.is_none()
829 && policy.non_retryable_error_types.is_empty()
830 {
831 return Err(ActivityOptionsError::new(
832 ActivityOptionsErrorKind::EmptyRetryPolicy,
833 Some("retry_policy"),
834 "retry_policy must configure at least one field",
835 ));
836 }
837 if policy.max_attempts == Some(0) {
838 return Err(ActivityOptionsError::new(
839 ActivityOptionsErrorKind::InvalidMaxAttempts,
840 Some("retry_policy.max_attempts"),
841 "max_attempts must be >= 1",
842 ));
843 }
844 if policy
845 .non_retryable_error_types
846 .iter()
847 .any(|error_type| error_type.trim().is_empty())
848 {
849 return Err(ActivityOptionsError::new(
850 ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
851 Some("retry_policy.non_retryable_error_types"),
852 "non_retryable_error_types must not contain empty values",
853 ));
854 }
855
856 let backoff_seconds = match &policy.backoff {
857 None => None,
858 Some(backoff) => {
859 let max_attempts = policy.max_attempts.ok_or_else(|| {
860 ActivityOptionsError::new(
861 ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
862 Some("retry_policy.backoff"),
863 "backoff requires max_attempts",
864 )
865 })?;
866 let retry_count = max_attempts.saturating_sub(1) as usize;
867 let intervals = match backoff {
868 ActivityBackoff::Explicit(intervals) => {
869 if intervals.len() > retry_count {
870 return Err(ActivityOptionsError::new(
871 ActivityOptionsErrorKind::TooManyBackoffIntervals,
872 Some("retry_policy.backoff"),
873 "backoff interval count must not exceed max_attempts - 1",
874 ));
875 }
876 intervals.clone()
877 }
878 ActivityBackoff::Exponential {
879 initial_interval,
880 coefficient,
881 maximum_interval,
882 } => {
883 if *coefficient < 1 {
884 return Err(ActivityOptionsError::new(
885 ActivityOptionsErrorKind::InvalidBackoffCoefficient,
886 Some("retry_policy.backoff.coefficient"),
887 "backoff coefficient must be >= 1",
888 ));
889 }
890 if retry_count > 10_000 {
891 return Err(ActivityOptionsError::new(
892 ActivityOptionsErrorKind::BackoffGenerationTooLarge,
893 Some("retry_policy.max_attempts"),
894 "generated backoff supports at most 10000 retry intervals",
895 ));
896 }
897 let mut current = *initial_interval;
898 let mut intervals = Vec::with_capacity(retry_count);
899 for _ in 0..retry_count {
900 let interval = maximum_interval
901 .map(|maximum| current.min(maximum))
902 .unwrap_or(current);
903 intervals.push(interval);
904 if maximum_interval.is_some_and(|maximum| interval == maximum) {
905 break;
906 }
907 current = current.checked_mul(*coefficient).ok_or_else(|| {
908 ActivityOptionsError::new(
909 ActivityOptionsErrorKind::BackoffOverflow,
910 Some("retry_policy.backoff"),
911 "generated backoff interval overflowed",
912 )
913 })?;
914 }
915 intervals
916 }
917 };
918 Some(
919 intervals
920 .into_iter()
921 .map(|interval| {
922 activity_protocol_seconds(interval).ok_or_else(|| {
923 ActivityOptionsError::new(
924 ActivityOptionsErrorKind::BackoffOverflow,
925 Some("retry_policy.backoff"),
926 "backoff interval is too large for the worker protocol",
927 )
928 })
929 })
930 .collect::<std::result::Result<Vec<_>, _>>()?,
931 )
932 }
933 };
934
935 let mut encoded = serde_json::Map::new();
936 if let Some(max_attempts) = policy.max_attempts {
937 encoded.insert("max_attempts".to_string(), json!(max_attempts));
938 }
939 if let Some(backoff_seconds) = backoff_seconds {
940 encoded.insert("backoff_seconds".to_string(), json!(backoff_seconds));
941 }
942 if !policy.non_retryable_error_types.is_empty() {
943 let mut canonical_error_types = Vec::new();
944 for error_type in policy
945 .non_retryable_error_types
946 .iter()
947 .map(|error_type| error_type.trim())
948 {
949 if !canonical_error_types.contains(&error_type) {
950 canonical_error_types.push(error_type);
951 }
952 }
953 encoded.insert(
954 "non_retryable_error_types".to_string(),
955 json!(canonical_error_types),
956 );
957 }
958 Ok(Value::Object(encoded))
959}
960
961#[derive(Clone, Debug, Error)]
964#[error("non-deterministic workflow replay ({reason}) at sequence {sequence:?}: {message}")]
965pub struct ReplayFailure {
966 pub reason: String,
967 pub sequence: Option<u64>,
968 pub expected: Option<String>,
969 pub actual: Option<String>,
970 pub message: String,
971}
972
973impl ReplayFailure {
974 fn new(
975 reason: impl Into<String>,
976 sequence: Option<u64>,
977 expected: Option<String>,
978 actual: Option<String>,
979 message: impl Into<String>,
980 ) -> Self {
981 Self {
982 reason: reason.into(),
983 sequence,
984 expected,
985 actual,
986 message: message.into(),
987 }
988 }
989}
990
991#[derive(Clone, Debug, Error)]
993#[error("query failed ({reason}, HTTP {status}): {message}")]
994pub struct QueryFailure {
995 pub status: u16,
996 pub reason: String,
997 pub message: String,
998 pub body: Value,
999}
1000
1001#[derive(Clone, Debug, Error)]
1003#[error("protocol rejected ({reason}, HTTP {status}): {message}")]
1004pub struct ProtocolFailure {
1005 pub status: u16,
1006 pub reason: String,
1007 pub message: String,
1008 pub supported_version: Option<String>,
1009 pub requested_version: Option<String>,
1010 pub body: Value,
1011}
1012
1013#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
1014pub struct PayloadEnvelope {
1015 pub codec: String,
1016 pub blob: String,
1017}
1018
1019impl PayloadEnvelope {
1020 pub fn avro<T: Serialize>(value: &T) -> Result<Self> {
1021 encode_payload(value, DEFAULT_CODEC)
1022 }
1023
1024 pub fn avro_value(value: &AvroValue) -> Result<Self> {
1027 encode_avro_value(value)
1028 }
1029}
1030
1031#[derive(Clone, Debug, PartialEq)]
1033pub enum AvroValue {
1034 Null,
1035 Boolean(bool),
1036 Long(i64),
1037 Double(f64),
1038 Bytes(Vec<u8>),
1039 String(String),
1040 Array(Vec<AvroValue>),
1041 Map(BTreeMap<String, AvroValue>),
1042}
1043
1044impl AvroValue {
1045 fn from_serialize<T: Serialize>(value: &T) -> Result<Self> {
1046 Self::from_serde_value(
1047 serde_value::to_value(value).map_err(|error| {
1048 Error::Codec(format!("could not adapt value for Avro: {error}"))
1049 })?,
1050 )
1051 }
1052
1053 fn from_serde_value(value: serde_value::Value) -> Result<Self> {
1054 use serde_value::Value as SerdeValue;
1055
1056 match value {
1057 SerdeValue::Unit => Ok(Self::Null),
1058 SerdeValue::Bool(value) => Ok(Self::Boolean(value)),
1059 SerdeValue::I8(value) => Ok(Self::Long(i64::from(value))),
1060 SerdeValue::I16(value) => Ok(Self::Long(i64::from(value))),
1061 SerdeValue::I32(value) => Ok(Self::Long(i64::from(value))),
1062 SerdeValue::I64(value) => Ok(Self::Long(value)),
1063 SerdeValue::U8(value) => Ok(Self::Long(i64::from(value))),
1064 SerdeValue::U16(value) => Ok(Self::Long(i64::from(value))),
1065 SerdeValue::U32(value) => Ok(Self::Long(i64::from(value))),
1066 SerdeValue::U64(value) => i64::try_from(value).map(Self::Long).map_err(|_| {
1067 Error::Codec(
1068 "integer_overflow: Avro Value long must be within signed 64-bit range"
1069 .to_string(),
1070 )
1071 }),
1072 SerdeValue::F32(value) => Self::finite_double(f64::from(value)),
1073 SerdeValue::F64(value) => Self::finite_double(value),
1074 SerdeValue::Char(value) => Ok(Self::String(value.to_string())),
1075 SerdeValue::String(value) => Ok(Self::String(value)),
1076 SerdeValue::Bytes(value) => Ok(Self::Bytes(value)),
1077 SerdeValue::Option(None) => Ok(Self::Null),
1078 SerdeValue::Option(Some(value)) | SerdeValue::Newtype(value) => {
1079 Self::from_serde_value(*value)
1080 }
1081 SerdeValue::Seq(values) => values
1082 .into_iter()
1083 .map(Self::from_serde_value)
1084 .collect::<Result<Vec<_>>>()
1085 .map(Self::Array),
1086 SerdeValue::Map(values) => values
1087 .into_iter()
1088 .map(|(key, value)| {
1089 let SerdeValue::String(key) = key else {
1090 return Err(Error::Codec(
1091 "invalid_map_key: Avro Value map keys must be strings".to_string(),
1092 ));
1093 };
1094
1095 Ok((key, Self::from_serde_value(value)?))
1096 })
1097 .collect::<Result<BTreeMap<_, _>>>()
1098 .map(Self::Map),
1099 }
1100 }
1101
1102 fn finite_double(value: f64) -> Result<Self> {
1103 if !value.is_finite() {
1104 return Err(Error::Codec(
1105 "non_finite_float: Avro Value doubles must be finite".to_string(),
1106 ));
1107 }
1108
1109 Ok(Self::Double(value))
1110 }
1111
1112 fn into_json(self) -> Result<Value> {
1113 match self {
1114 Self::Null => Ok(Value::Null),
1115 Self::Boolean(value) => Ok(Value::Bool(value)),
1116 Self::Long(value) => Ok(Value::Number(value.into())),
1117 Self::Double(value) => serde_json::Number::from_f64(value)
1118 .map(Value::Number)
1119 .ok_or_else(|| {
1120 Error::Codec(
1121 "non_finite_float: decoded Avro Value double is not finite".to_string(),
1122 )
1123 }),
1124 Self::Bytes(value) => Ok(json!({
1125 "$type": "bytes",
1126 "base64": BASE64.encode(value),
1127 })),
1128 Self::String(value) => Ok(Value::String(value)),
1129 Self::Array(values) => values
1130 .into_iter()
1131 .map(Self::into_json)
1132 .collect::<Result<Vec<_>>>()
1133 .map(Value::Array),
1134 Self::Map(values) => values
1135 .into_iter()
1136 .map(|(key, value)| Ok((key, value.into_json()?)))
1137 .collect::<Result<serde_json::Map<_, _>>>()
1138 .map(Value::Object),
1139 }
1140 }
1141
1142 fn into_serde_value(self) -> serde_value::Value {
1143 use serde_value::Value as SerdeValue;
1144
1145 match self {
1146 Self::Null => SerdeValue::Unit,
1147 Self::Boolean(value) => SerdeValue::Bool(value),
1148 Self::Long(value) => SerdeValue::I64(value),
1149 Self::Double(value) => SerdeValue::F64(value),
1150 Self::Bytes(value) => SerdeValue::Bytes(value),
1151 Self::String(value) => SerdeValue::String(value),
1152 Self::Array(values) => {
1153 SerdeValue::Seq(values.into_iter().map(Self::into_serde_value).collect())
1154 }
1155 Self::Map(values) => SerdeValue::Map(
1156 values
1157 .into_iter()
1158 .map(|(key, value)| (SerdeValue::String(key), value.into_serde_value()))
1159 .collect(),
1160 ),
1161 }
1162 }
1163
1164 pub fn deserialize<T: DeserializeOwned>(self) -> Result<T> {
1165 self.into_serde_value().deserialize_into().map_err(|error| {
1166 Error::Codec(format!(
1167 "avro_value_type_mismatch: could not adapt decoded value: {error}"
1168 ))
1169 })
1170 }
1171}
1172
1173impl Serialize for AvroValue {
1174 fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
1175 where
1176 S: Serializer,
1177 {
1178 match self {
1179 Self::Null => serializer.serialize_unit(),
1180 Self::Boolean(value) => serializer.serialize_bool(*value),
1181 Self::Long(value) => serializer.serialize_i64(*value),
1182 Self::Double(value) => serializer.serialize_f64(*value),
1183 Self::Bytes(value) => serializer.serialize_bytes(value),
1184 Self::String(value) => serializer.serialize_str(value),
1185 Self::Array(values) => {
1186 let mut sequence = serializer.serialize_seq(Some(values.len()))?;
1187 for value in values {
1188 sequence.serialize_element(value)?;
1189 }
1190 sequence.end()
1191 }
1192 Self::Map(values) => {
1193 let mut map = serializer.serialize_map(Some(values.len()))?;
1194 for (key, value) in values {
1195 map.serialize_entry(key, value)?;
1196 }
1197 map.end()
1198 }
1199 }
1200 }
1201}
1202
1203pub fn encode_avro_value(value: &AvroValue) -> Result<PayloadEnvelope> {
1204 let datum = avro_value_to_datum(value)?;
1205 let datum = to_avro_datum(avro_value_schema()?, datum)
1206 .map_err(|err| Error::Codec(format!("avro_value_encode_failed: {err}")))?;
1207 let mut bytes = Vec::with_capacity(datum.len() + 10);
1208 bytes.extend_from_slice(&AVRO_SINGLE_OBJECT_MAGIC);
1209 bytes.extend_from_slice(&AVRO_VALUE_SCHEMA_FINGERPRINT);
1210 bytes.extend_from_slice(&datum);
1211 Ok(PayloadEnvelope {
1212 codec: DEFAULT_CODEC.to_string(),
1213 blob: BASE64.encode(bytes),
1214 })
1215}
1216
1217pub fn decode_avro_value(envelope: &PayloadEnvelope) -> Result<AvroValue> {
1218 if envelope.codec != DEFAULT_CODEC {
1219 return Err(unsupported_payload_codec(&envelope.codec));
1220 }
1221 decode_avro_value_blob(&envelope.blob)
1222}
1223
1224pub fn encode_payload<T: Serialize>(value: &T, codec: &str) -> Result<PayloadEnvelope> {
1225 let blob = match codec {
1226 DEFAULT_CODEC => encode_avro_value(&AvroValue::from_serialize(value)?)?.blob,
1227 other => return Err(unsupported_payload_codec(other)),
1228 };
1229
1230 Ok(PayloadEnvelope {
1231 codec: codec.to_string(),
1232 blob,
1233 })
1234}
1235
1236pub fn decode_payload<T: DeserializeOwned>(envelope: &PayloadEnvelope) -> Result<T> {
1237 match envelope.codec.as_str() {
1238 DEFAULT_CODEC => decode_avro_value(envelope)?.deserialize(),
1239 other => Err(unsupported_payload_codec(other)),
1240 }
1241}
1242
1243#[cfg(test)]
1244fn encode_value_envelope(value: &Value, codec: &str) -> Result<Value> {
1245 Ok(serde_json::to_value(encode_payload(value, codec)?)?)
1246}
1247
1248fn decode_wire_value(value: &Value, fallback_codec: &str) -> Result<Value> {
1249 if value.is_null() {
1250 return Ok(Value::Null);
1251 }
1252
1253 if let Some(object) = value.as_object() {
1254 if let (Some(codec), Some(blob)) = (
1255 object.get("codec").and_then(Value::as_str),
1256 object.get("blob").and_then(Value::as_str),
1257 ) {
1258 return decode_blob(blob, codec);
1259 }
1260 }
1261
1262 if let Some(blob) = value.as_str() {
1263 return decode_blob(blob, fallback_codec);
1264 }
1265
1266 Err(untagged_payload_value())
1267}
1268
1269fn encode_typed_envelope(value: &AvroValue, codec: &str) -> Result<Value> {
1270 let envelope = match codec {
1271 DEFAULT_CODEC => encode_avro_value(value)?,
1272 other => return Err(unsupported_payload_codec(other)),
1273 };
1274 Ok(serde_json::to_value(envelope)?)
1275}
1276
1277fn decode_wire_avro_value(value: &Value, fallback_codec: &str) -> Result<AvroValue> {
1278 if value.is_null() {
1279 return Ok(AvroValue::Null);
1280 }
1281
1282 if let Some(object) = value.as_object() {
1283 if let (Some(codec), Some(blob)) = (
1284 object.get("codec").and_then(Value::as_str),
1285 object.get("blob").and_then(Value::as_str),
1286 ) {
1287 return match codec {
1288 DEFAULT_CODEC => decode_avro_value_blob(blob),
1289 other => Err(unsupported_payload_codec(other)),
1290 };
1291 }
1292 }
1293
1294 if let Some(blob) = value.as_str() {
1295 return match fallback_codec {
1296 DEFAULT_CODEC => decode_avro_value_blob(blob),
1297 other => Err(unsupported_payload_codec(other)),
1298 };
1299 }
1300
1301 Err(untagged_payload_value())
1302}
1303
1304fn normalize_avro_arguments(value: AvroValue) -> AvroValue {
1305 match value {
1306 AvroValue::Null => AvroValue::Array(Vec::new()),
1307 AvroValue::Array(_) => value,
1308 other => AvroValue::Array(vec![other]),
1309 }
1310}
1311
1312fn decode_blob(blob: &str, codec: &str) -> Result<Value> {
1313 match codec {
1314 DEFAULT_CODEC => decode_avro_value_blob(blob)?.into_json(),
1315 other => Err(unsupported_payload_codec(other)),
1316 }
1317}
1318
1319fn unsupported_payload_codec(codec: &str) -> Error {
1320 Error::Codec(format!(
1321 "unsupported_payload_codec: workflow payload codec {codec:?} is not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
1322 ))
1323}
1324
1325fn untagged_payload_value() -> Error {
1326 Error::Codec(
1327 "unsupported_payload_codec: untagged durable payload values are not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
1328 .to_string(),
1329 )
1330}
1331
1332fn decode_avro_value_blob(blob: &str) -> Result<AvroValue> {
1333 let bytes = BASE64.decode(blob).map_err(|err| {
1334 Error::Codec(format!(
1335 "invalid_payload_framing: expected strict base64 Avro single-object bytes: {err}"
1336 ))
1337 })?;
1338
1339 if serde_json::from_slice::<Value>(&bytes).is_ok() {
1340 return Err(unsupported_payload_codec("json"));
1341 }
1342
1343 if bytes.len() < 10 || bytes[..2] != AVRO_SINGLE_OBJECT_MAGIC {
1344 return Err(Error::Codec(
1345 "invalid_payload_framing: expected Avro single-object magic c301".to_string(),
1346 ));
1347 }
1348
1349 let fingerprint: [u8; 8] = bytes[2..10]
1350 .try_into()
1351 .map_err(|_| Error::Codec("invalid Avro fingerprint length".to_string()))?;
1352 if fingerprint != AVRO_VALUE_SCHEMA_FINGERPRINT {
1353 return Err(Error::Codec(format!(
1354 "unsupported_payload_schema: unknown CRC-64-AVRO fingerprint {}",
1355 fingerprint
1356 .iter()
1357 .map(|byte| format!("{byte:02x}"))
1358 .collect::<String>()
1359 )));
1360 }
1361
1362 let mut datum_reader = StrictAvroDatumReader::new(&bytes[10..]);
1363 let datum = from_avro_datum(avro_value_schema()?, &mut datum_reader, None);
1367 if datum_reader.truncated {
1368 return Err(Error::Codec(
1369 "invalid_payload_framing: truncated Avro Value datum".to_string(),
1370 ));
1371 }
1372 let datum = datum.map_err(|err| {
1373 Error::Codec(format!(
1374 "invalid_payload_framing: malformed Avro Value datum: {err}"
1375 ))
1376 })?;
1377 if datum_reader.remaining() != 0 {
1378 return Err(Error::Codec(format!(
1379 "invalid_payload_framing: {} trailing bytes after Avro Value datum",
1380 datum_reader.remaining()
1381 )));
1382 }
1383 avro_value_from_datum(datum)
1384}
1385
1386struct StrictAvroDatumReader<'a> {
1387 bytes: &'a [u8],
1388 offset: usize,
1389 truncated: bool,
1390}
1391
1392impl<'a> StrictAvroDatumReader<'a> {
1393 fn new(bytes: &'a [u8]) -> Self {
1394 Self {
1395 bytes,
1396 offset: 0,
1397 truncated: false,
1398 }
1399 }
1400
1401 fn remaining(&self) -> usize {
1402 self.bytes.len() - self.offset
1403 }
1404}
1405
1406impl Read for StrictAvroDatumReader<'_> {
1407 fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
1408 let count = buffer.len().min(self.remaining());
1409 buffer[..count].copy_from_slice(&self.bytes[self.offset..self.offset + count]);
1410 self.offset += count;
1411 if count < buffer.len() {
1412 self.truncated = true;
1413 }
1414
1415 Ok(count)
1416 }
1417}
1418
1419fn avro_value_to_datum(value: &AvroValue) -> Result<AvroDatum> {
1420 let branch = match value {
1421 AvroValue::Null => AvroDatum::Union(0, Box::new(AvroDatum::Null)),
1422 AvroValue::Boolean(value) => AvroDatum::Union(
1423 1,
1424 Box::new(AvroDatum::Record(vec![(
1425 "boolean".to_string(),
1426 AvroDatum::Boolean(*value),
1427 )])),
1428 ),
1429 AvroValue::Long(value) => AvroDatum::Union(
1430 2,
1431 Box::new(AvroDatum::Record(vec![(
1432 "long".to_string(),
1433 AvroDatum::Long(*value),
1434 )])),
1435 ),
1436 AvroValue::Double(value) => {
1437 if !value.is_finite() {
1438 return Err(Error::Codec(
1439 "non_finite_float: Avro Value doubles must be finite".to_string(),
1440 ));
1441 }
1442 AvroDatum::Union(
1443 3,
1444 Box::new(AvroDatum::Record(vec![(
1445 "double".to_string(),
1446 AvroDatum::Double(*value),
1447 )])),
1448 )
1449 }
1450 AvroValue::Bytes(value) => AvroDatum::Union(
1451 4,
1452 Box::new(AvroDatum::Record(vec![(
1453 "bytes".to_string(),
1454 AvroDatum::Bytes(value.clone()),
1455 )])),
1456 ),
1457 AvroValue::String(value) => AvroDatum::Union(
1458 5,
1459 Box::new(AvroDatum::Record(vec![(
1460 "string".to_string(),
1461 AvroDatum::String(value.clone()),
1462 )])),
1463 ),
1464 AvroValue::Array(values) => AvroDatum::Union(
1465 6,
1466 Box::new(AvroDatum::Record(vec![(
1467 "items".to_string(),
1468 AvroDatum::Array(
1469 values
1470 .iter()
1471 .map(avro_value_to_datum)
1472 .collect::<Result<Vec<_>>>()?,
1473 ),
1474 )])),
1475 ),
1476 AvroValue::Map(values) => AvroDatum::Union(
1477 7,
1478 Box::new(AvroDatum::Record(vec![(
1479 "entries".to_string(),
1480 AvroDatum::Map(
1481 values
1482 .iter()
1483 .map(|(key, value)| Ok((key.clone(), avro_value_to_datum(value)?)))
1484 .collect::<Result<HashMap<_, _>>>()?,
1485 ),
1486 )])),
1487 ),
1488 };
1489 Ok(AvroDatum::Record(vec![("value".to_string(), branch)]))
1490}
1491
1492fn avro_value_from_datum(datum: AvroDatum) -> Result<AvroValue> {
1493 let AvroDatum::Record(mut outer) = datum else {
1494 return Err(Error::Codec(
1495 "invalid_payload_framing: datum is not a Value record".to_string(),
1496 ));
1497 };
1498 let (_, branch) = outer
1499 .pop()
1500 .filter(|(name, _)| name == "value")
1501 .ok_or_else(|| Error::Codec("invalid_payload_framing: Value field missing".to_string()))?;
1502 let AvroDatum::Union(_, branch) = branch else {
1503 return Err(Error::Codec(
1504 "invalid_payload_framing: invalid Value union".to_string(),
1505 ));
1506 };
1507 match *branch {
1508 AvroDatum::Null => Ok(AvroValue::Null),
1509 AvroDatum::Record(mut fields) => {
1510 let (name, value) = fields.pop().ok_or_else(|| {
1511 Error::Codec("invalid_payload_framing: empty Value branch".to_string())
1512 })?;
1513 match (name.as_str(), value) {
1514 ("boolean", AvroDatum::Boolean(value)) => Ok(AvroValue::Boolean(value)),
1515 ("long", AvroDatum::Long(value)) => Ok(AvroValue::Long(value)),
1516 ("double", AvroDatum::Double(value)) if value.is_finite() => {
1517 Ok(AvroValue::Double(value))
1518 }
1519 ("bytes", AvroDatum::Bytes(value)) => Ok(AvroValue::Bytes(value)),
1520 ("string", AvroDatum::String(value)) => Ok(AvroValue::String(value)),
1521 ("items", AvroDatum::Array(values)) => values
1522 .into_iter()
1523 .map(avro_value_from_datum)
1524 .collect::<Result<Vec<_>>>()
1525 .map(AvroValue::Array),
1526 ("entries", AvroDatum::Map(values)) => values
1527 .into_iter()
1528 .map(|(key, value)| Ok((key, avro_value_from_datum(value)?)))
1529 .collect::<Result<BTreeMap<_, _>>>()
1530 .map(AvroValue::Map),
1531 _ => Err(Error::Codec(
1532 "invalid_payload_framing: unknown Value branch".to_string(),
1533 )),
1534 }
1535 }
1536 _ => Err(Error::Codec(
1537 "invalid_payload_framing: invalid Value branch".to_string(),
1538 )),
1539 }
1540}
1541
1542fn avro_value_schema() -> Result<&'static Schema> {
1543 match AVRO_VALUE_SCHEMA.get_or_init(|| {
1544 Schema::parse_str(AVRO_VALUE_SCHEMA_JSON)
1545 .map_err(|err| format!("could not parse Avro Value schema: {err}"))
1546 }) {
1547 Ok(schema) => Ok(schema),
1548 Err(message) => Err(Error::Codec(message.clone())),
1549 }
1550}
1551
1552#[derive(Clone, Debug)]
1553pub struct Client {
1554 http: reqwest::Client,
1555 base_url: String,
1556 token: Option<String>,
1557 control_token: Option<String>,
1558 worker_token: Option<String>,
1559 namespace: String,
1560}
1561
1562impl Client {
1563 pub fn new(base_url: impl Into<String>) -> Result<Self> {
1564 Self::builder(base_url).build()
1565 }
1566
1567 pub fn builder(base_url: impl Into<String>) -> ClientBuilder {
1568 ClientBuilder {
1569 base_url: base_url.into(),
1570 token: None,
1571 control_token: None,
1572 worker_token: None,
1573 namespace: "default".to_string(),
1574 timeout: Duration::from_secs(60),
1575 }
1576 }
1577
1578 pub async fn health(&self) -> Result<Value> {
1579 self.request_json(
1580 reqwest::Method::GET,
1581 "/health",
1582 RequestProtocol::ControlPlane,
1583 Option::<&Value>::None,
1584 )
1585 .await
1586 }
1587
1588 pub async fn cluster_info(&self) -> Result<Value> {
1589 self.request_json(
1590 reqwest::Method::GET,
1591 "/cluster/info",
1592 RequestProtocol::ControlPlane,
1593 Option::<&Value>::None,
1594 )
1595 .await
1596 }
1597
1598 pub async fn start_workflow<T: Serialize>(
1599 &self,
1600 workflow_type: &str,
1601 task_queue: &str,
1602 workflow_id: &str,
1603 input: T,
1604 ) -> Result<WorkflowHandle> {
1605 self.start_workflow_with_options(
1606 workflow_type,
1607 task_queue,
1608 workflow_id,
1609 WorkflowStartOptions::default(),
1610 input,
1611 )
1612 .await
1613 }
1614
1615 pub async fn start_workflow_with_options<T: Serialize>(
1618 &self,
1619 workflow_type: &str,
1620 task_queue: &str,
1621 workflow_id: &str,
1622 options: WorkflowStartOptions,
1623 input: T,
1624 ) -> Result<WorkflowHandle> {
1625 options.validate()?;
1626 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1627 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
1628 let body = json!({
1629 "workflow_id": workflow_id,
1630 "workflow_type": workflow_type,
1631 "task_queue": task_queue,
1632 "input": input_envelope,
1633 "execution_timeout_seconds": options.execution_timeout_seconds,
1634 "run_timeout_seconds": options.run_timeout_seconds
1635 });
1636
1637 let data: Value = self
1638 .request_json(
1639 reqwest::Method::POST,
1640 "/workflows",
1641 RequestProtocol::ControlPlane,
1642 Some(&body),
1643 )
1644 .await?;
1645
1646 Ok(WorkflowHandle {
1647 client: self.clone(),
1648 workflow_id: data
1649 .get("workflow_id")
1650 .and_then(Value::as_str)
1651 .unwrap_or(workflow_id)
1652 .to_string(),
1653 run_id: data
1654 .get("run_id")
1655 .and_then(Value::as_str)
1656 .map(str::to_string),
1657 workflow_type: data
1658 .get("workflow_type")
1659 .and_then(Value::as_str)
1660 .unwrap_or(workflow_type)
1661 .to_string(),
1662 })
1663 }
1664
1665 pub async fn signal_workflow<T: Serialize>(
1666 &self,
1667 workflow_id: &str,
1668 signal_name: &str,
1669 input: T,
1670 ) -> Result<Value> {
1671 self.signal_workflow_target(workflow_id, None, signal_name, input)
1672 .await
1673 }
1674
1675 pub async fn signal_workflow_run<T: Serialize>(
1677 &self,
1678 workflow_id: &str,
1679 run_id: &str,
1680 signal_name: &str,
1681 input: T,
1682 ) -> Result<Value> {
1683 self.signal_workflow_target(workflow_id, Some(run_id), signal_name, input)
1684 .await
1685 }
1686
1687 async fn signal_workflow_target<T: Serialize>(
1688 &self,
1689 workflow_id: &str,
1690 run_id: Option<&str>,
1691 signal_name: &str,
1692 input: T,
1693 ) -> Result<Value> {
1694 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1695 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
1696 let body = json!({
1697 "input": input_envelope
1698 });
1699 let path = match run_id {
1700 Some(run_id) => {
1701 format!("/workflows/{workflow_id}/runs/{run_id}/signal/{signal_name}")
1702 }
1703 None => format!("/workflows/{workflow_id}/signal/{signal_name}"),
1704 };
1705 self.request_json(
1706 reqwest::Method::POST,
1707 &path,
1708 RequestProtocol::ControlPlane,
1709 Some(&body),
1710 )
1711 .await
1712 }
1713
1714 pub async fn cancel_workflow(
1716 &self,
1717 workflow_id: &str,
1718 options: WorkflowCommandOptions,
1719 ) -> Result<WorkflowCommandResult> {
1720 self.workflow_command(workflow_id, None, WorkflowCommandKind::Cancel, options)
1721 .await
1722 }
1723
1724 pub async fn cancel_workflow_run(
1726 &self,
1727 workflow_id: &str,
1728 run_id: &str,
1729 options: WorkflowCommandOptions,
1730 ) -> Result<WorkflowCommandResult> {
1731 self.workflow_command(
1732 workflow_id,
1733 Some(run_id),
1734 WorkflowCommandKind::Cancel,
1735 options,
1736 )
1737 .await
1738 }
1739
1740 pub async fn terminate_workflow(
1742 &self,
1743 workflow_id: &str,
1744 options: WorkflowCommandOptions,
1745 ) -> Result<WorkflowCommandResult> {
1746 self.workflow_command(workflow_id, None, WorkflowCommandKind::Terminate, options)
1747 .await
1748 }
1749
1750 pub async fn terminate_workflow_run(
1752 &self,
1753 workflow_id: &str,
1754 run_id: &str,
1755 options: WorkflowCommandOptions,
1756 ) -> Result<WorkflowCommandResult> {
1757 self.workflow_command(
1758 workflow_id,
1759 Some(run_id),
1760 WorkflowCommandKind::Terminate,
1761 options,
1762 )
1763 .await
1764 }
1765
1766 async fn workflow_command(
1767 &self,
1768 workflow_id: &str,
1769 run_id: Option<&str>,
1770 command: WorkflowCommandKind,
1771 options: WorkflowCommandOptions,
1772 ) -> Result<WorkflowCommandResult> {
1773 let path = match run_id {
1774 Some(run_id) => format!(
1775 "/workflows/{workflow_id}/runs/{run_id}/{}",
1776 command.as_str()
1777 ),
1778 None => format!("/workflows/{workflow_id}/{}", command.as_str()),
1779 };
1780 let data = match self
1781 .request_json(
1782 reqwest::Method::POST,
1783 &path,
1784 RequestProtocol::ControlPlane,
1785 Some(&options),
1786 )
1787 .await
1788 {
1789 Ok(data) => data,
1790 Err(Error::Http { status, body }) => {
1791 return Err(Error::WorkflowCommandRejected(workflow_command_rejection(
1792 command,
1793 status,
1794 body,
1795 workflow_id,
1796 run_id,
1797 )));
1798 }
1799 Err(error) => return Err(error),
1800 };
1801
1802 Ok(workflow_command_result(command, data, workflow_id, run_id))
1803 }
1804
1805 pub async fn query_workflow<T: Serialize>(
1811 &self,
1812 workflow_id: &str,
1813 query_name: &str,
1814 input: T,
1815 ) -> Result<Value> {
1816 self.query_workflow_target(workflow_id, None, query_name, input)
1817 .await
1818 }
1819
1820 pub async fn query_workflow_run<T: Serialize>(
1822 &self,
1823 workflow_id: &str,
1824 run_id: &str,
1825 query_name: &str,
1826 input: T,
1827 ) -> Result<Value> {
1828 self.query_workflow_target(workflow_id, Some(run_id), query_name, input)
1829 .await
1830 }
1831
1832 pub async fn query_workflow_avro_value<T: Serialize>(
1834 &self,
1835 workflow_id: &str,
1836 query_name: &str,
1837 input: T,
1838 ) -> Result<AvroValue> {
1839 self.query_workflow_avro_value_target(workflow_id, None, query_name, input)
1840 .await
1841 }
1842
1843 pub async fn query_workflow_run_avro_value<T: Serialize>(
1845 &self,
1846 workflow_id: &str,
1847 run_id: &str,
1848 query_name: &str,
1849 input: T,
1850 ) -> Result<AvroValue> {
1851 self.query_workflow_avro_value_target(workflow_id, Some(run_id), query_name, input)
1852 .await
1853 }
1854
1855 async fn query_workflow_avro_value_target<T: Serialize>(
1856 &self,
1857 workflow_id: &str,
1858 run_id: Option<&str>,
1859 query_name: &str,
1860 input: T,
1861 ) -> Result<AvroValue> {
1862 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1863 let body = json!({"input": encode_typed_envelope(&input, DEFAULT_CODEC)?});
1864 let path = match run_id {
1865 Some(run_id) => {
1866 format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
1867 }
1868 None => format!("/workflows/{workflow_id}/query/{query_name}"),
1869 };
1870 let response: Value = match self
1871 .request_json(
1872 reqwest::Method::POST,
1873 &path,
1874 RequestProtocol::ControlPlane,
1875 Some(&body),
1876 )
1877 .await
1878 {
1879 Ok(response) => response,
1880 Err(Error::Http { status, body }) => {
1881 return Err(Error::QueryFailed(query_failure(status, body)));
1882 }
1883 Err(error) => return Err(error),
1884 };
1885
1886 let envelope = response
1887 .get("result_envelope")
1888 .filter(|envelope| !envelope.is_null())
1889 .ok_or_else(|| {
1890 Error::Codec(
1891 "missing_payload_envelope: typed query result requires result_envelope"
1892 .to_string(),
1893 )
1894 })?;
1895 decode_wire_avro_value(envelope, DEFAULT_CODEC)
1896 }
1897
1898 async fn query_workflow_target<T: Serialize>(
1899 &self,
1900 workflow_id: &str,
1901 run_id: Option<&str>,
1902 query_name: &str,
1903 input: T,
1904 ) -> Result<Value> {
1905 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1906 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
1907 let body = json!({
1908 "input": input_envelope
1909 });
1910 let path = match run_id {
1911 Some(run_id) => {
1912 format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
1913 }
1914 None => format!("/workflows/{workflow_id}/query/{query_name}"),
1915 };
1916 let response: Value = match self
1917 .request_json(
1918 reqwest::Method::POST,
1919 &path,
1920 RequestProtocol::ControlPlane,
1921 Some(&body),
1922 )
1923 .await
1924 {
1925 Ok(response) => response,
1926 Err(Error::Http { status, body }) => {
1927 return Err(Error::QueryFailed(query_failure(status, body)));
1928 }
1929 Err(error) => return Err(error),
1930 };
1931
1932 if let Some(envelope) = response
1933 .get("result_envelope")
1934 .filter(|envelope| !envelope.is_null())
1935 {
1936 return decode_wire_value(envelope, DEFAULT_CODEC);
1937 }
1938
1939 Ok(response.get("result").cloned().unwrap_or(Value::Null))
1940 }
1941
1942 pub async fn update_workflow<T: Serialize>(
1944 &self,
1945 workflow_id: &str,
1946 update_name: &str,
1947 input: T,
1948 request_id: Option<&str>,
1949 ) -> Result<Value> {
1950 let response = self
1951 .update_workflow_response(workflow_id, update_name, input, request_id)
1952 .await?;
1953 if let Some(envelope) = response
1954 .get("result_envelope")
1955 .filter(|envelope| !envelope.is_null())
1956 {
1957 return decode_wire_value(envelope, DEFAULT_CODEC);
1958 }
1959 Ok(response.get("result").cloned().unwrap_or(response))
1960 }
1961
1962 pub async fn update_workflow_avro_value<T: Serialize>(
1964 &self,
1965 workflow_id: &str,
1966 update_name: &str,
1967 input: T,
1968 request_id: Option<&str>,
1969 ) -> Result<AvroValue> {
1970 let response = self
1971 .update_workflow_response(workflow_id, update_name, input, request_id)
1972 .await?;
1973 let envelope = response
1974 .get("result_envelope")
1975 .filter(|envelope| !envelope.is_null())
1976 .ok_or_else(|| {
1977 Error::Codec(
1978 "missing_payload_envelope: typed update result requires result_envelope"
1979 .to_string(),
1980 )
1981 })?;
1982 decode_wire_avro_value(envelope, DEFAULT_CODEC)
1983 }
1984
1985 async fn update_workflow_response<T: Serialize>(
1986 &self,
1987 workflow_id: &str,
1988 update_name: &str,
1989 input: T,
1990 request_id: Option<&str>,
1991 ) -> Result<Value> {
1992 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1993 let mut body = json!({
1994 "input": encode_typed_envelope(&input, DEFAULT_CODEC)?,
1995 "wait_for": "completed",
1996 });
1997 if let Some(request_id) = request_id {
1998 body["request_id"] = json!(request_id);
1999 }
2000 self.request_json(
2001 reqwest::Method::POST,
2002 &format!("/workflows/{workflow_id}/update/{update_name}"),
2003 RequestProtocol::ControlPlane,
2004 Some(&body),
2005 )
2006 .await
2007 }
2008
2009 pub async fn describe_workflow(&self, workflow_id: &str) -> Result<WorkflowDescription> {
2010 let path = format!("/workflows/{workflow_id}");
2011 let mut data: WorkflowDescription = self
2012 .request_json(
2013 reqwest::Method::GET,
2014 &path,
2015 RequestProtocol::ControlPlane,
2016 Option::<&Value>::None,
2017 )
2018 .await?;
2019 data.decode_payloads()?;
2020 Ok(data)
2021 }
2022
2023 pub async fn describe_workflow_run(
2025 &self,
2026 workflow_id: &str,
2027 run_id: &str,
2028 ) -> Result<WorkflowDescription> {
2029 let path = format!("/workflows/{workflow_id}/runs/{run_id}");
2030 let mut data: WorkflowDescription = self
2031 .request_json(
2032 reqwest::Method::GET,
2033 &path,
2034 RequestProtocol::ControlPlane,
2035 Option::<&Value>::None,
2036 )
2037 .await?;
2038 data.decode_payloads()?;
2039 Ok(data)
2040 }
2041
2042 pub async fn register_worker(
2043 &self,
2044 worker_id: &str,
2045 task_queue: &str,
2046 supported_workflow_types: Vec<String>,
2047 supported_activity_types: Vec<String>,
2048 max_concurrent_workflow_tasks: usize,
2049 max_concurrent_activity_tasks: usize,
2050 ) -> Result<RegisterWorkerResponse> {
2051 self.register_worker_with_capabilities(
2052 worker_id,
2053 task_queue,
2054 supported_workflow_types,
2055 supported_activity_types,
2056 max_concurrent_workflow_tasks,
2057 max_concurrent_activity_tasks,
2058 Vec::new(),
2059 )
2060 .await
2061 }
2062
2063 pub async fn register_worker_with_capabilities(
2065 &self,
2066 worker_id: &str,
2067 task_queue: &str,
2068 supported_workflow_types: Vec<String>,
2069 supported_activity_types: Vec<String>,
2070 max_concurrent_workflow_tasks: usize,
2071 max_concurrent_activity_tasks: usize,
2072 capabilities: Vec<String>,
2073 ) -> Result<RegisterWorkerResponse> {
2074 self.register_worker_with_command_contracts(
2075 worker_id,
2076 task_queue,
2077 supported_workflow_types,
2078 supported_activity_types,
2079 max_concurrent_workflow_tasks,
2080 max_concurrent_activity_tasks,
2081 capabilities,
2082 Value::Object(serde_json::Map::new()),
2083 )
2084 .await
2085 }
2086
2087 #[allow(clippy::too_many_arguments)]
2094 pub async fn register_worker_with_command_contracts(
2095 &self,
2096 worker_id: &str,
2097 task_queue: &str,
2098 supported_workflow_types: Vec<String>,
2099 supported_activity_types: Vec<String>,
2100 max_concurrent_workflow_tasks: usize,
2101 max_concurrent_activity_tasks: usize,
2102 capabilities: Vec<String>,
2103 workflow_command_contracts: Value,
2104 ) -> Result<RegisterWorkerResponse> {
2105 if let Some(contracts) = workflow_command_contracts.as_object() {
2106 for (workflow_type, contract) in contracts {
2107 let Some(update_validators) = contract.get("update_validators") else {
2108 continue;
2109 };
2110 if !update_validators
2111 .as_array()
2112 .is_some_and(|validators| validators.is_empty())
2113 {
2114 return Err(Error::UnsupportedUpdateValidators {
2115 workflow_type: workflow_type.clone(),
2116 });
2117 }
2118 }
2119 }
2120
2121 let mut body = json!({
2122 "worker_id": worker_id,
2123 "task_queue": task_queue,
2124 "runtime": "rust",
2125 "sdk_version": SDK_VERSION,
2126 "supported_workflow_types": supported_workflow_types,
2127 "supported_activity_types": supported_activity_types,
2128 "capabilities": capabilities,
2129 "max_concurrent_workflow_tasks": max_concurrent_workflow_tasks,
2130 "max_concurrent_activity_tasks": max_concurrent_activity_tasks
2131 });
2132 if workflow_command_contracts
2133 .as_object()
2134 .is_some_and(|contracts| !contracts.is_empty())
2135 {
2136 body["workflow_command_contracts"] = workflow_command_contracts;
2137 }
2138
2139 self.request_json(
2140 reqwest::Method::POST,
2141 "/worker/register",
2142 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2143 Some(&body),
2144 )
2145 .await
2146 }
2147
2148 pub async fn deregister_worker_registration(
2154 &self,
2155 worker_id: &str,
2156 ) -> Result<WorkerDeregistrationEnvelope> {
2157 let path = format!(
2158 "/worker/registrations/{}",
2159 percent_encode_path_segment(worker_id)
2160 );
2161 self.request_json(
2162 reqwest::Method::DELETE,
2163 &path,
2164 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2165 Option::<&Value>::None,
2166 )
2167 .await
2168 }
2169
2170 pub async fn poll_query_task(
2172 &self,
2173 worker_id: &str,
2174 task_queue: &str,
2175 timeout: Duration,
2176 ) -> Result<Option<QueryTask>> {
2177 Ok(self
2178 .poll_query_task_response(worker_id, task_queue, timeout)
2179 .await?
2180 .task)
2181 }
2182
2183 pub async fn poll_query_task_response(
2185 &self,
2186 worker_id: &str,
2187 task_queue: &str,
2188 timeout: Duration,
2189 ) -> Result<PollQueryTaskResponse> {
2190 let poll_request_id = unique_request_id("rust-query-poll");
2191 self.poll_query_task_response_with_request_id(
2192 worker_id,
2193 task_queue,
2194 timeout,
2195 &poll_request_id,
2196 1,
2197 )
2198 .await
2199 }
2200
2201 async fn poll_query_task_response_with_request_id(
2202 &self,
2203 worker_id: &str,
2204 task_queue: &str,
2205 timeout: Duration,
2206 poll_request_id: &str,
2207 transport_retries: usize,
2208 ) -> Result<PollQueryTaskResponse> {
2209 let timeout_seconds = long_poll_timeout_seconds(timeout);
2210 let body = json!({
2211 "worker_id": worker_id,
2212 "task_queue": task_queue,
2213 "poll_request_id": poll_request_id,
2214 "timeout_seconds": timeout_seconds,
2215 });
2216 self.poll_request_json(
2217 "/worker/query-tasks/poll",
2218 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
2219 &body,
2220 timeout + Duration::from_secs(5),
2221 transport_retries,
2222 )
2223 .await
2224 }
2225
2226 pub async fn complete_query_task<T: Serialize>(
2228 &self,
2229 query_task_id: &str,
2230 lease_owner: &str,
2231 query_task_attempt: u64,
2232 result: T,
2233 codec: &str,
2234 ) -> Result<Value> {
2235 let typed_result = AvroValue::from_serialize(&result)?;
2236 let result_envelope = encode_typed_envelope(&typed_result, codec)?;
2237 self.complete_query_task_with_envelope(
2238 query_task_id,
2239 lease_owner,
2240 query_task_attempt,
2241 typed_result.into_json()?,
2242 result_envelope,
2243 )
2244 .await
2245 }
2246
2247 async fn complete_query_task_with_envelope(
2248 &self,
2249 query_task_id: &str,
2250 lease_owner: &str,
2251 query_task_attempt: u64,
2252 result: Value,
2253 result_envelope: Value,
2254 ) -> Result<Value> {
2255 let body = json!({
2256 "lease_owner": lease_owner,
2257 "query_task_attempt": query_task_attempt,
2258 "result": result,
2259 "result_envelope": result_envelope,
2260 });
2261 let path = format!("/worker/query-tasks/{query_task_id}/complete");
2262 let response = self
2263 .request_json(
2264 reqwest::Method::POST,
2265 &path,
2266 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
2267 Some(&body),
2268 )
2269 .await;
2270 query_task_response(response)
2271 }
2272
2273 pub async fn fail_query_task(
2275 &self,
2276 query_task_id: &str,
2277 lease_owner: &str,
2278 query_task_attempt: u64,
2279 message: impl Into<String>,
2280 reason: impl Into<String>,
2281 failure_type: impl Into<String>,
2282 ) -> Result<Value> {
2283 let body = json!({
2284 "lease_owner": lease_owner,
2285 "query_task_attempt": query_task_attempt,
2286 "failure": {
2287 "message": message.into(),
2288 "reason": reason.into(),
2289 "type": failure_type.into(),
2290 }
2291 });
2292 let path = format!("/worker/query-tasks/{query_task_id}/fail");
2293 let response = self
2294 .request_json(
2295 reqwest::Method::POST,
2296 &path,
2297 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
2298 Some(&body),
2299 )
2300 .await;
2301 query_task_response(response)
2302 }
2303
2304 pub async fn heartbeat_worker(
2305 &self,
2306 worker_id: &str,
2307 workflow_available: usize,
2308 activity_available: usize,
2309 ) -> Result<Value> {
2310 let body = json!({
2311 "worker_id": worker_id,
2312 "task_slots": {
2313 "workflow_available": workflow_available,
2314 "activity_available": activity_available
2315 },
2316 "process_metrics": {
2317 "process_id": std::process::id(),
2318 "process_uptime_seconds": 0
2319 }
2320 });
2321
2322 self.request_json(
2323 reqwest::Method::POST,
2324 "/worker/heartbeat",
2325 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2326 Some(&body),
2327 )
2328 .await
2329 }
2330
2331 pub async fn poll_workflow_task(
2332 &self,
2333 worker_id: &str,
2334 task_queue: &str,
2335 timeout: Duration,
2336 ) -> Result<Option<WorkflowTask>> {
2337 Ok(self
2338 .poll_workflow_task_response(worker_id, task_queue, timeout)
2339 .await?
2340 .task)
2341 }
2342
2343 pub async fn poll_workflow_task_response(
2344 &self,
2345 worker_id: &str,
2346 task_queue: &str,
2347 timeout: Duration,
2348 ) -> Result<PollWorkflowTaskResponse> {
2349 let poll_request_id = unique_request_id("rust-workflow-poll");
2350 self.poll_workflow_task_response_with_request_id(
2351 worker_id,
2352 task_queue,
2353 timeout,
2354 &poll_request_id,
2355 1,
2356 )
2357 .await
2358 }
2359
2360 async fn poll_workflow_task_response_with_request_id(
2361 &self,
2362 worker_id: &str,
2363 task_queue: &str,
2364 timeout: Duration,
2365 poll_request_id: &str,
2366 transport_retries: usize,
2367 ) -> Result<PollWorkflowTaskResponse> {
2368 let body = json!({
2369 "worker_id": worker_id,
2370 "task_queue": task_queue,
2371 "poll_request_id": poll_request_id,
2372 "timeout_seconds": long_poll_timeout_seconds(timeout),
2373 });
2374 let mut data: PollWorkflowTaskResponse = self
2375 .poll_request_json(
2376 "/worker/workflow-tasks/poll",
2377 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2378 &body,
2379 timeout + Duration::from_secs(5),
2380 transport_retries,
2381 )
2382 .await?;
2383
2384 if let Some(task) = data.task.as_mut() {
2385 self.fetch_remaining_workflow_history(worker_id, task)
2386 .await?;
2387 }
2388
2389 Ok(data)
2390 }
2391
2392 async fn fetch_remaining_workflow_history(
2393 &self,
2394 worker_id: &str,
2395 task: &mut WorkflowTask,
2396 ) -> Result<()> {
2397 let mut next_token = task.next_history_page_token.clone();
2398
2399 while let Some(token) = next_token.take().filter(|token| !token.is_empty()) {
2400 let lease_owner = task
2401 .lease_owner
2402 .clone()
2403 .unwrap_or_else(|| worker_id.to_string());
2404 let page = self
2405 .workflow_task_history_page(
2406 &task.task_id,
2407 &lease_owner,
2408 task.workflow_task_attempt,
2409 &token,
2410 )
2411 .await?;
2412
2413 task.append_history_page(page);
2414
2415 if task.next_history_page_token.as_deref() == Some(token.as_str()) {
2416 return Err(Error::Codec(
2417 "workflow history pagination returned the same page token".to_string(),
2418 ));
2419 }
2420
2421 next_token = task.next_history_page_token.clone();
2422 }
2423
2424 Ok(())
2425 }
2426
2427 async fn workflow_task_history_page(
2428 &self,
2429 task_id: &str,
2430 lease_owner: &str,
2431 workflow_task_attempt: u64,
2432 next_history_page_token: &str,
2433 ) -> Result<WorkflowTaskHistoryPage> {
2434 let body = json!({
2435 "lease_owner": lease_owner,
2436 "workflow_task_attempt": workflow_task_attempt,
2437 "next_history_page_token": next_history_page_token
2438 });
2439 let path = format!("/worker/workflow-tasks/{task_id}/history");
2440
2441 self.request_json(
2442 reqwest::Method::POST,
2443 &path,
2444 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2445 Some(&body),
2446 )
2447 .await
2448 }
2449
2450 pub async fn complete_workflow_task(
2451 &self,
2452 task_id: &str,
2453 lease_owner: &str,
2454 workflow_task_attempt: u64,
2455 commands: Vec<Value>,
2456 ) -> Result<Value> {
2457 let body = json!({
2458 "lease_owner": lease_owner,
2459 "workflow_task_attempt": workflow_task_attempt,
2460 "commands": commands
2461 });
2462 let path = format!("/worker/workflow-tasks/{task_id}/complete");
2463 self.request_json(
2464 reqwest::Method::POST,
2465 &path,
2466 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2467 Some(&body),
2468 )
2469 .await
2470 }
2471
2472 pub async fn fail_workflow_task(
2473 &self,
2474 task_id: &str,
2475 lease_owner: &str,
2476 workflow_task_attempt: u64,
2477 message: impl Into<String>,
2478 ) -> Result<Value> {
2479 self.fail_workflow_task_with_type(
2480 task_id,
2481 lease_owner,
2482 workflow_task_attempt,
2483 message,
2484 "RustWorkflowTaskFailure",
2485 )
2486 .await
2487 }
2488
2489 async fn fail_workflow_task_with_type(
2490 &self,
2491 task_id: &str,
2492 lease_owner: &str,
2493 workflow_task_attempt: u64,
2494 message: impl Into<String>,
2495 failure_type: &str,
2496 ) -> Result<Value> {
2497 let body = json!({
2498 "lease_owner": lease_owner,
2499 "workflow_task_attempt": workflow_task_attempt,
2500 "failure": {
2501 "message": message.into(),
2502 "type": failure_type
2503 }
2504 });
2505 let path = format!("/worker/workflow-tasks/{task_id}/fail");
2506 self.request_json(
2507 reqwest::Method::POST,
2508 &path,
2509 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2510 Some(&body),
2511 )
2512 .await
2513 }
2514
2515 pub async fn poll_activity_task(
2516 &self,
2517 worker_id: &str,
2518 task_queue: &str,
2519 timeout: Duration,
2520 ) -> Result<Option<ActivityTask>> {
2521 Ok(self
2522 .poll_activity_task_response(worker_id, task_queue, timeout)
2523 .await?
2524 .task)
2525 }
2526
2527 pub async fn poll_activity_task_response(
2529 &self,
2530 worker_id: &str,
2531 task_queue: &str,
2532 timeout: Duration,
2533 ) -> Result<PollActivityTaskResponse> {
2534 let poll_request_id = unique_request_id("rust-activity-poll");
2535 self.poll_activity_task_response_with_request_id(
2536 worker_id,
2537 task_queue,
2538 timeout,
2539 &poll_request_id,
2540 1,
2541 )
2542 .await
2543 }
2544
2545 async fn poll_activity_task_response_with_request_id(
2546 &self,
2547 worker_id: &str,
2548 task_queue: &str,
2549 timeout: Duration,
2550 poll_request_id: &str,
2551 transport_retries: usize,
2552 ) -> Result<PollActivityTaskResponse> {
2553 let body = json!({
2554 "worker_id": worker_id,
2555 "task_queue": task_queue,
2556 "poll_request_id": poll_request_id,
2557 "timeout_seconds": long_poll_timeout_seconds(timeout),
2558 });
2559 let data: PollActivityTaskResponse = self
2560 .poll_request_json(
2561 "/worker/activity-tasks/poll",
2562 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2563 &body,
2564 timeout + Duration::from_secs(5),
2565 transport_retries,
2566 )
2567 .await?;
2568 Ok(data)
2569 }
2570
2571 pub async fn complete_activity_task<T: Serialize>(
2572 &self,
2573 task_id: &str,
2574 activity_attempt_id: &str,
2575 lease_owner: &str,
2576 result: T,
2577 codec: &str,
2578 ) -> Result<Value> {
2579 let result = encode_typed_envelope(&AvroValue::from_serialize(&result)?, codec)?;
2580 let body = json!({
2581 "activity_attempt_id": activity_attempt_id,
2582 "lease_owner": lease_owner,
2583 "result": result
2584 });
2585 let path = format!("/worker/activity-tasks/{task_id}/complete");
2586 activity_task_response(
2587 self.request_json(
2588 reqwest::Method::POST,
2589 &path,
2590 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2591 Some(&body),
2592 )
2593 .await,
2594 "complete",
2595 task_id,
2596 activity_attempt_id,
2597 )
2598 }
2599
2600 pub async fn fail_activity_task(
2601 &self,
2602 task_id: &str,
2603 activity_attempt_id: &str,
2604 lease_owner: &str,
2605 message: impl Into<String>,
2606 non_retryable: bool,
2607 ) -> Result<Value> {
2608 let body = json!({
2609 "activity_attempt_id": activity_attempt_id,
2610 "lease_owner": lease_owner,
2611 "failure": {
2612 "message": message.into(),
2613 "type": "RustActivityFailure",
2614 "non_retryable": non_retryable
2615 }
2616 });
2617 let path = format!("/worker/activity-tasks/{task_id}/fail");
2618 activity_task_response(
2619 self.request_json(
2620 reqwest::Method::POST,
2621 &path,
2622 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2623 Some(&body),
2624 )
2625 .await,
2626 "fail",
2627 task_id,
2628 activity_attempt_id,
2629 )
2630 }
2631
2632 pub async fn heartbeat_activity_task<T: Serialize>(
2633 &self,
2634 task_id: &str,
2635 activity_attempt_id: &str,
2636 lease_owner: &str,
2637 details: T,
2638 ) -> Result<ActivityHeartbeatResponse> {
2639 let details = encode_typed_envelope(&AvroValue::from_serialize(&details)?, DEFAULT_CODEC)?;
2640 let body = json!({
2641 "activity_attempt_id": activity_attempt_id,
2642 "lease_owner": lease_owner,
2643 "details": details
2644 });
2645 let path = format!("/worker/activity-tasks/{task_id}/heartbeat");
2646 activity_task_response(
2647 self.request_json(
2648 reqwest::Method::POST,
2649 &path,
2650 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2651 Some(&body),
2652 )
2653 .await,
2654 "heartbeat",
2655 task_id,
2656 activity_attempt_id,
2657 )
2658 }
2659
2660 async fn request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
2661 &self,
2662 method: reqwest::Method,
2663 path: &str,
2664 protocol: RequestProtocol,
2665 body: Option<&B>,
2666 ) -> Result<T> {
2667 self.request_json_with_timeout(method, path, protocol, body, Duration::from_secs(60))
2668 .await
2669 }
2670
2671 async fn request_json_with_timeout<T: DeserializeOwned, B: Serialize + ?Sized>(
2672 &self,
2673 method: reqwest::Method,
2674 path: &str,
2675 protocol: RequestProtocol,
2676 body: Option<&B>,
2677 timeout: Duration,
2678 ) -> Result<T> {
2679 let auth_token = self.auth_token(protocol)?;
2680 let mut request = self
2681 .http
2682 .request(method, format!("{}/api{}", self.base_url, path))
2683 .timeout(timeout)
2684 .header(reqwest::header::ACCEPT, "application/json")
2685 .header(reqwest::header::CONTENT_TYPE, "application/json")
2686 .header("X-Namespace", &self.namespace);
2687
2688 match protocol {
2689 RequestProtocol::Worker(version) => {
2690 request = request.header("X-Durable-Workflow-Protocol-Version", version);
2691 }
2692 RequestProtocol::ControlPlane => {
2693 request = request.header(
2694 "X-Durable-Workflow-Control-Plane-Version",
2695 CONTROL_PLANE_VERSION,
2696 );
2697 }
2698 }
2699
2700 if let Some(token) = auth_token {
2701 request = request.bearer_auth(token);
2702 }
2703
2704 if let Some(body) = body {
2705 request = request.json(body);
2706 }
2707
2708 let response = request.send().await?;
2709 let status = response.status();
2710 let bytes = response.bytes().await?;
2711
2712 if !status.is_success() {
2713 let body = String::from_utf8_lossy(&bytes).to_string();
2714 if let Some(protocol) = protocol_failure(status, &body) {
2715 return Err(Error::Protocol(protocol));
2716 }
2717 return Err(Error::Http { status, body });
2718 }
2719
2720 if bytes.is_empty() {
2721 return Ok(serde_json::from_value(Value::Null)?);
2722 }
2723
2724 Ok(serde_json::from_slice(&bytes)?)
2725 }
2726
2727 async fn poll_request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
2728 &self,
2729 path: &str,
2730 protocol: RequestProtocol,
2731 body: &B,
2732 timeout: Duration,
2733 max_retries: usize,
2734 ) -> Result<T> {
2735 let mut retries = 0;
2736
2737 loop {
2738 let response = self
2739 .request_json_with_timeout(
2740 reqwest::Method::POST,
2741 path,
2742 protocol,
2743 Some(body),
2744 timeout,
2745 )
2746 .await;
2747
2748 match response {
2749 Err(Error::Transport(_)) if retries < max_retries => retries += 1,
2750 response => return worker_poll_response(response),
2751 }
2752 }
2753 }
2754
2755 fn auth_token(&self, protocol: RequestProtocol) -> Result<Option<&str>> {
2756 match protocol {
2757 RequestProtocol::Worker(_) => {
2758 if let Some(token) = self.worker_token.as_deref().or(self.token.as_deref()) {
2759 return Ok(Some(token));
2760 }
2761 if self.control_token.is_some() {
2762 return Err(Error::MissingRoleCredentials {
2763 role: "worker",
2764 opposite_role: "control",
2765 });
2766 }
2767 Ok(None)
2768 }
2769 RequestProtocol::ControlPlane => {
2770 if let Some(token) = self.control_token.as_deref().or(self.token.as_deref()) {
2771 return Ok(Some(token));
2772 }
2773 if self.worker_token.is_some() {
2774 return Err(Error::MissingRoleCredentials {
2775 role: "control",
2776 opposite_role: "worker",
2777 });
2778 }
2779 Ok(None)
2780 }
2781 }
2782 }
2783}
2784
2785fn query_failure(status: reqwest::StatusCode, raw_body: String) -> QueryFailure {
2786 let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
2787 let reason = body
2788 .get("reason")
2789 .and_then(Value::as_str)
2790 .unwrap_or("query_rejected")
2791 .to_string();
2792 let message = body
2793 .get("message")
2794 .or_else(|| body.get("error"))
2795 .and_then(Value::as_str)
2796 .unwrap_or("workflow query was rejected")
2797 .to_string();
2798
2799 QueryFailure {
2800 status: status.as_u16(),
2801 reason,
2802 message,
2803 body,
2804 }
2805}
2806
2807fn workflow_command_result(
2808 command: WorkflowCommandKind,
2809 data: Value,
2810 workflow_id: &str,
2811 run_id: Option<&str>,
2812) -> WorkflowCommandResult {
2813 WorkflowCommandResult {
2814 command,
2815 workflow_id: data
2816 .get("workflow_id")
2817 .and_then(Value::as_str)
2818 .unwrap_or(workflow_id)
2819 .to_string(),
2820 run_id: data
2821 .get("run_id")
2822 .and_then(Value::as_str)
2823 .or(run_id)
2824 .map(str::to_string),
2825 outcome: data
2826 .get("outcome")
2827 .and_then(Value::as_str)
2828 .map(str::to_string),
2829 reason: data
2830 .get("reason")
2831 .and_then(Value::as_str)
2832 .map(str::to_string),
2833 command_status: data
2834 .get("command_status")
2835 .and_then(Value::as_str)
2836 .map(str::to_string),
2837 raw: data,
2838 }
2839}
2840
2841fn workflow_command_rejection(
2842 command: WorkflowCommandKind,
2843 status: reqwest::StatusCode,
2844 raw_body: String,
2845 workflow_id: &str,
2846 run_id: Option<&str>,
2847) -> WorkflowCommandRejection {
2848 let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
2849 WorkflowCommandRejection {
2850 command,
2851 status: status.as_u16(),
2852 reason: body
2853 .get("reason")
2854 .and_then(Value::as_str)
2855 .unwrap_or("workflow_command_rejected")
2856 .to_string(),
2857 message: body
2858 .get("message")
2859 .or_else(|| body.get("error"))
2860 .and_then(Value::as_str)
2861 .unwrap_or("workflow lifecycle command was rejected")
2862 .to_string(),
2863 workflow_id: body
2864 .get("workflow_id")
2865 .and_then(Value::as_str)
2866 .unwrap_or(workflow_id)
2867 .to_string(),
2868 run_id: body
2869 .get("run_id")
2870 .and_then(Value::as_str)
2871 .or(run_id)
2872 .map(str::to_string),
2873 target_scope: body
2874 .get("target_scope")
2875 .and_then(Value::as_str)
2876 .map(str::to_string),
2877 body,
2878 }
2879}
2880
2881fn query_task_response(response: Result<Value>) -> Result<Value> {
2882 match response {
2883 Err(Error::Http { status, body }) => Err(Error::QueryFailed(query_failure(status, body))),
2884 response => response,
2885 }
2886}
2887
2888fn worker_poll_response<T: DeserializeOwned>(response: Result<T>) -> Result<T> {
2889 match response {
2890 Err(Error::Http { status, body })
2891 if status == reqwest::StatusCode::CONFLICT && worker_poll_body_is_stop(&body) =>
2892 {
2893 Ok(serde_json::from_str(&body)?)
2894 }
2895 response => response,
2896 }
2897}
2898
2899fn worker_poll_body_is_stop(body: &str) -> bool {
2900 serde_json::from_str::<Value>(body)
2901 .ok()
2902 .is_some_and(|body| {
2903 worker_poll_is_stop(
2904 body.get("poll_status").and_then(Value::as_str),
2905 body.get("reason").and_then(Value::as_str),
2906 )
2907 })
2908}
2909
2910fn worker_poll_is_stop(poll_status: Option<&str>, reason: Option<&str>) -> bool {
2911 matches!(poll_status, Some("draining" | "stopped"))
2912 || matches!(reason, Some("worker_draining" | "worker_stopped"))
2913}
2914
2915fn query_task_rejection_is_final(error: &Error) -> bool {
2916 matches!(
2917 error,
2918 Error::QueryFailed(failure)
2919 if QUERY_TASK_FINAL_REJECTION_REASONS.contains(&failure.reason.as_str())
2920 )
2921}
2922
2923fn activity_task_response<T>(
2924 response: Result<T>,
2925 operation: &str,
2926 task_id: &str,
2927 activity_attempt_id: &str,
2928) -> Result<T> {
2929 match response {
2930 Err(Error::Http { status, body }) => {
2931 let body = serde_json::from_str(&body).unwrap_or_else(|_| json!({"message": body}));
2932 Err(Error::ActivityTaskRejected(ActivityTaskRejection {
2933 operation: operation.to_string(),
2934 status: status.as_u16(),
2935 reason: body
2936 .get("reason")
2937 .and_then(Value::as_str)
2938 .unwrap_or("activity_task_rejected")
2939 .to_string(),
2940 task_id: body
2941 .get("task_id")
2942 .and_then(Value::as_str)
2943 .unwrap_or(task_id)
2944 .to_string(),
2945 activity_attempt_id: body
2946 .get("activity_attempt_id")
2947 .and_then(Value::as_str)
2948 .unwrap_or(activity_attempt_id)
2949 .to_string(),
2950 cancel_requested: body
2951 .get("cancel_requested")
2952 .and_then(Value::as_bool)
2953 .unwrap_or(false),
2954 can_continue: body.get("can_continue").and_then(Value::as_bool),
2955 run_closed_reason: body
2956 .get("run_closed_reason")
2957 .and_then(Value::as_str)
2958 .map(str::to_string),
2959 body,
2960 }))
2961 }
2962 response => response,
2963 }
2964}
2965
2966fn activity_task_rejection_is_final(error: &Error) -> bool {
2967 matches!(
2968 error,
2969 Error::ActivityTaskRejected(rejection)
2970 if matches!(
2971 rejection.reason.as_str(),
2972 "run_cancelled"
2973 | "run_terminated"
2974 | "attempt_closed"
2975 | "stale_attempt"
2976 | "activity_cancelled"
2977 | "task_cancelled"
2978 | "run_closed"
2979 | "activity_not_running"
2980 | "attempt_not_found"
2981 )
2982 )
2983}
2984
2985fn workflow_task_completion_is_terminal_timeout(
2986 error: &Error,
2987 task_id: &str,
2988 workflow_task_attempt: u64,
2989 run_id: Option<&str>,
2990) -> bool {
2991 let Error::Http { status, body } = error else {
2992 return false;
2993 };
2994 if *status != reqwest::StatusCode::CONFLICT {
2995 return false;
2996 }
2997
2998 let Some(run_id) = run_id else {
2999 return false;
3000 };
3001 let Ok(body) = serde_json::from_str::<Value>(body) else {
3002 return false;
3003 };
3004
3005 body.get("recorded").and_then(Value::as_bool) == Some(false)
3006 && body.get("reason").and_then(Value::as_str) == Some("run_timed_out")
3007 && body.get("run_status").and_then(Value::as_str) == Some("failed")
3008 && body.get("run_id").and_then(Value::as_str) == Some(run_id)
3009 && body.get("task_id").and_then(Value::as_str) == Some(task_id)
3010 && body.get("workflow_task_attempt").and_then(Value::as_u64) == Some(workflow_task_attempt)
3011}
3012
3013fn protocol_failure(status: reqwest::StatusCode, raw_body: &str) -> Option<ProtocolFailure> {
3014 let body: Value = serde_json::from_str(raw_body).ok()?;
3015 let reason = body.get("reason")?.as_str()?;
3016 if !matches!(
3017 reason,
3018 "missing_protocol_version"
3019 | "unsupported_protocol_version"
3020 | "missing_control_plane_version"
3021 | "unsupported_control_plane_version"
3022 ) {
3023 return None;
3024 }
3025
3026 Some(ProtocolFailure {
3027 status: status.as_u16(),
3028 reason: reason.to_string(),
3029 message: body
3030 .get("message")
3031 .or_else(|| body.get("error"))
3032 .and_then(Value::as_str)
3033 .unwrap_or("protocol version rejected")
3034 .to_string(),
3035 supported_version: body
3036 .get("supported_version")
3037 .and_then(Value::as_str)
3038 .map(str::to_string),
3039 requested_version: body
3040 .get("requested_version")
3041 .and_then(Value::as_str)
3042 .map(str::to_string),
3043 body,
3044 })
3045}
3046
3047fn long_poll_timeout_seconds(timeout: Duration) -> u64 {
3048 timeout
3049 .as_secs()
3050 .saturating_add(u64::from(timeout.subsec_nanos() > 0))
3051 .min(MAX_LONG_POLL_TIMEOUT_SECONDS)
3052}
3053
3054fn worker_operation_is_retryable(error: &Error) -> bool {
3055 match error {
3056 Error::Transport(error) => {
3057 error.is_timeout() || error.is_connect() || error.is_request() || error.is_body()
3058 }
3059 Error::Http { status, .. } => {
3060 matches!(
3061 *status,
3062 reqwest::StatusCode::REQUEST_TIMEOUT | reqwest::StatusCode::TOO_MANY_REQUESTS
3063 ) || status.is_server_error()
3064 }
3065 _ => false,
3066 }
3067}
3068
3069fn worker_retry_delay(policy: WorkerRetryPolicy, retry: usize) -> Duration {
3070 let exponent = retry.saturating_sub(1).min(31) as u32;
3071 policy
3072 .initial_backoff
3073 .saturating_mul(1_u32 << exponent)
3074 .min(policy.max_backoff)
3075}
3076
3077#[derive(Debug)]
3078pub struct ClientBuilder {
3079 base_url: String,
3080 token: Option<String>,
3081 control_token: Option<String>,
3082 worker_token: Option<String>,
3083 namespace: String,
3084 timeout: Duration,
3085}
3086
3087impl ClientBuilder {
3088 pub fn token(mut self, token: Option<String>) -> Self {
3089 self.token = token;
3090 self
3091 }
3092
3093 pub fn control_token(mut self, token: Option<String>) -> Self {
3094 self.control_token = token;
3095 self
3096 }
3097
3098 pub fn worker_token(mut self, token: Option<String>) -> Self {
3099 self.worker_token = token;
3100 self
3101 }
3102
3103 pub fn namespace(mut self, namespace: impl Into<String>) -> Self {
3104 self.namespace = namespace.into();
3105 self
3106 }
3107
3108 pub fn timeout(mut self, timeout: Duration) -> Self {
3109 self.timeout = timeout;
3110 self
3111 }
3112
3113 pub fn build(self) -> Result<Client> {
3114 let base_url = self.base_url.trim_end_matches('/').to_string();
3115 let has_sdk_api_suffix = reqwest::Url::parse(&base_url)
3116 .map(|url| url.path().trim_end_matches('/').ends_with("/api"))
3117 .unwrap_or_else(|_| base_url.ends_with("/api"));
3118
3119 if has_sdk_api_suffix {
3120 return Err(Error::InvalidBaseUrl);
3121 }
3122
3123 Ok(Client {
3124 http: reqwest::Client::builder().timeout(self.timeout).build()?,
3125 base_url,
3126 token: self.token,
3127 control_token: self.control_token,
3128 worker_token: self.worker_token,
3129 namespace: self.namespace,
3130 })
3131 }
3132}
3133
3134#[derive(Clone, Debug)]
3135pub struct WorkflowHandle {
3136 client: Client,
3137 pub workflow_id: String,
3138 pub run_id: Option<String>,
3139 pub workflow_type: String,
3140}
3141
3142impl WorkflowHandle {
3143 pub async fn describe(&self) -> Result<WorkflowDescription> {
3145 self.client.describe_workflow(&self.workflow_id).await
3146 }
3147
3148 pub async fn describe_selected_run(&self) -> Result<WorkflowDescription> {
3150 let run_id = self.run_id.as_deref().ok_or_else(|| {
3151 Error::Codec("run_id is required for selected-run description".to_string())
3152 })?;
3153 self.client
3154 .describe_workflow_run(&self.workflow_id, run_id)
3155 .await
3156 }
3157
3158 pub async fn signal<T: Serialize>(&self, signal_name: &str, input: T) -> Result<Value> {
3159 self.client
3160 .signal_workflow(&self.workflow_id, signal_name, input)
3161 .await
3162 }
3163
3164 pub async fn signal_selected_run<T: Serialize>(
3166 &self,
3167 signal_name: &str,
3168 input: T,
3169 ) -> Result<Value> {
3170 let run_id = self.run_id.as_deref().ok_or_else(|| {
3171 Error::Codec("run_id is required for selected-run signaling".to_string())
3172 })?;
3173 self.client
3174 .signal_workflow_run(&self.workflow_id, run_id, signal_name, input)
3175 .await
3176 }
3177
3178 pub async fn cancel(&self, options: WorkflowCommandOptions) -> Result<WorkflowCommandResult> {
3180 self.client
3181 .cancel_workflow(&self.workflow_id, options)
3182 .await
3183 }
3184
3185 pub async fn cancel_selected_run(
3187 &self,
3188 options: WorkflowCommandOptions,
3189 ) -> Result<WorkflowCommandResult> {
3190 let run_id = self.run_id.as_deref().ok_or_else(|| {
3191 Error::Codec("run_id is required for selected-run cancellation".to_string())
3192 })?;
3193 self.client
3194 .cancel_workflow_run(&self.workflow_id, run_id, options)
3195 .await
3196 }
3197
3198 pub async fn terminate(
3200 &self,
3201 options: WorkflowCommandOptions,
3202 ) -> Result<WorkflowCommandResult> {
3203 self.client
3204 .terminate_workflow(&self.workflow_id, options)
3205 .await
3206 }
3207
3208 pub async fn terminate_selected_run(
3210 &self,
3211 options: WorkflowCommandOptions,
3212 ) -> Result<WorkflowCommandResult> {
3213 let run_id = self.run_id.as_deref().ok_or_else(|| {
3214 Error::Codec("run_id is required for selected-run termination".to_string())
3215 })?;
3216 self.client
3217 .terminate_workflow_run(&self.workflow_id, run_id, options)
3218 .await
3219 }
3220
3221 pub async fn query<T: Serialize>(&self, query_name: &str, input: T) -> Result<Value> {
3223 self.client
3224 .query_workflow(&self.workflow_id, query_name, input)
3225 .await
3226 }
3227
3228 pub async fn query_avro_value<T: Serialize>(
3229 &self,
3230 query_name: &str,
3231 input: T,
3232 ) -> Result<AvroValue> {
3233 self.client
3234 .query_workflow_avro_value(&self.workflow_id, query_name, input)
3235 .await
3236 }
3237
3238 pub async fn update<T: Serialize>(
3239 &self,
3240 update_name: &str,
3241 input: T,
3242 request_id: Option<&str>,
3243 ) -> Result<Value> {
3244 self.client
3245 .update_workflow(&self.workflow_id, update_name, input, request_id)
3246 .await
3247 }
3248
3249 pub async fn update_avro_value<T: Serialize>(
3250 &self,
3251 update_name: &str,
3252 input: T,
3253 request_id: Option<&str>,
3254 ) -> Result<AvroValue> {
3255 self.client
3256 .update_workflow_avro_value(&self.workflow_id, update_name, input, request_id)
3257 .await
3258 }
3259
3260 pub async fn query_selected_run<T: Serialize>(
3262 &self,
3263 query_name: &str,
3264 input: T,
3265 ) -> Result<Value> {
3266 let run_id = self
3267 .run_id
3268 .as_deref()
3269 .ok_or_else(|| Error::Codec("run_id is required for selected-run query".to_string()))?;
3270 self.client
3271 .query_workflow_run(&self.workflow_id, run_id, query_name, input)
3272 .await
3273 }
3274
3275 pub async fn result(&self, options: WorkflowResultOptions) -> Result<Value> {
3277 self.result_target(options, None).await
3278 }
3279
3280 pub async fn result_avro_value(&self, options: WorkflowResultOptions) -> Result<AvroValue> {
3282 self.result_avro_value_target(options, None).await
3283 }
3284
3285 pub async fn result_selected_run(&self, options: WorkflowResultOptions) -> Result<Value> {
3287 let run_id = self.run_id.as_deref().ok_or_else(|| {
3288 Error::Codec("run_id is required for selected-run result".to_string())
3289 })?;
3290 self.result_target(options, Some(run_id)).await
3291 }
3292
3293 pub async fn result_selected_run_avro_value(
3295 &self,
3296 options: WorkflowResultOptions,
3297 ) -> Result<AvroValue> {
3298 let run_id = self.run_id.as_deref().ok_or_else(|| {
3299 Error::Codec("run_id is required for selected-run result".to_string())
3300 })?;
3301 self.result_avro_value_target(options, Some(run_id)).await
3302 }
3303
3304 async fn result_avro_value_target(
3305 &self,
3306 options: WorkflowResultOptions,
3307 selected_run_id: Option<&str>,
3308 ) -> Result<AvroValue> {
3309 let started = Instant::now();
3310
3311 loop {
3312 let description = match selected_run_id {
3313 Some(run_id) => {
3314 self.client
3315 .describe_workflow_run(&self.workflow_id, run_id)
3316 .await?
3317 }
3318 None => self.describe().await?,
3319 };
3320 if description.is_completed() {
3321 return description.output_avro_value.ok_or_else(|| {
3322 Error::Codec(
3323 "missing_payload_envelope: typed workflow result requires output_envelope"
3324 .to_string(),
3325 )
3326 });
3327 }
3328 if description.is_terminal() {
3329 let outcome =
3330 workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
3331 return Err(match outcome.kind {
3332 WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
3333 WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
3334 WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
3335 WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
3336 });
3337 }
3338 if started.elapsed() >= options.timeout {
3339 return Err(Error::Timeout);
3340 }
3341 tokio::time::sleep(options.poll_interval).await;
3342 }
3343 }
3344
3345 async fn result_target(
3346 &self,
3347 options: WorkflowResultOptions,
3348 selected_run_id: Option<&str>,
3349 ) -> Result<Value> {
3350 let started = Instant::now();
3351
3352 loop {
3353 let description = match selected_run_id {
3354 Some(run_id) => {
3355 self.client
3356 .describe_workflow_run(&self.workflow_id, run_id)
3357 .await?
3358 }
3359 None => self.describe().await?,
3360 };
3361 if description.is_completed() {
3362 return Ok(description.output.unwrap_or(Value::Null));
3363 }
3364
3365 if description.is_terminal() {
3366 let outcome =
3367 workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
3368 return Err(match outcome.kind {
3369 WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
3370 WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
3371 WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
3372 WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
3373 });
3374 }
3375
3376 if started.elapsed() >= options.timeout {
3377 return Err(Error::WorkflowTimedOut(WorkflowTerminalOutcome {
3378 kind: WorkflowTerminalKind::TimedOut,
3379 workflow_id: description
3380 .workflow_id
3381 .clone()
3382 .unwrap_or_else(|| self.workflow_id.clone()),
3383 run_id: description
3384 .run_id
3385 .clone()
3386 .or_else(|| selected_run_id.map(str::to_string)),
3387 reason: "result_wait_timeout".to_string(),
3388 failure_category: Some("client_timeout".to_string()),
3389 failure_id: None,
3390 exception_type: None,
3391 exception_class: None,
3392 non_retryable: None,
3393 message: Some(format!(
3394 "workflow result was not terminal within {:?}",
3395 options.timeout
3396 )),
3397 exception: None,
3398 raw: description.raw_value(),
3399 }));
3400 }
3401
3402 tokio::time::sleep(options.poll_interval).await;
3403 }
3404 }
3405}
3406
3407#[derive(Clone, Copy, Debug)]
3408pub struct WorkflowResultOptions {
3409 pub poll_interval: Duration,
3410 pub timeout: Duration,
3411}
3412
3413impl Default for WorkflowResultOptions {
3414 fn default() -> Self {
3415 Self {
3416 poll_interval: Duration::from_millis(500),
3417 timeout: Duration::from_secs(30),
3418 }
3419 }
3420}
3421
3422#[derive(Clone, Debug, Deserialize)]
3423pub struct WorkflowDescription {
3424 pub workflow_id: Option<String>,
3425 pub run_id: Option<String>,
3426 pub workflow_type: Option<String>,
3427 pub status: Option<String>,
3428 #[serde(default)]
3429 pub closed_reason: Option<String>,
3430 #[serde(default)]
3431 pub error: Option<String>,
3432 #[serde(default)]
3433 pub failure: Option<Value>,
3434 #[serde(default)]
3435 pub exception: Option<Value>,
3436 #[serde(default)]
3437 pub failures: Vec<Value>,
3438 #[serde(default)]
3439 pub output: Option<Value>,
3440 #[serde(default)]
3441 pub output_envelope: Option<Value>,
3442 #[serde(skip)]
3443 pub output_avro_value: Option<AvroValue>,
3444 #[serde(flatten)]
3445 pub raw: HashMap<String, Value>,
3446}
3447
3448impl WorkflowDescription {
3449 pub fn is_completed(&self) -> bool {
3450 matches!(self.status.as_deref(), Some("completed" | "Completed"))
3451 }
3452
3453 pub fn is_terminal(&self) -> bool {
3454 matches!(
3455 self.status.as_deref(),
3456 Some(
3457 "completed"
3458 | "Completed"
3459 | "failed"
3460 | "Failed"
3461 | "cancelled"
3462 | "Cancelled"
3463 | "terminated"
3464 | "Terminated"
3465 | "timed_out"
3466 | "TimedOut",
3467 )
3468 )
3469 }
3470
3471 fn decode_payloads(&mut self) -> Result<()> {
3472 if let Some(envelope) = &self.output_envelope {
3473 let value = decode_wire_avro_value(envelope, DEFAULT_CODEC)?;
3474 self.output = Some(value.clone().into_json()?);
3475 self.output_avro_value = Some(value);
3476 }
3477
3478 Ok(())
3479 }
3480
3481 fn raw_value(&self) -> Value {
3482 let mut data = self.raw.clone();
3483 data.insert(
3484 "workflow_id".to_string(),
3485 self.workflow_id
3486 .clone()
3487 .map(Value::String)
3488 .unwrap_or(Value::Null),
3489 );
3490 data.insert(
3491 "run_id".to_string(),
3492 self.run_id
3493 .clone()
3494 .map(Value::String)
3495 .unwrap_or(Value::Null),
3496 );
3497 data.insert(
3498 "workflow_type".to_string(),
3499 self.workflow_type
3500 .clone()
3501 .map(Value::String)
3502 .unwrap_or(Value::Null),
3503 );
3504 data.insert(
3505 "status".to_string(),
3506 self.status
3507 .clone()
3508 .map(Value::String)
3509 .unwrap_or(Value::Null),
3510 );
3511 data.insert(
3512 "closed_reason".to_string(),
3513 self.closed_reason
3514 .clone()
3515 .map(Value::String)
3516 .unwrap_or(Value::Null),
3517 );
3518 if let Some(failure) = &self.failure {
3519 data.insert("failure".to_string(), failure.clone());
3520 }
3521 if let Some(exception) = &self.exception {
3522 data.insert("exception".to_string(), exception.clone());
3523 }
3524 Value::Object(data.into_iter().collect())
3525 }
3526}
3527
3528fn workflow_terminal_outcome(
3529 description: &WorkflowDescription,
3530 workflow_id: &str,
3531 run_id: Option<&str>,
3532) -> WorkflowTerminalOutcome {
3533 let terminal_kind = description
3534 .closed_reason
3535 .as_deref()
3536 .or(description.status.as_deref())
3537 .unwrap_or("failed")
3538 .to_ascii_lowercase();
3539 let kind = match terminal_kind.as_str() {
3540 "cancelled" | "canceled" => WorkflowTerminalKind::Cancelled,
3541 "terminated" => WorkflowTerminalKind::Terminated,
3542 "timed_out" | "timedout" => WorkflowTerminalKind::TimedOut,
3543 _ => WorkflowTerminalKind::Failed,
3544 };
3545 let default_reason = match kind {
3546 WorkflowTerminalKind::Failed => "workflow_failed",
3547 WorkflowTerminalKind::Cancelled => "cancelled",
3548 WorkflowTerminalKind::Terminated => "terminated",
3549 WorkflowTerminalKind::TimedOut => "timed_out",
3550 };
3551 let failure = description
3552 .failure
3553 .as_ref()
3554 .filter(|value| value.is_object());
3555 let nested_failure = failure
3556 .and_then(|value| value.get("failures"))
3557 .and_then(Value::as_array)
3558 .and_then(|failures| failures.last())
3559 .or_else(|| description.failures.last());
3560 let exception = description
3561 .exception
3562 .clone()
3563 .or_else(|| failure.and_then(|value| value.get("exception")).cloned())
3564 .or_else(|| {
3565 nested_failure
3566 .and_then(|value| value.get("exception_payload"))
3567 .cloned()
3568 });
3569 let string_field = |name: &str| {
3570 failure
3571 .and_then(|value| value.get(name))
3572 .and_then(Value::as_str)
3573 .or_else(|| {
3574 nested_failure
3575 .and_then(|value| value.get(name))
3576 .and_then(Value::as_str)
3577 })
3578 .map(str::to_string)
3579 };
3580 let exception_field = |name: &str| {
3581 exception
3582 .as_ref()
3583 .and_then(|value| value.get(name))
3584 .and_then(Value::as_str)
3585 .map(str::to_string)
3586 };
3587 let message = description
3588 .error
3589 .clone()
3590 .or_else(|| string_field("message"))
3591 .or_else(|| exception_field("message"));
3592 let reason = description
3593 .raw
3594 .get("reason")
3595 .and_then(Value::as_str)
3596 .map(str::to_string)
3597 .or_else(|| {
3598 failure
3599 .and_then(|value| value.get("reason"))
3600 .and_then(Value::as_str)
3601 .map(str::to_string)
3602 })
3603 .or_else(|| description.closed_reason.clone())
3604 .unwrap_or_else(|| default_reason.to_string());
3605 let failure_id = string_field("failure_id").or_else(|| {
3606 nested_failure
3607 .and_then(|value| value.get("id"))
3608 .and_then(Value::as_str)
3609 .map(str::to_string)
3610 });
3611
3612 WorkflowTerminalOutcome {
3613 kind,
3614 workflow_id: description
3615 .workflow_id
3616 .clone()
3617 .unwrap_or_else(|| workflow_id.to_string()),
3618 run_id: description
3619 .run_id
3620 .clone()
3621 .or_else(|| run_id.map(str::to_string)),
3622 reason,
3623 failure_category: string_field("failure_category")
3624 .or_else(|| Some(default_reason.to_string())),
3625 failure_id,
3626 exception_type: string_field("exception_type").or_else(|| exception_field("type")),
3627 exception_class: string_field("exception_class").or_else(|| exception_field("class")),
3628 non_retryable: failure
3629 .and_then(|value| value.get("non_retryable"))
3630 .and_then(Value::as_bool)
3631 .or_else(|| {
3632 nested_failure
3633 .and_then(|value| value.get("non_retryable"))
3634 .and_then(Value::as_bool)
3635 }),
3636 message,
3637 exception,
3638 raw: description.raw_value(),
3639 }
3640}
3641
3642#[derive(Clone, Debug, Deserialize)]
3643pub struct RegisterWorkerResponse {
3644 pub worker_id: String,
3645 pub registered: bool,
3646 #[serde(default)]
3647 pub heartbeat_interval_seconds: Option<u64>,
3648 #[serde(default)]
3649 pub protocol_version: Option<String>,
3650 #[serde(default)]
3651 pub server_capabilities: Option<Value>,
3652}
3653
3654#[derive(Clone, Debug, Deserialize, PartialEq, Eq)]
3656pub struct WorkerDeregistrationEnvelope {
3657 pub worker_id: String,
3658 pub outcome: String,
3659 pub recovered_workflow_task_count: u64,
3660}
3661
3662#[derive(Clone, Debug, Deserialize)]
3663pub struct PollWorkflowTaskResponse {
3664 #[serde(default)]
3665 pub task: Option<WorkflowTask>,
3666 #[serde(default)]
3667 pub poll_status: Option<String>,
3668 #[serde(default)]
3669 pub reason: Option<String>,
3670 #[serde(default)]
3671 pub protocol_version: Option<String>,
3672 #[serde(default)]
3673 pub server_capabilities: Option<Value>,
3674}
3675
3676impl PollWorkflowTaskResponse {
3677 pub fn outcome(&self) -> WorkerPollOutcome {
3679 worker_poll_outcome(
3680 self.task.is_some(),
3681 self.poll_status.as_deref(),
3682 self.reason.as_deref(),
3683 )
3684 }
3685}
3686
3687#[derive(Clone, Debug, Deserialize)]
3688pub struct PollActivityTaskResponse {
3689 #[serde(default)]
3690 pub task: Option<ActivityTask>,
3691 #[serde(default)]
3692 pub poll_status: Option<String>,
3693 #[serde(default)]
3694 pub reason: Option<String>,
3695}
3696
3697impl PollActivityTaskResponse {
3698 pub fn outcome(&self) -> WorkerPollOutcome {
3700 worker_poll_outcome(
3701 self.task.is_some(),
3702 self.poll_status.as_deref(),
3703 self.reason.as_deref(),
3704 )
3705 }
3706}
3707
3708#[derive(Clone, Debug, Deserialize)]
3709pub struct PollQueryTaskResponse {
3710 #[serde(default)]
3711 pub task: Option<QueryTask>,
3712 #[serde(default)]
3713 pub poll_status: Option<String>,
3714 #[serde(default)]
3715 pub reason: Option<String>,
3716}
3717
3718impl PollQueryTaskResponse {
3719 pub fn outcome(&self) -> WorkerPollOutcome {
3721 worker_poll_outcome(
3722 self.task.is_some(),
3723 self.poll_status.as_deref(),
3724 self.reason.as_deref(),
3725 )
3726 }
3727}
3728
3729#[derive(Clone, Debug, PartialEq, Eq)]
3731pub enum WorkerPollOutcome {
3732 Task,
3734 Idle {
3736 poll_status: Option<String>,
3737 reason: Option<String>,
3738 },
3739 Stop {
3741 poll_status: Option<String>,
3742 reason: Option<String>,
3743 },
3744}
3745
3746impl WorkerPollOutcome {
3747 pub fn should_stop(&self) -> bool {
3748 matches!(self, Self::Stop { .. })
3749 }
3750}
3751
3752fn worker_poll_outcome(
3753 has_task: bool,
3754 poll_status: Option<&str>,
3755 reason: Option<&str>,
3756) -> WorkerPollOutcome {
3757 if worker_poll_is_stop(poll_status, reason) {
3758 return WorkerPollOutcome::Stop {
3759 poll_status: poll_status.map(str::to_string),
3760 reason: reason.map(str::to_string),
3761 };
3762 }
3763
3764 if has_task {
3765 WorkerPollOutcome::Task
3766 } else {
3767 WorkerPollOutcome::Idle {
3768 poll_status: poll_status.map(str::to_string),
3769 reason: reason.map(str::to_string),
3770 }
3771 }
3772}
3773
3774#[derive(Clone, Debug, Deserialize)]
3776pub struct QueryTask {
3777 pub query_task_id: String,
3778 #[serde(default = "default_workflow_task_attempt")]
3779 pub query_task_attempt: u64,
3780 #[serde(default)]
3781 pub lease_owner: Option<String>,
3782 #[serde(default)]
3783 pub workflow_id: Option<String>,
3784 #[serde(default)]
3785 pub run_id: Option<String>,
3786 pub workflow_type: String,
3787 pub query_name: String,
3788 #[serde(default = "default_payload_codec")]
3789 pub payload_codec: String,
3790 #[serde(default)]
3791 pub workflow_arguments: Option<Value>,
3792 #[serde(default)]
3793 pub query_arguments: Option<Value>,
3794 #[serde(default)]
3795 pub history_events: Vec<HistoryEvent>,
3796 #[serde(default)]
3797 pub history_export: Option<Value>,
3798 #[serde(default)]
3799 pub run_status: Option<String>,
3800}
3801
3802#[derive(Clone, Debug, Deserialize)]
3803pub struct WorkflowTask {
3804 pub task_id: String,
3805 #[serde(default)]
3806 pub workflow_id: Option<String>,
3807 #[serde(default)]
3808 pub run_id: Option<String>,
3809 pub workflow_type: String,
3810 #[serde(default = "default_payload_codec")]
3811 pub payload_codec: String,
3812 #[serde(default)]
3813 pub arguments: Option<Value>,
3814 #[serde(default)]
3815 pub history_events: Vec<HistoryEvent>,
3816 #[serde(default)]
3817 pub total_history_events: Option<u64>,
3818 #[serde(default)]
3819 pub history_size_bytes: Option<u64>,
3820 #[serde(default)]
3821 pub continue_as_new_recommended: Option<bool>,
3822 #[serde(default)]
3823 pub history_budget_pressure: Option<String>,
3824 #[serde(default)]
3825 pub next_history_page_token: Option<String>,
3826 #[serde(default = "default_workflow_task_attempt")]
3827 pub workflow_task_attempt: u64,
3828 #[serde(default)]
3829 pub workflow_signal_id: Option<String>,
3830 #[serde(default)]
3831 pub signal_name: Option<String>,
3832 #[serde(default)]
3833 pub signal_arguments: Option<Value>,
3834 #[serde(default)]
3835 pub workflow_update_id: Option<String>,
3836 #[serde(default)]
3837 pub update_name: Option<String>,
3838 #[serde(default)]
3839 pub lease_owner: Option<String>,
3840}
3841
3842impl WorkflowTask {
3843 fn append_history_page(&mut self, page: WorkflowTaskHistoryPage) {
3844 self.history_events.extend(page.history_events);
3845
3846 if page.total_history_events.is_some() {
3847 self.total_history_events = page.total_history_events;
3848 }
3849
3850 self.next_history_page_token = page
3851 .next_history_page_token
3852 .filter(|token| !token.is_empty());
3853 }
3854}
3855
3856#[derive(Clone, Debug, Deserialize)]
3857struct WorkflowTaskHistoryPage {
3858 #[serde(default)]
3859 history_events: Vec<HistoryEvent>,
3860 #[serde(default)]
3861 total_history_events: Option<u64>,
3862 #[serde(default)]
3863 next_history_page_token: Option<String>,
3864}
3865
3866#[derive(Clone, Debug, Deserialize)]
3867pub struct ActivityTask {
3868 pub task_id: String,
3869 #[serde(default)]
3870 pub activity_attempt_id: Option<String>,
3871 #[serde(default)]
3872 pub attempt_id: Option<String>,
3873 pub activity_type: String,
3874 #[serde(default = "default_payload_codec")]
3875 pub payload_codec: String,
3876 #[serde(default)]
3877 pub arguments: Option<Value>,
3878 #[serde(default = "default_attempt_number")]
3879 pub attempt_number: u64,
3880 #[serde(default)]
3881 pub lease_owner: Option<String>,
3882}
3883
3884#[derive(Clone, Debug, Deserialize)]
3885pub struct HistoryEvent {
3886 #[serde(alias = "type")]
3887 pub event_type: String,
3888 #[serde(default)]
3889 pub payload: Value,
3890 #[serde(flatten)]
3891 pub raw: HashMap<String, Value>,
3892}
3893
3894#[derive(Clone, Debug, PartialEq)]
3896pub struct QuerySignal {
3897 pub id: Option<String>,
3898 pub name: String,
3899 pub arguments: Vec<Value>,
3900 avro_arguments: Vec<AvroValue>,
3901 pub workflow_sequence: Option<u64>,
3902}
3903
3904impl QuerySignal {
3905 pub fn arguments_avro_value(&self) -> &[AvroValue] {
3907 &self.avro_arguments
3908 }
3909}
3910
3911#[derive(Clone, Debug)]
3917pub struct QueryContext {
3918 pub workflow_id: Option<String>,
3919 pub run_id: Option<String>,
3920 pub workflow_type: String,
3921 pub run_status: Option<String>,
3922 workflow_input: Value,
3923 workflow_input_avro_value: AvroValue,
3924 history_events: Arc<Vec<HistoryEvent>>,
3925 signal_events: Arc<Vec<QuerySignal>>,
3926}
3927
3928impl QueryContext {
3929 pub fn workflow_input(&self) -> &Value {
3931 &self.workflow_input
3932 }
3933
3934 pub fn workflow_input_avro_value(&self) -> &AvroValue {
3936 &self.workflow_input_avro_value
3937 }
3938
3939 pub fn history_events(&self) -> &[HistoryEvent] {
3941 self.history_events.as_slice()
3942 }
3943
3944 pub fn signal_events(&self) -> &[QuerySignal] {
3946 self.signal_events.as_slice()
3947 }
3948
3949 pub fn signals(&self, signal_name: &str) -> Vec<Vec<Value>> {
3951 self.signal_events
3952 .iter()
3953 .filter(|signal| signal.name == signal_name)
3954 .map(|signal| signal.arguments.clone())
3955 .collect()
3956 }
3957
3958 pub fn signals_avro_value(&self, signal_name: &str) -> Vec<Vec<AvroValue>> {
3960 self.signal_events
3961 .iter()
3962 .filter(|signal| signal.name == signal_name)
3963 .map(|signal| signal.avro_arguments.clone())
3964 .collect()
3965 }
3966}
3967
3968#[derive(Clone, Debug, Deserialize)]
3969pub struct ActivityHeartbeatResponse {
3970 #[serde(default)]
3971 pub cancel_requested: bool,
3972 #[serde(default)]
3973 pub heartbeat_recorded: bool,
3974 #[serde(default)]
3975 pub can_continue: Option<bool>,
3976 #[serde(default)]
3977 pub reason: Option<String>,
3978 #[serde(default)]
3979 pub run_closed_reason: Option<String>,
3980 #[serde(default)]
3981 pub run_closed_at: Option<String>,
3982 #[serde(default)]
3983 pub lease_expires_at: Option<String>,
3984 #[serde(default)]
3985 pub last_heartbeat_at: Option<String>,
3986}
3987
3988impl ActivityHeartbeatResponse {
3989 pub fn should_stop(&self) -> bool {
3991 self.cancel_requested || self.can_continue == Some(false)
3992 }
3993}
3994
3995fn default_payload_codec() -> String {
3996 DEFAULT_CODEC.to_string()
3997}
3998
3999fn default_workflow_task_attempt() -> u64 {
4000 1
4001}
4002
4003fn default_attempt_number() -> u64 {
4004 1
4005}
4006
4007type WorkflowFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
4008type WorkflowHandler = Arc<dyn Fn(WorkflowContext, AvroValue) -> WorkflowFuture + Send + Sync>;
4009type ErasedWorkflowState = Arc<dyn Any + Send + Sync>;
4010type WorkflowStateSnapshot = Arc<dyn Fn() -> Result<ErasedWorkflowState> + Send + Sync>;
4011type ReplayedWorkflowHandler =
4012 Arc<dyn Fn(WorkflowContext, AvroValue) -> ReplayedWorkflowInvocation + Send + Sync>;
4013type ActivityFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
4014type ActivityHandler = Arc<dyn Fn(ActivityContext, AvroValue) -> ActivityFuture + Send + Sync>;
4015type QueryFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
4016type QueryHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
4017type UpdateHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
4018type ReplayedQueryHandler = Arc<
4019 dyn Fn(QueryContext, ErasedWorkflowState, AvroValue) -> std::result::Result<QueryFuture, String>
4020 + Send
4021 + Sync,
4022>;
4023type WorkerHeartbeatObserver = Arc<dyn Fn(&WorkerHeartbeatObservation) + Send + Sync>;
4024
4025struct ReplayedWorkflowInvocation {
4026 future: WorkflowFuture,
4027 snapshot: WorkflowStateSnapshot,
4028}
4029
4030#[derive(Clone)]
4031struct RegisteredWorkflow {
4032 execute: WorkflowHandler,
4033 replay: Option<ReplayedWorkflowHandler>,
4034 state_type: Option<TypeId>,
4035}
4036
4037#[derive(Clone)]
4038enum RegisteredQuery {
4039 Snapshot(QueryHandler),
4040 Replayed {
4041 state_type: TypeId,
4042 handler: ReplayedQueryHandler,
4043 },
4044}
4045
4046#[derive(Clone, Debug)]
4047pub struct WorkerHeartbeatObservation {
4048 pub worker_id: String,
4049 pub task_queue: String,
4050 pub acknowledged_at_unix_millis: u64,
4051 pub acknowledgement: Value,
4052}
4053
4054#[derive(Clone, Copy, Debug)]
4061pub struct WorkerRetryPolicy {
4062 pub max_retries: usize,
4064 pub initial_backoff: Duration,
4066 pub max_backoff: Duration,
4068}
4069
4070impl Default for WorkerRetryPolicy {
4071 fn default() -> Self {
4072 Self {
4073 max_retries: 5,
4074 initial_backoff: Duration::from_millis(100),
4075 max_backoff: Duration::from_secs(5),
4076 }
4077 }
4078}
4079
4080#[derive(Clone, Copy, Debug, PartialEq, Eq)]
4081enum ManagedPollOutcome {
4082 Idle,
4083 Handled,
4084 Stop,
4085}
4086
4087#[derive(Clone)]
4088pub struct Worker {
4089 client: Client,
4090 worker_id: String,
4091 task_queue: String,
4092 workflows: HashMap<String, RegisteredWorkflow>,
4093 activities: HashMap<String, ActivityHandler>,
4094 queries: HashMap<String, HashMap<String, RegisteredQuery>>,
4095 updates: HashMap<String, HashMap<String, UpdateHandler>>,
4096 max_concurrent_workflow_tasks: usize,
4097 max_concurrent_activity_tasks: usize,
4098 poll_timeout: Duration,
4099 heartbeat_interval: Duration,
4100 retry_policy: WorkerRetryPolicy,
4101 heartbeat_observer: Option<WorkerHeartbeatObserver>,
4102}
4103
4104impl Worker {
4105 pub fn new(client: Client, task_queue: impl Into<String>) -> Self {
4106 Self {
4107 client,
4108 worker_id: default_worker_id(),
4109 task_queue: task_queue.into(),
4110 workflows: HashMap::new(),
4111 activities: HashMap::new(),
4112 queries: HashMap::new(),
4113 updates: HashMap::new(),
4114 max_concurrent_workflow_tasks: 10,
4115 max_concurrent_activity_tasks: 10,
4116 poll_timeout: Duration::from_secs(30),
4117 heartbeat_interval: Duration::from_secs(60),
4118 retry_policy: WorkerRetryPolicy::default(),
4119 heartbeat_observer: None,
4120 }
4121 }
4122
4123 pub fn worker_id(mut self, worker_id: impl Into<String>) -> Self {
4124 self.worker_id = worker_id.into();
4125 self
4126 }
4127
4128 pub fn poll_timeout(mut self, timeout: Duration) -> Self {
4129 self.poll_timeout = timeout;
4130 self
4131 }
4132
4133 pub fn heartbeat_interval(mut self, interval: Duration) -> Self {
4134 self.heartbeat_interval = interval;
4135 self
4136 }
4137
4138 pub fn retry_policy(mut self, policy: WorkerRetryPolicy) -> Self {
4140 self.retry_policy = policy;
4141 self
4142 }
4143
4144 pub fn on_worker_heartbeat<F>(mut self, observer: F) -> Self
4145 where
4146 F: Fn(&WorkerHeartbeatObservation) + Send + Sync + 'static,
4147 {
4148 self.heartbeat_observer = Some(Arc::new(observer));
4149 self
4150 }
4151
4152 pub fn max_concurrent_workflow_tasks(mut self, count: usize) -> Self {
4153 self.max_concurrent_workflow_tasks = count.max(1);
4154 self
4155 }
4156
4157 pub fn max_concurrent_activity_tasks(mut self, count: usize) -> Self {
4158 self.max_concurrent_activity_tasks = count.max(1);
4159 self
4160 }
4161
4162 pub fn register_workflow<F, Fut>(&mut self, workflow_type: impl Into<String>, handler: F)
4169 where
4170 F: Fn(WorkflowContext, Value) -> Fut + Send + Sync + 'static,
4171 Fut: Future<Output = Result<Value>> + Send + 'static,
4172 {
4173 let handler = Arc::new(handler);
4174 self.workflows.insert(
4175 workflow_type.into(),
4176 RegisteredWorkflow {
4177 execute: Arc::new(move |ctx, input| {
4178 let handler = Arc::clone(&handler);
4179 Box::pin(async move {
4180 let result = handler(ctx, input.into_json()?).await?;
4181 AvroValue::from_serialize(&result)
4182 })
4183 }),
4184 replay: None,
4185 state_type: None,
4186 },
4187 );
4188 }
4189
4190 pub fn register_workflow_avro_value<F, Fut>(
4192 &mut self,
4193 workflow_type: impl Into<String>,
4194 handler: F,
4195 ) where
4196 F: Fn(WorkflowContext, AvroValue) -> Fut + Send + Sync + 'static,
4197 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4198 {
4199 self.workflows.insert(
4200 workflow_type.into(),
4201 RegisteredWorkflow {
4202 execute: Arc::new(move |ctx, input| Box::pin(handler(ctx, input))),
4203 replay: None,
4204 state_type: None,
4205 },
4206 );
4207 }
4208
4209 pub fn register_replayed_workflow<S, Factory, F, Fut>(
4217 &mut self,
4218 workflow_type: impl Into<String>,
4219 state_factory: Factory,
4220 handler: F,
4221 ) where
4222 S: Clone + Send + Sync + 'static,
4223 Factory: Fn() -> S + Send + Sync + 'static,
4224 F: Fn(WorkflowContext, Value, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
4225 Fut: Future<Output = Result<Value>> + Send + 'static,
4226 {
4227 let state_factory = Arc::new(state_factory);
4228 let handler = Arc::new(handler);
4229
4230 let execute_factory = Arc::clone(&state_factory);
4231 let execute_handler = Arc::clone(&handler);
4232 let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4233 let state = WorkflowInstance::new(execute_factory());
4234 let handler = Arc::clone(&execute_handler);
4235 Box::pin(async move {
4236 let result = handler(ctx, input.into_json()?, state).await?;
4237 AvroValue::from_serialize(&result)
4238 }) as WorkflowFuture
4239 });
4240
4241 let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4242 let state = WorkflowInstance::new(state_factory());
4243 let snapshot_state = state.clone();
4244 let snapshot: WorkflowStateSnapshot =
4245 Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
4246 let replay_handler = Arc::clone(&handler);
4247 let future = async move {
4248 let result = replay_handler(ctx, input.into_json()?, state).await?;
4249 AvroValue::from_serialize(&result)
4250 };
4251 ReplayedWorkflowInvocation {
4252 future: Box::pin(future),
4253 snapshot,
4254 }
4255 });
4256
4257 self.workflows.insert(
4258 workflow_type.into(),
4259 RegisteredWorkflow {
4260 execute,
4261 replay: Some(replay),
4262 state_type: Some(TypeId::of::<S>()),
4263 },
4264 );
4265 }
4266
4267 pub fn register_replayed_workflow_avro_value<S, Factory, F, Fut>(
4269 &mut self,
4270 workflow_type: impl Into<String>,
4271 state_factory: Factory,
4272 handler: F,
4273 ) where
4274 S: Clone + Send + Sync + 'static,
4275 Factory: Fn() -> S + Send + Sync + 'static,
4276 F: Fn(WorkflowContext, AvroValue, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
4277 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4278 {
4279 let state_factory = Arc::new(state_factory);
4280 let handler = Arc::new(handler);
4281
4282 let execute_factory = Arc::clone(&state_factory);
4283 let execute_handler = Arc::clone(&handler);
4284 let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4285 let state = WorkflowInstance::new(execute_factory());
4286 Box::pin(execute_handler(ctx, input, state)) as WorkflowFuture
4287 });
4288
4289 let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4290 let state = WorkflowInstance::new(state_factory());
4291 let snapshot_state = state.clone();
4292 let snapshot: WorkflowStateSnapshot =
4293 Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
4294 ReplayedWorkflowInvocation {
4295 future: Box::pin(handler(ctx, input, state)),
4296 snapshot,
4297 }
4298 });
4299
4300 self.workflows.insert(
4301 workflow_type.into(),
4302 RegisteredWorkflow {
4303 execute,
4304 replay: Some(replay),
4305 state_type: Some(TypeId::of::<S>()),
4306 },
4307 );
4308 }
4309
4310 pub fn register_activity<F, Fut>(&mut self, activity_type: impl Into<String>, handler: F)
4311 where
4312 F: Fn(ActivityContext, Value) -> Fut + Send + Sync + 'static,
4313 Fut: Future<Output = Result<Value>> + Send + 'static,
4314 {
4315 let handler = Arc::new(handler);
4316 self.activities.insert(
4317 activity_type.into(),
4318 Arc::new(move |ctx, args| {
4319 let handler = Arc::clone(&handler);
4320 Box::pin(async move {
4321 let result = handler(ctx, args.into_json()?).await?;
4322 AvroValue::from_serialize(&result)
4323 })
4324 }),
4325 );
4326 }
4327
4328 pub fn register_activity_avro_value<F, Fut>(
4330 &mut self,
4331 activity_type: impl Into<String>,
4332 handler: F,
4333 ) where
4334 F: Fn(ActivityContext, AvroValue) -> Fut + Send + Sync + 'static,
4335 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4336 {
4337 self.activities.insert(
4338 activity_type.into(),
4339 Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
4340 );
4341 }
4342
4343 pub fn register_query<F, Fut>(
4349 &mut self,
4350 workflow_type: impl Into<String>,
4351 query_name: impl Into<String>,
4352 handler: F,
4353 ) where
4354 F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
4355 Fut: Future<Output = Result<Value>> + Send + 'static,
4356 {
4357 let handler = Arc::new(handler);
4358 self.queries
4359 .entry(workflow_type.into())
4360 .or_default()
4361 .insert(
4362 query_name.into(),
4363 RegisteredQuery::Snapshot(Arc::new(move |ctx, args| {
4364 let handler = Arc::clone(&handler);
4365 Box::pin(async move {
4366 let result = handler(ctx, args.into_json()?).await?;
4367 AvroValue::from_serialize(&result)
4368 })
4369 })),
4370 );
4371 }
4372
4373 pub fn register_query_avro_value<F, Fut>(
4375 &mut self,
4376 workflow_type: impl Into<String>,
4377 query_name: impl Into<String>,
4378 handler: F,
4379 ) where
4380 F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
4381 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4382 {
4383 self.queries
4384 .entry(workflow_type.into())
4385 .or_default()
4386 .insert(
4387 query_name.into(),
4388 RegisteredQuery::Snapshot(Arc::new(move |ctx, args| Box::pin(handler(ctx, args)))),
4389 );
4390 }
4391
4392 pub fn register_replayed_query<S, F, Fut>(
4399 &mut self,
4400 workflow_type: impl Into<String>,
4401 query_name: impl Into<String>,
4402 handler: F,
4403 ) where
4404 S: Clone + Send + Sync + 'static,
4405 F: Fn(QueryContext, Arc<S>, Value) -> Fut + Send + Sync + 'static,
4406 Fut: Future<Output = Result<Value>> + Send + 'static,
4407 {
4408 let handler = Arc::new(handler);
4409 let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
4410 let state = state.downcast::<S>().map_err(|_| {
4411 "registered query state type does not match the replayed workflow state".to_string()
4412 })?;
4413 let handler = Arc::clone(&handler);
4414 Ok(Box::pin(async move {
4415 let result = handler(ctx, state, args.into_json()?).await?;
4416 AvroValue::from_serialize(&result)
4417 }))
4418 });
4419
4420 self.queries
4421 .entry(workflow_type.into())
4422 .or_default()
4423 .insert(
4424 query_name.into(),
4425 RegisteredQuery::Replayed {
4426 state_type: TypeId::of::<S>(),
4427 handler: erased_handler,
4428 },
4429 );
4430 }
4431
4432 pub fn register_replayed_query_avro_value<S, F, Fut>(
4434 &mut self,
4435 workflow_type: impl Into<String>,
4436 query_name: impl Into<String>,
4437 handler: F,
4438 ) where
4439 S: Clone + Send + Sync + 'static,
4440 F: Fn(QueryContext, Arc<S>, AvroValue) -> Fut + Send + Sync + 'static,
4441 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4442 {
4443 let handler = Arc::new(handler);
4444 let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
4445 let state = state.downcast::<S>().map_err(|_| {
4446 "registered query state type does not match the replayed workflow state".to_string()
4447 })?;
4448 Ok(Box::pin(handler(ctx, state, args)))
4449 });
4450
4451 self.queries
4452 .entry(workflow_type.into())
4453 .or_default()
4454 .insert(
4455 query_name.into(),
4456 RegisteredQuery::Replayed {
4457 state_type: TypeId::of::<S>(),
4458 handler: erased_handler,
4459 },
4460 );
4461 }
4462
4463 pub fn register_update<F, Fut>(
4465 &mut self,
4466 workflow_type: impl Into<String>,
4467 update_name: impl Into<String>,
4468 handler: F,
4469 ) where
4470 F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
4471 Fut: Future<Output = Result<Value>> + Send + 'static,
4472 {
4473 let handler = Arc::new(handler);
4474 self.updates
4475 .entry(workflow_type.into())
4476 .or_default()
4477 .insert(
4478 update_name.into(),
4479 Arc::new(move |ctx, args| {
4480 let handler = Arc::clone(&handler);
4481 Box::pin(async move {
4482 let result = handler(ctx, args.into_json()?).await?;
4483 AvroValue::from_serialize(&result)
4484 })
4485 }),
4486 );
4487 }
4488
4489 pub fn register_update_avro_value<F, Fut>(
4491 &mut self,
4492 workflow_type: impl Into<String>,
4493 update_name: impl Into<String>,
4494 handler: F,
4495 ) where
4496 F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
4497 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4498 {
4499 self.updates
4500 .entry(workflow_type.into())
4501 .or_default()
4502 .insert(
4503 update_name.into(),
4504 Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
4505 );
4506 }
4507
4508 pub async fn register(&self) -> Result<RegisterWorkerResponse> {
4509 let mut command_contracts = serde_json::Map::new();
4510 for workflow_type in self.workflows.keys() {
4511 let mut queries = self
4512 .queries
4513 .get(workflow_type)
4514 .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
4515 .unwrap_or_default();
4516 queries.sort();
4517 let mut updates = self
4518 .updates
4519 .get(workflow_type)
4520 .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
4521 .unwrap_or_default();
4522 updates.sort();
4523 if !queries.is_empty() || !updates.is_empty() {
4524 command_contracts.insert(
4525 workflow_type.clone(),
4526 json!({
4527 "queries": queries,
4528 "updates": updates,
4529 "update_validators": [],
4530 }),
4531 );
4532 }
4533 }
4534
4535 self.client
4536 .register_worker_with_command_contracts(
4537 &self.worker_id,
4538 &self.task_queue,
4539 self.workflows.keys().cloned().collect(),
4540 self.activities.keys().cloned().collect(),
4541 self.max_concurrent_workflow_tasks,
4542 self.max_concurrent_activity_tasks,
4543 [
4544 (!self.queries.is_empty()).then(|| QUERY_TASKS_CAPABILITY.to_string()),
4545 (!self.updates.is_empty()).then(|| WORKFLOW_UPDATES_CAPABILITY.to_string()),
4546 ]
4547 .into_iter()
4548 .flatten()
4549 .collect(),
4550 Value::Object(command_contracts),
4551 )
4552 .await
4553 }
4554
4555 pub async fn run(&self) -> Result<()> {
4561 self.run_until(std::future::pending::<()>()).await
4562 }
4563
4564 pub async fn run_until<F>(&self, shutdown: F) -> Result<()>
4568 where
4569 F: Future<Output = ()>,
4570 {
4571 let registration = self.register().await?;
4572 if !registration.registered {
4573 return Err(Error::WorkerLoop(format!(
4574 "worker registration for {:?} was not accepted",
4575 self.worker_id
4576 )));
4577 }
4578 let registered_worker_id = registration.worker_id.clone();
4579 let primary = self.run_registered_until(shutdown, registration).await;
4580 let deregistration = self
4581 .client
4582 .deregister_worker_registration(®istered_worker_id)
4583 .await;
4584
4585 match (primary, deregistration) {
4586 (Ok(()), Ok(_)) => Ok(()),
4587 (Ok(()), Err(deregistration)) => Err(deregistration),
4588 (Err(primary), Ok(_)) => Err(primary),
4589 (Err(primary), Err(deregistration)) => Err(Error::WorkerShutdown {
4590 primary: Box::new(primary),
4591 deregistration: Box::new(deregistration),
4592 }),
4593 }
4594 }
4595
4596 async fn run_registered_until<F>(
4597 &self,
4598 shutdown: F,
4599 registration: RegisterWorkerResponse,
4600 ) -> Result<()>
4601 where
4602 F: Future<Output = ()>,
4603 {
4604 let heartbeat_interval = Duration::from_secs(
4605 registration
4606 .heartbeat_interval_seconds
4607 .unwrap_or(self.heartbeat_interval.as_secs().max(1)),
4608 );
4609 let heartbeat = tokio::time::sleep(Duration::ZERO);
4615 tokio::pin!(heartbeat);
4616 tokio::pin!(shutdown);
4617 let stop = Arc::new(AtomicBool::new(false));
4618 let mut workflow_poller = (!self.workflows.is_empty()).then(|| {
4622 let worker = self.clone();
4623 let stop = Arc::clone(&stop);
4624 tokio::spawn(async move { worker.poll_workflows_until_stopped(stop).await })
4625 });
4626 let mut activity_poller = (!self.activities.is_empty()).then(|| {
4627 let worker = self.clone();
4628 let stop = Arc::clone(&stop);
4629 tokio::spawn(async move { worker.poll_activities_until_stopped(stop).await })
4630 });
4631 let mut query_poller = (!self.queries.is_empty()).then(|| {
4632 let worker = self.clone();
4633 let stop = Arc::clone(&stop);
4634 tokio::spawn(async move { worker.poll_queries_until_stopped(stop).await })
4635 });
4636
4637 loop {
4638 tokio::select! {
4639 _ = &mut shutdown => {
4640 stop.store(true, Ordering::SeqCst);
4641 break;
4642 }
4643 _ = &mut heartbeat => {
4644 let result = self.retry_worker_operation(|| {
4645 self.client.heartbeat_worker(
4646 &self.worker_id,
4647 self.max_concurrent_workflow_tasks,
4648 self.max_concurrent_activity_tasks,
4649 )
4650 }).await;
4651 heartbeat
4652 .as_mut()
4653 .reset(tokio::time::Instant::now() + heartbeat_interval);
4654 match result {
4655 Ok(acknowledgement) => {
4656 if let Some(observer) = &self.heartbeat_observer {
4657 observer(&WorkerHeartbeatObservation {
4658 worker_id: self.worker_id.clone(),
4659 task_queue: self.task_queue.clone(),
4660 acknowledged_at_unix_millis: SystemTime::now()
4661 .duration_since(UNIX_EPOCH)
4662 .unwrap_or_default()
4663 .as_millis()
4664 .min(u64::MAX as u128)
4665 as u64,
4666 acknowledgement,
4667 });
4668 }
4669 }
4670 Err(error) => {
4671 stop.store(true, Ordering::SeqCst);
4672 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await?;
4673 return Err(error);
4674 }
4675 }
4676 }
4677 result = OptionFuture::from(workflow_poller.as_mut()), if workflow_poller.is_some() => {
4678 workflow_poller = None;
4679 let stopped_by_server = stop.load(Ordering::SeqCst);
4680 stop.store(true, Ordering::SeqCst);
4681 let poller_result = optional_poller_result("workflow", result);
4682 let join_result =
4683 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
4684 poller_result?;
4685 join_result?;
4686 if stopped_by_server {
4687 return Ok(());
4688 }
4689 return Err(Error::WorkerLoop(
4690 "workflow poller stopped unexpectedly".to_string(),
4691 ));
4692 }
4693 result = OptionFuture::from(activity_poller.as_mut()), if activity_poller.is_some() => {
4694 activity_poller = None;
4695 let stopped_by_server = stop.load(Ordering::SeqCst);
4696 stop.store(true, Ordering::SeqCst);
4697 let poller_result = optional_poller_result("activity", result);
4698 let join_result =
4699 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
4700 poller_result?;
4701 join_result?;
4702 if stopped_by_server {
4703 return Ok(());
4704 }
4705 return Err(Error::WorkerLoop(
4706 "activity poller stopped unexpectedly".to_string(),
4707 ));
4708 }
4709 result = OptionFuture::from(query_poller.as_mut()), if query_poller.is_some() => {
4710 query_poller = None;
4711 let stopped_by_server = stop.load(Ordering::SeqCst);
4712 stop.store(true, Ordering::SeqCst);
4713 let poller_result = optional_poller_result("query", result);
4714 let join_result =
4715 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
4716 poller_result?;
4717 join_result?;
4718 if stopped_by_server {
4719 return Ok(());
4720 }
4721 return Err(Error::WorkerLoop(
4722 "query poller stopped unexpectedly".to_string(),
4723 ));
4724 }
4725 }
4726 }
4727
4728 join_pollers(
4729 workflow_poller.take(),
4730 activity_poller.take(),
4731 query_poller.take(),
4732 )
4733 .await
4734 }
4735
4736 pub async fn run_once(&self) -> Result<usize> {
4750 let mut handled = 0;
4751 match self.poll_workflow_once().await? {
4752 ManagedPollOutcome::Handled => handled += 1,
4753 ManagedPollOutcome::Stop => return Ok(handled),
4754 ManagedPollOutcome::Idle => {}
4755 }
4756 match self.poll_activity_once().await? {
4757 ManagedPollOutcome::Handled => handled += 1,
4758 ManagedPollOutcome::Stop => return Ok(handled),
4759 ManagedPollOutcome::Idle => {}
4760 }
4761 if !self.queries.is_empty() {
4762 match self.poll_query_once().await? {
4763 ManagedPollOutcome::Handled => handled += 1,
4764 ManagedPollOutcome::Stop => return Ok(handled),
4765 ManagedPollOutcome::Idle => {}
4766 }
4767 }
4768 Ok(handled)
4769 }
4770
4771 async fn poll_workflow_once(&self) -> Result<ManagedPollOutcome> {
4772 let poll_request_id = unique_request_id("rust-workflow-poll");
4773 let response = self
4774 .retry_worker_operation(|| {
4775 self.client.poll_workflow_task_response_with_request_id(
4776 &self.worker_id,
4777 &self.task_queue,
4778 self.poll_timeout,
4779 &poll_request_id,
4780 0,
4781 )
4782 })
4783 .await?;
4784 if response.outcome().should_stop() {
4785 return Ok(ManagedPollOutcome::Stop);
4786 }
4787 let Some(task) = response.task else {
4788 return Ok(ManagedPollOutcome::Idle);
4789 };
4790
4791 let task_id = task.task_id.clone();
4792 let attempt = task.workflow_task_attempt;
4793 let run_id = task.run_id.clone();
4794 let lease_owner = task
4795 .lease_owner
4796 .clone()
4797 .unwrap_or_else(|| self.worker_id.clone());
4798
4799 match self.execute_workflow_task(task) {
4800 Ok(commands) if commands.is_empty() => {
4801 self.client
4807 .fail_workflow_task_with_type(
4808 &task_id,
4809 &lease_owner,
4810 attempt,
4811 WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE,
4812 WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE,
4813 )
4814 .await?;
4815 }
4816 Ok(commands) => {
4817 let completion = self
4818 .client
4819 .complete_workflow_task(&task_id, &lease_owner, attempt, commands)
4820 .await;
4821 if let Err(error) = completion {
4822 if !workflow_task_completion_is_terminal_timeout(
4823 &error,
4824 &task_id,
4825 attempt,
4826 run_id.as_deref(),
4827 ) {
4828 return Err(error);
4829 }
4830 }
4831 }
4832 Err(error) => {
4833 self.client
4834 .fail_workflow_task(&task_id, &lease_owner, attempt, error.to_string())
4835 .await?;
4836 }
4837 }
4838
4839 Ok(ManagedPollOutcome::Handled)
4840 }
4841
4842 async fn poll_workflows_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
4843 while !stop.load(Ordering::SeqCst) {
4844 if self.poll_workflow_once().await? == ManagedPollOutcome::Stop {
4845 stop.store(true, Ordering::SeqCst);
4846 break;
4847 }
4848 }
4849
4850 Ok(())
4851 }
4852
4853 async fn poll_activity_once(&self) -> Result<ManagedPollOutcome> {
4854 let poll_request_id = unique_request_id("rust-activity-poll");
4855 let response = self
4856 .retry_worker_operation(|| {
4857 self.client.poll_activity_task_response_with_request_id(
4858 &self.worker_id,
4859 &self.task_queue,
4860 self.poll_timeout,
4861 &poll_request_id,
4862 0,
4863 )
4864 })
4865 .await?;
4866 if response.outcome().should_stop() {
4867 return Ok(ManagedPollOutcome::Stop);
4868 }
4869 let Some(task) = response.task else {
4870 return Ok(ManagedPollOutcome::Idle);
4871 };
4872
4873 let task_id = task.task_id.clone();
4874 let attempt_id = task
4875 .activity_attempt_id
4876 .clone()
4877 .or(task.attempt_id.clone())
4878 .unwrap_or_default();
4879 let lease_owner = task
4880 .lease_owner
4881 .clone()
4882 .unwrap_or_else(|| self.worker_id.clone());
4883 let codec = task.payload_codec.clone();
4884 let result = self.execute_activity_task(task).await;
4885 match result {
4886 Ok(value) => {
4887 let completion = self
4888 .client
4889 .complete_activity_task(&task_id, &attempt_id, &lease_owner, value, &codec)
4890 .await;
4891 if let Err(error) = completion {
4892 if !activity_task_rejection_is_final(&error) {
4893 return Err(error);
4894 }
4895 }
4896 }
4897 Err(error) => {
4898 let failure = self
4899 .client
4900 .fail_activity_task(
4901 &task_id,
4902 &attempt_id,
4903 &lease_owner,
4904 error.to_string(),
4905 false,
4906 )
4907 .await;
4908 if let Err(error) = failure {
4909 if !activity_task_rejection_is_final(&error) {
4910 return Err(error);
4911 }
4912 }
4913 }
4914 }
4915
4916 Ok(ManagedPollOutcome::Handled)
4917 }
4918
4919 async fn poll_activities_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
4920 while !stop.load(Ordering::SeqCst) {
4921 if self.poll_activity_once().await? == ManagedPollOutcome::Stop {
4922 stop.store(true, Ordering::SeqCst);
4923 break;
4924 }
4925 }
4926
4927 Ok(())
4928 }
4929
4930 async fn poll_query_once(&self) -> Result<ManagedPollOutcome> {
4931 let poll_request_id = unique_request_id("rust-query-poll");
4932 let response = self
4933 .retry_worker_operation(|| {
4934 self.client.poll_query_task_response_with_request_id(
4935 &self.worker_id,
4936 &self.task_queue,
4937 self.poll_timeout,
4938 &poll_request_id,
4939 0,
4940 )
4941 })
4942 .await?;
4943 if response.outcome().should_stop() {
4944 return Ok(ManagedPollOutcome::Stop);
4945 }
4946 let Some(task) = response.task else {
4947 return Ok(ManagedPollOutcome::Idle);
4948 };
4949
4950 let query_task_id = task.query_task_id.clone();
4951 let attempt = task.query_task_attempt;
4952 let lease_owner = task
4953 .lease_owner
4954 .clone()
4955 .unwrap_or_else(|| self.worker_id.clone());
4956 let codec = task.payload_codec.clone();
4957
4958 match self.execute_query_task(task).await {
4959 Ok(value) => {
4960 let result_envelope = match encode_typed_envelope(&value, &codec) {
4961 Ok(result_envelope) => result_envelope,
4962 Err(error) => {
4963 let failure = self
4964 .client
4965 .fail_query_task(
4966 &query_task_id,
4967 &lease_owner,
4968 attempt,
4969 error.to_string(),
4970 "query_result_encode_failed",
4971 "QueryResultEncodeFailed",
4972 )
4973 .await;
4974 if let Err(error) = failure {
4975 if !query_task_rejection_is_final(&error) {
4976 return Err(error);
4977 }
4978 }
4979 return Ok(ManagedPollOutcome::Handled);
4980 }
4981 };
4982
4983 if let Err(error) = self
4984 .client
4985 .complete_query_task_with_envelope(
4986 &query_task_id,
4987 &lease_owner,
4988 attempt,
4989 value.clone().into_json()?,
4990 result_envelope,
4991 )
4992 .await
4993 {
4994 if !query_task_rejection_is_final(&error) {
4995 return Err(error);
4996 }
4997 }
4998 }
4999 Err(failure) => {
5000 let result = self
5001 .client
5002 .fail_query_task(
5003 &query_task_id,
5004 &lease_owner,
5005 attempt,
5006 failure.message,
5007 failure.reason,
5008 failure.failure_type,
5009 )
5010 .await;
5011 if let Err(error) = result {
5012 if !query_task_rejection_is_final(&error) {
5013 return Err(error);
5014 }
5015 }
5016 }
5017 }
5018
5019 Ok(ManagedPollOutcome::Handled)
5020 }
5021
5022 async fn poll_queries_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
5023 while !stop.load(Ordering::SeqCst) {
5024 if self.poll_query_once().await? == ManagedPollOutcome::Stop {
5025 stop.store(true, Ordering::SeqCst);
5026 break;
5027 }
5028 }
5029
5030 Ok(())
5031 }
5032
5033 async fn retry_worker_operation<T, F, Fut>(&self, mut operation: F) -> Result<T>
5034 where
5035 F: FnMut() -> Fut,
5036 Fut: Future<Output = Result<T>>,
5037 {
5038 let mut retries = 0;
5039
5040 loop {
5041 match operation().await {
5042 Err(error)
5043 if worker_operation_is_retryable(&error)
5044 && retries < self.retry_policy.max_retries =>
5045 {
5046 retries += 1;
5047 tokio::time::sleep(worker_retry_delay(self.retry_policy, retries)).await;
5048 }
5049 result => return result,
5050 }
5051 }
5052 }
5053
5054 async fn execute_query_task(
5055 &self,
5056 mut task: QueryTask,
5057 ) -> std::result::Result<AvroValue, QueryTaskExecutionFailure> {
5058 if task.payload_codec != DEFAULT_CODEC {
5059 return Err(QueryTaskExecutionFailure::new(
5060 "query_payload_decode_failed",
5061 unsupported_payload_codec(&task.payload_codec).to_string(),
5062 "QueryPayloadDecodeFailed",
5063 ));
5064 }
5065
5066 if !self.workflows.contains_key(&task.workflow_type) {
5067 return Err(QueryTaskExecutionFailure::new(
5068 "query_workflow_type_not_registered",
5069 format!("no workflow registered for type {:?}", task.workflow_type),
5070 "WorkflowTypeNotRegistered",
5071 ));
5072 }
5073
5074 let Some(handlers) = self.queries.get(&task.workflow_type) else {
5075 return Err(QueryTaskExecutionFailure::new(
5076 "query_handler_unavailable",
5077 format!(
5078 "query handlers are unavailable for workflow type {:?}",
5079 task.workflow_type
5080 ),
5081 "QueryHandlerUnavailable",
5082 ));
5083 };
5084 let Some(query) = handlers.get(&task.query_name) else {
5085 return Err(QueryTaskExecutionFailure::new(
5086 "rejected_unknown_query",
5087 format!("unknown query {:?}", task.query_name),
5088 "QueryFailed",
5089 ));
5090 };
5091
5092 let args = decode_task_avro_arguments(task.query_arguments.as_ref(), &task.payload_codec)
5093 .map_err(|error| {
5094 QueryTaskExecutionFailure::new(
5095 "query_payload_decode_failed",
5096 format!("cannot decode query arguments: {error}"),
5097 "QueryPayloadDecodeFailed",
5098 )
5099 })?;
5100 let workflow_input_typed =
5101 decode_task_avro_arguments(task.workflow_arguments.as_ref(), &task.payload_codec)
5102 .map_err(|error| {
5103 QueryTaskExecutionFailure::new(
5104 "query_workflow_state_unavailable",
5105 format!("cannot decode workflow start input: {error}"),
5106 "QueryWorkflowStateUnavailable",
5107 )
5108 })?;
5109 let workflow_input = workflow_input_typed.clone().into_json().map_err(|error| {
5110 QueryTaskExecutionFailure::new(
5111 "query_workflow_state_unavailable",
5112 format!("cannot project workflow start input: {error}"),
5113 "QueryWorkflowStateUnavailable",
5114 )
5115 })?;
5116 hydrate_query_history_from_export(&mut task).map_err(|error| {
5117 QueryTaskExecutionFailure::new(
5118 "query_workflow_state_unavailable",
5119 format!("cannot restore query history snapshot: {error}"),
5120 "QueryWorkflowStateUnavailable",
5121 )
5122 })?;
5123 enrich_query_history_from_export(&mut task).map_err(|error| {
5124 QueryTaskExecutionFailure::new(
5125 "query_workflow_state_unavailable",
5126 format!("cannot restore compact query history payloads: {error}"),
5127 "QueryWorkflowStateUnavailable",
5128 )
5129 })?;
5130 let signal_events = query_signal_events(&task).map_err(|error| {
5131 QueryTaskExecutionFailure::new(
5132 "query_workflow_state_unavailable",
5133 format!("cannot decode committed workflow signals: {error}"),
5134 "QueryWorkflowStateUnavailable",
5135 )
5136 })?;
5137 let history_events = Arc::new(std::mem::take(&mut task.history_events));
5138 let context = QueryContext {
5139 workflow_id: task.workflow_id,
5140 run_id: task.run_id,
5141 workflow_type: task.workflow_type.clone(),
5142 run_status: task.run_status,
5143 workflow_input,
5144 workflow_input_avro_value: workflow_input_typed.clone(),
5145 history_events: Arc::clone(&history_events),
5146 signal_events: Arc::new(signal_events),
5147 };
5148
5149 let future = match query {
5150 RegisteredQuery::Snapshot(handler) => handler(context, args),
5151 RegisteredQuery::Replayed {
5152 state_type,
5153 handler,
5154 } => {
5155 let workflow = self
5156 .workflows
5157 .get(&task.workflow_type)
5158 .expect("workflow registration was checked above");
5159 if workflow.state_type != Some(*state_type) {
5160 return Err(QueryTaskExecutionFailure::new(
5161 "query_workflow_state_unavailable",
5162 "replayed query state type does not match its workflow registration",
5163 "QueryWorkflowStateUnavailable",
5164 ));
5165 }
5166 let replay = workflow.replay.as_ref().ok_or_else(|| {
5167 QueryTaskExecutionFailure::new(
5168 "query_workflow_state_unavailable",
5169 format!(
5170 "workflow type {:?} is not registered for instance-state replay",
5171 task.workflow_type
5172 ),
5173 "QueryWorkflowStateUnavailable",
5174 )
5175 })?;
5176 let workflow_state = Arc::new(Mutex::new(
5177 WorkflowState::new_with_identity(
5178 history_events.as_ref().clone(),
5179 context.workflow_id.clone(),
5180 context.run_id.clone(),
5181 self.task_queue.clone(),
5182 task.payload_codec,
5183 None,
5184 )
5185 .map_err(|error| {
5186 QueryTaskExecutionFailure::new(
5187 "query_workflow_state_unavailable",
5188 format!("workflow replay failed before query: {error}"),
5189 "QueryWorkflowStateUnavailable",
5190 )
5191 })?,
5192 ));
5193 let workflow_context = WorkflowContext {
5194 state: workflow_state,
5195 };
5196 let mut invocation = replay(workflow_context.clone(), workflow_input_typed.clone());
5197 let mut cx = TaskContext::from_waker(noop_waker_ref());
5198 match invocation.future.as_mut().poll(&mut cx) {
5199 Poll::Ready(Ok(_)) => {
5200 workflow_context
5201 .ensure_history_consumed()
5202 .map_err(|error| {
5203 QueryTaskExecutionFailure::new(
5204 "query_workflow_state_unavailable",
5205 format!("workflow replay failed before query: {error}"),
5206 "QueryWorkflowStateUnavailable",
5207 )
5208 })?;
5209 }
5210 Poll::Ready(Err(error)) => {
5211 return Err(QueryTaskExecutionFailure::new(
5212 "query_workflow_state_unavailable",
5213 format!("workflow replay failed before query: {error}"),
5214 "QueryWorkflowStateUnavailable",
5215 ));
5216 }
5217 Poll::Pending => {
5218 let commands = workflow_context.take_commands().map_err(|error| {
5219 QueryTaskExecutionFailure::new(
5220 "query_workflow_state_unavailable",
5221 format!("workflow replay failed before query: {error}"),
5222 "QueryWorkflowStateUnavailable",
5223 )
5224 })?;
5225 if commands.is_empty()
5226 && !workflow_context
5227 .matched_recorded_pending()
5228 .map_err(|error| {
5229 QueryTaskExecutionFailure::new(
5230 "query_workflow_state_unavailable",
5231 format!("workflow replay failed before query: {error}"),
5232 "QueryWorkflowStateUnavailable",
5233 )
5234 })?
5235 {
5236 return Err(QueryTaskExecutionFailure::new(
5237 "query_workflow_state_unavailable",
5238 "workflow replay yielded without a durable command",
5239 "QueryWorkflowStateUnavailable",
5240 ));
5241 }
5242 }
5243 }
5244 let state = (invocation.snapshot)().map_err(|error| {
5245 QueryTaskExecutionFailure::new(
5246 "query_workflow_state_unavailable",
5247 format!("cannot snapshot replayed workflow state: {error}"),
5248 "QueryWorkflowStateUnavailable",
5249 )
5250 })?;
5251 handler(context, state, args).map_err(|message| {
5252 QueryTaskExecutionFailure::new(
5253 "query_workflow_state_unavailable",
5254 message,
5255 "QueryWorkflowStateUnavailable",
5256 )
5257 })?
5258 }
5259 };
5260
5261 future.await.map_err(|error| {
5262 QueryTaskExecutionFailure::new("query_rejected", error.to_string(), "QueryFailed")
5263 })
5264 }
5265
5266 fn execute_workflow_task(&self, task: WorkflowTask) -> Result<Vec<Value>> {
5267 if let Some(update_id) = task
5268 .workflow_update_id
5269 .as_deref()
5270 .filter(|update_id| !update_id.is_empty())
5271 {
5272 return self.execute_update_task(&task, update_id);
5273 }
5274
5275 let workflow = self
5276 .workflows
5277 .get(&task.workflow_type)
5278 .ok_or_else(|| Error::WorkflowNotRegistered(task.workflow_type.clone()))?;
5279 let input = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
5280 let resume_signal = decode_resume_signal(&task)?;
5281 let history_budget = WorkflowHistoryBudget {
5282 event_count: task
5283 .total_history_events
5284 .unwrap_or_else(|| u64::try_from(task.history_events.len()).unwrap_or(u64::MAX)),
5285 size_bytes: task.history_size_bytes,
5286 continue_as_new_recommended: task.continue_as_new_recommended.unwrap_or(false),
5287 pressure: task.history_budget_pressure.clone(),
5288 };
5289 let mut workflow_state = WorkflowState::new_with_identity(
5290 task.history_events,
5291 task.workflow_id,
5292 task.run_id,
5293 self.task_queue.clone(),
5294 task.payload_codec.clone(),
5295 resume_signal,
5296 )?;
5297 workflow_state.history_budget = history_budget;
5298 let state = Arc::new(Mutex::new(workflow_state));
5299 let ctx = WorkflowContext { state };
5300 let mut future = (workflow.execute)(ctx.clone(), input);
5301 let mut cx = TaskContext::from_waker(noop_waker_ref());
5302
5303 match future.as_mut().poll(&mut cx) {
5304 Poll::Ready(Ok(result)) => {
5305 ctx.ensure_history_consumed()?;
5306 let result = encode_typed_envelope(&result, &task.payload_codec)?;
5307 let mut commands = ctx.take_commands()?;
5308 commands.push(json!({
5309 "type": "complete_workflow",
5310 "result": result
5311 }));
5312 Ok(commands)
5313 }
5314 Poll::Ready(Err(error)) => {
5315 if let Error::ContinueAsNew(request) = error {
5316 let mut commands = ctx.take_commands()?;
5317 if let Some(command) = ctx.continue_as_new_command(request)? {
5318 commands.push(command);
5319 }
5320 ctx.ensure_history_consumed()?;
5321 return Ok(commands);
5322 }
5323 ctx.ensure_history_consumed()?;
5326 if workflow_task_integrity_error(&error) {
5327 return Err(error);
5332 }
5333 let mut commands = ctx.take_commands()?;
5334 commands.push(workflow_failure_command(&error));
5335 Ok(commands)
5336 }
5337 Poll::Pending => {
5338 let commands = ctx.take_commands()?;
5339 if commands.is_empty() && !ctx.matched_recorded_pending()? {
5340 Err(Error::WorkflowYieldedWithoutCommand)
5341 } else {
5342 Ok(commands)
5343 }
5344 }
5345 }
5346 }
5347
5348 fn execute_update_task(&self, task: &WorkflowTask, update_id: &str) -> Result<Vec<Value>> {
5349 if !self.workflows.contains_key(&task.workflow_type) {
5350 return Err(Error::WorkflowNotRegistered(task.workflow_type.clone()));
5351 }
5352
5353 let accepted = task.history_events.iter().rev().find_map(|event| {
5354 (event.event_type == "UpdateAccepted"
5355 && event.payload.get("update_id").and_then(Value::as_str) == Some(update_id))
5356 .then_some(&event.payload)
5357 });
5358 let update_name = accepted
5359 .and_then(|payload| payload.get("update_name"))
5360 .and_then(Value::as_str)
5361 .or(task.update_name.as_deref())
5362 .unwrap_or_default();
5363 let Some(handler) = self
5364 .updates
5365 .get(&task.workflow_type)
5366 .and_then(|handlers| handlers.get(update_name))
5367 else {
5368 return Ok(vec![json!({
5369 "type": "fail_update",
5370 "update_id": update_id,
5371 "message": format!(
5372 "no update handler is registered for {}.{update_name}",
5373 task.workflow_type
5374 ),
5375 "exception_type": "UnknownUpdate",
5376 "non_retryable": true,
5377 })]);
5378 };
5379 let arguments = accepted
5380 .and_then(|payload| payload.get("arguments"))
5381 .or(task.arguments.as_ref());
5382 let arguments = decode_task_avro_arguments(arguments, &task.payload_codec)?;
5383 let context = QueryContext {
5384 workflow_id: task.workflow_id.clone(),
5385 run_id: task.run_id.clone(),
5386 workflow_type: task.workflow_type.clone(),
5387 run_status: Some("running".to_string()),
5388 workflow_input: Value::Null,
5389 workflow_input_avro_value: AvroValue::Null,
5390 history_events: Arc::new(task.history_events.clone()),
5391 signal_events: Arc::new(Vec::new()),
5392 };
5393 let mut future = handler(context, arguments);
5394 let mut cx = TaskContext::from_waker(noop_waker_ref());
5395
5396 match future.as_mut().poll(&mut cx) {
5397 Poll::Ready(Ok(result)) => Ok(vec![json!({
5398 "type": "complete_update",
5399 "update_id": update_id,
5400 "result": encode_typed_envelope(&result, &task.payload_codec)?,
5401 })]),
5402 Poll::Ready(Err(error)) => Ok(vec![json!({
5403 "type": "fail_update",
5404 "update_id": update_id,
5405 "message": error.to_string(),
5406 "exception_type": "UpdateFailed",
5407 "non_retryable": true,
5408 })]),
5409 Poll::Pending => Err(Error::WorkflowYieldedWithoutCommand),
5410 }
5411 }
5412
5413 async fn execute_activity_task(&self, task: ActivityTask) -> Result<AvroValue> {
5414 let handler = self
5415 .activities
5416 .get(&task.activity_type)
5417 .ok_or_else(|| Error::ActivityNotRegistered(task.activity_type.clone()))?;
5418 let args = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
5419 let attempt_id = task
5420 .activity_attempt_id
5421 .clone()
5422 .or(task.attempt_id.clone())
5423 .unwrap_or_default();
5424 let lease_owner = task
5425 .lease_owner
5426 .clone()
5427 .unwrap_or_else(|| self.worker_id.clone());
5428 let ctx = ActivityContext {
5429 client: self.client.clone(),
5430 task_id: task.task_id,
5431 activity_attempt_id: attempt_id,
5432 lease_owner,
5433 activity_type: task.activity_type,
5434 attempt_number: task.attempt_number,
5435 task_queue: self.task_queue.clone(),
5436 worker_id: self.worker_id.clone(),
5437 };
5438
5439 handler(ctx, args).await
5440 }
5441}
5442
5443fn poller_result(
5444 kind: &str,
5445 result: std::result::Result<Result<()>, tokio::task::JoinError>,
5446) -> Result<()> {
5447 match result {
5448 Ok(result) => result,
5449 Err(error) => Err(Error::WorkerLoop(format!(
5450 "{kind} poller join error: {error}"
5451 ))),
5452 }
5453}
5454
5455fn optional_poller_result(
5456 kind: &str,
5457 result: Option<std::result::Result<Result<()>, tokio::task::JoinError>>,
5458) -> Result<()> {
5459 match result {
5460 Some(result) => poller_result(kind, result),
5461 None => Ok(()),
5462 }
5463}
5464
5465async fn join_pollers(
5466 workflow_poller: Option<tokio::task::JoinHandle<Result<()>>>,
5467 activity_poller: Option<tokio::task::JoinHandle<Result<()>>>,
5468 query_poller: Option<tokio::task::JoinHandle<Result<()>>>,
5469) -> Result<()> {
5470 let mut first_error = None;
5471
5472 if let Some(handle) = workflow_poller {
5473 if let Err(error) = poller_result("workflow", handle.await) {
5474 first_error.get_or_insert(error);
5475 }
5476 }
5477
5478 if let Some(handle) = activity_poller {
5479 if let Err(error) = poller_result("activity", handle.await) {
5480 first_error.get_or_insert(error);
5481 }
5482 }
5483
5484 if let Some(handle) = query_poller {
5485 if let Err(error) = poller_result("query", handle.await) {
5486 first_error.get_or_insert(error);
5487 }
5488 }
5489
5490 if let Some(error) = first_error {
5491 Err(error)
5492 } else {
5493 Ok(())
5494 }
5495}
5496
5497fn default_worker_id() -> String {
5498 let millis = SystemTime::now()
5499 .duration_since(UNIX_EPOCH)
5500 .unwrap_or_default()
5501 .as_millis();
5502 format!("rust-worker-{}-{millis}", std::process::id())
5503}
5504
5505fn percent_encode_path_segment(segment: &str) -> String {
5506 const HEX: &[u8; 16] = b"0123456789ABCDEF";
5507 let mut encoded = String::with_capacity(segment.len());
5508
5509 for byte in segment.bytes() {
5510 if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') {
5511 encoded.push(char::from(byte));
5512 } else {
5513 encoded.push('%');
5514 encoded.push(char::from(HEX[(byte >> 4) as usize]));
5515 encoded.push(char::from(HEX[(byte & 0x0f) as usize]));
5516 }
5517 }
5518
5519 encoded
5520}
5521
5522fn unique_request_id(prefix: &str) -> String {
5523 let nanos = SystemTime::now()
5524 .duration_since(UNIX_EPOCH)
5525 .unwrap_or_default()
5526 .as_nanos();
5527 format!("{prefix}-{}-{nanos}", std::process::id())
5528}
5529
5530#[derive(Debug)]
5531struct QueryTaskExecutionFailure {
5532 reason: String,
5533 message: String,
5534 failure_type: String,
5535}
5536
5537impl QueryTaskExecutionFailure {
5538 fn new(
5539 reason: impl Into<String>,
5540 message: impl Into<String>,
5541 failure_type: impl Into<String>,
5542 ) -> Self {
5543 Self {
5544 reason: reason.into(),
5545 message: message.into(),
5546 failure_type: failure_type.into(),
5547 }
5548 }
5549}
5550
5551#[derive(Clone, Debug)]
5557pub struct WorkflowInstance<S> {
5558 state: Arc<Mutex<S>>,
5559}
5560
5561impl<S> WorkflowInstance<S> {
5562 fn new(state: S) -> Self {
5563 Self {
5564 state: Arc::new(Mutex::new(state)),
5565 }
5566 }
5567
5568 pub fn read<R>(&self, reader: impl FnOnce(&S) -> R) -> Result<R> {
5570 let state = self
5571 .state
5572 .lock()
5573 .map_err(|_| Error::WorkflowStatePoisoned)?;
5574 Ok(reader(&state))
5575 }
5576
5577 pub fn update<R>(&self, transition: impl FnOnce(&mut S) -> R) -> Result<R> {
5579 let mut state = self
5580 .state
5581 .lock()
5582 .map_err(|_| Error::WorkflowStatePoisoned)?;
5583 Ok(transition(&mut state))
5584 }
5585}
5586
5587impl<S: Clone> WorkflowInstance<S> {
5588 fn snapshot(&self) -> Result<S> {
5589 self.read(Clone::clone)
5590 }
5591}
5592
5593#[derive(Clone, Debug)]
5594pub struct WorkflowContext {
5595 state: Arc<Mutex<WorkflowState>>,
5596}
5597
5598impl WorkflowContext {
5599 pub fn workflow_identity(&self) -> Result<WorkflowIdentity> {
5601 let state = self
5602 .state
5603 .lock()
5604 .map_err(|_| Error::WorkflowStatePoisoned)?;
5605 Ok(WorkflowIdentity {
5606 workflow_id: state.workflow_id.clone(),
5607 run_id: state.run_id.clone(),
5608 })
5609 }
5610
5611 pub fn history_budget(&self) -> Result<WorkflowHistoryBudget> {
5613 let state = self
5614 .state
5615 .lock()
5616 .map_err(|_| Error::WorkflowStatePoisoned)?;
5617 Ok(state.history_budget.clone())
5618 }
5619
5620 pub fn continue_as_new<T: Serialize>(&self, args: T) -> Result<Value> {
5626 self.continue_as_new_with_options(ContinueAsNewOptions::new(), args)
5627 }
5628
5629 pub fn continue_as_new_with_options<T: Serialize>(
5631 &self,
5632 options: ContinueAsNewOptions,
5633 args: T,
5634 ) -> Result<Value> {
5635 options.validate()?;
5636 Err(Error::ContinueAsNew(ContinueAsNewRequest {
5637 arguments: normalize_avro_arguments(AvroValue::from_serialize(&args)?),
5638 options,
5639 }))
5640 }
5641
5642 pub fn activity<T: Serialize>(
5643 &self,
5644 activity_type: impl Into<String>,
5645 args: T,
5646 ) -> ActivityCall {
5647 self.activity_with_options(activity_type, ActivityOptions::new(), args)
5648 }
5649
5650 pub fn activity_on_queue<T, Q>(
5651 &self,
5652 activity_type: impl Into<String>,
5653 task_queue: Option<Q>,
5654 args: T,
5655 ) -> ActivityCall
5656 where
5657 T: Serialize,
5658 Q: Into<String>,
5659 {
5660 let mut options = ActivityOptions::new();
5661 options.task_queue = task_queue.map(Into::into);
5662 self.activity_with_options(activity_type, options, args)
5663 }
5664
5665 pub fn activity_with_options<T: Serialize>(
5703 &self,
5704 activity_type: impl Into<String>,
5705 options: ActivityOptions,
5706 args: T,
5707 ) -> ActivityCall {
5708 ActivityCall {
5709 ctx: self.clone(),
5710 activity_type: activity_type.into(),
5711 options,
5712 args: Some(AvroValue::from_serialize(&args)),
5713 scheduled: false,
5714 }
5715 }
5716
5717 pub async fn activity_avro_value<T: Serialize>(
5718 &self,
5719 activity_type: impl Into<String>,
5720 args: T,
5721 ) -> Result<AvroValue> {
5722 let mut call = self.activity(activity_type, args);
5723 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
5724 }
5725
5726 pub async fn activity_avro_value_with_options<T: Serialize>(
5727 &self,
5728 activity_type: impl Into<String>,
5729 options: ActivityOptions,
5730 args: T,
5731 ) -> Result<AvroValue> {
5732 let mut call = self.activity_with_options(activity_type, options, args);
5733 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
5734 }
5735
5736 pub fn wait_signal(&self, signal_name: impl Into<String>) -> SignalCall {
5737 SignalCall {
5738 ctx: self.clone(),
5739 signal_name: signal_name.into(),
5740 opened_wait: false,
5741 matched_pending: false,
5742 }
5743 }
5744
5745 pub async fn wait_signal_avro_value(
5746 &self,
5747 signal_name: impl Into<String>,
5748 ) -> Result<Vec<AvroValue>> {
5749 let mut call = self.wait_signal(signal_name);
5750 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
5751 }
5752
5753 pub fn sleep(&self, duration: Duration) -> TimerCall {
5774 let delay_seconds = duration
5775 .as_secs()
5776 .checked_add(u64::from(duration.subsec_nanos() > 0));
5777 TimerCall {
5778 ctx: self.clone(),
5779 delay_seconds,
5780 scheduled: false,
5781 matched_pending: false,
5782 }
5783 }
5784
5785 pub fn start_timer(&self, duration: Duration) -> TimerCall {
5787 self.sleep(duration)
5788 }
5789
5790 pub fn side_effect<T, F>(&self, callback: F) -> Result<T>
5797 where
5798 T: Serialize + DeserializeOwned,
5799 F: FnOnce() -> T,
5800 {
5801 {
5802 let mut state = self
5803 .state
5804 .lock()
5805 .map_err(|_| Error::WorkflowStatePoisoned)?;
5806 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
5807 return match recorded {
5808 RecordedCommand::SideEffect { sequence, value } => {
5809 state.command_cursor += 1;
5810 value.deserialize().map_err(|error| {
5811 Error::NonDeterministicReplay(ReplayFailure::new(
5812 "side_effect_type_mismatch",
5813 Some(sequence),
5814 Some(std::any::type_name::<T>().to_string()),
5815 Some(error.to_string()),
5816 "recorded side-effect value is incompatible with the requested Rust type",
5817 ))
5818 })
5819 }
5820 other => Err(command_mismatch(&other, "side effect")),
5821 };
5822 }
5823 }
5824
5825 let value = callback();
5826 let avro_value = AvroValue::from_serialize(&value)?;
5827 let mut state = self
5828 .state
5829 .lock()
5830 .map_err(|_| Error::WorkflowStatePoisoned)?;
5831 let result = encode_typed_envelope(&avro_value, &state.payload_codec)?;
5832 state.commands.push(json!({
5833 "type": "record_side_effect",
5834 "result": result,
5835 }));
5836 Ok(value)
5837 }
5838
5839 pub fn side_effect_avro_value<F>(&self, callback: F) -> Result<AvroValue>
5841 where
5842 F: FnOnce() -> AvroValue,
5843 {
5844 {
5845 let mut state = self
5846 .state
5847 .lock()
5848 .map_err(|_| Error::WorkflowStatePoisoned)?;
5849 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
5850 return match recorded {
5851 RecordedCommand::SideEffect { value, .. } => {
5852 state.command_cursor += 1;
5853 Ok(value)
5854 }
5855 other => Err(command_mismatch(&other, "side effect")),
5856 };
5857 }
5858 }
5859
5860 let value = callback();
5861 let mut state = self
5862 .state
5863 .lock()
5864 .map_err(|_| Error::WorkflowStatePoisoned)?;
5865 let result = encode_typed_envelope(&value, &state.payload_codec)?;
5866 state.commands.push(json!({
5867 "type": "record_side_effect",
5868 "result": result,
5869 }));
5870 Ok(value)
5871 }
5872
5873 pub fn uuid_v4(&self) -> Result<Uuid> {
5875 self.side_effect(Uuid::new_v4)
5876 }
5877
5878 pub fn get_version(
5881 &self,
5882 change_id: impl Into<String>,
5883 min_supported: i32,
5884 max_supported: i32,
5885 ) -> Result<i32> {
5886 let change_id = change_id.into();
5887 if change_id.trim().is_empty() {
5888 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5889 "version_change_id_invalid",
5890 None,
5891 Some("non-empty change ID".to_string()),
5892 Some(change_id),
5893 "version markers require a stable non-empty change ID",
5894 )));
5895 }
5896 if min_supported > max_supported {
5897 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5898 "version_range_invalid",
5899 None,
5900 Some("min_supported <= max_supported".to_string()),
5901 Some(format!("{min_supported}..={max_supported}")),
5902 "version marker supported range is invalid",
5903 )));
5904 }
5905
5906 let mut state = self
5907 .state
5908 .lock()
5909 .map_err(|_| Error::WorkflowStatePoisoned)?;
5910 if let Some((version, sequence)) = state.version_markers.get(&change_id).copied() {
5911 ensure_version_supported(&change_id, version, min_supported, max_supported, sequence)?;
5912 return Ok(version);
5913 }
5914
5915 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
5916 return match recorded {
5917 RecordedCommand::VersionMarker {
5918 sequence,
5919 change_id: recorded_change_id,
5920 version,
5921 ..
5922 } => {
5923 if recorded_change_id != change_id {
5924 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5925 "version_change_id_mismatch",
5926 Some(sequence),
5927 Some(recorded_change_id),
5928 Some(change_id),
5929 "recorded version marker change ID differs from current workflow code",
5930 )));
5931 }
5932 ensure_version_supported(
5933 &change_id,
5934 version,
5935 min_supported,
5936 max_supported,
5937 sequence,
5938 )?;
5939 state.command_cursor += 1;
5940 state.version_markers.insert(change_id, (version, sequence));
5941 Ok(version)
5942 }
5943 other => Err(command_mismatch(
5944 &other,
5945 format!("version marker:{change_id}"),
5946 )),
5947 };
5948 }
5949
5950 let version = max_supported;
5951 state.commands.push(json!({
5952 "type": "record_version_marker",
5953 "change_id": change_id,
5954 "version": version,
5955 "min_supported": min_supported,
5956 "max_supported": max_supported,
5957 }));
5958 state.version_markers.insert(change_id, (version, 0));
5961 Ok(version)
5962 }
5963
5964 pub fn patched(&self, change_id: impl Into<String>) -> Result<bool> {
5966 Ok(self.get_version(change_id, -1, 1)? == 1)
5967 }
5968
5969 pub fn deprecate_patch(&self, change_id: impl Into<String>) -> Result<()> {
5971 self.get_version(change_id, -1, 1).map(|_| ())
5972 }
5973
5974 pub fn start_child_workflow<T: Serialize>(
6001 &self,
6002 workflow_type: impl Into<String>,
6003 options: ChildWorkflowOptions,
6004 args: T,
6005 ) -> ChildWorkflowCall {
6006 ChildWorkflowCall {
6007 ctx: self.clone(),
6008 workflow_type: workflow_type.into(),
6009 options,
6010 args: Some(AvroValue::from_serialize(&args)),
6011 scheduled: false,
6012 matched_pending: false,
6013 }
6014 }
6015
6016 pub async fn start_child_workflow_avro_value<T: Serialize>(
6017 &self,
6018 workflow_type: impl Into<String>,
6019 options: ChildWorkflowOptions,
6020 args: T,
6021 ) -> Result<ChildWorkflowAvroResult> {
6022 let mut call = self.start_child_workflow(workflow_type, options, args);
6023 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
6024 }
6025
6026 fn take_commands(&self) -> Result<Vec<Value>> {
6027 let mut state = self
6028 .state
6029 .lock()
6030 .map_err(|_| Error::WorkflowStatePoisoned)?;
6031 Ok(std::mem::take(&mut state.commands))
6032 }
6033
6034 fn continue_as_new_command(&self, request: ContinueAsNewRequest) -> Result<Option<Value>> {
6035 let mut state = self
6036 .state
6037 .lock()
6038 .map_err(|_| Error::WorkflowStatePoisoned)?;
6039
6040 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6041 return Err(command_mismatch(&recorded, "continue as new"));
6042 }
6043 if state.recorded_continue_as_new_sequence.is_some() {
6044 state.continue_as_new_consumed = true;
6045 return Ok(None);
6046 }
6047
6048 let arguments = encode_typed_envelope(&request.arguments, &state.payload_codec)?;
6049 let mut command = serde_json::Map::from_iter([
6050 ("type".to_string(), json!("continue_as_new")),
6051 ("arguments".to_string(), arguments),
6052 ("queue".to_string(), json!(state.task_queue.clone())),
6053 ]);
6054 if let Some(workflow_type) = request.options.workflow_type {
6055 command.insert("workflow_type".to_string(), json!(workflow_type));
6056 }
6057 if let Some(task_queue) = request.options.task_queue {
6058 command.insert("queue".to_string(), json!(task_queue));
6059 }
6060 Ok(Some(Value::Object(command)))
6061 }
6062
6063 fn matched_recorded_pending(&self) -> Result<bool> {
6064 let state = self
6065 .state
6066 .lock()
6067 .map_err(|_| Error::WorkflowStatePoisoned)?;
6068 Ok(state.matched_recorded_pending)
6069 }
6070
6071 fn ensure_history_consumed(&self) -> Result<()> {
6072 let state = self
6073 .state
6074 .lock()
6075 .map_err(|_| Error::WorkflowStatePoisoned)?;
6076 if let Some(command) = state.recorded_commands.get(state.command_cursor) {
6077 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
6078 "recorded_commands_unconsumed",
6079 Some(command.sequence()),
6080 Some(command.shape().to_string()),
6081 Some("workflow completion".to_string()),
6082 "workflow completed before consuming all recorded durable commands",
6083 )));
6084 }
6085 if let Some(sequence) = state
6086 .recorded_continue_as_new_sequence
6087 .filter(|_| !state.continue_as_new_consumed)
6088 {
6089 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
6090 "recorded_continue_as_new_unconsumed",
6091 Some(sequence),
6092 Some("continue as new".to_string()),
6093 Some("workflow completion".to_string()),
6094 "workflow completed without consuming its recorded continue-as-new transition",
6095 )));
6096 }
6097 Ok(())
6098 }
6099}
6100
6101#[derive(Debug)]
6102struct WorkflowState {
6103 workflow_id: Option<String>,
6104 run_id: Option<String>,
6105 task_queue: String,
6106 payload_codec: String,
6107 history_budget: WorkflowHistoryBudget,
6108 resume_signal: Option<ResumeSignal>,
6109 recorded_commands: Vec<RecordedCommand>,
6110 recorded_continue_as_new_sequence: Option<u64>,
6111 continue_as_new_consumed: bool,
6112 command_cursor: usize,
6113 matched_recorded_pending: bool,
6114 version_markers: HashMap<String, (i32, u64)>,
6115 commands: Vec<Value>,
6116}
6117
6118impl WorkflowState {
6119 #[cfg(test)]
6120 fn new(
6121 history: Vec<HistoryEvent>,
6122 task_queue: String,
6123 payload_codec: String,
6124 resume_signal: Option<ResumeSignal>,
6125 ) -> Result<Self> {
6126 Self::new_with_identity(
6127 history,
6128 None,
6129 None,
6130 task_queue,
6131 payload_codec,
6132 resume_signal,
6133 )
6134 }
6135
6136 fn new_with_identity(
6137 history: Vec<HistoryEvent>,
6138 workflow_id: Option<String>,
6139 run_id: Option<String>,
6140 task_queue: String,
6141 payload_codec: String,
6142 resume_signal: Option<ResumeSignal>,
6143 ) -> Result<Self> {
6144 let recorded_commands = recorded_commands(
6145 &history,
6146 &payload_codec,
6147 WorkflowIdentity {
6148 workflow_id: workflow_id.clone(),
6149 run_id: run_id.clone(),
6150 },
6151 )?;
6152 let recorded_continue_as_new = history
6153 .iter()
6154 .filter(|event| event.event_type == "WorkflowContinuedAsNew")
6155 .collect::<Vec<_>>();
6156 if recorded_continue_as_new.len() > 1 {
6157 return Err(invalid_recorded_history(
6158 "duplicate_continue_as_new_transition",
6159 recorded_continue_as_new
6160 .last()
6161 .and_then(|event| durable_event_sequence(event))
6162 .unwrap_or(0),
6163 "one WorkflowContinuedAsNew event",
6164 &format!(
6165 "{} WorkflowContinuedAsNew events",
6166 recorded_continue_as_new.len()
6167 ),
6168 "workflow history records one continue-as-new transition more than once",
6169 ));
6170 }
6171 let recorded_continue_as_new_sequence = recorded_continue_as_new
6172 .first()
6173 .map(|event| {
6174 durable_event_sequence(event).ok_or_else(|| {
6175 Error::NonDeterministicReplay(ReplayFailure::new(
6176 "continue_as_new_sequence_missing",
6177 None,
6178 Some("recorded transition sequence".to_string()),
6179 Some("missing sequence".to_string()),
6180 "WorkflowContinuedAsNew history is missing its recorded sequence",
6181 ))
6182 })
6183 })
6184 .transpose()?;
6185 let event_count = u64::try_from(history.len()).unwrap_or(u64::MAX);
6186 Ok(Self {
6187 workflow_id,
6188 run_id,
6189 task_queue,
6190 payload_codec,
6191 history_budget: WorkflowHistoryBudget {
6192 event_count,
6193 ..WorkflowHistoryBudget::default()
6194 },
6195 resume_signal,
6196 recorded_commands,
6197 recorded_continue_as_new_sequence,
6198 continue_as_new_consumed: false,
6199 command_cursor: 0,
6200 matched_recorded_pending: false,
6201 version_markers: HashMap::new(),
6202 commands: Vec::new(),
6203 })
6204 }
6205}
6206
6207#[derive(Clone, Debug)]
6208enum RecordedCommand {
6209 Activity {
6210 sequence: u64,
6211 activity_type: Option<String>,
6212 options: Option<RecordedActivityOptions>,
6213 outcome: Option<ActivityOutcome>,
6214 },
6215 Timer {
6216 sequence: u64,
6217 delay_seconds: u64,
6218 fired: bool,
6219 },
6220 ChildWorkflow {
6221 sequence: u64,
6222 workflow_type: Option<String>,
6223 outcome: Option<ChildWorkflowOutcome>,
6224 },
6225 SignalWait {
6226 sequence: u64,
6227 signal_name: String,
6228 value: Option<Vec<AvroValue>>,
6229 },
6230 SideEffect {
6231 sequence: u64,
6232 value: AvroValue,
6233 },
6234 VersionMarker {
6235 sequence: u64,
6236 change_id: String,
6237 version: i32,
6238 },
6239}
6240
6241#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
6242struct RecordedActivityOptions {
6243 task_queue: RecordedSnapshotValue<Option<String>>,
6244 execution_mode: RecordedSnapshotValue<Option<String>>,
6245 retry_policy: ActivityRetrySnapshot,
6246}
6247
6248#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
6249enum RecordedSnapshotValue<T> {
6250 Unknown,
6252 Known(T),
6253}
6254
6255impl<T: PartialEq> RecordedSnapshotValue<T> {
6256 fn matches_current(&self, current: &Self) -> bool {
6257 match self {
6258 Self::Unknown => true,
6259 Self::Known(recorded) => matches!(current, Self::Known(value) if value == recorded),
6260 }
6261 }
6262}
6263
6264#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
6265struct ActivityRetrySnapshot {
6266 snapshot_version: RecordedSnapshotValue<Option<u64>>,
6267 max_attempts: RecordedSnapshotValue<Option<u64>>,
6268 backoff_seconds: RecordedSnapshotValue<Vec<u64>>,
6269 start_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
6270 schedule_to_start_timeout: RecordedSnapshotValue<Option<u64>>,
6271 schedule_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
6272 heartbeat_timeout: RecordedSnapshotValue<Option<u64>>,
6273 non_retryable_error_types: RecordedSnapshotValue<Vec<String>>,
6274}
6275
6276impl ActivityRetrySnapshot {
6277 fn matches_current(&self, current: &Self) -> bool {
6278 self.snapshot_version
6279 .matches_current(¤t.snapshot_version)
6280 && self.max_attempts.matches_current(¤t.max_attempts)
6281 && self
6282 .backoff_seconds
6283 .matches_current(¤t.backoff_seconds)
6284 && self
6285 .start_to_close_timeout
6286 .matches_current(¤t.start_to_close_timeout)
6287 && self
6288 .schedule_to_start_timeout
6289 .matches_current(¤t.schedule_to_start_timeout)
6290 && self
6291 .schedule_to_close_timeout
6292 .matches_current(¤t.schedule_to_close_timeout)
6293 && self
6294 .heartbeat_timeout
6295 .matches_current(¤t.heartbeat_timeout)
6296 && self
6297 .non_retryable_error_types
6298 .matches_current(¤t.non_retryable_error_types)
6299 }
6300}
6301
6302fn recorded_optional_u64(
6303 object: Option<&serde_json::Map<String, Value>>,
6304 field: &str,
6305) -> RecordedSnapshotValue<Option<u64>> {
6306 match object.and_then(|object| object.get(field)) {
6307 None => RecordedSnapshotValue::Unknown,
6308 Some(Value::Null) => RecordedSnapshotValue::Known(None),
6309 Some(value) => RecordedSnapshotValue::Known(value_as_u64(value)),
6310 }
6311}
6312
6313fn recorded_optional_string(
6314 object: &serde_json::Map<String, Value>,
6315 field: &str,
6316) -> RecordedSnapshotValue<Option<String>> {
6317 match object.get(field) {
6318 None => RecordedSnapshotValue::Unknown,
6319 Some(Value::Null) => RecordedSnapshotValue::Known(None),
6320 Some(value) => RecordedSnapshotValue::Known(value.as_str().map(str::to_string)),
6321 }
6322}
6323
6324fn recorded_activity_retry_snapshot(policy: Option<&Value>) -> ActivityRetrySnapshot {
6325 let policy = policy.and_then(Value::as_object);
6326 let backoff_seconds = policy
6327 .and_then(|policy| policy.get("backoff_seconds"))
6328 .and_then(Value::as_array)
6329 .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
6330 .map_or(RecordedSnapshotValue::Unknown, RecordedSnapshotValue::Known);
6331 let mut non_retryable_error_types = Vec::new();
6332 for error_type in policy
6333 .and_then(|policy| policy.get("non_retryable_error_types"))
6334 .and_then(Value::as_array)
6335 .into_iter()
6336 .flatten()
6337 .filter_map(Value::as_str)
6338 .map(str::trim)
6339 .filter(|error_type| !error_type.is_empty())
6340 {
6341 if !non_retryable_error_types
6342 .iter()
6343 .any(|recorded| recorded == error_type)
6344 {
6345 non_retryable_error_types.push(error_type.to_string());
6346 }
6347 }
6348
6349 ActivityRetrySnapshot {
6350 snapshot_version: recorded_optional_u64(policy, "snapshot_version"),
6351 max_attempts: recorded_optional_u64(policy, "max_attempts"),
6352 backoff_seconds,
6353 start_to_close_timeout: recorded_optional_u64(policy, "start_to_close_timeout"),
6354 schedule_to_start_timeout: recorded_optional_u64(policy, "schedule_to_start_timeout"),
6355 schedule_to_close_timeout: recorded_optional_u64(policy, "schedule_to_close_timeout"),
6356 heartbeat_timeout: recorded_optional_u64(policy, "heartbeat_timeout"),
6357 non_retryable_error_types: if policy
6358 .is_some_and(|policy| policy.contains_key("non_retryable_error_types"))
6359 {
6360 RecordedSnapshotValue::Known(non_retryable_error_types)
6361 } else {
6362 RecordedSnapshotValue::Unknown
6363 },
6364 }
6365}
6366
6367fn current_activity_retry_snapshot(options: &ValidatedActivityOptions) -> ActivityRetrySnapshot {
6368 let policy = options.retry_policy.as_ref();
6369 let max_attempts = match policy.and_then(|policy| policy.get("max_attempts")) {
6370 Some(Value::Null) => None,
6371 Some(value) => value_as_u64(value),
6372 None => Some(1),
6373 };
6374 let backoff_seconds = policy
6375 .and_then(|policy| policy.get("backoff_seconds"))
6376 .and_then(Value::as_array)
6377 .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
6378 .unwrap_or_default();
6379 let non_retryable_error_types = policy
6380 .and_then(|policy| policy.get("non_retryable_error_types"))
6381 .and_then(Value::as_array)
6382 .into_iter()
6383 .flatten()
6384 .filter_map(Value::as_str)
6385 .map(str::to_string)
6386 .collect();
6387
6388 ActivityRetrySnapshot {
6389 snapshot_version: RecordedSnapshotValue::Known(Some(1)),
6390 max_attempts: RecordedSnapshotValue::Known(max_attempts),
6391 backoff_seconds: RecordedSnapshotValue::Known(backoff_seconds),
6392 start_to_close_timeout: RecordedSnapshotValue::Known(options.start_to_close_timeout),
6393 schedule_to_start_timeout: RecordedSnapshotValue::Known(options.schedule_to_start_timeout),
6394 schedule_to_close_timeout: RecordedSnapshotValue::Known(options.schedule_to_close_timeout),
6395 heartbeat_timeout: RecordedSnapshotValue::Known(options.heartbeat_timeout),
6396 non_retryable_error_types: RecordedSnapshotValue::Known(non_retryable_error_types),
6397 }
6398}
6399
6400fn activity_options_description(options: &RecordedActivityOptions) -> String {
6401 serde_json::to_string(options).unwrap_or_else(|_| format!("{options:?}"))
6402}
6403
6404impl RecordedCommand {
6405 fn sequence(&self) -> u64 {
6406 match self {
6407 Self::Activity { sequence, .. }
6408 | Self::Timer { sequence, .. }
6409 | Self::ChildWorkflow { sequence, .. }
6410 | Self::SignalWait { sequence, .. }
6411 | Self::SideEffect { sequence, .. }
6412 | Self::VersionMarker { sequence, .. } => *sequence,
6413 }
6414 }
6415
6416 fn shape(&self) -> &'static str {
6417 match self {
6418 Self::Activity { .. } => "activity",
6419 Self::Timer { .. } => "timer",
6420 Self::ChildWorkflow { .. } => "child workflow",
6421 Self::SignalWait { .. } => "signal wait",
6422 Self::SideEffect { .. } => "side effect",
6423 Self::VersionMarker { .. } => "version marker",
6424 }
6425 }
6426}
6427
6428fn ensure_version_supported(
6429 change_id: &str,
6430 version: i32,
6431 min_supported: i32,
6432 max_supported: i32,
6433 sequence: u64,
6434) -> Result<()> {
6435 if (min_supported..=max_supported).contains(&version) {
6436 return Ok(());
6437 }
6438 Err(Error::NonDeterministicReplay(ReplayFailure::new(
6439 "version_marker_incompatible_range",
6440 (sequence != 0).then_some(sequence),
6441 Some(format!("{min_supported}..={max_supported}")),
6442 Some(format!("{change_id}:{version}")),
6443 "recorded workflow version is outside the range supported by current code",
6444 )))
6445}
6446
6447#[derive(Clone, Debug)]
6448struct ResumeSignal {
6449 signal_name: String,
6450 arguments: Vec<AvroValue>,
6451}
6452
6453pub struct ActivityCall {
6454 ctx: WorkflowContext,
6455 activity_type: String,
6456 options: ActivityOptions,
6457 args: Option<Result<AvroValue>>,
6458 scheduled: bool,
6459}
6460
6461impl ActivityCall {
6462 fn poll_avro_value(
6463 mut self: Pin<&mut Self>,
6464 _cx: &mut TaskContext<'_>,
6465 ) -> Poll<Result<AvroValue>> {
6466 let ctx = self.ctx.clone();
6467 let mut state = match ctx.state.lock() {
6468 Ok(state) => state,
6469 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6470 };
6471
6472 if self.scheduled {
6473 return Poll::Pending;
6474 }
6475
6476 let options = match self.options.validate() {
6477 Ok(options) => options,
6478 Err(error) => {
6479 return Poll::Ready(Err(Error::InvalidActivityOptions(error)));
6480 }
6481 };
6482 let task_queue = options
6483 .task_queue
6484 .clone()
6485 .unwrap_or_else(|| state.task_queue.clone());
6486 let current_recorded_options = RecordedActivityOptions {
6487 task_queue: RecordedSnapshotValue::Known(Some(task_queue.clone())),
6488 execution_mode: RecordedSnapshotValue::Known(None),
6491 retry_policy: current_activity_retry_snapshot(&options),
6492 };
6493
6494 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6495 let sequence = recorded.sequence();
6496 match recorded {
6497 RecordedCommand::Activity {
6498 activity_type,
6499 options: recorded_options,
6500 outcome,
6501 ..
6502 } => {
6503 if let Some(recorded_type) = activity_type {
6504 if recorded_type != self.activity_type {
6505 return Poll::Ready(Err(Error::NonDeterministicReplay(
6506 ReplayFailure::new(
6507 "recorded_command_detail_mismatch",
6508 Some(sequence),
6509 Some(format!("activity:{recorded_type}")),
6510 Some(format!("activity:{}", self.activity_type)),
6511 "recorded activity type differs from the current workflow command",
6512 ),
6513 )));
6514 }
6515 }
6516 if let Some(recorded_options) = recorded_options {
6517 if !recorded_options
6518 .task_queue
6519 .matches_current(¤t_recorded_options.task_queue)
6520 {
6521 return Poll::Ready(Err(Error::NonDeterministicReplay(
6522 ReplayFailure::new(
6523 "activity_task_queue_mismatch",
6524 Some(sequence),
6525 Some(activity_options_description(&recorded_options)),
6526 Some(activity_options_description(¤t_recorded_options)),
6527 "recorded activity task queue differs from the current workflow command",
6528 ),
6529 )));
6530 }
6531 if !recorded_options
6532 .execution_mode
6533 .matches_current(¤t_recorded_options.execution_mode)
6534 {
6535 return Poll::Ready(Err(Error::NonDeterministicReplay(
6536 ReplayFailure::new(
6537 "activity_execution_mode_mismatch",
6538 Some(sequence),
6539 Some(activity_options_description(&recorded_options)),
6540 Some(activity_options_description(¤t_recorded_options)),
6541 "recorded activity execution mode differs from the current workflow command",
6542 ),
6543 )));
6544 }
6545 if !recorded_options
6546 .retry_policy
6547 .matches_current(¤t_recorded_options.retry_policy)
6548 {
6549 return Poll::Ready(Err(Error::NonDeterministicReplay(
6550 ReplayFailure::new(
6551 "activity_retry_policy_mismatch",
6552 Some(sequence),
6553 Some(activity_options_description(&recorded_options)),
6554 Some(activity_options_description(¤t_recorded_options)),
6555 "recorded activity retry policy differs from the current workflow command",
6556 ),
6557 )));
6558 }
6559 }
6560 state.command_cursor += 1;
6561 if let Some(outcome) = outcome {
6562 return Poll::Ready(outcome.map_err(Error::ActivityFailed));
6563 }
6564 state.matched_recorded_pending = true;
6565 self.scheduled = true;
6566 return Poll::Pending;
6567 }
6568 other => {
6569 return Poll::Ready(Err(command_mismatch(
6570 &other,
6571 format!("activity:{}", self.activity_type),
6572 )));
6573 }
6574 }
6575 }
6576
6577 if !self.scheduled {
6578 let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
6579 Ok(args) => args,
6580 Err(error) => return Poll::Ready(Err(error)),
6581 };
6582 let arguments = normalize_avro_arguments(args);
6583 let envelope = match encode_typed_envelope(&arguments, &state.payload_codec) {
6584 Ok(envelope) => envelope,
6585 Err(error) => return Poll::Ready(Err(error)),
6586 };
6587
6588 let mut command = serde_json::Map::from_iter([
6589 ("type".to_string(), json!("schedule_activity")),
6590 (
6591 "activity_type".to_string(),
6592 json!(self.activity_type.clone()),
6593 ),
6594 ("queue".to_string(), json!(task_queue)),
6595 ("arguments".to_string(), envelope),
6596 ]);
6597 for (field, value) in [
6598 ("start_to_close_timeout", options.start_to_close_timeout),
6599 (
6600 "schedule_to_start_timeout",
6601 options.schedule_to_start_timeout,
6602 ),
6603 (
6604 "schedule_to_close_timeout",
6605 options.schedule_to_close_timeout,
6606 ),
6607 ("heartbeat_timeout", options.heartbeat_timeout),
6608 ] {
6609 if let Some(value) = value {
6610 command.insert(field.to_string(), json!(value));
6611 }
6612 }
6613 if let Some(retry_policy) = options.retry_policy {
6614 command.insert("retry_policy".to_string(), retry_policy);
6615 }
6616 state.commands.push(Value::Object(command));
6617 self.scheduled = true;
6618 }
6619
6620 Poll::Pending
6621 }
6622}
6623
6624impl Future for ActivityCall {
6625 type Output = Result<Value>;
6626
6627 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6628 match self.poll_avro_value(cx) {
6629 Poll::Ready(Ok(value)) => Poll::Ready(value.into_json()),
6630 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
6631 Poll::Pending => Poll::Pending,
6632 }
6633 }
6634}
6635
6636pub struct TimerCall {
6638 ctx: WorkflowContext,
6639 delay_seconds: Option<u64>,
6640 scheduled: bool,
6641 matched_pending: bool,
6642}
6643
6644impl Future for TimerCall {
6645 type Output = Result<()>;
6646
6647 fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6648 if self.matched_pending {
6649 return Poll::Pending;
6650 }
6651
6652 let ctx = self.ctx.clone();
6653 let Some(requested_delay) = self.delay_seconds else {
6654 return Poll::Ready(Err(Error::TimerDurationOverflow));
6655 };
6656 let mut state = match ctx.state.lock() {
6657 Ok(state) => state,
6658 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6659 };
6660
6661 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6662 match recorded {
6663 RecordedCommand::Timer {
6664 sequence,
6665 delay_seconds,
6666 fired,
6667 ..
6668 } => {
6669 if delay_seconds != requested_delay {
6670 return Poll::Ready(Err(Error::NonDeterministicReplay(
6671 ReplayFailure::new(
6672 "timer_delay_mismatch",
6673 Some(sequence),
6674 Some(format!("timer:{delay_seconds}s")),
6675 Some(format!("timer:{requested_delay}s")),
6676 "recorded timer delay differs from the current workflow command",
6677 ),
6678 )));
6679 }
6680 state.command_cursor += 1;
6681 if fired {
6682 return Poll::Ready(Ok(()));
6683 }
6684 state.matched_recorded_pending = true;
6685 self.scheduled = true;
6686 self.matched_pending = true;
6687 return Poll::Pending;
6688 }
6689 other => return Poll::Ready(Err(command_mismatch(&other, "timer"))),
6690 }
6691 }
6692
6693 if !self.scheduled {
6694 state.commands.push(json!({
6695 "type": "start_timer",
6696 "delay_seconds": requested_delay,
6697 }));
6698 self.scheduled = true;
6699 }
6700
6701 Poll::Pending
6702 }
6703}
6704
6705pub struct ChildWorkflowCall {
6707 ctx: WorkflowContext,
6708 workflow_type: String,
6709 options: ChildWorkflowOptions,
6710 args: Option<Result<AvroValue>>,
6711 scheduled: bool,
6712 matched_pending: bool,
6713}
6714
6715impl ChildWorkflowCall {
6716 fn poll_avro_value(
6717 mut self: Pin<&mut Self>,
6718 _cx: &mut TaskContext<'_>,
6719 ) -> Poll<Result<ChildWorkflowAvroResult>> {
6720 if self.matched_pending {
6721 return Poll::Pending;
6722 }
6723
6724 let ctx = self.ctx.clone();
6725 let mut state = match ctx.state.lock() {
6726 Ok(state) => state,
6727 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6728 };
6729
6730 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6731 let sequence = recorded.sequence();
6732 match recorded {
6733 RecordedCommand::ChildWorkflow {
6734 workflow_type,
6735 outcome,
6736 ..
6737 } => {
6738 if let Some(recorded_type) = workflow_type {
6739 if recorded_type != self.workflow_type {
6740 return Poll::Ready(Err(Error::NonDeterministicReplay(
6741 ReplayFailure::new(
6742 "recorded_command_detail_mismatch",
6743 Some(sequence),
6744 Some(format!("child workflow:{recorded_type}")),
6745 Some(format!("child workflow:{}", self.workflow_type)),
6746 "recorded child workflow type differs from the current workflow command",
6747 ),
6748 )));
6749 }
6750 }
6751 state.command_cursor += 1;
6752 if let Some(outcome) = outcome {
6753 return Poll::Ready(outcome.map_err(Error::ChildWorkflowFailed));
6754 }
6755 state.matched_recorded_pending = true;
6756 self.scheduled = true;
6757 self.matched_pending = true;
6758 return Poll::Pending;
6759 }
6760 other => {
6761 return Poll::Ready(Err(command_mismatch(
6762 &other,
6763 format!("child workflow:{}", self.workflow_type),
6764 )));
6765 }
6766 }
6767 }
6768
6769 if !self.scheduled {
6770 if self.options.task_queue.trim().is_empty() {
6771 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
6772 "task_queue must not be empty".to_string(),
6773 )));
6774 }
6775 for (name, value) in [
6776 (
6777 "execution_timeout_seconds",
6778 self.options.execution_timeout_seconds,
6779 ),
6780 ("run_timeout_seconds", self.options.run_timeout_seconds),
6781 ] {
6782 if value == Some(0) {
6783 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(format!(
6784 "{name} must be at least 1"
6785 ))));
6786 }
6787 }
6788
6789 let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
6790 Ok(args) => args,
6791 Err(error) => return Poll::Ready(Err(error)),
6792 };
6793 let arguments = match encode_typed_envelope(
6794 &normalize_avro_arguments(args),
6795 &state.payload_codec,
6796 ) {
6797 Ok(arguments) => arguments,
6798 Err(error) => return Poll::Ready(Err(error)),
6799 };
6800 let mut command = json!({
6801 "type": "start_child_workflow",
6802 "workflow_type": self.workflow_type,
6803 "queue": self.options.task_queue,
6804 "parent_close_policy": self.options.parent_close_policy.as_str(),
6805 "arguments": arguments,
6806 });
6807 let object = command
6808 .as_object_mut()
6809 .expect("child workflow command is always an object");
6810 if let Some(policy) = &self.options.retry_policy {
6811 let mut retry_policy = serde_json::Map::new();
6812 if let Some(max_attempts) = policy.max_attempts {
6813 if max_attempts == 0 {
6814 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
6815 "retry_policy.max_attempts must be at least 1".to_string(),
6816 )));
6817 }
6818 retry_policy.insert("max_attempts".to_string(), json!(max_attempts));
6819 }
6820 if !policy.backoff_seconds.is_empty() {
6821 retry_policy
6822 .insert("backoff_seconds".to_string(), json!(policy.backoff_seconds));
6823 }
6824 if !policy.non_retryable_error_types.is_empty() {
6825 retry_policy.insert(
6826 "non_retryable_error_types".to_string(),
6827 json!(policy.non_retryable_error_types),
6828 );
6829 }
6830 if retry_policy.is_empty() {
6831 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
6832 "retry_policy must configure at least one field".to_string(),
6833 )));
6834 }
6835 object.insert("retry_policy".to_string(), Value::Object(retry_policy));
6836 }
6837 if let Some(seconds) = self.options.execution_timeout_seconds {
6838 object.insert("execution_timeout_seconds".to_string(), json!(seconds));
6839 }
6840 if let Some(seconds) = self.options.run_timeout_seconds {
6841 object.insert("run_timeout_seconds".to_string(), json!(seconds));
6842 }
6843 state.commands.push(command);
6844 self.scheduled = true;
6845 }
6846
6847 Poll::Pending
6848 }
6849}
6850
6851impl Future for ChildWorkflowCall {
6852 type Output = Result<ChildWorkflowResult>;
6853
6854 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6855 match self.poll_avro_value(cx) {
6856 Poll::Ready(Ok(result)) => match result.result.into_json() {
6857 Ok(projected) => Poll::Ready(Ok(ChildWorkflowResult {
6858 parent: result.parent,
6859 child: result.child,
6860 child_workflow_type: result.child_workflow_type,
6861 result: projected,
6862 })),
6863 Err(error) => Poll::Ready(Err(error)),
6864 },
6865 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
6866 Poll::Pending => Poll::Pending,
6867 }
6868 }
6869}
6870
6871fn command_mismatch(recorded: &RecordedCommand, actual: impl Into<String>) -> Error {
6872 Error::NonDeterministicReplay(ReplayFailure::new(
6873 "recorded_command_mismatch",
6874 Some(recorded.sequence()),
6875 Some(recorded.shape().to_string()),
6876 Some(actual.into()),
6877 "current workflow command does not match the recorded durable command sequence",
6878 ))
6879}
6880
6881pub struct SignalCall {
6882 ctx: WorkflowContext,
6883 signal_name: String,
6884 opened_wait: bool,
6885 matched_pending: bool,
6886}
6887
6888impl SignalCall {
6889 fn poll_avro_value(
6890 mut self: Pin<&mut Self>,
6891 _cx: &mut TaskContext<'_>,
6892 ) -> Poll<Result<Vec<AvroValue>>> {
6893 if self.matched_pending {
6894 return Poll::Pending;
6895 }
6896
6897 let ctx = self.ctx.clone();
6898 let mut state = match ctx.state.lock() {
6899 Ok(state) => state,
6900 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6901 };
6902
6903 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6904 match recorded {
6905 RecordedCommand::SignalWait {
6906 sequence,
6907 signal_name,
6908 value,
6909 } => {
6910 if signal_name != self.signal_name {
6911 return Poll::Ready(Err(Error::NonDeterministicReplay(
6912 ReplayFailure::new(
6913 "recorded_command_detail_mismatch",
6914 Some(sequence),
6915 Some(format!("signal wait:{signal_name}")),
6916 Some(format!("signal wait:{}", self.signal_name)),
6917 "recorded signal name differs from the current workflow command",
6918 ),
6919 )));
6920 }
6921
6922 state.command_cursor += 1;
6923 if let Some(value) = value {
6924 return Poll::Ready(Ok(value));
6925 }
6926 if state
6927 .resume_signal
6928 .as_ref()
6929 .is_some_and(|signal| signal.signal_name == self.signal_name)
6930 {
6931 let signal = state
6932 .resume_signal
6933 .take()
6934 .expect("matching resume signal is present");
6935 return Poll::Ready(Ok(signal.arguments));
6936 }
6937
6938 state.matched_recorded_pending = true;
6939 self.opened_wait = true;
6940 self.matched_pending = true;
6941 return Poll::Pending;
6942 }
6943 other => {
6944 return Poll::Ready(Err(command_mismatch(
6945 &other,
6946 format!("signal wait:{}", self.signal_name),
6947 )));
6948 }
6949 }
6950 }
6951
6952 if state
6953 .resume_signal
6954 .as_ref()
6955 .is_some_and(|signal| signal.signal_name == self.signal_name)
6956 {
6957 let signal = state
6958 .resume_signal
6959 .take()
6960 .expect("matching resume signal is present");
6961 return Poll::Ready(Ok(signal.arguments));
6962 }
6963
6964 if !self.opened_wait {
6965 state.commands.push(json!({
6966 "type": "open_signal_wait",
6967 "signal_name": self.signal_name
6968 }));
6969 self.opened_wait = true;
6970 }
6971
6972 Poll::Pending
6973 }
6974}
6975
6976impl Future for SignalCall {
6977 type Output = Result<Vec<Value>>;
6978
6979 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6980 match self.poll_avro_value(cx) {
6981 Poll::Ready(Ok(values)) => Poll::Ready(
6982 values
6983 .into_iter()
6984 .map(AvroValue::into_json)
6985 .collect::<Result<Vec<_>>>(),
6986 ),
6987 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
6988 Poll::Pending => Poll::Pending,
6989 }
6990 }
6991}
6992
6993#[derive(Clone, Debug)]
6994pub struct ActivityContext {
6995 client: Client,
6996 pub task_id: String,
6997 pub activity_attempt_id: String,
6998 pub lease_owner: String,
6999 pub activity_type: String,
7000 pub attempt_number: u64,
7001 pub task_queue: String,
7002 pub worker_id: String,
7003}
7004
7005impl ActivityContext {
7006 pub async fn heartbeat<T: Serialize>(&self, details: T) -> Result<ActivityHeartbeatResponse> {
7007 self.client
7008 .heartbeat_activity_task(
7009 &self.task_id,
7010 &self.activity_attempt_id,
7011 &self.lease_owner,
7012 details,
7013 )
7014 .await
7015 }
7016}
7017
7018fn decode_task_avro_arguments(value: Option<&Value>, codec: &str) -> Result<AvroValue> {
7019 match value {
7020 Some(value) => Ok(normalize_avro_arguments(decode_wire_avro_value(
7021 value, codec,
7022 )?)),
7023 None => Ok(AvroValue::Array(Vec::new())),
7024 }
7025}
7026
7027fn decode_resume_signal(task: &WorkflowTask) -> Result<Option<ResumeSignal>> {
7028 let Some(signal_name) = task
7029 .signal_name
7030 .as_deref()
7031 .filter(|value| !value.is_empty())
7032 else {
7033 return Ok(None);
7034 };
7035 let Some(arguments) = task.signal_arguments.as_ref() else {
7036 return Ok(None);
7037 };
7038
7039 let decoded = normalize_avro_arguments(decode_wire_avro_value(arguments, &task.payload_codec)?);
7040 let AvroValue::Array(arguments) = decoded else {
7041 unreachable!("normalize_avro_arguments always returns an array");
7042 };
7043
7044 Ok(Some(ResumeSignal {
7045 signal_name: signal_name.to_string(),
7046 arguments,
7047 }))
7048}
7049
7050fn recorded_commands(
7051 events: &[HistoryEvent],
7052 fallback_codec: &str,
7053 parent: WorkflowIdentity,
7054) -> Result<Vec<RecordedCommand>> {
7055 let mut events_by_sequence: BTreeMap<u64, Vec<&HistoryEvent>> = BTreeMap::new();
7056 let mut last_new_sequence = None;
7057
7058 for event in events {
7059 let is_activity = matches!(
7060 event.event_type.as_str(),
7061 "ActivityScheduled"
7062 | "ActivityStarted"
7063 | "ActivityHeartbeatRecorded"
7064 | "ActivityRetryScheduled"
7065 | "ActivityCompleted"
7066 | "ActivityFailed"
7067 | "ActivityCancelled"
7068 | "ActivityTimedOut"
7069 );
7070 let is_workflow_timer = matches!(
7071 event.event_type.as_str(),
7072 "TimerScheduled" | "TimerCancelled" | "TimerFired"
7073 ) && !is_internal_timer_event(event);
7074 let is_child_workflow = matches!(
7075 event.event_type.as_str(),
7076 "ChildWorkflowScheduled"
7077 | "ChildRunCompleted"
7078 | "ChildRunFailed"
7079 | "ChildRunCancelled"
7080 | "ChildRunTerminated"
7081 );
7082 let is_signal_wait = is_recorded_signal_wait_event(event);
7083 let is_side_effect = event.event_type == "SideEffectRecorded";
7084 let is_version_marker = event.event_type == "VersionMarkerRecorded";
7085 if !is_activity
7086 && !is_workflow_timer
7087 && !is_child_workflow
7088 && !is_signal_wait
7089 && !is_side_effect
7090 && !is_version_marker
7091 {
7092 continue;
7093 }
7094
7095 let sequence = durable_event_sequence(event).ok_or_else(|| {
7096 Error::NonDeterministicReplay(ReplayFailure::new(
7097 "durable_command_sequence_missing",
7098 None,
7099 Some("positive workflow sequence".to_string()),
7100 Some(event.event_type.clone()),
7101 "durable command history event has no workflow sequence",
7102 ))
7103 })?;
7104 if sequence == 0 {
7105 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
7106 "durable_command_sequence_invalid",
7107 Some(sequence),
7108 Some("positive workflow sequence".to_string()),
7109 Some(sequence.to_string()),
7110 "durable command history uses an invalid workflow sequence",
7111 )));
7112 }
7113 if !events_by_sequence.contains_key(&sequence) {
7114 if let Some(previous) = last_new_sequence {
7115 if sequence < previous {
7116 return Err(invalid_recorded_history(
7117 "durable_command_sequence_mismatch",
7118 sequence,
7119 &format!("workflow sequence greater than {previous}"),
7120 &sequence.to_string(),
7121 "durable commands are not strictly ordered by their recorded workflow sequence",
7122 ));
7123 }
7124 }
7125 last_new_sequence = Some(sequence);
7126 }
7127 events_by_sequence.entry(sequence).or_default().push(event);
7128 }
7129
7130 let commands: Vec<RecordedCommand> = events_by_sequence
7131 .into_iter()
7132 .map(|(sequence, sequence_events)| {
7133 let activity_events: Vec<_> = sequence_events
7134 .iter()
7135 .copied()
7136 .filter(|event| event.event_type.starts_with("Activity"))
7137 .collect();
7138 let timer_events: Vec<_> = sequence_events
7139 .iter()
7140 .copied()
7141 .filter(|event| event.event_type.starts_with("Timer"))
7142 .collect();
7143 let child_events: Vec<_> = sequence_events
7144 .iter()
7145 .copied()
7146 .filter(|event| {
7147 event.event_type == "ChildWorkflowScheduled"
7148 || event.event_type.starts_with("ChildRun")
7149 })
7150 .collect();
7151 let signal_wait_events: Vec<_> = sequence_events
7152 .iter()
7153 .copied()
7154 .filter(|event| is_recorded_signal_wait_event(event))
7155 .collect();
7156 let side_effect_events: Vec<_> = sequence_events
7157 .iter()
7158 .copied()
7159 .filter(|event| event.event_type == "SideEffectRecorded")
7160 .collect();
7161 let version_marker_events: Vec<_> = sequence_events
7162 .iter()
7163 .copied()
7164 .filter(|event| event.event_type == "VersionMarkerRecorded")
7165 .collect();
7166
7167 let command_kind_count = usize::from(!activity_events.is_empty())
7168 + usize::from(!timer_events.is_empty())
7169 + usize::from(!child_events.is_empty())
7170 + usize::from(!signal_wait_events.is_empty())
7171 + usize::from(!side_effect_events.is_empty())
7172 + usize::from(!version_marker_events.is_empty());
7173 if command_kind_count > 1 {
7174 let actual = [
7175 (!activity_events.is_empty()).then_some("activity"),
7176 (!timer_events.is_empty()).then_some("timer"),
7177 (!child_events.is_empty()).then_some("child workflow"),
7178 (!signal_wait_events.is_empty()).then_some("signal wait"),
7179 (!side_effect_events.is_empty()).then_some("side effect"),
7180 (!version_marker_events.is_empty()).then_some("version marker"),
7181 ]
7182 .into_iter()
7183 .flatten()
7184 .collect::<Vec<_>>()
7185 .join(" and ");
7186 return Err(invalid_recorded_history(
7187 "durable_command_sequence_collision",
7188 sequence,
7189 "one durable command kind",
7190 &actual,
7191 "one workflow sequence records more than one durable command kind",
7192 ));
7193 }
7194
7195 if !activity_events.is_empty() {
7196 let scheduled_count = activity_events
7197 .iter()
7198 .filter(|event| event.event_type == "ActivityScheduled")
7199 .count();
7200 if scheduled_count > 1 {
7201 return Err(invalid_recorded_history(
7202 "duplicate_activity_schedule",
7203 sequence,
7204 "at most one ActivityScheduled event",
7205 "multiple ActivityScheduled events",
7206 "activity history schedules more than one command at one workflow sequence",
7207 ));
7208 }
7209 let activity_type = activity_events.iter().find_map(|event| {
7210 event
7211 .payload
7212 .get("activity_type")
7213 .or_else(|| event.payload.get("activity_name"))
7214 .and_then(Value::as_str)
7215 .map(str::to_string)
7216 });
7217 if activity_events.iter().filter_map(|event| {
7218 event
7219 .payload
7220 .get("activity_type")
7221 .or_else(|| event.payload.get("activity_name"))
7222 .and_then(Value::as_str)
7223 }).any(|candidate| Some(candidate) != activity_type.as_deref()) {
7224 return Err(invalid_recorded_history(
7225 "activity_identity_mismatch",
7226 sequence,
7227 activity_type.as_deref().unwrap_or("one activity identity"),
7228 "conflicting activity identities",
7229 "activity lifecycle events at one workflow sequence disagree on identity",
7230 ));
7231 }
7232 let terminal: Vec<_> = activity_events
7233 .iter()
7234 .copied()
7235 .filter(|event| {
7236 matches!(
7237 event.event_type.as_str(),
7238 "ActivityCompleted"
7239 | "ActivityFailed"
7240 | "ActivityCancelled"
7241 | "ActivityTimedOut"
7242 )
7243 })
7244 .collect();
7245 if terminal.len() > 1 {
7246 return Err(invalid_recorded_history(
7247 "duplicate_activity_terminal_event",
7248 sequence,
7249 "at most one terminal activity event",
7250 "multiple terminal activity events",
7251 "activity history settles one command more than once",
7252 ));
7253 }
7254 let outcome = terminal
7255 .first()
7256 .map(|event| activity_outcome(event, fallback_codec, activity_type.clone()))
7257 .transpose()?;
7258 let options = activity_events
7259 .iter()
7260 .find(|event| event.event_type == "ActivityScheduled")
7261 .and_then(|event| event.payload.get("activity"))
7262 .and_then(Value::as_object)
7263 .map(|activity| RecordedActivityOptions {
7264 task_queue: recorded_optional_string(activity, "queue"),
7265 execution_mode: recorded_optional_string(activity, "execution_mode"),
7266 retry_policy: recorded_activity_retry_snapshot(
7267 activity.get("retry_policy"),
7268 ),
7269 });
7270 return Ok(RecordedCommand::Activity {
7271 sequence,
7272 activity_type,
7273 options,
7274 outcome,
7275 });
7276 }
7277
7278 if !child_events.is_empty() {
7279 let scheduled: Vec<_> = child_events
7280 .iter()
7281 .copied()
7282 .filter(|event| event.event_type == "ChildWorkflowScheduled")
7283 .collect();
7284 if scheduled.len() != 1 {
7285 return Err(invalid_recorded_history(
7286 "child_workflow_schedule_missing_or_duplicate",
7287 sequence,
7288 "one ChildWorkflowScheduled event",
7289 &format!("{} ChildWorkflowScheduled events", scheduled.len()),
7290 "child workflow replay requires exactly one recorded schedule event",
7291 ));
7292 }
7293 let workflow_type = child_events.iter().find_map(|event| {
7294 event
7295 .payload
7296 .get("child_workflow_type")
7297 .or_else(|| event.payload.get("workflow_type"))
7298 .and_then(Value::as_str)
7299 .filter(|value| !value.is_empty())
7300 .map(str::to_string)
7301 });
7302 if child_events
7303 .iter()
7304 .filter_map(|event| {
7305 event
7306 .payload
7307 .get("child_workflow_type")
7308 .or_else(|| event.payload.get("workflow_type"))
7309 .and_then(Value::as_str)
7310 })
7311 .any(|candidate| Some(candidate) != workflow_type.as_deref())
7312 {
7313 return Err(invalid_recorded_history(
7314 "child_workflow_identity_mismatch",
7315 sequence,
7316 workflow_type
7317 .as_deref()
7318 .unwrap_or("one child workflow type"),
7319 "conflicting child workflow types",
7320 "child workflow lifecycle events at one sequence disagree on type",
7321 ));
7322 }
7323 let mut outcomes = child_workflow_outcomes(
7324 &child_events.iter().map(|event| (*event).clone()).collect::<Vec<_>>(),
7325 fallback_codec,
7326 parent.clone(),
7327 )?;
7328 if outcomes.len() > 1 {
7329 return Err(invalid_recorded_history(
7330 "duplicate_child_workflow_terminal_event",
7331 sequence,
7332 "at most one terminal child event",
7333 "multiple terminal child events",
7334 "child workflow history settles one command more than once",
7335 ));
7336 }
7337 return Ok(RecordedCommand::ChildWorkflow {
7338 sequence,
7339 workflow_type,
7340 outcome: outcomes.pop(),
7341 });
7342 }
7343
7344 if !signal_wait_events.is_empty() {
7345 let opened: Vec<_> = signal_wait_events
7346 .iter()
7347 .copied()
7348 .filter(|event| event.event_type == "SignalWaitOpened")
7349 .collect();
7350 if opened.len() != 1 {
7351 return Err(invalid_recorded_history(
7352 "signal_wait_open_missing_or_duplicate",
7353 sequence,
7354 "one SignalWaitOpened event",
7355 &format!("{} SignalWaitOpened events", opened.len()),
7356 "signal replay requires exactly one canonical wait-open event",
7357 ));
7358 }
7359
7360 let applied: Vec<_> = signal_wait_events
7361 .iter()
7362 .copied()
7363 .filter(|event| event.event_type == "SignalApplied")
7364 .collect();
7365 if applied.len() > 1 {
7366 return Err(invalid_recorded_history(
7367 "duplicate_signal_wait_apply",
7368 sequence,
7369 "at most one SignalApplied event",
7370 "multiple SignalApplied events",
7371 "signal history applies one durable wait more than once",
7372 ));
7373 }
7374
7375 let signal_names = signal_wait_events
7376 .iter()
7377 .map(|event| required_signal_wait_name(event, sequence))
7378 .collect::<Result<Vec<_>>>()?;
7379 let signal_name = signal_names
7380 .first()
7381 .expect("signal wait events are not empty")
7382 .clone();
7383 if signal_names.iter().any(|candidate| candidate != &signal_name) {
7384 return Err(invalid_recorded_history(
7385 "signal_wait_identity_mismatch",
7386 sequence,
7387 &signal_name,
7388 "conflicting signal names",
7389 "signal wait lifecycle events at one workflow sequence disagree on identity",
7390 ));
7391 }
7392 let value = applied
7393 .first()
7394 .map(|event| decode_signal_event_arguments(event, fallback_codec))
7395 .transpose()?;
7396 return Ok(RecordedCommand::SignalWait {
7397 sequence,
7398 signal_name,
7399 value,
7400 });
7401 }
7402
7403 if !side_effect_events.is_empty() {
7404 if side_effect_events.len() != 1 {
7405 return Err(invalid_recorded_history(
7406 "duplicate_side_effect_record",
7407 sequence,
7408 "one SideEffectRecorded event",
7409 &format!("{} SideEffectRecorded events", side_effect_events.len()),
7410 "side-effect history records one workflow command more than once",
7411 ));
7412 }
7413 let event = side_effect_events[0];
7414 let result = event.payload.get("result").ok_or_else(|| {
7415 invalid_recorded_history(
7416 "side_effect_result_missing",
7417 sequence,
7418 "recorded result payload",
7419 "missing result",
7420 "side-effect history is missing its recorded value",
7421 )
7422 })?;
7423 let has_published_envelope = result.as_str().is_some()
7424 || result.as_object().is_some_and(|envelope| {
7425 envelope.get("codec").and_then(Value::as_str).is_some()
7426 && envelope.get("blob").and_then(Value::as_str).is_some()
7427 });
7428 if !has_published_envelope {
7429 return Err(invalid_recorded_history(
7430 "side_effect_payload_malformed",
7431 sequence,
7432 "payload blob or {codec, blob} envelope",
7433 &result.to_string(),
7434 "side-effect history result does not use a published payload envelope",
7435 ));
7436 }
7437 let codec = event
7438 .payload
7439 .get("payload_codec")
7440 .and_then(Value::as_str)
7441 .unwrap_or(fallback_codec);
7442 let value = decode_wire_avro_value(result, codec).map_err(|error| {
7443 if error.to_string().contains("unsupported_payload_codec") {
7444 return error;
7445 }
7446
7447 invalid_recorded_history(
7448 "side_effect_payload_incompatible",
7449 sequence,
7450 &format!("valid {codec} payload envelope"),
7451 &error.to_string(),
7452 "side-effect history payload cannot be decoded with its recorded codec",
7453 )
7454 })?;
7455 return Ok(RecordedCommand::SideEffect { sequence, value });
7456 }
7457
7458 if !version_marker_events.is_empty() {
7459 if version_marker_events.len() != 1 {
7460 return Err(invalid_recorded_history(
7461 "duplicate_version_marker_record",
7462 sequence,
7463 "one VersionMarkerRecorded event",
7464 &format!("{} VersionMarkerRecorded events", version_marker_events.len()),
7465 "version-marker history records one workflow command more than once",
7466 ));
7467 }
7468 let payload = &version_marker_events[0].payload;
7469 let change_id = payload
7470 .get("change_id")
7471 .and_then(Value::as_str)
7472 .filter(|value| !value.is_empty())
7473 .map(str::to_string)
7474 .ok_or_else(|| {
7475 invalid_recorded_history(
7476 "version_marker_field_missing",
7477 sequence,
7478 "non-empty change_id",
7479 "missing or invalid change_id",
7480 "version-marker history is missing its stable change ID",
7481 )
7482 })?;
7483 let version = required_version_i32(payload, "version", sequence)?;
7484 let min_supported = required_version_i32(payload, "min_supported", sequence)?;
7485 let max_supported = required_version_i32(payload, "max_supported", sequence)?;
7486 if min_supported > max_supported || version < min_supported || version > max_supported {
7487 return Err(invalid_recorded_history(
7488 "version_marker_history_range_invalid",
7489 sequence,
7490 "min_supported <= version <= max_supported",
7491 &format!("{min_supported} <= {version} <= {max_supported}"),
7492 "recorded version marker contains an internally incompatible range",
7493 ));
7494 }
7495 return Ok(RecordedCommand::VersionMarker {
7496 sequence,
7497 change_id,
7498 version,
7499 });
7500 }
7501
7502 let scheduled: Vec<_> = timer_events
7503 .iter()
7504 .copied()
7505 .filter(|event| event.event_type == "TimerScheduled")
7506 .collect();
7507 let fired: Vec<_> = timer_events
7508 .iter()
7509 .copied()
7510 .filter(|event| event.event_type == "TimerFired")
7511 .collect();
7512 if scheduled.len() != 1 {
7513 return Err(invalid_recorded_history(
7514 "timer_schedule_missing_or_duplicate",
7515 sequence,
7516 "one TimerScheduled event",
7517 &format!("{} TimerScheduled events", scheduled.len()),
7518 "timer replay requires exactly one recorded schedule event",
7519 ));
7520 }
7521 if fired.len() > 1 {
7522 return Err(invalid_recorded_history(
7523 "duplicate_timer_fire",
7524 sequence,
7525 "at most one TimerFired event",
7526 "multiple TimerFired events",
7527 "timer history contains more than one fire event for a workflow sequence",
7528 ));
7529 }
7530
7531 let scheduled = scheduled[0];
7532 let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
7533 let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
7534 if let Some(fired) = fired.first() {
7535 let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
7536 if fired_timer_id != timer_id {
7537 return Err(invalid_recorded_history(
7538 "timer_identity_mismatch",
7539 sequence,
7540 &timer_id,
7541 &fired_timer_id,
7542 "TimerFired does not correspond to the recorded TimerScheduled event",
7543 ));
7544 }
7545 let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
7546 if fired_delay != delay_seconds {
7547 return Err(invalid_recorded_history(
7548 "timer_history_delay_mismatch",
7549 sequence,
7550 &delay_seconds.to_string(),
7551 &fired_delay.to_string(),
7552 "TimerScheduled and TimerFired record different delays",
7553 ));
7554 }
7555 }
7556
7557 Ok(RecordedCommand::Timer {
7558 sequence,
7559 delay_seconds,
7560 fired: !fired.is_empty(),
7561 })
7562 })
7563 .collect::<Result<_>>()?;
7564
7565 let mut marker_sequences = HashMap::new();
7566 for command in &commands {
7567 if let RecordedCommand::VersionMarker {
7568 sequence,
7569 change_id,
7570 ..
7571 } = command
7572 {
7573 if let Some(first_sequence) = marker_sequences.insert(change_id.clone(), *sequence) {
7574 return Err(invalid_recorded_history(
7575 "duplicate_version_marker",
7576 *sequence,
7577 &format!("one marker for change ID {change_id:?}"),
7578 &format!("markers at sequences {first_sequence} and {sequence}"),
7579 "workflow history contains duplicate markers for one stable change ID",
7580 ));
7581 }
7582 }
7583 }
7584
7585 Ok(commands)
7586}
7587
7588fn required_version_i32(payload: &Value, field: &str, sequence: u64) -> Result<i32> {
7589 payload
7590 .get(field)
7591 .and_then(Value::as_i64)
7592 .and_then(|value| i32::try_from(value).ok())
7593 .ok_or_else(|| {
7594 invalid_recorded_history(
7595 "version_marker_field_missing",
7596 sequence,
7597 &format!("integer {field}"),
7598 "missing or out-of-range integer",
7599 "version-marker history is missing a required integer field",
7600 )
7601 })
7602}
7603
7604fn durable_event_sequence(event: &HistoryEvent) -> Option<u64> {
7605 event
7606 .payload
7607 .get("sequence")
7608 .or_else(|| event.payload.get("workflow_sequence"))
7609 .or_else(|| event.raw.get("sequence"))
7610 .or_else(|| event.raw.get("workflow_sequence"))
7611 .and_then(value_as_u64)
7612}
7613
7614fn is_internal_timer_event(event: &HistoryEvent) -> bool {
7615 matches!(
7616 event
7617 .payload
7618 .get("timer_kind")
7619 .or_else(|| event.raw.get("timer_kind"))
7620 .and_then(Value::as_str),
7621 Some("condition_timeout" | "signal_timeout")
7622 )
7623}
7624
7625fn required_signal_wait_name(event: &HistoryEvent, sequence: u64) -> Result<String> {
7626 event
7627 .payload
7628 .get("signal_name")
7629 .or_else(|| event.raw.get("signal_name"))
7630 .and_then(Value::as_str)
7631 .filter(|value| !value.is_empty())
7632 .map(str::to_string)
7633 .ok_or_else(|| {
7634 invalid_recorded_history(
7635 "signal_wait_name_missing",
7636 sequence,
7637 "non-empty signal_name",
7638 &event.event_type,
7639 "canonical signal-wait history is missing its signal identity",
7640 )
7641 })
7642}
7643
7644fn is_recorded_signal_wait_event(event: &HistoryEvent) -> bool {
7645 matches!(
7646 event.event_type.as_str(),
7647 "SignalWaitOpened" | "SignalApplied"
7648 )
7649}
7650
7651fn required_history_string(event: &HistoryEvent, field: &str, sequence: u64) -> Result<String> {
7652 event
7653 .payload
7654 .get(field)
7655 .and_then(Value::as_str)
7656 .filter(|value| !value.is_empty())
7657 .map(str::to_string)
7658 .ok_or_else(|| {
7659 invalid_recorded_history(
7660 "timer_history_field_missing",
7661 sequence,
7662 field,
7663 &event.event_type,
7664 "timer history is missing a required identity field",
7665 )
7666 })
7667}
7668
7669fn required_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<u64> {
7670 event
7671 .payload
7672 .get(field)
7673 .and_then(value_as_u64)
7674 .ok_or_else(|| {
7675 invalid_recorded_history(
7676 "timer_history_field_missing",
7677 sequence,
7678 field,
7679 &event.event_type,
7680 "timer history is missing a required numeric field",
7681 )
7682 })
7683}
7684
7685fn invalid_recorded_history(
7686 reason: &str,
7687 sequence: u64,
7688 expected: &str,
7689 actual: &str,
7690 message: &str,
7691) -> Error {
7692 Error::NonDeterministicReplay(ReplayFailure::new(
7693 reason,
7694 Some(sequence),
7695 Some(expected.to_string()),
7696 Some(actual.to_string()),
7697 message,
7698 ))
7699}
7700
7701type ActivityOutcome = std::result::Result<AvroValue, ActivityFailure>;
7702
7703fn activity_outcome(
7704 event: &HistoryEvent,
7705 fallback_codec: &str,
7706 recorded_activity_type: Option<String>,
7707) -> Result<ActivityOutcome> {
7708 if event.event_type == "ActivityCompleted" {
7709 let codec = event
7710 .payload
7711 .get("payload_codec")
7712 .and_then(Value::as_str)
7713 .unwrap_or(fallback_codec);
7714 return Ok(Ok(decode_wire_avro_value(
7715 event.payload.get("result").unwrap_or(&Value::Null),
7716 codec,
7717 )?));
7718 }
7719
7720 let payload = &event.payload;
7721 let (kind, fallback_reason, fallback_message) = match event.event_type.as_str() {
7722 "ActivityFailed" => (ActivityFailureKind::Failed, "activity", "activity failed"),
7723 "ActivityCancelled" => (
7724 ActivityFailureKind::Cancelled,
7725 "cancelled",
7726 "activity was cancelled",
7727 ),
7728 "ActivityTimedOut" => (
7729 ActivityFailureKind::TimedOut,
7730 "timeout",
7731 "activity timed out",
7732 ),
7733 _ => unreachable!("activity_outcome is called only for terminal activity events"),
7734 };
7735 let exception = payload
7736 .get("exception")
7737 .filter(|value| !value.is_null())
7738 .cloned();
7739 let failure_category = payload_string(payload, "failure_category");
7740 let timeout_kind = payload_string(payload, "timeout_kind");
7741 let reason = payload_string(payload, "reason").unwrap_or_else(|| match kind {
7742 ActivityFailureKind::Failed => failure_category
7743 .clone()
7744 .unwrap_or_else(|| fallback_reason.to_string()),
7745 ActivityFailureKind::Cancelled => fallback_reason.to_string(),
7746 ActivityFailureKind::TimedOut => timeout_kind
7747 .clone()
7748 .unwrap_or_else(|| fallback_reason.to_string()),
7749 });
7750 let message = payload_string(payload, "message")
7751 .or_else(|| {
7752 exception
7753 .as_ref()
7754 .and_then(|value| payload_string(value, "message"))
7755 })
7756 .unwrap_or_else(|| fallback_message.to_string());
7757
7758 Ok(Err(ActivityFailure {
7759 kind,
7760 reason,
7761 message,
7762 activity_execution_id: payload_string(payload, "activity_execution_id"),
7763 activity_attempt_id: payload_string(payload, "activity_attempt_id"),
7764 activity_type: payload_string(payload, "activity_type")
7765 .or_else(|| payload_string(payload, "activity_name"))
7766 .or(recorded_activity_type),
7767 activity_class: payload_string(payload, "activity_class"),
7768 attempt_number: payload.get("attempt_number").and_then(value_as_u64),
7769 failure_id: payload_string(payload, "failure_id"),
7770 failure_category,
7771 timeout_kind,
7772 non_retryable: payload
7773 .get("non_retryable")
7774 .and_then(Value::as_bool)
7775 .unwrap_or(false),
7776 exception_type: payload_string(payload, "exception_type").or_else(|| {
7777 exception
7778 .as_ref()
7779 .and_then(|value| payload_string(value, "type"))
7780 }),
7781 exception_class: payload_string(payload, "exception_class").or_else(|| {
7782 exception
7783 .as_ref()
7784 .and_then(|value| payload_string(value, "class"))
7785 }),
7786 code: payload
7787 .get("code")
7788 .filter(|value| !value.is_null())
7789 .cloned(),
7790 exception,
7791 }))
7792}
7793
7794type ChildWorkflowOutcome = std::result::Result<ChildWorkflowAvroResult, ChildWorkflowFailure>;
7795
7796fn child_workflow_outcomes(
7797 events: &[HistoryEvent],
7798 fallback_codec: &str,
7799 parent: WorkflowIdentity,
7800) -> Result<Vec<ChildWorkflowOutcome>> {
7801 let mut outcomes = Vec::new();
7802
7803 for event in events {
7804 let kind = match event.event_type.as_str() {
7805 "ChildRunCompleted" => None,
7806 "ChildRunFailed" => Some((
7807 ChildWorkflowFailureKind::Failed,
7808 "child_workflow",
7809 "child workflow failed",
7810 )),
7811 "ChildRunCancelled" => Some((
7812 ChildWorkflowFailureKind::Cancelled,
7813 "cancelled",
7814 "child workflow was cancelled",
7815 )),
7816 "ChildRunTerminated" => Some((
7817 ChildWorkflowFailureKind::Terminated,
7818 "terminated",
7819 "child workflow was terminated",
7820 )),
7821 _ => continue,
7822 };
7823 let payload = &event.payload;
7824 let child_workflow_id = payload_string(payload, "child_workflow_instance_id");
7825 let child_workflow_run_id = payload_string(payload, "child_workflow_run_id");
7826 let child_workflow_type = payload_string(payload, "child_workflow_type");
7827
7828 if let Some((kind, reason, fallback_message)) = kind {
7829 let exception = payload
7830 .get("exception")
7831 .filter(|value| !value.is_null())
7832 .cloned();
7833 let message = payload_string(payload, "message")
7834 .or_else(|| {
7835 exception
7836 .as_ref()
7837 .and_then(|value| payload_string(value, "message"))
7838 })
7839 .unwrap_or_else(|| fallback_message.to_string());
7840 let exception_type = payload_string(payload, "exception_type").or_else(|| {
7841 exception
7842 .as_ref()
7843 .and_then(|value| payload_string(value, "type"))
7844 });
7845 let exception_class = payload_string(payload, "exception_class").or_else(|| {
7846 exception
7847 .as_ref()
7848 .and_then(|value| payload_string(value, "class"))
7849 });
7850 outcomes.push(Err(ChildWorkflowFailure {
7851 kind,
7852 reason: reason.to_string(),
7853 message,
7854 parent_workflow_id: parent.workflow_id.clone(),
7855 parent_workflow_run_id: parent.run_id.clone(),
7856 child_workflow_id,
7857 child_workflow_run_id,
7858 child_workflow_type,
7859 failure_id: payload_string(payload, "failure_id"),
7860 failure_category: payload_string(payload, "failure_category"),
7861 exception_type,
7862 exception_class,
7863 non_retryable: payload
7864 .get("non_retryable")
7865 .and_then(Value::as_bool)
7866 .unwrap_or(false),
7867 code: payload
7868 .get("code")
7869 .filter(|value| !value.is_null())
7870 .cloned(),
7871 exception,
7872 }));
7873 continue;
7874 }
7875
7876 let codec = payload
7877 .get("payload_codec")
7878 .and_then(Value::as_str)
7879 .unwrap_or(fallback_codec);
7880 let result = payload
7881 .get("result")
7882 .or_else(|| payload.get("output"))
7883 .unwrap_or(&Value::Null);
7884 outcomes.push(Ok(ChildWorkflowAvroResult {
7885 parent: parent.clone(),
7886 child: WorkflowIdentity {
7887 workflow_id: child_workflow_id,
7888 run_id: child_workflow_run_id,
7889 },
7890 child_workflow_type,
7891 result: decode_wire_avro_value(result, codec)?,
7892 }));
7893 }
7894
7895 Ok(outcomes)
7896}
7897
7898fn payload_string(payload: &Value, key: &str) -> Option<String> {
7899 payload
7900 .get(key)
7901 .and_then(Value::as_str)
7902 .filter(|value| !value.is_empty())
7903 .map(str::to_string)
7904}
7905
7906fn workflow_failure_command(error: &Error) -> Value {
7907 let (exception_type, exception_class, properties) = match error {
7908 Error::ActivityFailed(failure) => (
7909 match failure.kind {
7910 ActivityFailureKind::Failed => "ActivityFailed",
7911 ActivityFailureKind::Cancelled => "ActivityCancelled",
7912 ActivityFailureKind::TimedOut => "ActivityTimedOut",
7913 },
7914 "durable_workflow::ActivityFailure",
7915 json!({
7916 "reason": failure.reason,
7917 "activity_execution_id": failure.activity_execution_id,
7918 "activity_attempt_id": failure.activity_attempt_id,
7919 "activity_type": failure.activity_type,
7920 "activity_class": failure.activity_class,
7921 "attempt_number": failure.attempt_number,
7922 "failure_id": failure.failure_id,
7923 "failure_category": failure.failure_category,
7924 "timeout_kind": failure.timeout_kind,
7925 "activity_non_retryable": failure.non_retryable,
7926 "activity_exception_type": failure.exception_type,
7927 "activity_exception_class": failure.exception_class,
7928 "activity_code": failure.code,
7929 "activity_exception": failure.exception,
7930 }),
7931 ),
7932 Error::ChildWorkflowFailed(failure) => (
7933 match failure.kind {
7934 ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
7935 ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
7936 ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
7937 },
7938 "durable_workflow::ChildWorkflowFailure",
7939 json!({
7940 "reason": failure.reason,
7941 "parent_workflow_id": failure.parent_workflow_id,
7942 "parent_workflow_run_id": failure.parent_workflow_run_id,
7943 "child_workflow_id": failure.child_workflow_id,
7944 "child_workflow_run_id": failure.child_workflow_run_id,
7945 "child_workflow_type": failure.child_workflow_type,
7946 "failure_id": failure.failure_id,
7947 "failure_category": failure.failure_category,
7948 "child_exception_type": failure.exception_type,
7949 "child_exception_class": failure.exception_class,
7950 "child_non_retryable": failure.non_retryable,
7951 "child_code": failure.code,
7952 "child_exception": failure.exception,
7953 }),
7954 ),
7955 Error::NonDeterministicReplay(_) => (
7956 "NonDeterministicReplay",
7957 "durable_workflow::Error",
7958 Value::Null,
7959 ),
7960 _ => ("RustWorkflowError", "durable_workflow::Error", Value::Null),
7961 };
7962 let non_retryable = match error {
7963 Error::ActivityFailed(failure) => failure.non_retryable,
7964 Error::ChildWorkflowFailed(failure) => failure.non_retryable,
7965 Error::NonDeterministicReplay(_) => true,
7966 _ => false,
7967 };
7968
7969 json!({
7970 "type": "fail_workflow",
7971 "message": error.to_string(),
7972 "exception_type": exception_type,
7973 "exception_class": exception_class,
7974 "non_retryable": non_retryable,
7975 "exception": {
7976 "type": exception_type,
7977 "class": exception_class,
7978 "message": error.to_string(),
7979 "properties": properties,
7980 }
7981 })
7982}
7983
7984fn workflow_task_integrity_error(error: &Error) -> bool {
7985 matches!(
7986 error,
7987 Error::NonDeterministicReplay(_) | Error::Protocol(_) | Error::WorkflowStatePoisoned
7988 )
7989}
7990
7991fn decode_signal_event_arguments(
7992 event: &HistoryEvent,
7993 fallback_codec: &str,
7994) -> Result<Vec<AvroValue>> {
7995 let codec = event
7996 .payload
7997 .get("payload_codec")
7998 .and_then(Value::as_str)
7999 .unwrap_or(fallback_codec);
8000 let raw = event
8001 .payload
8002 .get("value")
8003 .or_else(|| event.payload.get("input"))
8004 .or_else(|| event.payload.get("arguments"));
8005 let decoded = match raw.filter(|value| !value.is_null()) {
8006 Some(value) => decode_wire_avro_value(value, codec)?,
8007 None => AvroValue::Array(Vec::new()),
8008 };
8009 let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
8010 unreachable!("normalize_avro_arguments always returns an array");
8011 };
8012 Ok(arguments)
8013}
8014
8015fn hydrate_query_history_from_export(task: &mut QueryTask) -> Result<()> {
8016 let Some(export_events) = task
8017 .history_export
8018 .as_ref()
8019 .and_then(|export| export.get("history_events"))
8020 .and_then(Value::as_array)
8021 else {
8022 return Ok(());
8023 };
8024
8025 if export_events.len() > task.history_events.len() {
8026 task.history_events = serde_json::from_value(Value::Array(export_events.clone()))?;
8027 }
8028
8029 Ok(())
8030}
8031
8032fn enrich_query_history_from_export(task: &mut QueryTask) -> Result<()> {
8033 let Some(export) = task.history_export.as_ref() else {
8034 return Ok(());
8035 };
8036 let signals = export
8037 .get("signals")
8038 .and_then(Value::as_array)
8039 .cloned()
8040 .unwrap_or_default();
8041 let activities = export
8042 .get("activities")
8043 .and_then(Value::as_array)
8044 .cloned()
8045 .unwrap_or_default();
8046 let export_codec = export
8047 .get("payloads")
8048 .and_then(|payloads| payloads.get("codec"))
8049 .and_then(Value::as_str)
8050 .unwrap_or(&task.payload_codec)
8051 .to_string();
8052 let mut signal_name_offsets: HashMap<String, usize> = HashMap::new();
8053
8054 for event in &mut task.history_events {
8055 if event.event_type == "ActivityCompleted" {
8056 let sequence = event
8057 .payload
8058 .get("sequence")
8059 .or_else(|| event.payload.get("workflow_sequence"))
8060 .and_then(value_as_u64);
8061 let Some(activity) = sequence.and_then(|sequence| {
8062 activities.iter().find(|activity| {
8063 activity.get("sequence").and_then(value_as_u64) == Some(sequence)
8064 })
8065 }) else {
8066 continue;
8067 };
8068 let Some(payload) = event.payload.as_object_mut() else {
8069 continue;
8070 };
8071 if missing_payload(payload.get("result")) {
8072 if let Some(result) = activity
8073 .get("result")
8074 .filter(|value| !missing_payload(Some(value)))
8075 {
8076 payload.insert("result".to_string(), result.clone());
8077 }
8078 }
8079 for field in ["payload_codec", "activity_type"] {
8080 if payload
8081 .get(field)
8082 .and_then(Value::as_str)
8083 .unwrap_or_default()
8084 .is_empty()
8085 {
8086 if let Some(value) = activity.get(field) {
8087 payload.insert(field.to_string(), value.clone());
8088 }
8089 }
8090 }
8091 continue;
8092 }
8093
8094 if event.event_type != "SignalReceived" && event.event_type != "SignalApplied" {
8095 continue;
8096 }
8097 let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
8098 let command_id = event
8099 .payload
8100 .get("workflow_command_id")
8101 .or_else(|| event.raw.get("workflow_command_id"))
8102 .and_then(Value::as_str);
8103 let signal_name = event
8104 .payload
8105 .get("signal_name")
8106 .and_then(Value::as_str)
8107 .unwrap_or_default()
8108 .to_string();
8109 let matched = signals
8110 .iter()
8111 .find(|signal| {
8112 signal_id.is_some() && signal.get("id").and_then(Value::as_str) == signal_id
8113 })
8114 .or_else(|| {
8115 signals.iter().find(|signal| {
8116 command_id.is_some()
8117 && signal.get("command_id").and_then(Value::as_str) == command_id
8118 })
8119 })
8120 .or_else(|| {
8121 let offset = signal_name_offsets.entry(signal_name.clone()).or_default();
8122 let signal = signals
8123 .iter()
8124 .filter(|signal| {
8125 signal.get("name").and_then(Value::as_str) == Some(signal_name.as_str())
8126 })
8127 .nth(*offset);
8128 if signal.is_some() {
8129 *offset += 1;
8130 }
8131 signal
8132 });
8133 let Some(signal) = matched else {
8134 continue;
8135 };
8136 let signal_codec = signal
8137 .get("payload_codec")
8138 .and_then(Value::as_str)
8139 .unwrap_or(&export_codec);
8140 let Some(payload) = event.payload.as_object_mut() else {
8141 continue;
8142 };
8143 if missing_payload(payload.get("arguments")) {
8144 if let Some(arguments) = signal
8145 .get("arguments")
8146 .filter(|value| !missing_payload(Some(value)))
8147 {
8148 let envelope = match arguments {
8149 Value::String(blob) => json!({"codec": signal_codec, "blob": blob}),
8150 other => other.clone(),
8151 };
8152 payload.insert("arguments".to_string(), envelope);
8153 }
8154 }
8155 if payload
8156 .get("payload_codec")
8157 .and_then(Value::as_str)
8158 .unwrap_or_default()
8159 .is_empty()
8160 {
8161 payload.insert("payload_codec".to_string(), json!(signal_codec));
8162 }
8163 }
8164
8165 Ok(())
8166}
8167
8168fn missing_payload(value: Option<&Value>) -> bool {
8169 match value {
8170 None | Some(Value::Null) => true,
8171 Some(Value::String(value)) => value.is_empty(),
8172 Some(_) => false,
8173 }
8174}
8175
8176fn query_signal_events(task: &QueryTask) -> Result<Vec<QuerySignal>> {
8177 let export_signals = task
8178 .history_export
8179 .as_ref()
8180 .and_then(|export| export.get("signals"))
8181 .and_then(Value::as_array)
8182 .cloned()
8183 .unwrap_or_default();
8184 let export_codec = task
8185 .history_export
8186 .as_ref()
8187 .and_then(|export| export.get("payloads"))
8188 .and_then(|payloads| payloads.get("codec"))
8189 .and_then(Value::as_str)
8190 .unwrap_or(&task.payload_codec);
8191 let mut name_offsets: HashMap<String, usize> = HashMap::new();
8192 let mut signals = Vec::new();
8193
8194 for event in &task.history_events {
8195 if event.event_type != "SignalApplied" && event.event_type != "SignalReceived" {
8196 continue;
8197 }
8198
8199 let name = event
8200 .payload
8201 .get("signal_name")
8202 .and_then(Value::as_str)
8203 .unwrap_or_default();
8204 if name.is_empty() {
8205 continue;
8206 }
8207 let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
8208 let command_id = event
8209 .payload
8210 .get("workflow_command_id")
8211 .or_else(|| event.raw.get("workflow_command_id"))
8212 .and_then(Value::as_str);
8213 let matched_export = export_signals
8214 .iter()
8215 .find(|candidate| {
8216 signal_id.is_some() && candidate.get("id").and_then(Value::as_str) == signal_id
8217 })
8218 .or_else(|| {
8219 export_signals.iter().find(|candidate| {
8220 command_id.is_some()
8221 && candidate.get("command_id").and_then(Value::as_str) == command_id
8222 })
8223 })
8224 .or_else(|| {
8225 let offset = name_offsets.entry(name.to_string()).or_default();
8226 let candidate = export_signals
8227 .iter()
8228 .filter(|candidate| candidate.get("name").and_then(Value::as_str) == Some(name))
8229 .nth(*offset);
8230 if candidate.is_some() {
8231 *offset += 1;
8232 }
8233 candidate
8234 });
8235 let codec = event
8236 .payload
8237 .get("payload_codec")
8238 .and_then(Value::as_str)
8239 .or_else(|| {
8240 matched_export
8241 .and_then(|signal| signal.get("payload_codec"))
8242 .and_then(Value::as_str)
8243 })
8244 .unwrap_or(export_codec);
8245 let raw_arguments = event
8246 .payload
8247 .get("value")
8248 .or_else(|| event.payload.get("input"))
8249 .or_else(|| event.payload.get("arguments"))
8250 .filter(|value| !value.is_null())
8251 .or_else(|| matched_export.and_then(|signal| signal.get("arguments")));
8252 let (arguments, avro_arguments) = decode_query_signal_arguments(raw_arguments, codec)?;
8253 let workflow_sequence = event
8254 .payload
8255 .get("workflow_sequence")
8256 .and_then(value_as_u64)
8257 .or_else(|| {
8258 matched_export
8259 .and_then(|signal| signal.get("workflow_sequence"))
8260 .and_then(value_as_u64)
8261 });
8262
8263 signals.push(QuerySignal {
8264 id: signal_id.map(str::to_string).or_else(|| {
8265 matched_export
8266 .and_then(|signal| signal.get("id"))
8267 .and_then(Value::as_str)
8268 .map(str::to_string)
8269 }),
8270 name: name.to_string(),
8271 arguments,
8272 avro_arguments,
8273 workflow_sequence,
8274 });
8275 }
8276
8277 if signals.is_empty() {
8278 for signal in export_signals {
8279 if signal.get("status").and_then(Value::as_str) == Some("rejected") {
8280 continue;
8281 }
8282 let Some(name) = signal.get("name").and_then(Value::as_str) else {
8283 continue;
8284 };
8285 let codec = signal
8286 .get("payload_codec")
8287 .and_then(Value::as_str)
8288 .unwrap_or(export_codec);
8289 let (arguments, avro_arguments) =
8290 decode_query_signal_arguments(signal.get("arguments"), codec)?;
8291 signals.push(QuerySignal {
8292 id: signal.get("id").and_then(Value::as_str).map(str::to_string),
8293 name: name.to_string(),
8294 arguments,
8295 avro_arguments,
8296 workflow_sequence: signal.get("workflow_sequence").and_then(value_as_u64),
8297 });
8298 }
8299 signals.sort_by_key(|signal| signal.workflow_sequence.unwrap_or(u64::MAX));
8300 }
8301
8302 Ok(signals)
8303}
8304
8305fn decode_query_signal_arguments(
8306 raw: Option<&Value>,
8307 codec: &str,
8308) -> Result<(Vec<Value>, Vec<AvroValue>)> {
8309 let decoded = match raw.filter(|value| !value.is_null()) {
8310 Some(value) => decode_wire_avro_value(value, codec)?,
8311 None => AvroValue::Array(Vec::new()),
8312 };
8313 let AvroValue::Array(avro_arguments) = normalize_avro_arguments(decoded) else {
8314 unreachable!("normalize_avro_arguments always returns an array");
8315 };
8316 let arguments = avro_arguments
8317 .iter()
8318 .cloned()
8319 .map(AvroValue::into_json)
8320 .collect::<Result<Vec<_>>>()?;
8321 Ok((arguments, avro_arguments))
8322}
8323
8324fn value_as_u64(value: &Value) -> Option<u64> {
8325 value
8326 .as_u64()
8327 .or_else(|| value.as_str().and_then(|value| value.parse().ok()))
8328}
8329
8330#[cfg(test)]
8331mod tests {
8332 use super::*;
8333 use std::{
8334 io::{Read, Write},
8335 net::{SocketAddr, TcpListener, TcpStream},
8336 sync::atomic::AtomicUsize,
8337 thread,
8338 };
8339
8340 fn fixture_envelope(value: Value) -> Value {
8341 encode_value_envelope(&value, DEFAULT_CODEC).expect("encode Avro test fixture")
8342 }
8343
8344 fn fixture_blob(value: Value) -> String {
8345 encode_payload(&value, DEFAULT_CODEC)
8346 .expect("encode Avro test fixture")
8347 .blob
8348 }
8349
8350 #[test]
8351 fn client_builder_rejects_the_sdk_owned_api_suffix() {
8352 for base_url in [
8353 "http://127.0.0.1:8080/api",
8354 "http://localhost:8080/api/",
8355 "https://runtime.example.test/namespaces/orders/api",
8356 ] {
8357 let error = Client::builder(base_url)
8358 .build()
8359 .expect_err("SDK-owned /api suffix must be rejected during build");
8360
8361 assert!(matches!(error, Error::InvalidBaseUrl), "{base_url}");
8362 assert!(
8363 error.to_string().contains("SDK appends /api automatically"),
8364 "the validation error must explain how to fix the endpoint"
8365 );
8366 }
8367 }
8368
8369 #[test]
8370 fn client_builder_preserves_self_hosted_and_managed_runtime_prefixes() {
8371 for (base_url, expected) in [
8372 ("http://127.0.0.1:8080", "http://127.0.0.1:8080"),
8373 (
8374 "http://localhost:8080/durable-workflow/",
8375 "http://localhost:8080/durable-workflow",
8376 ),
8377 (
8378 "https://runtime.example.test/namespaces/orders",
8379 "https://runtime.example.test/namespaces/orders",
8380 ),
8381 (
8382 "https://runtime.example.test/gateway/api/namespaces/orders",
8383 "https://runtime.example.test/gateway/api/namespaces/orders",
8384 ),
8385 (
8386 "https://api.example.test/runtime/orders/",
8387 "https://api.example.test/runtime/orders",
8388 ),
8389 ] {
8390 let client = Client::builder(base_url)
8391 .build()
8392 .expect("Server and Cloud runtime base URL must remain valid");
8393
8394 assert_eq!(client.base_url, expected);
8395 }
8396 }
8397
8398 fn typed_fidelity_probe() -> AvroValue {
8399 AvroValue::Map(BTreeMap::from([
8400 ("bytes".to_string(), AvroValue::Bytes(vec![0, 0xff])),
8401 ("empty".to_string(), AvroValue::Map(BTreeMap::new())),
8402 (
8403 "numeric".to_string(),
8404 AvroValue::Map(BTreeMap::from([
8405 ("0".to_string(), AvroValue::String("zero".to_string())),
8406 ("1".to_string(), AvroValue::String("one".to_string())),
8407 ])),
8408 ),
8409 (
8410 "nested".to_string(),
8411 AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([(
8412 "enabled".to_string(),
8413 AvroValue::Boolean(true),
8414 )]))]),
8415 ),
8416 (
8417 "projection_collisions".to_string(),
8418 AvroValue::Array(projection_collision_probe()),
8419 ),
8420 ]))
8421 }
8422
8423 fn projection_collision_probe() -> Vec<AvroValue> {
8424 vec![
8425 AvroValue::Map(BTreeMap::from([
8426 ("$type".to_string(), AvroValue::String("bytes".to_string())),
8427 (
8428 "base64".to_string(),
8429 AvroValue::String("ordinary user text".to_string()),
8430 ),
8431 ])),
8432 AvroValue::Map(BTreeMap::from([
8433 ("$type".to_string(), AvroValue::String("map".to_string())),
8434 (
8435 "entries".to_string(),
8436 AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([
8437 ("key".to_string(), AvroValue::String("ordinary".to_string())),
8438 (
8439 "value".to_string(),
8440 AvroValue::String("user map".to_string()),
8441 ),
8442 ]))]),
8443 ),
8444 ])),
8445 ]
8446 }
8447
8448 #[derive(Clone, Debug, Default, PartialEq)]
8449 struct ReplayCounterState {
8450 loaded: Option<String>,
8451 count: i64,
8452 finished: bool,
8453 }
8454
8455 fn replay_counter_worker() -> Worker {
8456 let client = Client::new("http://127.0.0.1:8080").expect("client");
8457 let mut worker = Worker::new(client, "rust-workers");
8458 worker.register_replayed_workflow(
8459 "replay-counter",
8460 ReplayCounterState::default,
8461 |ctx, _input, state| async move {
8462 let loaded = ctx.activity("load-counter", json!([])).await?;
8463 state.update(|current| {
8464 current.loaded = loaded.as_str().map(str::to_string);
8465 })?;
8466 for _ in 0..2 {
8467 let signal = ctx.wait_signal("increment").await?;
8468 let amount = signal.first().and_then(Value::as_i64).unwrap_or_default();
8469 state.update(|current| current.count += amount)?;
8470 }
8471 state.update(|current| current.finished = true)?;
8472 state.read(|current| Ok(json!(current.count)))?
8473 },
8474 );
8475 worker.register_replayed_query::<ReplayCounterState, _, _>(
8476 "replay-counter",
8477 "current",
8478 |_ctx, state, _args| async move {
8479 Ok(json!({
8480 "loaded": state.loaded,
8481 "count": state.count,
8482 "finished": state.finished,
8483 }))
8484 },
8485 );
8486 worker.register_replayed_query::<ReplayCounterState, _, _>(
8487 "replay-counter",
8488 "detached-mutation",
8489 |_ctx, state, _args| async move {
8490 let mut detached = (*state).clone();
8491 detached.count = 999;
8492 Ok(json!(detached.count))
8493 },
8494 );
8495 worker.register_replayed_query::<ReplayCounterState, _, _>(
8496 "replay-counter",
8497 "failed-mutation",
8498 |_ctx, state, _args| async move {
8499 let mut detached = (*state).clone();
8500 detached.count = 999;
8501 Err(Error::WorkerLoop("query refused".to_string()))
8502 },
8503 );
8504 worker
8505 }
8506
8507 fn replay_counter_query(
8508 query_name: &str,
8509 history_events: Value,
8510 run_status: &str,
8511 ) -> QueryTask {
8512 let arguments = fixture_envelope(json!([]));
8513 serde_json::from_value(json!({
8514 "query_task_id": format!("query-{query_name}"),
8515 "workflow_type": "replay-counter",
8516 "query_name": query_name,
8517 "payload_codec": DEFAULT_CODEC,
8518 "workflow_arguments": arguments.clone(),
8519 "query_arguments": arguments,
8520 "history_events": history_events,
8521 "run_status": run_status,
8522 }))
8523 .expect("query task")
8524 }
8525
8526 fn workflow_context(history: Vec<HistoryEvent>) -> WorkflowContext {
8527 workflow_context_with_codec(history, DEFAULT_CODEC)
8528 }
8529
8530 fn workflow_context_with_codec(
8531 history: Vec<HistoryEvent>,
8532 payload_codec: &str,
8533 ) -> WorkflowContext {
8534 WorkflowContext {
8535 state: Arc::new(Mutex::new(
8536 WorkflowState::new_with_identity(
8537 history,
8538 None,
8539 None,
8540 "rust-workers".to_string(),
8541 payload_codec.to_string(),
8542 None,
8543 )
8544 .expect("valid workflow history"),
8545 )),
8546 }
8547 }
8548
8549 fn history_event(event_type: &str, payload: Value) -> HistoryEvent {
8550 HistoryEvent {
8551 event_type: event_type.to_string(),
8552 payload,
8553 raw: HashMap::new(),
8554 }
8555 }
8556
8557 fn workflow_task(
8558 workflow_type: &str,
8559 history_events: Vec<HistoryEvent>,
8560 payload_codec: &str,
8561 ) -> WorkflowTask {
8562 WorkflowTask {
8563 task_id: format!("wft-{workflow_type}"),
8564 workflow_id: Some(format!("wf-{workflow_type}")),
8565 run_id: Some(format!("run-{workflow_type}")),
8566 workflow_type: workflow_type.to_string(),
8567 payload_codec: payload_codec.to_string(),
8568 arguments: Some(
8569 encode_value_envelope(&json!([]), payload_codec).expect("workflow arguments"),
8570 ),
8571 total_history_events: Some(history_events.len() as u64),
8572 history_size_bytes: None,
8573 continue_as_new_recommended: None,
8574 history_budget_pressure: None,
8575 history_events,
8576 next_history_page_token: None,
8577 workflow_task_attempt: 1,
8578 workflow_signal_id: None,
8579 signal_name: None,
8580 signal_arguments: None,
8581 workflow_update_id: None,
8582 update_name: None,
8583 lease_owner: Some("rust-worker".to_string()),
8584 }
8585 }
8586
8587 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
8588 struct SideEffectProbe {
8589 request_id: String,
8590 attempt: u32,
8591 }
8592
8593 #[test]
8594 fn typed_side_effect_runs_callback_once_and_replay_skips_it() {
8595 let calls = AtomicUsize::new(0);
8596 let ctx = workflow_context(Vec::new());
8597 let value = ctx
8598 .side_effect(|| {
8599 calls.fetch_add(1, Ordering::SeqCst);
8600 SideEffectProbe {
8601 request_id: "request-42".to_string(),
8602 attempt: 3,
8603 }
8604 })
8605 .expect("first side effect");
8606 assert_eq!(value.attempt, 3);
8607 assert_eq!(calls.load(Ordering::SeqCst), 1);
8608 let commands = ctx.take_commands().expect("commands");
8609 assert_eq!(commands.len(), 1);
8610 assert_eq!(commands[0]["type"], "record_side_effect");
8611 assert_eq!(
8612 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
8613 serde_json::to_value(&value).expect("value")
8614 );
8615
8616 let replay = workflow_context(vec![history_event(
8617 "SideEffectRecorded",
8618 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
8619 )]);
8620 let replayed: SideEffectProbe = replay
8621 .side_effect(|| {
8622 calls.fetch_add(1, Ordering::SeqCst);
8623 panic!("committed side-effect callbacks must not run during replay")
8624 })
8625 .expect("replayed side effect");
8626 assert_eq!(replayed, value);
8627 assert_eq!(calls.load(Ordering::SeqCst), 1);
8628 assert!(replay.take_commands().expect("commands").is_empty());
8629 replay.ensure_history_consumed().expect("history consumed");
8630 }
8631
8632 #[test]
8633 fn side_effect_uses_avro_envelope_and_uuid_is_replay_stable() {
8634 let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
8635 let value = ctx
8636 .side_effect(|| SideEffectProbe {
8637 request_id: "avro-request".to_string(),
8638 attempt: 1,
8639 })
8640 .expect("Avro side effect");
8641 let uuid = ctx.uuid_v4().expect("deterministic UUID");
8642 let commands = ctx.take_commands().expect("commands");
8643 assert_eq!(commands.len(), 2);
8644 assert_eq!(commands[0]["result"]["codec"], DEFAULT_CODEC);
8645 assert_eq!(commands[1]["result"]["codec"], DEFAULT_CODEC);
8646 assert_eq!(
8647 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
8648 serde_json::to_value(&value).expect("value")
8649 );
8650
8651 let replay = workflow_context_with_codec(
8652 vec![
8653 history_event(
8654 "SideEffectRecorded",
8655 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
8656 ),
8657 history_event(
8658 "SideEffectRecorded",
8659 json!({"sequence": 2, "result": commands[1]["result"].clone()}),
8660 ),
8661 ],
8662 DEFAULT_CODEC,
8663 );
8664 let replayed: SideEffectProbe = replay
8665 .side_effect(|| panic!("Avro callback must not run"))
8666 .expect("replayed Avro value");
8667 let replayed_uuid = replay.uuid_v4().expect("replayed UUID");
8668 assert_eq!(replayed, value);
8669 assert_eq!(replayed_uuid, uuid);
8670 assert!(replay.take_commands().expect("commands").is_empty());
8671 }
8672
8673 #[test]
8674 fn typed_side_effect_replay_preserves_bytes_and_maps() {
8675 let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
8676 let value = ctx
8677 .side_effect_avro_value(typed_fidelity_probe)
8678 .expect("typed side effect");
8679 let commands = ctx.take_commands().expect("side-effect command");
8680 assert_eq!(
8681 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
8682 .expect("recorded side effect"),
8683 value
8684 );
8685
8686 let replay = workflow_context_with_codec(
8687 vec![history_event(
8688 "SideEffectRecorded",
8689 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
8690 )],
8691 DEFAULT_CODEC,
8692 );
8693 assert_eq!(
8694 replay
8695 .side_effect_avro_value(|| panic!("replay must not invoke callback"))
8696 .expect("replayed typed side effect"),
8697 value
8698 );
8699 }
8700
8701 #[test]
8702 fn ordered_side_effects_share_the_durable_command_stream() {
8703 let first = encode_value_envelope(&json!("first"), DEFAULT_CODEC).expect("first");
8704 let second = encode_value_envelope(&json!(29), DEFAULT_CODEC).expect("second");
8705 let ctx = workflow_context(vec![
8706 history_event(
8707 "SideEffectRecorded",
8708 json!({"sequence": 1, "result": first}),
8709 ),
8710 history_event(
8711 "SideEffectRecorded",
8712 json!({"sequence": 2, "result": second}),
8713 ),
8714 ]);
8715 let first: String = ctx
8716 .side_effect(|| panic!("first callback must not run"))
8717 .expect("first replay");
8718 let second: i32 = ctx
8719 .side_effect(|| panic!("second callback must not run"))
8720 .expect("second replay");
8721 assert_eq!(first, "first");
8722 assert_eq!(second, 29);
8723 ctx.ensure_history_consumed().expect("ordered history");
8724
8725 let reordered = workflow_context(vec![history_event(
8726 "VersionMarkerRecorded",
8727 json!({
8728 "sequence": 1,
8729 "change_id": "before-side-effect",
8730 "version": 1,
8731 "min_supported": 1,
8732 "max_supported": 1,
8733 }),
8734 )]);
8735 let error = reordered
8736 .side_effect(|| "new".to_string())
8737 .expect_err("command reordering must fail");
8738 assert!(matches!(
8739 error,
8740 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8741 if reason == "recorded_command_mismatch"
8742 ));
8743 }
8744
8745 #[test]
8746 fn version_markers_replay_across_upgrades_and_do_not_duplicate() {
8747 let ctx = workflow_context(Vec::new());
8748 assert_eq!(ctx.get_version("checkout-v2", 1, 2).expect("version"), 2);
8749 assert_eq!(ctx.get_version("checkout-v2", 1, 3).expect("cached"), 2);
8750 assert!(ctx.patched("new-search").expect("patch"));
8751 ctx.deprecate_patch("new-search").expect("deprecate patch");
8752 let commands = ctx.take_commands().expect("commands");
8753 assert_eq!(commands.len(), 2);
8754 assert_eq!(commands[0]["type"], "record_version_marker");
8755 assert_eq!(commands[0]["version"], 2);
8756 assert_eq!(commands[1]["change_id"], "new-search");
8757
8758 let replay = workflow_context(vec![history_event(
8759 "VersionMarkerRecorded",
8760 json!({
8761 "sequence": 1,
8762 "change_id": "checkout-v2",
8763 "version": 2,
8764 "min_supported": 1,
8765 "max_supported": 2,
8766 }),
8767 )]);
8768 assert_eq!(replay.get_version("checkout-v2", 1, 4).expect("upgrade"), 2);
8769 assert_eq!(replay.get_version("checkout-v2", 2, 5).expect("repeat"), 2);
8770 assert!(replay.take_commands().expect("commands").is_empty());
8771 replay.ensure_history_consumed().expect("history consumed");
8772 }
8773
8774 #[test]
8775 fn version_markers_reject_incompatible_or_malformed_history() {
8776 let incompatible = workflow_context(vec![history_event(
8777 "VersionMarkerRecorded",
8778 json!({
8779 "sequence": 1,
8780 "change_id": "checkout-v2",
8781 "version": 1,
8782 "min_supported": 1,
8783 "max_supported": 2,
8784 }),
8785 )]);
8786 let error = incompatible
8787 .get_version("checkout-v2", 2, 3)
8788 .expect_err("old version is unsupported");
8789 assert!(matches!(
8790 error,
8791 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8792 if reason == "version_marker_incompatible_range"
8793 ));
8794
8795 for (history, reason) in [
8796 (
8797 vec![history_event("SideEffectRecorded", json!({"sequence": 1}))],
8798 "side_effect_result_missing",
8799 ),
8800 (
8801 vec![history_event(
8802 "SideEffectRecorded",
8803 json!({
8804 "sequence": 1,
8805 "result": {"codec": "avro", "blob": "not-base64"},
8806 }),
8807 )],
8808 "side_effect_payload_incompatible",
8809 ),
8810 (
8811 vec![history_event(
8812 "SideEffectRecorded",
8813 json!({"sequence": 1, "result": {"unwrapped": true}}),
8814 )],
8815 "side_effect_payload_malformed",
8816 ),
8817 (
8818 vec![history_event(
8819 "VersionMarkerRecorded",
8820 json!({
8821 "sequence": 1,
8822 "change_id": "change",
8823 "version": 1,
8824 "min_supported": 2,
8825 "max_supported": 1,
8826 }),
8827 )],
8828 "version_marker_history_range_invalid",
8829 ),
8830 ] {
8831 let error = WorkflowState::new(
8832 history,
8833 "rust-workers".to_string(),
8834 DEFAULT_CODEC.to_string(),
8835 None,
8836 )
8837 .expect_err("malformed history must fail");
8838 assert!(matches!(
8839 error,
8840 Error::NonDeterministicReplay(ReplayFailure { reason: actual, .. })
8841 if actual == reason
8842 ));
8843 }
8844 }
8845
8846 #[test]
8847 fn duplicate_side_effects_and_version_markers_are_rejected() {
8848 let duplicate_side_effect = WorkflowState::new(
8849 vec![
8850 history_event(
8851 "SideEffectRecorded",
8852 json!({"sequence": 1, "result": fixture_envelope(json!(1))}),
8853 ),
8854 history_event(
8855 "SideEffectRecorded",
8856 json!({"sequence": 1, "result": fixture_envelope(json!(2))}),
8857 ),
8858 ],
8859 "rust-workers".to_string(),
8860 DEFAULT_CODEC.to_string(),
8861 None,
8862 )
8863 .expect_err("duplicate side effect");
8864 assert!(matches!(
8865 duplicate_side_effect,
8866 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8867 if reason == "duplicate_side_effect_record"
8868 ));
8869
8870 let marker = |sequence| {
8871 history_event(
8872 "VersionMarkerRecorded",
8873 json!({
8874 "sequence": sequence,
8875 "change_id": "same-change",
8876 "version": 1,
8877 "min_supported": 1,
8878 "max_supported": 1,
8879 }),
8880 )
8881 };
8882 let duplicate_marker = WorkflowState::new(
8883 vec![marker(1), marker(3)],
8884 "rust-workers".to_string(),
8885 DEFAULT_CODEC.to_string(),
8886 None,
8887 )
8888 .expect_err("duplicate marker");
8889 assert!(matches!(
8890 duplicate_marker,
8891 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8892 if reason == "duplicate_version_marker"
8893 ));
8894 }
8895
8896 #[test]
8897 fn cold_worker_replay_does_not_repeat_committed_side_effects_or_markers() {
8898 fn worker(calls: Arc<AtomicUsize>) -> Worker {
8899 let client = Client::new("http://127.0.0.1:8080").expect("client");
8900 let mut worker = Worker::new(client, "rust-workers");
8901 worker.register_workflow("rust.side-effect-version", move |ctx, _input| {
8902 let calls = Arc::clone(&calls);
8903 async move {
8904 let captured = ctx.side_effect(|| {
8905 calls.fetch_add(1, Ordering::SeqCst);
8906 "captured-once".to_string()
8907 })?;
8908 let version = ctx.get_version("cold-restart", 1, 2)?;
8909 Ok(json!({"captured": captured, "version": version}))
8910 }
8911 });
8912 worker
8913 }
8914
8915 fn task(history_events: Vec<HistoryEvent>) -> WorkflowTask {
8916 WorkflowTask {
8917 task_id: "wft-side-effect-version".to_string(),
8918 workflow_id: Some("wf-side-effect-version".to_string()),
8919 run_id: Some("run-side-effect-version".to_string()),
8920 workflow_type: "rust.side-effect-version".to_string(),
8921 payload_codec: DEFAULT_CODEC.to_string(),
8922 arguments: Some(
8923 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("arguments"),
8924 ),
8925 history_events,
8926 total_history_events: None,
8927 history_size_bytes: None,
8928 continue_as_new_recommended: None,
8929 history_budget_pressure: None,
8930 next_history_page_token: None,
8931 workflow_task_attempt: 1,
8932 workflow_signal_id: None,
8933 signal_name: None,
8934 signal_arguments: None,
8935 workflow_update_id: None,
8936 update_name: None,
8937 lease_owner: Some("rust-worker".to_string()),
8938 }
8939 }
8940
8941 let calls = Arc::new(AtomicUsize::new(0));
8942 let initial = worker(Arc::clone(&calls))
8943 .execute_workflow_task(task(Vec::new()))
8944 .expect("initial execution");
8945 assert_eq!(
8946 initial
8947 .iter()
8948 .map(|command| &command["type"])
8949 .collect::<Vec<_>>(),
8950 vec![
8951 "record_side_effect",
8952 "record_version_marker",
8953 "complete_workflow"
8954 ]
8955 );
8956 assert_eq!(calls.load(Ordering::SeqCst), 1);
8957
8958 let restarted = worker(Arc::clone(&calls));
8959 let replayed = restarted
8960 .execute_workflow_task(task(vec![
8961 history_event(
8962 "SideEffectRecorded",
8963 json!({"sequence": 1, "result": initial[0]["result"].clone()}),
8964 ),
8965 history_event(
8966 "VersionMarkerRecorded",
8967 json!({
8968 "sequence": 2,
8969 "change_id": "cold-restart",
8970 "version": 2,
8971 "min_supported": 1,
8972 "max_supported": 2,
8973 }),
8974 ),
8975 ]))
8976 .expect("cold replay");
8977 assert_eq!(replayed.len(), 1);
8978 assert_eq!(replayed[0]["type"], "complete_workflow");
8979 assert_eq!(calls.load(Ordering::SeqCst), 1);
8980 }
8981
8982 #[test]
8983 fn side_effect_replay_rejects_changed_rust_value_type() {
8984 let result = encode_value_envelope(&json!({"value": 42}), DEFAULT_CODEC).expect("result");
8985 let ctx = workflow_context(vec![history_event(
8986 "SideEffectRecorded",
8987 json!({"sequence": 1, "result": result}),
8988 )]);
8989 let error = ctx
8990 .side_effect::<Vec<String>, _>(|| panic!("callback must not run"))
8991 .expect_err("changed type must fail replay");
8992 assert!(matches!(
8993 error,
8994 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8995 if reason == "side_effect_type_mismatch"
8996 ));
8997 }
8998
8999 fn completed_retry_activity_history() -> Vec<HistoryEvent> {
9000 vec![
9001 history_event(
9002 "ActivityScheduled",
9003 json!({
9004 "sequence": 1,
9005 "activity_type": "flaky",
9006 "activity_execution_id": "act-1",
9007 "activity": {
9008 "id": "act-1",
9009 "sequence": 1,
9010 "type": "flaky",
9011 "queue": "critical-activities",
9012 "execution_mode": null,
9013 "retry_policy": {
9014 "snapshot_version": 1,
9015 "max_attempts": 3,
9016 "backoff_seconds": [2, 4],
9017 "start_to_close_timeout": 30,
9018 "schedule_to_start_timeout": 5,
9019 "schedule_to_close_timeout": 90,
9020 "heartbeat_timeout": 10,
9021 "non_retryable_error_types": ["PermanentError"]
9022 }
9023 }
9024 }),
9025 ),
9026 history_event(
9027 "ActivityStarted",
9028 json!({
9029 "sequence": 1,
9030 "activity_type": "flaky",
9031 "activity_execution_id": "act-1",
9032 "activity_attempt_id": "attempt-1",
9033 "attempt_number": 1
9034 }),
9035 ),
9036 history_event(
9037 "ActivityRetryScheduled",
9038 json!({
9039 "sequence": 1,
9040 "activity_type": "flaky",
9041 "activity_execution_id": "act-1",
9042 "activity_attempt_id": "attempt-1",
9043 "attempt_number": 1,
9044 "retry_after_attempt": 1,
9045 "retry_backoff_seconds": 2,
9046 "failure_category": "activity",
9047 "exception_type": "TransientError"
9048 }),
9049 ),
9050 history_event(
9051 "ActivityStarted",
9052 json!({
9053 "sequence": 1,
9054 "activity_type": "flaky",
9055 "activity_execution_id": "act-1",
9056 "activity_attempt_id": "attempt-2",
9057 "attempt_number": 2
9058 }),
9059 ),
9060 history_event(
9061 "ActivityCompleted",
9062 json!({
9063 "sequence": 1,
9064 "activity_type": "flaky",
9065 "activity_execution_id": "act-1",
9066 "activity_attempt_id": "attempt-2",
9067 "attempt_number": 2,
9068 "payload_codec": DEFAULT_CODEC,
9069 "result": fixture_envelope(json!({"status":"recovered"}))
9070 }),
9071 ),
9072 ]
9073 }
9074
9075 fn retry_activity_options() -> ActivityOptions {
9076 ActivityOptions::new()
9077 .task_queue("critical-activities")
9078 .retry_policy(
9079 ActivityRetryPolicy::new(3)
9080 .backoff_intervals([Duration::from_secs(2), Duration::from_secs(4)])
9081 .non_retryable_error_type("PermanentError"),
9082 )
9083 .start_to_close_timeout(Duration::from_secs(30))
9084 .schedule_to_start_timeout(Duration::from_secs(5))
9085 .schedule_to_close_timeout(Duration::from_secs(90))
9086 .heartbeat_timeout(Duration::from_secs(10))
9087 }
9088
9089 #[test]
9090 fn fixed_avro_value_round_trips_json_values() {
9091 let value = json!({"greeting": "hello", "count": 3, "ok": true});
9092 let envelope = PayloadEnvelope::avro(&value).expect("encode");
9093 assert_eq!(envelope.codec, DEFAULT_CODEC);
9094 assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
9095 }
9096
9097 #[tokio::test]
9098 async fn typed_worker_surfaces_preserve_bytes_and_map_list_identity() {
9099 let client = Client::new("http://127.0.0.1:8080").expect("client");
9100 let mut worker = Worker::new(client, "rust-workers");
9101 worker.register_workflow_avro_value("typed.echo", |_ctx, input| async move { Ok(input) });
9102 worker
9103 .register_activity_avro_value("typed.activity", |_ctx, input| async move { Ok(input) });
9104 worker.register_query_avro_value("typed.echo", "inspect", |_ctx, input| async move {
9105 Ok(input)
9106 });
9107 worker.register_update_avro_value("typed.echo", "replace", |_ctx, input| async move {
9108 Ok(input)
9109 });
9110 worker.register_workflow_avro_value("typed.signal", |ctx, _input| async move {
9111 Ok(AvroValue::Array(
9112 ctx.wait_signal_avro_value("changed").await?,
9113 ))
9114 });
9115
9116 let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
9117 let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed envelope");
9118
9119 let mut workflow = workflow_task("typed.echo", Vec::new(), DEFAULT_CODEC);
9120 workflow.arguments = Some(envelope.clone());
9121 let commands = worker
9122 .execute_workflow_task(workflow)
9123 .expect("typed workflow task");
9124 assert_eq!(commands[0]["type"], "complete_workflow");
9125 assert_eq!(
9126 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
9127 .expect("typed workflow result"),
9128 arguments
9129 );
9130
9131 let activity = ActivityTask {
9132 task_id: "activity-typed".to_string(),
9133 activity_attempt_id: Some("attempt-typed".to_string()),
9134 attempt_id: None,
9135 activity_type: "typed.activity".to_string(),
9136 payload_codec: DEFAULT_CODEC.to_string(),
9137 arguments: Some(envelope.clone()),
9138 attempt_number: 1,
9139 lease_owner: Some("rust-worker".to_string()),
9140 };
9141 assert_eq!(
9142 worker
9143 .execute_activity_task(activity)
9144 .await
9145 .expect("typed activity result"),
9146 arguments
9147 );
9148
9149 let query = QueryTask {
9150 query_task_id: "query-typed".to_string(),
9151 query_task_attempt: 1,
9152 lease_owner: Some("rust-worker".to_string()),
9153 workflow_id: Some("typed-1".to_string()),
9154 run_id: Some("run-typed".to_string()),
9155 workflow_type: "typed.echo".to_string(),
9156 query_name: "inspect".to_string(),
9157 payload_codec: DEFAULT_CODEC.to_string(),
9158 workflow_arguments: Some(
9159 encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
9160 .expect("workflow input"),
9161 ),
9162 query_arguments: Some(envelope.clone()),
9163 history_events: Vec::new(),
9164 history_export: None,
9165 run_status: Some("running".to_string()),
9166 };
9167 assert_eq!(
9168 worker
9169 .execute_query_task(query)
9170 .await
9171 .expect("typed query result"),
9172 arguments
9173 );
9174
9175 let mut update = workflow_task(
9176 "typed.echo",
9177 vec![history_event(
9178 "UpdateAccepted",
9179 json!({
9180 "update_id": "update-typed",
9181 "update_name": "replace",
9182 "arguments": envelope.clone(),
9183 }),
9184 )],
9185 DEFAULT_CODEC,
9186 );
9187 update.workflow_update_id = Some("update-typed".to_string());
9188 update.update_name = Some("replace".to_string());
9189 let commands = worker
9190 .execute_workflow_task(update)
9191 .expect("typed update task");
9192 assert_eq!(commands[0]["type"], "complete_update");
9193 assert_eq!(
9194 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
9195 .expect("typed update result"),
9196 arguments
9197 );
9198
9199 let mut signal = workflow_task(
9200 "typed.signal",
9201 vec![history_event(
9202 "SignalReceived",
9203 json!({
9204 "signal_id": "signal-typed",
9205 "signal_name": "changed",
9206 "arguments": envelope.clone(),
9207 }),
9208 )],
9209 DEFAULT_CODEC,
9210 );
9211 signal.workflow_signal_id = Some("signal-typed".to_string());
9212 signal.signal_name = Some("changed".to_string());
9213 signal.signal_arguments = Some(envelope);
9214 let commands = worker
9215 .execute_workflow_task(signal)
9216 .expect("typed signal resume");
9217 assert_eq!(
9218 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
9219 .expect("typed signal result"),
9220 arguments
9221 );
9222 }
9223
9224 #[tokio::test]
9225 async fn typed_helpers_never_parse_json_inspection_projection() {
9226 let collision_values = projection_collision_probe();
9227 let expected = AvroValue::Array(collision_values.clone());
9228 let envelope = encode_typed_envelope(&expected, DEFAULT_CODEC).expect("collision envelope");
9229
9230 let activity_context = workflow_context_with_codec(
9231 vec![history_event(
9232 "ActivityCompleted",
9233 json!({
9234 "sequence": 1,
9235 "activity_type": "collision.activity",
9236 "payload_codec": DEFAULT_CODEC,
9237 "result": envelope.clone(),
9238 }),
9239 )],
9240 DEFAULT_CODEC,
9241 );
9242 assert_eq!(
9243 activity_context
9244 .activity_avro_value("collision.activity", AvroValue::Array(Vec::new()))
9245 .await
9246 .expect("typed activity collision result"),
9247 expected
9248 );
9249
9250 let signal_context = workflow_context_with_codec(
9251 vec![
9252 history_event(
9253 "SignalWaitOpened",
9254 json!({"sequence": 1, "signal_name": "collision"}),
9255 ),
9256 history_event(
9257 "SignalApplied",
9258 json!({
9259 "sequence": 1,
9260 "signal_name": "collision",
9261 "payload_codec": DEFAULT_CODEC,
9262 "value": envelope.clone(),
9263 }),
9264 ),
9265 ],
9266 DEFAULT_CODEC,
9267 );
9268 assert_eq!(
9269 signal_context
9270 .wait_signal_avro_value("collision")
9271 .await
9272 .expect("typed signal collision arguments"),
9273 collision_values
9274 );
9275
9276 let child_context = workflow_context_with_codec(
9277 vec![
9278 history_event(
9279 "ChildWorkflowScheduled",
9280 json!({
9281 "sequence": 1,
9282 "child_workflow_instance_id": "collision-child",
9283 "child_workflow_run_id": "collision-run",
9284 "child_workflow_type": "collision.child",
9285 }),
9286 ),
9287 history_event(
9288 "ChildRunCompleted",
9289 json!({
9290 "sequence": 1,
9291 "child_workflow_instance_id": "collision-child",
9292 "child_workflow_run_id": "collision-run",
9293 "child_workflow_type": "collision.child",
9294 "payload_codec": DEFAULT_CODEC,
9295 "result": envelope,
9296 }),
9297 ),
9298 ],
9299 DEFAULT_CODEC,
9300 );
9301 let child = child_context
9302 .start_child_workflow_avro_value(
9303 "collision.child",
9304 ChildWorkflowOptions::new("collision-workers"),
9305 AvroValue::Array(Vec::new()),
9306 )
9307 .await
9308 .expect("typed child collision result");
9309 assert_eq!(child.result, expected);
9310 }
9311
9312 #[tokio::test]
9313 async fn replayed_typed_query_keeps_lossless_workflow_and_query_inputs() {
9314 let client = Client::new("http://127.0.0.1:8080").expect("client");
9315 let mut worker = Worker::new(client, "rust-workers");
9316 worker.register_replayed_workflow_avro_value(
9317 "typed.replayed",
9318 || (),
9319 |_ctx, input, _state| async move { Ok(input) },
9320 );
9321 worker.register_replayed_query_avro_value::<(), _, _>(
9322 "typed.replayed",
9323 "inspect",
9324 |ctx, _state, args| async move {
9325 let mut signals = ctx.signals_avro_value("collision");
9326 let signal = signals
9327 .pop()
9328 .map(AvroValue::Array)
9329 .unwrap_or_else(|| AvroValue::Array(Vec::new()));
9330 Ok(AvroValue::Array(vec![
9331 ctx.workflow_input_avro_value().clone(),
9332 signal,
9333 args,
9334 ]))
9335 },
9336 );
9337 let arguments = AvroValue::Array(projection_collision_probe());
9338 let signal_arguments =
9339 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed query signal arguments");
9340 let task = QueryTask {
9341 query_task_id: "query-typed-replay".to_string(),
9342 query_task_attempt: 1,
9343 lease_owner: Some("rust-worker".to_string()),
9344 workflow_id: Some("typed-replay".to_string()),
9345 run_id: Some("run-typed-replay".to_string()),
9346 workflow_type: "typed.replayed".to_string(),
9347 query_name: "inspect".to_string(),
9348 payload_codec: DEFAULT_CODEC.to_string(),
9349 workflow_arguments: Some(
9350 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("workflow arguments"),
9351 ),
9352 query_arguments: Some(
9353 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("query arguments"),
9354 ),
9355 history_events: vec![history_event(
9356 "SignalReceived",
9357 json!({
9358 "signal_id": "collision-signal",
9359 "signal_name": "collision",
9360 "workflow_sequence": 1,
9361 "payload_codec": DEFAULT_CODEC,
9362 "arguments": signal_arguments,
9363 }),
9364 )],
9365 history_export: None,
9366 run_status: Some("completed".to_string()),
9367 };
9368
9369 assert_eq!(
9370 worker
9371 .execute_query_task(task)
9372 .await
9373 .expect("typed replay query"),
9374 AvroValue::Array(vec![arguments.clone(), arguments.clone(), arguments])
9375 );
9376 }
9377
9378 #[test]
9379 fn public_avro_adapter_rejects_non_string_map_keys_before_json_conversion() {
9380 let value = BTreeMap::from([(1_i32, "integer key")]);
9381 let error = PayloadEnvelope::avro(&value)
9382 .expect_err("integer map keys must fail")
9383 .to_string();
9384
9385 assert!(error.contains("invalid_map_key"));
9386 }
9387
9388 #[test]
9389 fn json_tagged_payload_fails_closed_with_actionable_diagnostic() {
9390 let envelope = PayloadEnvelope {
9391 codec: "json".to_string(),
9392 blob: r#"{"greeting":"hello"}"#.to_string(),
9393 };
9394
9395 let error = decode_payload::<Value>(&envelope).expect_err("JSON payload must fail");
9396 let diagnostic = error.to_string();
9397 assert!(diagnostic.contains("unsupported_payload_codec"));
9398 assert!(diagnostic.contains("codec=\"avro\""));
9399 assert!(diagnostic.contains("HTTP document transport"));
9400 }
9401
9402 #[test]
9403 fn untagged_json_payload_value_fails_closed() {
9404 let error = decode_wire_value(&json!({"stale": true}), DEFAULT_CODEC)
9405 .expect_err("untagged JSON payload values must fail");
9406 let diagnostic = error.to_string();
9407 assert!(diagnostic.contains("unsupported_payload_codec"));
9408 assert!(diagnostic.contains("untagged durable payload"));
9409 assert!(diagnostic.contains("HTTP document transport"));
9410 }
9411
9412 #[test]
9413 fn prerelease_avro_payload_without_single_object_frame_is_rejected() {
9414 let envelope = PayloadEnvelope {
9415 codec: DEFAULT_CODEC.to_string(),
9416 blob: BASE64.encode([0x01]),
9417 };
9418
9419 let error = decode_payload::<Value>(&envelope).expect_err("prerelease payload must fail");
9420 assert!(error.to_string().contains("invalid_payload_framing"));
9421 }
9422
9423 #[test]
9424 fn workflow_context_schedules_activity_until_completion_is_in_history() {
9425 let ctx = WorkflowContext {
9426 state: Arc::new(Mutex::new(
9427 WorkflowState::new_with_identity(
9428 Vec::new(),
9429 Some("wf-parent".to_string()),
9430 Some("run-parent".to_string()),
9431 "rust-workers".to_string(),
9432 DEFAULT_CODEC.to_string(),
9433 None,
9434 )
9435 .expect("workflow state"),
9436 )),
9437 };
9438
9439 let mut call = Box::pin(ctx.activity("hello.activity", json!(["Ada"])));
9440 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9441 assert!(matches!(
9442 call.as_mut().poll(&mut task_context),
9443 Poll::Pending
9444 ));
9445
9446 let commands = ctx.take_commands().expect("commands");
9447 assert_eq!(commands[0]["type"], "schedule_activity");
9448 assert_eq!(commands[0]["activity_type"], "hello.activity");
9449 }
9450
9451 #[test]
9452 fn activity_options_encode_retry_policy_queue_and_every_timeout() {
9453 let ctx = workflow_context(Vec::new());
9454 let options = ActivityOptions::new()
9455 .task_queue("payments")
9456 .retry_policy(
9457 ActivityRetryPolicy::new(4)
9458 .exponential_backoff(Duration::from_secs(1), 3, Some(Duration::from_secs(10)))
9459 .non_retryable_error_type("ValidationError"),
9460 )
9461 .start_to_close_timeout(Duration::from_secs(120))
9462 .schedule_to_start_timeout(Duration::from_secs(10))
9463 .schedule_to_close_timeout(Duration::from_secs(300))
9464 .heartbeat_timeout(Duration::from_secs(15));
9465 let mut call = Box::pin(ctx.activity_with_options(
9466 "charge-card",
9467 options,
9468 json!([{"order_id": "o-1"}]),
9469 ));
9470 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9471
9472 assert!(matches!(
9473 call.as_mut().poll(&mut task_context),
9474 Poll::Pending
9475 ));
9476 assert!(matches!(
9477 call.as_mut().poll(&mut task_context),
9478 Poll::Pending
9479 ));
9480
9481 let commands = ctx.take_commands().expect("activity command");
9482 assert_eq!(commands.len(), 1, "one future emits one logical schedule");
9483 assert_eq!(commands[0]["queue"], "payments");
9484 assert_eq!(
9485 commands[0]["retry_policy"],
9486 json!({
9487 "max_attempts": 4,
9488 "backoff_seconds": [1, 3, 9],
9489 "non_retryable_error_types": ["ValidationError"],
9490 })
9491 );
9492 assert_eq!(commands[0]["start_to_close_timeout"], 120);
9493 assert_eq!(commands[0]["schedule_to_start_timeout"], 10);
9494 assert_eq!(commands[0]["schedule_to_close_timeout"], 300);
9495 assert_eq!(commands[0]["heartbeat_timeout"], 15);
9496 }
9497
9498 #[test]
9499 fn activity_options_encode_explicit_and_rounded_backoff_intervals() {
9500 let ctx = workflow_context(Vec::new());
9501 let options = ActivityOptions::new().retry_policy(
9502 ActivityRetryPolicy::new(3)
9503 .backoff_intervals([Duration::from_millis(1), Duration::from_millis(1_001)]),
9504 );
9505 let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
9506 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9507
9508 assert!(matches!(
9509 call.as_mut().poll(&mut task_context),
9510 Poll::Pending
9511 ));
9512 assert_eq!(
9513 ctx.take_commands().expect("command")[0]["retry_policy"]["backoff_seconds"],
9514 json!([1, 2])
9515 );
9516 }
9517
9518 #[test]
9519 fn invalid_activity_options_return_typed_errors_before_emitting_commands() {
9520 let cases = [
9521 (
9522 ActivityOptions::new().task_queue(" "),
9523 ActivityOptionsErrorKind::EmptyTaskQueue,
9524 ),
9525 (
9526 ActivityOptions::new().retry_policy(ActivityRetryPolicy::default()),
9527 ActivityOptionsErrorKind::EmptyRetryPolicy,
9528 ),
9529 (
9530 ActivityOptions::new().retry_policy(ActivityRetryPolicy::new(0)),
9531 ActivityOptionsErrorKind::InvalidMaxAttempts,
9532 ),
9533 (
9534 ActivityOptions::new().retry_policy(ActivityRetryPolicy {
9535 max_attempts: None,
9536 backoff: Some(ActivityBackoff::Explicit(vec![Duration::from_secs(1)])),
9537 non_retryable_error_types: Vec::new(),
9538 }),
9539 ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
9540 ),
9541 (
9542 ActivityOptions::new().retry_policy(
9543 ActivityRetryPolicy::new(2)
9544 .backoff_intervals([Duration::from_secs(1), Duration::from_secs(2)]),
9545 ),
9546 ActivityOptionsErrorKind::TooManyBackoffIntervals,
9547 ),
9548 (
9549 ActivityOptions::new().retry_policy(
9550 ActivityRetryPolicy::new(2).exponential_backoff(
9551 Duration::from_secs(1),
9552 0,
9553 None,
9554 ),
9555 ),
9556 ActivityOptionsErrorKind::InvalidBackoffCoefficient,
9557 ),
9558 (
9559 ActivityOptions::new()
9560 .retry_policy(ActivityRetryPolicy::new(2).non_retryable_error_type(" ")),
9561 ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
9562 ),
9563 (
9564 ActivityOptions::new().retry_policy(
9565 ActivityRetryPolicy::new(10_002).exponential_backoff(
9566 Duration::from_secs(1),
9567 1,
9568 None,
9569 ),
9570 ),
9571 ActivityOptionsErrorKind::BackoffGenerationTooLarge,
9572 ),
9573 (
9574 ActivityOptions::new().retry_policy(
9575 ActivityRetryPolicy::new(2)
9576 .backoff_intervals([Duration::from_secs(i64::MAX as u64 + 1)]),
9577 ),
9578 ActivityOptionsErrorKind::BackoffOverflow,
9579 ),
9580 ];
9581
9582 for (options, expected_kind) in cases {
9583 let ctx = workflow_context(Vec::new());
9584 let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
9585 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9586 let Poll::Ready(Err(Error::InvalidActivityOptions(error))) =
9587 call.as_mut().poll(&mut task_context)
9588 else {
9589 panic!("expected typed activity validation error");
9590 };
9591 assert_eq!(error.kind, expected_kind);
9592 assert!(ctx.take_commands().expect("commands").is_empty());
9593 }
9594 }
9595
9596 #[test]
9597 fn activity_options_validate_positive_and_ordered_timeouts() {
9598 let zero_timeout_cases = [
9599 ActivityOptions::new().start_to_close_timeout(Duration::ZERO),
9600 ActivityOptions::new().schedule_to_start_timeout(Duration::ZERO),
9601 ActivityOptions::new().schedule_to_close_timeout(Duration::ZERO),
9602 ActivityOptions::new().heartbeat_timeout(Duration::ZERO),
9603 ];
9604 for options in zero_timeout_cases {
9605 assert_eq!(
9606 options.validate().expect_err("zero timeout").kind,
9607 ActivityOptionsErrorKind::TimeoutNotPositive
9608 );
9609 }
9610
9611 let ordering_cases = [
9612 ActivityOptions::new()
9613 .heartbeat_timeout(Duration::from_secs(11))
9614 .start_to_close_timeout(Duration::from_secs(10)),
9615 ActivityOptions::new()
9616 .start_to_close_timeout(Duration::from_secs(31))
9617 .schedule_to_close_timeout(Duration::from_secs(30)),
9618 ActivityOptions::new()
9619 .schedule_to_start_timeout(Duration::from_secs(31))
9620 .schedule_to_close_timeout(Duration::from_secs(30)),
9621 ];
9622 for options in ordering_cases {
9623 assert_eq!(
9624 options.validate().expect_err("timeout order").kind,
9625 ActivityOptionsErrorKind::TimeoutOrder
9626 );
9627 }
9628
9629 assert_eq!(
9630 ActivityOptions::new()
9631 .start_to_close_timeout(Duration::from_secs(i64::MAX as u64 + 1))
9632 .validate()
9633 .expect_err("protocol integer overflow")
9634 .kind,
9635 ActivityOptionsErrorKind::TimeoutOverflow
9636 );
9637 }
9638
9639 #[test]
9640 fn replayed_activity_retry_history_completes_without_duplicate_schedule() {
9641 let ctx = workflow_context(completed_retry_activity_history());
9642 let mut call =
9643 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
9644 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9645
9646 assert!(matches!(
9647 call.as_mut().poll(&mut task_context),
9648 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
9649 ));
9650 assert!(ctx.take_commands().expect("commands").is_empty());
9651 ctx.ensure_history_consumed().expect("history consumed");
9652 }
9653
9654 #[test]
9655 fn duplicate_non_retryable_types_use_one_command_and_replay_representation() {
9656 let mut options = retry_activity_options();
9657 options
9658 .retry_policy
9659 .as_mut()
9660 .expect("retry policy")
9661 .non_retryable_error_types
9662 .extend([" PermanentError ".to_string(), "PermanentError".to_string()]);
9663
9664 let new_ctx = workflow_context(Vec::new());
9665 let mut new_call =
9666 Box::pin(new_ctx.activity_with_options("flaky", options.clone(), json!([])));
9667 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9668 assert!(matches!(
9669 new_call.as_mut().poll(&mut task_context),
9670 Poll::Pending
9671 ));
9672 let commands = new_ctx.take_commands().expect("commands");
9673 assert_eq!(commands.len(), 1);
9674 assert_eq!(
9675 commands[0]["retry_policy"]["non_retryable_error_types"],
9676 json!(["PermanentError"])
9677 );
9678
9679 let replay_ctx = workflow_context(completed_retry_activity_history());
9680 let mut replay_call =
9681 Box::pin(replay_ctx.activity_with_options("flaky", options, json!([])));
9682 assert!(matches!(
9683 replay_call.as_mut().poll(&mut task_context),
9684 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
9685 ));
9686 assert!(replay_ctx.take_commands().expect("commands").is_empty());
9687 replay_ctx
9688 .ensure_history_consumed()
9689 .expect("history consumed");
9690 }
9691
9692 #[test]
9693 fn replayed_intermediate_retry_remains_pending_across_restarts() {
9694 let history = completed_retry_activity_history()
9695 .into_iter()
9696 .take(3)
9697 .collect::<Vec<_>>();
9698
9699 for _restart in 0..2 {
9700 let ctx = workflow_context(history.clone());
9701 let mut call =
9702 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
9703 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9704 assert!(matches!(
9705 call.as_mut().poll(&mut task_context),
9706 Poll::Pending
9707 ));
9708 assert!(ctx.take_commands().expect("commands").is_empty());
9709 }
9710 }
9711
9712 #[test]
9713 fn replayed_activity_rejects_changed_queue_retry_and_every_timeout_field() {
9714 let mut changed_queue = retry_activity_options();
9715 changed_queue.task_queue = Some("different-queue".to_string());
9716
9717 let mut changed_max_attempts = retry_activity_options();
9718 let retry_policy = changed_max_attempts
9719 .retry_policy
9720 .as_mut()
9721 .expect("retry policy");
9722 retry_policy.max_attempts = Some(4);
9723
9724 let mut changed_backoff = retry_activity_options();
9725 let retry_policy = changed_backoff.retry_policy.as_mut().expect("retry policy");
9726 retry_policy.backoff = Some(ActivityBackoff::Explicit(vec![
9727 Duration::from_secs(3),
9728 Duration::from_secs(4),
9729 ]));
9730
9731 let mut changed_non_retryable_types = retry_activity_options();
9732 let retry_policy = changed_non_retryable_types
9733 .retry_policy
9734 .as_mut()
9735 .expect("retry policy");
9736 retry_policy.non_retryable_error_types = vec!["AnotherPermanentError".to_string()];
9737
9738 let mut changed_start_to_close = retry_activity_options();
9739 changed_start_to_close.start_to_close_timeout = Some(Duration::from_secs(31));
9740 let mut changed_schedule_to_start = retry_activity_options();
9741 changed_schedule_to_start.schedule_to_start_timeout = Some(Duration::from_secs(6));
9742 let mut changed_schedule_to_close = retry_activity_options();
9743 changed_schedule_to_close.schedule_to_close_timeout = Some(Duration::from_secs(91));
9744 let mut changed_heartbeat = retry_activity_options();
9745 changed_heartbeat.heartbeat_timeout = Some(Duration::from_secs(11));
9746
9747 let cases = [
9748 (changed_queue, "activity_task_queue_mismatch"),
9749 (changed_max_attempts, "activity_retry_policy_mismatch"),
9750 (changed_backoff, "activity_retry_policy_mismatch"),
9751 (
9752 changed_non_retryable_types,
9753 "activity_retry_policy_mismatch",
9754 ),
9755 (changed_start_to_close, "activity_retry_policy_mismatch"),
9756 (changed_schedule_to_start, "activity_retry_policy_mismatch"),
9757 (changed_schedule_to_close, "activity_retry_policy_mismatch"),
9758 (changed_heartbeat, "activity_retry_policy_mismatch"),
9759 ];
9760
9761 for (options, expected_reason) in cases {
9762 let ctx = workflow_context(completed_retry_activity_history());
9763 let mut call = Box::pin(ctx.activity_with_options("flaky", options, json!([])));
9764 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9765 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
9766 call.as_mut().poll(&mut task_context)
9767 else {
9768 panic!("changed activity options must fail replay");
9769 };
9770 assert_eq!(failure.reason, expected_reason);
9771 assert_eq!(failure.sequence, Some(1));
9772 assert!(ctx.take_commands().expect("commands").is_empty());
9773 }
9774 }
9775
9776 #[test]
9777 fn replayed_activity_rejects_changed_execution_mode_and_snapshot_version() {
9778 let cases = [
9779 (
9780 "execution_mode",
9781 json!("local"),
9782 "activity_execution_mode_mismatch",
9783 ),
9784 (
9785 "snapshot_version",
9786 json!(2),
9787 "activity_retry_policy_mismatch",
9788 ),
9789 ];
9790
9791 for (field, value, expected_reason) in cases {
9792 let mut history = completed_retry_activity_history();
9793 let activity = history[0].payload["activity"]
9794 .as_object_mut()
9795 .expect("activity snapshot");
9796 if field == "execution_mode" {
9797 activity.insert(field.to_string(), value);
9798 } else {
9799 activity["retry_policy"]
9800 .as_object_mut()
9801 .expect("retry snapshot")
9802 .insert(field.to_string(), value);
9803 }
9804
9805 let ctx = workflow_context(history);
9806 let mut call =
9807 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
9808 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9809 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
9810 call.as_mut().poll(&mut task_context)
9811 else {
9812 panic!("changed {field} must fail replay");
9813 };
9814 assert_eq!(failure.reason, expected_reason);
9815 assert_eq!(failure.sequence, Some(1));
9816 assert!(ctx.take_commands().expect("commands").is_empty());
9817 }
9818 }
9819
9820 #[test]
9821 fn replayed_legacy_activity_treats_missing_option_snapshot_as_unknown() {
9822 let mut history = completed_retry_activity_history();
9823 let activity = history[0].payload["activity"]
9824 .as_object_mut()
9825 .expect("activity snapshot");
9826 activity.remove("execution_mode");
9827 activity.remove("retry_policy");
9828
9829 let mut current = retry_activity_options();
9830 current.start_to_close_timeout = Some(Duration::from_secs(45));
9831 current.schedule_to_start_timeout = Some(Duration::from_secs(8));
9832 current.schedule_to_close_timeout = Some(Duration::from_secs(120));
9833 current.heartbeat_timeout = Some(Duration::from_secs(12));
9834
9835 let ctx = workflow_context(history);
9836 let mut call = Box::pin(ctx.activity_with_options("flaky", current, json!([])));
9837 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9838 assert!(matches!(
9839 call.as_mut().poll(&mut task_context),
9840 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
9841 ));
9842 assert!(ctx.take_commands().expect("commands").is_empty());
9843 ctx.ensure_history_consumed().expect("history consumed");
9844 }
9845
9846 #[test]
9847 fn terminal_activity_failed_after_start_returns_typed_failure() {
9848 let history = vec![
9849 history_event(
9850 "ActivityScheduled",
9851 json!({
9852 "sequence": 1,
9853 "activity_type": "flaky",
9854 "activity_execution_id": "act-terminal",
9855 "activity": {
9856 "id": "act-terminal",
9857 "sequence": 1,
9858 "type": "flaky",
9859 "queue": "critical-activities",
9860 "retry_policy": {
9861 "snapshot_version": 1,
9862 "max_attempts": 3,
9863 "backoff_seconds": [2, 4],
9864 "non_retryable_error_types": ["PermanentError"]
9865 }
9866 }
9867 }),
9868 ),
9869 history_event(
9870 "ActivityStarted",
9871 json!({
9872 "sequence": 1,
9873 "activity_type": "flaky",
9874 "activity_execution_id": "act-terminal",
9875 "activity_attempt_id": "attempt-1",
9876 "attempt_number": 1
9877 }),
9878 ),
9879 history_event(
9880 "ActivityFailed",
9881 json!({
9882 "sequence": 1,
9883 "activity_type": "flaky",
9884 "activity_execution_id": "act-terminal",
9885 "activity_attempt_id": "attempt-1",
9886 "attempt_number": 1,
9887 "failure_id": "failure-terminal",
9888 "failure_category": "activity",
9889 "exception_type": "PermanentError",
9890 "message": "cannot retry",
9891 "non_retryable": true
9892 }),
9893 ),
9894 ];
9895 let ctx = workflow_context(history);
9896 let mut call =
9897 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
9898 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9899
9900 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
9901 call.as_mut().poll(&mut task_context)
9902 else {
9903 panic!("terminal ActivityFailed must settle the activity future");
9904 };
9905 assert_eq!(failure.kind, ActivityFailureKind::Failed);
9906 assert_eq!(
9907 failure.activity_execution_id.as_deref(),
9908 Some("act-terminal")
9909 );
9910 assert_eq!(failure.exception_type.as_deref(), Some("PermanentError"));
9911 assert!(failure.non_retryable);
9912 assert!(ctx.take_commands().expect("commands").is_empty());
9913 ctx.ensure_history_consumed().expect("history consumed");
9914 }
9915
9916 #[test]
9917 fn activity_terminal_events_return_machine_readable_failures() {
9918 let cases = [
9919 (
9920 "ActivityFailed",
9921 json!({
9922 "sequence": 1,
9923 "activity_type": "charge-card",
9924 "activity_execution_id": "act-1",
9925 "activity_attempt_id": "attempt-2",
9926 "attempt_number": 2,
9927 "failure_id": "failure-1",
9928 "failure_category": "activity",
9929 "exception_type": "PaymentDeclined",
9930 "exception_class": "payments.PaymentDeclined",
9931 "message": "card declined",
9932 "non_retryable": true
9933 }),
9934 ActivityFailureKind::Failed,
9935 "activity",
9936 ),
9937 (
9938 "ActivityCancelled",
9939 json!({
9940 "sequence": 1,
9941 "activity_type": "charge-card",
9942 "activity_execution_id": "act-1",
9943 "activity_attempt_id": "attempt-1"
9944 }),
9945 ActivityFailureKind::Cancelled,
9946 "cancelled",
9947 ),
9948 ];
9949
9950 for (event_type, payload, expected_kind, expected_reason) in cases {
9951 let ctx = workflow_context(vec![history_event(event_type, payload)]);
9952 let mut call = Box::pin(ctx.activity("charge-card", json!([])));
9953 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9954 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
9955 call.as_mut().poll(&mut task_context)
9956 else {
9957 panic!("expected terminal activity failure");
9958 };
9959 assert_eq!(failure.kind, expected_kind);
9960 assert_eq!(failure.reason, expected_reason);
9961 assert_eq!(failure.activity_execution_id.as_deref(), Some("act-1"));
9962 assert_eq!(failure.activity_type.as_deref(), Some("charge-card"));
9963 }
9964 }
9965
9966 #[test]
9967 fn every_activity_timeout_class_is_typed() {
9968 for timeout_kind in [
9969 "start_to_close",
9970 "schedule_to_start",
9971 "schedule_to_close",
9972 "heartbeat",
9973 ] {
9974 let ctx = workflow_context(vec![history_event(
9975 "ActivityTimedOut",
9976 json!({
9977 "sequence": 1,
9978 "activity_type": "slow",
9979 "activity_execution_id": "act-timeout",
9980 "activity_attempt_id": "attempt-timeout",
9981 "failure_category": "timeout",
9982 "timeout_kind": timeout_kind,
9983 "message": "deadline expired"
9984 }),
9985 )]);
9986 let mut call = Box::pin(ctx.activity("slow", json!([])));
9987 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9988 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
9989 call.as_mut().poll(&mut task_context)
9990 else {
9991 panic!("expected timeout failure");
9992 };
9993 assert_eq!(failure.kind, ActivityFailureKind::TimedOut);
9994 assert_eq!(failure.reason, timeout_kind);
9995 assert_eq!(failure.timeout_kind.as_deref(), Some(timeout_kind));
9996 assert_eq!(failure.failure_category.as_deref(), Some("timeout"));
9997 }
9998 }
9999
10000 #[test]
10001 fn workflow_sleep_emits_one_durable_timer_and_rounds_up() {
10002 let ctx = workflow_context(Vec::new());
10003 let mut sleep = Box::pin(ctx.sleep(Duration::from_millis(1_001)));
10004 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10005
10006 assert!(matches!(
10007 sleep.as_mut().poll(&mut task_context),
10008 Poll::Pending
10009 ));
10010 assert!(matches!(
10011 sleep.as_mut().poll(&mut task_context),
10012 Poll::Pending
10013 ));
10014
10015 let commands = ctx.take_commands().expect("timer command");
10016 assert_eq!(
10017 commands,
10018 vec![json!({
10019 "type": "start_timer",
10020 "delay_seconds": 2,
10021 })]
10022 );
10023 }
10024
10025 #[test]
10026 fn workflow_sleep_replays_matching_schedule_and_fire_without_a_command() {
10027 let history = vec![
10028 history_event(
10029 "TimerScheduled",
10030 json!({
10031 "sequence": 1,
10032 "timer_id": "timer-1",
10033 "delay_seconds": 5,
10034 "fire_at": "2026-07-11T12:00:05Z",
10035 }),
10036 ),
10037 history_event(
10038 "TimerFired",
10039 json!({
10040 "sequence": 1,
10041 "timer_id": "timer-1",
10042 "delay_seconds": 5,
10043 "fire_at": "2026-07-11T12:00:05Z",
10044 "fired_at": "2026-07-11T12:00:05Z",
10045 }),
10046 ),
10047 ];
10048
10049 for _restart in 0..2 {
10050 let ctx = workflow_context(history.clone());
10051 let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(5)));
10052 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10053 assert!(matches!(
10054 sleep.as_mut().poll(&mut task_context),
10055 Poll::Ready(Ok(()))
10056 ));
10057 assert!(ctx.take_commands().expect("commands").is_empty());
10058 ctx.ensure_history_consumed().expect("history consumed");
10059 }
10060 }
10061
10062 #[test]
10063 fn workflow_sleep_rejects_changed_delay_during_replay() {
10064 let ctx = workflow_context(vec![
10065 history_event(
10066 "TimerScheduled",
10067 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10068 ),
10069 history_event(
10070 "TimerFired",
10071 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10072 ),
10073 ]);
10074 let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(500)));
10075 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10076
10077 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10078 sleep.as_mut().poll(&mut task_context)
10079 else {
10080 panic!("changed timer delay must be rejected");
10081 };
10082 assert_eq!(failure.reason, "timer_delay_mismatch");
10083 assert_eq!(failure.sequence, Some(1));
10084 }
10085
10086 #[test]
10087 fn workflow_history_rejects_unpaired_or_mismatched_timer_events() {
10088 let lone_fire = WorkflowState::new(
10089 vec![history_event(
10090 "TimerFired",
10091 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10092 )],
10093 "rust-workers".to_string(),
10094 DEFAULT_CODEC.to_string(),
10095 None,
10096 )
10097 .expect_err("TimerFired requires TimerScheduled");
10098 assert!(matches!(
10099 lone_fire,
10100 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10101 if reason == "timer_schedule_missing_or_duplicate"
10102 ));
10103
10104 let wrong_identity = WorkflowState::new(
10105 vec![
10106 history_event(
10107 "TimerScheduled",
10108 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10109 ),
10110 history_event(
10111 "TimerFired",
10112 json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
10113 ),
10114 ],
10115 "rust-workers".to_string(),
10116 DEFAULT_CODEC.to_string(),
10117 None,
10118 )
10119 .expect_err("fire must match scheduled timer identity");
10120 assert!(matches!(
10121 wrong_identity,
10122 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10123 if reason == "timer_identity_mismatch"
10124 ));
10125
10126 let duplicate_fire = WorkflowState::new(
10127 vec![
10128 history_event(
10129 "TimerScheduled",
10130 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10131 ),
10132 history_event(
10133 "TimerFired",
10134 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10135 ),
10136 history_event(
10137 "TimerFired",
10138 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10139 ),
10140 ],
10141 "rust-workers".to_string(),
10142 DEFAULT_CODEC.to_string(),
10143 None,
10144 )
10145 .expect_err("a durable timer cannot fire twice");
10146 assert!(matches!(
10147 duplicate_fire,
10148 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10149 if reason == "duplicate_timer_fire"
10150 ));
10151
10152 let wrong_fired_delay = WorkflowState::new(
10153 vec![
10154 history_event(
10155 "TimerScheduled",
10156 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10157 ),
10158 history_event(
10159 "TimerFired",
10160 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 6}),
10161 ),
10162 ],
10163 "rust-workers".to_string(),
10164 DEFAULT_CODEC.to_string(),
10165 None,
10166 )
10167 .expect_err("timer schedule and fire delays must agree");
10168 assert!(matches!(
10169 wrong_fired_delay,
10170 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10171 if reason == "timer_history_delay_mismatch"
10172 ));
10173 }
10174
10175 #[test]
10176 fn replay_rejects_activity_moved_before_recorded_timer() {
10177 let ctx = workflow_context(vec![
10178 history_event(
10179 "TimerScheduled",
10180 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10181 ),
10182 history_event(
10183 "TimerFired",
10184 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10185 ),
10186 history_event(
10187 "ActivityCompleted",
10188 json!({
10189 "sequence": 2,
10190 "activity_type": "after-timer",
10191 "payload_codec": DEFAULT_CODEC,
10192 "result": fixture_envelope(json!("done")),
10193 }),
10194 ),
10195 ]);
10196 let mut activity = Box::pin(ctx.activity("after-timer", json!([])));
10197 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10198
10199 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10200 activity.as_mut().poll(&mut task_context)
10201 else {
10202 panic!("reordered durable command must be rejected");
10203 };
10204 assert_eq!(failure.reason, "recorded_command_mismatch");
10205 assert_eq!(failure.sequence, Some(1));
10206 assert_eq!(failure.expected.as_deref(), Some("timer"));
10207 assert_eq!(failure.actual.as_deref(), Some("activity:after-timer"));
10208 }
10209
10210 #[test]
10211 fn workflow_context_emits_a_typed_named_signal_wait() {
10212 let ctx = workflow_context(Vec::new());
10213 let mut signal = Box::pin(ctx.wait_signal("finish"));
10214 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10215
10216 assert!(matches!(
10217 signal.as_mut().poll(&mut task_context),
10218 Poll::Pending
10219 ));
10220 assert_eq!(
10221 ctx.take_commands().expect("signal-wait command"),
10222 vec![json!({
10223 "type": "open_signal_wait",
10224 "signal_name": "finish",
10225 })]
10226 );
10227 }
10228
10229 #[test]
10230 fn condition_wait_history_does_not_resolve_a_typed_signal_wait() {
10231 let ctx = workflow_context(vec![
10232 history_event(
10233 "ConditionWaitOpened",
10234 json!({"sequence": 1, "condition_key": "signal:finish"}),
10235 ),
10236 history_event(
10237 "ConditionWaitSatisfied",
10238 json!({"sequence": 1, "condition_key": "signal:finish"}),
10239 ),
10240 history_event(
10241 "SignalReceived",
10242 json!({"signal_name": "finish", "arguments": []}),
10243 ),
10244 ]);
10245 let mut signal = Box::pin(ctx.wait_signal("finish"));
10246 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10247
10248 assert!(matches!(
10249 signal.as_mut().poll(&mut task_context),
10250 Poll::Pending
10251 ));
10252 assert_eq!(
10253 ctx.take_commands().expect("typed signal-wait command"),
10254 vec![json!({
10255 "type": "open_signal_wait",
10256 "signal_name": "finish",
10257 })]
10258 );
10259 }
10260
10261 #[test]
10262 fn replay_orders_signal_waits_and_timers_in_one_command_stream() {
10263 let signal_then_timer = vec![
10264 history_event(
10265 "SignalWaitOpened",
10266 json!({"sequence": 1, "signal_name": "go"}),
10267 ),
10268 history_event(
10269 "SignalApplied",
10270 json!({
10271 "sequence": 1,
10272 "signal_name": "go",
10273 "value": fixture_envelope(json!(["now"])),
10274 }),
10275 ),
10276 history_event(
10277 "TimerScheduled",
10278 json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
10279 ),
10280 history_event(
10281 "TimerFired",
10282 json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
10283 ),
10284 ];
10285
10286 let ctx = workflow_context(signal_then_timer.clone());
10287 let mut signal = Box::pin(ctx.wait_signal("go"));
10288 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10289 assert!(matches!(
10290 signal.as_mut().poll(&mut task_context),
10291 Poll::Ready(Ok(arguments)) if arguments == vec![json!("now")]
10292 ));
10293 let mut timer = Box::pin(ctx.sleep(Duration::from_secs(5)));
10294 assert!(matches!(
10295 timer.as_mut().poll(&mut task_context),
10296 Poll::Ready(Ok(()))
10297 ));
10298 ctx.ensure_history_consumed()
10299 .expect("signal and timer history consumed in order");
10300
10301 let reordered = workflow_context(signal_then_timer);
10302 let mut timer_first = Box::pin(reordered.sleep(Duration::from_secs(5)));
10303 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10304 timer_first.as_mut().poll(&mut task_context)
10305 else {
10306 panic!("timer cannot consume signal-wait-first history");
10307 };
10308 assert_eq!(failure.reason, "recorded_command_mismatch");
10309 assert_eq!(failure.sequence, Some(1));
10310 assert_eq!(failure.expected.as_deref(), Some("signal wait"));
10311
10312 let timer_then_signal = vec![
10313 history_event(
10314 "TimerScheduled",
10315 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10316 ),
10317 history_event(
10318 "TimerFired",
10319 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10320 ),
10321 history_event(
10322 "SignalWaitOpened",
10323 json!({"sequence": 2, "signal_name": "go"}),
10324 ),
10325 history_event(
10326 "SignalApplied",
10327 json!({
10328 "sequence": 2,
10329 "signal_name": "go",
10330 "value": fixture_envelope(json!([])),
10331 }),
10332 ),
10333 ];
10334 let reordered = workflow_context(timer_then_signal);
10335 let mut signal_first = Box::pin(reordered.wait_signal("go"));
10336 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10337 signal_first.as_mut().poll(&mut task_context)
10338 else {
10339 panic!("signal wait cannot consume timer-first history");
10340 };
10341 assert_eq!(failure.reason, "recorded_command_mismatch");
10342 assert_eq!(failure.sequence, Some(1));
10343 assert_eq!(failure.expected.as_deref(), Some("timer"));
10344 }
10345
10346 #[test]
10347 fn workflow_history_rejects_duplicate_or_colliding_command_sequences() {
10348 let duplicate_timer = WorkflowState::new(
10349 vec![
10350 history_event(
10351 "TimerScheduled",
10352 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10353 ),
10354 history_event(
10355 "TimerScheduled",
10356 json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
10357 ),
10358 ],
10359 "rust-workers".to_string(),
10360 DEFAULT_CODEC.to_string(),
10361 None,
10362 )
10363 .expect_err("one workflow sequence cannot schedule two timers");
10364 assert!(matches!(
10365 duplicate_timer,
10366 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10367 if reason == "timer_schedule_missing_or_duplicate"
10368 ));
10369
10370 let colliding_kinds = WorkflowState::new(
10371 vec![
10372 history_event(
10373 "TimerScheduled",
10374 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10375 ),
10376 history_event(
10377 "ActivityCompleted",
10378 json!({"sequence": 1, "activity_type": "same-sequence"}),
10379 ),
10380 ],
10381 "rust-workers".to_string(),
10382 DEFAULT_CODEC.to_string(),
10383 None,
10384 )
10385 .expect_err("one workflow sequence cannot identify two command kinds");
10386 assert!(matches!(
10387 colliding_kinds,
10388 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10389 if reason == "durable_command_sequence_collision"
10390 ));
10391
10392 let duplicate_signal_wait = WorkflowState::new(
10393 vec![
10394 history_event(
10395 "SignalWaitOpened",
10396 json!({"sequence": 1, "signal_name": "go"}),
10397 ),
10398 history_event(
10399 "SignalWaitOpened",
10400 json!({"sequence": 1, "signal_name": "go"}),
10401 ),
10402 ],
10403 "rust-workers".to_string(),
10404 DEFAULT_CODEC.to_string(),
10405 None,
10406 )
10407 .expect_err("one workflow sequence cannot open two signal waits");
10408 assert!(matches!(
10409 duplicate_signal_wait,
10410 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10411 if reason == "signal_wait_open_missing_or_duplicate"
10412 ));
10413 }
10414
10415 #[test]
10416 fn workflow_history_accepts_a_first_command_after_global_sequence_gaps() {
10417 let result = encode_value_envelope(&json!({"captured": true}), DEFAULT_CODEC)
10418 .expect("side-effect result");
10419 let ctx = workflow_context(vec![history_event(
10420 "SideEffectRecorded",
10421 json!({"sequence": 99, "result": result}),
10422 )]);
10423
10424 let replayed: Value = ctx
10425 .side_effect(|| panic!("recorded side effect must not run"))
10426 .expect("positive global workflow sequence is valid");
10427 assert_eq!(replayed, json!({"captured": true}));
10428 ctx.ensure_history_consumed().expect("history consumed");
10429 }
10430
10431 #[test]
10432 fn workflow_history_rejects_zero_and_descending_command_sequences() {
10433 let result =
10434 encode_value_envelope(&json!("captured"), DEFAULT_CODEC).expect("side-effect result");
10435 let zero = WorkflowState::new(
10436 vec![history_event(
10437 "SideEffectRecorded",
10438 json!({"sequence": 0, "result": result.clone()}),
10439 )],
10440 "rust-workers".to_string(),
10441 DEFAULT_CODEC.to_string(),
10442 None,
10443 )
10444 .expect_err("durable command sequences must be positive");
10445 assert!(matches!(
10446 zero,
10447 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10448 if reason == "durable_command_sequence_invalid"
10449 ));
10450
10451 let descending = WorkflowState::new(
10452 vec![
10453 history_event(
10454 "SideEffectRecorded",
10455 json!({"sequence": 3, "result": result}),
10456 ),
10457 history_event(
10458 "VersionMarkerRecorded",
10459 json!({
10460 "sequence": 2,
10461 "change_id": "descending-marker",
10462 "version": 1,
10463 "min_supported": 1,
10464 "max_supported": 1,
10465 }),
10466 ),
10467 ],
10468 "rust-workers".to_string(),
10469 DEFAULT_CODEC.to_string(),
10470 None,
10471 )
10472 .expect_err("new durable commands must remain strictly ordered");
10473 let Error::NonDeterministicReplay(failure) = descending else {
10474 panic!("expected typed replay failure");
10475 };
10476 assert_eq!(failure.reason, "durable_command_sequence_mismatch");
10477 assert_eq!(failure.sequence, Some(2));
10478 assert_eq!(
10479 failure.expected.as_deref(),
10480 Some("workflow sequence greater than 3")
10481 );
10482 assert_eq!(failure.actual.as_deref(), Some("2"));
10483 }
10484
10485 #[test]
10486 fn workflow_task_replay_completes_after_signals_create_sequence_gaps() {
10487 fn worker() -> Worker {
10488 let client = Client::new("http://127.0.0.1:8080").expect("client");
10489 let mut worker = Worker::new(client, "rust-workers");
10490 worker.register_workflow("rust.finish-after-gaps", |ctx, _input| async move {
10491 ctx.wait_signal("finish").await?;
10492 let marker: String =
10493 ctx.side_effect(|| panic!("recorded side effect must not run"))?;
10494 assert_eq!(marker, "after-finish");
10495 Ok(json!("finished"))
10496 });
10497 worker
10498 }
10499
10500 let marker = encode_value_envelope(&json!("after-finish"), DEFAULT_CODEC)
10501 .expect("side-effect result");
10502 let task = workflow_task(
10503 "rust.finish-after-gaps",
10504 vec![
10505 history_event(
10506 "SignalWaitOpened",
10507 json!({"sequence": 1, "signal_name": "finish"}),
10508 ),
10509 history_event(
10510 "SignalReceived",
10511 json!({
10512 "signal_id": "increment-3",
10513 "signal_name": "increment",
10514 "workflow_sequence": 2,
10515 "payload_codec": DEFAULT_CODEC,
10516 "arguments": fixture_envelope(json!([3])),
10517 }),
10518 ),
10519 history_event(
10520 "SignalReceived",
10521 json!({
10522 "signal_id": "increment-5",
10523 "signal_name": "increment",
10524 "workflow_sequence": 3,
10525 "payload_codec": DEFAULT_CODEC,
10526 "arguments": fixture_envelope(json!([5])),
10527 }),
10528 ),
10529 history_event(
10530 "SignalReceived",
10531 json!({
10532 "signal_id": "finish",
10533 "signal_name": "finish",
10534 "workflow_sequence": 4,
10535 "payload_codec": DEFAULT_CODEC,
10536 "arguments": fixture_envelope(json!([])),
10537 }),
10538 ),
10539 history_event(
10540 "SignalApplied",
10541 json!({
10542 "sequence": 1,
10543 "signal_id": "finish",
10544 "signal_name": "finish",
10545 "payload_codec": DEFAULT_CODEC,
10546 "value": fixture_envelope(json!([])),
10547 }),
10548 ),
10549 history_event(
10550 "SideEffectRecorded",
10551 json!({"sequence": 5, "result": marker}),
10552 ),
10553 ],
10554 DEFAULT_CODEC,
10555 );
10556
10557 for _original_or_cold_worker in 0..2 {
10558 let commands = worker()
10559 .execute_workflow_task(task.clone())
10560 .expect("signal gaps preserve deterministic replay");
10561 assert_eq!(commands.len(), 1, "replay emits only terminal completion");
10562 assert_eq!(commands[0]["type"], "complete_workflow");
10563 assert_eq!(
10564 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("workflow output"),
10565 json!("finished")
10566 );
10567 }
10568 }
10569
10570 #[test]
10571 fn workflow_sleep_rejects_unrepresentable_rounded_duration() {
10572 let ctx = workflow_context(Vec::new());
10573 let mut sleep = Box::pin(ctx.start_timer(Duration::new(u64::MAX, 1)));
10574 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10575 assert!(matches!(
10576 sleep.as_mut().poll(&mut task_context),
10577 Poll::Ready(Err(Error::TimerDurationOverflow))
10578 ));
10579 assert!(ctx.take_commands().expect("commands").is_empty());
10580 }
10581
10582 #[test]
10583 fn workflow_task_replay_completes_without_rescheduling_recorded_commands() {
10584 let client = Client::new("http://127.0.0.1:8080").expect("client");
10585 let mut worker = Worker::new(client, "rust-workers");
10586 worker.register_workflow("rust.timer", |ctx, _input| async move {
10587 ctx.sleep(Duration::from_secs(5)).await?;
10588 ctx.activity("after-timer", json!([])).await
10589 });
10590
10591 let task = |history_events| WorkflowTask {
10592 task_id: "wft-rust-timer-1".to_string(),
10593 workflow_id: Some("wf-rust-timer".to_string()),
10594 run_id: Some("run-rust-timer".to_string()),
10595 workflow_type: "rust.timer".to_string(),
10596 payload_codec: DEFAULT_CODEC.to_string(),
10597 arguments: Some(
10598 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
10599 ),
10600 history_events,
10601 total_history_events: None,
10602 history_size_bytes: None,
10603 continue_as_new_recommended: None,
10604 history_budget_pressure: None,
10605 next_history_page_token: None,
10606 workflow_task_attempt: 1,
10607 workflow_signal_id: None,
10608 signal_name: None,
10609 signal_arguments: None,
10610 workflow_update_id: None,
10611 update_name: None,
10612 lease_owner: Some("rust-worker".to_string()),
10613 };
10614
10615 let initial = worker
10616 .execute_workflow_task(task(Vec::new()))
10617 .expect("initial timer task");
10618 assert_eq!(
10619 initial,
10620 vec![json!({"type": "start_timer", "delay_seconds": 5})]
10621 );
10622
10623 let activity_result =
10624 encode_value_envelope(&json!("done"), DEFAULT_CODEC).expect("activity result");
10625 let replayed = worker
10626 .execute_workflow_task(task(vec![
10627 history_event(
10628 "TimerScheduled",
10629 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10630 ),
10631 history_event(
10632 "TimerFired",
10633 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10634 ),
10635 history_event(
10636 "ActivityCompleted",
10637 json!({
10638 "sequence": 2,
10639 "activity_type": "after-timer",
10640 "payload_codec": DEFAULT_CODEC,
10641 "result": activity_result,
10642 }),
10643 ),
10644 ]))
10645 .expect("replayed workflow task");
10646 assert_eq!(replayed.len(), 1);
10647 assert_eq!(replayed[0]["type"], "complete_workflow");
10648 assert_eq!(
10649 decode_wire_value(&replayed[0]["result"], DEFAULT_CODEC).expect("result"),
10650 json!("done")
10651 );
10652 }
10653
10654 #[test]
10655 fn workflow_continue_as_new_emits_arguments_type_and_queue_once() {
10656 let client = Client::new("http://127.0.0.1:8080").expect("client");
10657 let mut worker = Worker::new(client, "rust-workers");
10658 worker.register_workflow("rust.continue", |ctx, _input| async move {
10659 ctx.continue_as_new_with_options(
10660 ContinueAsNewOptions::new()
10661 .workflow_type("rust.next")
10662 .task_queue("next-workers"),
10663 json!([2, {"cursor": "next"}]),
10664 )
10665 });
10666
10667 let commands = worker
10668 .execute_workflow_task(workflow_task("rust.continue", Vec::new(), DEFAULT_CODEC))
10669 .expect("continue-as-new command");
10670
10671 assert_eq!(commands.len(), 1);
10672 assert_eq!(commands[0]["type"], "continue_as_new");
10673 assert_eq!(commands[0]["workflow_type"], "rust.next");
10674 assert_eq!(commands[0]["queue"], "next-workers");
10675 assert_eq!(
10676 decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC)
10677 .expect("continue-as-new arguments"),
10678 json!([2, {"cursor": "next"}])
10679 );
10680 }
10681
10682 #[test]
10683 fn continue_as_new_preserves_typed_arguments() {
10684 let client = Client::new("http://127.0.0.1:8080").expect("client");
10685 let mut worker = Worker::new(client, "rust-workers");
10686 worker.register_workflow_avro_value("rust.typed-continue", |ctx, _input| async move {
10687 ctx.continue_as_new(AvroValue::Array(vec![typed_fidelity_probe()]))?;
10688 unreachable!("continue-as-new returns a control-flow error")
10689 });
10690
10691 let commands = worker
10692 .execute_workflow_task(workflow_task(
10693 "rust.typed-continue",
10694 Vec::new(),
10695 DEFAULT_CODEC,
10696 ))
10697 .expect("typed continue-as-new command");
10698
10699 assert_eq!(commands[0]["type"], "continue_as_new");
10700 assert_eq!(
10701 decode_wire_avro_value(&commands[0]["arguments"], DEFAULT_CODEC)
10702 .expect("typed continue arguments"),
10703 AvroValue::Array(vec![typed_fidelity_probe()])
10704 );
10705 }
10706
10707 #[test]
10708 fn recorded_continue_as_new_is_consumed_without_duplicate_successor_command() {
10709 let client = Client::new("http://127.0.0.1:8080").expect("client");
10710 let mut worker = Worker::new(client, "rust-workers");
10711 worker.register_workflow("rust.continue", |ctx, _input| async move {
10712 ctx.continue_as_new(json!([2]))
10713 });
10714 let task = workflow_task(
10715 "rust.continue",
10716 vec![history_event(
10717 "WorkflowContinuedAsNew",
10718 json!({"sequence": 1, "continued_to_run_id": "run-next"}),
10719 )],
10720 DEFAULT_CODEC,
10721 );
10722
10723 for _worker_restart_or_redelivery in 0..2 {
10724 let commands = worker
10725 .execute_workflow_task(task.clone())
10726 .expect("recorded transition replays");
10727 assert!(
10728 commands.is_empty(),
10729 "replay must not emit another successor"
10730 );
10731 }
10732 }
10733
10734 #[test]
10735 fn continue_as_new_rejects_invalid_overrides_before_emitting_a_command() {
10736 let ctx = workflow_context(Vec::new());
10737 let error = ctx
10738 .continue_as_new_with_options(ContinueAsNewOptions::new().task_queue(" "), json!([1]))
10739 .expect_err("blank queue must be rejected");
10740
10741 let Error::InvalidContinueAsNewOptions(error) = error else {
10742 panic!("expected typed continue-as-new validation error");
10743 };
10744 assert_eq!(error.field, "task_queue");
10745 assert!(ctx.take_commands().expect("commands").is_empty());
10746 }
10747
10748 #[test]
10749 fn workflow_context_exposes_server_history_budget() {
10750 let client = Client::new("http://127.0.0.1:8080").expect("client");
10751 let mut worker = Worker::new(client, "rust-workers");
10752 worker.register_workflow("rust.history-budget", |ctx, _input| async move {
10753 let budget = ctx.history_budget()?;
10754 Ok(json!({
10755 "events": budget.event_count,
10756 "bytes": budget.size_bytes,
10757 "recommended": budget.continue_as_new_recommended,
10758 "pressure": budget.pressure,
10759 }))
10760 });
10761 let task: WorkflowTask = serde_json::from_value(json!({
10762 "task_id": "task-history-budget",
10763 "workflow_type": "rust.history-budget",
10764 "payload_codec": DEFAULT_CODEC,
10765 "history_events": [],
10766 "total_history_events": 480,
10767 "history_size_bytes": 1_048_576,
10768 "continue_as_new_recommended": true,
10769 "history_budget_pressure": "continue_as_new_recommended",
10770 }))
10771 .expect("published workflow task");
10772
10773 let commands = worker
10774 .execute_workflow_task(task)
10775 .expect("history-budget workflow");
10776 let result = decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("result");
10777 assert_eq!(result["events"], 480);
10778 assert_eq!(result["bytes"], 1_048_576);
10779 assert_eq!(result["recommended"], true);
10780 assert_eq!(result["pressure"], "continue_as_new_recommended");
10781 }
10782
10783 #[test]
10784 fn uncaught_workflow_handler_error_emits_terminal_failure_command() {
10785 let client = Client::new("http://127.0.0.1:8080").expect("client");
10786 let mut worker = Worker::new(client, "rust-workers");
10787 worker.register_workflow("rust.failing", |_ctx, _input| async move {
10788 Err(Error::Codec("rust_conformance_failure".to_string()))
10789 });
10790 let task = WorkflowTask {
10791 task_id: "wft-rust-failing-1".to_string(),
10792 workflow_id: Some("wf-rust-failing".to_string()),
10793 run_id: Some("run-rust-failing".to_string()),
10794 workflow_type: "rust.failing".to_string(),
10795 payload_codec: DEFAULT_CODEC.to_string(),
10796 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
10797 history_events: Vec::new(),
10798 total_history_events: Some(0),
10799 history_size_bytes: None,
10800 continue_as_new_recommended: None,
10801 history_budget_pressure: None,
10802 next_history_page_token: None,
10803 workflow_task_attempt: 1,
10804 workflow_signal_id: None,
10805 signal_name: None,
10806 signal_arguments: None,
10807 workflow_update_id: None,
10808 update_name: None,
10809 lease_owner: Some("rust-worker".to_string()),
10810 };
10811
10812 let commands = worker
10813 .execute_workflow_task(task)
10814 .expect("handler failure becomes a workflow command");
10815
10816 assert_eq!(commands.len(), 1);
10817 assert_eq!(commands[0]["type"], "fail_workflow");
10818 assert_eq!(commands[0]["exception_type"], "RustWorkflowError");
10819 assert_eq!(commands[0]["exception_class"], "durable_workflow::Error");
10820 assert_eq!(commands[0]["non_retryable"], false);
10821 assert_eq!(
10822 commands[0]["message"],
10823 "codec error: rust_conformance_failure"
10824 );
10825 assert_eq!(
10826 commands[0]["exception"]["message"],
10827 "codec error: rust_conformance_failure"
10828 );
10829 }
10830
10831 #[test]
10832 fn ordinary_handler_error_preserves_commands_queued_in_the_same_decision() {
10833 let client = Client::new("http://127.0.0.1:8080").expect("client");
10834 let mut worker = Worker::new(client, "rust-workers");
10835 worker.register_workflow("rust.failing-after-side-effect", |ctx, _input| async move {
10836 let _: String = ctx.side_effect(|| "captured".to_string())?;
10837 Err(Error::WorkerLoop("application failure".to_string()))
10838 });
10839
10840 let commands = worker
10841 .execute_workflow_task(workflow_task(
10842 "rust.failing-after-side-effect",
10843 Vec::new(),
10844 DEFAULT_CODEC,
10845 ))
10846 .expect("ordinary failure remains a workflow decision");
10847
10848 assert_eq!(commands.len(), 2);
10849 assert_eq!(commands[0]["type"], "record_side_effect");
10850 assert_eq!(commands[1]["type"], "fail_workflow");
10851 }
10852
10853 #[test]
10854 fn handler_error_cannot_hide_an_unconsumed_committed_side_effect() {
10855 let client = Client::new("http://127.0.0.1:8080").expect("client");
10856 let mut worker = Worker::new(client, "rust-workers");
10857 worker.register_workflow("rust.removed-side-effect", |_ctx, _input| async move {
10858 Err(Error::WorkerLoop("application failure".to_string()))
10859 });
10860 let result =
10861 encode_value_envelope(&json!("committed"), DEFAULT_CODEC).expect("side-effect result");
10862
10863 let error = worker
10864 .execute_workflow_task(workflow_task(
10865 "rust.removed-side-effect",
10866 vec![history_event(
10867 "SideEffectRecorded",
10868 json!({"sequence": 1, "result": result}),
10869 )],
10870 DEFAULT_CODEC,
10871 ))
10872 .expect_err("removed committed history must not become fail_workflow");
10873
10874 let Error::NonDeterministicReplay(failure) = error else {
10875 panic!("expected typed replay failure");
10876 };
10877 assert_eq!(failure.reason, "recorded_commands_unconsumed");
10878 assert_eq!(failure.sequence, Some(1));
10879 assert_eq!(failure.expected.as_deref(), Some("side effect"));
10880 }
10881
10882 #[test]
10883 fn replay_error_discards_side_effect_queued_before_incompatible_marker_check() {
10884 let client = Client::new("http://127.0.0.1:8080").expect("client");
10885 let mut worker = Worker::new(client, "rust-workers");
10886 worker.register_workflow(
10887 "rust.side-effect-before-marker-error",
10888 |ctx, _input| async move {
10889 assert_eq!(ctx.get_version("restart-safe", 1, 1)?, 1);
10890 let _: String = ctx.side_effect(|| "must-not-commit".to_string())?;
10891 ctx.get_version("restart-safe", 2, 2)?;
10892 Ok(Value::Null)
10893 },
10894 );
10895
10896 let error = worker
10897 .execute_workflow_task(workflow_task(
10898 "rust.side-effect-before-marker-error",
10899 vec![history_event(
10900 "VersionMarkerRecorded",
10901 json!({
10902 "sequence": 1,
10903 "change_id": "restart-safe",
10904 "version": 1,
10905 "min_supported": 1,
10906 "max_supported": 1,
10907 }),
10908 )],
10909 DEFAULT_CODEC,
10910 ))
10911 .expect_err("replay error must return no queued workflow commands");
10912
10913 let Error::NonDeterministicReplay(failure) = error else {
10914 panic!("expected typed replay failure");
10915 };
10916 assert_eq!(failure.reason, "version_marker_incompatible_range");
10917 assert_eq!(failure.sequence, Some(1));
10918 }
10919
10920 #[test]
10921 fn workflow_task_replay_keeps_recorded_unfired_timer_pending_without_rescheduling() {
10922 let client = Client::new("http://127.0.0.1:8080").expect("client");
10923 let mut worker = Worker::new(client, "rust-workers");
10924 worker.register_workflow("rust.timer.pending", |ctx, _input| async move {
10925 ctx.sleep(Duration::from_secs(5)).await?;
10926 Ok(json!({"status": "timer fired"}))
10927 });
10928
10929 let task = WorkflowTask {
10930 task_id: "wft-rust-timer-pending".to_string(),
10931 workflow_id: Some("wf-rust-timer".to_string()),
10932 run_id: Some("run-rust-timer".to_string()),
10933 workflow_type: "rust.timer.pending".to_string(),
10934 payload_codec: DEFAULT_CODEC.to_string(),
10935 arguments: Some(
10936 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
10937 ),
10938 history_events: vec![history_event(
10939 "TimerScheduled",
10940 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10941 )],
10942 total_history_events: Some(1),
10943 history_size_bytes: None,
10944 continue_as_new_recommended: None,
10945 history_budget_pressure: None,
10946 next_history_page_token: None,
10947 workflow_task_attempt: 1,
10948 workflow_signal_id: None,
10949 signal_name: None,
10950 signal_arguments: None,
10951 workflow_update_id: None,
10952 update_name: None,
10953 lease_owner: Some("rust-worker".to_string()),
10954 };
10955
10956 for _redelivery_or_restart in 0..2 {
10957 let commands = worker
10958 .execute_workflow_task(task.clone())
10959 .expect("recorded timer remains pending");
10960 assert!(
10961 commands.is_empty(),
10962 "recorded timer must not be rescheduled"
10963 );
10964 }
10965 }
10966
10967 #[test]
10968 fn workflow_task_rejects_recorded_command_removed_from_workflow_code() {
10969 let client = Client::new("http://127.0.0.1:8080").expect("client");
10970 let mut worker = Worker::new(client, "rust-workers");
10971 worker.register_workflow("rust.timer.removed", |_ctx, _input| async move {
10972 Ok(json!({"status": "completed"}))
10973 });
10974 let task = WorkflowTask {
10975 task_id: "wft-rust-timer-removed".to_string(),
10976 workflow_id: Some("wf-rust-timer".to_string()),
10977 run_id: Some("run-rust-timer".to_string()),
10978 workflow_type: "rust.timer.removed".to_string(),
10979 payload_codec: DEFAULT_CODEC.to_string(),
10980 arguments: Some(
10981 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
10982 ),
10983 history_events: vec![
10984 history_event(
10985 "TimerScheduled",
10986 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10987 ),
10988 history_event(
10989 "TimerFired",
10990 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10991 ),
10992 ],
10993 total_history_events: Some(2),
10994 history_size_bytes: None,
10995 continue_as_new_recommended: None,
10996 history_budget_pressure: None,
10997 next_history_page_token: None,
10998 workflow_task_attempt: 1,
10999 workflow_signal_id: None,
11000 signal_name: None,
11001 signal_arguments: None,
11002 workflow_update_id: None,
11003 update_name: None,
11004 lease_owner: Some("rust-worker".to_string()),
11005 };
11006
11007 let Error::NonDeterministicReplay(failure) = worker
11008 .execute_workflow_task(task)
11009 .expect_err("removed timer must fail replay")
11010 else {
11011 panic!("expected typed replay failure");
11012 };
11013 assert_eq!(failure.reason, "recorded_commands_unconsumed");
11014 assert_eq!(failure.sequence, Some(1));
11015 }
11016
11017 #[test]
11018 fn workflow_context_emits_explicit_child_workflow_contract() {
11019 let ctx = WorkflowContext {
11020 state: Arc::new(Mutex::new(
11021 WorkflowState::new_with_identity(
11022 Vec::new(),
11023 Some("wf-parent".to_string()),
11024 Some("run-parent".to_string()),
11025 "parent-workers".to_string(),
11026 DEFAULT_CODEC.to_string(),
11027 None,
11028 )
11029 .expect("workflow state"),
11030 )),
11031 };
11032 let options = ChildWorkflowOptions::new("python-workers")
11033 .parent_close_policy(ParentClosePolicy::RequestCancel)
11034 .retry_policy(ChildWorkflowRetryPolicy {
11035 max_attempts: Some(3),
11036 backoff_seconds: vec![1, 5],
11037 non_retryable_error_types: vec!["ValidationError".to_string()],
11038 })
11039 .execution_timeout_seconds(600)
11040 .run_timeout_seconds(120);
11041 let mut call = Box::pin(ctx.start_child_workflow(
11042 "python.fulfil-order",
11043 options,
11044 json!([{"order_id": "order-42"}]),
11045 ));
11046 let mut task_context = TaskContext::from_waker(noop_waker_ref());
11047
11048 assert!(matches!(
11049 call.as_mut().poll(&mut task_context),
11050 Poll::Pending
11051 ));
11052 let commands = ctx.take_commands().expect("commands");
11053 assert_eq!(commands.len(), 1);
11054 let command = &commands[0];
11055 assert_eq!(command["type"], "start_child_workflow");
11056 assert_eq!(command["workflow_type"], "python.fulfil-order");
11057 assert_eq!(command["queue"], "python-workers");
11058 assert_eq!(command["parent_close_policy"], "request_cancel");
11059 assert_eq!(command["retry_policy"]["max_attempts"], 3);
11060 assert_eq!(command["execution_timeout_seconds"], 600);
11061 assert_eq!(command["run_timeout_seconds"], 120);
11062 assert_eq!(
11063 decode_wire_value(&command["arguments"], DEFAULT_CODEC).expect("child args"),
11064 json!([{"order_id": "order-42"}])
11065 );
11066 }
11067
11068 fn child_parent_worker() -> Worker {
11069 let client = Client::new("http://127.0.0.1:8080").expect("client");
11070 let mut worker = Worker::new(client, "rust-parent-workers");
11071 worker.register_workflow("rust.parent", |ctx, _input| async move {
11072 let child = ctx
11073 .start_child_workflow(
11074 "python.child",
11075 ChildWorkflowOptions::new("python-child-workers")
11076 .parent_close_policy(ParentClosePolicy::Terminate),
11077 json!([{"codec_probe": [1, true, "rust"]}]),
11078 )
11079 .await?;
11080 Ok(json!({
11081 "parent_workflow_id": child.parent.workflow_id,
11082 "parent_run_id": child.parent.run_id,
11083 "child_workflow_id": child.child.workflow_id,
11084 "child_run_id": child.child.run_id,
11085 "child_workflow_type": child.child_workflow_type,
11086 "result": child.result,
11087 }))
11088 });
11089 worker
11090 }
11091
11092 fn child_parent_task(event_type: &str, payload: Value) -> WorkflowTask {
11093 WorkflowTask {
11094 task_id: "wft-child-parent".to_string(),
11095 workflow_id: Some("wf-parent".to_string()),
11096 run_id: Some("run-parent".to_string()),
11097 workflow_type: "rust.parent".to_string(),
11098 payload_codec: DEFAULT_CODEC.to_string(),
11099 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
11100 history_events: vec![
11101 HistoryEvent {
11102 event_type: "ChildWorkflowScheduled".to_string(),
11103 payload: json!({
11104 "sequence": 1,
11105 "child_call_id": "call-child",
11106 "child_workflow_instance_id": "wf-child",
11107 "child_workflow_run_id": "run-child",
11108 "child_workflow_type": "python.child",
11109 }),
11110 raw: HashMap::new(),
11111 },
11112 HistoryEvent {
11113 event_type: event_type.to_string(),
11114 payload,
11115 raw: HashMap::new(),
11116 },
11117 ],
11118 total_history_events: Some(2),
11119 history_size_bytes: None,
11120 continue_as_new_recommended: None,
11121 history_budget_pressure: None,
11122 next_history_page_token: None,
11123 workflow_task_attempt: 1,
11124 workflow_signal_id: None,
11125 signal_name: None,
11126 signal_arguments: None,
11127 workflow_update_id: None,
11128 update_name: None,
11129 lease_owner: Some("rust-worker".to_string()),
11130 }
11131 }
11132
11133 #[test]
11134 fn committed_child_result_replays_without_starting_a_duplicate() {
11135 let worker = child_parent_worker();
11136 let task = child_parent_task(
11137 "ChildRunCompleted",
11138 json!({
11139 "sequence": 1,
11140 "child_call_id": "call-child",
11141 "child_workflow_instance_id": "wf-child",
11142 "child_workflow_run_id": "run-child",
11143 "child_workflow_type": "python.child",
11144 "payload_codec": DEFAULT_CODEC,
11145 "result": fixture_envelope(json!({"from":"python","ok":true})),
11146 }),
11147 );
11148
11149 for _restart in 0..2 {
11150 let commands = worker
11151 .execute_workflow_task(task.clone())
11152 .expect("replayed parent task");
11153 assert_eq!(commands.len(), 1);
11154 assert_eq!(commands[0]["type"], "complete_workflow");
11155 assert!(!commands
11156 .iter()
11157 .any(|command| command["type"] == "start_child_workflow"));
11158 let output =
11159 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
11160 assert_eq!(output["parent_workflow_id"], "wf-parent");
11161 assert_eq!(output["parent_run_id"], "run-parent");
11162 assert_eq!(output["child_workflow_id"], "wf-child");
11163 assert_eq!(output["child_run_id"], "run-child");
11164 assert_eq!(output["result"], json!({"from": "python", "ok": true}));
11165 }
11166 }
11167
11168 #[test]
11169 fn typed_child_arguments_and_results_survive_replay() {
11170 let client = Client::new("http://127.0.0.1:8080").expect("client");
11171 let mut worker = Worker::new(client, "rust-parent-workers");
11172 worker.register_workflow_avro_value("rust.typed-parent", |ctx, _input| async move {
11173 let child = ctx
11174 .start_child_workflow_avro_value(
11175 "python.typed-child",
11176 ChildWorkflowOptions::new("python-workers"),
11177 AvroValue::Array(vec![typed_fidelity_probe()]),
11178 )
11179 .await?;
11180 Ok(child.result)
11181 });
11182
11183 let initial = worker
11184 .execute_workflow_task(workflow_task(
11185 "rust.typed-parent",
11186 Vec::new(),
11187 DEFAULT_CODEC,
11188 ))
11189 .expect("typed child start");
11190 assert_eq!(initial[0]["type"], "start_child_workflow");
11191 assert_eq!(
11192 decode_wire_avro_value(&initial[0]["arguments"], DEFAULT_CODEC)
11193 .expect("typed child arguments"),
11194 AvroValue::Array(vec![typed_fidelity_probe()])
11195 );
11196
11197 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
11198 .expect("typed child result");
11199 let task = workflow_task(
11200 "rust.typed-parent",
11201 vec![
11202 history_event(
11203 "ChildWorkflowScheduled",
11204 json!({
11205 "sequence": 1,
11206 "child_call_id": "call-typed",
11207 "child_workflow_instance_id": "wf-child",
11208 "child_workflow_run_id": "run-child",
11209 "child_workflow_type": "python.typed-child",
11210 }),
11211 ),
11212 history_event(
11213 "ChildRunCompleted",
11214 json!({
11215 "sequence": 1,
11216 "child_call_id": "call-typed",
11217 "child_workflow_instance_id": "wf-child",
11218 "child_workflow_run_id": "run-child",
11219 "child_workflow_type": "python.typed-child",
11220 "payload_codec": DEFAULT_CODEC,
11221 "result": result,
11222 }),
11223 ),
11224 ],
11225 DEFAULT_CODEC,
11226 );
11227
11228 let commands = worker
11229 .execute_workflow_task(task)
11230 .expect("typed child replay");
11231 assert_eq!(commands[0]["type"], "complete_workflow");
11232 assert_eq!(
11233 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
11234 .expect("typed parent result"),
11235 typed_fidelity_probe()
11236 );
11237 }
11238
11239 #[test]
11240 fn pending_child_replays_after_restart_without_starting_a_duplicate() {
11241 let worker = child_parent_worker();
11242 let mut task = child_parent_task("unused", Value::Null);
11243 task.history_events.truncate(1);
11244 task.total_history_events = Some(1);
11245
11246 for _redelivery_or_restart in 0..2 {
11247 let commands = worker
11248 .execute_workflow_task(task.clone())
11249 .expect("recorded child remains pending");
11250 assert!(
11251 commands.is_empty(),
11252 "recorded pending child must not be started again"
11253 );
11254 }
11255 }
11256
11257 #[test]
11258 fn child_cancellation_becomes_stable_parent_failure_command() {
11259 let worker = child_parent_worker();
11260 let task = child_parent_task(
11261 "ChildRunCancelled",
11262 json!({
11263 "sequence": 1,
11264 "child_workflow_instance_id": "wf-child",
11265 "child_workflow_run_id": "run-child",
11266 "child_workflow_type": "python.child",
11267 "failure_id": "failure-child",
11268 "failure_category": "cancelled",
11269 "message": "cancelled by parent-close policy",
11270 }),
11271 );
11272
11273 let commands = worker
11274 .execute_workflow_task(task)
11275 .expect("parent settlement");
11276 assert_eq!(commands.len(), 1);
11277 assert_eq!(commands[0]["type"], "fail_workflow");
11278 assert_eq!(commands[0]["exception_type"], "ChildWorkflowCancelled");
11279 assert_eq!(
11280 commands[0]["exception"]["properties"]["reason"],
11281 "cancelled"
11282 );
11283 assert_eq!(
11284 commands[0]["exception"]["properties"]["child_workflow_run_id"],
11285 "run-child"
11286 );
11287 }
11288
11289 #[test]
11290 fn workflow_can_handle_typed_child_failure() {
11291 let client = Client::new("http://127.0.0.1:8080").expect("client");
11292 let mut worker = Worker::new(client, "rust-parent-workers");
11293 worker.register_workflow("rust.handled-parent", |ctx, _input| async move {
11294 match ctx
11295 .start_child_workflow(
11296 "python.child",
11297 ChildWorkflowOptions::new("python-child-workers"),
11298 json!([]),
11299 )
11300 .await
11301 {
11302 Err(Error::ChildWorkflowFailed(failure)) => Ok(json!({
11303 "reason": failure.reason,
11304 "failure_id": failure.failure_id,
11305 "exception_class": failure.exception_class,
11306 "child_run_id": failure.child_workflow_run_id,
11307 })),
11308 Err(error) => Err(error),
11309 Ok(_) => Err(Error::WorkerLoop(
11310 "child unexpectedly succeeded".to_string(),
11311 )),
11312 }
11313 });
11314 let mut task = child_parent_task(
11315 "ChildRunFailed",
11316 json!({
11317 "sequence": 1,
11318 "child_workflow_instance_id": "wf-child",
11319 "child_workflow_run_id": "run-child",
11320 "child_workflow_type": "python.child",
11321 "failure_id": "failure-child",
11322 "failure_category": "child_workflow",
11323 "message": "payment rejected",
11324 "exception": {
11325 "type": "PaymentRejected",
11326 "class": "payments.PaymentRejected",
11327 "message": "payment rejected"
11328 }
11329 }),
11330 );
11331 task.workflow_type = "rust.handled-parent".to_string();
11332
11333 let commands = worker.execute_workflow_task(task).expect("handled failure");
11334 assert_eq!(commands[0]["type"], "complete_workflow");
11335 let output =
11336 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
11337 assert_eq!(output["reason"], "child_workflow");
11338 assert_eq!(output["failure_id"], "failure-child");
11339 assert_eq!(output["exception_class"], "payments.PaymentRejected");
11340 assert_eq!(output["child_run_id"], "run-child");
11341 }
11342
11343 #[test]
11344 fn rust_hello_world_uses_signal_arguments_from_resume_payload() {
11345 let client = Client::new("http://127.0.0.1:8080").expect("client");
11346 let mut worker = Worker::new(client, "rust-workers");
11347
11348 worker.register_workflow("rust.hello_workflow", |ctx, _input| async move {
11349 let signal = ctx.wait_signal("start").await?;
11350 let name = signal
11351 .first()
11352 .and_then(|value| value.as_str())
11353 .unwrap_or("world");
11354 let greeting = ctx.activity("rust.hello_activity", json!([name])).await?;
11355 Ok(json!({
11356 "greeting": greeting,
11357 "language": "rust"
11358 }))
11359 });
11360
11361 let signal_arguments =
11362 encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC).expect("signal arguments");
11363 let task = WorkflowTask {
11364 task_id: "wft-rust-signal-1".to_string(),
11365 workflow_id: Some("wf-rust-hello".to_string()),
11366 run_id: Some("run-rust-hello".to_string()),
11367 workflow_type: "rust.hello_workflow".to_string(),
11368 payload_codec: DEFAULT_CODEC.to_string(),
11369 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
11370 history_events: vec![HistoryEvent {
11371 event_type: "SignalReceived".to_string(),
11372 payload: json!({
11373 "signal_id": "sig-rust-1",
11374 "signal_name": "start"
11375 }),
11376 raw: HashMap::new(),
11377 }],
11378 total_history_events: Some(1),
11379 history_size_bytes: None,
11380 continue_as_new_recommended: None,
11381 history_budget_pressure: None,
11382 next_history_page_token: None,
11383 workflow_task_attempt: 1,
11384 workflow_signal_id: Some("sig-rust-1".to_string()),
11385 signal_name: Some("start".to_string()),
11386 signal_arguments: Some(signal_arguments),
11387 workflow_update_id: None,
11388 update_name: None,
11389 lease_owner: Some("rust-worker".to_string()),
11390 };
11391
11392 let commands = worker.execute_workflow_task(task).expect("workflow task");
11393
11394 assert_eq!(commands.len(), 1);
11395 assert_eq!(commands[0]["type"], "schedule_activity");
11396 assert_eq!(commands[0]["activity_type"], "rust.hello_activity");
11397 assert_eq!(
11398 decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC).expect("activity args"),
11399 json!(["Rust"])
11400 );
11401 }
11402
11403 #[test]
11404 fn workflow_task_appends_paginated_history_events() {
11405 let mut task = WorkflowTask {
11406 task_id: "wft-rust-pages-1".to_string(),
11407 workflow_id: Some("wf-rust-pages".to_string()),
11408 run_id: Some("run-rust-pages".to_string()),
11409 workflow_type: "rust.hello_workflow".to_string(),
11410 payload_codec: DEFAULT_CODEC.to_string(),
11411 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
11412 history_events: vec![HistoryEvent {
11413 event_type: "WorkflowStarted".to_string(),
11414 payload: json!({}),
11415 raw: HashMap::new(),
11416 }],
11417 total_history_events: Some(3),
11418 history_size_bytes: None,
11419 continue_as_new_recommended: None,
11420 history_budget_pressure: None,
11421 next_history_page_token: Some("MQ==".to_string()),
11422 workflow_task_attempt: 1,
11423 workflow_signal_id: None,
11424 signal_name: None,
11425 signal_arguments: None,
11426 workflow_update_id: None,
11427 update_name: None,
11428 lease_owner: Some("rust-worker".to_string()),
11429 };
11430
11431 task.append_history_page(WorkflowTaskHistoryPage {
11432 history_events: vec![
11433 HistoryEvent {
11434 event_type: "SignalReceived".to_string(),
11435 payload: json!({
11436 "signal_id": "sig-rust-1",
11437 "signal_name": "start",
11438 "arguments": encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC)
11439 .expect("signal arguments")
11440 }),
11441 raw: HashMap::new(),
11442 },
11443 HistoryEvent {
11444 event_type: "MarkerRecorded".to_string(),
11445 payload: json!({"sequence": 3}),
11446 raw: HashMap::new(),
11447 },
11448 ],
11449 total_history_events: Some(3),
11450 next_history_page_token: None,
11451 });
11452
11453 assert_eq!(task.history_events.len(), 3);
11454 assert_eq!(task.total_history_events, Some(3));
11455 assert_eq!(task.next_history_page_token, None);
11456
11457 let signal = task
11458 .history_events
11459 .iter()
11460 .find(|event| event.event_type == "SignalReceived")
11461 .expect("signal event");
11462 assert_eq!(
11463 decode_signal_event_arguments(signal, DEFAULT_CODEC).expect("signal arguments"),
11464 vec![AvroValue::String("Rust".to_string())]
11465 );
11466 }
11467
11468 #[tokio::test]
11469 async fn query_handler_reads_ordered_cross_codec_signals_without_commands() {
11470 let client = Client::new("http://127.0.0.1:8080").expect("client");
11471 let mut worker = Worker::new(client, "rust-workers");
11472 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
11473 worker.register_query("counter", "current", |ctx, _args| async move {
11474 let mut count = 0_i64;
11475 for signal in ctx.signal_events() {
11476 let value = signal
11477 .arguments
11478 .first()
11479 .and_then(Value::as_i64)
11480 .unwrap_or_default();
11481 match signal.name.as_str() {
11482 "increment" => count += value,
11483 "set" => count = value,
11484 _ => {}
11485 }
11486 }
11487 Ok(json!(count))
11488 });
11489
11490 let task = QueryTask {
11491 query_task_id: "query-rust-counter".to_string(),
11492 query_task_attempt: 1,
11493 lease_owner: Some("rust-worker".to_string()),
11494 workflow_id: Some("counter-1".to_string()),
11495 run_id: Some("run-counter-1".to_string()),
11496 workflow_type: "counter".to_string(),
11497 query_name: "current".to_string(),
11498 payload_codec: DEFAULT_CODEC.to_string(),
11499 workflow_arguments: Some(
11500 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
11501 ),
11502 query_arguments: Some(
11503 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("query arguments"),
11504 ),
11505 history_events: vec![
11506 HistoryEvent {
11507 event_type: "SignalReceived".to_string(),
11508 payload: json!({
11509 "signal_id": "php-signal-1",
11510 "signal_name": "increment",
11511 "workflow_sequence": 1,
11512 "payload_codec": DEFAULT_CODEC,
11513 "arguments": encode_value_envelope(&json!([3]), DEFAULT_CODEC).expect("php avro signal")
11514 }),
11515 raw: HashMap::new(),
11516 },
11517 HistoryEvent {
11518 event_type: "SignalReceived".to_string(),
11519 payload: json!({
11520 "signal_id": "python-signal-2",
11521 "signal_name": "increment",
11522 "workflow_sequence": 2,
11523 "payload_codec": DEFAULT_CODEC,
11524 "arguments": encode_value_envelope(&json!([5]), DEFAULT_CODEC).expect("python avro signal")
11525 }),
11526 raw: HashMap::new(),
11527 },
11528 HistoryEvent {
11529 event_type: "SignalReceived".to_string(),
11530 payload: json!({
11531 "signal_id": "rust-signal-3",
11532 "signal_name": "set",
11533 "workflow_sequence": 3,
11534 "payload_codec": DEFAULT_CODEC,
11535 "arguments": encode_value_envelope(&json!([0]), DEFAULT_CODEC).expect("rust avro signal")
11536 }),
11537 raw: HashMap::new(),
11538 },
11539 ],
11540 history_export: None,
11541 run_status: Some("completed".to_string()),
11542 };
11543
11544 let result = worker.execute_query_task(task).await.expect("query result");
11545 assert_eq!(result.into_json().expect("query projection"), json!(0));
11546 }
11547
11548 #[tokio::test]
11549 async fn replayed_queries_read_running_completed_and_cold_restarted_instance_state() {
11550 let worker = replay_counter_worker();
11551 let running_history = json!([
11552 {
11553 "type": "ActivityCompleted",
11554 "payload": {
11555 "sequence": 1,
11556 "activity_type": "load-counter",
11557 "payload_codec": DEFAULT_CODEC,
11558 "result": fixture_envelope(json!("loaded"))
11559 }
11560 },
11561 {
11562 "type": "SignalWaitOpened",
11563 "payload": {
11564 "sequence": 3,
11565 "signal_name": "increment"
11566 }
11567 },
11568 {
11569 "type": "SignalReceived",
11570 "payload": {
11571 "signal_id": "signal-3",
11572 "signal_name": "increment",
11573 "workflow_sequence": 2,
11574 "payload_codec": DEFAULT_CODEC,
11575 "arguments": fixture_envelope(json!([3]))
11576 }
11577 },
11578 {
11579 "type": "SignalApplied",
11580 "payload": {
11581 "sequence": 3,
11582 "signal_id": "signal-3",
11583 "signal_name": "increment",
11584 "payload_codec": DEFAULT_CODEC,
11585 "value": fixture_envelope(json!([3]))
11586 }
11587 }
11588 ]);
11589
11590 let running = worker
11591 .execute_query_task(replay_counter_query(
11592 "current",
11593 running_history.clone(),
11594 "running",
11595 ))
11596 .await
11597 .expect("running replay query");
11598 assert_eq!(
11599 running.clone().into_json().expect("query projection"),
11600 json!({"loaded": "loaded", "count": 3, "finished": false})
11601 );
11602
11603 let detached = worker
11604 .execute_query_task(replay_counter_query(
11605 "detached-mutation",
11606 running_history.clone(),
11607 "running",
11608 ))
11609 .await
11610 .expect("query mutates only its detached state clone");
11611 assert_eq!(detached.into_json().expect("query projection"), json!(999));
11612 let failed = worker
11613 .execute_query_task(replay_counter_query(
11614 "failed-mutation",
11615 running_history.clone(),
11616 "running",
11617 ))
11618 .await
11619 .expect_err("failed query");
11620 assert_eq!(failed.reason, "query_rejected");
11621 let unchanged = worker
11622 .execute_query_task(replay_counter_query("current", running_history, "running"))
11623 .await
11624 .expect("later query reconstructs unchanged state");
11625 assert_eq!(unchanged, running);
11626
11627 let restarted_worker = replay_counter_worker();
11628 let empty_arguments = fixture_envelope(json!([]));
11629 let loaded_result = fixture_envelope(json!("loaded"));
11630 let signal_three = fixture_blob(json!([3]));
11631 let signal_five = fixture_blob(json!([5]));
11632 let restarted_task: QueryTask = serde_json::from_value(json!({
11633 "query_task_id": "query-after-restart",
11634 "workflow_id": "counter-1",
11635 "run_id": "run-counter-1",
11636 "workflow_type": "replay-counter",
11637 "query_name": "current",
11638 "payload_codec": DEFAULT_CODEC,
11639 "workflow_arguments": empty_arguments.clone(),
11640 "query_arguments": empty_arguments,
11641 "history_events": [],
11642 "history_export": {
11643 "payloads": {"codec": DEFAULT_CODEC},
11644 "history_events": [
11645 {
11646 "type": "ActivityCompleted",
11647 "payload": {
11648 "sequence": 1,
11649 "activity_type": "load-counter",
11650 "payload_codec": DEFAULT_CODEC,
11651 "result": null
11652 }
11653 },
11654 {
11655 "type": "SignalWaitOpened",
11656 "payload": {
11657 "sequence": 3,
11658 "signal_name": "increment"
11659 }
11660 },
11661 {
11662 "type": "SignalReceived",
11663 "payload": {
11664 "signal_id": "signal-3",
11665 "signal_name": "increment",
11666 "workflow_sequence": 2
11667 }
11668 },
11669 {
11670 "type": "SignalApplied",
11671 "payload": {
11672 "sequence": 3,
11673 "signal_id": "signal-3",
11674 "signal_name": "increment"
11675 }
11676 },
11677 {
11678 "type": "SignalWaitOpened",
11679 "payload": {
11680 "sequence": 5,
11681 "signal_name": "increment"
11682 }
11683 },
11684 {
11685 "type": "SignalReceived",
11686 "payload": {
11687 "signal_id": "signal-5",
11688 "signal_name": "increment",
11689 "workflow_sequence": 4
11690 }
11691 },
11692 {
11693 "type": "SignalApplied",
11694 "payload": {
11695 "sequence": 5,
11696 "signal_id": "signal-5",
11697 "signal_name": "increment"
11698 }
11699 }
11700 ],
11701 "activities": [{
11702 "sequence": 1,
11703 "activity_type": "load-counter",
11704 "payload_codec": DEFAULT_CODEC,
11705 "result": loaded_result
11706 }],
11707 "signals": [
11708 {
11709 "id": "signal-3",
11710 "name": "increment",
11711 "workflow_sequence": 2,
11712 "payload_codec": DEFAULT_CODEC,
11713 "arguments": signal_three
11714 },
11715 {
11716 "id": "signal-5",
11717 "name": "increment",
11718 "workflow_sequence": 4,
11719 "payload_codec": DEFAULT_CODEC,
11720 "arguments": signal_five
11721 }
11722 ]
11723 },
11724 "run_status": "completed"
11725 }))
11726 .expect("cold replay query task");
11727 let completed = restarted_worker
11728 .execute_query_task(restarted_task)
11729 .await
11730 .expect("completed cold replay query");
11731 assert_eq!(
11732 completed.into_json().expect("query projection"),
11733 json!({"loaded": "loaded", "count": 8, "finished": true})
11734 );
11735 }
11736
11737 #[tokio::test]
11738 async fn replayed_query_replay_failures_are_machine_readable() {
11739 let worker = replay_counter_worker();
11740 let task = replay_counter_query(
11741 "current",
11742 json!([{
11743 "type": "ActivityCompleted",
11744 "payload": {
11745 "sequence": 1,
11746 "payload_codec": DEFAULT_CODEC,
11747 "result": {"codec": DEFAULT_CODEC, "blob": "{"}
11748 }
11749 }]),
11750 "running",
11751 );
11752 let failure = worker
11753 .execute_query_task(task)
11754 .await
11755 .expect_err("invalid replay history payload");
11756 assert_eq!(failure.reason, "query_workflow_state_unavailable");
11757 assert_eq!(failure.failure_type, "QueryWorkflowStateUnavailable");
11758 }
11759
11760 #[tokio::test]
11761 async fn query_task_restores_compact_history_from_export() {
11762 let client = Client::new("http://127.0.0.1:8080").expect("client");
11763 let mut worker = Worker::new(client, "rust-workers");
11764 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
11765 worker.register_query("counter", "current", |ctx, _args| async move {
11766 Ok(json!(ctx.signals("increment")[0][0]))
11767 });
11768 let empty_arguments = fixture_envelope(json!([]));
11769 let exported_signal = fixture_blob(json!([9]));
11770 let task: QueryTask = serde_json::from_value(json!({
11771 "query_task_id": "query-export",
11772 "workflow_type": "counter",
11773 "query_name": "current",
11774 "payload_codec": DEFAULT_CODEC,
11775 "workflow_arguments": empty_arguments.clone(),
11776 "query_arguments": empty_arguments,
11777 "history_events": [],
11778 "history_export": {
11779 "payloads": {"codec": DEFAULT_CODEC},
11780 "history_events": [{
11781 "type": "SignalReceived",
11782 "payload": {"signal_id": "signal-export", "signal_name": "increment"}
11783 }],
11784 "signals": [{
11785 "id": "signal-export",
11786 "name": "increment",
11787 "status": "applied",
11788 "workflow_sequence": 1,
11789 "payload_codec": DEFAULT_CODEC,
11790 "arguments": exported_signal
11791 }]
11792 }
11793 }))
11794 .expect("query task");
11795
11796 let result = worker.execute_query_task(task).await.expect("query result");
11797 assert_eq!(result.into_json().expect("query projection"), json!(9));
11798 }
11799
11800 #[tokio::test]
11801 async fn query_task_failures_have_stable_reasons() {
11802 let client = Client::new("http://127.0.0.1:8080").expect("client");
11803 let mut worker = Worker::new(client, "rust-workers");
11804 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
11805 worker.register_query(
11806 "counter",
11807 "current",
11808 |_ctx, _args| async move { Ok(json!(0)) },
11809 );
11810
11811 let base_task = QueryTask {
11812 query_task_id: "query-errors".to_string(),
11813 query_task_attempt: 1,
11814 lease_owner: None,
11815 workflow_id: Some("counter-errors".to_string()),
11816 run_id: Some("run-errors".to_string()),
11817 workflow_type: "counter".to_string(),
11818 query_name: "missing".to_string(),
11819 payload_codec: DEFAULT_CODEC.to_string(),
11820 workflow_arguments: Some(fixture_envelope(json!([]))),
11821 query_arguments: Some(fixture_envelope(json!([]))),
11822 history_events: Vec::new(),
11823 history_export: None,
11824 run_status: Some("running".to_string()),
11825 };
11826
11827 let unknown = worker
11828 .execute_query_task(base_task.clone())
11829 .await
11830 .expect_err("unknown query");
11831 assert_eq!(unknown.reason, "rejected_unknown_query");
11832
11833 let mut malformed = base_task;
11834 malformed.query_name = "current".to_string();
11835 malformed.query_arguments = Some(json!({"codec": DEFAULT_CODEC, "blob": "{"}));
11836 let malformed = worker
11837 .execute_query_task(malformed)
11838 .await
11839 .expect_err("malformed payload");
11840 assert_eq!(malformed.reason, "query_payload_decode_failed");
11841
11842 let client = Client::new("http://127.0.0.1:8080").expect("client");
11843 let mut unavailable_worker = Worker::new(client, "rust-workers");
11844 unavailable_worker
11845 .register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
11846 let empty_arguments = fixture_envelope(json!([]));
11847 let unavailable_task: QueryTask = serde_json::from_value(json!({
11848 "query_task_id": "query-unavailable",
11849 "workflow_type": "counter",
11850 "query_name": "current",
11851 "payload_codec": DEFAULT_CODEC,
11852 "workflow_arguments": empty_arguments.clone(),
11853 "query_arguments": empty_arguments
11854 }))
11855 .expect("query task");
11856 let unavailable = unavailable_worker
11857 .execute_query_task(unavailable_task)
11858 .await
11859 .expect_err("query handler unavailable");
11860 assert_eq!(unavailable.reason, "query_handler_unavailable");
11861 }
11862
11863 #[tokio::test]
11864 async fn client_query_decodes_result_and_typed_failure() {
11865 let server = MockWorkerServer::start();
11866 let client = Client::builder(server.base_url())
11867 .timeout(Duration::from_secs(2))
11868 .build()
11869 .expect("client");
11870
11871 let result = client
11872 .query_workflow("counter-1", "current", json!([]))
11873 .await
11874 .expect("query result");
11875 assert_eq!(result, json!({"count": 8}));
11876
11877 let error = client
11878 .query_workflow("counter-1", "missing", json!([]))
11879 .await
11880 .expect_err("unknown query");
11881 let Error::QueryFailed(failure) = error else {
11882 panic!("expected typed query failure");
11883 };
11884 assert_eq!(failure.status, 404);
11885 assert_eq!(failure.reason, "rejected_unknown_query");
11886 }
11887
11888 #[tokio::test]
11889 async fn public_client_surfaces_send_and_receive_lossless_avro_values() {
11890 let server = MockWorkerServer::start();
11891 let client = Client::builder(server.base_url())
11892 .timeout(Duration::from_secs(2))
11893 .build()
11894 .expect("client");
11895 let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
11896
11897 client
11898 .start_workflow(
11899 "typed.echo",
11900 "rust-workers",
11901 "typed-start",
11902 arguments.clone(),
11903 )
11904 .await
11905 .expect("typed workflow start");
11906 assert_eq!(
11907 decode_wire_avro_value(
11908 &server.request_body("/api/workflows")["input"],
11909 DEFAULT_CODEC,
11910 )
11911 .expect("typed start input"),
11912 arguments
11913 );
11914
11915 client
11916 .signal_workflow("typed-1", "changed", arguments.clone())
11917 .await
11918 .expect("typed signal");
11919 assert_eq!(
11920 decode_wire_avro_value(
11921 &server.request_body("/api/workflows/typed-1/signal/changed")["input"],
11922 DEFAULT_CODEC,
11923 )
11924 .expect("typed signal input"),
11925 arguments
11926 );
11927
11928 assert_eq!(
11929 client
11930 .query_workflow_avro_value("typed-1", "inspect", arguments.clone())
11931 .await
11932 .expect("typed query"),
11933 typed_fidelity_probe()
11934 );
11935 assert_eq!(
11936 decode_wire_avro_value(
11937 &server.request_body("/api/workflows/typed-1/query/inspect")["input"],
11938 DEFAULT_CODEC,
11939 )
11940 .expect("typed query input"),
11941 arguments
11942 );
11943
11944 assert_eq!(
11945 client
11946 .update_workflow_avro_value(
11947 "typed-1",
11948 "replace",
11949 arguments.clone(),
11950 Some("typed-request"),
11951 )
11952 .await
11953 .expect("typed update"),
11954 typed_fidelity_probe()
11955 );
11956 let update = server.request_body("/api/workflows/typed-1/update/replace");
11957 assert_eq!(update["request_id"], "typed-request");
11958 assert_eq!(
11959 decode_wire_avro_value(&update["input"], DEFAULT_CODEC).expect("typed update input"),
11960 arguments
11961 );
11962
11963 let handle = WorkflowHandle {
11964 client: client.clone(),
11965 workflow_id: "typed-1".to_string(),
11966 run_id: Some("run-typed-1".to_string()),
11967 workflow_type: "typed.echo".to_string(),
11968 };
11969 assert_eq!(
11970 handle
11971 .result_avro_value(WorkflowResultOptions::default())
11972 .await
11973 .expect("typed workflow result"),
11974 typed_fidelity_probe()
11975 );
11976
11977 client
11978 .complete_activity_task(
11979 "activity-typed",
11980 "attempt-typed",
11981 "rust-worker",
11982 typed_fidelity_probe(),
11983 DEFAULT_CODEC,
11984 )
11985 .await
11986 .expect("typed activity completion");
11987 assert_eq!(
11988 decode_wire_avro_value(
11989 &server.request_body("/api/worker/activity-tasks/activity-typed/complete")
11990 ["result"],
11991 DEFAULT_CODEC,
11992 )
11993 .expect("typed activity result"),
11994 typed_fidelity_probe()
11995 );
11996 client
11997 .fail_activity_task(
11998 "activity-typed",
11999 "attempt-typed",
12000 "rust-worker",
12001 "typed failure",
12002 true,
12003 )
12004 .await
12005 .expect("activity failure");
12006 }
12007
12008 #[tokio::test]
12009 async fn lifecycle_commands_support_instance_and_selected_run_targets() {
12010 let server = MockWorkerServer::start();
12011 let client = Client::builder(server.base_url())
12012 .timeout(Duration::from_secs(2))
12013 .build()
12014 .expect("client");
12015
12016 let options = WorkflowCommandOptions::new()
12017 .reason("cleanup requested")
12018 .request_id("cancel-17");
12019 let cancelled = client
12020 .cancel_workflow("wf-lifecycle", options)
12021 .await
12022 .expect("instance cancellation");
12023 assert_eq!(cancelled.command, WorkflowCommandKind::Cancel);
12024 assert_eq!(cancelled.run_id.as_deref(), Some("run-current"));
12025 assert_eq!(cancelled.outcome.as_deref(), Some("cancelled"));
12026 assert_eq!(
12027 server.request_body("/api/workflows/wf-lifecycle/cancel"),
12028 json!({"reason":"cleanup requested","request_id":"cancel-17"})
12029 );
12030
12031 let terminated = client
12032 .terminate_workflow(
12033 "wf-lifecycle",
12034 WorkflowCommandOptions::new().reason("forced stop"),
12035 )
12036 .await
12037 .expect("instance termination");
12038 assert_eq!(terminated.command, WorkflowCommandKind::Terminate);
12039 assert_eq!(terminated.outcome.as_deref(), Some("terminated"));
12040
12041 client
12042 .cancel_workflow_run(
12043 "wf-lifecycle",
12044 "run-current",
12045 WorkflowCommandOptions::default(),
12046 )
12047 .await
12048 .expect("selected run cancellation");
12049 client
12050 .terminate_workflow_run(
12051 "wf-lifecycle",
12052 "run-current",
12053 WorkflowCommandOptions::default(),
12054 )
12055 .await
12056 .expect("selected run termination");
12057
12058 for (command, error) in [
12059 (
12060 WorkflowCommandKind::Cancel,
12061 client
12062 .cancel_workflow_run(
12063 "wf-lifecycle",
12064 "run-stale",
12065 WorkflowCommandOptions::default(),
12066 )
12067 .await
12068 .expect_err("stale cancellation must be rejected"),
12069 ),
12070 (
12071 WorkflowCommandKind::Terminate,
12072 client
12073 .terminate_workflow_run(
12074 "wf-lifecycle",
12075 "run-stale",
12076 WorkflowCommandOptions::default(),
12077 )
12078 .await
12079 .expect_err("stale termination must be rejected"),
12080 ),
12081 ] {
12082 let Error::WorkflowCommandRejected(rejection) = error else {
12083 panic!("expected typed command rejection");
12084 };
12085 assert_eq!(rejection.command, command);
12086 assert_eq!(rejection.status, 409);
12087 assert_eq!(rejection.reason, "historical_run_command_rejected");
12088 assert_eq!(rejection.run_id.as_deref(), Some("run-stale"));
12089 assert_eq!(rejection.target_scope.as_deref(), Some("run"));
12090 }
12091 }
12092
12093 #[tokio::test]
12094 async fn workflow_start_options_send_server_enforced_deadlines() {
12095 let server = MockWorkerServer::start();
12096 let client = Client::builder(server.base_url())
12097 .timeout(Duration::from_secs(2))
12098 .build()
12099 .expect("client");
12100
12101 let handle = client
12102 .start_workflow_with_options(
12103 "rust.timeout",
12104 "rust-timeouts",
12105 "wf-start-options",
12106 WorkflowStartOptions::new()
12107 .execution_timeout_seconds(30)
12108 .run_timeout_seconds(1),
12109 json!([]),
12110 )
12111 .await
12112 .expect("workflow start");
12113
12114 assert_eq!(handle.run_id.as_deref(), Some("run-start-options"));
12115 let body = server.request_body("/api/workflows");
12116 assert_eq!(body["execution_timeout_seconds"], 30);
12117 assert_eq!(body["run_timeout_seconds"], 1);
12118
12119 let invalid = client
12120 .start_workflow_with_options(
12121 "rust.timeout",
12122 "rust-timeouts",
12123 "wf-invalid-options",
12124 WorkflowStartOptions::new()
12125 .execution_timeout_seconds(1)
12126 .run_timeout_seconds(2),
12127 json!([]),
12128 )
12129 .await
12130 .expect_err("invalid deadline ordering");
12131 assert!(invalid
12132 .to_string()
12133 .contains("run_timeout_seconds cannot exceed execution_timeout_seconds"));
12134 }
12135
12136 #[tokio::test]
12137 async fn workflow_result_returns_each_typed_terminal_outcome() {
12138 let server = MockWorkerServer::start();
12139 let client = Client::builder(server.base_url())
12140 .timeout(Duration::from_secs(2))
12141 .build()
12142 .expect("client");
12143 let options = WorkflowResultOptions {
12144 poll_interval: Duration::ZERO,
12145 timeout: Duration::from_secs(1),
12146 };
12147
12148 let failed = WorkflowHandle {
12149 client: client.clone(),
12150 workflow_id: "wf-failed".to_string(),
12151 run_id: Some("run-failed".to_string()),
12152 workflow_type: "failure".to_string(),
12153 }
12154 .result(options)
12155 .await
12156 .expect_err("failed outcome");
12157 let Error::WorkflowFailed(failure) = failed else {
12158 panic!("expected WorkflowFailed");
12159 };
12160 assert_eq!(failure.workflow_id, "wf-failed");
12161 assert_eq!(failure.run_id.as_deref(), Some("run-failed"));
12162 assert_eq!(failure.failure_id.as_deref(), Some("failure-17"));
12163 assert_eq!(failure.failure_category.as_deref(), Some("application"));
12164 assert_eq!(failure.exception_type.as_deref(), Some("PaymentError"));
12165 assert_eq!(
12166 failure.exception_class.as_deref(),
12167 Some("billing::PaymentError")
12168 );
12169 assert_eq!(failure.non_retryable, Some(true));
12170
12171 for (workflow_id, expected_kind, expected_reason) in [
12172 (
12173 "wf-cancelled",
12174 WorkflowTerminalKind::Cancelled,
12175 "cleanup requested",
12176 ),
12177 (
12178 "wf-terminated",
12179 WorkflowTerminalKind::Terminated,
12180 "forced stop",
12181 ),
12182 (
12183 "wf-timed-out",
12184 WorkflowTerminalKind::TimedOut,
12185 "run_timeout",
12186 ),
12187 ] {
12188 let error = WorkflowHandle {
12189 client: client.clone(),
12190 workflow_id: workflow_id.to_string(),
12191 run_id: None,
12192 workflow_type: "terminal".to_string(),
12193 }
12194 .result(options)
12195 .await
12196 .expect_err("typed terminal outcome");
12197 let outcome = match error {
12198 Error::WorkflowCancelled(outcome) => outcome,
12199 Error::WorkflowTerminated(outcome) => outcome,
12200 Error::WorkflowTimedOut(outcome) => outcome,
12201 other => panic!("unexpected terminal error: {other}"),
12202 };
12203 assert_eq!(outcome.kind, expected_kind);
12204 assert_eq!(outcome.workflow_id, workflow_id);
12205 assert_eq!(outcome.reason, expected_reason);
12206 }
12207
12208 let wait_timeout = WorkflowHandle {
12209 client,
12210 workflow_id: "wf-waiting".to_string(),
12211 run_id: Some("run-waiting".to_string()),
12212 workflow_type: "waiting".to_string(),
12213 }
12214 .result(WorkflowResultOptions {
12215 poll_interval: Duration::ZERO,
12216 timeout: Duration::ZERO,
12217 })
12218 .await
12219 .expect_err("client wait timeout");
12220 let Error::WorkflowTimedOut(timeout) = wait_timeout else {
12221 panic!("expected typed client timeout");
12222 };
12223 assert_eq!(timeout.reason, "result_wait_timeout");
12224 assert_eq!(timeout.failure_category.as_deref(), Some("client_timeout"));
12225 assert_eq!(timeout.run_id.as_deref(), Some("run-waiting"));
12226 }
12227
12228 #[tokio::test]
12229 async fn workflow_result_follows_chain_and_selected_result_preserves_history() {
12230 let server = MockWorkerServer::start();
12231 let client = Client::builder(server.base_url())
12232 .timeout(Duration::from_secs(2))
12233 .build()
12234 .expect("client");
12235
12236 let handle = WorkflowHandle {
12237 client,
12238 workflow_id: "wf-selected".to_string(),
12239 run_id: Some("run-selected".to_string()),
12240 workflow_type: "selected".to_string(),
12241 };
12242 let options = WorkflowResultOptions {
12243 poll_interval: Duration::ZERO,
12244 timeout: Duration::from_secs(1),
12245 };
12246
12247 let current = handle
12248 .result(options)
12249 .await
12250 .expect("instance result follows the current run");
12251 assert_eq!(current, json!("current run output"));
12252
12253 let error = handle
12254 .result_selected_run(options)
12255 .await
12256 .expect_err("the selected run is cancelled even though the current run completed");
12257
12258 let Error::WorkflowCancelled(outcome) = error else {
12259 panic!("expected selected run cancellation");
12260 };
12261 assert_eq!(outcome.run_id.as_deref(), Some("run-selected"));
12262 assert_eq!(outcome.reason, "selected run cancelled");
12263 assert_eq!(
12264 server.request_count("/api/workflows/wf-selected/runs/run-selected"),
12265 1
12266 );
12267 assert_eq!(server.request_count("/api/workflows/wf-selected"), 1);
12268 }
12269
12270 #[tokio::test]
12271 async fn poll_responses_decode_http_conflict_drain_as_a_stable_stop() {
12272 let server = MockWorkerServer::draining_polls();
12273 let client = Client::builder(server.base_url())
12274 .timeout(Duration::from_secs(2))
12275 .build()
12276 .expect("client");
12277
12278 let workflow = client
12279 .poll_workflow_task_response("draining-worker", "rust-workers", Duration::ZERO)
12280 .await
12281 .expect("workflow drain response");
12282 let activity = client
12283 .poll_activity_task_response("draining-worker", "rust-workers", Duration::ZERO)
12284 .await
12285 .expect("activity drain response");
12286 let query = client
12287 .poll_query_task_response("draining-worker", "rust-workers", Duration::ZERO)
12288 .await
12289 .expect("query drain response");
12290
12291 for outcome in [workflow.outcome(), activity.outcome(), query.outcome()] {
12292 assert_eq!(
12293 outcome,
12294 WorkerPollOutcome::Stop {
12295 poll_status: Some("draining".to_string()),
12296 reason: Some("worker_draining".to_string()),
12297 }
12298 );
12299 }
12300
12301 assert!(client
12302 .poll_workflow_task("draining-worker", "rust-workers", Duration::ZERO)
12303 .await
12304 .expect("compatibility poll")
12305 .is_none());
12306 }
12307
12308 #[tokio::test]
12309 async fn managed_worker_honors_drain_stop_for_every_task_family() {
12310 let server = MockWorkerServer::draining_polls();
12311 let client = Client::builder(server.base_url())
12312 .timeout(Duration::from_secs(2))
12313 .build()
12314 .expect("client");
12315
12316 let mut workflow_worker = Worker::new(client.clone(), "rust-workers")
12317 .worker_id("draining-workflow-worker")
12318 .poll_timeout(Duration::ZERO);
12319 workflow_worker.register_workflow("counter", |_ctx, _args| async { Ok(Value::Null) });
12320 workflow_worker
12321 .run()
12322 .await
12323 .expect("workflow drain is a clean stop");
12324
12325 let mut activity_worker = Worker::new(client.clone(), "rust-workers")
12326 .worker_id("draining-activity-worker")
12327 .poll_timeout(Duration::ZERO);
12328 activity_worker.register_activity("write", |_ctx, _args| async { Ok(Value::Null) });
12329 activity_worker
12330 .run()
12331 .await
12332 .expect("activity drain is a clean stop");
12333
12334 let mut query_worker = Worker::new(client, "rust-workers")
12335 .worker_id("draining-query-worker")
12336 .poll_timeout(Duration::ZERO);
12337 query_worker.register_query("counter", "current", |_ctx, _args| async {
12338 Ok(Value::Null)
12339 });
12340 query_worker
12341 .run()
12342 .await
12343 .expect("query drain is a clean stop");
12344 }
12345
12346 #[tokio::test]
12347 async fn activity_cancellation_and_late_completion_remain_machine_readable() {
12348 let server = MockWorkerServer::start();
12349 let client = Client::builder(server.base_url())
12350 .timeout(Duration::from_secs(2))
12351 .build()
12352 .expect("client");
12353
12354 let heartbeat = client
12355 .heartbeat_activity_task(
12356 "activity-cancel",
12357 "attempt-cancel",
12358 "rust-worker",
12359 typed_fidelity_probe(),
12360 )
12361 .await
12362 .expect("cancellation heartbeat");
12363 assert!(heartbeat.cancel_requested);
12364 assert!(heartbeat.should_stop());
12365 assert_eq!(heartbeat.reason.as_deref(), Some("run_cancelled"));
12366 assert_eq!(heartbeat.run_closed_reason.as_deref(), Some("cancelled"));
12367 let heartbeat_body =
12368 server.request_body("/api/worker/activity-tasks/activity-cancel/heartbeat");
12369 assert_eq!(heartbeat_body["details"]["codec"], DEFAULT_CODEC);
12370 assert_eq!(
12371 decode_wire_avro_value(&heartbeat_body["details"], DEFAULT_CODEC)
12372 .expect("typed heartbeat details"),
12373 typed_fidelity_probe()
12374 );
12375
12376 let error = client
12377 .complete_activity_task(
12378 "activity-cancel",
12379 "attempt-cancel",
12380 "rust-worker",
12381 json!({"late":true}),
12382 DEFAULT_CODEC,
12383 )
12384 .await
12385 .expect_err("late completion must be refused");
12386 assert!(activity_task_rejection_is_final(&error));
12387 let Error::ActivityTaskRejected(rejection) = error else {
12388 panic!("expected typed activity rejection");
12389 };
12390 assert_eq!(rejection.status, 409);
12391 assert_eq!(rejection.reason, "run_cancelled");
12392 assert!(rejection.cancel_requested);
12393 assert_eq!(rejection.can_continue, Some(false));
12394 }
12395
12396 #[tokio::test]
12397 async fn managed_worker_survives_late_completion_and_restart_during_cancellation() {
12398 let server = MockWorkerServer::cancelled_activity();
12399 let client = Client::builder(server.base_url())
12400 .timeout(Duration::from_secs(2))
12401 .build()
12402 .expect("client");
12403 let cancellation_observed = Arc::new(AtomicBool::new(false));
12404 let observed = Arc::clone(&cancellation_observed);
12405 let mut worker = Worker::new(client.clone(), "rust-workers")
12406 .worker_id("rust-cancel-worker")
12407 .poll_timeout(Duration::from_millis(10));
12408 worker.register_activity("cancel-aware", move |ctx, _args| {
12409 let observed = Arc::clone(&observed);
12410 async move {
12411 let heartbeat = ctx.heartbeat(json!({"stage":"running"})).await?;
12412 observed.store(heartbeat.should_stop(), Ordering::SeqCst);
12413 Ok(json!({"late":"completion"}))
12414 }
12415 });
12416
12417 assert_eq!(
12418 worker.run_once().await.expect("cancelled attempt handled"),
12419 1
12420 );
12421 assert!(cancellation_observed.load(Ordering::SeqCst));
12422 assert_eq!(
12423 server.request_count("/api/worker/activity-tasks/activity-cancel/complete"),
12424 1
12425 );
12426
12427 let mut restarted = Worker::new(client, "rust-workers")
12428 .worker_id("rust-cancel-worker-restarted")
12429 .poll_timeout(Duration::from_millis(10));
12430 restarted.register_activity("cancel-aware", |_ctx, _args| async move { Ok(Value::Null) });
12431 assert_eq!(
12432 restarted
12433 .run_once()
12434 .await
12435 .expect("replacement worker continues polling"),
12436 0
12437 );
12438 }
12439
12440 #[tokio::test]
12441 async fn managed_worker_absorbs_selected_run_terminal_timeout_completion_race() {
12442 let response = r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"outcome":"completed","recorded":false,"run_id":"run-selected-timeout","run_status":"failed","created_task_ids":[],"reason":"run_timed_out"}"#;
12443 let server = MockWorkerServer::workflow_completion("409 Conflict", response);
12444 let client = Client::builder(server.base_url())
12445 .timeout(Duration::from_secs(2))
12446 .build()
12447 .expect("client");
12448
12449 let direct_error = client
12450 .complete_workflow_task(
12451 "workflow-timeout-task",
12452 "timeout-worker",
12453 3,
12454 vec![json!({"type": "complete_workflow", "result": null})],
12455 )
12456 .await
12457 .expect_err("the low-level client preserves the completion rejection");
12458 let Error::Http { status, body } = direct_error else {
12459 panic!("expected the original HTTP completion rejection");
12460 };
12461 assert_eq!(status, reqwest::StatusCode::CONFLICT);
12462 assert_eq!(
12463 serde_json::from_str::<Value>(&body).expect("response body")["reason"],
12464 "run_timed_out"
12465 );
12466
12467 let mut worker = Worker::new(client, "rust-workers")
12468 .worker_id("timeout-worker")
12469 .poll_timeout(Duration::from_millis(10));
12470 worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
12471 Ok(json!({"late": "result"}))
12472 });
12473
12474 assert_eq!(
12475 worker
12476 .run_once()
12477 .await
12478 .expect("authoritative selected-run timeout settles the tick"),
12479 1
12480 );
12481 assert_eq!(
12482 server.request_count("/api/worker/workflow-tasks/workflow-timeout-task/complete"),
12483 2,
12484 "both the direct client proof and managed worker must see the rejection"
12485 );
12486 }
12487
12488 #[tokio::test]
12489 async fn managed_worker_does_not_swallow_nearby_completion_errors() {
12490 for (name, status, response) in [
12491 ("bare conflict", "409 Conflict", r#"{"message":"conflict"}"#),
12492 (
12493 "command was recorded",
12494 "409 Conflict",
12495 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":true,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12496 ),
12497 (
12498 "lease conflict",
12499 "409 Conflict",
12500 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"lease_expired"}"#,
12501 ),
12502 (
12503 "nonterminal run",
12504 "409 Conflict",
12505 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"waiting","reason":"run_timed_out"}"#,
12506 ),
12507 (
12508 "different selected run",
12509 "409 Conflict",
12510 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-reused-workflow-current","run_status":"failed","reason":"run_timed_out"}"#,
12511 ),
12512 (
12513 "different task attempt",
12514 "409 Conflict",
12515 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":4,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12516 ),
12517 (
12518 "authentication failure",
12519 "401 Unauthorized",
12520 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12521 ),
12522 (
12523 "authorization failure",
12524 "403 Forbidden",
12525 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12526 ),
12527 (
12528 "protocol failure",
12529 "400 Bad Request",
12530 r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.2","requested_version":"1.3"}"#,
12531 ),
12532 (
12533 "malformed command",
12534 "422 Unprocessable Entity",
12535 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12536 ),
12537 (
12538 "transient server failure",
12539 "503 Service Unavailable",
12540 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12541 ),
12542 ] {
12543 let server = MockWorkerServer::workflow_completion(status, response);
12544 let client = Client::builder(server.base_url())
12545 .timeout(Duration::from_secs(2))
12546 .build()
12547 .expect("client");
12548 let mut worker = Worker::new(client, "rust-workers")
12549 .worker_id("timeout-worker")
12550 .poll_timeout(Duration::from_millis(10));
12551 worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
12552 Ok(json!({"late": "result"}))
12553 });
12554
12555 let error = worker
12556 .run_once()
12557 .await
12558 .expect_err(&format!("{name} must remain an error"));
12559 assert!(
12560 matches!(error, Error::Http { .. } | Error::Protocol(_)),
12561 "{name} returned an unexpected error variant: {error}"
12562 );
12563 }
12564 }
12565
12566 #[tokio::test]
12567 async fn worker_deregistration_uses_worker_plane_method_path_headers_and_result() {
12568 let server = MockWorkerServer::start();
12569 let client = Client::builder(server.base_url())
12570 .worker_token(Some("worker-secret".to_string()))
12571 .namespace("orders")
12572 .timeout(Duration::from_secs(2))
12573 .build()
12574 .expect("client");
12575 let path = "/api/worker/registrations/worker%2F%CE%B1%20space";
12576
12577 let result = client
12578 .deregister_worker_registration("worker/α space")
12579 .await
12580 .expect("deregister worker registration");
12581
12582 assert_eq!(server.method_for(path).as_deref(), Some("DELETE"));
12583 assert_eq!(
12584 server.worker_protocol_for(path).as_deref(),
12585 Some(WORKER_PROTOCOL_VERSION)
12586 );
12587 assert_eq!(server.control_protocol_for(path), None);
12588 assert_eq!(server.namespace_for(path).as_deref(), Some("orders"));
12589 assert_eq!(
12590 server.authorization_for(path).as_deref(),
12591 Some("Bearer worker-secret")
12592 );
12593 assert_eq!(
12594 result,
12595 WorkerDeregistrationEnvelope {
12596 worker_id: "deregistered-worker".to_string(),
12597 outcome: "deregistered".to_string(),
12598 recovered_workflow_task_count: 2,
12599 }
12600 );
12601 }
12602
12603 #[tokio::test]
12604 async fn low_level_registration_rejects_update_validators_before_transport() {
12605 let server = MockWorkerServer::start();
12606 let client = Client::builder(server.base_url())
12607 .timeout(Duration::from_secs(2))
12608 .build()
12609 .expect("client");
12610
12611 for update_validators in [json!(["approve"]), json!("approve")] {
12612 let error = client
12613 .register_worker_with_command_contracts(
12614 "validator-claiming-worker",
12615 "rust-workers",
12616 vec!["orders".to_string()],
12617 vec![],
12618 1,
12619 1,
12620 vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
12621 json!({
12622 "orders": {
12623 "queries": ["current"],
12624 "updates": ["approve"],
12625 "update_validators": update_validators,
12626 },
12627 }),
12628 )
12629 .await
12630 .expect_err("unsupported validator claims must fail before registration");
12631
12632 let Error::UnsupportedUpdateValidators { workflow_type } = error else {
12633 panic!("expected typed unsupported-validator failure");
12634 };
12635 assert_eq!(workflow_type, "orders");
12636 }
12637 assert_eq!(server.request_count("/api/worker/register"), 0);
12638 }
12639
12640 #[tokio::test]
12641 async fn low_level_registration_preserves_query_and_update_contracts() {
12642 let server = MockWorkerServer::start();
12643 let client = Client::builder(server.base_url())
12644 .timeout(Duration::from_secs(2))
12645 .build()
12646 .expect("client");
12647 let contracts = json!({
12648 "orders": {
12649 "queries": ["current"],
12650 "updates": ["approve"],
12651 "update_validators": [],
12652 },
12653 "payments": {
12654 "queries": ["status"],
12655 "updates": ["capture"],
12656 },
12657 });
12658
12659 client
12660 .register_worker_with_command_contracts(
12661 "command-worker",
12662 "rust-workers",
12663 vec!["orders".to_string(), "payments".to_string()],
12664 vec![],
12665 1,
12666 1,
12667 vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
12668 contracts.clone(),
12669 )
12670 .await
12671 .expect("query and update contracts must remain supported");
12672
12673 assert_eq!(
12674 server.request_body("/api/worker/register")["workflow_command_contracts"],
12675 contracts
12676 );
12677 }
12678
12679 #[tokio::test]
12680 async fn role_scoped_tokens_are_never_used_for_the_opposite_plane() {
12681 let server = MockWorkerServer::start();
12682 let control_only = Client::builder(server.base_url())
12683 .control_token(Some("control-secret".to_string()))
12684 .build()
12685 .expect("control client");
12686
12687 let error = control_only
12688 .register_worker("worker", "queue", vec![], vec![], 1, 1)
12689 .await
12690 .expect_err("control token must not authorize a worker request");
12691 assert!(matches!(
12692 error,
12693 Error::MissingRoleCredentials { role: "worker", .. }
12694 ));
12695 assert_eq!(server.request_count("/api/worker/register"), 0);
12696
12697 let worker_only = Client::builder(server.base_url())
12698 .worker_token(Some("worker-secret".to_string()))
12699 .build()
12700 .expect("worker client");
12701 let error = worker_only
12702 .health()
12703 .await
12704 .expect_err("worker token must not authorize a control request");
12705 assert!(matches!(
12706 error,
12707 Error::MissingRoleCredentials {
12708 role: "control",
12709 ..
12710 }
12711 ));
12712 assert_eq!(server.request_count("/api/health"), 0);
12713 }
12714
12715 #[tokio::test]
12716 async fn shared_token_supports_worker_and_control_planes() {
12717 let server = MockWorkerServer::start();
12718 let client = Client::builder(server.base_url())
12719 .token(Some("shared-secret".to_string()))
12720 .build()
12721 .expect("client");
12722
12723 client.health().await.expect("control request");
12724 client
12725 .register_worker("worker", "queue", vec![], vec![], 1, 1)
12726 .await
12727 .expect("worker request");
12728
12729 assert_eq!(
12730 server.authorization_for("/api/health").as_deref(),
12731 Some("Bearer shared-secret")
12732 );
12733 assert_eq!(
12734 server.control_protocol_for("/api/health").as_deref(),
12735 Some(CONTROL_PLANE_VERSION)
12736 );
12737 assert_eq!(
12738 server.authorization_for("/api/worker/register").as_deref(),
12739 Some("Bearer shared-secret")
12740 );
12741 assert_eq!(
12742 server
12743 .worker_protocol_for("/api/worker/register")
12744 .as_deref(),
12745 Some(WORKER_PROTOCOL_VERSION)
12746 );
12747 }
12748
12749 #[tokio::test]
12750 async fn baseline_worker_endpoints_send_the_baseline_protocol() {
12751 let server = MockWorkerServer::start();
12752 let client = Client::builder(server.base_url())
12753 .timeout(Duration::from_secs(2))
12754 .build()
12755 .expect("client");
12756
12757 client
12758 .register_worker("capture-worker", "capture", vec![], vec![], 1, 1)
12759 .await
12760 .expect("register");
12761 client
12762 .heartbeat_worker("capture-worker", 1, 1)
12763 .await
12764 .expect("heartbeat");
12765 client
12766 .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
12767 .await
12768 .expect("workflow poll");
12769 client
12770 .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
12771 .await
12772 .expect("activity poll");
12773
12774 for path in [
12775 "/api/worker/register",
12776 "/api/worker/heartbeat",
12777 "/api/worker/workflow-tasks/poll",
12778 "/api/worker/activity-tasks/poll",
12779 ] {
12780 assert_eq!(
12781 server.worker_protocol_for(path).as_deref(),
12782 Some(WORKER_PROTOCOL_VERSION),
12783 "unexpected protocol for {path}"
12784 );
12785 }
12786
12787 assert_eq!(
12788 server.request_body("/api/worker/workflow-tasks/poll")["timeout_seconds"],
12789 1
12790 );
12791 assert_eq!(
12792 server.request_body("/api/worker/activity-tasks/poll")["timeout_seconds"],
12793 1
12794 );
12795 assert!(
12796 server.request_body("/api/worker/workflow-tasks/poll")["poll_request_id"]
12797 .as_str()
12798 .is_some_and(|id| id.starts_with("rust-workflow-poll-"))
12799 );
12800 assert!(
12801 server.request_body("/api/worker/activity-tasks/poll")["poll_request_id"]
12802 .as_str()
12803 .is_some_and(|id| id.starts_with("rust-activity-poll-"))
12804 );
12805 }
12806
12807 #[tokio::test]
12808 async fn query_task_endpoints_send_the_query_feature_protocol() {
12809 let server = MockWorkerServer::start();
12810 let client = Client::builder(server.base_url())
12811 .timeout(Duration::from_secs(2))
12812 .build()
12813 .expect("client");
12814
12815 client
12816 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
12817 .await
12818 .expect("query poll");
12819 client
12820 .complete_query_task(
12821 "query-capture",
12822 "capture-worker",
12823 1,
12824 json!(8),
12825 DEFAULT_CODEC,
12826 )
12827 .await
12828 .expect("query complete");
12829 client
12830 .fail_query_task(
12831 "query-capture",
12832 "capture-worker",
12833 1,
12834 "failed",
12835 "query_rejected",
12836 "QueryFailed",
12837 )
12838 .await
12839 .expect("query fail");
12840
12841 for path in [
12842 "/api/worker/query-tasks/poll",
12843 "/api/worker/query-tasks/query-capture/complete",
12844 "/api/worker/query-tasks/query-capture/fail",
12845 ] {
12846 assert_eq!(
12847 server.worker_protocol_for(path).as_deref(),
12848 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
12849 "unexpected protocol for {path}"
12850 );
12851 }
12852
12853 assert_eq!(
12854 server.request_body("/api/worker/query-tasks/poll")["timeout_seconds"],
12855 1
12856 );
12857 assert!(
12858 server.request_body("/api/worker/query-tasks/poll")["poll_request_id"]
12859 .as_str()
12860 .is_some_and(|id| id.starts_with("rust-query-poll-"))
12861 );
12862 }
12863
12864 #[tokio::test]
12865 async fn disconnected_client_polls_retry_once_with_the_same_request_id() {
12866 let server = MockWorkerServer::transient_worker_failures();
12867 let client = Client::builder(server.base_url())
12868 .timeout(Duration::from_secs(2))
12869 .build()
12870 .expect("client");
12871
12872 client
12873 .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
12874 .await
12875 .expect("workflow poll retry");
12876 client
12877 .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
12878 .await
12879 .expect("activity poll retry");
12880 client
12881 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
12882 .await
12883 .expect("query poll retry");
12884
12885 for path in [
12886 "/api/worker/workflow-tasks/poll",
12887 "/api/worker/activity-tasks/poll",
12888 "/api/worker/query-tasks/poll",
12889 ] {
12890 let bodies = server.request_bodies(path);
12891 assert_eq!(bodies.len(), 2, "{path} must be retried once");
12892 assert_eq!(
12893 bodies[0]["poll_request_id"], bodies[1]["poll_request_id"],
12894 "{path} must preserve the request binding across retry"
12895 );
12896 }
12897 }
12898
12899 #[tokio::test]
12900 async fn worker_poll_retries_preserve_request_id_across_consecutive_failures() {
12901 let server = MockWorkerServer::consecutive_poll_failures(2);
12902 let client = Client::builder(server.base_url())
12903 .timeout(Duration::from_secs(2))
12904 .build()
12905 .expect("client");
12906 let mut worker = Worker::new(client, "capture")
12907 .worker_id("capture-worker")
12908 .poll_timeout(Duration::from_millis(10))
12909 .retry_policy(WorkerRetryPolicy {
12910 max_retries: 2,
12911 initial_backoff: Duration::from_millis(1),
12912 max_backoff: Duration::from_millis(1),
12913 });
12914 worker.register_workflow(
12915 "capture.workflow",
12916 |_ctx, _input| async move { Ok(Value::Null) },
12917 );
12918 worker.register_activity(
12919 "capture.activity",
12920 |_ctx, _input| async move { Ok(Value::Null) },
12921 );
12922 worker.register_query("capture.workflow", "current", |_ctx, _args| async move {
12923 Ok(Value::Null)
12924 });
12925
12926 assert_eq!(worker.run_once().await.expect("poll retries"), 0);
12927
12928 for path in [
12929 "/api/worker/workflow-tasks/poll",
12930 "/api/worker/activity-tasks/poll",
12931 "/api/worker/query-tasks/poll",
12932 ] {
12933 let bodies = server.request_bodies(path);
12934 assert_eq!(bodies.len(), 3, "{path} must use exactly two retries");
12935 assert!(
12936 bodies
12937 .iter()
12938 .all(|body| body["poll_request_id"] == bodies[0]["poll_request_id"]),
12939 "{path} must preserve one request binding across every retry"
12940 );
12941 }
12942 }
12943
12944 #[tokio::test]
12945 async fn query_protocol_rejection_from_older_server_is_typed() {
12946 let server = MockWorkerServer::reject_query_protocol();
12947 let client = Client::builder(server.base_url())
12948 .timeout(Duration::from_secs(2))
12949 .build()
12950 .expect("client");
12951
12952 let error = client
12953 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
12954 .await
12955 .expect_err("server below query protocol floor must reject");
12956 let Error::Protocol(failure) = error else {
12957 panic!("expected typed protocol failure");
12958 };
12959
12960 assert_eq!(failure.status, 400);
12961 assert_eq!(failure.reason, "unsupported_protocol_version");
12962 assert_eq!(failure.supported_version.as_deref(), Some("1.7"));
12963 assert_eq!(
12964 failure.requested_version.as_deref(),
12965 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
12966 );
12967 assert_eq!(
12968 server
12969 .worker_protocol_for("/api/worker/query-tasks/poll")
12970 .as_deref(),
12971 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
12972 );
12973 }
12974
12975 #[tokio::test]
12976 async fn run_once_without_query_handlers_keeps_pre_query_server_compatibility() {
12977 let server = MockWorkerServer::reject_query_protocol();
12978 let client = Client::builder(server.base_url())
12979 .timeout(Duration::from_secs(2))
12980 .build()
12981 .expect("client");
12982 let mut worker = Worker::new(client, "rust-workers")
12983 .worker_id("baseline-worker")
12984 .poll_timeout(Duration::from_millis(10));
12985
12986 worker.register_workflow("baseline.workflow", |_ctx, _input| async move {
12987 Ok(Value::Null)
12988 });
12989
12990 assert_eq!(worker.run_once().await.expect("baseline run once"), 0);
12991 assert_eq!(
12992 server
12993 .worker_protocol_for("/api/worker/workflow-tasks/poll")
12994 .as_deref(),
12995 Some(WORKER_PROTOCOL_VERSION)
12996 );
12997 assert_eq!(
12998 server.worker_protocol_for("/api/worker/query-tasks/poll"),
12999 None,
13000 "a worker without query handlers must not use the query-task endpoint"
13001 );
13002 }
13003
13004 #[tokio::test]
13005 async fn completion_time_query_rejection_is_typed_without_stopping_worker() {
13006 let server = MockWorkerServer::reject_query_completion();
13007 let client = Client::builder(server.base_url())
13008 .timeout(Duration::from_secs(2))
13009 .build()
13010 .expect("client");
13011
13012 let error = client
13013 .complete_query_task("query-late", "late-worker", 1, json!(8), DEFAULT_CODEC)
13014 .await
13015 .expect_err("expired completion must be rejected");
13016 let Error::QueryFailed(failure) = error else {
13017 panic!("expected typed query failure");
13018 };
13019 assert_eq!(failure.status, 409);
13020 assert_eq!(failure.reason, "query_task_timed_out");
13021
13022 let mut worker = Worker::new(client, "rust-workers")
13023 .worker_id("late-worker")
13024 .poll_timeout(Duration::from_millis(10));
13025 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
13026 worker.register_query(
13027 "counter",
13028 "current",
13029 |_ctx, _args| async move { Ok(json!(8)) },
13030 );
13031
13032 assert_eq!(worker.run_once().await.expect("late task is handled"), 1);
13033 assert_eq!(
13034 worker
13035 .run_once()
13036 .await
13037 .expect("worker continues after late completion"),
13038 0
13039 );
13040 assert_eq!(
13041 server.request_count("/api/worker/query-tasks/query-late/complete"),
13042 2
13043 );
13044 assert_eq!(
13045 server.request_count("/api/worker/query-tasks/query-late/fail"),
13046 0,
13047 "a server completion rejection must not be reported as an encoding failure"
13048 );
13049 }
13050
13051 #[tokio::test]
13052 async fn normal_shutdown_joins_pollers_and_deregisters_once() {
13053 let server = MockWorkerServer::start();
13054 let client = Client::builder(server.base_url())
13055 .timeout(Duration::from_secs(2))
13056 .build()
13057 .expect("client");
13058 let mut worker = Worker::new(client, "rust-workers")
13059 .worker_id("joined-worker")
13060 .poll_timeout(Duration::from_millis(10));
13061 worker.register_workflow(
13062 "joined.workflow",
13063 |_ctx, _input| async move { Ok(Value::Null) },
13064 );
13065 worker.register_activity(
13066 "joined.activity",
13067 |_ctx, _input| async move { Ok(Value::Null) },
13068 );
13069 worker.register_query("joined.workflow", "state", |_ctx, _input| async move {
13070 Ok(Value::Null)
13071 });
13072
13073 worker
13074 .run_until(tokio::time::sleep(Duration::from_millis(20)))
13075 .await
13076 .expect("normal shutdown");
13077
13078 let deregistration_path = "/api/worker/registrations/mock-worker";
13079 assert_eq!(server.request_count(deregistration_path), 1);
13080 for poll_path in [
13081 "/api/worker/workflow-tasks/poll",
13082 "/api/worker/activity-tasks/poll",
13083 "/api/worker/query-tasks/poll",
13084 ] {
13085 assert!(server.request_count(poll_path) > 0, "missing {poll_path}");
13086 }
13087 assert_eq!(
13088 server.captured_paths().last().map(String::as_str),
13089 Some(deregistration_path),
13090 "deregistration must start only after every poller has joined"
13091 );
13092 }
13093
13094 #[tokio::test]
13095 async fn registration_failure_does_not_deregister() {
13096 let server = MockWorkerServer::rejected_registration();
13097 let client = Client::builder(server.base_url())
13098 .timeout(Duration::from_secs(2))
13099 .build()
13100 .expect("client");
13101 let worker = Worker::new(client, "rust-workers").worker_id("never-registered");
13102
13103 let error = worker
13104 .run_until(async {})
13105 .await
13106 .expect_err("registration must fail");
13107 assert!(matches!(
13108 error,
13109 Error::Http {
13110 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
13111 ..
13112 }
13113 ));
13114 assert!(server
13115 .captured_paths()
13116 .iter()
13117 .all(|path| !path.starts_with("/api/worker/registrations/")));
13118 }
13119
13120 #[tokio::test]
13121 async fn declined_registration_does_not_deregister() {
13122 let server = MockWorkerServer::declined_registration();
13123 let client = Client::builder(server.base_url())
13124 .timeout(Duration::from_secs(2))
13125 .build()
13126 .expect("client");
13127 let worker = Worker::new(client, "rust-workers").worker_id("declined-worker");
13128
13129 let error = worker
13130 .run_until(async {})
13131 .await
13132 .expect_err("declined registration must fail");
13133 assert!(matches!(error, Error::WorkerLoop(_)));
13134 assert!(error.to_string().contains("was not accepted"));
13135 assert!(server
13136 .captured_paths()
13137 .iter()
13138 .all(|path| !path.starts_with("/api/worker/registrations/")));
13139 }
13140
13141 #[tokio::test]
13142 async fn deregistration_http_failure_is_returned_after_normal_shutdown() {
13143 let server = MockWorkerServer::rejected_deregistration();
13144 let client = Client::builder(server.base_url())
13145 .timeout(Duration::from_secs(2))
13146 .build()
13147 .expect("client");
13148 let worker = Worker::new(client, "rust-workers").worker_id("forbidden-cleanup");
13149
13150 let error = worker
13151 .run_until(async {})
13152 .await
13153 .expect_err("deregistration must fail");
13154 assert!(matches!(
13155 error,
13156 Error::Http {
13157 status: reqwest::StatusCode::FORBIDDEN,
13158 ..
13159 }
13160 ));
13161 assert_eq!(
13162 server.request_count("/api/worker/registrations/mock-worker"),
13163 1
13164 );
13165 }
13166
13167 #[tokio::test]
13168 async fn deregistration_protocol_failure_is_returned_after_normal_shutdown() {
13169 let server = MockWorkerServer::rejected_deregistration_protocol();
13170 let client = Client::builder(server.base_url())
13171 .timeout(Duration::from_secs(2))
13172 .build()
13173 .expect("client");
13174 let worker = Worker::new(client, "rust-workers").worker_id("protocol-cleanup");
13175
13176 let error = worker
13177 .run_until(async {})
13178 .await
13179 .expect_err("protocol rejection must fail shutdown");
13180 let Error::Protocol(failure) = error else {
13181 panic!("expected typed protocol failure");
13182 };
13183 assert_eq!(failure.reason, "unsupported_protocol_version");
13184 assert_eq!(failure.requested_version.as_deref(), Some("1.2"));
13185 assert_eq!(
13186 server.request_count("/api/worker/registrations/mock-worker"),
13187 1
13188 );
13189 }
13190
13191 #[tokio::test]
13192 async fn primary_poller_error_retains_deregistration_failure_context() {
13193 let server = MockWorkerServer::unauthorized_polls_and_rejected_deregistration();
13194 let client = Client::builder(server.base_url())
13195 .timeout(Duration::from_secs(2))
13196 .build()
13197 .expect("client");
13198 let mut worker = Worker::new(client, "rust-workers")
13199 .worker_id("combined-failure")
13200 .poll_timeout(Duration::from_millis(10));
13201 worker.register_workflow("combined.workflow", |_ctx, _input| async move {
13202 Ok(Value::Null)
13203 });
13204
13205 let error = worker
13206 .run()
13207 .await
13208 .expect_err("worker and cleanup must fail");
13209 let summary = error.to_string();
13210 assert!(summary.contains("authentication_failed"));
13211 assert!(summary.contains("worker cannot deregister"));
13212 let Error::WorkerShutdown {
13213 primary,
13214 deregistration,
13215 } = error
13216 else {
13217 panic!("expected combined worker shutdown error");
13218 };
13219 assert!(matches!(
13220 *primary,
13221 Error::Http {
13222 status: reqwest::StatusCode::UNAUTHORIZED,
13223 ..
13224 }
13225 ));
13226 assert!(matches!(
13227 *deregistration,
13228 Error::Http {
13229 status: reqwest::StatusCode::FORBIDDEN,
13230 ..
13231 }
13232 ));
13233 assert_eq!(
13234 server.request_count("/api/worker/registrations/mock-worker"),
13235 1
13236 );
13237 }
13238
13239 #[tokio::test]
13240 async fn activity_only_worker_can_shutdown_without_workflow_poller() {
13241 let server = MockWorkerServer::start();
13242 let client = Client::builder(server.base_url())
13243 .timeout(Duration::from_secs(2))
13244 .build()
13245 .expect("client");
13246 let mut worker = Worker::new(client, "rust-workers")
13247 .worker_id("activity-only-worker")
13248 .poll_timeout(Duration::from_millis(10));
13249
13250 worker.register_activity(
13251 "activity.only",
13252 |_ctx, _args| async move { Ok(Value::Null) },
13253 );
13254
13255 worker.run_until(async {}).await.expect("run worker");
13256 }
13257
13258 #[tokio::test]
13259 async fn workflow_only_worker_can_shutdown_without_activity_poller() {
13260 let server = MockWorkerServer::start();
13261 let client = Client::builder(server.base_url())
13262 .timeout(Duration::from_secs(2))
13263 .build()
13264 .expect("client");
13265 let mut worker = Worker::new(client, "rust-workers")
13266 .worker_id("workflow-only-worker")
13267 .poll_timeout(Duration::from_millis(10));
13268
13269 worker.register_workflow(
13270 "workflow.only",
13271 |_ctx, _input| async move { Ok(Value::Null) },
13272 );
13273
13274 worker.run_until(async {}).await.expect("run worker");
13275 }
13276
13277 #[tokio::test]
13278 async fn worker_heartbeat_observer_receives_server_acknowledgements() {
13279 let server = MockWorkerServer::start();
13280 let client = Client::builder(server.base_url())
13281 .timeout(Duration::from_secs(2))
13282 .build()
13283 .expect("client");
13284 let observations = Arc::new(Mutex::new(Vec::new()));
13285 let observed = Arc::clone(&observations);
13286 let mut worker = Worker::new(client, "rust-workers")
13287 .worker_id("observed-heartbeat-worker")
13288 .poll_timeout(Duration::from_millis(10))
13289 .on_worker_heartbeat(move |observation| {
13290 observed
13291 .lock()
13292 .expect("heartbeat observations")
13293 .push(observation.clone());
13294 });
13295
13296 worker.register_workflow("workflow.observed", |_ctx, _input| async move {
13297 Ok(Value::Null)
13298 });
13299 let acknowledged = Arc::clone(&observations);
13300 worker
13301 .run_until(async move {
13302 tokio::time::timeout(Duration::from_secs(2), async move {
13303 loop {
13304 if !acknowledged
13305 .lock()
13306 .expect("heartbeat observations")
13307 .is_empty()
13308 {
13309 break;
13310 }
13311 tokio::time::sleep(Duration::from_millis(1)).await;
13312 }
13313 })
13314 .await
13315 .expect("heartbeat acknowledgement within timeout");
13316 })
13317 .await
13318 .expect("run worker");
13319
13320 let observations = observations.lock().expect("heartbeat observations");
13321 let first = observations.first().expect("heartbeat acknowledgement");
13322 assert_eq!(first.worker_id, "observed-heartbeat-worker");
13323 assert_eq!(first.task_queue, "rust-workers");
13324 assert!(first.acknowledged_at_unix_millis > 0);
13325 assert_eq!(first.acknowledgement, json!({}));
13326 }
13327
13328 #[tokio::test]
13329 async fn delayed_worker_heartbeat_keeps_cadence_and_pollers_live() {
13330 let server = MockWorkerServer::delayed_heartbeat_worker();
13331 let client = Client::builder(server.base_url())
13332 .timeout(Duration::from_secs(3))
13333 .build()
13334 .expect("client");
13335 let observations = Arc::new(Mutex::new(Vec::new()));
13336 let observed = Arc::clone(&observations);
13337 let mut worker = Worker::new(client, "rust-snapshot-workers")
13338 .worker_id("rust-snapshot-worker")
13339 .poll_timeout(Duration::from_millis(10))
13340 .on_worker_heartbeat(move |observation| {
13341 observed
13342 .lock()
13343 .expect("heartbeat observations")
13344 .push(observation.clone());
13345 });
13346
13347 worker.register_workflow("snapshot", |ctx, _input| async move {
13348 ctx.wait_signal("finish").await?;
13349 Ok(json!({"status": "finished"}))
13350 });
13351 worker.register_query("snapshot", "current", |ctx, _args| async move {
13352 Ok(json!(ctx
13353 .signals("increment")
13354 .iter()
13355 .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
13356 .sum::<i64>()))
13357 });
13358 worker.register_activity("cancel-aware", |_ctx, _args| async move {
13359 Ok(json!({"late": "completion"}))
13360 });
13361
13362 worker
13363 .run_until(tokio::time::sleep(Duration::from_millis(3_800)))
13364 .await
13365 .expect("delayed heartbeat must allow a clean worker shutdown");
13366
13367 let observations = observations.lock().expect("heartbeat observations");
13368 assert!(
13369 observations.len() >= 3,
13370 "the immediate heartbeat, delayed acknowledgement, and next cadence heartbeat must complete"
13371 );
13372 assert!(
13373 observations.windows(2).all(|pair| {
13374 pair[1].acknowledged_at_unix_millis
13375 .saturating_sub(pair[0].acknowledged_at_unix_millis)
13376 >= 850
13377 }),
13378 "successful acknowledgements must not catch up faster than the advertised one-second cadence: {observations:?}"
13379 );
13380 drop(observations);
13381
13382 let heartbeat_times = server.request_times("/api/worker/heartbeat");
13383 let delayed_request_at = *heartbeat_times
13384 .get(1)
13385 .expect("intentionally delayed heartbeat request");
13386 let delay_window_start = delayed_request_at + Duration::from_millis(100);
13387 let delay_window_end = delayed_request_at + Duration::from_millis(1_400);
13388 for path in [
13389 "/api/worker/workflow-tasks/poll",
13390 "/api/worker/activity-tasks/poll",
13391 "/api/worker/query-tasks/poll",
13392 ] {
13393 assert!(
13394 server
13395 .request_times(path)
13396 .iter()
13397 .any(|received_at| *received_at >= delay_window_start
13398 && *received_at <= delay_window_end),
13399 "{path} must keep polling while a heartbeat acknowledgement is delayed"
13400 );
13401 }
13402 assert!(
13403 server.request_count("/api/worker/workflow-tasks/snapshot-wait-3/fail") >= 1,
13404 "workflow work must be settled"
13405 );
13406 assert!(
13407 server.request_count("/api/worker/activity-tasks/activity-cancel/complete") >= 1,
13408 "activity work must be settled"
13409 );
13410 assert!(
13411 server.request_count("/api/worker/query-tasks/snapshot-current/complete") >= 1,
13412 "query work must be settled"
13413 );
13414 }
13415
13416 #[tokio::test]
13417 async fn retried_worker_heartbeat_restarts_the_advertised_cadence() {
13418 let server = MockWorkerServer::heartbeat_retry_worker();
13419 let client = Client::builder(server.base_url())
13420 .timeout(Duration::from_secs(2))
13421 .build()
13422 .expect("client");
13423 let observations = Arc::new(Mutex::new(Vec::new()));
13424 let observed = Arc::clone(&observations);
13425 let worker = Worker::new(client, "rust-workers")
13426 .worker_id("heartbeat-retry-worker")
13427 .retry_policy(WorkerRetryPolicy {
13428 max_retries: 1,
13429 initial_backoff: Duration::from_millis(300),
13430 max_backoff: Duration::from_millis(300),
13431 })
13432 .on_worker_heartbeat(move |observation| {
13433 observed
13434 .lock()
13435 .expect("heartbeat observations")
13436 .push(observation.clone());
13437 });
13438
13439 worker
13440 .run_until(tokio::time::sleep(Duration::from_millis(2_700)))
13441 .await
13442 .expect("retryable heartbeat failure must remain bounded and recover");
13443
13444 let observations = observations.lock().expect("heartbeat observations");
13445 assert!(observations.len() >= 3, "heartbeat retry must recover");
13446 assert!(
13447 observations.windows(2).all(|pair| {
13448 pair[1]
13449 .acknowledged_at_unix_millis
13450 .saturating_sub(pair[0].acknowledged_at_unix_millis)
13451 >= 850
13452 }),
13453 "a successful retry must start a fresh advertised cadence: {observations:?}"
13454 );
13455 assert_eq!(
13456 server.request_count("/api/worker/heartbeat"),
13457 observations.len() + 1,
13458 "one retryable failure must add exactly one bounded request"
13459 );
13460 }
13461
13462 #[tokio::test]
13463 async fn query_enabled_worker_ignores_unmatched_signals_then_completes_once() {
13464 let server = MockWorkerServer::waiting_query_worker();
13465 let client = Client::builder(server.base_url())
13466 .timeout(Duration::from_secs(2))
13467 .build()
13468 .expect("client");
13469 let observations = Arc::new(Mutex::new(Vec::new()));
13470 let observed = Arc::clone(&observations);
13471 let mut worker = Worker::new(client, "rust-snapshot-workers")
13472 .worker_id("rust-snapshot-worker")
13473 .poll_timeout(Duration::from_millis(10))
13474 .on_worker_heartbeat(move |observation| {
13475 observed
13476 .lock()
13477 .expect("heartbeat observations")
13478 .push(observation.clone());
13479 });
13480
13481 worker.register_workflow("snapshot", |ctx, _input| async move {
13482 ctx.wait_signal("finish").await?;
13483 Ok(json!({"status": "finished"}))
13484 });
13485 worker.register_query("snapshot", "current", |ctx, _args| async move {
13486 let current = ctx
13487 .signals("increment")
13488 .iter()
13489 .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
13490 .sum::<i64>();
13491 Ok(json!(current))
13492 });
13493 worker.register_update("snapshot", "replace", |_ctx, args| async move { Ok(args) });
13494
13495 worker
13496 .run_until(tokio::time::sleep(Duration::from_millis(3_200)))
13497 .await
13498 .expect("pending workflow and query poller must remain live until shutdown");
13499
13500 assert!(
13501 observations.lock().expect("heartbeat observations").len() >= 4,
13502 "the immediate heartbeat and at least three advertised one-second intervals must be acknowledged"
13503 );
13504 assert!(
13505 server.request_count("/api/worker/workflow-tasks/poll") >= 3,
13506 "workflow polling must continue after empty replay acknowledgements"
13507 );
13508 assert!(
13509 server.request_count("/api/worker/query-tasks/poll") >= 2,
13510 "query polling must continue after serving the current query"
13511 );
13512 assert_eq!(
13513 server.request_body("/api/worker/register")["capabilities"],
13514 json!([QUERY_TASKS_CAPABILITY, WORKFLOW_UPDATES_CAPABILITY])
13515 );
13516 assert_eq!(
13517 server.request_body("/api/worker/register")["workflow_command_contracts"]["snapshot"],
13518 json!({
13519 "queries": ["current"],
13520 "updates": ["replace"],
13521 "update_validators": [],
13522 })
13523 );
13524
13525 let opened = server.request_body("/api/worker/workflow-tasks/snapshot-open/complete");
13526 assert_eq!(
13527 opened["commands"],
13528 json!([{
13529 "type": "open_signal_wait",
13530 "signal_name": "finish",
13531 }])
13532 );
13533
13534 for task_id in ["snapshot-wait-3", "snapshot-wait-5"] {
13535 let fail_path = format!("/api/worker/workflow-tasks/{task_id}/fail");
13536 let completion_path = format!("/api/worker/workflow-tasks/{task_id}/complete");
13537 let failure = server.request_body(&fail_path);
13538 assert_eq!(
13539 failure["failure"]["type"],
13540 WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE
13541 );
13542 assert_eq!(server.request_count(&completion_path), 0);
13543 }
13544
13545 let query_completion =
13546 server.request_body("/api/worker/query-tasks/snapshot-current/complete");
13547 assert_eq!(query_completion["result"], json!(8));
13548
13549 let terminal_path = "/api/worker/workflow-tasks/snapshot-finish/complete";
13550 assert_eq!(
13551 server.request_count(terminal_path),
13552 1,
13553 "the matching signal must settle the workflow exactly once"
13554 );
13555 let terminal = server.request_body(terminal_path);
13556 assert_eq!(terminal["commands"].as_array().map(Vec::len), Some(1));
13557 assert_eq!(terminal["commands"][0]["type"], "complete_workflow");
13558 assert_eq!(
13559 decode_wire_value(&terminal["commands"][0]["result"], DEFAULT_CODEC)
13560 .expect("terminal workflow result"),
13561 json!({"status": "finished"})
13562 );
13563 }
13564
13565 #[tokio::test]
13566 async fn worker_retries_poll_and_heartbeat_transport_failures_independently() {
13567 let server = MockWorkerServer::transient_worker_failures();
13568 let client = Client::builder(server.base_url())
13569 .timeout(Duration::from_secs(2))
13570 .build()
13571 .expect("client");
13572 let mut worker = Worker::new(client, "rust-workers")
13573 .worker_id("retry-worker")
13574 .poll_timeout(Duration::from_millis(10))
13575 .retry_policy(WorkerRetryPolicy {
13576 max_retries: 2,
13577 initial_backoff: Duration::from_millis(1),
13578 max_backoff: Duration::from_millis(1),
13579 });
13580 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
13581 worker.register_activity(
13582 "counter.activity",
13583 |_ctx, _input| async move { Ok(Value::Null) },
13584 );
13585 worker.register_query(
13586 "counter",
13587 "current",
13588 |_ctx, _args| async move { Ok(json!(8)) },
13589 );
13590
13591 worker
13592 .run_until(tokio::time::sleep(Duration::from_millis(75)))
13593 .await
13594 .expect("transient failures must not stop the worker");
13595
13596 for path in [
13597 "/api/worker/heartbeat",
13598 "/api/worker/workflow-tasks/poll",
13599 "/api/worker/activity-tasks/poll",
13600 "/api/worker/query-tasks/poll",
13601 ] {
13602 assert!(
13603 server.request_count(path) >= 2,
13604 "{path} must continue after its transient failure"
13605 );
13606 }
13607 }
13608
13609 #[tokio::test]
13610 async fn worker_bounds_transport_retries() {
13611 let server = MockWorkerServer::unavailable_polls();
13612 let client = Client::builder(server.base_url())
13613 .timeout(Duration::from_secs(2))
13614 .build()
13615 .expect("client");
13616 let mut worker = Worker::new(client, "rust-workers")
13617 .worker_id("bounded-retry-worker")
13618 .poll_timeout(Duration::from_millis(10))
13619 .retry_policy(WorkerRetryPolicy {
13620 max_retries: 2,
13621 initial_backoff: Duration::from_millis(1),
13622 max_backoff: Duration::from_millis(1),
13623 });
13624 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
13625
13626 let error = worker.run().await.expect_err("retry bound must terminate");
13627 assert!(matches!(error, Error::Transport(_)));
13628 assert_eq!(
13629 server.request_count("/api/worker/workflow-tasks/poll"),
13630 3,
13631 "one initial request plus exactly two retries"
13632 );
13633 }
13634
13635 #[tokio::test]
13636 async fn worker_retry_policy_can_disable_poll_retries() {
13637 let server = MockWorkerServer::unavailable_polls();
13638 let client = Client::builder(server.base_url())
13639 .timeout(Duration::from_secs(2))
13640 .build()
13641 .expect("client");
13642 let mut worker = Worker::new(client, "rust-workers")
13643 .worker_id("no-retry-worker")
13644 .poll_timeout(Duration::from_millis(10))
13645 .retry_policy(WorkerRetryPolicy {
13646 max_retries: 0,
13647 initial_backoff: Duration::from_millis(1),
13648 max_backoff: Duration::from_millis(1),
13649 });
13650 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
13651
13652 let error = worker
13653 .run_once()
13654 .await
13655 .expect_err("disabled retries must return the first transport failure");
13656 assert!(matches!(error, Error::Transport(_)));
13657 assert_eq!(
13658 server.request_count("/api/worker/workflow-tasks/poll"),
13659 1,
13660 "max_retries=0 must send only the initial request"
13661 );
13662 }
13663
13664 #[tokio::test]
13665 async fn worker_does_not_retry_authentication_failures() {
13666 let server = MockWorkerServer::unauthorized_polls();
13667 let client = Client::builder(server.base_url())
13668 .timeout(Duration::from_secs(2))
13669 .build()
13670 .expect("client");
13671 let mut worker = Worker::new(client, "rust-workers")
13672 .worker_id("unauthorized-worker")
13673 .poll_timeout(Duration::from_millis(10));
13674 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
13675
13676 let error = worker
13677 .run()
13678 .await
13679 .expect_err("authentication must terminate");
13680 let Error::Http { status, body } = error else {
13681 panic!("expected stable HTTP authentication error");
13682 };
13683 assert_eq!(status, reqwest::StatusCode::UNAUTHORIZED);
13684 assert!(body.contains("authentication_failed"));
13685 assert_eq!(
13686 server.request_count("/api/worker/workflow-tasks/poll"),
13687 1,
13688 "authentication failures must not be retried"
13689 );
13690 }
13691
13692 #[derive(Clone, Debug)]
13693 struct CapturedRequest {
13694 method: String,
13695 path: String,
13696 authorization: Option<String>,
13697 namespace: Option<String>,
13698 worker_protocol: Option<String>,
13699 control_protocol: Option<String>,
13700 body: String,
13701 received_at: Instant,
13702 }
13703
13704 struct MockWorkerServer {
13705 addr: SocketAddr,
13706 stop: Arc<AtomicBool>,
13707 requests: Arc<Mutex<Vec<CapturedRequest>>>,
13708 thread: Option<thread::JoinHandle<()>>,
13709 }
13710
13711 #[derive(Clone, Copy, Default)]
13712 struct MockWorkerBehavior {
13713 reject_query_protocol: bool,
13714 reject_query_completion: bool,
13715 waiting_query_worker: bool,
13716 decline_registration: bool,
13717 complete_named_signal: bool,
13718 poll_failures_per_path: usize,
13719 heartbeat_failures: usize,
13720 heartbeat_failure_request: Option<usize>,
13721 delayed_heartbeat_request: Option<usize>,
13722 heartbeat_response_delay: Duration,
13723 concurrent_requests: bool,
13724 unauthorized_polls: bool,
13725 reject_registration: bool,
13726 reject_deregistration: bool,
13727 reject_deregistration_protocol: bool,
13728 cancelled_activity: bool,
13729 draining_polls: bool,
13730 workflow_completion_status: Option<&'static str>,
13731 workflow_completion_body: Option<&'static str>,
13732 }
13733
13734 impl MockWorkerServer {
13735 fn start() -> Self {
13736 Self::start_with_behavior(MockWorkerBehavior::default())
13737 }
13738
13739 fn reject_query_protocol() -> Self {
13740 Self::start_with_behavior(MockWorkerBehavior {
13741 reject_query_protocol: true,
13742 ..MockWorkerBehavior::default()
13743 })
13744 }
13745
13746 fn reject_query_completion() -> Self {
13747 Self::start_with_behavior(MockWorkerBehavior {
13748 reject_query_completion: true,
13749 ..MockWorkerBehavior::default()
13750 })
13751 }
13752
13753 fn waiting_query_worker() -> Self {
13754 Self::start_with_behavior(MockWorkerBehavior {
13755 waiting_query_worker: true,
13756 complete_named_signal: true,
13757 ..MockWorkerBehavior::default()
13758 })
13759 }
13760
13761 fn transient_worker_failures() -> Self {
13762 Self::start_with_behavior(MockWorkerBehavior {
13763 poll_failures_per_path: 1,
13764 heartbeat_failures: 1,
13765 ..MockWorkerBehavior::default()
13766 })
13767 }
13768
13769 fn consecutive_poll_failures(count: usize) -> Self {
13770 Self::start_with_behavior(MockWorkerBehavior {
13771 poll_failures_per_path: count,
13772 ..MockWorkerBehavior::default()
13773 })
13774 }
13775
13776 fn delayed_heartbeat_worker() -> Self {
13777 Self::start_with_behavior(MockWorkerBehavior {
13778 waiting_query_worker: true,
13779 delayed_heartbeat_request: Some(2),
13780 heartbeat_response_delay: Duration::from_millis(1_500),
13781 concurrent_requests: true,
13782 cancelled_activity: true,
13783 ..MockWorkerBehavior::default()
13784 })
13785 }
13786
13787 fn heartbeat_retry_worker() -> Self {
13788 Self::start_with_behavior(MockWorkerBehavior {
13789 waiting_query_worker: true,
13790 heartbeat_failure_request: Some(2),
13791 concurrent_requests: true,
13792 ..MockWorkerBehavior::default()
13793 })
13794 }
13795
13796 fn unavailable_polls() -> Self {
13797 Self::start_with_behavior(MockWorkerBehavior {
13798 poll_failures_per_path: usize::MAX,
13799 ..MockWorkerBehavior::default()
13800 })
13801 }
13802
13803 fn unauthorized_polls() -> Self {
13804 Self::start_with_behavior(MockWorkerBehavior {
13805 unauthorized_polls: true,
13806 ..MockWorkerBehavior::default()
13807 })
13808 }
13809
13810 fn rejected_registration() -> Self {
13811 Self::start_with_behavior(MockWorkerBehavior {
13812 reject_registration: true,
13813 ..MockWorkerBehavior::default()
13814 })
13815 }
13816
13817 fn declined_registration() -> Self {
13818 Self::start_with_behavior(MockWorkerBehavior {
13819 decline_registration: true,
13820 ..MockWorkerBehavior::default()
13821 })
13822 }
13823
13824 fn rejected_deregistration() -> Self {
13825 Self::start_with_behavior(MockWorkerBehavior {
13826 reject_deregistration: true,
13827 ..MockWorkerBehavior::default()
13828 })
13829 }
13830
13831 fn rejected_deregistration_protocol() -> Self {
13832 Self::start_with_behavior(MockWorkerBehavior {
13833 reject_deregistration_protocol: true,
13834 ..MockWorkerBehavior::default()
13835 })
13836 }
13837
13838 fn unauthorized_polls_and_rejected_deregistration() -> Self {
13839 Self::start_with_behavior(MockWorkerBehavior {
13840 unauthorized_polls: true,
13841 reject_deregistration: true,
13842 ..MockWorkerBehavior::default()
13843 })
13844 }
13845
13846 fn cancelled_activity() -> Self {
13847 Self::start_with_behavior(MockWorkerBehavior {
13848 cancelled_activity: true,
13849 ..MockWorkerBehavior::default()
13850 })
13851 }
13852
13853 fn draining_polls() -> Self {
13854 Self::start_with_behavior(MockWorkerBehavior {
13855 draining_polls: true,
13856 ..MockWorkerBehavior::default()
13857 })
13858 }
13859
13860 fn workflow_completion(status: &'static str, body: &'static str) -> Self {
13861 Self::start_with_behavior(MockWorkerBehavior {
13862 workflow_completion_status: Some(status),
13863 workflow_completion_body: Some(body),
13864 ..MockWorkerBehavior::default()
13865 })
13866 }
13867
13868 fn start_with_behavior(behavior: MockWorkerBehavior) -> Self {
13869 let listener = TcpListener::bind("127.0.0.1:0").expect("bind mock server");
13870 listener
13871 .set_nonblocking(true)
13872 .expect("configure mock listener");
13873 let addr = listener.local_addr().expect("mock server address");
13874 let stop = Arc::new(AtomicBool::new(false));
13875 let server_stop = Arc::clone(&stop);
13876 let requests = Arc::new(Mutex::new(Vec::new()));
13877 let server_requests = Arc::clone(&requests);
13878 let thread = thread::spawn(move || {
13879 let mut request_threads = Vec::new();
13880 while !server_stop.load(Ordering::SeqCst) {
13881 match listener.accept() {
13882 Ok((mut stream, _)) => {
13883 if behavior.concurrent_requests {
13884 let requests = Arc::clone(&server_requests);
13885 request_threads.push(thread::spawn(move || {
13886 handle_mock_worker_request(&mut stream, &requests, behavior)
13887 }));
13888 } else {
13889 handle_mock_worker_request(&mut stream, &server_requests, behavior);
13890 }
13891 }
13892 Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
13893 let mut index = 0;
13894 while index < request_threads.len() {
13895 if request_threads[index].is_finished() {
13896 request_threads
13897 .swap_remove(index)
13898 .join()
13899 .expect("join mock request");
13900 } else {
13901 index += 1;
13902 }
13903 }
13904 thread::sleep(Duration::from_millis(5));
13905 }
13906 Err(_) => break,
13907 }
13908 }
13909 for request_thread in request_threads {
13910 request_thread.join().expect("join mock request");
13911 }
13912 });
13913
13914 Self {
13915 addr,
13916 stop,
13917 requests,
13918 thread: Some(thread),
13919 }
13920 }
13921
13922 fn base_url(&self) -> String {
13923 format!("http://{}", self.addr)
13924 }
13925
13926 fn worker_protocol_for(&self, path: &str) -> Option<String> {
13927 self.requests
13928 .lock()
13929 .expect("captured requests")
13930 .iter()
13931 .find(|request| request.path == path)
13932 .and_then(|request| request.worker_protocol.clone())
13933 }
13934
13935 fn control_protocol_for(&self, path: &str) -> Option<String> {
13936 self.requests
13937 .lock()
13938 .expect("captured requests")
13939 .iter()
13940 .find(|request| request.path == path)
13941 .and_then(|request| request.control_protocol.clone())
13942 }
13943
13944 fn method_for(&self, path: &str) -> Option<String> {
13945 self.requests
13946 .lock()
13947 .expect("captured requests")
13948 .iter()
13949 .find(|request| request.path == path)
13950 .map(|request| request.method.clone())
13951 }
13952
13953 fn authorization_for(&self, path: &str) -> Option<String> {
13954 self.requests
13955 .lock()
13956 .expect("captured requests")
13957 .iter()
13958 .find(|request| request.path == path)
13959 .and_then(|request| request.authorization.clone())
13960 }
13961
13962 fn namespace_for(&self, path: &str) -> Option<String> {
13963 self.requests
13964 .lock()
13965 .expect("captured requests")
13966 .iter()
13967 .find(|request| request.path == path)
13968 .and_then(|request| request.namespace.clone())
13969 }
13970
13971 fn request_count(&self, path: &str) -> usize {
13972 self.requests
13973 .lock()
13974 .expect("captured requests")
13975 .iter()
13976 .filter(|request| request.path == path)
13977 .count()
13978 }
13979
13980 fn captured_paths(&self) -> Vec<String> {
13981 self.requests
13982 .lock()
13983 .expect("captured requests")
13984 .iter()
13985 .map(|request| request.path.clone())
13986 .collect()
13987 }
13988
13989 fn request_times(&self, path: &str) -> Vec<Instant> {
13990 self.requests
13991 .lock()
13992 .expect("captured requests")
13993 .iter()
13994 .filter(|request| request.path == path)
13995 .map(|request| request.received_at)
13996 .collect()
13997 }
13998
13999 fn request_body(&self, path: &str) -> Value {
14000 let requests = self.requests.lock().expect("captured requests");
14001 let body = &requests
14002 .iter()
14003 .find(|request| request.path == path)
14004 .unwrap_or_else(|| panic!("missing request for {path}"))
14005 .body;
14006 serde_json::from_str(body).unwrap_or_else(|error| {
14007 panic!("invalid JSON request body for {path}: {error}: {body:?}")
14008 })
14009 }
14010
14011 fn request_bodies(&self, path: &str) -> Vec<Value> {
14012 self.requests
14013 .lock()
14014 .expect("captured requests")
14015 .iter()
14016 .filter(|request| request.path == path)
14017 .map(|request| {
14018 serde_json::from_str(&request.body).unwrap_or_else(|error| {
14019 panic!(
14020 "invalid JSON request body for {path}: {error}: {:?}",
14021 request.body
14022 )
14023 })
14024 })
14025 .collect()
14026 }
14027 }
14028
14029 impl Drop for MockWorkerServer {
14030 fn drop(&mut self) {
14031 self.stop.store(true, Ordering::SeqCst);
14032 let _ = TcpStream::connect(self.addr);
14033
14034 if let Some(thread) = self.thread.take() {
14035 thread.join().expect("join mock server");
14036 }
14037 }
14038 }
14039
14040 fn handle_mock_worker_request(
14041 stream: &mut TcpStream,
14042 requests: &Arc<Mutex<Vec<CapturedRequest>>>,
14043 behavior: MockWorkerBehavior,
14044 ) {
14045 let _ = stream.set_read_timeout(Some(Duration::from_millis(200)));
14046 let mut buffer = [0_u8; 8192];
14047 let mut request = Vec::new();
14048
14049 loop {
14050 match stream.read(&mut buffer) {
14051 Ok(0) => break,
14052 Ok(read) => {
14053 request.extend_from_slice(&buffer[..read]);
14054 if mock_request_is_complete(&request) {
14055 break;
14056 }
14057 }
14058 Err(error)
14059 if matches!(
14060 error.kind(),
14061 std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
14062 ) =>
14063 {
14064 break;
14065 }
14066 Err(_) => return,
14067 }
14068 }
14069
14070 let request = String::from_utf8_lossy(&request);
14071 let body = request
14072 .split_once("\r\n\r\n")
14073 .map(|(_, body)| body)
14074 .unwrap_or_default();
14075 let path = request
14076 .lines()
14077 .next()
14078 .and_then(|line| line.split_whitespace().nth(1))
14079 .unwrap_or_default();
14080 let method = request
14081 .lines()
14082 .next()
14083 .and_then(|line| line.split_whitespace().next())
14084 .unwrap_or_default();
14085 let authorization = request.lines().find_map(|line| {
14086 let (name, value) = line.split_once(':')?;
14087 name.eq_ignore_ascii_case("Authorization")
14088 .then(|| value.trim().to_string())
14089 });
14090 let namespace = request.lines().find_map(|line| {
14091 let (name, value) = line.split_once(':')?;
14092 name.eq_ignore_ascii_case("X-Namespace")
14093 .then(|| value.trim().to_string())
14094 });
14095 let worker_protocol = request.lines().find_map(|line| {
14096 let (name, value) = line.split_once(':')?;
14097 name.eq_ignore_ascii_case("X-Durable-Workflow-Protocol-Version")
14098 .then(|| value.trim().to_string())
14099 });
14100 let control_protocol = request.lines().find_map(|line| {
14101 let (name, value) = line.split_once(':')?;
14102 name.eq_ignore_ascii_case("X-Durable-Workflow-Control-Plane-Version")
14103 .then(|| value.trim().to_string())
14104 });
14105 let request_number = {
14106 let mut requests = requests.lock().expect("captured requests");
14107 requests.push(CapturedRequest {
14108 method: method.to_string(),
14109 path: path.to_string(),
14110 authorization,
14111 namespace,
14112 worker_protocol: worker_protocol.clone(),
14113 control_protocol,
14114 body: body.to_string(),
14115 received_at: Instant::now(),
14116 });
14117 requests
14118 .iter()
14119 .filter(|request| request.path == path)
14120 .count()
14121 };
14122
14123 if behavior.reject_registration && path == "/api/worker/register" {
14124 write_mock_response(
14125 stream,
14126 "503 Service Unavailable",
14127 r#"{"reason":"registration_unavailable","message":"registration failed"}"#,
14128 );
14129 return;
14130 }
14131
14132 if path.starts_with("/api/worker/registrations/") {
14133 if behavior.reject_deregistration_protocol {
14134 write_mock_response(
14135 stream,
14136 "400 Bad Request",
14137 r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.1","requested_version":"1.2"}"#,
14138 );
14139 } else if behavior.reject_deregistration {
14140 write_mock_response(
14141 stream,
14142 "403 Forbidden",
14143 r#"{"reason":"authorization_failed","message":"worker cannot deregister"}"#,
14144 );
14145 } else {
14146 write_mock_response(
14147 stream,
14148 "200 OK",
14149 r#"{"worker_id":"deregistered-worker","outcome":"deregistered","recovered_workflow_task_count":2}"#,
14150 );
14151 }
14152 return;
14153 }
14154
14155 let is_poll = matches!(
14156 path,
14157 "/api/worker/workflow-tasks/poll"
14158 | "/api/worker/activity-tasks/poll"
14159 | "/api/worker/query-tasks/poll"
14160 );
14161 if is_poll && request_number <= behavior.poll_failures_per_path {
14162 return;
14163 }
14164 if path == "/api/worker/heartbeat" && request_number <= behavior.heartbeat_failures {
14165 return;
14166 }
14167 if path == "/api/worker/heartbeat"
14168 && behavior.heartbeat_failure_request == Some(request_number)
14169 {
14170 return;
14171 }
14172 if path == "/api/worker/heartbeat"
14173 && behavior.delayed_heartbeat_request == Some(request_number)
14174 {
14175 thread::sleep(behavior.heartbeat_response_delay);
14176 }
14177 if behavior.unauthorized_polls && is_poll {
14178 write_mock_response(
14179 stream,
14180 "401 Unauthorized",
14181 r#"{"reason":"authentication_failed","message":"invalid worker token"}"#,
14182 );
14183 return;
14184 }
14185 if behavior.draining_polls && is_poll {
14186 write_mock_response(
14187 stream,
14188 "409 Conflict",
14189 r#"{"task":null,"poll_status":"draining","reason":"worker_draining","worker_status":"draining","drain_intent":"draining"}"#,
14190 );
14191 return;
14192 }
14193
14194 if behavior.reject_query_protocol && path.starts_with("/api/worker/query-tasks/") {
14195 let requested_version = worker_protocol.as_deref().unwrap_or("missing");
14196 let body = format!(
14197 r#"{{"reason":"unsupported_protocol_version","message":"Query tasks require worker protocol 1.8 or newer.","supported_version":"1.7","requested_version":"{requested_version}"}}"#
14198 );
14199 write_mock_response(stream, "400 Bad Request", &body);
14200 return;
14201 }
14202
14203 if behavior.reject_query_completion && path == "/api/worker/query-tasks/query-late/complete"
14204 {
14205 write_mock_response(
14206 stream,
14207 "409 Conflict",
14208 r#"{"reason":"query_task_timed_out","message":"query task timed out before completion"}"#,
14209 );
14210 return;
14211 }
14212
14213 if behavior.workflow_completion_status.is_some()
14214 && path == "/api/worker/workflow-tasks/poll"
14215 && request_number == 1
14216 {
14217 write_mock_response(
14218 stream,
14219 "200 OK",
14220 r#"{"task":{"task_id":"workflow-timeout-task","workflow_id":"reused-workflow-id","run_id":"run-selected-timeout","workflow_type":"timeout.workflow","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"workflow_task_attempt":3,"lease_owner":"timeout-worker"}}"#,
14221 );
14222 return;
14223 }
14224
14225 if path == "/api/worker/workflow-tasks/workflow-timeout-task/complete" {
14226 if let (Some(status), Some(body)) = (
14227 behavior.workflow_completion_status,
14228 behavior.workflow_completion_body,
14229 ) {
14230 write_mock_response(stream, status, body);
14231 return;
14232 }
14233 }
14234
14235 if behavior.waiting_query_worker {
14236 if behavior.complete_named_signal
14237 && path == "/api/worker/workflow-tasks/poll"
14238 && request_number == 1
14239 {
14240 let body = json!({
14241 "task": {
14242 "task_id": "snapshot-open",
14243 "workflow_id": "snapshot-1",
14244 "run_id": "snapshot-run-1",
14245 "workflow_type": "snapshot",
14246 "payload_codec": DEFAULT_CODEC,
14247 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
14248 .expect("Avro workflow arguments"),
14249 "history_events": [],
14250 "workflow_task_attempt": 1,
14251 "lease_owner": "rust-snapshot-worker"
14252 }
14253 })
14254 .to_string();
14255 write_mock_response(stream, "200 OK", &body);
14256 return;
14257 }
14258
14259 let signal_request = request_number - usize::from(behavior.complete_named_signal);
14260 let signal_request_limit = 2 + usize::from(behavior.complete_named_signal);
14261 if path == "/api/worker/workflow-tasks/poll"
14262 && signal_request >= 1
14263 && signal_request <= signal_request_limit
14264 {
14265 let finish = behavior.complete_named_signal && signal_request == 3;
14266 let amounts = if signal_request == 1 {
14267 vec![3]
14268 } else {
14269 vec![3, 5]
14270 };
14271 let task_id = if signal_request == 1 {
14272 "snapshot-wait-3"
14273 } else if finish {
14274 "snapshot-finish"
14275 } else {
14276 "snapshot-wait-5"
14277 };
14278 let mut history_events = std::iter::once(json!({
14279 "event_type": "SignalWaitOpened",
14280 "payload": {"sequence": 1, "signal_name": "finish"}
14281 }))
14282 .chain(amounts.iter().enumerate().map(|(index, amount)| {
14283 json!({
14284 "event_type": "SignalReceived",
14285 "payload": {
14286 "signal_id": format!("increment-{amount}"),
14287 "signal_name": "increment",
14288 "workflow_sequence": index + 2,
14289 "payload_codec": DEFAULT_CODEC,
14290 "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
14291 .expect("Avro signal envelope")
14292 }
14293 })
14294 }))
14295 .collect::<Vec<_>>();
14296 let (resume_id, resume_name, resume_arguments) = if finish {
14297 history_events.push(json!({
14298 "event_type": "SignalReceived",
14299 "payload": {
14300 "signal_id": "finish",
14301 "signal_name": "finish",
14302 "workflow_sequence": 4,
14303 "payload_codec": DEFAULT_CODEC,
14304 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
14305 .expect("Avro finish signal envelope")
14306 }
14307 }));
14308 (
14309 "finish".to_string(),
14310 "finish".to_string(),
14311 encode_value_envelope(&json!([]), DEFAULT_CODEC)
14312 .expect("Avro finish resume signal"),
14313 )
14314 } else {
14315 let amount = amounts.last().expect("amount");
14316 (
14317 format!("increment-{amount}"),
14318 "increment".to_string(),
14319 encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
14320 .expect("Avro increment resume signal"),
14321 )
14322 };
14323 let body = json!({
14324 "task": {
14325 "task_id": task_id,
14326 "workflow_id": "snapshot-1",
14327 "run_id": "snapshot-run-1",
14328 "workflow_type": "snapshot",
14329 "payload_codec": DEFAULT_CODEC,
14330 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
14331 .expect("Avro workflow arguments"),
14332 "history_events": history_events,
14333 "workflow_task_attempt": 1,
14334 "workflow_signal_id": resume_id,
14335 "signal_name": resume_name,
14336 "signal_arguments": resume_arguments,
14337 "lease_owner": "rust-snapshot-worker"
14338 }
14339 })
14340 .to_string();
14341 write_mock_response(stream, "200 OK", &body);
14342 return;
14343 }
14344
14345 if path == "/api/worker/query-tasks/poll" && request_number == 1 {
14346 let history_events = [3, 5]
14347 .into_iter()
14348 .enumerate()
14349 .map(|(index, amount)| {
14350 json!({
14351 "event_type": "SignalReceived",
14352 "payload": {
14353 "signal_id": format!("increment-{amount}"),
14354 "signal_name": "increment",
14355 "workflow_sequence": index + 2,
14356 "payload_codec": DEFAULT_CODEC,
14357 "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
14358 .expect("Avro query signal envelope")
14359 }
14360 })
14361 })
14362 .collect::<Vec<_>>();
14363 let body = json!({
14364 "task": {
14365 "query_task_id": "snapshot-current",
14366 "query_task_attempt": 1,
14367 "lease_owner": "rust-snapshot-worker",
14368 "workflow_id": "snapshot-1",
14369 "run_id": "snapshot-run-1",
14370 "workflow_type": "snapshot",
14371 "query_name": "current",
14372 "payload_codec": DEFAULT_CODEC,
14373 "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
14374 .expect("Avro workflow arguments"),
14375 "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
14376 .expect("Avro query arguments"),
14377 "history_events": history_events,
14378 "run_status": "waiting"
14379 }
14380 })
14381 .to_string();
14382 write_mock_response(stream, "200 OK", &body);
14383 return;
14384 }
14385
14386 if path == "/api/worker/workflow-tasks/snapshot-wait-3/fail"
14387 || path == "/api/worker/workflow-tasks/snapshot-wait-5/fail"
14388 {
14389 write_mock_response(
14390 stream,
14391 "200 OK",
14392 r#"{"outcome":"waiting_for_history","recorded":true}"#,
14393 );
14394 return;
14395 }
14396
14397 if path == "/api/worker/workflow-tasks/snapshot-open/complete" {
14398 write_mock_response(stream, "200 OK", r#"{"outcome":"waiting","recorded":true}"#);
14399 return;
14400 }
14401
14402 if path == "/api/worker/workflow-tasks/snapshot-finish/complete" {
14403 write_mock_response(
14404 stream,
14405 "200 OK",
14406 r#"{"outcome":"completed","run_status":"completed","recorded":true}"#,
14407 );
14408 return;
14409 }
14410
14411 if path == "/api/worker/query-tasks/snapshot-current/complete" {
14412 write_mock_response(stream, "200 OK", r#"{"outcome":"completed"}"#);
14413 return;
14414 }
14415 }
14416
14417 if matches!(
14418 path,
14419 "/api/workflows/typed-1/query/inspect" | "/api/workflows/typed-1/update/replace"
14420 ) {
14421 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
14422 .expect("typed mock result");
14423 let body = json!({
14424 "result": typed_fidelity_probe().into_json().expect("result projection"),
14425 "result_envelope": result,
14426 })
14427 .to_string();
14428 write_mock_response(stream, "200 OK", &body);
14429 return;
14430 }
14431
14432 if path == "/api/workflows/typed-1" {
14433 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
14434 .expect("typed mock result");
14435 let body = json!({
14436 "workflow_id": "typed-1",
14437 "run_id": "run-typed-1",
14438 "workflow_type": "typed.echo",
14439 "status": "completed",
14440 "output": typed_fidelity_probe().into_json().expect("output projection"),
14441 "output_envelope": result,
14442 })
14443 .to_string();
14444 write_mock_response(stream, "200 OK", &body);
14445 return;
14446 }
14447
14448 let (status, body) = match path {
14449 "/api/health" => ("200 OK", r#"{"status":"ok"}"#),
14450 "/api/workflows" => (
14451 "201 Created",
14452 r#"{"workflow_id":"wf-start-options","run_id":"run-start-options","workflow_type":"rust.timeout"}"#,
14453 ),
14454 "/api/worker/register" if behavior.decline_registration => (
14455 "200 OK",
14456 r#"{"worker_id":"declined-worker","registered":false}"#,
14457 ),
14458 "/api/worker/register" if behavior.waiting_query_worker => (
14459 "200 OK",
14460 r#"{"worker_id":"rust-snapshot-worker","registered":true,"heartbeat_interval_seconds":1}"#,
14461 ),
14462 "/api/worker/register" => (
14463 "200 OK",
14464 r#"{"worker_id":"mock-worker","registered":true,"heartbeat_interval_seconds":3600}"#,
14465 ),
14466 "/api/worker/heartbeat" => ("200 OK", "{}"),
14467 "/api/worker/activity-tasks/poll"
14468 if behavior.cancelled_activity && request_number == 1 =>
14469 {
14470 (
14471 "200 OK",
14472 r#"{"task":{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","activity_type":"cancel-aware","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"attempt_number":1,"lease_owner":"rust-cancel-worker"}}"#,
14473 )
14474 }
14475 "/api/worker/activity-tasks/poll" | "/api/worker/workflow-tasks/poll" => {
14476 ("200 OK", r#"{"task":null}"#)
14477 }
14478 "/api/worker/query-tasks/poll"
14479 if behavior.reject_query_completion && request_number == 1 =>
14480 {
14481 (
14482 "200 OK",
14483 r#"{"task":{"query_task_id":"query-late","query_task_attempt":1,"lease_owner":"late-worker","workflow_id":"counter-late","run_id":"run-late","workflow_type":"counter","query_name":"current","payload_codec":"avro","workflow_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"query_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"run_status":"running"}}"#,
14484 )
14485 }
14486 "/api/worker/query-tasks/poll" => ("200 OK", r#"{"task":null}"#),
14487 "/api/worker/query-tasks/query-capture/complete"
14488 | "/api/worker/query-tasks/query-capture/fail" => ("200 OK", "{}"),
14489 "/api/worker/activity-tasks/activity-cancel/heartbeat" => (
14490 "200 OK",
14491 r#"{"activity_attempt_id":"attempt-cancel","cancel_requested":true,"can_continue":false,"reason":"run_cancelled","run_closed_reason":"cancelled","heartbeat_recorded":false}"#,
14492 ),
14493 "/api/worker/activity-tasks/activity-cancel/complete" => (
14494 "409 Conflict",
14495 r#"{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","reason":"run_cancelled","cancel_requested":true,"can_continue":false,"run_closed_reason":"cancelled"}"#,
14496 ),
14497 "/api/worker/activity-tasks/activity-typed/complete"
14498 | "/api/worker/activity-tasks/activity-typed/fail"
14499 | "/api/workflows/typed-1/signal/changed" => ("200 OK", "{}"),
14500 "/api/workflows/counter-1/query/current" => (
14501 "200 OK",
14502 r#"{"workflow_id":"counter-1","query_name":"current","result":{"count":8},"result_envelope":{"codec":"avro","blob":"wwHioz3/VYAiNw4CCmNvdW50BBAA"}}"#,
14503 ),
14504 "/api/workflows/counter-1/query/missing" => (
14505 "404 Not Found",
14506 r#"{"workflow_id":"counter-1","query_name":"missing","reason":"rejected_unknown_query","message":"unknown query"}"#,
14507 ),
14508 "/api/workflows/wf-lifecycle/cancel" => (
14509 "200 OK",
14510 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","reason":"cleanup requested","command_status":"accepted"}"#,
14511 ),
14512 "/api/workflows/wf-lifecycle/terminate" => (
14513 "200 OK",
14514 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","reason":"forced stop","command_status":"accepted"}"#,
14515 ),
14516 "/api/workflows/wf-lifecycle/runs/run-current/cancel" => (
14517 "200 OK",
14518 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","command_status":"accepted"}"#,
14519 ),
14520 "/api/workflows/wf-lifecycle/runs/run-current/terminate" => (
14521 "200 OK",
14522 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","command_status":"accepted"}"#,
14523 ),
14524 "/api/workflows/wf-lifecycle/runs/run-stale/cancel"
14525 | "/api/workflows/wf-lifecycle/runs/run-stale/terminate" => (
14526 "409 Conflict",
14527 r#"{"workflow_id":"wf-lifecycle","run_id":"run-stale","reason":"historical_run_command_rejected","target_scope":"run","message":"Commands cannot target historical runs."}"#,
14528 ),
14529 "/api/workflows/wf-failed" | "/api/workflows/wf-failed/runs/run-failed" => (
14530 "200 OK",
14531 r#"{"workflow_id":"wf-failed","run_id":"run-failed","status":"failed","closed_reason":"failed","error":"payment failed","failure":{"message":"payment failed","failure_category":"application","exception_type":"PaymentError","exception_class":"billing::PaymentError","non_retryable":true,"exception":{"type":"PaymentError","class":"billing::PaymentError","message":"payment failed"},"failures":[{"id":"failure-17","failure_category":"application"}]}}"#,
14532 ),
14533 "/api/workflows/wf-cancelled" => (
14534 "200 OK",
14535 r#"{"workflow_id":"wf-cancelled","run_id":"run-cancelled","status":"cancelled","closed_reason":"cancelled","reason":"cleanup requested"}"#,
14536 ),
14537 "/api/workflows/wf-terminated" => (
14538 "200 OK",
14539 r#"{"workflow_id":"wf-terminated","run_id":"run-terminated","status":"terminated","closed_reason":"terminated","reason":"forced stop"}"#,
14540 ),
14541 "/api/workflows/wf-timed-out" => (
14542 "200 OK",
14543 r#"{"workflow_id":"wf-timed-out","run_id":"run-timed-out","status":"failed","closed_reason":"timed_out","reason":"run_timeout"}"#,
14544 ),
14545 "/api/workflows/wf-waiting" | "/api/workflows/wf-waiting/runs/run-waiting" => (
14546 "200 OK",
14547 r#"{"workflow_id":"wf-waiting","run_id":"run-waiting","status":"waiting"}"#,
14548 ),
14549 "/api/workflows/wf-selected" => (
14550 "200 OK",
14551 r#"{"workflow_id":"wf-selected","run_id":"run-current","status":"completed","output":"current run output"}"#,
14552 ),
14553 "/api/workflows/wf-selected/runs/run-selected" => (
14554 "200 OK",
14555 r#"{"workflow_id":"wf-selected","run_id":"run-selected","status":"cancelled","closed_reason":"cancelled","reason":"selected run cancelled"}"#,
14556 ),
14557 _ => ("404 Not Found", r#"{"message":"not found"}"#),
14558 };
14559 write_mock_response(stream, status, body);
14560 }
14561
14562 fn mock_request_is_complete(request: &[u8]) -> bool {
14563 let Some(header_end) = request
14564 .windows(4)
14565 .position(|window| window == b"\r\n\r\n")
14566 .map(|position| position + 4)
14567 else {
14568 return false;
14569 };
14570 let headers = String::from_utf8_lossy(&request[..header_end]);
14571 let content_length = headers.lines().find_map(|line| {
14572 let (name, value) = line.split_once(':')?;
14573 name.eq_ignore_ascii_case("content-length")
14574 .then(|| value.trim().parse::<usize>().ok())
14575 .flatten()
14576 });
14577
14578 request.len() >= header_end + content_length.unwrap_or(0)
14579 }
14580
14581 fn write_mock_response(stream: &mut TcpStream, status: &str, body: &str) {
14582 let response = format!(
14583 "HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
14584 body.len()
14585 );
14586
14587 let _ = stream.write_all(response.as_bytes());
14588 let _ = stream.flush();
14589 }
14590}