1#![doc = include_str!("../README.md")]
2
3use std::{
4 any::{Any, TypeId},
5 collections::{BTreeMap, HashMap},
6 future::Future,
7 io::{self, Read},
8 pin::Pin,
9 sync::{
10 atomic::{AtomicBool, Ordering},
11 Arc, Mutex, OnceLock,
12 },
13 task::{Context as TaskContext, Poll},
14 time::{Duration, Instant, SystemTime, UNIX_EPOCH},
15};
16
17use apache_avro::{from_avro_datum, to_avro_datum, types::Value as AvroDatum, Schema};
18use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
19use futures_util::{future::OptionFuture, task::noop_waker_ref};
20use serde::{
21 de::DeserializeOwned,
22 ser::{SerializeMap, SerializeSeq},
23 Deserialize, Serialize, Serializer,
24};
25pub use serde_json::{json, Value};
26use thiserror::Error;
27pub use uuid::Uuid;
28
29pub const WORKER_PROTOCOL_VERSION: &str = "1.2";
30pub const CONTROL_PLANE_VERSION: &str = "2";
31pub const DEFAULT_CODEC: &str = "avro";
32pub const SDK_VERSION: &str = concat!("durable-workflow-rust/", env!("CARGO_PKG_VERSION"));
33pub const QUERY_TASKS_CAPABILITY: &str = "query_tasks";
35pub const WORKFLOW_UPDATES_CAPABILITY: &str = "workflow_updates";
37pub const QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
39
40const MAX_LONG_POLL_TIMEOUT_SECONDS: u64 = 60;
41const WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE: &str =
42 "Workflow task waiting for scheduled history.";
43const WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE: &str = "WorkflowTaskWaitingForHistory";
44
45const QUERY_TASK_FINAL_REJECTION_REASONS: &[&str] = &[
46 "lease_expired",
47 "query_task_not_found",
48 "query_task_not_leased",
49 "query_task_timed_out",
50];
51
52pub const AVRO_VALUE_SCHEMA_JSON: &str =
54 include_str!("../schema/durable_workflow.protocol.Value.v1.avsc");
55pub const AVRO_VALUE_SCHEMA_FINGERPRINT_HEX: &str = "e2a33dff55802237";
56pub const AVRO_VALUE_SCHEMA_FINGERPRINT: [u8; 8] = [0xe2, 0xa3, 0x3d, 0xff, 0x55, 0x80, 0x22, 0x37];
57const AVRO_SINGLE_OBJECT_MAGIC: [u8; 2] = [0xc3, 0x01];
58
59static AVRO_VALUE_SCHEMA: OnceLock<std::result::Result<Schema, String>> = OnceLock::new();
60
61#[derive(Clone, Copy)]
62enum RequestProtocol {
63 ControlPlane,
64 Worker(&'static str),
65}
66
67pub type Result<T> = std::result::Result<T, Error>;
68
69#[derive(Debug, Error)]
70pub enum Error {
71 #[error("transport error: {0}")]
72 Transport(#[from] reqwest::Error),
73 #[error(
74 "invalid Durable Workflow base URL: omit the SDK-owned /api suffix and pass the Server or Cloud runtime base URL; the SDK appends /api automatically"
75 )]
76 InvalidBaseUrl,
77 #[error("json error: {0}")]
78 Json(#[from] serde_json::Error),
79 #[error("http {status}: {body}")]
80 Http {
81 status: reqwest::StatusCode,
82 body: String,
83 },
84 #[error("codec error: {0}")]
85 Codec(String),
86 #[error(transparent)]
87 QueryFailed(QueryFailure),
88 #[error(transparent)]
89 Protocol(ProtocolFailure),
90 #[error(transparent)]
91 NonDeterministicReplay(ReplayFailure),
92 #[error(transparent)]
93 ChildWorkflowFailed(ChildWorkflowFailure),
94 #[error(transparent)]
95 ActivityFailed(ActivityFailure),
96 #[error(transparent)]
97 WorkflowCommandRejected(WorkflowCommandRejection),
98 #[error(transparent)]
99 WorkflowFailed(WorkflowTerminalOutcome),
100 #[error(transparent)]
101 WorkflowCancelled(WorkflowTerminalOutcome),
102 #[error(transparent)]
103 WorkflowTerminated(WorkflowTerminalOutcome),
104 #[error(transparent)]
105 WorkflowTimedOut(WorkflowTerminalOutcome),
106 #[error(transparent)]
107 ActivityTaskRejected(ActivityTaskRejection),
108 #[error("workflow handler {0:?} is not registered")]
109 WorkflowNotRegistered(String),
110 #[error("activity handler {0:?} is not registered")]
111 ActivityNotRegistered(String),
112 #[error("workflow future yielded without emitting a durable command")]
113 WorkflowYieldedWithoutCommand,
114 #[error("workflow state lock is poisoned")]
115 WorkflowStatePoisoned,
116 #[error("timer duration is too large for the worker protocol")]
117 TimerDurationOverflow,
118 #[error("operation timed out")]
119 Timeout,
120 #[error(
121 "missing {role}-plane credentials: configure ClientBuilder::{role}_token or ClientBuilder::token; a {opposite_role}-plane token cannot authorize this request"
122 )]
123 MissingRoleCredentials {
124 role: &'static str,
125 opposite_role: &'static str,
126 },
127 #[error("worker loop error: {0}")]
128 WorkerLoop(String),
129 #[error(
130 "workflow command contract for {workflow_type:?} declares update validators, but this Rust SDK cannot execute synchronous pre-accept update validation"
131 )]
132 UnsupportedUpdateValidators { workflow_type: String },
133 #[error("{primary}; worker deregistration also failed: {deregistration}")]
134 WorkerShutdown {
135 primary: Box<Error>,
136 deregistration: Box<Error>,
137 },
138 #[error("invalid child workflow options: {0}")]
139 InvalidChildWorkflowOptions(String),
140 #[error(transparent)]
141 InvalidActivityOptions(ActivityOptionsError),
142 #[error(transparent)]
143 InvalidContinueAsNewOptions(#[from] ContinueAsNewOptionsError),
144 #[doc(hidden)]
145 #[error("workflow requested continue as new")]
146 ContinueAsNew(ContinueAsNewRequest),
147}
148
149#[derive(Clone, Copy, Debug, PartialEq, Eq)]
151pub enum WorkflowCommandKind {
152 Cancel,
153 Terminate,
154}
155
156impl WorkflowCommandKind {
157 fn as_str(self) -> &'static str {
158 match self {
159 Self::Cancel => "cancel",
160 Self::Terminate => "terminate",
161 }
162 }
163}
164
165#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
167pub struct WorkflowCommandOptions {
168 #[serde(skip_serializing_if = "Option::is_none")]
169 pub reason: Option<String>,
170 #[serde(skip_serializing_if = "Option::is_none")]
171 pub request_id: Option<String>,
172}
173
174#[derive(Clone, Debug, PartialEq, Eq)]
181pub struct WorkflowStartOptions {
182 pub execution_timeout_seconds: u64,
183 pub run_timeout_seconds: u64,
184}
185
186impl Default for WorkflowStartOptions {
187 fn default() -> Self {
188 Self {
189 execution_timeout_seconds: 3600,
190 run_timeout_seconds: 600,
191 }
192 }
193}
194
195impl WorkflowStartOptions {
196 pub fn new() -> Self {
197 Self::default()
198 }
199
200 pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
201 self.execution_timeout_seconds = seconds;
202 self
203 }
204
205 pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
206 self.run_timeout_seconds = seconds;
207 self
208 }
209
210 fn validate(&self) -> Result<()> {
211 if self.execution_timeout_seconds == 0 {
212 return Err(Error::Codec(
213 "execution_timeout_seconds must be at least 1".to_string(),
214 ));
215 }
216 if self.run_timeout_seconds == 0 {
217 return Err(Error::Codec(
218 "run_timeout_seconds must be at least 1".to_string(),
219 ));
220 }
221 if self.run_timeout_seconds > self.execution_timeout_seconds {
222 return Err(Error::Codec(
223 "run_timeout_seconds cannot exceed execution_timeout_seconds".to_string(),
224 ));
225 }
226
227 Ok(())
228 }
229}
230
231#[derive(Clone, Debug, Default, PartialEq, Eq)]
236pub struct ContinueAsNewOptions {
237 pub workflow_type: Option<String>,
238 pub task_queue: Option<String>,
239}
240
241impl ContinueAsNewOptions {
242 pub fn new() -> Self {
243 Self::default()
244 }
245
246 pub fn workflow_type(mut self, workflow_type: impl Into<String>) -> Self {
247 self.workflow_type = Some(workflow_type.into());
248 self
249 }
250
251 pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
252 self.task_queue = Some(task_queue.into());
253 self
254 }
255
256 fn validate(&self) -> std::result::Result<(), ContinueAsNewOptionsError> {
257 for (field, value) in [
258 ("workflow_type", self.workflow_type.as_deref()),
259 ("task_queue", self.task_queue.as_deref()),
260 ] {
261 if value.is_some_and(|value| value.trim().is_empty()) {
262 return Err(ContinueAsNewOptionsError {
263 field,
264 message: format!("{field} must not be empty"),
265 });
266 }
267 }
268 Ok(())
269 }
270}
271
272#[derive(Clone, Debug, Error, PartialEq, Eq)]
274#[error("invalid continue-as-new option {field}: {message}")]
275pub struct ContinueAsNewOptionsError {
276 pub field: &'static str,
277 pub message: String,
278}
279
280#[derive(Clone, Debug, Default, PartialEq, Eq)]
282pub struct WorkflowHistoryBudget {
283 pub event_count: u64,
284 pub size_bytes: Option<u64>,
285 pub continue_as_new_recommended: bool,
286 pub pressure: Option<String>,
287}
288
289#[doc(hidden)]
290#[derive(Clone, Debug)]
291pub struct ContinueAsNewRequest {
292 arguments: AvroValue,
293 options: ContinueAsNewOptions,
294}
295
296impl WorkflowCommandOptions {
297 pub fn new() -> Self {
298 Self::default()
299 }
300
301 pub fn reason(mut self, reason: impl Into<String>) -> Self {
302 self.reason = Some(reason.into());
303 self
304 }
305
306 pub fn request_id(mut self, request_id: impl Into<String>) -> Self {
307 self.request_id = Some(request_id.into());
308 self
309 }
310}
311
312#[derive(Clone, Debug, PartialEq)]
314pub struct WorkflowCommandResult {
315 pub command: WorkflowCommandKind,
316 pub workflow_id: String,
317 pub run_id: Option<String>,
318 pub outcome: Option<String>,
319 pub reason: Option<String>,
320 pub command_status: Option<String>,
321 pub raw: Value,
322}
323
324#[derive(Clone, Debug, Error)]
326#[error("workflow {command:?} rejected ({reason}, HTTP {status}): {message}")]
327pub struct WorkflowCommandRejection {
328 pub command: WorkflowCommandKind,
329 pub status: u16,
330 pub reason: String,
331 pub message: String,
332 pub workflow_id: String,
333 pub run_id: Option<String>,
334 pub target_scope: Option<String>,
335 pub body: Value,
336}
337
338#[derive(Clone, Copy, Debug, PartialEq, Eq)]
340pub enum WorkflowTerminalKind {
341 Failed,
342 Cancelled,
343 Terminated,
344 TimedOut,
345}
346
347#[derive(Clone, Debug, Error)]
353#[error("workflow {workflow_id} run {run_id:?} ended as {kind:?} ({reason})")]
354pub struct WorkflowTerminalOutcome {
355 pub kind: WorkflowTerminalKind,
356 pub workflow_id: String,
357 pub run_id: Option<String>,
358 pub reason: String,
359 pub failure_category: Option<String>,
360 pub failure_id: Option<String>,
361 pub exception_type: Option<String>,
362 pub exception_class: Option<String>,
363 pub non_retryable: Option<bool>,
364 pub message: Option<String>,
365 pub exception: Option<Value>,
366 pub raw: Value,
367}
368
369#[derive(Clone, Debug, Error)]
371#[error("activity task {operation} rejected ({reason}, HTTP {status})")]
372pub struct ActivityTaskRejection {
373 pub operation: String,
374 pub status: u16,
375 pub reason: String,
376 pub task_id: String,
377 pub activity_attempt_id: String,
378 pub cancel_requested: bool,
379 pub can_continue: Option<bool>,
380 pub run_closed_reason: Option<String>,
381 pub body: Value,
382}
383
384#[derive(Clone, Copy, Debug, PartialEq, Eq)]
386pub enum ActivityOptionsErrorKind {
387 EmptyTaskQueue,
388 EmptyRetryPolicy,
389 InvalidMaxAttempts,
390 BackoffWithoutRetryBudget,
391 TooManyBackoffIntervals,
392 InvalidBackoffCoefficient,
393 BackoffGenerationTooLarge,
394 BackoffOverflow,
395 EmptyNonRetryableErrorType,
396 TimeoutNotPositive,
397 TimeoutOverflow,
398 TimeoutOrder,
399}
400
401#[derive(Clone, Debug, Error, PartialEq, Eq)]
403#[error("invalid activity options ({kind:?}, {field:?}): {message}")]
404pub struct ActivityOptionsError {
405 pub kind: ActivityOptionsErrorKind,
406 pub field: Option<&'static str>,
407 pub message: String,
408}
409
410impl ActivityOptionsError {
411 fn new(
412 kind: ActivityOptionsErrorKind,
413 field: Option<&'static str>,
414 message: impl Into<String>,
415 ) -> Self {
416 Self {
417 kind,
418 field,
419 message: message.into(),
420 }
421 }
422}
423
424#[derive(Clone, Copy, Debug, PartialEq, Eq)]
426pub enum ActivityFailureKind {
427 Failed,
428 Cancelled,
429 TimedOut,
430}
431
432#[derive(Clone, Debug, Error)]
437#[error("activity failed ({reason}): {message}")]
438pub struct ActivityFailure {
439 pub kind: ActivityFailureKind,
440 pub reason: String,
441 pub message: String,
442 pub activity_execution_id: Option<String>,
443 pub activity_attempt_id: Option<String>,
444 pub activity_type: Option<String>,
445 pub activity_class: Option<String>,
446 pub attempt_number: Option<u64>,
447 pub failure_id: Option<String>,
448 pub failure_category: Option<String>,
449 pub timeout_kind: Option<String>,
450 pub non_retryable: bool,
451 pub exception_type: Option<String>,
452 pub exception_class: Option<String>,
453 pub code: Option<Value>,
454 pub exception: Option<Value>,
455}
456
457#[derive(Clone, Copy, Debug, PartialEq, Eq)]
459pub enum ChildWorkflowFailureKind {
460 Failed,
461 Cancelled,
462 Terminated,
463}
464
465#[derive(Clone, Debug, Error)]
471#[error("child workflow failed ({reason}): {message}")]
472pub struct ChildWorkflowFailure {
473 pub kind: ChildWorkflowFailureKind,
474 pub reason: String,
475 pub message: String,
476 pub parent_workflow_id: Option<String>,
477 pub parent_workflow_run_id: Option<String>,
478 pub child_workflow_id: Option<String>,
479 pub child_workflow_run_id: Option<String>,
480 pub child_workflow_type: Option<String>,
481 pub failure_id: Option<String>,
482 pub failure_category: Option<String>,
483 pub exception_type: Option<String>,
484 pub exception_class: Option<String>,
485 pub non_retryable: bool,
486 pub code: Option<Value>,
487 pub exception: Option<Value>,
488}
489
490#[derive(Clone, Debug, PartialEq, Eq)]
492pub struct WorkflowIdentity {
493 pub workflow_id: Option<String>,
494 pub run_id: Option<String>,
495}
496
497#[derive(Clone, Debug, PartialEq)]
499pub struct ChildWorkflowResult {
500 pub parent: WorkflowIdentity,
501 pub child: WorkflowIdentity,
502 pub child_workflow_type: Option<String>,
503 pub result: Value,
504}
505
506#[derive(Clone, Debug, PartialEq)]
508pub struct ChildWorkflowAvroResult {
509 pub parent: WorkflowIdentity,
510 pub child: WorkflowIdentity,
511 pub child_workflow_type: Option<String>,
512 pub result: AvroValue,
513}
514
515#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
517pub enum ParentClosePolicy {
518 #[default]
519 Abandon,
520 RequestCancel,
521 Terminate,
522}
523
524impl ParentClosePolicy {
525 fn as_str(self) -> &'static str {
526 match self {
527 Self::Abandon => "abandon",
528 Self::RequestCancel => "request_cancel",
529 Self::Terminate => "terminate",
530 }
531 }
532}
533
534#[derive(Clone, Debug, Default, PartialEq, Eq)]
536pub struct ChildWorkflowRetryPolicy {
537 pub max_attempts: Option<u32>,
538 pub backoff_seconds: Vec<u64>,
539 pub non_retryable_error_types: Vec<String>,
540}
541
542#[derive(Clone, Debug, PartialEq, Eq)]
546pub struct ChildWorkflowOptions {
547 pub task_queue: String,
548 pub parent_close_policy: ParentClosePolicy,
549 pub retry_policy: Option<ChildWorkflowRetryPolicy>,
550 pub execution_timeout_seconds: Option<u64>,
551 pub run_timeout_seconds: Option<u64>,
552}
553
554impl ChildWorkflowOptions {
555 pub fn new(task_queue: impl Into<String>) -> Self {
556 Self {
557 task_queue: task_queue.into(),
558 parent_close_policy: ParentClosePolicy::Abandon,
559 retry_policy: None,
560 execution_timeout_seconds: None,
561 run_timeout_seconds: None,
562 }
563 }
564
565 pub fn parent_close_policy(mut self, policy: ParentClosePolicy) -> Self {
566 self.parent_close_policy = policy;
567 self
568 }
569
570 pub fn retry_policy(mut self, policy: ChildWorkflowRetryPolicy) -> Self {
571 self.retry_policy = Some(policy);
572 self
573 }
574
575 pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
576 self.execution_timeout_seconds = Some(seconds);
577 self
578 }
579
580 pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
581 self.run_timeout_seconds = Some(seconds);
582 self
583 }
584}
585
586#[derive(Clone, Debug, PartialEq, Eq)]
588pub enum ActivityBackoff {
589 Explicit(Vec<Duration>),
592 Exponential {
594 initial_interval: Duration,
595 coefficient: u32,
596 maximum_interval: Option<Duration>,
597 },
598}
599
600#[derive(Clone, Debug, Default, PartialEq, Eq)]
602pub struct ActivityRetryPolicy {
603 pub max_attempts: Option<u32>,
604 pub backoff: Option<ActivityBackoff>,
605 pub non_retryable_error_types: Vec<String>,
606}
607
608impl ActivityRetryPolicy {
609 pub fn new(max_attempts: u32) -> Self {
611 Self {
612 max_attempts: Some(max_attempts),
613 ..Self::default()
614 }
615 }
616
617 pub fn backoff_intervals(mut self, intervals: impl IntoIterator<Item = Duration>) -> Self {
618 self.backoff = Some(ActivityBackoff::Explicit(intervals.into_iter().collect()));
619 self
620 }
621
622 pub fn exponential_backoff(
623 mut self,
624 initial_interval: Duration,
625 coefficient: u32,
626 maximum_interval: Option<Duration>,
627 ) -> Self {
628 self.backoff = Some(ActivityBackoff::Exponential {
629 initial_interval,
630 coefficient,
631 maximum_interval,
632 });
633 self
634 }
635
636 pub fn non_retryable_error_type(mut self, error_type: impl Into<String>) -> Self {
637 self.non_retryable_error_types.push(error_type.into());
638 self
639 }
640
641 pub fn non_retryable_error_types(
642 mut self,
643 error_types: impl IntoIterator<Item = impl Into<String>>,
644 ) -> Self {
645 self.non_retryable_error_types
646 .extend(error_types.into_iter().map(Into::into));
647 self
648 }
649}
650
651#[derive(Clone, Debug, Default, PartialEq, Eq)]
656pub struct ActivityOptions {
657 pub task_queue: Option<String>,
658 pub retry_policy: Option<ActivityRetryPolicy>,
659 pub start_to_close_timeout: Option<Duration>,
660 pub schedule_to_start_timeout: Option<Duration>,
661 pub schedule_to_close_timeout: Option<Duration>,
662 pub heartbeat_timeout: Option<Duration>,
663}
664
665impl ActivityOptions {
666 pub fn new() -> Self {
667 Self::default()
668 }
669
670 pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
671 self.task_queue = Some(task_queue.into());
672 self
673 }
674
675 pub fn retry_policy(mut self, policy: ActivityRetryPolicy) -> Self {
676 self.retry_policy = Some(policy);
677 self
678 }
679
680 pub fn start_to_close_timeout(mut self, timeout: Duration) -> Self {
681 self.start_to_close_timeout = Some(timeout);
682 self
683 }
684
685 pub fn schedule_to_start_timeout(mut self, timeout: Duration) -> Self {
686 self.schedule_to_start_timeout = Some(timeout);
687 self
688 }
689
690 pub fn schedule_to_close_timeout(mut self, timeout: Duration) -> Self {
691 self.schedule_to_close_timeout = Some(timeout);
692 self
693 }
694
695 pub fn heartbeat_timeout(mut self, timeout: Duration) -> Self {
696 self.heartbeat_timeout = Some(timeout);
697 self
698 }
699
700 fn validate(&self) -> std::result::Result<ValidatedActivityOptions, ActivityOptionsError> {
701 if self
702 .task_queue
703 .as_deref()
704 .is_some_and(|queue| queue.trim().is_empty())
705 {
706 return Err(ActivityOptionsError::new(
707 ActivityOptionsErrorKind::EmptyTaskQueue,
708 Some("task_queue"),
709 "task_queue must not be empty",
710 ));
711 }
712
713 for (field, value) in [
714 ("start_to_close_timeout", self.start_to_close_timeout),
715 ("schedule_to_start_timeout", self.schedule_to_start_timeout),
716 ("schedule_to_close_timeout", self.schedule_to_close_timeout),
717 ("heartbeat_timeout", self.heartbeat_timeout),
718 ] {
719 if value.is_some_and(|value| value.is_zero()) {
720 return Err(ActivityOptionsError::new(
721 ActivityOptionsErrorKind::TimeoutNotPositive,
722 Some(field),
723 format!("{field} must be positive"),
724 ));
725 }
726 }
727
728 validate_timeout_order(
729 "heartbeat_timeout",
730 self.heartbeat_timeout,
731 "start_to_close_timeout",
732 self.start_to_close_timeout,
733 )?;
734 validate_timeout_order(
735 "start_to_close_timeout",
736 self.start_to_close_timeout,
737 "schedule_to_close_timeout",
738 self.schedule_to_close_timeout,
739 )?;
740 validate_timeout_order(
741 "schedule_to_start_timeout",
742 self.schedule_to_start_timeout,
743 "schedule_to_close_timeout",
744 self.schedule_to_close_timeout,
745 )?;
746
747 Ok(ValidatedActivityOptions {
748 task_queue: self.task_queue.clone(),
749 retry_policy: self
750 .retry_policy
751 .as_ref()
752 .map(validate_activity_retry_policy)
753 .transpose()?,
754 start_to_close_timeout: timeout_seconds(
755 "start_to_close_timeout",
756 self.start_to_close_timeout,
757 )?,
758 schedule_to_start_timeout: timeout_seconds(
759 "schedule_to_start_timeout",
760 self.schedule_to_start_timeout,
761 )?,
762 schedule_to_close_timeout: timeout_seconds(
763 "schedule_to_close_timeout",
764 self.schedule_to_close_timeout,
765 )?,
766 heartbeat_timeout: timeout_seconds("heartbeat_timeout", self.heartbeat_timeout)?,
767 })
768 }
769}
770
771#[derive(Clone, Debug)]
772struct ValidatedActivityOptions {
773 task_queue: Option<String>,
774 retry_policy: Option<Value>,
775 start_to_close_timeout: Option<u64>,
776 schedule_to_start_timeout: Option<u64>,
777 schedule_to_close_timeout: Option<u64>,
778 heartbeat_timeout: Option<u64>,
779}
780
781fn validate_timeout_order(
782 smaller_name: &'static str,
783 smaller: Option<Duration>,
784 larger_name: &'static str,
785 larger: Option<Duration>,
786) -> std::result::Result<(), ActivityOptionsError> {
787 if matches!((smaller, larger), (Some(smaller), Some(larger)) if smaller > larger) {
788 return Err(ActivityOptionsError::new(
789 ActivityOptionsErrorKind::TimeoutOrder,
790 Some(smaller_name),
791 format!("{smaller_name} must be <= {larger_name}"),
792 ));
793 }
794 Ok(())
795}
796
797fn timeout_seconds(
798 field: &'static str,
799 value: Option<Duration>,
800) -> std::result::Result<Option<u64>, ActivityOptionsError> {
801 value
802 .map(|value| {
803 activity_protocol_seconds(value).ok_or_else(|| {
804 ActivityOptionsError::new(
805 ActivityOptionsErrorKind::TimeoutOverflow,
806 Some(field),
807 format!("{field} is too large for the worker protocol"),
808 )
809 })
810 })
811 .transpose()
812}
813
814fn duration_seconds_ceil(value: Duration) -> Option<u64> {
815 value
816 .as_secs()
817 .checked_add(u64::from(value.subsec_nanos() > 0))
818}
819
820fn activity_protocol_seconds(value: Duration) -> Option<u64> {
821 duration_seconds_ceil(value).filter(|seconds| *seconds <= i64::MAX as u64)
822}
823
824fn validate_activity_retry_policy(
825 policy: &ActivityRetryPolicy,
826) -> std::result::Result<Value, ActivityOptionsError> {
827 if policy.max_attempts.is_none()
828 && policy.backoff.is_none()
829 && policy.non_retryable_error_types.is_empty()
830 {
831 return Err(ActivityOptionsError::new(
832 ActivityOptionsErrorKind::EmptyRetryPolicy,
833 Some("retry_policy"),
834 "retry_policy must configure at least one field",
835 ));
836 }
837 if policy.max_attempts == Some(0) {
838 return Err(ActivityOptionsError::new(
839 ActivityOptionsErrorKind::InvalidMaxAttempts,
840 Some("retry_policy.max_attempts"),
841 "max_attempts must be >= 1",
842 ));
843 }
844 if policy
845 .non_retryable_error_types
846 .iter()
847 .any(|error_type| error_type.trim().is_empty())
848 {
849 return Err(ActivityOptionsError::new(
850 ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
851 Some("retry_policy.non_retryable_error_types"),
852 "non_retryable_error_types must not contain empty values",
853 ));
854 }
855
856 let backoff_seconds = match &policy.backoff {
857 None => None,
858 Some(backoff) => {
859 let max_attempts = policy.max_attempts.ok_or_else(|| {
860 ActivityOptionsError::new(
861 ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
862 Some("retry_policy.backoff"),
863 "backoff requires max_attempts",
864 )
865 })?;
866 let retry_count = max_attempts.saturating_sub(1) as usize;
867 let intervals = match backoff {
868 ActivityBackoff::Explicit(intervals) => {
869 if intervals.len() > retry_count {
870 return Err(ActivityOptionsError::new(
871 ActivityOptionsErrorKind::TooManyBackoffIntervals,
872 Some("retry_policy.backoff"),
873 "backoff interval count must not exceed max_attempts - 1",
874 ));
875 }
876 intervals.clone()
877 }
878 ActivityBackoff::Exponential {
879 initial_interval,
880 coefficient,
881 maximum_interval,
882 } => {
883 if *coefficient < 1 {
884 return Err(ActivityOptionsError::new(
885 ActivityOptionsErrorKind::InvalidBackoffCoefficient,
886 Some("retry_policy.backoff.coefficient"),
887 "backoff coefficient must be >= 1",
888 ));
889 }
890 if retry_count > 10_000 {
891 return Err(ActivityOptionsError::new(
892 ActivityOptionsErrorKind::BackoffGenerationTooLarge,
893 Some("retry_policy.max_attempts"),
894 "generated backoff supports at most 10000 retry intervals",
895 ));
896 }
897 let mut current = *initial_interval;
898 let mut intervals = Vec::with_capacity(retry_count);
899 for _ in 0..retry_count {
900 let interval = maximum_interval
901 .map(|maximum| current.min(maximum))
902 .unwrap_or(current);
903 intervals.push(interval);
904 if maximum_interval.is_some_and(|maximum| interval == maximum) {
905 break;
906 }
907 current = current.checked_mul(*coefficient).ok_or_else(|| {
908 ActivityOptionsError::new(
909 ActivityOptionsErrorKind::BackoffOverflow,
910 Some("retry_policy.backoff"),
911 "generated backoff interval overflowed",
912 )
913 })?;
914 }
915 intervals
916 }
917 };
918 Some(
919 intervals
920 .into_iter()
921 .map(|interval| {
922 activity_protocol_seconds(interval).ok_or_else(|| {
923 ActivityOptionsError::new(
924 ActivityOptionsErrorKind::BackoffOverflow,
925 Some("retry_policy.backoff"),
926 "backoff interval is too large for the worker protocol",
927 )
928 })
929 })
930 .collect::<std::result::Result<Vec<_>, _>>()?,
931 )
932 }
933 };
934
935 let mut encoded = serde_json::Map::new();
936 if let Some(max_attempts) = policy.max_attempts {
937 encoded.insert("max_attempts".to_string(), json!(max_attempts));
938 }
939 if let Some(backoff_seconds) = backoff_seconds {
940 encoded.insert("backoff_seconds".to_string(), json!(backoff_seconds));
941 }
942 if !policy.non_retryable_error_types.is_empty() {
943 let mut canonical_error_types = Vec::new();
944 for error_type in policy
945 .non_retryable_error_types
946 .iter()
947 .map(|error_type| error_type.trim())
948 {
949 if !canonical_error_types.contains(&error_type) {
950 canonical_error_types.push(error_type);
951 }
952 }
953 encoded.insert(
954 "non_retryable_error_types".to_string(),
955 json!(canonical_error_types),
956 );
957 }
958 Ok(Value::Object(encoded))
959}
960
961#[derive(Clone, Debug, Error)]
964#[error("non-deterministic workflow replay ({reason}) at sequence {sequence:?}: {message}")]
965pub struct ReplayFailure {
966 pub reason: String,
967 pub sequence: Option<u64>,
968 pub expected: Option<String>,
969 pub actual: Option<String>,
970 pub message: String,
971}
972
973impl ReplayFailure {
974 fn new(
975 reason: impl Into<String>,
976 sequence: Option<u64>,
977 expected: Option<String>,
978 actual: Option<String>,
979 message: impl Into<String>,
980 ) -> Self {
981 Self {
982 reason: reason.into(),
983 sequence,
984 expected,
985 actual,
986 message: message.into(),
987 }
988 }
989}
990
991#[derive(Clone, Debug, Error)]
993#[error("query failed ({reason}, HTTP {status}): {message}")]
994pub struct QueryFailure {
995 pub status: u16,
996 pub reason: String,
997 pub message: String,
998 pub body: Value,
999}
1000
1001#[derive(Clone, Debug, Error)]
1003#[error("protocol rejected ({reason}, HTTP {status}): {message}")]
1004pub struct ProtocolFailure {
1005 pub status: u16,
1006 pub reason: String,
1007 pub message: String,
1008 pub supported_version: Option<String>,
1009 pub requested_version: Option<String>,
1010 pub body: Value,
1011}
1012
1013#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
1014pub struct PayloadEnvelope {
1015 pub codec: String,
1016 pub blob: String,
1017}
1018
1019impl PayloadEnvelope {
1020 pub fn avro<T: Serialize>(value: &T) -> Result<Self> {
1021 encode_payload(value, DEFAULT_CODEC)
1022 }
1023
1024 pub fn avro_value(value: &AvroValue) -> Result<Self> {
1027 encode_avro_value(value)
1028 }
1029}
1030
1031#[derive(Clone, Debug, PartialEq)]
1033pub enum AvroValue {
1034 Null,
1035 Boolean(bool),
1036 Long(i64),
1037 Double(f64),
1038 Bytes(Vec<u8>),
1039 String(String),
1040 Array(Vec<AvroValue>),
1041 Map(BTreeMap<String, AvroValue>),
1042}
1043
1044impl AvroValue {
1045 fn from_serialize<T: Serialize>(value: &T) -> Result<Self> {
1046 Self::from_serde_value(
1047 serde_value::to_value(value).map_err(|error| {
1048 Error::Codec(format!("could not adapt value for Avro: {error}"))
1049 })?,
1050 )
1051 }
1052
1053 fn from_serde_value(value: serde_value::Value) -> Result<Self> {
1054 use serde_value::Value as SerdeValue;
1055
1056 match value {
1057 SerdeValue::Unit => Ok(Self::Null),
1058 SerdeValue::Bool(value) => Ok(Self::Boolean(value)),
1059 SerdeValue::I8(value) => Ok(Self::Long(i64::from(value))),
1060 SerdeValue::I16(value) => Ok(Self::Long(i64::from(value))),
1061 SerdeValue::I32(value) => Ok(Self::Long(i64::from(value))),
1062 SerdeValue::I64(value) => Ok(Self::Long(value)),
1063 SerdeValue::U8(value) => Ok(Self::Long(i64::from(value))),
1064 SerdeValue::U16(value) => Ok(Self::Long(i64::from(value))),
1065 SerdeValue::U32(value) => Ok(Self::Long(i64::from(value))),
1066 SerdeValue::U64(value) => i64::try_from(value).map(Self::Long).map_err(|_| {
1067 Error::Codec(
1068 "integer_overflow: Avro Value long must be within signed 64-bit range"
1069 .to_string(),
1070 )
1071 }),
1072 SerdeValue::F32(value) => Self::finite_double(f64::from(value)),
1073 SerdeValue::F64(value) => Self::finite_double(value),
1074 SerdeValue::Char(value) => Ok(Self::String(value.to_string())),
1075 SerdeValue::String(value) => Ok(Self::String(value)),
1076 SerdeValue::Bytes(value) => Ok(Self::Bytes(value)),
1077 SerdeValue::Option(None) => Ok(Self::Null),
1078 SerdeValue::Option(Some(value)) | SerdeValue::Newtype(value) => {
1079 Self::from_serde_value(*value)
1080 }
1081 SerdeValue::Seq(values) => values
1082 .into_iter()
1083 .map(Self::from_serde_value)
1084 .collect::<Result<Vec<_>>>()
1085 .map(Self::Array),
1086 SerdeValue::Map(values) => values
1087 .into_iter()
1088 .map(|(key, value)| {
1089 let SerdeValue::String(key) = key else {
1090 return Err(Error::Codec(
1091 "invalid_map_key: Avro Value map keys must be strings".to_string(),
1092 ));
1093 };
1094
1095 Ok((key, Self::from_serde_value(value)?))
1096 })
1097 .collect::<Result<BTreeMap<_, _>>>()
1098 .map(Self::Map),
1099 }
1100 }
1101
1102 fn finite_double(value: f64) -> Result<Self> {
1103 if !value.is_finite() {
1104 return Err(Error::Codec(
1105 "non_finite_float: Avro Value doubles must be finite".to_string(),
1106 ));
1107 }
1108
1109 Ok(Self::Double(value))
1110 }
1111
1112 fn into_json(self) -> Result<Value> {
1113 match self {
1114 Self::Null => Ok(Value::Null),
1115 Self::Boolean(value) => Ok(Value::Bool(value)),
1116 Self::Long(value) => Ok(Value::Number(value.into())),
1117 Self::Double(value) => serde_json::Number::from_f64(value)
1118 .map(Value::Number)
1119 .ok_or_else(|| {
1120 Error::Codec(
1121 "non_finite_float: decoded Avro Value double is not finite".to_string(),
1122 )
1123 }),
1124 Self::Bytes(value) => Ok(json!({
1125 "$type": "bytes",
1126 "base64": BASE64.encode(value),
1127 })),
1128 Self::String(value) => Ok(Value::String(value)),
1129 Self::Array(values) => values
1130 .into_iter()
1131 .map(Self::into_json)
1132 .collect::<Result<Vec<_>>>()
1133 .map(Value::Array),
1134 Self::Map(values) => values
1135 .into_iter()
1136 .map(|(key, value)| Ok((key, value.into_json()?)))
1137 .collect::<Result<serde_json::Map<_, _>>>()
1138 .map(Value::Object),
1139 }
1140 }
1141
1142 fn into_serde_value(self) -> serde_value::Value {
1143 use serde_value::Value as SerdeValue;
1144
1145 match self {
1146 Self::Null => SerdeValue::Unit,
1147 Self::Boolean(value) => SerdeValue::Bool(value),
1148 Self::Long(value) => SerdeValue::I64(value),
1149 Self::Double(value) => SerdeValue::F64(value),
1150 Self::Bytes(value) => SerdeValue::Bytes(value),
1151 Self::String(value) => SerdeValue::String(value),
1152 Self::Array(values) => {
1153 SerdeValue::Seq(values.into_iter().map(Self::into_serde_value).collect())
1154 }
1155 Self::Map(values) => SerdeValue::Map(
1156 values
1157 .into_iter()
1158 .map(|(key, value)| (SerdeValue::String(key), value.into_serde_value()))
1159 .collect(),
1160 ),
1161 }
1162 }
1163
1164 pub fn deserialize<T: DeserializeOwned>(self) -> Result<T> {
1165 self.into_serde_value().deserialize_into().map_err(|error| {
1166 Error::Codec(format!(
1167 "avro_value_type_mismatch: could not adapt decoded value: {error}"
1168 ))
1169 })
1170 }
1171}
1172
1173impl Serialize for AvroValue {
1174 fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
1175 where
1176 S: Serializer,
1177 {
1178 match self {
1179 Self::Null => serializer.serialize_unit(),
1180 Self::Boolean(value) => serializer.serialize_bool(*value),
1181 Self::Long(value) => serializer.serialize_i64(*value),
1182 Self::Double(value) => serializer.serialize_f64(*value),
1183 Self::Bytes(value) => serializer.serialize_bytes(value),
1184 Self::String(value) => serializer.serialize_str(value),
1185 Self::Array(values) => {
1186 let mut sequence = serializer.serialize_seq(Some(values.len()))?;
1187 for value in values {
1188 sequence.serialize_element(value)?;
1189 }
1190 sequence.end()
1191 }
1192 Self::Map(values) => {
1193 let mut map = serializer.serialize_map(Some(values.len()))?;
1194 for (key, value) in values {
1195 map.serialize_entry(key, value)?;
1196 }
1197 map.end()
1198 }
1199 }
1200 }
1201}
1202
1203pub fn encode_avro_value(value: &AvroValue) -> Result<PayloadEnvelope> {
1204 let datum = avro_value_to_datum(value)?;
1205 let datum = to_avro_datum(avro_value_schema()?, datum)
1206 .map_err(|err| Error::Codec(format!("avro_value_encode_failed: {err}")))?;
1207 let mut bytes = Vec::with_capacity(datum.len() + 10);
1208 bytes.extend_from_slice(&AVRO_SINGLE_OBJECT_MAGIC);
1209 bytes.extend_from_slice(&AVRO_VALUE_SCHEMA_FINGERPRINT);
1210 bytes.extend_from_slice(&datum);
1211 Ok(PayloadEnvelope {
1212 codec: DEFAULT_CODEC.to_string(),
1213 blob: BASE64.encode(bytes),
1214 })
1215}
1216
1217pub fn decode_avro_value(envelope: &PayloadEnvelope) -> Result<AvroValue> {
1218 if envelope.codec != DEFAULT_CODEC {
1219 return Err(unsupported_payload_codec(&envelope.codec));
1220 }
1221 decode_avro_value_blob(&envelope.blob)
1222}
1223
1224pub fn encode_payload<T: Serialize>(value: &T, codec: &str) -> Result<PayloadEnvelope> {
1225 let blob = match codec {
1226 DEFAULT_CODEC => encode_avro_value(&AvroValue::from_serialize(value)?)?.blob,
1227 other => return Err(unsupported_payload_codec(other)),
1228 };
1229
1230 Ok(PayloadEnvelope {
1231 codec: codec.to_string(),
1232 blob,
1233 })
1234}
1235
1236pub fn decode_payload<T: DeserializeOwned>(envelope: &PayloadEnvelope) -> Result<T> {
1237 match envelope.codec.as_str() {
1238 DEFAULT_CODEC => decode_avro_value(envelope)?.deserialize(),
1239 other => Err(unsupported_payload_codec(other)),
1240 }
1241}
1242
1243#[cfg(test)]
1244fn encode_value_envelope(value: &Value, codec: &str) -> Result<Value> {
1245 Ok(serde_json::to_value(encode_payload(value, codec)?)?)
1246}
1247
1248fn decode_wire_value(value: &Value, fallback_codec: &str) -> Result<Value> {
1249 validate_payload_codec(fallback_codec)?;
1250
1251 if value.is_null() {
1252 return Ok(Value::Null);
1253 }
1254
1255 if let Some((codec, blob)) = payload_envelope_parts(value)? {
1256 return decode_blob(blob, codec);
1257 }
1258
1259 if let Some(blob) = value.as_str() {
1260 return decode_blob(blob, fallback_codec);
1261 }
1262
1263 Err(untagged_payload_value())
1264}
1265
1266fn encode_typed_envelope(value: &AvroValue, codec: &str) -> Result<Value> {
1267 let envelope = match codec {
1268 DEFAULT_CODEC => encode_avro_value(value)?,
1269 other => return Err(unsupported_payload_codec(other)),
1270 };
1271 Ok(serde_json::to_value(envelope)?)
1272}
1273
1274fn decode_wire_avro_value(value: &Value, fallback_codec: &str) -> Result<AvroValue> {
1275 validate_payload_codec(fallback_codec)?;
1276
1277 if value.is_null() {
1278 return Ok(AvroValue::Null);
1279 }
1280
1281 if let Some((codec, blob)) = payload_envelope_parts(value)? {
1282 validate_payload_codec(codec)?;
1283 return decode_avro_value_blob(blob);
1284 }
1285
1286 if let Some(blob) = value.as_str() {
1287 return match fallback_codec {
1288 DEFAULT_CODEC => decode_avro_value_blob(blob),
1289 other => Err(unsupported_payload_codec(other)),
1290 };
1291 }
1292
1293 Err(untagged_payload_value())
1294}
1295
1296fn normalize_avro_arguments(value: AvroValue) -> AvroValue {
1297 match value {
1298 AvroValue::Null => AvroValue::Array(Vec::new()),
1299 AvroValue::Array(_) => value,
1300 other => AvroValue::Array(vec![other]),
1301 }
1302}
1303
1304fn decode_blob(blob: &str, codec: &str) -> Result<Value> {
1305 match codec {
1306 DEFAULT_CODEC => decode_avro_value_blob(blob)?.into_json(),
1307 other => Err(unsupported_payload_codec(other)),
1308 }
1309}
1310
1311fn validate_payload_codec(codec: &str) -> Result<()> {
1312 match codec {
1313 DEFAULT_CODEC => Ok(()),
1314 other => Err(unsupported_payload_codec(other)),
1315 }
1316}
1317
1318fn payload_envelope_parts(value: &Value) -> Result<Option<(&str, &str)>> {
1319 let Some(object) = value.as_object() else {
1320 return Ok(None);
1321 };
1322 if !object.contains_key("codec") && !object.contains_key("blob") {
1323 return Ok(None);
1324 }
1325
1326 let codec = object
1327 .get("codec")
1328 .and_then(Value::as_str)
1329 .ok_or_else(invalid_payload_envelope)?;
1330 validate_payload_codec(codec)?;
1331 let blob = object
1332 .get("blob")
1333 .and_then(Value::as_str)
1334 .ok_or_else(invalid_payload_envelope)?;
1335 Ok(Some((codec, blob)))
1336}
1337
1338fn invalid_payload_envelope() -> Error {
1339 Error::Codec(
1340 "invalid_payload_envelope: durable payloads must use an object with string codec=\"avro\" and blob fields"
1341 .to_string(),
1342 )
1343}
1344
1345fn validate_workflow_task_commands(commands: &[Value]) -> Result<()> {
1346 for command in commands {
1347 let Some(command) = command.as_object() else {
1348 continue;
1349 };
1350 let Some(command_type) = command.get("type").and_then(Value::as_str) else {
1351 continue;
1352 };
1353 let Some(payload_field) = workflow_command_payload_field(command_type) else {
1354 continue;
1355 };
1356
1357 if let Some(codec) = command.get("payload_codec") {
1358 let codec = codec.as_str().ok_or_else(invalid_payload_envelope)?;
1359 validate_payload_codec(codec)?;
1360 }
1361
1362 let payload = command
1363 .get(payload_field)
1364 .ok_or_else(invalid_payload_envelope)?;
1365 validate_outbound_payload_envelope(payload)?;
1366 }
1367 Ok(())
1368}
1369
1370fn workflow_command_payload_field(command_type: &str) -> Option<&'static str> {
1371 match command_type {
1372 "complete_workflow" | "complete_update" | "record_side_effect" => Some("result"),
1373 "schedule_activity" | "start_child_workflow" | "continue_as_new" => Some("arguments"),
1374 "start_service_operation" => Some("request_payload"),
1375 _ => None,
1376 }
1377}
1378
1379fn validate_outbound_payload_envelope(value: &Value) -> Result<()> {
1380 let Some((codec, blob)) = payload_envelope_parts(value)? else {
1381 return Err(untagged_payload_value());
1382 };
1383 validate_payload_codec(codec)?;
1384 decode_avro_value_blob(blob)?;
1385 Ok(())
1386}
1387
1388fn unsupported_payload_codec(codec: &str) -> Error {
1389 Error::Codec(format!(
1390 "unsupported_payload_codec: workflow payload codec {codec:?} is not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
1391 ))
1392}
1393
1394fn untagged_payload_value() -> Error {
1395 Error::Codec(
1396 "unsupported_payload_codec: untagged durable payload values are not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
1397 .to_string(),
1398 )
1399}
1400
1401fn decode_avro_value_blob(blob: &str) -> Result<AvroValue> {
1402 let bytes = BASE64.decode(blob).map_err(|err| {
1403 Error::Codec(format!(
1404 "invalid_payload_framing: expected strict base64 Avro single-object bytes: {err}"
1405 ))
1406 })?;
1407
1408 if serde_json::from_slice::<Value>(&bytes).is_ok() {
1409 return Err(unsupported_payload_codec("json"));
1410 }
1411
1412 if bytes.len() < 10 || bytes[..2] != AVRO_SINGLE_OBJECT_MAGIC {
1413 return Err(Error::Codec(
1414 "invalid_payload_framing: expected Avro single-object magic c301".to_string(),
1415 ));
1416 }
1417
1418 let fingerprint: [u8; 8] = bytes[2..10]
1419 .try_into()
1420 .map_err(|_| Error::Codec("invalid Avro fingerprint length".to_string()))?;
1421 if fingerprint != AVRO_VALUE_SCHEMA_FINGERPRINT {
1422 return Err(Error::Codec(format!(
1423 "unsupported_payload_schema: unknown CRC-64-AVRO fingerprint {}",
1424 fingerprint
1425 .iter()
1426 .map(|byte| format!("{byte:02x}"))
1427 .collect::<String>()
1428 )));
1429 }
1430
1431 let mut datum_reader = StrictAvroDatumReader::new(&bytes[10..]);
1432 let datum = from_avro_datum(avro_value_schema()?, &mut datum_reader, None);
1436 if datum_reader.truncated {
1437 return Err(Error::Codec(
1438 "invalid_payload_framing: truncated Avro Value datum".to_string(),
1439 ));
1440 }
1441 let datum = datum.map_err(|err| {
1442 Error::Codec(format!(
1443 "invalid_payload_framing: malformed Avro Value datum: {err}"
1444 ))
1445 })?;
1446 if datum_reader.remaining() != 0 {
1447 return Err(Error::Codec(format!(
1448 "invalid_payload_framing: {} trailing bytes after Avro Value datum",
1449 datum_reader.remaining()
1450 )));
1451 }
1452 avro_value_from_datum(datum)
1453}
1454
1455struct StrictAvroDatumReader<'a> {
1456 bytes: &'a [u8],
1457 offset: usize,
1458 truncated: bool,
1459}
1460
1461impl<'a> StrictAvroDatumReader<'a> {
1462 fn new(bytes: &'a [u8]) -> Self {
1463 Self {
1464 bytes,
1465 offset: 0,
1466 truncated: false,
1467 }
1468 }
1469
1470 fn remaining(&self) -> usize {
1471 self.bytes.len() - self.offset
1472 }
1473}
1474
1475impl Read for StrictAvroDatumReader<'_> {
1476 fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
1477 let count = buffer.len().min(self.remaining());
1478 buffer[..count].copy_from_slice(&self.bytes[self.offset..self.offset + count]);
1479 self.offset += count;
1480 if count < buffer.len() {
1481 self.truncated = true;
1482 }
1483
1484 Ok(count)
1485 }
1486}
1487
1488fn avro_value_to_datum(value: &AvroValue) -> Result<AvroDatum> {
1489 let branch = match value {
1490 AvroValue::Null => AvroDatum::Union(0, Box::new(AvroDatum::Null)),
1491 AvroValue::Boolean(value) => AvroDatum::Union(
1492 1,
1493 Box::new(AvroDatum::Record(vec![(
1494 "boolean".to_string(),
1495 AvroDatum::Boolean(*value),
1496 )])),
1497 ),
1498 AvroValue::Long(value) => AvroDatum::Union(
1499 2,
1500 Box::new(AvroDatum::Record(vec![(
1501 "long".to_string(),
1502 AvroDatum::Long(*value),
1503 )])),
1504 ),
1505 AvroValue::Double(value) => {
1506 if !value.is_finite() {
1507 return Err(Error::Codec(
1508 "non_finite_float: Avro Value doubles must be finite".to_string(),
1509 ));
1510 }
1511 AvroDatum::Union(
1512 3,
1513 Box::new(AvroDatum::Record(vec![(
1514 "double".to_string(),
1515 AvroDatum::Double(*value),
1516 )])),
1517 )
1518 }
1519 AvroValue::Bytes(value) => AvroDatum::Union(
1520 4,
1521 Box::new(AvroDatum::Record(vec![(
1522 "bytes".to_string(),
1523 AvroDatum::Bytes(value.clone()),
1524 )])),
1525 ),
1526 AvroValue::String(value) => AvroDatum::Union(
1527 5,
1528 Box::new(AvroDatum::Record(vec![(
1529 "string".to_string(),
1530 AvroDatum::String(value.clone()),
1531 )])),
1532 ),
1533 AvroValue::Array(values) => AvroDatum::Union(
1534 6,
1535 Box::new(AvroDatum::Record(vec![(
1536 "items".to_string(),
1537 AvroDatum::Array(
1538 values
1539 .iter()
1540 .map(avro_value_to_datum)
1541 .collect::<Result<Vec<_>>>()?,
1542 ),
1543 )])),
1544 ),
1545 AvroValue::Map(values) => AvroDatum::Union(
1546 7,
1547 Box::new(AvroDatum::Record(vec![(
1548 "entries".to_string(),
1549 AvroDatum::Map(
1550 values
1551 .iter()
1552 .map(|(key, value)| Ok((key.clone(), avro_value_to_datum(value)?)))
1553 .collect::<Result<HashMap<_, _>>>()?,
1554 ),
1555 )])),
1556 ),
1557 };
1558 Ok(AvroDatum::Record(vec![("value".to_string(), branch)]))
1559}
1560
1561fn avro_value_from_datum(datum: AvroDatum) -> Result<AvroValue> {
1562 let AvroDatum::Record(mut outer) = datum else {
1563 return Err(Error::Codec(
1564 "invalid_payload_framing: datum is not a Value record".to_string(),
1565 ));
1566 };
1567 let (_, branch) = outer
1568 .pop()
1569 .filter(|(name, _)| name == "value")
1570 .ok_or_else(|| Error::Codec("invalid_payload_framing: Value field missing".to_string()))?;
1571 let AvroDatum::Union(_, branch) = branch else {
1572 return Err(Error::Codec(
1573 "invalid_payload_framing: invalid Value union".to_string(),
1574 ));
1575 };
1576 match *branch {
1577 AvroDatum::Null => Ok(AvroValue::Null),
1578 AvroDatum::Record(mut fields) => {
1579 let (name, value) = fields.pop().ok_or_else(|| {
1580 Error::Codec("invalid_payload_framing: empty Value branch".to_string())
1581 })?;
1582 match (name.as_str(), value) {
1583 ("boolean", AvroDatum::Boolean(value)) => Ok(AvroValue::Boolean(value)),
1584 ("long", AvroDatum::Long(value)) => Ok(AvroValue::Long(value)),
1585 ("double", AvroDatum::Double(value)) if value.is_finite() => {
1586 Ok(AvroValue::Double(value))
1587 }
1588 ("bytes", AvroDatum::Bytes(value)) => Ok(AvroValue::Bytes(value)),
1589 ("string", AvroDatum::String(value)) => Ok(AvroValue::String(value)),
1590 ("items", AvroDatum::Array(values)) => values
1591 .into_iter()
1592 .map(avro_value_from_datum)
1593 .collect::<Result<Vec<_>>>()
1594 .map(AvroValue::Array),
1595 ("entries", AvroDatum::Map(values)) => values
1596 .into_iter()
1597 .map(|(key, value)| Ok((key, avro_value_from_datum(value)?)))
1598 .collect::<Result<BTreeMap<_, _>>>()
1599 .map(AvroValue::Map),
1600 _ => Err(Error::Codec(
1601 "invalid_payload_framing: unknown Value branch".to_string(),
1602 )),
1603 }
1604 }
1605 _ => Err(Error::Codec(
1606 "invalid_payload_framing: invalid Value branch".to_string(),
1607 )),
1608 }
1609}
1610
1611fn avro_value_schema() -> Result<&'static Schema> {
1612 match AVRO_VALUE_SCHEMA.get_or_init(|| {
1613 Schema::parse_str(AVRO_VALUE_SCHEMA_JSON)
1614 .map_err(|err| format!("could not parse Avro Value schema: {err}"))
1615 }) {
1616 Ok(schema) => Ok(schema),
1617 Err(message) => Err(Error::Codec(message.clone())),
1618 }
1619}
1620
1621#[derive(Clone, Debug)]
1622pub struct Client {
1623 http: reqwest::Client,
1624 base_url: String,
1625 token: Option<String>,
1626 control_token: Option<String>,
1627 worker_token: Option<String>,
1628 namespace: String,
1629}
1630
1631impl Client {
1632 pub fn new(base_url: impl Into<String>) -> Result<Self> {
1633 Self::builder(base_url).build()
1634 }
1635
1636 pub fn builder(base_url: impl Into<String>) -> ClientBuilder {
1637 ClientBuilder {
1638 base_url: base_url.into(),
1639 token: None,
1640 control_token: None,
1641 worker_token: None,
1642 namespace: "default".to_string(),
1643 timeout: Duration::from_secs(60),
1644 }
1645 }
1646
1647 pub async fn health(&self) -> Result<Value> {
1648 self.request_json(
1649 reqwest::Method::GET,
1650 "/health",
1651 RequestProtocol::ControlPlane,
1652 Option::<&Value>::None,
1653 )
1654 .await
1655 }
1656
1657 pub async fn cluster_info(&self) -> Result<Value> {
1658 self.request_json(
1659 reqwest::Method::GET,
1660 "/cluster/info",
1661 RequestProtocol::ControlPlane,
1662 Option::<&Value>::None,
1663 )
1664 .await
1665 }
1666
1667 pub async fn start_workflow<T: Serialize>(
1668 &self,
1669 workflow_type: &str,
1670 task_queue: &str,
1671 workflow_id: &str,
1672 input: T,
1673 ) -> Result<WorkflowHandle> {
1674 self.start_workflow_with_options(
1675 workflow_type,
1676 task_queue,
1677 workflow_id,
1678 WorkflowStartOptions::default(),
1679 input,
1680 )
1681 .await
1682 }
1683
1684 pub async fn start_workflow_with_options<T: Serialize>(
1687 &self,
1688 workflow_type: &str,
1689 task_queue: &str,
1690 workflow_id: &str,
1691 options: WorkflowStartOptions,
1692 input: T,
1693 ) -> Result<WorkflowHandle> {
1694 options.validate()?;
1695 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1696 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
1697 let body = json!({
1698 "workflow_id": workflow_id,
1699 "workflow_type": workflow_type,
1700 "task_queue": task_queue,
1701 "input": input_envelope,
1702 "execution_timeout_seconds": options.execution_timeout_seconds,
1703 "run_timeout_seconds": options.run_timeout_seconds
1704 });
1705
1706 let data: Value = self
1707 .request_json(
1708 reqwest::Method::POST,
1709 "/workflows",
1710 RequestProtocol::ControlPlane,
1711 Some(&body),
1712 )
1713 .await?;
1714
1715 Ok(WorkflowHandle {
1716 client: self.clone(),
1717 workflow_id: data
1718 .get("workflow_id")
1719 .and_then(Value::as_str)
1720 .unwrap_or(workflow_id)
1721 .to_string(),
1722 run_id: data
1723 .get("run_id")
1724 .and_then(Value::as_str)
1725 .map(str::to_string),
1726 workflow_type: data
1727 .get("workflow_type")
1728 .and_then(Value::as_str)
1729 .unwrap_or(workflow_type)
1730 .to_string(),
1731 })
1732 }
1733
1734 pub async fn signal_workflow<T: Serialize>(
1735 &self,
1736 workflow_id: &str,
1737 signal_name: &str,
1738 input: T,
1739 ) -> Result<Value> {
1740 self.signal_workflow_target(workflow_id, None, signal_name, input)
1741 .await
1742 }
1743
1744 pub async fn signal_workflow_run<T: Serialize>(
1746 &self,
1747 workflow_id: &str,
1748 run_id: &str,
1749 signal_name: &str,
1750 input: T,
1751 ) -> Result<Value> {
1752 self.signal_workflow_target(workflow_id, Some(run_id), signal_name, input)
1753 .await
1754 }
1755
1756 async fn signal_workflow_target<T: Serialize>(
1757 &self,
1758 workflow_id: &str,
1759 run_id: Option<&str>,
1760 signal_name: &str,
1761 input: T,
1762 ) -> Result<Value> {
1763 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1764 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
1765 let body = json!({
1766 "input": input_envelope
1767 });
1768 let path = match run_id {
1769 Some(run_id) => {
1770 format!("/workflows/{workflow_id}/runs/{run_id}/signal/{signal_name}")
1771 }
1772 None => format!("/workflows/{workflow_id}/signal/{signal_name}"),
1773 };
1774 self.request_json(
1775 reqwest::Method::POST,
1776 &path,
1777 RequestProtocol::ControlPlane,
1778 Some(&body),
1779 )
1780 .await
1781 }
1782
1783 pub async fn cancel_workflow(
1785 &self,
1786 workflow_id: &str,
1787 options: WorkflowCommandOptions,
1788 ) -> Result<WorkflowCommandResult> {
1789 self.workflow_command(workflow_id, None, WorkflowCommandKind::Cancel, options)
1790 .await
1791 }
1792
1793 pub async fn cancel_workflow_run(
1795 &self,
1796 workflow_id: &str,
1797 run_id: &str,
1798 options: WorkflowCommandOptions,
1799 ) -> Result<WorkflowCommandResult> {
1800 self.workflow_command(
1801 workflow_id,
1802 Some(run_id),
1803 WorkflowCommandKind::Cancel,
1804 options,
1805 )
1806 .await
1807 }
1808
1809 pub async fn terminate_workflow(
1811 &self,
1812 workflow_id: &str,
1813 options: WorkflowCommandOptions,
1814 ) -> Result<WorkflowCommandResult> {
1815 self.workflow_command(workflow_id, None, WorkflowCommandKind::Terminate, options)
1816 .await
1817 }
1818
1819 pub async fn terminate_workflow_run(
1821 &self,
1822 workflow_id: &str,
1823 run_id: &str,
1824 options: WorkflowCommandOptions,
1825 ) -> Result<WorkflowCommandResult> {
1826 self.workflow_command(
1827 workflow_id,
1828 Some(run_id),
1829 WorkflowCommandKind::Terminate,
1830 options,
1831 )
1832 .await
1833 }
1834
1835 async fn workflow_command(
1836 &self,
1837 workflow_id: &str,
1838 run_id: Option<&str>,
1839 command: WorkflowCommandKind,
1840 options: WorkflowCommandOptions,
1841 ) -> Result<WorkflowCommandResult> {
1842 let path = match run_id {
1843 Some(run_id) => format!(
1844 "/workflows/{workflow_id}/runs/{run_id}/{}",
1845 command.as_str()
1846 ),
1847 None => format!("/workflows/{workflow_id}/{}", command.as_str()),
1848 };
1849 let data = match self
1850 .request_json(
1851 reqwest::Method::POST,
1852 &path,
1853 RequestProtocol::ControlPlane,
1854 Some(&options),
1855 )
1856 .await
1857 {
1858 Ok(data) => data,
1859 Err(Error::Http { status, body }) => {
1860 return Err(Error::WorkflowCommandRejected(workflow_command_rejection(
1861 command,
1862 status,
1863 body,
1864 workflow_id,
1865 run_id,
1866 )));
1867 }
1868 Err(error) => return Err(error),
1869 };
1870
1871 Ok(workflow_command_result(command, data, workflow_id, run_id))
1872 }
1873
1874 pub async fn query_workflow<T: Serialize>(
1880 &self,
1881 workflow_id: &str,
1882 query_name: &str,
1883 input: T,
1884 ) -> Result<Value> {
1885 self.query_workflow_target(workflow_id, None, query_name, input)
1886 .await
1887 }
1888
1889 pub async fn query_workflow_run<T: Serialize>(
1891 &self,
1892 workflow_id: &str,
1893 run_id: &str,
1894 query_name: &str,
1895 input: T,
1896 ) -> Result<Value> {
1897 self.query_workflow_target(workflow_id, Some(run_id), query_name, input)
1898 .await
1899 }
1900
1901 pub async fn query_workflow_avro_value<T: Serialize>(
1903 &self,
1904 workflow_id: &str,
1905 query_name: &str,
1906 input: T,
1907 ) -> Result<AvroValue> {
1908 self.query_workflow_avro_value_target(workflow_id, None, query_name, input)
1909 .await
1910 }
1911
1912 pub async fn query_workflow_run_avro_value<T: Serialize>(
1914 &self,
1915 workflow_id: &str,
1916 run_id: &str,
1917 query_name: &str,
1918 input: T,
1919 ) -> Result<AvroValue> {
1920 self.query_workflow_avro_value_target(workflow_id, Some(run_id), query_name, input)
1921 .await
1922 }
1923
1924 async fn query_workflow_avro_value_target<T: Serialize>(
1925 &self,
1926 workflow_id: &str,
1927 run_id: Option<&str>,
1928 query_name: &str,
1929 input: T,
1930 ) -> Result<AvroValue> {
1931 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1932 let body = json!({"input": encode_typed_envelope(&input, DEFAULT_CODEC)?});
1933 let path = match run_id {
1934 Some(run_id) => {
1935 format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
1936 }
1937 None => format!("/workflows/{workflow_id}/query/{query_name}"),
1938 };
1939 let response: Value = match self
1940 .request_json(
1941 reqwest::Method::POST,
1942 &path,
1943 RequestProtocol::ControlPlane,
1944 Some(&body),
1945 )
1946 .await
1947 {
1948 Ok(response) => response,
1949 Err(Error::Http { status, body }) => {
1950 return Err(Error::QueryFailed(query_failure(status, body)));
1951 }
1952 Err(error) => return Err(error),
1953 };
1954
1955 let envelope = response
1956 .get("result_envelope")
1957 .filter(|envelope| !envelope.is_null())
1958 .ok_or_else(|| {
1959 Error::Codec(
1960 "missing_payload_envelope: typed query result requires result_envelope"
1961 .to_string(),
1962 )
1963 })?;
1964 decode_wire_avro_value(envelope, DEFAULT_CODEC)
1965 }
1966
1967 async fn query_workflow_target<T: Serialize>(
1968 &self,
1969 workflow_id: &str,
1970 run_id: Option<&str>,
1971 query_name: &str,
1972 input: T,
1973 ) -> Result<Value> {
1974 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1975 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
1976 let body = json!({
1977 "input": input_envelope
1978 });
1979 let path = match run_id {
1980 Some(run_id) => {
1981 format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
1982 }
1983 None => format!("/workflows/{workflow_id}/query/{query_name}"),
1984 };
1985 let response: Value = match self
1986 .request_json(
1987 reqwest::Method::POST,
1988 &path,
1989 RequestProtocol::ControlPlane,
1990 Some(&body),
1991 )
1992 .await
1993 {
1994 Ok(response) => response,
1995 Err(Error::Http { status, body }) => {
1996 return Err(Error::QueryFailed(query_failure(status, body)));
1997 }
1998 Err(error) => return Err(error),
1999 };
2000
2001 if let Some(envelope) = response
2002 .get("result_envelope")
2003 .filter(|envelope| !envelope.is_null())
2004 {
2005 return decode_wire_value(envelope, DEFAULT_CODEC);
2006 }
2007
2008 Ok(response.get("result").cloned().unwrap_or(Value::Null))
2009 }
2010
2011 pub async fn update_workflow<T: Serialize>(
2013 &self,
2014 workflow_id: &str,
2015 update_name: &str,
2016 input: T,
2017 request_id: Option<&str>,
2018 ) -> Result<Value> {
2019 let response = self
2020 .update_workflow_response(workflow_id, update_name, input, request_id)
2021 .await?;
2022 if let Some(envelope) = response
2023 .get("result_envelope")
2024 .filter(|envelope| !envelope.is_null())
2025 {
2026 return decode_wire_value(envelope, DEFAULT_CODEC);
2027 }
2028 Ok(response.get("result").cloned().unwrap_or(response))
2029 }
2030
2031 pub async fn update_workflow_avro_value<T: Serialize>(
2033 &self,
2034 workflow_id: &str,
2035 update_name: &str,
2036 input: T,
2037 request_id: Option<&str>,
2038 ) -> Result<AvroValue> {
2039 let response = self
2040 .update_workflow_response(workflow_id, update_name, input, request_id)
2041 .await?;
2042 let envelope = response
2043 .get("result_envelope")
2044 .filter(|envelope| !envelope.is_null())
2045 .ok_or_else(|| {
2046 Error::Codec(
2047 "missing_payload_envelope: typed update result requires result_envelope"
2048 .to_string(),
2049 )
2050 })?;
2051 decode_wire_avro_value(envelope, DEFAULT_CODEC)
2052 }
2053
2054 async fn update_workflow_response<T: Serialize>(
2055 &self,
2056 workflow_id: &str,
2057 update_name: &str,
2058 input: T,
2059 request_id: Option<&str>,
2060 ) -> Result<Value> {
2061 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2062 let mut body = json!({
2063 "input": encode_typed_envelope(&input, DEFAULT_CODEC)?,
2064 "wait_for": "completed",
2065 });
2066 if let Some(request_id) = request_id {
2067 body["request_id"] = json!(request_id);
2068 }
2069 self.request_json(
2070 reqwest::Method::POST,
2071 &format!("/workflows/{workflow_id}/update/{update_name}"),
2072 RequestProtocol::ControlPlane,
2073 Some(&body),
2074 )
2075 .await
2076 }
2077
2078 pub async fn describe_workflow(&self, workflow_id: &str) -> Result<WorkflowDescription> {
2079 let path = format!("/workflows/{workflow_id}");
2080 let mut data: WorkflowDescription = self
2081 .request_json(
2082 reqwest::Method::GET,
2083 &path,
2084 RequestProtocol::ControlPlane,
2085 Option::<&Value>::None,
2086 )
2087 .await?;
2088 data.decode_payloads()?;
2089 Ok(data)
2090 }
2091
2092 pub async fn describe_workflow_run(
2094 &self,
2095 workflow_id: &str,
2096 run_id: &str,
2097 ) -> Result<WorkflowDescription> {
2098 let path = format!("/workflows/{workflow_id}/runs/{run_id}");
2099 let mut data: WorkflowDescription = self
2100 .request_json(
2101 reqwest::Method::GET,
2102 &path,
2103 RequestProtocol::ControlPlane,
2104 Option::<&Value>::None,
2105 )
2106 .await?;
2107 data.decode_payloads()?;
2108 Ok(data)
2109 }
2110
2111 pub async fn register_worker(
2112 &self,
2113 worker_id: &str,
2114 task_queue: &str,
2115 supported_workflow_types: Vec<String>,
2116 supported_activity_types: Vec<String>,
2117 max_concurrent_workflow_tasks: usize,
2118 max_concurrent_activity_tasks: usize,
2119 ) -> Result<RegisterWorkerResponse> {
2120 self.register_worker_with_capabilities(
2121 worker_id,
2122 task_queue,
2123 supported_workflow_types,
2124 supported_activity_types,
2125 max_concurrent_workflow_tasks,
2126 max_concurrent_activity_tasks,
2127 Vec::new(),
2128 )
2129 .await
2130 }
2131
2132 pub async fn register_worker_with_capabilities(
2134 &self,
2135 worker_id: &str,
2136 task_queue: &str,
2137 supported_workflow_types: Vec<String>,
2138 supported_activity_types: Vec<String>,
2139 max_concurrent_workflow_tasks: usize,
2140 max_concurrent_activity_tasks: usize,
2141 capabilities: Vec<String>,
2142 ) -> Result<RegisterWorkerResponse> {
2143 self.register_worker_with_command_contracts(
2144 worker_id,
2145 task_queue,
2146 supported_workflow_types,
2147 supported_activity_types,
2148 max_concurrent_workflow_tasks,
2149 max_concurrent_activity_tasks,
2150 capabilities,
2151 Value::Object(serde_json::Map::new()),
2152 )
2153 .await
2154 }
2155
2156 #[allow(clippy::too_many_arguments)]
2163 pub async fn register_worker_with_command_contracts(
2164 &self,
2165 worker_id: &str,
2166 task_queue: &str,
2167 supported_workflow_types: Vec<String>,
2168 supported_activity_types: Vec<String>,
2169 max_concurrent_workflow_tasks: usize,
2170 max_concurrent_activity_tasks: usize,
2171 capabilities: Vec<String>,
2172 workflow_command_contracts: Value,
2173 ) -> Result<RegisterWorkerResponse> {
2174 if let Some(contracts) = workflow_command_contracts.as_object() {
2175 for (workflow_type, contract) in contracts {
2176 let Some(update_validators) = contract.get("update_validators") else {
2177 continue;
2178 };
2179 if !update_validators
2180 .as_array()
2181 .is_some_and(|validators| validators.is_empty())
2182 {
2183 return Err(Error::UnsupportedUpdateValidators {
2184 workflow_type: workflow_type.clone(),
2185 });
2186 }
2187 }
2188 }
2189
2190 let mut body = json!({
2191 "worker_id": worker_id,
2192 "task_queue": task_queue,
2193 "runtime": "rust",
2194 "sdk_version": SDK_VERSION,
2195 "supported_workflow_types": supported_workflow_types,
2196 "supported_activity_types": supported_activity_types,
2197 "capabilities": capabilities,
2198 "max_concurrent_workflow_tasks": max_concurrent_workflow_tasks,
2199 "max_concurrent_activity_tasks": max_concurrent_activity_tasks
2200 });
2201 if workflow_command_contracts
2202 .as_object()
2203 .is_some_and(|contracts| !contracts.is_empty())
2204 {
2205 body["workflow_command_contracts"] = workflow_command_contracts;
2206 }
2207
2208 self.request_json(
2209 reqwest::Method::POST,
2210 "/worker/register",
2211 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2212 Some(&body),
2213 )
2214 .await
2215 }
2216
2217 pub async fn deregister_worker_registration(
2223 &self,
2224 worker_id: &str,
2225 ) -> Result<WorkerDeregistrationEnvelope> {
2226 let path = format!(
2227 "/worker/registrations/{}",
2228 percent_encode_path_segment(worker_id)
2229 );
2230 self.request_json(
2231 reqwest::Method::DELETE,
2232 &path,
2233 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2234 Option::<&Value>::None,
2235 )
2236 .await
2237 }
2238
2239 pub async fn poll_query_task(
2241 &self,
2242 worker_id: &str,
2243 task_queue: &str,
2244 timeout: Duration,
2245 ) -> Result<Option<QueryTask>> {
2246 Ok(self
2247 .poll_query_task_response(worker_id, task_queue, timeout)
2248 .await?
2249 .task)
2250 }
2251
2252 pub async fn poll_query_task_response(
2254 &self,
2255 worker_id: &str,
2256 task_queue: &str,
2257 timeout: Duration,
2258 ) -> Result<PollQueryTaskResponse> {
2259 let poll_request_id = unique_request_id("rust-query-poll");
2260 self.poll_query_task_response_with_request_id(
2261 worker_id,
2262 task_queue,
2263 timeout,
2264 &poll_request_id,
2265 1,
2266 )
2267 .await
2268 }
2269
2270 async fn poll_query_task_response_with_request_id(
2271 &self,
2272 worker_id: &str,
2273 task_queue: &str,
2274 timeout: Duration,
2275 poll_request_id: &str,
2276 transport_retries: usize,
2277 ) -> Result<PollQueryTaskResponse> {
2278 let timeout_seconds = long_poll_timeout_seconds(timeout);
2279 let body = json!({
2280 "worker_id": worker_id,
2281 "task_queue": task_queue,
2282 "poll_request_id": poll_request_id,
2283 "timeout_seconds": timeout_seconds,
2284 });
2285 self.poll_request_json(
2286 "/worker/query-tasks/poll",
2287 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
2288 &body,
2289 timeout + Duration::from_secs(5),
2290 transport_retries,
2291 )
2292 .await
2293 }
2294
2295 pub async fn complete_query_task<T: Serialize>(
2297 &self,
2298 query_task_id: &str,
2299 lease_owner: &str,
2300 query_task_attempt: u64,
2301 result: T,
2302 codec: &str,
2303 ) -> Result<Value> {
2304 let typed_result = AvroValue::from_serialize(&result)?;
2305 let result_envelope = encode_typed_envelope(&typed_result, codec)?;
2306 self.complete_query_task_with_envelope(
2307 query_task_id,
2308 lease_owner,
2309 query_task_attempt,
2310 typed_result.into_json()?,
2311 result_envelope,
2312 )
2313 .await
2314 }
2315
2316 async fn complete_query_task_with_envelope(
2317 &self,
2318 query_task_id: &str,
2319 lease_owner: &str,
2320 query_task_attempt: u64,
2321 result: Value,
2322 result_envelope: Value,
2323 ) -> Result<Value> {
2324 let body = json!({
2325 "lease_owner": lease_owner,
2326 "query_task_attempt": query_task_attempt,
2327 "result": result,
2328 "result_envelope": result_envelope,
2329 });
2330 let path = format!("/worker/query-tasks/{query_task_id}/complete");
2331 let response = self
2332 .request_json(
2333 reqwest::Method::POST,
2334 &path,
2335 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
2336 Some(&body),
2337 )
2338 .await;
2339 query_task_response(response)
2340 }
2341
2342 pub async fn fail_query_task(
2344 &self,
2345 query_task_id: &str,
2346 lease_owner: &str,
2347 query_task_attempt: u64,
2348 message: impl Into<String>,
2349 reason: impl Into<String>,
2350 failure_type: impl Into<String>,
2351 ) -> Result<Value> {
2352 let body = json!({
2353 "lease_owner": lease_owner,
2354 "query_task_attempt": query_task_attempt,
2355 "failure": {
2356 "message": message.into(),
2357 "reason": reason.into(),
2358 "type": failure_type.into(),
2359 }
2360 });
2361 let path = format!("/worker/query-tasks/{query_task_id}/fail");
2362 let response = self
2363 .request_json(
2364 reqwest::Method::POST,
2365 &path,
2366 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
2367 Some(&body),
2368 )
2369 .await;
2370 query_task_response(response)
2371 }
2372
2373 pub async fn heartbeat_worker(
2374 &self,
2375 worker_id: &str,
2376 workflow_available: usize,
2377 activity_available: usize,
2378 ) -> Result<Value> {
2379 let body = json!({
2380 "worker_id": worker_id,
2381 "task_slots": {
2382 "workflow_available": workflow_available,
2383 "activity_available": activity_available
2384 },
2385 "process_metrics": {
2386 "process_id": std::process::id(),
2387 "process_uptime_seconds": 0
2388 }
2389 });
2390
2391 self.request_json(
2392 reqwest::Method::POST,
2393 "/worker/heartbeat",
2394 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2395 Some(&body),
2396 )
2397 .await
2398 }
2399
2400 pub async fn poll_workflow_task(
2401 &self,
2402 worker_id: &str,
2403 task_queue: &str,
2404 timeout: Duration,
2405 ) -> Result<Option<WorkflowTask>> {
2406 Ok(self
2407 .poll_workflow_task_response(worker_id, task_queue, timeout)
2408 .await?
2409 .task)
2410 }
2411
2412 pub async fn poll_workflow_task_response(
2413 &self,
2414 worker_id: &str,
2415 task_queue: &str,
2416 timeout: Duration,
2417 ) -> Result<PollWorkflowTaskResponse> {
2418 let poll_request_id = unique_request_id("rust-workflow-poll");
2419 self.poll_workflow_task_response_with_request_id(
2420 worker_id,
2421 task_queue,
2422 timeout,
2423 &poll_request_id,
2424 1,
2425 )
2426 .await
2427 }
2428
2429 async fn poll_workflow_task_response_with_request_id(
2430 &self,
2431 worker_id: &str,
2432 task_queue: &str,
2433 timeout: Duration,
2434 poll_request_id: &str,
2435 transport_retries: usize,
2436 ) -> Result<PollWorkflowTaskResponse> {
2437 let body = json!({
2438 "worker_id": worker_id,
2439 "task_queue": task_queue,
2440 "poll_request_id": poll_request_id,
2441 "timeout_seconds": long_poll_timeout_seconds(timeout),
2442 });
2443 let mut data: PollWorkflowTaskResponse = self
2444 .poll_request_json(
2445 "/worker/workflow-tasks/poll",
2446 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2447 &body,
2448 timeout + Duration::from_secs(5),
2449 transport_retries,
2450 )
2451 .await?;
2452
2453 if let Some(task) = data.task.as_mut() {
2454 self.fetch_remaining_workflow_history(worker_id, task)
2455 .await?;
2456 }
2457
2458 Ok(data)
2459 }
2460
2461 async fn fetch_remaining_workflow_history(
2462 &self,
2463 worker_id: &str,
2464 task: &mut WorkflowTask,
2465 ) -> Result<()> {
2466 let mut next_token = task.next_history_page_token.clone();
2467
2468 while let Some(token) = next_token.take().filter(|token| !token.is_empty()) {
2469 let lease_owner = task
2470 .lease_owner
2471 .clone()
2472 .unwrap_or_else(|| worker_id.to_string());
2473 let page = self
2474 .workflow_task_history_page(
2475 &task.task_id,
2476 &lease_owner,
2477 task.workflow_task_attempt,
2478 &token,
2479 )
2480 .await?;
2481
2482 task.append_history_page(page);
2483
2484 if task.next_history_page_token.as_deref() == Some(token.as_str()) {
2485 return Err(Error::Codec(
2486 "workflow history pagination returned the same page token".to_string(),
2487 ));
2488 }
2489
2490 next_token = task.next_history_page_token.clone();
2491 }
2492
2493 Ok(())
2494 }
2495
2496 async fn workflow_task_history_page(
2497 &self,
2498 task_id: &str,
2499 lease_owner: &str,
2500 workflow_task_attempt: u64,
2501 next_history_page_token: &str,
2502 ) -> Result<WorkflowTaskHistoryPage> {
2503 let body = json!({
2504 "lease_owner": lease_owner,
2505 "workflow_task_attempt": workflow_task_attempt,
2506 "next_history_page_token": next_history_page_token
2507 });
2508 let path = format!("/worker/workflow-tasks/{task_id}/history");
2509
2510 self.request_json(
2511 reqwest::Method::POST,
2512 &path,
2513 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2514 Some(&body),
2515 )
2516 .await
2517 }
2518
2519 pub async fn complete_workflow_task(
2520 &self,
2521 task_id: &str,
2522 lease_owner: &str,
2523 workflow_task_attempt: u64,
2524 commands: Vec<Value>,
2525 ) -> Result<Value> {
2526 validate_workflow_task_commands(&commands)?;
2527 let body = json!({
2528 "lease_owner": lease_owner,
2529 "workflow_task_attempt": workflow_task_attempt,
2530 "commands": commands
2531 });
2532 let path = format!("/worker/workflow-tasks/{task_id}/complete");
2533 self.request_json(
2534 reqwest::Method::POST,
2535 &path,
2536 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2537 Some(&body),
2538 )
2539 .await
2540 }
2541
2542 pub async fn fail_workflow_task(
2543 &self,
2544 task_id: &str,
2545 lease_owner: &str,
2546 workflow_task_attempt: u64,
2547 message: impl Into<String>,
2548 ) -> Result<Value> {
2549 self.fail_workflow_task_with_type(
2550 task_id,
2551 lease_owner,
2552 workflow_task_attempt,
2553 message,
2554 "RustWorkflowTaskFailure",
2555 )
2556 .await
2557 }
2558
2559 async fn fail_workflow_task_with_type(
2560 &self,
2561 task_id: &str,
2562 lease_owner: &str,
2563 workflow_task_attempt: u64,
2564 message: impl Into<String>,
2565 failure_type: &str,
2566 ) -> Result<Value> {
2567 let body = json!({
2568 "lease_owner": lease_owner,
2569 "workflow_task_attempt": workflow_task_attempt,
2570 "failure": {
2571 "message": message.into(),
2572 "type": failure_type
2573 }
2574 });
2575 let path = format!("/worker/workflow-tasks/{task_id}/fail");
2576 self.request_json(
2577 reqwest::Method::POST,
2578 &path,
2579 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2580 Some(&body),
2581 )
2582 .await
2583 }
2584
2585 pub async fn poll_activity_task(
2586 &self,
2587 worker_id: &str,
2588 task_queue: &str,
2589 timeout: Duration,
2590 ) -> Result<Option<ActivityTask>> {
2591 Ok(self
2592 .poll_activity_task_response(worker_id, task_queue, timeout)
2593 .await?
2594 .task)
2595 }
2596
2597 pub async fn poll_activity_task_response(
2599 &self,
2600 worker_id: &str,
2601 task_queue: &str,
2602 timeout: Duration,
2603 ) -> Result<PollActivityTaskResponse> {
2604 let poll_request_id = unique_request_id("rust-activity-poll");
2605 self.poll_activity_task_response_with_request_id(
2606 worker_id,
2607 task_queue,
2608 timeout,
2609 &poll_request_id,
2610 1,
2611 )
2612 .await
2613 }
2614
2615 async fn poll_activity_task_response_with_request_id(
2616 &self,
2617 worker_id: &str,
2618 task_queue: &str,
2619 timeout: Duration,
2620 poll_request_id: &str,
2621 transport_retries: usize,
2622 ) -> Result<PollActivityTaskResponse> {
2623 let body = json!({
2624 "worker_id": worker_id,
2625 "task_queue": task_queue,
2626 "poll_request_id": poll_request_id,
2627 "timeout_seconds": long_poll_timeout_seconds(timeout),
2628 });
2629 let data: PollActivityTaskResponse = self
2630 .poll_request_json(
2631 "/worker/activity-tasks/poll",
2632 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2633 &body,
2634 timeout + Duration::from_secs(5),
2635 transport_retries,
2636 )
2637 .await?;
2638 Ok(data)
2639 }
2640
2641 pub async fn complete_activity_task<T: Serialize>(
2642 &self,
2643 task_id: &str,
2644 activity_attempt_id: &str,
2645 lease_owner: &str,
2646 result: T,
2647 codec: &str,
2648 ) -> Result<Value> {
2649 let result = encode_typed_envelope(&AvroValue::from_serialize(&result)?, codec)?;
2650 let body = json!({
2651 "activity_attempt_id": activity_attempt_id,
2652 "lease_owner": lease_owner,
2653 "result": result
2654 });
2655 let path = format!("/worker/activity-tasks/{task_id}/complete");
2656 activity_task_response(
2657 self.request_json(
2658 reqwest::Method::POST,
2659 &path,
2660 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2661 Some(&body),
2662 )
2663 .await,
2664 "complete",
2665 task_id,
2666 activity_attempt_id,
2667 )
2668 }
2669
2670 pub async fn fail_activity_task(
2671 &self,
2672 task_id: &str,
2673 activity_attempt_id: &str,
2674 lease_owner: &str,
2675 message: impl Into<String>,
2676 non_retryable: bool,
2677 ) -> Result<Value> {
2678 let body = json!({
2679 "activity_attempt_id": activity_attempt_id,
2680 "lease_owner": lease_owner,
2681 "failure": {
2682 "message": message.into(),
2683 "type": "RustActivityFailure",
2684 "non_retryable": non_retryable
2685 }
2686 });
2687 let path = format!("/worker/activity-tasks/{task_id}/fail");
2688 activity_task_response(
2689 self.request_json(
2690 reqwest::Method::POST,
2691 &path,
2692 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2693 Some(&body),
2694 )
2695 .await,
2696 "fail",
2697 task_id,
2698 activity_attempt_id,
2699 )
2700 }
2701
2702 pub async fn heartbeat_activity_task<T: Serialize>(
2703 &self,
2704 task_id: &str,
2705 activity_attempt_id: &str,
2706 lease_owner: &str,
2707 details: T,
2708 ) -> Result<ActivityHeartbeatResponse> {
2709 let details = encode_typed_envelope(&AvroValue::from_serialize(&details)?, DEFAULT_CODEC)?;
2710 let body = json!({
2711 "activity_attempt_id": activity_attempt_id,
2712 "lease_owner": lease_owner,
2713 "details": details
2714 });
2715 let path = format!("/worker/activity-tasks/{task_id}/heartbeat");
2716 activity_task_response(
2717 self.request_json(
2718 reqwest::Method::POST,
2719 &path,
2720 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2721 Some(&body),
2722 )
2723 .await,
2724 "heartbeat",
2725 task_id,
2726 activity_attempt_id,
2727 )
2728 }
2729
2730 async fn request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
2731 &self,
2732 method: reqwest::Method,
2733 path: &str,
2734 protocol: RequestProtocol,
2735 body: Option<&B>,
2736 ) -> Result<T> {
2737 self.request_json_with_timeout(method, path, protocol, body, Duration::from_secs(60))
2738 .await
2739 }
2740
2741 async fn request_json_with_timeout<T: DeserializeOwned, B: Serialize + ?Sized>(
2742 &self,
2743 method: reqwest::Method,
2744 path: &str,
2745 protocol: RequestProtocol,
2746 body: Option<&B>,
2747 timeout: Duration,
2748 ) -> Result<T> {
2749 let auth_token = self.auth_token(protocol)?;
2750 let mut request = self
2751 .http
2752 .request(method, format!("{}/api{}", self.base_url, path))
2753 .timeout(timeout)
2754 .header(reqwest::header::ACCEPT, "application/json")
2755 .header(reqwest::header::CONTENT_TYPE, "application/json")
2756 .header("X-Namespace", &self.namespace);
2757
2758 match protocol {
2759 RequestProtocol::Worker(version) => {
2760 request = request.header("X-Durable-Workflow-Protocol-Version", version);
2761 }
2762 RequestProtocol::ControlPlane => {
2763 request = request.header(
2764 "X-Durable-Workflow-Control-Plane-Version",
2765 CONTROL_PLANE_VERSION,
2766 );
2767 }
2768 }
2769
2770 if let Some(token) = auth_token {
2771 request = request.bearer_auth(token);
2772 }
2773
2774 if let Some(body) = body {
2775 request = request.json(body);
2776 }
2777
2778 let response = request.send().await?;
2779 let status = response.status();
2780 let bytes = response.bytes().await?;
2781
2782 if !status.is_success() {
2783 let body = String::from_utf8_lossy(&bytes).to_string();
2784 if let Some(protocol) = protocol_failure(status, &body) {
2785 return Err(Error::Protocol(protocol));
2786 }
2787 return Err(Error::Http { status, body });
2788 }
2789
2790 if bytes.is_empty() {
2791 return Ok(serde_json::from_value(Value::Null)?);
2792 }
2793
2794 Ok(serde_json::from_slice(&bytes)?)
2795 }
2796
2797 async fn poll_request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
2798 &self,
2799 path: &str,
2800 protocol: RequestProtocol,
2801 body: &B,
2802 timeout: Duration,
2803 max_retries: usize,
2804 ) -> Result<T> {
2805 let mut retries = 0;
2806
2807 loop {
2808 let response = self
2809 .request_json_with_timeout(
2810 reqwest::Method::POST,
2811 path,
2812 protocol,
2813 Some(body),
2814 timeout,
2815 )
2816 .await;
2817
2818 match response {
2819 Err(Error::Transport(_)) if retries < max_retries => retries += 1,
2820 response => return worker_poll_response(response),
2821 }
2822 }
2823 }
2824
2825 fn auth_token(&self, protocol: RequestProtocol) -> Result<Option<&str>> {
2826 match protocol {
2827 RequestProtocol::Worker(_) => {
2828 if let Some(token) = self.worker_token.as_deref().or(self.token.as_deref()) {
2829 return Ok(Some(token));
2830 }
2831 if self.control_token.is_some() {
2832 return Err(Error::MissingRoleCredentials {
2833 role: "worker",
2834 opposite_role: "control",
2835 });
2836 }
2837 Ok(None)
2838 }
2839 RequestProtocol::ControlPlane => {
2840 if let Some(token) = self.control_token.as_deref().or(self.token.as_deref()) {
2841 return Ok(Some(token));
2842 }
2843 if self.worker_token.is_some() {
2844 return Err(Error::MissingRoleCredentials {
2845 role: "control",
2846 opposite_role: "worker",
2847 });
2848 }
2849 Ok(None)
2850 }
2851 }
2852 }
2853}
2854
2855fn query_failure(status: reqwest::StatusCode, raw_body: String) -> QueryFailure {
2856 let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
2857 let reason = body
2858 .get("reason")
2859 .and_then(Value::as_str)
2860 .unwrap_or("query_rejected")
2861 .to_string();
2862 let message = body
2863 .get("message")
2864 .or_else(|| body.get("error"))
2865 .and_then(Value::as_str)
2866 .unwrap_or("workflow query was rejected")
2867 .to_string();
2868
2869 QueryFailure {
2870 status: status.as_u16(),
2871 reason,
2872 message,
2873 body,
2874 }
2875}
2876
2877fn workflow_command_result(
2878 command: WorkflowCommandKind,
2879 data: Value,
2880 workflow_id: &str,
2881 run_id: Option<&str>,
2882) -> WorkflowCommandResult {
2883 WorkflowCommandResult {
2884 command,
2885 workflow_id: data
2886 .get("workflow_id")
2887 .and_then(Value::as_str)
2888 .unwrap_or(workflow_id)
2889 .to_string(),
2890 run_id: data
2891 .get("run_id")
2892 .and_then(Value::as_str)
2893 .or(run_id)
2894 .map(str::to_string),
2895 outcome: data
2896 .get("outcome")
2897 .and_then(Value::as_str)
2898 .map(str::to_string),
2899 reason: data
2900 .get("reason")
2901 .and_then(Value::as_str)
2902 .map(str::to_string),
2903 command_status: data
2904 .get("command_status")
2905 .and_then(Value::as_str)
2906 .map(str::to_string),
2907 raw: data,
2908 }
2909}
2910
2911fn workflow_command_rejection(
2912 command: WorkflowCommandKind,
2913 status: reqwest::StatusCode,
2914 raw_body: String,
2915 workflow_id: &str,
2916 run_id: Option<&str>,
2917) -> WorkflowCommandRejection {
2918 let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
2919 WorkflowCommandRejection {
2920 command,
2921 status: status.as_u16(),
2922 reason: body
2923 .get("reason")
2924 .and_then(Value::as_str)
2925 .unwrap_or("workflow_command_rejected")
2926 .to_string(),
2927 message: body
2928 .get("message")
2929 .or_else(|| body.get("error"))
2930 .and_then(Value::as_str)
2931 .unwrap_or("workflow lifecycle command was rejected")
2932 .to_string(),
2933 workflow_id: body
2934 .get("workflow_id")
2935 .and_then(Value::as_str)
2936 .unwrap_or(workflow_id)
2937 .to_string(),
2938 run_id: body
2939 .get("run_id")
2940 .and_then(Value::as_str)
2941 .or(run_id)
2942 .map(str::to_string),
2943 target_scope: body
2944 .get("target_scope")
2945 .and_then(Value::as_str)
2946 .map(str::to_string),
2947 body,
2948 }
2949}
2950
2951fn query_task_response(response: Result<Value>) -> Result<Value> {
2952 match response {
2953 Err(Error::Http { status, body }) => Err(Error::QueryFailed(query_failure(status, body))),
2954 response => response,
2955 }
2956}
2957
2958fn worker_poll_response<T: DeserializeOwned>(response: Result<T>) -> Result<T> {
2959 match response {
2960 Err(Error::Http { status, body })
2961 if status == reqwest::StatusCode::CONFLICT && worker_poll_body_is_stop(&body) =>
2962 {
2963 Ok(serde_json::from_str(&body)?)
2964 }
2965 response => response,
2966 }
2967}
2968
2969fn worker_poll_body_is_stop(body: &str) -> bool {
2970 serde_json::from_str::<Value>(body)
2971 .ok()
2972 .is_some_and(|body| {
2973 worker_poll_is_stop(
2974 body.get("poll_status").and_then(Value::as_str),
2975 body.get("reason").and_then(Value::as_str),
2976 )
2977 })
2978}
2979
2980fn worker_poll_is_stop(poll_status: Option<&str>, reason: Option<&str>) -> bool {
2981 matches!(poll_status, Some("draining" | "stopped"))
2982 || matches!(reason, Some("worker_draining" | "worker_stopped"))
2983}
2984
2985fn query_task_rejection_is_final(error: &Error) -> bool {
2986 matches!(
2987 error,
2988 Error::QueryFailed(failure)
2989 if QUERY_TASK_FINAL_REJECTION_REASONS.contains(&failure.reason.as_str())
2990 )
2991}
2992
2993fn activity_task_response<T>(
2994 response: Result<T>,
2995 operation: &str,
2996 task_id: &str,
2997 activity_attempt_id: &str,
2998) -> Result<T> {
2999 match response {
3000 Err(Error::Http { status, body }) => {
3001 let body = serde_json::from_str(&body).unwrap_or_else(|_| json!({"message": body}));
3002 Err(Error::ActivityTaskRejected(ActivityTaskRejection {
3003 operation: operation.to_string(),
3004 status: status.as_u16(),
3005 reason: body
3006 .get("reason")
3007 .and_then(Value::as_str)
3008 .unwrap_or("activity_task_rejected")
3009 .to_string(),
3010 task_id: body
3011 .get("task_id")
3012 .and_then(Value::as_str)
3013 .unwrap_or(task_id)
3014 .to_string(),
3015 activity_attempt_id: body
3016 .get("activity_attempt_id")
3017 .and_then(Value::as_str)
3018 .unwrap_or(activity_attempt_id)
3019 .to_string(),
3020 cancel_requested: body
3021 .get("cancel_requested")
3022 .and_then(Value::as_bool)
3023 .unwrap_or(false),
3024 can_continue: body.get("can_continue").and_then(Value::as_bool),
3025 run_closed_reason: body
3026 .get("run_closed_reason")
3027 .and_then(Value::as_str)
3028 .map(str::to_string),
3029 body,
3030 }))
3031 }
3032 response => response,
3033 }
3034}
3035
3036fn activity_task_rejection_is_final(error: &Error) -> bool {
3037 matches!(
3038 error,
3039 Error::ActivityTaskRejected(rejection)
3040 if matches!(
3041 rejection.reason.as_str(),
3042 "run_cancelled"
3043 | "run_terminated"
3044 | "attempt_closed"
3045 | "stale_attempt"
3046 | "activity_cancelled"
3047 | "task_cancelled"
3048 | "run_closed"
3049 | "activity_not_running"
3050 | "attempt_not_found"
3051 )
3052 )
3053}
3054
3055fn workflow_task_completion_is_terminal_timeout(
3056 error: &Error,
3057 task_id: &str,
3058 workflow_task_attempt: u64,
3059 run_id: Option<&str>,
3060) -> bool {
3061 let Error::Http { status, body } = error else {
3062 return false;
3063 };
3064 if *status != reqwest::StatusCode::CONFLICT {
3065 return false;
3066 }
3067
3068 let Some(run_id) = run_id else {
3069 return false;
3070 };
3071 let Ok(body) = serde_json::from_str::<Value>(body) else {
3072 return false;
3073 };
3074
3075 body.get("recorded").and_then(Value::as_bool) == Some(false)
3076 && body.get("reason").and_then(Value::as_str) == Some("run_timed_out")
3077 && body.get("run_status").and_then(Value::as_str) == Some("failed")
3078 && body.get("run_id").and_then(Value::as_str) == Some(run_id)
3079 && body.get("task_id").and_then(Value::as_str) == Some(task_id)
3080 && body.get("workflow_task_attempt").and_then(Value::as_u64) == Some(workflow_task_attempt)
3081}
3082
3083fn protocol_failure(status: reqwest::StatusCode, raw_body: &str) -> Option<ProtocolFailure> {
3084 let body: Value = serde_json::from_str(raw_body).ok()?;
3085 let reason = body.get("reason")?.as_str()?;
3086 if !matches!(
3087 reason,
3088 "missing_protocol_version"
3089 | "unsupported_protocol_version"
3090 | "missing_control_plane_version"
3091 | "unsupported_control_plane_version"
3092 ) {
3093 return None;
3094 }
3095
3096 Some(ProtocolFailure {
3097 status: status.as_u16(),
3098 reason: reason.to_string(),
3099 message: body
3100 .get("message")
3101 .or_else(|| body.get("error"))
3102 .and_then(Value::as_str)
3103 .unwrap_or("protocol version rejected")
3104 .to_string(),
3105 supported_version: body
3106 .get("supported_version")
3107 .and_then(Value::as_str)
3108 .map(str::to_string),
3109 requested_version: body
3110 .get("requested_version")
3111 .and_then(Value::as_str)
3112 .map(str::to_string),
3113 body,
3114 })
3115}
3116
3117fn long_poll_timeout_seconds(timeout: Duration) -> u64 {
3118 timeout
3119 .as_secs()
3120 .saturating_add(u64::from(timeout.subsec_nanos() > 0))
3121 .min(MAX_LONG_POLL_TIMEOUT_SECONDS)
3122}
3123
3124fn worker_operation_is_retryable(error: &Error) -> bool {
3125 match error {
3126 Error::Transport(error) => {
3127 error.is_timeout() || error.is_connect() || error.is_request() || error.is_body()
3128 }
3129 Error::Http { status, .. } => {
3130 matches!(
3131 *status,
3132 reqwest::StatusCode::REQUEST_TIMEOUT | reqwest::StatusCode::TOO_MANY_REQUESTS
3133 ) || status.is_server_error()
3134 }
3135 _ => false,
3136 }
3137}
3138
3139fn worker_retry_delay(policy: WorkerRetryPolicy, retry: usize) -> Duration {
3140 let exponent = retry.saturating_sub(1).min(31) as u32;
3141 policy
3142 .initial_backoff
3143 .saturating_mul(1_u32 << exponent)
3144 .min(policy.max_backoff)
3145}
3146
3147#[derive(Debug)]
3148pub struct ClientBuilder {
3149 base_url: String,
3150 token: Option<String>,
3151 control_token: Option<String>,
3152 worker_token: Option<String>,
3153 namespace: String,
3154 timeout: Duration,
3155}
3156
3157impl ClientBuilder {
3158 pub fn token(mut self, token: Option<String>) -> Self {
3159 self.token = token;
3160 self
3161 }
3162
3163 pub fn control_token(mut self, token: Option<String>) -> Self {
3164 self.control_token = token;
3165 self
3166 }
3167
3168 pub fn worker_token(mut self, token: Option<String>) -> Self {
3169 self.worker_token = token;
3170 self
3171 }
3172
3173 pub fn namespace(mut self, namespace: impl Into<String>) -> Self {
3174 self.namespace = namespace.into();
3175 self
3176 }
3177
3178 pub fn timeout(mut self, timeout: Duration) -> Self {
3179 self.timeout = timeout;
3180 self
3181 }
3182
3183 pub fn build(self) -> Result<Client> {
3184 let base_url = self.base_url.trim_end_matches('/').to_string();
3185 let has_sdk_api_suffix = reqwest::Url::parse(&base_url)
3186 .map(|url| url.path().trim_end_matches('/').ends_with("/api"))
3187 .unwrap_or_else(|_| base_url.ends_with("/api"));
3188
3189 if has_sdk_api_suffix {
3190 return Err(Error::InvalidBaseUrl);
3191 }
3192
3193 Ok(Client {
3194 http: reqwest::Client::builder().timeout(self.timeout).build()?,
3195 base_url,
3196 token: self.token,
3197 control_token: self.control_token,
3198 worker_token: self.worker_token,
3199 namespace: self.namespace,
3200 })
3201 }
3202}
3203
3204#[derive(Clone, Debug)]
3205pub struct WorkflowHandle {
3206 client: Client,
3207 pub workflow_id: String,
3208 pub run_id: Option<String>,
3209 pub workflow_type: String,
3210}
3211
3212impl WorkflowHandle {
3213 pub async fn describe(&self) -> Result<WorkflowDescription> {
3215 self.client.describe_workflow(&self.workflow_id).await
3216 }
3217
3218 pub async fn describe_selected_run(&self) -> Result<WorkflowDescription> {
3220 let run_id = self.run_id.as_deref().ok_or_else(|| {
3221 Error::Codec("run_id is required for selected-run description".to_string())
3222 })?;
3223 self.client
3224 .describe_workflow_run(&self.workflow_id, run_id)
3225 .await
3226 }
3227
3228 pub async fn signal<T: Serialize>(&self, signal_name: &str, input: T) -> Result<Value> {
3229 self.client
3230 .signal_workflow(&self.workflow_id, signal_name, input)
3231 .await
3232 }
3233
3234 pub async fn signal_selected_run<T: Serialize>(
3236 &self,
3237 signal_name: &str,
3238 input: T,
3239 ) -> Result<Value> {
3240 let run_id = self.run_id.as_deref().ok_or_else(|| {
3241 Error::Codec("run_id is required for selected-run signaling".to_string())
3242 })?;
3243 self.client
3244 .signal_workflow_run(&self.workflow_id, run_id, signal_name, input)
3245 .await
3246 }
3247
3248 pub async fn cancel(&self, options: WorkflowCommandOptions) -> Result<WorkflowCommandResult> {
3250 self.client
3251 .cancel_workflow(&self.workflow_id, options)
3252 .await
3253 }
3254
3255 pub async fn cancel_selected_run(
3257 &self,
3258 options: WorkflowCommandOptions,
3259 ) -> Result<WorkflowCommandResult> {
3260 let run_id = self.run_id.as_deref().ok_or_else(|| {
3261 Error::Codec("run_id is required for selected-run cancellation".to_string())
3262 })?;
3263 self.client
3264 .cancel_workflow_run(&self.workflow_id, run_id, options)
3265 .await
3266 }
3267
3268 pub async fn terminate(
3270 &self,
3271 options: WorkflowCommandOptions,
3272 ) -> Result<WorkflowCommandResult> {
3273 self.client
3274 .terminate_workflow(&self.workflow_id, options)
3275 .await
3276 }
3277
3278 pub async fn terminate_selected_run(
3280 &self,
3281 options: WorkflowCommandOptions,
3282 ) -> Result<WorkflowCommandResult> {
3283 let run_id = self.run_id.as_deref().ok_or_else(|| {
3284 Error::Codec("run_id is required for selected-run termination".to_string())
3285 })?;
3286 self.client
3287 .terminate_workflow_run(&self.workflow_id, run_id, options)
3288 .await
3289 }
3290
3291 pub async fn query<T: Serialize>(&self, query_name: &str, input: T) -> Result<Value> {
3293 self.client
3294 .query_workflow(&self.workflow_id, query_name, input)
3295 .await
3296 }
3297
3298 pub async fn query_avro_value<T: Serialize>(
3299 &self,
3300 query_name: &str,
3301 input: T,
3302 ) -> Result<AvroValue> {
3303 self.client
3304 .query_workflow_avro_value(&self.workflow_id, query_name, input)
3305 .await
3306 }
3307
3308 pub async fn update<T: Serialize>(
3309 &self,
3310 update_name: &str,
3311 input: T,
3312 request_id: Option<&str>,
3313 ) -> Result<Value> {
3314 self.client
3315 .update_workflow(&self.workflow_id, update_name, input, request_id)
3316 .await
3317 }
3318
3319 pub async fn update_avro_value<T: Serialize>(
3320 &self,
3321 update_name: &str,
3322 input: T,
3323 request_id: Option<&str>,
3324 ) -> Result<AvroValue> {
3325 self.client
3326 .update_workflow_avro_value(&self.workflow_id, update_name, input, request_id)
3327 .await
3328 }
3329
3330 pub async fn query_selected_run<T: Serialize>(
3332 &self,
3333 query_name: &str,
3334 input: T,
3335 ) -> Result<Value> {
3336 let run_id = self
3337 .run_id
3338 .as_deref()
3339 .ok_or_else(|| Error::Codec("run_id is required for selected-run query".to_string()))?;
3340 self.client
3341 .query_workflow_run(&self.workflow_id, run_id, query_name, input)
3342 .await
3343 }
3344
3345 pub async fn result(&self, options: WorkflowResultOptions) -> Result<Value> {
3347 self.result_target(options, None).await
3348 }
3349
3350 pub async fn result_avro_value(&self, options: WorkflowResultOptions) -> Result<AvroValue> {
3352 self.result_avro_value_target(options, None).await
3353 }
3354
3355 pub async fn result_selected_run(&self, options: WorkflowResultOptions) -> Result<Value> {
3357 let run_id = self.run_id.as_deref().ok_or_else(|| {
3358 Error::Codec("run_id is required for selected-run result".to_string())
3359 })?;
3360 self.result_target(options, Some(run_id)).await
3361 }
3362
3363 pub async fn result_selected_run_avro_value(
3365 &self,
3366 options: WorkflowResultOptions,
3367 ) -> Result<AvroValue> {
3368 let run_id = self.run_id.as_deref().ok_or_else(|| {
3369 Error::Codec("run_id is required for selected-run result".to_string())
3370 })?;
3371 self.result_avro_value_target(options, Some(run_id)).await
3372 }
3373
3374 async fn result_avro_value_target(
3375 &self,
3376 options: WorkflowResultOptions,
3377 selected_run_id: Option<&str>,
3378 ) -> Result<AvroValue> {
3379 let started = Instant::now();
3380
3381 loop {
3382 let description = match selected_run_id {
3383 Some(run_id) => {
3384 self.client
3385 .describe_workflow_run(&self.workflow_id, run_id)
3386 .await?
3387 }
3388 None => self.describe().await?,
3389 };
3390 if description.is_completed() {
3391 return description.output_avro_value.ok_or_else(|| {
3392 Error::Codec(
3393 "missing_payload_envelope: typed workflow result requires output_envelope"
3394 .to_string(),
3395 )
3396 });
3397 }
3398 if description.is_terminal() {
3399 let outcome =
3400 workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
3401 return Err(match outcome.kind {
3402 WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
3403 WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
3404 WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
3405 WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
3406 });
3407 }
3408 if started.elapsed() >= options.timeout {
3409 return Err(Error::Timeout);
3410 }
3411 tokio::time::sleep(options.poll_interval).await;
3412 }
3413 }
3414
3415 async fn result_target(
3416 &self,
3417 options: WorkflowResultOptions,
3418 selected_run_id: Option<&str>,
3419 ) -> Result<Value> {
3420 let started = Instant::now();
3421
3422 loop {
3423 let description = match selected_run_id {
3424 Some(run_id) => {
3425 self.client
3426 .describe_workflow_run(&self.workflow_id, run_id)
3427 .await?
3428 }
3429 None => self.describe().await?,
3430 };
3431 if description.is_completed() {
3432 return Ok(description.output.unwrap_or(Value::Null));
3433 }
3434
3435 if description.is_terminal() {
3436 let outcome =
3437 workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
3438 return Err(match outcome.kind {
3439 WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
3440 WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
3441 WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
3442 WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
3443 });
3444 }
3445
3446 if started.elapsed() >= options.timeout {
3447 return Err(Error::WorkflowTimedOut(WorkflowTerminalOutcome {
3448 kind: WorkflowTerminalKind::TimedOut,
3449 workflow_id: description
3450 .workflow_id
3451 .clone()
3452 .unwrap_or_else(|| self.workflow_id.clone()),
3453 run_id: description
3454 .run_id
3455 .clone()
3456 .or_else(|| selected_run_id.map(str::to_string)),
3457 reason: "result_wait_timeout".to_string(),
3458 failure_category: Some("client_timeout".to_string()),
3459 failure_id: None,
3460 exception_type: None,
3461 exception_class: None,
3462 non_retryable: None,
3463 message: Some(format!(
3464 "workflow result was not terminal within {:?}",
3465 options.timeout
3466 )),
3467 exception: None,
3468 raw: description.raw_value(),
3469 }));
3470 }
3471
3472 tokio::time::sleep(options.poll_interval).await;
3473 }
3474 }
3475}
3476
3477#[derive(Clone, Copy, Debug)]
3478pub struct WorkflowResultOptions {
3479 pub poll_interval: Duration,
3480 pub timeout: Duration,
3481}
3482
3483impl Default for WorkflowResultOptions {
3484 fn default() -> Self {
3485 Self {
3486 poll_interval: Duration::from_millis(500),
3487 timeout: Duration::from_secs(30),
3488 }
3489 }
3490}
3491
3492#[derive(Clone, Debug, Deserialize)]
3493pub struct WorkflowDescription {
3494 pub workflow_id: Option<String>,
3495 pub run_id: Option<String>,
3496 pub workflow_type: Option<String>,
3497 pub status: Option<String>,
3498 #[serde(default)]
3499 pub closed_reason: Option<String>,
3500 #[serde(default)]
3501 pub error: Option<String>,
3502 #[serde(default)]
3503 pub failure: Option<Value>,
3504 #[serde(default)]
3505 pub exception: Option<Value>,
3506 #[serde(default)]
3507 pub failures: Vec<Value>,
3508 #[serde(default)]
3509 pub output: Option<Value>,
3510 #[serde(default)]
3511 pub output_envelope: Option<Value>,
3512 #[serde(skip)]
3513 pub output_avro_value: Option<AvroValue>,
3514 #[serde(flatten)]
3515 pub raw: HashMap<String, Value>,
3516}
3517
3518impl WorkflowDescription {
3519 pub fn is_completed(&self) -> bool {
3520 matches!(self.status.as_deref(), Some("completed" | "Completed"))
3521 }
3522
3523 pub fn is_terminal(&self) -> bool {
3524 matches!(
3525 self.status.as_deref(),
3526 Some(
3527 "completed"
3528 | "Completed"
3529 | "failed"
3530 | "Failed"
3531 | "cancelled"
3532 | "Cancelled"
3533 | "terminated"
3534 | "Terminated"
3535 | "timed_out"
3536 | "TimedOut",
3537 )
3538 )
3539 }
3540
3541 fn decode_payloads(&mut self) -> Result<()> {
3542 if let Some(envelope) = &self.output_envelope {
3543 let value = decode_wire_avro_value(envelope, DEFAULT_CODEC)?;
3544 self.output = Some(value.clone().into_json()?);
3545 self.output_avro_value = Some(value);
3546 }
3547
3548 Ok(())
3549 }
3550
3551 fn raw_value(&self) -> Value {
3552 let mut data = self.raw.clone();
3553 data.insert(
3554 "workflow_id".to_string(),
3555 self.workflow_id
3556 .clone()
3557 .map(Value::String)
3558 .unwrap_or(Value::Null),
3559 );
3560 data.insert(
3561 "run_id".to_string(),
3562 self.run_id
3563 .clone()
3564 .map(Value::String)
3565 .unwrap_or(Value::Null),
3566 );
3567 data.insert(
3568 "workflow_type".to_string(),
3569 self.workflow_type
3570 .clone()
3571 .map(Value::String)
3572 .unwrap_or(Value::Null),
3573 );
3574 data.insert(
3575 "status".to_string(),
3576 self.status
3577 .clone()
3578 .map(Value::String)
3579 .unwrap_or(Value::Null),
3580 );
3581 data.insert(
3582 "closed_reason".to_string(),
3583 self.closed_reason
3584 .clone()
3585 .map(Value::String)
3586 .unwrap_or(Value::Null),
3587 );
3588 if let Some(failure) = &self.failure {
3589 data.insert("failure".to_string(), failure.clone());
3590 }
3591 if let Some(exception) = &self.exception {
3592 data.insert("exception".to_string(), exception.clone());
3593 }
3594 Value::Object(data.into_iter().collect())
3595 }
3596}
3597
3598fn workflow_terminal_outcome(
3599 description: &WorkflowDescription,
3600 workflow_id: &str,
3601 run_id: Option<&str>,
3602) -> WorkflowTerminalOutcome {
3603 let terminal_kind = description
3604 .closed_reason
3605 .as_deref()
3606 .or(description.status.as_deref())
3607 .unwrap_or("failed")
3608 .to_ascii_lowercase();
3609 let kind = match terminal_kind.as_str() {
3610 "cancelled" | "canceled" => WorkflowTerminalKind::Cancelled,
3611 "terminated" => WorkflowTerminalKind::Terminated,
3612 "timed_out" | "timedout" => WorkflowTerminalKind::TimedOut,
3613 _ => WorkflowTerminalKind::Failed,
3614 };
3615 let default_reason = match kind {
3616 WorkflowTerminalKind::Failed => "workflow_failed",
3617 WorkflowTerminalKind::Cancelled => "cancelled",
3618 WorkflowTerminalKind::Terminated => "terminated",
3619 WorkflowTerminalKind::TimedOut => "timed_out",
3620 };
3621 let failure = description
3622 .failure
3623 .as_ref()
3624 .filter(|value| value.is_object());
3625 let nested_failure = failure
3626 .and_then(|value| value.get("failures"))
3627 .and_then(Value::as_array)
3628 .and_then(|failures| failures.last())
3629 .or_else(|| description.failures.last());
3630 let exception = description
3631 .exception
3632 .clone()
3633 .or_else(|| failure.and_then(|value| value.get("exception")).cloned())
3634 .or_else(|| {
3635 nested_failure
3636 .and_then(|value| value.get("exception_payload"))
3637 .cloned()
3638 });
3639 let string_field = |name: &str| {
3640 failure
3641 .and_then(|value| value.get(name))
3642 .and_then(Value::as_str)
3643 .or_else(|| {
3644 nested_failure
3645 .and_then(|value| value.get(name))
3646 .and_then(Value::as_str)
3647 })
3648 .map(str::to_string)
3649 };
3650 let exception_field = |name: &str| {
3651 exception
3652 .as_ref()
3653 .and_then(|value| value.get(name))
3654 .and_then(Value::as_str)
3655 .map(str::to_string)
3656 };
3657 let message = description
3658 .error
3659 .clone()
3660 .or_else(|| string_field("message"))
3661 .or_else(|| exception_field("message"));
3662 let reason = description
3663 .raw
3664 .get("reason")
3665 .and_then(Value::as_str)
3666 .map(str::to_string)
3667 .or_else(|| {
3668 failure
3669 .and_then(|value| value.get("reason"))
3670 .and_then(Value::as_str)
3671 .map(str::to_string)
3672 })
3673 .or_else(|| description.closed_reason.clone())
3674 .unwrap_or_else(|| default_reason.to_string());
3675 let failure_id = string_field("failure_id").or_else(|| {
3676 nested_failure
3677 .and_then(|value| value.get("id"))
3678 .and_then(Value::as_str)
3679 .map(str::to_string)
3680 });
3681
3682 WorkflowTerminalOutcome {
3683 kind,
3684 workflow_id: description
3685 .workflow_id
3686 .clone()
3687 .unwrap_or_else(|| workflow_id.to_string()),
3688 run_id: description
3689 .run_id
3690 .clone()
3691 .or_else(|| run_id.map(str::to_string)),
3692 reason,
3693 failure_category: string_field("failure_category")
3694 .or_else(|| Some(default_reason.to_string())),
3695 failure_id,
3696 exception_type: string_field("exception_type").or_else(|| exception_field("type")),
3697 exception_class: string_field("exception_class").or_else(|| exception_field("class")),
3698 non_retryable: failure
3699 .and_then(|value| value.get("non_retryable"))
3700 .and_then(Value::as_bool)
3701 .or_else(|| {
3702 nested_failure
3703 .and_then(|value| value.get("non_retryable"))
3704 .and_then(Value::as_bool)
3705 }),
3706 message,
3707 exception,
3708 raw: description.raw_value(),
3709 }
3710}
3711
3712#[derive(Clone, Debug, Deserialize)]
3713pub struct RegisterWorkerResponse {
3714 pub worker_id: String,
3715 pub registered: bool,
3716 #[serde(default)]
3717 pub heartbeat_interval_seconds: Option<u64>,
3718 #[serde(default)]
3719 pub protocol_version: Option<String>,
3720 #[serde(default)]
3721 pub server_capabilities: Option<Value>,
3722}
3723
3724#[derive(Clone, Debug, Deserialize, PartialEq, Eq)]
3726pub struct WorkerDeregistrationEnvelope {
3727 pub worker_id: String,
3728 pub outcome: String,
3729 pub recovered_workflow_task_count: u64,
3730}
3731
3732#[derive(Clone, Debug, Deserialize)]
3733pub struct PollWorkflowTaskResponse {
3734 #[serde(default)]
3735 pub task: Option<WorkflowTask>,
3736 #[serde(default)]
3737 pub poll_status: Option<String>,
3738 #[serde(default)]
3739 pub reason: Option<String>,
3740 #[serde(default)]
3741 pub protocol_version: Option<String>,
3742 #[serde(default)]
3743 pub server_capabilities: Option<Value>,
3744}
3745
3746impl PollWorkflowTaskResponse {
3747 pub fn outcome(&self) -> WorkerPollOutcome {
3749 worker_poll_outcome(
3750 self.task.is_some(),
3751 self.poll_status.as_deref(),
3752 self.reason.as_deref(),
3753 )
3754 }
3755}
3756
3757#[derive(Clone, Debug, Deserialize)]
3758pub struct PollActivityTaskResponse {
3759 #[serde(default)]
3760 pub task: Option<ActivityTask>,
3761 #[serde(default)]
3762 pub poll_status: Option<String>,
3763 #[serde(default)]
3764 pub reason: Option<String>,
3765}
3766
3767impl PollActivityTaskResponse {
3768 pub fn outcome(&self) -> WorkerPollOutcome {
3770 worker_poll_outcome(
3771 self.task.is_some(),
3772 self.poll_status.as_deref(),
3773 self.reason.as_deref(),
3774 )
3775 }
3776}
3777
3778#[derive(Clone, Debug, Deserialize)]
3779pub struct PollQueryTaskResponse {
3780 #[serde(default)]
3781 pub task: Option<QueryTask>,
3782 #[serde(default)]
3783 pub poll_status: Option<String>,
3784 #[serde(default)]
3785 pub reason: Option<String>,
3786}
3787
3788impl PollQueryTaskResponse {
3789 pub fn outcome(&self) -> WorkerPollOutcome {
3791 worker_poll_outcome(
3792 self.task.is_some(),
3793 self.poll_status.as_deref(),
3794 self.reason.as_deref(),
3795 )
3796 }
3797}
3798
3799#[derive(Clone, Debug, PartialEq, Eq)]
3801pub enum WorkerPollOutcome {
3802 Task,
3804 Idle {
3806 poll_status: Option<String>,
3807 reason: Option<String>,
3808 },
3809 Stop {
3811 poll_status: Option<String>,
3812 reason: Option<String>,
3813 },
3814}
3815
3816impl WorkerPollOutcome {
3817 pub fn should_stop(&self) -> bool {
3818 matches!(self, Self::Stop { .. })
3819 }
3820}
3821
3822fn worker_poll_outcome(
3823 has_task: bool,
3824 poll_status: Option<&str>,
3825 reason: Option<&str>,
3826) -> WorkerPollOutcome {
3827 if worker_poll_is_stop(poll_status, reason) {
3828 return WorkerPollOutcome::Stop {
3829 poll_status: poll_status.map(str::to_string),
3830 reason: reason.map(str::to_string),
3831 };
3832 }
3833
3834 if has_task {
3835 WorkerPollOutcome::Task
3836 } else {
3837 WorkerPollOutcome::Idle {
3838 poll_status: poll_status.map(str::to_string),
3839 reason: reason.map(str::to_string),
3840 }
3841 }
3842}
3843
3844#[derive(Clone, Debug, Deserialize)]
3846pub struct QueryTask {
3847 pub query_task_id: String,
3848 #[serde(default = "default_workflow_task_attempt")]
3849 pub query_task_attempt: u64,
3850 #[serde(default)]
3851 pub lease_owner: Option<String>,
3852 #[serde(default)]
3853 pub workflow_id: Option<String>,
3854 #[serde(default)]
3855 pub run_id: Option<String>,
3856 pub workflow_type: String,
3857 pub query_name: String,
3858 #[serde(default = "default_payload_codec")]
3859 pub payload_codec: String,
3860 #[serde(default)]
3861 pub workflow_arguments: Option<Value>,
3862 #[serde(default)]
3863 pub query_arguments: Option<Value>,
3864 #[serde(default)]
3865 pub history_events: Vec<HistoryEvent>,
3866 #[serde(default)]
3867 pub history_export: Option<Value>,
3868 #[serde(default)]
3869 pub run_status: Option<String>,
3870}
3871
3872#[derive(Clone, Debug, Deserialize)]
3873pub struct WorkflowTask {
3874 pub task_id: String,
3875 #[serde(default)]
3876 pub workflow_id: Option<String>,
3877 #[serde(default)]
3878 pub run_id: Option<String>,
3879 pub workflow_type: String,
3880 #[serde(default = "default_payload_codec")]
3881 pub payload_codec: String,
3882 #[serde(default)]
3883 pub arguments: Option<Value>,
3884 #[serde(default)]
3885 pub history_events: Vec<HistoryEvent>,
3886 #[serde(default)]
3887 pub total_history_events: Option<u64>,
3888 #[serde(default)]
3889 pub history_size_bytes: Option<u64>,
3890 #[serde(default)]
3891 pub continue_as_new_recommended: Option<bool>,
3892 #[serde(default)]
3893 pub history_budget_pressure: Option<String>,
3894 #[serde(default)]
3895 pub next_history_page_token: Option<String>,
3896 #[serde(default = "default_workflow_task_attempt")]
3897 pub workflow_task_attempt: u64,
3898 #[serde(default)]
3899 pub workflow_signal_id: Option<String>,
3900 #[serde(default)]
3901 pub signal_name: Option<String>,
3902 #[serde(default)]
3903 pub signal_arguments: Option<Value>,
3904 #[serde(default)]
3905 pub workflow_update_id: Option<String>,
3906 #[serde(default)]
3907 pub update_name: Option<String>,
3908 #[serde(default)]
3909 pub lease_owner: Option<String>,
3910}
3911
3912impl WorkflowTask {
3913 fn append_history_page(&mut self, page: WorkflowTaskHistoryPage) {
3914 self.history_events.extend(page.history_events);
3915
3916 if page.total_history_events.is_some() {
3917 self.total_history_events = page.total_history_events;
3918 }
3919
3920 self.next_history_page_token = page
3921 .next_history_page_token
3922 .filter(|token| !token.is_empty());
3923 }
3924}
3925
3926#[derive(Clone, Debug, Deserialize)]
3927struct WorkflowTaskHistoryPage {
3928 #[serde(default)]
3929 history_events: Vec<HistoryEvent>,
3930 #[serde(default)]
3931 total_history_events: Option<u64>,
3932 #[serde(default)]
3933 next_history_page_token: Option<String>,
3934}
3935
3936#[derive(Clone, Debug, Deserialize)]
3937pub struct ActivityTask {
3938 pub task_id: String,
3939 #[serde(default)]
3940 pub activity_attempt_id: Option<String>,
3941 #[serde(default)]
3942 pub attempt_id: Option<String>,
3943 pub activity_type: String,
3944 #[serde(default = "default_payload_codec")]
3945 pub payload_codec: String,
3946 #[serde(default)]
3947 pub arguments: Option<Value>,
3948 #[serde(default = "default_attempt_number")]
3949 pub attempt_number: u64,
3950 #[serde(default)]
3951 pub lease_owner: Option<String>,
3952}
3953
3954#[derive(Clone, Debug, Deserialize)]
3955pub struct HistoryEvent {
3956 #[serde(alias = "type")]
3957 pub event_type: String,
3958 #[serde(default)]
3959 pub payload: Value,
3960 #[serde(flatten)]
3961 pub raw: HashMap<String, Value>,
3962}
3963
3964#[derive(Clone, Debug, PartialEq)]
3966pub struct QuerySignal {
3967 pub id: Option<String>,
3968 pub name: String,
3969 pub arguments: Vec<Value>,
3970 avro_arguments: Vec<AvroValue>,
3971 pub workflow_sequence: Option<u64>,
3972}
3973
3974impl QuerySignal {
3975 pub fn arguments_avro_value(&self) -> &[AvroValue] {
3977 &self.avro_arguments
3978 }
3979}
3980
3981#[derive(Clone, Debug)]
3987pub struct QueryContext {
3988 pub workflow_id: Option<String>,
3989 pub run_id: Option<String>,
3990 pub workflow_type: String,
3991 pub run_status: Option<String>,
3992 workflow_input: Value,
3993 workflow_input_avro_value: AvroValue,
3994 history_events: Arc<Vec<HistoryEvent>>,
3995 signal_events: Arc<Vec<QuerySignal>>,
3996}
3997
3998impl QueryContext {
3999 pub fn workflow_input(&self) -> &Value {
4001 &self.workflow_input
4002 }
4003
4004 pub fn workflow_input_avro_value(&self) -> &AvroValue {
4006 &self.workflow_input_avro_value
4007 }
4008
4009 pub fn history_events(&self) -> &[HistoryEvent] {
4011 self.history_events.as_slice()
4012 }
4013
4014 pub fn signal_events(&self) -> &[QuerySignal] {
4016 self.signal_events.as_slice()
4017 }
4018
4019 pub fn signals(&self, signal_name: &str) -> Vec<Vec<Value>> {
4021 self.signal_events
4022 .iter()
4023 .filter(|signal| signal.name == signal_name)
4024 .map(|signal| signal.arguments.clone())
4025 .collect()
4026 }
4027
4028 pub fn signals_avro_value(&self, signal_name: &str) -> Vec<Vec<AvroValue>> {
4030 self.signal_events
4031 .iter()
4032 .filter(|signal| signal.name == signal_name)
4033 .map(|signal| signal.avro_arguments.clone())
4034 .collect()
4035 }
4036}
4037
4038#[derive(Clone, Debug, Deserialize)]
4039pub struct ActivityHeartbeatResponse {
4040 #[serde(default)]
4041 pub cancel_requested: bool,
4042 #[serde(default)]
4043 pub heartbeat_recorded: bool,
4044 #[serde(default)]
4045 pub can_continue: Option<bool>,
4046 #[serde(default)]
4047 pub reason: Option<String>,
4048 #[serde(default)]
4049 pub run_closed_reason: Option<String>,
4050 #[serde(default)]
4051 pub run_closed_at: Option<String>,
4052 #[serde(default)]
4053 pub lease_expires_at: Option<String>,
4054 #[serde(default)]
4055 pub last_heartbeat_at: Option<String>,
4056}
4057
4058impl ActivityHeartbeatResponse {
4059 pub fn should_stop(&self) -> bool {
4061 self.cancel_requested || self.can_continue == Some(false)
4062 }
4063}
4064
4065fn default_payload_codec() -> String {
4066 DEFAULT_CODEC.to_string()
4067}
4068
4069fn default_workflow_task_attempt() -> u64 {
4070 1
4071}
4072
4073fn default_attempt_number() -> u64 {
4074 1
4075}
4076
4077type WorkflowFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
4078type WorkflowHandler = Arc<dyn Fn(WorkflowContext, AvroValue) -> WorkflowFuture + Send + Sync>;
4079type ErasedWorkflowState = Arc<dyn Any + Send + Sync>;
4080type WorkflowStateSnapshot = Arc<dyn Fn() -> Result<ErasedWorkflowState> + Send + Sync>;
4081type ReplayedWorkflowHandler =
4082 Arc<dyn Fn(WorkflowContext, AvroValue) -> ReplayedWorkflowInvocation + Send + Sync>;
4083type ActivityFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
4084type ActivityHandler = Arc<dyn Fn(ActivityContext, AvroValue) -> ActivityFuture + Send + Sync>;
4085type QueryFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
4086type QueryHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
4087type UpdateHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
4088type ReplayedQueryHandler = Arc<
4089 dyn Fn(QueryContext, ErasedWorkflowState, AvroValue) -> std::result::Result<QueryFuture, String>
4090 + Send
4091 + Sync,
4092>;
4093type WorkerHeartbeatObserver = Arc<dyn Fn(&WorkerHeartbeatObservation) + Send + Sync>;
4094
4095struct ReplayedWorkflowInvocation {
4096 future: WorkflowFuture,
4097 snapshot: WorkflowStateSnapshot,
4098}
4099
4100#[derive(Clone)]
4101struct RegisteredWorkflow {
4102 execute: WorkflowHandler,
4103 replay: Option<ReplayedWorkflowHandler>,
4104 state_type: Option<TypeId>,
4105}
4106
4107#[derive(Clone)]
4108enum RegisteredQuery {
4109 Snapshot(QueryHandler),
4110 Replayed {
4111 state_type: TypeId,
4112 handler: ReplayedQueryHandler,
4113 },
4114}
4115
4116#[derive(Clone, Debug)]
4117pub struct WorkerHeartbeatObservation {
4118 pub worker_id: String,
4119 pub task_queue: String,
4120 pub acknowledged_at_unix_millis: u64,
4121 pub acknowledgement: Value,
4122}
4123
4124#[derive(Clone, Copy, Debug)]
4131pub struct WorkerRetryPolicy {
4132 pub max_retries: usize,
4134 pub initial_backoff: Duration,
4136 pub max_backoff: Duration,
4138}
4139
4140impl Default for WorkerRetryPolicy {
4141 fn default() -> Self {
4142 Self {
4143 max_retries: 5,
4144 initial_backoff: Duration::from_millis(100),
4145 max_backoff: Duration::from_secs(5),
4146 }
4147 }
4148}
4149
4150#[derive(Clone, Copy, Debug, PartialEq, Eq)]
4151enum ManagedPollOutcome {
4152 Idle,
4153 Handled,
4154 Stop,
4155}
4156
4157#[derive(Clone)]
4158pub struct Worker {
4159 client: Client,
4160 worker_id: String,
4161 task_queue: String,
4162 workflows: HashMap<String, RegisteredWorkflow>,
4163 activities: HashMap<String, ActivityHandler>,
4164 queries: HashMap<String, HashMap<String, RegisteredQuery>>,
4165 updates: HashMap<String, HashMap<String, UpdateHandler>>,
4166 max_concurrent_workflow_tasks: usize,
4167 max_concurrent_activity_tasks: usize,
4168 poll_timeout: Duration,
4169 heartbeat_interval: Duration,
4170 retry_policy: WorkerRetryPolicy,
4171 heartbeat_observer: Option<WorkerHeartbeatObserver>,
4172}
4173
4174impl Worker {
4175 pub fn new(client: Client, task_queue: impl Into<String>) -> Self {
4176 Self {
4177 client,
4178 worker_id: default_worker_id(),
4179 task_queue: task_queue.into(),
4180 workflows: HashMap::new(),
4181 activities: HashMap::new(),
4182 queries: HashMap::new(),
4183 updates: HashMap::new(),
4184 max_concurrent_workflow_tasks: 10,
4185 max_concurrent_activity_tasks: 10,
4186 poll_timeout: Duration::from_secs(30),
4187 heartbeat_interval: Duration::from_secs(60),
4188 retry_policy: WorkerRetryPolicy::default(),
4189 heartbeat_observer: None,
4190 }
4191 }
4192
4193 pub fn worker_id(mut self, worker_id: impl Into<String>) -> Self {
4194 self.worker_id = worker_id.into();
4195 self
4196 }
4197
4198 pub fn poll_timeout(mut self, timeout: Duration) -> Self {
4199 self.poll_timeout = timeout;
4200 self
4201 }
4202
4203 pub fn heartbeat_interval(mut self, interval: Duration) -> Self {
4204 self.heartbeat_interval = interval;
4205 self
4206 }
4207
4208 pub fn retry_policy(mut self, policy: WorkerRetryPolicy) -> Self {
4210 self.retry_policy = policy;
4211 self
4212 }
4213
4214 pub fn on_worker_heartbeat<F>(mut self, observer: F) -> Self
4215 where
4216 F: Fn(&WorkerHeartbeatObservation) + Send + Sync + 'static,
4217 {
4218 self.heartbeat_observer = Some(Arc::new(observer));
4219 self
4220 }
4221
4222 pub fn max_concurrent_workflow_tasks(mut self, count: usize) -> Self {
4223 self.max_concurrent_workflow_tasks = count.max(1);
4224 self
4225 }
4226
4227 pub fn max_concurrent_activity_tasks(mut self, count: usize) -> Self {
4228 self.max_concurrent_activity_tasks = count.max(1);
4229 self
4230 }
4231
4232 pub fn register_workflow<F, Fut>(&mut self, workflow_type: impl Into<String>, handler: F)
4239 where
4240 F: Fn(WorkflowContext, Value) -> Fut + Send + Sync + 'static,
4241 Fut: Future<Output = Result<Value>> + Send + 'static,
4242 {
4243 let handler = Arc::new(handler);
4244 self.workflows.insert(
4245 workflow_type.into(),
4246 RegisteredWorkflow {
4247 execute: Arc::new(move |ctx, input| {
4248 let handler = Arc::clone(&handler);
4249 Box::pin(async move {
4250 let result = handler(ctx, input.into_json()?).await?;
4251 AvroValue::from_serialize(&result)
4252 })
4253 }),
4254 replay: None,
4255 state_type: None,
4256 },
4257 );
4258 }
4259
4260 pub fn register_workflow_avro_value<F, Fut>(
4262 &mut self,
4263 workflow_type: impl Into<String>,
4264 handler: F,
4265 ) where
4266 F: Fn(WorkflowContext, AvroValue) -> Fut + Send + Sync + 'static,
4267 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4268 {
4269 self.workflows.insert(
4270 workflow_type.into(),
4271 RegisteredWorkflow {
4272 execute: Arc::new(move |ctx, input| Box::pin(handler(ctx, input))),
4273 replay: None,
4274 state_type: None,
4275 },
4276 );
4277 }
4278
4279 pub fn register_replayed_workflow<S, Factory, F, Fut>(
4287 &mut self,
4288 workflow_type: impl Into<String>,
4289 state_factory: Factory,
4290 handler: F,
4291 ) where
4292 S: Clone + Send + Sync + 'static,
4293 Factory: Fn() -> S + Send + Sync + 'static,
4294 F: Fn(WorkflowContext, Value, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
4295 Fut: Future<Output = Result<Value>> + Send + 'static,
4296 {
4297 let state_factory = Arc::new(state_factory);
4298 let handler = Arc::new(handler);
4299
4300 let execute_factory = Arc::clone(&state_factory);
4301 let execute_handler = Arc::clone(&handler);
4302 let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4303 let state = WorkflowInstance::new(execute_factory());
4304 let handler = Arc::clone(&execute_handler);
4305 Box::pin(async move {
4306 let result = handler(ctx, input.into_json()?, state).await?;
4307 AvroValue::from_serialize(&result)
4308 }) as WorkflowFuture
4309 });
4310
4311 let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4312 let state = WorkflowInstance::new(state_factory());
4313 let snapshot_state = state.clone();
4314 let snapshot: WorkflowStateSnapshot =
4315 Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
4316 let replay_handler = Arc::clone(&handler);
4317 let future = async move {
4318 let result = replay_handler(ctx, input.into_json()?, state).await?;
4319 AvroValue::from_serialize(&result)
4320 };
4321 ReplayedWorkflowInvocation {
4322 future: Box::pin(future),
4323 snapshot,
4324 }
4325 });
4326
4327 self.workflows.insert(
4328 workflow_type.into(),
4329 RegisteredWorkflow {
4330 execute,
4331 replay: Some(replay),
4332 state_type: Some(TypeId::of::<S>()),
4333 },
4334 );
4335 }
4336
4337 pub fn register_replayed_workflow_avro_value<S, Factory, F, Fut>(
4339 &mut self,
4340 workflow_type: impl Into<String>,
4341 state_factory: Factory,
4342 handler: F,
4343 ) where
4344 S: Clone + Send + Sync + 'static,
4345 Factory: Fn() -> S + Send + Sync + 'static,
4346 F: Fn(WorkflowContext, AvroValue, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
4347 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4348 {
4349 let state_factory = Arc::new(state_factory);
4350 let handler = Arc::new(handler);
4351
4352 let execute_factory = Arc::clone(&state_factory);
4353 let execute_handler = Arc::clone(&handler);
4354 let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4355 let state = WorkflowInstance::new(execute_factory());
4356 Box::pin(execute_handler(ctx, input, state)) as WorkflowFuture
4357 });
4358
4359 let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4360 let state = WorkflowInstance::new(state_factory());
4361 let snapshot_state = state.clone();
4362 let snapshot: WorkflowStateSnapshot =
4363 Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
4364 ReplayedWorkflowInvocation {
4365 future: Box::pin(handler(ctx, input, state)),
4366 snapshot,
4367 }
4368 });
4369
4370 self.workflows.insert(
4371 workflow_type.into(),
4372 RegisteredWorkflow {
4373 execute,
4374 replay: Some(replay),
4375 state_type: Some(TypeId::of::<S>()),
4376 },
4377 );
4378 }
4379
4380 pub fn register_activity<F, Fut>(&mut self, activity_type: impl Into<String>, handler: F)
4381 where
4382 F: Fn(ActivityContext, Value) -> Fut + Send + Sync + 'static,
4383 Fut: Future<Output = Result<Value>> + Send + 'static,
4384 {
4385 let handler = Arc::new(handler);
4386 self.activities.insert(
4387 activity_type.into(),
4388 Arc::new(move |ctx, args| {
4389 let handler = Arc::clone(&handler);
4390 Box::pin(async move {
4391 let result = handler(ctx, args.into_json()?).await?;
4392 AvroValue::from_serialize(&result)
4393 })
4394 }),
4395 );
4396 }
4397
4398 pub fn register_activity_avro_value<F, Fut>(
4400 &mut self,
4401 activity_type: impl Into<String>,
4402 handler: F,
4403 ) where
4404 F: Fn(ActivityContext, AvroValue) -> Fut + Send + Sync + 'static,
4405 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4406 {
4407 self.activities.insert(
4408 activity_type.into(),
4409 Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
4410 );
4411 }
4412
4413 pub fn register_query<F, Fut>(
4419 &mut self,
4420 workflow_type: impl Into<String>,
4421 query_name: impl Into<String>,
4422 handler: F,
4423 ) where
4424 F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
4425 Fut: Future<Output = Result<Value>> + Send + 'static,
4426 {
4427 let handler = Arc::new(handler);
4428 self.queries
4429 .entry(workflow_type.into())
4430 .or_default()
4431 .insert(
4432 query_name.into(),
4433 RegisteredQuery::Snapshot(Arc::new(move |ctx, args| {
4434 let handler = Arc::clone(&handler);
4435 Box::pin(async move {
4436 let result = handler(ctx, args.into_json()?).await?;
4437 AvroValue::from_serialize(&result)
4438 })
4439 })),
4440 );
4441 }
4442
4443 pub fn register_query_avro_value<F, Fut>(
4445 &mut self,
4446 workflow_type: impl Into<String>,
4447 query_name: impl Into<String>,
4448 handler: F,
4449 ) where
4450 F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
4451 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4452 {
4453 self.queries
4454 .entry(workflow_type.into())
4455 .or_default()
4456 .insert(
4457 query_name.into(),
4458 RegisteredQuery::Snapshot(Arc::new(move |ctx, args| Box::pin(handler(ctx, args)))),
4459 );
4460 }
4461
4462 pub fn register_replayed_query<S, F, Fut>(
4469 &mut self,
4470 workflow_type: impl Into<String>,
4471 query_name: impl Into<String>,
4472 handler: F,
4473 ) where
4474 S: Clone + Send + Sync + 'static,
4475 F: Fn(QueryContext, Arc<S>, Value) -> Fut + Send + Sync + 'static,
4476 Fut: Future<Output = Result<Value>> + Send + 'static,
4477 {
4478 let handler = Arc::new(handler);
4479 let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
4480 let state = state.downcast::<S>().map_err(|_| {
4481 "registered query state type does not match the replayed workflow state".to_string()
4482 })?;
4483 let handler = Arc::clone(&handler);
4484 Ok(Box::pin(async move {
4485 let result = handler(ctx, state, args.into_json()?).await?;
4486 AvroValue::from_serialize(&result)
4487 }))
4488 });
4489
4490 self.queries
4491 .entry(workflow_type.into())
4492 .or_default()
4493 .insert(
4494 query_name.into(),
4495 RegisteredQuery::Replayed {
4496 state_type: TypeId::of::<S>(),
4497 handler: erased_handler,
4498 },
4499 );
4500 }
4501
4502 pub fn register_replayed_query_avro_value<S, F, Fut>(
4504 &mut self,
4505 workflow_type: impl Into<String>,
4506 query_name: impl Into<String>,
4507 handler: F,
4508 ) where
4509 S: Clone + Send + Sync + 'static,
4510 F: Fn(QueryContext, Arc<S>, AvroValue) -> Fut + Send + Sync + 'static,
4511 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4512 {
4513 let handler = Arc::new(handler);
4514 let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
4515 let state = state.downcast::<S>().map_err(|_| {
4516 "registered query state type does not match the replayed workflow state".to_string()
4517 })?;
4518 Ok(Box::pin(handler(ctx, state, args)))
4519 });
4520
4521 self.queries
4522 .entry(workflow_type.into())
4523 .or_default()
4524 .insert(
4525 query_name.into(),
4526 RegisteredQuery::Replayed {
4527 state_type: TypeId::of::<S>(),
4528 handler: erased_handler,
4529 },
4530 );
4531 }
4532
4533 pub fn register_update<F, Fut>(
4535 &mut self,
4536 workflow_type: impl Into<String>,
4537 update_name: impl Into<String>,
4538 handler: F,
4539 ) where
4540 F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
4541 Fut: Future<Output = Result<Value>> + Send + 'static,
4542 {
4543 let handler = Arc::new(handler);
4544 self.updates
4545 .entry(workflow_type.into())
4546 .or_default()
4547 .insert(
4548 update_name.into(),
4549 Arc::new(move |ctx, args| {
4550 let handler = Arc::clone(&handler);
4551 Box::pin(async move {
4552 let result = handler(ctx, args.into_json()?).await?;
4553 AvroValue::from_serialize(&result)
4554 })
4555 }),
4556 );
4557 }
4558
4559 pub fn register_update_avro_value<F, Fut>(
4561 &mut self,
4562 workflow_type: impl Into<String>,
4563 update_name: impl Into<String>,
4564 handler: F,
4565 ) where
4566 F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
4567 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4568 {
4569 self.updates
4570 .entry(workflow_type.into())
4571 .or_default()
4572 .insert(
4573 update_name.into(),
4574 Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
4575 );
4576 }
4577
4578 pub async fn register(&self) -> Result<RegisterWorkerResponse> {
4579 let mut command_contracts = serde_json::Map::new();
4580 for workflow_type in self.workflows.keys() {
4581 let mut queries = self
4582 .queries
4583 .get(workflow_type)
4584 .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
4585 .unwrap_or_default();
4586 queries.sort();
4587 let mut updates = self
4588 .updates
4589 .get(workflow_type)
4590 .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
4591 .unwrap_or_default();
4592 updates.sort();
4593 if !queries.is_empty() || !updates.is_empty() {
4594 command_contracts.insert(
4595 workflow_type.clone(),
4596 json!({
4597 "queries": queries,
4598 "updates": updates,
4599 "update_validators": [],
4600 }),
4601 );
4602 }
4603 }
4604
4605 self.client
4606 .register_worker_with_command_contracts(
4607 &self.worker_id,
4608 &self.task_queue,
4609 self.workflows.keys().cloned().collect(),
4610 self.activities.keys().cloned().collect(),
4611 self.max_concurrent_workflow_tasks,
4612 self.max_concurrent_activity_tasks,
4613 [
4614 (!self.queries.is_empty()).then(|| QUERY_TASKS_CAPABILITY.to_string()),
4615 (!self.updates.is_empty()).then(|| WORKFLOW_UPDATES_CAPABILITY.to_string()),
4616 ]
4617 .into_iter()
4618 .flatten()
4619 .collect(),
4620 Value::Object(command_contracts),
4621 )
4622 .await
4623 }
4624
4625 pub async fn run(&self) -> Result<()> {
4631 self.run_until(std::future::pending::<()>()).await
4632 }
4633
4634 pub async fn run_until<F>(&self, shutdown: F) -> Result<()>
4638 where
4639 F: Future<Output = ()>,
4640 {
4641 let registration = self.register().await?;
4642 if !registration.registered {
4643 return Err(Error::WorkerLoop(format!(
4644 "worker registration for {:?} was not accepted",
4645 self.worker_id
4646 )));
4647 }
4648 let registered_worker_id = registration.worker_id.clone();
4649 let primary = self.run_registered_until(shutdown, registration).await;
4650 let deregistration = self
4651 .client
4652 .deregister_worker_registration(®istered_worker_id)
4653 .await;
4654
4655 match (primary, deregistration) {
4656 (Ok(()), Ok(_)) => Ok(()),
4657 (Ok(()), Err(deregistration)) => Err(deregistration),
4658 (Err(primary), Ok(_)) => Err(primary),
4659 (Err(primary), Err(deregistration)) => Err(Error::WorkerShutdown {
4660 primary: Box::new(primary),
4661 deregistration: Box::new(deregistration),
4662 }),
4663 }
4664 }
4665
4666 async fn run_registered_until<F>(
4667 &self,
4668 shutdown: F,
4669 registration: RegisterWorkerResponse,
4670 ) -> Result<()>
4671 where
4672 F: Future<Output = ()>,
4673 {
4674 let heartbeat_interval = Duration::from_secs(
4675 registration
4676 .heartbeat_interval_seconds
4677 .unwrap_or(self.heartbeat_interval.as_secs().max(1)),
4678 );
4679 let heartbeat = tokio::time::sleep(Duration::ZERO);
4685 tokio::pin!(heartbeat);
4686 tokio::pin!(shutdown);
4687 let stop = Arc::new(AtomicBool::new(false));
4688 let mut workflow_poller = (!self.workflows.is_empty()).then(|| {
4692 let worker = self.clone();
4693 let stop = Arc::clone(&stop);
4694 tokio::spawn(async move { worker.poll_workflows_until_stopped(stop).await })
4695 });
4696 let mut activity_poller = (!self.activities.is_empty()).then(|| {
4697 let worker = self.clone();
4698 let stop = Arc::clone(&stop);
4699 tokio::spawn(async move { worker.poll_activities_until_stopped(stop).await })
4700 });
4701 let mut query_poller = (!self.queries.is_empty()).then(|| {
4702 let worker = self.clone();
4703 let stop = Arc::clone(&stop);
4704 tokio::spawn(async move { worker.poll_queries_until_stopped(stop).await })
4705 });
4706
4707 loop {
4708 tokio::select! {
4709 _ = &mut shutdown => {
4710 stop.store(true, Ordering::SeqCst);
4711 break;
4712 }
4713 _ = &mut heartbeat => {
4714 let result = self.retry_worker_operation(|| {
4715 self.client.heartbeat_worker(
4716 &self.worker_id,
4717 self.max_concurrent_workflow_tasks,
4718 self.max_concurrent_activity_tasks,
4719 )
4720 }).await;
4721 heartbeat
4722 .as_mut()
4723 .reset(tokio::time::Instant::now() + heartbeat_interval);
4724 match result {
4725 Ok(acknowledgement) => {
4726 if let Some(observer) = &self.heartbeat_observer {
4727 observer(&WorkerHeartbeatObservation {
4728 worker_id: self.worker_id.clone(),
4729 task_queue: self.task_queue.clone(),
4730 acknowledged_at_unix_millis: SystemTime::now()
4731 .duration_since(UNIX_EPOCH)
4732 .unwrap_or_default()
4733 .as_millis()
4734 .min(u64::MAX as u128)
4735 as u64,
4736 acknowledgement,
4737 });
4738 }
4739 }
4740 Err(error) => {
4741 stop.store(true, Ordering::SeqCst);
4742 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await?;
4743 return Err(error);
4744 }
4745 }
4746 }
4747 result = OptionFuture::from(workflow_poller.as_mut()), if workflow_poller.is_some() => {
4748 workflow_poller = None;
4749 let stopped_by_server = stop.load(Ordering::SeqCst);
4750 stop.store(true, Ordering::SeqCst);
4751 let poller_result = optional_poller_result("workflow", result);
4752 let join_result =
4753 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
4754 poller_result?;
4755 join_result?;
4756 if stopped_by_server {
4757 return Ok(());
4758 }
4759 return Err(Error::WorkerLoop(
4760 "workflow poller stopped unexpectedly".to_string(),
4761 ));
4762 }
4763 result = OptionFuture::from(activity_poller.as_mut()), if activity_poller.is_some() => {
4764 activity_poller = None;
4765 let stopped_by_server = stop.load(Ordering::SeqCst);
4766 stop.store(true, Ordering::SeqCst);
4767 let poller_result = optional_poller_result("activity", result);
4768 let join_result =
4769 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
4770 poller_result?;
4771 join_result?;
4772 if stopped_by_server {
4773 return Ok(());
4774 }
4775 return Err(Error::WorkerLoop(
4776 "activity poller stopped unexpectedly".to_string(),
4777 ));
4778 }
4779 result = OptionFuture::from(query_poller.as_mut()), if query_poller.is_some() => {
4780 query_poller = None;
4781 let stopped_by_server = stop.load(Ordering::SeqCst);
4782 stop.store(true, Ordering::SeqCst);
4783 let poller_result = optional_poller_result("query", result);
4784 let join_result =
4785 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
4786 poller_result?;
4787 join_result?;
4788 if stopped_by_server {
4789 return Ok(());
4790 }
4791 return Err(Error::WorkerLoop(
4792 "query poller stopped unexpectedly".to_string(),
4793 ));
4794 }
4795 }
4796 }
4797
4798 join_pollers(
4799 workflow_poller.take(),
4800 activity_poller.take(),
4801 query_poller.take(),
4802 )
4803 .await
4804 }
4805
4806 pub async fn run_once(&self) -> Result<usize> {
4820 let mut handled = 0;
4821 match self.poll_workflow_once().await? {
4822 ManagedPollOutcome::Handled => handled += 1,
4823 ManagedPollOutcome::Stop => return Ok(handled),
4824 ManagedPollOutcome::Idle => {}
4825 }
4826 match self.poll_activity_once().await? {
4827 ManagedPollOutcome::Handled => handled += 1,
4828 ManagedPollOutcome::Stop => return Ok(handled),
4829 ManagedPollOutcome::Idle => {}
4830 }
4831 if !self.queries.is_empty() {
4832 match self.poll_query_once().await? {
4833 ManagedPollOutcome::Handled => handled += 1,
4834 ManagedPollOutcome::Stop => return Ok(handled),
4835 ManagedPollOutcome::Idle => {}
4836 }
4837 }
4838 Ok(handled)
4839 }
4840
4841 async fn poll_workflow_once(&self) -> Result<ManagedPollOutcome> {
4842 let poll_request_id = unique_request_id("rust-workflow-poll");
4843 let response = self
4844 .retry_worker_operation(|| {
4845 self.client.poll_workflow_task_response_with_request_id(
4846 &self.worker_id,
4847 &self.task_queue,
4848 self.poll_timeout,
4849 &poll_request_id,
4850 0,
4851 )
4852 })
4853 .await?;
4854 if response.outcome().should_stop() {
4855 return Ok(ManagedPollOutcome::Stop);
4856 }
4857 let Some(task) = response.task else {
4858 return Ok(ManagedPollOutcome::Idle);
4859 };
4860
4861 let task_id = task.task_id.clone();
4862 let attempt = task.workflow_task_attempt;
4863 let run_id = task.run_id.clone();
4864 let lease_owner = task
4865 .lease_owner
4866 .clone()
4867 .unwrap_or_else(|| self.worker_id.clone());
4868
4869 match self.execute_workflow_task(task) {
4870 Ok(commands) if commands.is_empty() => {
4871 self.client
4877 .fail_workflow_task_with_type(
4878 &task_id,
4879 &lease_owner,
4880 attempt,
4881 WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE,
4882 WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE,
4883 )
4884 .await?;
4885 }
4886 Ok(commands) => {
4887 let completion = self
4888 .client
4889 .complete_workflow_task(&task_id, &lease_owner, attempt, commands)
4890 .await;
4891 if let Err(error) = completion {
4892 if !workflow_task_completion_is_terminal_timeout(
4893 &error,
4894 &task_id,
4895 attempt,
4896 run_id.as_deref(),
4897 ) {
4898 return Err(error);
4899 }
4900 }
4901 }
4902 Err(error) => {
4903 self.client
4904 .fail_workflow_task(&task_id, &lease_owner, attempt, error.to_string())
4905 .await?;
4906 }
4907 }
4908
4909 Ok(ManagedPollOutcome::Handled)
4910 }
4911
4912 async fn poll_workflows_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
4913 while !stop.load(Ordering::SeqCst) {
4914 if self.poll_workflow_once().await? == ManagedPollOutcome::Stop {
4915 stop.store(true, Ordering::SeqCst);
4916 break;
4917 }
4918 }
4919
4920 Ok(())
4921 }
4922
4923 async fn poll_activity_once(&self) -> Result<ManagedPollOutcome> {
4924 let poll_request_id = unique_request_id("rust-activity-poll");
4925 let response = self
4926 .retry_worker_operation(|| {
4927 self.client.poll_activity_task_response_with_request_id(
4928 &self.worker_id,
4929 &self.task_queue,
4930 self.poll_timeout,
4931 &poll_request_id,
4932 0,
4933 )
4934 })
4935 .await?;
4936 if response.outcome().should_stop() {
4937 return Ok(ManagedPollOutcome::Stop);
4938 }
4939 let Some(task) = response.task else {
4940 return Ok(ManagedPollOutcome::Idle);
4941 };
4942
4943 let task_id = task.task_id.clone();
4944 let attempt_id = task
4945 .activity_attempt_id
4946 .clone()
4947 .or(task.attempt_id.clone())
4948 .unwrap_or_default();
4949 let lease_owner = task
4950 .lease_owner
4951 .clone()
4952 .unwrap_or_else(|| self.worker_id.clone());
4953 let codec = task.payload_codec.clone();
4954 let result = self.execute_activity_task(task).await;
4955 match result {
4956 Ok(value) => {
4957 let completion = self
4958 .client
4959 .complete_activity_task(&task_id, &attempt_id, &lease_owner, value, &codec)
4960 .await;
4961 if let Err(error) = completion {
4962 if !activity_task_rejection_is_final(&error) {
4963 return Err(error);
4964 }
4965 }
4966 }
4967 Err(error) => {
4968 let failure = self
4969 .client
4970 .fail_activity_task(
4971 &task_id,
4972 &attempt_id,
4973 &lease_owner,
4974 error.to_string(),
4975 false,
4976 )
4977 .await;
4978 if let Err(error) = failure {
4979 if !activity_task_rejection_is_final(&error) {
4980 return Err(error);
4981 }
4982 }
4983 }
4984 }
4985
4986 Ok(ManagedPollOutcome::Handled)
4987 }
4988
4989 async fn poll_activities_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
4990 while !stop.load(Ordering::SeqCst) {
4991 if self.poll_activity_once().await? == ManagedPollOutcome::Stop {
4992 stop.store(true, Ordering::SeqCst);
4993 break;
4994 }
4995 }
4996
4997 Ok(())
4998 }
4999
5000 async fn poll_query_once(&self) -> Result<ManagedPollOutcome> {
5001 let poll_request_id = unique_request_id("rust-query-poll");
5002 let response = self
5003 .retry_worker_operation(|| {
5004 self.client.poll_query_task_response_with_request_id(
5005 &self.worker_id,
5006 &self.task_queue,
5007 self.poll_timeout,
5008 &poll_request_id,
5009 0,
5010 )
5011 })
5012 .await?;
5013 if response.outcome().should_stop() {
5014 return Ok(ManagedPollOutcome::Stop);
5015 }
5016 let Some(task) = response.task else {
5017 return Ok(ManagedPollOutcome::Idle);
5018 };
5019
5020 let query_task_id = task.query_task_id.clone();
5021 let attempt = task.query_task_attempt;
5022 let lease_owner = task
5023 .lease_owner
5024 .clone()
5025 .unwrap_or_else(|| self.worker_id.clone());
5026 let codec = task.payload_codec.clone();
5027
5028 match self.execute_query_task(task).await {
5029 Ok(value) => {
5030 let result_envelope = match encode_typed_envelope(&value, &codec) {
5031 Ok(result_envelope) => result_envelope,
5032 Err(error) => {
5033 let failure = self
5034 .client
5035 .fail_query_task(
5036 &query_task_id,
5037 &lease_owner,
5038 attempt,
5039 error.to_string(),
5040 "query_result_encode_failed",
5041 "QueryResultEncodeFailed",
5042 )
5043 .await;
5044 if let Err(error) = failure {
5045 if !query_task_rejection_is_final(&error) {
5046 return Err(error);
5047 }
5048 }
5049 return Ok(ManagedPollOutcome::Handled);
5050 }
5051 };
5052
5053 if let Err(error) = self
5054 .client
5055 .complete_query_task_with_envelope(
5056 &query_task_id,
5057 &lease_owner,
5058 attempt,
5059 value.clone().into_json()?,
5060 result_envelope,
5061 )
5062 .await
5063 {
5064 if !query_task_rejection_is_final(&error) {
5065 return Err(error);
5066 }
5067 }
5068 }
5069 Err(failure) => {
5070 let result = self
5071 .client
5072 .fail_query_task(
5073 &query_task_id,
5074 &lease_owner,
5075 attempt,
5076 failure.message,
5077 failure.reason,
5078 failure.failure_type,
5079 )
5080 .await;
5081 if let Err(error) = result {
5082 if !query_task_rejection_is_final(&error) {
5083 return Err(error);
5084 }
5085 }
5086 }
5087 }
5088
5089 Ok(ManagedPollOutcome::Handled)
5090 }
5091
5092 async fn poll_queries_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
5093 while !stop.load(Ordering::SeqCst) {
5094 if self.poll_query_once().await? == ManagedPollOutcome::Stop {
5095 stop.store(true, Ordering::SeqCst);
5096 break;
5097 }
5098 }
5099
5100 Ok(())
5101 }
5102
5103 async fn retry_worker_operation<T, F, Fut>(&self, mut operation: F) -> Result<T>
5104 where
5105 F: FnMut() -> Fut,
5106 Fut: Future<Output = Result<T>>,
5107 {
5108 let mut retries = 0;
5109
5110 loop {
5111 match operation().await {
5112 Err(error)
5113 if worker_operation_is_retryable(&error)
5114 && retries < self.retry_policy.max_retries =>
5115 {
5116 retries += 1;
5117 tokio::time::sleep(worker_retry_delay(self.retry_policy, retries)).await;
5118 }
5119 result => return result,
5120 }
5121 }
5122 }
5123
5124 async fn execute_query_task(
5125 &self,
5126 mut task: QueryTask,
5127 ) -> std::result::Result<AvroValue, QueryTaskExecutionFailure> {
5128 validate_query_task_payloads(&task).map_err(|error| {
5129 QueryTaskExecutionFailure::new(
5130 "query_payload_decode_failed",
5131 error.to_string(),
5132 "QueryPayloadDecodeFailed",
5133 )
5134 })?;
5135
5136 if !self.workflows.contains_key(&task.workflow_type) {
5137 return Err(QueryTaskExecutionFailure::new(
5138 "query_workflow_type_not_registered",
5139 format!("no workflow registered for type {:?}", task.workflow_type),
5140 "WorkflowTypeNotRegistered",
5141 ));
5142 }
5143
5144 let Some(handlers) = self.queries.get(&task.workflow_type) else {
5145 return Err(QueryTaskExecutionFailure::new(
5146 "query_handler_unavailable",
5147 format!(
5148 "query handlers are unavailable for workflow type {:?}",
5149 task.workflow_type
5150 ),
5151 "QueryHandlerUnavailable",
5152 ));
5153 };
5154 let Some(query) = handlers.get(&task.query_name) else {
5155 return Err(QueryTaskExecutionFailure::new(
5156 "rejected_unknown_query",
5157 format!("unknown query {:?}", task.query_name),
5158 "QueryFailed",
5159 ));
5160 };
5161
5162 let args = decode_task_avro_arguments(task.query_arguments.as_ref(), &task.payload_codec)
5163 .map_err(|error| {
5164 QueryTaskExecutionFailure::new(
5165 "query_payload_decode_failed",
5166 format!("cannot decode query arguments: {error}"),
5167 "QueryPayloadDecodeFailed",
5168 )
5169 })?;
5170 let workflow_input_typed =
5171 decode_task_avro_arguments(task.workflow_arguments.as_ref(), &task.payload_codec)
5172 .map_err(|error| {
5173 QueryTaskExecutionFailure::new(
5174 "query_workflow_state_unavailable",
5175 format!("cannot decode workflow start input: {error}"),
5176 "QueryWorkflowStateUnavailable",
5177 )
5178 })?;
5179 let workflow_input = workflow_input_typed.clone().into_json().map_err(|error| {
5180 QueryTaskExecutionFailure::new(
5181 "query_workflow_state_unavailable",
5182 format!("cannot project workflow start input: {error}"),
5183 "QueryWorkflowStateUnavailable",
5184 )
5185 })?;
5186 hydrate_query_history_from_export(&mut task).map_err(|error| {
5187 QueryTaskExecutionFailure::new(
5188 "query_workflow_state_unavailable",
5189 format!("cannot restore query history snapshot: {error}"),
5190 "QueryWorkflowStateUnavailable",
5191 )
5192 })?;
5193 enrich_query_history_from_export(&mut task).map_err(|error| {
5194 QueryTaskExecutionFailure::new(
5195 "query_workflow_state_unavailable",
5196 format!("cannot restore compact query history payloads: {error}"),
5197 "QueryWorkflowStateUnavailable",
5198 )
5199 })?;
5200 let signal_events = query_signal_events(&task).map_err(|error| {
5201 QueryTaskExecutionFailure::new(
5202 "query_workflow_state_unavailable",
5203 format!("cannot decode committed workflow signals: {error}"),
5204 "QueryWorkflowStateUnavailable",
5205 )
5206 })?;
5207 let history_events = Arc::new(std::mem::take(&mut task.history_events));
5208 let context = QueryContext {
5209 workflow_id: task.workflow_id,
5210 run_id: task.run_id,
5211 workflow_type: task.workflow_type.clone(),
5212 run_status: task.run_status,
5213 workflow_input,
5214 workflow_input_avro_value: workflow_input_typed.clone(),
5215 history_events: Arc::clone(&history_events),
5216 signal_events: Arc::new(signal_events),
5217 };
5218
5219 let future = match query {
5220 RegisteredQuery::Snapshot(handler) => handler(context, args),
5221 RegisteredQuery::Replayed {
5222 state_type,
5223 handler,
5224 } => {
5225 let workflow = self
5226 .workflows
5227 .get(&task.workflow_type)
5228 .expect("workflow registration was checked above");
5229 if workflow.state_type != Some(*state_type) {
5230 return Err(QueryTaskExecutionFailure::new(
5231 "query_workflow_state_unavailable",
5232 "replayed query state type does not match its workflow registration",
5233 "QueryWorkflowStateUnavailable",
5234 ));
5235 }
5236 let replay = workflow.replay.as_ref().ok_or_else(|| {
5237 QueryTaskExecutionFailure::new(
5238 "query_workflow_state_unavailable",
5239 format!(
5240 "workflow type {:?} is not registered for instance-state replay",
5241 task.workflow_type
5242 ),
5243 "QueryWorkflowStateUnavailable",
5244 )
5245 })?;
5246 let workflow_state = Arc::new(Mutex::new(
5247 WorkflowState::new_with_identity(
5248 history_events.as_ref().clone(),
5249 context.workflow_id.clone(),
5250 context.run_id.clone(),
5251 self.task_queue.clone(),
5252 task.payload_codec,
5253 None,
5254 )
5255 .map_err(|error| {
5256 QueryTaskExecutionFailure::new(
5257 "query_workflow_state_unavailable",
5258 format!("workflow replay failed before query: {error}"),
5259 "QueryWorkflowStateUnavailable",
5260 )
5261 })?,
5262 ));
5263 let workflow_context = WorkflowContext {
5264 state: workflow_state,
5265 };
5266 let mut invocation = replay(workflow_context.clone(), workflow_input_typed.clone());
5267 let mut cx = TaskContext::from_waker(noop_waker_ref());
5268 match invocation.future.as_mut().poll(&mut cx) {
5269 Poll::Ready(Ok(_)) => {
5270 workflow_context
5271 .ensure_history_consumed()
5272 .map_err(|error| {
5273 QueryTaskExecutionFailure::new(
5274 "query_workflow_state_unavailable",
5275 format!("workflow replay failed before query: {error}"),
5276 "QueryWorkflowStateUnavailable",
5277 )
5278 })?;
5279 }
5280 Poll::Ready(Err(error)) => {
5281 return Err(QueryTaskExecutionFailure::new(
5282 "query_workflow_state_unavailable",
5283 format!("workflow replay failed before query: {error}"),
5284 "QueryWorkflowStateUnavailable",
5285 ));
5286 }
5287 Poll::Pending => {
5288 let commands = workflow_context.take_commands().map_err(|error| {
5289 QueryTaskExecutionFailure::new(
5290 "query_workflow_state_unavailable",
5291 format!("workflow replay failed before query: {error}"),
5292 "QueryWorkflowStateUnavailable",
5293 )
5294 })?;
5295 if commands.is_empty()
5296 && !workflow_context
5297 .matched_recorded_pending()
5298 .map_err(|error| {
5299 QueryTaskExecutionFailure::new(
5300 "query_workflow_state_unavailable",
5301 format!("workflow replay failed before query: {error}"),
5302 "QueryWorkflowStateUnavailable",
5303 )
5304 })?
5305 {
5306 return Err(QueryTaskExecutionFailure::new(
5307 "query_workflow_state_unavailable",
5308 "workflow replay yielded without a durable command",
5309 "QueryWorkflowStateUnavailable",
5310 ));
5311 }
5312 }
5313 }
5314 let state = (invocation.snapshot)().map_err(|error| {
5315 QueryTaskExecutionFailure::new(
5316 "query_workflow_state_unavailable",
5317 format!("cannot snapshot replayed workflow state: {error}"),
5318 "QueryWorkflowStateUnavailable",
5319 )
5320 })?;
5321 handler(context, state, args).map_err(|message| {
5322 QueryTaskExecutionFailure::new(
5323 "query_workflow_state_unavailable",
5324 message,
5325 "QueryWorkflowStateUnavailable",
5326 )
5327 })?
5328 }
5329 };
5330
5331 future.await.map_err(|error| {
5332 QueryTaskExecutionFailure::new("query_rejected", error.to_string(), "QueryFailed")
5333 })
5334 }
5335
5336 fn execute_workflow_task(&self, task: WorkflowTask) -> Result<Vec<Value>> {
5337 validate_workflow_task_payloads(&task)?;
5338
5339 if let Some(update_id) = task
5340 .workflow_update_id
5341 .as_deref()
5342 .filter(|update_id| !update_id.is_empty())
5343 {
5344 return self.execute_update_task(&task, update_id);
5345 }
5346
5347 let workflow = self
5348 .workflows
5349 .get(&task.workflow_type)
5350 .ok_or_else(|| Error::WorkflowNotRegistered(task.workflow_type.clone()))?;
5351 let input = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
5352 let resume_signal = decode_resume_signal(&task)?;
5353 let history_budget = WorkflowHistoryBudget {
5354 event_count: task
5355 .total_history_events
5356 .unwrap_or_else(|| u64::try_from(task.history_events.len()).unwrap_or(u64::MAX)),
5357 size_bytes: task.history_size_bytes,
5358 continue_as_new_recommended: task.continue_as_new_recommended.unwrap_or(false),
5359 pressure: task.history_budget_pressure.clone(),
5360 };
5361 let mut workflow_state = WorkflowState::new_with_identity(
5362 task.history_events,
5363 task.workflow_id,
5364 task.run_id,
5365 self.task_queue.clone(),
5366 task.payload_codec.clone(),
5367 resume_signal,
5368 )?;
5369 workflow_state.history_budget = history_budget;
5370 let state = Arc::new(Mutex::new(workflow_state));
5371 let ctx = WorkflowContext { state };
5372 let mut future = (workflow.execute)(ctx.clone(), input);
5373 let mut cx = TaskContext::from_waker(noop_waker_ref());
5374
5375 match future.as_mut().poll(&mut cx) {
5376 Poll::Ready(Ok(result)) => {
5377 ctx.ensure_history_consumed()?;
5378 let result = encode_typed_envelope(&result, &task.payload_codec)?;
5379 let mut commands = ctx.take_commands()?;
5380 commands.push(json!({
5381 "type": "complete_workflow",
5382 "result": result
5383 }));
5384 Ok(commands)
5385 }
5386 Poll::Ready(Err(error)) => {
5387 if let Error::ContinueAsNew(request) = error {
5388 let mut commands = ctx.take_commands()?;
5389 if let Some(command) = ctx.continue_as_new_command(request)? {
5390 commands.push(command);
5391 }
5392 ctx.ensure_history_consumed()?;
5393 return Ok(commands);
5394 }
5395 ctx.ensure_history_consumed()?;
5398 if workflow_task_integrity_error(&error) {
5399 return Err(error);
5404 }
5405 let mut commands = ctx.take_commands()?;
5406 commands.push(workflow_failure_command(&error));
5407 Ok(commands)
5408 }
5409 Poll::Pending => {
5410 let commands = ctx.take_commands()?;
5411 if commands.is_empty() && !ctx.matched_recorded_pending()? {
5412 Err(Error::WorkflowYieldedWithoutCommand)
5413 } else {
5414 Ok(commands)
5415 }
5416 }
5417 }
5418 }
5419
5420 fn execute_update_task(&self, task: &WorkflowTask, update_id: &str) -> Result<Vec<Value>> {
5421 if !self.workflows.contains_key(&task.workflow_type) {
5422 return Err(Error::WorkflowNotRegistered(task.workflow_type.clone()));
5423 }
5424
5425 let accepted = task.history_events.iter().rev().find_map(|event| {
5426 (event.event_type == "UpdateAccepted"
5427 && event.payload.get("update_id").and_then(Value::as_str) == Some(update_id))
5428 .then_some(&event.payload)
5429 });
5430 let update_name = accepted
5431 .and_then(|payload| payload.get("update_name"))
5432 .and_then(Value::as_str)
5433 .or(task.update_name.as_deref())
5434 .unwrap_or_default();
5435 let Some(handler) = self
5436 .updates
5437 .get(&task.workflow_type)
5438 .and_then(|handlers| handlers.get(update_name))
5439 else {
5440 return Ok(vec![json!({
5441 "type": "fail_update",
5442 "update_id": update_id,
5443 "message": format!(
5444 "no update handler is registered for {}.{update_name}",
5445 task.workflow_type
5446 ),
5447 "exception_type": "UnknownUpdate",
5448 "non_retryable": true,
5449 })]);
5450 };
5451 let arguments = accepted
5452 .and_then(|payload| payload.get("arguments"))
5453 .or(task.arguments.as_ref());
5454 let arguments = decode_task_avro_arguments(arguments, &task.payload_codec)?;
5455 let context = QueryContext {
5456 workflow_id: task.workflow_id.clone(),
5457 run_id: task.run_id.clone(),
5458 workflow_type: task.workflow_type.clone(),
5459 run_status: Some("running".to_string()),
5460 workflow_input: Value::Null,
5461 workflow_input_avro_value: AvroValue::Null,
5462 history_events: Arc::new(task.history_events.clone()),
5463 signal_events: Arc::new(Vec::new()),
5464 };
5465 let mut future = handler(context, arguments);
5466 let mut cx = TaskContext::from_waker(noop_waker_ref());
5467
5468 match future.as_mut().poll(&mut cx) {
5469 Poll::Ready(Ok(result)) => Ok(vec![json!({
5470 "type": "complete_update",
5471 "update_id": update_id,
5472 "result": encode_typed_envelope(&result, &task.payload_codec)?,
5473 })]),
5474 Poll::Ready(Err(error)) => Ok(vec![json!({
5475 "type": "fail_update",
5476 "update_id": update_id,
5477 "message": error.to_string(),
5478 "exception_type": "UpdateFailed",
5479 "non_retryable": true,
5480 })]),
5481 Poll::Pending => Err(Error::WorkflowYieldedWithoutCommand),
5482 }
5483 }
5484
5485 async fn execute_activity_task(&self, task: ActivityTask) -> Result<AvroValue> {
5486 validate_activity_task_payloads(&task)?;
5487
5488 let handler = self
5489 .activities
5490 .get(&task.activity_type)
5491 .ok_or_else(|| Error::ActivityNotRegistered(task.activity_type.clone()))?;
5492 let args = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
5493 let attempt_id = task
5494 .activity_attempt_id
5495 .clone()
5496 .or(task.attempt_id.clone())
5497 .unwrap_or_default();
5498 let lease_owner = task
5499 .lease_owner
5500 .clone()
5501 .unwrap_or_else(|| self.worker_id.clone());
5502 let ctx = ActivityContext {
5503 client: self.client.clone(),
5504 task_id: task.task_id,
5505 activity_attempt_id: attempt_id,
5506 lease_owner,
5507 activity_type: task.activity_type,
5508 attempt_number: task.attempt_number,
5509 task_queue: self.task_queue.clone(),
5510 worker_id: self.worker_id.clone(),
5511 };
5512
5513 handler(ctx, args).await
5514 }
5515}
5516
5517fn poller_result(
5518 kind: &str,
5519 result: std::result::Result<Result<()>, tokio::task::JoinError>,
5520) -> Result<()> {
5521 match result {
5522 Ok(result) => result,
5523 Err(error) => Err(Error::WorkerLoop(format!(
5524 "{kind} poller join error: {error}"
5525 ))),
5526 }
5527}
5528
5529fn optional_poller_result(
5530 kind: &str,
5531 result: Option<std::result::Result<Result<()>, tokio::task::JoinError>>,
5532) -> Result<()> {
5533 match result {
5534 Some(result) => poller_result(kind, result),
5535 None => Ok(()),
5536 }
5537}
5538
5539async fn join_pollers(
5540 workflow_poller: Option<tokio::task::JoinHandle<Result<()>>>,
5541 activity_poller: Option<tokio::task::JoinHandle<Result<()>>>,
5542 query_poller: Option<tokio::task::JoinHandle<Result<()>>>,
5543) -> Result<()> {
5544 let mut first_error = None;
5545
5546 if let Some(handle) = workflow_poller {
5547 if let Err(error) = poller_result("workflow", handle.await) {
5548 first_error.get_or_insert(error);
5549 }
5550 }
5551
5552 if let Some(handle) = activity_poller {
5553 if let Err(error) = poller_result("activity", handle.await) {
5554 first_error.get_or_insert(error);
5555 }
5556 }
5557
5558 if let Some(handle) = query_poller {
5559 if let Err(error) = poller_result("query", handle.await) {
5560 first_error.get_or_insert(error);
5561 }
5562 }
5563
5564 if let Some(error) = first_error {
5565 Err(error)
5566 } else {
5567 Ok(())
5568 }
5569}
5570
5571fn default_worker_id() -> String {
5572 let millis = SystemTime::now()
5573 .duration_since(UNIX_EPOCH)
5574 .unwrap_or_default()
5575 .as_millis();
5576 format!("rust-worker-{}-{millis}", std::process::id())
5577}
5578
5579fn percent_encode_path_segment(segment: &str) -> String {
5580 const HEX: &[u8; 16] = b"0123456789ABCDEF";
5581 let mut encoded = String::with_capacity(segment.len());
5582
5583 for byte in segment.bytes() {
5584 if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') {
5585 encoded.push(char::from(byte));
5586 } else {
5587 encoded.push('%');
5588 encoded.push(char::from(HEX[(byte >> 4) as usize]));
5589 encoded.push(char::from(HEX[(byte & 0x0f) as usize]));
5590 }
5591 }
5592
5593 encoded
5594}
5595
5596fn unique_request_id(prefix: &str) -> String {
5597 let nanos = SystemTime::now()
5598 .duration_since(UNIX_EPOCH)
5599 .unwrap_or_default()
5600 .as_nanos();
5601 format!("{prefix}-{}-{nanos}", std::process::id())
5602}
5603
5604#[derive(Debug)]
5605struct QueryTaskExecutionFailure {
5606 reason: String,
5607 message: String,
5608 failure_type: String,
5609}
5610
5611impl QueryTaskExecutionFailure {
5612 fn new(
5613 reason: impl Into<String>,
5614 message: impl Into<String>,
5615 failure_type: impl Into<String>,
5616 ) -> Self {
5617 Self {
5618 reason: reason.into(),
5619 message: message.into(),
5620 failure_type: failure_type.into(),
5621 }
5622 }
5623}
5624
5625#[derive(Clone, Debug)]
5631pub struct WorkflowInstance<S> {
5632 state: Arc<Mutex<S>>,
5633}
5634
5635impl<S> WorkflowInstance<S> {
5636 fn new(state: S) -> Self {
5637 Self {
5638 state: Arc::new(Mutex::new(state)),
5639 }
5640 }
5641
5642 pub fn read<R>(&self, reader: impl FnOnce(&S) -> R) -> Result<R> {
5644 let state = self
5645 .state
5646 .lock()
5647 .map_err(|_| Error::WorkflowStatePoisoned)?;
5648 Ok(reader(&state))
5649 }
5650
5651 pub fn update<R>(&self, transition: impl FnOnce(&mut S) -> R) -> Result<R> {
5653 let mut state = self
5654 .state
5655 .lock()
5656 .map_err(|_| Error::WorkflowStatePoisoned)?;
5657 Ok(transition(&mut state))
5658 }
5659}
5660
5661impl<S: Clone> WorkflowInstance<S> {
5662 fn snapshot(&self) -> Result<S> {
5663 self.read(Clone::clone)
5664 }
5665}
5666
5667#[derive(Clone, Debug)]
5668pub struct WorkflowContext {
5669 state: Arc<Mutex<WorkflowState>>,
5670}
5671
5672impl WorkflowContext {
5673 pub fn workflow_identity(&self) -> Result<WorkflowIdentity> {
5675 let state = self
5676 .state
5677 .lock()
5678 .map_err(|_| Error::WorkflowStatePoisoned)?;
5679 Ok(WorkflowIdentity {
5680 workflow_id: state.workflow_id.clone(),
5681 run_id: state.run_id.clone(),
5682 })
5683 }
5684
5685 pub fn history_budget(&self) -> Result<WorkflowHistoryBudget> {
5687 let state = self
5688 .state
5689 .lock()
5690 .map_err(|_| Error::WorkflowStatePoisoned)?;
5691 Ok(state.history_budget.clone())
5692 }
5693
5694 pub fn continue_as_new<T: Serialize>(&self, args: T) -> Result<Value> {
5700 self.continue_as_new_with_options(ContinueAsNewOptions::new(), args)
5701 }
5702
5703 pub fn continue_as_new_with_options<T: Serialize>(
5705 &self,
5706 options: ContinueAsNewOptions,
5707 args: T,
5708 ) -> Result<Value> {
5709 options.validate()?;
5710 Err(Error::ContinueAsNew(ContinueAsNewRequest {
5711 arguments: normalize_avro_arguments(AvroValue::from_serialize(&args)?),
5712 options,
5713 }))
5714 }
5715
5716 pub fn activity<T: Serialize>(
5717 &self,
5718 activity_type: impl Into<String>,
5719 args: T,
5720 ) -> ActivityCall {
5721 self.activity_with_options(activity_type, ActivityOptions::new(), args)
5722 }
5723
5724 pub fn activity_on_queue<T, Q>(
5725 &self,
5726 activity_type: impl Into<String>,
5727 task_queue: Option<Q>,
5728 args: T,
5729 ) -> ActivityCall
5730 where
5731 T: Serialize,
5732 Q: Into<String>,
5733 {
5734 let mut options = ActivityOptions::new();
5735 options.task_queue = task_queue.map(Into::into);
5736 self.activity_with_options(activity_type, options, args)
5737 }
5738
5739 pub fn activity_with_options<T: Serialize>(
5777 &self,
5778 activity_type: impl Into<String>,
5779 options: ActivityOptions,
5780 args: T,
5781 ) -> ActivityCall {
5782 ActivityCall {
5783 ctx: self.clone(),
5784 activity_type: activity_type.into(),
5785 options,
5786 args: Some(AvroValue::from_serialize(&args)),
5787 scheduled: false,
5788 }
5789 }
5790
5791 pub async fn activity_avro_value<T: Serialize>(
5792 &self,
5793 activity_type: impl Into<String>,
5794 args: T,
5795 ) -> Result<AvroValue> {
5796 let mut call = self.activity(activity_type, args);
5797 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
5798 }
5799
5800 pub async fn activity_avro_value_with_options<T: Serialize>(
5801 &self,
5802 activity_type: impl Into<String>,
5803 options: ActivityOptions,
5804 args: T,
5805 ) -> Result<AvroValue> {
5806 let mut call = self.activity_with_options(activity_type, options, args);
5807 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
5808 }
5809
5810 pub fn wait_signal(&self, signal_name: impl Into<String>) -> SignalCall {
5811 SignalCall {
5812 ctx: self.clone(),
5813 signal_name: signal_name.into(),
5814 opened_wait: false,
5815 matched_pending: false,
5816 }
5817 }
5818
5819 pub async fn wait_signal_avro_value(
5820 &self,
5821 signal_name: impl Into<String>,
5822 ) -> Result<Vec<AvroValue>> {
5823 let mut call = self.wait_signal(signal_name);
5824 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
5825 }
5826
5827 pub fn sleep(&self, duration: Duration) -> TimerCall {
5848 let delay_seconds = duration
5849 .as_secs()
5850 .checked_add(u64::from(duration.subsec_nanos() > 0));
5851 TimerCall {
5852 ctx: self.clone(),
5853 delay_seconds,
5854 scheduled: false,
5855 matched_pending: false,
5856 }
5857 }
5858
5859 pub fn start_timer(&self, duration: Duration) -> TimerCall {
5861 self.sleep(duration)
5862 }
5863
5864 pub fn side_effect<T, F>(&self, callback: F) -> Result<T>
5871 where
5872 T: Serialize + DeserializeOwned,
5873 F: FnOnce() -> T,
5874 {
5875 {
5876 let mut state = self
5877 .state
5878 .lock()
5879 .map_err(|_| Error::WorkflowStatePoisoned)?;
5880 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
5881 return match recorded {
5882 RecordedCommand::SideEffect { sequence, value } => {
5883 state.command_cursor += 1;
5884 value.deserialize().map_err(|error| {
5885 Error::NonDeterministicReplay(ReplayFailure::new(
5886 "side_effect_type_mismatch",
5887 Some(sequence),
5888 Some(std::any::type_name::<T>().to_string()),
5889 Some(error.to_string()),
5890 "recorded side-effect value is incompatible with the requested Rust type",
5891 ))
5892 })
5893 }
5894 other => Err(command_mismatch(&other, "side effect")),
5895 };
5896 }
5897 }
5898
5899 let value = callback();
5900 let avro_value = AvroValue::from_serialize(&value)?;
5901 let mut state = self
5902 .state
5903 .lock()
5904 .map_err(|_| Error::WorkflowStatePoisoned)?;
5905 let result = encode_typed_envelope(&avro_value, &state.payload_codec)?;
5906 state.commands.push(json!({
5907 "type": "record_side_effect",
5908 "result": result,
5909 }));
5910 Ok(value)
5911 }
5912
5913 pub fn side_effect_avro_value<F>(&self, callback: F) -> Result<AvroValue>
5915 where
5916 F: FnOnce() -> AvroValue,
5917 {
5918 {
5919 let mut state = self
5920 .state
5921 .lock()
5922 .map_err(|_| Error::WorkflowStatePoisoned)?;
5923 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
5924 return match recorded {
5925 RecordedCommand::SideEffect { value, .. } => {
5926 state.command_cursor += 1;
5927 Ok(value)
5928 }
5929 other => Err(command_mismatch(&other, "side effect")),
5930 };
5931 }
5932 }
5933
5934 let value = callback();
5935 let mut state = self
5936 .state
5937 .lock()
5938 .map_err(|_| Error::WorkflowStatePoisoned)?;
5939 let result = encode_typed_envelope(&value, &state.payload_codec)?;
5940 state.commands.push(json!({
5941 "type": "record_side_effect",
5942 "result": result,
5943 }));
5944 Ok(value)
5945 }
5946
5947 pub fn uuid_v4(&self) -> Result<Uuid> {
5949 self.side_effect(Uuid::new_v4)
5950 }
5951
5952 pub fn get_version(
5955 &self,
5956 change_id: impl Into<String>,
5957 min_supported: i32,
5958 max_supported: i32,
5959 ) -> Result<i32> {
5960 let change_id = change_id.into();
5961 if change_id.trim().is_empty() {
5962 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5963 "version_change_id_invalid",
5964 None,
5965 Some("non-empty change ID".to_string()),
5966 Some(change_id),
5967 "version markers require a stable non-empty change ID",
5968 )));
5969 }
5970 if min_supported > max_supported {
5971 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5972 "version_range_invalid",
5973 None,
5974 Some("min_supported <= max_supported".to_string()),
5975 Some(format!("{min_supported}..={max_supported}")),
5976 "version marker supported range is invalid",
5977 )));
5978 }
5979
5980 let mut state = self
5981 .state
5982 .lock()
5983 .map_err(|_| Error::WorkflowStatePoisoned)?;
5984 if let Some((version, sequence)) = state.version_markers.get(&change_id).copied() {
5985 ensure_version_supported(&change_id, version, min_supported, max_supported, sequence)?;
5986 return Ok(version);
5987 }
5988
5989 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
5990 return match recorded {
5991 RecordedCommand::VersionMarker {
5992 sequence,
5993 change_id: recorded_change_id,
5994 version,
5995 ..
5996 } => {
5997 if recorded_change_id != change_id {
5998 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5999 "version_change_id_mismatch",
6000 Some(sequence),
6001 Some(recorded_change_id),
6002 Some(change_id),
6003 "recorded version marker change ID differs from current workflow code",
6004 )));
6005 }
6006 ensure_version_supported(
6007 &change_id,
6008 version,
6009 min_supported,
6010 max_supported,
6011 sequence,
6012 )?;
6013 state.command_cursor += 1;
6014 state.version_markers.insert(change_id, (version, sequence));
6015 Ok(version)
6016 }
6017 other => Err(command_mismatch(
6018 &other,
6019 format!("version marker:{change_id}"),
6020 )),
6021 };
6022 }
6023
6024 let version = max_supported;
6025 state.commands.push(json!({
6026 "type": "record_version_marker",
6027 "change_id": change_id,
6028 "version": version,
6029 "min_supported": min_supported,
6030 "max_supported": max_supported,
6031 }));
6032 state.version_markers.insert(change_id, (version, 0));
6035 Ok(version)
6036 }
6037
6038 pub fn patched(&self, change_id: impl Into<String>) -> Result<bool> {
6040 Ok(self.get_version(change_id, -1, 1)? == 1)
6041 }
6042
6043 pub fn deprecate_patch(&self, change_id: impl Into<String>) -> Result<()> {
6045 self.get_version(change_id, -1, 1).map(|_| ())
6046 }
6047
6048 pub fn start_child_workflow<T: Serialize>(
6075 &self,
6076 workflow_type: impl Into<String>,
6077 options: ChildWorkflowOptions,
6078 args: T,
6079 ) -> ChildWorkflowCall {
6080 ChildWorkflowCall {
6081 ctx: self.clone(),
6082 workflow_type: workflow_type.into(),
6083 options,
6084 args: Some(AvroValue::from_serialize(&args)),
6085 scheduled: false,
6086 matched_pending: false,
6087 }
6088 }
6089
6090 pub async fn start_child_workflow_avro_value<T: Serialize>(
6091 &self,
6092 workflow_type: impl Into<String>,
6093 options: ChildWorkflowOptions,
6094 args: T,
6095 ) -> Result<ChildWorkflowAvroResult> {
6096 let mut call = self.start_child_workflow(workflow_type, options, args);
6097 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
6098 }
6099
6100 fn take_commands(&self) -> Result<Vec<Value>> {
6101 let mut state = self
6102 .state
6103 .lock()
6104 .map_err(|_| Error::WorkflowStatePoisoned)?;
6105 Ok(std::mem::take(&mut state.commands))
6106 }
6107
6108 fn continue_as_new_command(&self, request: ContinueAsNewRequest) -> Result<Option<Value>> {
6109 let mut state = self
6110 .state
6111 .lock()
6112 .map_err(|_| Error::WorkflowStatePoisoned)?;
6113
6114 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6115 return Err(command_mismatch(&recorded, "continue as new"));
6116 }
6117 if state.recorded_continue_as_new_sequence.is_some() {
6118 state.continue_as_new_consumed = true;
6119 return Ok(None);
6120 }
6121
6122 let arguments = encode_typed_envelope(&request.arguments, &state.payload_codec)?;
6123 let mut command = serde_json::Map::from_iter([
6124 ("type".to_string(), json!("continue_as_new")),
6125 ("arguments".to_string(), arguments),
6126 ("queue".to_string(), json!(state.task_queue.clone())),
6127 ]);
6128 if let Some(workflow_type) = request.options.workflow_type {
6129 command.insert("workflow_type".to_string(), json!(workflow_type));
6130 }
6131 if let Some(task_queue) = request.options.task_queue {
6132 command.insert("queue".to_string(), json!(task_queue));
6133 }
6134 Ok(Some(Value::Object(command)))
6135 }
6136
6137 fn matched_recorded_pending(&self) -> Result<bool> {
6138 let state = self
6139 .state
6140 .lock()
6141 .map_err(|_| Error::WorkflowStatePoisoned)?;
6142 Ok(state.matched_recorded_pending)
6143 }
6144
6145 fn ensure_history_consumed(&self) -> Result<()> {
6146 let state = self
6147 .state
6148 .lock()
6149 .map_err(|_| Error::WorkflowStatePoisoned)?;
6150 if let Some(command) = state.recorded_commands.get(state.command_cursor) {
6151 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
6152 "recorded_commands_unconsumed",
6153 Some(command.sequence()),
6154 Some(command.shape().to_string()),
6155 Some("workflow completion".to_string()),
6156 "workflow completed before consuming all recorded durable commands",
6157 )));
6158 }
6159 if let Some(sequence) = state
6160 .recorded_continue_as_new_sequence
6161 .filter(|_| !state.continue_as_new_consumed)
6162 {
6163 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
6164 "recorded_continue_as_new_unconsumed",
6165 Some(sequence),
6166 Some("continue as new".to_string()),
6167 Some("workflow completion".to_string()),
6168 "workflow completed without consuming its recorded continue-as-new transition",
6169 )));
6170 }
6171 Ok(())
6172 }
6173}
6174
6175#[derive(Debug)]
6176struct WorkflowState {
6177 workflow_id: Option<String>,
6178 run_id: Option<String>,
6179 task_queue: String,
6180 payload_codec: String,
6181 history_budget: WorkflowHistoryBudget,
6182 resume_signal: Option<ResumeSignal>,
6183 recorded_commands: Vec<RecordedCommand>,
6184 recorded_continue_as_new_sequence: Option<u64>,
6185 continue_as_new_consumed: bool,
6186 command_cursor: usize,
6187 matched_recorded_pending: bool,
6188 version_markers: HashMap<String, (i32, u64)>,
6189 commands: Vec<Value>,
6190}
6191
6192impl WorkflowState {
6193 #[cfg(test)]
6194 fn new(
6195 history: Vec<HistoryEvent>,
6196 task_queue: String,
6197 payload_codec: String,
6198 resume_signal: Option<ResumeSignal>,
6199 ) -> Result<Self> {
6200 Self::new_with_identity(
6201 history,
6202 None,
6203 None,
6204 task_queue,
6205 payload_codec,
6206 resume_signal,
6207 )
6208 }
6209
6210 fn new_with_identity(
6211 history: Vec<HistoryEvent>,
6212 workflow_id: Option<String>,
6213 run_id: Option<String>,
6214 task_queue: String,
6215 payload_codec: String,
6216 resume_signal: Option<ResumeSignal>,
6217 ) -> Result<Self> {
6218 let recorded_commands = recorded_commands(
6219 &history,
6220 &payload_codec,
6221 WorkflowIdentity {
6222 workflow_id: workflow_id.clone(),
6223 run_id: run_id.clone(),
6224 },
6225 )?;
6226 let recorded_continue_as_new = history
6227 .iter()
6228 .filter(|event| event.event_type == "WorkflowContinuedAsNew")
6229 .collect::<Vec<_>>();
6230 if recorded_continue_as_new.len() > 1 {
6231 return Err(invalid_recorded_history(
6232 "duplicate_continue_as_new_transition",
6233 recorded_continue_as_new
6234 .last()
6235 .and_then(|event| durable_event_sequence(event))
6236 .unwrap_or(0),
6237 "one WorkflowContinuedAsNew event",
6238 &format!(
6239 "{} WorkflowContinuedAsNew events",
6240 recorded_continue_as_new.len()
6241 ),
6242 "workflow history records one continue-as-new transition more than once",
6243 ));
6244 }
6245 let recorded_continue_as_new_sequence = recorded_continue_as_new
6246 .first()
6247 .map(|event| {
6248 durable_event_sequence(event).ok_or_else(|| {
6249 Error::NonDeterministicReplay(ReplayFailure::new(
6250 "continue_as_new_sequence_missing",
6251 None,
6252 Some("recorded transition sequence".to_string()),
6253 Some("missing sequence".to_string()),
6254 "WorkflowContinuedAsNew history is missing its recorded sequence",
6255 ))
6256 })
6257 })
6258 .transpose()?;
6259 let event_count = u64::try_from(history.len()).unwrap_or(u64::MAX);
6260 Ok(Self {
6261 workflow_id,
6262 run_id,
6263 task_queue,
6264 payload_codec,
6265 history_budget: WorkflowHistoryBudget {
6266 event_count,
6267 ..WorkflowHistoryBudget::default()
6268 },
6269 resume_signal,
6270 recorded_commands,
6271 recorded_continue_as_new_sequence,
6272 continue_as_new_consumed: false,
6273 command_cursor: 0,
6274 matched_recorded_pending: false,
6275 version_markers: HashMap::new(),
6276 commands: Vec::new(),
6277 })
6278 }
6279}
6280
6281#[derive(Clone, Debug)]
6282enum RecordedCommand {
6283 Activity {
6284 sequence: u64,
6285 activity_type: Option<String>,
6286 options: Option<RecordedActivityOptions>,
6287 outcome: Option<ActivityOutcome>,
6288 },
6289 Timer {
6290 sequence: u64,
6291 delay_seconds: u64,
6292 fired: bool,
6293 },
6294 ChildWorkflow {
6295 sequence: u64,
6296 workflow_type: Option<String>,
6297 outcome: Option<ChildWorkflowOutcome>,
6298 },
6299 SignalWait {
6300 sequence: u64,
6301 signal_name: String,
6302 value: Option<Vec<AvroValue>>,
6303 },
6304 SideEffect {
6305 sequence: u64,
6306 value: AvroValue,
6307 },
6308 VersionMarker {
6309 sequence: u64,
6310 change_id: String,
6311 version: i32,
6312 },
6313}
6314
6315#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
6316struct RecordedActivityOptions {
6317 task_queue: RecordedSnapshotValue<Option<String>>,
6318 execution_mode: RecordedSnapshotValue<Option<String>>,
6319 retry_policy: ActivityRetrySnapshot,
6320}
6321
6322#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
6323enum RecordedSnapshotValue<T> {
6324 Unknown,
6326 Known(T),
6327}
6328
6329impl<T: PartialEq> RecordedSnapshotValue<T> {
6330 fn matches_current(&self, current: &Self) -> bool {
6331 match self {
6332 Self::Unknown => true,
6333 Self::Known(recorded) => matches!(current, Self::Known(value) if value == recorded),
6334 }
6335 }
6336}
6337
6338#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
6339struct ActivityRetrySnapshot {
6340 snapshot_version: RecordedSnapshotValue<Option<u64>>,
6341 max_attempts: RecordedSnapshotValue<Option<u64>>,
6342 backoff_seconds: RecordedSnapshotValue<Vec<u64>>,
6343 start_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
6344 schedule_to_start_timeout: RecordedSnapshotValue<Option<u64>>,
6345 schedule_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
6346 heartbeat_timeout: RecordedSnapshotValue<Option<u64>>,
6347 non_retryable_error_types: RecordedSnapshotValue<Vec<String>>,
6348}
6349
6350impl ActivityRetrySnapshot {
6351 fn matches_current(&self, current: &Self) -> bool {
6352 self.snapshot_version
6353 .matches_current(¤t.snapshot_version)
6354 && self.max_attempts.matches_current(¤t.max_attempts)
6355 && self
6356 .backoff_seconds
6357 .matches_current(¤t.backoff_seconds)
6358 && self
6359 .start_to_close_timeout
6360 .matches_current(¤t.start_to_close_timeout)
6361 && self
6362 .schedule_to_start_timeout
6363 .matches_current(¤t.schedule_to_start_timeout)
6364 && self
6365 .schedule_to_close_timeout
6366 .matches_current(¤t.schedule_to_close_timeout)
6367 && self
6368 .heartbeat_timeout
6369 .matches_current(¤t.heartbeat_timeout)
6370 && self
6371 .non_retryable_error_types
6372 .matches_current(¤t.non_retryable_error_types)
6373 }
6374}
6375
6376fn recorded_optional_u64(
6377 object: Option<&serde_json::Map<String, Value>>,
6378 field: &str,
6379) -> RecordedSnapshotValue<Option<u64>> {
6380 match object.and_then(|object| object.get(field)) {
6381 None => RecordedSnapshotValue::Unknown,
6382 Some(Value::Null) => RecordedSnapshotValue::Known(None),
6383 Some(value) => RecordedSnapshotValue::Known(value_as_u64(value)),
6384 }
6385}
6386
6387fn recorded_optional_string(
6388 object: &serde_json::Map<String, Value>,
6389 field: &str,
6390) -> RecordedSnapshotValue<Option<String>> {
6391 match object.get(field) {
6392 None => RecordedSnapshotValue::Unknown,
6393 Some(Value::Null) => RecordedSnapshotValue::Known(None),
6394 Some(value) => RecordedSnapshotValue::Known(value.as_str().map(str::to_string)),
6395 }
6396}
6397
6398fn recorded_activity_retry_snapshot(policy: Option<&Value>) -> ActivityRetrySnapshot {
6399 let policy = policy.and_then(Value::as_object);
6400 let backoff_seconds = policy
6401 .and_then(|policy| policy.get("backoff_seconds"))
6402 .and_then(Value::as_array)
6403 .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
6404 .map_or(RecordedSnapshotValue::Unknown, RecordedSnapshotValue::Known);
6405 let mut non_retryable_error_types = Vec::new();
6406 for error_type in policy
6407 .and_then(|policy| policy.get("non_retryable_error_types"))
6408 .and_then(Value::as_array)
6409 .into_iter()
6410 .flatten()
6411 .filter_map(Value::as_str)
6412 .map(str::trim)
6413 .filter(|error_type| !error_type.is_empty())
6414 {
6415 if !non_retryable_error_types
6416 .iter()
6417 .any(|recorded| recorded == error_type)
6418 {
6419 non_retryable_error_types.push(error_type.to_string());
6420 }
6421 }
6422
6423 ActivityRetrySnapshot {
6424 snapshot_version: recorded_optional_u64(policy, "snapshot_version"),
6425 max_attempts: recorded_optional_u64(policy, "max_attempts"),
6426 backoff_seconds,
6427 start_to_close_timeout: recorded_optional_u64(policy, "start_to_close_timeout"),
6428 schedule_to_start_timeout: recorded_optional_u64(policy, "schedule_to_start_timeout"),
6429 schedule_to_close_timeout: recorded_optional_u64(policy, "schedule_to_close_timeout"),
6430 heartbeat_timeout: recorded_optional_u64(policy, "heartbeat_timeout"),
6431 non_retryable_error_types: if policy
6432 .is_some_and(|policy| policy.contains_key("non_retryable_error_types"))
6433 {
6434 RecordedSnapshotValue::Known(non_retryable_error_types)
6435 } else {
6436 RecordedSnapshotValue::Unknown
6437 },
6438 }
6439}
6440
6441fn current_activity_retry_snapshot(options: &ValidatedActivityOptions) -> ActivityRetrySnapshot {
6442 let policy = options.retry_policy.as_ref();
6443 let max_attempts = match policy.and_then(|policy| policy.get("max_attempts")) {
6444 Some(Value::Null) => None,
6445 Some(value) => value_as_u64(value),
6446 None => Some(1),
6447 };
6448 let backoff_seconds = policy
6449 .and_then(|policy| policy.get("backoff_seconds"))
6450 .and_then(Value::as_array)
6451 .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
6452 .unwrap_or_default();
6453 let non_retryable_error_types = policy
6454 .and_then(|policy| policy.get("non_retryable_error_types"))
6455 .and_then(Value::as_array)
6456 .into_iter()
6457 .flatten()
6458 .filter_map(Value::as_str)
6459 .map(str::to_string)
6460 .collect();
6461
6462 ActivityRetrySnapshot {
6463 snapshot_version: RecordedSnapshotValue::Known(Some(1)),
6464 max_attempts: RecordedSnapshotValue::Known(max_attempts),
6465 backoff_seconds: RecordedSnapshotValue::Known(backoff_seconds),
6466 start_to_close_timeout: RecordedSnapshotValue::Known(options.start_to_close_timeout),
6467 schedule_to_start_timeout: RecordedSnapshotValue::Known(options.schedule_to_start_timeout),
6468 schedule_to_close_timeout: RecordedSnapshotValue::Known(options.schedule_to_close_timeout),
6469 heartbeat_timeout: RecordedSnapshotValue::Known(options.heartbeat_timeout),
6470 non_retryable_error_types: RecordedSnapshotValue::Known(non_retryable_error_types),
6471 }
6472}
6473
6474fn activity_options_description(options: &RecordedActivityOptions) -> String {
6475 serde_json::to_string(options).unwrap_or_else(|_| format!("{options:?}"))
6476}
6477
6478impl RecordedCommand {
6479 fn sequence(&self) -> u64 {
6480 match self {
6481 Self::Activity { sequence, .. }
6482 | Self::Timer { sequence, .. }
6483 | Self::ChildWorkflow { sequence, .. }
6484 | Self::SignalWait { sequence, .. }
6485 | Self::SideEffect { sequence, .. }
6486 | Self::VersionMarker { sequence, .. } => *sequence,
6487 }
6488 }
6489
6490 fn shape(&self) -> &'static str {
6491 match self {
6492 Self::Activity { .. } => "activity",
6493 Self::Timer { .. } => "timer",
6494 Self::ChildWorkflow { .. } => "child workflow",
6495 Self::SignalWait { .. } => "signal wait",
6496 Self::SideEffect { .. } => "side effect",
6497 Self::VersionMarker { .. } => "version marker",
6498 }
6499 }
6500}
6501
6502fn ensure_version_supported(
6503 change_id: &str,
6504 version: i32,
6505 min_supported: i32,
6506 max_supported: i32,
6507 sequence: u64,
6508) -> Result<()> {
6509 if (min_supported..=max_supported).contains(&version) {
6510 return Ok(());
6511 }
6512 Err(Error::NonDeterministicReplay(ReplayFailure::new(
6513 "version_marker_incompatible_range",
6514 (sequence != 0).then_some(sequence),
6515 Some(format!("{min_supported}..={max_supported}")),
6516 Some(format!("{change_id}:{version}")),
6517 "recorded workflow version is outside the range supported by current code",
6518 )))
6519}
6520
6521#[derive(Clone, Debug)]
6522struct ResumeSignal {
6523 signal_name: String,
6524 arguments: Vec<AvroValue>,
6525}
6526
6527pub struct ActivityCall {
6528 ctx: WorkflowContext,
6529 activity_type: String,
6530 options: ActivityOptions,
6531 args: Option<Result<AvroValue>>,
6532 scheduled: bool,
6533}
6534
6535impl ActivityCall {
6536 fn poll_avro_value(
6537 mut self: Pin<&mut Self>,
6538 _cx: &mut TaskContext<'_>,
6539 ) -> Poll<Result<AvroValue>> {
6540 let ctx = self.ctx.clone();
6541 let mut state = match ctx.state.lock() {
6542 Ok(state) => state,
6543 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6544 };
6545
6546 if self.scheduled {
6547 return Poll::Pending;
6548 }
6549
6550 let options = match self.options.validate() {
6551 Ok(options) => options,
6552 Err(error) => {
6553 return Poll::Ready(Err(Error::InvalidActivityOptions(error)));
6554 }
6555 };
6556 let task_queue = options
6557 .task_queue
6558 .clone()
6559 .unwrap_or_else(|| state.task_queue.clone());
6560 let current_recorded_options = RecordedActivityOptions {
6561 task_queue: RecordedSnapshotValue::Known(Some(task_queue.clone())),
6562 execution_mode: RecordedSnapshotValue::Known(None),
6565 retry_policy: current_activity_retry_snapshot(&options),
6566 };
6567
6568 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6569 let sequence = recorded.sequence();
6570 match recorded {
6571 RecordedCommand::Activity {
6572 activity_type,
6573 options: recorded_options,
6574 outcome,
6575 ..
6576 } => {
6577 if let Some(recorded_type) = activity_type {
6578 if recorded_type != self.activity_type {
6579 return Poll::Ready(Err(Error::NonDeterministicReplay(
6580 ReplayFailure::new(
6581 "recorded_command_detail_mismatch",
6582 Some(sequence),
6583 Some(format!("activity:{recorded_type}")),
6584 Some(format!("activity:{}", self.activity_type)),
6585 "recorded activity type differs from the current workflow command",
6586 ),
6587 )));
6588 }
6589 }
6590 if let Some(recorded_options) = recorded_options {
6591 if !recorded_options
6592 .task_queue
6593 .matches_current(¤t_recorded_options.task_queue)
6594 {
6595 return Poll::Ready(Err(Error::NonDeterministicReplay(
6596 ReplayFailure::new(
6597 "activity_task_queue_mismatch",
6598 Some(sequence),
6599 Some(activity_options_description(&recorded_options)),
6600 Some(activity_options_description(¤t_recorded_options)),
6601 "recorded activity task queue differs from the current workflow command",
6602 ),
6603 )));
6604 }
6605 if !recorded_options
6606 .execution_mode
6607 .matches_current(¤t_recorded_options.execution_mode)
6608 {
6609 return Poll::Ready(Err(Error::NonDeterministicReplay(
6610 ReplayFailure::new(
6611 "activity_execution_mode_mismatch",
6612 Some(sequence),
6613 Some(activity_options_description(&recorded_options)),
6614 Some(activity_options_description(¤t_recorded_options)),
6615 "recorded activity execution mode differs from the current workflow command",
6616 ),
6617 )));
6618 }
6619 if !recorded_options
6620 .retry_policy
6621 .matches_current(¤t_recorded_options.retry_policy)
6622 {
6623 return Poll::Ready(Err(Error::NonDeterministicReplay(
6624 ReplayFailure::new(
6625 "activity_retry_policy_mismatch",
6626 Some(sequence),
6627 Some(activity_options_description(&recorded_options)),
6628 Some(activity_options_description(¤t_recorded_options)),
6629 "recorded activity retry policy differs from the current workflow command",
6630 ),
6631 )));
6632 }
6633 }
6634 state.command_cursor += 1;
6635 if let Some(outcome) = outcome {
6636 return Poll::Ready(outcome.map_err(Error::ActivityFailed));
6637 }
6638 state.matched_recorded_pending = true;
6639 self.scheduled = true;
6640 return Poll::Pending;
6641 }
6642 other => {
6643 return Poll::Ready(Err(command_mismatch(
6644 &other,
6645 format!("activity:{}", self.activity_type),
6646 )));
6647 }
6648 }
6649 }
6650
6651 if !self.scheduled {
6652 let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
6653 Ok(args) => args,
6654 Err(error) => return Poll::Ready(Err(error)),
6655 };
6656 let arguments = normalize_avro_arguments(args);
6657 let envelope = match encode_typed_envelope(&arguments, &state.payload_codec) {
6658 Ok(envelope) => envelope,
6659 Err(error) => return Poll::Ready(Err(error)),
6660 };
6661
6662 let mut command = serde_json::Map::from_iter([
6663 ("type".to_string(), json!("schedule_activity")),
6664 (
6665 "activity_type".to_string(),
6666 json!(self.activity_type.clone()),
6667 ),
6668 ("queue".to_string(), json!(task_queue)),
6669 ("arguments".to_string(), envelope),
6670 ]);
6671 for (field, value) in [
6672 ("start_to_close_timeout", options.start_to_close_timeout),
6673 (
6674 "schedule_to_start_timeout",
6675 options.schedule_to_start_timeout,
6676 ),
6677 (
6678 "schedule_to_close_timeout",
6679 options.schedule_to_close_timeout,
6680 ),
6681 ("heartbeat_timeout", options.heartbeat_timeout),
6682 ] {
6683 if let Some(value) = value {
6684 command.insert(field.to_string(), json!(value));
6685 }
6686 }
6687 if let Some(retry_policy) = options.retry_policy {
6688 command.insert("retry_policy".to_string(), retry_policy);
6689 }
6690 state.commands.push(Value::Object(command));
6691 self.scheduled = true;
6692 }
6693
6694 Poll::Pending
6695 }
6696}
6697
6698impl Future for ActivityCall {
6699 type Output = Result<Value>;
6700
6701 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6702 match self.poll_avro_value(cx) {
6703 Poll::Ready(Ok(value)) => Poll::Ready(value.into_json()),
6704 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
6705 Poll::Pending => Poll::Pending,
6706 }
6707 }
6708}
6709
6710pub struct TimerCall {
6712 ctx: WorkflowContext,
6713 delay_seconds: Option<u64>,
6714 scheduled: bool,
6715 matched_pending: bool,
6716}
6717
6718impl Future for TimerCall {
6719 type Output = Result<()>;
6720
6721 fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6722 if self.matched_pending {
6723 return Poll::Pending;
6724 }
6725
6726 let ctx = self.ctx.clone();
6727 let Some(requested_delay) = self.delay_seconds else {
6728 return Poll::Ready(Err(Error::TimerDurationOverflow));
6729 };
6730 let mut state = match ctx.state.lock() {
6731 Ok(state) => state,
6732 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6733 };
6734
6735 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6736 match recorded {
6737 RecordedCommand::Timer {
6738 sequence,
6739 delay_seconds,
6740 fired,
6741 ..
6742 } => {
6743 if delay_seconds != requested_delay {
6744 return Poll::Ready(Err(Error::NonDeterministicReplay(
6745 ReplayFailure::new(
6746 "timer_delay_mismatch",
6747 Some(sequence),
6748 Some(format!("timer:{delay_seconds}s")),
6749 Some(format!("timer:{requested_delay}s")),
6750 "recorded timer delay differs from the current workflow command",
6751 ),
6752 )));
6753 }
6754 state.command_cursor += 1;
6755 if fired {
6756 return Poll::Ready(Ok(()));
6757 }
6758 state.matched_recorded_pending = true;
6759 self.scheduled = true;
6760 self.matched_pending = true;
6761 return Poll::Pending;
6762 }
6763 other => return Poll::Ready(Err(command_mismatch(&other, "timer"))),
6764 }
6765 }
6766
6767 if !self.scheduled {
6768 state.commands.push(json!({
6769 "type": "start_timer",
6770 "delay_seconds": requested_delay,
6771 }));
6772 self.scheduled = true;
6773 }
6774
6775 Poll::Pending
6776 }
6777}
6778
6779pub struct ChildWorkflowCall {
6781 ctx: WorkflowContext,
6782 workflow_type: String,
6783 options: ChildWorkflowOptions,
6784 args: Option<Result<AvroValue>>,
6785 scheduled: bool,
6786 matched_pending: bool,
6787}
6788
6789impl ChildWorkflowCall {
6790 fn poll_avro_value(
6791 mut self: Pin<&mut Self>,
6792 _cx: &mut TaskContext<'_>,
6793 ) -> Poll<Result<ChildWorkflowAvroResult>> {
6794 if self.matched_pending {
6795 return Poll::Pending;
6796 }
6797
6798 let ctx = self.ctx.clone();
6799 let mut state = match ctx.state.lock() {
6800 Ok(state) => state,
6801 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6802 };
6803
6804 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6805 let sequence = recorded.sequence();
6806 match recorded {
6807 RecordedCommand::ChildWorkflow {
6808 workflow_type,
6809 outcome,
6810 ..
6811 } => {
6812 if let Some(recorded_type) = workflow_type {
6813 if recorded_type != self.workflow_type {
6814 return Poll::Ready(Err(Error::NonDeterministicReplay(
6815 ReplayFailure::new(
6816 "recorded_command_detail_mismatch",
6817 Some(sequence),
6818 Some(format!("child workflow:{recorded_type}")),
6819 Some(format!("child workflow:{}", self.workflow_type)),
6820 "recorded child workflow type differs from the current workflow command",
6821 ),
6822 )));
6823 }
6824 }
6825 state.command_cursor += 1;
6826 if let Some(outcome) = outcome {
6827 return Poll::Ready(outcome.map_err(Error::ChildWorkflowFailed));
6828 }
6829 state.matched_recorded_pending = true;
6830 self.scheduled = true;
6831 self.matched_pending = true;
6832 return Poll::Pending;
6833 }
6834 other => {
6835 return Poll::Ready(Err(command_mismatch(
6836 &other,
6837 format!("child workflow:{}", self.workflow_type),
6838 )));
6839 }
6840 }
6841 }
6842
6843 if !self.scheduled {
6844 if self.options.task_queue.trim().is_empty() {
6845 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
6846 "task_queue must not be empty".to_string(),
6847 )));
6848 }
6849 for (name, value) in [
6850 (
6851 "execution_timeout_seconds",
6852 self.options.execution_timeout_seconds,
6853 ),
6854 ("run_timeout_seconds", self.options.run_timeout_seconds),
6855 ] {
6856 if value == Some(0) {
6857 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(format!(
6858 "{name} must be at least 1"
6859 ))));
6860 }
6861 }
6862
6863 let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
6864 Ok(args) => args,
6865 Err(error) => return Poll::Ready(Err(error)),
6866 };
6867 let arguments = match encode_typed_envelope(
6868 &normalize_avro_arguments(args),
6869 &state.payload_codec,
6870 ) {
6871 Ok(arguments) => arguments,
6872 Err(error) => return Poll::Ready(Err(error)),
6873 };
6874 let mut command = json!({
6875 "type": "start_child_workflow",
6876 "workflow_type": self.workflow_type,
6877 "queue": self.options.task_queue,
6878 "parent_close_policy": self.options.parent_close_policy.as_str(),
6879 "arguments": arguments,
6880 });
6881 let object = command
6882 .as_object_mut()
6883 .expect("child workflow command is always an object");
6884 if let Some(policy) = &self.options.retry_policy {
6885 let mut retry_policy = serde_json::Map::new();
6886 if let Some(max_attempts) = policy.max_attempts {
6887 if max_attempts == 0 {
6888 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
6889 "retry_policy.max_attempts must be at least 1".to_string(),
6890 )));
6891 }
6892 retry_policy.insert("max_attempts".to_string(), json!(max_attempts));
6893 }
6894 if !policy.backoff_seconds.is_empty() {
6895 retry_policy
6896 .insert("backoff_seconds".to_string(), json!(policy.backoff_seconds));
6897 }
6898 if !policy.non_retryable_error_types.is_empty() {
6899 retry_policy.insert(
6900 "non_retryable_error_types".to_string(),
6901 json!(policy.non_retryable_error_types),
6902 );
6903 }
6904 if retry_policy.is_empty() {
6905 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
6906 "retry_policy must configure at least one field".to_string(),
6907 )));
6908 }
6909 object.insert("retry_policy".to_string(), Value::Object(retry_policy));
6910 }
6911 if let Some(seconds) = self.options.execution_timeout_seconds {
6912 object.insert("execution_timeout_seconds".to_string(), json!(seconds));
6913 }
6914 if let Some(seconds) = self.options.run_timeout_seconds {
6915 object.insert("run_timeout_seconds".to_string(), json!(seconds));
6916 }
6917 state.commands.push(command);
6918 self.scheduled = true;
6919 }
6920
6921 Poll::Pending
6922 }
6923}
6924
6925impl Future for ChildWorkflowCall {
6926 type Output = Result<ChildWorkflowResult>;
6927
6928 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6929 match self.poll_avro_value(cx) {
6930 Poll::Ready(Ok(result)) => match result.result.into_json() {
6931 Ok(projected) => Poll::Ready(Ok(ChildWorkflowResult {
6932 parent: result.parent,
6933 child: result.child,
6934 child_workflow_type: result.child_workflow_type,
6935 result: projected,
6936 })),
6937 Err(error) => Poll::Ready(Err(error)),
6938 },
6939 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
6940 Poll::Pending => Poll::Pending,
6941 }
6942 }
6943}
6944
6945fn command_mismatch(recorded: &RecordedCommand, actual: impl Into<String>) -> Error {
6946 Error::NonDeterministicReplay(ReplayFailure::new(
6947 "recorded_command_mismatch",
6948 Some(recorded.sequence()),
6949 Some(recorded.shape().to_string()),
6950 Some(actual.into()),
6951 "current workflow command does not match the recorded durable command sequence",
6952 ))
6953}
6954
6955pub struct SignalCall {
6956 ctx: WorkflowContext,
6957 signal_name: String,
6958 opened_wait: bool,
6959 matched_pending: bool,
6960}
6961
6962impl SignalCall {
6963 fn poll_avro_value(
6964 mut self: Pin<&mut Self>,
6965 _cx: &mut TaskContext<'_>,
6966 ) -> Poll<Result<Vec<AvroValue>>> {
6967 if self.matched_pending {
6968 return Poll::Pending;
6969 }
6970
6971 let ctx = self.ctx.clone();
6972 let mut state = match ctx.state.lock() {
6973 Ok(state) => state,
6974 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6975 };
6976
6977 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6978 match recorded {
6979 RecordedCommand::SignalWait {
6980 sequence,
6981 signal_name,
6982 value,
6983 } => {
6984 if signal_name != self.signal_name {
6985 return Poll::Ready(Err(Error::NonDeterministicReplay(
6986 ReplayFailure::new(
6987 "recorded_command_detail_mismatch",
6988 Some(sequence),
6989 Some(format!("signal wait:{signal_name}")),
6990 Some(format!("signal wait:{}", self.signal_name)),
6991 "recorded signal name differs from the current workflow command",
6992 ),
6993 )));
6994 }
6995
6996 state.command_cursor += 1;
6997 if let Some(value) = value {
6998 return Poll::Ready(Ok(value));
6999 }
7000 if state
7001 .resume_signal
7002 .as_ref()
7003 .is_some_and(|signal| signal.signal_name == self.signal_name)
7004 {
7005 let signal = state
7006 .resume_signal
7007 .take()
7008 .expect("matching resume signal is present");
7009 return Poll::Ready(Ok(signal.arguments));
7010 }
7011
7012 state.matched_recorded_pending = true;
7013 self.opened_wait = true;
7014 self.matched_pending = true;
7015 return Poll::Pending;
7016 }
7017 other => {
7018 return Poll::Ready(Err(command_mismatch(
7019 &other,
7020 format!("signal wait:{}", self.signal_name),
7021 )));
7022 }
7023 }
7024 }
7025
7026 if state
7027 .resume_signal
7028 .as_ref()
7029 .is_some_and(|signal| signal.signal_name == self.signal_name)
7030 {
7031 let signal = state
7032 .resume_signal
7033 .take()
7034 .expect("matching resume signal is present");
7035 return Poll::Ready(Ok(signal.arguments));
7036 }
7037
7038 if !self.opened_wait {
7039 state.commands.push(json!({
7040 "type": "open_signal_wait",
7041 "signal_name": self.signal_name
7042 }));
7043 self.opened_wait = true;
7044 }
7045
7046 Poll::Pending
7047 }
7048}
7049
7050impl Future for SignalCall {
7051 type Output = Result<Vec<Value>>;
7052
7053 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
7054 match self.poll_avro_value(cx) {
7055 Poll::Ready(Ok(values)) => Poll::Ready(
7056 values
7057 .into_iter()
7058 .map(AvroValue::into_json)
7059 .collect::<Result<Vec<_>>>(),
7060 ),
7061 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
7062 Poll::Pending => Poll::Pending,
7063 }
7064 }
7065}
7066
7067#[derive(Clone, Debug)]
7068pub struct ActivityContext {
7069 client: Client,
7070 pub task_id: String,
7071 pub activity_attempt_id: String,
7072 pub lease_owner: String,
7073 pub activity_type: String,
7074 pub attempt_number: u64,
7075 pub task_queue: String,
7076 pub worker_id: String,
7077}
7078
7079impl ActivityContext {
7080 pub async fn heartbeat<T: Serialize>(&self, details: T) -> Result<ActivityHeartbeatResponse> {
7081 self.client
7082 .heartbeat_activity_task(
7083 &self.task_id,
7084 &self.activity_attempt_id,
7085 &self.lease_owner,
7086 details,
7087 )
7088 .await
7089 }
7090}
7091
7092fn decode_task_avro_arguments(value: Option<&Value>, codec: &str) -> Result<AvroValue> {
7093 validate_payload_codec(codec)?;
7094 match value {
7095 Some(value) => Ok(normalize_avro_arguments(decode_wire_avro_value(
7096 value, codec,
7097 )?)),
7098 None => Ok(AvroValue::Array(Vec::new())),
7099 }
7100}
7101
7102fn decode_resume_signal(task: &WorkflowTask) -> Result<Option<ResumeSignal>> {
7103 let Some(signal_name) = task
7104 .signal_name
7105 .as_deref()
7106 .filter(|value| !value.is_empty())
7107 else {
7108 return Ok(None);
7109 };
7110 let decoded = decode_task_avro_arguments(task.signal_arguments.as_ref(), &task.payload_codec)?;
7111 let AvroValue::Array(arguments) = decoded else {
7112 unreachable!("normalize_avro_arguments always returns an array");
7113 };
7114
7115 Ok(Some(ResumeSignal {
7116 signal_name: signal_name.to_string(),
7117 arguments,
7118 }))
7119}
7120
7121fn validate_workflow_task_payloads(task: &WorkflowTask) -> Result<()> {
7122 validate_payload_codec(&task.payload_codec)?;
7123 validate_optional_inbound_payload(task.arguments.as_ref(), &task.payload_codec)?;
7124 validate_optional_inbound_payload(task.signal_arguments.as_ref(), &task.payload_codec)?;
7125 for event in &task.history_events {
7126 validate_history_event_payloads(event, &task.payload_codec)?;
7127 }
7128 Ok(())
7129}
7130
7131fn validate_activity_task_payloads(task: &ActivityTask) -> Result<()> {
7132 validate_payload_codec(&task.payload_codec)?;
7133 validate_optional_inbound_payload(task.arguments.as_ref(), &task.payload_codec)
7134}
7135
7136fn validate_query_task_payloads(task: &QueryTask) -> Result<()> {
7137 validate_payload_codec(&task.payload_codec)?;
7138 validate_optional_inbound_payload(task.workflow_arguments.as_ref(), &task.payload_codec)?;
7139 validate_optional_inbound_payload(task.query_arguments.as_ref(), &task.payload_codec)?;
7140 for event in &task.history_events {
7141 validate_history_event_payloads(event, &task.payload_codec)?;
7142 }
7143
7144 let Some(export) = task.history_export.as_ref() else {
7145 return Ok(());
7146 };
7147 let export_codec = match export.get("payloads") {
7148 Some(payloads) => declared_payload_codec(payloads, "codec")?,
7149 None => None,
7150 }
7151 .unwrap_or(&task.payload_codec);
7152 validate_payload_codec(export_codec)?;
7153
7154 if let Some(events) = export.get("history_events").and_then(Value::as_array) {
7155 for event in events {
7156 let event_type = event
7157 .get("event_type")
7158 .or_else(|| event.get("type"))
7159 .and_then(Value::as_str)
7160 .unwrap_or_default();
7161 if let Some(payload) = event.get("payload") {
7162 validate_history_payloads(event_type, payload, export_codec)?;
7163 }
7164 }
7165 }
7166 for signal in export
7167 .get("signals")
7168 .and_then(Value::as_array)
7169 .into_iter()
7170 .flatten()
7171 {
7172 let codec = declared_payload_codec(signal, "payload_codec")?.unwrap_or(export_codec);
7173 validate_payload_codec(codec)?;
7174 validate_optional_inbound_payload(signal.get("arguments"), codec)?;
7175 }
7176 for activity in export
7177 .get("activities")
7178 .and_then(Value::as_array)
7179 .into_iter()
7180 .flatten()
7181 {
7182 let codec = declared_payload_codec(activity, "payload_codec")?.unwrap_or(export_codec);
7183 validate_payload_codec(codec)?;
7184 validate_optional_inbound_payload(activity.get("arguments"), codec)?;
7185 validate_optional_inbound_payload(activity.get("result"), codec)?;
7186 }
7187 Ok(())
7188}
7189
7190fn validate_history_event_payloads(event: &HistoryEvent, fallback_codec: &str) -> Result<()> {
7191 validate_history_payloads(&event.event_type, &event.payload, fallback_codec)
7192}
7193
7194fn validate_history_payloads(
7195 event_type: &str,
7196 payload: &Value,
7197 fallback_codec: &str,
7198) -> Result<()> {
7199 let codec = declared_payload_codec(payload, "payload_codec")?.unwrap_or(fallback_codec);
7200 validate_payload_codec(codec)?;
7201 for field in history_payload_fields(event_type) {
7202 validate_optional_inbound_payload(payload.get(*field), codec)?;
7203 }
7204 Ok(())
7205}
7206
7207const SIGNAL_HISTORY_PAYLOAD_FIELDS: &[&str] = &["value", "input", "arguments"];
7208
7209fn history_payload_fields(event_type: &str) -> &'static [&'static str] {
7210 match event_type {
7211 "ActivityCompleted" => &["result"],
7212 "SignalReceived" | "SignalApplied" => SIGNAL_HISTORY_PAYLOAD_FIELDS,
7213 "UpdateAccepted" | "UpdateRejected" | "UpdateApplied" => &["arguments"],
7214 "UpdateCompleted" | "SideEffectRecorded" => &["result"],
7215 "ChildRunCompleted" => &["result", "output"],
7216 "WorkflowCompleted" => &["output"],
7217 "ServiceCallStarted"
7218 | "ServiceCallCompleted"
7219 | "ServiceCallFailed"
7220 | "ServiceCallCancelled" => &["request_payload", "response_payload"],
7221 _ => &[],
7222 }
7223}
7224
7225fn signal_history_payload(payload: &Value) -> Option<&Value> {
7226 SIGNAL_HISTORY_PAYLOAD_FIELDS
7227 .iter()
7228 .find_map(|field| payload.get(*field))
7229}
7230
7231fn declared_payload_codec<'a>(value: &'a Value, field: &str) -> Result<Option<&'a str>> {
7232 match value.get(field) {
7233 None => Ok(None),
7234 Some(Value::String(codec)) => Ok(Some(codec)),
7235 Some(_) => Err(invalid_payload_envelope()),
7236 }
7237}
7238
7239fn validate_optional_inbound_payload(value: Option<&Value>, codec: &str) -> Result<()> {
7240 validate_payload_codec(codec)?;
7241 if let Some(value) = value.filter(|value| !value.is_null()) {
7242 decode_wire_avro_value(value, codec)?;
7243 }
7244 Ok(())
7245}
7246
7247fn recorded_commands(
7248 events: &[HistoryEvent],
7249 fallback_codec: &str,
7250 parent: WorkflowIdentity,
7251) -> Result<Vec<RecordedCommand>> {
7252 let mut events_by_sequence: BTreeMap<u64, Vec<&HistoryEvent>> = BTreeMap::new();
7253 let mut last_new_sequence = None;
7254
7255 for event in events {
7256 let is_activity = matches!(
7257 event.event_type.as_str(),
7258 "ActivityScheduled"
7259 | "ActivityStarted"
7260 | "ActivityHeartbeatRecorded"
7261 | "ActivityRetryScheduled"
7262 | "ActivityCompleted"
7263 | "ActivityFailed"
7264 | "ActivityCancelled"
7265 | "ActivityTimedOut"
7266 );
7267 let is_workflow_timer = matches!(
7268 event.event_type.as_str(),
7269 "TimerScheduled" | "TimerCancelled" | "TimerFired"
7270 ) && !is_internal_timer_event(event);
7271 let is_child_workflow = matches!(
7272 event.event_type.as_str(),
7273 "ChildWorkflowScheduled"
7274 | "ChildRunCompleted"
7275 | "ChildRunFailed"
7276 | "ChildRunCancelled"
7277 | "ChildRunTerminated"
7278 );
7279 let is_signal_wait = is_recorded_signal_wait_event(event);
7280 let is_side_effect = event.event_type == "SideEffectRecorded";
7281 let is_version_marker = event.event_type == "VersionMarkerRecorded";
7282 if !is_activity
7283 && !is_workflow_timer
7284 && !is_child_workflow
7285 && !is_signal_wait
7286 && !is_side_effect
7287 && !is_version_marker
7288 {
7289 continue;
7290 }
7291
7292 let sequence = durable_event_sequence(event).ok_or_else(|| {
7293 Error::NonDeterministicReplay(ReplayFailure::new(
7294 "durable_command_sequence_missing",
7295 None,
7296 Some("positive workflow sequence".to_string()),
7297 Some(event.event_type.clone()),
7298 "durable command history event has no workflow sequence",
7299 ))
7300 })?;
7301 if sequence == 0 {
7302 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
7303 "durable_command_sequence_invalid",
7304 Some(sequence),
7305 Some("positive workflow sequence".to_string()),
7306 Some(sequence.to_string()),
7307 "durable command history uses an invalid workflow sequence",
7308 )));
7309 }
7310 if !events_by_sequence.contains_key(&sequence) {
7311 if let Some(previous) = last_new_sequence {
7312 if sequence < previous {
7313 return Err(invalid_recorded_history(
7314 "durable_command_sequence_mismatch",
7315 sequence,
7316 &format!("workflow sequence greater than {previous}"),
7317 &sequence.to_string(),
7318 "durable commands are not strictly ordered by their recorded workflow sequence",
7319 ));
7320 }
7321 }
7322 last_new_sequence = Some(sequence);
7323 }
7324 events_by_sequence.entry(sequence).or_default().push(event);
7325 }
7326
7327 let commands: Vec<RecordedCommand> = events_by_sequence
7328 .into_iter()
7329 .map(|(sequence, sequence_events)| {
7330 let activity_events: Vec<_> = sequence_events
7331 .iter()
7332 .copied()
7333 .filter(|event| event.event_type.starts_with("Activity"))
7334 .collect();
7335 let timer_events: Vec<_> = sequence_events
7336 .iter()
7337 .copied()
7338 .filter(|event| event.event_type.starts_with("Timer"))
7339 .collect();
7340 let child_events: Vec<_> = sequence_events
7341 .iter()
7342 .copied()
7343 .filter(|event| {
7344 event.event_type == "ChildWorkflowScheduled"
7345 || event.event_type.starts_with("ChildRun")
7346 })
7347 .collect();
7348 let signal_wait_events: Vec<_> = sequence_events
7349 .iter()
7350 .copied()
7351 .filter(|event| is_recorded_signal_wait_event(event))
7352 .collect();
7353 let side_effect_events: Vec<_> = sequence_events
7354 .iter()
7355 .copied()
7356 .filter(|event| event.event_type == "SideEffectRecorded")
7357 .collect();
7358 let version_marker_events: Vec<_> = sequence_events
7359 .iter()
7360 .copied()
7361 .filter(|event| event.event_type == "VersionMarkerRecorded")
7362 .collect();
7363
7364 let command_kind_count = usize::from(!activity_events.is_empty())
7365 + usize::from(!timer_events.is_empty())
7366 + usize::from(!child_events.is_empty())
7367 + usize::from(!signal_wait_events.is_empty())
7368 + usize::from(!side_effect_events.is_empty())
7369 + usize::from(!version_marker_events.is_empty());
7370 if command_kind_count > 1 {
7371 let actual = [
7372 (!activity_events.is_empty()).then_some("activity"),
7373 (!timer_events.is_empty()).then_some("timer"),
7374 (!child_events.is_empty()).then_some("child workflow"),
7375 (!signal_wait_events.is_empty()).then_some("signal wait"),
7376 (!side_effect_events.is_empty()).then_some("side effect"),
7377 (!version_marker_events.is_empty()).then_some("version marker"),
7378 ]
7379 .into_iter()
7380 .flatten()
7381 .collect::<Vec<_>>()
7382 .join(" and ");
7383 return Err(invalid_recorded_history(
7384 "durable_command_sequence_collision",
7385 sequence,
7386 "one durable command kind",
7387 &actual,
7388 "one workflow sequence records more than one durable command kind",
7389 ));
7390 }
7391
7392 if !activity_events.is_empty() {
7393 let scheduled_count = activity_events
7394 .iter()
7395 .filter(|event| event.event_type == "ActivityScheduled")
7396 .count();
7397 if scheduled_count > 1 {
7398 return Err(invalid_recorded_history(
7399 "duplicate_activity_schedule",
7400 sequence,
7401 "at most one ActivityScheduled event",
7402 "multiple ActivityScheduled events",
7403 "activity history schedules more than one command at one workflow sequence",
7404 ));
7405 }
7406 let activity_type = activity_events.iter().find_map(|event| {
7407 event
7408 .payload
7409 .get("activity_type")
7410 .or_else(|| event.payload.get("activity_name"))
7411 .and_then(Value::as_str)
7412 .map(str::to_string)
7413 });
7414 if activity_events.iter().filter_map(|event| {
7415 event
7416 .payload
7417 .get("activity_type")
7418 .or_else(|| event.payload.get("activity_name"))
7419 .and_then(Value::as_str)
7420 }).any(|candidate| Some(candidate) != activity_type.as_deref()) {
7421 return Err(invalid_recorded_history(
7422 "activity_identity_mismatch",
7423 sequence,
7424 activity_type.as_deref().unwrap_or("one activity identity"),
7425 "conflicting activity identities",
7426 "activity lifecycle events at one workflow sequence disagree on identity",
7427 ));
7428 }
7429 let terminal: Vec<_> = activity_events
7430 .iter()
7431 .copied()
7432 .filter(|event| {
7433 matches!(
7434 event.event_type.as_str(),
7435 "ActivityCompleted"
7436 | "ActivityFailed"
7437 | "ActivityCancelled"
7438 | "ActivityTimedOut"
7439 )
7440 })
7441 .collect();
7442 if terminal.len() > 1 {
7443 return Err(invalid_recorded_history(
7444 "duplicate_activity_terminal_event",
7445 sequence,
7446 "at most one terminal activity event",
7447 "multiple terminal activity events",
7448 "activity history settles one command more than once",
7449 ));
7450 }
7451 let outcome = terminal
7452 .first()
7453 .map(|event| activity_outcome(event, fallback_codec, activity_type.clone()))
7454 .transpose()?;
7455 let options = activity_events
7456 .iter()
7457 .find(|event| event.event_type == "ActivityScheduled")
7458 .and_then(|event| event.payload.get("activity"))
7459 .and_then(Value::as_object)
7460 .map(|activity| RecordedActivityOptions {
7461 task_queue: recorded_optional_string(activity, "queue"),
7462 execution_mode: recorded_optional_string(activity, "execution_mode"),
7463 retry_policy: recorded_activity_retry_snapshot(
7464 activity.get("retry_policy"),
7465 ),
7466 });
7467 return Ok(RecordedCommand::Activity {
7468 sequence,
7469 activity_type,
7470 options,
7471 outcome,
7472 });
7473 }
7474
7475 if !child_events.is_empty() {
7476 let scheduled: Vec<_> = child_events
7477 .iter()
7478 .copied()
7479 .filter(|event| event.event_type == "ChildWorkflowScheduled")
7480 .collect();
7481 if scheduled.len() != 1 {
7482 return Err(invalid_recorded_history(
7483 "child_workflow_schedule_missing_or_duplicate",
7484 sequence,
7485 "one ChildWorkflowScheduled event",
7486 &format!("{} ChildWorkflowScheduled events", scheduled.len()),
7487 "child workflow replay requires exactly one recorded schedule event",
7488 ));
7489 }
7490 let workflow_type = child_events.iter().find_map(|event| {
7491 event
7492 .payload
7493 .get("child_workflow_type")
7494 .or_else(|| event.payload.get("workflow_type"))
7495 .and_then(Value::as_str)
7496 .filter(|value| !value.is_empty())
7497 .map(str::to_string)
7498 });
7499 if child_events
7500 .iter()
7501 .filter_map(|event| {
7502 event
7503 .payload
7504 .get("child_workflow_type")
7505 .or_else(|| event.payload.get("workflow_type"))
7506 .and_then(Value::as_str)
7507 })
7508 .any(|candidate| Some(candidate) != workflow_type.as_deref())
7509 {
7510 return Err(invalid_recorded_history(
7511 "child_workflow_identity_mismatch",
7512 sequence,
7513 workflow_type
7514 .as_deref()
7515 .unwrap_or("one child workflow type"),
7516 "conflicting child workflow types",
7517 "child workflow lifecycle events at one sequence disagree on type",
7518 ));
7519 }
7520 let mut outcomes = child_workflow_outcomes(
7521 &child_events.iter().map(|event| (*event).clone()).collect::<Vec<_>>(),
7522 fallback_codec,
7523 parent.clone(),
7524 )?;
7525 if outcomes.len() > 1 {
7526 return Err(invalid_recorded_history(
7527 "duplicate_child_workflow_terminal_event",
7528 sequence,
7529 "at most one terminal child event",
7530 "multiple terminal child events",
7531 "child workflow history settles one command more than once",
7532 ));
7533 }
7534 return Ok(RecordedCommand::ChildWorkflow {
7535 sequence,
7536 workflow_type,
7537 outcome: outcomes.pop(),
7538 });
7539 }
7540
7541 if !signal_wait_events.is_empty() {
7542 let opened: Vec<_> = signal_wait_events
7543 .iter()
7544 .copied()
7545 .filter(|event| event.event_type == "SignalWaitOpened")
7546 .collect();
7547 if opened.len() != 1 {
7548 return Err(invalid_recorded_history(
7549 "signal_wait_open_missing_or_duplicate",
7550 sequence,
7551 "one SignalWaitOpened event",
7552 &format!("{} SignalWaitOpened events", opened.len()),
7553 "signal replay requires exactly one canonical wait-open event",
7554 ));
7555 }
7556
7557 let applied: Vec<_> = signal_wait_events
7558 .iter()
7559 .copied()
7560 .filter(|event| event.event_type == "SignalApplied")
7561 .collect();
7562 if applied.len() > 1 {
7563 return Err(invalid_recorded_history(
7564 "duplicate_signal_wait_apply",
7565 sequence,
7566 "at most one SignalApplied event",
7567 "multiple SignalApplied events",
7568 "signal history applies one durable wait more than once",
7569 ));
7570 }
7571
7572 let signal_names = signal_wait_events
7573 .iter()
7574 .map(|event| required_signal_wait_name(event, sequence))
7575 .collect::<Result<Vec<_>>>()?;
7576 let signal_name = signal_names
7577 .first()
7578 .expect("signal wait events are not empty")
7579 .clone();
7580 if signal_names.iter().any(|candidate| candidate != &signal_name) {
7581 return Err(invalid_recorded_history(
7582 "signal_wait_identity_mismatch",
7583 sequence,
7584 &signal_name,
7585 "conflicting signal names",
7586 "signal wait lifecycle events at one workflow sequence disagree on identity",
7587 ));
7588 }
7589 let value = applied
7590 .first()
7591 .map(|event| decode_signal_event_arguments(event, fallback_codec))
7592 .transpose()?;
7593 return Ok(RecordedCommand::SignalWait {
7594 sequence,
7595 signal_name,
7596 value,
7597 });
7598 }
7599
7600 if !side_effect_events.is_empty() {
7601 if side_effect_events.len() != 1 {
7602 return Err(invalid_recorded_history(
7603 "duplicate_side_effect_record",
7604 sequence,
7605 "one SideEffectRecorded event",
7606 &format!("{} SideEffectRecorded events", side_effect_events.len()),
7607 "side-effect history records one workflow command more than once",
7608 ));
7609 }
7610 let event = side_effect_events[0];
7611 let result = event.payload.get("result").ok_or_else(|| {
7612 invalid_recorded_history(
7613 "side_effect_result_missing",
7614 sequence,
7615 "recorded result payload",
7616 "missing result",
7617 "side-effect history is missing its recorded value",
7618 )
7619 })?;
7620 let has_published_envelope = result.as_str().is_some()
7621 || result.as_object().is_some_and(|envelope| {
7622 envelope.get("codec").and_then(Value::as_str).is_some()
7623 && envelope.get("blob").and_then(Value::as_str).is_some()
7624 });
7625 if !has_published_envelope {
7626 return Err(invalid_recorded_history(
7627 "side_effect_payload_malformed",
7628 sequence,
7629 "payload blob or {codec, blob} envelope",
7630 &result.to_string(),
7631 "side-effect history result does not use a published payload envelope",
7632 ));
7633 }
7634 let codec = event
7635 .payload
7636 .get("payload_codec")
7637 .and_then(Value::as_str)
7638 .unwrap_or(fallback_codec);
7639 let value = decode_wire_avro_value(result, codec).map_err(|error| {
7640 if error.to_string().contains("unsupported_payload_codec") {
7641 return error;
7642 }
7643
7644 invalid_recorded_history(
7645 "side_effect_payload_incompatible",
7646 sequence,
7647 &format!("valid {codec} payload envelope"),
7648 &error.to_string(),
7649 "side-effect history payload cannot be decoded with its recorded codec",
7650 )
7651 })?;
7652 return Ok(RecordedCommand::SideEffect { sequence, value });
7653 }
7654
7655 if !version_marker_events.is_empty() {
7656 if version_marker_events.len() != 1 {
7657 return Err(invalid_recorded_history(
7658 "duplicate_version_marker_record",
7659 sequence,
7660 "one VersionMarkerRecorded event",
7661 &format!("{} VersionMarkerRecorded events", version_marker_events.len()),
7662 "version-marker history records one workflow command more than once",
7663 ));
7664 }
7665 let payload = &version_marker_events[0].payload;
7666 let change_id = payload
7667 .get("change_id")
7668 .and_then(Value::as_str)
7669 .filter(|value| !value.is_empty())
7670 .map(str::to_string)
7671 .ok_or_else(|| {
7672 invalid_recorded_history(
7673 "version_marker_field_missing",
7674 sequence,
7675 "non-empty change_id",
7676 "missing or invalid change_id",
7677 "version-marker history is missing its stable change ID",
7678 )
7679 })?;
7680 let version = required_version_i32(payload, "version", sequence)?;
7681 let min_supported = required_version_i32(payload, "min_supported", sequence)?;
7682 let max_supported = required_version_i32(payload, "max_supported", sequence)?;
7683 if min_supported > max_supported || version < min_supported || version > max_supported {
7684 return Err(invalid_recorded_history(
7685 "version_marker_history_range_invalid",
7686 sequence,
7687 "min_supported <= version <= max_supported",
7688 &format!("{min_supported} <= {version} <= {max_supported}"),
7689 "recorded version marker contains an internally incompatible range",
7690 ));
7691 }
7692 return Ok(RecordedCommand::VersionMarker {
7693 sequence,
7694 change_id,
7695 version,
7696 });
7697 }
7698
7699 let scheduled: Vec<_> = timer_events
7700 .iter()
7701 .copied()
7702 .filter(|event| event.event_type == "TimerScheduled")
7703 .collect();
7704 let fired: Vec<_> = timer_events
7705 .iter()
7706 .copied()
7707 .filter(|event| event.event_type == "TimerFired")
7708 .collect();
7709 if scheduled.len() != 1 {
7710 return Err(invalid_recorded_history(
7711 "timer_schedule_missing_or_duplicate",
7712 sequence,
7713 "one TimerScheduled event",
7714 &format!("{} TimerScheduled events", scheduled.len()),
7715 "timer replay requires exactly one recorded schedule event",
7716 ));
7717 }
7718 if fired.len() > 1 {
7719 return Err(invalid_recorded_history(
7720 "duplicate_timer_fire",
7721 sequence,
7722 "at most one TimerFired event",
7723 "multiple TimerFired events",
7724 "timer history contains more than one fire event for a workflow sequence",
7725 ));
7726 }
7727
7728 let scheduled = scheduled[0];
7729 let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
7730 let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
7731 if let Some(fired) = fired.first() {
7732 let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
7733 if fired_timer_id != timer_id {
7734 return Err(invalid_recorded_history(
7735 "timer_identity_mismatch",
7736 sequence,
7737 &timer_id,
7738 &fired_timer_id,
7739 "TimerFired does not correspond to the recorded TimerScheduled event",
7740 ));
7741 }
7742 let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
7743 if fired_delay != delay_seconds {
7744 return Err(invalid_recorded_history(
7745 "timer_history_delay_mismatch",
7746 sequence,
7747 &delay_seconds.to_string(),
7748 &fired_delay.to_string(),
7749 "TimerScheduled and TimerFired record different delays",
7750 ));
7751 }
7752 }
7753
7754 Ok(RecordedCommand::Timer {
7755 sequence,
7756 delay_seconds,
7757 fired: !fired.is_empty(),
7758 })
7759 })
7760 .collect::<Result<_>>()?;
7761
7762 let mut marker_sequences = HashMap::new();
7763 for command in &commands {
7764 if let RecordedCommand::VersionMarker {
7765 sequence,
7766 change_id,
7767 ..
7768 } = command
7769 {
7770 if let Some(first_sequence) = marker_sequences.insert(change_id.clone(), *sequence) {
7771 return Err(invalid_recorded_history(
7772 "duplicate_version_marker",
7773 *sequence,
7774 &format!("one marker for change ID {change_id:?}"),
7775 &format!("markers at sequences {first_sequence} and {sequence}"),
7776 "workflow history contains duplicate markers for one stable change ID",
7777 ));
7778 }
7779 }
7780 }
7781
7782 Ok(commands)
7783}
7784
7785fn required_version_i32(payload: &Value, field: &str, sequence: u64) -> Result<i32> {
7786 payload
7787 .get(field)
7788 .and_then(Value::as_i64)
7789 .and_then(|value| i32::try_from(value).ok())
7790 .ok_or_else(|| {
7791 invalid_recorded_history(
7792 "version_marker_field_missing",
7793 sequence,
7794 &format!("integer {field}"),
7795 "missing or out-of-range integer",
7796 "version-marker history is missing a required integer field",
7797 )
7798 })
7799}
7800
7801fn durable_event_sequence(event: &HistoryEvent) -> Option<u64> {
7802 event
7803 .payload
7804 .get("sequence")
7805 .or_else(|| event.payload.get("workflow_sequence"))
7806 .or_else(|| event.raw.get("sequence"))
7807 .or_else(|| event.raw.get("workflow_sequence"))
7808 .and_then(value_as_u64)
7809}
7810
7811fn is_internal_timer_event(event: &HistoryEvent) -> bool {
7812 matches!(
7813 event
7814 .payload
7815 .get("timer_kind")
7816 .or_else(|| event.raw.get("timer_kind"))
7817 .and_then(Value::as_str),
7818 Some("condition_timeout" | "signal_timeout")
7819 )
7820}
7821
7822fn required_signal_wait_name(event: &HistoryEvent, sequence: u64) -> Result<String> {
7823 event
7824 .payload
7825 .get("signal_name")
7826 .or_else(|| event.raw.get("signal_name"))
7827 .and_then(Value::as_str)
7828 .filter(|value| !value.is_empty())
7829 .map(str::to_string)
7830 .ok_or_else(|| {
7831 invalid_recorded_history(
7832 "signal_wait_name_missing",
7833 sequence,
7834 "non-empty signal_name",
7835 &event.event_type,
7836 "canonical signal-wait history is missing its signal identity",
7837 )
7838 })
7839}
7840
7841fn is_recorded_signal_wait_event(event: &HistoryEvent) -> bool {
7842 matches!(
7843 event.event_type.as_str(),
7844 "SignalWaitOpened" | "SignalApplied"
7845 )
7846}
7847
7848fn required_history_string(event: &HistoryEvent, field: &str, sequence: u64) -> Result<String> {
7849 event
7850 .payload
7851 .get(field)
7852 .and_then(Value::as_str)
7853 .filter(|value| !value.is_empty())
7854 .map(str::to_string)
7855 .ok_or_else(|| {
7856 invalid_recorded_history(
7857 "timer_history_field_missing",
7858 sequence,
7859 field,
7860 &event.event_type,
7861 "timer history is missing a required identity field",
7862 )
7863 })
7864}
7865
7866fn required_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<u64> {
7867 event
7868 .payload
7869 .get(field)
7870 .and_then(value_as_u64)
7871 .ok_or_else(|| {
7872 invalid_recorded_history(
7873 "timer_history_field_missing",
7874 sequence,
7875 field,
7876 &event.event_type,
7877 "timer history is missing a required numeric field",
7878 )
7879 })
7880}
7881
7882fn invalid_recorded_history(
7883 reason: &str,
7884 sequence: u64,
7885 expected: &str,
7886 actual: &str,
7887 message: &str,
7888) -> Error {
7889 Error::NonDeterministicReplay(ReplayFailure::new(
7890 reason,
7891 Some(sequence),
7892 Some(expected.to_string()),
7893 Some(actual.to_string()),
7894 message,
7895 ))
7896}
7897
7898type ActivityOutcome = std::result::Result<AvroValue, ActivityFailure>;
7899
7900fn activity_outcome(
7901 event: &HistoryEvent,
7902 fallback_codec: &str,
7903 recorded_activity_type: Option<String>,
7904) -> Result<ActivityOutcome> {
7905 if event.event_type == "ActivityCompleted" {
7906 let codec = event
7907 .payload
7908 .get("payload_codec")
7909 .and_then(Value::as_str)
7910 .unwrap_or(fallback_codec);
7911 return Ok(Ok(decode_wire_avro_value(
7912 event.payload.get("result").unwrap_or(&Value::Null),
7913 codec,
7914 )?));
7915 }
7916
7917 let payload = &event.payload;
7918 let (kind, fallback_reason, fallback_message) = match event.event_type.as_str() {
7919 "ActivityFailed" => (ActivityFailureKind::Failed, "activity", "activity failed"),
7920 "ActivityCancelled" => (
7921 ActivityFailureKind::Cancelled,
7922 "cancelled",
7923 "activity was cancelled",
7924 ),
7925 "ActivityTimedOut" => (
7926 ActivityFailureKind::TimedOut,
7927 "timeout",
7928 "activity timed out",
7929 ),
7930 _ => unreachable!("activity_outcome is called only for terminal activity events"),
7931 };
7932 let exception = payload
7933 .get("exception")
7934 .filter(|value| !value.is_null())
7935 .cloned();
7936 let failure_category = payload_string(payload, "failure_category");
7937 let timeout_kind = payload_string(payload, "timeout_kind");
7938 let reason = payload_string(payload, "reason").unwrap_or_else(|| match kind {
7939 ActivityFailureKind::Failed => failure_category
7940 .clone()
7941 .unwrap_or_else(|| fallback_reason.to_string()),
7942 ActivityFailureKind::Cancelled => fallback_reason.to_string(),
7943 ActivityFailureKind::TimedOut => timeout_kind
7944 .clone()
7945 .unwrap_or_else(|| fallback_reason.to_string()),
7946 });
7947 let message = payload_string(payload, "message")
7948 .or_else(|| {
7949 exception
7950 .as_ref()
7951 .and_then(|value| payload_string(value, "message"))
7952 })
7953 .unwrap_or_else(|| fallback_message.to_string());
7954
7955 Ok(Err(ActivityFailure {
7956 kind,
7957 reason,
7958 message,
7959 activity_execution_id: payload_string(payload, "activity_execution_id"),
7960 activity_attempt_id: payload_string(payload, "activity_attempt_id"),
7961 activity_type: payload_string(payload, "activity_type")
7962 .or_else(|| payload_string(payload, "activity_name"))
7963 .or(recorded_activity_type),
7964 activity_class: payload_string(payload, "activity_class"),
7965 attempt_number: payload.get("attempt_number").and_then(value_as_u64),
7966 failure_id: payload_string(payload, "failure_id"),
7967 failure_category,
7968 timeout_kind,
7969 non_retryable: payload
7970 .get("non_retryable")
7971 .and_then(Value::as_bool)
7972 .unwrap_or(false),
7973 exception_type: payload_string(payload, "exception_type").or_else(|| {
7974 exception
7975 .as_ref()
7976 .and_then(|value| payload_string(value, "type"))
7977 }),
7978 exception_class: payload_string(payload, "exception_class").or_else(|| {
7979 exception
7980 .as_ref()
7981 .and_then(|value| payload_string(value, "class"))
7982 }),
7983 code: payload
7984 .get("code")
7985 .filter(|value| !value.is_null())
7986 .cloned(),
7987 exception,
7988 }))
7989}
7990
7991type ChildWorkflowOutcome = std::result::Result<ChildWorkflowAvroResult, ChildWorkflowFailure>;
7992
7993fn child_workflow_outcomes(
7994 events: &[HistoryEvent],
7995 fallback_codec: &str,
7996 parent: WorkflowIdentity,
7997) -> Result<Vec<ChildWorkflowOutcome>> {
7998 let mut outcomes = Vec::new();
7999
8000 for event in events {
8001 let kind = match event.event_type.as_str() {
8002 "ChildRunCompleted" => None,
8003 "ChildRunFailed" => Some((
8004 ChildWorkflowFailureKind::Failed,
8005 "child_workflow",
8006 "child workflow failed",
8007 )),
8008 "ChildRunCancelled" => Some((
8009 ChildWorkflowFailureKind::Cancelled,
8010 "cancelled",
8011 "child workflow was cancelled",
8012 )),
8013 "ChildRunTerminated" => Some((
8014 ChildWorkflowFailureKind::Terminated,
8015 "terminated",
8016 "child workflow was terminated",
8017 )),
8018 _ => continue,
8019 };
8020 let payload = &event.payload;
8021 let child_workflow_id = payload_string(payload, "child_workflow_instance_id");
8022 let child_workflow_run_id = payload_string(payload, "child_workflow_run_id");
8023 let child_workflow_type = payload_string(payload, "child_workflow_type");
8024
8025 if let Some((kind, reason, fallback_message)) = kind {
8026 let exception = payload
8027 .get("exception")
8028 .filter(|value| !value.is_null())
8029 .cloned();
8030 let message = payload_string(payload, "message")
8031 .or_else(|| {
8032 exception
8033 .as_ref()
8034 .and_then(|value| payload_string(value, "message"))
8035 })
8036 .unwrap_or_else(|| fallback_message.to_string());
8037 let exception_type = payload_string(payload, "exception_type").or_else(|| {
8038 exception
8039 .as_ref()
8040 .and_then(|value| payload_string(value, "type"))
8041 });
8042 let exception_class = payload_string(payload, "exception_class").or_else(|| {
8043 exception
8044 .as_ref()
8045 .and_then(|value| payload_string(value, "class"))
8046 });
8047 outcomes.push(Err(ChildWorkflowFailure {
8048 kind,
8049 reason: reason.to_string(),
8050 message,
8051 parent_workflow_id: parent.workflow_id.clone(),
8052 parent_workflow_run_id: parent.run_id.clone(),
8053 child_workflow_id,
8054 child_workflow_run_id,
8055 child_workflow_type,
8056 failure_id: payload_string(payload, "failure_id"),
8057 failure_category: payload_string(payload, "failure_category"),
8058 exception_type,
8059 exception_class,
8060 non_retryable: payload
8061 .get("non_retryable")
8062 .and_then(Value::as_bool)
8063 .unwrap_or(false),
8064 code: payload
8065 .get("code")
8066 .filter(|value| !value.is_null())
8067 .cloned(),
8068 exception,
8069 }));
8070 continue;
8071 }
8072
8073 let codec = payload
8074 .get("payload_codec")
8075 .and_then(Value::as_str)
8076 .unwrap_or(fallback_codec);
8077 let result = payload
8078 .get("result")
8079 .or_else(|| payload.get("output"))
8080 .unwrap_or(&Value::Null);
8081 outcomes.push(Ok(ChildWorkflowAvroResult {
8082 parent: parent.clone(),
8083 child: WorkflowIdentity {
8084 workflow_id: child_workflow_id,
8085 run_id: child_workflow_run_id,
8086 },
8087 child_workflow_type,
8088 result: decode_wire_avro_value(result, codec)?,
8089 }));
8090 }
8091
8092 Ok(outcomes)
8093}
8094
8095fn payload_string(payload: &Value, key: &str) -> Option<String> {
8096 payload
8097 .get(key)
8098 .and_then(Value::as_str)
8099 .filter(|value| !value.is_empty())
8100 .map(str::to_string)
8101}
8102
8103fn workflow_failure_command(error: &Error) -> Value {
8104 let (exception_type, exception_class, properties) = match error {
8105 Error::ActivityFailed(failure) => (
8106 match failure.kind {
8107 ActivityFailureKind::Failed => "ActivityFailed",
8108 ActivityFailureKind::Cancelled => "ActivityCancelled",
8109 ActivityFailureKind::TimedOut => "ActivityTimedOut",
8110 },
8111 "durable_workflow::ActivityFailure",
8112 json!({
8113 "reason": failure.reason,
8114 "activity_execution_id": failure.activity_execution_id,
8115 "activity_attempt_id": failure.activity_attempt_id,
8116 "activity_type": failure.activity_type,
8117 "activity_class": failure.activity_class,
8118 "attempt_number": failure.attempt_number,
8119 "failure_id": failure.failure_id,
8120 "failure_category": failure.failure_category,
8121 "timeout_kind": failure.timeout_kind,
8122 "activity_non_retryable": failure.non_retryable,
8123 "activity_exception_type": failure.exception_type,
8124 "activity_exception_class": failure.exception_class,
8125 "activity_code": failure.code,
8126 "activity_exception": failure.exception,
8127 }),
8128 ),
8129 Error::ChildWorkflowFailed(failure) => (
8130 match failure.kind {
8131 ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
8132 ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
8133 ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
8134 },
8135 "durable_workflow::ChildWorkflowFailure",
8136 json!({
8137 "reason": failure.reason,
8138 "parent_workflow_id": failure.parent_workflow_id,
8139 "parent_workflow_run_id": failure.parent_workflow_run_id,
8140 "child_workflow_id": failure.child_workflow_id,
8141 "child_workflow_run_id": failure.child_workflow_run_id,
8142 "child_workflow_type": failure.child_workflow_type,
8143 "failure_id": failure.failure_id,
8144 "failure_category": failure.failure_category,
8145 "child_exception_type": failure.exception_type,
8146 "child_exception_class": failure.exception_class,
8147 "child_non_retryable": failure.non_retryable,
8148 "child_code": failure.code,
8149 "child_exception": failure.exception,
8150 }),
8151 ),
8152 Error::NonDeterministicReplay(_) => (
8153 "NonDeterministicReplay",
8154 "durable_workflow::Error",
8155 Value::Null,
8156 ),
8157 _ => ("RustWorkflowError", "durable_workflow::Error", Value::Null),
8158 };
8159 let non_retryable = match error {
8160 Error::ActivityFailed(failure) => failure.non_retryable,
8161 Error::ChildWorkflowFailed(failure) => failure.non_retryable,
8162 Error::NonDeterministicReplay(_) => true,
8163 _ => false,
8164 };
8165
8166 json!({
8167 "type": "fail_workflow",
8168 "message": error.to_string(),
8169 "exception_type": exception_type,
8170 "exception_class": exception_class,
8171 "non_retryable": non_retryable,
8172 "exception": {
8173 "type": exception_type,
8174 "class": exception_class,
8175 "message": error.to_string(),
8176 "properties": properties,
8177 }
8178 })
8179}
8180
8181fn workflow_task_integrity_error(error: &Error) -> bool {
8182 matches!(
8183 error,
8184 Error::NonDeterministicReplay(_) | Error::Protocol(_) | Error::WorkflowStatePoisoned
8185 )
8186}
8187
8188fn decode_signal_event_arguments(
8189 event: &HistoryEvent,
8190 fallback_codec: &str,
8191) -> Result<Vec<AvroValue>> {
8192 let codec = declared_payload_codec(&event.payload, "payload_codec")?.unwrap_or(fallback_codec);
8193 validate_payload_codec(codec)?;
8194 let raw = signal_history_payload(&event.payload);
8195 let decoded = match raw.filter(|value| !value.is_null()) {
8196 Some(value) => decode_wire_avro_value(value, codec)?,
8197 None => AvroValue::Array(Vec::new()),
8198 };
8199 let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
8200 unreachable!("normalize_avro_arguments always returns an array");
8201 };
8202 Ok(arguments)
8203}
8204
8205fn hydrate_query_history_from_export(task: &mut QueryTask) -> Result<()> {
8206 let Some(export_events) = task
8207 .history_export
8208 .as_ref()
8209 .and_then(|export| export.get("history_events"))
8210 .and_then(Value::as_array)
8211 else {
8212 return Ok(());
8213 };
8214
8215 if export_events.len() > task.history_events.len() {
8216 task.history_events = serde_json::from_value(Value::Array(export_events.clone()))?;
8217 }
8218
8219 Ok(())
8220}
8221
8222fn enrich_query_history_from_export(task: &mut QueryTask) -> Result<()> {
8223 let Some(export) = task.history_export.as_ref() else {
8224 return Ok(());
8225 };
8226 let signals = export
8227 .get("signals")
8228 .and_then(Value::as_array)
8229 .cloned()
8230 .unwrap_or_default();
8231 let activities = export
8232 .get("activities")
8233 .and_then(Value::as_array)
8234 .cloned()
8235 .unwrap_or_default();
8236 let export_codec = export
8237 .get("payloads")
8238 .and_then(|payloads| payloads.get("codec"))
8239 .and_then(Value::as_str)
8240 .unwrap_or(&task.payload_codec)
8241 .to_string();
8242 let mut signal_name_offsets: HashMap<String, usize> = HashMap::new();
8243
8244 for event in &mut task.history_events {
8245 if event.event_type == "ActivityCompleted" {
8246 let sequence = event
8247 .payload
8248 .get("sequence")
8249 .or_else(|| event.payload.get("workflow_sequence"))
8250 .and_then(value_as_u64);
8251 let Some(activity) = sequence.and_then(|sequence| {
8252 activities.iter().find(|activity| {
8253 activity.get("sequence").and_then(value_as_u64) == Some(sequence)
8254 })
8255 }) else {
8256 continue;
8257 };
8258 let Some(payload) = event.payload.as_object_mut() else {
8259 continue;
8260 };
8261 if missing_payload(payload.get("result")) {
8262 if let Some(result) = activity
8263 .get("result")
8264 .filter(|value| !missing_payload(Some(value)))
8265 {
8266 payload.insert("result".to_string(), result.clone());
8267 }
8268 }
8269 for field in ["payload_codec", "activity_type"] {
8270 if payload
8271 .get(field)
8272 .and_then(Value::as_str)
8273 .unwrap_or_default()
8274 .is_empty()
8275 {
8276 if let Some(value) = activity.get(field) {
8277 payload.insert(field.to_string(), value.clone());
8278 }
8279 }
8280 }
8281 continue;
8282 }
8283
8284 if event.event_type != "SignalReceived" && event.event_type != "SignalApplied" {
8285 continue;
8286 }
8287 let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
8288 let command_id = event
8289 .payload
8290 .get("workflow_command_id")
8291 .or_else(|| event.raw.get("workflow_command_id"))
8292 .and_then(Value::as_str);
8293 let signal_name = event
8294 .payload
8295 .get("signal_name")
8296 .and_then(Value::as_str)
8297 .unwrap_or_default()
8298 .to_string();
8299 let matched = signals
8300 .iter()
8301 .find(|signal| {
8302 signal_id.is_some() && signal.get("id").and_then(Value::as_str) == signal_id
8303 })
8304 .or_else(|| {
8305 signals.iter().find(|signal| {
8306 command_id.is_some()
8307 && signal.get("command_id").and_then(Value::as_str) == command_id
8308 })
8309 })
8310 .or_else(|| {
8311 let offset = signal_name_offsets.entry(signal_name.clone()).or_default();
8312 let signal = signals
8313 .iter()
8314 .filter(|signal| {
8315 signal.get("name").and_then(Value::as_str) == Some(signal_name.as_str())
8316 })
8317 .nth(*offset);
8318 if signal.is_some() {
8319 *offset += 1;
8320 }
8321 signal
8322 });
8323 let Some(signal) = matched else {
8324 continue;
8325 };
8326 let signal_codec = signal
8327 .get("payload_codec")
8328 .and_then(Value::as_str)
8329 .unwrap_or(&export_codec);
8330 let Some(payload) = event.payload.as_object_mut() else {
8331 continue;
8332 };
8333 if missing_payload(payload.get("arguments")) {
8334 if let Some(arguments) = signal
8335 .get("arguments")
8336 .filter(|value| !missing_payload(Some(value)))
8337 {
8338 let envelope = match arguments {
8339 Value::String(blob) => json!({"codec": signal_codec, "blob": blob}),
8340 other => other.clone(),
8341 };
8342 payload.insert("arguments".to_string(), envelope);
8343 }
8344 }
8345 if payload
8346 .get("payload_codec")
8347 .and_then(Value::as_str)
8348 .unwrap_or_default()
8349 .is_empty()
8350 {
8351 payload.insert("payload_codec".to_string(), json!(signal_codec));
8352 }
8353 }
8354
8355 Ok(())
8356}
8357
8358fn missing_payload(value: Option<&Value>) -> bool {
8359 match value {
8360 None | Some(Value::Null) => true,
8361 Some(Value::String(value)) => value.is_empty(),
8362 Some(_) => false,
8363 }
8364}
8365
8366fn query_signal_events(task: &QueryTask) -> Result<Vec<QuerySignal>> {
8367 let export_signals = task
8368 .history_export
8369 .as_ref()
8370 .and_then(|export| export.get("signals"))
8371 .and_then(Value::as_array)
8372 .cloned()
8373 .unwrap_or_default();
8374 let export_codec = task
8375 .history_export
8376 .as_ref()
8377 .and_then(|export| export.get("payloads"))
8378 .and_then(|payloads| payloads.get("codec"))
8379 .and_then(Value::as_str)
8380 .unwrap_or(&task.payload_codec);
8381 let mut name_offsets: HashMap<String, usize> = HashMap::new();
8382 let mut signals = Vec::new();
8383
8384 for event in &task.history_events {
8385 if event.event_type != "SignalApplied" && event.event_type != "SignalReceived" {
8386 continue;
8387 }
8388
8389 let name = event
8390 .payload
8391 .get("signal_name")
8392 .and_then(Value::as_str)
8393 .unwrap_or_default();
8394 if name.is_empty() {
8395 continue;
8396 }
8397 let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
8398 let command_id = event
8399 .payload
8400 .get("workflow_command_id")
8401 .or_else(|| event.raw.get("workflow_command_id"))
8402 .and_then(Value::as_str);
8403 let matched_export = export_signals
8404 .iter()
8405 .find(|candidate| {
8406 signal_id.is_some() && candidate.get("id").and_then(Value::as_str) == signal_id
8407 })
8408 .or_else(|| {
8409 export_signals.iter().find(|candidate| {
8410 command_id.is_some()
8411 && candidate.get("command_id").and_then(Value::as_str) == command_id
8412 })
8413 })
8414 .or_else(|| {
8415 let offset = name_offsets.entry(name.to_string()).or_default();
8416 let candidate = export_signals
8417 .iter()
8418 .filter(|candidate| candidate.get("name").and_then(Value::as_str) == Some(name))
8419 .nth(*offset);
8420 if candidate.is_some() {
8421 *offset += 1;
8422 }
8423 candidate
8424 });
8425 let codec = event
8426 .payload
8427 .get("payload_codec")
8428 .and_then(Value::as_str)
8429 .or_else(|| {
8430 matched_export
8431 .and_then(|signal| signal.get("payload_codec"))
8432 .and_then(Value::as_str)
8433 })
8434 .unwrap_or(export_codec);
8435 let raw_arguments = signal_history_payload(&event.payload)
8436 .filter(|value| !value.is_null())
8437 .or_else(|| matched_export.and_then(|signal| signal.get("arguments")));
8438 let (arguments, avro_arguments) = decode_query_signal_arguments(raw_arguments, codec)?;
8439 let workflow_sequence = event
8440 .payload
8441 .get("workflow_sequence")
8442 .and_then(value_as_u64)
8443 .or_else(|| {
8444 matched_export
8445 .and_then(|signal| signal.get("workflow_sequence"))
8446 .and_then(value_as_u64)
8447 });
8448
8449 signals.push(QuerySignal {
8450 id: signal_id.map(str::to_string).or_else(|| {
8451 matched_export
8452 .and_then(|signal| signal.get("id"))
8453 .and_then(Value::as_str)
8454 .map(str::to_string)
8455 }),
8456 name: name.to_string(),
8457 arguments,
8458 avro_arguments,
8459 workflow_sequence,
8460 });
8461 }
8462
8463 if signals.is_empty() {
8464 for signal in export_signals {
8465 if signal.get("status").and_then(Value::as_str) == Some("rejected") {
8466 continue;
8467 }
8468 let Some(name) = signal.get("name").and_then(Value::as_str) else {
8469 continue;
8470 };
8471 let codec = signal
8472 .get("payload_codec")
8473 .and_then(Value::as_str)
8474 .unwrap_or(export_codec);
8475 let (arguments, avro_arguments) =
8476 decode_query_signal_arguments(signal.get("arguments"), codec)?;
8477 signals.push(QuerySignal {
8478 id: signal.get("id").and_then(Value::as_str).map(str::to_string),
8479 name: name.to_string(),
8480 arguments,
8481 avro_arguments,
8482 workflow_sequence: signal.get("workflow_sequence").and_then(value_as_u64),
8483 });
8484 }
8485 signals.sort_by_key(|signal| signal.workflow_sequence.unwrap_or(u64::MAX));
8486 }
8487
8488 Ok(signals)
8489}
8490
8491fn decode_query_signal_arguments(
8492 raw: Option<&Value>,
8493 codec: &str,
8494) -> Result<(Vec<Value>, Vec<AvroValue>)> {
8495 validate_payload_codec(codec)?;
8496 let decoded = match raw.filter(|value| !value.is_null()) {
8497 Some(value) => decode_wire_avro_value(value, codec)?,
8498 None => AvroValue::Array(Vec::new()),
8499 };
8500 let AvroValue::Array(avro_arguments) = normalize_avro_arguments(decoded) else {
8501 unreachable!("normalize_avro_arguments always returns an array");
8502 };
8503 let arguments = avro_arguments
8504 .iter()
8505 .cloned()
8506 .map(AvroValue::into_json)
8507 .collect::<Result<Vec<_>>>()?;
8508 Ok((arguments, avro_arguments))
8509}
8510
8511fn value_as_u64(value: &Value) -> Option<u64> {
8512 value
8513 .as_u64()
8514 .or_else(|| value.as_str().and_then(|value| value.parse().ok()))
8515}
8516
8517#[cfg(test)]
8518mod tests {
8519 use super::*;
8520 use std::{
8521 io::{Read, Write},
8522 net::{SocketAddr, TcpListener, TcpStream},
8523 sync::atomic::AtomicUsize,
8524 thread,
8525 };
8526
8527 fn fixture_envelope(value: Value) -> Value {
8528 encode_value_envelope(&value, DEFAULT_CODEC).expect("encode Avro test fixture")
8529 }
8530
8531 fn fixture_blob(value: Value) -> String {
8532 encode_payload(&value, DEFAULT_CODEC)
8533 .expect("encode Avro test fixture")
8534 .blob
8535 }
8536
8537 #[test]
8538 fn client_builder_rejects_the_sdk_owned_api_suffix() {
8539 for base_url in [
8540 "http://127.0.0.1:8080/api",
8541 "http://localhost:8080/api/",
8542 "https://runtime.example.test/namespaces/orders/api",
8543 ] {
8544 let error = Client::builder(base_url)
8545 .build()
8546 .expect_err("SDK-owned /api suffix must be rejected during build");
8547
8548 assert!(matches!(error, Error::InvalidBaseUrl), "{base_url}");
8549 assert!(
8550 error.to_string().contains("SDK appends /api automatically"),
8551 "the validation error must explain how to fix the endpoint"
8552 );
8553 }
8554 }
8555
8556 #[test]
8557 fn client_builder_preserves_self_hosted_and_managed_runtime_prefixes() {
8558 for (base_url, expected) in [
8559 ("http://127.0.0.1:8080", "http://127.0.0.1:8080"),
8560 (
8561 "http://localhost:8080/durable-workflow/",
8562 "http://localhost:8080/durable-workflow",
8563 ),
8564 (
8565 "https://runtime.example.test/namespaces/orders",
8566 "https://runtime.example.test/namespaces/orders",
8567 ),
8568 (
8569 "https://runtime.example.test/gateway/api/namespaces/orders",
8570 "https://runtime.example.test/gateway/api/namespaces/orders",
8571 ),
8572 (
8573 "https://api.example.test/runtime/orders/",
8574 "https://api.example.test/runtime/orders",
8575 ),
8576 ] {
8577 let client = Client::builder(base_url)
8578 .build()
8579 .expect("Server and Cloud runtime base URL must remain valid");
8580
8581 assert_eq!(client.base_url, expected);
8582 }
8583 }
8584
8585 fn typed_fidelity_probe() -> AvroValue {
8586 AvroValue::Map(BTreeMap::from([
8587 ("bytes".to_string(), AvroValue::Bytes(vec![0, 0xff])),
8588 ("empty".to_string(), AvroValue::Map(BTreeMap::new())),
8589 (
8590 "numeric".to_string(),
8591 AvroValue::Map(BTreeMap::from([
8592 ("0".to_string(), AvroValue::String("zero".to_string())),
8593 ("1".to_string(), AvroValue::String("one".to_string())),
8594 ])),
8595 ),
8596 (
8597 "nested".to_string(),
8598 AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([(
8599 "enabled".to_string(),
8600 AvroValue::Boolean(true),
8601 )]))]),
8602 ),
8603 (
8604 "projection_collisions".to_string(),
8605 AvroValue::Array(projection_collision_probe()),
8606 ),
8607 ]))
8608 }
8609
8610 fn projection_collision_probe() -> Vec<AvroValue> {
8611 vec![
8612 AvroValue::Map(BTreeMap::from([
8613 ("$type".to_string(), AvroValue::String("bytes".to_string())),
8614 (
8615 "base64".to_string(),
8616 AvroValue::String("ordinary user text".to_string()),
8617 ),
8618 ])),
8619 AvroValue::Map(BTreeMap::from([
8620 ("$type".to_string(), AvroValue::String("map".to_string())),
8621 (
8622 "entries".to_string(),
8623 AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([
8624 ("key".to_string(), AvroValue::String("ordinary".to_string())),
8625 (
8626 "value".to_string(),
8627 AvroValue::String("user map".to_string()),
8628 ),
8629 ]))]),
8630 ),
8631 ])),
8632 ]
8633 }
8634
8635 #[derive(Clone, Debug, Default, PartialEq)]
8636 struct ReplayCounterState {
8637 loaded: Option<String>,
8638 count: i64,
8639 finished: bool,
8640 }
8641
8642 fn replay_counter_worker() -> Worker {
8643 let client = Client::new("http://127.0.0.1:8080").expect("client");
8644 let mut worker = Worker::new(client, "rust-workers");
8645 worker.register_replayed_workflow(
8646 "replay-counter",
8647 ReplayCounterState::default,
8648 |ctx, _input, state| async move {
8649 let loaded = ctx.activity("load-counter", json!([])).await?;
8650 state.update(|current| {
8651 current.loaded = loaded.as_str().map(str::to_string);
8652 })?;
8653 for _ in 0..2 {
8654 let signal = ctx.wait_signal("increment").await?;
8655 let amount = signal.first().and_then(Value::as_i64).unwrap_or_default();
8656 state.update(|current| current.count += amount)?;
8657 }
8658 state.update(|current| current.finished = true)?;
8659 state.read(|current| Ok(json!(current.count)))?
8660 },
8661 );
8662 worker.register_replayed_query::<ReplayCounterState, _, _>(
8663 "replay-counter",
8664 "current",
8665 |_ctx, state, _args| async move {
8666 Ok(json!({
8667 "loaded": state.loaded,
8668 "count": state.count,
8669 "finished": state.finished,
8670 }))
8671 },
8672 );
8673 worker.register_replayed_query::<ReplayCounterState, _, _>(
8674 "replay-counter",
8675 "detached-mutation",
8676 |_ctx, state, _args| async move {
8677 let mut detached = (*state).clone();
8678 detached.count = 999;
8679 Ok(json!(detached.count))
8680 },
8681 );
8682 worker.register_replayed_query::<ReplayCounterState, _, _>(
8683 "replay-counter",
8684 "failed-mutation",
8685 |_ctx, state, _args| async move {
8686 let mut detached = (*state).clone();
8687 detached.count = 999;
8688 Err(Error::WorkerLoop("query refused".to_string()))
8689 },
8690 );
8691 worker
8692 }
8693
8694 fn replay_counter_query(
8695 query_name: &str,
8696 history_events: Value,
8697 run_status: &str,
8698 ) -> QueryTask {
8699 let arguments = fixture_envelope(json!([]));
8700 serde_json::from_value(json!({
8701 "query_task_id": format!("query-{query_name}"),
8702 "workflow_type": "replay-counter",
8703 "query_name": query_name,
8704 "payload_codec": DEFAULT_CODEC,
8705 "workflow_arguments": arguments.clone(),
8706 "query_arguments": arguments,
8707 "history_events": history_events,
8708 "run_status": run_status,
8709 }))
8710 .expect("query task")
8711 }
8712
8713 fn workflow_context(history: Vec<HistoryEvent>) -> WorkflowContext {
8714 workflow_context_with_codec(history, DEFAULT_CODEC)
8715 }
8716
8717 fn workflow_context_with_codec(
8718 history: Vec<HistoryEvent>,
8719 payload_codec: &str,
8720 ) -> WorkflowContext {
8721 WorkflowContext {
8722 state: Arc::new(Mutex::new(
8723 WorkflowState::new_with_identity(
8724 history,
8725 None,
8726 None,
8727 "rust-workers".to_string(),
8728 payload_codec.to_string(),
8729 None,
8730 )
8731 .expect("valid workflow history"),
8732 )),
8733 }
8734 }
8735
8736 fn history_event(event_type: &str, payload: Value) -> HistoryEvent {
8737 HistoryEvent {
8738 event_type: event_type.to_string(),
8739 payload,
8740 raw: HashMap::new(),
8741 }
8742 }
8743
8744 fn workflow_task(
8745 workflow_type: &str,
8746 history_events: Vec<HistoryEvent>,
8747 payload_codec: &str,
8748 ) -> WorkflowTask {
8749 WorkflowTask {
8750 task_id: format!("wft-{workflow_type}"),
8751 workflow_id: Some(format!("wf-{workflow_type}")),
8752 run_id: Some(format!("run-{workflow_type}")),
8753 workflow_type: workflow_type.to_string(),
8754 payload_codec: payload_codec.to_string(),
8755 arguments: Some(
8756 encode_value_envelope(&json!([]), payload_codec).expect("workflow arguments"),
8757 ),
8758 total_history_events: Some(history_events.len() as u64),
8759 history_size_bytes: None,
8760 continue_as_new_recommended: None,
8761 history_budget_pressure: None,
8762 history_events,
8763 next_history_page_token: None,
8764 workflow_task_attempt: 1,
8765 workflow_signal_id: None,
8766 signal_name: None,
8767 signal_arguments: None,
8768 workflow_update_id: None,
8769 update_name: None,
8770 lease_owner: Some("rust-worker".to_string()),
8771 }
8772 }
8773
8774 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
8775 struct SideEffectProbe {
8776 request_id: String,
8777 attempt: u32,
8778 }
8779
8780 #[test]
8781 fn typed_side_effect_runs_callback_once_and_replay_skips_it() {
8782 let calls = AtomicUsize::new(0);
8783 let ctx = workflow_context(Vec::new());
8784 let value = ctx
8785 .side_effect(|| {
8786 calls.fetch_add(1, Ordering::SeqCst);
8787 SideEffectProbe {
8788 request_id: "request-42".to_string(),
8789 attempt: 3,
8790 }
8791 })
8792 .expect("first side effect");
8793 assert_eq!(value.attempt, 3);
8794 assert_eq!(calls.load(Ordering::SeqCst), 1);
8795 let commands = ctx.take_commands().expect("commands");
8796 assert_eq!(commands.len(), 1);
8797 assert_eq!(commands[0]["type"], "record_side_effect");
8798 assert_eq!(
8799 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
8800 serde_json::to_value(&value).expect("value")
8801 );
8802
8803 let replay = workflow_context(vec![history_event(
8804 "SideEffectRecorded",
8805 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
8806 )]);
8807 let replayed: SideEffectProbe = replay
8808 .side_effect(|| {
8809 calls.fetch_add(1, Ordering::SeqCst);
8810 panic!("committed side-effect callbacks must not run during replay")
8811 })
8812 .expect("replayed side effect");
8813 assert_eq!(replayed, value);
8814 assert_eq!(calls.load(Ordering::SeqCst), 1);
8815 assert!(replay.take_commands().expect("commands").is_empty());
8816 replay.ensure_history_consumed().expect("history consumed");
8817 }
8818
8819 #[test]
8820 fn side_effect_uses_avro_envelope_and_uuid_is_replay_stable() {
8821 let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
8822 let value = ctx
8823 .side_effect(|| SideEffectProbe {
8824 request_id: "avro-request".to_string(),
8825 attempt: 1,
8826 })
8827 .expect("Avro side effect");
8828 let uuid = ctx.uuid_v4().expect("deterministic UUID");
8829 let commands = ctx.take_commands().expect("commands");
8830 assert_eq!(commands.len(), 2);
8831 assert_eq!(commands[0]["result"]["codec"], DEFAULT_CODEC);
8832 assert_eq!(commands[1]["result"]["codec"], DEFAULT_CODEC);
8833 assert_eq!(
8834 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
8835 serde_json::to_value(&value).expect("value")
8836 );
8837
8838 let replay = workflow_context_with_codec(
8839 vec![
8840 history_event(
8841 "SideEffectRecorded",
8842 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
8843 ),
8844 history_event(
8845 "SideEffectRecorded",
8846 json!({"sequence": 2, "result": commands[1]["result"].clone()}),
8847 ),
8848 ],
8849 DEFAULT_CODEC,
8850 );
8851 let replayed: SideEffectProbe = replay
8852 .side_effect(|| panic!("Avro callback must not run"))
8853 .expect("replayed Avro value");
8854 let replayed_uuid = replay.uuid_v4().expect("replayed UUID");
8855 assert_eq!(replayed, value);
8856 assert_eq!(replayed_uuid, uuid);
8857 assert!(replay.take_commands().expect("commands").is_empty());
8858 }
8859
8860 #[test]
8861 fn typed_side_effect_replay_preserves_bytes_and_maps() {
8862 let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
8863 let value = ctx
8864 .side_effect_avro_value(typed_fidelity_probe)
8865 .expect("typed side effect");
8866 let commands = ctx.take_commands().expect("side-effect command");
8867 assert_eq!(
8868 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
8869 .expect("recorded side effect"),
8870 value
8871 );
8872
8873 let replay = workflow_context_with_codec(
8874 vec![history_event(
8875 "SideEffectRecorded",
8876 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
8877 )],
8878 DEFAULT_CODEC,
8879 );
8880 assert_eq!(
8881 replay
8882 .side_effect_avro_value(|| panic!("replay must not invoke callback"))
8883 .expect("replayed typed side effect"),
8884 value
8885 );
8886 }
8887
8888 #[test]
8889 fn ordered_side_effects_share_the_durable_command_stream() {
8890 let first = encode_value_envelope(&json!("first"), DEFAULT_CODEC).expect("first");
8891 let second = encode_value_envelope(&json!(29), DEFAULT_CODEC).expect("second");
8892 let ctx = workflow_context(vec![
8893 history_event(
8894 "SideEffectRecorded",
8895 json!({"sequence": 1, "result": first}),
8896 ),
8897 history_event(
8898 "SideEffectRecorded",
8899 json!({"sequence": 2, "result": second}),
8900 ),
8901 ]);
8902 let first: String = ctx
8903 .side_effect(|| panic!("first callback must not run"))
8904 .expect("first replay");
8905 let second: i32 = ctx
8906 .side_effect(|| panic!("second callback must not run"))
8907 .expect("second replay");
8908 assert_eq!(first, "first");
8909 assert_eq!(second, 29);
8910 ctx.ensure_history_consumed().expect("ordered history");
8911
8912 let reordered = workflow_context(vec![history_event(
8913 "VersionMarkerRecorded",
8914 json!({
8915 "sequence": 1,
8916 "change_id": "before-side-effect",
8917 "version": 1,
8918 "min_supported": 1,
8919 "max_supported": 1,
8920 }),
8921 )]);
8922 let error = reordered
8923 .side_effect(|| "new".to_string())
8924 .expect_err("command reordering must fail");
8925 assert!(matches!(
8926 error,
8927 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8928 if reason == "recorded_command_mismatch"
8929 ));
8930 }
8931
8932 #[test]
8933 fn version_markers_replay_across_upgrades_and_do_not_duplicate() {
8934 let ctx = workflow_context(Vec::new());
8935 assert_eq!(ctx.get_version("checkout-v2", 1, 2).expect("version"), 2);
8936 assert_eq!(ctx.get_version("checkout-v2", 1, 3).expect("cached"), 2);
8937 assert!(ctx.patched("new-search").expect("patch"));
8938 ctx.deprecate_patch("new-search").expect("deprecate patch");
8939 let commands = ctx.take_commands().expect("commands");
8940 assert_eq!(commands.len(), 2);
8941 assert_eq!(commands[0]["type"], "record_version_marker");
8942 assert_eq!(commands[0]["version"], 2);
8943 assert_eq!(commands[1]["change_id"], "new-search");
8944
8945 let replay = workflow_context(vec![history_event(
8946 "VersionMarkerRecorded",
8947 json!({
8948 "sequence": 1,
8949 "change_id": "checkout-v2",
8950 "version": 2,
8951 "min_supported": 1,
8952 "max_supported": 2,
8953 }),
8954 )]);
8955 assert_eq!(replay.get_version("checkout-v2", 1, 4).expect("upgrade"), 2);
8956 assert_eq!(replay.get_version("checkout-v2", 2, 5).expect("repeat"), 2);
8957 assert!(replay.take_commands().expect("commands").is_empty());
8958 replay.ensure_history_consumed().expect("history consumed");
8959 }
8960
8961 #[test]
8962 fn version_markers_reject_incompatible_or_malformed_history() {
8963 let incompatible = workflow_context(vec![history_event(
8964 "VersionMarkerRecorded",
8965 json!({
8966 "sequence": 1,
8967 "change_id": "checkout-v2",
8968 "version": 1,
8969 "min_supported": 1,
8970 "max_supported": 2,
8971 }),
8972 )]);
8973 let error = incompatible
8974 .get_version("checkout-v2", 2, 3)
8975 .expect_err("old version is unsupported");
8976 assert!(matches!(
8977 error,
8978 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8979 if reason == "version_marker_incompatible_range"
8980 ));
8981
8982 for (history, reason) in [
8983 (
8984 vec![history_event("SideEffectRecorded", json!({"sequence": 1}))],
8985 "side_effect_result_missing",
8986 ),
8987 (
8988 vec![history_event(
8989 "SideEffectRecorded",
8990 json!({
8991 "sequence": 1,
8992 "result": {"codec": "avro", "blob": "not-base64"},
8993 }),
8994 )],
8995 "side_effect_payload_incompatible",
8996 ),
8997 (
8998 vec![history_event(
8999 "SideEffectRecorded",
9000 json!({"sequence": 1, "result": {"unwrapped": true}}),
9001 )],
9002 "side_effect_payload_malformed",
9003 ),
9004 (
9005 vec![history_event(
9006 "VersionMarkerRecorded",
9007 json!({
9008 "sequence": 1,
9009 "change_id": "change",
9010 "version": 1,
9011 "min_supported": 2,
9012 "max_supported": 1,
9013 }),
9014 )],
9015 "version_marker_history_range_invalid",
9016 ),
9017 ] {
9018 let error = WorkflowState::new(
9019 history,
9020 "rust-workers".to_string(),
9021 DEFAULT_CODEC.to_string(),
9022 None,
9023 )
9024 .expect_err("malformed history must fail");
9025 assert!(matches!(
9026 error,
9027 Error::NonDeterministicReplay(ReplayFailure { reason: actual, .. })
9028 if actual == reason
9029 ));
9030 }
9031 }
9032
9033 #[test]
9034 fn duplicate_side_effects_and_version_markers_are_rejected() {
9035 let duplicate_side_effect = WorkflowState::new(
9036 vec![
9037 history_event(
9038 "SideEffectRecorded",
9039 json!({"sequence": 1, "result": fixture_envelope(json!(1))}),
9040 ),
9041 history_event(
9042 "SideEffectRecorded",
9043 json!({"sequence": 1, "result": fixture_envelope(json!(2))}),
9044 ),
9045 ],
9046 "rust-workers".to_string(),
9047 DEFAULT_CODEC.to_string(),
9048 None,
9049 )
9050 .expect_err("duplicate side effect");
9051 assert!(matches!(
9052 duplicate_side_effect,
9053 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
9054 if reason == "duplicate_side_effect_record"
9055 ));
9056
9057 let marker = |sequence| {
9058 history_event(
9059 "VersionMarkerRecorded",
9060 json!({
9061 "sequence": sequence,
9062 "change_id": "same-change",
9063 "version": 1,
9064 "min_supported": 1,
9065 "max_supported": 1,
9066 }),
9067 )
9068 };
9069 let duplicate_marker = WorkflowState::new(
9070 vec![marker(1), marker(3)],
9071 "rust-workers".to_string(),
9072 DEFAULT_CODEC.to_string(),
9073 None,
9074 )
9075 .expect_err("duplicate marker");
9076 assert!(matches!(
9077 duplicate_marker,
9078 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
9079 if reason == "duplicate_version_marker"
9080 ));
9081 }
9082
9083 #[test]
9084 fn cold_worker_replay_does_not_repeat_committed_side_effects_or_markers() {
9085 fn worker(calls: Arc<AtomicUsize>) -> Worker {
9086 let client = Client::new("http://127.0.0.1:8080").expect("client");
9087 let mut worker = Worker::new(client, "rust-workers");
9088 worker.register_workflow("rust.side-effect-version", move |ctx, _input| {
9089 let calls = Arc::clone(&calls);
9090 async move {
9091 let captured = ctx.side_effect(|| {
9092 calls.fetch_add(1, Ordering::SeqCst);
9093 "captured-once".to_string()
9094 })?;
9095 let version = ctx.get_version("cold-restart", 1, 2)?;
9096 Ok(json!({"captured": captured, "version": version}))
9097 }
9098 });
9099 worker
9100 }
9101
9102 fn task(history_events: Vec<HistoryEvent>) -> WorkflowTask {
9103 WorkflowTask {
9104 task_id: "wft-side-effect-version".to_string(),
9105 workflow_id: Some("wf-side-effect-version".to_string()),
9106 run_id: Some("run-side-effect-version".to_string()),
9107 workflow_type: "rust.side-effect-version".to_string(),
9108 payload_codec: DEFAULT_CODEC.to_string(),
9109 arguments: Some(
9110 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("arguments"),
9111 ),
9112 history_events,
9113 total_history_events: None,
9114 history_size_bytes: None,
9115 continue_as_new_recommended: None,
9116 history_budget_pressure: None,
9117 next_history_page_token: None,
9118 workflow_task_attempt: 1,
9119 workflow_signal_id: None,
9120 signal_name: None,
9121 signal_arguments: None,
9122 workflow_update_id: None,
9123 update_name: None,
9124 lease_owner: Some("rust-worker".to_string()),
9125 }
9126 }
9127
9128 let calls = Arc::new(AtomicUsize::new(0));
9129 let initial = worker(Arc::clone(&calls))
9130 .execute_workflow_task(task(Vec::new()))
9131 .expect("initial execution");
9132 assert_eq!(
9133 initial
9134 .iter()
9135 .map(|command| &command["type"])
9136 .collect::<Vec<_>>(),
9137 vec![
9138 "record_side_effect",
9139 "record_version_marker",
9140 "complete_workflow"
9141 ]
9142 );
9143 assert_eq!(calls.load(Ordering::SeqCst), 1);
9144
9145 let restarted = worker(Arc::clone(&calls));
9146 let replayed = restarted
9147 .execute_workflow_task(task(vec![
9148 history_event(
9149 "SideEffectRecorded",
9150 json!({"sequence": 1, "result": initial[0]["result"].clone()}),
9151 ),
9152 history_event(
9153 "VersionMarkerRecorded",
9154 json!({
9155 "sequence": 2,
9156 "change_id": "cold-restart",
9157 "version": 2,
9158 "min_supported": 1,
9159 "max_supported": 2,
9160 }),
9161 ),
9162 ]))
9163 .expect("cold replay");
9164 assert_eq!(replayed.len(), 1);
9165 assert_eq!(replayed[0]["type"], "complete_workflow");
9166 assert_eq!(calls.load(Ordering::SeqCst), 1);
9167 }
9168
9169 #[test]
9170 fn side_effect_replay_rejects_changed_rust_value_type() {
9171 let result = encode_value_envelope(&json!({"value": 42}), DEFAULT_CODEC).expect("result");
9172 let ctx = workflow_context(vec![history_event(
9173 "SideEffectRecorded",
9174 json!({"sequence": 1, "result": result}),
9175 )]);
9176 let error = ctx
9177 .side_effect::<Vec<String>, _>(|| panic!("callback must not run"))
9178 .expect_err("changed type must fail replay");
9179 assert!(matches!(
9180 error,
9181 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
9182 if reason == "side_effect_type_mismatch"
9183 ));
9184 }
9185
9186 fn completed_retry_activity_history() -> Vec<HistoryEvent> {
9187 vec![
9188 history_event(
9189 "ActivityScheduled",
9190 json!({
9191 "sequence": 1,
9192 "activity_type": "flaky",
9193 "activity_execution_id": "act-1",
9194 "activity": {
9195 "id": "act-1",
9196 "sequence": 1,
9197 "type": "flaky",
9198 "queue": "critical-activities",
9199 "execution_mode": null,
9200 "retry_policy": {
9201 "snapshot_version": 1,
9202 "max_attempts": 3,
9203 "backoff_seconds": [2, 4],
9204 "start_to_close_timeout": 30,
9205 "schedule_to_start_timeout": 5,
9206 "schedule_to_close_timeout": 90,
9207 "heartbeat_timeout": 10,
9208 "non_retryable_error_types": ["PermanentError"]
9209 }
9210 }
9211 }),
9212 ),
9213 history_event(
9214 "ActivityStarted",
9215 json!({
9216 "sequence": 1,
9217 "activity_type": "flaky",
9218 "activity_execution_id": "act-1",
9219 "activity_attempt_id": "attempt-1",
9220 "attempt_number": 1
9221 }),
9222 ),
9223 history_event(
9224 "ActivityRetryScheduled",
9225 json!({
9226 "sequence": 1,
9227 "activity_type": "flaky",
9228 "activity_execution_id": "act-1",
9229 "activity_attempt_id": "attempt-1",
9230 "attempt_number": 1,
9231 "retry_after_attempt": 1,
9232 "retry_backoff_seconds": 2,
9233 "failure_category": "activity",
9234 "exception_type": "TransientError"
9235 }),
9236 ),
9237 history_event(
9238 "ActivityStarted",
9239 json!({
9240 "sequence": 1,
9241 "activity_type": "flaky",
9242 "activity_execution_id": "act-1",
9243 "activity_attempt_id": "attempt-2",
9244 "attempt_number": 2
9245 }),
9246 ),
9247 history_event(
9248 "ActivityCompleted",
9249 json!({
9250 "sequence": 1,
9251 "activity_type": "flaky",
9252 "activity_execution_id": "act-1",
9253 "activity_attempt_id": "attempt-2",
9254 "attempt_number": 2,
9255 "payload_codec": DEFAULT_CODEC,
9256 "result": fixture_envelope(json!({"status":"recovered"}))
9257 }),
9258 ),
9259 ]
9260 }
9261
9262 fn retry_activity_options() -> ActivityOptions {
9263 ActivityOptions::new()
9264 .task_queue("critical-activities")
9265 .retry_policy(
9266 ActivityRetryPolicy::new(3)
9267 .backoff_intervals([Duration::from_secs(2), Duration::from_secs(4)])
9268 .non_retryable_error_type("PermanentError"),
9269 )
9270 .start_to_close_timeout(Duration::from_secs(30))
9271 .schedule_to_start_timeout(Duration::from_secs(5))
9272 .schedule_to_close_timeout(Duration::from_secs(90))
9273 .heartbeat_timeout(Duration::from_secs(10))
9274 }
9275
9276 #[test]
9277 fn fixed_avro_value_round_trips_json_values() {
9278 let value = json!({"greeting": "hello", "count": 3, "ok": true});
9279 let envelope = PayloadEnvelope::avro(&value).expect("encode");
9280 assert_eq!(envelope.codec, DEFAULT_CODEC);
9281 assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
9282 }
9283
9284 #[tokio::test]
9285 async fn typed_worker_surfaces_preserve_bytes_and_map_list_identity() {
9286 let client = Client::new("http://127.0.0.1:8080").expect("client");
9287 let mut worker = Worker::new(client, "rust-workers");
9288 worker.register_workflow_avro_value("typed.echo", |_ctx, input| async move { Ok(input) });
9289 worker
9290 .register_activity_avro_value("typed.activity", |_ctx, input| async move { Ok(input) });
9291 worker.register_query_avro_value("typed.echo", "inspect", |_ctx, input| async move {
9292 Ok(input)
9293 });
9294 worker.register_update_avro_value("typed.echo", "replace", |_ctx, input| async move {
9295 Ok(input)
9296 });
9297 worker.register_workflow_avro_value("typed.signal", |ctx, _input| async move {
9298 Ok(AvroValue::Array(
9299 ctx.wait_signal_avro_value("changed").await?,
9300 ))
9301 });
9302
9303 let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
9304 let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed envelope");
9305
9306 let mut workflow = workflow_task("typed.echo", Vec::new(), DEFAULT_CODEC);
9307 workflow.arguments = Some(envelope.clone());
9308 let commands = worker
9309 .execute_workflow_task(workflow)
9310 .expect("typed workflow task");
9311 assert_eq!(commands[0]["type"], "complete_workflow");
9312 assert_eq!(
9313 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
9314 .expect("typed workflow result"),
9315 arguments
9316 );
9317
9318 let activity = ActivityTask {
9319 task_id: "activity-typed".to_string(),
9320 activity_attempt_id: Some("attempt-typed".to_string()),
9321 attempt_id: None,
9322 activity_type: "typed.activity".to_string(),
9323 payload_codec: DEFAULT_CODEC.to_string(),
9324 arguments: Some(envelope.clone()),
9325 attempt_number: 1,
9326 lease_owner: Some("rust-worker".to_string()),
9327 };
9328 assert_eq!(
9329 worker
9330 .execute_activity_task(activity)
9331 .await
9332 .expect("typed activity result"),
9333 arguments
9334 );
9335
9336 let query = QueryTask {
9337 query_task_id: "query-typed".to_string(),
9338 query_task_attempt: 1,
9339 lease_owner: Some("rust-worker".to_string()),
9340 workflow_id: Some("typed-1".to_string()),
9341 run_id: Some("run-typed".to_string()),
9342 workflow_type: "typed.echo".to_string(),
9343 query_name: "inspect".to_string(),
9344 payload_codec: DEFAULT_CODEC.to_string(),
9345 workflow_arguments: Some(
9346 encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
9347 .expect("workflow input"),
9348 ),
9349 query_arguments: Some(envelope.clone()),
9350 history_events: Vec::new(),
9351 history_export: None,
9352 run_status: Some("running".to_string()),
9353 };
9354 assert_eq!(
9355 worker
9356 .execute_query_task(query)
9357 .await
9358 .expect("typed query result"),
9359 arguments
9360 );
9361
9362 let mut update = workflow_task(
9363 "typed.echo",
9364 vec![history_event(
9365 "UpdateAccepted",
9366 json!({
9367 "update_id": "update-typed",
9368 "update_name": "replace",
9369 "arguments": envelope.clone(),
9370 }),
9371 )],
9372 DEFAULT_CODEC,
9373 );
9374 update.workflow_update_id = Some("update-typed".to_string());
9375 update.update_name = Some("replace".to_string());
9376 let commands = worker
9377 .execute_workflow_task(update)
9378 .expect("typed update task");
9379 assert_eq!(commands[0]["type"], "complete_update");
9380 assert_eq!(
9381 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
9382 .expect("typed update result"),
9383 arguments
9384 );
9385
9386 let mut signal = workflow_task(
9387 "typed.signal",
9388 vec![history_event(
9389 "SignalReceived",
9390 json!({
9391 "signal_id": "signal-typed",
9392 "signal_name": "changed",
9393 "arguments": envelope.clone(),
9394 }),
9395 )],
9396 DEFAULT_CODEC,
9397 );
9398 signal.workflow_signal_id = Some("signal-typed".to_string());
9399 signal.signal_name = Some("changed".to_string());
9400 signal.signal_arguments = Some(envelope);
9401 let commands = worker
9402 .execute_workflow_task(signal)
9403 .expect("typed signal resume");
9404 assert_eq!(
9405 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
9406 .expect("typed signal result"),
9407 arguments
9408 );
9409 }
9410
9411 #[tokio::test]
9412 async fn typed_helpers_never_parse_json_inspection_projection() {
9413 let collision_values = projection_collision_probe();
9414 let expected = AvroValue::Array(collision_values.clone());
9415 let envelope = encode_typed_envelope(&expected, DEFAULT_CODEC).expect("collision envelope");
9416
9417 let activity_context = workflow_context_with_codec(
9418 vec![history_event(
9419 "ActivityCompleted",
9420 json!({
9421 "sequence": 1,
9422 "activity_type": "collision.activity",
9423 "payload_codec": DEFAULT_CODEC,
9424 "result": envelope.clone(),
9425 }),
9426 )],
9427 DEFAULT_CODEC,
9428 );
9429 assert_eq!(
9430 activity_context
9431 .activity_avro_value("collision.activity", AvroValue::Array(Vec::new()))
9432 .await
9433 .expect("typed activity collision result"),
9434 expected
9435 );
9436
9437 let signal_context = workflow_context_with_codec(
9438 vec![
9439 history_event(
9440 "SignalWaitOpened",
9441 json!({"sequence": 1, "signal_name": "collision"}),
9442 ),
9443 history_event(
9444 "SignalApplied",
9445 json!({
9446 "sequence": 1,
9447 "signal_name": "collision",
9448 "payload_codec": DEFAULT_CODEC,
9449 "value": envelope.clone(),
9450 }),
9451 ),
9452 ],
9453 DEFAULT_CODEC,
9454 );
9455 assert_eq!(
9456 signal_context
9457 .wait_signal_avro_value("collision")
9458 .await
9459 .expect("typed signal collision arguments"),
9460 collision_values
9461 );
9462
9463 let child_context = workflow_context_with_codec(
9464 vec![
9465 history_event(
9466 "ChildWorkflowScheduled",
9467 json!({
9468 "sequence": 1,
9469 "child_workflow_instance_id": "collision-child",
9470 "child_workflow_run_id": "collision-run",
9471 "child_workflow_type": "collision.child",
9472 }),
9473 ),
9474 history_event(
9475 "ChildRunCompleted",
9476 json!({
9477 "sequence": 1,
9478 "child_workflow_instance_id": "collision-child",
9479 "child_workflow_run_id": "collision-run",
9480 "child_workflow_type": "collision.child",
9481 "payload_codec": DEFAULT_CODEC,
9482 "result": envelope,
9483 }),
9484 ),
9485 ],
9486 DEFAULT_CODEC,
9487 );
9488 let child = child_context
9489 .start_child_workflow_avro_value(
9490 "collision.child",
9491 ChildWorkflowOptions::new("collision-workers"),
9492 AvroValue::Array(Vec::new()),
9493 )
9494 .await
9495 .expect("typed child collision result");
9496 assert_eq!(child.result, expected);
9497 }
9498
9499 #[tokio::test]
9500 async fn replayed_typed_query_keeps_lossless_workflow_and_query_inputs() {
9501 let client = Client::new("http://127.0.0.1:8080").expect("client");
9502 let mut worker = Worker::new(client, "rust-workers");
9503 worker.register_replayed_workflow_avro_value(
9504 "typed.replayed",
9505 || (),
9506 |_ctx, input, _state| async move { Ok(input) },
9507 );
9508 worker.register_replayed_query_avro_value::<(), _, _>(
9509 "typed.replayed",
9510 "inspect",
9511 |ctx, _state, args| async move {
9512 let mut signals = ctx.signals_avro_value("collision");
9513 let signal = signals
9514 .pop()
9515 .map(AvroValue::Array)
9516 .unwrap_or_else(|| AvroValue::Array(Vec::new()));
9517 Ok(AvroValue::Array(vec![
9518 ctx.workflow_input_avro_value().clone(),
9519 signal,
9520 args,
9521 ]))
9522 },
9523 );
9524 let arguments = AvroValue::Array(projection_collision_probe());
9525 let signal_arguments =
9526 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed query signal arguments");
9527 let task = QueryTask {
9528 query_task_id: "query-typed-replay".to_string(),
9529 query_task_attempt: 1,
9530 lease_owner: Some("rust-worker".to_string()),
9531 workflow_id: Some("typed-replay".to_string()),
9532 run_id: Some("run-typed-replay".to_string()),
9533 workflow_type: "typed.replayed".to_string(),
9534 query_name: "inspect".to_string(),
9535 payload_codec: DEFAULT_CODEC.to_string(),
9536 workflow_arguments: Some(
9537 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("workflow arguments"),
9538 ),
9539 query_arguments: Some(
9540 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("query arguments"),
9541 ),
9542 history_events: vec![history_event(
9543 "SignalReceived",
9544 json!({
9545 "signal_id": "collision-signal",
9546 "signal_name": "collision",
9547 "workflow_sequence": 1,
9548 "payload_codec": DEFAULT_CODEC,
9549 "arguments": signal_arguments,
9550 }),
9551 )],
9552 history_export: None,
9553 run_status: Some("completed".to_string()),
9554 };
9555
9556 assert_eq!(
9557 worker
9558 .execute_query_task(task)
9559 .await
9560 .expect("typed replay query"),
9561 AvroValue::Array(vec![arguments.clone(), arguments.clone(), arguments])
9562 );
9563 }
9564
9565 #[test]
9566 fn public_avro_adapter_rejects_non_string_map_keys_before_json_conversion() {
9567 let value = BTreeMap::from([(1_i32, "integer key")]);
9568 let error = PayloadEnvelope::avro(&value)
9569 .expect_err("integer map keys must fail")
9570 .to_string();
9571
9572 assert!(error.contains("invalid_map_key"));
9573 }
9574
9575 #[test]
9576 fn json_tagged_payload_fails_closed_with_actionable_diagnostic() {
9577 let envelope = PayloadEnvelope {
9578 codec: "json".to_string(),
9579 blob: r#"{"greeting":"hello"}"#.to_string(),
9580 };
9581
9582 let error = decode_payload::<Value>(&envelope).expect_err("JSON payload must fail");
9583 let diagnostic = error.to_string();
9584 assert!(diagnostic.contains("unsupported_payload_codec"));
9585 assert!(diagnostic.contains("codec=\"avro\""));
9586 assert!(diagnostic.contains("HTTP document transport"));
9587 }
9588
9589 #[test]
9590 fn untagged_json_payload_value_fails_closed() {
9591 let error = decode_wire_value(&json!({"stale": true}), DEFAULT_CODEC)
9592 .expect_err("untagged JSON payload values must fail");
9593 let diagnostic = error.to_string();
9594 assert!(diagnostic.contains("unsupported_payload_codec"));
9595 assert!(diagnostic.contains("untagged durable payload"));
9596 assert!(diagnostic.contains("HTTP document transport"));
9597 }
9598
9599 #[test]
9600 fn prerelease_avro_payload_without_single_object_frame_is_rejected() {
9601 let envelope = PayloadEnvelope {
9602 codec: DEFAULT_CODEC.to_string(),
9603 blob: BASE64.encode([0x01]),
9604 };
9605
9606 let error = decode_payload::<Value>(&envelope).expect_err("prerelease payload must fail");
9607 assert!(error.to_string().contains("invalid_payload_framing"));
9608 }
9609
9610 #[tokio::test]
9611 async fn workflow_completion_rejects_invalid_payload_slots_without_transport() {
9612 let server = MockWorkerServer::start();
9613 let client = Client::builder(server.base_url())
9614 .timeout(Duration::from_secs(2))
9615 .build()
9616 .expect("client");
9617 let invalid_commands = [
9618 json!({
9619 "type": "complete_workflow",
9620 "result": {"codec": "json", "blob": null}
9621 }),
9622 json!({
9623 "type": "schedule_activity",
9624 "arguments": {"codec": "yaml", "blob": "ignored"}
9625 }),
9626 json!({
9627 "type": "start_child_workflow",
9628 "arguments": {"codec": DEFAULT_CODEC, "blob": null}
9629 }),
9630 json!({"type": "continue_as_new", "arguments": []}),
9631 json!({"type": "complete_update"}),
9632 json!({"type": "record_side_effect", "result": null}),
9633 json!({
9634 "type": "start_service_operation",
9635 "payload_codec": DEFAULT_CODEC,
9636 "request_payload": "raw-avro-bytes"
9637 }),
9638 ];
9639
9640 for command in invalid_commands {
9641 let error = client
9642 .complete_workflow_task("invalid-codec", "rust-worker", 1, vec![command])
9643 .await
9644 .expect_err("invalid durable payload must fail locally");
9645 let diagnostic = error.to_string();
9646 assert!(
9647 diagnostic.contains("unsupported_payload_codec")
9648 || diagnostic.contains("invalid_payload_envelope")
9649 || diagnostic.contains("untagged durable payload"),
9650 "unexpected validation diagnostic: {diagnostic}"
9651 );
9652 }
9653
9654 assert_eq!(
9655 server.request_count("/api/worker/workflow-tasks/invalid-codec/complete"),
9656 0,
9657 "invalid command payloads must not reach HTTP transport"
9658 );
9659 }
9660
9661 #[test]
9662 fn workflow_completion_validates_only_protocol_owned_payload_slots() {
9663 let envelope = fixture_envelope(json!({"codec": "customer-value"}));
9664 let commands = [
9665 json!({"type": "complete_workflow", "result": envelope.clone()}),
9666 json!({"type": "schedule_activity", "arguments": envelope.clone()}),
9667 json!({"type": "start_child_workflow", "arguments": envelope.clone()}),
9668 json!({"type": "continue_as_new", "arguments": envelope.clone()}),
9669 json!({"type": "complete_update", "result": envelope.clone()}),
9670 json!({"type": "record_side_effect", "result": envelope.clone()}),
9671 json!({
9672 "type": "start_service_operation",
9673 "payload_codec": DEFAULT_CODEC,
9674 "request_payload": envelope.clone()
9675 }),
9676 json!({
9677 "type": "complete_workflow",
9678 "result": envelope,
9679 "metadata": {
9680 "codec": "json",
9681 "payload_codec": "customer-codec",
9682 "result": {"codec": "yaml", "blob": null}
9683 }
9684 }),
9685 ];
9686
9687 validate_workflow_task_commands(&commands)
9688 .expect("customer metadata must not become a protocol codec declaration");
9689 }
9690
9691 #[test]
9692 fn valid_avro_tasks_normalize_absent_and_null_arguments_to_empty_lists() {
9693 assert_eq!(
9694 decode_task_avro_arguments(None, DEFAULT_CODEC).expect("absent arguments"),
9695 AvroValue::Array(Vec::new())
9696 );
9697 assert_eq!(
9698 decode_task_avro_arguments(Some(&Value::Null), DEFAULT_CODEC).expect("null arguments"),
9699 AvroValue::Array(Vec::new())
9700 );
9701
9702 let mut signal = workflow_task("missing", Vec::new(), DEFAULT_CODEC);
9703 signal.signal_name = Some("empty-signal".to_string());
9704 signal.signal_arguments = None;
9705 let decoded = decode_resume_signal(&signal)
9706 .expect("valid Avro signal")
9707 .expect("named signal resumes the workflow");
9708 assert!(decoded.arguments.is_empty());
9709 }
9710
9711 #[tokio::test]
9712 async fn invalid_inbound_codecs_precede_handlers_and_unrelated_outcomes() {
9713 let client = Client::new("http://127.0.0.1:8080").expect("client");
9714 let mut worker = Worker::new(client, "rust-workers");
9715 let handler_calls = Arc::new(AtomicUsize::new(0));
9716
9717 let calls = Arc::clone(&handler_calls);
9718 worker.register_workflow("codec.workflow", move |_ctx, _args| {
9719 calls.fetch_add(1, Ordering::SeqCst);
9720 async move { Ok(Value::Null) }
9721 });
9722 let calls = Arc::clone(&handler_calls);
9723 worker.register_activity("codec.activity", move |_ctx, _args| {
9724 calls.fetch_add(1, Ordering::SeqCst);
9725 async move { Ok(Value::Null) }
9726 });
9727 let calls = Arc::clone(&handler_calls);
9728 worker.register_update("codec.workflow", "known", move |_ctx, _args| {
9729 calls.fetch_add(1, Ordering::SeqCst);
9730 async move { Ok(Value::Null) }
9731 });
9732 let calls = Arc::clone(&handler_calls);
9733 worker.register_query("codec.workflow", "known", move |_ctx, _args| {
9734 calls.fetch_add(1, Ordering::SeqCst);
9735 async move { Ok(Value::Null) }
9736 });
9737
9738 let mut workflow = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
9739 workflow.payload_codec = "json".to_string();
9740 workflow.arguments = None;
9741 let error = worker
9742 .execute_workflow_task(workflow)
9743 .expect_err("task codec must be checked before workflow invocation");
9744 assert!(error.to_string().contains("unsupported_payload_codec"));
9745
9746 let activity = ActivityTask {
9747 task_id: "activity-invalid-codec".to_string(),
9748 activity_attempt_id: None,
9749 attempt_id: None,
9750 activity_type: "codec.activity".to_string(),
9751 payload_codec: "unknown".to_string(),
9752 arguments: None,
9753 attempt_number: 1,
9754 lease_owner: None,
9755 };
9756 let error = worker
9757 .execute_activity_task(activity)
9758 .await
9759 .expect_err("task codec must be checked before activity invocation");
9760 assert!(error.to_string().contains("unsupported_payload_codec"));
9761
9762 let mut update = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
9763 update.workflow_update_id = Some("update-invalid-codec".to_string());
9764 update.update_name = Some("known".to_string());
9765 update.history_events.push(history_event(
9766 "UpdateAccepted",
9767 json!({
9768 "update_id": "update-invalid-codec",
9769 "update_name": "known",
9770 "arguments": {"codec": "json", "blob": null}
9771 }),
9772 ));
9773 let error = worker
9774 .execute_workflow_task(update)
9775 .expect_err("nested update codec must be checked before handler lookup");
9776 assert!(error.to_string().contains("unsupported_payload_codec"));
9777
9778 let query: QueryTask = serde_json::from_value(json!({
9779 "query_task_id": "query-invalid-codec",
9780 "workflow_type": "codec.workflow",
9781 "query_name": "known",
9782 "payload_codec": DEFAULT_CODEC,
9783 "workflow_arguments": null,
9784 "query_arguments": null,
9785 "history_export": {
9786 "payloads": {"codec": DEFAULT_CODEC},
9787 "signals": [{
9788 "name": "empty",
9789 "payload_codec": "json",
9790 "arguments": null
9791 }]
9792 }
9793 }))
9794 .expect("query task");
9795 let failure = worker
9796 .execute_query_task(query)
9797 .await
9798 .expect_err("exported signal codec must be checked before query invocation");
9799 assert_eq!(failure.reason, "query_payload_decode_failed");
9800 assert!(failure.message.contains("unsupported_payload_codec"));
9801
9802 let exported_history: QueryTask = serde_json::from_value(json!({
9803 "query_task_id": "query-invalid-history-codec",
9804 "workflow_type": "codec.workflow",
9805 "query_name": "known",
9806 "payload_codec": DEFAULT_CODEC,
9807 "history_export": {
9808 "payloads": {"codec": DEFAULT_CODEC},
9809 "history_events": [{
9810 "type": "ActivityCompleted",
9811 "payload": {"payload_codec": "unknown", "result": null}
9812 }]
9813 }
9814 }))
9815 .expect("query task");
9816 let failure = worker
9817 .execute_query_task(exported_history)
9818 .await
9819 .expect_err("exported history codec must be checked before query invocation");
9820 assert_eq!(failure.reason, "query_payload_decode_failed");
9821 assert!(failure.message.contains("unsupported_payload_codec"));
9822 assert_eq!(handler_calls.load(Ordering::SeqCst), 0);
9823
9824 let mut unknown_workflow = workflow_task("missing", Vec::new(), DEFAULT_CODEC);
9825 unknown_workflow.arguments = None;
9826 unknown_workflow.history_events.push(history_event(
9827 "SignalReceived",
9828 json!({
9829 "signal_name": "empty",
9830 "payload_codec": "json",
9831 "arguments": null
9832 }),
9833 ));
9834 let error = worker
9835 .execute_workflow_task(unknown_workflow)
9836 .expect_err("history codec must precede unknown workflow outcome");
9837 assert!(error.to_string().contains("unsupported_payload_codec"));
9838
9839 let unknown_activity = ActivityTask {
9840 task_id: "activity-unknown".to_string(),
9841 activity_attempt_id: None,
9842 attempt_id: None,
9843 activity_type: "missing".to_string(),
9844 payload_codec: "json".to_string(),
9845 arguments: None,
9846 attempt_number: 1,
9847 lease_owner: None,
9848 };
9849 let error = worker
9850 .execute_activity_task(unknown_activity)
9851 .await
9852 .expect_err("codec must precede unknown activity outcome");
9853 assert!(error.to_string().contains("unsupported_payload_codec"));
9854
9855 let mut unknown_update = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
9856 unknown_update.payload_codec = "json".to_string();
9857 unknown_update.arguments = None;
9858 unknown_update.workflow_update_id = Some("update-unknown".to_string());
9859 unknown_update.update_name = Some("missing".to_string());
9860 let error = worker
9861 .execute_workflow_task(unknown_update)
9862 .expect_err("codec must precede fail_update shortcut");
9863 assert!(error.to_string().contains("unsupported_payload_codec"));
9864
9865 let unknown_query: QueryTask = serde_json::from_value(json!({
9866 "query_task_id": "query-unknown",
9867 "workflow_type": "missing",
9868 "query_name": "missing",
9869 "payload_codec": "json",
9870 "workflow_arguments": null,
9871 "query_arguments": null
9872 }))
9873 .expect("query task");
9874 let failure = worker
9875 .execute_query_task(unknown_query)
9876 .await
9877 .expect_err("codec must precede unknown query outcome");
9878 assert_eq!(failure.reason, "query_payload_decode_failed");
9879 assert!(failure.message.contains("unsupported_payload_codec"));
9880 }
9881
9882 #[tokio::test]
9883 async fn invalid_signal_history_payload_aliases_precede_shortcuts() {
9884 let client = Client::new("http://127.0.0.1:8080").expect("client");
9885 let worker = Worker::new(client, "rust-workers");
9886
9887 for event_type in ["SignalReceived", "SignalApplied"] {
9888 for (payload_field, codec) in [
9889 ("value", "json"),
9890 ("input", "unknown"),
9891 ("arguments", "json"),
9892 ] {
9893 let payload = json!({
9894 "signal_name": "empty",
9895 payload_field: {"codec": codec, "blob": null}
9896 });
9897 let workflow = workflow_task(
9898 "missing",
9899 vec![history_event(event_type, payload.clone())],
9900 DEFAULT_CODEC,
9901 );
9902 let error = worker
9903 .execute_workflow_task(workflow)
9904 .expect_err("signal payload codec must precede unknown workflow outcome");
9905 assert!(
9906 error.to_string().contains("unsupported_payload_codec"),
9907 "{event_type}.{payload_field} returned an unrelated workflow error: {error}"
9908 );
9909
9910 let query: QueryTask = serde_json::from_value(json!({
9911 "query_task_id": format!("query-{event_type}-{payload_field}"),
9912 "workflow_type": "missing",
9913 "query_name": "missing",
9914 "payload_codec": DEFAULT_CODEC,
9915 "workflow_arguments": null,
9916 "query_arguments": null,
9917 "history_events": [{
9918 "event_type": event_type,
9919 "payload": payload
9920 }]
9921 }))
9922 .expect("query task");
9923 let failure = worker
9924 .execute_query_task(query)
9925 .await
9926 .expect_err("signal payload codec must precede unknown query outcome");
9927 assert_eq!(
9928 failure.reason, "query_payload_decode_failed",
9929 "{event_type}.{payload_field} returned an unrelated query outcome"
9930 );
9931 assert!(
9932 failure.message.contains("unsupported_payload_codec"),
9933 "{event_type}.{payload_field} returned an unrelated query error: {}",
9934 failure.message
9935 );
9936 }
9937 }
9938 }
9939
9940 #[test]
9941 fn workflow_context_schedules_activity_until_completion_is_in_history() {
9942 let ctx = WorkflowContext {
9943 state: Arc::new(Mutex::new(
9944 WorkflowState::new_with_identity(
9945 Vec::new(),
9946 Some("wf-parent".to_string()),
9947 Some("run-parent".to_string()),
9948 "rust-workers".to_string(),
9949 DEFAULT_CODEC.to_string(),
9950 None,
9951 )
9952 .expect("workflow state"),
9953 )),
9954 };
9955
9956 let mut call = Box::pin(ctx.activity("hello.activity", json!(["Ada"])));
9957 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9958 assert!(matches!(
9959 call.as_mut().poll(&mut task_context),
9960 Poll::Pending
9961 ));
9962
9963 let commands = ctx.take_commands().expect("commands");
9964 assert_eq!(commands[0]["type"], "schedule_activity");
9965 assert_eq!(commands[0]["activity_type"], "hello.activity");
9966 }
9967
9968 #[test]
9969 fn activity_options_encode_retry_policy_queue_and_every_timeout() {
9970 let ctx = workflow_context(Vec::new());
9971 let options = ActivityOptions::new()
9972 .task_queue("payments")
9973 .retry_policy(
9974 ActivityRetryPolicy::new(4)
9975 .exponential_backoff(Duration::from_secs(1), 3, Some(Duration::from_secs(10)))
9976 .non_retryable_error_type("ValidationError"),
9977 )
9978 .start_to_close_timeout(Duration::from_secs(120))
9979 .schedule_to_start_timeout(Duration::from_secs(10))
9980 .schedule_to_close_timeout(Duration::from_secs(300))
9981 .heartbeat_timeout(Duration::from_secs(15));
9982 let mut call = Box::pin(ctx.activity_with_options(
9983 "charge-card",
9984 options,
9985 json!([{"order_id": "o-1"}]),
9986 ));
9987 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9988
9989 assert!(matches!(
9990 call.as_mut().poll(&mut task_context),
9991 Poll::Pending
9992 ));
9993 assert!(matches!(
9994 call.as_mut().poll(&mut task_context),
9995 Poll::Pending
9996 ));
9997
9998 let commands = ctx.take_commands().expect("activity command");
9999 assert_eq!(commands.len(), 1, "one future emits one logical schedule");
10000 assert_eq!(commands[0]["queue"], "payments");
10001 assert_eq!(
10002 commands[0]["retry_policy"],
10003 json!({
10004 "max_attempts": 4,
10005 "backoff_seconds": [1, 3, 9],
10006 "non_retryable_error_types": ["ValidationError"],
10007 })
10008 );
10009 assert_eq!(commands[0]["start_to_close_timeout"], 120);
10010 assert_eq!(commands[0]["schedule_to_start_timeout"], 10);
10011 assert_eq!(commands[0]["schedule_to_close_timeout"], 300);
10012 assert_eq!(commands[0]["heartbeat_timeout"], 15);
10013 }
10014
10015 #[test]
10016 fn activity_options_encode_explicit_and_rounded_backoff_intervals() {
10017 let ctx = workflow_context(Vec::new());
10018 let options = ActivityOptions::new().retry_policy(
10019 ActivityRetryPolicy::new(3)
10020 .backoff_intervals([Duration::from_millis(1), Duration::from_millis(1_001)]),
10021 );
10022 let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
10023 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10024
10025 assert!(matches!(
10026 call.as_mut().poll(&mut task_context),
10027 Poll::Pending
10028 ));
10029 assert_eq!(
10030 ctx.take_commands().expect("command")[0]["retry_policy"]["backoff_seconds"],
10031 json!([1, 2])
10032 );
10033 }
10034
10035 #[test]
10036 fn invalid_activity_options_return_typed_errors_before_emitting_commands() {
10037 let cases = [
10038 (
10039 ActivityOptions::new().task_queue(" "),
10040 ActivityOptionsErrorKind::EmptyTaskQueue,
10041 ),
10042 (
10043 ActivityOptions::new().retry_policy(ActivityRetryPolicy::default()),
10044 ActivityOptionsErrorKind::EmptyRetryPolicy,
10045 ),
10046 (
10047 ActivityOptions::new().retry_policy(ActivityRetryPolicy::new(0)),
10048 ActivityOptionsErrorKind::InvalidMaxAttempts,
10049 ),
10050 (
10051 ActivityOptions::new().retry_policy(ActivityRetryPolicy {
10052 max_attempts: None,
10053 backoff: Some(ActivityBackoff::Explicit(vec![Duration::from_secs(1)])),
10054 non_retryable_error_types: Vec::new(),
10055 }),
10056 ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
10057 ),
10058 (
10059 ActivityOptions::new().retry_policy(
10060 ActivityRetryPolicy::new(2)
10061 .backoff_intervals([Duration::from_secs(1), Duration::from_secs(2)]),
10062 ),
10063 ActivityOptionsErrorKind::TooManyBackoffIntervals,
10064 ),
10065 (
10066 ActivityOptions::new().retry_policy(
10067 ActivityRetryPolicy::new(2).exponential_backoff(
10068 Duration::from_secs(1),
10069 0,
10070 None,
10071 ),
10072 ),
10073 ActivityOptionsErrorKind::InvalidBackoffCoefficient,
10074 ),
10075 (
10076 ActivityOptions::new()
10077 .retry_policy(ActivityRetryPolicy::new(2).non_retryable_error_type(" ")),
10078 ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
10079 ),
10080 (
10081 ActivityOptions::new().retry_policy(
10082 ActivityRetryPolicy::new(10_002).exponential_backoff(
10083 Duration::from_secs(1),
10084 1,
10085 None,
10086 ),
10087 ),
10088 ActivityOptionsErrorKind::BackoffGenerationTooLarge,
10089 ),
10090 (
10091 ActivityOptions::new().retry_policy(
10092 ActivityRetryPolicy::new(2)
10093 .backoff_intervals([Duration::from_secs(i64::MAX as u64 + 1)]),
10094 ),
10095 ActivityOptionsErrorKind::BackoffOverflow,
10096 ),
10097 ];
10098
10099 for (options, expected_kind) in cases {
10100 let ctx = workflow_context(Vec::new());
10101 let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
10102 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10103 let Poll::Ready(Err(Error::InvalidActivityOptions(error))) =
10104 call.as_mut().poll(&mut task_context)
10105 else {
10106 panic!("expected typed activity validation error");
10107 };
10108 assert_eq!(error.kind, expected_kind);
10109 assert!(ctx.take_commands().expect("commands").is_empty());
10110 }
10111 }
10112
10113 #[test]
10114 fn activity_options_validate_positive_and_ordered_timeouts() {
10115 let zero_timeout_cases = [
10116 ActivityOptions::new().start_to_close_timeout(Duration::ZERO),
10117 ActivityOptions::new().schedule_to_start_timeout(Duration::ZERO),
10118 ActivityOptions::new().schedule_to_close_timeout(Duration::ZERO),
10119 ActivityOptions::new().heartbeat_timeout(Duration::ZERO),
10120 ];
10121 for options in zero_timeout_cases {
10122 assert_eq!(
10123 options.validate().expect_err("zero timeout").kind,
10124 ActivityOptionsErrorKind::TimeoutNotPositive
10125 );
10126 }
10127
10128 let ordering_cases = [
10129 ActivityOptions::new()
10130 .heartbeat_timeout(Duration::from_secs(11))
10131 .start_to_close_timeout(Duration::from_secs(10)),
10132 ActivityOptions::new()
10133 .start_to_close_timeout(Duration::from_secs(31))
10134 .schedule_to_close_timeout(Duration::from_secs(30)),
10135 ActivityOptions::new()
10136 .schedule_to_start_timeout(Duration::from_secs(31))
10137 .schedule_to_close_timeout(Duration::from_secs(30)),
10138 ];
10139 for options in ordering_cases {
10140 assert_eq!(
10141 options.validate().expect_err("timeout order").kind,
10142 ActivityOptionsErrorKind::TimeoutOrder
10143 );
10144 }
10145
10146 assert_eq!(
10147 ActivityOptions::new()
10148 .start_to_close_timeout(Duration::from_secs(i64::MAX as u64 + 1))
10149 .validate()
10150 .expect_err("protocol integer overflow")
10151 .kind,
10152 ActivityOptionsErrorKind::TimeoutOverflow
10153 );
10154 }
10155
10156 #[test]
10157 fn replayed_activity_retry_history_completes_without_duplicate_schedule() {
10158 let ctx = workflow_context(completed_retry_activity_history());
10159 let mut call =
10160 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
10161 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10162
10163 assert!(matches!(
10164 call.as_mut().poll(&mut task_context),
10165 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
10166 ));
10167 assert!(ctx.take_commands().expect("commands").is_empty());
10168 ctx.ensure_history_consumed().expect("history consumed");
10169 }
10170
10171 #[test]
10172 fn duplicate_non_retryable_types_use_one_command_and_replay_representation() {
10173 let mut options = retry_activity_options();
10174 options
10175 .retry_policy
10176 .as_mut()
10177 .expect("retry policy")
10178 .non_retryable_error_types
10179 .extend([" PermanentError ".to_string(), "PermanentError".to_string()]);
10180
10181 let new_ctx = workflow_context(Vec::new());
10182 let mut new_call =
10183 Box::pin(new_ctx.activity_with_options("flaky", options.clone(), json!([])));
10184 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10185 assert!(matches!(
10186 new_call.as_mut().poll(&mut task_context),
10187 Poll::Pending
10188 ));
10189 let commands = new_ctx.take_commands().expect("commands");
10190 assert_eq!(commands.len(), 1);
10191 assert_eq!(
10192 commands[0]["retry_policy"]["non_retryable_error_types"],
10193 json!(["PermanentError"])
10194 );
10195
10196 let replay_ctx = workflow_context(completed_retry_activity_history());
10197 let mut replay_call =
10198 Box::pin(replay_ctx.activity_with_options("flaky", options, json!([])));
10199 assert!(matches!(
10200 replay_call.as_mut().poll(&mut task_context),
10201 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
10202 ));
10203 assert!(replay_ctx.take_commands().expect("commands").is_empty());
10204 replay_ctx
10205 .ensure_history_consumed()
10206 .expect("history consumed");
10207 }
10208
10209 #[test]
10210 fn replayed_intermediate_retry_remains_pending_across_restarts() {
10211 let history = completed_retry_activity_history()
10212 .into_iter()
10213 .take(3)
10214 .collect::<Vec<_>>();
10215
10216 for _restart in 0..2 {
10217 let ctx = workflow_context(history.clone());
10218 let mut call =
10219 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
10220 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10221 assert!(matches!(
10222 call.as_mut().poll(&mut task_context),
10223 Poll::Pending
10224 ));
10225 assert!(ctx.take_commands().expect("commands").is_empty());
10226 }
10227 }
10228
10229 #[test]
10230 fn replayed_activity_rejects_changed_queue_retry_and_every_timeout_field() {
10231 let mut changed_queue = retry_activity_options();
10232 changed_queue.task_queue = Some("different-queue".to_string());
10233
10234 let mut changed_max_attempts = retry_activity_options();
10235 let retry_policy = changed_max_attempts
10236 .retry_policy
10237 .as_mut()
10238 .expect("retry policy");
10239 retry_policy.max_attempts = Some(4);
10240
10241 let mut changed_backoff = retry_activity_options();
10242 let retry_policy = changed_backoff.retry_policy.as_mut().expect("retry policy");
10243 retry_policy.backoff = Some(ActivityBackoff::Explicit(vec![
10244 Duration::from_secs(3),
10245 Duration::from_secs(4),
10246 ]));
10247
10248 let mut changed_non_retryable_types = retry_activity_options();
10249 let retry_policy = changed_non_retryable_types
10250 .retry_policy
10251 .as_mut()
10252 .expect("retry policy");
10253 retry_policy.non_retryable_error_types = vec!["AnotherPermanentError".to_string()];
10254
10255 let mut changed_start_to_close = retry_activity_options();
10256 changed_start_to_close.start_to_close_timeout = Some(Duration::from_secs(31));
10257 let mut changed_schedule_to_start = retry_activity_options();
10258 changed_schedule_to_start.schedule_to_start_timeout = Some(Duration::from_secs(6));
10259 let mut changed_schedule_to_close = retry_activity_options();
10260 changed_schedule_to_close.schedule_to_close_timeout = Some(Duration::from_secs(91));
10261 let mut changed_heartbeat = retry_activity_options();
10262 changed_heartbeat.heartbeat_timeout = Some(Duration::from_secs(11));
10263
10264 let cases = [
10265 (changed_queue, "activity_task_queue_mismatch"),
10266 (changed_max_attempts, "activity_retry_policy_mismatch"),
10267 (changed_backoff, "activity_retry_policy_mismatch"),
10268 (
10269 changed_non_retryable_types,
10270 "activity_retry_policy_mismatch",
10271 ),
10272 (changed_start_to_close, "activity_retry_policy_mismatch"),
10273 (changed_schedule_to_start, "activity_retry_policy_mismatch"),
10274 (changed_schedule_to_close, "activity_retry_policy_mismatch"),
10275 (changed_heartbeat, "activity_retry_policy_mismatch"),
10276 ];
10277
10278 for (options, expected_reason) in cases {
10279 let ctx = workflow_context(completed_retry_activity_history());
10280 let mut call = Box::pin(ctx.activity_with_options("flaky", options, json!([])));
10281 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10282 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10283 call.as_mut().poll(&mut task_context)
10284 else {
10285 panic!("changed activity options must fail replay");
10286 };
10287 assert_eq!(failure.reason, expected_reason);
10288 assert_eq!(failure.sequence, Some(1));
10289 assert!(ctx.take_commands().expect("commands").is_empty());
10290 }
10291 }
10292
10293 #[test]
10294 fn replayed_activity_rejects_changed_execution_mode_and_snapshot_version() {
10295 let cases = [
10296 (
10297 "execution_mode",
10298 json!("local"),
10299 "activity_execution_mode_mismatch",
10300 ),
10301 (
10302 "snapshot_version",
10303 json!(2),
10304 "activity_retry_policy_mismatch",
10305 ),
10306 ];
10307
10308 for (field, value, expected_reason) in cases {
10309 let mut history = completed_retry_activity_history();
10310 let activity = history[0].payload["activity"]
10311 .as_object_mut()
10312 .expect("activity snapshot");
10313 if field == "execution_mode" {
10314 activity.insert(field.to_string(), value);
10315 } else {
10316 activity["retry_policy"]
10317 .as_object_mut()
10318 .expect("retry snapshot")
10319 .insert(field.to_string(), value);
10320 }
10321
10322 let ctx = workflow_context(history);
10323 let mut call =
10324 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
10325 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10326 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10327 call.as_mut().poll(&mut task_context)
10328 else {
10329 panic!("changed {field} must fail replay");
10330 };
10331 assert_eq!(failure.reason, expected_reason);
10332 assert_eq!(failure.sequence, Some(1));
10333 assert!(ctx.take_commands().expect("commands").is_empty());
10334 }
10335 }
10336
10337 #[test]
10338 fn replayed_legacy_activity_treats_missing_option_snapshot_as_unknown() {
10339 let mut history = completed_retry_activity_history();
10340 let activity = history[0].payload["activity"]
10341 .as_object_mut()
10342 .expect("activity snapshot");
10343 activity.remove("execution_mode");
10344 activity.remove("retry_policy");
10345
10346 let mut current = retry_activity_options();
10347 current.start_to_close_timeout = Some(Duration::from_secs(45));
10348 current.schedule_to_start_timeout = Some(Duration::from_secs(8));
10349 current.schedule_to_close_timeout = Some(Duration::from_secs(120));
10350 current.heartbeat_timeout = Some(Duration::from_secs(12));
10351
10352 let ctx = workflow_context(history);
10353 let mut call = Box::pin(ctx.activity_with_options("flaky", current, json!([])));
10354 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10355 assert!(matches!(
10356 call.as_mut().poll(&mut task_context),
10357 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
10358 ));
10359 assert!(ctx.take_commands().expect("commands").is_empty());
10360 ctx.ensure_history_consumed().expect("history consumed");
10361 }
10362
10363 #[test]
10364 fn terminal_activity_failed_after_start_returns_typed_failure() {
10365 let history = vec![
10366 history_event(
10367 "ActivityScheduled",
10368 json!({
10369 "sequence": 1,
10370 "activity_type": "flaky",
10371 "activity_execution_id": "act-terminal",
10372 "activity": {
10373 "id": "act-terminal",
10374 "sequence": 1,
10375 "type": "flaky",
10376 "queue": "critical-activities",
10377 "retry_policy": {
10378 "snapshot_version": 1,
10379 "max_attempts": 3,
10380 "backoff_seconds": [2, 4],
10381 "non_retryable_error_types": ["PermanentError"]
10382 }
10383 }
10384 }),
10385 ),
10386 history_event(
10387 "ActivityStarted",
10388 json!({
10389 "sequence": 1,
10390 "activity_type": "flaky",
10391 "activity_execution_id": "act-terminal",
10392 "activity_attempt_id": "attempt-1",
10393 "attempt_number": 1
10394 }),
10395 ),
10396 history_event(
10397 "ActivityFailed",
10398 json!({
10399 "sequence": 1,
10400 "activity_type": "flaky",
10401 "activity_execution_id": "act-terminal",
10402 "activity_attempt_id": "attempt-1",
10403 "attempt_number": 1,
10404 "failure_id": "failure-terminal",
10405 "failure_category": "activity",
10406 "exception_type": "PermanentError",
10407 "message": "cannot retry",
10408 "non_retryable": true
10409 }),
10410 ),
10411 ];
10412 let ctx = workflow_context(history);
10413 let mut call =
10414 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
10415 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10416
10417 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
10418 call.as_mut().poll(&mut task_context)
10419 else {
10420 panic!("terminal ActivityFailed must settle the activity future");
10421 };
10422 assert_eq!(failure.kind, ActivityFailureKind::Failed);
10423 assert_eq!(
10424 failure.activity_execution_id.as_deref(),
10425 Some("act-terminal")
10426 );
10427 assert_eq!(failure.exception_type.as_deref(), Some("PermanentError"));
10428 assert!(failure.non_retryable);
10429 assert!(ctx.take_commands().expect("commands").is_empty());
10430 ctx.ensure_history_consumed().expect("history consumed");
10431 }
10432
10433 #[test]
10434 fn activity_terminal_events_return_machine_readable_failures() {
10435 let cases = [
10436 (
10437 "ActivityFailed",
10438 json!({
10439 "sequence": 1,
10440 "activity_type": "charge-card",
10441 "activity_execution_id": "act-1",
10442 "activity_attempt_id": "attempt-2",
10443 "attempt_number": 2,
10444 "failure_id": "failure-1",
10445 "failure_category": "activity",
10446 "exception_type": "PaymentDeclined",
10447 "exception_class": "payments.PaymentDeclined",
10448 "message": "card declined",
10449 "non_retryable": true
10450 }),
10451 ActivityFailureKind::Failed,
10452 "activity",
10453 ),
10454 (
10455 "ActivityCancelled",
10456 json!({
10457 "sequence": 1,
10458 "activity_type": "charge-card",
10459 "activity_execution_id": "act-1",
10460 "activity_attempt_id": "attempt-1"
10461 }),
10462 ActivityFailureKind::Cancelled,
10463 "cancelled",
10464 ),
10465 ];
10466
10467 for (event_type, payload, expected_kind, expected_reason) in cases {
10468 let ctx = workflow_context(vec![history_event(event_type, payload)]);
10469 let mut call = Box::pin(ctx.activity("charge-card", json!([])));
10470 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10471 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
10472 call.as_mut().poll(&mut task_context)
10473 else {
10474 panic!("expected terminal activity failure");
10475 };
10476 assert_eq!(failure.kind, expected_kind);
10477 assert_eq!(failure.reason, expected_reason);
10478 assert_eq!(failure.activity_execution_id.as_deref(), Some("act-1"));
10479 assert_eq!(failure.activity_type.as_deref(), Some("charge-card"));
10480 }
10481 }
10482
10483 #[test]
10484 fn every_activity_timeout_class_is_typed() {
10485 for timeout_kind in [
10486 "start_to_close",
10487 "schedule_to_start",
10488 "schedule_to_close",
10489 "heartbeat",
10490 ] {
10491 let ctx = workflow_context(vec![history_event(
10492 "ActivityTimedOut",
10493 json!({
10494 "sequence": 1,
10495 "activity_type": "slow",
10496 "activity_execution_id": "act-timeout",
10497 "activity_attempt_id": "attempt-timeout",
10498 "failure_category": "timeout",
10499 "timeout_kind": timeout_kind,
10500 "message": "deadline expired"
10501 }),
10502 )]);
10503 let mut call = Box::pin(ctx.activity("slow", json!([])));
10504 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10505 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
10506 call.as_mut().poll(&mut task_context)
10507 else {
10508 panic!("expected timeout failure");
10509 };
10510 assert_eq!(failure.kind, ActivityFailureKind::TimedOut);
10511 assert_eq!(failure.reason, timeout_kind);
10512 assert_eq!(failure.timeout_kind.as_deref(), Some(timeout_kind));
10513 assert_eq!(failure.failure_category.as_deref(), Some("timeout"));
10514 }
10515 }
10516
10517 #[test]
10518 fn workflow_sleep_emits_one_durable_timer_and_rounds_up() {
10519 let ctx = workflow_context(Vec::new());
10520 let mut sleep = Box::pin(ctx.sleep(Duration::from_millis(1_001)));
10521 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10522
10523 assert!(matches!(
10524 sleep.as_mut().poll(&mut task_context),
10525 Poll::Pending
10526 ));
10527 assert!(matches!(
10528 sleep.as_mut().poll(&mut task_context),
10529 Poll::Pending
10530 ));
10531
10532 let commands = ctx.take_commands().expect("timer command");
10533 assert_eq!(
10534 commands,
10535 vec![json!({
10536 "type": "start_timer",
10537 "delay_seconds": 2,
10538 })]
10539 );
10540 }
10541
10542 #[test]
10543 fn workflow_sleep_replays_matching_schedule_and_fire_without_a_command() {
10544 let history = vec![
10545 history_event(
10546 "TimerScheduled",
10547 json!({
10548 "sequence": 1,
10549 "timer_id": "timer-1",
10550 "delay_seconds": 5,
10551 "fire_at": "2026-07-11T12:00:05Z",
10552 }),
10553 ),
10554 history_event(
10555 "TimerFired",
10556 json!({
10557 "sequence": 1,
10558 "timer_id": "timer-1",
10559 "delay_seconds": 5,
10560 "fire_at": "2026-07-11T12:00:05Z",
10561 "fired_at": "2026-07-11T12:00:05Z",
10562 }),
10563 ),
10564 ];
10565
10566 for _restart in 0..2 {
10567 let ctx = workflow_context(history.clone());
10568 let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(5)));
10569 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10570 assert!(matches!(
10571 sleep.as_mut().poll(&mut task_context),
10572 Poll::Ready(Ok(()))
10573 ));
10574 assert!(ctx.take_commands().expect("commands").is_empty());
10575 ctx.ensure_history_consumed().expect("history consumed");
10576 }
10577 }
10578
10579 #[test]
10580 fn workflow_sleep_rejects_changed_delay_during_replay() {
10581 let ctx = workflow_context(vec![
10582 history_event(
10583 "TimerScheduled",
10584 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10585 ),
10586 history_event(
10587 "TimerFired",
10588 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10589 ),
10590 ]);
10591 let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(500)));
10592 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10593
10594 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10595 sleep.as_mut().poll(&mut task_context)
10596 else {
10597 panic!("changed timer delay must be rejected");
10598 };
10599 assert_eq!(failure.reason, "timer_delay_mismatch");
10600 assert_eq!(failure.sequence, Some(1));
10601 }
10602
10603 #[test]
10604 fn workflow_history_rejects_unpaired_or_mismatched_timer_events() {
10605 let lone_fire = WorkflowState::new(
10606 vec![history_event(
10607 "TimerFired",
10608 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10609 )],
10610 "rust-workers".to_string(),
10611 DEFAULT_CODEC.to_string(),
10612 None,
10613 )
10614 .expect_err("TimerFired requires TimerScheduled");
10615 assert!(matches!(
10616 lone_fire,
10617 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10618 if reason == "timer_schedule_missing_or_duplicate"
10619 ));
10620
10621 let wrong_identity = WorkflowState::new(
10622 vec![
10623 history_event(
10624 "TimerScheduled",
10625 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10626 ),
10627 history_event(
10628 "TimerFired",
10629 json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
10630 ),
10631 ],
10632 "rust-workers".to_string(),
10633 DEFAULT_CODEC.to_string(),
10634 None,
10635 )
10636 .expect_err("fire must match scheduled timer identity");
10637 assert!(matches!(
10638 wrong_identity,
10639 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10640 if reason == "timer_identity_mismatch"
10641 ));
10642
10643 let duplicate_fire = WorkflowState::new(
10644 vec![
10645 history_event(
10646 "TimerScheduled",
10647 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10648 ),
10649 history_event(
10650 "TimerFired",
10651 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10652 ),
10653 history_event(
10654 "TimerFired",
10655 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10656 ),
10657 ],
10658 "rust-workers".to_string(),
10659 DEFAULT_CODEC.to_string(),
10660 None,
10661 )
10662 .expect_err("a durable timer cannot fire twice");
10663 assert!(matches!(
10664 duplicate_fire,
10665 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10666 if reason == "duplicate_timer_fire"
10667 ));
10668
10669 let wrong_fired_delay = WorkflowState::new(
10670 vec![
10671 history_event(
10672 "TimerScheduled",
10673 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10674 ),
10675 history_event(
10676 "TimerFired",
10677 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 6}),
10678 ),
10679 ],
10680 "rust-workers".to_string(),
10681 DEFAULT_CODEC.to_string(),
10682 None,
10683 )
10684 .expect_err("timer schedule and fire delays must agree");
10685 assert!(matches!(
10686 wrong_fired_delay,
10687 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10688 if reason == "timer_history_delay_mismatch"
10689 ));
10690 }
10691
10692 #[test]
10693 fn replay_rejects_activity_moved_before_recorded_timer() {
10694 let ctx = workflow_context(vec![
10695 history_event(
10696 "TimerScheduled",
10697 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10698 ),
10699 history_event(
10700 "TimerFired",
10701 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10702 ),
10703 history_event(
10704 "ActivityCompleted",
10705 json!({
10706 "sequence": 2,
10707 "activity_type": "after-timer",
10708 "payload_codec": DEFAULT_CODEC,
10709 "result": fixture_envelope(json!("done")),
10710 }),
10711 ),
10712 ]);
10713 let mut activity = Box::pin(ctx.activity("after-timer", json!([])));
10714 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10715
10716 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10717 activity.as_mut().poll(&mut task_context)
10718 else {
10719 panic!("reordered durable command must be rejected");
10720 };
10721 assert_eq!(failure.reason, "recorded_command_mismatch");
10722 assert_eq!(failure.sequence, Some(1));
10723 assert_eq!(failure.expected.as_deref(), Some("timer"));
10724 assert_eq!(failure.actual.as_deref(), Some("activity:after-timer"));
10725 }
10726
10727 #[test]
10728 fn workflow_context_emits_a_typed_named_signal_wait() {
10729 let ctx = workflow_context(Vec::new());
10730 let mut signal = Box::pin(ctx.wait_signal("finish"));
10731 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10732
10733 assert!(matches!(
10734 signal.as_mut().poll(&mut task_context),
10735 Poll::Pending
10736 ));
10737 assert_eq!(
10738 ctx.take_commands().expect("signal-wait command"),
10739 vec![json!({
10740 "type": "open_signal_wait",
10741 "signal_name": "finish",
10742 })]
10743 );
10744 }
10745
10746 #[test]
10747 fn condition_wait_history_does_not_resolve_a_typed_signal_wait() {
10748 let ctx = workflow_context(vec![
10749 history_event(
10750 "ConditionWaitOpened",
10751 json!({"sequence": 1, "condition_key": "signal:finish"}),
10752 ),
10753 history_event(
10754 "ConditionWaitSatisfied",
10755 json!({"sequence": 1, "condition_key": "signal:finish"}),
10756 ),
10757 history_event(
10758 "SignalReceived",
10759 json!({"signal_name": "finish", "arguments": []}),
10760 ),
10761 ]);
10762 let mut signal = Box::pin(ctx.wait_signal("finish"));
10763 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10764
10765 assert!(matches!(
10766 signal.as_mut().poll(&mut task_context),
10767 Poll::Pending
10768 ));
10769 assert_eq!(
10770 ctx.take_commands().expect("typed signal-wait command"),
10771 vec![json!({
10772 "type": "open_signal_wait",
10773 "signal_name": "finish",
10774 })]
10775 );
10776 }
10777
10778 #[test]
10779 fn replay_orders_signal_waits_and_timers_in_one_command_stream() {
10780 let signal_then_timer = vec![
10781 history_event(
10782 "SignalWaitOpened",
10783 json!({"sequence": 1, "signal_name": "go"}),
10784 ),
10785 history_event(
10786 "SignalApplied",
10787 json!({
10788 "sequence": 1,
10789 "signal_name": "go",
10790 "value": fixture_envelope(json!(["now"])),
10791 }),
10792 ),
10793 history_event(
10794 "TimerScheduled",
10795 json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
10796 ),
10797 history_event(
10798 "TimerFired",
10799 json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
10800 ),
10801 ];
10802
10803 let ctx = workflow_context(signal_then_timer.clone());
10804 let mut signal = Box::pin(ctx.wait_signal("go"));
10805 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10806 assert!(matches!(
10807 signal.as_mut().poll(&mut task_context),
10808 Poll::Ready(Ok(arguments)) if arguments == vec![json!("now")]
10809 ));
10810 let mut timer = Box::pin(ctx.sleep(Duration::from_secs(5)));
10811 assert!(matches!(
10812 timer.as_mut().poll(&mut task_context),
10813 Poll::Ready(Ok(()))
10814 ));
10815 ctx.ensure_history_consumed()
10816 .expect("signal and timer history consumed in order");
10817
10818 let reordered = workflow_context(signal_then_timer);
10819 let mut timer_first = Box::pin(reordered.sleep(Duration::from_secs(5)));
10820 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10821 timer_first.as_mut().poll(&mut task_context)
10822 else {
10823 panic!("timer cannot consume signal-wait-first history");
10824 };
10825 assert_eq!(failure.reason, "recorded_command_mismatch");
10826 assert_eq!(failure.sequence, Some(1));
10827 assert_eq!(failure.expected.as_deref(), Some("signal wait"));
10828
10829 let timer_then_signal = vec![
10830 history_event(
10831 "TimerScheduled",
10832 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10833 ),
10834 history_event(
10835 "TimerFired",
10836 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10837 ),
10838 history_event(
10839 "SignalWaitOpened",
10840 json!({"sequence": 2, "signal_name": "go"}),
10841 ),
10842 history_event(
10843 "SignalApplied",
10844 json!({
10845 "sequence": 2,
10846 "signal_name": "go",
10847 "value": fixture_envelope(json!([])),
10848 }),
10849 ),
10850 ];
10851 let reordered = workflow_context(timer_then_signal);
10852 let mut signal_first = Box::pin(reordered.wait_signal("go"));
10853 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10854 signal_first.as_mut().poll(&mut task_context)
10855 else {
10856 panic!("signal wait cannot consume timer-first history");
10857 };
10858 assert_eq!(failure.reason, "recorded_command_mismatch");
10859 assert_eq!(failure.sequence, Some(1));
10860 assert_eq!(failure.expected.as_deref(), Some("timer"));
10861 }
10862
10863 #[test]
10864 fn workflow_history_rejects_duplicate_or_colliding_command_sequences() {
10865 let duplicate_timer = WorkflowState::new(
10866 vec![
10867 history_event(
10868 "TimerScheduled",
10869 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10870 ),
10871 history_event(
10872 "TimerScheduled",
10873 json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
10874 ),
10875 ],
10876 "rust-workers".to_string(),
10877 DEFAULT_CODEC.to_string(),
10878 None,
10879 )
10880 .expect_err("one workflow sequence cannot schedule two timers");
10881 assert!(matches!(
10882 duplicate_timer,
10883 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10884 if reason == "timer_schedule_missing_or_duplicate"
10885 ));
10886
10887 let colliding_kinds = WorkflowState::new(
10888 vec![
10889 history_event(
10890 "TimerScheduled",
10891 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10892 ),
10893 history_event(
10894 "ActivityCompleted",
10895 json!({"sequence": 1, "activity_type": "same-sequence"}),
10896 ),
10897 ],
10898 "rust-workers".to_string(),
10899 DEFAULT_CODEC.to_string(),
10900 None,
10901 )
10902 .expect_err("one workflow sequence cannot identify two command kinds");
10903 assert!(matches!(
10904 colliding_kinds,
10905 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10906 if reason == "durable_command_sequence_collision"
10907 ));
10908
10909 let duplicate_signal_wait = WorkflowState::new(
10910 vec![
10911 history_event(
10912 "SignalWaitOpened",
10913 json!({"sequence": 1, "signal_name": "go"}),
10914 ),
10915 history_event(
10916 "SignalWaitOpened",
10917 json!({"sequence": 1, "signal_name": "go"}),
10918 ),
10919 ],
10920 "rust-workers".to_string(),
10921 DEFAULT_CODEC.to_string(),
10922 None,
10923 )
10924 .expect_err("one workflow sequence cannot open two signal waits");
10925 assert!(matches!(
10926 duplicate_signal_wait,
10927 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10928 if reason == "signal_wait_open_missing_or_duplicate"
10929 ));
10930 }
10931
10932 #[test]
10933 fn workflow_history_accepts_a_first_command_after_global_sequence_gaps() {
10934 let result = encode_value_envelope(&json!({"captured": true}), DEFAULT_CODEC)
10935 .expect("side-effect result");
10936 let ctx = workflow_context(vec![history_event(
10937 "SideEffectRecorded",
10938 json!({"sequence": 99, "result": result}),
10939 )]);
10940
10941 let replayed: Value = ctx
10942 .side_effect(|| panic!("recorded side effect must not run"))
10943 .expect("positive global workflow sequence is valid");
10944 assert_eq!(replayed, json!({"captured": true}));
10945 ctx.ensure_history_consumed().expect("history consumed");
10946 }
10947
10948 #[test]
10949 fn workflow_history_rejects_zero_and_descending_command_sequences() {
10950 let result =
10951 encode_value_envelope(&json!("captured"), DEFAULT_CODEC).expect("side-effect result");
10952 let zero = WorkflowState::new(
10953 vec![history_event(
10954 "SideEffectRecorded",
10955 json!({"sequence": 0, "result": result.clone()}),
10956 )],
10957 "rust-workers".to_string(),
10958 DEFAULT_CODEC.to_string(),
10959 None,
10960 )
10961 .expect_err("durable command sequences must be positive");
10962 assert!(matches!(
10963 zero,
10964 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10965 if reason == "durable_command_sequence_invalid"
10966 ));
10967
10968 let descending = WorkflowState::new(
10969 vec![
10970 history_event(
10971 "SideEffectRecorded",
10972 json!({"sequence": 3, "result": result}),
10973 ),
10974 history_event(
10975 "VersionMarkerRecorded",
10976 json!({
10977 "sequence": 2,
10978 "change_id": "descending-marker",
10979 "version": 1,
10980 "min_supported": 1,
10981 "max_supported": 1,
10982 }),
10983 ),
10984 ],
10985 "rust-workers".to_string(),
10986 DEFAULT_CODEC.to_string(),
10987 None,
10988 )
10989 .expect_err("new durable commands must remain strictly ordered");
10990 let Error::NonDeterministicReplay(failure) = descending else {
10991 panic!("expected typed replay failure");
10992 };
10993 assert_eq!(failure.reason, "durable_command_sequence_mismatch");
10994 assert_eq!(failure.sequence, Some(2));
10995 assert_eq!(
10996 failure.expected.as_deref(),
10997 Some("workflow sequence greater than 3")
10998 );
10999 assert_eq!(failure.actual.as_deref(), Some("2"));
11000 }
11001
11002 #[test]
11003 fn workflow_task_replay_completes_after_signals_create_sequence_gaps() {
11004 fn worker() -> Worker {
11005 let client = Client::new("http://127.0.0.1:8080").expect("client");
11006 let mut worker = Worker::new(client, "rust-workers");
11007 worker.register_workflow("rust.finish-after-gaps", |ctx, _input| async move {
11008 ctx.wait_signal("finish").await?;
11009 let marker: String =
11010 ctx.side_effect(|| panic!("recorded side effect must not run"))?;
11011 assert_eq!(marker, "after-finish");
11012 Ok(json!("finished"))
11013 });
11014 worker
11015 }
11016
11017 let marker = encode_value_envelope(&json!("after-finish"), DEFAULT_CODEC)
11018 .expect("side-effect result");
11019 let task = workflow_task(
11020 "rust.finish-after-gaps",
11021 vec![
11022 history_event(
11023 "SignalWaitOpened",
11024 json!({"sequence": 1, "signal_name": "finish"}),
11025 ),
11026 history_event(
11027 "SignalReceived",
11028 json!({
11029 "signal_id": "increment-3",
11030 "signal_name": "increment",
11031 "workflow_sequence": 2,
11032 "payload_codec": DEFAULT_CODEC,
11033 "arguments": fixture_envelope(json!([3])),
11034 }),
11035 ),
11036 history_event(
11037 "SignalReceived",
11038 json!({
11039 "signal_id": "increment-5",
11040 "signal_name": "increment",
11041 "workflow_sequence": 3,
11042 "payload_codec": DEFAULT_CODEC,
11043 "arguments": fixture_envelope(json!([5])),
11044 }),
11045 ),
11046 history_event(
11047 "SignalReceived",
11048 json!({
11049 "signal_id": "finish",
11050 "signal_name": "finish",
11051 "workflow_sequence": 4,
11052 "payload_codec": DEFAULT_CODEC,
11053 "arguments": fixture_envelope(json!([])),
11054 }),
11055 ),
11056 history_event(
11057 "SignalApplied",
11058 json!({
11059 "sequence": 1,
11060 "signal_id": "finish",
11061 "signal_name": "finish",
11062 "payload_codec": DEFAULT_CODEC,
11063 "value": fixture_envelope(json!([])),
11064 }),
11065 ),
11066 history_event(
11067 "SideEffectRecorded",
11068 json!({"sequence": 5, "result": marker}),
11069 ),
11070 ],
11071 DEFAULT_CODEC,
11072 );
11073
11074 for _original_or_cold_worker in 0..2 {
11075 let commands = worker()
11076 .execute_workflow_task(task.clone())
11077 .expect("signal gaps preserve deterministic replay");
11078 assert_eq!(commands.len(), 1, "replay emits only terminal completion");
11079 assert_eq!(commands[0]["type"], "complete_workflow");
11080 assert_eq!(
11081 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("workflow output"),
11082 json!("finished")
11083 );
11084 }
11085 }
11086
11087 #[test]
11088 fn workflow_sleep_rejects_unrepresentable_rounded_duration() {
11089 let ctx = workflow_context(Vec::new());
11090 let mut sleep = Box::pin(ctx.start_timer(Duration::new(u64::MAX, 1)));
11091 let mut task_context = TaskContext::from_waker(noop_waker_ref());
11092 assert!(matches!(
11093 sleep.as_mut().poll(&mut task_context),
11094 Poll::Ready(Err(Error::TimerDurationOverflow))
11095 ));
11096 assert!(ctx.take_commands().expect("commands").is_empty());
11097 }
11098
11099 #[test]
11100 fn workflow_task_replay_completes_without_rescheduling_recorded_commands() {
11101 let client = Client::new("http://127.0.0.1:8080").expect("client");
11102 let mut worker = Worker::new(client, "rust-workers");
11103 worker.register_workflow("rust.timer", |ctx, _input| async move {
11104 ctx.sleep(Duration::from_secs(5)).await?;
11105 ctx.activity("after-timer", json!([])).await
11106 });
11107
11108 let task = |history_events| WorkflowTask {
11109 task_id: "wft-rust-timer-1".to_string(),
11110 workflow_id: Some("wf-rust-timer".to_string()),
11111 run_id: Some("run-rust-timer".to_string()),
11112 workflow_type: "rust.timer".to_string(),
11113 payload_codec: DEFAULT_CODEC.to_string(),
11114 arguments: Some(
11115 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
11116 ),
11117 history_events,
11118 total_history_events: None,
11119 history_size_bytes: None,
11120 continue_as_new_recommended: None,
11121 history_budget_pressure: None,
11122 next_history_page_token: None,
11123 workflow_task_attempt: 1,
11124 workflow_signal_id: None,
11125 signal_name: None,
11126 signal_arguments: None,
11127 workflow_update_id: None,
11128 update_name: None,
11129 lease_owner: Some("rust-worker".to_string()),
11130 };
11131
11132 let initial = worker
11133 .execute_workflow_task(task(Vec::new()))
11134 .expect("initial timer task");
11135 assert_eq!(
11136 initial,
11137 vec![json!({"type": "start_timer", "delay_seconds": 5})]
11138 );
11139
11140 let activity_result =
11141 encode_value_envelope(&json!("done"), DEFAULT_CODEC).expect("activity result");
11142 let replayed = worker
11143 .execute_workflow_task(task(vec![
11144 history_event(
11145 "TimerScheduled",
11146 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
11147 ),
11148 history_event(
11149 "TimerFired",
11150 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
11151 ),
11152 history_event(
11153 "ActivityCompleted",
11154 json!({
11155 "sequence": 2,
11156 "activity_type": "after-timer",
11157 "payload_codec": DEFAULT_CODEC,
11158 "result": activity_result,
11159 }),
11160 ),
11161 ]))
11162 .expect("replayed workflow task");
11163 assert_eq!(replayed.len(), 1);
11164 assert_eq!(replayed[0]["type"], "complete_workflow");
11165 assert_eq!(
11166 decode_wire_value(&replayed[0]["result"], DEFAULT_CODEC).expect("result"),
11167 json!("done")
11168 );
11169 }
11170
11171 #[test]
11172 fn workflow_continue_as_new_emits_arguments_type_and_queue_once() {
11173 let client = Client::new("http://127.0.0.1:8080").expect("client");
11174 let mut worker = Worker::new(client, "rust-workers");
11175 worker.register_workflow("rust.continue", |ctx, _input| async move {
11176 ctx.continue_as_new_with_options(
11177 ContinueAsNewOptions::new()
11178 .workflow_type("rust.next")
11179 .task_queue("next-workers"),
11180 json!([2, {"cursor": "next"}]),
11181 )
11182 });
11183
11184 let commands = worker
11185 .execute_workflow_task(workflow_task("rust.continue", Vec::new(), DEFAULT_CODEC))
11186 .expect("continue-as-new command");
11187
11188 assert_eq!(commands.len(), 1);
11189 assert_eq!(commands[0]["type"], "continue_as_new");
11190 assert_eq!(commands[0]["workflow_type"], "rust.next");
11191 assert_eq!(commands[0]["queue"], "next-workers");
11192 assert_eq!(
11193 decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC)
11194 .expect("continue-as-new arguments"),
11195 json!([2, {"cursor": "next"}])
11196 );
11197 }
11198
11199 #[test]
11200 fn continue_as_new_preserves_typed_arguments() {
11201 let client = Client::new("http://127.0.0.1:8080").expect("client");
11202 let mut worker = Worker::new(client, "rust-workers");
11203 worker.register_workflow_avro_value("rust.typed-continue", |ctx, _input| async move {
11204 ctx.continue_as_new(AvroValue::Array(vec![typed_fidelity_probe()]))?;
11205 unreachable!("continue-as-new returns a control-flow error")
11206 });
11207
11208 let commands = worker
11209 .execute_workflow_task(workflow_task(
11210 "rust.typed-continue",
11211 Vec::new(),
11212 DEFAULT_CODEC,
11213 ))
11214 .expect("typed continue-as-new command");
11215
11216 assert_eq!(commands[0]["type"], "continue_as_new");
11217 assert_eq!(
11218 decode_wire_avro_value(&commands[0]["arguments"], DEFAULT_CODEC)
11219 .expect("typed continue arguments"),
11220 AvroValue::Array(vec![typed_fidelity_probe()])
11221 );
11222 }
11223
11224 #[test]
11225 fn recorded_continue_as_new_is_consumed_without_duplicate_successor_command() {
11226 let client = Client::new("http://127.0.0.1:8080").expect("client");
11227 let mut worker = Worker::new(client, "rust-workers");
11228 worker.register_workflow("rust.continue", |ctx, _input| async move {
11229 ctx.continue_as_new(json!([2]))
11230 });
11231 let task = workflow_task(
11232 "rust.continue",
11233 vec![history_event(
11234 "WorkflowContinuedAsNew",
11235 json!({"sequence": 1, "continued_to_run_id": "run-next"}),
11236 )],
11237 DEFAULT_CODEC,
11238 );
11239
11240 for _worker_restart_or_redelivery in 0..2 {
11241 let commands = worker
11242 .execute_workflow_task(task.clone())
11243 .expect("recorded transition replays");
11244 assert!(
11245 commands.is_empty(),
11246 "replay must not emit another successor"
11247 );
11248 }
11249 }
11250
11251 #[test]
11252 fn continue_as_new_rejects_invalid_overrides_before_emitting_a_command() {
11253 let ctx = workflow_context(Vec::new());
11254 let error = ctx
11255 .continue_as_new_with_options(ContinueAsNewOptions::new().task_queue(" "), json!([1]))
11256 .expect_err("blank queue must be rejected");
11257
11258 let Error::InvalidContinueAsNewOptions(error) = error else {
11259 panic!("expected typed continue-as-new validation error");
11260 };
11261 assert_eq!(error.field, "task_queue");
11262 assert!(ctx.take_commands().expect("commands").is_empty());
11263 }
11264
11265 #[test]
11266 fn workflow_context_exposes_server_history_budget() {
11267 let client = Client::new("http://127.0.0.1:8080").expect("client");
11268 let mut worker = Worker::new(client, "rust-workers");
11269 worker.register_workflow("rust.history-budget", |ctx, _input| async move {
11270 let budget = ctx.history_budget()?;
11271 Ok(json!({
11272 "events": budget.event_count,
11273 "bytes": budget.size_bytes,
11274 "recommended": budget.continue_as_new_recommended,
11275 "pressure": budget.pressure,
11276 }))
11277 });
11278 let task: WorkflowTask = serde_json::from_value(json!({
11279 "task_id": "task-history-budget",
11280 "workflow_type": "rust.history-budget",
11281 "payload_codec": DEFAULT_CODEC,
11282 "history_events": [],
11283 "total_history_events": 480,
11284 "history_size_bytes": 1_048_576,
11285 "continue_as_new_recommended": true,
11286 "history_budget_pressure": "continue_as_new_recommended",
11287 }))
11288 .expect("published workflow task");
11289
11290 let commands = worker
11291 .execute_workflow_task(task)
11292 .expect("history-budget workflow");
11293 let result = decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("result");
11294 assert_eq!(result["events"], 480);
11295 assert_eq!(result["bytes"], 1_048_576);
11296 assert_eq!(result["recommended"], true);
11297 assert_eq!(result["pressure"], "continue_as_new_recommended");
11298 }
11299
11300 #[test]
11301 fn uncaught_workflow_handler_error_emits_terminal_failure_command() {
11302 let client = Client::new("http://127.0.0.1:8080").expect("client");
11303 let mut worker = Worker::new(client, "rust-workers");
11304 worker.register_workflow("rust.failing", |_ctx, _input| async move {
11305 Err(Error::Codec("rust_conformance_failure".to_string()))
11306 });
11307 let task = WorkflowTask {
11308 task_id: "wft-rust-failing-1".to_string(),
11309 workflow_id: Some("wf-rust-failing".to_string()),
11310 run_id: Some("run-rust-failing".to_string()),
11311 workflow_type: "rust.failing".to_string(),
11312 payload_codec: DEFAULT_CODEC.to_string(),
11313 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
11314 history_events: Vec::new(),
11315 total_history_events: Some(0),
11316 history_size_bytes: None,
11317 continue_as_new_recommended: None,
11318 history_budget_pressure: None,
11319 next_history_page_token: None,
11320 workflow_task_attempt: 1,
11321 workflow_signal_id: None,
11322 signal_name: None,
11323 signal_arguments: None,
11324 workflow_update_id: None,
11325 update_name: None,
11326 lease_owner: Some("rust-worker".to_string()),
11327 };
11328
11329 let commands = worker
11330 .execute_workflow_task(task)
11331 .expect("handler failure becomes a workflow command");
11332
11333 assert_eq!(commands.len(), 1);
11334 assert_eq!(commands[0]["type"], "fail_workflow");
11335 assert_eq!(commands[0]["exception_type"], "RustWorkflowError");
11336 assert_eq!(commands[0]["exception_class"], "durable_workflow::Error");
11337 assert_eq!(commands[0]["non_retryable"], false);
11338 assert_eq!(
11339 commands[0]["message"],
11340 "codec error: rust_conformance_failure"
11341 );
11342 assert_eq!(
11343 commands[0]["exception"]["message"],
11344 "codec error: rust_conformance_failure"
11345 );
11346 }
11347
11348 #[test]
11349 fn ordinary_handler_error_preserves_commands_queued_in_the_same_decision() {
11350 let client = Client::new("http://127.0.0.1:8080").expect("client");
11351 let mut worker = Worker::new(client, "rust-workers");
11352 worker.register_workflow("rust.failing-after-side-effect", |ctx, _input| async move {
11353 let _: String = ctx.side_effect(|| "captured".to_string())?;
11354 Err(Error::WorkerLoop("application failure".to_string()))
11355 });
11356
11357 let commands = worker
11358 .execute_workflow_task(workflow_task(
11359 "rust.failing-after-side-effect",
11360 Vec::new(),
11361 DEFAULT_CODEC,
11362 ))
11363 .expect("ordinary failure remains a workflow decision");
11364
11365 assert_eq!(commands.len(), 2);
11366 assert_eq!(commands[0]["type"], "record_side_effect");
11367 assert_eq!(commands[1]["type"], "fail_workflow");
11368 }
11369
11370 #[test]
11371 fn handler_error_cannot_hide_an_unconsumed_committed_side_effect() {
11372 let client = Client::new("http://127.0.0.1:8080").expect("client");
11373 let mut worker = Worker::new(client, "rust-workers");
11374 worker.register_workflow("rust.removed-side-effect", |_ctx, _input| async move {
11375 Err(Error::WorkerLoop("application failure".to_string()))
11376 });
11377 let result =
11378 encode_value_envelope(&json!("committed"), DEFAULT_CODEC).expect("side-effect result");
11379
11380 let error = worker
11381 .execute_workflow_task(workflow_task(
11382 "rust.removed-side-effect",
11383 vec![history_event(
11384 "SideEffectRecorded",
11385 json!({"sequence": 1, "result": result}),
11386 )],
11387 DEFAULT_CODEC,
11388 ))
11389 .expect_err("removed committed history must not become fail_workflow");
11390
11391 let Error::NonDeterministicReplay(failure) = error else {
11392 panic!("expected typed replay failure");
11393 };
11394 assert_eq!(failure.reason, "recorded_commands_unconsumed");
11395 assert_eq!(failure.sequence, Some(1));
11396 assert_eq!(failure.expected.as_deref(), Some("side effect"));
11397 }
11398
11399 #[test]
11400 fn replay_error_discards_side_effect_queued_before_incompatible_marker_check() {
11401 let client = Client::new("http://127.0.0.1:8080").expect("client");
11402 let mut worker = Worker::new(client, "rust-workers");
11403 worker.register_workflow(
11404 "rust.side-effect-before-marker-error",
11405 |ctx, _input| async move {
11406 assert_eq!(ctx.get_version("restart-safe", 1, 1)?, 1);
11407 let _: String = ctx.side_effect(|| "must-not-commit".to_string())?;
11408 ctx.get_version("restart-safe", 2, 2)?;
11409 Ok(Value::Null)
11410 },
11411 );
11412
11413 let error = worker
11414 .execute_workflow_task(workflow_task(
11415 "rust.side-effect-before-marker-error",
11416 vec![history_event(
11417 "VersionMarkerRecorded",
11418 json!({
11419 "sequence": 1,
11420 "change_id": "restart-safe",
11421 "version": 1,
11422 "min_supported": 1,
11423 "max_supported": 1,
11424 }),
11425 )],
11426 DEFAULT_CODEC,
11427 ))
11428 .expect_err("replay error must return no queued workflow commands");
11429
11430 let Error::NonDeterministicReplay(failure) = error else {
11431 panic!("expected typed replay failure");
11432 };
11433 assert_eq!(failure.reason, "version_marker_incompatible_range");
11434 assert_eq!(failure.sequence, Some(1));
11435 }
11436
11437 #[test]
11438 fn workflow_task_replay_keeps_recorded_unfired_timer_pending_without_rescheduling() {
11439 let client = Client::new("http://127.0.0.1:8080").expect("client");
11440 let mut worker = Worker::new(client, "rust-workers");
11441 worker.register_workflow("rust.timer.pending", |ctx, _input| async move {
11442 ctx.sleep(Duration::from_secs(5)).await?;
11443 Ok(json!({"status": "timer fired"}))
11444 });
11445
11446 let task = WorkflowTask {
11447 task_id: "wft-rust-timer-pending".to_string(),
11448 workflow_id: Some("wf-rust-timer".to_string()),
11449 run_id: Some("run-rust-timer".to_string()),
11450 workflow_type: "rust.timer.pending".to_string(),
11451 payload_codec: DEFAULT_CODEC.to_string(),
11452 arguments: Some(
11453 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
11454 ),
11455 history_events: vec![history_event(
11456 "TimerScheduled",
11457 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
11458 )],
11459 total_history_events: Some(1),
11460 history_size_bytes: None,
11461 continue_as_new_recommended: None,
11462 history_budget_pressure: None,
11463 next_history_page_token: None,
11464 workflow_task_attempt: 1,
11465 workflow_signal_id: None,
11466 signal_name: None,
11467 signal_arguments: None,
11468 workflow_update_id: None,
11469 update_name: None,
11470 lease_owner: Some("rust-worker".to_string()),
11471 };
11472
11473 for _redelivery_or_restart in 0..2 {
11474 let commands = worker
11475 .execute_workflow_task(task.clone())
11476 .expect("recorded timer remains pending");
11477 assert!(
11478 commands.is_empty(),
11479 "recorded timer must not be rescheduled"
11480 );
11481 }
11482 }
11483
11484 #[test]
11485 fn workflow_task_rejects_recorded_command_removed_from_workflow_code() {
11486 let client = Client::new("http://127.0.0.1:8080").expect("client");
11487 let mut worker = Worker::new(client, "rust-workers");
11488 worker.register_workflow("rust.timer.removed", |_ctx, _input| async move {
11489 Ok(json!({"status": "completed"}))
11490 });
11491 let task = WorkflowTask {
11492 task_id: "wft-rust-timer-removed".to_string(),
11493 workflow_id: Some("wf-rust-timer".to_string()),
11494 run_id: Some("run-rust-timer".to_string()),
11495 workflow_type: "rust.timer.removed".to_string(),
11496 payload_codec: DEFAULT_CODEC.to_string(),
11497 arguments: Some(
11498 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
11499 ),
11500 history_events: vec![
11501 history_event(
11502 "TimerScheduled",
11503 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
11504 ),
11505 history_event(
11506 "TimerFired",
11507 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
11508 ),
11509 ],
11510 total_history_events: Some(2),
11511 history_size_bytes: None,
11512 continue_as_new_recommended: None,
11513 history_budget_pressure: None,
11514 next_history_page_token: None,
11515 workflow_task_attempt: 1,
11516 workflow_signal_id: None,
11517 signal_name: None,
11518 signal_arguments: None,
11519 workflow_update_id: None,
11520 update_name: None,
11521 lease_owner: Some("rust-worker".to_string()),
11522 };
11523
11524 let Error::NonDeterministicReplay(failure) = worker
11525 .execute_workflow_task(task)
11526 .expect_err("removed timer must fail replay")
11527 else {
11528 panic!("expected typed replay failure");
11529 };
11530 assert_eq!(failure.reason, "recorded_commands_unconsumed");
11531 assert_eq!(failure.sequence, Some(1));
11532 }
11533
11534 #[test]
11535 fn workflow_context_emits_explicit_child_workflow_contract() {
11536 let ctx = WorkflowContext {
11537 state: Arc::new(Mutex::new(
11538 WorkflowState::new_with_identity(
11539 Vec::new(),
11540 Some("wf-parent".to_string()),
11541 Some("run-parent".to_string()),
11542 "parent-workers".to_string(),
11543 DEFAULT_CODEC.to_string(),
11544 None,
11545 )
11546 .expect("workflow state"),
11547 )),
11548 };
11549 let options = ChildWorkflowOptions::new("python-workers")
11550 .parent_close_policy(ParentClosePolicy::RequestCancel)
11551 .retry_policy(ChildWorkflowRetryPolicy {
11552 max_attempts: Some(3),
11553 backoff_seconds: vec![1, 5],
11554 non_retryable_error_types: vec!["ValidationError".to_string()],
11555 })
11556 .execution_timeout_seconds(600)
11557 .run_timeout_seconds(120);
11558 let mut call = Box::pin(ctx.start_child_workflow(
11559 "python.fulfil-order",
11560 options,
11561 json!([{"order_id": "order-42"}]),
11562 ));
11563 let mut task_context = TaskContext::from_waker(noop_waker_ref());
11564
11565 assert!(matches!(
11566 call.as_mut().poll(&mut task_context),
11567 Poll::Pending
11568 ));
11569 let commands = ctx.take_commands().expect("commands");
11570 assert_eq!(commands.len(), 1);
11571 let command = &commands[0];
11572 assert_eq!(command["type"], "start_child_workflow");
11573 assert_eq!(command["workflow_type"], "python.fulfil-order");
11574 assert_eq!(command["queue"], "python-workers");
11575 assert_eq!(command["parent_close_policy"], "request_cancel");
11576 assert_eq!(command["retry_policy"]["max_attempts"], 3);
11577 assert_eq!(command["execution_timeout_seconds"], 600);
11578 assert_eq!(command["run_timeout_seconds"], 120);
11579 assert_eq!(
11580 decode_wire_value(&command["arguments"], DEFAULT_CODEC).expect("child args"),
11581 json!([{"order_id": "order-42"}])
11582 );
11583 }
11584
11585 fn child_parent_worker() -> Worker {
11586 let client = Client::new("http://127.0.0.1:8080").expect("client");
11587 let mut worker = Worker::new(client, "rust-parent-workers");
11588 worker.register_workflow("rust.parent", |ctx, _input| async move {
11589 let child = ctx
11590 .start_child_workflow(
11591 "python.child",
11592 ChildWorkflowOptions::new("python-child-workers")
11593 .parent_close_policy(ParentClosePolicy::Terminate),
11594 json!([{"codec_probe": [1, true, "rust"]}]),
11595 )
11596 .await?;
11597 Ok(json!({
11598 "parent_workflow_id": child.parent.workflow_id,
11599 "parent_run_id": child.parent.run_id,
11600 "child_workflow_id": child.child.workflow_id,
11601 "child_run_id": child.child.run_id,
11602 "child_workflow_type": child.child_workflow_type,
11603 "result": child.result,
11604 }))
11605 });
11606 worker
11607 }
11608
11609 fn child_parent_task(event_type: &str, payload: Value) -> WorkflowTask {
11610 WorkflowTask {
11611 task_id: "wft-child-parent".to_string(),
11612 workflow_id: Some("wf-parent".to_string()),
11613 run_id: Some("run-parent".to_string()),
11614 workflow_type: "rust.parent".to_string(),
11615 payload_codec: DEFAULT_CODEC.to_string(),
11616 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
11617 history_events: vec![
11618 HistoryEvent {
11619 event_type: "ChildWorkflowScheduled".to_string(),
11620 payload: json!({
11621 "sequence": 1,
11622 "child_call_id": "call-child",
11623 "child_workflow_instance_id": "wf-child",
11624 "child_workflow_run_id": "run-child",
11625 "child_workflow_type": "python.child",
11626 }),
11627 raw: HashMap::new(),
11628 },
11629 HistoryEvent {
11630 event_type: event_type.to_string(),
11631 payload,
11632 raw: HashMap::new(),
11633 },
11634 ],
11635 total_history_events: Some(2),
11636 history_size_bytes: None,
11637 continue_as_new_recommended: None,
11638 history_budget_pressure: None,
11639 next_history_page_token: None,
11640 workflow_task_attempt: 1,
11641 workflow_signal_id: None,
11642 signal_name: None,
11643 signal_arguments: None,
11644 workflow_update_id: None,
11645 update_name: None,
11646 lease_owner: Some("rust-worker".to_string()),
11647 }
11648 }
11649
11650 #[test]
11651 fn committed_child_result_replays_without_starting_a_duplicate() {
11652 let worker = child_parent_worker();
11653 let task = child_parent_task(
11654 "ChildRunCompleted",
11655 json!({
11656 "sequence": 1,
11657 "child_call_id": "call-child",
11658 "child_workflow_instance_id": "wf-child",
11659 "child_workflow_run_id": "run-child",
11660 "child_workflow_type": "python.child",
11661 "payload_codec": DEFAULT_CODEC,
11662 "result": fixture_envelope(json!({"from":"python","ok":true})),
11663 }),
11664 );
11665
11666 for _restart in 0..2 {
11667 let commands = worker
11668 .execute_workflow_task(task.clone())
11669 .expect("replayed parent task");
11670 assert_eq!(commands.len(), 1);
11671 assert_eq!(commands[0]["type"], "complete_workflow");
11672 assert!(!commands
11673 .iter()
11674 .any(|command| command["type"] == "start_child_workflow"));
11675 let output =
11676 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
11677 assert_eq!(output["parent_workflow_id"], "wf-parent");
11678 assert_eq!(output["parent_run_id"], "run-parent");
11679 assert_eq!(output["child_workflow_id"], "wf-child");
11680 assert_eq!(output["child_run_id"], "run-child");
11681 assert_eq!(output["result"], json!({"from": "python", "ok": true}));
11682 }
11683 }
11684
11685 #[test]
11686 fn typed_child_arguments_and_results_survive_replay() {
11687 let client = Client::new("http://127.0.0.1:8080").expect("client");
11688 let mut worker = Worker::new(client, "rust-parent-workers");
11689 worker.register_workflow_avro_value("rust.typed-parent", |ctx, _input| async move {
11690 let child = ctx
11691 .start_child_workflow_avro_value(
11692 "python.typed-child",
11693 ChildWorkflowOptions::new("python-workers"),
11694 AvroValue::Array(vec![typed_fidelity_probe()]),
11695 )
11696 .await?;
11697 Ok(child.result)
11698 });
11699
11700 let initial = worker
11701 .execute_workflow_task(workflow_task(
11702 "rust.typed-parent",
11703 Vec::new(),
11704 DEFAULT_CODEC,
11705 ))
11706 .expect("typed child start");
11707 assert_eq!(initial[0]["type"], "start_child_workflow");
11708 assert_eq!(
11709 decode_wire_avro_value(&initial[0]["arguments"], DEFAULT_CODEC)
11710 .expect("typed child arguments"),
11711 AvroValue::Array(vec![typed_fidelity_probe()])
11712 );
11713
11714 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
11715 .expect("typed child result");
11716 let task = workflow_task(
11717 "rust.typed-parent",
11718 vec![
11719 history_event(
11720 "ChildWorkflowScheduled",
11721 json!({
11722 "sequence": 1,
11723 "child_call_id": "call-typed",
11724 "child_workflow_instance_id": "wf-child",
11725 "child_workflow_run_id": "run-child",
11726 "child_workflow_type": "python.typed-child",
11727 }),
11728 ),
11729 history_event(
11730 "ChildRunCompleted",
11731 json!({
11732 "sequence": 1,
11733 "child_call_id": "call-typed",
11734 "child_workflow_instance_id": "wf-child",
11735 "child_workflow_run_id": "run-child",
11736 "child_workflow_type": "python.typed-child",
11737 "payload_codec": DEFAULT_CODEC,
11738 "result": result,
11739 }),
11740 ),
11741 ],
11742 DEFAULT_CODEC,
11743 );
11744
11745 let commands = worker
11746 .execute_workflow_task(task)
11747 .expect("typed child replay");
11748 assert_eq!(commands[0]["type"], "complete_workflow");
11749 assert_eq!(
11750 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
11751 .expect("typed parent result"),
11752 typed_fidelity_probe()
11753 );
11754 }
11755
11756 #[test]
11757 fn pending_child_replays_after_restart_without_starting_a_duplicate() {
11758 let worker = child_parent_worker();
11759 let mut task = child_parent_task("unused", Value::Null);
11760 task.history_events.truncate(1);
11761 task.total_history_events = Some(1);
11762
11763 for _redelivery_or_restart in 0..2 {
11764 let commands = worker
11765 .execute_workflow_task(task.clone())
11766 .expect("recorded child remains pending");
11767 assert!(
11768 commands.is_empty(),
11769 "recorded pending child must not be started again"
11770 );
11771 }
11772 }
11773
11774 #[test]
11775 fn child_cancellation_becomes_stable_parent_failure_command() {
11776 let worker = child_parent_worker();
11777 let task = child_parent_task(
11778 "ChildRunCancelled",
11779 json!({
11780 "sequence": 1,
11781 "child_workflow_instance_id": "wf-child",
11782 "child_workflow_run_id": "run-child",
11783 "child_workflow_type": "python.child",
11784 "failure_id": "failure-child",
11785 "failure_category": "cancelled",
11786 "message": "cancelled by parent-close policy",
11787 }),
11788 );
11789
11790 let commands = worker
11791 .execute_workflow_task(task)
11792 .expect("parent settlement");
11793 assert_eq!(commands.len(), 1);
11794 assert_eq!(commands[0]["type"], "fail_workflow");
11795 assert_eq!(commands[0]["exception_type"], "ChildWorkflowCancelled");
11796 assert_eq!(
11797 commands[0]["exception"]["properties"]["reason"],
11798 "cancelled"
11799 );
11800 assert_eq!(
11801 commands[0]["exception"]["properties"]["child_workflow_run_id"],
11802 "run-child"
11803 );
11804 }
11805
11806 #[test]
11807 fn workflow_can_handle_typed_child_failure() {
11808 let client = Client::new("http://127.0.0.1:8080").expect("client");
11809 let mut worker = Worker::new(client, "rust-parent-workers");
11810 worker.register_workflow("rust.handled-parent", |ctx, _input| async move {
11811 match ctx
11812 .start_child_workflow(
11813 "python.child",
11814 ChildWorkflowOptions::new("python-child-workers"),
11815 json!([]),
11816 )
11817 .await
11818 {
11819 Err(Error::ChildWorkflowFailed(failure)) => Ok(json!({
11820 "reason": failure.reason,
11821 "failure_id": failure.failure_id,
11822 "exception_class": failure.exception_class,
11823 "child_run_id": failure.child_workflow_run_id,
11824 })),
11825 Err(error) => Err(error),
11826 Ok(_) => Err(Error::WorkerLoop(
11827 "child unexpectedly succeeded".to_string(),
11828 )),
11829 }
11830 });
11831 let mut task = child_parent_task(
11832 "ChildRunFailed",
11833 json!({
11834 "sequence": 1,
11835 "child_workflow_instance_id": "wf-child",
11836 "child_workflow_run_id": "run-child",
11837 "child_workflow_type": "python.child",
11838 "failure_id": "failure-child",
11839 "failure_category": "child_workflow",
11840 "message": "payment rejected",
11841 "exception": {
11842 "type": "PaymentRejected",
11843 "class": "payments.PaymentRejected",
11844 "message": "payment rejected"
11845 }
11846 }),
11847 );
11848 task.workflow_type = "rust.handled-parent".to_string();
11849
11850 let commands = worker.execute_workflow_task(task).expect("handled failure");
11851 assert_eq!(commands[0]["type"], "complete_workflow");
11852 let output =
11853 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
11854 assert_eq!(output["reason"], "child_workflow");
11855 assert_eq!(output["failure_id"], "failure-child");
11856 assert_eq!(output["exception_class"], "payments.PaymentRejected");
11857 assert_eq!(output["child_run_id"], "run-child");
11858 }
11859
11860 #[test]
11861 fn rust_hello_world_uses_signal_arguments_from_resume_payload() {
11862 let client = Client::new("http://127.0.0.1:8080").expect("client");
11863 let mut worker = Worker::new(client, "rust-workers");
11864
11865 worker.register_workflow("rust.hello_workflow", |ctx, _input| async move {
11866 let signal = ctx.wait_signal("start").await?;
11867 let name = signal
11868 .first()
11869 .and_then(|value| value.as_str())
11870 .unwrap_or("world");
11871 let greeting = ctx.activity("rust.hello_activity", json!([name])).await?;
11872 Ok(json!({
11873 "greeting": greeting,
11874 "language": "rust"
11875 }))
11876 });
11877
11878 let signal_arguments =
11879 encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC).expect("signal arguments");
11880 let task = WorkflowTask {
11881 task_id: "wft-rust-signal-1".to_string(),
11882 workflow_id: Some("wf-rust-hello".to_string()),
11883 run_id: Some("run-rust-hello".to_string()),
11884 workflow_type: "rust.hello_workflow".to_string(),
11885 payload_codec: DEFAULT_CODEC.to_string(),
11886 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
11887 history_events: vec![HistoryEvent {
11888 event_type: "SignalReceived".to_string(),
11889 payload: json!({
11890 "signal_id": "sig-rust-1",
11891 "signal_name": "start"
11892 }),
11893 raw: HashMap::new(),
11894 }],
11895 total_history_events: Some(1),
11896 history_size_bytes: None,
11897 continue_as_new_recommended: None,
11898 history_budget_pressure: None,
11899 next_history_page_token: None,
11900 workflow_task_attempt: 1,
11901 workflow_signal_id: Some("sig-rust-1".to_string()),
11902 signal_name: Some("start".to_string()),
11903 signal_arguments: Some(signal_arguments),
11904 workflow_update_id: None,
11905 update_name: None,
11906 lease_owner: Some("rust-worker".to_string()),
11907 };
11908
11909 let commands = worker.execute_workflow_task(task).expect("workflow task");
11910
11911 assert_eq!(commands.len(), 1);
11912 assert_eq!(commands[0]["type"], "schedule_activity");
11913 assert_eq!(commands[0]["activity_type"], "rust.hello_activity");
11914 assert_eq!(
11915 decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC).expect("activity args"),
11916 json!(["Rust"])
11917 );
11918 }
11919
11920 #[test]
11921 fn workflow_task_appends_paginated_history_events() {
11922 let mut task = WorkflowTask {
11923 task_id: "wft-rust-pages-1".to_string(),
11924 workflow_id: Some("wf-rust-pages".to_string()),
11925 run_id: Some("run-rust-pages".to_string()),
11926 workflow_type: "rust.hello_workflow".to_string(),
11927 payload_codec: DEFAULT_CODEC.to_string(),
11928 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
11929 history_events: vec![HistoryEvent {
11930 event_type: "WorkflowStarted".to_string(),
11931 payload: json!({}),
11932 raw: HashMap::new(),
11933 }],
11934 total_history_events: Some(3),
11935 history_size_bytes: None,
11936 continue_as_new_recommended: None,
11937 history_budget_pressure: None,
11938 next_history_page_token: Some("MQ==".to_string()),
11939 workflow_task_attempt: 1,
11940 workflow_signal_id: None,
11941 signal_name: None,
11942 signal_arguments: None,
11943 workflow_update_id: None,
11944 update_name: None,
11945 lease_owner: Some("rust-worker".to_string()),
11946 };
11947
11948 task.append_history_page(WorkflowTaskHistoryPage {
11949 history_events: vec![
11950 HistoryEvent {
11951 event_type: "SignalReceived".to_string(),
11952 payload: json!({
11953 "signal_id": "sig-rust-1",
11954 "signal_name": "start",
11955 "arguments": encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC)
11956 .expect("signal arguments")
11957 }),
11958 raw: HashMap::new(),
11959 },
11960 HistoryEvent {
11961 event_type: "MarkerRecorded".to_string(),
11962 payload: json!({"sequence": 3}),
11963 raw: HashMap::new(),
11964 },
11965 ],
11966 total_history_events: Some(3),
11967 next_history_page_token: None,
11968 });
11969
11970 assert_eq!(task.history_events.len(), 3);
11971 assert_eq!(task.total_history_events, Some(3));
11972 assert_eq!(task.next_history_page_token, None);
11973
11974 let signal = task
11975 .history_events
11976 .iter()
11977 .find(|event| event.event_type == "SignalReceived")
11978 .expect("signal event");
11979 assert_eq!(
11980 decode_signal_event_arguments(signal, DEFAULT_CODEC).expect("signal arguments"),
11981 vec![AvroValue::String("Rust".to_string())]
11982 );
11983 }
11984
11985 #[tokio::test]
11986 async fn query_handler_reads_ordered_cross_codec_signals_without_commands() {
11987 let client = Client::new("http://127.0.0.1:8080").expect("client");
11988 let mut worker = Worker::new(client, "rust-workers");
11989 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
11990 worker.register_query("counter", "current", |ctx, _args| async move {
11991 let mut count = 0_i64;
11992 for signal in ctx.signal_events() {
11993 let value = signal
11994 .arguments
11995 .first()
11996 .and_then(Value::as_i64)
11997 .unwrap_or_default();
11998 match signal.name.as_str() {
11999 "increment" => count += value,
12000 "set" => count = value,
12001 _ => {}
12002 }
12003 }
12004 Ok(json!(count))
12005 });
12006
12007 let task = QueryTask {
12008 query_task_id: "query-rust-counter".to_string(),
12009 query_task_attempt: 1,
12010 lease_owner: Some("rust-worker".to_string()),
12011 workflow_id: Some("counter-1".to_string()),
12012 run_id: Some("run-counter-1".to_string()),
12013 workflow_type: "counter".to_string(),
12014 query_name: "current".to_string(),
12015 payload_codec: DEFAULT_CODEC.to_string(),
12016 workflow_arguments: Some(
12017 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
12018 ),
12019 query_arguments: Some(
12020 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("query arguments"),
12021 ),
12022 history_events: vec![
12023 HistoryEvent {
12024 event_type: "SignalReceived".to_string(),
12025 payload: json!({
12026 "signal_id": "php-signal-1",
12027 "signal_name": "increment",
12028 "workflow_sequence": 1,
12029 "payload_codec": DEFAULT_CODEC,
12030 "arguments": encode_value_envelope(&json!([3]), DEFAULT_CODEC).expect("php avro signal")
12031 }),
12032 raw: HashMap::new(),
12033 },
12034 HistoryEvent {
12035 event_type: "SignalReceived".to_string(),
12036 payload: json!({
12037 "signal_id": "python-signal-2",
12038 "signal_name": "increment",
12039 "workflow_sequence": 2,
12040 "payload_codec": DEFAULT_CODEC,
12041 "arguments": encode_value_envelope(&json!([5]), DEFAULT_CODEC).expect("python avro signal")
12042 }),
12043 raw: HashMap::new(),
12044 },
12045 HistoryEvent {
12046 event_type: "SignalReceived".to_string(),
12047 payload: json!({
12048 "signal_id": "rust-signal-3",
12049 "signal_name": "set",
12050 "workflow_sequence": 3,
12051 "payload_codec": DEFAULT_CODEC,
12052 "arguments": encode_value_envelope(&json!([0]), DEFAULT_CODEC).expect("rust avro signal")
12053 }),
12054 raw: HashMap::new(),
12055 },
12056 ],
12057 history_export: None,
12058 run_status: Some("completed".to_string()),
12059 };
12060
12061 let result = worker.execute_query_task(task).await.expect("query result");
12062 assert_eq!(result.into_json().expect("query projection"), json!(0));
12063 }
12064
12065 #[tokio::test]
12066 async fn replayed_queries_read_running_completed_and_cold_restarted_instance_state() {
12067 let worker = replay_counter_worker();
12068 let running_history = json!([
12069 {
12070 "type": "ActivityCompleted",
12071 "payload": {
12072 "sequence": 1,
12073 "activity_type": "load-counter",
12074 "payload_codec": DEFAULT_CODEC,
12075 "result": fixture_envelope(json!("loaded"))
12076 }
12077 },
12078 {
12079 "type": "SignalWaitOpened",
12080 "payload": {
12081 "sequence": 3,
12082 "signal_name": "increment"
12083 }
12084 },
12085 {
12086 "type": "SignalReceived",
12087 "payload": {
12088 "signal_id": "signal-3",
12089 "signal_name": "increment",
12090 "workflow_sequence": 2,
12091 "payload_codec": DEFAULT_CODEC,
12092 "arguments": fixture_envelope(json!([3]))
12093 }
12094 },
12095 {
12096 "type": "SignalApplied",
12097 "payload": {
12098 "sequence": 3,
12099 "signal_id": "signal-3",
12100 "signal_name": "increment",
12101 "payload_codec": DEFAULT_CODEC,
12102 "value": fixture_envelope(json!([3]))
12103 }
12104 }
12105 ]);
12106
12107 let running = worker
12108 .execute_query_task(replay_counter_query(
12109 "current",
12110 running_history.clone(),
12111 "running",
12112 ))
12113 .await
12114 .expect("running replay query");
12115 assert_eq!(
12116 running.clone().into_json().expect("query projection"),
12117 json!({"loaded": "loaded", "count": 3, "finished": false})
12118 );
12119
12120 let detached = worker
12121 .execute_query_task(replay_counter_query(
12122 "detached-mutation",
12123 running_history.clone(),
12124 "running",
12125 ))
12126 .await
12127 .expect("query mutates only its detached state clone");
12128 assert_eq!(detached.into_json().expect("query projection"), json!(999));
12129 let failed = worker
12130 .execute_query_task(replay_counter_query(
12131 "failed-mutation",
12132 running_history.clone(),
12133 "running",
12134 ))
12135 .await
12136 .expect_err("failed query");
12137 assert_eq!(failed.reason, "query_rejected");
12138 let unchanged = worker
12139 .execute_query_task(replay_counter_query("current", running_history, "running"))
12140 .await
12141 .expect("later query reconstructs unchanged state");
12142 assert_eq!(unchanged, running);
12143
12144 let restarted_worker = replay_counter_worker();
12145 let empty_arguments = fixture_envelope(json!([]));
12146 let loaded_result = fixture_envelope(json!("loaded"));
12147 let signal_three = fixture_blob(json!([3]));
12148 let signal_five = fixture_blob(json!([5]));
12149 let restarted_task: QueryTask = serde_json::from_value(json!({
12150 "query_task_id": "query-after-restart",
12151 "workflow_id": "counter-1",
12152 "run_id": "run-counter-1",
12153 "workflow_type": "replay-counter",
12154 "query_name": "current",
12155 "payload_codec": DEFAULT_CODEC,
12156 "workflow_arguments": empty_arguments.clone(),
12157 "query_arguments": empty_arguments,
12158 "history_events": [],
12159 "history_export": {
12160 "payloads": {"codec": DEFAULT_CODEC},
12161 "history_events": [
12162 {
12163 "type": "ActivityCompleted",
12164 "payload": {
12165 "sequence": 1,
12166 "activity_type": "load-counter",
12167 "payload_codec": DEFAULT_CODEC,
12168 "result": null
12169 }
12170 },
12171 {
12172 "type": "SignalWaitOpened",
12173 "payload": {
12174 "sequence": 3,
12175 "signal_name": "increment"
12176 }
12177 },
12178 {
12179 "type": "SignalReceived",
12180 "payload": {
12181 "signal_id": "signal-3",
12182 "signal_name": "increment",
12183 "workflow_sequence": 2
12184 }
12185 },
12186 {
12187 "type": "SignalApplied",
12188 "payload": {
12189 "sequence": 3,
12190 "signal_id": "signal-3",
12191 "signal_name": "increment"
12192 }
12193 },
12194 {
12195 "type": "SignalWaitOpened",
12196 "payload": {
12197 "sequence": 5,
12198 "signal_name": "increment"
12199 }
12200 },
12201 {
12202 "type": "SignalReceived",
12203 "payload": {
12204 "signal_id": "signal-5",
12205 "signal_name": "increment",
12206 "workflow_sequence": 4
12207 }
12208 },
12209 {
12210 "type": "SignalApplied",
12211 "payload": {
12212 "sequence": 5,
12213 "signal_id": "signal-5",
12214 "signal_name": "increment"
12215 }
12216 }
12217 ],
12218 "activities": [{
12219 "sequence": 1,
12220 "activity_type": "load-counter",
12221 "payload_codec": DEFAULT_CODEC,
12222 "result": loaded_result
12223 }],
12224 "signals": [
12225 {
12226 "id": "signal-3",
12227 "name": "increment",
12228 "workflow_sequence": 2,
12229 "payload_codec": DEFAULT_CODEC,
12230 "arguments": signal_three
12231 },
12232 {
12233 "id": "signal-5",
12234 "name": "increment",
12235 "workflow_sequence": 4,
12236 "payload_codec": DEFAULT_CODEC,
12237 "arguments": signal_five
12238 }
12239 ]
12240 },
12241 "run_status": "completed"
12242 }))
12243 .expect("cold replay query task");
12244 let completed = restarted_worker
12245 .execute_query_task(restarted_task)
12246 .await
12247 .expect("completed cold replay query");
12248 assert_eq!(
12249 completed.into_json().expect("query projection"),
12250 json!({"loaded": "loaded", "count": 8, "finished": true})
12251 );
12252 }
12253
12254 #[tokio::test]
12255 async fn replayed_query_replay_failures_are_machine_readable() {
12256 let worker = replay_counter_worker();
12257 let task = replay_counter_query(
12258 "current",
12259 json!([{
12260 "type": "ActivityCompleted",
12261 "payload": {
12262 "sequence": 1,
12263 "payload_codec": DEFAULT_CODEC,
12264 "result": {"codec": DEFAULT_CODEC, "blob": "{"}
12265 }
12266 }]),
12267 "running",
12268 );
12269 let failure = worker
12270 .execute_query_task(task)
12271 .await
12272 .expect_err("invalid replay history payload");
12273 assert_eq!(failure.reason, "query_payload_decode_failed");
12274 assert_eq!(failure.failure_type, "QueryPayloadDecodeFailed");
12275 assert!(failure.message.contains("invalid_payload_framing"));
12276 }
12277
12278 #[tokio::test]
12279 async fn query_task_restores_compact_history_from_export() {
12280 let client = Client::new("http://127.0.0.1:8080").expect("client");
12281 let mut worker = Worker::new(client, "rust-workers");
12282 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
12283 worker.register_query("counter", "current", |ctx, _args| async move {
12284 Ok(json!(ctx.signals("increment")[0][0]))
12285 });
12286 let empty_arguments = fixture_envelope(json!([]));
12287 let exported_signal = fixture_blob(json!([9]));
12288 let task: QueryTask = serde_json::from_value(json!({
12289 "query_task_id": "query-export",
12290 "workflow_type": "counter",
12291 "query_name": "current",
12292 "payload_codec": DEFAULT_CODEC,
12293 "workflow_arguments": empty_arguments.clone(),
12294 "query_arguments": empty_arguments,
12295 "history_events": [],
12296 "history_export": {
12297 "payloads": {"codec": DEFAULT_CODEC},
12298 "history_events": [{
12299 "type": "SignalReceived",
12300 "payload": {"signal_id": "signal-export", "signal_name": "increment"}
12301 }],
12302 "signals": [{
12303 "id": "signal-export",
12304 "name": "increment",
12305 "status": "applied",
12306 "workflow_sequence": 1,
12307 "payload_codec": DEFAULT_CODEC,
12308 "arguments": exported_signal
12309 }]
12310 }
12311 }))
12312 .expect("query task");
12313
12314 let result = worker.execute_query_task(task).await.expect("query result");
12315 assert_eq!(result.into_json().expect("query projection"), json!(9));
12316 }
12317
12318 #[tokio::test]
12319 async fn query_task_failures_have_stable_reasons() {
12320 let client = Client::new("http://127.0.0.1:8080").expect("client");
12321 let mut worker = Worker::new(client, "rust-workers");
12322 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
12323 worker.register_query(
12324 "counter",
12325 "current",
12326 |_ctx, _args| async move { Ok(json!(0)) },
12327 );
12328
12329 let base_task = QueryTask {
12330 query_task_id: "query-errors".to_string(),
12331 query_task_attempt: 1,
12332 lease_owner: None,
12333 workflow_id: Some("counter-errors".to_string()),
12334 run_id: Some("run-errors".to_string()),
12335 workflow_type: "counter".to_string(),
12336 query_name: "missing".to_string(),
12337 payload_codec: DEFAULT_CODEC.to_string(),
12338 workflow_arguments: Some(fixture_envelope(json!([]))),
12339 query_arguments: Some(fixture_envelope(json!([]))),
12340 history_events: Vec::new(),
12341 history_export: None,
12342 run_status: Some("running".to_string()),
12343 };
12344
12345 let unknown = worker
12346 .execute_query_task(base_task.clone())
12347 .await
12348 .expect_err("unknown query");
12349 assert_eq!(unknown.reason, "rejected_unknown_query");
12350
12351 let mut malformed = base_task;
12352 malformed.query_name = "current".to_string();
12353 malformed.query_arguments = Some(json!({"codec": DEFAULT_CODEC, "blob": "{"}));
12354 let malformed = worker
12355 .execute_query_task(malformed)
12356 .await
12357 .expect_err("malformed payload");
12358 assert_eq!(malformed.reason, "query_payload_decode_failed");
12359
12360 let client = Client::new("http://127.0.0.1:8080").expect("client");
12361 let mut unavailable_worker = Worker::new(client, "rust-workers");
12362 unavailable_worker
12363 .register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
12364 let empty_arguments = fixture_envelope(json!([]));
12365 let unavailable_task: QueryTask = serde_json::from_value(json!({
12366 "query_task_id": "query-unavailable",
12367 "workflow_type": "counter",
12368 "query_name": "current",
12369 "payload_codec": DEFAULT_CODEC,
12370 "workflow_arguments": empty_arguments.clone(),
12371 "query_arguments": empty_arguments
12372 }))
12373 .expect("query task");
12374 let unavailable = unavailable_worker
12375 .execute_query_task(unavailable_task)
12376 .await
12377 .expect_err("query handler unavailable");
12378 assert_eq!(unavailable.reason, "query_handler_unavailable");
12379 }
12380
12381 #[tokio::test]
12382 async fn client_query_decodes_result_and_typed_failure() {
12383 let server = MockWorkerServer::start();
12384 let client = Client::builder(server.base_url())
12385 .timeout(Duration::from_secs(2))
12386 .build()
12387 .expect("client");
12388
12389 let result = client
12390 .query_workflow("counter-1", "current", json!([]))
12391 .await
12392 .expect("query result");
12393 assert_eq!(result, json!({"count": 8}));
12394
12395 let error = client
12396 .query_workflow("counter-1", "missing", json!([]))
12397 .await
12398 .expect_err("unknown query");
12399 let Error::QueryFailed(failure) = error else {
12400 panic!("expected typed query failure");
12401 };
12402 assert_eq!(failure.status, 404);
12403 assert_eq!(failure.reason, "rejected_unknown_query");
12404 }
12405
12406 #[tokio::test]
12407 async fn public_client_surfaces_send_and_receive_lossless_avro_values() {
12408 let server = MockWorkerServer::start();
12409 let client = Client::builder(server.base_url())
12410 .timeout(Duration::from_secs(2))
12411 .build()
12412 .expect("client");
12413 let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
12414
12415 client
12416 .start_workflow(
12417 "typed.echo",
12418 "rust-workers",
12419 "typed-start",
12420 arguments.clone(),
12421 )
12422 .await
12423 .expect("typed workflow start");
12424 assert_eq!(
12425 decode_wire_avro_value(
12426 &server.request_body("/api/workflows")["input"],
12427 DEFAULT_CODEC,
12428 )
12429 .expect("typed start input"),
12430 arguments
12431 );
12432
12433 client
12434 .signal_workflow("typed-1", "changed", arguments.clone())
12435 .await
12436 .expect("typed signal");
12437 assert_eq!(
12438 decode_wire_avro_value(
12439 &server.request_body("/api/workflows/typed-1/signal/changed")["input"],
12440 DEFAULT_CODEC,
12441 )
12442 .expect("typed signal input"),
12443 arguments
12444 );
12445
12446 assert_eq!(
12447 client
12448 .query_workflow_avro_value("typed-1", "inspect", arguments.clone())
12449 .await
12450 .expect("typed query"),
12451 typed_fidelity_probe()
12452 );
12453 assert_eq!(
12454 decode_wire_avro_value(
12455 &server.request_body("/api/workflows/typed-1/query/inspect")["input"],
12456 DEFAULT_CODEC,
12457 )
12458 .expect("typed query input"),
12459 arguments
12460 );
12461
12462 assert_eq!(
12463 client
12464 .update_workflow_avro_value(
12465 "typed-1",
12466 "replace",
12467 arguments.clone(),
12468 Some("typed-request"),
12469 )
12470 .await
12471 .expect("typed update"),
12472 typed_fidelity_probe()
12473 );
12474 let update = server.request_body("/api/workflows/typed-1/update/replace");
12475 assert_eq!(update["request_id"], "typed-request");
12476 assert_eq!(
12477 decode_wire_avro_value(&update["input"], DEFAULT_CODEC).expect("typed update input"),
12478 arguments
12479 );
12480
12481 let handle = WorkflowHandle {
12482 client: client.clone(),
12483 workflow_id: "typed-1".to_string(),
12484 run_id: Some("run-typed-1".to_string()),
12485 workflow_type: "typed.echo".to_string(),
12486 };
12487 assert_eq!(
12488 handle
12489 .result_avro_value(WorkflowResultOptions::default())
12490 .await
12491 .expect("typed workflow result"),
12492 typed_fidelity_probe()
12493 );
12494
12495 client
12496 .complete_activity_task(
12497 "activity-typed",
12498 "attempt-typed",
12499 "rust-worker",
12500 typed_fidelity_probe(),
12501 DEFAULT_CODEC,
12502 )
12503 .await
12504 .expect("typed activity completion");
12505 assert_eq!(
12506 decode_wire_avro_value(
12507 &server.request_body("/api/worker/activity-tasks/activity-typed/complete")
12508 ["result"],
12509 DEFAULT_CODEC,
12510 )
12511 .expect("typed activity result"),
12512 typed_fidelity_probe()
12513 );
12514 client
12515 .fail_activity_task(
12516 "activity-typed",
12517 "attempt-typed",
12518 "rust-worker",
12519 "typed failure",
12520 true,
12521 )
12522 .await
12523 .expect("activity failure");
12524 }
12525
12526 #[tokio::test]
12527 async fn lifecycle_commands_support_instance_and_selected_run_targets() {
12528 let server = MockWorkerServer::start();
12529 let client = Client::builder(server.base_url())
12530 .timeout(Duration::from_secs(2))
12531 .build()
12532 .expect("client");
12533
12534 let options = WorkflowCommandOptions::new()
12535 .reason("cleanup requested")
12536 .request_id("cancel-17");
12537 let cancelled = client
12538 .cancel_workflow("wf-lifecycle", options)
12539 .await
12540 .expect("instance cancellation");
12541 assert_eq!(cancelled.command, WorkflowCommandKind::Cancel);
12542 assert_eq!(cancelled.run_id.as_deref(), Some("run-current"));
12543 assert_eq!(cancelled.outcome.as_deref(), Some("cancelled"));
12544 assert_eq!(
12545 server.request_body("/api/workflows/wf-lifecycle/cancel"),
12546 json!({"reason":"cleanup requested","request_id":"cancel-17"})
12547 );
12548
12549 let terminated = client
12550 .terminate_workflow(
12551 "wf-lifecycle",
12552 WorkflowCommandOptions::new().reason("forced stop"),
12553 )
12554 .await
12555 .expect("instance termination");
12556 assert_eq!(terminated.command, WorkflowCommandKind::Terminate);
12557 assert_eq!(terminated.outcome.as_deref(), Some("terminated"));
12558
12559 client
12560 .cancel_workflow_run(
12561 "wf-lifecycle",
12562 "run-current",
12563 WorkflowCommandOptions::default(),
12564 )
12565 .await
12566 .expect("selected run cancellation");
12567 client
12568 .terminate_workflow_run(
12569 "wf-lifecycle",
12570 "run-current",
12571 WorkflowCommandOptions::default(),
12572 )
12573 .await
12574 .expect("selected run termination");
12575
12576 for (command, error) in [
12577 (
12578 WorkflowCommandKind::Cancel,
12579 client
12580 .cancel_workflow_run(
12581 "wf-lifecycle",
12582 "run-stale",
12583 WorkflowCommandOptions::default(),
12584 )
12585 .await
12586 .expect_err("stale cancellation must be rejected"),
12587 ),
12588 (
12589 WorkflowCommandKind::Terminate,
12590 client
12591 .terminate_workflow_run(
12592 "wf-lifecycle",
12593 "run-stale",
12594 WorkflowCommandOptions::default(),
12595 )
12596 .await
12597 .expect_err("stale termination must be rejected"),
12598 ),
12599 ] {
12600 let Error::WorkflowCommandRejected(rejection) = error else {
12601 panic!("expected typed command rejection");
12602 };
12603 assert_eq!(rejection.command, command);
12604 assert_eq!(rejection.status, 409);
12605 assert_eq!(rejection.reason, "historical_run_command_rejected");
12606 assert_eq!(rejection.run_id.as_deref(), Some("run-stale"));
12607 assert_eq!(rejection.target_scope.as_deref(), Some("run"));
12608 }
12609 }
12610
12611 #[tokio::test]
12612 async fn workflow_start_options_send_server_enforced_deadlines() {
12613 let server = MockWorkerServer::start();
12614 let client = Client::builder(server.base_url())
12615 .timeout(Duration::from_secs(2))
12616 .build()
12617 .expect("client");
12618
12619 let handle = client
12620 .start_workflow_with_options(
12621 "rust.timeout",
12622 "rust-timeouts",
12623 "wf-start-options",
12624 WorkflowStartOptions::new()
12625 .execution_timeout_seconds(30)
12626 .run_timeout_seconds(1),
12627 json!([]),
12628 )
12629 .await
12630 .expect("workflow start");
12631
12632 assert_eq!(handle.run_id.as_deref(), Some("run-start-options"));
12633 let body = server.request_body("/api/workflows");
12634 assert_eq!(body["execution_timeout_seconds"], 30);
12635 assert_eq!(body["run_timeout_seconds"], 1);
12636
12637 let invalid = client
12638 .start_workflow_with_options(
12639 "rust.timeout",
12640 "rust-timeouts",
12641 "wf-invalid-options",
12642 WorkflowStartOptions::new()
12643 .execution_timeout_seconds(1)
12644 .run_timeout_seconds(2),
12645 json!([]),
12646 )
12647 .await
12648 .expect_err("invalid deadline ordering");
12649 assert!(invalid
12650 .to_string()
12651 .contains("run_timeout_seconds cannot exceed execution_timeout_seconds"));
12652 }
12653
12654 #[tokio::test]
12655 async fn workflow_result_returns_each_typed_terminal_outcome() {
12656 let server = MockWorkerServer::start();
12657 let client = Client::builder(server.base_url())
12658 .timeout(Duration::from_secs(2))
12659 .build()
12660 .expect("client");
12661 let options = WorkflowResultOptions {
12662 poll_interval: Duration::ZERO,
12663 timeout: Duration::from_secs(1),
12664 };
12665
12666 let failed = WorkflowHandle {
12667 client: client.clone(),
12668 workflow_id: "wf-failed".to_string(),
12669 run_id: Some("run-failed".to_string()),
12670 workflow_type: "failure".to_string(),
12671 }
12672 .result(options)
12673 .await
12674 .expect_err("failed outcome");
12675 let Error::WorkflowFailed(failure) = failed else {
12676 panic!("expected WorkflowFailed");
12677 };
12678 assert_eq!(failure.workflow_id, "wf-failed");
12679 assert_eq!(failure.run_id.as_deref(), Some("run-failed"));
12680 assert_eq!(failure.failure_id.as_deref(), Some("failure-17"));
12681 assert_eq!(failure.failure_category.as_deref(), Some("application"));
12682 assert_eq!(failure.exception_type.as_deref(), Some("PaymentError"));
12683 assert_eq!(
12684 failure.exception_class.as_deref(),
12685 Some("billing::PaymentError")
12686 );
12687 assert_eq!(failure.non_retryable, Some(true));
12688
12689 for (workflow_id, expected_kind, expected_reason) in [
12690 (
12691 "wf-cancelled",
12692 WorkflowTerminalKind::Cancelled,
12693 "cleanup requested",
12694 ),
12695 (
12696 "wf-terminated",
12697 WorkflowTerminalKind::Terminated,
12698 "forced stop",
12699 ),
12700 (
12701 "wf-timed-out",
12702 WorkflowTerminalKind::TimedOut,
12703 "run_timeout",
12704 ),
12705 ] {
12706 let error = WorkflowHandle {
12707 client: client.clone(),
12708 workflow_id: workflow_id.to_string(),
12709 run_id: None,
12710 workflow_type: "terminal".to_string(),
12711 }
12712 .result(options)
12713 .await
12714 .expect_err("typed terminal outcome");
12715 let outcome = match error {
12716 Error::WorkflowCancelled(outcome) => outcome,
12717 Error::WorkflowTerminated(outcome) => outcome,
12718 Error::WorkflowTimedOut(outcome) => outcome,
12719 other => panic!("unexpected terminal error: {other}"),
12720 };
12721 assert_eq!(outcome.kind, expected_kind);
12722 assert_eq!(outcome.workflow_id, workflow_id);
12723 assert_eq!(outcome.reason, expected_reason);
12724 }
12725
12726 let wait_timeout = WorkflowHandle {
12727 client,
12728 workflow_id: "wf-waiting".to_string(),
12729 run_id: Some("run-waiting".to_string()),
12730 workflow_type: "waiting".to_string(),
12731 }
12732 .result(WorkflowResultOptions {
12733 poll_interval: Duration::ZERO,
12734 timeout: Duration::ZERO,
12735 })
12736 .await
12737 .expect_err("client wait timeout");
12738 let Error::WorkflowTimedOut(timeout) = wait_timeout else {
12739 panic!("expected typed client timeout");
12740 };
12741 assert_eq!(timeout.reason, "result_wait_timeout");
12742 assert_eq!(timeout.failure_category.as_deref(), Some("client_timeout"));
12743 assert_eq!(timeout.run_id.as_deref(), Some("run-waiting"));
12744 }
12745
12746 #[tokio::test]
12747 async fn workflow_result_follows_chain_and_selected_result_preserves_history() {
12748 let server = MockWorkerServer::start();
12749 let client = Client::builder(server.base_url())
12750 .timeout(Duration::from_secs(2))
12751 .build()
12752 .expect("client");
12753
12754 let handle = WorkflowHandle {
12755 client,
12756 workflow_id: "wf-selected".to_string(),
12757 run_id: Some("run-selected".to_string()),
12758 workflow_type: "selected".to_string(),
12759 };
12760 let options = WorkflowResultOptions {
12761 poll_interval: Duration::ZERO,
12762 timeout: Duration::from_secs(1),
12763 };
12764
12765 let current = handle
12766 .result(options)
12767 .await
12768 .expect("instance result follows the current run");
12769 assert_eq!(current, json!("current run output"));
12770
12771 let error = handle
12772 .result_selected_run(options)
12773 .await
12774 .expect_err("the selected run is cancelled even though the current run completed");
12775
12776 let Error::WorkflowCancelled(outcome) = error else {
12777 panic!("expected selected run cancellation");
12778 };
12779 assert_eq!(outcome.run_id.as_deref(), Some("run-selected"));
12780 assert_eq!(outcome.reason, "selected run cancelled");
12781 assert_eq!(
12782 server.request_count("/api/workflows/wf-selected/runs/run-selected"),
12783 1
12784 );
12785 assert_eq!(server.request_count("/api/workflows/wf-selected"), 1);
12786 }
12787
12788 #[tokio::test]
12789 async fn poll_responses_decode_http_conflict_drain_as_a_stable_stop() {
12790 let server = MockWorkerServer::draining_polls();
12791 let client = Client::builder(server.base_url())
12792 .timeout(Duration::from_secs(2))
12793 .build()
12794 .expect("client");
12795
12796 let workflow = client
12797 .poll_workflow_task_response("draining-worker", "rust-workers", Duration::ZERO)
12798 .await
12799 .expect("workflow drain response");
12800 let activity = client
12801 .poll_activity_task_response("draining-worker", "rust-workers", Duration::ZERO)
12802 .await
12803 .expect("activity drain response");
12804 let query = client
12805 .poll_query_task_response("draining-worker", "rust-workers", Duration::ZERO)
12806 .await
12807 .expect("query drain response");
12808
12809 for outcome in [workflow.outcome(), activity.outcome(), query.outcome()] {
12810 assert_eq!(
12811 outcome,
12812 WorkerPollOutcome::Stop {
12813 poll_status: Some("draining".to_string()),
12814 reason: Some("worker_draining".to_string()),
12815 }
12816 );
12817 }
12818
12819 assert!(client
12820 .poll_workflow_task("draining-worker", "rust-workers", Duration::ZERO)
12821 .await
12822 .expect("compatibility poll")
12823 .is_none());
12824 }
12825
12826 #[tokio::test]
12827 async fn managed_worker_honors_drain_stop_for_every_task_family() {
12828 let server = MockWorkerServer::draining_polls();
12829 let client = Client::builder(server.base_url())
12830 .timeout(Duration::from_secs(2))
12831 .build()
12832 .expect("client");
12833
12834 let mut workflow_worker = Worker::new(client.clone(), "rust-workers")
12835 .worker_id("draining-workflow-worker")
12836 .poll_timeout(Duration::ZERO);
12837 workflow_worker.register_workflow("counter", |_ctx, _args| async { Ok(Value::Null) });
12838 workflow_worker
12839 .run()
12840 .await
12841 .expect("workflow drain is a clean stop");
12842
12843 let mut activity_worker = Worker::new(client.clone(), "rust-workers")
12844 .worker_id("draining-activity-worker")
12845 .poll_timeout(Duration::ZERO);
12846 activity_worker.register_activity("write", |_ctx, _args| async { Ok(Value::Null) });
12847 activity_worker
12848 .run()
12849 .await
12850 .expect("activity drain is a clean stop");
12851
12852 let mut query_worker = Worker::new(client, "rust-workers")
12853 .worker_id("draining-query-worker")
12854 .poll_timeout(Duration::ZERO);
12855 query_worker.register_query("counter", "current", |_ctx, _args| async {
12856 Ok(Value::Null)
12857 });
12858 query_worker
12859 .run()
12860 .await
12861 .expect("query drain is a clean stop");
12862 }
12863
12864 #[tokio::test]
12865 async fn activity_cancellation_and_late_completion_remain_machine_readable() {
12866 let server = MockWorkerServer::start();
12867 let client = Client::builder(server.base_url())
12868 .timeout(Duration::from_secs(2))
12869 .build()
12870 .expect("client");
12871
12872 let heartbeat = client
12873 .heartbeat_activity_task(
12874 "activity-cancel",
12875 "attempt-cancel",
12876 "rust-worker",
12877 typed_fidelity_probe(),
12878 )
12879 .await
12880 .expect("cancellation heartbeat");
12881 assert!(heartbeat.cancel_requested);
12882 assert!(heartbeat.should_stop());
12883 assert_eq!(heartbeat.reason.as_deref(), Some("run_cancelled"));
12884 assert_eq!(heartbeat.run_closed_reason.as_deref(), Some("cancelled"));
12885 let heartbeat_body =
12886 server.request_body("/api/worker/activity-tasks/activity-cancel/heartbeat");
12887 assert_eq!(heartbeat_body["details"]["codec"], DEFAULT_CODEC);
12888 assert_eq!(
12889 decode_wire_avro_value(&heartbeat_body["details"], DEFAULT_CODEC)
12890 .expect("typed heartbeat details"),
12891 typed_fidelity_probe()
12892 );
12893
12894 let error = client
12895 .complete_activity_task(
12896 "activity-cancel",
12897 "attempt-cancel",
12898 "rust-worker",
12899 json!({"late":true}),
12900 DEFAULT_CODEC,
12901 )
12902 .await
12903 .expect_err("late completion must be refused");
12904 assert!(activity_task_rejection_is_final(&error));
12905 let Error::ActivityTaskRejected(rejection) = error else {
12906 panic!("expected typed activity rejection");
12907 };
12908 assert_eq!(rejection.status, 409);
12909 assert_eq!(rejection.reason, "run_cancelled");
12910 assert!(rejection.cancel_requested);
12911 assert_eq!(rejection.can_continue, Some(false));
12912 }
12913
12914 #[tokio::test]
12915 async fn managed_worker_survives_late_completion_and_restart_during_cancellation() {
12916 let server = MockWorkerServer::cancelled_activity();
12917 let client = Client::builder(server.base_url())
12918 .timeout(Duration::from_secs(2))
12919 .build()
12920 .expect("client");
12921 let cancellation_observed = Arc::new(AtomicBool::new(false));
12922 let observed = Arc::clone(&cancellation_observed);
12923 let mut worker = Worker::new(client.clone(), "rust-workers")
12924 .worker_id("rust-cancel-worker")
12925 .poll_timeout(Duration::from_millis(10));
12926 worker.register_activity("cancel-aware", move |ctx, _args| {
12927 let observed = Arc::clone(&observed);
12928 async move {
12929 let heartbeat = ctx.heartbeat(json!({"stage":"running"})).await?;
12930 observed.store(heartbeat.should_stop(), Ordering::SeqCst);
12931 Ok(json!({"late":"completion"}))
12932 }
12933 });
12934
12935 assert_eq!(
12936 worker.run_once().await.expect("cancelled attempt handled"),
12937 1
12938 );
12939 assert!(cancellation_observed.load(Ordering::SeqCst));
12940 assert_eq!(
12941 server.request_count("/api/worker/activity-tasks/activity-cancel/complete"),
12942 1
12943 );
12944
12945 let mut restarted = Worker::new(client, "rust-workers")
12946 .worker_id("rust-cancel-worker-restarted")
12947 .poll_timeout(Duration::from_millis(10));
12948 restarted.register_activity("cancel-aware", |_ctx, _args| async move { Ok(Value::Null) });
12949 assert_eq!(
12950 restarted
12951 .run_once()
12952 .await
12953 .expect("replacement worker continues polling"),
12954 0
12955 );
12956 }
12957
12958 #[tokio::test]
12959 async fn managed_worker_absorbs_selected_run_terminal_timeout_completion_race() {
12960 let response = r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"outcome":"completed","recorded":false,"run_id":"run-selected-timeout","run_status":"failed","created_task_ids":[],"reason":"run_timed_out"}"#;
12961 let server = MockWorkerServer::workflow_completion("409 Conflict", response);
12962 let client = Client::builder(server.base_url())
12963 .timeout(Duration::from_secs(2))
12964 .build()
12965 .expect("client");
12966
12967 let direct_error = client
12968 .complete_workflow_task(
12969 "workflow-timeout-task",
12970 "timeout-worker",
12971 3,
12972 vec![json!({
12973 "type": "complete_workflow",
12974 "result": fixture_envelope(Value::Null)
12975 })],
12976 )
12977 .await
12978 .expect_err("the low-level client preserves the completion rejection");
12979 let Error::Http { status, body } = direct_error else {
12980 panic!("expected the original HTTP completion rejection");
12981 };
12982 assert_eq!(status, reqwest::StatusCode::CONFLICT);
12983 assert_eq!(
12984 serde_json::from_str::<Value>(&body).expect("response body")["reason"],
12985 "run_timed_out"
12986 );
12987
12988 let mut worker = Worker::new(client, "rust-workers")
12989 .worker_id("timeout-worker")
12990 .poll_timeout(Duration::from_millis(10));
12991 worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
12992 Ok(json!({"late": "result"}))
12993 });
12994
12995 assert_eq!(
12996 worker
12997 .run_once()
12998 .await
12999 .expect("authoritative selected-run timeout settles the tick"),
13000 1
13001 );
13002 assert_eq!(
13003 server.request_count("/api/worker/workflow-tasks/workflow-timeout-task/complete"),
13004 2,
13005 "both the direct client proof and managed worker must see the rejection"
13006 );
13007 }
13008
13009 #[tokio::test]
13010 async fn managed_worker_does_not_swallow_nearby_completion_errors() {
13011 for (name, status, response) in [
13012 ("bare conflict", "409 Conflict", r#"{"message":"conflict"}"#),
13013 (
13014 "command was recorded",
13015 "409 Conflict",
13016 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":true,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
13017 ),
13018 (
13019 "lease conflict",
13020 "409 Conflict",
13021 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"lease_expired"}"#,
13022 ),
13023 (
13024 "nonterminal run",
13025 "409 Conflict",
13026 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"waiting","reason":"run_timed_out"}"#,
13027 ),
13028 (
13029 "different selected run",
13030 "409 Conflict",
13031 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-reused-workflow-current","run_status":"failed","reason":"run_timed_out"}"#,
13032 ),
13033 (
13034 "different task attempt",
13035 "409 Conflict",
13036 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":4,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
13037 ),
13038 (
13039 "authentication failure",
13040 "401 Unauthorized",
13041 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
13042 ),
13043 (
13044 "authorization failure",
13045 "403 Forbidden",
13046 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
13047 ),
13048 (
13049 "protocol failure",
13050 "400 Bad Request",
13051 r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.2","requested_version":"1.3"}"#,
13052 ),
13053 (
13054 "malformed command",
13055 "422 Unprocessable Entity",
13056 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
13057 ),
13058 (
13059 "transient server failure",
13060 "503 Service Unavailable",
13061 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
13062 ),
13063 ] {
13064 let server = MockWorkerServer::workflow_completion(status, response);
13065 let client = Client::builder(server.base_url())
13066 .timeout(Duration::from_secs(2))
13067 .build()
13068 .expect("client");
13069 let mut worker = Worker::new(client, "rust-workers")
13070 .worker_id("timeout-worker")
13071 .poll_timeout(Duration::from_millis(10));
13072 worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
13073 Ok(json!({"late": "result"}))
13074 });
13075
13076 let error = worker
13077 .run_once()
13078 .await
13079 .expect_err(&format!("{name} must remain an error"));
13080 assert!(
13081 matches!(error, Error::Http { .. } | Error::Protocol(_)),
13082 "{name} returned an unexpected error variant: {error}"
13083 );
13084 }
13085 }
13086
13087 #[tokio::test]
13088 async fn worker_deregistration_uses_worker_plane_method_path_headers_and_result() {
13089 let server = MockWorkerServer::start();
13090 let client = Client::builder(server.base_url())
13091 .worker_token(Some("worker-secret".to_string()))
13092 .namespace("orders")
13093 .timeout(Duration::from_secs(2))
13094 .build()
13095 .expect("client");
13096 let path = "/api/worker/registrations/worker%2F%CE%B1%20space";
13097
13098 let result = client
13099 .deregister_worker_registration("worker/α space")
13100 .await
13101 .expect("deregister worker registration");
13102
13103 assert_eq!(server.method_for(path).as_deref(), Some("DELETE"));
13104 assert_eq!(
13105 server.worker_protocol_for(path).as_deref(),
13106 Some(WORKER_PROTOCOL_VERSION)
13107 );
13108 assert_eq!(server.control_protocol_for(path), None);
13109 assert_eq!(server.namespace_for(path).as_deref(), Some("orders"));
13110 assert_eq!(
13111 server.authorization_for(path).as_deref(),
13112 Some("Bearer worker-secret")
13113 );
13114 assert_eq!(
13115 result,
13116 WorkerDeregistrationEnvelope {
13117 worker_id: "deregistered-worker".to_string(),
13118 outcome: "deregistered".to_string(),
13119 recovered_workflow_task_count: 2,
13120 }
13121 );
13122 }
13123
13124 #[tokio::test]
13125 async fn low_level_registration_rejects_update_validators_before_transport() {
13126 let server = MockWorkerServer::start();
13127 let client = Client::builder(server.base_url())
13128 .timeout(Duration::from_secs(2))
13129 .build()
13130 .expect("client");
13131
13132 for update_validators in [json!(["approve"]), json!("approve")] {
13133 let error = client
13134 .register_worker_with_command_contracts(
13135 "validator-claiming-worker",
13136 "rust-workers",
13137 vec!["orders".to_string()],
13138 vec![],
13139 1,
13140 1,
13141 vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
13142 json!({
13143 "orders": {
13144 "queries": ["current"],
13145 "updates": ["approve"],
13146 "update_validators": update_validators,
13147 },
13148 }),
13149 )
13150 .await
13151 .expect_err("unsupported validator claims must fail before registration");
13152
13153 let Error::UnsupportedUpdateValidators { workflow_type } = error else {
13154 panic!("expected typed unsupported-validator failure");
13155 };
13156 assert_eq!(workflow_type, "orders");
13157 }
13158 assert_eq!(server.request_count("/api/worker/register"), 0);
13159 }
13160
13161 #[tokio::test]
13162 async fn low_level_registration_preserves_query_and_update_contracts() {
13163 let server = MockWorkerServer::start();
13164 let client = Client::builder(server.base_url())
13165 .timeout(Duration::from_secs(2))
13166 .build()
13167 .expect("client");
13168 let contracts = json!({
13169 "orders": {
13170 "queries": ["current"],
13171 "updates": ["approve"],
13172 "update_validators": [],
13173 },
13174 "payments": {
13175 "queries": ["status"],
13176 "updates": ["capture"],
13177 },
13178 });
13179
13180 client
13181 .register_worker_with_command_contracts(
13182 "command-worker",
13183 "rust-workers",
13184 vec!["orders".to_string(), "payments".to_string()],
13185 vec![],
13186 1,
13187 1,
13188 vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
13189 contracts.clone(),
13190 )
13191 .await
13192 .expect("query and update contracts must remain supported");
13193
13194 assert_eq!(
13195 server.request_body("/api/worker/register")["workflow_command_contracts"],
13196 contracts
13197 );
13198 }
13199
13200 #[tokio::test]
13201 async fn role_scoped_tokens_are_never_used_for_the_opposite_plane() {
13202 let server = MockWorkerServer::start();
13203 let control_only = Client::builder(server.base_url())
13204 .control_token(Some("control-secret".to_string()))
13205 .build()
13206 .expect("control client");
13207
13208 let error = control_only
13209 .register_worker("worker", "queue", vec![], vec![], 1, 1)
13210 .await
13211 .expect_err("control token must not authorize a worker request");
13212 assert!(matches!(
13213 error,
13214 Error::MissingRoleCredentials { role: "worker", .. }
13215 ));
13216 assert_eq!(server.request_count("/api/worker/register"), 0);
13217
13218 let worker_only = Client::builder(server.base_url())
13219 .worker_token(Some("worker-secret".to_string()))
13220 .build()
13221 .expect("worker client");
13222 let error = worker_only
13223 .health()
13224 .await
13225 .expect_err("worker token must not authorize a control request");
13226 assert!(matches!(
13227 error,
13228 Error::MissingRoleCredentials {
13229 role: "control",
13230 ..
13231 }
13232 ));
13233 assert_eq!(server.request_count("/api/health"), 0);
13234 }
13235
13236 #[tokio::test]
13237 async fn shared_token_supports_worker_and_control_planes() {
13238 let server = MockWorkerServer::start();
13239 let client = Client::builder(server.base_url())
13240 .token(Some("shared-secret".to_string()))
13241 .build()
13242 .expect("client");
13243
13244 client.health().await.expect("control request");
13245 client
13246 .register_worker("worker", "queue", vec![], vec![], 1, 1)
13247 .await
13248 .expect("worker request");
13249
13250 assert_eq!(
13251 server.authorization_for("/api/health").as_deref(),
13252 Some("Bearer shared-secret")
13253 );
13254 assert_eq!(
13255 server.control_protocol_for("/api/health").as_deref(),
13256 Some(CONTROL_PLANE_VERSION)
13257 );
13258 assert_eq!(
13259 server.authorization_for("/api/worker/register").as_deref(),
13260 Some("Bearer shared-secret")
13261 );
13262 assert_eq!(
13263 server
13264 .worker_protocol_for("/api/worker/register")
13265 .as_deref(),
13266 Some(WORKER_PROTOCOL_VERSION)
13267 );
13268 }
13269
13270 #[tokio::test]
13271 async fn baseline_worker_endpoints_send_the_baseline_protocol() {
13272 let server = MockWorkerServer::start();
13273 let client = Client::builder(server.base_url())
13274 .timeout(Duration::from_secs(2))
13275 .build()
13276 .expect("client");
13277
13278 client
13279 .register_worker("capture-worker", "capture", vec![], vec![], 1, 1)
13280 .await
13281 .expect("register");
13282 client
13283 .heartbeat_worker("capture-worker", 1, 1)
13284 .await
13285 .expect("heartbeat");
13286 client
13287 .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
13288 .await
13289 .expect("workflow poll");
13290 client
13291 .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
13292 .await
13293 .expect("activity poll");
13294
13295 for path in [
13296 "/api/worker/register",
13297 "/api/worker/heartbeat",
13298 "/api/worker/workflow-tasks/poll",
13299 "/api/worker/activity-tasks/poll",
13300 ] {
13301 assert_eq!(
13302 server.worker_protocol_for(path).as_deref(),
13303 Some(WORKER_PROTOCOL_VERSION),
13304 "unexpected protocol for {path}"
13305 );
13306 }
13307
13308 assert_eq!(
13309 server.request_body("/api/worker/workflow-tasks/poll")["timeout_seconds"],
13310 1
13311 );
13312 assert_eq!(
13313 server.request_body("/api/worker/activity-tasks/poll")["timeout_seconds"],
13314 1
13315 );
13316 assert!(
13317 server.request_body("/api/worker/workflow-tasks/poll")["poll_request_id"]
13318 .as_str()
13319 .is_some_and(|id| id.starts_with("rust-workflow-poll-"))
13320 );
13321 assert!(
13322 server.request_body("/api/worker/activity-tasks/poll")["poll_request_id"]
13323 .as_str()
13324 .is_some_and(|id| id.starts_with("rust-activity-poll-"))
13325 );
13326 }
13327
13328 #[tokio::test]
13329 async fn query_task_endpoints_send_the_query_feature_protocol() {
13330 let server = MockWorkerServer::start();
13331 let client = Client::builder(server.base_url())
13332 .timeout(Duration::from_secs(2))
13333 .build()
13334 .expect("client");
13335
13336 client
13337 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
13338 .await
13339 .expect("query poll");
13340 client
13341 .complete_query_task(
13342 "query-capture",
13343 "capture-worker",
13344 1,
13345 json!(8),
13346 DEFAULT_CODEC,
13347 )
13348 .await
13349 .expect("query complete");
13350 client
13351 .fail_query_task(
13352 "query-capture",
13353 "capture-worker",
13354 1,
13355 "failed",
13356 "query_rejected",
13357 "QueryFailed",
13358 )
13359 .await
13360 .expect("query fail");
13361
13362 for path in [
13363 "/api/worker/query-tasks/poll",
13364 "/api/worker/query-tasks/query-capture/complete",
13365 "/api/worker/query-tasks/query-capture/fail",
13366 ] {
13367 assert_eq!(
13368 server.worker_protocol_for(path).as_deref(),
13369 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
13370 "unexpected protocol for {path}"
13371 );
13372 }
13373
13374 assert_eq!(
13375 server.request_body("/api/worker/query-tasks/poll")["timeout_seconds"],
13376 1
13377 );
13378 assert!(
13379 server.request_body("/api/worker/query-tasks/poll")["poll_request_id"]
13380 .as_str()
13381 .is_some_and(|id| id.starts_with("rust-query-poll-"))
13382 );
13383 }
13384
13385 #[tokio::test]
13386 async fn disconnected_client_polls_retry_once_with_the_same_request_id() {
13387 let server = MockWorkerServer::transient_worker_failures();
13388 let client = Client::builder(server.base_url())
13389 .timeout(Duration::from_secs(2))
13390 .build()
13391 .expect("client");
13392
13393 client
13394 .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
13395 .await
13396 .expect("workflow poll retry");
13397 client
13398 .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
13399 .await
13400 .expect("activity poll retry");
13401 client
13402 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
13403 .await
13404 .expect("query poll retry");
13405
13406 for path in [
13407 "/api/worker/workflow-tasks/poll",
13408 "/api/worker/activity-tasks/poll",
13409 "/api/worker/query-tasks/poll",
13410 ] {
13411 let bodies = server.request_bodies(path);
13412 assert_eq!(bodies.len(), 2, "{path} must be retried once");
13413 assert_eq!(
13414 bodies[0]["poll_request_id"], bodies[1]["poll_request_id"],
13415 "{path} must preserve the request binding across retry"
13416 );
13417 }
13418 }
13419
13420 #[tokio::test]
13421 async fn worker_poll_retries_preserve_request_id_across_consecutive_failures() {
13422 let server = MockWorkerServer::consecutive_poll_failures(2);
13423 let client = Client::builder(server.base_url())
13424 .timeout(Duration::from_secs(2))
13425 .build()
13426 .expect("client");
13427 let mut worker = Worker::new(client, "capture")
13428 .worker_id("capture-worker")
13429 .poll_timeout(Duration::from_millis(10))
13430 .retry_policy(WorkerRetryPolicy {
13431 max_retries: 2,
13432 initial_backoff: Duration::from_millis(1),
13433 max_backoff: Duration::from_millis(1),
13434 });
13435 worker.register_workflow(
13436 "capture.workflow",
13437 |_ctx, _input| async move { Ok(Value::Null) },
13438 );
13439 worker.register_activity(
13440 "capture.activity",
13441 |_ctx, _input| async move { Ok(Value::Null) },
13442 );
13443 worker.register_query("capture.workflow", "current", |_ctx, _args| async move {
13444 Ok(Value::Null)
13445 });
13446
13447 assert_eq!(worker.run_once().await.expect("poll retries"), 0);
13448
13449 for path in [
13450 "/api/worker/workflow-tasks/poll",
13451 "/api/worker/activity-tasks/poll",
13452 "/api/worker/query-tasks/poll",
13453 ] {
13454 let bodies = server.request_bodies(path);
13455 assert_eq!(bodies.len(), 3, "{path} must use exactly two retries");
13456 assert!(
13457 bodies
13458 .iter()
13459 .all(|body| body["poll_request_id"] == bodies[0]["poll_request_id"]),
13460 "{path} must preserve one request binding across every retry"
13461 );
13462 }
13463 }
13464
13465 #[tokio::test]
13466 async fn query_protocol_rejection_from_older_server_is_typed() {
13467 let server = MockWorkerServer::reject_query_protocol();
13468 let client = Client::builder(server.base_url())
13469 .timeout(Duration::from_secs(2))
13470 .build()
13471 .expect("client");
13472
13473 let error = client
13474 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
13475 .await
13476 .expect_err("server below query protocol floor must reject");
13477 let Error::Protocol(failure) = error else {
13478 panic!("expected typed protocol failure");
13479 };
13480
13481 assert_eq!(failure.status, 400);
13482 assert_eq!(failure.reason, "unsupported_protocol_version");
13483 assert_eq!(failure.supported_version.as_deref(), Some("1.7"));
13484 assert_eq!(
13485 failure.requested_version.as_deref(),
13486 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
13487 );
13488 assert_eq!(
13489 server
13490 .worker_protocol_for("/api/worker/query-tasks/poll")
13491 .as_deref(),
13492 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
13493 );
13494 }
13495
13496 #[tokio::test]
13497 async fn run_once_without_query_handlers_keeps_pre_query_server_compatibility() {
13498 let server = MockWorkerServer::reject_query_protocol();
13499 let client = Client::builder(server.base_url())
13500 .timeout(Duration::from_secs(2))
13501 .build()
13502 .expect("client");
13503 let mut worker = Worker::new(client, "rust-workers")
13504 .worker_id("baseline-worker")
13505 .poll_timeout(Duration::from_millis(10));
13506
13507 worker.register_workflow("baseline.workflow", |_ctx, _input| async move {
13508 Ok(Value::Null)
13509 });
13510
13511 assert_eq!(worker.run_once().await.expect("baseline run once"), 0);
13512 assert_eq!(
13513 server
13514 .worker_protocol_for("/api/worker/workflow-tasks/poll")
13515 .as_deref(),
13516 Some(WORKER_PROTOCOL_VERSION)
13517 );
13518 assert_eq!(
13519 server.worker_protocol_for("/api/worker/query-tasks/poll"),
13520 None,
13521 "a worker without query handlers must not use the query-task endpoint"
13522 );
13523 }
13524
13525 #[tokio::test]
13526 async fn completion_time_query_rejection_is_typed_without_stopping_worker() {
13527 let server = MockWorkerServer::reject_query_completion();
13528 let client = Client::builder(server.base_url())
13529 .timeout(Duration::from_secs(2))
13530 .build()
13531 .expect("client");
13532
13533 let error = client
13534 .complete_query_task("query-late", "late-worker", 1, json!(8), DEFAULT_CODEC)
13535 .await
13536 .expect_err("expired completion must be rejected");
13537 let Error::QueryFailed(failure) = error else {
13538 panic!("expected typed query failure");
13539 };
13540 assert_eq!(failure.status, 409);
13541 assert_eq!(failure.reason, "query_task_timed_out");
13542
13543 let mut worker = Worker::new(client, "rust-workers")
13544 .worker_id("late-worker")
13545 .poll_timeout(Duration::from_millis(10));
13546 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
13547 worker.register_query(
13548 "counter",
13549 "current",
13550 |_ctx, _args| async move { Ok(json!(8)) },
13551 );
13552
13553 assert_eq!(worker.run_once().await.expect("late task is handled"), 1);
13554 assert_eq!(
13555 worker
13556 .run_once()
13557 .await
13558 .expect("worker continues after late completion"),
13559 0
13560 );
13561 assert_eq!(
13562 server.request_count("/api/worker/query-tasks/query-late/complete"),
13563 2
13564 );
13565 assert_eq!(
13566 server.request_count("/api/worker/query-tasks/query-late/fail"),
13567 0,
13568 "a server completion rejection must not be reported as an encoding failure"
13569 );
13570 }
13571
13572 #[tokio::test]
13573 async fn normal_shutdown_joins_pollers_and_deregisters_once() {
13574 let server = MockWorkerServer::start();
13575 let client = Client::builder(server.base_url())
13576 .timeout(Duration::from_secs(2))
13577 .build()
13578 .expect("client");
13579 let mut worker = Worker::new(client, "rust-workers")
13580 .worker_id("joined-worker")
13581 .poll_timeout(Duration::from_millis(10));
13582 worker.register_workflow(
13583 "joined.workflow",
13584 |_ctx, _input| async move { Ok(Value::Null) },
13585 );
13586 worker.register_activity(
13587 "joined.activity",
13588 |_ctx, _input| async move { Ok(Value::Null) },
13589 );
13590 worker.register_query("joined.workflow", "state", |_ctx, _input| async move {
13591 Ok(Value::Null)
13592 });
13593
13594 worker
13595 .run_until(tokio::time::sleep(Duration::from_millis(20)))
13596 .await
13597 .expect("normal shutdown");
13598
13599 let deregistration_path = "/api/worker/registrations/mock-worker";
13600 assert_eq!(server.request_count(deregistration_path), 1);
13601 for poll_path in [
13602 "/api/worker/workflow-tasks/poll",
13603 "/api/worker/activity-tasks/poll",
13604 "/api/worker/query-tasks/poll",
13605 ] {
13606 assert!(server.request_count(poll_path) > 0, "missing {poll_path}");
13607 }
13608 assert_eq!(
13609 server.captured_paths().last().map(String::as_str),
13610 Some(deregistration_path),
13611 "deregistration must start only after every poller has joined"
13612 );
13613 }
13614
13615 #[tokio::test]
13616 async fn registration_failure_does_not_deregister() {
13617 let server = MockWorkerServer::rejected_registration();
13618 let client = Client::builder(server.base_url())
13619 .timeout(Duration::from_secs(2))
13620 .build()
13621 .expect("client");
13622 let worker = Worker::new(client, "rust-workers").worker_id("never-registered");
13623
13624 let error = worker
13625 .run_until(async {})
13626 .await
13627 .expect_err("registration must fail");
13628 assert!(matches!(
13629 error,
13630 Error::Http {
13631 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
13632 ..
13633 }
13634 ));
13635 assert!(server
13636 .captured_paths()
13637 .iter()
13638 .all(|path| !path.starts_with("/api/worker/registrations/")));
13639 }
13640
13641 #[tokio::test]
13642 async fn declined_registration_does_not_deregister() {
13643 let server = MockWorkerServer::declined_registration();
13644 let client = Client::builder(server.base_url())
13645 .timeout(Duration::from_secs(2))
13646 .build()
13647 .expect("client");
13648 let worker = Worker::new(client, "rust-workers").worker_id("declined-worker");
13649
13650 let error = worker
13651 .run_until(async {})
13652 .await
13653 .expect_err("declined registration must fail");
13654 assert!(matches!(error, Error::WorkerLoop(_)));
13655 assert!(error.to_string().contains("was not accepted"));
13656 assert!(server
13657 .captured_paths()
13658 .iter()
13659 .all(|path| !path.starts_with("/api/worker/registrations/")));
13660 }
13661
13662 #[tokio::test]
13663 async fn deregistration_http_failure_is_returned_after_normal_shutdown() {
13664 let server = MockWorkerServer::rejected_deregistration();
13665 let client = Client::builder(server.base_url())
13666 .timeout(Duration::from_secs(2))
13667 .build()
13668 .expect("client");
13669 let worker = Worker::new(client, "rust-workers").worker_id("forbidden-cleanup");
13670
13671 let error = worker
13672 .run_until(async {})
13673 .await
13674 .expect_err("deregistration must fail");
13675 assert!(matches!(
13676 error,
13677 Error::Http {
13678 status: reqwest::StatusCode::FORBIDDEN,
13679 ..
13680 }
13681 ));
13682 assert_eq!(
13683 server.request_count("/api/worker/registrations/mock-worker"),
13684 1
13685 );
13686 }
13687
13688 #[tokio::test]
13689 async fn deregistration_protocol_failure_is_returned_after_normal_shutdown() {
13690 let server = MockWorkerServer::rejected_deregistration_protocol();
13691 let client = Client::builder(server.base_url())
13692 .timeout(Duration::from_secs(2))
13693 .build()
13694 .expect("client");
13695 let worker = Worker::new(client, "rust-workers").worker_id("protocol-cleanup");
13696
13697 let error = worker
13698 .run_until(async {})
13699 .await
13700 .expect_err("protocol rejection must fail shutdown");
13701 let Error::Protocol(failure) = error else {
13702 panic!("expected typed protocol failure");
13703 };
13704 assert_eq!(failure.reason, "unsupported_protocol_version");
13705 assert_eq!(failure.requested_version.as_deref(), Some("1.2"));
13706 assert_eq!(
13707 server.request_count("/api/worker/registrations/mock-worker"),
13708 1
13709 );
13710 }
13711
13712 #[tokio::test]
13713 async fn primary_poller_error_retains_deregistration_failure_context() {
13714 let server = MockWorkerServer::unauthorized_polls_and_rejected_deregistration();
13715 let client = Client::builder(server.base_url())
13716 .timeout(Duration::from_secs(2))
13717 .build()
13718 .expect("client");
13719 let mut worker = Worker::new(client, "rust-workers")
13720 .worker_id("combined-failure")
13721 .poll_timeout(Duration::from_millis(10));
13722 worker.register_workflow("combined.workflow", |_ctx, _input| async move {
13723 Ok(Value::Null)
13724 });
13725
13726 let error = worker
13727 .run()
13728 .await
13729 .expect_err("worker and cleanup must fail");
13730 let summary = error.to_string();
13731 assert!(summary.contains("authentication_failed"));
13732 assert!(summary.contains("worker cannot deregister"));
13733 let Error::WorkerShutdown {
13734 primary,
13735 deregistration,
13736 } = error
13737 else {
13738 panic!("expected combined worker shutdown error");
13739 };
13740 assert!(matches!(
13741 *primary,
13742 Error::Http {
13743 status: reqwest::StatusCode::UNAUTHORIZED,
13744 ..
13745 }
13746 ));
13747 assert!(matches!(
13748 *deregistration,
13749 Error::Http {
13750 status: reqwest::StatusCode::FORBIDDEN,
13751 ..
13752 }
13753 ));
13754 assert_eq!(
13755 server.request_count("/api/worker/registrations/mock-worker"),
13756 1
13757 );
13758 }
13759
13760 #[tokio::test]
13761 async fn activity_only_worker_can_shutdown_without_workflow_poller() {
13762 let server = MockWorkerServer::start();
13763 let client = Client::builder(server.base_url())
13764 .timeout(Duration::from_secs(2))
13765 .build()
13766 .expect("client");
13767 let mut worker = Worker::new(client, "rust-workers")
13768 .worker_id("activity-only-worker")
13769 .poll_timeout(Duration::from_millis(10));
13770
13771 worker.register_activity(
13772 "activity.only",
13773 |_ctx, _args| async move { Ok(Value::Null) },
13774 );
13775
13776 worker.run_until(async {}).await.expect("run worker");
13777 }
13778
13779 #[tokio::test]
13780 async fn workflow_only_worker_can_shutdown_without_activity_poller() {
13781 let server = MockWorkerServer::start();
13782 let client = Client::builder(server.base_url())
13783 .timeout(Duration::from_secs(2))
13784 .build()
13785 .expect("client");
13786 let mut worker = Worker::new(client, "rust-workers")
13787 .worker_id("workflow-only-worker")
13788 .poll_timeout(Duration::from_millis(10));
13789
13790 worker.register_workflow(
13791 "workflow.only",
13792 |_ctx, _input| async move { Ok(Value::Null) },
13793 );
13794
13795 worker.run_until(async {}).await.expect("run worker");
13796 }
13797
13798 #[tokio::test]
13799 async fn worker_heartbeat_observer_receives_server_acknowledgements() {
13800 let server = MockWorkerServer::start();
13801 let client = Client::builder(server.base_url())
13802 .timeout(Duration::from_secs(2))
13803 .build()
13804 .expect("client");
13805 let observations = Arc::new(Mutex::new(Vec::new()));
13806 let observed = Arc::clone(&observations);
13807 let mut worker = Worker::new(client, "rust-workers")
13808 .worker_id("observed-heartbeat-worker")
13809 .poll_timeout(Duration::from_millis(10))
13810 .on_worker_heartbeat(move |observation| {
13811 observed
13812 .lock()
13813 .expect("heartbeat observations")
13814 .push(observation.clone());
13815 });
13816
13817 worker.register_workflow("workflow.observed", |_ctx, _input| async move {
13818 Ok(Value::Null)
13819 });
13820 let acknowledged = Arc::clone(&observations);
13821 worker
13822 .run_until(async move {
13823 tokio::time::timeout(Duration::from_secs(2), async move {
13824 loop {
13825 if !acknowledged
13826 .lock()
13827 .expect("heartbeat observations")
13828 .is_empty()
13829 {
13830 break;
13831 }
13832 tokio::time::sleep(Duration::from_millis(1)).await;
13833 }
13834 })
13835 .await
13836 .expect("heartbeat acknowledgement within timeout");
13837 })
13838 .await
13839 .expect("run worker");
13840
13841 let observations = observations.lock().expect("heartbeat observations");
13842 let first = observations.first().expect("heartbeat acknowledgement");
13843 assert_eq!(first.worker_id, "observed-heartbeat-worker");
13844 assert_eq!(first.task_queue, "rust-workers");
13845 assert!(first.acknowledged_at_unix_millis > 0);
13846 assert_eq!(first.acknowledgement, json!({}));
13847 }
13848
13849 #[tokio::test]
13850 async fn delayed_worker_heartbeat_keeps_cadence_and_pollers_live() {
13851 let server = MockWorkerServer::delayed_heartbeat_worker();
13852 let client = Client::builder(server.base_url())
13853 .timeout(Duration::from_secs(3))
13854 .build()
13855 .expect("client");
13856 let observations = Arc::new(Mutex::new(Vec::new()));
13857 let observed = Arc::clone(&observations);
13858 let mut worker = Worker::new(client, "rust-snapshot-workers")
13859 .worker_id("rust-snapshot-worker")
13860 .poll_timeout(Duration::from_millis(10))
13861 .on_worker_heartbeat(move |observation| {
13862 observed
13863 .lock()
13864 .expect("heartbeat observations")
13865 .push(observation.clone());
13866 });
13867
13868 worker.register_workflow("snapshot", |ctx, _input| async move {
13869 ctx.wait_signal("finish").await?;
13870 Ok(json!({"status": "finished"}))
13871 });
13872 worker.register_query("snapshot", "current", |ctx, _args| async move {
13873 Ok(json!(ctx
13874 .signals("increment")
13875 .iter()
13876 .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
13877 .sum::<i64>()))
13878 });
13879 worker.register_activity("cancel-aware", |_ctx, _args| async move {
13880 Ok(json!({"late": "completion"}))
13881 });
13882
13883 worker
13884 .run_until(tokio::time::sleep(Duration::from_millis(3_800)))
13885 .await
13886 .expect("delayed heartbeat must allow a clean worker shutdown");
13887
13888 let observations = observations.lock().expect("heartbeat observations");
13889 assert!(
13890 observations.len() >= 3,
13891 "the immediate heartbeat, delayed acknowledgement, and next cadence heartbeat must complete"
13892 );
13893 assert!(
13894 observations.windows(2).all(|pair| {
13895 pair[1].acknowledged_at_unix_millis
13896 .saturating_sub(pair[0].acknowledged_at_unix_millis)
13897 >= 850
13898 }),
13899 "successful acknowledgements must not catch up faster than the advertised one-second cadence: {observations:?}"
13900 );
13901 drop(observations);
13902
13903 let heartbeat_times = server.request_times("/api/worker/heartbeat");
13904 let delayed_request_at = *heartbeat_times
13905 .get(1)
13906 .expect("intentionally delayed heartbeat request");
13907 let delay_window_start = delayed_request_at + Duration::from_millis(100);
13908 let delay_window_end = delayed_request_at + Duration::from_millis(1_400);
13909 for path in [
13910 "/api/worker/workflow-tasks/poll",
13911 "/api/worker/activity-tasks/poll",
13912 "/api/worker/query-tasks/poll",
13913 ] {
13914 assert!(
13915 server
13916 .request_times(path)
13917 .iter()
13918 .any(|received_at| *received_at >= delay_window_start
13919 && *received_at <= delay_window_end),
13920 "{path} must keep polling while a heartbeat acknowledgement is delayed"
13921 );
13922 }
13923 assert!(
13924 server.request_count("/api/worker/workflow-tasks/snapshot-wait-3/fail") >= 1,
13925 "workflow work must be settled"
13926 );
13927 assert!(
13928 server.request_count("/api/worker/activity-tasks/activity-cancel/complete") >= 1,
13929 "activity work must be settled"
13930 );
13931 assert!(
13932 server.request_count("/api/worker/query-tasks/snapshot-current/complete") >= 1,
13933 "query work must be settled"
13934 );
13935 }
13936
13937 #[tokio::test]
13938 async fn retried_worker_heartbeat_restarts_the_advertised_cadence() {
13939 let server = MockWorkerServer::heartbeat_retry_worker();
13940 let client = Client::builder(server.base_url())
13941 .timeout(Duration::from_secs(2))
13942 .build()
13943 .expect("client");
13944 let observations = Arc::new(Mutex::new(Vec::new()));
13945 let observed = Arc::clone(&observations);
13946 let worker = Worker::new(client, "rust-workers")
13947 .worker_id("heartbeat-retry-worker")
13948 .retry_policy(WorkerRetryPolicy {
13949 max_retries: 1,
13950 initial_backoff: Duration::from_millis(300),
13951 max_backoff: Duration::from_millis(300),
13952 })
13953 .on_worker_heartbeat(move |observation| {
13954 observed
13955 .lock()
13956 .expect("heartbeat observations")
13957 .push(observation.clone());
13958 });
13959
13960 worker
13961 .run_until(tokio::time::sleep(Duration::from_millis(2_700)))
13962 .await
13963 .expect("retryable heartbeat failure must remain bounded and recover");
13964
13965 let observations = observations.lock().expect("heartbeat observations");
13966 assert!(observations.len() >= 3, "heartbeat retry must recover");
13967 assert!(
13968 observations.windows(2).all(|pair| {
13969 pair[1]
13970 .acknowledged_at_unix_millis
13971 .saturating_sub(pair[0].acknowledged_at_unix_millis)
13972 >= 850
13973 }),
13974 "a successful retry must start a fresh advertised cadence: {observations:?}"
13975 );
13976 assert_eq!(
13977 server.request_count("/api/worker/heartbeat"),
13978 observations.len() + 1,
13979 "one retryable failure must add exactly one bounded request"
13980 );
13981 }
13982
13983 #[tokio::test]
13984 async fn query_enabled_worker_ignores_unmatched_signals_then_completes_once() {
13985 let server = MockWorkerServer::waiting_query_worker();
13986 let client = Client::builder(server.base_url())
13987 .timeout(Duration::from_secs(2))
13988 .build()
13989 .expect("client");
13990 let observations = Arc::new(Mutex::new(Vec::new()));
13991 let observed = Arc::clone(&observations);
13992 let mut worker = Worker::new(client, "rust-snapshot-workers")
13993 .worker_id("rust-snapshot-worker")
13994 .poll_timeout(Duration::from_millis(10))
13995 .on_worker_heartbeat(move |observation| {
13996 observed
13997 .lock()
13998 .expect("heartbeat observations")
13999 .push(observation.clone());
14000 });
14001
14002 worker.register_workflow("snapshot", |ctx, _input| async move {
14003 ctx.wait_signal("finish").await?;
14004 Ok(json!({"status": "finished"}))
14005 });
14006 worker.register_query("snapshot", "current", |ctx, _args| async move {
14007 let current = ctx
14008 .signals("increment")
14009 .iter()
14010 .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
14011 .sum::<i64>();
14012 Ok(json!(current))
14013 });
14014 worker.register_update("snapshot", "replace", |_ctx, args| async move { Ok(args) });
14015
14016 worker
14017 .run_until(tokio::time::sleep(Duration::from_millis(3_200)))
14018 .await
14019 .expect("pending workflow and query poller must remain live until shutdown");
14020
14021 assert!(
14022 observations.lock().expect("heartbeat observations").len() >= 4,
14023 "the immediate heartbeat and at least three advertised one-second intervals must be acknowledged"
14024 );
14025 assert!(
14026 server.request_count("/api/worker/workflow-tasks/poll") >= 3,
14027 "workflow polling must continue after empty replay acknowledgements"
14028 );
14029 assert!(
14030 server.request_count("/api/worker/query-tasks/poll") >= 2,
14031 "query polling must continue after serving the current query"
14032 );
14033 assert_eq!(
14034 server.request_body("/api/worker/register")["capabilities"],
14035 json!([QUERY_TASKS_CAPABILITY, WORKFLOW_UPDATES_CAPABILITY])
14036 );
14037 assert_eq!(
14038 server.request_body("/api/worker/register")["workflow_command_contracts"]["snapshot"],
14039 json!({
14040 "queries": ["current"],
14041 "updates": ["replace"],
14042 "update_validators": [],
14043 })
14044 );
14045
14046 let opened = server.request_body("/api/worker/workflow-tasks/snapshot-open/complete");
14047 assert_eq!(
14048 opened["commands"],
14049 json!([{
14050 "type": "open_signal_wait",
14051 "signal_name": "finish",
14052 }])
14053 );
14054
14055 for task_id in ["snapshot-wait-3", "snapshot-wait-5"] {
14056 let fail_path = format!("/api/worker/workflow-tasks/{task_id}/fail");
14057 let completion_path = format!("/api/worker/workflow-tasks/{task_id}/complete");
14058 let failure = server.request_body(&fail_path);
14059 assert_eq!(
14060 failure["failure"]["type"],
14061 WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE
14062 );
14063 assert_eq!(server.request_count(&completion_path), 0);
14064 }
14065
14066 let query_completion =
14067 server.request_body("/api/worker/query-tasks/snapshot-current/complete");
14068 assert_eq!(query_completion["result"], json!(8));
14069
14070 let terminal_path = "/api/worker/workflow-tasks/snapshot-finish/complete";
14071 assert_eq!(
14072 server.request_count(terminal_path),
14073 1,
14074 "the matching signal must settle the workflow exactly once"
14075 );
14076 let terminal = server.request_body(terminal_path);
14077 assert_eq!(terminal["commands"].as_array().map(Vec::len), Some(1));
14078 assert_eq!(terminal["commands"][0]["type"], "complete_workflow");
14079 assert_eq!(
14080 decode_wire_value(&terminal["commands"][0]["result"], DEFAULT_CODEC)
14081 .expect("terminal workflow result"),
14082 json!({"status": "finished"})
14083 );
14084 }
14085
14086 #[tokio::test]
14087 async fn worker_retries_poll_and_heartbeat_transport_failures_independently() {
14088 let server = MockWorkerServer::transient_worker_failures();
14089 let client = Client::builder(server.base_url())
14090 .timeout(Duration::from_secs(2))
14091 .build()
14092 .expect("client");
14093 let mut worker = Worker::new(client, "rust-workers")
14094 .worker_id("retry-worker")
14095 .poll_timeout(Duration::from_millis(10))
14096 .retry_policy(WorkerRetryPolicy {
14097 max_retries: 2,
14098 initial_backoff: Duration::from_millis(1),
14099 max_backoff: Duration::from_millis(1),
14100 });
14101 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
14102 worker.register_activity(
14103 "counter.activity",
14104 |_ctx, _input| async move { Ok(Value::Null) },
14105 );
14106 worker.register_query(
14107 "counter",
14108 "current",
14109 |_ctx, _args| async move { Ok(json!(8)) },
14110 );
14111
14112 worker
14113 .run_until(tokio::time::sleep(Duration::from_millis(75)))
14114 .await
14115 .expect("transient failures must not stop the worker");
14116
14117 for path in [
14118 "/api/worker/heartbeat",
14119 "/api/worker/workflow-tasks/poll",
14120 "/api/worker/activity-tasks/poll",
14121 "/api/worker/query-tasks/poll",
14122 ] {
14123 assert!(
14124 server.request_count(path) >= 2,
14125 "{path} must continue after its transient failure"
14126 );
14127 }
14128 }
14129
14130 #[tokio::test]
14131 async fn worker_bounds_transport_retries() {
14132 let server = MockWorkerServer::unavailable_polls();
14133 let client = Client::builder(server.base_url())
14134 .timeout(Duration::from_secs(2))
14135 .build()
14136 .expect("client");
14137 let mut worker = Worker::new(client, "rust-workers")
14138 .worker_id("bounded-retry-worker")
14139 .poll_timeout(Duration::from_millis(10))
14140 .retry_policy(WorkerRetryPolicy {
14141 max_retries: 2,
14142 initial_backoff: Duration::from_millis(1),
14143 max_backoff: Duration::from_millis(1),
14144 });
14145 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
14146
14147 let error = worker.run().await.expect_err("retry bound must terminate");
14148 assert!(matches!(error, Error::Transport(_)));
14149 assert_eq!(
14150 server.request_count("/api/worker/workflow-tasks/poll"),
14151 3,
14152 "one initial request plus exactly two retries"
14153 );
14154 }
14155
14156 #[tokio::test]
14157 async fn worker_retry_policy_can_disable_poll_retries() {
14158 let server = MockWorkerServer::unavailable_polls();
14159 let client = Client::builder(server.base_url())
14160 .timeout(Duration::from_secs(2))
14161 .build()
14162 .expect("client");
14163 let mut worker = Worker::new(client, "rust-workers")
14164 .worker_id("no-retry-worker")
14165 .poll_timeout(Duration::from_millis(10))
14166 .retry_policy(WorkerRetryPolicy {
14167 max_retries: 0,
14168 initial_backoff: Duration::from_millis(1),
14169 max_backoff: Duration::from_millis(1),
14170 });
14171 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
14172
14173 let error = worker
14174 .run_once()
14175 .await
14176 .expect_err("disabled retries must return the first transport failure");
14177 assert!(matches!(error, Error::Transport(_)));
14178 assert_eq!(
14179 server.request_count("/api/worker/workflow-tasks/poll"),
14180 1,
14181 "max_retries=0 must send only the initial request"
14182 );
14183 }
14184
14185 #[tokio::test]
14186 async fn worker_does_not_retry_authentication_failures() {
14187 let server = MockWorkerServer::unauthorized_polls();
14188 let client = Client::builder(server.base_url())
14189 .timeout(Duration::from_secs(2))
14190 .build()
14191 .expect("client");
14192 let mut worker = Worker::new(client, "rust-workers")
14193 .worker_id("unauthorized-worker")
14194 .poll_timeout(Duration::from_millis(10));
14195 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
14196
14197 let error = worker
14198 .run()
14199 .await
14200 .expect_err("authentication must terminate");
14201 let Error::Http { status, body } = error else {
14202 panic!("expected stable HTTP authentication error");
14203 };
14204 assert_eq!(status, reqwest::StatusCode::UNAUTHORIZED);
14205 assert!(body.contains("authentication_failed"));
14206 assert_eq!(
14207 server.request_count("/api/worker/workflow-tasks/poll"),
14208 1,
14209 "authentication failures must not be retried"
14210 );
14211 }
14212
14213 #[derive(Clone, Debug)]
14214 struct CapturedRequest {
14215 method: String,
14216 path: String,
14217 authorization: Option<String>,
14218 namespace: Option<String>,
14219 worker_protocol: Option<String>,
14220 control_protocol: Option<String>,
14221 body: String,
14222 received_at: Instant,
14223 }
14224
14225 struct MockWorkerServer {
14226 addr: SocketAddr,
14227 stop: Arc<AtomicBool>,
14228 requests: Arc<Mutex<Vec<CapturedRequest>>>,
14229 thread: Option<thread::JoinHandle<()>>,
14230 }
14231
14232 #[derive(Clone, Copy, Default)]
14233 struct MockWorkerBehavior {
14234 reject_query_protocol: bool,
14235 reject_query_completion: bool,
14236 waiting_query_worker: bool,
14237 decline_registration: bool,
14238 complete_named_signal: bool,
14239 poll_failures_per_path: usize,
14240 heartbeat_failures: usize,
14241 heartbeat_failure_request: Option<usize>,
14242 delayed_heartbeat_request: Option<usize>,
14243 heartbeat_response_delay: Duration,
14244 concurrent_requests: bool,
14245 unauthorized_polls: bool,
14246 reject_registration: bool,
14247 reject_deregistration: bool,
14248 reject_deregistration_protocol: bool,
14249 cancelled_activity: bool,
14250 draining_polls: bool,
14251 workflow_completion_status: Option<&'static str>,
14252 workflow_completion_body: Option<&'static str>,
14253 }
14254
14255 impl MockWorkerServer {
14256 fn start() -> Self {
14257 Self::start_with_behavior(MockWorkerBehavior::default())
14258 }
14259
14260 fn reject_query_protocol() -> Self {
14261 Self::start_with_behavior(MockWorkerBehavior {
14262 reject_query_protocol: true,
14263 ..MockWorkerBehavior::default()
14264 })
14265 }
14266
14267 fn reject_query_completion() -> Self {
14268 Self::start_with_behavior(MockWorkerBehavior {
14269 reject_query_completion: true,
14270 ..MockWorkerBehavior::default()
14271 })
14272 }
14273
14274 fn waiting_query_worker() -> Self {
14275 Self::start_with_behavior(MockWorkerBehavior {
14276 waiting_query_worker: true,
14277 complete_named_signal: true,
14278 ..MockWorkerBehavior::default()
14279 })
14280 }
14281
14282 fn transient_worker_failures() -> Self {
14283 Self::start_with_behavior(MockWorkerBehavior {
14284 poll_failures_per_path: 1,
14285 heartbeat_failures: 1,
14286 ..MockWorkerBehavior::default()
14287 })
14288 }
14289
14290 fn consecutive_poll_failures(count: usize) -> Self {
14291 Self::start_with_behavior(MockWorkerBehavior {
14292 poll_failures_per_path: count,
14293 ..MockWorkerBehavior::default()
14294 })
14295 }
14296
14297 fn delayed_heartbeat_worker() -> Self {
14298 Self::start_with_behavior(MockWorkerBehavior {
14299 waiting_query_worker: true,
14300 delayed_heartbeat_request: Some(2),
14301 heartbeat_response_delay: Duration::from_millis(1_500),
14302 concurrent_requests: true,
14303 cancelled_activity: true,
14304 ..MockWorkerBehavior::default()
14305 })
14306 }
14307
14308 fn heartbeat_retry_worker() -> Self {
14309 Self::start_with_behavior(MockWorkerBehavior {
14310 waiting_query_worker: true,
14311 heartbeat_failure_request: Some(2),
14312 concurrent_requests: true,
14313 ..MockWorkerBehavior::default()
14314 })
14315 }
14316
14317 fn unavailable_polls() -> Self {
14318 Self::start_with_behavior(MockWorkerBehavior {
14319 poll_failures_per_path: usize::MAX,
14320 ..MockWorkerBehavior::default()
14321 })
14322 }
14323
14324 fn unauthorized_polls() -> Self {
14325 Self::start_with_behavior(MockWorkerBehavior {
14326 unauthorized_polls: true,
14327 ..MockWorkerBehavior::default()
14328 })
14329 }
14330
14331 fn rejected_registration() -> Self {
14332 Self::start_with_behavior(MockWorkerBehavior {
14333 reject_registration: true,
14334 ..MockWorkerBehavior::default()
14335 })
14336 }
14337
14338 fn declined_registration() -> Self {
14339 Self::start_with_behavior(MockWorkerBehavior {
14340 decline_registration: true,
14341 ..MockWorkerBehavior::default()
14342 })
14343 }
14344
14345 fn rejected_deregistration() -> Self {
14346 Self::start_with_behavior(MockWorkerBehavior {
14347 reject_deregistration: true,
14348 ..MockWorkerBehavior::default()
14349 })
14350 }
14351
14352 fn rejected_deregistration_protocol() -> Self {
14353 Self::start_with_behavior(MockWorkerBehavior {
14354 reject_deregistration_protocol: true,
14355 ..MockWorkerBehavior::default()
14356 })
14357 }
14358
14359 fn unauthorized_polls_and_rejected_deregistration() -> Self {
14360 Self::start_with_behavior(MockWorkerBehavior {
14361 unauthorized_polls: true,
14362 reject_deregistration: true,
14363 ..MockWorkerBehavior::default()
14364 })
14365 }
14366
14367 fn cancelled_activity() -> Self {
14368 Self::start_with_behavior(MockWorkerBehavior {
14369 cancelled_activity: true,
14370 ..MockWorkerBehavior::default()
14371 })
14372 }
14373
14374 fn draining_polls() -> Self {
14375 Self::start_with_behavior(MockWorkerBehavior {
14376 draining_polls: true,
14377 ..MockWorkerBehavior::default()
14378 })
14379 }
14380
14381 fn workflow_completion(status: &'static str, body: &'static str) -> Self {
14382 Self::start_with_behavior(MockWorkerBehavior {
14383 workflow_completion_status: Some(status),
14384 workflow_completion_body: Some(body),
14385 ..MockWorkerBehavior::default()
14386 })
14387 }
14388
14389 fn start_with_behavior(behavior: MockWorkerBehavior) -> Self {
14390 let listener = TcpListener::bind("127.0.0.1:0").expect("bind mock server");
14391 listener
14392 .set_nonblocking(true)
14393 .expect("configure mock listener");
14394 let addr = listener.local_addr().expect("mock server address");
14395 let stop = Arc::new(AtomicBool::new(false));
14396 let server_stop = Arc::clone(&stop);
14397 let requests = Arc::new(Mutex::new(Vec::new()));
14398 let server_requests = Arc::clone(&requests);
14399 let thread = thread::spawn(move || {
14400 let mut request_threads = Vec::new();
14401 while !server_stop.load(Ordering::SeqCst) {
14402 match listener.accept() {
14403 Ok((mut stream, _)) => {
14404 if behavior.concurrent_requests {
14405 let requests = Arc::clone(&server_requests);
14406 request_threads.push(thread::spawn(move || {
14407 handle_mock_worker_request(&mut stream, &requests, behavior)
14408 }));
14409 } else {
14410 handle_mock_worker_request(&mut stream, &server_requests, behavior);
14411 }
14412 }
14413 Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
14414 let mut index = 0;
14415 while index < request_threads.len() {
14416 if request_threads[index].is_finished() {
14417 request_threads
14418 .swap_remove(index)
14419 .join()
14420 .expect("join mock request");
14421 } else {
14422 index += 1;
14423 }
14424 }
14425 thread::sleep(Duration::from_millis(5));
14426 }
14427 Err(_) => break,
14428 }
14429 }
14430 for request_thread in request_threads {
14431 request_thread.join().expect("join mock request");
14432 }
14433 });
14434
14435 Self {
14436 addr,
14437 stop,
14438 requests,
14439 thread: Some(thread),
14440 }
14441 }
14442
14443 fn base_url(&self) -> String {
14444 format!("http://{}", self.addr)
14445 }
14446
14447 fn worker_protocol_for(&self, path: &str) -> Option<String> {
14448 self.requests
14449 .lock()
14450 .expect("captured requests")
14451 .iter()
14452 .find(|request| request.path == path)
14453 .and_then(|request| request.worker_protocol.clone())
14454 }
14455
14456 fn control_protocol_for(&self, path: &str) -> Option<String> {
14457 self.requests
14458 .lock()
14459 .expect("captured requests")
14460 .iter()
14461 .find(|request| request.path == path)
14462 .and_then(|request| request.control_protocol.clone())
14463 }
14464
14465 fn method_for(&self, path: &str) -> Option<String> {
14466 self.requests
14467 .lock()
14468 .expect("captured requests")
14469 .iter()
14470 .find(|request| request.path == path)
14471 .map(|request| request.method.clone())
14472 }
14473
14474 fn authorization_for(&self, path: &str) -> Option<String> {
14475 self.requests
14476 .lock()
14477 .expect("captured requests")
14478 .iter()
14479 .find(|request| request.path == path)
14480 .and_then(|request| request.authorization.clone())
14481 }
14482
14483 fn namespace_for(&self, path: &str) -> Option<String> {
14484 self.requests
14485 .lock()
14486 .expect("captured requests")
14487 .iter()
14488 .find(|request| request.path == path)
14489 .and_then(|request| request.namespace.clone())
14490 }
14491
14492 fn request_count(&self, path: &str) -> usize {
14493 self.requests
14494 .lock()
14495 .expect("captured requests")
14496 .iter()
14497 .filter(|request| request.path == path)
14498 .count()
14499 }
14500
14501 fn captured_paths(&self) -> Vec<String> {
14502 self.requests
14503 .lock()
14504 .expect("captured requests")
14505 .iter()
14506 .map(|request| request.path.clone())
14507 .collect()
14508 }
14509
14510 fn request_times(&self, path: &str) -> Vec<Instant> {
14511 self.requests
14512 .lock()
14513 .expect("captured requests")
14514 .iter()
14515 .filter(|request| request.path == path)
14516 .map(|request| request.received_at)
14517 .collect()
14518 }
14519
14520 fn request_body(&self, path: &str) -> Value {
14521 let requests = self.requests.lock().expect("captured requests");
14522 let body = &requests
14523 .iter()
14524 .find(|request| request.path == path)
14525 .unwrap_or_else(|| panic!("missing request for {path}"))
14526 .body;
14527 serde_json::from_str(body).unwrap_or_else(|error| {
14528 panic!("invalid JSON request body for {path}: {error}: {body:?}")
14529 })
14530 }
14531
14532 fn request_bodies(&self, path: &str) -> Vec<Value> {
14533 self.requests
14534 .lock()
14535 .expect("captured requests")
14536 .iter()
14537 .filter(|request| request.path == path)
14538 .map(|request| {
14539 serde_json::from_str(&request.body).unwrap_or_else(|error| {
14540 panic!(
14541 "invalid JSON request body for {path}: {error}: {:?}",
14542 request.body
14543 )
14544 })
14545 })
14546 .collect()
14547 }
14548 }
14549
14550 impl Drop for MockWorkerServer {
14551 fn drop(&mut self) {
14552 self.stop.store(true, Ordering::SeqCst);
14553 let _ = TcpStream::connect(self.addr);
14554
14555 if let Some(thread) = self.thread.take() {
14556 thread.join().expect("join mock server");
14557 }
14558 }
14559 }
14560
14561 fn handle_mock_worker_request(
14562 stream: &mut TcpStream,
14563 requests: &Arc<Mutex<Vec<CapturedRequest>>>,
14564 behavior: MockWorkerBehavior,
14565 ) {
14566 let _ = stream.set_read_timeout(Some(Duration::from_millis(200)));
14567 let mut buffer = [0_u8; 8192];
14568 let mut request = Vec::new();
14569
14570 loop {
14571 match stream.read(&mut buffer) {
14572 Ok(0) => break,
14573 Ok(read) => {
14574 request.extend_from_slice(&buffer[..read]);
14575 if mock_request_is_complete(&request) {
14576 break;
14577 }
14578 }
14579 Err(error)
14580 if matches!(
14581 error.kind(),
14582 std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
14583 ) =>
14584 {
14585 break;
14586 }
14587 Err(_) => return,
14588 }
14589 }
14590
14591 let request = String::from_utf8_lossy(&request);
14592 let body = request
14593 .split_once("\r\n\r\n")
14594 .map(|(_, body)| body)
14595 .unwrap_or_default();
14596 let path = request
14597 .lines()
14598 .next()
14599 .and_then(|line| line.split_whitespace().nth(1))
14600 .unwrap_or_default();
14601 let method = request
14602 .lines()
14603 .next()
14604 .and_then(|line| line.split_whitespace().next())
14605 .unwrap_or_default();
14606 let authorization = request.lines().find_map(|line| {
14607 let (name, value) = line.split_once(':')?;
14608 name.eq_ignore_ascii_case("Authorization")
14609 .then(|| value.trim().to_string())
14610 });
14611 let namespace = request.lines().find_map(|line| {
14612 let (name, value) = line.split_once(':')?;
14613 name.eq_ignore_ascii_case("X-Namespace")
14614 .then(|| value.trim().to_string())
14615 });
14616 let worker_protocol = request.lines().find_map(|line| {
14617 let (name, value) = line.split_once(':')?;
14618 name.eq_ignore_ascii_case("X-Durable-Workflow-Protocol-Version")
14619 .then(|| value.trim().to_string())
14620 });
14621 let control_protocol = request.lines().find_map(|line| {
14622 let (name, value) = line.split_once(':')?;
14623 name.eq_ignore_ascii_case("X-Durable-Workflow-Control-Plane-Version")
14624 .then(|| value.trim().to_string())
14625 });
14626 let request_number = {
14627 let mut requests = requests.lock().expect("captured requests");
14628 requests.push(CapturedRequest {
14629 method: method.to_string(),
14630 path: path.to_string(),
14631 authorization,
14632 namespace,
14633 worker_protocol: worker_protocol.clone(),
14634 control_protocol,
14635 body: body.to_string(),
14636 received_at: Instant::now(),
14637 });
14638 requests
14639 .iter()
14640 .filter(|request| request.path == path)
14641 .count()
14642 };
14643
14644 if behavior.reject_registration && path == "/api/worker/register" {
14645 write_mock_response(
14646 stream,
14647 "503 Service Unavailable",
14648 r#"{"reason":"registration_unavailable","message":"registration failed"}"#,
14649 );
14650 return;
14651 }
14652
14653 if path.starts_with("/api/worker/registrations/") {
14654 if behavior.reject_deregistration_protocol {
14655 write_mock_response(
14656 stream,
14657 "400 Bad Request",
14658 r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.1","requested_version":"1.2"}"#,
14659 );
14660 } else if behavior.reject_deregistration {
14661 write_mock_response(
14662 stream,
14663 "403 Forbidden",
14664 r#"{"reason":"authorization_failed","message":"worker cannot deregister"}"#,
14665 );
14666 } else {
14667 write_mock_response(
14668 stream,
14669 "200 OK",
14670 r#"{"worker_id":"deregistered-worker","outcome":"deregistered","recovered_workflow_task_count":2}"#,
14671 );
14672 }
14673 return;
14674 }
14675
14676 let is_poll = matches!(
14677 path,
14678 "/api/worker/workflow-tasks/poll"
14679 | "/api/worker/activity-tasks/poll"
14680 | "/api/worker/query-tasks/poll"
14681 );
14682 if is_poll && request_number <= behavior.poll_failures_per_path {
14683 return;
14684 }
14685 if path == "/api/worker/heartbeat" && request_number <= behavior.heartbeat_failures {
14686 return;
14687 }
14688 if path == "/api/worker/heartbeat"
14689 && behavior.heartbeat_failure_request == Some(request_number)
14690 {
14691 return;
14692 }
14693 if path == "/api/worker/heartbeat"
14694 && behavior.delayed_heartbeat_request == Some(request_number)
14695 {
14696 thread::sleep(behavior.heartbeat_response_delay);
14697 }
14698 if behavior.unauthorized_polls && is_poll {
14699 write_mock_response(
14700 stream,
14701 "401 Unauthorized",
14702 r#"{"reason":"authentication_failed","message":"invalid worker token"}"#,
14703 );
14704 return;
14705 }
14706 if behavior.draining_polls && is_poll {
14707 write_mock_response(
14708 stream,
14709 "409 Conflict",
14710 r#"{"task":null,"poll_status":"draining","reason":"worker_draining","worker_status":"draining","drain_intent":"draining"}"#,
14711 );
14712 return;
14713 }
14714
14715 if behavior.reject_query_protocol && path.starts_with("/api/worker/query-tasks/") {
14716 let requested_version = worker_protocol.as_deref().unwrap_or("missing");
14717 let body = format!(
14718 r#"{{"reason":"unsupported_protocol_version","message":"Query tasks require worker protocol 1.8 or newer.","supported_version":"1.7","requested_version":"{requested_version}"}}"#
14719 );
14720 write_mock_response(stream, "400 Bad Request", &body);
14721 return;
14722 }
14723
14724 if behavior.reject_query_completion && path == "/api/worker/query-tasks/query-late/complete"
14725 {
14726 write_mock_response(
14727 stream,
14728 "409 Conflict",
14729 r#"{"reason":"query_task_timed_out","message":"query task timed out before completion"}"#,
14730 );
14731 return;
14732 }
14733
14734 if behavior.workflow_completion_status.is_some()
14735 && path == "/api/worker/workflow-tasks/poll"
14736 && request_number == 1
14737 {
14738 write_mock_response(
14739 stream,
14740 "200 OK",
14741 r#"{"task":{"task_id":"workflow-timeout-task","workflow_id":"reused-workflow-id","run_id":"run-selected-timeout","workflow_type":"timeout.workflow","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"workflow_task_attempt":3,"lease_owner":"timeout-worker"}}"#,
14742 );
14743 return;
14744 }
14745
14746 if path == "/api/worker/workflow-tasks/workflow-timeout-task/complete" {
14747 if let (Some(status), Some(body)) = (
14748 behavior.workflow_completion_status,
14749 behavior.workflow_completion_body,
14750 ) {
14751 write_mock_response(stream, status, body);
14752 return;
14753 }
14754 }
14755
14756 if behavior.waiting_query_worker {
14757 if behavior.complete_named_signal
14758 && path == "/api/worker/workflow-tasks/poll"
14759 && request_number == 1
14760 {
14761 let body = json!({
14762 "task": {
14763 "task_id": "snapshot-open",
14764 "workflow_id": "snapshot-1",
14765 "run_id": "snapshot-run-1",
14766 "workflow_type": "snapshot",
14767 "payload_codec": DEFAULT_CODEC,
14768 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
14769 .expect("Avro workflow arguments"),
14770 "history_events": [],
14771 "workflow_task_attempt": 1,
14772 "lease_owner": "rust-snapshot-worker"
14773 }
14774 })
14775 .to_string();
14776 write_mock_response(stream, "200 OK", &body);
14777 return;
14778 }
14779
14780 let signal_request = request_number - usize::from(behavior.complete_named_signal);
14781 let signal_request_limit = 2 + usize::from(behavior.complete_named_signal);
14782 if path == "/api/worker/workflow-tasks/poll"
14783 && signal_request >= 1
14784 && signal_request <= signal_request_limit
14785 {
14786 let finish = behavior.complete_named_signal && signal_request == 3;
14787 let amounts = if signal_request == 1 {
14788 vec![3]
14789 } else {
14790 vec![3, 5]
14791 };
14792 let task_id = if signal_request == 1 {
14793 "snapshot-wait-3"
14794 } else if finish {
14795 "snapshot-finish"
14796 } else {
14797 "snapshot-wait-5"
14798 };
14799 let mut history_events = std::iter::once(json!({
14800 "event_type": "SignalWaitOpened",
14801 "payload": {"sequence": 1, "signal_name": "finish"}
14802 }))
14803 .chain(amounts.iter().enumerate().map(|(index, amount)| {
14804 json!({
14805 "event_type": "SignalReceived",
14806 "payload": {
14807 "signal_id": format!("increment-{amount}"),
14808 "signal_name": "increment",
14809 "workflow_sequence": index + 2,
14810 "payload_codec": DEFAULT_CODEC,
14811 "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
14812 .expect("Avro signal envelope")
14813 }
14814 })
14815 }))
14816 .collect::<Vec<_>>();
14817 let (resume_id, resume_name, resume_arguments) = if finish {
14818 history_events.push(json!({
14819 "event_type": "SignalReceived",
14820 "payload": {
14821 "signal_id": "finish",
14822 "signal_name": "finish",
14823 "workflow_sequence": 4,
14824 "payload_codec": DEFAULT_CODEC,
14825 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
14826 .expect("Avro finish signal envelope")
14827 }
14828 }));
14829 (
14830 "finish".to_string(),
14831 "finish".to_string(),
14832 encode_value_envelope(&json!([]), DEFAULT_CODEC)
14833 .expect("Avro finish resume signal"),
14834 )
14835 } else {
14836 let amount = amounts.last().expect("amount");
14837 (
14838 format!("increment-{amount}"),
14839 "increment".to_string(),
14840 encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
14841 .expect("Avro increment resume signal"),
14842 )
14843 };
14844 let body = json!({
14845 "task": {
14846 "task_id": task_id,
14847 "workflow_id": "snapshot-1",
14848 "run_id": "snapshot-run-1",
14849 "workflow_type": "snapshot",
14850 "payload_codec": DEFAULT_CODEC,
14851 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
14852 .expect("Avro workflow arguments"),
14853 "history_events": history_events,
14854 "workflow_task_attempt": 1,
14855 "workflow_signal_id": resume_id,
14856 "signal_name": resume_name,
14857 "signal_arguments": resume_arguments,
14858 "lease_owner": "rust-snapshot-worker"
14859 }
14860 })
14861 .to_string();
14862 write_mock_response(stream, "200 OK", &body);
14863 return;
14864 }
14865
14866 if path == "/api/worker/query-tasks/poll" && request_number == 1 {
14867 let history_events = [3, 5]
14868 .into_iter()
14869 .enumerate()
14870 .map(|(index, amount)| {
14871 json!({
14872 "event_type": "SignalReceived",
14873 "payload": {
14874 "signal_id": format!("increment-{amount}"),
14875 "signal_name": "increment",
14876 "workflow_sequence": index + 2,
14877 "payload_codec": DEFAULT_CODEC,
14878 "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
14879 .expect("Avro query signal envelope")
14880 }
14881 })
14882 })
14883 .collect::<Vec<_>>();
14884 let body = json!({
14885 "task": {
14886 "query_task_id": "snapshot-current",
14887 "query_task_attempt": 1,
14888 "lease_owner": "rust-snapshot-worker",
14889 "workflow_id": "snapshot-1",
14890 "run_id": "snapshot-run-1",
14891 "workflow_type": "snapshot",
14892 "query_name": "current",
14893 "payload_codec": DEFAULT_CODEC,
14894 "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
14895 .expect("Avro workflow arguments"),
14896 "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
14897 .expect("Avro query arguments"),
14898 "history_events": history_events,
14899 "run_status": "waiting"
14900 }
14901 })
14902 .to_string();
14903 write_mock_response(stream, "200 OK", &body);
14904 return;
14905 }
14906
14907 if path == "/api/worker/workflow-tasks/snapshot-wait-3/fail"
14908 || path == "/api/worker/workflow-tasks/snapshot-wait-5/fail"
14909 {
14910 write_mock_response(
14911 stream,
14912 "200 OK",
14913 r#"{"outcome":"waiting_for_history","recorded":true}"#,
14914 );
14915 return;
14916 }
14917
14918 if path == "/api/worker/workflow-tasks/snapshot-open/complete" {
14919 write_mock_response(stream, "200 OK", r#"{"outcome":"waiting","recorded":true}"#);
14920 return;
14921 }
14922
14923 if path == "/api/worker/workflow-tasks/snapshot-finish/complete" {
14924 write_mock_response(
14925 stream,
14926 "200 OK",
14927 r#"{"outcome":"completed","run_status":"completed","recorded":true}"#,
14928 );
14929 return;
14930 }
14931
14932 if path == "/api/worker/query-tasks/snapshot-current/complete" {
14933 write_mock_response(stream, "200 OK", r#"{"outcome":"completed"}"#);
14934 return;
14935 }
14936 }
14937
14938 if matches!(
14939 path,
14940 "/api/workflows/typed-1/query/inspect" | "/api/workflows/typed-1/update/replace"
14941 ) {
14942 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
14943 .expect("typed mock result");
14944 let body = json!({
14945 "result": typed_fidelity_probe().into_json().expect("result projection"),
14946 "result_envelope": result,
14947 })
14948 .to_string();
14949 write_mock_response(stream, "200 OK", &body);
14950 return;
14951 }
14952
14953 if path == "/api/workflows/typed-1" {
14954 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
14955 .expect("typed mock result");
14956 let body = json!({
14957 "workflow_id": "typed-1",
14958 "run_id": "run-typed-1",
14959 "workflow_type": "typed.echo",
14960 "status": "completed",
14961 "output": typed_fidelity_probe().into_json().expect("output projection"),
14962 "output_envelope": result,
14963 })
14964 .to_string();
14965 write_mock_response(stream, "200 OK", &body);
14966 return;
14967 }
14968
14969 let (status, body) = match path {
14970 "/api/health" => ("200 OK", r#"{"status":"ok"}"#),
14971 "/api/workflows" => (
14972 "201 Created",
14973 r#"{"workflow_id":"wf-start-options","run_id":"run-start-options","workflow_type":"rust.timeout"}"#,
14974 ),
14975 "/api/worker/register" if behavior.decline_registration => (
14976 "200 OK",
14977 r#"{"worker_id":"declined-worker","registered":false}"#,
14978 ),
14979 "/api/worker/register" if behavior.waiting_query_worker => (
14980 "200 OK",
14981 r#"{"worker_id":"rust-snapshot-worker","registered":true,"heartbeat_interval_seconds":1}"#,
14982 ),
14983 "/api/worker/register" => (
14984 "200 OK",
14985 r#"{"worker_id":"mock-worker","registered":true,"heartbeat_interval_seconds":3600}"#,
14986 ),
14987 "/api/worker/heartbeat" => ("200 OK", "{}"),
14988 "/api/worker/activity-tasks/poll"
14989 if behavior.cancelled_activity && request_number == 1 =>
14990 {
14991 (
14992 "200 OK",
14993 r#"{"task":{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","activity_type":"cancel-aware","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"attempt_number":1,"lease_owner":"rust-cancel-worker"}}"#,
14994 )
14995 }
14996 "/api/worker/activity-tasks/poll" | "/api/worker/workflow-tasks/poll" => {
14997 ("200 OK", r#"{"task":null}"#)
14998 }
14999 "/api/worker/query-tasks/poll"
15000 if behavior.reject_query_completion && request_number == 1 =>
15001 {
15002 (
15003 "200 OK",
15004 r#"{"task":{"query_task_id":"query-late","query_task_attempt":1,"lease_owner":"late-worker","workflow_id":"counter-late","run_id":"run-late","workflow_type":"counter","query_name":"current","payload_codec":"avro","workflow_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"query_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"run_status":"running"}}"#,
15005 )
15006 }
15007 "/api/worker/query-tasks/poll" => ("200 OK", r#"{"task":null}"#),
15008 "/api/worker/query-tasks/query-capture/complete"
15009 | "/api/worker/query-tasks/query-capture/fail" => ("200 OK", "{}"),
15010 "/api/worker/activity-tasks/activity-cancel/heartbeat" => (
15011 "200 OK",
15012 r#"{"activity_attempt_id":"attempt-cancel","cancel_requested":true,"can_continue":false,"reason":"run_cancelled","run_closed_reason":"cancelled","heartbeat_recorded":false}"#,
15013 ),
15014 "/api/worker/activity-tasks/activity-cancel/complete" => (
15015 "409 Conflict",
15016 r#"{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","reason":"run_cancelled","cancel_requested":true,"can_continue":false,"run_closed_reason":"cancelled"}"#,
15017 ),
15018 "/api/worker/activity-tasks/activity-typed/complete"
15019 | "/api/worker/activity-tasks/activity-typed/fail"
15020 | "/api/workflows/typed-1/signal/changed" => ("200 OK", "{}"),
15021 "/api/workflows/counter-1/query/current" => (
15022 "200 OK",
15023 r#"{"workflow_id":"counter-1","query_name":"current","result":{"count":8},"result_envelope":{"codec":"avro","blob":"wwHioz3/VYAiNw4CCmNvdW50BBAA"}}"#,
15024 ),
15025 "/api/workflows/counter-1/query/missing" => (
15026 "404 Not Found",
15027 r#"{"workflow_id":"counter-1","query_name":"missing","reason":"rejected_unknown_query","message":"unknown query"}"#,
15028 ),
15029 "/api/workflows/wf-lifecycle/cancel" => (
15030 "200 OK",
15031 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","reason":"cleanup requested","command_status":"accepted"}"#,
15032 ),
15033 "/api/workflows/wf-lifecycle/terminate" => (
15034 "200 OK",
15035 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","reason":"forced stop","command_status":"accepted"}"#,
15036 ),
15037 "/api/workflows/wf-lifecycle/runs/run-current/cancel" => (
15038 "200 OK",
15039 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","command_status":"accepted"}"#,
15040 ),
15041 "/api/workflows/wf-lifecycle/runs/run-current/terminate" => (
15042 "200 OK",
15043 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","command_status":"accepted"}"#,
15044 ),
15045 "/api/workflows/wf-lifecycle/runs/run-stale/cancel"
15046 | "/api/workflows/wf-lifecycle/runs/run-stale/terminate" => (
15047 "409 Conflict",
15048 r#"{"workflow_id":"wf-lifecycle","run_id":"run-stale","reason":"historical_run_command_rejected","target_scope":"run","message":"Commands cannot target historical runs."}"#,
15049 ),
15050 "/api/workflows/wf-failed" | "/api/workflows/wf-failed/runs/run-failed" => (
15051 "200 OK",
15052 r#"{"workflow_id":"wf-failed","run_id":"run-failed","status":"failed","closed_reason":"failed","error":"payment failed","failure":{"message":"payment failed","failure_category":"application","exception_type":"PaymentError","exception_class":"billing::PaymentError","non_retryable":true,"exception":{"type":"PaymentError","class":"billing::PaymentError","message":"payment failed"},"failures":[{"id":"failure-17","failure_category":"application"}]}}"#,
15053 ),
15054 "/api/workflows/wf-cancelled" => (
15055 "200 OK",
15056 r#"{"workflow_id":"wf-cancelled","run_id":"run-cancelled","status":"cancelled","closed_reason":"cancelled","reason":"cleanup requested"}"#,
15057 ),
15058 "/api/workflows/wf-terminated" => (
15059 "200 OK",
15060 r#"{"workflow_id":"wf-terminated","run_id":"run-terminated","status":"terminated","closed_reason":"terminated","reason":"forced stop"}"#,
15061 ),
15062 "/api/workflows/wf-timed-out" => (
15063 "200 OK",
15064 r#"{"workflow_id":"wf-timed-out","run_id":"run-timed-out","status":"failed","closed_reason":"timed_out","reason":"run_timeout"}"#,
15065 ),
15066 "/api/workflows/wf-waiting" | "/api/workflows/wf-waiting/runs/run-waiting" => (
15067 "200 OK",
15068 r#"{"workflow_id":"wf-waiting","run_id":"run-waiting","status":"waiting"}"#,
15069 ),
15070 "/api/workflows/wf-selected" => (
15071 "200 OK",
15072 r#"{"workflow_id":"wf-selected","run_id":"run-current","status":"completed","output":"current run output"}"#,
15073 ),
15074 "/api/workflows/wf-selected/runs/run-selected" => (
15075 "200 OK",
15076 r#"{"workflow_id":"wf-selected","run_id":"run-selected","status":"cancelled","closed_reason":"cancelled","reason":"selected run cancelled"}"#,
15077 ),
15078 _ => ("404 Not Found", r#"{"message":"not found"}"#),
15079 };
15080 write_mock_response(stream, status, body);
15081 }
15082
15083 fn mock_request_is_complete(request: &[u8]) -> bool {
15084 let Some(header_end) = request
15085 .windows(4)
15086 .position(|window| window == b"\r\n\r\n")
15087 .map(|position| position + 4)
15088 else {
15089 return false;
15090 };
15091 let headers = String::from_utf8_lossy(&request[..header_end]);
15092 let content_length = headers.lines().find_map(|line| {
15093 let (name, value) = line.split_once(':')?;
15094 name.eq_ignore_ascii_case("content-length")
15095 .then(|| value.trim().parse::<usize>().ok())
15096 .flatten()
15097 });
15098
15099 request.len() >= header_end + content_length.unwrap_or(0)
15100 }
15101
15102 fn write_mock_response(stream: &mut TcpStream, status: &str, body: &str) {
15103 let response = format!(
15104 "HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
15105 body.len()
15106 );
15107
15108 let _ = stream.write_all(response.as_bytes());
15109 let _ = stream.flush();
15110 }
15111}