1#![doc = include_str!("../README.md")]
2
3use std::{
4 any::{Any, TypeId},
5 collections::{BTreeMap, HashMap},
6 future::Future,
7 io::{self, Read},
8 pin::Pin,
9 sync::{
10 atomic::{AtomicBool, Ordering},
11 Arc, Mutex, OnceLock,
12 },
13 task::{Context as TaskContext, Poll},
14 time::{Duration, Instant, SystemTime, UNIX_EPOCH},
15};
16
17use apache_avro::{from_avro_datum, to_avro_datum, types::Value as AvroDatum, Schema};
18use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
19use futures_util::{future::OptionFuture, task::noop_waker_ref};
20use serde::{
21 de::DeserializeOwned,
22 ser::{SerializeMap, SerializeSeq},
23 Deserialize, Serialize, Serializer,
24};
25pub use serde_json::{json, Value};
26use thiserror::Error;
27pub use uuid::Uuid;
28
29pub const WORKER_PROTOCOL_VERSION: &str = "1.2";
30pub const CONTROL_PLANE_VERSION: &str = "2";
31pub const DEFAULT_CODEC: &str = "avro";
32pub const JSON_CODEC: &str = "json";
33pub const SDK_VERSION: &str = concat!("durable-workflow-rust/", env!("CARGO_PKG_VERSION"));
34pub const QUERY_TASKS_CAPABILITY: &str = "query_tasks";
36pub const WORKFLOW_UPDATES_CAPABILITY: &str = "workflow_updates";
38pub const QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
40
41const MAX_LONG_POLL_TIMEOUT_SECONDS: u64 = 60;
42const WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE: &str =
43 "Workflow task waiting for scheduled history.";
44const WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE: &str = "WorkflowTaskWaitingForHistory";
45
46const QUERY_TASK_FINAL_REJECTION_REASONS: &[&str] = &[
47 "lease_expired",
48 "query_task_not_found",
49 "query_task_not_leased",
50 "query_task_timed_out",
51];
52
53pub const AVRO_VALUE_SCHEMA_JSON: &str =
55 include_str!("../schema/durable_workflow.protocol.Value.v1.avsc");
56pub const AVRO_VALUE_SCHEMA_FINGERPRINT_HEX: &str = "e2a33dff55802237";
57pub const AVRO_VALUE_SCHEMA_FINGERPRINT: [u8; 8] = [0xe2, 0xa3, 0x3d, 0xff, 0x55, 0x80, 0x22, 0x37];
58const AVRO_SINGLE_OBJECT_MAGIC: [u8; 2] = [0xc3, 0x01];
59
60static AVRO_VALUE_SCHEMA: OnceLock<std::result::Result<Schema, String>> = OnceLock::new();
61
62#[derive(Clone, Copy)]
63enum RequestProtocol {
64 ControlPlane,
65 Worker(&'static str),
66}
67
68pub type Result<T> = std::result::Result<T, Error>;
69
70#[derive(Debug, Error)]
71pub enum Error {
72 #[error("transport error: {0}")]
73 Transport(#[from] reqwest::Error),
74 #[error(
75 "invalid Durable Workflow base URL: omit the SDK-owned /api suffix and pass the Server or Cloud runtime base URL; the SDK appends /api automatically"
76 )]
77 InvalidBaseUrl,
78 #[error("json error: {0}")]
79 Json(#[from] serde_json::Error),
80 #[error("http {status}: {body}")]
81 Http {
82 status: reqwest::StatusCode,
83 body: String,
84 },
85 #[error("codec error: {0}")]
86 Codec(String),
87 #[error(transparent)]
88 QueryFailed(QueryFailure),
89 #[error(transparent)]
90 Protocol(ProtocolFailure),
91 #[error(transparent)]
92 NonDeterministicReplay(ReplayFailure),
93 #[error(transparent)]
94 ChildWorkflowFailed(ChildWorkflowFailure),
95 #[error(transparent)]
96 ActivityFailed(ActivityFailure),
97 #[error(transparent)]
98 WorkflowCommandRejected(WorkflowCommandRejection),
99 #[error(transparent)]
100 WorkflowFailed(WorkflowTerminalOutcome),
101 #[error(transparent)]
102 WorkflowCancelled(WorkflowTerminalOutcome),
103 #[error(transparent)]
104 WorkflowTerminated(WorkflowTerminalOutcome),
105 #[error(transparent)]
106 WorkflowTimedOut(WorkflowTerminalOutcome),
107 #[error(transparent)]
108 ActivityTaskRejected(ActivityTaskRejection),
109 #[error("workflow handler {0:?} is not registered")]
110 WorkflowNotRegistered(String),
111 #[error("activity handler {0:?} is not registered")]
112 ActivityNotRegistered(String),
113 #[error("workflow future yielded without emitting a durable command")]
114 WorkflowYieldedWithoutCommand,
115 #[error("workflow state lock is poisoned")]
116 WorkflowStatePoisoned,
117 #[error("timer duration is too large for the worker protocol")]
118 TimerDurationOverflow,
119 #[error("operation timed out")]
120 Timeout,
121 #[error(
122 "missing {role}-plane credentials: configure ClientBuilder::{role}_token or ClientBuilder::token; a {opposite_role}-plane token cannot authorize this request"
123 )]
124 MissingRoleCredentials {
125 role: &'static str,
126 opposite_role: &'static str,
127 },
128 #[error("worker loop error: {0}")]
129 WorkerLoop(String),
130 #[error("{primary}; worker deregistration also failed: {deregistration}")]
131 WorkerShutdown {
132 primary: Box<Error>,
133 deregistration: Box<Error>,
134 },
135 #[error("invalid child workflow options: {0}")]
136 InvalidChildWorkflowOptions(String),
137 #[error(transparent)]
138 InvalidActivityOptions(ActivityOptionsError),
139 #[error(transparent)]
140 InvalidContinueAsNewOptions(#[from] ContinueAsNewOptionsError),
141 #[doc(hidden)]
142 #[error("workflow requested continue as new")]
143 ContinueAsNew(ContinueAsNewRequest),
144}
145
146#[derive(Clone, Copy, Debug, PartialEq, Eq)]
148pub enum WorkflowCommandKind {
149 Cancel,
150 Terminate,
151}
152
153impl WorkflowCommandKind {
154 fn as_str(self) -> &'static str {
155 match self {
156 Self::Cancel => "cancel",
157 Self::Terminate => "terminate",
158 }
159 }
160}
161
162#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
164pub struct WorkflowCommandOptions {
165 #[serde(skip_serializing_if = "Option::is_none")]
166 pub reason: Option<String>,
167 #[serde(skip_serializing_if = "Option::is_none")]
168 pub request_id: Option<String>,
169}
170
171#[derive(Clone, Debug, PartialEq, Eq)]
178pub struct WorkflowStartOptions {
179 pub execution_timeout_seconds: u64,
180 pub run_timeout_seconds: u64,
181}
182
183impl Default for WorkflowStartOptions {
184 fn default() -> Self {
185 Self {
186 execution_timeout_seconds: 3600,
187 run_timeout_seconds: 600,
188 }
189 }
190}
191
192impl WorkflowStartOptions {
193 pub fn new() -> Self {
194 Self::default()
195 }
196
197 pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
198 self.execution_timeout_seconds = seconds;
199 self
200 }
201
202 pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
203 self.run_timeout_seconds = seconds;
204 self
205 }
206
207 fn validate(&self) -> Result<()> {
208 if self.execution_timeout_seconds == 0 {
209 return Err(Error::Codec(
210 "execution_timeout_seconds must be at least 1".to_string(),
211 ));
212 }
213 if self.run_timeout_seconds == 0 {
214 return Err(Error::Codec(
215 "run_timeout_seconds must be at least 1".to_string(),
216 ));
217 }
218 if self.run_timeout_seconds > self.execution_timeout_seconds {
219 return Err(Error::Codec(
220 "run_timeout_seconds cannot exceed execution_timeout_seconds".to_string(),
221 ));
222 }
223
224 Ok(())
225 }
226}
227
228#[derive(Clone, Debug, Default, PartialEq, Eq)]
233pub struct ContinueAsNewOptions {
234 pub workflow_type: Option<String>,
235 pub task_queue: Option<String>,
236}
237
238impl ContinueAsNewOptions {
239 pub fn new() -> Self {
240 Self::default()
241 }
242
243 pub fn workflow_type(mut self, workflow_type: impl Into<String>) -> Self {
244 self.workflow_type = Some(workflow_type.into());
245 self
246 }
247
248 pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
249 self.task_queue = Some(task_queue.into());
250 self
251 }
252
253 fn validate(&self) -> std::result::Result<(), ContinueAsNewOptionsError> {
254 for (field, value) in [
255 ("workflow_type", self.workflow_type.as_deref()),
256 ("task_queue", self.task_queue.as_deref()),
257 ] {
258 if value.is_some_and(|value| value.trim().is_empty()) {
259 return Err(ContinueAsNewOptionsError {
260 field,
261 message: format!("{field} must not be empty"),
262 });
263 }
264 }
265 Ok(())
266 }
267}
268
269#[derive(Clone, Debug, Error, PartialEq, Eq)]
271#[error("invalid continue-as-new option {field}: {message}")]
272pub struct ContinueAsNewOptionsError {
273 pub field: &'static str,
274 pub message: String,
275}
276
277#[derive(Clone, Debug, Default, PartialEq, Eq)]
279pub struct WorkflowHistoryBudget {
280 pub event_count: u64,
281 pub size_bytes: Option<u64>,
282 pub continue_as_new_recommended: bool,
283 pub pressure: Option<String>,
284}
285
286#[doc(hidden)]
287#[derive(Clone, Debug)]
288pub struct ContinueAsNewRequest {
289 arguments: AvroValue,
290 options: ContinueAsNewOptions,
291}
292
293impl WorkflowCommandOptions {
294 pub fn new() -> Self {
295 Self::default()
296 }
297
298 pub fn reason(mut self, reason: impl Into<String>) -> Self {
299 self.reason = Some(reason.into());
300 self
301 }
302
303 pub fn request_id(mut self, request_id: impl Into<String>) -> Self {
304 self.request_id = Some(request_id.into());
305 self
306 }
307}
308
309#[derive(Clone, Debug, PartialEq)]
311pub struct WorkflowCommandResult {
312 pub command: WorkflowCommandKind,
313 pub workflow_id: String,
314 pub run_id: Option<String>,
315 pub outcome: Option<String>,
316 pub reason: Option<String>,
317 pub command_status: Option<String>,
318 pub raw: Value,
319}
320
321#[derive(Clone, Debug, Error)]
323#[error("workflow {command:?} rejected ({reason}, HTTP {status}): {message}")]
324pub struct WorkflowCommandRejection {
325 pub command: WorkflowCommandKind,
326 pub status: u16,
327 pub reason: String,
328 pub message: String,
329 pub workflow_id: String,
330 pub run_id: Option<String>,
331 pub target_scope: Option<String>,
332 pub body: Value,
333}
334
335#[derive(Clone, Copy, Debug, PartialEq, Eq)]
337pub enum WorkflowTerminalKind {
338 Failed,
339 Cancelled,
340 Terminated,
341 TimedOut,
342}
343
344#[derive(Clone, Debug, Error)]
350#[error("workflow {workflow_id} run {run_id:?} ended as {kind:?} ({reason})")]
351pub struct WorkflowTerminalOutcome {
352 pub kind: WorkflowTerminalKind,
353 pub workflow_id: String,
354 pub run_id: Option<String>,
355 pub reason: String,
356 pub failure_category: Option<String>,
357 pub failure_id: Option<String>,
358 pub exception_type: Option<String>,
359 pub exception_class: Option<String>,
360 pub non_retryable: Option<bool>,
361 pub message: Option<String>,
362 pub exception: Option<Value>,
363 pub raw: Value,
364}
365
366#[derive(Clone, Debug, Error)]
368#[error("activity task {operation} rejected ({reason}, HTTP {status})")]
369pub struct ActivityTaskRejection {
370 pub operation: String,
371 pub status: u16,
372 pub reason: String,
373 pub task_id: String,
374 pub activity_attempt_id: String,
375 pub cancel_requested: bool,
376 pub can_continue: Option<bool>,
377 pub run_closed_reason: Option<String>,
378 pub body: Value,
379}
380
381#[derive(Clone, Copy, Debug, PartialEq, Eq)]
383pub enum ActivityOptionsErrorKind {
384 EmptyTaskQueue,
385 EmptyRetryPolicy,
386 InvalidMaxAttempts,
387 BackoffWithoutRetryBudget,
388 TooManyBackoffIntervals,
389 InvalidBackoffCoefficient,
390 BackoffGenerationTooLarge,
391 BackoffOverflow,
392 EmptyNonRetryableErrorType,
393 TimeoutNotPositive,
394 TimeoutOverflow,
395 TimeoutOrder,
396}
397
398#[derive(Clone, Debug, Error, PartialEq, Eq)]
400#[error("invalid activity options ({kind:?}, {field:?}): {message}")]
401pub struct ActivityOptionsError {
402 pub kind: ActivityOptionsErrorKind,
403 pub field: Option<&'static str>,
404 pub message: String,
405}
406
407impl ActivityOptionsError {
408 fn new(
409 kind: ActivityOptionsErrorKind,
410 field: Option<&'static str>,
411 message: impl Into<String>,
412 ) -> Self {
413 Self {
414 kind,
415 field,
416 message: message.into(),
417 }
418 }
419}
420
421#[derive(Clone, Copy, Debug, PartialEq, Eq)]
423pub enum ActivityFailureKind {
424 Failed,
425 Cancelled,
426 TimedOut,
427}
428
429#[derive(Clone, Debug, Error)]
434#[error("activity failed ({reason}): {message}")]
435pub struct ActivityFailure {
436 pub kind: ActivityFailureKind,
437 pub reason: String,
438 pub message: String,
439 pub activity_execution_id: Option<String>,
440 pub activity_attempt_id: Option<String>,
441 pub activity_type: Option<String>,
442 pub activity_class: Option<String>,
443 pub attempt_number: Option<u64>,
444 pub failure_id: Option<String>,
445 pub failure_category: Option<String>,
446 pub timeout_kind: Option<String>,
447 pub non_retryable: bool,
448 pub exception_type: Option<String>,
449 pub exception_class: Option<String>,
450 pub code: Option<Value>,
451 pub exception: Option<Value>,
452}
453
454#[derive(Clone, Copy, Debug, PartialEq, Eq)]
456pub enum ChildWorkflowFailureKind {
457 Failed,
458 Cancelled,
459 Terminated,
460}
461
462#[derive(Clone, Debug, Error)]
468#[error("child workflow failed ({reason}): {message}")]
469pub struct ChildWorkflowFailure {
470 pub kind: ChildWorkflowFailureKind,
471 pub reason: String,
472 pub message: String,
473 pub parent_workflow_id: Option<String>,
474 pub parent_workflow_run_id: Option<String>,
475 pub child_workflow_id: Option<String>,
476 pub child_workflow_run_id: Option<String>,
477 pub child_workflow_type: Option<String>,
478 pub failure_id: Option<String>,
479 pub failure_category: Option<String>,
480 pub exception_type: Option<String>,
481 pub exception_class: Option<String>,
482 pub non_retryable: bool,
483 pub code: Option<Value>,
484 pub exception: Option<Value>,
485}
486
487#[derive(Clone, Debug, PartialEq, Eq)]
489pub struct WorkflowIdentity {
490 pub workflow_id: Option<String>,
491 pub run_id: Option<String>,
492}
493
494#[derive(Clone, Debug, PartialEq)]
496pub struct ChildWorkflowResult {
497 pub parent: WorkflowIdentity,
498 pub child: WorkflowIdentity,
499 pub child_workflow_type: Option<String>,
500 pub result: Value,
501}
502
503#[derive(Clone, Debug, PartialEq)]
505pub struct ChildWorkflowAvroResult {
506 pub parent: WorkflowIdentity,
507 pub child: WorkflowIdentity,
508 pub child_workflow_type: Option<String>,
509 pub result: AvroValue,
510}
511
512#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
514pub enum ParentClosePolicy {
515 #[default]
516 Abandon,
517 RequestCancel,
518 Terminate,
519}
520
521impl ParentClosePolicy {
522 fn as_str(self) -> &'static str {
523 match self {
524 Self::Abandon => "abandon",
525 Self::RequestCancel => "request_cancel",
526 Self::Terminate => "terminate",
527 }
528 }
529}
530
531#[derive(Clone, Debug, Default, PartialEq, Eq)]
533pub struct ChildWorkflowRetryPolicy {
534 pub max_attempts: Option<u32>,
535 pub backoff_seconds: Vec<u64>,
536 pub non_retryable_error_types: Vec<String>,
537}
538
539#[derive(Clone, Debug, PartialEq, Eq)]
543pub struct ChildWorkflowOptions {
544 pub task_queue: String,
545 pub parent_close_policy: ParentClosePolicy,
546 pub retry_policy: Option<ChildWorkflowRetryPolicy>,
547 pub execution_timeout_seconds: Option<u64>,
548 pub run_timeout_seconds: Option<u64>,
549}
550
551impl ChildWorkflowOptions {
552 pub fn new(task_queue: impl Into<String>) -> Self {
553 Self {
554 task_queue: task_queue.into(),
555 parent_close_policy: ParentClosePolicy::Abandon,
556 retry_policy: None,
557 execution_timeout_seconds: None,
558 run_timeout_seconds: None,
559 }
560 }
561
562 pub fn parent_close_policy(mut self, policy: ParentClosePolicy) -> Self {
563 self.parent_close_policy = policy;
564 self
565 }
566
567 pub fn retry_policy(mut self, policy: ChildWorkflowRetryPolicy) -> Self {
568 self.retry_policy = Some(policy);
569 self
570 }
571
572 pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
573 self.execution_timeout_seconds = Some(seconds);
574 self
575 }
576
577 pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
578 self.run_timeout_seconds = Some(seconds);
579 self
580 }
581}
582
583#[derive(Clone, Debug, PartialEq, Eq)]
585pub enum ActivityBackoff {
586 Explicit(Vec<Duration>),
589 Exponential {
591 initial_interval: Duration,
592 coefficient: u32,
593 maximum_interval: Option<Duration>,
594 },
595}
596
597#[derive(Clone, Debug, Default, PartialEq, Eq)]
599pub struct ActivityRetryPolicy {
600 pub max_attempts: Option<u32>,
601 pub backoff: Option<ActivityBackoff>,
602 pub non_retryable_error_types: Vec<String>,
603}
604
605impl ActivityRetryPolicy {
606 pub fn new(max_attempts: u32) -> Self {
608 Self {
609 max_attempts: Some(max_attempts),
610 ..Self::default()
611 }
612 }
613
614 pub fn backoff_intervals(mut self, intervals: impl IntoIterator<Item = Duration>) -> Self {
615 self.backoff = Some(ActivityBackoff::Explicit(intervals.into_iter().collect()));
616 self
617 }
618
619 pub fn exponential_backoff(
620 mut self,
621 initial_interval: Duration,
622 coefficient: u32,
623 maximum_interval: Option<Duration>,
624 ) -> Self {
625 self.backoff = Some(ActivityBackoff::Exponential {
626 initial_interval,
627 coefficient,
628 maximum_interval,
629 });
630 self
631 }
632
633 pub fn non_retryable_error_type(mut self, error_type: impl Into<String>) -> Self {
634 self.non_retryable_error_types.push(error_type.into());
635 self
636 }
637
638 pub fn non_retryable_error_types(
639 mut self,
640 error_types: impl IntoIterator<Item = impl Into<String>>,
641 ) -> Self {
642 self.non_retryable_error_types
643 .extend(error_types.into_iter().map(Into::into));
644 self
645 }
646}
647
648#[derive(Clone, Debug, Default, PartialEq, Eq)]
653pub struct ActivityOptions {
654 pub task_queue: Option<String>,
655 pub retry_policy: Option<ActivityRetryPolicy>,
656 pub start_to_close_timeout: Option<Duration>,
657 pub schedule_to_start_timeout: Option<Duration>,
658 pub schedule_to_close_timeout: Option<Duration>,
659 pub heartbeat_timeout: Option<Duration>,
660}
661
662impl ActivityOptions {
663 pub fn new() -> Self {
664 Self::default()
665 }
666
667 pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
668 self.task_queue = Some(task_queue.into());
669 self
670 }
671
672 pub fn retry_policy(mut self, policy: ActivityRetryPolicy) -> Self {
673 self.retry_policy = Some(policy);
674 self
675 }
676
677 pub fn start_to_close_timeout(mut self, timeout: Duration) -> Self {
678 self.start_to_close_timeout = Some(timeout);
679 self
680 }
681
682 pub fn schedule_to_start_timeout(mut self, timeout: Duration) -> Self {
683 self.schedule_to_start_timeout = Some(timeout);
684 self
685 }
686
687 pub fn schedule_to_close_timeout(mut self, timeout: Duration) -> Self {
688 self.schedule_to_close_timeout = Some(timeout);
689 self
690 }
691
692 pub fn heartbeat_timeout(mut self, timeout: Duration) -> Self {
693 self.heartbeat_timeout = Some(timeout);
694 self
695 }
696
697 fn validate(&self) -> std::result::Result<ValidatedActivityOptions, ActivityOptionsError> {
698 if self
699 .task_queue
700 .as_deref()
701 .is_some_and(|queue| queue.trim().is_empty())
702 {
703 return Err(ActivityOptionsError::new(
704 ActivityOptionsErrorKind::EmptyTaskQueue,
705 Some("task_queue"),
706 "task_queue must not be empty",
707 ));
708 }
709
710 for (field, value) in [
711 ("start_to_close_timeout", self.start_to_close_timeout),
712 ("schedule_to_start_timeout", self.schedule_to_start_timeout),
713 ("schedule_to_close_timeout", self.schedule_to_close_timeout),
714 ("heartbeat_timeout", self.heartbeat_timeout),
715 ] {
716 if value.is_some_and(|value| value.is_zero()) {
717 return Err(ActivityOptionsError::new(
718 ActivityOptionsErrorKind::TimeoutNotPositive,
719 Some(field),
720 format!("{field} must be positive"),
721 ));
722 }
723 }
724
725 validate_timeout_order(
726 "heartbeat_timeout",
727 self.heartbeat_timeout,
728 "start_to_close_timeout",
729 self.start_to_close_timeout,
730 )?;
731 validate_timeout_order(
732 "start_to_close_timeout",
733 self.start_to_close_timeout,
734 "schedule_to_close_timeout",
735 self.schedule_to_close_timeout,
736 )?;
737 validate_timeout_order(
738 "schedule_to_start_timeout",
739 self.schedule_to_start_timeout,
740 "schedule_to_close_timeout",
741 self.schedule_to_close_timeout,
742 )?;
743
744 Ok(ValidatedActivityOptions {
745 task_queue: self.task_queue.clone(),
746 retry_policy: self
747 .retry_policy
748 .as_ref()
749 .map(validate_activity_retry_policy)
750 .transpose()?,
751 start_to_close_timeout: timeout_seconds(
752 "start_to_close_timeout",
753 self.start_to_close_timeout,
754 )?,
755 schedule_to_start_timeout: timeout_seconds(
756 "schedule_to_start_timeout",
757 self.schedule_to_start_timeout,
758 )?,
759 schedule_to_close_timeout: timeout_seconds(
760 "schedule_to_close_timeout",
761 self.schedule_to_close_timeout,
762 )?,
763 heartbeat_timeout: timeout_seconds("heartbeat_timeout", self.heartbeat_timeout)?,
764 })
765 }
766}
767
768#[derive(Clone, Debug)]
769struct ValidatedActivityOptions {
770 task_queue: Option<String>,
771 retry_policy: Option<Value>,
772 start_to_close_timeout: Option<u64>,
773 schedule_to_start_timeout: Option<u64>,
774 schedule_to_close_timeout: Option<u64>,
775 heartbeat_timeout: Option<u64>,
776}
777
778fn validate_timeout_order(
779 smaller_name: &'static str,
780 smaller: Option<Duration>,
781 larger_name: &'static str,
782 larger: Option<Duration>,
783) -> std::result::Result<(), ActivityOptionsError> {
784 if matches!((smaller, larger), (Some(smaller), Some(larger)) if smaller > larger) {
785 return Err(ActivityOptionsError::new(
786 ActivityOptionsErrorKind::TimeoutOrder,
787 Some(smaller_name),
788 format!("{smaller_name} must be <= {larger_name}"),
789 ));
790 }
791 Ok(())
792}
793
794fn timeout_seconds(
795 field: &'static str,
796 value: Option<Duration>,
797) -> std::result::Result<Option<u64>, ActivityOptionsError> {
798 value
799 .map(|value| {
800 activity_protocol_seconds(value).ok_or_else(|| {
801 ActivityOptionsError::new(
802 ActivityOptionsErrorKind::TimeoutOverflow,
803 Some(field),
804 format!("{field} is too large for the worker protocol"),
805 )
806 })
807 })
808 .transpose()
809}
810
811fn duration_seconds_ceil(value: Duration) -> Option<u64> {
812 value
813 .as_secs()
814 .checked_add(u64::from(value.subsec_nanos() > 0))
815}
816
817fn activity_protocol_seconds(value: Duration) -> Option<u64> {
818 duration_seconds_ceil(value).filter(|seconds| *seconds <= i64::MAX as u64)
819}
820
821fn validate_activity_retry_policy(
822 policy: &ActivityRetryPolicy,
823) -> std::result::Result<Value, ActivityOptionsError> {
824 if policy.max_attempts.is_none()
825 && policy.backoff.is_none()
826 && policy.non_retryable_error_types.is_empty()
827 {
828 return Err(ActivityOptionsError::new(
829 ActivityOptionsErrorKind::EmptyRetryPolicy,
830 Some("retry_policy"),
831 "retry_policy must configure at least one field",
832 ));
833 }
834 if policy.max_attempts == Some(0) {
835 return Err(ActivityOptionsError::new(
836 ActivityOptionsErrorKind::InvalidMaxAttempts,
837 Some("retry_policy.max_attempts"),
838 "max_attempts must be >= 1",
839 ));
840 }
841 if policy
842 .non_retryable_error_types
843 .iter()
844 .any(|error_type| error_type.trim().is_empty())
845 {
846 return Err(ActivityOptionsError::new(
847 ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
848 Some("retry_policy.non_retryable_error_types"),
849 "non_retryable_error_types must not contain empty values",
850 ));
851 }
852
853 let backoff_seconds = match &policy.backoff {
854 None => None,
855 Some(backoff) => {
856 let max_attempts = policy.max_attempts.ok_or_else(|| {
857 ActivityOptionsError::new(
858 ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
859 Some("retry_policy.backoff"),
860 "backoff requires max_attempts",
861 )
862 })?;
863 let retry_count = max_attempts.saturating_sub(1) as usize;
864 let intervals = match backoff {
865 ActivityBackoff::Explicit(intervals) => {
866 if intervals.len() > retry_count {
867 return Err(ActivityOptionsError::new(
868 ActivityOptionsErrorKind::TooManyBackoffIntervals,
869 Some("retry_policy.backoff"),
870 "backoff interval count must not exceed max_attempts - 1",
871 ));
872 }
873 intervals.clone()
874 }
875 ActivityBackoff::Exponential {
876 initial_interval,
877 coefficient,
878 maximum_interval,
879 } => {
880 if *coefficient < 1 {
881 return Err(ActivityOptionsError::new(
882 ActivityOptionsErrorKind::InvalidBackoffCoefficient,
883 Some("retry_policy.backoff.coefficient"),
884 "backoff coefficient must be >= 1",
885 ));
886 }
887 if retry_count > 10_000 {
888 return Err(ActivityOptionsError::new(
889 ActivityOptionsErrorKind::BackoffGenerationTooLarge,
890 Some("retry_policy.max_attempts"),
891 "generated backoff supports at most 10000 retry intervals",
892 ));
893 }
894 let mut current = *initial_interval;
895 let mut intervals = Vec::with_capacity(retry_count);
896 for _ in 0..retry_count {
897 let interval = maximum_interval
898 .map(|maximum| current.min(maximum))
899 .unwrap_or(current);
900 intervals.push(interval);
901 if maximum_interval.is_some_and(|maximum| interval == maximum) {
902 break;
903 }
904 current = current.checked_mul(*coefficient).ok_or_else(|| {
905 ActivityOptionsError::new(
906 ActivityOptionsErrorKind::BackoffOverflow,
907 Some("retry_policy.backoff"),
908 "generated backoff interval overflowed",
909 )
910 })?;
911 }
912 intervals
913 }
914 };
915 Some(
916 intervals
917 .into_iter()
918 .map(|interval| {
919 activity_protocol_seconds(interval).ok_or_else(|| {
920 ActivityOptionsError::new(
921 ActivityOptionsErrorKind::BackoffOverflow,
922 Some("retry_policy.backoff"),
923 "backoff interval is too large for the worker protocol",
924 )
925 })
926 })
927 .collect::<std::result::Result<Vec<_>, _>>()?,
928 )
929 }
930 };
931
932 let mut encoded = serde_json::Map::new();
933 if let Some(max_attempts) = policy.max_attempts {
934 encoded.insert("max_attempts".to_string(), json!(max_attempts));
935 }
936 if let Some(backoff_seconds) = backoff_seconds {
937 encoded.insert("backoff_seconds".to_string(), json!(backoff_seconds));
938 }
939 if !policy.non_retryable_error_types.is_empty() {
940 let mut canonical_error_types = Vec::new();
941 for error_type in policy
942 .non_retryable_error_types
943 .iter()
944 .map(|error_type| error_type.trim())
945 {
946 if !canonical_error_types.contains(&error_type) {
947 canonical_error_types.push(error_type);
948 }
949 }
950 encoded.insert(
951 "non_retryable_error_types".to_string(),
952 json!(canonical_error_types),
953 );
954 }
955 Ok(Value::Object(encoded))
956}
957
958#[derive(Clone, Debug, Error)]
961#[error("non-deterministic workflow replay ({reason}) at sequence {sequence:?}: {message}")]
962pub struct ReplayFailure {
963 pub reason: String,
964 pub sequence: Option<u64>,
965 pub expected: Option<String>,
966 pub actual: Option<String>,
967 pub message: String,
968}
969
970impl ReplayFailure {
971 fn new(
972 reason: impl Into<String>,
973 sequence: Option<u64>,
974 expected: Option<String>,
975 actual: Option<String>,
976 message: impl Into<String>,
977 ) -> Self {
978 Self {
979 reason: reason.into(),
980 sequence,
981 expected,
982 actual,
983 message: message.into(),
984 }
985 }
986}
987
988#[derive(Clone, Debug, Error)]
990#[error("query failed ({reason}, HTTP {status}): {message}")]
991pub struct QueryFailure {
992 pub status: u16,
993 pub reason: String,
994 pub message: String,
995 pub body: Value,
996}
997
998#[derive(Clone, Debug, Error)]
1000#[error("protocol rejected ({reason}, HTTP {status}): {message}")]
1001pub struct ProtocolFailure {
1002 pub status: u16,
1003 pub reason: String,
1004 pub message: String,
1005 pub supported_version: Option<String>,
1006 pub requested_version: Option<String>,
1007 pub body: Value,
1008}
1009
1010#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
1011pub struct PayloadEnvelope {
1012 pub codec: String,
1013 pub blob: String,
1014}
1015
1016impl PayloadEnvelope {
1017 pub fn avro<T: Serialize>(value: &T) -> Result<Self> {
1018 encode_payload(value, DEFAULT_CODEC)
1019 }
1020
1021 pub fn json<T: Serialize>(value: &T) -> Result<Self> {
1022 encode_payload(value, JSON_CODEC)
1023 }
1024
1025 pub fn avro_value(value: &AvroValue) -> Result<Self> {
1028 encode_avro_value(value)
1029 }
1030}
1031
1032#[derive(Clone, Debug, PartialEq)]
1034pub enum AvroValue {
1035 Null,
1036 Boolean(bool),
1037 Long(i64),
1038 Double(f64),
1039 Bytes(Vec<u8>),
1040 String(String),
1041 Array(Vec<AvroValue>),
1042 Map(BTreeMap<String, AvroValue>),
1043}
1044
1045impl AvroValue {
1046 fn from_serialize<T: Serialize>(value: &T) -> Result<Self> {
1047 Self::from_serde_value(
1048 serde_value::to_value(value).map_err(|error| {
1049 Error::Codec(format!("could not adapt value for Avro: {error}"))
1050 })?,
1051 )
1052 }
1053
1054 fn from_serde_value(value: serde_value::Value) -> Result<Self> {
1055 use serde_value::Value as SerdeValue;
1056
1057 match value {
1058 SerdeValue::Unit => Ok(Self::Null),
1059 SerdeValue::Bool(value) => Ok(Self::Boolean(value)),
1060 SerdeValue::I8(value) => Ok(Self::Long(i64::from(value))),
1061 SerdeValue::I16(value) => Ok(Self::Long(i64::from(value))),
1062 SerdeValue::I32(value) => Ok(Self::Long(i64::from(value))),
1063 SerdeValue::I64(value) => Ok(Self::Long(value)),
1064 SerdeValue::U8(value) => Ok(Self::Long(i64::from(value))),
1065 SerdeValue::U16(value) => Ok(Self::Long(i64::from(value))),
1066 SerdeValue::U32(value) => Ok(Self::Long(i64::from(value))),
1067 SerdeValue::U64(value) => i64::try_from(value).map(Self::Long).map_err(|_| {
1068 Error::Codec(
1069 "integer_overflow: Avro Value long must be within signed 64-bit range"
1070 .to_string(),
1071 )
1072 }),
1073 SerdeValue::F32(value) => Self::finite_double(f64::from(value)),
1074 SerdeValue::F64(value) => Self::finite_double(value),
1075 SerdeValue::Char(value) => Ok(Self::String(value.to_string())),
1076 SerdeValue::String(value) => Ok(Self::String(value)),
1077 SerdeValue::Bytes(value) => Ok(Self::Bytes(value)),
1078 SerdeValue::Option(None) => Ok(Self::Null),
1079 SerdeValue::Option(Some(value)) | SerdeValue::Newtype(value) => {
1080 Self::from_serde_value(*value)
1081 }
1082 SerdeValue::Seq(values) => values
1083 .into_iter()
1084 .map(Self::from_serde_value)
1085 .collect::<Result<Vec<_>>>()
1086 .map(Self::Array),
1087 SerdeValue::Map(values) => values
1088 .into_iter()
1089 .map(|(key, value)| {
1090 let SerdeValue::String(key) = key else {
1091 return Err(Error::Codec(
1092 "invalid_map_key: Avro Value map keys must be strings".to_string(),
1093 ));
1094 };
1095
1096 Ok((key, Self::from_serde_value(value)?))
1097 })
1098 .collect::<Result<BTreeMap<_, _>>>()
1099 .map(Self::Map),
1100 }
1101 }
1102
1103 fn finite_double(value: f64) -> Result<Self> {
1104 if !value.is_finite() {
1105 return Err(Error::Codec(
1106 "non_finite_float: Avro Value doubles must be finite".to_string(),
1107 ));
1108 }
1109
1110 Ok(Self::Double(value))
1111 }
1112
1113 fn into_json(self) -> Result<Value> {
1114 match self {
1115 Self::Null => Ok(Value::Null),
1116 Self::Boolean(value) => Ok(Value::Bool(value)),
1117 Self::Long(value) => Ok(Value::Number(value.into())),
1118 Self::Double(value) => serde_json::Number::from_f64(value)
1119 .map(Value::Number)
1120 .ok_or_else(|| {
1121 Error::Codec(
1122 "non_finite_float: decoded Avro Value double is not finite".to_string(),
1123 )
1124 }),
1125 Self::Bytes(value) => Ok(json!({
1126 "$type": "bytes",
1127 "base64": BASE64.encode(value),
1128 })),
1129 Self::String(value) => Ok(Value::String(value)),
1130 Self::Array(values) => values
1131 .into_iter()
1132 .map(Self::into_json)
1133 .collect::<Result<Vec<_>>>()
1134 .map(Value::Array),
1135 Self::Map(values) => values
1136 .into_iter()
1137 .map(|(key, value)| Ok((key, value.into_json()?)))
1138 .collect::<Result<serde_json::Map<_, _>>>()
1139 .map(Value::Object),
1140 }
1141 }
1142
1143 fn into_serde_value(self) -> serde_value::Value {
1144 use serde_value::Value as SerdeValue;
1145
1146 match self {
1147 Self::Null => SerdeValue::Unit,
1148 Self::Boolean(value) => SerdeValue::Bool(value),
1149 Self::Long(value) => SerdeValue::I64(value),
1150 Self::Double(value) => SerdeValue::F64(value),
1151 Self::Bytes(value) => SerdeValue::Bytes(value),
1152 Self::String(value) => SerdeValue::String(value),
1153 Self::Array(values) => {
1154 SerdeValue::Seq(values.into_iter().map(Self::into_serde_value).collect())
1155 }
1156 Self::Map(values) => SerdeValue::Map(
1157 values
1158 .into_iter()
1159 .map(|(key, value)| (SerdeValue::String(key), value.into_serde_value()))
1160 .collect(),
1161 ),
1162 }
1163 }
1164
1165 pub fn deserialize<T: DeserializeOwned>(self) -> Result<T> {
1166 self.into_serde_value().deserialize_into().map_err(|error| {
1167 Error::Codec(format!(
1168 "avro_value_type_mismatch: could not adapt decoded value: {error}"
1169 ))
1170 })
1171 }
1172}
1173
1174impl Serialize for AvroValue {
1175 fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
1176 where
1177 S: Serializer,
1178 {
1179 match self {
1180 Self::Null => serializer.serialize_unit(),
1181 Self::Boolean(value) => serializer.serialize_bool(*value),
1182 Self::Long(value) => serializer.serialize_i64(*value),
1183 Self::Double(value) => serializer.serialize_f64(*value),
1184 Self::Bytes(value) => serializer.serialize_bytes(value),
1185 Self::String(value) => serializer.serialize_str(value),
1186 Self::Array(values) => {
1187 let mut sequence = serializer.serialize_seq(Some(values.len()))?;
1188 for value in values {
1189 sequence.serialize_element(value)?;
1190 }
1191 sequence.end()
1192 }
1193 Self::Map(values) => {
1194 let mut map = serializer.serialize_map(Some(values.len()))?;
1195 for (key, value) in values {
1196 map.serialize_entry(key, value)?;
1197 }
1198 map.end()
1199 }
1200 }
1201 }
1202}
1203
1204pub fn encode_avro_value(value: &AvroValue) -> Result<PayloadEnvelope> {
1205 let datum = avro_value_to_datum(value)?;
1206 let datum = to_avro_datum(avro_value_schema()?, datum)
1207 .map_err(|err| Error::Codec(format!("avro_value_encode_failed: {err}")))?;
1208 let mut bytes = Vec::with_capacity(datum.len() + 10);
1209 bytes.extend_from_slice(&AVRO_SINGLE_OBJECT_MAGIC);
1210 bytes.extend_from_slice(&AVRO_VALUE_SCHEMA_FINGERPRINT);
1211 bytes.extend_from_slice(&datum);
1212 Ok(PayloadEnvelope {
1213 codec: DEFAULT_CODEC.to_string(),
1214 blob: BASE64.encode(bytes),
1215 })
1216}
1217
1218pub fn decode_avro_value(envelope: &PayloadEnvelope) -> Result<AvroValue> {
1219 if envelope.codec != DEFAULT_CODEC {
1220 return Err(Error::Codec(format!(
1221 "unsupported payload codec {:?}",
1222 envelope.codec
1223 )));
1224 }
1225 decode_avro_value_blob(&envelope.blob)
1226}
1227
1228pub fn encode_payload<T: Serialize>(value: &T, codec: &str) -> Result<PayloadEnvelope> {
1229 let blob = match codec {
1230 JSON_CODEC => serde_json::to_string(value)?,
1231 DEFAULT_CODEC => encode_avro_value(&AvroValue::from_serialize(value)?)?.blob,
1232 other => return Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1233 };
1234
1235 Ok(PayloadEnvelope {
1236 codec: codec.to_string(),
1237 blob,
1238 })
1239}
1240
1241pub fn decode_payload<T: DeserializeOwned>(envelope: &PayloadEnvelope) -> Result<T> {
1242 match envelope.codec.as_str() {
1243 JSON_CODEC => Ok(serde_json::from_str(&envelope.blob)?),
1244 DEFAULT_CODEC => decode_avro_value(envelope)?.deserialize(),
1245 other => Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1246 }
1247}
1248
1249#[cfg(test)]
1250fn encode_value_envelope(value: &Value, codec: &str) -> Result<Value> {
1251 Ok(serde_json::to_value(encode_payload(value, codec)?)?)
1252}
1253
1254fn decode_wire_value(value: &Value, fallback_codec: &str) -> Result<Value> {
1255 if value.is_null() {
1256 return Ok(Value::Null);
1257 }
1258
1259 if let Some(object) = value.as_object() {
1260 if let (Some(codec), Some(blob)) = (
1261 object.get("codec").and_then(Value::as_str),
1262 object.get("blob").and_then(Value::as_str),
1263 ) {
1264 return decode_blob(blob, codec);
1265 }
1266 }
1267
1268 if let Some(blob) = value.as_str() {
1269 return decode_blob(blob, fallback_codec);
1270 }
1271
1272 Ok(value.clone())
1273}
1274
1275fn encode_typed_envelope(value: &AvroValue, codec: &str) -> Result<Value> {
1276 let envelope = match codec {
1277 DEFAULT_CODEC => encode_avro_value(value)?,
1278 JSON_CODEC => PayloadEnvelope {
1279 codec: JSON_CODEC.to_string(),
1280 blob: serde_json::to_string(&value.clone().into_json()?)?,
1281 },
1282 other => return Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1283 };
1284 Ok(serde_json::to_value(envelope)?)
1285}
1286
1287fn decode_wire_avro_value(value: &Value, fallback_codec: &str) -> Result<AvroValue> {
1288 if value.is_null() {
1289 return Ok(AvroValue::Null);
1290 }
1291
1292 if let Some(object) = value.as_object() {
1293 if let (Some(codec), Some(blob)) = (
1294 object.get("codec").and_then(Value::as_str),
1295 object.get("blob").and_then(Value::as_str),
1296 ) {
1297 return match codec {
1298 DEFAULT_CODEC => decode_avro_value_blob(blob),
1299 JSON_CODEC => AvroValue::from_serialize(&serde_json::from_str::<Value>(blob)?),
1300 other => Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1301 };
1302 }
1303 }
1304
1305 if let Some(blob) = value.as_str() {
1306 return match fallback_codec {
1307 DEFAULT_CODEC => decode_avro_value_blob(blob),
1308 JSON_CODEC => AvroValue::from_serialize(&serde_json::from_str::<Value>(blob)?),
1309 other => Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1310 };
1311 }
1312
1313 AvroValue::from_serialize(value)
1314}
1315
1316fn normalize_avro_arguments(value: AvroValue) -> AvroValue {
1317 match value {
1318 AvroValue::Null => AvroValue::Array(Vec::new()),
1319 AvroValue::Array(_) => value,
1320 other => AvroValue::Array(vec![other]),
1321 }
1322}
1323
1324fn decode_blob(blob: &str, codec: &str) -> Result<Value> {
1325 match codec {
1326 JSON_CODEC => Ok(serde_json::from_str(blob)?),
1327 DEFAULT_CODEC => decode_avro_value_blob(blob)?.into_json(),
1328 other => Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1329 }
1330}
1331
1332fn decode_avro_value_blob(blob: &str) -> Result<AvroValue> {
1333 let bytes = BASE64.decode(blob).map_err(|err| {
1334 Error::Codec(format!(
1335 "invalid_payload_framing: expected strict base64 Avro single-object bytes: {err}"
1336 ))
1337 })?;
1338
1339 if bytes.len() < 10 || bytes[..2] != AVRO_SINGLE_OBJECT_MAGIC {
1340 return Err(Error::Codec(
1341 "invalid_payload_framing: expected Avro single-object magic c301".to_string(),
1342 ));
1343 }
1344
1345 let fingerprint: [u8; 8] = bytes[2..10]
1346 .try_into()
1347 .map_err(|_| Error::Codec("invalid Avro fingerprint length".to_string()))?;
1348 if fingerprint != AVRO_VALUE_SCHEMA_FINGERPRINT {
1349 return Err(Error::Codec(format!(
1350 "unsupported_payload_schema: unknown CRC-64-AVRO fingerprint {}",
1351 fingerprint
1352 .iter()
1353 .map(|byte| format!("{byte:02x}"))
1354 .collect::<String>()
1355 )));
1356 }
1357
1358 let mut datum_reader = StrictAvroDatumReader::new(&bytes[10..]);
1359 let datum = from_avro_datum(avro_value_schema()?, &mut datum_reader, None);
1363 if datum_reader.truncated {
1364 return Err(Error::Codec(
1365 "invalid_payload_framing: truncated Avro Value datum".to_string(),
1366 ));
1367 }
1368 let datum = datum.map_err(|err| {
1369 Error::Codec(format!(
1370 "invalid_payload_framing: malformed Avro Value datum: {err}"
1371 ))
1372 })?;
1373 if datum_reader.remaining() != 0 {
1374 return Err(Error::Codec(format!(
1375 "invalid_payload_framing: {} trailing bytes after Avro Value datum",
1376 datum_reader.remaining()
1377 )));
1378 }
1379 avro_value_from_datum(datum)
1380}
1381
1382struct StrictAvroDatumReader<'a> {
1383 bytes: &'a [u8],
1384 offset: usize,
1385 truncated: bool,
1386}
1387
1388impl<'a> StrictAvroDatumReader<'a> {
1389 fn new(bytes: &'a [u8]) -> Self {
1390 Self {
1391 bytes,
1392 offset: 0,
1393 truncated: false,
1394 }
1395 }
1396
1397 fn remaining(&self) -> usize {
1398 self.bytes.len() - self.offset
1399 }
1400}
1401
1402impl Read for StrictAvroDatumReader<'_> {
1403 fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
1404 let count = buffer.len().min(self.remaining());
1405 buffer[..count].copy_from_slice(&self.bytes[self.offset..self.offset + count]);
1406 self.offset += count;
1407 if count < buffer.len() {
1408 self.truncated = true;
1409 }
1410
1411 Ok(count)
1412 }
1413}
1414
1415fn avro_value_to_datum(value: &AvroValue) -> Result<AvroDatum> {
1416 let branch = match value {
1417 AvroValue::Null => AvroDatum::Union(0, Box::new(AvroDatum::Null)),
1418 AvroValue::Boolean(value) => AvroDatum::Union(
1419 1,
1420 Box::new(AvroDatum::Record(vec![(
1421 "boolean".to_string(),
1422 AvroDatum::Boolean(*value),
1423 )])),
1424 ),
1425 AvroValue::Long(value) => AvroDatum::Union(
1426 2,
1427 Box::new(AvroDatum::Record(vec![(
1428 "long".to_string(),
1429 AvroDatum::Long(*value),
1430 )])),
1431 ),
1432 AvroValue::Double(value) => {
1433 if !value.is_finite() {
1434 return Err(Error::Codec(
1435 "non_finite_float: Avro Value doubles must be finite".to_string(),
1436 ));
1437 }
1438 AvroDatum::Union(
1439 3,
1440 Box::new(AvroDatum::Record(vec![(
1441 "double".to_string(),
1442 AvroDatum::Double(*value),
1443 )])),
1444 )
1445 }
1446 AvroValue::Bytes(value) => AvroDatum::Union(
1447 4,
1448 Box::new(AvroDatum::Record(vec![(
1449 "bytes".to_string(),
1450 AvroDatum::Bytes(value.clone()),
1451 )])),
1452 ),
1453 AvroValue::String(value) => AvroDatum::Union(
1454 5,
1455 Box::new(AvroDatum::Record(vec![(
1456 "string".to_string(),
1457 AvroDatum::String(value.clone()),
1458 )])),
1459 ),
1460 AvroValue::Array(values) => AvroDatum::Union(
1461 6,
1462 Box::new(AvroDatum::Record(vec![(
1463 "items".to_string(),
1464 AvroDatum::Array(
1465 values
1466 .iter()
1467 .map(avro_value_to_datum)
1468 .collect::<Result<Vec<_>>>()?,
1469 ),
1470 )])),
1471 ),
1472 AvroValue::Map(values) => AvroDatum::Union(
1473 7,
1474 Box::new(AvroDatum::Record(vec![(
1475 "entries".to_string(),
1476 AvroDatum::Map(
1477 values
1478 .iter()
1479 .map(|(key, value)| Ok((key.clone(), avro_value_to_datum(value)?)))
1480 .collect::<Result<HashMap<_, _>>>()?,
1481 ),
1482 )])),
1483 ),
1484 };
1485 Ok(AvroDatum::Record(vec![("value".to_string(), branch)]))
1486}
1487
1488fn avro_value_from_datum(datum: AvroDatum) -> Result<AvroValue> {
1489 let AvroDatum::Record(mut outer) = datum else {
1490 return Err(Error::Codec(
1491 "invalid_payload_framing: datum is not a Value record".to_string(),
1492 ));
1493 };
1494 let (_, branch) = outer
1495 .pop()
1496 .filter(|(name, _)| name == "value")
1497 .ok_or_else(|| Error::Codec("invalid_payload_framing: Value field missing".to_string()))?;
1498 let AvroDatum::Union(_, branch) = branch else {
1499 return Err(Error::Codec(
1500 "invalid_payload_framing: invalid Value union".to_string(),
1501 ));
1502 };
1503 match *branch {
1504 AvroDatum::Null => Ok(AvroValue::Null),
1505 AvroDatum::Record(mut fields) => {
1506 let (name, value) = fields.pop().ok_or_else(|| {
1507 Error::Codec("invalid_payload_framing: empty Value branch".to_string())
1508 })?;
1509 match (name.as_str(), value) {
1510 ("boolean", AvroDatum::Boolean(value)) => Ok(AvroValue::Boolean(value)),
1511 ("long", AvroDatum::Long(value)) => Ok(AvroValue::Long(value)),
1512 ("double", AvroDatum::Double(value)) if value.is_finite() => {
1513 Ok(AvroValue::Double(value))
1514 }
1515 ("bytes", AvroDatum::Bytes(value)) => Ok(AvroValue::Bytes(value)),
1516 ("string", AvroDatum::String(value)) => Ok(AvroValue::String(value)),
1517 ("items", AvroDatum::Array(values)) => values
1518 .into_iter()
1519 .map(avro_value_from_datum)
1520 .collect::<Result<Vec<_>>>()
1521 .map(AvroValue::Array),
1522 ("entries", AvroDatum::Map(values)) => values
1523 .into_iter()
1524 .map(|(key, value)| Ok((key, avro_value_from_datum(value)?)))
1525 .collect::<Result<BTreeMap<_, _>>>()
1526 .map(AvroValue::Map),
1527 _ => Err(Error::Codec(
1528 "invalid_payload_framing: unknown Value branch".to_string(),
1529 )),
1530 }
1531 }
1532 _ => Err(Error::Codec(
1533 "invalid_payload_framing: invalid Value branch".to_string(),
1534 )),
1535 }
1536}
1537
1538fn avro_value_schema() -> Result<&'static Schema> {
1539 match AVRO_VALUE_SCHEMA.get_or_init(|| {
1540 Schema::parse_str(AVRO_VALUE_SCHEMA_JSON)
1541 .map_err(|err| format!("could not parse Avro Value schema: {err}"))
1542 }) {
1543 Ok(schema) => Ok(schema),
1544 Err(message) => Err(Error::Codec(message.clone())),
1545 }
1546}
1547
1548#[derive(Clone, Debug)]
1549pub struct Client {
1550 http: reqwest::Client,
1551 base_url: String,
1552 token: Option<String>,
1553 control_token: Option<String>,
1554 worker_token: Option<String>,
1555 namespace: String,
1556}
1557
1558impl Client {
1559 pub fn new(base_url: impl Into<String>) -> Result<Self> {
1560 Self::builder(base_url).build()
1561 }
1562
1563 pub fn builder(base_url: impl Into<String>) -> ClientBuilder {
1564 ClientBuilder {
1565 base_url: base_url.into(),
1566 token: None,
1567 control_token: None,
1568 worker_token: None,
1569 namespace: "default".to_string(),
1570 timeout: Duration::from_secs(60),
1571 }
1572 }
1573
1574 pub async fn health(&self) -> Result<Value> {
1575 self.request_json(
1576 reqwest::Method::GET,
1577 "/health",
1578 RequestProtocol::ControlPlane,
1579 Option::<&Value>::None,
1580 )
1581 .await
1582 }
1583
1584 pub async fn cluster_info(&self) -> Result<Value> {
1585 self.request_json(
1586 reqwest::Method::GET,
1587 "/cluster/info",
1588 RequestProtocol::ControlPlane,
1589 Option::<&Value>::None,
1590 )
1591 .await
1592 }
1593
1594 pub async fn start_workflow<T: Serialize>(
1595 &self,
1596 workflow_type: &str,
1597 task_queue: &str,
1598 workflow_id: &str,
1599 input: T,
1600 ) -> Result<WorkflowHandle> {
1601 self.start_workflow_with_options(
1602 workflow_type,
1603 task_queue,
1604 workflow_id,
1605 WorkflowStartOptions::default(),
1606 input,
1607 )
1608 .await
1609 }
1610
1611 pub async fn start_workflow_with_options<T: Serialize>(
1614 &self,
1615 workflow_type: &str,
1616 task_queue: &str,
1617 workflow_id: &str,
1618 options: WorkflowStartOptions,
1619 input: T,
1620 ) -> Result<WorkflowHandle> {
1621 options.validate()?;
1622 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1623 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
1624 let body = json!({
1625 "workflow_id": workflow_id,
1626 "workflow_type": workflow_type,
1627 "task_queue": task_queue,
1628 "input": input_envelope,
1629 "execution_timeout_seconds": options.execution_timeout_seconds,
1630 "run_timeout_seconds": options.run_timeout_seconds
1631 });
1632
1633 let data: Value = self
1634 .request_json(
1635 reqwest::Method::POST,
1636 "/workflows",
1637 RequestProtocol::ControlPlane,
1638 Some(&body),
1639 )
1640 .await?;
1641
1642 Ok(WorkflowHandle {
1643 client: self.clone(),
1644 workflow_id: data
1645 .get("workflow_id")
1646 .and_then(Value::as_str)
1647 .unwrap_or(workflow_id)
1648 .to_string(),
1649 run_id: data
1650 .get("run_id")
1651 .and_then(Value::as_str)
1652 .map(str::to_string),
1653 workflow_type: data
1654 .get("workflow_type")
1655 .and_then(Value::as_str)
1656 .unwrap_or(workflow_type)
1657 .to_string(),
1658 })
1659 }
1660
1661 pub async fn signal_workflow<T: Serialize>(
1662 &self,
1663 workflow_id: &str,
1664 signal_name: &str,
1665 input: T,
1666 ) -> Result<Value> {
1667 self.signal_workflow_target(workflow_id, None, signal_name, input)
1668 .await
1669 }
1670
1671 pub async fn signal_workflow_run<T: Serialize>(
1673 &self,
1674 workflow_id: &str,
1675 run_id: &str,
1676 signal_name: &str,
1677 input: T,
1678 ) -> Result<Value> {
1679 self.signal_workflow_target(workflow_id, Some(run_id), signal_name, input)
1680 .await
1681 }
1682
1683 async fn signal_workflow_target<T: Serialize>(
1684 &self,
1685 workflow_id: &str,
1686 run_id: Option<&str>,
1687 signal_name: &str,
1688 input: T,
1689 ) -> Result<Value> {
1690 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1691 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
1692 let body = json!({
1693 "input": input_envelope
1694 });
1695 let path = match run_id {
1696 Some(run_id) => {
1697 format!("/workflows/{workflow_id}/runs/{run_id}/signal/{signal_name}")
1698 }
1699 None => format!("/workflows/{workflow_id}/signal/{signal_name}"),
1700 };
1701 self.request_json(
1702 reqwest::Method::POST,
1703 &path,
1704 RequestProtocol::ControlPlane,
1705 Some(&body),
1706 )
1707 .await
1708 }
1709
1710 pub async fn cancel_workflow(
1712 &self,
1713 workflow_id: &str,
1714 options: WorkflowCommandOptions,
1715 ) -> Result<WorkflowCommandResult> {
1716 self.workflow_command(workflow_id, None, WorkflowCommandKind::Cancel, options)
1717 .await
1718 }
1719
1720 pub async fn cancel_workflow_run(
1722 &self,
1723 workflow_id: &str,
1724 run_id: &str,
1725 options: WorkflowCommandOptions,
1726 ) -> Result<WorkflowCommandResult> {
1727 self.workflow_command(
1728 workflow_id,
1729 Some(run_id),
1730 WorkflowCommandKind::Cancel,
1731 options,
1732 )
1733 .await
1734 }
1735
1736 pub async fn terminate_workflow(
1738 &self,
1739 workflow_id: &str,
1740 options: WorkflowCommandOptions,
1741 ) -> Result<WorkflowCommandResult> {
1742 self.workflow_command(workflow_id, None, WorkflowCommandKind::Terminate, options)
1743 .await
1744 }
1745
1746 pub async fn terminate_workflow_run(
1748 &self,
1749 workflow_id: &str,
1750 run_id: &str,
1751 options: WorkflowCommandOptions,
1752 ) -> Result<WorkflowCommandResult> {
1753 self.workflow_command(
1754 workflow_id,
1755 Some(run_id),
1756 WorkflowCommandKind::Terminate,
1757 options,
1758 )
1759 .await
1760 }
1761
1762 async fn workflow_command(
1763 &self,
1764 workflow_id: &str,
1765 run_id: Option<&str>,
1766 command: WorkflowCommandKind,
1767 options: WorkflowCommandOptions,
1768 ) -> Result<WorkflowCommandResult> {
1769 let path = match run_id {
1770 Some(run_id) => format!(
1771 "/workflows/{workflow_id}/runs/{run_id}/{}",
1772 command.as_str()
1773 ),
1774 None => format!("/workflows/{workflow_id}/{}", command.as_str()),
1775 };
1776 let data = match self
1777 .request_json(
1778 reqwest::Method::POST,
1779 &path,
1780 RequestProtocol::ControlPlane,
1781 Some(&options),
1782 )
1783 .await
1784 {
1785 Ok(data) => data,
1786 Err(Error::Http { status, body }) => {
1787 return Err(Error::WorkflowCommandRejected(workflow_command_rejection(
1788 command,
1789 status,
1790 body,
1791 workflow_id,
1792 run_id,
1793 )));
1794 }
1795 Err(error) => return Err(error),
1796 };
1797
1798 Ok(workflow_command_result(command, data, workflow_id, run_id))
1799 }
1800
1801 pub async fn query_workflow<T: Serialize>(
1807 &self,
1808 workflow_id: &str,
1809 query_name: &str,
1810 input: T,
1811 ) -> Result<Value> {
1812 self.query_workflow_target(workflow_id, None, query_name, input)
1813 .await
1814 }
1815
1816 pub async fn query_workflow_run<T: Serialize>(
1818 &self,
1819 workflow_id: &str,
1820 run_id: &str,
1821 query_name: &str,
1822 input: T,
1823 ) -> Result<Value> {
1824 self.query_workflow_target(workflow_id, Some(run_id), query_name, input)
1825 .await
1826 }
1827
1828 pub async fn query_workflow_avro_value<T: Serialize>(
1830 &self,
1831 workflow_id: &str,
1832 query_name: &str,
1833 input: T,
1834 ) -> Result<AvroValue> {
1835 self.query_workflow_avro_value_target(workflow_id, None, query_name, input)
1836 .await
1837 }
1838
1839 pub async fn query_workflow_run_avro_value<T: Serialize>(
1841 &self,
1842 workflow_id: &str,
1843 run_id: &str,
1844 query_name: &str,
1845 input: T,
1846 ) -> Result<AvroValue> {
1847 self.query_workflow_avro_value_target(workflow_id, Some(run_id), query_name, input)
1848 .await
1849 }
1850
1851 async fn query_workflow_avro_value_target<T: Serialize>(
1852 &self,
1853 workflow_id: &str,
1854 run_id: Option<&str>,
1855 query_name: &str,
1856 input: T,
1857 ) -> Result<AvroValue> {
1858 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1859 let body = json!({"input": encode_typed_envelope(&input, DEFAULT_CODEC)?});
1860 let path = match run_id {
1861 Some(run_id) => {
1862 format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
1863 }
1864 None => format!("/workflows/{workflow_id}/query/{query_name}"),
1865 };
1866 let response: Value = match self
1867 .request_json(
1868 reqwest::Method::POST,
1869 &path,
1870 RequestProtocol::ControlPlane,
1871 Some(&body),
1872 )
1873 .await
1874 {
1875 Ok(response) => response,
1876 Err(Error::Http { status, body }) => {
1877 return Err(Error::QueryFailed(query_failure(status, body)));
1878 }
1879 Err(error) => return Err(error),
1880 };
1881
1882 let envelope = response
1883 .get("result_envelope")
1884 .filter(|envelope| !envelope.is_null())
1885 .ok_or_else(|| {
1886 Error::Codec(
1887 "missing_payload_envelope: typed query result requires result_envelope"
1888 .to_string(),
1889 )
1890 })?;
1891 decode_wire_avro_value(envelope, DEFAULT_CODEC)
1892 }
1893
1894 async fn query_workflow_target<T: Serialize>(
1895 &self,
1896 workflow_id: &str,
1897 run_id: Option<&str>,
1898 query_name: &str,
1899 input: T,
1900 ) -> Result<Value> {
1901 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1902 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
1903 let body = json!({
1904 "input": input_envelope
1905 });
1906 let path = match run_id {
1907 Some(run_id) => {
1908 format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
1909 }
1910 None => format!("/workflows/{workflow_id}/query/{query_name}"),
1911 };
1912 let response: Value = match self
1913 .request_json(
1914 reqwest::Method::POST,
1915 &path,
1916 RequestProtocol::ControlPlane,
1917 Some(&body),
1918 )
1919 .await
1920 {
1921 Ok(response) => response,
1922 Err(Error::Http { status, body }) => {
1923 return Err(Error::QueryFailed(query_failure(status, body)));
1924 }
1925 Err(error) => return Err(error),
1926 };
1927
1928 if let Some(envelope) = response
1929 .get("result_envelope")
1930 .filter(|envelope| !envelope.is_null())
1931 {
1932 return decode_wire_value(envelope, DEFAULT_CODEC);
1933 }
1934
1935 Ok(response.get("result").cloned().unwrap_or(Value::Null))
1936 }
1937
1938 pub async fn update_workflow<T: Serialize>(
1940 &self,
1941 workflow_id: &str,
1942 update_name: &str,
1943 input: T,
1944 request_id: Option<&str>,
1945 ) -> Result<Value> {
1946 let response = self
1947 .update_workflow_response(workflow_id, update_name, input, request_id)
1948 .await?;
1949 if let Some(envelope) = response
1950 .get("result_envelope")
1951 .filter(|envelope| !envelope.is_null())
1952 {
1953 return decode_wire_value(envelope, DEFAULT_CODEC);
1954 }
1955 Ok(response.get("result").cloned().unwrap_or(response))
1956 }
1957
1958 pub async fn update_workflow_avro_value<T: Serialize>(
1960 &self,
1961 workflow_id: &str,
1962 update_name: &str,
1963 input: T,
1964 request_id: Option<&str>,
1965 ) -> Result<AvroValue> {
1966 let response = self
1967 .update_workflow_response(workflow_id, update_name, input, request_id)
1968 .await?;
1969 let envelope = response
1970 .get("result_envelope")
1971 .filter(|envelope| !envelope.is_null())
1972 .ok_or_else(|| {
1973 Error::Codec(
1974 "missing_payload_envelope: typed update result requires result_envelope"
1975 .to_string(),
1976 )
1977 })?;
1978 decode_wire_avro_value(envelope, DEFAULT_CODEC)
1979 }
1980
1981 async fn update_workflow_response<T: Serialize>(
1982 &self,
1983 workflow_id: &str,
1984 update_name: &str,
1985 input: T,
1986 request_id: Option<&str>,
1987 ) -> Result<Value> {
1988 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1989 let mut body = json!({
1990 "input": encode_typed_envelope(&input, DEFAULT_CODEC)?,
1991 "wait_for": "completed",
1992 });
1993 if let Some(request_id) = request_id {
1994 body["request_id"] = json!(request_id);
1995 }
1996 self.request_json(
1997 reqwest::Method::POST,
1998 &format!("/workflows/{workflow_id}/update/{update_name}"),
1999 RequestProtocol::ControlPlane,
2000 Some(&body),
2001 )
2002 .await
2003 }
2004
2005 pub async fn describe_workflow(&self, workflow_id: &str) -> Result<WorkflowDescription> {
2006 let path = format!("/workflows/{workflow_id}");
2007 let mut data: WorkflowDescription = self
2008 .request_json(
2009 reqwest::Method::GET,
2010 &path,
2011 RequestProtocol::ControlPlane,
2012 Option::<&Value>::None,
2013 )
2014 .await?;
2015 data.decode_payloads()?;
2016 Ok(data)
2017 }
2018
2019 pub async fn describe_workflow_run(
2021 &self,
2022 workflow_id: &str,
2023 run_id: &str,
2024 ) -> Result<WorkflowDescription> {
2025 let path = format!("/workflows/{workflow_id}/runs/{run_id}");
2026 let mut data: WorkflowDescription = self
2027 .request_json(
2028 reqwest::Method::GET,
2029 &path,
2030 RequestProtocol::ControlPlane,
2031 Option::<&Value>::None,
2032 )
2033 .await?;
2034 data.decode_payloads()?;
2035 Ok(data)
2036 }
2037
2038 pub async fn register_worker(
2039 &self,
2040 worker_id: &str,
2041 task_queue: &str,
2042 supported_workflow_types: Vec<String>,
2043 supported_activity_types: Vec<String>,
2044 max_concurrent_workflow_tasks: usize,
2045 max_concurrent_activity_tasks: usize,
2046 ) -> Result<RegisterWorkerResponse> {
2047 self.register_worker_with_capabilities(
2048 worker_id,
2049 task_queue,
2050 supported_workflow_types,
2051 supported_activity_types,
2052 max_concurrent_workflow_tasks,
2053 max_concurrent_activity_tasks,
2054 Vec::new(),
2055 )
2056 .await
2057 }
2058
2059 pub async fn register_worker_with_capabilities(
2061 &self,
2062 worker_id: &str,
2063 task_queue: &str,
2064 supported_workflow_types: Vec<String>,
2065 supported_activity_types: Vec<String>,
2066 max_concurrent_workflow_tasks: usize,
2067 max_concurrent_activity_tasks: usize,
2068 capabilities: Vec<String>,
2069 ) -> Result<RegisterWorkerResponse> {
2070 self.register_worker_with_command_contracts(
2071 worker_id,
2072 task_queue,
2073 supported_workflow_types,
2074 supported_activity_types,
2075 max_concurrent_workflow_tasks,
2076 max_concurrent_activity_tasks,
2077 capabilities,
2078 Value::Object(serde_json::Map::new()),
2079 )
2080 .await
2081 }
2082
2083 #[allow(clippy::too_many_arguments)]
2085 pub async fn register_worker_with_command_contracts(
2086 &self,
2087 worker_id: &str,
2088 task_queue: &str,
2089 supported_workflow_types: Vec<String>,
2090 supported_activity_types: Vec<String>,
2091 max_concurrent_workflow_tasks: usize,
2092 max_concurrent_activity_tasks: usize,
2093 capabilities: Vec<String>,
2094 workflow_command_contracts: Value,
2095 ) -> Result<RegisterWorkerResponse> {
2096 let mut body = json!({
2097 "worker_id": worker_id,
2098 "task_queue": task_queue,
2099 "runtime": "rust",
2100 "sdk_version": SDK_VERSION,
2101 "supported_workflow_types": supported_workflow_types,
2102 "supported_activity_types": supported_activity_types,
2103 "capabilities": capabilities,
2104 "max_concurrent_workflow_tasks": max_concurrent_workflow_tasks,
2105 "max_concurrent_activity_tasks": max_concurrent_activity_tasks
2106 });
2107 if workflow_command_contracts
2108 .as_object()
2109 .is_some_and(|contracts| !contracts.is_empty())
2110 {
2111 body["workflow_command_contracts"] = workflow_command_contracts;
2112 }
2113
2114 self.request_json(
2115 reqwest::Method::POST,
2116 "/worker/register",
2117 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2118 Some(&body),
2119 )
2120 .await
2121 }
2122
2123 pub async fn deregister_worker_registration(
2129 &self,
2130 worker_id: &str,
2131 ) -> Result<WorkerDeregistrationEnvelope> {
2132 let path = format!(
2133 "/worker/registrations/{}",
2134 percent_encode_path_segment(worker_id)
2135 );
2136 self.request_json(
2137 reqwest::Method::DELETE,
2138 &path,
2139 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2140 Option::<&Value>::None,
2141 )
2142 .await
2143 }
2144
2145 pub async fn poll_query_task(
2147 &self,
2148 worker_id: &str,
2149 task_queue: &str,
2150 timeout: Duration,
2151 ) -> Result<Option<QueryTask>> {
2152 Ok(self
2153 .poll_query_task_response(worker_id, task_queue, timeout)
2154 .await?
2155 .task)
2156 }
2157
2158 pub async fn poll_query_task_response(
2160 &self,
2161 worker_id: &str,
2162 task_queue: &str,
2163 timeout: Duration,
2164 ) -> Result<PollQueryTaskResponse> {
2165 let poll_request_id = unique_request_id("rust-query-poll");
2166 self.poll_query_task_response_with_request_id(
2167 worker_id,
2168 task_queue,
2169 timeout,
2170 &poll_request_id,
2171 1,
2172 )
2173 .await
2174 }
2175
2176 async fn poll_query_task_response_with_request_id(
2177 &self,
2178 worker_id: &str,
2179 task_queue: &str,
2180 timeout: Duration,
2181 poll_request_id: &str,
2182 transport_retries: usize,
2183 ) -> Result<PollQueryTaskResponse> {
2184 let timeout_seconds = long_poll_timeout_seconds(timeout);
2185 let body = json!({
2186 "worker_id": worker_id,
2187 "task_queue": task_queue,
2188 "poll_request_id": poll_request_id,
2189 "timeout_seconds": timeout_seconds,
2190 });
2191 self.poll_request_json(
2192 "/worker/query-tasks/poll",
2193 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
2194 &body,
2195 timeout + Duration::from_secs(5),
2196 transport_retries,
2197 )
2198 .await
2199 }
2200
2201 pub async fn complete_query_task<T: Serialize>(
2203 &self,
2204 query_task_id: &str,
2205 lease_owner: &str,
2206 query_task_attempt: u64,
2207 result: T,
2208 codec: &str,
2209 ) -> Result<Value> {
2210 let typed_result = AvroValue::from_serialize(&result)?;
2211 let result_envelope = encode_typed_envelope(&typed_result, codec)?;
2212 self.complete_query_task_with_envelope(
2213 query_task_id,
2214 lease_owner,
2215 query_task_attempt,
2216 typed_result.into_json()?,
2217 result_envelope,
2218 )
2219 .await
2220 }
2221
2222 async fn complete_query_task_with_envelope(
2223 &self,
2224 query_task_id: &str,
2225 lease_owner: &str,
2226 query_task_attempt: u64,
2227 result: Value,
2228 result_envelope: Value,
2229 ) -> Result<Value> {
2230 let body = json!({
2231 "lease_owner": lease_owner,
2232 "query_task_attempt": query_task_attempt,
2233 "result": result,
2234 "result_envelope": result_envelope,
2235 });
2236 let path = format!("/worker/query-tasks/{query_task_id}/complete");
2237 let response = self
2238 .request_json(
2239 reqwest::Method::POST,
2240 &path,
2241 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
2242 Some(&body),
2243 )
2244 .await;
2245 query_task_response(response)
2246 }
2247
2248 pub async fn fail_query_task(
2250 &self,
2251 query_task_id: &str,
2252 lease_owner: &str,
2253 query_task_attempt: u64,
2254 message: impl Into<String>,
2255 reason: impl Into<String>,
2256 failure_type: impl Into<String>,
2257 ) -> Result<Value> {
2258 let body = json!({
2259 "lease_owner": lease_owner,
2260 "query_task_attempt": query_task_attempt,
2261 "failure": {
2262 "message": message.into(),
2263 "reason": reason.into(),
2264 "type": failure_type.into(),
2265 }
2266 });
2267 let path = format!("/worker/query-tasks/{query_task_id}/fail");
2268 let response = self
2269 .request_json(
2270 reqwest::Method::POST,
2271 &path,
2272 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
2273 Some(&body),
2274 )
2275 .await;
2276 query_task_response(response)
2277 }
2278
2279 pub async fn heartbeat_worker(
2280 &self,
2281 worker_id: &str,
2282 workflow_available: usize,
2283 activity_available: usize,
2284 ) -> Result<Value> {
2285 let body = json!({
2286 "worker_id": worker_id,
2287 "task_slots": {
2288 "workflow_available": workflow_available,
2289 "activity_available": activity_available
2290 },
2291 "process_metrics": {
2292 "process_id": std::process::id(),
2293 "process_uptime_seconds": 0
2294 }
2295 });
2296
2297 self.request_json(
2298 reqwest::Method::POST,
2299 "/worker/heartbeat",
2300 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2301 Some(&body),
2302 )
2303 .await
2304 }
2305
2306 pub async fn poll_workflow_task(
2307 &self,
2308 worker_id: &str,
2309 task_queue: &str,
2310 timeout: Duration,
2311 ) -> Result<Option<WorkflowTask>> {
2312 Ok(self
2313 .poll_workflow_task_response(worker_id, task_queue, timeout)
2314 .await?
2315 .task)
2316 }
2317
2318 pub async fn poll_workflow_task_response(
2319 &self,
2320 worker_id: &str,
2321 task_queue: &str,
2322 timeout: Duration,
2323 ) -> Result<PollWorkflowTaskResponse> {
2324 let poll_request_id = unique_request_id("rust-workflow-poll");
2325 self.poll_workflow_task_response_with_request_id(
2326 worker_id,
2327 task_queue,
2328 timeout,
2329 &poll_request_id,
2330 1,
2331 )
2332 .await
2333 }
2334
2335 async fn poll_workflow_task_response_with_request_id(
2336 &self,
2337 worker_id: &str,
2338 task_queue: &str,
2339 timeout: Duration,
2340 poll_request_id: &str,
2341 transport_retries: usize,
2342 ) -> Result<PollWorkflowTaskResponse> {
2343 let body = json!({
2344 "worker_id": worker_id,
2345 "task_queue": task_queue,
2346 "poll_request_id": poll_request_id,
2347 "timeout_seconds": long_poll_timeout_seconds(timeout),
2348 });
2349 let mut data: PollWorkflowTaskResponse = self
2350 .poll_request_json(
2351 "/worker/workflow-tasks/poll",
2352 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2353 &body,
2354 timeout + Duration::from_secs(5),
2355 transport_retries,
2356 )
2357 .await?;
2358
2359 if let Some(task) = data.task.as_mut() {
2360 self.fetch_remaining_workflow_history(worker_id, task)
2361 .await?;
2362 }
2363
2364 Ok(data)
2365 }
2366
2367 async fn fetch_remaining_workflow_history(
2368 &self,
2369 worker_id: &str,
2370 task: &mut WorkflowTask,
2371 ) -> Result<()> {
2372 let mut next_token = task.next_history_page_token.clone();
2373
2374 while let Some(token) = next_token.take().filter(|token| !token.is_empty()) {
2375 let lease_owner = task
2376 .lease_owner
2377 .clone()
2378 .unwrap_or_else(|| worker_id.to_string());
2379 let page = self
2380 .workflow_task_history_page(
2381 &task.task_id,
2382 &lease_owner,
2383 task.workflow_task_attempt,
2384 &token,
2385 )
2386 .await?;
2387
2388 task.append_history_page(page);
2389
2390 if task.next_history_page_token.as_deref() == Some(token.as_str()) {
2391 return Err(Error::Codec(
2392 "workflow history pagination returned the same page token".to_string(),
2393 ));
2394 }
2395
2396 next_token = task.next_history_page_token.clone();
2397 }
2398
2399 Ok(())
2400 }
2401
2402 async fn workflow_task_history_page(
2403 &self,
2404 task_id: &str,
2405 lease_owner: &str,
2406 workflow_task_attempt: u64,
2407 next_history_page_token: &str,
2408 ) -> Result<WorkflowTaskHistoryPage> {
2409 let body = json!({
2410 "lease_owner": lease_owner,
2411 "workflow_task_attempt": workflow_task_attempt,
2412 "next_history_page_token": next_history_page_token
2413 });
2414 let path = format!("/worker/workflow-tasks/{task_id}/history");
2415
2416 self.request_json(
2417 reqwest::Method::POST,
2418 &path,
2419 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2420 Some(&body),
2421 )
2422 .await
2423 }
2424
2425 pub async fn complete_workflow_task(
2426 &self,
2427 task_id: &str,
2428 lease_owner: &str,
2429 workflow_task_attempt: u64,
2430 commands: Vec<Value>,
2431 ) -> Result<Value> {
2432 let body = json!({
2433 "lease_owner": lease_owner,
2434 "workflow_task_attempt": workflow_task_attempt,
2435 "commands": commands
2436 });
2437 let path = format!("/worker/workflow-tasks/{task_id}/complete");
2438 self.request_json(
2439 reqwest::Method::POST,
2440 &path,
2441 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2442 Some(&body),
2443 )
2444 .await
2445 }
2446
2447 pub async fn fail_workflow_task(
2448 &self,
2449 task_id: &str,
2450 lease_owner: &str,
2451 workflow_task_attempt: u64,
2452 message: impl Into<String>,
2453 ) -> Result<Value> {
2454 self.fail_workflow_task_with_type(
2455 task_id,
2456 lease_owner,
2457 workflow_task_attempt,
2458 message,
2459 "RustWorkflowTaskFailure",
2460 )
2461 .await
2462 }
2463
2464 async fn fail_workflow_task_with_type(
2465 &self,
2466 task_id: &str,
2467 lease_owner: &str,
2468 workflow_task_attempt: u64,
2469 message: impl Into<String>,
2470 failure_type: &str,
2471 ) -> Result<Value> {
2472 let body = json!({
2473 "lease_owner": lease_owner,
2474 "workflow_task_attempt": workflow_task_attempt,
2475 "failure": {
2476 "message": message.into(),
2477 "type": failure_type
2478 }
2479 });
2480 let path = format!("/worker/workflow-tasks/{task_id}/fail");
2481 self.request_json(
2482 reqwest::Method::POST,
2483 &path,
2484 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2485 Some(&body),
2486 )
2487 .await
2488 }
2489
2490 pub async fn poll_activity_task(
2491 &self,
2492 worker_id: &str,
2493 task_queue: &str,
2494 timeout: Duration,
2495 ) -> Result<Option<ActivityTask>> {
2496 Ok(self
2497 .poll_activity_task_response(worker_id, task_queue, timeout)
2498 .await?
2499 .task)
2500 }
2501
2502 pub async fn poll_activity_task_response(
2504 &self,
2505 worker_id: &str,
2506 task_queue: &str,
2507 timeout: Duration,
2508 ) -> Result<PollActivityTaskResponse> {
2509 let poll_request_id = unique_request_id("rust-activity-poll");
2510 self.poll_activity_task_response_with_request_id(
2511 worker_id,
2512 task_queue,
2513 timeout,
2514 &poll_request_id,
2515 1,
2516 )
2517 .await
2518 }
2519
2520 async fn poll_activity_task_response_with_request_id(
2521 &self,
2522 worker_id: &str,
2523 task_queue: &str,
2524 timeout: Duration,
2525 poll_request_id: &str,
2526 transport_retries: usize,
2527 ) -> Result<PollActivityTaskResponse> {
2528 let body = json!({
2529 "worker_id": worker_id,
2530 "task_queue": task_queue,
2531 "poll_request_id": poll_request_id,
2532 "timeout_seconds": long_poll_timeout_seconds(timeout),
2533 });
2534 let data: PollActivityTaskResponse = self
2535 .poll_request_json(
2536 "/worker/activity-tasks/poll",
2537 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2538 &body,
2539 timeout + Duration::from_secs(5),
2540 transport_retries,
2541 )
2542 .await?;
2543 Ok(data)
2544 }
2545
2546 pub async fn complete_activity_task<T: Serialize>(
2547 &self,
2548 task_id: &str,
2549 activity_attempt_id: &str,
2550 lease_owner: &str,
2551 result: T,
2552 codec: &str,
2553 ) -> Result<Value> {
2554 let result = encode_typed_envelope(&AvroValue::from_serialize(&result)?, codec)?;
2555 let body = json!({
2556 "activity_attempt_id": activity_attempt_id,
2557 "lease_owner": lease_owner,
2558 "result": result
2559 });
2560 let path = format!("/worker/activity-tasks/{task_id}/complete");
2561 activity_task_response(
2562 self.request_json(
2563 reqwest::Method::POST,
2564 &path,
2565 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2566 Some(&body),
2567 )
2568 .await,
2569 "complete",
2570 task_id,
2571 activity_attempt_id,
2572 )
2573 }
2574
2575 pub async fn fail_activity_task(
2576 &self,
2577 task_id: &str,
2578 activity_attempt_id: &str,
2579 lease_owner: &str,
2580 message: impl Into<String>,
2581 non_retryable: bool,
2582 ) -> Result<Value> {
2583 let body = json!({
2584 "activity_attempt_id": activity_attempt_id,
2585 "lease_owner": lease_owner,
2586 "failure": {
2587 "message": message.into(),
2588 "type": "RustActivityFailure",
2589 "non_retryable": non_retryable
2590 }
2591 });
2592 let path = format!("/worker/activity-tasks/{task_id}/fail");
2593 activity_task_response(
2594 self.request_json(
2595 reqwest::Method::POST,
2596 &path,
2597 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2598 Some(&body),
2599 )
2600 .await,
2601 "fail",
2602 task_id,
2603 activity_attempt_id,
2604 )
2605 }
2606
2607 pub async fn heartbeat_activity_task<T: Serialize>(
2608 &self,
2609 task_id: &str,
2610 activity_attempt_id: &str,
2611 lease_owner: &str,
2612 details: T,
2613 ) -> Result<ActivityHeartbeatResponse> {
2614 let details = encode_typed_envelope(&AvroValue::from_serialize(&details)?, DEFAULT_CODEC)?;
2615 let body = json!({
2616 "activity_attempt_id": activity_attempt_id,
2617 "lease_owner": lease_owner,
2618 "details": details
2619 });
2620 let path = format!("/worker/activity-tasks/{task_id}/heartbeat");
2621 activity_task_response(
2622 self.request_json(
2623 reqwest::Method::POST,
2624 &path,
2625 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2626 Some(&body),
2627 )
2628 .await,
2629 "heartbeat",
2630 task_id,
2631 activity_attempt_id,
2632 )
2633 }
2634
2635 async fn request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
2636 &self,
2637 method: reqwest::Method,
2638 path: &str,
2639 protocol: RequestProtocol,
2640 body: Option<&B>,
2641 ) -> Result<T> {
2642 self.request_json_with_timeout(method, path, protocol, body, Duration::from_secs(60))
2643 .await
2644 }
2645
2646 async fn request_json_with_timeout<T: DeserializeOwned, B: Serialize + ?Sized>(
2647 &self,
2648 method: reqwest::Method,
2649 path: &str,
2650 protocol: RequestProtocol,
2651 body: Option<&B>,
2652 timeout: Duration,
2653 ) -> Result<T> {
2654 let auth_token = self.auth_token(protocol)?;
2655 let mut request = self
2656 .http
2657 .request(method, format!("{}/api{}", self.base_url, path))
2658 .timeout(timeout)
2659 .header(reqwest::header::ACCEPT, "application/json")
2660 .header(reqwest::header::CONTENT_TYPE, "application/json")
2661 .header("X-Namespace", &self.namespace);
2662
2663 match protocol {
2664 RequestProtocol::Worker(version) => {
2665 request = request.header("X-Durable-Workflow-Protocol-Version", version);
2666 }
2667 RequestProtocol::ControlPlane => {
2668 request = request.header(
2669 "X-Durable-Workflow-Control-Plane-Version",
2670 CONTROL_PLANE_VERSION,
2671 );
2672 }
2673 }
2674
2675 if let Some(token) = auth_token {
2676 request = request.bearer_auth(token);
2677 }
2678
2679 if let Some(body) = body {
2680 request = request.json(body);
2681 }
2682
2683 let response = request.send().await?;
2684 let status = response.status();
2685 let bytes = response.bytes().await?;
2686
2687 if !status.is_success() {
2688 let body = String::from_utf8_lossy(&bytes).to_string();
2689 if let Some(protocol) = protocol_failure(status, &body) {
2690 return Err(Error::Protocol(protocol));
2691 }
2692 return Err(Error::Http { status, body });
2693 }
2694
2695 if bytes.is_empty() {
2696 return Ok(serde_json::from_value(Value::Null)?);
2697 }
2698
2699 Ok(serde_json::from_slice(&bytes)?)
2700 }
2701
2702 async fn poll_request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
2703 &self,
2704 path: &str,
2705 protocol: RequestProtocol,
2706 body: &B,
2707 timeout: Duration,
2708 max_retries: usize,
2709 ) -> Result<T> {
2710 let mut retries = 0;
2711
2712 loop {
2713 let response = self
2714 .request_json_with_timeout(
2715 reqwest::Method::POST,
2716 path,
2717 protocol,
2718 Some(body),
2719 timeout,
2720 )
2721 .await;
2722
2723 match response {
2724 Err(Error::Transport(_)) if retries < max_retries => retries += 1,
2725 response => return worker_poll_response(response),
2726 }
2727 }
2728 }
2729
2730 fn auth_token(&self, protocol: RequestProtocol) -> Result<Option<&str>> {
2731 match protocol {
2732 RequestProtocol::Worker(_) => {
2733 if let Some(token) = self.worker_token.as_deref().or(self.token.as_deref()) {
2734 return Ok(Some(token));
2735 }
2736 if self.control_token.is_some() {
2737 return Err(Error::MissingRoleCredentials {
2738 role: "worker",
2739 opposite_role: "control",
2740 });
2741 }
2742 Ok(None)
2743 }
2744 RequestProtocol::ControlPlane => {
2745 if let Some(token) = self.control_token.as_deref().or(self.token.as_deref()) {
2746 return Ok(Some(token));
2747 }
2748 if self.worker_token.is_some() {
2749 return Err(Error::MissingRoleCredentials {
2750 role: "control",
2751 opposite_role: "worker",
2752 });
2753 }
2754 Ok(None)
2755 }
2756 }
2757 }
2758}
2759
2760fn query_failure(status: reqwest::StatusCode, raw_body: String) -> QueryFailure {
2761 let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
2762 let reason = body
2763 .get("reason")
2764 .and_then(Value::as_str)
2765 .unwrap_or("query_rejected")
2766 .to_string();
2767 let message = body
2768 .get("message")
2769 .or_else(|| body.get("error"))
2770 .and_then(Value::as_str)
2771 .unwrap_or("workflow query was rejected")
2772 .to_string();
2773
2774 QueryFailure {
2775 status: status.as_u16(),
2776 reason,
2777 message,
2778 body,
2779 }
2780}
2781
2782fn workflow_command_result(
2783 command: WorkflowCommandKind,
2784 data: Value,
2785 workflow_id: &str,
2786 run_id: Option<&str>,
2787) -> WorkflowCommandResult {
2788 WorkflowCommandResult {
2789 command,
2790 workflow_id: data
2791 .get("workflow_id")
2792 .and_then(Value::as_str)
2793 .unwrap_or(workflow_id)
2794 .to_string(),
2795 run_id: data
2796 .get("run_id")
2797 .and_then(Value::as_str)
2798 .or(run_id)
2799 .map(str::to_string),
2800 outcome: data
2801 .get("outcome")
2802 .and_then(Value::as_str)
2803 .map(str::to_string),
2804 reason: data
2805 .get("reason")
2806 .and_then(Value::as_str)
2807 .map(str::to_string),
2808 command_status: data
2809 .get("command_status")
2810 .and_then(Value::as_str)
2811 .map(str::to_string),
2812 raw: data,
2813 }
2814}
2815
2816fn workflow_command_rejection(
2817 command: WorkflowCommandKind,
2818 status: reqwest::StatusCode,
2819 raw_body: String,
2820 workflow_id: &str,
2821 run_id: Option<&str>,
2822) -> WorkflowCommandRejection {
2823 let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
2824 WorkflowCommandRejection {
2825 command,
2826 status: status.as_u16(),
2827 reason: body
2828 .get("reason")
2829 .and_then(Value::as_str)
2830 .unwrap_or("workflow_command_rejected")
2831 .to_string(),
2832 message: body
2833 .get("message")
2834 .or_else(|| body.get("error"))
2835 .and_then(Value::as_str)
2836 .unwrap_or("workflow lifecycle command was rejected")
2837 .to_string(),
2838 workflow_id: body
2839 .get("workflow_id")
2840 .and_then(Value::as_str)
2841 .unwrap_or(workflow_id)
2842 .to_string(),
2843 run_id: body
2844 .get("run_id")
2845 .and_then(Value::as_str)
2846 .or(run_id)
2847 .map(str::to_string),
2848 target_scope: body
2849 .get("target_scope")
2850 .and_then(Value::as_str)
2851 .map(str::to_string),
2852 body,
2853 }
2854}
2855
2856fn query_task_response(response: Result<Value>) -> Result<Value> {
2857 match response {
2858 Err(Error::Http { status, body }) => Err(Error::QueryFailed(query_failure(status, body))),
2859 response => response,
2860 }
2861}
2862
2863fn worker_poll_response<T: DeserializeOwned>(response: Result<T>) -> Result<T> {
2864 match response {
2865 Err(Error::Http { status, body })
2866 if status == reqwest::StatusCode::CONFLICT && worker_poll_body_is_stop(&body) =>
2867 {
2868 Ok(serde_json::from_str(&body)?)
2869 }
2870 response => response,
2871 }
2872}
2873
2874fn worker_poll_body_is_stop(body: &str) -> bool {
2875 serde_json::from_str::<Value>(body)
2876 .ok()
2877 .is_some_and(|body| {
2878 worker_poll_is_stop(
2879 body.get("poll_status").and_then(Value::as_str),
2880 body.get("reason").and_then(Value::as_str),
2881 )
2882 })
2883}
2884
2885fn worker_poll_is_stop(poll_status: Option<&str>, reason: Option<&str>) -> bool {
2886 matches!(poll_status, Some("draining" | "stopped"))
2887 || matches!(reason, Some("worker_draining" | "worker_stopped"))
2888}
2889
2890fn query_task_rejection_is_final(error: &Error) -> bool {
2891 matches!(
2892 error,
2893 Error::QueryFailed(failure)
2894 if QUERY_TASK_FINAL_REJECTION_REASONS.contains(&failure.reason.as_str())
2895 )
2896}
2897
2898fn activity_task_response<T>(
2899 response: Result<T>,
2900 operation: &str,
2901 task_id: &str,
2902 activity_attempt_id: &str,
2903) -> Result<T> {
2904 match response {
2905 Err(Error::Http { status, body }) => {
2906 let body = serde_json::from_str(&body).unwrap_or_else(|_| json!({"message": body}));
2907 Err(Error::ActivityTaskRejected(ActivityTaskRejection {
2908 operation: operation.to_string(),
2909 status: status.as_u16(),
2910 reason: body
2911 .get("reason")
2912 .and_then(Value::as_str)
2913 .unwrap_or("activity_task_rejected")
2914 .to_string(),
2915 task_id: body
2916 .get("task_id")
2917 .and_then(Value::as_str)
2918 .unwrap_or(task_id)
2919 .to_string(),
2920 activity_attempt_id: body
2921 .get("activity_attempt_id")
2922 .and_then(Value::as_str)
2923 .unwrap_or(activity_attempt_id)
2924 .to_string(),
2925 cancel_requested: body
2926 .get("cancel_requested")
2927 .and_then(Value::as_bool)
2928 .unwrap_or(false),
2929 can_continue: body.get("can_continue").and_then(Value::as_bool),
2930 run_closed_reason: body
2931 .get("run_closed_reason")
2932 .and_then(Value::as_str)
2933 .map(str::to_string),
2934 body,
2935 }))
2936 }
2937 response => response,
2938 }
2939}
2940
2941fn activity_task_rejection_is_final(error: &Error) -> bool {
2942 matches!(
2943 error,
2944 Error::ActivityTaskRejected(rejection)
2945 if matches!(
2946 rejection.reason.as_str(),
2947 "run_cancelled"
2948 | "run_terminated"
2949 | "attempt_closed"
2950 | "stale_attempt"
2951 | "activity_cancelled"
2952 | "task_cancelled"
2953 | "run_closed"
2954 | "activity_not_running"
2955 | "attempt_not_found"
2956 )
2957 )
2958}
2959
2960fn workflow_task_completion_is_terminal_timeout(
2961 error: &Error,
2962 task_id: &str,
2963 workflow_task_attempt: u64,
2964 run_id: Option<&str>,
2965) -> bool {
2966 let Error::Http { status, body } = error else {
2967 return false;
2968 };
2969 if *status != reqwest::StatusCode::CONFLICT {
2970 return false;
2971 }
2972
2973 let Some(run_id) = run_id else {
2974 return false;
2975 };
2976 let Ok(body) = serde_json::from_str::<Value>(body) else {
2977 return false;
2978 };
2979
2980 body.get("recorded").and_then(Value::as_bool) == Some(false)
2981 && body.get("reason").and_then(Value::as_str) == Some("run_timed_out")
2982 && body.get("run_status").and_then(Value::as_str) == Some("failed")
2983 && body.get("run_id").and_then(Value::as_str) == Some(run_id)
2984 && body.get("task_id").and_then(Value::as_str) == Some(task_id)
2985 && body.get("workflow_task_attempt").and_then(Value::as_u64) == Some(workflow_task_attempt)
2986}
2987
2988fn protocol_failure(status: reqwest::StatusCode, raw_body: &str) -> Option<ProtocolFailure> {
2989 let body: Value = serde_json::from_str(raw_body).ok()?;
2990 let reason = body.get("reason")?.as_str()?;
2991 if !matches!(
2992 reason,
2993 "missing_protocol_version"
2994 | "unsupported_protocol_version"
2995 | "missing_control_plane_version"
2996 | "unsupported_control_plane_version"
2997 ) {
2998 return None;
2999 }
3000
3001 Some(ProtocolFailure {
3002 status: status.as_u16(),
3003 reason: reason.to_string(),
3004 message: body
3005 .get("message")
3006 .or_else(|| body.get("error"))
3007 .and_then(Value::as_str)
3008 .unwrap_or("protocol version rejected")
3009 .to_string(),
3010 supported_version: body
3011 .get("supported_version")
3012 .and_then(Value::as_str)
3013 .map(str::to_string),
3014 requested_version: body
3015 .get("requested_version")
3016 .and_then(Value::as_str)
3017 .map(str::to_string),
3018 body,
3019 })
3020}
3021
3022fn long_poll_timeout_seconds(timeout: Duration) -> u64 {
3023 timeout
3024 .as_secs()
3025 .saturating_add(u64::from(timeout.subsec_nanos() > 0))
3026 .min(MAX_LONG_POLL_TIMEOUT_SECONDS)
3027}
3028
3029fn worker_operation_is_retryable(error: &Error) -> bool {
3030 match error {
3031 Error::Transport(error) => {
3032 error.is_timeout() || error.is_connect() || error.is_request() || error.is_body()
3033 }
3034 Error::Http { status, .. } => {
3035 matches!(
3036 *status,
3037 reqwest::StatusCode::REQUEST_TIMEOUT | reqwest::StatusCode::TOO_MANY_REQUESTS
3038 ) || status.is_server_error()
3039 }
3040 _ => false,
3041 }
3042}
3043
3044fn worker_retry_delay(policy: WorkerRetryPolicy, retry: usize) -> Duration {
3045 let exponent = retry.saturating_sub(1).min(31) as u32;
3046 policy
3047 .initial_backoff
3048 .saturating_mul(1_u32 << exponent)
3049 .min(policy.max_backoff)
3050}
3051
3052#[derive(Debug)]
3053pub struct ClientBuilder {
3054 base_url: String,
3055 token: Option<String>,
3056 control_token: Option<String>,
3057 worker_token: Option<String>,
3058 namespace: String,
3059 timeout: Duration,
3060}
3061
3062impl ClientBuilder {
3063 pub fn token(mut self, token: Option<String>) -> Self {
3064 self.token = token;
3065 self
3066 }
3067
3068 pub fn control_token(mut self, token: Option<String>) -> Self {
3069 self.control_token = token;
3070 self
3071 }
3072
3073 pub fn worker_token(mut self, token: Option<String>) -> Self {
3074 self.worker_token = token;
3075 self
3076 }
3077
3078 pub fn namespace(mut self, namespace: impl Into<String>) -> Self {
3079 self.namespace = namespace.into();
3080 self
3081 }
3082
3083 pub fn timeout(mut self, timeout: Duration) -> Self {
3084 self.timeout = timeout;
3085 self
3086 }
3087
3088 pub fn build(self) -> Result<Client> {
3089 let base_url = self.base_url.trim_end_matches('/').to_string();
3090 let has_sdk_api_suffix = reqwest::Url::parse(&base_url)
3091 .map(|url| url.path().trim_end_matches('/').ends_with("/api"))
3092 .unwrap_or_else(|_| base_url.ends_with("/api"));
3093
3094 if has_sdk_api_suffix {
3095 return Err(Error::InvalidBaseUrl);
3096 }
3097
3098 Ok(Client {
3099 http: reqwest::Client::builder().timeout(self.timeout).build()?,
3100 base_url,
3101 token: self.token,
3102 control_token: self.control_token,
3103 worker_token: self.worker_token,
3104 namespace: self.namespace,
3105 })
3106 }
3107}
3108
3109#[derive(Clone, Debug)]
3110pub struct WorkflowHandle {
3111 client: Client,
3112 pub workflow_id: String,
3113 pub run_id: Option<String>,
3114 pub workflow_type: String,
3115}
3116
3117impl WorkflowHandle {
3118 pub async fn describe(&self) -> Result<WorkflowDescription> {
3120 self.client.describe_workflow(&self.workflow_id).await
3121 }
3122
3123 pub async fn describe_selected_run(&self) -> Result<WorkflowDescription> {
3125 let run_id = self.run_id.as_deref().ok_or_else(|| {
3126 Error::Codec("run_id is required for selected-run description".to_string())
3127 })?;
3128 self.client
3129 .describe_workflow_run(&self.workflow_id, run_id)
3130 .await
3131 }
3132
3133 pub async fn signal<T: Serialize>(&self, signal_name: &str, input: T) -> Result<Value> {
3134 self.client
3135 .signal_workflow(&self.workflow_id, signal_name, input)
3136 .await
3137 }
3138
3139 pub async fn signal_selected_run<T: Serialize>(
3141 &self,
3142 signal_name: &str,
3143 input: T,
3144 ) -> Result<Value> {
3145 let run_id = self.run_id.as_deref().ok_or_else(|| {
3146 Error::Codec("run_id is required for selected-run signaling".to_string())
3147 })?;
3148 self.client
3149 .signal_workflow_run(&self.workflow_id, run_id, signal_name, input)
3150 .await
3151 }
3152
3153 pub async fn cancel(&self, options: WorkflowCommandOptions) -> Result<WorkflowCommandResult> {
3155 self.client
3156 .cancel_workflow(&self.workflow_id, options)
3157 .await
3158 }
3159
3160 pub async fn cancel_selected_run(
3162 &self,
3163 options: WorkflowCommandOptions,
3164 ) -> Result<WorkflowCommandResult> {
3165 let run_id = self.run_id.as_deref().ok_or_else(|| {
3166 Error::Codec("run_id is required for selected-run cancellation".to_string())
3167 })?;
3168 self.client
3169 .cancel_workflow_run(&self.workflow_id, run_id, options)
3170 .await
3171 }
3172
3173 pub async fn terminate(
3175 &self,
3176 options: WorkflowCommandOptions,
3177 ) -> Result<WorkflowCommandResult> {
3178 self.client
3179 .terminate_workflow(&self.workflow_id, options)
3180 .await
3181 }
3182
3183 pub async fn terminate_selected_run(
3185 &self,
3186 options: WorkflowCommandOptions,
3187 ) -> Result<WorkflowCommandResult> {
3188 let run_id = self.run_id.as_deref().ok_or_else(|| {
3189 Error::Codec("run_id is required for selected-run termination".to_string())
3190 })?;
3191 self.client
3192 .terminate_workflow_run(&self.workflow_id, run_id, options)
3193 .await
3194 }
3195
3196 pub async fn query<T: Serialize>(&self, query_name: &str, input: T) -> Result<Value> {
3198 self.client
3199 .query_workflow(&self.workflow_id, query_name, input)
3200 .await
3201 }
3202
3203 pub async fn query_avro_value<T: Serialize>(
3204 &self,
3205 query_name: &str,
3206 input: T,
3207 ) -> Result<AvroValue> {
3208 self.client
3209 .query_workflow_avro_value(&self.workflow_id, query_name, input)
3210 .await
3211 }
3212
3213 pub async fn update<T: Serialize>(
3214 &self,
3215 update_name: &str,
3216 input: T,
3217 request_id: Option<&str>,
3218 ) -> Result<Value> {
3219 self.client
3220 .update_workflow(&self.workflow_id, update_name, input, request_id)
3221 .await
3222 }
3223
3224 pub async fn update_avro_value<T: Serialize>(
3225 &self,
3226 update_name: &str,
3227 input: T,
3228 request_id: Option<&str>,
3229 ) -> Result<AvroValue> {
3230 self.client
3231 .update_workflow_avro_value(&self.workflow_id, update_name, input, request_id)
3232 .await
3233 }
3234
3235 pub async fn query_selected_run<T: Serialize>(
3237 &self,
3238 query_name: &str,
3239 input: T,
3240 ) -> Result<Value> {
3241 let run_id = self
3242 .run_id
3243 .as_deref()
3244 .ok_or_else(|| Error::Codec("run_id is required for selected-run query".to_string()))?;
3245 self.client
3246 .query_workflow_run(&self.workflow_id, run_id, query_name, input)
3247 .await
3248 }
3249
3250 pub async fn result(&self, options: WorkflowResultOptions) -> Result<Value> {
3252 self.result_target(options, None).await
3253 }
3254
3255 pub async fn result_avro_value(&self, options: WorkflowResultOptions) -> Result<AvroValue> {
3257 self.result_avro_value_target(options, None).await
3258 }
3259
3260 pub async fn result_selected_run(&self, options: WorkflowResultOptions) -> Result<Value> {
3262 let run_id = self.run_id.as_deref().ok_or_else(|| {
3263 Error::Codec("run_id is required for selected-run result".to_string())
3264 })?;
3265 self.result_target(options, Some(run_id)).await
3266 }
3267
3268 pub async fn result_selected_run_avro_value(
3270 &self,
3271 options: WorkflowResultOptions,
3272 ) -> Result<AvroValue> {
3273 let run_id = self.run_id.as_deref().ok_or_else(|| {
3274 Error::Codec("run_id is required for selected-run result".to_string())
3275 })?;
3276 self.result_avro_value_target(options, Some(run_id)).await
3277 }
3278
3279 async fn result_avro_value_target(
3280 &self,
3281 options: WorkflowResultOptions,
3282 selected_run_id: Option<&str>,
3283 ) -> Result<AvroValue> {
3284 let started = Instant::now();
3285
3286 loop {
3287 let description = match selected_run_id {
3288 Some(run_id) => {
3289 self.client
3290 .describe_workflow_run(&self.workflow_id, run_id)
3291 .await?
3292 }
3293 None => self.describe().await?,
3294 };
3295 if description.is_completed() {
3296 return description.output_avro_value.ok_or_else(|| {
3297 Error::Codec(
3298 "missing_payload_envelope: typed workflow result requires output_envelope"
3299 .to_string(),
3300 )
3301 });
3302 }
3303 if description.is_terminal() {
3304 let outcome =
3305 workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
3306 return Err(match outcome.kind {
3307 WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
3308 WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
3309 WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
3310 WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
3311 });
3312 }
3313 if started.elapsed() >= options.timeout {
3314 return Err(Error::Timeout);
3315 }
3316 tokio::time::sleep(options.poll_interval).await;
3317 }
3318 }
3319
3320 async fn result_target(
3321 &self,
3322 options: WorkflowResultOptions,
3323 selected_run_id: Option<&str>,
3324 ) -> Result<Value> {
3325 let started = Instant::now();
3326
3327 loop {
3328 let description = match selected_run_id {
3329 Some(run_id) => {
3330 self.client
3331 .describe_workflow_run(&self.workflow_id, run_id)
3332 .await?
3333 }
3334 None => self.describe().await?,
3335 };
3336 if description.is_completed() {
3337 return Ok(description.output.unwrap_or(Value::Null));
3338 }
3339
3340 if description.is_terminal() {
3341 let outcome =
3342 workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
3343 return Err(match outcome.kind {
3344 WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
3345 WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
3346 WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
3347 WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
3348 });
3349 }
3350
3351 if started.elapsed() >= options.timeout {
3352 return Err(Error::WorkflowTimedOut(WorkflowTerminalOutcome {
3353 kind: WorkflowTerminalKind::TimedOut,
3354 workflow_id: description
3355 .workflow_id
3356 .clone()
3357 .unwrap_or_else(|| self.workflow_id.clone()),
3358 run_id: description
3359 .run_id
3360 .clone()
3361 .or_else(|| selected_run_id.map(str::to_string)),
3362 reason: "result_wait_timeout".to_string(),
3363 failure_category: Some("client_timeout".to_string()),
3364 failure_id: None,
3365 exception_type: None,
3366 exception_class: None,
3367 non_retryable: None,
3368 message: Some(format!(
3369 "workflow result was not terminal within {:?}",
3370 options.timeout
3371 )),
3372 exception: None,
3373 raw: description.raw_value(),
3374 }));
3375 }
3376
3377 tokio::time::sleep(options.poll_interval).await;
3378 }
3379 }
3380}
3381
3382#[derive(Clone, Copy, Debug)]
3383pub struct WorkflowResultOptions {
3384 pub poll_interval: Duration,
3385 pub timeout: Duration,
3386}
3387
3388impl Default for WorkflowResultOptions {
3389 fn default() -> Self {
3390 Self {
3391 poll_interval: Duration::from_millis(500),
3392 timeout: Duration::from_secs(30),
3393 }
3394 }
3395}
3396
3397#[derive(Clone, Debug, Deserialize)]
3398pub struct WorkflowDescription {
3399 pub workflow_id: Option<String>,
3400 pub run_id: Option<String>,
3401 pub workflow_type: Option<String>,
3402 pub status: Option<String>,
3403 #[serde(default)]
3404 pub closed_reason: Option<String>,
3405 #[serde(default)]
3406 pub error: Option<String>,
3407 #[serde(default)]
3408 pub failure: Option<Value>,
3409 #[serde(default)]
3410 pub exception: Option<Value>,
3411 #[serde(default)]
3412 pub failures: Vec<Value>,
3413 #[serde(default)]
3414 pub output: Option<Value>,
3415 #[serde(default)]
3416 pub output_envelope: Option<Value>,
3417 #[serde(skip)]
3418 pub output_avro_value: Option<AvroValue>,
3419 #[serde(flatten)]
3420 pub raw: HashMap<String, Value>,
3421}
3422
3423impl WorkflowDescription {
3424 pub fn is_completed(&self) -> bool {
3425 matches!(self.status.as_deref(), Some("completed" | "Completed"))
3426 }
3427
3428 pub fn is_terminal(&self) -> bool {
3429 matches!(
3430 self.status.as_deref(),
3431 Some(
3432 "completed"
3433 | "Completed"
3434 | "failed"
3435 | "Failed"
3436 | "cancelled"
3437 | "Cancelled"
3438 | "terminated"
3439 | "Terminated"
3440 | "timed_out"
3441 | "TimedOut",
3442 )
3443 )
3444 }
3445
3446 fn decode_payloads(&mut self) -> Result<()> {
3447 if let Some(envelope) = &self.output_envelope {
3448 let value = decode_wire_avro_value(envelope, DEFAULT_CODEC)?;
3449 self.output = Some(value.clone().into_json()?);
3450 self.output_avro_value = Some(value);
3451 }
3452
3453 Ok(())
3454 }
3455
3456 fn raw_value(&self) -> Value {
3457 let mut data = self.raw.clone();
3458 data.insert(
3459 "workflow_id".to_string(),
3460 self.workflow_id
3461 .clone()
3462 .map(Value::String)
3463 .unwrap_or(Value::Null),
3464 );
3465 data.insert(
3466 "run_id".to_string(),
3467 self.run_id
3468 .clone()
3469 .map(Value::String)
3470 .unwrap_or(Value::Null),
3471 );
3472 data.insert(
3473 "workflow_type".to_string(),
3474 self.workflow_type
3475 .clone()
3476 .map(Value::String)
3477 .unwrap_or(Value::Null),
3478 );
3479 data.insert(
3480 "status".to_string(),
3481 self.status
3482 .clone()
3483 .map(Value::String)
3484 .unwrap_or(Value::Null),
3485 );
3486 data.insert(
3487 "closed_reason".to_string(),
3488 self.closed_reason
3489 .clone()
3490 .map(Value::String)
3491 .unwrap_or(Value::Null),
3492 );
3493 if let Some(failure) = &self.failure {
3494 data.insert("failure".to_string(), failure.clone());
3495 }
3496 if let Some(exception) = &self.exception {
3497 data.insert("exception".to_string(), exception.clone());
3498 }
3499 Value::Object(data.into_iter().collect())
3500 }
3501}
3502
3503fn workflow_terminal_outcome(
3504 description: &WorkflowDescription,
3505 workflow_id: &str,
3506 run_id: Option<&str>,
3507) -> WorkflowTerminalOutcome {
3508 let terminal_kind = description
3509 .closed_reason
3510 .as_deref()
3511 .or(description.status.as_deref())
3512 .unwrap_or("failed")
3513 .to_ascii_lowercase();
3514 let kind = match terminal_kind.as_str() {
3515 "cancelled" | "canceled" => WorkflowTerminalKind::Cancelled,
3516 "terminated" => WorkflowTerminalKind::Terminated,
3517 "timed_out" | "timedout" => WorkflowTerminalKind::TimedOut,
3518 _ => WorkflowTerminalKind::Failed,
3519 };
3520 let default_reason = match kind {
3521 WorkflowTerminalKind::Failed => "workflow_failed",
3522 WorkflowTerminalKind::Cancelled => "cancelled",
3523 WorkflowTerminalKind::Terminated => "terminated",
3524 WorkflowTerminalKind::TimedOut => "timed_out",
3525 };
3526 let failure = description
3527 .failure
3528 .as_ref()
3529 .filter(|value| value.is_object());
3530 let nested_failure = failure
3531 .and_then(|value| value.get("failures"))
3532 .and_then(Value::as_array)
3533 .and_then(|failures| failures.last())
3534 .or_else(|| description.failures.last());
3535 let exception = description
3536 .exception
3537 .clone()
3538 .or_else(|| failure.and_then(|value| value.get("exception")).cloned())
3539 .or_else(|| {
3540 nested_failure
3541 .and_then(|value| value.get("exception_payload"))
3542 .cloned()
3543 });
3544 let string_field = |name: &str| {
3545 failure
3546 .and_then(|value| value.get(name))
3547 .and_then(Value::as_str)
3548 .or_else(|| {
3549 nested_failure
3550 .and_then(|value| value.get(name))
3551 .and_then(Value::as_str)
3552 })
3553 .map(str::to_string)
3554 };
3555 let exception_field = |name: &str| {
3556 exception
3557 .as_ref()
3558 .and_then(|value| value.get(name))
3559 .and_then(Value::as_str)
3560 .map(str::to_string)
3561 };
3562 let message = description
3563 .error
3564 .clone()
3565 .or_else(|| string_field("message"))
3566 .or_else(|| exception_field("message"));
3567 let reason = description
3568 .raw
3569 .get("reason")
3570 .and_then(Value::as_str)
3571 .map(str::to_string)
3572 .or_else(|| {
3573 failure
3574 .and_then(|value| value.get("reason"))
3575 .and_then(Value::as_str)
3576 .map(str::to_string)
3577 })
3578 .or_else(|| description.closed_reason.clone())
3579 .unwrap_or_else(|| default_reason.to_string());
3580 let failure_id = string_field("failure_id").or_else(|| {
3581 nested_failure
3582 .and_then(|value| value.get("id"))
3583 .and_then(Value::as_str)
3584 .map(str::to_string)
3585 });
3586
3587 WorkflowTerminalOutcome {
3588 kind,
3589 workflow_id: description
3590 .workflow_id
3591 .clone()
3592 .unwrap_or_else(|| workflow_id.to_string()),
3593 run_id: description
3594 .run_id
3595 .clone()
3596 .or_else(|| run_id.map(str::to_string)),
3597 reason,
3598 failure_category: string_field("failure_category")
3599 .or_else(|| Some(default_reason.to_string())),
3600 failure_id,
3601 exception_type: string_field("exception_type").or_else(|| exception_field("type")),
3602 exception_class: string_field("exception_class").or_else(|| exception_field("class")),
3603 non_retryable: failure
3604 .and_then(|value| value.get("non_retryable"))
3605 .and_then(Value::as_bool)
3606 .or_else(|| {
3607 nested_failure
3608 .and_then(|value| value.get("non_retryable"))
3609 .and_then(Value::as_bool)
3610 }),
3611 message,
3612 exception,
3613 raw: description.raw_value(),
3614 }
3615}
3616
3617#[derive(Clone, Debug, Deserialize)]
3618pub struct RegisterWorkerResponse {
3619 pub worker_id: String,
3620 pub registered: bool,
3621 #[serde(default)]
3622 pub heartbeat_interval_seconds: Option<u64>,
3623 #[serde(default)]
3624 pub protocol_version: Option<String>,
3625 #[serde(default)]
3626 pub server_capabilities: Option<Value>,
3627}
3628
3629#[derive(Clone, Debug, Deserialize, PartialEq, Eq)]
3631pub struct WorkerDeregistrationEnvelope {
3632 pub worker_id: String,
3633 pub outcome: String,
3634 pub recovered_workflow_task_count: u64,
3635}
3636
3637#[derive(Clone, Debug, Deserialize)]
3638pub struct PollWorkflowTaskResponse {
3639 #[serde(default)]
3640 pub task: Option<WorkflowTask>,
3641 #[serde(default)]
3642 pub poll_status: Option<String>,
3643 #[serde(default)]
3644 pub reason: Option<String>,
3645 #[serde(default)]
3646 pub protocol_version: Option<String>,
3647 #[serde(default)]
3648 pub server_capabilities: Option<Value>,
3649}
3650
3651impl PollWorkflowTaskResponse {
3652 pub fn outcome(&self) -> WorkerPollOutcome {
3654 worker_poll_outcome(
3655 self.task.is_some(),
3656 self.poll_status.as_deref(),
3657 self.reason.as_deref(),
3658 )
3659 }
3660}
3661
3662#[derive(Clone, Debug, Deserialize)]
3663pub struct PollActivityTaskResponse {
3664 #[serde(default)]
3665 pub task: Option<ActivityTask>,
3666 #[serde(default)]
3667 pub poll_status: Option<String>,
3668 #[serde(default)]
3669 pub reason: Option<String>,
3670}
3671
3672impl PollActivityTaskResponse {
3673 pub fn outcome(&self) -> WorkerPollOutcome {
3675 worker_poll_outcome(
3676 self.task.is_some(),
3677 self.poll_status.as_deref(),
3678 self.reason.as_deref(),
3679 )
3680 }
3681}
3682
3683#[derive(Clone, Debug, Deserialize)]
3684pub struct PollQueryTaskResponse {
3685 #[serde(default)]
3686 pub task: Option<QueryTask>,
3687 #[serde(default)]
3688 pub poll_status: Option<String>,
3689 #[serde(default)]
3690 pub reason: Option<String>,
3691}
3692
3693impl PollQueryTaskResponse {
3694 pub fn outcome(&self) -> WorkerPollOutcome {
3696 worker_poll_outcome(
3697 self.task.is_some(),
3698 self.poll_status.as_deref(),
3699 self.reason.as_deref(),
3700 )
3701 }
3702}
3703
3704#[derive(Clone, Debug, PartialEq, Eq)]
3706pub enum WorkerPollOutcome {
3707 Task,
3709 Idle {
3711 poll_status: Option<String>,
3712 reason: Option<String>,
3713 },
3714 Stop {
3716 poll_status: Option<String>,
3717 reason: Option<String>,
3718 },
3719}
3720
3721impl WorkerPollOutcome {
3722 pub fn should_stop(&self) -> bool {
3723 matches!(self, Self::Stop { .. })
3724 }
3725}
3726
3727fn worker_poll_outcome(
3728 has_task: bool,
3729 poll_status: Option<&str>,
3730 reason: Option<&str>,
3731) -> WorkerPollOutcome {
3732 if worker_poll_is_stop(poll_status, reason) {
3733 return WorkerPollOutcome::Stop {
3734 poll_status: poll_status.map(str::to_string),
3735 reason: reason.map(str::to_string),
3736 };
3737 }
3738
3739 if has_task {
3740 WorkerPollOutcome::Task
3741 } else {
3742 WorkerPollOutcome::Idle {
3743 poll_status: poll_status.map(str::to_string),
3744 reason: reason.map(str::to_string),
3745 }
3746 }
3747}
3748
3749#[derive(Clone, Debug, Deserialize)]
3751pub struct QueryTask {
3752 pub query_task_id: String,
3753 #[serde(default = "default_workflow_task_attempt")]
3754 pub query_task_attempt: u64,
3755 #[serde(default)]
3756 pub lease_owner: Option<String>,
3757 #[serde(default)]
3758 pub workflow_id: Option<String>,
3759 #[serde(default)]
3760 pub run_id: Option<String>,
3761 pub workflow_type: String,
3762 pub query_name: String,
3763 #[serde(default = "default_payload_codec")]
3764 pub payload_codec: String,
3765 #[serde(default)]
3766 pub workflow_arguments: Option<Value>,
3767 #[serde(default)]
3768 pub query_arguments: Option<Value>,
3769 #[serde(default)]
3770 pub history_events: Vec<HistoryEvent>,
3771 #[serde(default)]
3772 pub history_export: Option<Value>,
3773 #[serde(default)]
3774 pub run_status: Option<String>,
3775}
3776
3777#[derive(Clone, Debug, Deserialize)]
3778pub struct WorkflowTask {
3779 pub task_id: String,
3780 #[serde(default)]
3781 pub workflow_id: Option<String>,
3782 #[serde(default)]
3783 pub run_id: Option<String>,
3784 pub workflow_type: String,
3785 #[serde(default = "default_payload_codec")]
3786 pub payload_codec: String,
3787 #[serde(default)]
3788 pub arguments: Option<Value>,
3789 #[serde(default)]
3790 pub history_events: Vec<HistoryEvent>,
3791 #[serde(default)]
3792 pub total_history_events: Option<u64>,
3793 #[serde(default)]
3794 pub history_size_bytes: Option<u64>,
3795 #[serde(default)]
3796 pub continue_as_new_recommended: Option<bool>,
3797 #[serde(default)]
3798 pub history_budget_pressure: Option<String>,
3799 #[serde(default)]
3800 pub next_history_page_token: Option<String>,
3801 #[serde(default = "default_workflow_task_attempt")]
3802 pub workflow_task_attempt: u64,
3803 #[serde(default)]
3804 pub workflow_signal_id: Option<String>,
3805 #[serde(default)]
3806 pub signal_name: Option<String>,
3807 #[serde(default)]
3808 pub signal_arguments: Option<Value>,
3809 #[serde(default)]
3810 pub workflow_update_id: Option<String>,
3811 #[serde(default)]
3812 pub update_name: Option<String>,
3813 #[serde(default)]
3814 pub lease_owner: Option<String>,
3815}
3816
3817impl WorkflowTask {
3818 fn append_history_page(&mut self, page: WorkflowTaskHistoryPage) {
3819 self.history_events.extend(page.history_events);
3820
3821 if page.total_history_events.is_some() {
3822 self.total_history_events = page.total_history_events;
3823 }
3824
3825 self.next_history_page_token = page
3826 .next_history_page_token
3827 .filter(|token| !token.is_empty());
3828 }
3829}
3830
3831#[derive(Clone, Debug, Deserialize)]
3832struct WorkflowTaskHistoryPage {
3833 #[serde(default)]
3834 history_events: Vec<HistoryEvent>,
3835 #[serde(default)]
3836 total_history_events: Option<u64>,
3837 #[serde(default)]
3838 next_history_page_token: Option<String>,
3839}
3840
3841#[derive(Clone, Debug, Deserialize)]
3842pub struct ActivityTask {
3843 pub task_id: String,
3844 #[serde(default)]
3845 pub activity_attempt_id: Option<String>,
3846 #[serde(default)]
3847 pub attempt_id: Option<String>,
3848 pub activity_type: String,
3849 #[serde(default = "default_payload_codec")]
3850 pub payload_codec: String,
3851 #[serde(default)]
3852 pub arguments: Option<Value>,
3853 #[serde(default = "default_attempt_number")]
3854 pub attempt_number: u64,
3855 #[serde(default)]
3856 pub lease_owner: Option<String>,
3857}
3858
3859#[derive(Clone, Debug, Deserialize)]
3860pub struct HistoryEvent {
3861 #[serde(alias = "type")]
3862 pub event_type: String,
3863 #[serde(default)]
3864 pub payload: Value,
3865 #[serde(flatten)]
3866 pub raw: HashMap<String, Value>,
3867}
3868
3869#[derive(Clone, Debug, PartialEq)]
3871pub struct QuerySignal {
3872 pub id: Option<String>,
3873 pub name: String,
3874 pub arguments: Vec<Value>,
3875 avro_arguments: Vec<AvroValue>,
3876 pub workflow_sequence: Option<u64>,
3877}
3878
3879impl QuerySignal {
3880 pub fn arguments_avro_value(&self) -> &[AvroValue] {
3882 &self.avro_arguments
3883 }
3884}
3885
3886#[derive(Clone, Debug)]
3892pub struct QueryContext {
3893 pub workflow_id: Option<String>,
3894 pub run_id: Option<String>,
3895 pub workflow_type: String,
3896 pub run_status: Option<String>,
3897 workflow_input: Value,
3898 workflow_input_avro_value: AvroValue,
3899 history_events: Arc<Vec<HistoryEvent>>,
3900 signal_events: Arc<Vec<QuerySignal>>,
3901}
3902
3903impl QueryContext {
3904 pub fn workflow_input(&self) -> &Value {
3906 &self.workflow_input
3907 }
3908
3909 pub fn workflow_input_avro_value(&self) -> &AvroValue {
3911 &self.workflow_input_avro_value
3912 }
3913
3914 pub fn history_events(&self) -> &[HistoryEvent] {
3916 self.history_events.as_slice()
3917 }
3918
3919 pub fn signal_events(&self) -> &[QuerySignal] {
3921 self.signal_events.as_slice()
3922 }
3923
3924 pub fn signals(&self, signal_name: &str) -> Vec<Vec<Value>> {
3926 self.signal_events
3927 .iter()
3928 .filter(|signal| signal.name == signal_name)
3929 .map(|signal| signal.arguments.clone())
3930 .collect()
3931 }
3932
3933 pub fn signals_avro_value(&self, signal_name: &str) -> Vec<Vec<AvroValue>> {
3935 self.signal_events
3936 .iter()
3937 .filter(|signal| signal.name == signal_name)
3938 .map(|signal| signal.avro_arguments.clone())
3939 .collect()
3940 }
3941}
3942
3943#[derive(Clone, Debug, Deserialize)]
3944pub struct ActivityHeartbeatResponse {
3945 #[serde(default)]
3946 pub cancel_requested: bool,
3947 #[serde(default)]
3948 pub heartbeat_recorded: bool,
3949 #[serde(default)]
3950 pub can_continue: Option<bool>,
3951 #[serde(default)]
3952 pub reason: Option<String>,
3953 #[serde(default)]
3954 pub run_closed_reason: Option<String>,
3955 #[serde(default)]
3956 pub run_closed_at: Option<String>,
3957 #[serde(default)]
3958 pub lease_expires_at: Option<String>,
3959 #[serde(default)]
3960 pub last_heartbeat_at: Option<String>,
3961}
3962
3963impl ActivityHeartbeatResponse {
3964 pub fn should_stop(&self) -> bool {
3966 self.cancel_requested || self.can_continue == Some(false)
3967 }
3968}
3969
3970fn default_payload_codec() -> String {
3971 DEFAULT_CODEC.to_string()
3972}
3973
3974fn default_workflow_task_attempt() -> u64 {
3975 1
3976}
3977
3978fn default_attempt_number() -> u64 {
3979 1
3980}
3981
3982type WorkflowFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
3983type WorkflowHandler = Arc<dyn Fn(WorkflowContext, AvroValue) -> WorkflowFuture + Send + Sync>;
3984type ErasedWorkflowState = Arc<dyn Any + Send + Sync>;
3985type WorkflowStateSnapshot = Arc<dyn Fn() -> Result<ErasedWorkflowState> + Send + Sync>;
3986type ReplayedWorkflowHandler =
3987 Arc<dyn Fn(WorkflowContext, AvroValue) -> ReplayedWorkflowInvocation + Send + Sync>;
3988type ActivityFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
3989type ActivityHandler = Arc<dyn Fn(ActivityContext, AvroValue) -> ActivityFuture + Send + Sync>;
3990type QueryFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
3991type QueryHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
3992type UpdateHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
3993type ReplayedQueryHandler = Arc<
3994 dyn Fn(QueryContext, ErasedWorkflowState, AvroValue) -> std::result::Result<QueryFuture, String>
3995 + Send
3996 + Sync,
3997>;
3998type WorkerHeartbeatObserver = Arc<dyn Fn(&WorkerHeartbeatObservation) + Send + Sync>;
3999
4000struct ReplayedWorkflowInvocation {
4001 future: WorkflowFuture,
4002 snapshot: WorkflowStateSnapshot,
4003}
4004
4005#[derive(Clone)]
4006struct RegisteredWorkflow {
4007 execute: WorkflowHandler,
4008 replay: Option<ReplayedWorkflowHandler>,
4009 state_type: Option<TypeId>,
4010}
4011
4012#[derive(Clone)]
4013enum RegisteredQuery {
4014 Snapshot(QueryHandler),
4015 Replayed {
4016 state_type: TypeId,
4017 handler: ReplayedQueryHandler,
4018 },
4019}
4020
4021#[derive(Clone, Debug)]
4022pub struct WorkerHeartbeatObservation {
4023 pub worker_id: String,
4024 pub task_queue: String,
4025 pub acknowledged_at_unix_millis: u64,
4026 pub acknowledgement: Value,
4027}
4028
4029#[derive(Clone, Copy, Debug)]
4036pub struct WorkerRetryPolicy {
4037 pub max_retries: usize,
4039 pub initial_backoff: Duration,
4041 pub max_backoff: Duration,
4043}
4044
4045impl Default for WorkerRetryPolicy {
4046 fn default() -> Self {
4047 Self {
4048 max_retries: 5,
4049 initial_backoff: Duration::from_millis(100),
4050 max_backoff: Duration::from_secs(5),
4051 }
4052 }
4053}
4054
4055#[derive(Clone, Copy, Debug, PartialEq, Eq)]
4056enum ManagedPollOutcome {
4057 Idle,
4058 Handled,
4059 Stop,
4060}
4061
4062#[derive(Clone)]
4063pub struct Worker {
4064 client: Client,
4065 worker_id: String,
4066 task_queue: String,
4067 workflows: HashMap<String, RegisteredWorkflow>,
4068 activities: HashMap<String, ActivityHandler>,
4069 queries: HashMap<String, HashMap<String, RegisteredQuery>>,
4070 updates: HashMap<String, HashMap<String, UpdateHandler>>,
4071 max_concurrent_workflow_tasks: usize,
4072 max_concurrent_activity_tasks: usize,
4073 poll_timeout: Duration,
4074 heartbeat_interval: Duration,
4075 retry_policy: WorkerRetryPolicy,
4076 heartbeat_observer: Option<WorkerHeartbeatObserver>,
4077}
4078
4079impl Worker {
4080 pub fn new(client: Client, task_queue: impl Into<String>) -> Self {
4081 Self {
4082 client,
4083 worker_id: default_worker_id(),
4084 task_queue: task_queue.into(),
4085 workflows: HashMap::new(),
4086 activities: HashMap::new(),
4087 queries: HashMap::new(),
4088 updates: HashMap::new(),
4089 max_concurrent_workflow_tasks: 10,
4090 max_concurrent_activity_tasks: 10,
4091 poll_timeout: Duration::from_secs(30),
4092 heartbeat_interval: Duration::from_secs(60),
4093 retry_policy: WorkerRetryPolicy::default(),
4094 heartbeat_observer: None,
4095 }
4096 }
4097
4098 pub fn worker_id(mut self, worker_id: impl Into<String>) -> Self {
4099 self.worker_id = worker_id.into();
4100 self
4101 }
4102
4103 pub fn poll_timeout(mut self, timeout: Duration) -> Self {
4104 self.poll_timeout = timeout;
4105 self
4106 }
4107
4108 pub fn heartbeat_interval(mut self, interval: Duration) -> Self {
4109 self.heartbeat_interval = interval;
4110 self
4111 }
4112
4113 pub fn retry_policy(mut self, policy: WorkerRetryPolicy) -> Self {
4115 self.retry_policy = policy;
4116 self
4117 }
4118
4119 pub fn on_worker_heartbeat<F>(mut self, observer: F) -> Self
4120 where
4121 F: Fn(&WorkerHeartbeatObservation) + Send + Sync + 'static,
4122 {
4123 self.heartbeat_observer = Some(Arc::new(observer));
4124 self
4125 }
4126
4127 pub fn max_concurrent_workflow_tasks(mut self, count: usize) -> Self {
4128 self.max_concurrent_workflow_tasks = count.max(1);
4129 self
4130 }
4131
4132 pub fn max_concurrent_activity_tasks(mut self, count: usize) -> Self {
4133 self.max_concurrent_activity_tasks = count.max(1);
4134 self
4135 }
4136
4137 pub fn register_workflow<F, Fut>(&mut self, workflow_type: impl Into<String>, handler: F)
4144 where
4145 F: Fn(WorkflowContext, Value) -> Fut + Send + Sync + 'static,
4146 Fut: Future<Output = Result<Value>> + Send + 'static,
4147 {
4148 let handler = Arc::new(handler);
4149 self.workflows.insert(
4150 workflow_type.into(),
4151 RegisteredWorkflow {
4152 execute: Arc::new(move |ctx, input| {
4153 let handler = Arc::clone(&handler);
4154 Box::pin(async move {
4155 let result = handler(ctx, input.into_json()?).await?;
4156 AvroValue::from_serialize(&result)
4157 })
4158 }),
4159 replay: None,
4160 state_type: None,
4161 },
4162 );
4163 }
4164
4165 pub fn register_workflow_avro_value<F, Fut>(
4167 &mut self,
4168 workflow_type: impl Into<String>,
4169 handler: F,
4170 ) where
4171 F: Fn(WorkflowContext, AvroValue) -> Fut + Send + Sync + 'static,
4172 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4173 {
4174 self.workflows.insert(
4175 workflow_type.into(),
4176 RegisteredWorkflow {
4177 execute: Arc::new(move |ctx, input| Box::pin(handler(ctx, input))),
4178 replay: None,
4179 state_type: None,
4180 },
4181 );
4182 }
4183
4184 pub fn register_replayed_workflow<S, Factory, F, Fut>(
4192 &mut self,
4193 workflow_type: impl Into<String>,
4194 state_factory: Factory,
4195 handler: F,
4196 ) where
4197 S: Clone + Send + Sync + 'static,
4198 Factory: Fn() -> S + Send + Sync + 'static,
4199 F: Fn(WorkflowContext, Value, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
4200 Fut: Future<Output = Result<Value>> + Send + 'static,
4201 {
4202 let state_factory = Arc::new(state_factory);
4203 let handler = Arc::new(handler);
4204
4205 let execute_factory = Arc::clone(&state_factory);
4206 let execute_handler = Arc::clone(&handler);
4207 let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4208 let state = WorkflowInstance::new(execute_factory());
4209 let handler = Arc::clone(&execute_handler);
4210 Box::pin(async move {
4211 let result = handler(ctx, input.into_json()?, state).await?;
4212 AvroValue::from_serialize(&result)
4213 }) as WorkflowFuture
4214 });
4215
4216 let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4217 let state = WorkflowInstance::new(state_factory());
4218 let snapshot_state = state.clone();
4219 let snapshot: WorkflowStateSnapshot =
4220 Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
4221 let replay_handler = Arc::clone(&handler);
4222 let future = async move {
4223 let result = replay_handler(ctx, input.into_json()?, state).await?;
4224 AvroValue::from_serialize(&result)
4225 };
4226 ReplayedWorkflowInvocation {
4227 future: Box::pin(future),
4228 snapshot,
4229 }
4230 });
4231
4232 self.workflows.insert(
4233 workflow_type.into(),
4234 RegisteredWorkflow {
4235 execute,
4236 replay: Some(replay),
4237 state_type: Some(TypeId::of::<S>()),
4238 },
4239 );
4240 }
4241
4242 pub fn register_replayed_workflow_avro_value<S, Factory, F, Fut>(
4244 &mut self,
4245 workflow_type: impl Into<String>,
4246 state_factory: Factory,
4247 handler: F,
4248 ) where
4249 S: Clone + Send + Sync + 'static,
4250 Factory: Fn() -> S + Send + Sync + 'static,
4251 F: Fn(WorkflowContext, AvroValue, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
4252 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4253 {
4254 let state_factory = Arc::new(state_factory);
4255 let handler = Arc::new(handler);
4256
4257 let execute_factory = Arc::clone(&state_factory);
4258 let execute_handler = Arc::clone(&handler);
4259 let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4260 let state = WorkflowInstance::new(execute_factory());
4261 Box::pin(execute_handler(ctx, input, state)) as WorkflowFuture
4262 });
4263
4264 let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4265 let state = WorkflowInstance::new(state_factory());
4266 let snapshot_state = state.clone();
4267 let snapshot: WorkflowStateSnapshot =
4268 Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
4269 ReplayedWorkflowInvocation {
4270 future: Box::pin(handler(ctx, input, state)),
4271 snapshot,
4272 }
4273 });
4274
4275 self.workflows.insert(
4276 workflow_type.into(),
4277 RegisteredWorkflow {
4278 execute,
4279 replay: Some(replay),
4280 state_type: Some(TypeId::of::<S>()),
4281 },
4282 );
4283 }
4284
4285 pub fn register_activity<F, Fut>(&mut self, activity_type: impl Into<String>, handler: F)
4286 where
4287 F: Fn(ActivityContext, Value) -> Fut + Send + Sync + 'static,
4288 Fut: Future<Output = Result<Value>> + Send + 'static,
4289 {
4290 let handler = Arc::new(handler);
4291 self.activities.insert(
4292 activity_type.into(),
4293 Arc::new(move |ctx, args| {
4294 let handler = Arc::clone(&handler);
4295 Box::pin(async move {
4296 let result = handler(ctx, args.into_json()?).await?;
4297 AvroValue::from_serialize(&result)
4298 })
4299 }),
4300 );
4301 }
4302
4303 pub fn register_activity_avro_value<F, Fut>(
4305 &mut self,
4306 activity_type: impl Into<String>,
4307 handler: F,
4308 ) where
4309 F: Fn(ActivityContext, AvroValue) -> Fut + Send + Sync + 'static,
4310 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4311 {
4312 self.activities.insert(
4313 activity_type.into(),
4314 Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
4315 );
4316 }
4317
4318 pub fn register_query<F, Fut>(
4324 &mut self,
4325 workflow_type: impl Into<String>,
4326 query_name: impl Into<String>,
4327 handler: F,
4328 ) where
4329 F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
4330 Fut: Future<Output = Result<Value>> + Send + 'static,
4331 {
4332 let handler = Arc::new(handler);
4333 self.queries
4334 .entry(workflow_type.into())
4335 .or_default()
4336 .insert(
4337 query_name.into(),
4338 RegisteredQuery::Snapshot(Arc::new(move |ctx, args| {
4339 let handler = Arc::clone(&handler);
4340 Box::pin(async move {
4341 let result = handler(ctx, args.into_json()?).await?;
4342 AvroValue::from_serialize(&result)
4343 })
4344 })),
4345 );
4346 }
4347
4348 pub fn register_query_avro_value<F, Fut>(
4350 &mut self,
4351 workflow_type: impl Into<String>,
4352 query_name: impl Into<String>,
4353 handler: F,
4354 ) where
4355 F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
4356 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4357 {
4358 self.queries
4359 .entry(workflow_type.into())
4360 .or_default()
4361 .insert(
4362 query_name.into(),
4363 RegisteredQuery::Snapshot(Arc::new(move |ctx, args| Box::pin(handler(ctx, args)))),
4364 );
4365 }
4366
4367 pub fn register_replayed_query<S, F, Fut>(
4374 &mut self,
4375 workflow_type: impl Into<String>,
4376 query_name: impl Into<String>,
4377 handler: F,
4378 ) where
4379 S: Clone + Send + Sync + 'static,
4380 F: Fn(QueryContext, Arc<S>, Value) -> Fut + Send + Sync + 'static,
4381 Fut: Future<Output = Result<Value>> + Send + 'static,
4382 {
4383 let handler = Arc::new(handler);
4384 let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
4385 let state = state.downcast::<S>().map_err(|_| {
4386 "registered query state type does not match the replayed workflow state".to_string()
4387 })?;
4388 let handler = Arc::clone(&handler);
4389 Ok(Box::pin(async move {
4390 let result = handler(ctx, state, args.into_json()?).await?;
4391 AvroValue::from_serialize(&result)
4392 }))
4393 });
4394
4395 self.queries
4396 .entry(workflow_type.into())
4397 .or_default()
4398 .insert(
4399 query_name.into(),
4400 RegisteredQuery::Replayed {
4401 state_type: TypeId::of::<S>(),
4402 handler: erased_handler,
4403 },
4404 );
4405 }
4406
4407 pub fn register_replayed_query_avro_value<S, F, Fut>(
4409 &mut self,
4410 workflow_type: impl Into<String>,
4411 query_name: impl Into<String>,
4412 handler: F,
4413 ) where
4414 S: Clone + Send + Sync + 'static,
4415 F: Fn(QueryContext, Arc<S>, AvroValue) -> Fut + Send + Sync + 'static,
4416 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4417 {
4418 let handler = Arc::new(handler);
4419 let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
4420 let state = state.downcast::<S>().map_err(|_| {
4421 "registered query state type does not match the replayed workflow state".to_string()
4422 })?;
4423 Ok(Box::pin(handler(ctx, state, args)))
4424 });
4425
4426 self.queries
4427 .entry(workflow_type.into())
4428 .or_default()
4429 .insert(
4430 query_name.into(),
4431 RegisteredQuery::Replayed {
4432 state_type: TypeId::of::<S>(),
4433 handler: erased_handler,
4434 },
4435 );
4436 }
4437
4438 pub fn register_update<F, Fut>(
4440 &mut self,
4441 workflow_type: impl Into<String>,
4442 update_name: impl Into<String>,
4443 handler: F,
4444 ) where
4445 F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
4446 Fut: Future<Output = Result<Value>> + Send + 'static,
4447 {
4448 let handler = Arc::new(handler);
4449 self.updates
4450 .entry(workflow_type.into())
4451 .or_default()
4452 .insert(
4453 update_name.into(),
4454 Arc::new(move |ctx, args| {
4455 let handler = Arc::clone(&handler);
4456 Box::pin(async move {
4457 let result = handler(ctx, args.into_json()?).await?;
4458 AvroValue::from_serialize(&result)
4459 })
4460 }),
4461 );
4462 }
4463
4464 pub fn register_update_avro_value<F, Fut>(
4466 &mut self,
4467 workflow_type: impl Into<String>,
4468 update_name: impl Into<String>,
4469 handler: F,
4470 ) where
4471 F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
4472 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4473 {
4474 self.updates
4475 .entry(workflow_type.into())
4476 .or_default()
4477 .insert(
4478 update_name.into(),
4479 Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
4480 );
4481 }
4482
4483 pub async fn register(&self) -> Result<RegisterWorkerResponse> {
4484 let mut command_contracts = serde_json::Map::new();
4485 for workflow_type in self.workflows.keys() {
4486 let mut queries = self
4487 .queries
4488 .get(workflow_type)
4489 .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
4490 .unwrap_or_default();
4491 queries.sort();
4492 let mut updates = self
4493 .updates
4494 .get(workflow_type)
4495 .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
4496 .unwrap_or_default();
4497 updates.sort();
4498 if !queries.is_empty() || !updates.is_empty() {
4499 command_contracts.insert(
4500 workflow_type.clone(),
4501 json!({
4502 "queries": queries,
4503 "updates": updates,
4504 "update_validators": [],
4505 }),
4506 );
4507 }
4508 }
4509
4510 self.client
4511 .register_worker_with_command_contracts(
4512 &self.worker_id,
4513 &self.task_queue,
4514 self.workflows.keys().cloned().collect(),
4515 self.activities.keys().cloned().collect(),
4516 self.max_concurrent_workflow_tasks,
4517 self.max_concurrent_activity_tasks,
4518 [
4519 (!self.queries.is_empty()).then(|| QUERY_TASKS_CAPABILITY.to_string()),
4520 (!self.updates.is_empty()).then(|| WORKFLOW_UPDATES_CAPABILITY.to_string()),
4521 ]
4522 .into_iter()
4523 .flatten()
4524 .collect(),
4525 Value::Object(command_contracts),
4526 )
4527 .await
4528 }
4529
4530 pub async fn run(&self) -> Result<()> {
4536 self.run_until(std::future::pending::<()>()).await
4537 }
4538
4539 pub async fn run_until<F>(&self, shutdown: F) -> Result<()>
4543 where
4544 F: Future<Output = ()>,
4545 {
4546 let registration = self.register().await?;
4547 if !registration.registered {
4548 return Err(Error::WorkerLoop(format!(
4549 "worker registration for {:?} was not accepted",
4550 self.worker_id
4551 )));
4552 }
4553 let registered_worker_id = registration.worker_id.clone();
4554 let primary = self.run_registered_until(shutdown, registration).await;
4555 let deregistration = self
4556 .client
4557 .deregister_worker_registration(®istered_worker_id)
4558 .await;
4559
4560 match (primary, deregistration) {
4561 (Ok(()), Ok(_)) => Ok(()),
4562 (Ok(()), Err(deregistration)) => Err(deregistration),
4563 (Err(primary), Ok(_)) => Err(primary),
4564 (Err(primary), Err(deregistration)) => Err(Error::WorkerShutdown {
4565 primary: Box::new(primary),
4566 deregistration: Box::new(deregistration),
4567 }),
4568 }
4569 }
4570
4571 async fn run_registered_until<F>(
4572 &self,
4573 shutdown: F,
4574 registration: RegisterWorkerResponse,
4575 ) -> Result<()>
4576 where
4577 F: Future<Output = ()>,
4578 {
4579 let heartbeat_interval = Duration::from_secs(
4580 registration
4581 .heartbeat_interval_seconds
4582 .unwrap_or(self.heartbeat_interval.as_secs().max(1)),
4583 );
4584 let heartbeat = tokio::time::sleep(Duration::ZERO);
4590 tokio::pin!(heartbeat);
4591 tokio::pin!(shutdown);
4592 let stop = Arc::new(AtomicBool::new(false));
4593 let mut workflow_poller = (!self.workflows.is_empty()).then(|| {
4597 let worker = self.clone();
4598 let stop = Arc::clone(&stop);
4599 tokio::spawn(async move { worker.poll_workflows_until_stopped(stop).await })
4600 });
4601 let mut activity_poller = (!self.activities.is_empty()).then(|| {
4602 let worker = self.clone();
4603 let stop = Arc::clone(&stop);
4604 tokio::spawn(async move { worker.poll_activities_until_stopped(stop).await })
4605 });
4606 let mut query_poller = (!self.queries.is_empty()).then(|| {
4607 let worker = self.clone();
4608 let stop = Arc::clone(&stop);
4609 tokio::spawn(async move { worker.poll_queries_until_stopped(stop).await })
4610 });
4611
4612 loop {
4613 tokio::select! {
4614 _ = &mut shutdown => {
4615 stop.store(true, Ordering::SeqCst);
4616 break;
4617 }
4618 _ = &mut heartbeat => {
4619 let result = self.retry_worker_operation(|| {
4620 self.client.heartbeat_worker(
4621 &self.worker_id,
4622 self.max_concurrent_workflow_tasks,
4623 self.max_concurrent_activity_tasks,
4624 )
4625 }).await;
4626 heartbeat
4627 .as_mut()
4628 .reset(tokio::time::Instant::now() + heartbeat_interval);
4629 match result {
4630 Ok(acknowledgement) => {
4631 if let Some(observer) = &self.heartbeat_observer {
4632 observer(&WorkerHeartbeatObservation {
4633 worker_id: self.worker_id.clone(),
4634 task_queue: self.task_queue.clone(),
4635 acknowledged_at_unix_millis: SystemTime::now()
4636 .duration_since(UNIX_EPOCH)
4637 .unwrap_or_default()
4638 .as_millis()
4639 .min(u64::MAX as u128)
4640 as u64,
4641 acknowledgement,
4642 });
4643 }
4644 }
4645 Err(error) => {
4646 stop.store(true, Ordering::SeqCst);
4647 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await?;
4648 return Err(error);
4649 }
4650 }
4651 }
4652 result = OptionFuture::from(workflow_poller.as_mut()), if workflow_poller.is_some() => {
4653 workflow_poller = None;
4654 let stopped_by_server = stop.load(Ordering::SeqCst);
4655 stop.store(true, Ordering::SeqCst);
4656 let poller_result = optional_poller_result("workflow", result);
4657 let join_result =
4658 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
4659 poller_result?;
4660 join_result?;
4661 if stopped_by_server {
4662 return Ok(());
4663 }
4664 return Err(Error::WorkerLoop(
4665 "workflow poller stopped unexpectedly".to_string(),
4666 ));
4667 }
4668 result = OptionFuture::from(activity_poller.as_mut()), if activity_poller.is_some() => {
4669 activity_poller = None;
4670 let stopped_by_server = stop.load(Ordering::SeqCst);
4671 stop.store(true, Ordering::SeqCst);
4672 let poller_result = optional_poller_result("activity", result);
4673 let join_result =
4674 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
4675 poller_result?;
4676 join_result?;
4677 if stopped_by_server {
4678 return Ok(());
4679 }
4680 return Err(Error::WorkerLoop(
4681 "activity poller stopped unexpectedly".to_string(),
4682 ));
4683 }
4684 result = OptionFuture::from(query_poller.as_mut()), if query_poller.is_some() => {
4685 query_poller = None;
4686 let stopped_by_server = stop.load(Ordering::SeqCst);
4687 stop.store(true, Ordering::SeqCst);
4688 let poller_result = optional_poller_result("query", result);
4689 let join_result =
4690 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
4691 poller_result?;
4692 join_result?;
4693 if stopped_by_server {
4694 return Ok(());
4695 }
4696 return Err(Error::WorkerLoop(
4697 "query poller stopped unexpectedly".to_string(),
4698 ));
4699 }
4700 }
4701 }
4702
4703 join_pollers(
4704 workflow_poller.take(),
4705 activity_poller.take(),
4706 query_poller.take(),
4707 )
4708 .await
4709 }
4710
4711 pub async fn run_once(&self) -> Result<usize> {
4725 let mut handled = 0;
4726 match self.poll_workflow_once().await? {
4727 ManagedPollOutcome::Handled => handled += 1,
4728 ManagedPollOutcome::Stop => return Ok(handled),
4729 ManagedPollOutcome::Idle => {}
4730 }
4731 match self.poll_activity_once().await? {
4732 ManagedPollOutcome::Handled => handled += 1,
4733 ManagedPollOutcome::Stop => return Ok(handled),
4734 ManagedPollOutcome::Idle => {}
4735 }
4736 if !self.queries.is_empty() {
4737 match self.poll_query_once().await? {
4738 ManagedPollOutcome::Handled => handled += 1,
4739 ManagedPollOutcome::Stop => return Ok(handled),
4740 ManagedPollOutcome::Idle => {}
4741 }
4742 }
4743 Ok(handled)
4744 }
4745
4746 async fn poll_workflow_once(&self) -> Result<ManagedPollOutcome> {
4747 let poll_request_id = unique_request_id("rust-workflow-poll");
4748 let response = self
4749 .retry_worker_operation(|| {
4750 self.client.poll_workflow_task_response_with_request_id(
4751 &self.worker_id,
4752 &self.task_queue,
4753 self.poll_timeout,
4754 &poll_request_id,
4755 0,
4756 )
4757 })
4758 .await?;
4759 if response.outcome().should_stop() {
4760 return Ok(ManagedPollOutcome::Stop);
4761 }
4762 let Some(task) = response.task else {
4763 return Ok(ManagedPollOutcome::Idle);
4764 };
4765
4766 let task_id = task.task_id.clone();
4767 let attempt = task.workflow_task_attempt;
4768 let run_id = task.run_id.clone();
4769 let lease_owner = task
4770 .lease_owner
4771 .clone()
4772 .unwrap_or_else(|| self.worker_id.clone());
4773
4774 match self.execute_workflow_task(task) {
4775 Ok(commands) if commands.is_empty() => {
4776 self.client
4782 .fail_workflow_task_with_type(
4783 &task_id,
4784 &lease_owner,
4785 attempt,
4786 WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE,
4787 WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE,
4788 )
4789 .await?;
4790 }
4791 Ok(commands) => {
4792 let completion = self
4793 .client
4794 .complete_workflow_task(&task_id, &lease_owner, attempt, commands)
4795 .await;
4796 if let Err(error) = completion {
4797 if !workflow_task_completion_is_terminal_timeout(
4798 &error,
4799 &task_id,
4800 attempt,
4801 run_id.as_deref(),
4802 ) {
4803 return Err(error);
4804 }
4805 }
4806 }
4807 Err(error) => {
4808 self.client
4809 .fail_workflow_task(&task_id, &lease_owner, attempt, error.to_string())
4810 .await?;
4811 }
4812 }
4813
4814 Ok(ManagedPollOutcome::Handled)
4815 }
4816
4817 async fn poll_workflows_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
4818 while !stop.load(Ordering::SeqCst) {
4819 if self.poll_workflow_once().await? == ManagedPollOutcome::Stop {
4820 stop.store(true, Ordering::SeqCst);
4821 break;
4822 }
4823 }
4824
4825 Ok(())
4826 }
4827
4828 async fn poll_activity_once(&self) -> Result<ManagedPollOutcome> {
4829 let poll_request_id = unique_request_id("rust-activity-poll");
4830 let response = self
4831 .retry_worker_operation(|| {
4832 self.client.poll_activity_task_response_with_request_id(
4833 &self.worker_id,
4834 &self.task_queue,
4835 self.poll_timeout,
4836 &poll_request_id,
4837 0,
4838 )
4839 })
4840 .await?;
4841 if response.outcome().should_stop() {
4842 return Ok(ManagedPollOutcome::Stop);
4843 }
4844 let Some(task) = response.task else {
4845 return Ok(ManagedPollOutcome::Idle);
4846 };
4847
4848 let task_id = task.task_id.clone();
4849 let attempt_id = task
4850 .activity_attempt_id
4851 .clone()
4852 .or(task.attempt_id.clone())
4853 .unwrap_or_default();
4854 let lease_owner = task
4855 .lease_owner
4856 .clone()
4857 .unwrap_or_else(|| self.worker_id.clone());
4858 let codec = task.payload_codec.clone();
4859 let result = self.execute_activity_task(task).await;
4860 match result {
4861 Ok(value) => {
4862 let completion = self
4863 .client
4864 .complete_activity_task(&task_id, &attempt_id, &lease_owner, value, &codec)
4865 .await;
4866 if let Err(error) = completion {
4867 if !activity_task_rejection_is_final(&error) {
4868 return Err(error);
4869 }
4870 }
4871 }
4872 Err(error) => {
4873 let failure = self
4874 .client
4875 .fail_activity_task(
4876 &task_id,
4877 &attempt_id,
4878 &lease_owner,
4879 error.to_string(),
4880 false,
4881 )
4882 .await;
4883 if let Err(error) = failure {
4884 if !activity_task_rejection_is_final(&error) {
4885 return Err(error);
4886 }
4887 }
4888 }
4889 }
4890
4891 Ok(ManagedPollOutcome::Handled)
4892 }
4893
4894 async fn poll_activities_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
4895 while !stop.load(Ordering::SeqCst) {
4896 if self.poll_activity_once().await? == ManagedPollOutcome::Stop {
4897 stop.store(true, Ordering::SeqCst);
4898 break;
4899 }
4900 }
4901
4902 Ok(())
4903 }
4904
4905 async fn poll_query_once(&self) -> Result<ManagedPollOutcome> {
4906 let poll_request_id = unique_request_id("rust-query-poll");
4907 let response = self
4908 .retry_worker_operation(|| {
4909 self.client.poll_query_task_response_with_request_id(
4910 &self.worker_id,
4911 &self.task_queue,
4912 self.poll_timeout,
4913 &poll_request_id,
4914 0,
4915 )
4916 })
4917 .await?;
4918 if response.outcome().should_stop() {
4919 return Ok(ManagedPollOutcome::Stop);
4920 }
4921 let Some(task) = response.task else {
4922 return Ok(ManagedPollOutcome::Idle);
4923 };
4924
4925 let query_task_id = task.query_task_id.clone();
4926 let attempt = task.query_task_attempt;
4927 let lease_owner = task
4928 .lease_owner
4929 .clone()
4930 .unwrap_or_else(|| self.worker_id.clone());
4931 let codec = task.payload_codec.clone();
4932
4933 match self.execute_query_task(task).await {
4934 Ok(value) => {
4935 let result_envelope = match encode_typed_envelope(&value, &codec) {
4936 Ok(result_envelope) => result_envelope,
4937 Err(error) => {
4938 let failure = self
4939 .client
4940 .fail_query_task(
4941 &query_task_id,
4942 &lease_owner,
4943 attempt,
4944 error.to_string(),
4945 "query_result_encode_failed",
4946 "QueryResultEncodeFailed",
4947 )
4948 .await;
4949 if let Err(error) = failure {
4950 if !query_task_rejection_is_final(&error) {
4951 return Err(error);
4952 }
4953 }
4954 return Ok(ManagedPollOutcome::Handled);
4955 }
4956 };
4957
4958 if let Err(error) = self
4959 .client
4960 .complete_query_task_with_envelope(
4961 &query_task_id,
4962 &lease_owner,
4963 attempt,
4964 value.clone().into_json()?,
4965 result_envelope,
4966 )
4967 .await
4968 {
4969 if !query_task_rejection_is_final(&error) {
4970 return Err(error);
4971 }
4972 }
4973 }
4974 Err(failure) => {
4975 let result = self
4976 .client
4977 .fail_query_task(
4978 &query_task_id,
4979 &lease_owner,
4980 attempt,
4981 failure.message,
4982 failure.reason,
4983 failure.failure_type,
4984 )
4985 .await;
4986 if let Err(error) = result {
4987 if !query_task_rejection_is_final(&error) {
4988 return Err(error);
4989 }
4990 }
4991 }
4992 }
4993
4994 Ok(ManagedPollOutcome::Handled)
4995 }
4996
4997 async fn poll_queries_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
4998 while !stop.load(Ordering::SeqCst) {
4999 if self.poll_query_once().await? == ManagedPollOutcome::Stop {
5000 stop.store(true, Ordering::SeqCst);
5001 break;
5002 }
5003 }
5004
5005 Ok(())
5006 }
5007
5008 async fn retry_worker_operation<T, F, Fut>(&self, mut operation: F) -> Result<T>
5009 where
5010 F: FnMut() -> Fut,
5011 Fut: Future<Output = Result<T>>,
5012 {
5013 let mut retries = 0;
5014
5015 loop {
5016 match operation().await {
5017 Err(error)
5018 if worker_operation_is_retryable(&error)
5019 && retries < self.retry_policy.max_retries =>
5020 {
5021 retries += 1;
5022 tokio::time::sleep(worker_retry_delay(self.retry_policy, retries)).await;
5023 }
5024 result => return result,
5025 }
5026 }
5027 }
5028
5029 async fn execute_query_task(
5030 &self,
5031 mut task: QueryTask,
5032 ) -> std::result::Result<AvroValue, QueryTaskExecutionFailure> {
5033 if !matches!(task.payload_codec.as_str(), DEFAULT_CODEC | JSON_CODEC) {
5034 return Err(QueryTaskExecutionFailure::new(
5035 "query_payload_decode_failed",
5036 format!(
5037 "cannot decode query payload with unsupported codec {:?}",
5038 task.payload_codec
5039 ),
5040 "QueryPayloadDecodeFailed",
5041 ));
5042 }
5043
5044 if !self.workflows.contains_key(&task.workflow_type) {
5045 return Err(QueryTaskExecutionFailure::new(
5046 "query_workflow_type_not_registered",
5047 format!("no workflow registered for type {:?}", task.workflow_type),
5048 "WorkflowTypeNotRegistered",
5049 ));
5050 }
5051
5052 let Some(handlers) = self.queries.get(&task.workflow_type) else {
5053 return Err(QueryTaskExecutionFailure::new(
5054 "query_handler_unavailable",
5055 format!(
5056 "query handlers are unavailable for workflow type {:?}",
5057 task.workflow_type
5058 ),
5059 "QueryHandlerUnavailable",
5060 ));
5061 };
5062 let Some(query) = handlers.get(&task.query_name) else {
5063 return Err(QueryTaskExecutionFailure::new(
5064 "rejected_unknown_query",
5065 format!("unknown query {:?}", task.query_name),
5066 "QueryFailed",
5067 ));
5068 };
5069
5070 let args = decode_task_avro_arguments(task.query_arguments.as_ref(), &task.payload_codec)
5071 .map_err(|error| {
5072 QueryTaskExecutionFailure::new(
5073 "query_payload_decode_failed",
5074 format!("cannot decode query arguments: {error}"),
5075 "QueryPayloadDecodeFailed",
5076 )
5077 })?;
5078 let workflow_input_typed =
5079 decode_task_avro_arguments(task.workflow_arguments.as_ref(), &task.payload_codec)
5080 .map_err(|error| {
5081 QueryTaskExecutionFailure::new(
5082 "query_workflow_state_unavailable",
5083 format!("cannot decode workflow start input: {error}"),
5084 "QueryWorkflowStateUnavailable",
5085 )
5086 })?;
5087 let workflow_input = workflow_input_typed.clone().into_json().map_err(|error| {
5088 QueryTaskExecutionFailure::new(
5089 "query_workflow_state_unavailable",
5090 format!("cannot project workflow start input: {error}"),
5091 "QueryWorkflowStateUnavailable",
5092 )
5093 })?;
5094 hydrate_query_history_from_export(&mut task).map_err(|error| {
5095 QueryTaskExecutionFailure::new(
5096 "query_workflow_state_unavailable",
5097 format!("cannot restore query history snapshot: {error}"),
5098 "QueryWorkflowStateUnavailable",
5099 )
5100 })?;
5101 enrich_query_history_from_export(&mut task).map_err(|error| {
5102 QueryTaskExecutionFailure::new(
5103 "query_workflow_state_unavailable",
5104 format!("cannot restore compact query history payloads: {error}"),
5105 "QueryWorkflowStateUnavailable",
5106 )
5107 })?;
5108 let signal_events = query_signal_events(&task).map_err(|error| {
5109 QueryTaskExecutionFailure::new(
5110 "query_workflow_state_unavailable",
5111 format!("cannot decode committed workflow signals: {error}"),
5112 "QueryWorkflowStateUnavailable",
5113 )
5114 })?;
5115 let history_events = Arc::new(std::mem::take(&mut task.history_events));
5116 let context = QueryContext {
5117 workflow_id: task.workflow_id,
5118 run_id: task.run_id,
5119 workflow_type: task.workflow_type.clone(),
5120 run_status: task.run_status,
5121 workflow_input,
5122 workflow_input_avro_value: workflow_input_typed.clone(),
5123 history_events: Arc::clone(&history_events),
5124 signal_events: Arc::new(signal_events),
5125 };
5126
5127 let future = match query {
5128 RegisteredQuery::Snapshot(handler) => handler(context, args),
5129 RegisteredQuery::Replayed {
5130 state_type,
5131 handler,
5132 } => {
5133 let workflow = self
5134 .workflows
5135 .get(&task.workflow_type)
5136 .expect("workflow registration was checked above");
5137 if workflow.state_type != Some(*state_type) {
5138 return Err(QueryTaskExecutionFailure::new(
5139 "query_workflow_state_unavailable",
5140 "replayed query state type does not match its workflow registration",
5141 "QueryWorkflowStateUnavailable",
5142 ));
5143 }
5144 let replay = workflow.replay.as_ref().ok_or_else(|| {
5145 QueryTaskExecutionFailure::new(
5146 "query_workflow_state_unavailable",
5147 format!(
5148 "workflow type {:?} is not registered for instance-state replay",
5149 task.workflow_type
5150 ),
5151 "QueryWorkflowStateUnavailable",
5152 )
5153 })?;
5154 let workflow_state = Arc::new(Mutex::new(
5155 WorkflowState::new_with_identity(
5156 history_events.as_ref().clone(),
5157 context.workflow_id.clone(),
5158 context.run_id.clone(),
5159 self.task_queue.clone(),
5160 task.payload_codec,
5161 None,
5162 )
5163 .map_err(|error| {
5164 QueryTaskExecutionFailure::new(
5165 "query_workflow_state_unavailable",
5166 format!("workflow replay failed before query: {error}"),
5167 "QueryWorkflowStateUnavailable",
5168 )
5169 })?,
5170 ));
5171 let workflow_context = WorkflowContext {
5172 state: workflow_state,
5173 };
5174 let mut invocation = replay(workflow_context.clone(), workflow_input_typed.clone());
5175 let mut cx = TaskContext::from_waker(noop_waker_ref());
5176 match invocation.future.as_mut().poll(&mut cx) {
5177 Poll::Ready(Ok(_)) => {
5178 workflow_context
5179 .ensure_history_consumed()
5180 .map_err(|error| {
5181 QueryTaskExecutionFailure::new(
5182 "query_workflow_state_unavailable",
5183 format!("workflow replay failed before query: {error}"),
5184 "QueryWorkflowStateUnavailable",
5185 )
5186 })?;
5187 }
5188 Poll::Ready(Err(error)) => {
5189 return Err(QueryTaskExecutionFailure::new(
5190 "query_workflow_state_unavailable",
5191 format!("workflow replay failed before query: {error}"),
5192 "QueryWorkflowStateUnavailable",
5193 ));
5194 }
5195 Poll::Pending => {
5196 let commands = workflow_context.take_commands().map_err(|error| {
5197 QueryTaskExecutionFailure::new(
5198 "query_workflow_state_unavailable",
5199 format!("workflow replay failed before query: {error}"),
5200 "QueryWorkflowStateUnavailable",
5201 )
5202 })?;
5203 if commands.is_empty()
5204 && !workflow_context
5205 .matched_recorded_pending()
5206 .map_err(|error| {
5207 QueryTaskExecutionFailure::new(
5208 "query_workflow_state_unavailable",
5209 format!("workflow replay failed before query: {error}"),
5210 "QueryWorkflowStateUnavailable",
5211 )
5212 })?
5213 {
5214 return Err(QueryTaskExecutionFailure::new(
5215 "query_workflow_state_unavailable",
5216 "workflow replay yielded without a durable command",
5217 "QueryWorkflowStateUnavailable",
5218 ));
5219 }
5220 }
5221 }
5222 let state = (invocation.snapshot)().map_err(|error| {
5223 QueryTaskExecutionFailure::new(
5224 "query_workflow_state_unavailable",
5225 format!("cannot snapshot replayed workflow state: {error}"),
5226 "QueryWorkflowStateUnavailable",
5227 )
5228 })?;
5229 handler(context, state, args).map_err(|message| {
5230 QueryTaskExecutionFailure::new(
5231 "query_workflow_state_unavailable",
5232 message,
5233 "QueryWorkflowStateUnavailable",
5234 )
5235 })?
5236 }
5237 };
5238
5239 future.await.map_err(|error| {
5240 QueryTaskExecutionFailure::new("query_rejected", error.to_string(), "QueryFailed")
5241 })
5242 }
5243
5244 fn execute_workflow_task(&self, task: WorkflowTask) -> Result<Vec<Value>> {
5245 if let Some(update_id) = task
5246 .workflow_update_id
5247 .as_deref()
5248 .filter(|update_id| !update_id.is_empty())
5249 {
5250 return self.execute_update_task(&task, update_id);
5251 }
5252
5253 let workflow = self
5254 .workflows
5255 .get(&task.workflow_type)
5256 .ok_or_else(|| Error::WorkflowNotRegistered(task.workflow_type.clone()))?;
5257 let input = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
5258 let resume_signal = decode_resume_signal(&task)?;
5259 let history_budget = WorkflowHistoryBudget {
5260 event_count: task
5261 .total_history_events
5262 .unwrap_or_else(|| u64::try_from(task.history_events.len()).unwrap_or(u64::MAX)),
5263 size_bytes: task.history_size_bytes,
5264 continue_as_new_recommended: task.continue_as_new_recommended.unwrap_or(false),
5265 pressure: task.history_budget_pressure.clone(),
5266 };
5267 let mut workflow_state = WorkflowState::new_with_identity(
5268 task.history_events,
5269 task.workflow_id,
5270 task.run_id,
5271 self.task_queue.clone(),
5272 task.payload_codec.clone(),
5273 resume_signal,
5274 )?;
5275 workflow_state.history_budget = history_budget;
5276 let state = Arc::new(Mutex::new(workflow_state));
5277 let ctx = WorkflowContext { state };
5278 let mut future = (workflow.execute)(ctx.clone(), input);
5279 let mut cx = TaskContext::from_waker(noop_waker_ref());
5280
5281 match future.as_mut().poll(&mut cx) {
5282 Poll::Ready(Ok(result)) => {
5283 ctx.ensure_history_consumed()?;
5284 let result = encode_typed_envelope(&result, &task.payload_codec)?;
5285 let mut commands = ctx.take_commands()?;
5286 commands.push(json!({
5287 "type": "complete_workflow",
5288 "result": result
5289 }));
5290 Ok(commands)
5291 }
5292 Poll::Ready(Err(error)) => {
5293 if let Error::ContinueAsNew(request) = error {
5294 let mut commands = ctx.take_commands()?;
5295 if let Some(command) = ctx.continue_as_new_command(request)? {
5296 commands.push(command);
5297 }
5298 ctx.ensure_history_consumed()?;
5299 return Ok(commands);
5300 }
5301 ctx.ensure_history_consumed()?;
5304 if workflow_task_integrity_error(&error) {
5305 return Err(error);
5310 }
5311 let mut commands = ctx.take_commands()?;
5312 commands.push(workflow_failure_command(&error));
5313 Ok(commands)
5314 }
5315 Poll::Pending => {
5316 let commands = ctx.take_commands()?;
5317 if commands.is_empty() && !ctx.matched_recorded_pending()? {
5318 Err(Error::WorkflowYieldedWithoutCommand)
5319 } else {
5320 Ok(commands)
5321 }
5322 }
5323 }
5324 }
5325
5326 fn execute_update_task(&self, task: &WorkflowTask, update_id: &str) -> Result<Vec<Value>> {
5327 if !self.workflows.contains_key(&task.workflow_type) {
5328 return Err(Error::WorkflowNotRegistered(task.workflow_type.clone()));
5329 }
5330
5331 let accepted = task.history_events.iter().rev().find_map(|event| {
5332 (event.event_type == "UpdateAccepted"
5333 && event.payload.get("update_id").and_then(Value::as_str) == Some(update_id))
5334 .then_some(&event.payload)
5335 });
5336 let update_name = accepted
5337 .and_then(|payload| payload.get("update_name"))
5338 .and_then(Value::as_str)
5339 .or(task.update_name.as_deref())
5340 .unwrap_or_default();
5341 let Some(handler) = self
5342 .updates
5343 .get(&task.workflow_type)
5344 .and_then(|handlers| handlers.get(update_name))
5345 else {
5346 return Ok(vec![json!({
5347 "type": "fail_update",
5348 "update_id": update_id,
5349 "message": format!(
5350 "no update handler is registered for {}.{update_name}",
5351 task.workflow_type
5352 ),
5353 "exception_type": "UnknownUpdate",
5354 "non_retryable": true,
5355 })]);
5356 };
5357 let arguments = accepted
5358 .and_then(|payload| payload.get("arguments"))
5359 .or(task.arguments.as_ref());
5360 let arguments = decode_task_avro_arguments(arguments, &task.payload_codec)?;
5361 let context = QueryContext {
5362 workflow_id: task.workflow_id.clone(),
5363 run_id: task.run_id.clone(),
5364 workflow_type: task.workflow_type.clone(),
5365 run_status: Some("running".to_string()),
5366 workflow_input: Value::Null,
5367 workflow_input_avro_value: AvroValue::Null,
5368 history_events: Arc::new(task.history_events.clone()),
5369 signal_events: Arc::new(Vec::new()),
5370 };
5371 let mut future = handler(context, arguments);
5372 let mut cx = TaskContext::from_waker(noop_waker_ref());
5373
5374 match future.as_mut().poll(&mut cx) {
5375 Poll::Ready(Ok(result)) => Ok(vec![json!({
5376 "type": "complete_update",
5377 "update_id": update_id,
5378 "result": encode_typed_envelope(&result, &task.payload_codec)?,
5379 })]),
5380 Poll::Ready(Err(error)) => Ok(vec![json!({
5381 "type": "fail_update",
5382 "update_id": update_id,
5383 "message": error.to_string(),
5384 "exception_type": "UpdateFailed",
5385 "non_retryable": true,
5386 })]),
5387 Poll::Pending => Err(Error::WorkflowYieldedWithoutCommand),
5388 }
5389 }
5390
5391 async fn execute_activity_task(&self, task: ActivityTask) -> Result<AvroValue> {
5392 let handler = self
5393 .activities
5394 .get(&task.activity_type)
5395 .ok_or_else(|| Error::ActivityNotRegistered(task.activity_type.clone()))?;
5396 let args = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
5397 let attempt_id = task
5398 .activity_attempt_id
5399 .clone()
5400 .or(task.attempt_id.clone())
5401 .unwrap_or_default();
5402 let lease_owner = task
5403 .lease_owner
5404 .clone()
5405 .unwrap_or_else(|| self.worker_id.clone());
5406 let ctx = ActivityContext {
5407 client: self.client.clone(),
5408 task_id: task.task_id,
5409 activity_attempt_id: attempt_id,
5410 lease_owner,
5411 activity_type: task.activity_type,
5412 attempt_number: task.attempt_number,
5413 task_queue: self.task_queue.clone(),
5414 worker_id: self.worker_id.clone(),
5415 };
5416
5417 handler(ctx, args).await
5418 }
5419}
5420
5421fn poller_result(
5422 kind: &str,
5423 result: std::result::Result<Result<()>, tokio::task::JoinError>,
5424) -> Result<()> {
5425 match result {
5426 Ok(result) => result,
5427 Err(error) => Err(Error::WorkerLoop(format!(
5428 "{kind} poller join error: {error}"
5429 ))),
5430 }
5431}
5432
5433fn optional_poller_result(
5434 kind: &str,
5435 result: Option<std::result::Result<Result<()>, tokio::task::JoinError>>,
5436) -> Result<()> {
5437 match result {
5438 Some(result) => poller_result(kind, result),
5439 None => Ok(()),
5440 }
5441}
5442
5443async fn join_pollers(
5444 workflow_poller: Option<tokio::task::JoinHandle<Result<()>>>,
5445 activity_poller: Option<tokio::task::JoinHandle<Result<()>>>,
5446 query_poller: Option<tokio::task::JoinHandle<Result<()>>>,
5447) -> Result<()> {
5448 let mut first_error = None;
5449
5450 if let Some(handle) = workflow_poller {
5451 if let Err(error) = poller_result("workflow", handle.await) {
5452 first_error.get_or_insert(error);
5453 }
5454 }
5455
5456 if let Some(handle) = activity_poller {
5457 if let Err(error) = poller_result("activity", handle.await) {
5458 first_error.get_or_insert(error);
5459 }
5460 }
5461
5462 if let Some(handle) = query_poller {
5463 if let Err(error) = poller_result("query", handle.await) {
5464 first_error.get_or_insert(error);
5465 }
5466 }
5467
5468 if let Some(error) = first_error {
5469 Err(error)
5470 } else {
5471 Ok(())
5472 }
5473}
5474
5475fn default_worker_id() -> String {
5476 let millis = SystemTime::now()
5477 .duration_since(UNIX_EPOCH)
5478 .unwrap_or_default()
5479 .as_millis();
5480 format!("rust-worker-{}-{millis}", std::process::id())
5481}
5482
5483fn percent_encode_path_segment(segment: &str) -> String {
5484 const HEX: &[u8; 16] = b"0123456789ABCDEF";
5485 let mut encoded = String::with_capacity(segment.len());
5486
5487 for byte in segment.bytes() {
5488 if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') {
5489 encoded.push(char::from(byte));
5490 } else {
5491 encoded.push('%');
5492 encoded.push(char::from(HEX[(byte >> 4) as usize]));
5493 encoded.push(char::from(HEX[(byte & 0x0f) as usize]));
5494 }
5495 }
5496
5497 encoded
5498}
5499
5500fn unique_request_id(prefix: &str) -> String {
5501 let nanos = SystemTime::now()
5502 .duration_since(UNIX_EPOCH)
5503 .unwrap_or_default()
5504 .as_nanos();
5505 format!("{prefix}-{}-{nanos}", std::process::id())
5506}
5507
5508#[derive(Debug)]
5509struct QueryTaskExecutionFailure {
5510 reason: String,
5511 message: String,
5512 failure_type: String,
5513}
5514
5515impl QueryTaskExecutionFailure {
5516 fn new(
5517 reason: impl Into<String>,
5518 message: impl Into<String>,
5519 failure_type: impl Into<String>,
5520 ) -> Self {
5521 Self {
5522 reason: reason.into(),
5523 message: message.into(),
5524 failure_type: failure_type.into(),
5525 }
5526 }
5527}
5528
5529#[derive(Clone, Debug)]
5535pub struct WorkflowInstance<S> {
5536 state: Arc<Mutex<S>>,
5537}
5538
5539impl<S> WorkflowInstance<S> {
5540 fn new(state: S) -> Self {
5541 Self {
5542 state: Arc::new(Mutex::new(state)),
5543 }
5544 }
5545
5546 pub fn read<R>(&self, reader: impl FnOnce(&S) -> R) -> Result<R> {
5548 let state = self
5549 .state
5550 .lock()
5551 .map_err(|_| Error::WorkflowStatePoisoned)?;
5552 Ok(reader(&state))
5553 }
5554
5555 pub fn update<R>(&self, transition: impl FnOnce(&mut S) -> R) -> Result<R> {
5557 let mut state = self
5558 .state
5559 .lock()
5560 .map_err(|_| Error::WorkflowStatePoisoned)?;
5561 Ok(transition(&mut state))
5562 }
5563}
5564
5565impl<S: Clone> WorkflowInstance<S> {
5566 fn snapshot(&self) -> Result<S> {
5567 self.read(Clone::clone)
5568 }
5569}
5570
5571#[derive(Clone, Debug)]
5572pub struct WorkflowContext {
5573 state: Arc<Mutex<WorkflowState>>,
5574}
5575
5576impl WorkflowContext {
5577 pub fn workflow_identity(&self) -> Result<WorkflowIdentity> {
5579 let state = self
5580 .state
5581 .lock()
5582 .map_err(|_| Error::WorkflowStatePoisoned)?;
5583 Ok(WorkflowIdentity {
5584 workflow_id: state.workflow_id.clone(),
5585 run_id: state.run_id.clone(),
5586 })
5587 }
5588
5589 pub fn history_budget(&self) -> Result<WorkflowHistoryBudget> {
5591 let state = self
5592 .state
5593 .lock()
5594 .map_err(|_| Error::WorkflowStatePoisoned)?;
5595 Ok(state.history_budget.clone())
5596 }
5597
5598 pub fn continue_as_new<T: Serialize>(&self, args: T) -> Result<Value> {
5604 self.continue_as_new_with_options(ContinueAsNewOptions::new(), args)
5605 }
5606
5607 pub fn continue_as_new_with_options<T: Serialize>(
5609 &self,
5610 options: ContinueAsNewOptions,
5611 args: T,
5612 ) -> Result<Value> {
5613 options.validate()?;
5614 Err(Error::ContinueAsNew(ContinueAsNewRequest {
5615 arguments: normalize_avro_arguments(AvroValue::from_serialize(&args)?),
5616 options,
5617 }))
5618 }
5619
5620 pub fn activity<T: Serialize>(
5621 &self,
5622 activity_type: impl Into<String>,
5623 args: T,
5624 ) -> ActivityCall {
5625 self.activity_with_options(activity_type, ActivityOptions::new(), args)
5626 }
5627
5628 pub fn activity_on_queue<T, Q>(
5629 &self,
5630 activity_type: impl Into<String>,
5631 task_queue: Option<Q>,
5632 args: T,
5633 ) -> ActivityCall
5634 where
5635 T: Serialize,
5636 Q: Into<String>,
5637 {
5638 let mut options = ActivityOptions::new();
5639 options.task_queue = task_queue.map(Into::into);
5640 self.activity_with_options(activity_type, options, args)
5641 }
5642
5643 pub fn activity_with_options<T: Serialize>(
5681 &self,
5682 activity_type: impl Into<String>,
5683 options: ActivityOptions,
5684 args: T,
5685 ) -> ActivityCall {
5686 ActivityCall {
5687 ctx: self.clone(),
5688 activity_type: activity_type.into(),
5689 options,
5690 args: Some(AvroValue::from_serialize(&args)),
5691 scheduled: false,
5692 }
5693 }
5694
5695 pub async fn activity_avro_value<T: Serialize>(
5696 &self,
5697 activity_type: impl Into<String>,
5698 args: T,
5699 ) -> Result<AvroValue> {
5700 let mut call = self.activity(activity_type, args);
5701 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
5702 }
5703
5704 pub async fn activity_avro_value_with_options<T: Serialize>(
5705 &self,
5706 activity_type: impl Into<String>,
5707 options: ActivityOptions,
5708 args: T,
5709 ) -> Result<AvroValue> {
5710 let mut call = self.activity_with_options(activity_type, options, args);
5711 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
5712 }
5713
5714 pub fn wait_signal(&self, signal_name: impl Into<String>) -> SignalCall {
5715 SignalCall {
5716 ctx: self.clone(),
5717 signal_name: signal_name.into(),
5718 opened_wait: false,
5719 matched_pending: false,
5720 }
5721 }
5722
5723 pub async fn wait_signal_avro_value(
5724 &self,
5725 signal_name: impl Into<String>,
5726 ) -> Result<Vec<AvroValue>> {
5727 let mut call = self.wait_signal(signal_name);
5728 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
5729 }
5730
5731 pub fn sleep(&self, duration: Duration) -> TimerCall {
5752 let delay_seconds = duration
5753 .as_secs()
5754 .checked_add(u64::from(duration.subsec_nanos() > 0));
5755 TimerCall {
5756 ctx: self.clone(),
5757 delay_seconds,
5758 scheduled: false,
5759 matched_pending: false,
5760 }
5761 }
5762
5763 pub fn start_timer(&self, duration: Duration) -> TimerCall {
5765 self.sleep(duration)
5766 }
5767
5768 pub fn side_effect<T, F>(&self, callback: F) -> Result<T>
5775 where
5776 T: Serialize + DeserializeOwned,
5777 F: FnOnce() -> T,
5778 {
5779 {
5780 let mut state = self
5781 .state
5782 .lock()
5783 .map_err(|_| Error::WorkflowStatePoisoned)?;
5784 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
5785 return match recorded {
5786 RecordedCommand::SideEffect { sequence, value } => {
5787 state.command_cursor += 1;
5788 value.deserialize().map_err(|error| {
5789 Error::NonDeterministicReplay(ReplayFailure::new(
5790 "side_effect_type_mismatch",
5791 Some(sequence),
5792 Some(std::any::type_name::<T>().to_string()),
5793 Some(error.to_string()),
5794 "recorded side-effect value is incompatible with the requested Rust type",
5795 ))
5796 })
5797 }
5798 other => Err(command_mismatch(&other, "side effect")),
5799 };
5800 }
5801 }
5802
5803 let value = callback();
5804 let avro_value = AvroValue::from_serialize(&value)?;
5805 let mut state = self
5806 .state
5807 .lock()
5808 .map_err(|_| Error::WorkflowStatePoisoned)?;
5809 let result = encode_typed_envelope(&avro_value, &state.payload_codec)?;
5810 state.commands.push(json!({
5811 "type": "record_side_effect",
5812 "result": result,
5813 }));
5814 Ok(value)
5815 }
5816
5817 pub fn side_effect_avro_value<F>(&self, callback: F) -> Result<AvroValue>
5819 where
5820 F: FnOnce() -> AvroValue,
5821 {
5822 {
5823 let mut state = self
5824 .state
5825 .lock()
5826 .map_err(|_| Error::WorkflowStatePoisoned)?;
5827 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
5828 return match recorded {
5829 RecordedCommand::SideEffect { value, .. } => {
5830 state.command_cursor += 1;
5831 Ok(value)
5832 }
5833 other => Err(command_mismatch(&other, "side effect")),
5834 };
5835 }
5836 }
5837
5838 let value = callback();
5839 let mut state = self
5840 .state
5841 .lock()
5842 .map_err(|_| Error::WorkflowStatePoisoned)?;
5843 let result = encode_typed_envelope(&value, &state.payload_codec)?;
5844 state.commands.push(json!({
5845 "type": "record_side_effect",
5846 "result": result,
5847 }));
5848 Ok(value)
5849 }
5850
5851 pub fn uuid_v4(&self) -> Result<Uuid> {
5853 self.side_effect(Uuid::new_v4)
5854 }
5855
5856 pub fn get_version(
5859 &self,
5860 change_id: impl Into<String>,
5861 min_supported: i32,
5862 max_supported: i32,
5863 ) -> Result<i32> {
5864 let change_id = change_id.into();
5865 if change_id.trim().is_empty() {
5866 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5867 "version_change_id_invalid",
5868 None,
5869 Some("non-empty change ID".to_string()),
5870 Some(change_id),
5871 "version markers require a stable non-empty change ID",
5872 )));
5873 }
5874 if min_supported > max_supported {
5875 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5876 "version_range_invalid",
5877 None,
5878 Some("min_supported <= max_supported".to_string()),
5879 Some(format!("{min_supported}..={max_supported}")),
5880 "version marker supported range is invalid",
5881 )));
5882 }
5883
5884 let mut state = self
5885 .state
5886 .lock()
5887 .map_err(|_| Error::WorkflowStatePoisoned)?;
5888 if let Some((version, sequence)) = state.version_markers.get(&change_id).copied() {
5889 ensure_version_supported(&change_id, version, min_supported, max_supported, sequence)?;
5890 return Ok(version);
5891 }
5892
5893 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
5894 return match recorded {
5895 RecordedCommand::VersionMarker {
5896 sequence,
5897 change_id: recorded_change_id,
5898 version,
5899 ..
5900 } => {
5901 if recorded_change_id != change_id {
5902 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5903 "version_change_id_mismatch",
5904 Some(sequence),
5905 Some(recorded_change_id),
5906 Some(change_id),
5907 "recorded version marker change ID differs from current workflow code",
5908 )));
5909 }
5910 ensure_version_supported(
5911 &change_id,
5912 version,
5913 min_supported,
5914 max_supported,
5915 sequence,
5916 )?;
5917 state.command_cursor += 1;
5918 state.version_markers.insert(change_id, (version, sequence));
5919 Ok(version)
5920 }
5921 other => Err(command_mismatch(
5922 &other,
5923 format!("version marker:{change_id}"),
5924 )),
5925 };
5926 }
5927
5928 let version = max_supported;
5929 state.commands.push(json!({
5930 "type": "record_version_marker",
5931 "change_id": change_id,
5932 "version": version,
5933 "min_supported": min_supported,
5934 "max_supported": max_supported,
5935 }));
5936 state.version_markers.insert(change_id, (version, 0));
5939 Ok(version)
5940 }
5941
5942 pub fn patched(&self, change_id: impl Into<String>) -> Result<bool> {
5944 Ok(self.get_version(change_id, -1, 1)? == 1)
5945 }
5946
5947 pub fn deprecate_patch(&self, change_id: impl Into<String>) -> Result<()> {
5949 self.get_version(change_id, -1, 1).map(|_| ())
5950 }
5951
5952 pub fn start_child_workflow<T: Serialize>(
5979 &self,
5980 workflow_type: impl Into<String>,
5981 options: ChildWorkflowOptions,
5982 args: T,
5983 ) -> ChildWorkflowCall {
5984 ChildWorkflowCall {
5985 ctx: self.clone(),
5986 workflow_type: workflow_type.into(),
5987 options,
5988 args: Some(AvroValue::from_serialize(&args)),
5989 scheduled: false,
5990 matched_pending: false,
5991 }
5992 }
5993
5994 pub async fn start_child_workflow_avro_value<T: Serialize>(
5995 &self,
5996 workflow_type: impl Into<String>,
5997 options: ChildWorkflowOptions,
5998 args: T,
5999 ) -> Result<ChildWorkflowAvroResult> {
6000 let mut call = self.start_child_workflow(workflow_type, options, args);
6001 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
6002 }
6003
6004 fn take_commands(&self) -> Result<Vec<Value>> {
6005 let mut state = self
6006 .state
6007 .lock()
6008 .map_err(|_| Error::WorkflowStatePoisoned)?;
6009 Ok(std::mem::take(&mut state.commands))
6010 }
6011
6012 fn continue_as_new_command(&self, request: ContinueAsNewRequest) -> Result<Option<Value>> {
6013 let mut state = self
6014 .state
6015 .lock()
6016 .map_err(|_| Error::WorkflowStatePoisoned)?;
6017
6018 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6019 return Err(command_mismatch(&recorded, "continue as new"));
6020 }
6021 if state.recorded_continue_as_new_sequence.is_some() {
6022 state.continue_as_new_consumed = true;
6023 return Ok(None);
6024 }
6025
6026 let arguments = encode_typed_envelope(&request.arguments, &state.payload_codec)?;
6027 let mut command = serde_json::Map::from_iter([
6028 ("type".to_string(), json!("continue_as_new")),
6029 ("arguments".to_string(), arguments),
6030 ("queue".to_string(), json!(state.task_queue.clone())),
6031 ]);
6032 if let Some(workflow_type) = request.options.workflow_type {
6033 command.insert("workflow_type".to_string(), json!(workflow_type));
6034 }
6035 if let Some(task_queue) = request.options.task_queue {
6036 command.insert("queue".to_string(), json!(task_queue));
6037 }
6038 Ok(Some(Value::Object(command)))
6039 }
6040
6041 fn matched_recorded_pending(&self) -> Result<bool> {
6042 let state = self
6043 .state
6044 .lock()
6045 .map_err(|_| Error::WorkflowStatePoisoned)?;
6046 Ok(state.matched_recorded_pending)
6047 }
6048
6049 fn ensure_history_consumed(&self) -> Result<()> {
6050 let state = self
6051 .state
6052 .lock()
6053 .map_err(|_| Error::WorkflowStatePoisoned)?;
6054 if let Some(command) = state.recorded_commands.get(state.command_cursor) {
6055 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
6056 "recorded_commands_unconsumed",
6057 Some(command.sequence()),
6058 Some(command.shape().to_string()),
6059 Some("workflow completion".to_string()),
6060 "workflow completed before consuming all recorded durable commands",
6061 )));
6062 }
6063 if let Some(sequence) = state
6064 .recorded_continue_as_new_sequence
6065 .filter(|_| !state.continue_as_new_consumed)
6066 {
6067 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
6068 "recorded_continue_as_new_unconsumed",
6069 Some(sequence),
6070 Some("continue as new".to_string()),
6071 Some("workflow completion".to_string()),
6072 "workflow completed without consuming its recorded continue-as-new transition",
6073 )));
6074 }
6075 Ok(())
6076 }
6077}
6078
6079#[derive(Debug)]
6080struct WorkflowState {
6081 workflow_id: Option<String>,
6082 run_id: Option<String>,
6083 task_queue: String,
6084 payload_codec: String,
6085 history_budget: WorkflowHistoryBudget,
6086 resume_signal: Option<ResumeSignal>,
6087 recorded_commands: Vec<RecordedCommand>,
6088 recorded_continue_as_new_sequence: Option<u64>,
6089 continue_as_new_consumed: bool,
6090 command_cursor: usize,
6091 matched_recorded_pending: bool,
6092 version_markers: HashMap<String, (i32, u64)>,
6093 commands: Vec<Value>,
6094}
6095
6096impl WorkflowState {
6097 #[cfg(test)]
6098 fn new(
6099 history: Vec<HistoryEvent>,
6100 task_queue: String,
6101 payload_codec: String,
6102 resume_signal: Option<ResumeSignal>,
6103 ) -> Result<Self> {
6104 Self::new_with_identity(
6105 history,
6106 None,
6107 None,
6108 task_queue,
6109 payload_codec,
6110 resume_signal,
6111 )
6112 }
6113
6114 fn new_with_identity(
6115 history: Vec<HistoryEvent>,
6116 workflow_id: Option<String>,
6117 run_id: Option<String>,
6118 task_queue: String,
6119 payload_codec: String,
6120 resume_signal: Option<ResumeSignal>,
6121 ) -> Result<Self> {
6122 let recorded_commands = recorded_commands(
6123 &history,
6124 &payload_codec,
6125 WorkflowIdentity {
6126 workflow_id: workflow_id.clone(),
6127 run_id: run_id.clone(),
6128 },
6129 )?;
6130 let recorded_continue_as_new = history
6131 .iter()
6132 .filter(|event| event.event_type == "WorkflowContinuedAsNew")
6133 .collect::<Vec<_>>();
6134 if recorded_continue_as_new.len() > 1 {
6135 return Err(invalid_recorded_history(
6136 "duplicate_continue_as_new_transition",
6137 recorded_continue_as_new
6138 .last()
6139 .and_then(|event| durable_event_sequence(event))
6140 .unwrap_or(0),
6141 "one WorkflowContinuedAsNew event",
6142 &format!(
6143 "{} WorkflowContinuedAsNew events",
6144 recorded_continue_as_new.len()
6145 ),
6146 "workflow history records one continue-as-new transition more than once",
6147 ));
6148 }
6149 let recorded_continue_as_new_sequence = recorded_continue_as_new
6150 .first()
6151 .map(|event| {
6152 durable_event_sequence(event).ok_or_else(|| {
6153 Error::NonDeterministicReplay(ReplayFailure::new(
6154 "continue_as_new_sequence_missing",
6155 None,
6156 Some("recorded transition sequence".to_string()),
6157 Some("missing sequence".to_string()),
6158 "WorkflowContinuedAsNew history is missing its recorded sequence",
6159 ))
6160 })
6161 })
6162 .transpose()?;
6163 let event_count = u64::try_from(history.len()).unwrap_or(u64::MAX);
6164 Ok(Self {
6165 workflow_id,
6166 run_id,
6167 task_queue,
6168 payload_codec,
6169 history_budget: WorkflowHistoryBudget {
6170 event_count,
6171 ..WorkflowHistoryBudget::default()
6172 },
6173 resume_signal,
6174 recorded_commands,
6175 recorded_continue_as_new_sequence,
6176 continue_as_new_consumed: false,
6177 command_cursor: 0,
6178 matched_recorded_pending: false,
6179 version_markers: HashMap::new(),
6180 commands: Vec::new(),
6181 })
6182 }
6183}
6184
6185#[derive(Clone, Debug)]
6186enum RecordedCommand {
6187 Activity {
6188 sequence: u64,
6189 activity_type: Option<String>,
6190 options: Option<RecordedActivityOptions>,
6191 outcome: Option<ActivityOutcome>,
6192 },
6193 Timer {
6194 sequence: u64,
6195 delay_seconds: u64,
6196 fired: bool,
6197 },
6198 ChildWorkflow {
6199 sequence: u64,
6200 workflow_type: Option<String>,
6201 outcome: Option<ChildWorkflowOutcome>,
6202 },
6203 SignalWait {
6204 sequence: u64,
6205 signal_name: String,
6206 value: Option<Vec<AvroValue>>,
6207 },
6208 SideEffect {
6209 sequence: u64,
6210 value: AvroValue,
6211 },
6212 VersionMarker {
6213 sequence: u64,
6214 change_id: String,
6215 version: i32,
6216 },
6217}
6218
6219#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
6220struct RecordedActivityOptions {
6221 task_queue: RecordedSnapshotValue<Option<String>>,
6222 execution_mode: RecordedSnapshotValue<Option<String>>,
6223 retry_policy: ActivityRetrySnapshot,
6224}
6225
6226#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
6227enum RecordedSnapshotValue<T> {
6228 Unknown,
6230 Known(T),
6231}
6232
6233impl<T: PartialEq> RecordedSnapshotValue<T> {
6234 fn matches_current(&self, current: &Self) -> bool {
6235 match self {
6236 Self::Unknown => true,
6237 Self::Known(recorded) => matches!(current, Self::Known(value) if value == recorded),
6238 }
6239 }
6240}
6241
6242#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
6243struct ActivityRetrySnapshot {
6244 snapshot_version: RecordedSnapshotValue<Option<u64>>,
6245 max_attempts: RecordedSnapshotValue<Option<u64>>,
6246 backoff_seconds: RecordedSnapshotValue<Vec<u64>>,
6247 start_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
6248 schedule_to_start_timeout: RecordedSnapshotValue<Option<u64>>,
6249 schedule_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
6250 heartbeat_timeout: RecordedSnapshotValue<Option<u64>>,
6251 non_retryable_error_types: RecordedSnapshotValue<Vec<String>>,
6252}
6253
6254impl ActivityRetrySnapshot {
6255 fn matches_current(&self, current: &Self) -> bool {
6256 self.snapshot_version
6257 .matches_current(¤t.snapshot_version)
6258 && self.max_attempts.matches_current(¤t.max_attempts)
6259 && self
6260 .backoff_seconds
6261 .matches_current(¤t.backoff_seconds)
6262 && self
6263 .start_to_close_timeout
6264 .matches_current(¤t.start_to_close_timeout)
6265 && self
6266 .schedule_to_start_timeout
6267 .matches_current(¤t.schedule_to_start_timeout)
6268 && self
6269 .schedule_to_close_timeout
6270 .matches_current(¤t.schedule_to_close_timeout)
6271 && self
6272 .heartbeat_timeout
6273 .matches_current(¤t.heartbeat_timeout)
6274 && self
6275 .non_retryable_error_types
6276 .matches_current(¤t.non_retryable_error_types)
6277 }
6278}
6279
6280fn recorded_optional_u64(
6281 object: Option<&serde_json::Map<String, Value>>,
6282 field: &str,
6283) -> RecordedSnapshotValue<Option<u64>> {
6284 match object.and_then(|object| object.get(field)) {
6285 None => RecordedSnapshotValue::Unknown,
6286 Some(Value::Null) => RecordedSnapshotValue::Known(None),
6287 Some(value) => RecordedSnapshotValue::Known(value_as_u64(value)),
6288 }
6289}
6290
6291fn recorded_optional_string(
6292 object: &serde_json::Map<String, Value>,
6293 field: &str,
6294) -> RecordedSnapshotValue<Option<String>> {
6295 match object.get(field) {
6296 None => RecordedSnapshotValue::Unknown,
6297 Some(Value::Null) => RecordedSnapshotValue::Known(None),
6298 Some(value) => RecordedSnapshotValue::Known(value.as_str().map(str::to_string)),
6299 }
6300}
6301
6302fn recorded_activity_retry_snapshot(policy: Option<&Value>) -> ActivityRetrySnapshot {
6303 let policy = policy.and_then(Value::as_object);
6304 let backoff_seconds = policy
6305 .and_then(|policy| policy.get("backoff_seconds"))
6306 .and_then(Value::as_array)
6307 .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
6308 .map_or(RecordedSnapshotValue::Unknown, RecordedSnapshotValue::Known);
6309 let mut non_retryable_error_types = Vec::new();
6310 for error_type in policy
6311 .and_then(|policy| policy.get("non_retryable_error_types"))
6312 .and_then(Value::as_array)
6313 .into_iter()
6314 .flatten()
6315 .filter_map(Value::as_str)
6316 .map(str::trim)
6317 .filter(|error_type| !error_type.is_empty())
6318 {
6319 if !non_retryable_error_types
6320 .iter()
6321 .any(|recorded| recorded == error_type)
6322 {
6323 non_retryable_error_types.push(error_type.to_string());
6324 }
6325 }
6326
6327 ActivityRetrySnapshot {
6328 snapshot_version: recorded_optional_u64(policy, "snapshot_version"),
6329 max_attempts: recorded_optional_u64(policy, "max_attempts"),
6330 backoff_seconds,
6331 start_to_close_timeout: recorded_optional_u64(policy, "start_to_close_timeout"),
6332 schedule_to_start_timeout: recorded_optional_u64(policy, "schedule_to_start_timeout"),
6333 schedule_to_close_timeout: recorded_optional_u64(policy, "schedule_to_close_timeout"),
6334 heartbeat_timeout: recorded_optional_u64(policy, "heartbeat_timeout"),
6335 non_retryable_error_types: if policy
6336 .is_some_and(|policy| policy.contains_key("non_retryable_error_types"))
6337 {
6338 RecordedSnapshotValue::Known(non_retryable_error_types)
6339 } else {
6340 RecordedSnapshotValue::Unknown
6341 },
6342 }
6343}
6344
6345fn current_activity_retry_snapshot(options: &ValidatedActivityOptions) -> ActivityRetrySnapshot {
6346 let policy = options.retry_policy.as_ref();
6347 let max_attempts = match policy.and_then(|policy| policy.get("max_attempts")) {
6348 Some(Value::Null) => None,
6349 Some(value) => value_as_u64(value),
6350 None => Some(1),
6351 };
6352 let backoff_seconds = policy
6353 .and_then(|policy| policy.get("backoff_seconds"))
6354 .and_then(Value::as_array)
6355 .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
6356 .unwrap_or_default();
6357 let non_retryable_error_types = policy
6358 .and_then(|policy| policy.get("non_retryable_error_types"))
6359 .and_then(Value::as_array)
6360 .into_iter()
6361 .flatten()
6362 .filter_map(Value::as_str)
6363 .map(str::to_string)
6364 .collect();
6365
6366 ActivityRetrySnapshot {
6367 snapshot_version: RecordedSnapshotValue::Known(Some(1)),
6368 max_attempts: RecordedSnapshotValue::Known(max_attempts),
6369 backoff_seconds: RecordedSnapshotValue::Known(backoff_seconds),
6370 start_to_close_timeout: RecordedSnapshotValue::Known(options.start_to_close_timeout),
6371 schedule_to_start_timeout: RecordedSnapshotValue::Known(options.schedule_to_start_timeout),
6372 schedule_to_close_timeout: RecordedSnapshotValue::Known(options.schedule_to_close_timeout),
6373 heartbeat_timeout: RecordedSnapshotValue::Known(options.heartbeat_timeout),
6374 non_retryable_error_types: RecordedSnapshotValue::Known(non_retryable_error_types),
6375 }
6376}
6377
6378fn activity_options_description(options: &RecordedActivityOptions) -> String {
6379 serde_json::to_string(options).unwrap_or_else(|_| format!("{options:?}"))
6380}
6381
6382impl RecordedCommand {
6383 fn sequence(&self) -> u64 {
6384 match self {
6385 Self::Activity { sequence, .. }
6386 | Self::Timer { sequence, .. }
6387 | Self::ChildWorkflow { sequence, .. }
6388 | Self::SignalWait { sequence, .. }
6389 | Self::SideEffect { sequence, .. }
6390 | Self::VersionMarker { sequence, .. } => *sequence,
6391 }
6392 }
6393
6394 fn shape(&self) -> &'static str {
6395 match self {
6396 Self::Activity { .. } => "activity",
6397 Self::Timer { .. } => "timer",
6398 Self::ChildWorkflow { .. } => "child workflow",
6399 Self::SignalWait { .. } => "signal wait",
6400 Self::SideEffect { .. } => "side effect",
6401 Self::VersionMarker { .. } => "version marker",
6402 }
6403 }
6404}
6405
6406fn ensure_version_supported(
6407 change_id: &str,
6408 version: i32,
6409 min_supported: i32,
6410 max_supported: i32,
6411 sequence: u64,
6412) -> Result<()> {
6413 if (min_supported..=max_supported).contains(&version) {
6414 return Ok(());
6415 }
6416 Err(Error::NonDeterministicReplay(ReplayFailure::new(
6417 "version_marker_incompatible_range",
6418 (sequence != 0).then_some(sequence),
6419 Some(format!("{min_supported}..={max_supported}")),
6420 Some(format!("{change_id}:{version}")),
6421 "recorded workflow version is outside the range supported by current code",
6422 )))
6423}
6424
6425#[derive(Clone, Debug)]
6426struct ResumeSignal {
6427 signal_name: String,
6428 arguments: Vec<AvroValue>,
6429}
6430
6431pub struct ActivityCall {
6432 ctx: WorkflowContext,
6433 activity_type: String,
6434 options: ActivityOptions,
6435 args: Option<Result<AvroValue>>,
6436 scheduled: bool,
6437}
6438
6439impl ActivityCall {
6440 fn poll_avro_value(
6441 mut self: Pin<&mut Self>,
6442 _cx: &mut TaskContext<'_>,
6443 ) -> Poll<Result<AvroValue>> {
6444 let ctx = self.ctx.clone();
6445 let mut state = match ctx.state.lock() {
6446 Ok(state) => state,
6447 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6448 };
6449
6450 if self.scheduled {
6451 return Poll::Pending;
6452 }
6453
6454 let options = match self.options.validate() {
6455 Ok(options) => options,
6456 Err(error) => {
6457 return Poll::Ready(Err(Error::InvalidActivityOptions(error)));
6458 }
6459 };
6460 let task_queue = options
6461 .task_queue
6462 .clone()
6463 .unwrap_or_else(|| state.task_queue.clone());
6464 let current_recorded_options = RecordedActivityOptions {
6465 task_queue: RecordedSnapshotValue::Known(Some(task_queue.clone())),
6466 execution_mode: RecordedSnapshotValue::Known(None),
6469 retry_policy: current_activity_retry_snapshot(&options),
6470 };
6471
6472 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6473 let sequence = recorded.sequence();
6474 match recorded {
6475 RecordedCommand::Activity {
6476 activity_type,
6477 options: recorded_options,
6478 outcome,
6479 ..
6480 } => {
6481 if let Some(recorded_type) = activity_type {
6482 if recorded_type != self.activity_type {
6483 return Poll::Ready(Err(Error::NonDeterministicReplay(
6484 ReplayFailure::new(
6485 "recorded_command_detail_mismatch",
6486 Some(sequence),
6487 Some(format!("activity:{recorded_type}")),
6488 Some(format!("activity:{}", self.activity_type)),
6489 "recorded activity type differs from the current workflow command",
6490 ),
6491 )));
6492 }
6493 }
6494 if let Some(recorded_options) = recorded_options {
6495 if !recorded_options
6496 .task_queue
6497 .matches_current(¤t_recorded_options.task_queue)
6498 {
6499 return Poll::Ready(Err(Error::NonDeterministicReplay(
6500 ReplayFailure::new(
6501 "activity_task_queue_mismatch",
6502 Some(sequence),
6503 Some(activity_options_description(&recorded_options)),
6504 Some(activity_options_description(¤t_recorded_options)),
6505 "recorded activity task queue differs from the current workflow command",
6506 ),
6507 )));
6508 }
6509 if !recorded_options
6510 .execution_mode
6511 .matches_current(¤t_recorded_options.execution_mode)
6512 {
6513 return Poll::Ready(Err(Error::NonDeterministicReplay(
6514 ReplayFailure::new(
6515 "activity_execution_mode_mismatch",
6516 Some(sequence),
6517 Some(activity_options_description(&recorded_options)),
6518 Some(activity_options_description(¤t_recorded_options)),
6519 "recorded activity execution mode differs from the current workflow command",
6520 ),
6521 )));
6522 }
6523 if !recorded_options
6524 .retry_policy
6525 .matches_current(¤t_recorded_options.retry_policy)
6526 {
6527 return Poll::Ready(Err(Error::NonDeterministicReplay(
6528 ReplayFailure::new(
6529 "activity_retry_policy_mismatch",
6530 Some(sequence),
6531 Some(activity_options_description(&recorded_options)),
6532 Some(activity_options_description(¤t_recorded_options)),
6533 "recorded activity retry policy differs from the current workflow command",
6534 ),
6535 )));
6536 }
6537 }
6538 state.command_cursor += 1;
6539 if let Some(outcome) = outcome {
6540 return Poll::Ready(outcome.map_err(Error::ActivityFailed));
6541 }
6542 state.matched_recorded_pending = true;
6543 self.scheduled = true;
6544 return Poll::Pending;
6545 }
6546 other => {
6547 return Poll::Ready(Err(command_mismatch(
6548 &other,
6549 format!("activity:{}", self.activity_type),
6550 )));
6551 }
6552 }
6553 }
6554
6555 if !self.scheduled {
6556 let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
6557 Ok(args) => args,
6558 Err(error) => return Poll::Ready(Err(error)),
6559 };
6560 let arguments = normalize_avro_arguments(args);
6561 let envelope = match encode_typed_envelope(&arguments, &state.payload_codec) {
6562 Ok(envelope) => envelope,
6563 Err(error) => return Poll::Ready(Err(error)),
6564 };
6565
6566 let mut command = serde_json::Map::from_iter([
6567 ("type".to_string(), json!("schedule_activity")),
6568 (
6569 "activity_type".to_string(),
6570 json!(self.activity_type.clone()),
6571 ),
6572 ("queue".to_string(), json!(task_queue)),
6573 ("arguments".to_string(), envelope),
6574 ]);
6575 for (field, value) in [
6576 ("start_to_close_timeout", options.start_to_close_timeout),
6577 (
6578 "schedule_to_start_timeout",
6579 options.schedule_to_start_timeout,
6580 ),
6581 (
6582 "schedule_to_close_timeout",
6583 options.schedule_to_close_timeout,
6584 ),
6585 ("heartbeat_timeout", options.heartbeat_timeout),
6586 ] {
6587 if let Some(value) = value {
6588 command.insert(field.to_string(), json!(value));
6589 }
6590 }
6591 if let Some(retry_policy) = options.retry_policy {
6592 command.insert("retry_policy".to_string(), retry_policy);
6593 }
6594 state.commands.push(Value::Object(command));
6595 self.scheduled = true;
6596 }
6597
6598 Poll::Pending
6599 }
6600}
6601
6602impl Future for ActivityCall {
6603 type Output = Result<Value>;
6604
6605 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6606 match self.poll_avro_value(cx) {
6607 Poll::Ready(Ok(value)) => Poll::Ready(value.into_json()),
6608 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
6609 Poll::Pending => Poll::Pending,
6610 }
6611 }
6612}
6613
6614pub struct TimerCall {
6616 ctx: WorkflowContext,
6617 delay_seconds: Option<u64>,
6618 scheduled: bool,
6619 matched_pending: bool,
6620}
6621
6622impl Future for TimerCall {
6623 type Output = Result<()>;
6624
6625 fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6626 if self.matched_pending {
6627 return Poll::Pending;
6628 }
6629
6630 let ctx = self.ctx.clone();
6631 let Some(requested_delay) = self.delay_seconds else {
6632 return Poll::Ready(Err(Error::TimerDurationOverflow));
6633 };
6634 let mut state = match ctx.state.lock() {
6635 Ok(state) => state,
6636 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6637 };
6638
6639 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6640 match recorded {
6641 RecordedCommand::Timer {
6642 sequence,
6643 delay_seconds,
6644 fired,
6645 ..
6646 } => {
6647 if delay_seconds != requested_delay {
6648 return Poll::Ready(Err(Error::NonDeterministicReplay(
6649 ReplayFailure::new(
6650 "timer_delay_mismatch",
6651 Some(sequence),
6652 Some(format!("timer:{delay_seconds}s")),
6653 Some(format!("timer:{requested_delay}s")),
6654 "recorded timer delay differs from the current workflow command",
6655 ),
6656 )));
6657 }
6658 state.command_cursor += 1;
6659 if fired {
6660 return Poll::Ready(Ok(()));
6661 }
6662 state.matched_recorded_pending = true;
6663 self.scheduled = true;
6664 self.matched_pending = true;
6665 return Poll::Pending;
6666 }
6667 other => return Poll::Ready(Err(command_mismatch(&other, "timer"))),
6668 }
6669 }
6670
6671 if !self.scheduled {
6672 state.commands.push(json!({
6673 "type": "start_timer",
6674 "delay_seconds": requested_delay,
6675 }));
6676 self.scheduled = true;
6677 }
6678
6679 Poll::Pending
6680 }
6681}
6682
6683pub struct ChildWorkflowCall {
6685 ctx: WorkflowContext,
6686 workflow_type: String,
6687 options: ChildWorkflowOptions,
6688 args: Option<Result<AvroValue>>,
6689 scheduled: bool,
6690 matched_pending: bool,
6691}
6692
6693impl ChildWorkflowCall {
6694 fn poll_avro_value(
6695 mut self: Pin<&mut Self>,
6696 _cx: &mut TaskContext<'_>,
6697 ) -> Poll<Result<ChildWorkflowAvroResult>> {
6698 if self.matched_pending {
6699 return Poll::Pending;
6700 }
6701
6702 let ctx = self.ctx.clone();
6703 let mut state = match ctx.state.lock() {
6704 Ok(state) => state,
6705 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6706 };
6707
6708 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6709 let sequence = recorded.sequence();
6710 match recorded {
6711 RecordedCommand::ChildWorkflow {
6712 workflow_type,
6713 outcome,
6714 ..
6715 } => {
6716 if let Some(recorded_type) = workflow_type {
6717 if recorded_type != self.workflow_type {
6718 return Poll::Ready(Err(Error::NonDeterministicReplay(
6719 ReplayFailure::new(
6720 "recorded_command_detail_mismatch",
6721 Some(sequence),
6722 Some(format!("child workflow:{recorded_type}")),
6723 Some(format!("child workflow:{}", self.workflow_type)),
6724 "recorded child workflow type differs from the current workflow command",
6725 ),
6726 )));
6727 }
6728 }
6729 state.command_cursor += 1;
6730 if let Some(outcome) = outcome {
6731 return Poll::Ready(outcome.map_err(Error::ChildWorkflowFailed));
6732 }
6733 state.matched_recorded_pending = true;
6734 self.scheduled = true;
6735 self.matched_pending = true;
6736 return Poll::Pending;
6737 }
6738 other => {
6739 return Poll::Ready(Err(command_mismatch(
6740 &other,
6741 format!("child workflow:{}", self.workflow_type),
6742 )));
6743 }
6744 }
6745 }
6746
6747 if !self.scheduled {
6748 if self.options.task_queue.trim().is_empty() {
6749 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
6750 "task_queue must not be empty".to_string(),
6751 )));
6752 }
6753 for (name, value) in [
6754 (
6755 "execution_timeout_seconds",
6756 self.options.execution_timeout_seconds,
6757 ),
6758 ("run_timeout_seconds", self.options.run_timeout_seconds),
6759 ] {
6760 if value == Some(0) {
6761 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(format!(
6762 "{name} must be at least 1"
6763 ))));
6764 }
6765 }
6766
6767 let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
6768 Ok(args) => args,
6769 Err(error) => return Poll::Ready(Err(error)),
6770 };
6771 let arguments = match encode_typed_envelope(
6772 &normalize_avro_arguments(args),
6773 &state.payload_codec,
6774 ) {
6775 Ok(arguments) => arguments,
6776 Err(error) => return Poll::Ready(Err(error)),
6777 };
6778 let mut command = json!({
6779 "type": "start_child_workflow",
6780 "workflow_type": self.workflow_type,
6781 "queue": self.options.task_queue,
6782 "parent_close_policy": self.options.parent_close_policy.as_str(),
6783 "arguments": arguments,
6784 });
6785 let object = command
6786 .as_object_mut()
6787 .expect("child workflow command is always an object");
6788 if let Some(policy) = &self.options.retry_policy {
6789 let mut retry_policy = serde_json::Map::new();
6790 if let Some(max_attempts) = policy.max_attempts {
6791 if max_attempts == 0 {
6792 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
6793 "retry_policy.max_attempts must be at least 1".to_string(),
6794 )));
6795 }
6796 retry_policy.insert("max_attempts".to_string(), json!(max_attempts));
6797 }
6798 if !policy.backoff_seconds.is_empty() {
6799 retry_policy
6800 .insert("backoff_seconds".to_string(), json!(policy.backoff_seconds));
6801 }
6802 if !policy.non_retryable_error_types.is_empty() {
6803 retry_policy.insert(
6804 "non_retryable_error_types".to_string(),
6805 json!(policy.non_retryable_error_types),
6806 );
6807 }
6808 if retry_policy.is_empty() {
6809 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
6810 "retry_policy must configure at least one field".to_string(),
6811 )));
6812 }
6813 object.insert("retry_policy".to_string(), Value::Object(retry_policy));
6814 }
6815 if let Some(seconds) = self.options.execution_timeout_seconds {
6816 object.insert("execution_timeout_seconds".to_string(), json!(seconds));
6817 }
6818 if let Some(seconds) = self.options.run_timeout_seconds {
6819 object.insert("run_timeout_seconds".to_string(), json!(seconds));
6820 }
6821 state.commands.push(command);
6822 self.scheduled = true;
6823 }
6824
6825 Poll::Pending
6826 }
6827}
6828
6829impl Future for ChildWorkflowCall {
6830 type Output = Result<ChildWorkflowResult>;
6831
6832 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6833 match self.poll_avro_value(cx) {
6834 Poll::Ready(Ok(result)) => match result.result.into_json() {
6835 Ok(projected) => Poll::Ready(Ok(ChildWorkflowResult {
6836 parent: result.parent,
6837 child: result.child,
6838 child_workflow_type: result.child_workflow_type,
6839 result: projected,
6840 })),
6841 Err(error) => Poll::Ready(Err(error)),
6842 },
6843 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
6844 Poll::Pending => Poll::Pending,
6845 }
6846 }
6847}
6848
6849fn command_mismatch(recorded: &RecordedCommand, actual: impl Into<String>) -> Error {
6850 Error::NonDeterministicReplay(ReplayFailure::new(
6851 "recorded_command_mismatch",
6852 Some(recorded.sequence()),
6853 Some(recorded.shape().to_string()),
6854 Some(actual.into()),
6855 "current workflow command does not match the recorded durable command sequence",
6856 ))
6857}
6858
6859pub struct SignalCall {
6860 ctx: WorkflowContext,
6861 signal_name: String,
6862 opened_wait: bool,
6863 matched_pending: bool,
6864}
6865
6866impl SignalCall {
6867 fn poll_avro_value(
6868 mut self: Pin<&mut Self>,
6869 _cx: &mut TaskContext<'_>,
6870 ) -> Poll<Result<Vec<AvroValue>>> {
6871 if self.matched_pending {
6872 return Poll::Pending;
6873 }
6874
6875 let ctx = self.ctx.clone();
6876 let mut state = match ctx.state.lock() {
6877 Ok(state) => state,
6878 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6879 };
6880
6881 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6882 match recorded {
6883 RecordedCommand::SignalWait {
6884 sequence,
6885 signal_name,
6886 value,
6887 } => {
6888 if signal_name != self.signal_name {
6889 return Poll::Ready(Err(Error::NonDeterministicReplay(
6890 ReplayFailure::new(
6891 "recorded_command_detail_mismatch",
6892 Some(sequence),
6893 Some(format!("signal wait:{signal_name}")),
6894 Some(format!("signal wait:{}", self.signal_name)),
6895 "recorded signal name differs from the current workflow command",
6896 ),
6897 )));
6898 }
6899
6900 state.command_cursor += 1;
6901 if let Some(value) = value {
6902 return Poll::Ready(Ok(value));
6903 }
6904 if state
6905 .resume_signal
6906 .as_ref()
6907 .is_some_and(|signal| signal.signal_name == self.signal_name)
6908 {
6909 let signal = state
6910 .resume_signal
6911 .take()
6912 .expect("matching resume signal is present");
6913 return Poll::Ready(Ok(signal.arguments));
6914 }
6915
6916 state.matched_recorded_pending = true;
6917 self.opened_wait = true;
6918 self.matched_pending = true;
6919 return Poll::Pending;
6920 }
6921 other => {
6922 return Poll::Ready(Err(command_mismatch(
6923 &other,
6924 format!("signal wait:{}", self.signal_name),
6925 )));
6926 }
6927 }
6928 }
6929
6930 if state
6931 .resume_signal
6932 .as_ref()
6933 .is_some_and(|signal| signal.signal_name == self.signal_name)
6934 {
6935 let signal = state
6936 .resume_signal
6937 .take()
6938 .expect("matching resume signal is present");
6939 return Poll::Ready(Ok(signal.arguments));
6940 }
6941
6942 if !self.opened_wait {
6943 state.commands.push(json!({
6944 "type": "open_signal_wait",
6945 "signal_name": self.signal_name
6946 }));
6947 self.opened_wait = true;
6948 }
6949
6950 Poll::Pending
6951 }
6952}
6953
6954impl Future for SignalCall {
6955 type Output = Result<Vec<Value>>;
6956
6957 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6958 match self.poll_avro_value(cx) {
6959 Poll::Ready(Ok(values)) => Poll::Ready(
6960 values
6961 .into_iter()
6962 .map(AvroValue::into_json)
6963 .collect::<Result<Vec<_>>>(),
6964 ),
6965 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
6966 Poll::Pending => Poll::Pending,
6967 }
6968 }
6969}
6970
6971#[derive(Clone, Debug)]
6972pub struct ActivityContext {
6973 client: Client,
6974 pub task_id: String,
6975 pub activity_attempt_id: String,
6976 pub lease_owner: String,
6977 pub activity_type: String,
6978 pub attempt_number: u64,
6979 pub task_queue: String,
6980 pub worker_id: String,
6981}
6982
6983impl ActivityContext {
6984 pub async fn heartbeat<T: Serialize>(&self, details: T) -> Result<ActivityHeartbeatResponse> {
6985 self.client
6986 .heartbeat_activity_task(
6987 &self.task_id,
6988 &self.activity_attempt_id,
6989 &self.lease_owner,
6990 details,
6991 )
6992 .await
6993 }
6994}
6995
6996fn decode_task_avro_arguments(value: Option<&Value>, codec: &str) -> Result<AvroValue> {
6997 match value {
6998 Some(value) => Ok(normalize_avro_arguments(decode_wire_avro_value(
6999 value, codec,
7000 )?)),
7001 None => Ok(AvroValue::Array(Vec::new())),
7002 }
7003}
7004
7005fn decode_resume_signal(task: &WorkflowTask) -> Result<Option<ResumeSignal>> {
7006 let Some(signal_name) = task
7007 .signal_name
7008 .as_deref()
7009 .filter(|value| !value.is_empty())
7010 else {
7011 return Ok(None);
7012 };
7013 let Some(arguments) = task.signal_arguments.as_ref() else {
7014 return Ok(None);
7015 };
7016
7017 let decoded = normalize_avro_arguments(decode_wire_avro_value(arguments, &task.payload_codec)?);
7018 let AvroValue::Array(arguments) = decoded else {
7019 unreachable!("normalize_avro_arguments always returns an array");
7020 };
7021
7022 Ok(Some(ResumeSignal {
7023 signal_name: signal_name.to_string(),
7024 arguments,
7025 }))
7026}
7027
7028fn recorded_commands(
7029 events: &[HistoryEvent],
7030 fallback_codec: &str,
7031 parent: WorkflowIdentity,
7032) -> Result<Vec<RecordedCommand>> {
7033 let mut events_by_sequence: BTreeMap<u64, Vec<&HistoryEvent>> = BTreeMap::new();
7034 let mut last_new_sequence = None;
7035
7036 for event in events {
7037 let is_activity = matches!(
7038 event.event_type.as_str(),
7039 "ActivityScheduled"
7040 | "ActivityStarted"
7041 | "ActivityHeartbeatRecorded"
7042 | "ActivityRetryScheduled"
7043 | "ActivityCompleted"
7044 | "ActivityFailed"
7045 | "ActivityCancelled"
7046 | "ActivityTimedOut"
7047 );
7048 let is_workflow_timer = matches!(
7049 event.event_type.as_str(),
7050 "TimerScheduled" | "TimerCancelled" | "TimerFired"
7051 ) && !is_internal_timer_event(event);
7052 let is_child_workflow = matches!(
7053 event.event_type.as_str(),
7054 "ChildWorkflowScheduled"
7055 | "ChildRunCompleted"
7056 | "ChildRunFailed"
7057 | "ChildRunCancelled"
7058 | "ChildRunTerminated"
7059 );
7060 let is_signal_wait = is_recorded_signal_wait_event(event);
7061 let is_side_effect = event.event_type == "SideEffectRecorded";
7062 let is_version_marker = event.event_type == "VersionMarkerRecorded";
7063 if !is_activity
7064 && !is_workflow_timer
7065 && !is_child_workflow
7066 && !is_signal_wait
7067 && !is_side_effect
7068 && !is_version_marker
7069 {
7070 continue;
7071 }
7072
7073 let sequence = durable_event_sequence(event).ok_or_else(|| {
7074 Error::NonDeterministicReplay(ReplayFailure::new(
7075 "durable_command_sequence_missing",
7076 None,
7077 Some("positive workflow sequence".to_string()),
7078 Some(event.event_type.clone()),
7079 "durable command history event has no workflow sequence",
7080 ))
7081 })?;
7082 if sequence == 0 {
7083 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
7084 "durable_command_sequence_invalid",
7085 Some(sequence),
7086 Some("positive workflow sequence".to_string()),
7087 Some(sequence.to_string()),
7088 "durable command history uses an invalid workflow sequence",
7089 )));
7090 }
7091 if !events_by_sequence.contains_key(&sequence) {
7092 if let Some(previous) = last_new_sequence {
7093 if sequence < previous {
7094 return Err(invalid_recorded_history(
7095 "durable_command_sequence_mismatch",
7096 sequence,
7097 &format!("workflow sequence greater than {previous}"),
7098 &sequence.to_string(),
7099 "durable commands are not strictly ordered by their recorded workflow sequence",
7100 ));
7101 }
7102 }
7103 last_new_sequence = Some(sequence);
7104 }
7105 events_by_sequence.entry(sequence).or_default().push(event);
7106 }
7107
7108 let commands: Vec<RecordedCommand> = events_by_sequence
7109 .into_iter()
7110 .map(|(sequence, sequence_events)| {
7111 let activity_events: Vec<_> = sequence_events
7112 .iter()
7113 .copied()
7114 .filter(|event| event.event_type.starts_with("Activity"))
7115 .collect();
7116 let timer_events: Vec<_> = sequence_events
7117 .iter()
7118 .copied()
7119 .filter(|event| event.event_type.starts_with("Timer"))
7120 .collect();
7121 let child_events: Vec<_> = sequence_events
7122 .iter()
7123 .copied()
7124 .filter(|event| {
7125 event.event_type == "ChildWorkflowScheduled"
7126 || event.event_type.starts_with("ChildRun")
7127 })
7128 .collect();
7129 let signal_wait_events: Vec<_> = sequence_events
7130 .iter()
7131 .copied()
7132 .filter(|event| is_recorded_signal_wait_event(event))
7133 .collect();
7134 let side_effect_events: Vec<_> = sequence_events
7135 .iter()
7136 .copied()
7137 .filter(|event| event.event_type == "SideEffectRecorded")
7138 .collect();
7139 let version_marker_events: Vec<_> = sequence_events
7140 .iter()
7141 .copied()
7142 .filter(|event| event.event_type == "VersionMarkerRecorded")
7143 .collect();
7144
7145 let command_kind_count = usize::from(!activity_events.is_empty())
7146 + usize::from(!timer_events.is_empty())
7147 + usize::from(!child_events.is_empty())
7148 + usize::from(!signal_wait_events.is_empty())
7149 + usize::from(!side_effect_events.is_empty())
7150 + usize::from(!version_marker_events.is_empty());
7151 if command_kind_count > 1 {
7152 let actual = [
7153 (!activity_events.is_empty()).then_some("activity"),
7154 (!timer_events.is_empty()).then_some("timer"),
7155 (!child_events.is_empty()).then_some("child workflow"),
7156 (!signal_wait_events.is_empty()).then_some("signal wait"),
7157 (!side_effect_events.is_empty()).then_some("side effect"),
7158 (!version_marker_events.is_empty()).then_some("version marker"),
7159 ]
7160 .into_iter()
7161 .flatten()
7162 .collect::<Vec<_>>()
7163 .join(" and ");
7164 return Err(invalid_recorded_history(
7165 "durable_command_sequence_collision",
7166 sequence,
7167 "one durable command kind",
7168 &actual,
7169 "one workflow sequence records more than one durable command kind",
7170 ));
7171 }
7172
7173 if !activity_events.is_empty() {
7174 let scheduled_count = activity_events
7175 .iter()
7176 .filter(|event| event.event_type == "ActivityScheduled")
7177 .count();
7178 if scheduled_count > 1 {
7179 return Err(invalid_recorded_history(
7180 "duplicate_activity_schedule",
7181 sequence,
7182 "at most one ActivityScheduled event",
7183 "multiple ActivityScheduled events",
7184 "activity history schedules more than one command at one workflow sequence",
7185 ));
7186 }
7187 let activity_type = activity_events.iter().find_map(|event| {
7188 event
7189 .payload
7190 .get("activity_type")
7191 .or_else(|| event.payload.get("activity_name"))
7192 .and_then(Value::as_str)
7193 .map(str::to_string)
7194 });
7195 if activity_events.iter().filter_map(|event| {
7196 event
7197 .payload
7198 .get("activity_type")
7199 .or_else(|| event.payload.get("activity_name"))
7200 .and_then(Value::as_str)
7201 }).any(|candidate| Some(candidate) != activity_type.as_deref()) {
7202 return Err(invalid_recorded_history(
7203 "activity_identity_mismatch",
7204 sequence,
7205 activity_type.as_deref().unwrap_or("one activity identity"),
7206 "conflicting activity identities",
7207 "activity lifecycle events at one workflow sequence disagree on identity",
7208 ));
7209 }
7210 let terminal: Vec<_> = activity_events
7211 .iter()
7212 .copied()
7213 .filter(|event| {
7214 matches!(
7215 event.event_type.as_str(),
7216 "ActivityCompleted"
7217 | "ActivityFailed"
7218 | "ActivityCancelled"
7219 | "ActivityTimedOut"
7220 )
7221 })
7222 .collect();
7223 if terminal.len() > 1 {
7224 return Err(invalid_recorded_history(
7225 "duplicate_activity_terminal_event",
7226 sequence,
7227 "at most one terminal activity event",
7228 "multiple terminal activity events",
7229 "activity history settles one command more than once",
7230 ));
7231 }
7232 let outcome = terminal
7233 .first()
7234 .map(|event| activity_outcome(event, fallback_codec, activity_type.clone()))
7235 .transpose()?;
7236 let options = activity_events
7237 .iter()
7238 .find(|event| event.event_type == "ActivityScheduled")
7239 .and_then(|event| event.payload.get("activity"))
7240 .and_then(Value::as_object)
7241 .map(|activity| RecordedActivityOptions {
7242 task_queue: recorded_optional_string(activity, "queue"),
7243 execution_mode: recorded_optional_string(activity, "execution_mode"),
7244 retry_policy: recorded_activity_retry_snapshot(
7245 activity.get("retry_policy"),
7246 ),
7247 });
7248 return Ok(RecordedCommand::Activity {
7249 sequence,
7250 activity_type,
7251 options,
7252 outcome,
7253 });
7254 }
7255
7256 if !child_events.is_empty() {
7257 let scheduled: Vec<_> = child_events
7258 .iter()
7259 .copied()
7260 .filter(|event| event.event_type == "ChildWorkflowScheduled")
7261 .collect();
7262 if scheduled.len() != 1 {
7263 return Err(invalid_recorded_history(
7264 "child_workflow_schedule_missing_or_duplicate",
7265 sequence,
7266 "one ChildWorkflowScheduled event",
7267 &format!("{} ChildWorkflowScheduled events", scheduled.len()),
7268 "child workflow replay requires exactly one recorded schedule event",
7269 ));
7270 }
7271 let workflow_type = child_events.iter().find_map(|event| {
7272 event
7273 .payload
7274 .get("child_workflow_type")
7275 .or_else(|| event.payload.get("workflow_type"))
7276 .and_then(Value::as_str)
7277 .filter(|value| !value.is_empty())
7278 .map(str::to_string)
7279 });
7280 if child_events
7281 .iter()
7282 .filter_map(|event| {
7283 event
7284 .payload
7285 .get("child_workflow_type")
7286 .or_else(|| event.payload.get("workflow_type"))
7287 .and_then(Value::as_str)
7288 })
7289 .any(|candidate| Some(candidate) != workflow_type.as_deref())
7290 {
7291 return Err(invalid_recorded_history(
7292 "child_workflow_identity_mismatch",
7293 sequence,
7294 workflow_type
7295 .as_deref()
7296 .unwrap_or("one child workflow type"),
7297 "conflicting child workflow types",
7298 "child workflow lifecycle events at one sequence disagree on type",
7299 ));
7300 }
7301 let mut outcomes = child_workflow_outcomes(
7302 &child_events.iter().map(|event| (*event).clone()).collect::<Vec<_>>(),
7303 fallback_codec,
7304 parent.clone(),
7305 )?;
7306 if outcomes.len() > 1 {
7307 return Err(invalid_recorded_history(
7308 "duplicate_child_workflow_terminal_event",
7309 sequence,
7310 "at most one terminal child event",
7311 "multiple terminal child events",
7312 "child workflow history settles one command more than once",
7313 ));
7314 }
7315 return Ok(RecordedCommand::ChildWorkflow {
7316 sequence,
7317 workflow_type,
7318 outcome: outcomes.pop(),
7319 });
7320 }
7321
7322 if !signal_wait_events.is_empty() {
7323 let opened: Vec<_> = signal_wait_events
7324 .iter()
7325 .copied()
7326 .filter(|event| event.event_type == "SignalWaitOpened")
7327 .collect();
7328 if opened.len() != 1 {
7329 return Err(invalid_recorded_history(
7330 "signal_wait_open_missing_or_duplicate",
7331 sequence,
7332 "one SignalWaitOpened event",
7333 &format!("{} SignalWaitOpened events", opened.len()),
7334 "signal replay requires exactly one canonical wait-open event",
7335 ));
7336 }
7337
7338 let applied: Vec<_> = signal_wait_events
7339 .iter()
7340 .copied()
7341 .filter(|event| event.event_type == "SignalApplied")
7342 .collect();
7343 if applied.len() > 1 {
7344 return Err(invalid_recorded_history(
7345 "duplicate_signal_wait_apply",
7346 sequence,
7347 "at most one SignalApplied event",
7348 "multiple SignalApplied events",
7349 "signal history applies one durable wait more than once",
7350 ));
7351 }
7352
7353 let signal_names = signal_wait_events
7354 .iter()
7355 .map(|event| required_signal_wait_name(event, sequence))
7356 .collect::<Result<Vec<_>>>()?;
7357 let signal_name = signal_names
7358 .first()
7359 .expect("signal wait events are not empty")
7360 .clone();
7361 if signal_names.iter().any(|candidate| candidate != &signal_name) {
7362 return Err(invalid_recorded_history(
7363 "signal_wait_identity_mismatch",
7364 sequence,
7365 &signal_name,
7366 "conflicting signal names",
7367 "signal wait lifecycle events at one workflow sequence disagree on identity",
7368 ));
7369 }
7370 let value = applied
7371 .first()
7372 .map(|event| decode_signal_event_arguments(event, fallback_codec))
7373 .transpose()?;
7374 return Ok(RecordedCommand::SignalWait {
7375 sequence,
7376 signal_name,
7377 value,
7378 });
7379 }
7380
7381 if !side_effect_events.is_empty() {
7382 if side_effect_events.len() != 1 {
7383 return Err(invalid_recorded_history(
7384 "duplicate_side_effect_record",
7385 sequence,
7386 "one SideEffectRecorded event",
7387 &format!("{} SideEffectRecorded events", side_effect_events.len()),
7388 "side-effect history records one workflow command more than once",
7389 ));
7390 }
7391 let event = side_effect_events[0];
7392 let result = event.payload.get("result").ok_or_else(|| {
7393 invalid_recorded_history(
7394 "side_effect_result_missing",
7395 sequence,
7396 "recorded result payload",
7397 "missing result",
7398 "side-effect history is missing its recorded value",
7399 )
7400 })?;
7401 let has_published_envelope = result.as_str().is_some()
7402 || result.as_object().is_some_and(|envelope| {
7403 envelope.get("codec").and_then(Value::as_str).is_some()
7404 && envelope.get("blob").and_then(Value::as_str).is_some()
7405 });
7406 if !has_published_envelope {
7407 return Err(invalid_recorded_history(
7408 "side_effect_payload_malformed",
7409 sequence,
7410 "payload blob or {codec, blob} envelope",
7411 &result.to_string(),
7412 "side-effect history result does not use a published payload envelope",
7413 ));
7414 }
7415 let codec = event
7416 .payload
7417 .get("payload_codec")
7418 .and_then(Value::as_str)
7419 .unwrap_or(fallback_codec);
7420 let value = decode_wire_avro_value(result, codec).map_err(|error| {
7421 invalid_recorded_history(
7422 "side_effect_payload_incompatible",
7423 sequence,
7424 &format!("valid {codec} payload envelope"),
7425 &error.to_string(),
7426 "side-effect history payload cannot be decoded with its recorded codec",
7427 )
7428 })?;
7429 return Ok(RecordedCommand::SideEffect { sequence, value });
7430 }
7431
7432 if !version_marker_events.is_empty() {
7433 if version_marker_events.len() != 1 {
7434 return Err(invalid_recorded_history(
7435 "duplicate_version_marker_record",
7436 sequence,
7437 "one VersionMarkerRecorded event",
7438 &format!("{} VersionMarkerRecorded events", version_marker_events.len()),
7439 "version-marker history records one workflow command more than once",
7440 ));
7441 }
7442 let payload = &version_marker_events[0].payload;
7443 let change_id = payload
7444 .get("change_id")
7445 .and_then(Value::as_str)
7446 .filter(|value| !value.is_empty())
7447 .map(str::to_string)
7448 .ok_or_else(|| {
7449 invalid_recorded_history(
7450 "version_marker_field_missing",
7451 sequence,
7452 "non-empty change_id",
7453 "missing or invalid change_id",
7454 "version-marker history is missing its stable change ID",
7455 )
7456 })?;
7457 let version = required_version_i32(payload, "version", sequence)?;
7458 let min_supported = required_version_i32(payload, "min_supported", sequence)?;
7459 let max_supported = required_version_i32(payload, "max_supported", sequence)?;
7460 if min_supported > max_supported || version < min_supported || version > max_supported {
7461 return Err(invalid_recorded_history(
7462 "version_marker_history_range_invalid",
7463 sequence,
7464 "min_supported <= version <= max_supported",
7465 &format!("{min_supported} <= {version} <= {max_supported}"),
7466 "recorded version marker contains an internally incompatible range",
7467 ));
7468 }
7469 return Ok(RecordedCommand::VersionMarker {
7470 sequence,
7471 change_id,
7472 version,
7473 });
7474 }
7475
7476 let scheduled: Vec<_> = timer_events
7477 .iter()
7478 .copied()
7479 .filter(|event| event.event_type == "TimerScheduled")
7480 .collect();
7481 let fired: Vec<_> = timer_events
7482 .iter()
7483 .copied()
7484 .filter(|event| event.event_type == "TimerFired")
7485 .collect();
7486 if scheduled.len() != 1 {
7487 return Err(invalid_recorded_history(
7488 "timer_schedule_missing_or_duplicate",
7489 sequence,
7490 "one TimerScheduled event",
7491 &format!("{} TimerScheduled events", scheduled.len()),
7492 "timer replay requires exactly one recorded schedule event",
7493 ));
7494 }
7495 if fired.len() > 1 {
7496 return Err(invalid_recorded_history(
7497 "duplicate_timer_fire",
7498 sequence,
7499 "at most one TimerFired event",
7500 "multiple TimerFired events",
7501 "timer history contains more than one fire event for a workflow sequence",
7502 ));
7503 }
7504
7505 let scheduled = scheduled[0];
7506 let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
7507 let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
7508 if let Some(fired) = fired.first() {
7509 let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
7510 if fired_timer_id != timer_id {
7511 return Err(invalid_recorded_history(
7512 "timer_identity_mismatch",
7513 sequence,
7514 &timer_id,
7515 &fired_timer_id,
7516 "TimerFired does not correspond to the recorded TimerScheduled event",
7517 ));
7518 }
7519 let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
7520 if fired_delay != delay_seconds {
7521 return Err(invalid_recorded_history(
7522 "timer_history_delay_mismatch",
7523 sequence,
7524 &delay_seconds.to_string(),
7525 &fired_delay.to_string(),
7526 "TimerScheduled and TimerFired record different delays",
7527 ));
7528 }
7529 }
7530
7531 Ok(RecordedCommand::Timer {
7532 sequence,
7533 delay_seconds,
7534 fired: !fired.is_empty(),
7535 })
7536 })
7537 .collect::<Result<_>>()?;
7538
7539 let mut marker_sequences = HashMap::new();
7540 for command in &commands {
7541 if let RecordedCommand::VersionMarker {
7542 sequence,
7543 change_id,
7544 ..
7545 } = command
7546 {
7547 if let Some(first_sequence) = marker_sequences.insert(change_id.clone(), *sequence) {
7548 return Err(invalid_recorded_history(
7549 "duplicate_version_marker",
7550 *sequence,
7551 &format!("one marker for change ID {change_id:?}"),
7552 &format!("markers at sequences {first_sequence} and {sequence}"),
7553 "workflow history contains duplicate markers for one stable change ID",
7554 ));
7555 }
7556 }
7557 }
7558
7559 Ok(commands)
7560}
7561
7562fn required_version_i32(payload: &Value, field: &str, sequence: u64) -> Result<i32> {
7563 payload
7564 .get(field)
7565 .and_then(Value::as_i64)
7566 .and_then(|value| i32::try_from(value).ok())
7567 .ok_or_else(|| {
7568 invalid_recorded_history(
7569 "version_marker_field_missing",
7570 sequence,
7571 &format!("integer {field}"),
7572 "missing or out-of-range integer",
7573 "version-marker history is missing a required integer field",
7574 )
7575 })
7576}
7577
7578fn durable_event_sequence(event: &HistoryEvent) -> Option<u64> {
7579 event
7580 .payload
7581 .get("sequence")
7582 .or_else(|| event.payload.get("workflow_sequence"))
7583 .or_else(|| event.raw.get("sequence"))
7584 .or_else(|| event.raw.get("workflow_sequence"))
7585 .and_then(value_as_u64)
7586}
7587
7588fn is_internal_timer_event(event: &HistoryEvent) -> bool {
7589 matches!(
7590 event
7591 .payload
7592 .get("timer_kind")
7593 .or_else(|| event.raw.get("timer_kind"))
7594 .and_then(Value::as_str),
7595 Some("condition_timeout" | "signal_timeout")
7596 )
7597}
7598
7599fn required_signal_wait_name(event: &HistoryEvent, sequence: u64) -> Result<String> {
7600 event
7601 .payload
7602 .get("signal_name")
7603 .or_else(|| event.raw.get("signal_name"))
7604 .and_then(Value::as_str)
7605 .filter(|value| !value.is_empty())
7606 .map(str::to_string)
7607 .ok_or_else(|| {
7608 invalid_recorded_history(
7609 "signal_wait_name_missing",
7610 sequence,
7611 "non-empty signal_name",
7612 &event.event_type,
7613 "canonical signal-wait history is missing its signal identity",
7614 )
7615 })
7616}
7617
7618fn is_recorded_signal_wait_event(event: &HistoryEvent) -> bool {
7619 matches!(
7620 event.event_type.as_str(),
7621 "SignalWaitOpened" | "SignalApplied"
7622 )
7623}
7624
7625fn required_history_string(event: &HistoryEvent, field: &str, sequence: u64) -> Result<String> {
7626 event
7627 .payload
7628 .get(field)
7629 .and_then(Value::as_str)
7630 .filter(|value| !value.is_empty())
7631 .map(str::to_string)
7632 .ok_or_else(|| {
7633 invalid_recorded_history(
7634 "timer_history_field_missing",
7635 sequence,
7636 field,
7637 &event.event_type,
7638 "timer history is missing a required identity field",
7639 )
7640 })
7641}
7642
7643fn required_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<u64> {
7644 event
7645 .payload
7646 .get(field)
7647 .and_then(value_as_u64)
7648 .ok_or_else(|| {
7649 invalid_recorded_history(
7650 "timer_history_field_missing",
7651 sequence,
7652 field,
7653 &event.event_type,
7654 "timer history is missing a required numeric field",
7655 )
7656 })
7657}
7658
7659fn invalid_recorded_history(
7660 reason: &str,
7661 sequence: u64,
7662 expected: &str,
7663 actual: &str,
7664 message: &str,
7665) -> Error {
7666 Error::NonDeterministicReplay(ReplayFailure::new(
7667 reason,
7668 Some(sequence),
7669 Some(expected.to_string()),
7670 Some(actual.to_string()),
7671 message,
7672 ))
7673}
7674
7675type ActivityOutcome = std::result::Result<AvroValue, ActivityFailure>;
7676
7677fn activity_outcome(
7678 event: &HistoryEvent,
7679 fallback_codec: &str,
7680 recorded_activity_type: Option<String>,
7681) -> Result<ActivityOutcome> {
7682 if event.event_type == "ActivityCompleted" {
7683 let codec = event
7684 .payload
7685 .get("payload_codec")
7686 .and_then(Value::as_str)
7687 .unwrap_or(fallback_codec);
7688 return Ok(Ok(decode_wire_avro_value(
7689 event.payload.get("result").unwrap_or(&Value::Null),
7690 codec,
7691 )?));
7692 }
7693
7694 let payload = &event.payload;
7695 let (kind, fallback_reason, fallback_message) = match event.event_type.as_str() {
7696 "ActivityFailed" => (ActivityFailureKind::Failed, "activity", "activity failed"),
7697 "ActivityCancelled" => (
7698 ActivityFailureKind::Cancelled,
7699 "cancelled",
7700 "activity was cancelled",
7701 ),
7702 "ActivityTimedOut" => (
7703 ActivityFailureKind::TimedOut,
7704 "timeout",
7705 "activity timed out",
7706 ),
7707 _ => unreachable!("activity_outcome is called only for terminal activity events"),
7708 };
7709 let exception = payload
7710 .get("exception")
7711 .filter(|value| !value.is_null())
7712 .cloned();
7713 let failure_category = payload_string(payload, "failure_category");
7714 let timeout_kind = payload_string(payload, "timeout_kind");
7715 let reason = payload_string(payload, "reason").unwrap_or_else(|| match kind {
7716 ActivityFailureKind::Failed => failure_category
7717 .clone()
7718 .unwrap_or_else(|| fallback_reason.to_string()),
7719 ActivityFailureKind::Cancelled => fallback_reason.to_string(),
7720 ActivityFailureKind::TimedOut => timeout_kind
7721 .clone()
7722 .unwrap_or_else(|| fallback_reason.to_string()),
7723 });
7724 let message = payload_string(payload, "message")
7725 .or_else(|| {
7726 exception
7727 .as_ref()
7728 .and_then(|value| payload_string(value, "message"))
7729 })
7730 .unwrap_or_else(|| fallback_message.to_string());
7731
7732 Ok(Err(ActivityFailure {
7733 kind,
7734 reason,
7735 message,
7736 activity_execution_id: payload_string(payload, "activity_execution_id"),
7737 activity_attempt_id: payload_string(payload, "activity_attempt_id"),
7738 activity_type: payload_string(payload, "activity_type")
7739 .or_else(|| payload_string(payload, "activity_name"))
7740 .or(recorded_activity_type),
7741 activity_class: payload_string(payload, "activity_class"),
7742 attempt_number: payload.get("attempt_number").and_then(value_as_u64),
7743 failure_id: payload_string(payload, "failure_id"),
7744 failure_category,
7745 timeout_kind,
7746 non_retryable: payload
7747 .get("non_retryable")
7748 .and_then(Value::as_bool)
7749 .unwrap_or(false),
7750 exception_type: payload_string(payload, "exception_type").or_else(|| {
7751 exception
7752 .as_ref()
7753 .and_then(|value| payload_string(value, "type"))
7754 }),
7755 exception_class: payload_string(payload, "exception_class").or_else(|| {
7756 exception
7757 .as_ref()
7758 .and_then(|value| payload_string(value, "class"))
7759 }),
7760 code: payload
7761 .get("code")
7762 .filter(|value| !value.is_null())
7763 .cloned(),
7764 exception,
7765 }))
7766}
7767
7768type ChildWorkflowOutcome = std::result::Result<ChildWorkflowAvroResult, ChildWorkflowFailure>;
7769
7770fn child_workflow_outcomes(
7771 events: &[HistoryEvent],
7772 fallback_codec: &str,
7773 parent: WorkflowIdentity,
7774) -> Result<Vec<ChildWorkflowOutcome>> {
7775 let mut outcomes = Vec::new();
7776
7777 for event in events {
7778 let kind = match event.event_type.as_str() {
7779 "ChildRunCompleted" => None,
7780 "ChildRunFailed" => Some((
7781 ChildWorkflowFailureKind::Failed,
7782 "child_workflow",
7783 "child workflow failed",
7784 )),
7785 "ChildRunCancelled" => Some((
7786 ChildWorkflowFailureKind::Cancelled,
7787 "cancelled",
7788 "child workflow was cancelled",
7789 )),
7790 "ChildRunTerminated" => Some((
7791 ChildWorkflowFailureKind::Terminated,
7792 "terminated",
7793 "child workflow was terminated",
7794 )),
7795 _ => continue,
7796 };
7797 let payload = &event.payload;
7798 let child_workflow_id = payload_string(payload, "child_workflow_instance_id");
7799 let child_workflow_run_id = payload_string(payload, "child_workflow_run_id");
7800 let child_workflow_type = payload_string(payload, "child_workflow_type");
7801
7802 if let Some((kind, reason, fallback_message)) = kind {
7803 let exception = payload
7804 .get("exception")
7805 .filter(|value| !value.is_null())
7806 .cloned();
7807 let message = payload_string(payload, "message")
7808 .or_else(|| {
7809 exception
7810 .as_ref()
7811 .and_then(|value| payload_string(value, "message"))
7812 })
7813 .unwrap_or_else(|| fallback_message.to_string());
7814 let exception_type = payload_string(payload, "exception_type").or_else(|| {
7815 exception
7816 .as_ref()
7817 .and_then(|value| payload_string(value, "type"))
7818 });
7819 let exception_class = payload_string(payload, "exception_class").or_else(|| {
7820 exception
7821 .as_ref()
7822 .and_then(|value| payload_string(value, "class"))
7823 });
7824 outcomes.push(Err(ChildWorkflowFailure {
7825 kind,
7826 reason: reason.to_string(),
7827 message,
7828 parent_workflow_id: parent.workflow_id.clone(),
7829 parent_workflow_run_id: parent.run_id.clone(),
7830 child_workflow_id,
7831 child_workflow_run_id,
7832 child_workflow_type,
7833 failure_id: payload_string(payload, "failure_id"),
7834 failure_category: payload_string(payload, "failure_category"),
7835 exception_type,
7836 exception_class,
7837 non_retryable: payload
7838 .get("non_retryable")
7839 .and_then(Value::as_bool)
7840 .unwrap_or(false),
7841 code: payload
7842 .get("code")
7843 .filter(|value| !value.is_null())
7844 .cloned(),
7845 exception,
7846 }));
7847 continue;
7848 }
7849
7850 let codec = payload
7851 .get("payload_codec")
7852 .and_then(Value::as_str)
7853 .unwrap_or(fallback_codec);
7854 let result = payload
7855 .get("result")
7856 .or_else(|| payload.get("output"))
7857 .unwrap_or(&Value::Null);
7858 outcomes.push(Ok(ChildWorkflowAvroResult {
7859 parent: parent.clone(),
7860 child: WorkflowIdentity {
7861 workflow_id: child_workflow_id,
7862 run_id: child_workflow_run_id,
7863 },
7864 child_workflow_type,
7865 result: decode_wire_avro_value(result, codec)?,
7866 }));
7867 }
7868
7869 Ok(outcomes)
7870}
7871
7872fn payload_string(payload: &Value, key: &str) -> Option<String> {
7873 payload
7874 .get(key)
7875 .and_then(Value::as_str)
7876 .filter(|value| !value.is_empty())
7877 .map(str::to_string)
7878}
7879
7880fn workflow_failure_command(error: &Error) -> Value {
7881 let (exception_type, exception_class, properties) = match error {
7882 Error::ActivityFailed(failure) => (
7883 match failure.kind {
7884 ActivityFailureKind::Failed => "ActivityFailed",
7885 ActivityFailureKind::Cancelled => "ActivityCancelled",
7886 ActivityFailureKind::TimedOut => "ActivityTimedOut",
7887 },
7888 "durable_workflow::ActivityFailure",
7889 json!({
7890 "reason": failure.reason,
7891 "activity_execution_id": failure.activity_execution_id,
7892 "activity_attempt_id": failure.activity_attempt_id,
7893 "activity_type": failure.activity_type,
7894 "activity_class": failure.activity_class,
7895 "attempt_number": failure.attempt_number,
7896 "failure_id": failure.failure_id,
7897 "failure_category": failure.failure_category,
7898 "timeout_kind": failure.timeout_kind,
7899 "activity_non_retryable": failure.non_retryable,
7900 "activity_exception_type": failure.exception_type,
7901 "activity_exception_class": failure.exception_class,
7902 "activity_code": failure.code,
7903 "activity_exception": failure.exception,
7904 }),
7905 ),
7906 Error::ChildWorkflowFailed(failure) => (
7907 match failure.kind {
7908 ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
7909 ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
7910 ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
7911 },
7912 "durable_workflow::ChildWorkflowFailure",
7913 json!({
7914 "reason": failure.reason,
7915 "parent_workflow_id": failure.parent_workflow_id,
7916 "parent_workflow_run_id": failure.parent_workflow_run_id,
7917 "child_workflow_id": failure.child_workflow_id,
7918 "child_workflow_run_id": failure.child_workflow_run_id,
7919 "child_workflow_type": failure.child_workflow_type,
7920 "failure_id": failure.failure_id,
7921 "failure_category": failure.failure_category,
7922 "child_exception_type": failure.exception_type,
7923 "child_exception_class": failure.exception_class,
7924 "child_non_retryable": failure.non_retryable,
7925 "child_code": failure.code,
7926 "child_exception": failure.exception,
7927 }),
7928 ),
7929 Error::NonDeterministicReplay(_) => (
7930 "NonDeterministicReplay",
7931 "durable_workflow::Error",
7932 Value::Null,
7933 ),
7934 _ => ("RustWorkflowError", "durable_workflow::Error", Value::Null),
7935 };
7936 let non_retryable = match error {
7937 Error::ActivityFailed(failure) => failure.non_retryable,
7938 Error::ChildWorkflowFailed(failure) => failure.non_retryable,
7939 Error::NonDeterministicReplay(_) => true,
7940 _ => false,
7941 };
7942
7943 json!({
7944 "type": "fail_workflow",
7945 "message": error.to_string(),
7946 "exception_type": exception_type,
7947 "exception_class": exception_class,
7948 "non_retryable": non_retryable,
7949 "exception": {
7950 "type": exception_type,
7951 "class": exception_class,
7952 "message": error.to_string(),
7953 "properties": properties,
7954 }
7955 })
7956}
7957
7958fn workflow_task_integrity_error(error: &Error) -> bool {
7959 matches!(
7960 error,
7961 Error::NonDeterministicReplay(_) | Error::Protocol(_) | Error::WorkflowStatePoisoned
7962 )
7963}
7964
7965fn decode_signal_event_arguments(
7966 event: &HistoryEvent,
7967 fallback_codec: &str,
7968) -> Result<Vec<AvroValue>> {
7969 let codec = event
7970 .payload
7971 .get("payload_codec")
7972 .and_then(Value::as_str)
7973 .unwrap_or(fallback_codec);
7974 let raw = event
7975 .payload
7976 .get("value")
7977 .or_else(|| event.payload.get("input"))
7978 .or_else(|| event.payload.get("arguments"));
7979 let decoded = match raw.filter(|value| !value.is_null()) {
7980 Some(value) => decode_wire_avro_value(value, codec)?,
7981 None => AvroValue::Array(Vec::new()),
7982 };
7983 let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
7984 unreachable!("normalize_avro_arguments always returns an array");
7985 };
7986 Ok(arguments)
7987}
7988
7989fn hydrate_query_history_from_export(task: &mut QueryTask) -> Result<()> {
7990 let Some(export_events) = task
7991 .history_export
7992 .as_ref()
7993 .and_then(|export| export.get("history_events"))
7994 .and_then(Value::as_array)
7995 else {
7996 return Ok(());
7997 };
7998
7999 if export_events.len() > task.history_events.len() {
8000 task.history_events = serde_json::from_value(Value::Array(export_events.clone()))?;
8001 }
8002
8003 Ok(())
8004}
8005
8006fn enrich_query_history_from_export(task: &mut QueryTask) -> Result<()> {
8007 let Some(export) = task.history_export.as_ref() else {
8008 return Ok(());
8009 };
8010 let signals = export
8011 .get("signals")
8012 .and_then(Value::as_array)
8013 .cloned()
8014 .unwrap_or_default();
8015 let activities = export
8016 .get("activities")
8017 .and_then(Value::as_array)
8018 .cloned()
8019 .unwrap_or_default();
8020 let export_codec = export
8021 .get("payloads")
8022 .and_then(|payloads| payloads.get("codec"))
8023 .and_then(Value::as_str)
8024 .unwrap_or(&task.payload_codec)
8025 .to_string();
8026 let mut signal_name_offsets: HashMap<String, usize> = HashMap::new();
8027
8028 for event in &mut task.history_events {
8029 if event.event_type == "ActivityCompleted" {
8030 let sequence = event
8031 .payload
8032 .get("sequence")
8033 .or_else(|| event.payload.get("workflow_sequence"))
8034 .and_then(value_as_u64);
8035 let Some(activity) = sequence.and_then(|sequence| {
8036 activities.iter().find(|activity| {
8037 activity.get("sequence").and_then(value_as_u64) == Some(sequence)
8038 })
8039 }) else {
8040 continue;
8041 };
8042 let Some(payload) = event.payload.as_object_mut() else {
8043 continue;
8044 };
8045 if missing_payload(payload.get("result")) {
8046 if let Some(result) = activity
8047 .get("result")
8048 .filter(|value| !missing_payload(Some(value)))
8049 {
8050 payload.insert("result".to_string(), result.clone());
8051 }
8052 }
8053 for field in ["payload_codec", "activity_type"] {
8054 if payload
8055 .get(field)
8056 .and_then(Value::as_str)
8057 .unwrap_or_default()
8058 .is_empty()
8059 {
8060 if let Some(value) = activity.get(field) {
8061 payload.insert(field.to_string(), value.clone());
8062 }
8063 }
8064 }
8065 continue;
8066 }
8067
8068 if event.event_type != "SignalReceived" && event.event_type != "SignalApplied" {
8069 continue;
8070 }
8071 let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
8072 let command_id = event
8073 .payload
8074 .get("workflow_command_id")
8075 .or_else(|| event.raw.get("workflow_command_id"))
8076 .and_then(Value::as_str);
8077 let signal_name = event
8078 .payload
8079 .get("signal_name")
8080 .and_then(Value::as_str)
8081 .unwrap_or_default()
8082 .to_string();
8083 let matched = signals
8084 .iter()
8085 .find(|signal| {
8086 signal_id.is_some() && signal.get("id").and_then(Value::as_str) == signal_id
8087 })
8088 .or_else(|| {
8089 signals.iter().find(|signal| {
8090 command_id.is_some()
8091 && signal.get("command_id").and_then(Value::as_str) == command_id
8092 })
8093 })
8094 .or_else(|| {
8095 let offset = signal_name_offsets.entry(signal_name.clone()).or_default();
8096 let signal = signals
8097 .iter()
8098 .filter(|signal| {
8099 signal.get("name").and_then(Value::as_str) == Some(signal_name.as_str())
8100 })
8101 .nth(*offset);
8102 if signal.is_some() {
8103 *offset += 1;
8104 }
8105 signal
8106 });
8107 let Some(signal) = matched else {
8108 continue;
8109 };
8110 let signal_codec = signal
8111 .get("payload_codec")
8112 .and_then(Value::as_str)
8113 .unwrap_or(&export_codec);
8114 let Some(payload) = event.payload.as_object_mut() else {
8115 continue;
8116 };
8117 if missing_payload(payload.get("arguments")) {
8118 if let Some(arguments) = signal
8119 .get("arguments")
8120 .filter(|value| !missing_payload(Some(value)))
8121 {
8122 let envelope = match arguments {
8123 Value::String(blob) => json!({"codec": signal_codec, "blob": blob}),
8124 other => other.clone(),
8125 };
8126 payload.insert("arguments".to_string(), envelope);
8127 }
8128 }
8129 if payload
8130 .get("payload_codec")
8131 .and_then(Value::as_str)
8132 .unwrap_or_default()
8133 .is_empty()
8134 {
8135 payload.insert("payload_codec".to_string(), json!(signal_codec));
8136 }
8137 }
8138
8139 Ok(())
8140}
8141
8142fn missing_payload(value: Option<&Value>) -> bool {
8143 match value {
8144 None | Some(Value::Null) => true,
8145 Some(Value::String(value)) => value.is_empty(),
8146 Some(_) => false,
8147 }
8148}
8149
8150fn query_signal_events(task: &QueryTask) -> Result<Vec<QuerySignal>> {
8151 let export_signals = task
8152 .history_export
8153 .as_ref()
8154 .and_then(|export| export.get("signals"))
8155 .and_then(Value::as_array)
8156 .cloned()
8157 .unwrap_or_default();
8158 let export_codec = task
8159 .history_export
8160 .as_ref()
8161 .and_then(|export| export.get("payloads"))
8162 .and_then(|payloads| payloads.get("codec"))
8163 .and_then(Value::as_str)
8164 .unwrap_or(&task.payload_codec);
8165 let mut name_offsets: HashMap<String, usize> = HashMap::new();
8166 let mut signals = Vec::new();
8167
8168 for event in &task.history_events {
8169 if event.event_type != "SignalApplied" && event.event_type != "SignalReceived" {
8170 continue;
8171 }
8172
8173 let name = event
8174 .payload
8175 .get("signal_name")
8176 .and_then(Value::as_str)
8177 .unwrap_or_default();
8178 if name.is_empty() {
8179 continue;
8180 }
8181 let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
8182 let command_id = event
8183 .payload
8184 .get("workflow_command_id")
8185 .or_else(|| event.raw.get("workflow_command_id"))
8186 .and_then(Value::as_str);
8187 let matched_export = export_signals
8188 .iter()
8189 .find(|candidate| {
8190 signal_id.is_some() && candidate.get("id").and_then(Value::as_str) == signal_id
8191 })
8192 .or_else(|| {
8193 export_signals.iter().find(|candidate| {
8194 command_id.is_some()
8195 && candidate.get("command_id").and_then(Value::as_str) == command_id
8196 })
8197 })
8198 .or_else(|| {
8199 let offset = name_offsets.entry(name.to_string()).or_default();
8200 let candidate = export_signals
8201 .iter()
8202 .filter(|candidate| candidate.get("name").and_then(Value::as_str) == Some(name))
8203 .nth(*offset);
8204 if candidate.is_some() {
8205 *offset += 1;
8206 }
8207 candidate
8208 });
8209 let codec = event
8210 .payload
8211 .get("payload_codec")
8212 .and_then(Value::as_str)
8213 .or_else(|| {
8214 matched_export
8215 .and_then(|signal| signal.get("payload_codec"))
8216 .and_then(Value::as_str)
8217 })
8218 .unwrap_or(export_codec);
8219 let raw_arguments = event
8220 .payload
8221 .get("value")
8222 .or_else(|| event.payload.get("input"))
8223 .or_else(|| event.payload.get("arguments"))
8224 .filter(|value| !value.is_null())
8225 .or_else(|| matched_export.and_then(|signal| signal.get("arguments")));
8226 let (arguments, avro_arguments) = decode_query_signal_arguments(raw_arguments, codec)?;
8227 let workflow_sequence = event
8228 .payload
8229 .get("workflow_sequence")
8230 .and_then(value_as_u64)
8231 .or_else(|| {
8232 matched_export
8233 .and_then(|signal| signal.get("workflow_sequence"))
8234 .and_then(value_as_u64)
8235 });
8236
8237 signals.push(QuerySignal {
8238 id: signal_id.map(str::to_string).or_else(|| {
8239 matched_export
8240 .and_then(|signal| signal.get("id"))
8241 .and_then(Value::as_str)
8242 .map(str::to_string)
8243 }),
8244 name: name.to_string(),
8245 arguments,
8246 avro_arguments,
8247 workflow_sequence,
8248 });
8249 }
8250
8251 if signals.is_empty() {
8252 for signal in export_signals {
8253 if signal.get("status").and_then(Value::as_str) == Some("rejected") {
8254 continue;
8255 }
8256 let Some(name) = signal.get("name").and_then(Value::as_str) else {
8257 continue;
8258 };
8259 let codec = signal
8260 .get("payload_codec")
8261 .and_then(Value::as_str)
8262 .unwrap_or(export_codec);
8263 let (arguments, avro_arguments) =
8264 decode_query_signal_arguments(signal.get("arguments"), codec)?;
8265 signals.push(QuerySignal {
8266 id: signal.get("id").and_then(Value::as_str).map(str::to_string),
8267 name: name.to_string(),
8268 arguments,
8269 avro_arguments,
8270 workflow_sequence: signal.get("workflow_sequence").and_then(value_as_u64),
8271 });
8272 }
8273 signals.sort_by_key(|signal| signal.workflow_sequence.unwrap_or(u64::MAX));
8274 }
8275
8276 Ok(signals)
8277}
8278
8279fn decode_query_signal_arguments(
8280 raw: Option<&Value>,
8281 codec: &str,
8282) -> Result<(Vec<Value>, Vec<AvroValue>)> {
8283 let decoded = match raw.filter(|value| !value.is_null()) {
8284 Some(value) => decode_wire_avro_value(value, codec)?,
8285 None => AvroValue::Array(Vec::new()),
8286 };
8287 let AvroValue::Array(avro_arguments) = normalize_avro_arguments(decoded) else {
8288 unreachable!("normalize_avro_arguments always returns an array");
8289 };
8290 let arguments = avro_arguments
8291 .iter()
8292 .cloned()
8293 .map(AvroValue::into_json)
8294 .collect::<Result<Vec<_>>>()?;
8295 Ok((arguments, avro_arguments))
8296}
8297
8298fn value_as_u64(value: &Value) -> Option<u64> {
8299 value
8300 .as_u64()
8301 .or_else(|| value.as_str().and_then(|value| value.parse().ok()))
8302}
8303
8304#[cfg(test)]
8305mod tests {
8306 use super::*;
8307 use std::{
8308 io::{Read, Write},
8309 net::{SocketAddr, TcpListener, TcpStream},
8310 sync::atomic::AtomicUsize,
8311 thread,
8312 };
8313
8314 #[test]
8315 fn client_builder_rejects_the_sdk_owned_api_suffix() {
8316 for base_url in [
8317 "http://127.0.0.1:8080/api",
8318 "http://localhost:8080/api/",
8319 "https://runtime.example.test/namespaces/orders/api",
8320 ] {
8321 let error = Client::builder(base_url)
8322 .build()
8323 .expect_err("SDK-owned /api suffix must be rejected during build");
8324
8325 assert!(matches!(error, Error::InvalidBaseUrl), "{base_url}");
8326 assert!(
8327 error.to_string().contains("SDK appends /api automatically"),
8328 "the validation error must explain how to fix the endpoint"
8329 );
8330 }
8331 }
8332
8333 #[test]
8334 fn client_builder_preserves_self_hosted_and_managed_runtime_prefixes() {
8335 for (base_url, expected) in [
8336 ("http://127.0.0.1:8080", "http://127.0.0.1:8080"),
8337 (
8338 "http://localhost:8080/durable-workflow/",
8339 "http://localhost:8080/durable-workflow",
8340 ),
8341 (
8342 "https://runtime.example.test/namespaces/orders",
8343 "https://runtime.example.test/namespaces/orders",
8344 ),
8345 (
8346 "https://runtime.example.test/gateway/api/namespaces/orders",
8347 "https://runtime.example.test/gateway/api/namespaces/orders",
8348 ),
8349 (
8350 "https://api.example.test/runtime/orders/",
8351 "https://api.example.test/runtime/orders",
8352 ),
8353 ] {
8354 let client = Client::builder(base_url)
8355 .build()
8356 .expect("Server and Cloud runtime base URL must remain valid");
8357
8358 assert_eq!(client.base_url, expected);
8359 }
8360 }
8361
8362 fn typed_fidelity_probe() -> AvroValue {
8363 AvroValue::Map(BTreeMap::from([
8364 ("bytes".to_string(), AvroValue::Bytes(vec![0, 0xff])),
8365 ("empty".to_string(), AvroValue::Map(BTreeMap::new())),
8366 (
8367 "numeric".to_string(),
8368 AvroValue::Map(BTreeMap::from([
8369 ("0".to_string(), AvroValue::String("zero".to_string())),
8370 ("1".to_string(), AvroValue::String("one".to_string())),
8371 ])),
8372 ),
8373 (
8374 "nested".to_string(),
8375 AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([(
8376 "enabled".to_string(),
8377 AvroValue::Boolean(true),
8378 )]))]),
8379 ),
8380 (
8381 "projection_collisions".to_string(),
8382 AvroValue::Array(projection_collision_probe()),
8383 ),
8384 ]))
8385 }
8386
8387 fn projection_collision_probe() -> Vec<AvroValue> {
8388 vec![
8389 AvroValue::Map(BTreeMap::from([
8390 ("$type".to_string(), AvroValue::String("bytes".to_string())),
8391 (
8392 "base64".to_string(),
8393 AvroValue::String("ordinary user text".to_string()),
8394 ),
8395 ])),
8396 AvroValue::Map(BTreeMap::from([
8397 ("$type".to_string(), AvroValue::String("map".to_string())),
8398 (
8399 "entries".to_string(),
8400 AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([
8401 ("key".to_string(), AvroValue::String("ordinary".to_string())),
8402 (
8403 "value".to_string(),
8404 AvroValue::String("user map".to_string()),
8405 ),
8406 ]))]),
8407 ),
8408 ])),
8409 ]
8410 }
8411
8412 #[derive(Clone, Debug, Default, PartialEq)]
8413 struct ReplayCounterState {
8414 loaded: Option<String>,
8415 count: i64,
8416 finished: bool,
8417 }
8418
8419 fn replay_counter_worker() -> Worker {
8420 let client = Client::new("http://127.0.0.1:8080").expect("client");
8421 let mut worker = Worker::new(client, "rust-workers");
8422 worker.register_replayed_workflow(
8423 "replay-counter",
8424 ReplayCounterState::default,
8425 |ctx, _input, state| async move {
8426 let loaded = ctx.activity("load-counter", json!([])).await?;
8427 state.update(|current| {
8428 current.loaded = loaded.as_str().map(str::to_string);
8429 })?;
8430 for _ in 0..2 {
8431 let signal = ctx.wait_signal("increment").await?;
8432 let amount = signal.first().and_then(Value::as_i64).unwrap_or_default();
8433 state.update(|current| current.count += amount)?;
8434 }
8435 state.update(|current| current.finished = true)?;
8436 state.read(|current| Ok(json!(current.count)))?
8437 },
8438 );
8439 worker.register_replayed_query::<ReplayCounterState, _, _>(
8440 "replay-counter",
8441 "current",
8442 |_ctx, state, _args| async move {
8443 Ok(json!({
8444 "loaded": state.loaded,
8445 "count": state.count,
8446 "finished": state.finished,
8447 }))
8448 },
8449 );
8450 worker.register_replayed_query::<ReplayCounterState, _, _>(
8451 "replay-counter",
8452 "detached-mutation",
8453 |_ctx, state, _args| async move {
8454 let mut detached = (*state).clone();
8455 detached.count = 999;
8456 Ok(json!(detached.count))
8457 },
8458 );
8459 worker.register_replayed_query::<ReplayCounterState, _, _>(
8460 "replay-counter",
8461 "failed-mutation",
8462 |_ctx, state, _args| async move {
8463 let mut detached = (*state).clone();
8464 detached.count = 999;
8465 Err(Error::WorkerLoop("query refused".to_string()))
8466 },
8467 );
8468 worker
8469 }
8470
8471 fn replay_counter_query(
8472 query_name: &str,
8473 history_events: Value,
8474 run_status: &str,
8475 ) -> QueryTask {
8476 serde_json::from_value(json!({
8477 "query_task_id": format!("query-{query_name}"),
8478 "workflow_type": "replay-counter",
8479 "query_name": query_name,
8480 "payload_codec": "json",
8481 "workflow_arguments": {"codec": "json", "blob": "[]"},
8482 "query_arguments": {"codec": "json", "blob": "[]"},
8483 "history_events": history_events,
8484 "run_status": run_status,
8485 }))
8486 .expect("query task")
8487 }
8488
8489 fn workflow_context(history: Vec<HistoryEvent>) -> WorkflowContext {
8490 workflow_context_with_codec(history, JSON_CODEC)
8491 }
8492
8493 fn workflow_context_with_codec(
8494 history: Vec<HistoryEvent>,
8495 payload_codec: &str,
8496 ) -> WorkflowContext {
8497 WorkflowContext {
8498 state: Arc::new(Mutex::new(
8499 WorkflowState::new_with_identity(
8500 history,
8501 None,
8502 None,
8503 "rust-workers".to_string(),
8504 payload_codec.to_string(),
8505 None,
8506 )
8507 .expect("valid workflow history"),
8508 )),
8509 }
8510 }
8511
8512 fn history_event(event_type: &str, payload: Value) -> HistoryEvent {
8513 HistoryEvent {
8514 event_type: event_type.to_string(),
8515 payload,
8516 raw: HashMap::new(),
8517 }
8518 }
8519
8520 fn workflow_task(
8521 workflow_type: &str,
8522 history_events: Vec<HistoryEvent>,
8523 payload_codec: &str,
8524 ) -> WorkflowTask {
8525 WorkflowTask {
8526 task_id: format!("wft-{workflow_type}"),
8527 workflow_id: Some(format!("wf-{workflow_type}")),
8528 run_id: Some(format!("run-{workflow_type}")),
8529 workflow_type: workflow_type.to_string(),
8530 payload_codec: payload_codec.to_string(),
8531 arguments: Some(
8532 encode_value_envelope(&json!([]), payload_codec).expect("workflow arguments"),
8533 ),
8534 total_history_events: Some(history_events.len() as u64),
8535 history_size_bytes: None,
8536 continue_as_new_recommended: None,
8537 history_budget_pressure: None,
8538 history_events,
8539 next_history_page_token: None,
8540 workflow_task_attempt: 1,
8541 workflow_signal_id: None,
8542 signal_name: None,
8543 signal_arguments: None,
8544 workflow_update_id: None,
8545 update_name: None,
8546 lease_owner: Some("rust-worker".to_string()),
8547 }
8548 }
8549
8550 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
8551 struct SideEffectProbe {
8552 request_id: String,
8553 attempt: u32,
8554 }
8555
8556 #[test]
8557 fn typed_side_effect_runs_callback_once_and_replay_skips_it() {
8558 let calls = AtomicUsize::new(0);
8559 let ctx = workflow_context(Vec::new());
8560 let value = ctx
8561 .side_effect(|| {
8562 calls.fetch_add(1, Ordering::SeqCst);
8563 SideEffectProbe {
8564 request_id: "request-42".to_string(),
8565 attempt: 3,
8566 }
8567 })
8568 .expect("first side effect");
8569 assert_eq!(value.attempt, 3);
8570 assert_eq!(calls.load(Ordering::SeqCst), 1);
8571 let commands = ctx.take_commands().expect("commands");
8572 assert_eq!(commands.len(), 1);
8573 assert_eq!(commands[0]["type"], "record_side_effect");
8574 assert_eq!(
8575 decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("JSON result"),
8576 serde_json::to_value(&value).expect("value")
8577 );
8578
8579 let replay = workflow_context(vec![history_event(
8580 "SideEffectRecorded",
8581 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
8582 )]);
8583 let replayed: SideEffectProbe = replay
8584 .side_effect(|| {
8585 calls.fetch_add(1, Ordering::SeqCst);
8586 panic!("committed side-effect callbacks must not run during replay")
8587 })
8588 .expect("replayed side effect");
8589 assert_eq!(replayed, value);
8590 assert_eq!(calls.load(Ordering::SeqCst), 1);
8591 assert!(replay.take_commands().expect("commands").is_empty());
8592 replay.ensure_history_consumed().expect("history consumed");
8593 }
8594
8595 #[test]
8596 fn side_effect_uses_avro_envelope_and_uuid_is_replay_stable() {
8597 let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
8598 let value = ctx
8599 .side_effect(|| SideEffectProbe {
8600 request_id: "avro-request".to_string(),
8601 attempt: 1,
8602 })
8603 .expect("Avro side effect");
8604 let uuid = ctx.uuid_v4().expect("deterministic UUID");
8605 let commands = ctx.take_commands().expect("commands");
8606 assert_eq!(commands.len(), 2);
8607 assert_eq!(commands[0]["result"]["codec"], DEFAULT_CODEC);
8608 assert_eq!(commands[1]["result"]["codec"], DEFAULT_CODEC);
8609 assert_eq!(
8610 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
8611 serde_json::to_value(&value).expect("value")
8612 );
8613
8614 let replay = workflow_context_with_codec(
8615 vec![
8616 history_event(
8617 "SideEffectRecorded",
8618 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
8619 ),
8620 history_event(
8621 "SideEffectRecorded",
8622 json!({"sequence": 2, "result": commands[1]["result"].clone()}),
8623 ),
8624 ],
8625 DEFAULT_CODEC,
8626 );
8627 let replayed: SideEffectProbe = replay
8628 .side_effect(|| panic!("Avro callback must not run"))
8629 .expect("replayed Avro value");
8630 let replayed_uuid = replay.uuid_v4().expect("replayed UUID");
8631 assert_eq!(replayed, value);
8632 assert_eq!(replayed_uuid, uuid);
8633 assert!(replay.take_commands().expect("commands").is_empty());
8634 }
8635
8636 #[test]
8637 fn typed_side_effect_replay_preserves_bytes_and_maps() {
8638 let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
8639 let value = ctx
8640 .side_effect_avro_value(typed_fidelity_probe)
8641 .expect("typed side effect");
8642 let commands = ctx.take_commands().expect("side-effect command");
8643 assert_eq!(
8644 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
8645 .expect("recorded side effect"),
8646 value
8647 );
8648
8649 let replay = workflow_context_with_codec(
8650 vec![history_event(
8651 "SideEffectRecorded",
8652 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
8653 )],
8654 DEFAULT_CODEC,
8655 );
8656 assert_eq!(
8657 replay
8658 .side_effect_avro_value(|| panic!("replay must not invoke callback"))
8659 .expect("replayed typed side effect"),
8660 value
8661 );
8662 }
8663
8664 #[test]
8665 fn ordered_side_effects_share_the_durable_command_stream() {
8666 let first = encode_value_envelope(&json!("first"), JSON_CODEC).expect("first");
8667 let second = encode_value_envelope(&json!(29), JSON_CODEC).expect("second");
8668 let ctx = workflow_context(vec![
8669 history_event(
8670 "SideEffectRecorded",
8671 json!({"sequence": 1, "result": first}),
8672 ),
8673 history_event(
8674 "SideEffectRecorded",
8675 json!({"sequence": 2, "result": second}),
8676 ),
8677 ]);
8678 let first: String = ctx
8679 .side_effect(|| panic!("first callback must not run"))
8680 .expect("first replay");
8681 let second: i32 = ctx
8682 .side_effect(|| panic!("second callback must not run"))
8683 .expect("second replay");
8684 assert_eq!(first, "first");
8685 assert_eq!(second, 29);
8686 ctx.ensure_history_consumed().expect("ordered history");
8687
8688 let reordered = workflow_context(vec![history_event(
8689 "VersionMarkerRecorded",
8690 json!({
8691 "sequence": 1,
8692 "change_id": "before-side-effect",
8693 "version": 1,
8694 "min_supported": 1,
8695 "max_supported": 1,
8696 }),
8697 )]);
8698 let error = reordered
8699 .side_effect(|| "new".to_string())
8700 .expect_err("command reordering must fail");
8701 assert!(matches!(
8702 error,
8703 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8704 if reason == "recorded_command_mismatch"
8705 ));
8706 }
8707
8708 #[test]
8709 fn version_markers_replay_across_upgrades_and_do_not_duplicate() {
8710 let ctx = workflow_context(Vec::new());
8711 assert_eq!(ctx.get_version("checkout-v2", 1, 2).expect("version"), 2);
8712 assert_eq!(ctx.get_version("checkout-v2", 1, 3).expect("cached"), 2);
8713 assert!(ctx.patched("new-search").expect("patch"));
8714 ctx.deprecate_patch("new-search").expect("deprecate patch");
8715 let commands = ctx.take_commands().expect("commands");
8716 assert_eq!(commands.len(), 2);
8717 assert_eq!(commands[0]["type"], "record_version_marker");
8718 assert_eq!(commands[0]["version"], 2);
8719 assert_eq!(commands[1]["change_id"], "new-search");
8720
8721 let replay = workflow_context(vec![history_event(
8722 "VersionMarkerRecorded",
8723 json!({
8724 "sequence": 1,
8725 "change_id": "checkout-v2",
8726 "version": 2,
8727 "min_supported": 1,
8728 "max_supported": 2,
8729 }),
8730 )]);
8731 assert_eq!(replay.get_version("checkout-v2", 1, 4).expect("upgrade"), 2);
8732 assert_eq!(replay.get_version("checkout-v2", 2, 5).expect("repeat"), 2);
8733 assert!(replay.take_commands().expect("commands").is_empty());
8734 replay.ensure_history_consumed().expect("history consumed");
8735 }
8736
8737 #[test]
8738 fn version_markers_reject_incompatible_or_malformed_history() {
8739 let incompatible = workflow_context(vec![history_event(
8740 "VersionMarkerRecorded",
8741 json!({
8742 "sequence": 1,
8743 "change_id": "checkout-v2",
8744 "version": 1,
8745 "min_supported": 1,
8746 "max_supported": 2,
8747 }),
8748 )]);
8749 let error = incompatible
8750 .get_version("checkout-v2", 2, 3)
8751 .expect_err("old version is unsupported");
8752 assert!(matches!(
8753 error,
8754 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8755 if reason == "version_marker_incompatible_range"
8756 ));
8757
8758 for (history, reason) in [
8759 (
8760 vec![history_event("SideEffectRecorded", json!({"sequence": 1}))],
8761 "side_effect_result_missing",
8762 ),
8763 (
8764 vec![history_event(
8765 "SideEffectRecorded",
8766 json!({
8767 "sequence": 1,
8768 "result": {"codec": "avro", "blob": "not-base64"},
8769 }),
8770 )],
8771 "side_effect_payload_incompatible",
8772 ),
8773 (
8774 vec![history_event(
8775 "SideEffectRecorded",
8776 json!({"sequence": 1, "result": {"unwrapped": true}}),
8777 )],
8778 "side_effect_payload_malformed",
8779 ),
8780 (
8781 vec![history_event(
8782 "VersionMarkerRecorded",
8783 json!({
8784 "sequence": 1,
8785 "change_id": "change",
8786 "version": 1,
8787 "min_supported": 2,
8788 "max_supported": 1,
8789 }),
8790 )],
8791 "version_marker_history_range_invalid",
8792 ),
8793 ] {
8794 let error = WorkflowState::new(
8795 history,
8796 "rust-workers".to_string(),
8797 JSON_CODEC.to_string(),
8798 None,
8799 )
8800 .expect_err("malformed history must fail");
8801 assert!(matches!(
8802 error,
8803 Error::NonDeterministicReplay(ReplayFailure { reason: actual, .. })
8804 if actual == reason
8805 ));
8806 }
8807 }
8808
8809 #[test]
8810 fn duplicate_side_effects_and_version_markers_are_rejected() {
8811 let duplicate_side_effect = WorkflowState::new(
8812 vec![
8813 history_event(
8814 "SideEffectRecorded",
8815 json!({"sequence": 1, "result": {"codec": "json", "blob": "1"}}),
8816 ),
8817 history_event(
8818 "SideEffectRecorded",
8819 json!({"sequence": 1, "result": {"codec": "json", "blob": "2"}}),
8820 ),
8821 ],
8822 "rust-workers".to_string(),
8823 JSON_CODEC.to_string(),
8824 None,
8825 )
8826 .expect_err("duplicate side effect");
8827 assert!(matches!(
8828 duplicate_side_effect,
8829 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8830 if reason == "duplicate_side_effect_record"
8831 ));
8832
8833 let marker = |sequence| {
8834 history_event(
8835 "VersionMarkerRecorded",
8836 json!({
8837 "sequence": sequence,
8838 "change_id": "same-change",
8839 "version": 1,
8840 "min_supported": 1,
8841 "max_supported": 1,
8842 }),
8843 )
8844 };
8845 let duplicate_marker = WorkflowState::new(
8846 vec![marker(1), marker(3)],
8847 "rust-workers".to_string(),
8848 JSON_CODEC.to_string(),
8849 None,
8850 )
8851 .expect_err("duplicate marker");
8852 assert!(matches!(
8853 duplicate_marker,
8854 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8855 if reason == "duplicate_version_marker"
8856 ));
8857 }
8858
8859 #[test]
8860 fn cold_worker_replay_does_not_repeat_committed_side_effects_or_markers() {
8861 fn worker(calls: Arc<AtomicUsize>) -> Worker {
8862 let client = Client::new("http://127.0.0.1:8080").expect("client");
8863 let mut worker = Worker::new(client, "rust-workers");
8864 worker.register_workflow("rust.side-effect-version", move |ctx, _input| {
8865 let calls = Arc::clone(&calls);
8866 async move {
8867 let captured = ctx.side_effect(|| {
8868 calls.fetch_add(1, Ordering::SeqCst);
8869 "captured-once".to_string()
8870 })?;
8871 let version = ctx.get_version("cold-restart", 1, 2)?;
8872 Ok(json!({"captured": captured, "version": version}))
8873 }
8874 });
8875 worker
8876 }
8877
8878 fn task(history_events: Vec<HistoryEvent>) -> WorkflowTask {
8879 WorkflowTask {
8880 task_id: "wft-side-effect-version".to_string(),
8881 workflow_id: Some("wf-side-effect-version".to_string()),
8882 run_id: Some("run-side-effect-version".to_string()),
8883 workflow_type: "rust.side-effect-version".to_string(),
8884 payload_codec: JSON_CODEC.to_string(),
8885 arguments: Some(encode_value_envelope(&json!([]), JSON_CODEC).expect("arguments")),
8886 history_events,
8887 total_history_events: None,
8888 history_size_bytes: None,
8889 continue_as_new_recommended: None,
8890 history_budget_pressure: None,
8891 next_history_page_token: None,
8892 workflow_task_attempt: 1,
8893 workflow_signal_id: None,
8894 signal_name: None,
8895 signal_arguments: None,
8896 workflow_update_id: None,
8897 update_name: None,
8898 lease_owner: Some("rust-worker".to_string()),
8899 }
8900 }
8901
8902 let calls = Arc::new(AtomicUsize::new(0));
8903 let initial = worker(Arc::clone(&calls))
8904 .execute_workflow_task(task(Vec::new()))
8905 .expect("initial execution");
8906 assert_eq!(
8907 initial
8908 .iter()
8909 .map(|command| &command["type"])
8910 .collect::<Vec<_>>(),
8911 vec![
8912 "record_side_effect",
8913 "record_version_marker",
8914 "complete_workflow"
8915 ]
8916 );
8917 assert_eq!(calls.load(Ordering::SeqCst), 1);
8918
8919 let restarted = worker(Arc::clone(&calls));
8920 let replayed = restarted
8921 .execute_workflow_task(task(vec![
8922 history_event(
8923 "SideEffectRecorded",
8924 json!({"sequence": 1, "result": initial[0]["result"].clone()}),
8925 ),
8926 history_event(
8927 "VersionMarkerRecorded",
8928 json!({
8929 "sequence": 2,
8930 "change_id": "cold-restart",
8931 "version": 2,
8932 "min_supported": 1,
8933 "max_supported": 2,
8934 }),
8935 ),
8936 ]))
8937 .expect("cold replay");
8938 assert_eq!(replayed.len(), 1);
8939 assert_eq!(replayed[0]["type"], "complete_workflow");
8940 assert_eq!(calls.load(Ordering::SeqCst), 1);
8941 }
8942
8943 #[test]
8944 fn side_effect_replay_rejects_changed_rust_value_type() {
8945 let result = encode_value_envelope(&json!({"value": 42}), JSON_CODEC).expect("result");
8946 let ctx = workflow_context(vec![history_event(
8947 "SideEffectRecorded",
8948 json!({"sequence": 1, "result": result}),
8949 )]);
8950 let error = ctx
8951 .side_effect::<Vec<String>, _>(|| panic!("callback must not run"))
8952 .expect_err("changed type must fail replay");
8953 assert!(matches!(
8954 error,
8955 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8956 if reason == "side_effect_type_mismatch"
8957 ));
8958 }
8959
8960 fn completed_retry_activity_history() -> Vec<HistoryEvent> {
8961 vec![
8962 history_event(
8963 "ActivityScheduled",
8964 json!({
8965 "sequence": 1,
8966 "activity_type": "flaky",
8967 "activity_execution_id": "act-1",
8968 "activity": {
8969 "id": "act-1",
8970 "sequence": 1,
8971 "type": "flaky",
8972 "queue": "critical-activities",
8973 "execution_mode": null,
8974 "retry_policy": {
8975 "snapshot_version": 1,
8976 "max_attempts": 3,
8977 "backoff_seconds": [2, 4],
8978 "start_to_close_timeout": 30,
8979 "schedule_to_start_timeout": 5,
8980 "schedule_to_close_timeout": 90,
8981 "heartbeat_timeout": 10,
8982 "non_retryable_error_types": ["PermanentError"]
8983 }
8984 }
8985 }),
8986 ),
8987 history_event(
8988 "ActivityStarted",
8989 json!({
8990 "sequence": 1,
8991 "activity_type": "flaky",
8992 "activity_execution_id": "act-1",
8993 "activity_attempt_id": "attempt-1",
8994 "attempt_number": 1
8995 }),
8996 ),
8997 history_event(
8998 "ActivityRetryScheduled",
8999 json!({
9000 "sequence": 1,
9001 "activity_type": "flaky",
9002 "activity_execution_id": "act-1",
9003 "activity_attempt_id": "attempt-1",
9004 "attempt_number": 1,
9005 "retry_after_attempt": 1,
9006 "retry_backoff_seconds": 2,
9007 "failure_category": "activity",
9008 "exception_type": "TransientError"
9009 }),
9010 ),
9011 history_event(
9012 "ActivityStarted",
9013 json!({
9014 "sequence": 1,
9015 "activity_type": "flaky",
9016 "activity_execution_id": "act-1",
9017 "activity_attempt_id": "attempt-2",
9018 "attempt_number": 2
9019 }),
9020 ),
9021 history_event(
9022 "ActivityCompleted",
9023 json!({
9024 "sequence": 1,
9025 "activity_type": "flaky",
9026 "activity_execution_id": "act-1",
9027 "activity_attempt_id": "attempt-2",
9028 "attempt_number": 2,
9029 "payload_codec": "json",
9030 "result": {"codec": "json", "blob": "{\"status\":\"recovered\"}"}
9031 }),
9032 ),
9033 ]
9034 }
9035
9036 fn retry_activity_options() -> ActivityOptions {
9037 ActivityOptions::new()
9038 .task_queue("critical-activities")
9039 .retry_policy(
9040 ActivityRetryPolicy::new(3)
9041 .backoff_intervals([Duration::from_secs(2), Duration::from_secs(4)])
9042 .non_retryable_error_type("PermanentError"),
9043 )
9044 .start_to_close_timeout(Duration::from_secs(30))
9045 .schedule_to_start_timeout(Duration::from_secs(5))
9046 .schedule_to_close_timeout(Duration::from_secs(90))
9047 .heartbeat_timeout(Duration::from_secs(10))
9048 }
9049
9050 #[test]
9051 fn fixed_avro_value_round_trips_json_values() {
9052 let value = json!({"greeting": "hello", "count": 3, "ok": true});
9053 let envelope = PayloadEnvelope::avro(&value).expect("encode");
9054 assert_eq!(envelope.codec, DEFAULT_CODEC);
9055 assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
9056 }
9057
9058 #[tokio::test]
9059 async fn typed_worker_surfaces_preserve_bytes_and_map_list_identity() {
9060 let client = Client::new("http://127.0.0.1:8080").expect("client");
9061 let mut worker = Worker::new(client, "rust-workers");
9062 worker.register_workflow_avro_value("typed.echo", |_ctx, input| async move { Ok(input) });
9063 worker
9064 .register_activity_avro_value("typed.activity", |_ctx, input| async move { Ok(input) });
9065 worker.register_query_avro_value("typed.echo", "inspect", |_ctx, input| async move {
9066 Ok(input)
9067 });
9068 worker.register_update_avro_value("typed.echo", "replace", |_ctx, input| async move {
9069 Ok(input)
9070 });
9071 worker.register_workflow_avro_value("typed.signal", |ctx, _input| async move {
9072 Ok(AvroValue::Array(
9073 ctx.wait_signal_avro_value("changed").await?,
9074 ))
9075 });
9076
9077 let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
9078 let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed envelope");
9079
9080 let mut workflow = workflow_task("typed.echo", Vec::new(), DEFAULT_CODEC);
9081 workflow.arguments = Some(envelope.clone());
9082 let commands = worker
9083 .execute_workflow_task(workflow)
9084 .expect("typed workflow task");
9085 assert_eq!(commands[0]["type"], "complete_workflow");
9086 assert_eq!(
9087 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
9088 .expect("typed workflow result"),
9089 arguments
9090 );
9091
9092 let activity = ActivityTask {
9093 task_id: "activity-typed".to_string(),
9094 activity_attempt_id: Some("attempt-typed".to_string()),
9095 attempt_id: None,
9096 activity_type: "typed.activity".to_string(),
9097 payload_codec: DEFAULT_CODEC.to_string(),
9098 arguments: Some(envelope.clone()),
9099 attempt_number: 1,
9100 lease_owner: Some("rust-worker".to_string()),
9101 };
9102 assert_eq!(
9103 worker
9104 .execute_activity_task(activity)
9105 .await
9106 .expect("typed activity result"),
9107 arguments
9108 );
9109
9110 let query = QueryTask {
9111 query_task_id: "query-typed".to_string(),
9112 query_task_attempt: 1,
9113 lease_owner: Some("rust-worker".to_string()),
9114 workflow_id: Some("typed-1".to_string()),
9115 run_id: Some("run-typed".to_string()),
9116 workflow_type: "typed.echo".to_string(),
9117 query_name: "inspect".to_string(),
9118 payload_codec: DEFAULT_CODEC.to_string(),
9119 workflow_arguments: Some(
9120 encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
9121 .expect("workflow input"),
9122 ),
9123 query_arguments: Some(envelope.clone()),
9124 history_events: Vec::new(),
9125 history_export: None,
9126 run_status: Some("running".to_string()),
9127 };
9128 assert_eq!(
9129 worker
9130 .execute_query_task(query)
9131 .await
9132 .expect("typed query result"),
9133 arguments
9134 );
9135
9136 let mut update = workflow_task(
9137 "typed.echo",
9138 vec![history_event(
9139 "UpdateAccepted",
9140 json!({
9141 "update_id": "update-typed",
9142 "update_name": "replace",
9143 "arguments": envelope.clone(),
9144 }),
9145 )],
9146 DEFAULT_CODEC,
9147 );
9148 update.workflow_update_id = Some("update-typed".to_string());
9149 update.update_name = Some("replace".to_string());
9150 let commands = worker
9151 .execute_workflow_task(update)
9152 .expect("typed update task");
9153 assert_eq!(commands[0]["type"], "complete_update");
9154 assert_eq!(
9155 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
9156 .expect("typed update result"),
9157 arguments
9158 );
9159
9160 let mut signal = workflow_task(
9161 "typed.signal",
9162 vec![history_event(
9163 "SignalReceived",
9164 json!({
9165 "signal_id": "signal-typed",
9166 "signal_name": "changed",
9167 "arguments": envelope.clone(),
9168 }),
9169 )],
9170 DEFAULT_CODEC,
9171 );
9172 signal.workflow_signal_id = Some("signal-typed".to_string());
9173 signal.signal_name = Some("changed".to_string());
9174 signal.signal_arguments = Some(envelope);
9175 let commands = worker
9176 .execute_workflow_task(signal)
9177 .expect("typed signal resume");
9178 assert_eq!(
9179 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
9180 .expect("typed signal result"),
9181 arguments
9182 );
9183 }
9184
9185 #[tokio::test]
9186 async fn typed_helpers_never_parse_json_inspection_projection() {
9187 let collision_values = projection_collision_probe();
9188 let expected = AvroValue::Array(collision_values.clone());
9189 let envelope = encode_typed_envelope(&expected, DEFAULT_CODEC).expect("collision envelope");
9190
9191 let activity_context = workflow_context_with_codec(
9192 vec![history_event(
9193 "ActivityCompleted",
9194 json!({
9195 "sequence": 1,
9196 "activity_type": "collision.activity",
9197 "payload_codec": DEFAULT_CODEC,
9198 "result": envelope.clone(),
9199 }),
9200 )],
9201 DEFAULT_CODEC,
9202 );
9203 assert_eq!(
9204 activity_context
9205 .activity_avro_value("collision.activity", AvroValue::Array(Vec::new()))
9206 .await
9207 .expect("typed activity collision result"),
9208 expected
9209 );
9210
9211 let signal_context = workflow_context_with_codec(
9212 vec![
9213 history_event(
9214 "SignalWaitOpened",
9215 json!({"sequence": 1, "signal_name": "collision"}),
9216 ),
9217 history_event(
9218 "SignalApplied",
9219 json!({
9220 "sequence": 1,
9221 "signal_name": "collision",
9222 "payload_codec": DEFAULT_CODEC,
9223 "value": envelope.clone(),
9224 }),
9225 ),
9226 ],
9227 DEFAULT_CODEC,
9228 );
9229 assert_eq!(
9230 signal_context
9231 .wait_signal_avro_value("collision")
9232 .await
9233 .expect("typed signal collision arguments"),
9234 collision_values
9235 );
9236
9237 let child_context = workflow_context_with_codec(
9238 vec![
9239 history_event(
9240 "ChildWorkflowScheduled",
9241 json!({
9242 "sequence": 1,
9243 "child_workflow_instance_id": "collision-child",
9244 "child_workflow_run_id": "collision-run",
9245 "child_workflow_type": "collision.child",
9246 }),
9247 ),
9248 history_event(
9249 "ChildRunCompleted",
9250 json!({
9251 "sequence": 1,
9252 "child_workflow_instance_id": "collision-child",
9253 "child_workflow_run_id": "collision-run",
9254 "child_workflow_type": "collision.child",
9255 "payload_codec": DEFAULT_CODEC,
9256 "result": envelope,
9257 }),
9258 ),
9259 ],
9260 DEFAULT_CODEC,
9261 );
9262 let child = child_context
9263 .start_child_workflow_avro_value(
9264 "collision.child",
9265 ChildWorkflowOptions::new("collision-workers"),
9266 AvroValue::Array(Vec::new()),
9267 )
9268 .await
9269 .expect("typed child collision result");
9270 assert_eq!(child.result, expected);
9271 }
9272
9273 #[tokio::test]
9274 async fn replayed_typed_query_keeps_lossless_workflow_and_query_inputs() {
9275 let client = Client::new("http://127.0.0.1:8080").expect("client");
9276 let mut worker = Worker::new(client, "rust-workers");
9277 worker.register_replayed_workflow_avro_value(
9278 "typed.replayed",
9279 || (),
9280 |_ctx, input, _state| async move { Ok(input) },
9281 );
9282 worker.register_replayed_query_avro_value::<(), _, _>(
9283 "typed.replayed",
9284 "inspect",
9285 |ctx, _state, args| async move {
9286 let mut signals = ctx.signals_avro_value("collision");
9287 let signal = signals
9288 .pop()
9289 .map(AvroValue::Array)
9290 .unwrap_or_else(|| AvroValue::Array(Vec::new()));
9291 Ok(AvroValue::Array(vec![
9292 ctx.workflow_input_avro_value().clone(),
9293 signal,
9294 args,
9295 ]))
9296 },
9297 );
9298 let arguments = AvroValue::Array(projection_collision_probe());
9299 let signal_arguments =
9300 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed query signal arguments");
9301 let task = QueryTask {
9302 query_task_id: "query-typed-replay".to_string(),
9303 query_task_attempt: 1,
9304 lease_owner: Some("rust-worker".to_string()),
9305 workflow_id: Some("typed-replay".to_string()),
9306 run_id: Some("run-typed-replay".to_string()),
9307 workflow_type: "typed.replayed".to_string(),
9308 query_name: "inspect".to_string(),
9309 payload_codec: DEFAULT_CODEC.to_string(),
9310 workflow_arguments: Some(
9311 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("workflow arguments"),
9312 ),
9313 query_arguments: Some(
9314 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("query arguments"),
9315 ),
9316 history_events: vec![history_event(
9317 "SignalReceived",
9318 json!({
9319 "signal_id": "collision-signal",
9320 "signal_name": "collision",
9321 "workflow_sequence": 1,
9322 "payload_codec": DEFAULT_CODEC,
9323 "arguments": signal_arguments,
9324 }),
9325 )],
9326 history_export: None,
9327 run_status: Some("completed".to_string()),
9328 };
9329
9330 assert_eq!(
9331 worker
9332 .execute_query_task(task)
9333 .await
9334 .expect("typed replay query"),
9335 AvroValue::Array(vec![arguments.clone(), arguments.clone(), arguments])
9336 );
9337 }
9338
9339 #[test]
9340 fn public_avro_adapter_rejects_non_string_map_keys_before_json_conversion() {
9341 let value = BTreeMap::from([(1_i32, "integer key")]);
9342 let error = PayloadEnvelope::avro(&value)
9343 .expect_err("integer map keys must fail")
9344 .to_string();
9345
9346 assert!(error.contains("invalid_map_key"));
9347 }
9348
9349 #[test]
9350 fn json_codec_remains_plain_json() {
9351 let value = json!({"greeting": "hello", "count": 3, "ok": true});
9352 let envelope = PayloadEnvelope::json(&value).expect("encode");
9353
9354 assert_eq!(envelope.codec, JSON_CODEC);
9355 assert_eq!(envelope.blob, serde_json::to_string(&value).expect("json"));
9356 assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
9357 }
9358
9359 #[test]
9360 fn prerelease_avro_payload_without_single_object_frame_is_rejected() {
9361 let envelope = PayloadEnvelope {
9362 codec: DEFAULT_CODEC.to_string(),
9363 blob: BASE64.encode([0x01]),
9364 };
9365
9366 let error = decode_payload::<Value>(&envelope).expect_err("prerelease payload must fail");
9367 assert!(error.to_string().contains("invalid_payload_framing"));
9368 }
9369
9370 #[test]
9371 fn workflow_context_schedules_activity_until_completion_is_in_history() {
9372 let ctx = WorkflowContext {
9373 state: Arc::new(Mutex::new(
9374 WorkflowState::new_with_identity(
9375 Vec::new(),
9376 Some("wf-parent".to_string()),
9377 Some("run-parent".to_string()),
9378 "rust-workers".to_string(),
9379 DEFAULT_CODEC.to_string(),
9380 None,
9381 )
9382 .expect("workflow state"),
9383 )),
9384 };
9385
9386 let mut call = Box::pin(ctx.activity("hello.activity", json!(["Ada"])));
9387 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9388 assert!(matches!(
9389 call.as_mut().poll(&mut task_context),
9390 Poll::Pending
9391 ));
9392
9393 let commands = ctx.take_commands().expect("commands");
9394 assert_eq!(commands[0]["type"], "schedule_activity");
9395 assert_eq!(commands[0]["activity_type"], "hello.activity");
9396 }
9397
9398 #[test]
9399 fn activity_options_encode_retry_policy_queue_and_every_timeout() {
9400 let ctx = workflow_context(Vec::new());
9401 let options = ActivityOptions::new()
9402 .task_queue("payments")
9403 .retry_policy(
9404 ActivityRetryPolicy::new(4)
9405 .exponential_backoff(Duration::from_secs(1), 3, Some(Duration::from_secs(10)))
9406 .non_retryable_error_type("ValidationError"),
9407 )
9408 .start_to_close_timeout(Duration::from_secs(120))
9409 .schedule_to_start_timeout(Duration::from_secs(10))
9410 .schedule_to_close_timeout(Duration::from_secs(300))
9411 .heartbeat_timeout(Duration::from_secs(15));
9412 let mut call = Box::pin(ctx.activity_with_options(
9413 "charge-card",
9414 options,
9415 json!([{"order_id": "o-1"}]),
9416 ));
9417 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9418
9419 assert!(matches!(
9420 call.as_mut().poll(&mut task_context),
9421 Poll::Pending
9422 ));
9423 assert!(matches!(
9424 call.as_mut().poll(&mut task_context),
9425 Poll::Pending
9426 ));
9427
9428 let commands = ctx.take_commands().expect("activity command");
9429 assert_eq!(commands.len(), 1, "one future emits one logical schedule");
9430 assert_eq!(commands[0]["queue"], "payments");
9431 assert_eq!(
9432 commands[0]["retry_policy"],
9433 json!({
9434 "max_attempts": 4,
9435 "backoff_seconds": [1, 3, 9],
9436 "non_retryable_error_types": ["ValidationError"],
9437 })
9438 );
9439 assert_eq!(commands[0]["start_to_close_timeout"], 120);
9440 assert_eq!(commands[0]["schedule_to_start_timeout"], 10);
9441 assert_eq!(commands[0]["schedule_to_close_timeout"], 300);
9442 assert_eq!(commands[0]["heartbeat_timeout"], 15);
9443 }
9444
9445 #[test]
9446 fn activity_options_encode_explicit_and_rounded_backoff_intervals() {
9447 let ctx = workflow_context(Vec::new());
9448 let options = ActivityOptions::new().retry_policy(
9449 ActivityRetryPolicy::new(3)
9450 .backoff_intervals([Duration::from_millis(1), Duration::from_millis(1_001)]),
9451 );
9452 let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
9453 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9454
9455 assert!(matches!(
9456 call.as_mut().poll(&mut task_context),
9457 Poll::Pending
9458 ));
9459 assert_eq!(
9460 ctx.take_commands().expect("command")[0]["retry_policy"]["backoff_seconds"],
9461 json!([1, 2])
9462 );
9463 }
9464
9465 #[test]
9466 fn invalid_activity_options_return_typed_errors_before_emitting_commands() {
9467 let cases = [
9468 (
9469 ActivityOptions::new().task_queue(" "),
9470 ActivityOptionsErrorKind::EmptyTaskQueue,
9471 ),
9472 (
9473 ActivityOptions::new().retry_policy(ActivityRetryPolicy::default()),
9474 ActivityOptionsErrorKind::EmptyRetryPolicy,
9475 ),
9476 (
9477 ActivityOptions::new().retry_policy(ActivityRetryPolicy::new(0)),
9478 ActivityOptionsErrorKind::InvalidMaxAttempts,
9479 ),
9480 (
9481 ActivityOptions::new().retry_policy(ActivityRetryPolicy {
9482 max_attempts: None,
9483 backoff: Some(ActivityBackoff::Explicit(vec![Duration::from_secs(1)])),
9484 non_retryable_error_types: Vec::new(),
9485 }),
9486 ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
9487 ),
9488 (
9489 ActivityOptions::new().retry_policy(
9490 ActivityRetryPolicy::new(2)
9491 .backoff_intervals([Duration::from_secs(1), Duration::from_secs(2)]),
9492 ),
9493 ActivityOptionsErrorKind::TooManyBackoffIntervals,
9494 ),
9495 (
9496 ActivityOptions::new().retry_policy(
9497 ActivityRetryPolicy::new(2).exponential_backoff(
9498 Duration::from_secs(1),
9499 0,
9500 None,
9501 ),
9502 ),
9503 ActivityOptionsErrorKind::InvalidBackoffCoefficient,
9504 ),
9505 (
9506 ActivityOptions::new()
9507 .retry_policy(ActivityRetryPolicy::new(2).non_retryable_error_type(" ")),
9508 ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
9509 ),
9510 (
9511 ActivityOptions::new().retry_policy(
9512 ActivityRetryPolicy::new(10_002).exponential_backoff(
9513 Duration::from_secs(1),
9514 1,
9515 None,
9516 ),
9517 ),
9518 ActivityOptionsErrorKind::BackoffGenerationTooLarge,
9519 ),
9520 (
9521 ActivityOptions::new().retry_policy(
9522 ActivityRetryPolicy::new(2)
9523 .backoff_intervals([Duration::from_secs(i64::MAX as u64 + 1)]),
9524 ),
9525 ActivityOptionsErrorKind::BackoffOverflow,
9526 ),
9527 ];
9528
9529 for (options, expected_kind) in cases {
9530 let ctx = workflow_context(Vec::new());
9531 let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
9532 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9533 let Poll::Ready(Err(Error::InvalidActivityOptions(error))) =
9534 call.as_mut().poll(&mut task_context)
9535 else {
9536 panic!("expected typed activity validation error");
9537 };
9538 assert_eq!(error.kind, expected_kind);
9539 assert!(ctx.take_commands().expect("commands").is_empty());
9540 }
9541 }
9542
9543 #[test]
9544 fn activity_options_validate_positive_and_ordered_timeouts() {
9545 let zero_timeout_cases = [
9546 ActivityOptions::new().start_to_close_timeout(Duration::ZERO),
9547 ActivityOptions::new().schedule_to_start_timeout(Duration::ZERO),
9548 ActivityOptions::new().schedule_to_close_timeout(Duration::ZERO),
9549 ActivityOptions::new().heartbeat_timeout(Duration::ZERO),
9550 ];
9551 for options in zero_timeout_cases {
9552 assert_eq!(
9553 options.validate().expect_err("zero timeout").kind,
9554 ActivityOptionsErrorKind::TimeoutNotPositive
9555 );
9556 }
9557
9558 let ordering_cases = [
9559 ActivityOptions::new()
9560 .heartbeat_timeout(Duration::from_secs(11))
9561 .start_to_close_timeout(Duration::from_secs(10)),
9562 ActivityOptions::new()
9563 .start_to_close_timeout(Duration::from_secs(31))
9564 .schedule_to_close_timeout(Duration::from_secs(30)),
9565 ActivityOptions::new()
9566 .schedule_to_start_timeout(Duration::from_secs(31))
9567 .schedule_to_close_timeout(Duration::from_secs(30)),
9568 ];
9569 for options in ordering_cases {
9570 assert_eq!(
9571 options.validate().expect_err("timeout order").kind,
9572 ActivityOptionsErrorKind::TimeoutOrder
9573 );
9574 }
9575
9576 assert_eq!(
9577 ActivityOptions::new()
9578 .start_to_close_timeout(Duration::from_secs(i64::MAX as u64 + 1))
9579 .validate()
9580 .expect_err("protocol integer overflow")
9581 .kind,
9582 ActivityOptionsErrorKind::TimeoutOverflow
9583 );
9584 }
9585
9586 #[test]
9587 fn replayed_activity_retry_history_completes_without_duplicate_schedule() {
9588 let ctx = workflow_context(completed_retry_activity_history());
9589 let mut call =
9590 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
9591 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9592
9593 assert!(matches!(
9594 call.as_mut().poll(&mut task_context),
9595 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
9596 ));
9597 assert!(ctx.take_commands().expect("commands").is_empty());
9598 ctx.ensure_history_consumed().expect("history consumed");
9599 }
9600
9601 #[test]
9602 fn duplicate_non_retryable_types_use_one_command_and_replay_representation() {
9603 let mut options = retry_activity_options();
9604 options
9605 .retry_policy
9606 .as_mut()
9607 .expect("retry policy")
9608 .non_retryable_error_types
9609 .extend([" PermanentError ".to_string(), "PermanentError".to_string()]);
9610
9611 let new_ctx = workflow_context(Vec::new());
9612 let mut new_call =
9613 Box::pin(new_ctx.activity_with_options("flaky", options.clone(), json!([])));
9614 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9615 assert!(matches!(
9616 new_call.as_mut().poll(&mut task_context),
9617 Poll::Pending
9618 ));
9619 let commands = new_ctx.take_commands().expect("commands");
9620 assert_eq!(commands.len(), 1);
9621 assert_eq!(
9622 commands[0]["retry_policy"]["non_retryable_error_types"],
9623 json!(["PermanentError"])
9624 );
9625
9626 let replay_ctx = workflow_context(completed_retry_activity_history());
9627 let mut replay_call =
9628 Box::pin(replay_ctx.activity_with_options("flaky", options, json!([])));
9629 assert!(matches!(
9630 replay_call.as_mut().poll(&mut task_context),
9631 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
9632 ));
9633 assert!(replay_ctx.take_commands().expect("commands").is_empty());
9634 replay_ctx
9635 .ensure_history_consumed()
9636 .expect("history consumed");
9637 }
9638
9639 #[test]
9640 fn replayed_intermediate_retry_remains_pending_across_restarts() {
9641 let history = completed_retry_activity_history()
9642 .into_iter()
9643 .take(3)
9644 .collect::<Vec<_>>();
9645
9646 for _restart in 0..2 {
9647 let ctx = workflow_context(history.clone());
9648 let mut call =
9649 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
9650 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9651 assert!(matches!(
9652 call.as_mut().poll(&mut task_context),
9653 Poll::Pending
9654 ));
9655 assert!(ctx.take_commands().expect("commands").is_empty());
9656 }
9657 }
9658
9659 #[test]
9660 fn replayed_activity_rejects_changed_queue_retry_and_every_timeout_field() {
9661 let mut changed_queue = retry_activity_options();
9662 changed_queue.task_queue = Some("different-queue".to_string());
9663
9664 let mut changed_max_attempts = retry_activity_options();
9665 let retry_policy = changed_max_attempts
9666 .retry_policy
9667 .as_mut()
9668 .expect("retry policy");
9669 retry_policy.max_attempts = Some(4);
9670
9671 let mut changed_backoff = retry_activity_options();
9672 let retry_policy = changed_backoff.retry_policy.as_mut().expect("retry policy");
9673 retry_policy.backoff = Some(ActivityBackoff::Explicit(vec![
9674 Duration::from_secs(3),
9675 Duration::from_secs(4),
9676 ]));
9677
9678 let mut changed_non_retryable_types = retry_activity_options();
9679 let retry_policy = changed_non_retryable_types
9680 .retry_policy
9681 .as_mut()
9682 .expect("retry policy");
9683 retry_policy.non_retryable_error_types = vec!["AnotherPermanentError".to_string()];
9684
9685 let mut changed_start_to_close = retry_activity_options();
9686 changed_start_to_close.start_to_close_timeout = Some(Duration::from_secs(31));
9687 let mut changed_schedule_to_start = retry_activity_options();
9688 changed_schedule_to_start.schedule_to_start_timeout = Some(Duration::from_secs(6));
9689 let mut changed_schedule_to_close = retry_activity_options();
9690 changed_schedule_to_close.schedule_to_close_timeout = Some(Duration::from_secs(91));
9691 let mut changed_heartbeat = retry_activity_options();
9692 changed_heartbeat.heartbeat_timeout = Some(Duration::from_secs(11));
9693
9694 let cases = [
9695 (changed_queue, "activity_task_queue_mismatch"),
9696 (changed_max_attempts, "activity_retry_policy_mismatch"),
9697 (changed_backoff, "activity_retry_policy_mismatch"),
9698 (
9699 changed_non_retryable_types,
9700 "activity_retry_policy_mismatch",
9701 ),
9702 (changed_start_to_close, "activity_retry_policy_mismatch"),
9703 (changed_schedule_to_start, "activity_retry_policy_mismatch"),
9704 (changed_schedule_to_close, "activity_retry_policy_mismatch"),
9705 (changed_heartbeat, "activity_retry_policy_mismatch"),
9706 ];
9707
9708 for (options, expected_reason) in cases {
9709 let ctx = workflow_context(completed_retry_activity_history());
9710 let mut call = Box::pin(ctx.activity_with_options("flaky", options, json!([])));
9711 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9712 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
9713 call.as_mut().poll(&mut task_context)
9714 else {
9715 panic!("changed activity options must fail replay");
9716 };
9717 assert_eq!(failure.reason, expected_reason);
9718 assert_eq!(failure.sequence, Some(1));
9719 assert!(ctx.take_commands().expect("commands").is_empty());
9720 }
9721 }
9722
9723 #[test]
9724 fn replayed_activity_rejects_changed_execution_mode_and_snapshot_version() {
9725 let cases = [
9726 (
9727 "execution_mode",
9728 json!("local"),
9729 "activity_execution_mode_mismatch",
9730 ),
9731 (
9732 "snapshot_version",
9733 json!(2),
9734 "activity_retry_policy_mismatch",
9735 ),
9736 ];
9737
9738 for (field, value, expected_reason) in cases {
9739 let mut history = completed_retry_activity_history();
9740 let activity = history[0].payload["activity"]
9741 .as_object_mut()
9742 .expect("activity snapshot");
9743 if field == "execution_mode" {
9744 activity.insert(field.to_string(), value);
9745 } else {
9746 activity["retry_policy"]
9747 .as_object_mut()
9748 .expect("retry snapshot")
9749 .insert(field.to_string(), value);
9750 }
9751
9752 let ctx = workflow_context(history);
9753 let mut call =
9754 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
9755 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9756 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
9757 call.as_mut().poll(&mut task_context)
9758 else {
9759 panic!("changed {field} must fail replay");
9760 };
9761 assert_eq!(failure.reason, expected_reason);
9762 assert_eq!(failure.sequence, Some(1));
9763 assert!(ctx.take_commands().expect("commands").is_empty());
9764 }
9765 }
9766
9767 #[test]
9768 fn replayed_legacy_activity_treats_missing_option_snapshot_as_unknown() {
9769 let mut history = completed_retry_activity_history();
9770 let activity = history[0].payload["activity"]
9771 .as_object_mut()
9772 .expect("activity snapshot");
9773 activity.remove("execution_mode");
9774 activity.remove("retry_policy");
9775
9776 let mut current = retry_activity_options();
9777 current.start_to_close_timeout = Some(Duration::from_secs(45));
9778 current.schedule_to_start_timeout = Some(Duration::from_secs(8));
9779 current.schedule_to_close_timeout = Some(Duration::from_secs(120));
9780 current.heartbeat_timeout = Some(Duration::from_secs(12));
9781
9782 let ctx = workflow_context(history);
9783 let mut call = Box::pin(ctx.activity_with_options("flaky", current, json!([])));
9784 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9785 assert!(matches!(
9786 call.as_mut().poll(&mut task_context),
9787 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
9788 ));
9789 assert!(ctx.take_commands().expect("commands").is_empty());
9790 ctx.ensure_history_consumed().expect("history consumed");
9791 }
9792
9793 #[test]
9794 fn terminal_activity_failed_after_start_returns_typed_failure() {
9795 let history = vec![
9796 history_event(
9797 "ActivityScheduled",
9798 json!({
9799 "sequence": 1,
9800 "activity_type": "flaky",
9801 "activity_execution_id": "act-terminal",
9802 "activity": {
9803 "id": "act-terminal",
9804 "sequence": 1,
9805 "type": "flaky",
9806 "queue": "critical-activities",
9807 "retry_policy": {
9808 "snapshot_version": 1,
9809 "max_attempts": 3,
9810 "backoff_seconds": [2, 4],
9811 "non_retryable_error_types": ["PermanentError"]
9812 }
9813 }
9814 }),
9815 ),
9816 history_event(
9817 "ActivityStarted",
9818 json!({
9819 "sequence": 1,
9820 "activity_type": "flaky",
9821 "activity_execution_id": "act-terminal",
9822 "activity_attempt_id": "attempt-1",
9823 "attempt_number": 1
9824 }),
9825 ),
9826 history_event(
9827 "ActivityFailed",
9828 json!({
9829 "sequence": 1,
9830 "activity_type": "flaky",
9831 "activity_execution_id": "act-terminal",
9832 "activity_attempt_id": "attempt-1",
9833 "attempt_number": 1,
9834 "failure_id": "failure-terminal",
9835 "failure_category": "activity",
9836 "exception_type": "PermanentError",
9837 "message": "cannot retry",
9838 "non_retryable": true
9839 }),
9840 ),
9841 ];
9842 let ctx = workflow_context(history);
9843 let mut call =
9844 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
9845 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9846
9847 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
9848 call.as_mut().poll(&mut task_context)
9849 else {
9850 panic!("terminal ActivityFailed must settle the activity future");
9851 };
9852 assert_eq!(failure.kind, ActivityFailureKind::Failed);
9853 assert_eq!(
9854 failure.activity_execution_id.as_deref(),
9855 Some("act-terminal")
9856 );
9857 assert_eq!(failure.exception_type.as_deref(), Some("PermanentError"));
9858 assert!(failure.non_retryable);
9859 assert!(ctx.take_commands().expect("commands").is_empty());
9860 ctx.ensure_history_consumed().expect("history consumed");
9861 }
9862
9863 #[test]
9864 fn activity_terminal_events_return_machine_readable_failures() {
9865 let cases = [
9866 (
9867 "ActivityFailed",
9868 json!({
9869 "sequence": 1,
9870 "activity_type": "charge-card",
9871 "activity_execution_id": "act-1",
9872 "activity_attempt_id": "attempt-2",
9873 "attempt_number": 2,
9874 "failure_id": "failure-1",
9875 "failure_category": "activity",
9876 "exception_type": "PaymentDeclined",
9877 "exception_class": "payments.PaymentDeclined",
9878 "message": "card declined",
9879 "non_retryable": true
9880 }),
9881 ActivityFailureKind::Failed,
9882 "activity",
9883 ),
9884 (
9885 "ActivityCancelled",
9886 json!({
9887 "sequence": 1,
9888 "activity_type": "charge-card",
9889 "activity_execution_id": "act-1",
9890 "activity_attempt_id": "attempt-1"
9891 }),
9892 ActivityFailureKind::Cancelled,
9893 "cancelled",
9894 ),
9895 ];
9896
9897 for (event_type, payload, expected_kind, expected_reason) in cases {
9898 let ctx = workflow_context(vec![history_event(event_type, payload)]);
9899 let mut call = Box::pin(ctx.activity("charge-card", json!([])));
9900 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9901 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
9902 call.as_mut().poll(&mut task_context)
9903 else {
9904 panic!("expected terminal activity failure");
9905 };
9906 assert_eq!(failure.kind, expected_kind);
9907 assert_eq!(failure.reason, expected_reason);
9908 assert_eq!(failure.activity_execution_id.as_deref(), Some("act-1"));
9909 assert_eq!(failure.activity_type.as_deref(), Some("charge-card"));
9910 }
9911 }
9912
9913 #[test]
9914 fn every_activity_timeout_class_is_typed() {
9915 for timeout_kind in [
9916 "start_to_close",
9917 "schedule_to_start",
9918 "schedule_to_close",
9919 "heartbeat",
9920 ] {
9921 let ctx = workflow_context(vec![history_event(
9922 "ActivityTimedOut",
9923 json!({
9924 "sequence": 1,
9925 "activity_type": "slow",
9926 "activity_execution_id": "act-timeout",
9927 "activity_attempt_id": "attempt-timeout",
9928 "failure_category": "timeout",
9929 "timeout_kind": timeout_kind,
9930 "message": "deadline expired"
9931 }),
9932 )]);
9933 let mut call = Box::pin(ctx.activity("slow", json!([])));
9934 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9935 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
9936 call.as_mut().poll(&mut task_context)
9937 else {
9938 panic!("expected timeout failure");
9939 };
9940 assert_eq!(failure.kind, ActivityFailureKind::TimedOut);
9941 assert_eq!(failure.reason, timeout_kind);
9942 assert_eq!(failure.timeout_kind.as_deref(), Some(timeout_kind));
9943 assert_eq!(failure.failure_category.as_deref(), Some("timeout"));
9944 }
9945 }
9946
9947 #[test]
9948 fn workflow_sleep_emits_one_durable_timer_and_rounds_up() {
9949 let ctx = workflow_context(Vec::new());
9950 let mut sleep = Box::pin(ctx.sleep(Duration::from_millis(1_001)));
9951 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9952
9953 assert!(matches!(
9954 sleep.as_mut().poll(&mut task_context),
9955 Poll::Pending
9956 ));
9957 assert!(matches!(
9958 sleep.as_mut().poll(&mut task_context),
9959 Poll::Pending
9960 ));
9961
9962 let commands = ctx.take_commands().expect("timer command");
9963 assert_eq!(
9964 commands,
9965 vec![json!({
9966 "type": "start_timer",
9967 "delay_seconds": 2,
9968 })]
9969 );
9970 }
9971
9972 #[test]
9973 fn workflow_sleep_replays_matching_schedule_and_fire_without_a_command() {
9974 let history = vec![
9975 history_event(
9976 "TimerScheduled",
9977 json!({
9978 "sequence": 1,
9979 "timer_id": "timer-1",
9980 "delay_seconds": 5,
9981 "fire_at": "2026-07-11T12:00:05Z",
9982 }),
9983 ),
9984 history_event(
9985 "TimerFired",
9986 json!({
9987 "sequence": 1,
9988 "timer_id": "timer-1",
9989 "delay_seconds": 5,
9990 "fire_at": "2026-07-11T12:00:05Z",
9991 "fired_at": "2026-07-11T12:00:05Z",
9992 }),
9993 ),
9994 ];
9995
9996 for _restart in 0..2 {
9997 let ctx = workflow_context(history.clone());
9998 let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(5)));
9999 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10000 assert!(matches!(
10001 sleep.as_mut().poll(&mut task_context),
10002 Poll::Ready(Ok(()))
10003 ));
10004 assert!(ctx.take_commands().expect("commands").is_empty());
10005 ctx.ensure_history_consumed().expect("history consumed");
10006 }
10007 }
10008
10009 #[test]
10010 fn workflow_sleep_rejects_changed_delay_during_replay() {
10011 let ctx = workflow_context(vec![
10012 history_event(
10013 "TimerScheduled",
10014 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10015 ),
10016 history_event(
10017 "TimerFired",
10018 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10019 ),
10020 ]);
10021 let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(500)));
10022 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10023
10024 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10025 sleep.as_mut().poll(&mut task_context)
10026 else {
10027 panic!("changed timer delay must be rejected");
10028 };
10029 assert_eq!(failure.reason, "timer_delay_mismatch");
10030 assert_eq!(failure.sequence, Some(1));
10031 }
10032
10033 #[test]
10034 fn workflow_history_rejects_unpaired_or_mismatched_timer_events() {
10035 let lone_fire = WorkflowState::new(
10036 vec![history_event(
10037 "TimerFired",
10038 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10039 )],
10040 "rust-workers".to_string(),
10041 JSON_CODEC.to_string(),
10042 None,
10043 )
10044 .expect_err("TimerFired requires TimerScheduled");
10045 assert!(matches!(
10046 lone_fire,
10047 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10048 if reason == "timer_schedule_missing_or_duplicate"
10049 ));
10050
10051 let wrong_identity = WorkflowState::new(
10052 vec![
10053 history_event(
10054 "TimerScheduled",
10055 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10056 ),
10057 history_event(
10058 "TimerFired",
10059 json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
10060 ),
10061 ],
10062 "rust-workers".to_string(),
10063 JSON_CODEC.to_string(),
10064 None,
10065 )
10066 .expect_err("fire must match scheduled timer identity");
10067 assert!(matches!(
10068 wrong_identity,
10069 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10070 if reason == "timer_identity_mismatch"
10071 ));
10072
10073 let duplicate_fire = WorkflowState::new(
10074 vec![
10075 history_event(
10076 "TimerScheduled",
10077 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10078 ),
10079 history_event(
10080 "TimerFired",
10081 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10082 ),
10083 history_event(
10084 "TimerFired",
10085 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10086 ),
10087 ],
10088 "rust-workers".to_string(),
10089 JSON_CODEC.to_string(),
10090 None,
10091 )
10092 .expect_err("a durable timer cannot fire twice");
10093 assert!(matches!(
10094 duplicate_fire,
10095 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10096 if reason == "duplicate_timer_fire"
10097 ));
10098
10099 let wrong_fired_delay = WorkflowState::new(
10100 vec![
10101 history_event(
10102 "TimerScheduled",
10103 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10104 ),
10105 history_event(
10106 "TimerFired",
10107 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 6}),
10108 ),
10109 ],
10110 "rust-workers".to_string(),
10111 JSON_CODEC.to_string(),
10112 None,
10113 )
10114 .expect_err("timer schedule and fire delays must agree");
10115 assert!(matches!(
10116 wrong_fired_delay,
10117 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10118 if reason == "timer_history_delay_mismatch"
10119 ));
10120 }
10121
10122 #[test]
10123 fn replay_rejects_activity_moved_before_recorded_timer() {
10124 let ctx = workflow_context(vec![
10125 history_event(
10126 "TimerScheduled",
10127 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10128 ),
10129 history_event(
10130 "TimerFired",
10131 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10132 ),
10133 history_event(
10134 "ActivityCompleted",
10135 json!({
10136 "sequence": 2,
10137 "activity_type": "after-timer",
10138 "payload_codec": "json",
10139 "result": {"codec": "json", "blob": "\"done\""},
10140 }),
10141 ),
10142 ]);
10143 let mut activity = Box::pin(ctx.activity("after-timer", json!([])));
10144 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10145
10146 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10147 activity.as_mut().poll(&mut task_context)
10148 else {
10149 panic!("reordered durable command must be rejected");
10150 };
10151 assert_eq!(failure.reason, "recorded_command_mismatch");
10152 assert_eq!(failure.sequence, Some(1));
10153 assert_eq!(failure.expected.as_deref(), Some("timer"));
10154 assert_eq!(failure.actual.as_deref(), Some("activity:after-timer"));
10155 }
10156
10157 #[test]
10158 fn workflow_context_emits_a_typed_named_signal_wait() {
10159 let ctx = workflow_context(Vec::new());
10160 let mut signal = Box::pin(ctx.wait_signal("finish"));
10161 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10162
10163 assert!(matches!(
10164 signal.as_mut().poll(&mut task_context),
10165 Poll::Pending
10166 ));
10167 assert_eq!(
10168 ctx.take_commands().expect("signal-wait command"),
10169 vec![json!({
10170 "type": "open_signal_wait",
10171 "signal_name": "finish",
10172 })]
10173 );
10174 }
10175
10176 #[test]
10177 fn condition_wait_history_does_not_resolve_a_typed_signal_wait() {
10178 let ctx = workflow_context(vec![
10179 history_event(
10180 "ConditionWaitOpened",
10181 json!({"sequence": 1, "condition_key": "signal:finish"}),
10182 ),
10183 history_event(
10184 "ConditionWaitSatisfied",
10185 json!({"sequence": 1, "condition_key": "signal:finish"}),
10186 ),
10187 history_event(
10188 "SignalReceived",
10189 json!({"signal_name": "finish", "arguments": []}),
10190 ),
10191 ]);
10192 let mut signal = Box::pin(ctx.wait_signal("finish"));
10193 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10194
10195 assert!(matches!(
10196 signal.as_mut().poll(&mut task_context),
10197 Poll::Pending
10198 ));
10199 assert_eq!(
10200 ctx.take_commands().expect("typed signal-wait command"),
10201 vec![json!({
10202 "type": "open_signal_wait",
10203 "signal_name": "finish",
10204 })]
10205 );
10206 }
10207
10208 #[test]
10209 fn replay_orders_signal_waits_and_timers_in_one_command_stream() {
10210 let signal_then_timer = vec![
10211 history_event(
10212 "SignalWaitOpened",
10213 json!({"sequence": 1, "signal_name": "go"}),
10214 ),
10215 history_event(
10216 "SignalApplied",
10217 json!({
10218 "sequence": 1,
10219 "signal_name": "go",
10220 "value": {"codec": "json", "blob": "[\"now\"]"},
10221 }),
10222 ),
10223 history_event(
10224 "TimerScheduled",
10225 json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
10226 ),
10227 history_event(
10228 "TimerFired",
10229 json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
10230 ),
10231 ];
10232
10233 let ctx = workflow_context(signal_then_timer.clone());
10234 let mut signal = Box::pin(ctx.wait_signal("go"));
10235 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10236 assert!(matches!(
10237 signal.as_mut().poll(&mut task_context),
10238 Poll::Ready(Ok(arguments)) if arguments == vec![json!("now")]
10239 ));
10240 let mut timer = Box::pin(ctx.sleep(Duration::from_secs(5)));
10241 assert!(matches!(
10242 timer.as_mut().poll(&mut task_context),
10243 Poll::Ready(Ok(()))
10244 ));
10245 ctx.ensure_history_consumed()
10246 .expect("signal and timer history consumed in order");
10247
10248 let reordered = workflow_context(signal_then_timer);
10249 let mut timer_first = Box::pin(reordered.sleep(Duration::from_secs(5)));
10250 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10251 timer_first.as_mut().poll(&mut task_context)
10252 else {
10253 panic!("timer cannot consume signal-wait-first history");
10254 };
10255 assert_eq!(failure.reason, "recorded_command_mismatch");
10256 assert_eq!(failure.sequence, Some(1));
10257 assert_eq!(failure.expected.as_deref(), Some("signal wait"));
10258
10259 let timer_then_signal = vec![
10260 history_event(
10261 "TimerScheduled",
10262 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10263 ),
10264 history_event(
10265 "TimerFired",
10266 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10267 ),
10268 history_event(
10269 "SignalWaitOpened",
10270 json!({"sequence": 2, "signal_name": "go"}),
10271 ),
10272 history_event(
10273 "SignalApplied",
10274 json!({
10275 "sequence": 2,
10276 "signal_name": "go",
10277 "value": {"codec": "json", "blob": "[]"},
10278 }),
10279 ),
10280 ];
10281 let reordered = workflow_context(timer_then_signal);
10282 let mut signal_first = Box::pin(reordered.wait_signal("go"));
10283 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10284 signal_first.as_mut().poll(&mut task_context)
10285 else {
10286 panic!("signal wait cannot consume timer-first history");
10287 };
10288 assert_eq!(failure.reason, "recorded_command_mismatch");
10289 assert_eq!(failure.sequence, Some(1));
10290 assert_eq!(failure.expected.as_deref(), Some("timer"));
10291 }
10292
10293 #[test]
10294 fn workflow_history_rejects_duplicate_or_colliding_command_sequences() {
10295 let duplicate_timer = WorkflowState::new(
10296 vec![
10297 history_event(
10298 "TimerScheduled",
10299 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10300 ),
10301 history_event(
10302 "TimerScheduled",
10303 json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
10304 ),
10305 ],
10306 "rust-workers".to_string(),
10307 JSON_CODEC.to_string(),
10308 None,
10309 )
10310 .expect_err("one workflow sequence cannot schedule two timers");
10311 assert!(matches!(
10312 duplicate_timer,
10313 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10314 if reason == "timer_schedule_missing_or_duplicate"
10315 ));
10316
10317 let colliding_kinds = WorkflowState::new(
10318 vec![
10319 history_event(
10320 "TimerScheduled",
10321 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10322 ),
10323 history_event(
10324 "ActivityCompleted",
10325 json!({"sequence": 1, "activity_type": "same-sequence"}),
10326 ),
10327 ],
10328 "rust-workers".to_string(),
10329 JSON_CODEC.to_string(),
10330 None,
10331 )
10332 .expect_err("one workflow sequence cannot identify two command kinds");
10333 assert!(matches!(
10334 colliding_kinds,
10335 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10336 if reason == "durable_command_sequence_collision"
10337 ));
10338
10339 let duplicate_signal_wait = WorkflowState::new(
10340 vec![
10341 history_event(
10342 "SignalWaitOpened",
10343 json!({"sequence": 1, "signal_name": "go"}),
10344 ),
10345 history_event(
10346 "SignalWaitOpened",
10347 json!({"sequence": 1, "signal_name": "go"}),
10348 ),
10349 ],
10350 "rust-workers".to_string(),
10351 JSON_CODEC.to_string(),
10352 None,
10353 )
10354 .expect_err("one workflow sequence cannot open two signal waits");
10355 assert!(matches!(
10356 duplicate_signal_wait,
10357 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10358 if reason == "signal_wait_open_missing_or_duplicate"
10359 ));
10360 }
10361
10362 #[test]
10363 fn workflow_history_accepts_a_first_command_after_global_sequence_gaps() {
10364 let result = encode_value_envelope(&json!({"captured": true}), JSON_CODEC)
10365 .expect("side-effect result");
10366 let ctx = workflow_context(vec![history_event(
10367 "SideEffectRecorded",
10368 json!({"sequence": 99, "result": result}),
10369 )]);
10370
10371 let replayed: Value = ctx
10372 .side_effect(|| panic!("recorded side effect must not run"))
10373 .expect("positive global workflow sequence is valid");
10374 assert_eq!(replayed, json!({"captured": true}));
10375 ctx.ensure_history_consumed().expect("history consumed");
10376 }
10377
10378 #[test]
10379 fn workflow_history_rejects_zero_and_descending_command_sequences() {
10380 let result =
10381 encode_value_envelope(&json!("captured"), JSON_CODEC).expect("side-effect result");
10382 let zero = WorkflowState::new(
10383 vec![history_event(
10384 "SideEffectRecorded",
10385 json!({"sequence": 0, "result": result.clone()}),
10386 )],
10387 "rust-workers".to_string(),
10388 JSON_CODEC.to_string(),
10389 None,
10390 )
10391 .expect_err("durable command sequences must be positive");
10392 assert!(matches!(
10393 zero,
10394 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10395 if reason == "durable_command_sequence_invalid"
10396 ));
10397
10398 let descending = WorkflowState::new(
10399 vec![
10400 history_event(
10401 "SideEffectRecorded",
10402 json!({"sequence": 3, "result": result}),
10403 ),
10404 history_event(
10405 "VersionMarkerRecorded",
10406 json!({
10407 "sequence": 2,
10408 "change_id": "descending-marker",
10409 "version": 1,
10410 "min_supported": 1,
10411 "max_supported": 1,
10412 }),
10413 ),
10414 ],
10415 "rust-workers".to_string(),
10416 JSON_CODEC.to_string(),
10417 None,
10418 )
10419 .expect_err("new durable commands must remain strictly ordered");
10420 let Error::NonDeterministicReplay(failure) = descending else {
10421 panic!("expected typed replay failure");
10422 };
10423 assert_eq!(failure.reason, "durable_command_sequence_mismatch");
10424 assert_eq!(failure.sequence, Some(2));
10425 assert_eq!(
10426 failure.expected.as_deref(),
10427 Some("workflow sequence greater than 3")
10428 );
10429 assert_eq!(failure.actual.as_deref(), Some("2"));
10430 }
10431
10432 #[test]
10433 fn workflow_task_replay_completes_after_signals_create_sequence_gaps() {
10434 fn worker() -> Worker {
10435 let client = Client::new("http://127.0.0.1:8080").expect("client");
10436 let mut worker = Worker::new(client, "rust-workers");
10437 worker.register_workflow("rust.finish-after-gaps", |ctx, _input| async move {
10438 ctx.wait_signal("finish").await?;
10439 let marker: String =
10440 ctx.side_effect(|| panic!("recorded side effect must not run"))?;
10441 assert_eq!(marker, "after-finish");
10442 Ok(json!("finished"))
10443 });
10444 worker
10445 }
10446
10447 let marker =
10448 encode_value_envelope(&json!("after-finish"), JSON_CODEC).expect("side-effect result");
10449 let task = workflow_task(
10450 "rust.finish-after-gaps",
10451 vec![
10452 history_event(
10453 "SignalWaitOpened",
10454 json!({"sequence": 1, "signal_name": "finish"}),
10455 ),
10456 history_event(
10457 "SignalReceived",
10458 json!({
10459 "signal_id": "increment-3",
10460 "signal_name": "increment",
10461 "workflow_sequence": 2,
10462 "payload_codec": "json",
10463 "arguments": {"codec": "json", "blob": "[3]"},
10464 }),
10465 ),
10466 history_event(
10467 "SignalReceived",
10468 json!({
10469 "signal_id": "increment-5",
10470 "signal_name": "increment",
10471 "workflow_sequence": 3,
10472 "payload_codec": "json",
10473 "arguments": {"codec": "json", "blob": "[5]"},
10474 }),
10475 ),
10476 history_event(
10477 "SignalReceived",
10478 json!({
10479 "signal_id": "finish",
10480 "signal_name": "finish",
10481 "workflow_sequence": 4,
10482 "payload_codec": "json",
10483 "arguments": {"codec": "json", "blob": "[]"},
10484 }),
10485 ),
10486 history_event(
10487 "SignalApplied",
10488 json!({
10489 "sequence": 1,
10490 "signal_id": "finish",
10491 "signal_name": "finish",
10492 "payload_codec": "json",
10493 "value": {"codec": "json", "blob": "[]"},
10494 }),
10495 ),
10496 history_event(
10497 "SideEffectRecorded",
10498 json!({"sequence": 5, "result": marker}),
10499 ),
10500 ],
10501 JSON_CODEC,
10502 );
10503
10504 for _original_or_cold_worker in 0..2 {
10505 let commands = worker()
10506 .execute_workflow_task(task.clone())
10507 .expect("signal gaps preserve deterministic replay");
10508 assert_eq!(commands.len(), 1, "replay emits only terminal completion");
10509 assert_eq!(commands[0]["type"], "complete_workflow");
10510 assert_eq!(
10511 decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("workflow output"),
10512 json!("finished")
10513 );
10514 }
10515 }
10516
10517 #[test]
10518 fn workflow_sleep_rejects_unrepresentable_rounded_duration() {
10519 let ctx = workflow_context(Vec::new());
10520 let mut sleep = Box::pin(ctx.start_timer(Duration::new(u64::MAX, 1)));
10521 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10522 assert!(matches!(
10523 sleep.as_mut().poll(&mut task_context),
10524 Poll::Ready(Err(Error::TimerDurationOverflow))
10525 ));
10526 assert!(ctx.take_commands().expect("commands").is_empty());
10527 }
10528
10529 #[test]
10530 fn workflow_task_replay_completes_without_rescheduling_recorded_commands() {
10531 let client = Client::new("http://127.0.0.1:8080").expect("client");
10532 let mut worker = Worker::new(client, "rust-workers");
10533 worker.register_workflow("rust.timer", |ctx, _input| async move {
10534 ctx.sleep(Duration::from_secs(5)).await?;
10535 ctx.activity("after-timer", json!([])).await
10536 });
10537
10538 let task = |history_events| WorkflowTask {
10539 task_id: "wft-rust-timer-1".to_string(),
10540 workflow_id: Some("wf-rust-timer".to_string()),
10541 run_id: Some("run-rust-timer".to_string()),
10542 workflow_type: "rust.timer".to_string(),
10543 payload_codec: JSON_CODEC.to_string(),
10544 arguments: Some(json!({"codec": "json", "blob": "[]"})),
10545 history_events,
10546 total_history_events: None,
10547 history_size_bytes: None,
10548 continue_as_new_recommended: None,
10549 history_budget_pressure: None,
10550 next_history_page_token: None,
10551 workflow_task_attempt: 1,
10552 workflow_signal_id: None,
10553 signal_name: None,
10554 signal_arguments: None,
10555 workflow_update_id: None,
10556 update_name: None,
10557 lease_owner: Some("rust-worker".to_string()),
10558 };
10559
10560 let initial = worker
10561 .execute_workflow_task(task(Vec::new()))
10562 .expect("initial timer task");
10563 assert_eq!(
10564 initial,
10565 vec![json!({"type": "start_timer", "delay_seconds": 5})]
10566 );
10567
10568 let replayed = worker
10569 .execute_workflow_task(task(vec![
10570 history_event(
10571 "TimerScheduled",
10572 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10573 ),
10574 history_event(
10575 "TimerFired",
10576 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10577 ),
10578 history_event(
10579 "ActivityCompleted",
10580 json!({
10581 "sequence": 2,
10582 "activity_type": "after-timer",
10583 "payload_codec": "json",
10584 "result": {"codec": "json", "blob": "\"done\""},
10585 }),
10586 ),
10587 ]))
10588 .expect("replayed workflow task");
10589 assert_eq!(replayed.len(), 1);
10590 assert_eq!(replayed[0]["type"], "complete_workflow");
10591 assert_eq!(
10592 decode_wire_value(&replayed[0]["result"], JSON_CODEC).expect("result"),
10593 json!("done")
10594 );
10595 }
10596
10597 #[test]
10598 fn workflow_continue_as_new_emits_arguments_type_and_queue_once() {
10599 let client = Client::new("http://127.0.0.1:8080").expect("client");
10600 let mut worker = Worker::new(client, "rust-workers");
10601 worker.register_workflow("rust.continue", |ctx, _input| async move {
10602 ctx.continue_as_new_with_options(
10603 ContinueAsNewOptions::new()
10604 .workflow_type("rust.next")
10605 .task_queue("next-workers"),
10606 json!([2, {"cursor": "next"}]),
10607 )
10608 });
10609
10610 let commands = worker
10611 .execute_workflow_task(workflow_task("rust.continue", Vec::new(), DEFAULT_CODEC))
10612 .expect("continue-as-new command");
10613
10614 assert_eq!(commands.len(), 1);
10615 assert_eq!(commands[0]["type"], "continue_as_new");
10616 assert_eq!(commands[0]["workflow_type"], "rust.next");
10617 assert_eq!(commands[0]["queue"], "next-workers");
10618 assert_eq!(
10619 decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC)
10620 .expect("continue-as-new arguments"),
10621 json!([2, {"cursor": "next"}])
10622 );
10623 }
10624
10625 #[test]
10626 fn continue_as_new_preserves_typed_arguments() {
10627 let client = Client::new("http://127.0.0.1:8080").expect("client");
10628 let mut worker = Worker::new(client, "rust-workers");
10629 worker.register_workflow_avro_value("rust.typed-continue", |ctx, _input| async move {
10630 ctx.continue_as_new(AvroValue::Array(vec![typed_fidelity_probe()]))?;
10631 unreachable!("continue-as-new returns a control-flow error")
10632 });
10633
10634 let commands = worker
10635 .execute_workflow_task(workflow_task(
10636 "rust.typed-continue",
10637 Vec::new(),
10638 DEFAULT_CODEC,
10639 ))
10640 .expect("typed continue-as-new command");
10641
10642 assert_eq!(commands[0]["type"], "continue_as_new");
10643 assert_eq!(
10644 decode_wire_avro_value(&commands[0]["arguments"], DEFAULT_CODEC)
10645 .expect("typed continue arguments"),
10646 AvroValue::Array(vec![typed_fidelity_probe()])
10647 );
10648 }
10649
10650 #[test]
10651 fn recorded_continue_as_new_is_consumed_without_duplicate_successor_command() {
10652 let client = Client::new("http://127.0.0.1:8080").expect("client");
10653 let mut worker = Worker::new(client, "rust-workers");
10654 worker.register_workflow("rust.continue", |ctx, _input| async move {
10655 ctx.continue_as_new(json!([2]))
10656 });
10657 let task = workflow_task(
10658 "rust.continue",
10659 vec![history_event(
10660 "WorkflowContinuedAsNew",
10661 json!({"sequence": 1, "continued_to_run_id": "run-next"}),
10662 )],
10663 JSON_CODEC,
10664 );
10665
10666 for _worker_restart_or_redelivery in 0..2 {
10667 let commands = worker
10668 .execute_workflow_task(task.clone())
10669 .expect("recorded transition replays");
10670 assert!(
10671 commands.is_empty(),
10672 "replay must not emit another successor"
10673 );
10674 }
10675 }
10676
10677 #[test]
10678 fn continue_as_new_rejects_invalid_overrides_before_emitting_a_command() {
10679 let ctx = workflow_context(Vec::new());
10680 let error = ctx
10681 .continue_as_new_with_options(ContinueAsNewOptions::new().task_queue(" "), json!([1]))
10682 .expect_err("blank queue must be rejected");
10683
10684 let Error::InvalidContinueAsNewOptions(error) = error else {
10685 panic!("expected typed continue-as-new validation error");
10686 };
10687 assert_eq!(error.field, "task_queue");
10688 assert!(ctx.take_commands().expect("commands").is_empty());
10689 }
10690
10691 #[test]
10692 fn workflow_context_exposes_server_history_budget() {
10693 let client = Client::new("http://127.0.0.1:8080").expect("client");
10694 let mut worker = Worker::new(client, "rust-workers");
10695 worker.register_workflow("rust.history-budget", |ctx, _input| async move {
10696 let budget = ctx.history_budget()?;
10697 Ok(json!({
10698 "events": budget.event_count,
10699 "bytes": budget.size_bytes,
10700 "recommended": budget.continue_as_new_recommended,
10701 "pressure": budget.pressure,
10702 }))
10703 });
10704 let task: WorkflowTask = serde_json::from_value(json!({
10705 "task_id": "task-history-budget",
10706 "workflow_type": "rust.history-budget",
10707 "payload_codec": JSON_CODEC,
10708 "history_events": [],
10709 "total_history_events": 480,
10710 "history_size_bytes": 1_048_576,
10711 "continue_as_new_recommended": true,
10712 "history_budget_pressure": "continue_as_new_recommended",
10713 }))
10714 .expect("published workflow task");
10715
10716 let commands = worker
10717 .execute_workflow_task(task)
10718 .expect("history-budget workflow");
10719 let result = decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("result");
10720 assert_eq!(result["events"], 480);
10721 assert_eq!(result["bytes"], 1_048_576);
10722 assert_eq!(result["recommended"], true);
10723 assert_eq!(result["pressure"], "continue_as_new_recommended");
10724 }
10725
10726 #[test]
10727 fn uncaught_workflow_handler_error_emits_terminal_failure_command() {
10728 let client = Client::new("http://127.0.0.1:8080").expect("client");
10729 let mut worker = Worker::new(client, "rust-workers");
10730 worker.register_workflow("rust.failing", |_ctx, _input| async move {
10731 Err(Error::Codec("rust_conformance_failure".to_string()))
10732 });
10733 let task = WorkflowTask {
10734 task_id: "wft-rust-failing-1".to_string(),
10735 workflow_id: Some("wf-rust-failing".to_string()),
10736 run_id: Some("run-rust-failing".to_string()),
10737 workflow_type: "rust.failing".to_string(),
10738 payload_codec: JSON_CODEC.to_string(),
10739 arguments: Some(encode_value_envelope(&json!([]), JSON_CODEC).expect("input")),
10740 history_events: Vec::new(),
10741 total_history_events: Some(0),
10742 history_size_bytes: None,
10743 continue_as_new_recommended: None,
10744 history_budget_pressure: None,
10745 next_history_page_token: None,
10746 workflow_task_attempt: 1,
10747 workflow_signal_id: None,
10748 signal_name: None,
10749 signal_arguments: None,
10750 workflow_update_id: None,
10751 update_name: None,
10752 lease_owner: Some("rust-worker".to_string()),
10753 };
10754
10755 let commands = worker
10756 .execute_workflow_task(task)
10757 .expect("handler failure becomes a workflow command");
10758
10759 assert_eq!(commands.len(), 1);
10760 assert_eq!(commands[0]["type"], "fail_workflow");
10761 assert_eq!(commands[0]["exception_type"], "RustWorkflowError");
10762 assert_eq!(commands[0]["exception_class"], "durable_workflow::Error");
10763 assert_eq!(commands[0]["non_retryable"], false);
10764 assert_eq!(
10765 commands[0]["message"],
10766 "codec error: rust_conformance_failure"
10767 );
10768 assert_eq!(
10769 commands[0]["exception"]["message"],
10770 "codec error: rust_conformance_failure"
10771 );
10772 }
10773
10774 #[test]
10775 fn ordinary_handler_error_preserves_commands_queued_in_the_same_decision() {
10776 let client = Client::new("http://127.0.0.1:8080").expect("client");
10777 let mut worker = Worker::new(client, "rust-workers");
10778 worker.register_workflow("rust.failing-after-side-effect", |ctx, _input| async move {
10779 let _: String = ctx.side_effect(|| "captured".to_string())?;
10780 Err(Error::WorkerLoop("application failure".to_string()))
10781 });
10782
10783 let commands = worker
10784 .execute_workflow_task(workflow_task(
10785 "rust.failing-after-side-effect",
10786 Vec::new(),
10787 JSON_CODEC,
10788 ))
10789 .expect("ordinary failure remains a workflow decision");
10790
10791 assert_eq!(commands.len(), 2);
10792 assert_eq!(commands[0]["type"], "record_side_effect");
10793 assert_eq!(commands[1]["type"], "fail_workflow");
10794 }
10795
10796 #[test]
10797 fn handler_error_cannot_hide_an_unconsumed_committed_side_effect() {
10798 let client = Client::new("http://127.0.0.1:8080").expect("client");
10799 let mut worker = Worker::new(client, "rust-workers");
10800 worker.register_workflow("rust.removed-side-effect", |_ctx, _input| async move {
10801 Err(Error::WorkerLoop("application failure".to_string()))
10802 });
10803 let result =
10804 encode_value_envelope(&json!("committed"), JSON_CODEC).expect("side-effect result");
10805
10806 let error = worker
10807 .execute_workflow_task(workflow_task(
10808 "rust.removed-side-effect",
10809 vec![history_event(
10810 "SideEffectRecorded",
10811 json!({"sequence": 1, "result": result}),
10812 )],
10813 JSON_CODEC,
10814 ))
10815 .expect_err("removed committed history must not become fail_workflow");
10816
10817 let Error::NonDeterministicReplay(failure) = error else {
10818 panic!("expected typed replay failure");
10819 };
10820 assert_eq!(failure.reason, "recorded_commands_unconsumed");
10821 assert_eq!(failure.sequence, Some(1));
10822 assert_eq!(failure.expected.as_deref(), Some("side effect"));
10823 }
10824
10825 #[test]
10826 fn replay_error_discards_side_effect_queued_before_incompatible_marker_check() {
10827 let client = Client::new("http://127.0.0.1:8080").expect("client");
10828 let mut worker = Worker::new(client, "rust-workers");
10829 worker.register_workflow(
10830 "rust.side-effect-before-marker-error",
10831 |ctx, _input| async move {
10832 assert_eq!(ctx.get_version("restart-safe", 1, 1)?, 1);
10833 let _: String = ctx.side_effect(|| "must-not-commit".to_string())?;
10834 ctx.get_version("restart-safe", 2, 2)?;
10835 Ok(Value::Null)
10836 },
10837 );
10838
10839 let error = worker
10840 .execute_workflow_task(workflow_task(
10841 "rust.side-effect-before-marker-error",
10842 vec![history_event(
10843 "VersionMarkerRecorded",
10844 json!({
10845 "sequence": 1,
10846 "change_id": "restart-safe",
10847 "version": 1,
10848 "min_supported": 1,
10849 "max_supported": 1,
10850 }),
10851 )],
10852 JSON_CODEC,
10853 ))
10854 .expect_err("replay error must return no queued workflow commands");
10855
10856 let Error::NonDeterministicReplay(failure) = error else {
10857 panic!("expected typed replay failure");
10858 };
10859 assert_eq!(failure.reason, "version_marker_incompatible_range");
10860 assert_eq!(failure.sequence, Some(1));
10861 }
10862
10863 #[test]
10864 fn workflow_task_replay_keeps_recorded_unfired_timer_pending_without_rescheduling() {
10865 let client = Client::new("http://127.0.0.1:8080").expect("client");
10866 let mut worker = Worker::new(client, "rust-workers");
10867 worker.register_workflow("rust.timer.pending", |ctx, _input| async move {
10868 ctx.sleep(Duration::from_secs(5)).await?;
10869 Ok(json!({"status": "timer fired"}))
10870 });
10871
10872 let task = WorkflowTask {
10873 task_id: "wft-rust-timer-pending".to_string(),
10874 workflow_id: Some("wf-rust-timer".to_string()),
10875 run_id: Some("run-rust-timer".to_string()),
10876 workflow_type: "rust.timer.pending".to_string(),
10877 payload_codec: JSON_CODEC.to_string(),
10878 arguments: Some(json!({"codec": "json", "blob": "[]"})),
10879 history_events: vec![history_event(
10880 "TimerScheduled",
10881 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10882 )],
10883 total_history_events: Some(1),
10884 history_size_bytes: None,
10885 continue_as_new_recommended: None,
10886 history_budget_pressure: None,
10887 next_history_page_token: None,
10888 workflow_task_attempt: 1,
10889 workflow_signal_id: None,
10890 signal_name: None,
10891 signal_arguments: None,
10892 workflow_update_id: None,
10893 update_name: None,
10894 lease_owner: Some("rust-worker".to_string()),
10895 };
10896
10897 for _redelivery_or_restart in 0..2 {
10898 let commands = worker
10899 .execute_workflow_task(task.clone())
10900 .expect("recorded timer remains pending");
10901 assert!(
10902 commands.is_empty(),
10903 "recorded timer must not be rescheduled"
10904 );
10905 }
10906 }
10907
10908 #[test]
10909 fn workflow_task_rejects_recorded_command_removed_from_workflow_code() {
10910 let client = Client::new("http://127.0.0.1:8080").expect("client");
10911 let mut worker = Worker::new(client, "rust-workers");
10912 worker.register_workflow("rust.timer.removed", |_ctx, _input| async move {
10913 Ok(json!({"status": "completed"}))
10914 });
10915 let task = WorkflowTask {
10916 task_id: "wft-rust-timer-removed".to_string(),
10917 workflow_id: Some("wf-rust-timer".to_string()),
10918 run_id: Some("run-rust-timer".to_string()),
10919 workflow_type: "rust.timer.removed".to_string(),
10920 payload_codec: JSON_CODEC.to_string(),
10921 arguments: Some(json!({"codec": "json", "blob": "[]"})),
10922 history_events: vec![
10923 history_event(
10924 "TimerScheduled",
10925 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10926 ),
10927 history_event(
10928 "TimerFired",
10929 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10930 ),
10931 ],
10932 total_history_events: Some(2),
10933 history_size_bytes: None,
10934 continue_as_new_recommended: None,
10935 history_budget_pressure: None,
10936 next_history_page_token: None,
10937 workflow_task_attempt: 1,
10938 workflow_signal_id: None,
10939 signal_name: None,
10940 signal_arguments: None,
10941 workflow_update_id: None,
10942 update_name: None,
10943 lease_owner: Some("rust-worker".to_string()),
10944 };
10945
10946 let Error::NonDeterministicReplay(failure) = worker
10947 .execute_workflow_task(task)
10948 .expect_err("removed timer must fail replay")
10949 else {
10950 panic!("expected typed replay failure");
10951 };
10952 assert_eq!(failure.reason, "recorded_commands_unconsumed");
10953 assert_eq!(failure.sequence, Some(1));
10954 }
10955
10956 #[test]
10957 fn workflow_context_emits_explicit_child_workflow_contract() {
10958 let ctx = WorkflowContext {
10959 state: Arc::new(Mutex::new(
10960 WorkflowState::new_with_identity(
10961 Vec::new(),
10962 Some("wf-parent".to_string()),
10963 Some("run-parent".to_string()),
10964 "parent-workers".to_string(),
10965 JSON_CODEC.to_string(),
10966 None,
10967 )
10968 .expect("workflow state"),
10969 )),
10970 };
10971 let options = ChildWorkflowOptions::new("python-workers")
10972 .parent_close_policy(ParentClosePolicy::RequestCancel)
10973 .retry_policy(ChildWorkflowRetryPolicy {
10974 max_attempts: Some(3),
10975 backoff_seconds: vec![1, 5],
10976 non_retryable_error_types: vec!["ValidationError".to_string()],
10977 })
10978 .execution_timeout_seconds(600)
10979 .run_timeout_seconds(120);
10980 let mut call = Box::pin(ctx.start_child_workflow(
10981 "python.fulfil-order",
10982 options,
10983 json!([{"order_id": "order-42"}]),
10984 ));
10985 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10986
10987 assert!(matches!(
10988 call.as_mut().poll(&mut task_context),
10989 Poll::Pending
10990 ));
10991 let commands = ctx.take_commands().expect("commands");
10992 assert_eq!(commands.len(), 1);
10993 let command = &commands[0];
10994 assert_eq!(command["type"], "start_child_workflow");
10995 assert_eq!(command["workflow_type"], "python.fulfil-order");
10996 assert_eq!(command["queue"], "python-workers");
10997 assert_eq!(command["parent_close_policy"], "request_cancel");
10998 assert_eq!(command["retry_policy"]["max_attempts"], 3);
10999 assert_eq!(command["execution_timeout_seconds"], 600);
11000 assert_eq!(command["run_timeout_seconds"], 120);
11001 assert_eq!(
11002 decode_wire_value(&command["arguments"], JSON_CODEC).expect("child args"),
11003 json!([{"order_id": "order-42"}])
11004 );
11005 }
11006
11007 fn child_parent_worker() -> Worker {
11008 let client = Client::new("http://127.0.0.1:8080").expect("client");
11009 let mut worker = Worker::new(client, "rust-parent-workers");
11010 worker.register_workflow("rust.parent", |ctx, _input| async move {
11011 let child = ctx
11012 .start_child_workflow(
11013 "python.child",
11014 ChildWorkflowOptions::new("python-child-workers")
11015 .parent_close_policy(ParentClosePolicy::Terminate),
11016 json!([{"codec_probe": [1, true, "rust"]}]),
11017 )
11018 .await?;
11019 Ok(json!({
11020 "parent_workflow_id": child.parent.workflow_id,
11021 "parent_run_id": child.parent.run_id,
11022 "child_workflow_id": child.child.workflow_id,
11023 "child_run_id": child.child.run_id,
11024 "child_workflow_type": child.child_workflow_type,
11025 "result": child.result,
11026 }))
11027 });
11028 worker
11029 }
11030
11031 fn child_parent_task(event_type: &str, payload: Value) -> WorkflowTask {
11032 WorkflowTask {
11033 task_id: "wft-child-parent".to_string(),
11034 workflow_id: Some("wf-parent".to_string()),
11035 run_id: Some("run-parent".to_string()),
11036 workflow_type: "rust.parent".to_string(),
11037 payload_codec: JSON_CODEC.to_string(),
11038 arguments: Some(encode_value_envelope(&json!([]), JSON_CODEC).expect("input")),
11039 history_events: vec![
11040 HistoryEvent {
11041 event_type: "ChildWorkflowScheduled".to_string(),
11042 payload: json!({
11043 "sequence": 1,
11044 "child_call_id": "call-child",
11045 "child_workflow_instance_id": "wf-child",
11046 "child_workflow_run_id": "run-child",
11047 "child_workflow_type": "python.child",
11048 }),
11049 raw: HashMap::new(),
11050 },
11051 HistoryEvent {
11052 event_type: event_type.to_string(),
11053 payload,
11054 raw: HashMap::new(),
11055 },
11056 ],
11057 total_history_events: Some(2),
11058 history_size_bytes: None,
11059 continue_as_new_recommended: None,
11060 history_budget_pressure: None,
11061 next_history_page_token: None,
11062 workflow_task_attempt: 1,
11063 workflow_signal_id: None,
11064 signal_name: None,
11065 signal_arguments: None,
11066 workflow_update_id: None,
11067 update_name: None,
11068 lease_owner: Some("rust-worker".to_string()),
11069 }
11070 }
11071
11072 #[test]
11073 fn committed_child_result_replays_without_starting_a_duplicate() {
11074 let worker = child_parent_worker();
11075 let task = child_parent_task(
11076 "ChildRunCompleted",
11077 json!({
11078 "sequence": 1,
11079 "child_call_id": "call-child",
11080 "child_workflow_instance_id": "wf-child",
11081 "child_workflow_run_id": "run-child",
11082 "child_workflow_type": "python.child",
11083 "payload_codec": "json",
11084 "result": {"codec": "json", "blob": "{\"from\":\"python\",\"ok\":true}"},
11085 }),
11086 );
11087
11088 for _restart in 0..2 {
11089 let commands = worker
11090 .execute_workflow_task(task.clone())
11091 .expect("replayed parent task");
11092 assert_eq!(commands.len(), 1);
11093 assert_eq!(commands[0]["type"], "complete_workflow");
11094 assert!(!commands
11095 .iter()
11096 .any(|command| command["type"] == "start_child_workflow"));
11097 let output =
11098 decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("parent output");
11099 assert_eq!(output["parent_workflow_id"], "wf-parent");
11100 assert_eq!(output["parent_run_id"], "run-parent");
11101 assert_eq!(output["child_workflow_id"], "wf-child");
11102 assert_eq!(output["child_run_id"], "run-child");
11103 assert_eq!(output["result"], json!({"from": "python", "ok": true}));
11104 }
11105 }
11106
11107 #[test]
11108 fn typed_child_arguments_and_results_survive_replay() {
11109 let client = Client::new("http://127.0.0.1:8080").expect("client");
11110 let mut worker = Worker::new(client, "rust-parent-workers");
11111 worker.register_workflow_avro_value("rust.typed-parent", |ctx, _input| async move {
11112 let child = ctx
11113 .start_child_workflow_avro_value(
11114 "python.typed-child",
11115 ChildWorkflowOptions::new("python-workers"),
11116 AvroValue::Array(vec![typed_fidelity_probe()]),
11117 )
11118 .await?;
11119 Ok(child.result)
11120 });
11121
11122 let initial = worker
11123 .execute_workflow_task(workflow_task(
11124 "rust.typed-parent",
11125 Vec::new(),
11126 DEFAULT_CODEC,
11127 ))
11128 .expect("typed child start");
11129 assert_eq!(initial[0]["type"], "start_child_workflow");
11130 assert_eq!(
11131 decode_wire_avro_value(&initial[0]["arguments"], DEFAULT_CODEC)
11132 .expect("typed child arguments"),
11133 AvroValue::Array(vec![typed_fidelity_probe()])
11134 );
11135
11136 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
11137 .expect("typed child result");
11138 let task = workflow_task(
11139 "rust.typed-parent",
11140 vec![
11141 history_event(
11142 "ChildWorkflowScheduled",
11143 json!({
11144 "sequence": 1,
11145 "child_call_id": "call-typed",
11146 "child_workflow_instance_id": "wf-child",
11147 "child_workflow_run_id": "run-child",
11148 "child_workflow_type": "python.typed-child",
11149 }),
11150 ),
11151 history_event(
11152 "ChildRunCompleted",
11153 json!({
11154 "sequence": 1,
11155 "child_call_id": "call-typed",
11156 "child_workflow_instance_id": "wf-child",
11157 "child_workflow_run_id": "run-child",
11158 "child_workflow_type": "python.typed-child",
11159 "payload_codec": DEFAULT_CODEC,
11160 "result": result,
11161 }),
11162 ),
11163 ],
11164 DEFAULT_CODEC,
11165 );
11166
11167 let commands = worker
11168 .execute_workflow_task(task)
11169 .expect("typed child replay");
11170 assert_eq!(commands[0]["type"], "complete_workflow");
11171 assert_eq!(
11172 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
11173 .expect("typed parent result"),
11174 typed_fidelity_probe()
11175 );
11176 }
11177
11178 #[test]
11179 fn pending_child_replays_after_restart_without_starting_a_duplicate() {
11180 let worker = child_parent_worker();
11181 let mut task = child_parent_task("unused", Value::Null);
11182 task.history_events.truncate(1);
11183 task.total_history_events = Some(1);
11184
11185 for _redelivery_or_restart in 0..2 {
11186 let commands = worker
11187 .execute_workflow_task(task.clone())
11188 .expect("recorded child remains pending");
11189 assert!(
11190 commands.is_empty(),
11191 "recorded pending child must not be started again"
11192 );
11193 }
11194 }
11195
11196 #[test]
11197 fn child_cancellation_becomes_stable_parent_failure_command() {
11198 let worker = child_parent_worker();
11199 let task = child_parent_task(
11200 "ChildRunCancelled",
11201 json!({
11202 "sequence": 1,
11203 "child_workflow_instance_id": "wf-child",
11204 "child_workflow_run_id": "run-child",
11205 "child_workflow_type": "python.child",
11206 "failure_id": "failure-child",
11207 "failure_category": "cancelled",
11208 "message": "cancelled by parent-close policy",
11209 }),
11210 );
11211
11212 let commands = worker
11213 .execute_workflow_task(task)
11214 .expect("parent settlement");
11215 assert_eq!(commands.len(), 1);
11216 assert_eq!(commands[0]["type"], "fail_workflow");
11217 assert_eq!(commands[0]["exception_type"], "ChildWorkflowCancelled");
11218 assert_eq!(
11219 commands[0]["exception"]["properties"]["reason"],
11220 "cancelled"
11221 );
11222 assert_eq!(
11223 commands[0]["exception"]["properties"]["child_workflow_run_id"],
11224 "run-child"
11225 );
11226 }
11227
11228 #[test]
11229 fn workflow_can_handle_typed_child_failure() {
11230 let client = Client::new("http://127.0.0.1:8080").expect("client");
11231 let mut worker = Worker::new(client, "rust-parent-workers");
11232 worker.register_workflow("rust.handled-parent", |ctx, _input| async move {
11233 match ctx
11234 .start_child_workflow(
11235 "python.child",
11236 ChildWorkflowOptions::new("python-child-workers"),
11237 json!([]),
11238 )
11239 .await
11240 {
11241 Err(Error::ChildWorkflowFailed(failure)) => Ok(json!({
11242 "reason": failure.reason,
11243 "failure_id": failure.failure_id,
11244 "exception_class": failure.exception_class,
11245 "child_run_id": failure.child_workflow_run_id,
11246 })),
11247 Err(error) => Err(error),
11248 Ok(_) => Err(Error::WorkerLoop(
11249 "child unexpectedly succeeded".to_string(),
11250 )),
11251 }
11252 });
11253 let mut task = child_parent_task(
11254 "ChildRunFailed",
11255 json!({
11256 "sequence": 1,
11257 "child_workflow_instance_id": "wf-child",
11258 "child_workflow_run_id": "run-child",
11259 "child_workflow_type": "python.child",
11260 "failure_id": "failure-child",
11261 "failure_category": "child_workflow",
11262 "message": "payment rejected",
11263 "exception": {
11264 "type": "PaymentRejected",
11265 "class": "payments.PaymentRejected",
11266 "message": "payment rejected"
11267 }
11268 }),
11269 );
11270 task.workflow_type = "rust.handled-parent".to_string();
11271
11272 let commands = worker.execute_workflow_task(task).expect("handled failure");
11273 assert_eq!(commands[0]["type"], "complete_workflow");
11274 let output = decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("parent output");
11275 assert_eq!(output["reason"], "child_workflow");
11276 assert_eq!(output["failure_id"], "failure-child");
11277 assert_eq!(output["exception_class"], "payments.PaymentRejected");
11278 assert_eq!(output["child_run_id"], "run-child");
11279 }
11280
11281 #[test]
11282 fn rust_hello_world_uses_signal_arguments_from_resume_payload() {
11283 let client = Client::new("http://127.0.0.1:8080").expect("client");
11284 let mut worker = Worker::new(client, "rust-workers");
11285
11286 worker.register_workflow("rust.hello_workflow", |ctx, _input| async move {
11287 let signal = ctx.wait_signal("start").await?;
11288 let name = signal
11289 .first()
11290 .and_then(|value| value.as_str())
11291 .unwrap_or("world");
11292 let greeting = ctx.activity("rust.hello_activity", json!([name])).await?;
11293 Ok(json!({
11294 "greeting": greeting,
11295 "language": "rust"
11296 }))
11297 });
11298
11299 let signal_arguments =
11300 encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC).expect("signal arguments");
11301 let task = WorkflowTask {
11302 task_id: "wft-rust-signal-1".to_string(),
11303 workflow_id: Some("wf-rust-hello".to_string()),
11304 run_id: Some("run-rust-hello".to_string()),
11305 workflow_type: "rust.hello_workflow".to_string(),
11306 payload_codec: DEFAULT_CODEC.to_string(),
11307 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
11308 history_events: vec![HistoryEvent {
11309 event_type: "SignalReceived".to_string(),
11310 payload: json!({
11311 "signal_id": "sig-rust-1",
11312 "signal_name": "start"
11313 }),
11314 raw: HashMap::new(),
11315 }],
11316 total_history_events: Some(1),
11317 history_size_bytes: None,
11318 continue_as_new_recommended: None,
11319 history_budget_pressure: None,
11320 next_history_page_token: None,
11321 workflow_task_attempt: 1,
11322 workflow_signal_id: Some("sig-rust-1".to_string()),
11323 signal_name: Some("start".to_string()),
11324 signal_arguments: Some(signal_arguments),
11325 workflow_update_id: None,
11326 update_name: None,
11327 lease_owner: Some("rust-worker".to_string()),
11328 };
11329
11330 let commands = worker.execute_workflow_task(task).expect("workflow task");
11331
11332 assert_eq!(commands.len(), 1);
11333 assert_eq!(commands[0]["type"], "schedule_activity");
11334 assert_eq!(commands[0]["activity_type"], "rust.hello_activity");
11335 assert_eq!(
11336 decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC).expect("activity args"),
11337 json!(["Rust"])
11338 );
11339 }
11340
11341 #[test]
11342 fn workflow_task_appends_paginated_history_events() {
11343 let mut task = WorkflowTask {
11344 task_id: "wft-rust-pages-1".to_string(),
11345 workflow_id: Some("wf-rust-pages".to_string()),
11346 run_id: Some("run-rust-pages".to_string()),
11347 workflow_type: "rust.hello_workflow".to_string(),
11348 payload_codec: DEFAULT_CODEC.to_string(),
11349 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
11350 history_events: vec![HistoryEvent {
11351 event_type: "WorkflowStarted".to_string(),
11352 payload: json!({}),
11353 raw: HashMap::new(),
11354 }],
11355 total_history_events: Some(3),
11356 history_size_bytes: None,
11357 continue_as_new_recommended: None,
11358 history_budget_pressure: None,
11359 next_history_page_token: Some("MQ==".to_string()),
11360 workflow_task_attempt: 1,
11361 workflow_signal_id: None,
11362 signal_name: None,
11363 signal_arguments: None,
11364 workflow_update_id: None,
11365 update_name: None,
11366 lease_owner: Some("rust-worker".to_string()),
11367 };
11368
11369 task.append_history_page(WorkflowTaskHistoryPage {
11370 history_events: vec![
11371 HistoryEvent {
11372 event_type: "SignalReceived".to_string(),
11373 payload: json!({
11374 "signal_id": "sig-rust-1",
11375 "signal_name": "start",
11376 "arguments": encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC)
11377 .expect("signal arguments")
11378 }),
11379 raw: HashMap::new(),
11380 },
11381 HistoryEvent {
11382 event_type: "MarkerRecorded".to_string(),
11383 payload: json!({"sequence": 3}),
11384 raw: HashMap::new(),
11385 },
11386 ],
11387 total_history_events: Some(3),
11388 next_history_page_token: None,
11389 });
11390
11391 assert_eq!(task.history_events.len(), 3);
11392 assert_eq!(task.total_history_events, Some(3));
11393 assert_eq!(task.next_history_page_token, None);
11394
11395 let signal = task
11396 .history_events
11397 .iter()
11398 .find(|event| event.event_type == "SignalReceived")
11399 .expect("signal event");
11400 assert_eq!(
11401 decode_signal_event_arguments(signal, DEFAULT_CODEC).expect("signal arguments"),
11402 vec![AvroValue::String("Rust".to_string())]
11403 );
11404 }
11405
11406 #[tokio::test]
11407 async fn query_handler_reads_ordered_cross_codec_signals_without_commands() {
11408 let client = Client::new("http://127.0.0.1:8080").expect("client");
11409 let mut worker = Worker::new(client, "rust-workers");
11410 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
11411 worker.register_query("counter", "current", |ctx, _args| async move {
11412 let mut count = 0_i64;
11413 for signal in ctx.signal_events() {
11414 let value = signal
11415 .arguments
11416 .first()
11417 .and_then(Value::as_i64)
11418 .unwrap_or_default();
11419 match signal.name.as_str() {
11420 "increment" => count += value,
11421 "set" => count = value,
11422 _ => {}
11423 }
11424 }
11425 Ok(json!(count))
11426 });
11427
11428 let task = QueryTask {
11429 query_task_id: "query-rust-counter".to_string(),
11430 query_task_attempt: 1,
11431 lease_owner: Some("rust-worker".to_string()),
11432 workflow_id: Some("counter-1".to_string()),
11433 run_id: Some("run-counter-1".to_string()),
11434 workflow_type: "counter".to_string(),
11435 query_name: "current".to_string(),
11436 payload_codec: DEFAULT_CODEC.to_string(),
11437 workflow_arguments: Some(
11438 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
11439 ),
11440 query_arguments: Some(
11441 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("query arguments"),
11442 ),
11443 history_events: vec![
11444 HistoryEvent {
11445 event_type: "SignalReceived".to_string(),
11446 payload: json!({
11447 "signal_id": "php-signal-1",
11448 "signal_name": "increment",
11449 "workflow_sequence": 1,
11450 "payload_codec": DEFAULT_CODEC,
11451 "arguments": encode_value_envelope(&json!([3]), DEFAULT_CODEC).expect("php avro signal")
11452 }),
11453 raw: HashMap::new(),
11454 },
11455 HistoryEvent {
11456 event_type: "SignalReceived".to_string(),
11457 payload: json!({
11458 "signal_id": "python-signal-2",
11459 "signal_name": "increment",
11460 "workflow_sequence": 2,
11461 "payload_codec": JSON_CODEC,
11462 "arguments": encode_value_envelope(&json!([5]), JSON_CODEC).expect("python json signal")
11463 }),
11464 raw: HashMap::new(),
11465 },
11466 HistoryEvent {
11467 event_type: "SignalReceived".to_string(),
11468 payload: json!({
11469 "signal_id": "rust-signal-3",
11470 "signal_name": "set",
11471 "workflow_sequence": 3,
11472 "payload_codec": DEFAULT_CODEC,
11473 "arguments": encode_value_envelope(&json!([0]), DEFAULT_CODEC).expect("rust avro signal")
11474 }),
11475 raw: HashMap::new(),
11476 },
11477 ],
11478 history_export: None,
11479 run_status: Some("completed".to_string()),
11480 };
11481
11482 let result = worker.execute_query_task(task).await.expect("query result");
11483 assert_eq!(result.into_json().expect("query projection"), json!(0));
11484 }
11485
11486 #[tokio::test]
11487 async fn replayed_queries_read_running_completed_and_cold_restarted_instance_state() {
11488 let worker = replay_counter_worker();
11489 let running_history = json!([
11490 {
11491 "type": "ActivityCompleted",
11492 "payload": {
11493 "sequence": 1,
11494 "activity_type": "load-counter",
11495 "payload_codec": "json",
11496 "result": {"codec": "json", "blob": "\"loaded\""}
11497 }
11498 },
11499 {
11500 "type": "SignalWaitOpened",
11501 "payload": {
11502 "sequence": 3,
11503 "signal_name": "increment"
11504 }
11505 },
11506 {
11507 "type": "SignalReceived",
11508 "payload": {
11509 "signal_id": "signal-3",
11510 "signal_name": "increment",
11511 "workflow_sequence": 2,
11512 "payload_codec": "json",
11513 "arguments": {"codec": "json", "blob": "[3]"}
11514 }
11515 },
11516 {
11517 "type": "SignalApplied",
11518 "payload": {
11519 "sequence": 3,
11520 "signal_id": "signal-3",
11521 "signal_name": "increment",
11522 "payload_codec": "json",
11523 "value": {"codec": "json", "blob": "[3]"}
11524 }
11525 }
11526 ]);
11527
11528 let running = worker
11529 .execute_query_task(replay_counter_query(
11530 "current",
11531 running_history.clone(),
11532 "running",
11533 ))
11534 .await
11535 .expect("running replay query");
11536 assert_eq!(
11537 running.clone().into_json().expect("query projection"),
11538 json!({"loaded": "loaded", "count": 3, "finished": false})
11539 );
11540
11541 let detached = worker
11542 .execute_query_task(replay_counter_query(
11543 "detached-mutation",
11544 running_history.clone(),
11545 "running",
11546 ))
11547 .await
11548 .expect("query mutates only its detached state clone");
11549 assert_eq!(detached.into_json().expect("query projection"), json!(999));
11550 let failed = worker
11551 .execute_query_task(replay_counter_query(
11552 "failed-mutation",
11553 running_history.clone(),
11554 "running",
11555 ))
11556 .await
11557 .expect_err("failed query");
11558 assert_eq!(failed.reason, "query_rejected");
11559 let unchanged = worker
11560 .execute_query_task(replay_counter_query("current", running_history, "running"))
11561 .await
11562 .expect("later query reconstructs unchanged state");
11563 assert_eq!(unchanged, running);
11564
11565 let restarted_worker = replay_counter_worker();
11566 let restarted_task: QueryTask = serde_json::from_value(json!({
11567 "query_task_id": "query-after-restart",
11568 "workflow_id": "counter-1",
11569 "run_id": "run-counter-1",
11570 "workflow_type": "replay-counter",
11571 "query_name": "current",
11572 "payload_codec": "json",
11573 "workflow_arguments": {"codec": "json", "blob": "[]"},
11574 "query_arguments": {"codec": "json", "blob": "[]"},
11575 "history_events": [],
11576 "history_export": {
11577 "payloads": {"codec": "json"},
11578 "history_events": [
11579 {
11580 "type": "ActivityCompleted",
11581 "payload": {
11582 "sequence": 1,
11583 "activity_type": "load-counter",
11584 "payload_codec": "json",
11585 "result": null
11586 }
11587 },
11588 {
11589 "type": "SignalWaitOpened",
11590 "payload": {
11591 "sequence": 3,
11592 "signal_name": "increment"
11593 }
11594 },
11595 {
11596 "type": "SignalReceived",
11597 "payload": {
11598 "signal_id": "signal-3",
11599 "signal_name": "increment",
11600 "workflow_sequence": 2
11601 }
11602 },
11603 {
11604 "type": "SignalApplied",
11605 "payload": {
11606 "sequence": 3,
11607 "signal_id": "signal-3",
11608 "signal_name": "increment"
11609 }
11610 },
11611 {
11612 "type": "SignalWaitOpened",
11613 "payload": {
11614 "sequence": 5,
11615 "signal_name": "increment"
11616 }
11617 },
11618 {
11619 "type": "SignalReceived",
11620 "payload": {
11621 "signal_id": "signal-5",
11622 "signal_name": "increment",
11623 "workflow_sequence": 4
11624 }
11625 },
11626 {
11627 "type": "SignalApplied",
11628 "payload": {
11629 "sequence": 5,
11630 "signal_id": "signal-5",
11631 "signal_name": "increment"
11632 }
11633 }
11634 ],
11635 "activities": [{
11636 "sequence": 1,
11637 "activity_type": "load-counter",
11638 "payload_codec": "json",
11639 "result": {"codec": "json", "blob": "\"loaded\""}
11640 }],
11641 "signals": [
11642 {
11643 "id": "signal-3",
11644 "name": "increment",
11645 "workflow_sequence": 2,
11646 "payload_codec": "json",
11647 "arguments": "[3]"
11648 },
11649 {
11650 "id": "signal-5",
11651 "name": "increment",
11652 "workflow_sequence": 4,
11653 "payload_codec": "json",
11654 "arguments": "[5]"
11655 }
11656 ]
11657 },
11658 "run_status": "completed"
11659 }))
11660 .expect("cold replay query task");
11661 let completed = restarted_worker
11662 .execute_query_task(restarted_task)
11663 .await
11664 .expect("completed cold replay query");
11665 assert_eq!(
11666 completed.into_json().expect("query projection"),
11667 json!({"loaded": "loaded", "count": 8, "finished": true})
11668 );
11669 }
11670
11671 #[tokio::test]
11672 async fn replayed_query_replay_failures_are_machine_readable() {
11673 let worker = replay_counter_worker();
11674 let task = replay_counter_query(
11675 "current",
11676 json!([{
11677 "type": "ActivityCompleted",
11678 "payload": {
11679 "sequence": 1,
11680 "payload_codec": "json",
11681 "result": {"codec": "json", "blob": "{"}
11682 }
11683 }]),
11684 "running",
11685 );
11686 let failure = worker
11687 .execute_query_task(task)
11688 .await
11689 .expect_err("invalid replay history payload");
11690 assert_eq!(failure.reason, "query_workflow_state_unavailable");
11691 assert_eq!(failure.failure_type, "QueryWorkflowStateUnavailable");
11692 }
11693
11694 #[tokio::test]
11695 async fn query_task_restores_compact_history_from_export() {
11696 let client = Client::new("http://127.0.0.1:8080").expect("client");
11697 let mut worker = Worker::new(client, "rust-workers");
11698 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
11699 worker.register_query("counter", "current", |ctx, _args| async move {
11700 Ok(json!(ctx.signals("increment")[0][0]))
11701 });
11702 let task: QueryTask = serde_json::from_value(json!({
11703 "query_task_id": "query-export",
11704 "workflow_type": "counter",
11705 "query_name": "current",
11706 "payload_codec": "json",
11707 "workflow_arguments": {"codec": "json", "blob": "[]"},
11708 "query_arguments": {"codec": "json", "blob": "[]"},
11709 "history_events": [],
11710 "history_export": {
11711 "payloads": {"codec": "json"},
11712 "history_events": [{
11713 "type": "SignalReceived",
11714 "payload": {"signal_id": "signal-export", "signal_name": "increment"}
11715 }],
11716 "signals": [{
11717 "id": "signal-export",
11718 "name": "increment",
11719 "status": "applied",
11720 "workflow_sequence": 1,
11721 "payload_codec": "json",
11722 "arguments": "[9]"
11723 }]
11724 }
11725 }))
11726 .expect("query task");
11727
11728 let result = worker.execute_query_task(task).await.expect("query result");
11729 assert_eq!(result.into_json().expect("query projection"), json!(9));
11730 }
11731
11732 #[tokio::test]
11733 async fn query_task_failures_have_stable_reasons() {
11734 let client = Client::new("http://127.0.0.1:8080").expect("client");
11735 let mut worker = Worker::new(client, "rust-workers");
11736 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
11737 worker.register_query(
11738 "counter",
11739 "current",
11740 |_ctx, _args| async move { Ok(json!(0)) },
11741 );
11742
11743 let base_task = QueryTask {
11744 query_task_id: "query-errors".to_string(),
11745 query_task_attempt: 1,
11746 lease_owner: None,
11747 workflow_id: Some("counter-errors".to_string()),
11748 run_id: Some("run-errors".to_string()),
11749 workflow_type: "counter".to_string(),
11750 query_name: "missing".to_string(),
11751 payload_codec: JSON_CODEC.to_string(),
11752 workflow_arguments: Some(json!({"codec": "json", "blob": "[]"})),
11753 query_arguments: Some(json!({"codec": "json", "blob": "[]"})),
11754 history_events: Vec::new(),
11755 history_export: None,
11756 run_status: Some("running".to_string()),
11757 };
11758
11759 let unknown = worker
11760 .execute_query_task(base_task.clone())
11761 .await
11762 .expect_err("unknown query");
11763 assert_eq!(unknown.reason, "rejected_unknown_query");
11764
11765 let mut malformed = base_task;
11766 malformed.query_name = "current".to_string();
11767 malformed.query_arguments = Some(json!({"codec": "json", "blob": "{"}));
11768 let malformed = worker
11769 .execute_query_task(malformed)
11770 .await
11771 .expect_err("malformed payload");
11772 assert_eq!(malformed.reason, "query_payload_decode_failed");
11773
11774 let client = Client::new("http://127.0.0.1:8080").expect("client");
11775 let mut unavailable_worker = Worker::new(client, "rust-workers");
11776 unavailable_worker
11777 .register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
11778 let unavailable_task: QueryTask = serde_json::from_value(json!({
11779 "query_task_id": "query-unavailable",
11780 "workflow_type": "counter",
11781 "query_name": "current",
11782 "payload_codec": "json",
11783 "workflow_arguments": {"codec": "json", "blob": "[]"},
11784 "query_arguments": {"codec": "json", "blob": "[]"}
11785 }))
11786 .expect("query task");
11787 let unavailable = unavailable_worker
11788 .execute_query_task(unavailable_task)
11789 .await
11790 .expect_err("query handler unavailable");
11791 assert_eq!(unavailable.reason, "query_handler_unavailable");
11792 }
11793
11794 #[tokio::test]
11795 async fn client_query_decodes_result_and_typed_failure() {
11796 let server = MockWorkerServer::start();
11797 let client = Client::builder(server.base_url())
11798 .timeout(Duration::from_secs(2))
11799 .build()
11800 .expect("client");
11801
11802 let result = client
11803 .query_workflow("counter-1", "current", json!([]))
11804 .await
11805 .expect("query result");
11806 assert_eq!(result, json!({"count": 8}));
11807
11808 let error = client
11809 .query_workflow("counter-1", "missing", json!([]))
11810 .await
11811 .expect_err("unknown query");
11812 let Error::QueryFailed(failure) = error else {
11813 panic!("expected typed query failure");
11814 };
11815 assert_eq!(failure.status, 404);
11816 assert_eq!(failure.reason, "rejected_unknown_query");
11817 }
11818
11819 #[tokio::test]
11820 async fn public_client_surfaces_send_and_receive_lossless_avro_values() {
11821 let server = MockWorkerServer::start();
11822 let client = Client::builder(server.base_url())
11823 .timeout(Duration::from_secs(2))
11824 .build()
11825 .expect("client");
11826 let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
11827
11828 client
11829 .start_workflow(
11830 "typed.echo",
11831 "rust-workers",
11832 "typed-start",
11833 arguments.clone(),
11834 )
11835 .await
11836 .expect("typed workflow start");
11837 assert_eq!(
11838 decode_wire_avro_value(
11839 &server.request_body("/api/workflows")["input"],
11840 DEFAULT_CODEC,
11841 )
11842 .expect("typed start input"),
11843 arguments
11844 );
11845
11846 client
11847 .signal_workflow("typed-1", "changed", arguments.clone())
11848 .await
11849 .expect("typed signal");
11850 assert_eq!(
11851 decode_wire_avro_value(
11852 &server.request_body("/api/workflows/typed-1/signal/changed")["input"],
11853 DEFAULT_CODEC,
11854 )
11855 .expect("typed signal input"),
11856 arguments
11857 );
11858
11859 assert_eq!(
11860 client
11861 .query_workflow_avro_value("typed-1", "inspect", arguments.clone())
11862 .await
11863 .expect("typed query"),
11864 typed_fidelity_probe()
11865 );
11866 assert_eq!(
11867 decode_wire_avro_value(
11868 &server.request_body("/api/workflows/typed-1/query/inspect")["input"],
11869 DEFAULT_CODEC,
11870 )
11871 .expect("typed query input"),
11872 arguments
11873 );
11874
11875 assert_eq!(
11876 client
11877 .update_workflow_avro_value(
11878 "typed-1",
11879 "replace",
11880 arguments.clone(),
11881 Some("typed-request"),
11882 )
11883 .await
11884 .expect("typed update"),
11885 typed_fidelity_probe()
11886 );
11887 let update = server.request_body("/api/workflows/typed-1/update/replace");
11888 assert_eq!(update["request_id"], "typed-request");
11889 assert_eq!(
11890 decode_wire_avro_value(&update["input"], DEFAULT_CODEC).expect("typed update input"),
11891 arguments
11892 );
11893
11894 let handle = WorkflowHandle {
11895 client: client.clone(),
11896 workflow_id: "typed-1".to_string(),
11897 run_id: Some("run-typed-1".to_string()),
11898 workflow_type: "typed.echo".to_string(),
11899 };
11900 assert_eq!(
11901 handle
11902 .result_avro_value(WorkflowResultOptions::default())
11903 .await
11904 .expect("typed workflow result"),
11905 typed_fidelity_probe()
11906 );
11907
11908 client
11909 .complete_activity_task(
11910 "activity-typed",
11911 "attempt-typed",
11912 "rust-worker",
11913 typed_fidelity_probe(),
11914 DEFAULT_CODEC,
11915 )
11916 .await
11917 .expect("typed activity completion");
11918 assert_eq!(
11919 decode_wire_avro_value(
11920 &server.request_body("/api/worker/activity-tasks/activity-typed/complete")
11921 ["result"],
11922 DEFAULT_CODEC,
11923 )
11924 .expect("typed activity result"),
11925 typed_fidelity_probe()
11926 );
11927 client
11928 .fail_activity_task(
11929 "activity-typed",
11930 "attempt-typed",
11931 "rust-worker",
11932 "typed failure",
11933 true,
11934 )
11935 .await
11936 .expect("activity failure");
11937 }
11938
11939 #[tokio::test]
11940 async fn lifecycle_commands_support_instance_and_selected_run_targets() {
11941 let server = MockWorkerServer::start();
11942 let client = Client::builder(server.base_url())
11943 .timeout(Duration::from_secs(2))
11944 .build()
11945 .expect("client");
11946
11947 let options = WorkflowCommandOptions::new()
11948 .reason("cleanup requested")
11949 .request_id("cancel-17");
11950 let cancelled = client
11951 .cancel_workflow("wf-lifecycle", options)
11952 .await
11953 .expect("instance cancellation");
11954 assert_eq!(cancelled.command, WorkflowCommandKind::Cancel);
11955 assert_eq!(cancelled.run_id.as_deref(), Some("run-current"));
11956 assert_eq!(cancelled.outcome.as_deref(), Some("cancelled"));
11957 assert_eq!(
11958 server.request_body("/api/workflows/wf-lifecycle/cancel"),
11959 json!({"reason":"cleanup requested","request_id":"cancel-17"})
11960 );
11961
11962 let terminated = client
11963 .terminate_workflow(
11964 "wf-lifecycle",
11965 WorkflowCommandOptions::new().reason("forced stop"),
11966 )
11967 .await
11968 .expect("instance termination");
11969 assert_eq!(terminated.command, WorkflowCommandKind::Terminate);
11970 assert_eq!(terminated.outcome.as_deref(), Some("terminated"));
11971
11972 client
11973 .cancel_workflow_run(
11974 "wf-lifecycle",
11975 "run-current",
11976 WorkflowCommandOptions::default(),
11977 )
11978 .await
11979 .expect("selected run cancellation");
11980 client
11981 .terminate_workflow_run(
11982 "wf-lifecycle",
11983 "run-current",
11984 WorkflowCommandOptions::default(),
11985 )
11986 .await
11987 .expect("selected run termination");
11988
11989 for (command, error) in [
11990 (
11991 WorkflowCommandKind::Cancel,
11992 client
11993 .cancel_workflow_run(
11994 "wf-lifecycle",
11995 "run-stale",
11996 WorkflowCommandOptions::default(),
11997 )
11998 .await
11999 .expect_err("stale cancellation must be rejected"),
12000 ),
12001 (
12002 WorkflowCommandKind::Terminate,
12003 client
12004 .terminate_workflow_run(
12005 "wf-lifecycle",
12006 "run-stale",
12007 WorkflowCommandOptions::default(),
12008 )
12009 .await
12010 .expect_err("stale termination must be rejected"),
12011 ),
12012 ] {
12013 let Error::WorkflowCommandRejected(rejection) = error else {
12014 panic!("expected typed command rejection");
12015 };
12016 assert_eq!(rejection.command, command);
12017 assert_eq!(rejection.status, 409);
12018 assert_eq!(rejection.reason, "historical_run_command_rejected");
12019 assert_eq!(rejection.run_id.as_deref(), Some("run-stale"));
12020 assert_eq!(rejection.target_scope.as_deref(), Some("run"));
12021 }
12022 }
12023
12024 #[tokio::test]
12025 async fn workflow_start_options_send_server_enforced_deadlines() {
12026 let server = MockWorkerServer::start();
12027 let client = Client::builder(server.base_url())
12028 .timeout(Duration::from_secs(2))
12029 .build()
12030 .expect("client");
12031
12032 let handle = client
12033 .start_workflow_with_options(
12034 "rust.timeout",
12035 "rust-timeouts",
12036 "wf-start-options",
12037 WorkflowStartOptions::new()
12038 .execution_timeout_seconds(30)
12039 .run_timeout_seconds(1),
12040 json!([]),
12041 )
12042 .await
12043 .expect("workflow start");
12044
12045 assert_eq!(handle.run_id.as_deref(), Some("run-start-options"));
12046 let body = server.request_body("/api/workflows");
12047 assert_eq!(body["execution_timeout_seconds"], 30);
12048 assert_eq!(body["run_timeout_seconds"], 1);
12049
12050 let invalid = client
12051 .start_workflow_with_options(
12052 "rust.timeout",
12053 "rust-timeouts",
12054 "wf-invalid-options",
12055 WorkflowStartOptions::new()
12056 .execution_timeout_seconds(1)
12057 .run_timeout_seconds(2),
12058 json!([]),
12059 )
12060 .await
12061 .expect_err("invalid deadline ordering");
12062 assert!(invalid
12063 .to_string()
12064 .contains("run_timeout_seconds cannot exceed execution_timeout_seconds"));
12065 }
12066
12067 #[tokio::test]
12068 async fn workflow_result_returns_each_typed_terminal_outcome() {
12069 let server = MockWorkerServer::start();
12070 let client = Client::builder(server.base_url())
12071 .timeout(Duration::from_secs(2))
12072 .build()
12073 .expect("client");
12074 let options = WorkflowResultOptions {
12075 poll_interval: Duration::ZERO,
12076 timeout: Duration::from_secs(1),
12077 };
12078
12079 let failed = WorkflowHandle {
12080 client: client.clone(),
12081 workflow_id: "wf-failed".to_string(),
12082 run_id: Some("run-failed".to_string()),
12083 workflow_type: "failure".to_string(),
12084 }
12085 .result(options)
12086 .await
12087 .expect_err("failed outcome");
12088 let Error::WorkflowFailed(failure) = failed else {
12089 panic!("expected WorkflowFailed");
12090 };
12091 assert_eq!(failure.workflow_id, "wf-failed");
12092 assert_eq!(failure.run_id.as_deref(), Some("run-failed"));
12093 assert_eq!(failure.failure_id.as_deref(), Some("failure-17"));
12094 assert_eq!(failure.failure_category.as_deref(), Some("application"));
12095 assert_eq!(failure.exception_type.as_deref(), Some("PaymentError"));
12096 assert_eq!(
12097 failure.exception_class.as_deref(),
12098 Some("billing::PaymentError")
12099 );
12100 assert_eq!(failure.non_retryable, Some(true));
12101
12102 for (workflow_id, expected_kind, expected_reason) in [
12103 (
12104 "wf-cancelled",
12105 WorkflowTerminalKind::Cancelled,
12106 "cleanup requested",
12107 ),
12108 (
12109 "wf-terminated",
12110 WorkflowTerminalKind::Terminated,
12111 "forced stop",
12112 ),
12113 (
12114 "wf-timed-out",
12115 WorkflowTerminalKind::TimedOut,
12116 "run_timeout",
12117 ),
12118 ] {
12119 let error = WorkflowHandle {
12120 client: client.clone(),
12121 workflow_id: workflow_id.to_string(),
12122 run_id: None,
12123 workflow_type: "terminal".to_string(),
12124 }
12125 .result(options)
12126 .await
12127 .expect_err("typed terminal outcome");
12128 let outcome = match error {
12129 Error::WorkflowCancelled(outcome) => outcome,
12130 Error::WorkflowTerminated(outcome) => outcome,
12131 Error::WorkflowTimedOut(outcome) => outcome,
12132 other => panic!("unexpected terminal error: {other}"),
12133 };
12134 assert_eq!(outcome.kind, expected_kind);
12135 assert_eq!(outcome.workflow_id, workflow_id);
12136 assert_eq!(outcome.reason, expected_reason);
12137 }
12138
12139 let wait_timeout = WorkflowHandle {
12140 client,
12141 workflow_id: "wf-waiting".to_string(),
12142 run_id: Some("run-waiting".to_string()),
12143 workflow_type: "waiting".to_string(),
12144 }
12145 .result(WorkflowResultOptions {
12146 poll_interval: Duration::ZERO,
12147 timeout: Duration::ZERO,
12148 })
12149 .await
12150 .expect_err("client wait timeout");
12151 let Error::WorkflowTimedOut(timeout) = wait_timeout else {
12152 panic!("expected typed client timeout");
12153 };
12154 assert_eq!(timeout.reason, "result_wait_timeout");
12155 assert_eq!(timeout.failure_category.as_deref(), Some("client_timeout"));
12156 assert_eq!(timeout.run_id.as_deref(), Some("run-waiting"));
12157 }
12158
12159 #[tokio::test]
12160 async fn workflow_result_follows_chain_and_selected_result_preserves_history() {
12161 let server = MockWorkerServer::start();
12162 let client = Client::builder(server.base_url())
12163 .timeout(Duration::from_secs(2))
12164 .build()
12165 .expect("client");
12166
12167 let handle = WorkflowHandle {
12168 client,
12169 workflow_id: "wf-selected".to_string(),
12170 run_id: Some("run-selected".to_string()),
12171 workflow_type: "selected".to_string(),
12172 };
12173 let options = WorkflowResultOptions {
12174 poll_interval: Duration::ZERO,
12175 timeout: Duration::from_secs(1),
12176 };
12177
12178 let current = handle
12179 .result(options)
12180 .await
12181 .expect("instance result follows the current run");
12182 assert_eq!(current, json!("current run output"));
12183
12184 let error = handle
12185 .result_selected_run(options)
12186 .await
12187 .expect_err("the selected run is cancelled even though the current run completed");
12188
12189 let Error::WorkflowCancelled(outcome) = error else {
12190 panic!("expected selected run cancellation");
12191 };
12192 assert_eq!(outcome.run_id.as_deref(), Some("run-selected"));
12193 assert_eq!(outcome.reason, "selected run cancelled");
12194 assert_eq!(
12195 server.request_count("/api/workflows/wf-selected/runs/run-selected"),
12196 1
12197 );
12198 assert_eq!(server.request_count("/api/workflows/wf-selected"), 1);
12199 }
12200
12201 #[tokio::test]
12202 async fn poll_responses_decode_http_conflict_drain_as_a_stable_stop() {
12203 let server = MockWorkerServer::draining_polls();
12204 let client = Client::builder(server.base_url())
12205 .timeout(Duration::from_secs(2))
12206 .build()
12207 .expect("client");
12208
12209 let workflow = client
12210 .poll_workflow_task_response("draining-worker", "rust-workers", Duration::ZERO)
12211 .await
12212 .expect("workflow drain response");
12213 let activity = client
12214 .poll_activity_task_response("draining-worker", "rust-workers", Duration::ZERO)
12215 .await
12216 .expect("activity drain response");
12217 let query = client
12218 .poll_query_task_response("draining-worker", "rust-workers", Duration::ZERO)
12219 .await
12220 .expect("query drain response");
12221
12222 for outcome in [workflow.outcome(), activity.outcome(), query.outcome()] {
12223 assert_eq!(
12224 outcome,
12225 WorkerPollOutcome::Stop {
12226 poll_status: Some("draining".to_string()),
12227 reason: Some("worker_draining".to_string()),
12228 }
12229 );
12230 }
12231
12232 assert!(client
12233 .poll_workflow_task("draining-worker", "rust-workers", Duration::ZERO)
12234 .await
12235 .expect("compatibility poll")
12236 .is_none());
12237 }
12238
12239 #[tokio::test]
12240 async fn managed_worker_honors_drain_stop_for_every_task_family() {
12241 let server = MockWorkerServer::draining_polls();
12242 let client = Client::builder(server.base_url())
12243 .timeout(Duration::from_secs(2))
12244 .build()
12245 .expect("client");
12246
12247 let mut workflow_worker = Worker::new(client.clone(), "rust-workers")
12248 .worker_id("draining-workflow-worker")
12249 .poll_timeout(Duration::ZERO);
12250 workflow_worker.register_workflow("counter", |_ctx, _args| async { Ok(Value::Null) });
12251 workflow_worker
12252 .run()
12253 .await
12254 .expect("workflow drain is a clean stop");
12255
12256 let mut activity_worker = Worker::new(client.clone(), "rust-workers")
12257 .worker_id("draining-activity-worker")
12258 .poll_timeout(Duration::ZERO);
12259 activity_worker.register_activity("write", |_ctx, _args| async { Ok(Value::Null) });
12260 activity_worker
12261 .run()
12262 .await
12263 .expect("activity drain is a clean stop");
12264
12265 let mut query_worker = Worker::new(client, "rust-workers")
12266 .worker_id("draining-query-worker")
12267 .poll_timeout(Duration::ZERO);
12268 query_worker.register_query("counter", "current", |_ctx, _args| async {
12269 Ok(Value::Null)
12270 });
12271 query_worker
12272 .run()
12273 .await
12274 .expect("query drain is a clean stop");
12275 }
12276
12277 #[tokio::test]
12278 async fn activity_cancellation_and_late_completion_remain_machine_readable() {
12279 let server = MockWorkerServer::start();
12280 let client = Client::builder(server.base_url())
12281 .timeout(Duration::from_secs(2))
12282 .build()
12283 .expect("client");
12284
12285 let heartbeat = client
12286 .heartbeat_activity_task(
12287 "activity-cancel",
12288 "attempt-cancel",
12289 "rust-worker",
12290 typed_fidelity_probe(),
12291 )
12292 .await
12293 .expect("cancellation heartbeat");
12294 assert!(heartbeat.cancel_requested);
12295 assert!(heartbeat.should_stop());
12296 assert_eq!(heartbeat.reason.as_deref(), Some("run_cancelled"));
12297 assert_eq!(heartbeat.run_closed_reason.as_deref(), Some("cancelled"));
12298 let heartbeat_body =
12299 server.request_body("/api/worker/activity-tasks/activity-cancel/heartbeat");
12300 assert_eq!(heartbeat_body["details"]["codec"], DEFAULT_CODEC);
12301 assert_eq!(
12302 decode_wire_avro_value(&heartbeat_body["details"], DEFAULT_CODEC)
12303 .expect("typed heartbeat details"),
12304 typed_fidelity_probe()
12305 );
12306
12307 let error = client
12308 .complete_activity_task(
12309 "activity-cancel",
12310 "attempt-cancel",
12311 "rust-worker",
12312 json!({"late":true}),
12313 JSON_CODEC,
12314 )
12315 .await
12316 .expect_err("late completion must be refused");
12317 assert!(activity_task_rejection_is_final(&error));
12318 let Error::ActivityTaskRejected(rejection) = error else {
12319 panic!("expected typed activity rejection");
12320 };
12321 assert_eq!(rejection.status, 409);
12322 assert_eq!(rejection.reason, "run_cancelled");
12323 assert!(rejection.cancel_requested);
12324 assert_eq!(rejection.can_continue, Some(false));
12325 }
12326
12327 #[tokio::test]
12328 async fn managed_worker_survives_late_completion_and_restart_during_cancellation() {
12329 let server = MockWorkerServer::cancelled_activity();
12330 let client = Client::builder(server.base_url())
12331 .timeout(Duration::from_secs(2))
12332 .build()
12333 .expect("client");
12334 let cancellation_observed = Arc::new(AtomicBool::new(false));
12335 let observed = Arc::clone(&cancellation_observed);
12336 let mut worker = Worker::new(client.clone(), "rust-workers")
12337 .worker_id("rust-cancel-worker")
12338 .poll_timeout(Duration::from_millis(10));
12339 worker.register_activity("cancel-aware", move |ctx, _args| {
12340 let observed = Arc::clone(&observed);
12341 async move {
12342 let heartbeat = ctx.heartbeat(json!({"stage":"running"})).await?;
12343 observed.store(heartbeat.should_stop(), Ordering::SeqCst);
12344 Ok(json!({"late":"completion"}))
12345 }
12346 });
12347
12348 assert_eq!(
12349 worker.run_once().await.expect("cancelled attempt handled"),
12350 1
12351 );
12352 assert!(cancellation_observed.load(Ordering::SeqCst));
12353 assert_eq!(
12354 server.request_count("/api/worker/activity-tasks/activity-cancel/complete"),
12355 1
12356 );
12357
12358 let mut restarted = Worker::new(client, "rust-workers")
12359 .worker_id("rust-cancel-worker-restarted")
12360 .poll_timeout(Duration::from_millis(10));
12361 restarted.register_activity("cancel-aware", |_ctx, _args| async move { Ok(Value::Null) });
12362 assert_eq!(
12363 restarted
12364 .run_once()
12365 .await
12366 .expect("replacement worker continues polling"),
12367 0
12368 );
12369 }
12370
12371 #[tokio::test]
12372 async fn managed_worker_absorbs_selected_run_terminal_timeout_completion_race() {
12373 let response = r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"outcome":"completed","recorded":false,"run_id":"run-selected-timeout","run_status":"failed","created_task_ids":[],"reason":"run_timed_out"}"#;
12374 let server = MockWorkerServer::workflow_completion("409 Conflict", response);
12375 let client = Client::builder(server.base_url())
12376 .timeout(Duration::from_secs(2))
12377 .build()
12378 .expect("client");
12379
12380 let direct_error = client
12381 .complete_workflow_task(
12382 "workflow-timeout-task",
12383 "timeout-worker",
12384 3,
12385 vec![json!({"type": "complete_workflow", "result": null})],
12386 )
12387 .await
12388 .expect_err("the low-level client preserves the completion rejection");
12389 let Error::Http { status, body } = direct_error else {
12390 panic!("expected the original HTTP completion rejection");
12391 };
12392 assert_eq!(status, reqwest::StatusCode::CONFLICT);
12393 assert_eq!(
12394 serde_json::from_str::<Value>(&body).expect("response body")["reason"],
12395 "run_timed_out"
12396 );
12397
12398 let mut worker = Worker::new(client, "rust-workers")
12399 .worker_id("timeout-worker")
12400 .poll_timeout(Duration::from_millis(10));
12401 worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
12402 Ok(json!({"late": "result"}))
12403 });
12404
12405 assert_eq!(
12406 worker
12407 .run_once()
12408 .await
12409 .expect("authoritative selected-run timeout settles the tick"),
12410 1
12411 );
12412 assert_eq!(
12413 server.request_count("/api/worker/workflow-tasks/workflow-timeout-task/complete"),
12414 2,
12415 "both the direct client proof and managed worker must see the rejection"
12416 );
12417 }
12418
12419 #[tokio::test]
12420 async fn managed_worker_does_not_swallow_nearby_completion_errors() {
12421 for (name, status, response) in [
12422 ("bare conflict", "409 Conflict", r#"{"message":"conflict"}"#),
12423 (
12424 "command was recorded",
12425 "409 Conflict",
12426 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":true,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12427 ),
12428 (
12429 "lease conflict",
12430 "409 Conflict",
12431 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"lease_expired"}"#,
12432 ),
12433 (
12434 "nonterminal run",
12435 "409 Conflict",
12436 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"waiting","reason":"run_timed_out"}"#,
12437 ),
12438 (
12439 "different selected run",
12440 "409 Conflict",
12441 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-reused-workflow-current","run_status":"failed","reason":"run_timed_out"}"#,
12442 ),
12443 (
12444 "different task attempt",
12445 "409 Conflict",
12446 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":4,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12447 ),
12448 (
12449 "authentication failure",
12450 "401 Unauthorized",
12451 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12452 ),
12453 (
12454 "authorization failure",
12455 "403 Forbidden",
12456 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12457 ),
12458 (
12459 "protocol failure",
12460 "400 Bad Request",
12461 r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.2","requested_version":"1.3"}"#,
12462 ),
12463 (
12464 "malformed command",
12465 "422 Unprocessable Entity",
12466 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12467 ),
12468 (
12469 "transient server failure",
12470 "503 Service Unavailable",
12471 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12472 ),
12473 ] {
12474 let server = MockWorkerServer::workflow_completion(status, response);
12475 let client = Client::builder(server.base_url())
12476 .timeout(Duration::from_secs(2))
12477 .build()
12478 .expect("client");
12479 let mut worker = Worker::new(client, "rust-workers")
12480 .worker_id("timeout-worker")
12481 .poll_timeout(Duration::from_millis(10));
12482 worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
12483 Ok(json!({"late": "result"}))
12484 });
12485
12486 let error = worker
12487 .run_once()
12488 .await
12489 .expect_err(&format!("{name} must remain an error"));
12490 assert!(
12491 matches!(error, Error::Http { .. } | Error::Protocol(_)),
12492 "{name} returned an unexpected error variant: {error}"
12493 );
12494 }
12495 }
12496
12497 #[tokio::test]
12498 async fn worker_deregistration_uses_worker_plane_method_path_headers_and_result() {
12499 let server = MockWorkerServer::start();
12500 let client = Client::builder(server.base_url())
12501 .worker_token(Some("worker-secret".to_string()))
12502 .namespace("orders")
12503 .timeout(Duration::from_secs(2))
12504 .build()
12505 .expect("client");
12506 let path = "/api/worker/registrations/worker%2F%CE%B1%20space";
12507
12508 let result = client
12509 .deregister_worker_registration("worker/α space")
12510 .await
12511 .expect("deregister worker registration");
12512
12513 assert_eq!(server.method_for(path).as_deref(), Some("DELETE"));
12514 assert_eq!(
12515 server.worker_protocol_for(path).as_deref(),
12516 Some(WORKER_PROTOCOL_VERSION)
12517 );
12518 assert_eq!(server.control_protocol_for(path), None);
12519 assert_eq!(server.namespace_for(path).as_deref(), Some("orders"));
12520 assert_eq!(
12521 server.authorization_for(path).as_deref(),
12522 Some("Bearer worker-secret")
12523 );
12524 assert_eq!(
12525 result,
12526 WorkerDeregistrationEnvelope {
12527 worker_id: "deregistered-worker".to_string(),
12528 outcome: "deregistered".to_string(),
12529 recovered_workflow_task_count: 2,
12530 }
12531 );
12532 }
12533
12534 #[tokio::test]
12535 async fn role_scoped_tokens_are_never_used_for_the_opposite_plane() {
12536 let server = MockWorkerServer::start();
12537 let control_only = Client::builder(server.base_url())
12538 .control_token(Some("control-secret".to_string()))
12539 .build()
12540 .expect("control client");
12541
12542 let error = control_only
12543 .register_worker("worker", "queue", vec![], vec![], 1, 1)
12544 .await
12545 .expect_err("control token must not authorize a worker request");
12546 assert!(matches!(
12547 error,
12548 Error::MissingRoleCredentials { role: "worker", .. }
12549 ));
12550 assert_eq!(server.request_count("/api/worker/register"), 0);
12551
12552 let worker_only = Client::builder(server.base_url())
12553 .worker_token(Some("worker-secret".to_string()))
12554 .build()
12555 .expect("worker client");
12556 let error = worker_only
12557 .health()
12558 .await
12559 .expect_err("worker token must not authorize a control request");
12560 assert!(matches!(
12561 error,
12562 Error::MissingRoleCredentials {
12563 role: "control",
12564 ..
12565 }
12566 ));
12567 assert_eq!(server.request_count("/api/health"), 0);
12568 }
12569
12570 #[tokio::test]
12571 async fn shared_token_supports_worker_and_control_planes() {
12572 let server = MockWorkerServer::start();
12573 let client = Client::builder(server.base_url())
12574 .token(Some("shared-secret".to_string()))
12575 .build()
12576 .expect("client");
12577
12578 client.health().await.expect("control request");
12579 client
12580 .register_worker("worker", "queue", vec![], vec![], 1, 1)
12581 .await
12582 .expect("worker request");
12583
12584 assert_eq!(
12585 server.authorization_for("/api/health").as_deref(),
12586 Some("Bearer shared-secret")
12587 );
12588 assert_eq!(
12589 server.control_protocol_for("/api/health").as_deref(),
12590 Some(CONTROL_PLANE_VERSION)
12591 );
12592 assert_eq!(
12593 server.authorization_for("/api/worker/register").as_deref(),
12594 Some("Bearer shared-secret")
12595 );
12596 assert_eq!(
12597 server
12598 .worker_protocol_for("/api/worker/register")
12599 .as_deref(),
12600 Some(WORKER_PROTOCOL_VERSION)
12601 );
12602 }
12603
12604 #[tokio::test]
12605 async fn baseline_worker_endpoints_send_the_baseline_protocol() {
12606 let server = MockWorkerServer::start();
12607 let client = Client::builder(server.base_url())
12608 .timeout(Duration::from_secs(2))
12609 .build()
12610 .expect("client");
12611
12612 client
12613 .register_worker("capture-worker", "capture", vec![], vec![], 1, 1)
12614 .await
12615 .expect("register");
12616 client
12617 .heartbeat_worker("capture-worker", 1, 1)
12618 .await
12619 .expect("heartbeat");
12620 client
12621 .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
12622 .await
12623 .expect("workflow poll");
12624 client
12625 .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
12626 .await
12627 .expect("activity poll");
12628
12629 for path in [
12630 "/api/worker/register",
12631 "/api/worker/heartbeat",
12632 "/api/worker/workflow-tasks/poll",
12633 "/api/worker/activity-tasks/poll",
12634 ] {
12635 assert_eq!(
12636 server.worker_protocol_for(path).as_deref(),
12637 Some(WORKER_PROTOCOL_VERSION),
12638 "unexpected protocol for {path}"
12639 );
12640 }
12641
12642 assert_eq!(
12643 server.request_body("/api/worker/workflow-tasks/poll")["timeout_seconds"],
12644 1
12645 );
12646 assert_eq!(
12647 server.request_body("/api/worker/activity-tasks/poll")["timeout_seconds"],
12648 1
12649 );
12650 assert!(
12651 server.request_body("/api/worker/workflow-tasks/poll")["poll_request_id"]
12652 .as_str()
12653 .is_some_and(|id| id.starts_with("rust-workflow-poll-"))
12654 );
12655 assert!(
12656 server.request_body("/api/worker/activity-tasks/poll")["poll_request_id"]
12657 .as_str()
12658 .is_some_and(|id| id.starts_with("rust-activity-poll-"))
12659 );
12660 }
12661
12662 #[tokio::test]
12663 async fn query_task_endpoints_send_the_query_feature_protocol() {
12664 let server = MockWorkerServer::start();
12665 let client = Client::builder(server.base_url())
12666 .timeout(Duration::from_secs(2))
12667 .build()
12668 .expect("client");
12669
12670 client
12671 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
12672 .await
12673 .expect("query poll");
12674 client
12675 .complete_query_task("query-capture", "capture-worker", 1, json!(8), JSON_CODEC)
12676 .await
12677 .expect("query complete");
12678 client
12679 .fail_query_task(
12680 "query-capture",
12681 "capture-worker",
12682 1,
12683 "failed",
12684 "query_rejected",
12685 "QueryFailed",
12686 )
12687 .await
12688 .expect("query fail");
12689
12690 for path in [
12691 "/api/worker/query-tasks/poll",
12692 "/api/worker/query-tasks/query-capture/complete",
12693 "/api/worker/query-tasks/query-capture/fail",
12694 ] {
12695 assert_eq!(
12696 server.worker_protocol_for(path).as_deref(),
12697 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
12698 "unexpected protocol for {path}"
12699 );
12700 }
12701
12702 assert_eq!(
12703 server.request_body("/api/worker/query-tasks/poll")["timeout_seconds"],
12704 1
12705 );
12706 assert!(
12707 server.request_body("/api/worker/query-tasks/poll")["poll_request_id"]
12708 .as_str()
12709 .is_some_and(|id| id.starts_with("rust-query-poll-"))
12710 );
12711 }
12712
12713 #[tokio::test]
12714 async fn disconnected_client_polls_retry_once_with_the_same_request_id() {
12715 let server = MockWorkerServer::transient_worker_failures();
12716 let client = Client::builder(server.base_url())
12717 .timeout(Duration::from_secs(2))
12718 .build()
12719 .expect("client");
12720
12721 client
12722 .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
12723 .await
12724 .expect("workflow poll retry");
12725 client
12726 .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
12727 .await
12728 .expect("activity poll retry");
12729 client
12730 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
12731 .await
12732 .expect("query poll retry");
12733
12734 for path in [
12735 "/api/worker/workflow-tasks/poll",
12736 "/api/worker/activity-tasks/poll",
12737 "/api/worker/query-tasks/poll",
12738 ] {
12739 let bodies = server.request_bodies(path);
12740 assert_eq!(bodies.len(), 2, "{path} must be retried once");
12741 assert_eq!(
12742 bodies[0]["poll_request_id"], bodies[1]["poll_request_id"],
12743 "{path} must preserve the request binding across retry"
12744 );
12745 }
12746 }
12747
12748 #[tokio::test]
12749 async fn worker_poll_retries_preserve_request_id_across_consecutive_failures() {
12750 let server = MockWorkerServer::consecutive_poll_failures(2);
12751 let client = Client::builder(server.base_url())
12752 .timeout(Duration::from_secs(2))
12753 .build()
12754 .expect("client");
12755 let mut worker = Worker::new(client, "capture")
12756 .worker_id("capture-worker")
12757 .poll_timeout(Duration::from_millis(10))
12758 .retry_policy(WorkerRetryPolicy {
12759 max_retries: 2,
12760 initial_backoff: Duration::from_millis(1),
12761 max_backoff: Duration::from_millis(1),
12762 });
12763 worker.register_workflow(
12764 "capture.workflow",
12765 |_ctx, _input| async move { Ok(Value::Null) },
12766 );
12767 worker.register_activity(
12768 "capture.activity",
12769 |_ctx, _input| async move { Ok(Value::Null) },
12770 );
12771 worker.register_query("capture.workflow", "current", |_ctx, _args| async move {
12772 Ok(Value::Null)
12773 });
12774
12775 assert_eq!(worker.run_once().await.expect("poll retries"), 0);
12776
12777 for path in [
12778 "/api/worker/workflow-tasks/poll",
12779 "/api/worker/activity-tasks/poll",
12780 "/api/worker/query-tasks/poll",
12781 ] {
12782 let bodies = server.request_bodies(path);
12783 assert_eq!(bodies.len(), 3, "{path} must use exactly two retries");
12784 assert!(
12785 bodies
12786 .iter()
12787 .all(|body| body["poll_request_id"] == bodies[0]["poll_request_id"]),
12788 "{path} must preserve one request binding across every retry"
12789 );
12790 }
12791 }
12792
12793 #[tokio::test]
12794 async fn query_protocol_rejection_from_older_server_is_typed() {
12795 let server = MockWorkerServer::reject_query_protocol();
12796 let client = Client::builder(server.base_url())
12797 .timeout(Duration::from_secs(2))
12798 .build()
12799 .expect("client");
12800
12801 let error = client
12802 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
12803 .await
12804 .expect_err("server below query protocol floor must reject");
12805 let Error::Protocol(failure) = error else {
12806 panic!("expected typed protocol failure");
12807 };
12808
12809 assert_eq!(failure.status, 400);
12810 assert_eq!(failure.reason, "unsupported_protocol_version");
12811 assert_eq!(failure.supported_version.as_deref(), Some("1.7"));
12812 assert_eq!(
12813 failure.requested_version.as_deref(),
12814 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
12815 );
12816 assert_eq!(
12817 server
12818 .worker_protocol_for("/api/worker/query-tasks/poll")
12819 .as_deref(),
12820 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
12821 );
12822 }
12823
12824 #[tokio::test]
12825 async fn run_once_without_query_handlers_keeps_pre_query_server_compatibility() {
12826 let server = MockWorkerServer::reject_query_protocol();
12827 let client = Client::builder(server.base_url())
12828 .timeout(Duration::from_secs(2))
12829 .build()
12830 .expect("client");
12831 let mut worker = Worker::new(client, "rust-workers")
12832 .worker_id("baseline-worker")
12833 .poll_timeout(Duration::from_millis(10));
12834
12835 worker.register_workflow("baseline.workflow", |_ctx, _input| async move {
12836 Ok(Value::Null)
12837 });
12838
12839 assert_eq!(worker.run_once().await.expect("baseline run once"), 0);
12840 assert_eq!(
12841 server
12842 .worker_protocol_for("/api/worker/workflow-tasks/poll")
12843 .as_deref(),
12844 Some(WORKER_PROTOCOL_VERSION)
12845 );
12846 assert_eq!(
12847 server.worker_protocol_for("/api/worker/query-tasks/poll"),
12848 None,
12849 "a worker without query handlers must not use the query-task endpoint"
12850 );
12851 }
12852
12853 #[tokio::test]
12854 async fn completion_time_query_rejection_is_typed_without_stopping_worker() {
12855 let server = MockWorkerServer::reject_query_completion();
12856 let client = Client::builder(server.base_url())
12857 .timeout(Duration::from_secs(2))
12858 .build()
12859 .expect("client");
12860
12861 let error = client
12862 .complete_query_task("query-late", "late-worker", 1, json!(8), JSON_CODEC)
12863 .await
12864 .expect_err("expired completion must be rejected");
12865 let Error::QueryFailed(failure) = error else {
12866 panic!("expected typed query failure");
12867 };
12868 assert_eq!(failure.status, 409);
12869 assert_eq!(failure.reason, "query_task_timed_out");
12870
12871 let mut worker = Worker::new(client, "rust-workers")
12872 .worker_id("late-worker")
12873 .poll_timeout(Duration::from_millis(10));
12874 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
12875 worker.register_query(
12876 "counter",
12877 "current",
12878 |_ctx, _args| async move { Ok(json!(8)) },
12879 );
12880
12881 assert_eq!(worker.run_once().await.expect("late task is handled"), 1);
12882 assert_eq!(
12883 worker
12884 .run_once()
12885 .await
12886 .expect("worker continues after late completion"),
12887 0
12888 );
12889 assert_eq!(
12890 server.request_count("/api/worker/query-tasks/query-late/complete"),
12891 2
12892 );
12893 assert_eq!(
12894 server.request_count("/api/worker/query-tasks/query-late/fail"),
12895 0,
12896 "a server completion rejection must not be reported as an encoding failure"
12897 );
12898 }
12899
12900 #[tokio::test]
12901 async fn normal_shutdown_joins_pollers_and_deregisters_once() {
12902 let server = MockWorkerServer::start();
12903 let client = Client::builder(server.base_url())
12904 .timeout(Duration::from_secs(2))
12905 .build()
12906 .expect("client");
12907 let mut worker = Worker::new(client, "rust-workers")
12908 .worker_id("joined-worker")
12909 .poll_timeout(Duration::from_millis(10));
12910 worker.register_workflow(
12911 "joined.workflow",
12912 |_ctx, _input| async move { Ok(Value::Null) },
12913 );
12914 worker.register_activity(
12915 "joined.activity",
12916 |_ctx, _input| async move { Ok(Value::Null) },
12917 );
12918 worker.register_query("joined.workflow", "state", |_ctx, _input| async move {
12919 Ok(Value::Null)
12920 });
12921
12922 worker
12923 .run_until(tokio::time::sleep(Duration::from_millis(20)))
12924 .await
12925 .expect("normal shutdown");
12926
12927 let deregistration_path = "/api/worker/registrations/mock-worker";
12928 assert_eq!(server.request_count(deregistration_path), 1);
12929 for poll_path in [
12930 "/api/worker/workflow-tasks/poll",
12931 "/api/worker/activity-tasks/poll",
12932 "/api/worker/query-tasks/poll",
12933 ] {
12934 assert!(server.request_count(poll_path) > 0, "missing {poll_path}");
12935 }
12936 assert_eq!(
12937 server.captured_paths().last().map(String::as_str),
12938 Some(deregistration_path),
12939 "deregistration must start only after every poller has joined"
12940 );
12941 }
12942
12943 #[tokio::test]
12944 async fn registration_failure_does_not_deregister() {
12945 let server = MockWorkerServer::rejected_registration();
12946 let client = Client::builder(server.base_url())
12947 .timeout(Duration::from_secs(2))
12948 .build()
12949 .expect("client");
12950 let worker = Worker::new(client, "rust-workers").worker_id("never-registered");
12951
12952 let error = worker
12953 .run_until(async {})
12954 .await
12955 .expect_err("registration must fail");
12956 assert!(matches!(
12957 error,
12958 Error::Http {
12959 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
12960 ..
12961 }
12962 ));
12963 assert!(server
12964 .captured_paths()
12965 .iter()
12966 .all(|path| !path.starts_with("/api/worker/registrations/")));
12967 }
12968
12969 #[tokio::test]
12970 async fn declined_registration_does_not_deregister() {
12971 let server = MockWorkerServer::declined_registration();
12972 let client = Client::builder(server.base_url())
12973 .timeout(Duration::from_secs(2))
12974 .build()
12975 .expect("client");
12976 let worker = Worker::new(client, "rust-workers").worker_id("declined-worker");
12977
12978 let error = worker
12979 .run_until(async {})
12980 .await
12981 .expect_err("declined registration must fail");
12982 assert!(matches!(error, Error::WorkerLoop(_)));
12983 assert!(error.to_string().contains("was not accepted"));
12984 assert!(server
12985 .captured_paths()
12986 .iter()
12987 .all(|path| !path.starts_with("/api/worker/registrations/")));
12988 }
12989
12990 #[tokio::test]
12991 async fn deregistration_http_failure_is_returned_after_normal_shutdown() {
12992 let server = MockWorkerServer::rejected_deregistration();
12993 let client = Client::builder(server.base_url())
12994 .timeout(Duration::from_secs(2))
12995 .build()
12996 .expect("client");
12997 let worker = Worker::new(client, "rust-workers").worker_id("forbidden-cleanup");
12998
12999 let error = worker
13000 .run_until(async {})
13001 .await
13002 .expect_err("deregistration must fail");
13003 assert!(matches!(
13004 error,
13005 Error::Http {
13006 status: reqwest::StatusCode::FORBIDDEN,
13007 ..
13008 }
13009 ));
13010 assert_eq!(
13011 server.request_count("/api/worker/registrations/mock-worker"),
13012 1
13013 );
13014 }
13015
13016 #[tokio::test]
13017 async fn deregistration_protocol_failure_is_returned_after_normal_shutdown() {
13018 let server = MockWorkerServer::rejected_deregistration_protocol();
13019 let client = Client::builder(server.base_url())
13020 .timeout(Duration::from_secs(2))
13021 .build()
13022 .expect("client");
13023 let worker = Worker::new(client, "rust-workers").worker_id("protocol-cleanup");
13024
13025 let error = worker
13026 .run_until(async {})
13027 .await
13028 .expect_err("protocol rejection must fail shutdown");
13029 let Error::Protocol(failure) = error else {
13030 panic!("expected typed protocol failure");
13031 };
13032 assert_eq!(failure.reason, "unsupported_protocol_version");
13033 assert_eq!(failure.requested_version.as_deref(), Some("1.2"));
13034 assert_eq!(
13035 server.request_count("/api/worker/registrations/mock-worker"),
13036 1
13037 );
13038 }
13039
13040 #[tokio::test]
13041 async fn primary_poller_error_retains_deregistration_failure_context() {
13042 let server = MockWorkerServer::unauthorized_polls_and_rejected_deregistration();
13043 let client = Client::builder(server.base_url())
13044 .timeout(Duration::from_secs(2))
13045 .build()
13046 .expect("client");
13047 let mut worker = Worker::new(client, "rust-workers")
13048 .worker_id("combined-failure")
13049 .poll_timeout(Duration::from_millis(10));
13050 worker.register_workflow("combined.workflow", |_ctx, _input| async move {
13051 Ok(Value::Null)
13052 });
13053
13054 let error = worker
13055 .run()
13056 .await
13057 .expect_err("worker and cleanup must fail");
13058 let summary = error.to_string();
13059 assert!(summary.contains("authentication_failed"));
13060 assert!(summary.contains("worker cannot deregister"));
13061 let Error::WorkerShutdown {
13062 primary,
13063 deregistration,
13064 } = error
13065 else {
13066 panic!("expected combined worker shutdown error");
13067 };
13068 assert!(matches!(
13069 *primary,
13070 Error::Http {
13071 status: reqwest::StatusCode::UNAUTHORIZED,
13072 ..
13073 }
13074 ));
13075 assert!(matches!(
13076 *deregistration,
13077 Error::Http {
13078 status: reqwest::StatusCode::FORBIDDEN,
13079 ..
13080 }
13081 ));
13082 assert_eq!(
13083 server.request_count("/api/worker/registrations/mock-worker"),
13084 1
13085 );
13086 }
13087
13088 #[tokio::test]
13089 async fn activity_only_worker_can_shutdown_without_workflow_poller() {
13090 let server = MockWorkerServer::start();
13091 let client = Client::builder(server.base_url())
13092 .timeout(Duration::from_secs(2))
13093 .build()
13094 .expect("client");
13095 let mut worker = Worker::new(client, "rust-workers")
13096 .worker_id("activity-only-worker")
13097 .poll_timeout(Duration::from_millis(10));
13098
13099 worker.register_activity(
13100 "activity.only",
13101 |_ctx, _args| async move { Ok(Value::Null) },
13102 );
13103
13104 worker.run_until(async {}).await.expect("run worker");
13105 }
13106
13107 #[tokio::test]
13108 async fn workflow_only_worker_can_shutdown_without_activity_poller() {
13109 let server = MockWorkerServer::start();
13110 let client = Client::builder(server.base_url())
13111 .timeout(Duration::from_secs(2))
13112 .build()
13113 .expect("client");
13114 let mut worker = Worker::new(client, "rust-workers")
13115 .worker_id("workflow-only-worker")
13116 .poll_timeout(Duration::from_millis(10));
13117
13118 worker.register_workflow(
13119 "workflow.only",
13120 |_ctx, _input| async move { Ok(Value::Null) },
13121 );
13122
13123 worker.run_until(async {}).await.expect("run worker");
13124 }
13125
13126 #[tokio::test]
13127 async fn worker_heartbeat_observer_receives_server_acknowledgements() {
13128 let server = MockWorkerServer::start();
13129 let client = Client::builder(server.base_url())
13130 .timeout(Duration::from_secs(2))
13131 .build()
13132 .expect("client");
13133 let observations = Arc::new(Mutex::new(Vec::new()));
13134 let observed = Arc::clone(&observations);
13135 let mut worker = Worker::new(client, "rust-workers")
13136 .worker_id("observed-heartbeat-worker")
13137 .poll_timeout(Duration::from_millis(10))
13138 .on_worker_heartbeat(move |observation| {
13139 observed
13140 .lock()
13141 .expect("heartbeat observations")
13142 .push(observation.clone());
13143 });
13144
13145 worker.register_workflow("workflow.observed", |_ctx, _input| async move {
13146 Ok(Value::Null)
13147 });
13148 let acknowledged = Arc::clone(&observations);
13149 worker
13150 .run_until(async move {
13151 tokio::time::timeout(Duration::from_secs(2), async move {
13152 loop {
13153 if !acknowledged
13154 .lock()
13155 .expect("heartbeat observations")
13156 .is_empty()
13157 {
13158 break;
13159 }
13160 tokio::time::sleep(Duration::from_millis(1)).await;
13161 }
13162 })
13163 .await
13164 .expect("heartbeat acknowledgement within timeout");
13165 })
13166 .await
13167 .expect("run worker");
13168
13169 let observations = observations.lock().expect("heartbeat observations");
13170 let first = observations.first().expect("heartbeat acknowledgement");
13171 assert_eq!(first.worker_id, "observed-heartbeat-worker");
13172 assert_eq!(first.task_queue, "rust-workers");
13173 assert!(first.acknowledged_at_unix_millis > 0);
13174 assert_eq!(first.acknowledgement, json!({}));
13175 }
13176
13177 #[tokio::test]
13178 async fn delayed_worker_heartbeat_keeps_cadence_and_pollers_live() {
13179 let server = MockWorkerServer::delayed_heartbeat_worker();
13180 let client = Client::builder(server.base_url())
13181 .timeout(Duration::from_secs(3))
13182 .build()
13183 .expect("client");
13184 let observations = Arc::new(Mutex::new(Vec::new()));
13185 let observed = Arc::clone(&observations);
13186 let mut worker = Worker::new(client, "rust-snapshot-workers")
13187 .worker_id("rust-snapshot-worker")
13188 .poll_timeout(Duration::from_millis(10))
13189 .on_worker_heartbeat(move |observation| {
13190 observed
13191 .lock()
13192 .expect("heartbeat observations")
13193 .push(observation.clone());
13194 });
13195
13196 worker.register_workflow("snapshot", |ctx, _input| async move {
13197 ctx.wait_signal("finish").await?;
13198 Ok(json!({"status": "finished"}))
13199 });
13200 worker.register_query("snapshot", "current", |ctx, _args| async move {
13201 Ok(json!(ctx
13202 .signals("increment")
13203 .iter()
13204 .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
13205 .sum::<i64>()))
13206 });
13207 worker.register_activity("cancel-aware", |_ctx, _args| async move {
13208 Ok(json!({"late": "completion"}))
13209 });
13210
13211 worker
13212 .run_until(tokio::time::sleep(Duration::from_millis(3_800)))
13213 .await
13214 .expect("delayed heartbeat must allow a clean worker shutdown");
13215
13216 let observations = observations.lock().expect("heartbeat observations");
13217 assert!(
13218 observations.len() >= 3,
13219 "the immediate heartbeat, delayed acknowledgement, and next cadence heartbeat must complete"
13220 );
13221 assert!(
13222 observations.windows(2).all(|pair| {
13223 pair[1].acknowledged_at_unix_millis
13224 .saturating_sub(pair[0].acknowledged_at_unix_millis)
13225 >= 850
13226 }),
13227 "successful acknowledgements must not catch up faster than the advertised one-second cadence: {observations:?}"
13228 );
13229 drop(observations);
13230
13231 let heartbeat_times = server.request_times("/api/worker/heartbeat");
13232 let delayed_request_at = *heartbeat_times
13233 .get(1)
13234 .expect("intentionally delayed heartbeat request");
13235 let delay_window_start = delayed_request_at + Duration::from_millis(100);
13236 let delay_window_end = delayed_request_at + Duration::from_millis(1_400);
13237 for path in [
13238 "/api/worker/workflow-tasks/poll",
13239 "/api/worker/activity-tasks/poll",
13240 "/api/worker/query-tasks/poll",
13241 ] {
13242 assert!(
13243 server
13244 .request_times(path)
13245 .iter()
13246 .any(|received_at| *received_at >= delay_window_start
13247 && *received_at <= delay_window_end),
13248 "{path} must keep polling while a heartbeat acknowledgement is delayed"
13249 );
13250 }
13251 assert!(
13252 server.request_count("/api/worker/workflow-tasks/snapshot-wait-3/fail") >= 1,
13253 "workflow work must be settled"
13254 );
13255 assert!(
13256 server.request_count("/api/worker/activity-tasks/activity-cancel/complete") >= 1,
13257 "activity work must be settled"
13258 );
13259 assert!(
13260 server.request_count("/api/worker/query-tasks/snapshot-current/complete") >= 1,
13261 "query work must be settled"
13262 );
13263 }
13264
13265 #[tokio::test]
13266 async fn retried_worker_heartbeat_restarts_the_advertised_cadence() {
13267 let server = MockWorkerServer::heartbeat_retry_worker();
13268 let client = Client::builder(server.base_url())
13269 .timeout(Duration::from_secs(2))
13270 .build()
13271 .expect("client");
13272 let observations = Arc::new(Mutex::new(Vec::new()));
13273 let observed = Arc::clone(&observations);
13274 let worker = Worker::new(client, "rust-workers")
13275 .worker_id("heartbeat-retry-worker")
13276 .retry_policy(WorkerRetryPolicy {
13277 max_retries: 1,
13278 initial_backoff: Duration::from_millis(300),
13279 max_backoff: Duration::from_millis(300),
13280 })
13281 .on_worker_heartbeat(move |observation| {
13282 observed
13283 .lock()
13284 .expect("heartbeat observations")
13285 .push(observation.clone());
13286 });
13287
13288 worker
13289 .run_until(tokio::time::sleep(Duration::from_millis(2_700)))
13290 .await
13291 .expect("retryable heartbeat failure must remain bounded and recover");
13292
13293 let observations = observations.lock().expect("heartbeat observations");
13294 assert!(observations.len() >= 3, "heartbeat retry must recover");
13295 assert!(
13296 observations.windows(2).all(|pair| {
13297 pair[1]
13298 .acknowledged_at_unix_millis
13299 .saturating_sub(pair[0].acknowledged_at_unix_millis)
13300 >= 850
13301 }),
13302 "a successful retry must start a fresh advertised cadence: {observations:?}"
13303 );
13304 assert_eq!(
13305 server.request_count("/api/worker/heartbeat"),
13306 observations.len() + 1,
13307 "one retryable failure must add exactly one bounded request"
13308 );
13309 }
13310
13311 #[tokio::test]
13312 async fn query_enabled_worker_ignores_unmatched_signals_then_completes_once() {
13313 let server = MockWorkerServer::waiting_query_worker();
13314 let client = Client::builder(server.base_url())
13315 .timeout(Duration::from_secs(2))
13316 .build()
13317 .expect("client");
13318 let observations = Arc::new(Mutex::new(Vec::new()));
13319 let observed = Arc::clone(&observations);
13320 let mut worker = Worker::new(client, "rust-snapshot-workers")
13321 .worker_id("rust-snapshot-worker")
13322 .poll_timeout(Duration::from_millis(10))
13323 .on_worker_heartbeat(move |observation| {
13324 observed
13325 .lock()
13326 .expect("heartbeat observations")
13327 .push(observation.clone());
13328 });
13329
13330 worker.register_workflow("snapshot", |ctx, _input| async move {
13331 ctx.wait_signal("finish").await?;
13332 Ok(json!({"status": "finished"}))
13333 });
13334 worker.register_query("snapshot", "current", |ctx, _args| async move {
13335 let current = ctx
13336 .signals("increment")
13337 .iter()
13338 .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
13339 .sum::<i64>();
13340 Ok(json!(current))
13341 });
13342 worker.register_update("snapshot", "replace", |_ctx, args| async move { Ok(args) });
13343
13344 worker
13345 .run_until(tokio::time::sleep(Duration::from_millis(3_200)))
13346 .await
13347 .expect("pending workflow and query poller must remain live until shutdown");
13348
13349 assert!(
13350 observations.lock().expect("heartbeat observations").len() >= 4,
13351 "the immediate heartbeat and at least three advertised one-second intervals must be acknowledged"
13352 );
13353 assert!(
13354 server.request_count("/api/worker/workflow-tasks/poll") >= 3,
13355 "workflow polling must continue after empty replay acknowledgements"
13356 );
13357 assert!(
13358 server.request_count("/api/worker/query-tasks/poll") >= 2,
13359 "query polling must continue after serving the current query"
13360 );
13361 assert_eq!(
13362 server.request_body("/api/worker/register")["capabilities"],
13363 json!([QUERY_TASKS_CAPABILITY, WORKFLOW_UPDATES_CAPABILITY])
13364 );
13365 assert_eq!(
13366 server.request_body("/api/worker/register")["workflow_command_contracts"]["snapshot"],
13367 json!({
13368 "queries": ["current"],
13369 "updates": ["replace"],
13370 "update_validators": [],
13371 })
13372 );
13373
13374 let opened = server.request_body("/api/worker/workflow-tasks/snapshot-open/complete");
13375 assert_eq!(
13376 opened["commands"],
13377 json!([{
13378 "type": "open_signal_wait",
13379 "signal_name": "finish",
13380 }])
13381 );
13382
13383 for task_id in ["snapshot-wait-3", "snapshot-wait-5"] {
13384 let fail_path = format!("/api/worker/workflow-tasks/{task_id}/fail");
13385 let completion_path = format!("/api/worker/workflow-tasks/{task_id}/complete");
13386 let failure = server.request_body(&fail_path);
13387 assert_eq!(
13388 failure["failure"]["type"],
13389 WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE
13390 );
13391 assert_eq!(server.request_count(&completion_path), 0);
13392 }
13393
13394 let query_completion =
13395 server.request_body("/api/worker/query-tasks/snapshot-current/complete");
13396 assert_eq!(query_completion["result"], json!(8));
13397
13398 let terminal_path = "/api/worker/workflow-tasks/snapshot-finish/complete";
13399 assert_eq!(
13400 server.request_count(terminal_path),
13401 1,
13402 "the matching signal must settle the workflow exactly once"
13403 );
13404 let terminal = server.request_body(terminal_path);
13405 assert_eq!(terminal["commands"].as_array().map(Vec::len), Some(1));
13406 assert_eq!(terminal["commands"][0]["type"], "complete_workflow");
13407 assert_eq!(
13408 decode_wire_value(&terminal["commands"][0]["result"], DEFAULT_CODEC)
13409 .expect("terminal workflow result"),
13410 json!({"status": "finished"})
13411 );
13412 }
13413
13414 #[tokio::test]
13415 async fn worker_retries_poll_and_heartbeat_transport_failures_independently() {
13416 let server = MockWorkerServer::transient_worker_failures();
13417 let client = Client::builder(server.base_url())
13418 .timeout(Duration::from_secs(2))
13419 .build()
13420 .expect("client");
13421 let mut worker = Worker::new(client, "rust-workers")
13422 .worker_id("retry-worker")
13423 .poll_timeout(Duration::from_millis(10))
13424 .retry_policy(WorkerRetryPolicy {
13425 max_retries: 2,
13426 initial_backoff: Duration::from_millis(1),
13427 max_backoff: Duration::from_millis(1),
13428 });
13429 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
13430 worker.register_activity(
13431 "counter.activity",
13432 |_ctx, _input| async move { Ok(Value::Null) },
13433 );
13434 worker.register_query(
13435 "counter",
13436 "current",
13437 |_ctx, _args| async move { Ok(json!(8)) },
13438 );
13439
13440 worker
13441 .run_until(tokio::time::sleep(Duration::from_millis(75)))
13442 .await
13443 .expect("transient failures must not stop the worker");
13444
13445 for path in [
13446 "/api/worker/heartbeat",
13447 "/api/worker/workflow-tasks/poll",
13448 "/api/worker/activity-tasks/poll",
13449 "/api/worker/query-tasks/poll",
13450 ] {
13451 assert!(
13452 server.request_count(path) >= 2,
13453 "{path} must continue after its transient failure"
13454 );
13455 }
13456 }
13457
13458 #[tokio::test]
13459 async fn worker_bounds_transport_retries() {
13460 let server = MockWorkerServer::unavailable_polls();
13461 let client = Client::builder(server.base_url())
13462 .timeout(Duration::from_secs(2))
13463 .build()
13464 .expect("client");
13465 let mut worker = Worker::new(client, "rust-workers")
13466 .worker_id("bounded-retry-worker")
13467 .poll_timeout(Duration::from_millis(10))
13468 .retry_policy(WorkerRetryPolicy {
13469 max_retries: 2,
13470 initial_backoff: Duration::from_millis(1),
13471 max_backoff: Duration::from_millis(1),
13472 });
13473 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
13474
13475 let error = worker.run().await.expect_err("retry bound must terminate");
13476 assert!(matches!(error, Error::Transport(_)));
13477 assert_eq!(
13478 server.request_count("/api/worker/workflow-tasks/poll"),
13479 3,
13480 "one initial request plus exactly two retries"
13481 );
13482 }
13483
13484 #[tokio::test]
13485 async fn worker_retry_policy_can_disable_poll_retries() {
13486 let server = MockWorkerServer::unavailable_polls();
13487 let client = Client::builder(server.base_url())
13488 .timeout(Duration::from_secs(2))
13489 .build()
13490 .expect("client");
13491 let mut worker = Worker::new(client, "rust-workers")
13492 .worker_id("no-retry-worker")
13493 .poll_timeout(Duration::from_millis(10))
13494 .retry_policy(WorkerRetryPolicy {
13495 max_retries: 0,
13496 initial_backoff: Duration::from_millis(1),
13497 max_backoff: Duration::from_millis(1),
13498 });
13499 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
13500
13501 let error = worker
13502 .run_once()
13503 .await
13504 .expect_err("disabled retries must return the first transport failure");
13505 assert!(matches!(error, Error::Transport(_)));
13506 assert_eq!(
13507 server.request_count("/api/worker/workflow-tasks/poll"),
13508 1,
13509 "max_retries=0 must send only the initial request"
13510 );
13511 }
13512
13513 #[tokio::test]
13514 async fn worker_does_not_retry_authentication_failures() {
13515 let server = MockWorkerServer::unauthorized_polls();
13516 let client = Client::builder(server.base_url())
13517 .timeout(Duration::from_secs(2))
13518 .build()
13519 .expect("client");
13520 let mut worker = Worker::new(client, "rust-workers")
13521 .worker_id("unauthorized-worker")
13522 .poll_timeout(Duration::from_millis(10));
13523 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
13524
13525 let error = worker
13526 .run()
13527 .await
13528 .expect_err("authentication must terminate");
13529 let Error::Http { status, body } = error else {
13530 panic!("expected stable HTTP authentication error");
13531 };
13532 assert_eq!(status, reqwest::StatusCode::UNAUTHORIZED);
13533 assert!(body.contains("authentication_failed"));
13534 assert_eq!(
13535 server.request_count("/api/worker/workflow-tasks/poll"),
13536 1,
13537 "authentication failures must not be retried"
13538 );
13539 }
13540
13541 #[derive(Clone, Debug)]
13542 struct CapturedRequest {
13543 method: String,
13544 path: String,
13545 authorization: Option<String>,
13546 namespace: Option<String>,
13547 worker_protocol: Option<String>,
13548 control_protocol: Option<String>,
13549 body: String,
13550 received_at: Instant,
13551 }
13552
13553 struct MockWorkerServer {
13554 addr: SocketAddr,
13555 stop: Arc<AtomicBool>,
13556 requests: Arc<Mutex<Vec<CapturedRequest>>>,
13557 thread: Option<thread::JoinHandle<()>>,
13558 }
13559
13560 #[derive(Clone, Copy, Default)]
13561 struct MockWorkerBehavior {
13562 reject_query_protocol: bool,
13563 reject_query_completion: bool,
13564 waiting_query_worker: bool,
13565 decline_registration: bool,
13566 complete_named_signal: bool,
13567 poll_failures_per_path: usize,
13568 heartbeat_failures: usize,
13569 heartbeat_failure_request: Option<usize>,
13570 delayed_heartbeat_request: Option<usize>,
13571 heartbeat_response_delay: Duration,
13572 concurrent_requests: bool,
13573 unauthorized_polls: bool,
13574 reject_registration: bool,
13575 reject_deregistration: bool,
13576 reject_deregistration_protocol: bool,
13577 cancelled_activity: bool,
13578 draining_polls: bool,
13579 workflow_completion_status: Option<&'static str>,
13580 workflow_completion_body: Option<&'static str>,
13581 }
13582
13583 impl MockWorkerServer {
13584 fn start() -> Self {
13585 Self::start_with_behavior(MockWorkerBehavior::default())
13586 }
13587
13588 fn reject_query_protocol() -> Self {
13589 Self::start_with_behavior(MockWorkerBehavior {
13590 reject_query_protocol: true,
13591 ..MockWorkerBehavior::default()
13592 })
13593 }
13594
13595 fn reject_query_completion() -> Self {
13596 Self::start_with_behavior(MockWorkerBehavior {
13597 reject_query_completion: true,
13598 ..MockWorkerBehavior::default()
13599 })
13600 }
13601
13602 fn waiting_query_worker() -> Self {
13603 Self::start_with_behavior(MockWorkerBehavior {
13604 waiting_query_worker: true,
13605 complete_named_signal: true,
13606 ..MockWorkerBehavior::default()
13607 })
13608 }
13609
13610 fn transient_worker_failures() -> Self {
13611 Self::start_with_behavior(MockWorkerBehavior {
13612 poll_failures_per_path: 1,
13613 heartbeat_failures: 1,
13614 ..MockWorkerBehavior::default()
13615 })
13616 }
13617
13618 fn consecutive_poll_failures(count: usize) -> Self {
13619 Self::start_with_behavior(MockWorkerBehavior {
13620 poll_failures_per_path: count,
13621 ..MockWorkerBehavior::default()
13622 })
13623 }
13624
13625 fn delayed_heartbeat_worker() -> Self {
13626 Self::start_with_behavior(MockWorkerBehavior {
13627 waiting_query_worker: true,
13628 delayed_heartbeat_request: Some(2),
13629 heartbeat_response_delay: Duration::from_millis(1_500),
13630 concurrent_requests: true,
13631 cancelled_activity: true,
13632 ..MockWorkerBehavior::default()
13633 })
13634 }
13635
13636 fn heartbeat_retry_worker() -> Self {
13637 Self::start_with_behavior(MockWorkerBehavior {
13638 waiting_query_worker: true,
13639 heartbeat_failure_request: Some(2),
13640 concurrent_requests: true,
13641 ..MockWorkerBehavior::default()
13642 })
13643 }
13644
13645 fn unavailable_polls() -> Self {
13646 Self::start_with_behavior(MockWorkerBehavior {
13647 poll_failures_per_path: usize::MAX,
13648 ..MockWorkerBehavior::default()
13649 })
13650 }
13651
13652 fn unauthorized_polls() -> Self {
13653 Self::start_with_behavior(MockWorkerBehavior {
13654 unauthorized_polls: true,
13655 ..MockWorkerBehavior::default()
13656 })
13657 }
13658
13659 fn rejected_registration() -> Self {
13660 Self::start_with_behavior(MockWorkerBehavior {
13661 reject_registration: true,
13662 ..MockWorkerBehavior::default()
13663 })
13664 }
13665
13666 fn declined_registration() -> Self {
13667 Self::start_with_behavior(MockWorkerBehavior {
13668 decline_registration: true,
13669 ..MockWorkerBehavior::default()
13670 })
13671 }
13672
13673 fn rejected_deregistration() -> Self {
13674 Self::start_with_behavior(MockWorkerBehavior {
13675 reject_deregistration: true,
13676 ..MockWorkerBehavior::default()
13677 })
13678 }
13679
13680 fn rejected_deregistration_protocol() -> Self {
13681 Self::start_with_behavior(MockWorkerBehavior {
13682 reject_deregistration_protocol: true,
13683 ..MockWorkerBehavior::default()
13684 })
13685 }
13686
13687 fn unauthorized_polls_and_rejected_deregistration() -> Self {
13688 Self::start_with_behavior(MockWorkerBehavior {
13689 unauthorized_polls: true,
13690 reject_deregistration: true,
13691 ..MockWorkerBehavior::default()
13692 })
13693 }
13694
13695 fn cancelled_activity() -> Self {
13696 Self::start_with_behavior(MockWorkerBehavior {
13697 cancelled_activity: true,
13698 ..MockWorkerBehavior::default()
13699 })
13700 }
13701
13702 fn draining_polls() -> Self {
13703 Self::start_with_behavior(MockWorkerBehavior {
13704 draining_polls: true,
13705 ..MockWorkerBehavior::default()
13706 })
13707 }
13708
13709 fn workflow_completion(status: &'static str, body: &'static str) -> Self {
13710 Self::start_with_behavior(MockWorkerBehavior {
13711 workflow_completion_status: Some(status),
13712 workflow_completion_body: Some(body),
13713 ..MockWorkerBehavior::default()
13714 })
13715 }
13716
13717 fn start_with_behavior(behavior: MockWorkerBehavior) -> Self {
13718 let listener = TcpListener::bind("127.0.0.1:0").expect("bind mock server");
13719 listener
13720 .set_nonblocking(true)
13721 .expect("configure mock listener");
13722 let addr = listener.local_addr().expect("mock server address");
13723 let stop = Arc::new(AtomicBool::new(false));
13724 let server_stop = Arc::clone(&stop);
13725 let requests = Arc::new(Mutex::new(Vec::new()));
13726 let server_requests = Arc::clone(&requests);
13727 let thread = thread::spawn(move || {
13728 let mut request_threads = Vec::new();
13729 while !server_stop.load(Ordering::SeqCst) {
13730 match listener.accept() {
13731 Ok((mut stream, _)) => {
13732 if behavior.concurrent_requests {
13733 let requests = Arc::clone(&server_requests);
13734 request_threads.push(thread::spawn(move || {
13735 handle_mock_worker_request(&mut stream, &requests, behavior)
13736 }));
13737 } else {
13738 handle_mock_worker_request(&mut stream, &server_requests, behavior);
13739 }
13740 }
13741 Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
13742 let mut index = 0;
13743 while index < request_threads.len() {
13744 if request_threads[index].is_finished() {
13745 request_threads
13746 .swap_remove(index)
13747 .join()
13748 .expect("join mock request");
13749 } else {
13750 index += 1;
13751 }
13752 }
13753 thread::sleep(Duration::from_millis(5));
13754 }
13755 Err(_) => break,
13756 }
13757 }
13758 for request_thread in request_threads {
13759 request_thread.join().expect("join mock request");
13760 }
13761 });
13762
13763 Self {
13764 addr,
13765 stop,
13766 requests,
13767 thread: Some(thread),
13768 }
13769 }
13770
13771 fn base_url(&self) -> String {
13772 format!("http://{}", self.addr)
13773 }
13774
13775 fn worker_protocol_for(&self, path: &str) -> Option<String> {
13776 self.requests
13777 .lock()
13778 .expect("captured requests")
13779 .iter()
13780 .find(|request| request.path == path)
13781 .and_then(|request| request.worker_protocol.clone())
13782 }
13783
13784 fn control_protocol_for(&self, path: &str) -> Option<String> {
13785 self.requests
13786 .lock()
13787 .expect("captured requests")
13788 .iter()
13789 .find(|request| request.path == path)
13790 .and_then(|request| request.control_protocol.clone())
13791 }
13792
13793 fn method_for(&self, path: &str) -> Option<String> {
13794 self.requests
13795 .lock()
13796 .expect("captured requests")
13797 .iter()
13798 .find(|request| request.path == path)
13799 .map(|request| request.method.clone())
13800 }
13801
13802 fn authorization_for(&self, path: &str) -> Option<String> {
13803 self.requests
13804 .lock()
13805 .expect("captured requests")
13806 .iter()
13807 .find(|request| request.path == path)
13808 .and_then(|request| request.authorization.clone())
13809 }
13810
13811 fn namespace_for(&self, path: &str) -> Option<String> {
13812 self.requests
13813 .lock()
13814 .expect("captured requests")
13815 .iter()
13816 .find(|request| request.path == path)
13817 .and_then(|request| request.namespace.clone())
13818 }
13819
13820 fn request_count(&self, path: &str) -> usize {
13821 self.requests
13822 .lock()
13823 .expect("captured requests")
13824 .iter()
13825 .filter(|request| request.path == path)
13826 .count()
13827 }
13828
13829 fn captured_paths(&self) -> Vec<String> {
13830 self.requests
13831 .lock()
13832 .expect("captured requests")
13833 .iter()
13834 .map(|request| request.path.clone())
13835 .collect()
13836 }
13837
13838 fn request_times(&self, path: &str) -> Vec<Instant> {
13839 self.requests
13840 .lock()
13841 .expect("captured requests")
13842 .iter()
13843 .filter(|request| request.path == path)
13844 .map(|request| request.received_at)
13845 .collect()
13846 }
13847
13848 fn request_body(&self, path: &str) -> Value {
13849 let requests = self.requests.lock().expect("captured requests");
13850 let body = &requests
13851 .iter()
13852 .find(|request| request.path == path)
13853 .unwrap_or_else(|| panic!("missing request for {path}"))
13854 .body;
13855 serde_json::from_str(body).unwrap_or_else(|error| {
13856 panic!("invalid JSON request body for {path}: {error}: {body:?}")
13857 })
13858 }
13859
13860 fn request_bodies(&self, path: &str) -> Vec<Value> {
13861 self.requests
13862 .lock()
13863 .expect("captured requests")
13864 .iter()
13865 .filter(|request| request.path == path)
13866 .map(|request| {
13867 serde_json::from_str(&request.body).unwrap_or_else(|error| {
13868 panic!(
13869 "invalid JSON request body for {path}: {error}: {:?}",
13870 request.body
13871 )
13872 })
13873 })
13874 .collect()
13875 }
13876 }
13877
13878 impl Drop for MockWorkerServer {
13879 fn drop(&mut self) {
13880 self.stop.store(true, Ordering::SeqCst);
13881 let _ = TcpStream::connect(self.addr);
13882
13883 if let Some(thread) = self.thread.take() {
13884 thread.join().expect("join mock server");
13885 }
13886 }
13887 }
13888
13889 fn handle_mock_worker_request(
13890 stream: &mut TcpStream,
13891 requests: &Arc<Mutex<Vec<CapturedRequest>>>,
13892 behavior: MockWorkerBehavior,
13893 ) {
13894 let _ = stream.set_read_timeout(Some(Duration::from_millis(200)));
13895 let mut buffer = [0_u8; 8192];
13896 let mut request = Vec::new();
13897
13898 loop {
13899 match stream.read(&mut buffer) {
13900 Ok(0) => break,
13901 Ok(read) => {
13902 request.extend_from_slice(&buffer[..read]);
13903 if mock_request_is_complete(&request) {
13904 break;
13905 }
13906 }
13907 Err(error)
13908 if matches!(
13909 error.kind(),
13910 std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
13911 ) =>
13912 {
13913 break;
13914 }
13915 Err(_) => return,
13916 }
13917 }
13918
13919 let request = String::from_utf8_lossy(&request);
13920 let body = request
13921 .split_once("\r\n\r\n")
13922 .map(|(_, body)| body)
13923 .unwrap_or_default();
13924 let path = request
13925 .lines()
13926 .next()
13927 .and_then(|line| line.split_whitespace().nth(1))
13928 .unwrap_or_default();
13929 let method = request
13930 .lines()
13931 .next()
13932 .and_then(|line| line.split_whitespace().next())
13933 .unwrap_or_default();
13934 let authorization = request.lines().find_map(|line| {
13935 let (name, value) = line.split_once(':')?;
13936 name.eq_ignore_ascii_case("Authorization")
13937 .then(|| value.trim().to_string())
13938 });
13939 let namespace = request.lines().find_map(|line| {
13940 let (name, value) = line.split_once(':')?;
13941 name.eq_ignore_ascii_case("X-Namespace")
13942 .then(|| value.trim().to_string())
13943 });
13944 let worker_protocol = request.lines().find_map(|line| {
13945 let (name, value) = line.split_once(':')?;
13946 name.eq_ignore_ascii_case("X-Durable-Workflow-Protocol-Version")
13947 .then(|| value.trim().to_string())
13948 });
13949 let control_protocol = request.lines().find_map(|line| {
13950 let (name, value) = line.split_once(':')?;
13951 name.eq_ignore_ascii_case("X-Durable-Workflow-Control-Plane-Version")
13952 .then(|| value.trim().to_string())
13953 });
13954 let request_number = {
13955 let mut requests = requests.lock().expect("captured requests");
13956 requests.push(CapturedRequest {
13957 method: method.to_string(),
13958 path: path.to_string(),
13959 authorization,
13960 namespace,
13961 worker_protocol: worker_protocol.clone(),
13962 control_protocol,
13963 body: body.to_string(),
13964 received_at: Instant::now(),
13965 });
13966 requests
13967 .iter()
13968 .filter(|request| request.path == path)
13969 .count()
13970 };
13971
13972 if behavior.reject_registration && path == "/api/worker/register" {
13973 write_mock_response(
13974 stream,
13975 "503 Service Unavailable",
13976 r#"{"reason":"registration_unavailable","message":"registration failed"}"#,
13977 );
13978 return;
13979 }
13980
13981 if path.starts_with("/api/worker/registrations/") {
13982 if behavior.reject_deregistration_protocol {
13983 write_mock_response(
13984 stream,
13985 "400 Bad Request",
13986 r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.1","requested_version":"1.2"}"#,
13987 );
13988 } else if behavior.reject_deregistration {
13989 write_mock_response(
13990 stream,
13991 "403 Forbidden",
13992 r#"{"reason":"authorization_failed","message":"worker cannot deregister"}"#,
13993 );
13994 } else {
13995 write_mock_response(
13996 stream,
13997 "200 OK",
13998 r#"{"worker_id":"deregistered-worker","outcome":"deregistered","recovered_workflow_task_count":2}"#,
13999 );
14000 }
14001 return;
14002 }
14003
14004 let is_poll = matches!(
14005 path,
14006 "/api/worker/workflow-tasks/poll"
14007 | "/api/worker/activity-tasks/poll"
14008 | "/api/worker/query-tasks/poll"
14009 );
14010 if is_poll && request_number <= behavior.poll_failures_per_path {
14011 return;
14012 }
14013 if path == "/api/worker/heartbeat" && request_number <= behavior.heartbeat_failures {
14014 return;
14015 }
14016 if path == "/api/worker/heartbeat"
14017 && behavior.heartbeat_failure_request == Some(request_number)
14018 {
14019 return;
14020 }
14021 if path == "/api/worker/heartbeat"
14022 && behavior.delayed_heartbeat_request == Some(request_number)
14023 {
14024 thread::sleep(behavior.heartbeat_response_delay);
14025 }
14026 if behavior.unauthorized_polls && is_poll {
14027 write_mock_response(
14028 stream,
14029 "401 Unauthorized",
14030 r#"{"reason":"authentication_failed","message":"invalid worker token"}"#,
14031 );
14032 return;
14033 }
14034 if behavior.draining_polls && is_poll {
14035 write_mock_response(
14036 stream,
14037 "409 Conflict",
14038 r#"{"task":null,"poll_status":"draining","reason":"worker_draining","worker_status":"draining","drain_intent":"draining"}"#,
14039 );
14040 return;
14041 }
14042
14043 if behavior.reject_query_protocol && path.starts_with("/api/worker/query-tasks/") {
14044 let requested_version = worker_protocol.as_deref().unwrap_or("missing");
14045 let body = format!(
14046 r#"{{"reason":"unsupported_protocol_version","message":"Query tasks require worker protocol 1.8 or newer.","supported_version":"1.7","requested_version":"{requested_version}"}}"#
14047 );
14048 write_mock_response(stream, "400 Bad Request", &body);
14049 return;
14050 }
14051
14052 if behavior.reject_query_completion && path == "/api/worker/query-tasks/query-late/complete"
14053 {
14054 write_mock_response(
14055 stream,
14056 "409 Conflict",
14057 r#"{"reason":"query_task_timed_out","message":"query task timed out before completion"}"#,
14058 );
14059 return;
14060 }
14061
14062 if behavior.workflow_completion_status.is_some()
14063 && path == "/api/worker/workflow-tasks/poll"
14064 && request_number == 1
14065 {
14066 write_mock_response(
14067 stream,
14068 "200 OK",
14069 r#"{"task":{"task_id":"workflow-timeout-task","workflow_id":"reused-workflow-id","run_id":"run-selected-timeout","workflow_type":"timeout.workflow","payload_codec":"json","arguments":{"codec":"json","blob":"[]"},"history_events":[],"workflow_task_attempt":3,"lease_owner":"timeout-worker"}}"#,
14070 );
14071 return;
14072 }
14073
14074 if path == "/api/worker/workflow-tasks/workflow-timeout-task/complete" {
14075 if let (Some(status), Some(body)) = (
14076 behavior.workflow_completion_status,
14077 behavior.workflow_completion_body,
14078 ) {
14079 write_mock_response(stream, status, body);
14080 return;
14081 }
14082 }
14083
14084 if behavior.waiting_query_worker {
14085 if behavior.complete_named_signal
14086 && path == "/api/worker/workflow-tasks/poll"
14087 && request_number == 1
14088 {
14089 let body = json!({
14090 "task": {
14091 "task_id": "snapshot-open",
14092 "workflow_id": "snapshot-1",
14093 "run_id": "snapshot-run-1",
14094 "workflow_type": "snapshot",
14095 "payload_codec": DEFAULT_CODEC,
14096 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
14097 .expect("Avro workflow arguments"),
14098 "history_events": [],
14099 "workflow_task_attempt": 1,
14100 "lease_owner": "rust-snapshot-worker"
14101 }
14102 })
14103 .to_string();
14104 write_mock_response(stream, "200 OK", &body);
14105 return;
14106 }
14107
14108 let signal_request = request_number - usize::from(behavior.complete_named_signal);
14109 let signal_request_limit = 2 + usize::from(behavior.complete_named_signal);
14110 if path == "/api/worker/workflow-tasks/poll"
14111 && signal_request >= 1
14112 && signal_request <= signal_request_limit
14113 {
14114 let finish = behavior.complete_named_signal && signal_request == 3;
14115 let amounts = if signal_request == 1 {
14116 vec![3]
14117 } else {
14118 vec![3, 5]
14119 };
14120 let task_id = if signal_request == 1 {
14121 "snapshot-wait-3"
14122 } else if finish {
14123 "snapshot-finish"
14124 } else {
14125 "snapshot-wait-5"
14126 };
14127 let mut history_events = std::iter::once(json!({
14128 "event_type": "SignalWaitOpened",
14129 "payload": {"sequence": 1, "signal_name": "finish"}
14130 }))
14131 .chain(amounts.iter().enumerate().map(|(index, amount)| {
14132 json!({
14133 "event_type": "SignalReceived",
14134 "payload": {
14135 "signal_id": format!("increment-{amount}"),
14136 "signal_name": "increment",
14137 "workflow_sequence": index + 2,
14138 "payload_codec": DEFAULT_CODEC,
14139 "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
14140 .expect("Avro signal envelope")
14141 }
14142 })
14143 }))
14144 .collect::<Vec<_>>();
14145 let (resume_id, resume_name, resume_arguments) = if finish {
14146 history_events.push(json!({
14147 "event_type": "SignalReceived",
14148 "payload": {
14149 "signal_id": "finish",
14150 "signal_name": "finish",
14151 "workflow_sequence": 4,
14152 "payload_codec": DEFAULT_CODEC,
14153 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
14154 .expect("Avro finish signal envelope")
14155 }
14156 }));
14157 (
14158 "finish".to_string(),
14159 "finish".to_string(),
14160 encode_value_envelope(&json!([]), DEFAULT_CODEC)
14161 .expect("Avro finish resume signal"),
14162 )
14163 } else {
14164 let amount = amounts.last().expect("amount");
14165 (
14166 format!("increment-{amount}"),
14167 "increment".to_string(),
14168 encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
14169 .expect("Avro increment resume signal"),
14170 )
14171 };
14172 let body = json!({
14173 "task": {
14174 "task_id": task_id,
14175 "workflow_id": "snapshot-1",
14176 "run_id": "snapshot-run-1",
14177 "workflow_type": "snapshot",
14178 "payload_codec": DEFAULT_CODEC,
14179 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
14180 .expect("Avro workflow arguments"),
14181 "history_events": history_events,
14182 "workflow_task_attempt": 1,
14183 "workflow_signal_id": resume_id,
14184 "signal_name": resume_name,
14185 "signal_arguments": resume_arguments,
14186 "lease_owner": "rust-snapshot-worker"
14187 }
14188 })
14189 .to_string();
14190 write_mock_response(stream, "200 OK", &body);
14191 return;
14192 }
14193
14194 if path == "/api/worker/query-tasks/poll" && request_number == 1 {
14195 let history_events = [3, 5]
14196 .into_iter()
14197 .enumerate()
14198 .map(|(index, amount)| {
14199 json!({
14200 "event_type": "SignalReceived",
14201 "payload": {
14202 "signal_id": format!("increment-{amount}"),
14203 "signal_name": "increment",
14204 "workflow_sequence": index + 2,
14205 "payload_codec": DEFAULT_CODEC,
14206 "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
14207 .expect("Avro query signal envelope")
14208 }
14209 })
14210 })
14211 .collect::<Vec<_>>();
14212 let body = json!({
14213 "task": {
14214 "query_task_id": "snapshot-current",
14215 "query_task_attempt": 1,
14216 "lease_owner": "rust-snapshot-worker",
14217 "workflow_id": "snapshot-1",
14218 "run_id": "snapshot-run-1",
14219 "workflow_type": "snapshot",
14220 "query_name": "current",
14221 "payload_codec": DEFAULT_CODEC,
14222 "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
14223 .expect("Avro workflow arguments"),
14224 "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
14225 .expect("Avro query arguments"),
14226 "history_events": history_events,
14227 "run_status": "waiting"
14228 }
14229 })
14230 .to_string();
14231 write_mock_response(stream, "200 OK", &body);
14232 return;
14233 }
14234
14235 if path == "/api/worker/workflow-tasks/snapshot-wait-3/fail"
14236 || path == "/api/worker/workflow-tasks/snapshot-wait-5/fail"
14237 {
14238 write_mock_response(
14239 stream,
14240 "200 OK",
14241 r#"{"outcome":"waiting_for_history","recorded":true}"#,
14242 );
14243 return;
14244 }
14245
14246 if path == "/api/worker/workflow-tasks/snapshot-open/complete" {
14247 write_mock_response(stream, "200 OK", r#"{"outcome":"waiting","recorded":true}"#);
14248 return;
14249 }
14250
14251 if path == "/api/worker/workflow-tasks/snapshot-finish/complete" {
14252 write_mock_response(
14253 stream,
14254 "200 OK",
14255 r#"{"outcome":"completed","run_status":"completed","recorded":true}"#,
14256 );
14257 return;
14258 }
14259
14260 if path == "/api/worker/query-tasks/snapshot-current/complete" {
14261 write_mock_response(stream, "200 OK", r#"{"outcome":"completed"}"#);
14262 return;
14263 }
14264 }
14265
14266 if matches!(
14267 path,
14268 "/api/workflows/typed-1/query/inspect" | "/api/workflows/typed-1/update/replace"
14269 ) {
14270 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
14271 .expect("typed mock result");
14272 let body = json!({
14273 "result": typed_fidelity_probe().into_json().expect("result projection"),
14274 "result_envelope": result,
14275 })
14276 .to_string();
14277 write_mock_response(stream, "200 OK", &body);
14278 return;
14279 }
14280
14281 if path == "/api/workflows/typed-1" {
14282 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
14283 .expect("typed mock result");
14284 let body = json!({
14285 "workflow_id": "typed-1",
14286 "run_id": "run-typed-1",
14287 "workflow_type": "typed.echo",
14288 "status": "completed",
14289 "output": typed_fidelity_probe().into_json().expect("output projection"),
14290 "output_envelope": result,
14291 })
14292 .to_string();
14293 write_mock_response(stream, "200 OK", &body);
14294 return;
14295 }
14296
14297 let (status, body) = match path {
14298 "/api/health" => ("200 OK", r#"{"status":"ok"}"#),
14299 "/api/workflows" => (
14300 "201 Created",
14301 r#"{"workflow_id":"wf-start-options","run_id":"run-start-options","workflow_type":"rust.timeout"}"#,
14302 ),
14303 "/api/worker/register" if behavior.decline_registration => (
14304 "200 OK",
14305 r#"{"worker_id":"declined-worker","registered":false}"#,
14306 ),
14307 "/api/worker/register" if behavior.waiting_query_worker => (
14308 "200 OK",
14309 r#"{"worker_id":"rust-snapshot-worker","registered":true,"heartbeat_interval_seconds":1}"#,
14310 ),
14311 "/api/worker/register" => (
14312 "200 OK",
14313 r#"{"worker_id":"mock-worker","registered":true,"heartbeat_interval_seconds":3600}"#,
14314 ),
14315 "/api/worker/heartbeat" => ("200 OK", "{}"),
14316 "/api/worker/activity-tasks/poll"
14317 if behavior.cancelled_activity && request_number == 1 =>
14318 {
14319 (
14320 "200 OK",
14321 r#"{"task":{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","activity_type":"cancel-aware","payload_codec":"json","arguments":{"codec":"json","blob":"[]"},"attempt_number":1,"lease_owner":"rust-cancel-worker"}}"#,
14322 )
14323 }
14324 "/api/worker/activity-tasks/poll" | "/api/worker/workflow-tasks/poll" => {
14325 ("200 OK", r#"{"task":null}"#)
14326 }
14327 "/api/worker/query-tasks/poll"
14328 if behavior.reject_query_completion && request_number == 1 =>
14329 {
14330 (
14331 "200 OK",
14332 r#"{"task":{"query_task_id":"query-late","query_task_attempt":1,"lease_owner":"late-worker","workflow_id":"counter-late","run_id":"run-late","workflow_type":"counter","query_name":"current","payload_codec":"json","workflow_arguments":{"codec":"json","blob":"[]"},"query_arguments":{"codec":"json","blob":"[]"},"history_events":[],"run_status":"running"}}"#,
14333 )
14334 }
14335 "/api/worker/query-tasks/poll" => ("200 OK", r#"{"task":null}"#),
14336 "/api/worker/query-tasks/query-capture/complete"
14337 | "/api/worker/query-tasks/query-capture/fail" => ("200 OK", "{}"),
14338 "/api/worker/activity-tasks/activity-cancel/heartbeat" => (
14339 "200 OK",
14340 r#"{"activity_attempt_id":"attempt-cancel","cancel_requested":true,"can_continue":false,"reason":"run_cancelled","run_closed_reason":"cancelled","heartbeat_recorded":false}"#,
14341 ),
14342 "/api/worker/activity-tasks/activity-cancel/complete" => (
14343 "409 Conflict",
14344 r#"{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","reason":"run_cancelled","cancel_requested":true,"can_continue":false,"run_closed_reason":"cancelled"}"#,
14345 ),
14346 "/api/worker/activity-tasks/activity-typed/complete"
14347 | "/api/worker/activity-tasks/activity-typed/fail"
14348 | "/api/workflows/typed-1/signal/changed" => ("200 OK", "{}"),
14349 "/api/workflows/counter-1/query/current" => (
14350 "200 OK",
14351 r#"{"workflow_id":"counter-1","query_name":"current","result":{"count":8},"result_envelope":{"codec":"json","blob":"{\"count\":8}"}}"#,
14352 ),
14353 "/api/workflows/counter-1/query/missing" => (
14354 "404 Not Found",
14355 r#"{"workflow_id":"counter-1","query_name":"missing","reason":"rejected_unknown_query","message":"unknown query"}"#,
14356 ),
14357 "/api/workflows/wf-lifecycle/cancel" => (
14358 "200 OK",
14359 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","reason":"cleanup requested","command_status":"accepted"}"#,
14360 ),
14361 "/api/workflows/wf-lifecycle/terminate" => (
14362 "200 OK",
14363 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","reason":"forced stop","command_status":"accepted"}"#,
14364 ),
14365 "/api/workflows/wf-lifecycle/runs/run-current/cancel" => (
14366 "200 OK",
14367 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","command_status":"accepted"}"#,
14368 ),
14369 "/api/workflows/wf-lifecycle/runs/run-current/terminate" => (
14370 "200 OK",
14371 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","command_status":"accepted"}"#,
14372 ),
14373 "/api/workflows/wf-lifecycle/runs/run-stale/cancel"
14374 | "/api/workflows/wf-lifecycle/runs/run-stale/terminate" => (
14375 "409 Conflict",
14376 r#"{"workflow_id":"wf-lifecycle","run_id":"run-stale","reason":"historical_run_command_rejected","target_scope":"run","message":"Commands cannot target historical runs."}"#,
14377 ),
14378 "/api/workflows/wf-failed" | "/api/workflows/wf-failed/runs/run-failed" => (
14379 "200 OK",
14380 r#"{"workflow_id":"wf-failed","run_id":"run-failed","status":"failed","closed_reason":"failed","error":"payment failed","failure":{"message":"payment failed","failure_category":"application","exception_type":"PaymentError","exception_class":"billing::PaymentError","non_retryable":true,"exception":{"type":"PaymentError","class":"billing::PaymentError","message":"payment failed"},"failures":[{"id":"failure-17","failure_category":"application"}]}}"#,
14381 ),
14382 "/api/workflows/wf-cancelled" => (
14383 "200 OK",
14384 r#"{"workflow_id":"wf-cancelled","run_id":"run-cancelled","status":"cancelled","closed_reason":"cancelled","reason":"cleanup requested"}"#,
14385 ),
14386 "/api/workflows/wf-terminated" => (
14387 "200 OK",
14388 r#"{"workflow_id":"wf-terminated","run_id":"run-terminated","status":"terminated","closed_reason":"terminated","reason":"forced stop"}"#,
14389 ),
14390 "/api/workflows/wf-timed-out" => (
14391 "200 OK",
14392 r#"{"workflow_id":"wf-timed-out","run_id":"run-timed-out","status":"failed","closed_reason":"timed_out","reason":"run_timeout"}"#,
14393 ),
14394 "/api/workflows/wf-waiting" | "/api/workflows/wf-waiting/runs/run-waiting" => (
14395 "200 OK",
14396 r#"{"workflow_id":"wf-waiting","run_id":"run-waiting","status":"waiting"}"#,
14397 ),
14398 "/api/workflows/wf-selected" => (
14399 "200 OK",
14400 r#"{"workflow_id":"wf-selected","run_id":"run-current","status":"completed","output":"current run output"}"#,
14401 ),
14402 "/api/workflows/wf-selected/runs/run-selected" => (
14403 "200 OK",
14404 r#"{"workflow_id":"wf-selected","run_id":"run-selected","status":"cancelled","closed_reason":"cancelled","reason":"selected run cancelled"}"#,
14405 ),
14406 _ => ("404 Not Found", r#"{"message":"not found"}"#),
14407 };
14408 write_mock_response(stream, status, body);
14409 }
14410
14411 fn mock_request_is_complete(request: &[u8]) -> bool {
14412 let Some(header_end) = request
14413 .windows(4)
14414 .position(|window| window == b"\r\n\r\n")
14415 .map(|position| position + 4)
14416 else {
14417 return false;
14418 };
14419 let headers = String::from_utf8_lossy(&request[..header_end]);
14420 let content_length = headers.lines().find_map(|line| {
14421 let (name, value) = line.split_once(':')?;
14422 name.eq_ignore_ascii_case("content-length")
14423 .then(|| value.trim().parse::<usize>().ok())
14424 .flatten()
14425 });
14426
14427 request.len() >= header_end + content_length.unwrap_or(0)
14428 }
14429
14430 fn write_mock_response(stream: &mut TcpStream, status: &str, body: &str) {
14431 let response = format!(
14432 "HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
14433 body.len()
14434 );
14435
14436 let _ = stream.write_all(response.as_bytes());
14437 let _ = stream.flush();
14438 }
14439}