1#![doc = include_str!("../README.md")]
2
3use std::{
4 any::{Any, TypeId},
5 collections::{BTreeMap, HashMap},
6 future::Future,
7 io::{self, Read},
8 pin::Pin,
9 sync::{
10 atomic::{AtomicBool, Ordering},
11 Arc, Mutex, OnceLock,
12 },
13 task::{Context as TaskContext, Poll},
14 time::{Duration, Instant, SystemTime, UNIX_EPOCH},
15};
16
17use apache_avro::{from_avro_datum, to_avro_datum, types::Value as AvroDatum, Schema};
18use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
19use futures_util::{future::OptionFuture, task::noop_waker_ref};
20use serde::{
21 de::DeserializeOwned,
22 ser::{SerializeMap, SerializeSeq},
23 Deserialize, Serialize, Serializer,
24};
25pub use serde_json::{json, Value};
26use thiserror::Error;
27pub use uuid::Uuid;
28
29pub const WORKER_PROTOCOL_VERSION: &str = "1.2";
30pub const CONTROL_PLANE_VERSION: &str = "2";
31pub const DEFAULT_CODEC: &str = "avro";
32pub const JSON_CODEC: &str = "json";
33pub const SDK_VERSION: &str = concat!("durable-workflow-rust/", env!("CARGO_PKG_VERSION"));
34pub const QUERY_TASKS_CAPABILITY: &str = "query_tasks";
36pub const WORKFLOW_UPDATES_CAPABILITY: &str = "workflow_updates";
38pub const QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
40
41const MAX_LONG_POLL_TIMEOUT_SECONDS: u64 = 60;
42const WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE: &str =
43 "Workflow task waiting for scheduled history.";
44const WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE: &str = "WorkflowTaskWaitingForHistory";
45
46const QUERY_TASK_FINAL_REJECTION_REASONS: &[&str] = &[
47 "lease_expired",
48 "query_task_not_found",
49 "query_task_not_leased",
50 "query_task_timed_out",
51];
52
53pub const AVRO_VALUE_SCHEMA_JSON: &str =
55 include_str!("../schema/durable_workflow.protocol.Value.v1.avsc");
56pub const AVRO_VALUE_SCHEMA_FINGERPRINT_HEX: &str = "e2a33dff55802237";
57pub const AVRO_VALUE_SCHEMA_FINGERPRINT: [u8; 8] = [0xe2, 0xa3, 0x3d, 0xff, 0x55, 0x80, 0x22, 0x37];
58const AVRO_SINGLE_OBJECT_MAGIC: [u8; 2] = [0xc3, 0x01];
59
60static AVRO_VALUE_SCHEMA: OnceLock<std::result::Result<Schema, String>> = OnceLock::new();
61
62#[derive(Clone, Copy)]
63enum RequestProtocol {
64 ControlPlane,
65 Worker(&'static str),
66}
67
68pub type Result<T> = std::result::Result<T, Error>;
69
70#[derive(Debug, Error)]
71pub enum Error {
72 #[error("transport error: {0}")]
73 Transport(#[from] reqwest::Error),
74 #[error(
75 "invalid Durable Workflow base URL: omit the SDK-owned /api suffix and pass the Server or Cloud runtime base URL; the SDK appends /api automatically"
76 )]
77 InvalidBaseUrl,
78 #[error("json error: {0}")]
79 Json(#[from] serde_json::Error),
80 #[error("http {status}: {body}")]
81 Http {
82 status: reqwest::StatusCode,
83 body: String,
84 },
85 #[error("codec error: {0}")]
86 Codec(String),
87 #[error(transparent)]
88 QueryFailed(QueryFailure),
89 #[error(transparent)]
90 Protocol(ProtocolFailure),
91 #[error(transparent)]
92 NonDeterministicReplay(ReplayFailure),
93 #[error(transparent)]
94 ChildWorkflowFailed(ChildWorkflowFailure),
95 #[error(transparent)]
96 ActivityFailed(ActivityFailure),
97 #[error(transparent)]
98 WorkflowCommandRejected(WorkflowCommandRejection),
99 #[error(transparent)]
100 WorkflowFailed(WorkflowTerminalOutcome),
101 #[error(transparent)]
102 WorkflowCancelled(WorkflowTerminalOutcome),
103 #[error(transparent)]
104 WorkflowTerminated(WorkflowTerminalOutcome),
105 #[error(transparent)]
106 WorkflowTimedOut(WorkflowTerminalOutcome),
107 #[error(transparent)]
108 ActivityTaskRejected(ActivityTaskRejection),
109 #[error("workflow handler {0:?} is not registered")]
110 WorkflowNotRegistered(String),
111 #[error("activity handler {0:?} is not registered")]
112 ActivityNotRegistered(String),
113 #[error("workflow future yielded without emitting a durable command")]
114 WorkflowYieldedWithoutCommand,
115 #[error("workflow state lock is poisoned")]
116 WorkflowStatePoisoned,
117 #[error("timer duration is too large for the worker protocol")]
118 TimerDurationOverflow,
119 #[error("operation timed out")]
120 Timeout,
121 #[error(
122 "missing {role}-plane credentials: configure ClientBuilder::{role}_token or ClientBuilder::token; a {opposite_role}-plane token cannot authorize this request"
123 )]
124 MissingRoleCredentials {
125 role: &'static str,
126 opposite_role: &'static str,
127 },
128 #[error("worker loop error: {0}")]
129 WorkerLoop(String),
130 #[error("{primary}; worker deregistration also failed: {deregistration}")]
131 WorkerShutdown {
132 primary: Box<Error>,
133 deregistration: Box<Error>,
134 },
135 #[error("invalid child workflow options: {0}")]
136 InvalidChildWorkflowOptions(String),
137 #[error(transparent)]
138 InvalidActivityOptions(ActivityOptionsError),
139 #[error(transparent)]
140 InvalidContinueAsNewOptions(#[from] ContinueAsNewOptionsError),
141 #[doc(hidden)]
142 #[error("workflow requested continue as new")]
143 ContinueAsNew(ContinueAsNewRequest),
144}
145
146#[derive(Clone, Copy, Debug, PartialEq, Eq)]
148pub enum WorkflowCommandKind {
149 Cancel,
150 Terminate,
151}
152
153impl WorkflowCommandKind {
154 fn as_str(self) -> &'static str {
155 match self {
156 Self::Cancel => "cancel",
157 Self::Terminate => "terminate",
158 }
159 }
160}
161
162#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
164pub struct WorkflowCommandOptions {
165 #[serde(skip_serializing_if = "Option::is_none")]
166 pub reason: Option<String>,
167 #[serde(skip_serializing_if = "Option::is_none")]
168 pub request_id: Option<String>,
169}
170
171#[derive(Clone, Debug, PartialEq, Eq)]
178pub struct WorkflowStartOptions {
179 pub execution_timeout_seconds: u64,
180 pub run_timeout_seconds: u64,
181}
182
183impl Default for WorkflowStartOptions {
184 fn default() -> Self {
185 Self {
186 execution_timeout_seconds: 3600,
187 run_timeout_seconds: 600,
188 }
189 }
190}
191
192impl WorkflowStartOptions {
193 pub fn new() -> Self {
194 Self::default()
195 }
196
197 pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
198 self.execution_timeout_seconds = seconds;
199 self
200 }
201
202 pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
203 self.run_timeout_seconds = seconds;
204 self
205 }
206
207 fn validate(&self) -> Result<()> {
208 if self.execution_timeout_seconds == 0 {
209 return Err(Error::Codec(
210 "execution_timeout_seconds must be at least 1".to_string(),
211 ));
212 }
213 if self.run_timeout_seconds == 0 {
214 return Err(Error::Codec(
215 "run_timeout_seconds must be at least 1".to_string(),
216 ));
217 }
218 if self.run_timeout_seconds > self.execution_timeout_seconds {
219 return Err(Error::Codec(
220 "run_timeout_seconds cannot exceed execution_timeout_seconds".to_string(),
221 ));
222 }
223
224 Ok(())
225 }
226}
227
228#[derive(Clone, Debug, Default, PartialEq, Eq)]
233pub struct ContinueAsNewOptions {
234 pub workflow_type: Option<String>,
235 pub task_queue: Option<String>,
236}
237
238impl ContinueAsNewOptions {
239 pub fn new() -> Self {
240 Self::default()
241 }
242
243 pub fn workflow_type(mut self, workflow_type: impl Into<String>) -> Self {
244 self.workflow_type = Some(workflow_type.into());
245 self
246 }
247
248 pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
249 self.task_queue = Some(task_queue.into());
250 self
251 }
252
253 fn validate(&self) -> std::result::Result<(), ContinueAsNewOptionsError> {
254 for (field, value) in [
255 ("workflow_type", self.workflow_type.as_deref()),
256 ("task_queue", self.task_queue.as_deref()),
257 ] {
258 if value.is_some_and(|value| value.trim().is_empty()) {
259 return Err(ContinueAsNewOptionsError {
260 field,
261 message: format!("{field} must not be empty"),
262 });
263 }
264 }
265 Ok(())
266 }
267}
268
269#[derive(Clone, Debug, Error, PartialEq, Eq)]
271#[error("invalid continue-as-new option {field}: {message}")]
272pub struct ContinueAsNewOptionsError {
273 pub field: &'static str,
274 pub message: String,
275}
276
277#[derive(Clone, Debug, Default, PartialEq, Eq)]
279pub struct WorkflowHistoryBudget {
280 pub event_count: u64,
281 pub size_bytes: Option<u64>,
282 pub continue_as_new_recommended: bool,
283 pub pressure: Option<String>,
284}
285
286#[doc(hidden)]
287#[derive(Clone, Debug)]
288pub struct ContinueAsNewRequest {
289 arguments: AvroValue,
290 options: ContinueAsNewOptions,
291}
292
293impl WorkflowCommandOptions {
294 pub fn new() -> Self {
295 Self::default()
296 }
297
298 pub fn reason(mut self, reason: impl Into<String>) -> Self {
299 self.reason = Some(reason.into());
300 self
301 }
302
303 pub fn request_id(mut self, request_id: impl Into<String>) -> Self {
304 self.request_id = Some(request_id.into());
305 self
306 }
307}
308
309#[derive(Clone, Debug, PartialEq)]
311pub struct WorkflowCommandResult {
312 pub command: WorkflowCommandKind,
313 pub workflow_id: String,
314 pub run_id: Option<String>,
315 pub outcome: Option<String>,
316 pub reason: Option<String>,
317 pub command_status: Option<String>,
318 pub raw: Value,
319}
320
321#[derive(Clone, Debug, Error)]
323#[error("workflow {command:?} rejected ({reason}, HTTP {status}): {message}")]
324pub struct WorkflowCommandRejection {
325 pub command: WorkflowCommandKind,
326 pub status: u16,
327 pub reason: String,
328 pub message: String,
329 pub workflow_id: String,
330 pub run_id: Option<String>,
331 pub target_scope: Option<String>,
332 pub body: Value,
333}
334
335#[derive(Clone, Copy, Debug, PartialEq, Eq)]
337pub enum WorkflowTerminalKind {
338 Failed,
339 Cancelled,
340 Terminated,
341 TimedOut,
342}
343
344#[derive(Clone, Debug, Error)]
350#[error("workflow {workflow_id} run {run_id:?} ended as {kind:?} ({reason})")]
351pub struct WorkflowTerminalOutcome {
352 pub kind: WorkflowTerminalKind,
353 pub workflow_id: String,
354 pub run_id: Option<String>,
355 pub reason: String,
356 pub failure_category: Option<String>,
357 pub failure_id: Option<String>,
358 pub exception_type: Option<String>,
359 pub exception_class: Option<String>,
360 pub non_retryable: Option<bool>,
361 pub message: Option<String>,
362 pub exception: Option<Value>,
363 pub raw: Value,
364}
365
366#[derive(Clone, Debug, Error)]
368#[error("activity task {operation} rejected ({reason}, HTTP {status})")]
369pub struct ActivityTaskRejection {
370 pub operation: String,
371 pub status: u16,
372 pub reason: String,
373 pub task_id: String,
374 pub activity_attempt_id: String,
375 pub cancel_requested: bool,
376 pub can_continue: Option<bool>,
377 pub run_closed_reason: Option<String>,
378 pub body: Value,
379}
380
381#[derive(Clone, Copy, Debug, PartialEq, Eq)]
383pub enum ActivityOptionsErrorKind {
384 EmptyTaskQueue,
385 EmptyRetryPolicy,
386 InvalidMaxAttempts,
387 BackoffWithoutRetryBudget,
388 TooManyBackoffIntervals,
389 InvalidBackoffCoefficient,
390 BackoffGenerationTooLarge,
391 BackoffOverflow,
392 EmptyNonRetryableErrorType,
393 TimeoutNotPositive,
394 TimeoutOverflow,
395 TimeoutOrder,
396}
397
398#[derive(Clone, Debug, Error, PartialEq, Eq)]
400#[error("invalid activity options ({kind:?}, {field:?}): {message}")]
401pub struct ActivityOptionsError {
402 pub kind: ActivityOptionsErrorKind,
403 pub field: Option<&'static str>,
404 pub message: String,
405}
406
407impl ActivityOptionsError {
408 fn new(
409 kind: ActivityOptionsErrorKind,
410 field: Option<&'static str>,
411 message: impl Into<String>,
412 ) -> Self {
413 Self {
414 kind,
415 field,
416 message: message.into(),
417 }
418 }
419}
420
421#[derive(Clone, Copy, Debug, PartialEq, Eq)]
423pub enum ActivityFailureKind {
424 Failed,
425 Cancelled,
426 TimedOut,
427}
428
429#[derive(Clone, Debug, Error)]
434#[error("activity failed ({reason}): {message}")]
435pub struct ActivityFailure {
436 pub kind: ActivityFailureKind,
437 pub reason: String,
438 pub message: String,
439 pub activity_execution_id: Option<String>,
440 pub activity_attempt_id: Option<String>,
441 pub activity_type: Option<String>,
442 pub activity_class: Option<String>,
443 pub attempt_number: Option<u64>,
444 pub failure_id: Option<String>,
445 pub failure_category: Option<String>,
446 pub timeout_kind: Option<String>,
447 pub non_retryable: bool,
448 pub exception_type: Option<String>,
449 pub exception_class: Option<String>,
450 pub code: Option<Value>,
451 pub exception: Option<Value>,
452}
453
454#[derive(Clone, Copy, Debug, PartialEq, Eq)]
456pub enum ChildWorkflowFailureKind {
457 Failed,
458 Cancelled,
459 Terminated,
460}
461
462#[derive(Clone, Debug, Error)]
468#[error("child workflow failed ({reason}): {message}")]
469pub struct ChildWorkflowFailure {
470 pub kind: ChildWorkflowFailureKind,
471 pub reason: String,
472 pub message: String,
473 pub parent_workflow_id: Option<String>,
474 pub parent_workflow_run_id: Option<String>,
475 pub child_workflow_id: Option<String>,
476 pub child_workflow_run_id: Option<String>,
477 pub child_workflow_type: Option<String>,
478 pub failure_id: Option<String>,
479 pub failure_category: Option<String>,
480 pub exception_type: Option<String>,
481 pub exception_class: Option<String>,
482 pub non_retryable: bool,
483 pub code: Option<Value>,
484 pub exception: Option<Value>,
485}
486
487#[derive(Clone, Debug, PartialEq, Eq)]
489pub struct WorkflowIdentity {
490 pub workflow_id: Option<String>,
491 pub run_id: Option<String>,
492}
493
494#[derive(Clone, Debug, PartialEq)]
496pub struct ChildWorkflowResult {
497 pub parent: WorkflowIdentity,
498 pub child: WorkflowIdentity,
499 pub child_workflow_type: Option<String>,
500 pub result: Value,
501}
502
503#[derive(Clone, Debug, PartialEq)]
505pub struct ChildWorkflowAvroResult {
506 pub parent: WorkflowIdentity,
507 pub child: WorkflowIdentity,
508 pub child_workflow_type: Option<String>,
509 pub result: AvroValue,
510}
511
512#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
514pub enum ParentClosePolicy {
515 #[default]
516 Abandon,
517 RequestCancel,
518 Terminate,
519}
520
521impl ParentClosePolicy {
522 fn as_str(self) -> &'static str {
523 match self {
524 Self::Abandon => "abandon",
525 Self::RequestCancel => "request_cancel",
526 Self::Terminate => "terminate",
527 }
528 }
529}
530
531#[derive(Clone, Debug, Default, PartialEq, Eq)]
533pub struct ChildWorkflowRetryPolicy {
534 pub max_attempts: Option<u32>,
535 pub backoff_seconds: Vec<u64>,
536 pub non_retryable_error_types: Vec<String>,
537}
538
539#[derive(Clone, Debug, PartialEq, Eq)]
543pub struct ChildWorkflowOptions {
544 pub task_queue: String,
545 pub parent_close_policy: ParentClosePolicy,
546 pub retry_policy: Option<ChildWorkflowRetryPolicy>,
547 pub execution_timeout_seconds: Option<u64>,
548 pub run_timeout_seconds: Option<u64>,
549}
550
551impl ChildWorkflowOptions {
552 pub fn new(task_queue: impl Into<String>) -> Self {
553 Self {
554 task_queue: task_queue.into(),
555 parent_close_policy: ParentClosePolicy::Abandon,
556 retry_policy: None,
557 execution_timeout_seconds: None,
558 run_timeout_seconds: None,
559 }
560 }
561
562 pub fn parent_close_policy(mut self, policy: ParentClosePolicy) -> Self {
563 self.parent_close_policy = policy;
564 self
565 }
566
567 pub fn retry_policy(mut self, policy: ChildWorkflowRetryPolicy) -> Self {
568 self.retry_policy = Some(policy);
569 self
570 }
571
572 pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
573 self.execution_timeout_seconds = Some(seconds);
574 self
575 }
576
577 pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
578 self.run_timeout_seconds = Some(seconds);
579 self
580 }
581}
582
583#[derive(Clone, Debug, PartialEq, Eq)]
585pub enum ActivityBackoff {
586 Explicit(Vec<Duration>),
589 Exponential {
591 initial_interval: Duration,
592 coefficient: u32,
593 maximum_interval: Option<Duration>,
594 },
595}
596
597#[derive(Clone, Debug, Default, PartialEq, Eq)]
599pub struct ActivityRetryPolicy {
600 pub max_attempts: Option<u32>,
601 pub backoff: Option<ActivityBackoff>,
602 pub non_retryable_error_types: Vec<String>,
603}
604
605impl ActivityRetryPolicy {
606 pub fn new(max_attempts: u32) -> Self {
608 Self {
609 max_attempts: Some(max_attempts),
610 ..Self::default()
611 }
612 }
613
614 pub fn backoff_intervals(mut self, intervals: impl IntoIterator<Item = Duration>) -> Self {
615 self.backoff = Some(ActivityBackoff::Explicit(intervals.into_iter().collect()));
616 self
617 }
618
619 pub fn exponential_backoff(
620 mut self,
621 initial_interval: Duration,
622 coefficient: u32,
623 maximum_interval: Option<Duration>,
624 ) -> Self {
625 self.backoff = Some(ActivityBackoff::Exponential {
626 initial_interval,
627 coefficient,
628 maximum_interval,
629 });
630 self
631 }
632
633 pub fn non_retryable_error_type(mut self, error_type: impl Into<String>) -> Self {
634 self.non_retryable_error_types.push(error_type.into());
635 self
636 }
637
638 pub fn non_retryable_error_types(
639 mut self,
640 error_types: impl IntoIterator<Item = impl Into<String>>,
641 ) -> Self {
642 self.non_retryable_error_types
643 .extend(error_types.into_iter().map(Into::into));
644 self
645 }
646}
647
648#[derive(Clone, Debug, Default, PartialEq, Eq)]
653pub struct ActivityOptions {
654 pub task_queue: Option<String>,
655 pub retry_policy: Option<ActivityRetryPolicy>,
656 pub start_to_close_timeout: Option<Duration>,
657 pub schedule_to_start_timeout: Option<Duration>,
658 pub schedule_to_close_timeout: Option<Duration>,
659 pub heartbeat_timeout: Option<Duration>,
660}
661
662impl ActivityOptions {
663 pub fn new() -> Self {
664 Self::default()
665 }
666
667 pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
668 self.task_queue = Some(task_queue.into());
669 self
670 }
671
672 pub fn retry_policy(mut self, policy: ActivityRetryPolicy) -> Self {
673 self.retry_policy = Some(policy);
674 self
675 }
676
677 pub fn start_to_close_timeout(mut self, timeout: Duration) -> Self {
678 self.start_to_close_timeout = Some(timeout);
679 self
680 }
681
682 pub fn schedule_to_start_timeout(mut self, timeout: Duration) -> Self {
683 self.schedule_to_start_timeout = Some(timeout);
684 self
685 }
686
687 pub fn schedule_to_close_timeout(mut self, timeout: Duration) -> Self {
688 self.schedule_to_close_timeout = Some(timeout);
689 self
690 }
691
692 pub fn heartbeat_timeout(mut self, timeout: Duration) -> Self {
693 self.heartbeat_timeout = Some(timeout);
694 self
695 }
696
697 fn validate(&self) -> std::result::Result<ValidatedActivityOptions, ActivityOptionsError> {
698 if self
699 .task_queue
700 .as_deref()
701 .is_some_and(|queue| queue.trim().is_empty())
702 {
703 return Err(ActivityOptionsError::new(
704 ActivityOptionsErrorKind::EmptyTaskQueue,
705 Some("task_queue"),
706 "task_queue must not be empty",
707 ));
708 }
709
710 for (field, value) in [
711 ("start_to_close_timeout", self.start_to_close_timeout),
712 ("schedule_to_start_timeout", self.schedule_to_start_timeout),
713 ("schedule_to_close_timeout", self.schedule_to_close_timeout),
714 ("heartbeat_timeout", self.heartbeat_timeout),
715 ] {
716 if value.is_some_and(|value| value.is_zero()) {
717 return Err(ActivityOptionsError::new(
718 ActivityOptionsErrorKind::TimeoutNotPositive,
719 Some(field),
720 format!("{field} must be positive"),
721 ));
722 }
723 }
724
725 validate_timeout_order(
726 "heartbeat_timeout",
727 self.heartbeat_timeout,
728 "start_to_close_timeout",
729 self.start_to_close_timeout,
730 )?;
731 validate_timeout_order(
732 "start_to_close_timeout",
733 self.start_to_close_timeout,
734 "schedule_to_close_timeout",
735 self.schedule_to_close_timeout,
736 )?;
737 validate_timeout_order(
738 "schedule_to_start_timeout",
739 self.schedule_to_start_timeout,
740 "schedule_to_close_timeout",
741 self.schedule_to_close_timeout,
742 )?;
743
744 Ok(ValidatedActivityOptions {
745 task_queue: self.task_queue.clone(),
746 retry_policy: self
747 .retry_policy
748 .as_ref()
749 .map(validate_activity_retry_policy)
750 .transpose()?,
751 start_to_close_timeout: timeout_seconds(
752 "start_to_close_timeout",
753 self.start_to_close_timeout,
754 )?,
755 schedule_to_start_timeout: timeout_seconds(
756 "schedule_to_start_timeout",
757 self.schedule_to_start_timeout,
758 )?,
759 schedule_to_close_timeout: timeout_seconds(
760 "schedule_to_close_timeout",
761 self.schedule_to_close_timeout,
762 )?,
763 heartbeat_timeout: timeout_seconds("heartbeat_timeout", self.heartbeat_timeout)?,
764 })
765 }
766}
767
768#[derive(Clone, Debug)]
769struct ValidatedActivityOptions {
770 task_queue: Option<String>,
771 retry_policy: Option<Value>,
772 start_to_close_timeout: Option<u64>,
773 schedule_to_start_timeout: Option<u64>,
774 schedule_to_close_timeout: Option<u64>,
775 heartbeat_timeout: Option<u64>,
776}
777
778fn validate_timeout_order(
779 smaller_name: &'static str,
780 smaller: Option<Duration>,
781 larger_name: &'static str,
782 larger: Option<Duration>,
783) -> std::result::Result<(), ActivityOptionsError> {
784 if matches!((smaller, larger), (Some(smaller), Some(larger)) if smaller > larger) {
785 return Err(ActivityOptionsError::new(
786 ActivityOptionsErrorKind::TimeoutOrder,
787 Some(smaller_name),
788 format!("{smaller_name} must be <= {larger_name}"),
789 ));
790 }
791 Ok(())
792}
793
794fn timeout_seconds(
795 field: &'static str,
796 value: Option<Duration>,
797) -> std::result::Result<Option<u64>, ActivityOptionsError> {
798 value
799 .map(|value| {
800 activity_protocol_seconds(value).ok_or_else(|| {
801 ActivityOptionsError::new(
802 ActivityOptionsErrorKind::TimeoutOverflow,
803 Some(field),
804 format!("{field} is too large for the worker protocol"),
805 )
806 })
807 })
808 .transpose()
809}
810
811fn duration_seconds_ceil(value: Duration) -> Option<u64> {
812 value
813 .as_secs()
814 .checked_add(u64::from(value.subsec_nanos() > 0))
815}
816
817fn activity_protocol_seconds(value: Duration) -> Option<u64> {
818 duration_seconds_ceil(value).filter(|seconds| *seconds <= i64::MAX as u64)
819}
820
821fn validate_activity_retry_policy(
822 policy: &ActivityRetryPolicy,
823) -> std::result::Result<Value, ActivityOptionsError> {
824 if policy.max_attempts.is_none()
825 && policy.backoff.is_none()
826 && policy.non_retryable_error_types.is_empty()
827 {
828 return Err(ActivityOptionsError::new(
829 ActivityOptionsErrorKind::EmptyRetryPolicy,
830 Some("retry_policy"),
831 "retry_policy must configure at least one field",
832 ));
833 }
834 if policy.max_attempts == Some(0) {
835 return Err(ActivityOptionsError::new(
836 ActivityOptionsErrorKind::InvalidMaxAttempts,
837 Some("retry_policy.max_attempts"),
838 "max_attempts must be >= 1",
839 ));
840 }
841 if policy
842 .non_retryable_error_types
843 .iter()
844 .any(|error_type| error_type.trim().is_empty())
845 {
846 return Err(ActivityOptionsError::new(
847 ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
848 Some("retry_policy.non_retryable_error_types"),
849 "non_retryable_error_types must not contain empty values",
850 ));
851 }
852
853 let backoff_seconds = match &policy.backoff {
854 None => None,
855 Some(backoff) => {
856 let max_attempts = policy.max_attempts.ok_or_else(|| {
857 ActivityOptionsError::new(
858 ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
859 Some("retry_policy.backoff"),
860 "backoff requires max_attempts",
861 )
862 })?;
863 let retry_count = max_attempts.saturating_sub(1) as usize;
864 let intervals = match backoff {
865 ActivityBackoff::Explicit(intervals) => {
866 if intervals.len() > retry_count {
867 return Err(ActivityOptionsError::new(
868 ActivityOptionsErrorKind::TooManyBackoffIntervals,
869 Some("retry_policy.backoff"),
870 "backoff interval count must not exceed max_attempts - 1",
871 ));
872 }
873 intervals.clone()
874 }
875 ActivityBackoff::Exponential {
876 initial_interval,
877 coefficient,
878 maximum_interval,
879 } => {
880 if *coefficient < 1 {
881 return Err(ActivityOptionsError::new(
882 ActivityOptionsErrorKind::InvalidBackoffCoefficient,
883 Some("retry_policy.backoff.coefficient"),
884 "backoff coefficient must be >= 1",
885 ));
886 }
887 if retry_count > 10_000 {
888 return Err(ActivityOptionsError::new(
889 ActivityOptionsErrorKind::BackoffGenerationTooLarge,
890 Some("retry_policy.max_attempts"),
891 "generated backoff supports at most 10000 retry intervals",
892 ));
893 }
894 let mut current = *initial_interval;
895 let mut intervals = Vec::with_capacity(retry_count);
896 for _ in 0..retry_count {
897 let interval = maximum_interval
898 .map(|maximum| current.min(maximum))
899 .unwrap_or(current);
900 intervals.push(interval);
901 if maximum_interval.is_some_and(|maximum| interval == maximum) {
902 break;
903 }
904 current = current.checked_mul(*coefficient).ok_or_else(|| {
905 ActivityOptionsError::new(
906 ActivityOptionsErrorKind::BackoffOverflow,
907 Some("retry_policy.backoff"),
908 "generated backoff interval overflowed",
909 )
910 })?;
911 }
912 intervals
913 }
914 };
915 Some(
916 intervals
917 .into_iter()
918 .map(|interval| {
919 activity_protocol_seconds(interval).ok_or_else(|| {
920 ActivityOptionsError::new(
921 ActivityOptionsErrorKind::BackoffOverflow,
922 Some("retry_policy.backoff"),
923 "backoff interval is too large for the worker protocol",
924 )
925 })
926 })
927 .collect::<std::result::Result<Vec<_>, _>>()?,
928 )
929 }
930 };
931
932 let mut encoded = serde_json::Map::new();
933 if let Some(max_attempts) = policy.max_attempts {
934 encoded.insert("max_attempts".to_string(), json!(max_attempts));
935 }
936 if let Some(backoff_seconds) = backoff_seconds {
937 encoded.insert("backoff_seconds".to_string(), json!(backoff_seconds));
938 }
939 if !policy.non_retryable_error_types.is_empty() {
940 let mut canonical_error_types = Vec::new();
941 for error_type in policy
942 .non_retryable_error_types
943 .iter()
944 .map(|error_type| error_type.trim())
945 {
946 if !canonical_error_types.contains(&error_type) {
947 canonical_error_types.push(error_type);
948 }
949 }
950 encoded.insert(
951 "non_retryable_error_types".to_string(),
952 json!(canonical_error_types),
953 );
954 }
955 Ok(Value::Object(encoded))
956}
957
958#[derive(Clone, Debug, Error)]
961#[error("non-deterministic workflow replay ({reason}) at sequence {sequence:?}: {message}")]
962pub struct ReplayFailure {
963 pub reason: String,
964 pub sequence: Option<u64>,
965 pub expected: Option<String>,
966 pub actual: Option<String>,
967 pub message: String,
968}
969
970impl ReplayFailure {
971 fn new(
972 reason: impl Into<String>,
973 sequence: Option<u64>,
974 expected: Option<String>,
975 actual: Option<String>,
976 message: impl Into<String>,
977 ) -> Self {
978 Self {
979 reason: reason.into(),
980 sequence,
981 expected,
982 actual,
983 message: message.into(),
984 }
985 }
986}
987
988#[derive(Clone, Debug, Error)]
990#[error("query failed ({reason}, HTTP {status}): {message}")]
991pub struct QueryFailure {
992 pub status: u16,
993 pub reason: String,
994 pub message: String,
995 pub body: Value,
996}
997
998#[derive(Clone, Debug, Error)]
1000#[error("protocol rejected ({reason}, HTTP {status}): {message}")]
1001pub struct ProtocolFailure {
1002 pub status: u16,
1003 pub reason: String,
1004 pub message: String,
1005 pub supported_version: Option<String>,
1006 pub requested_version: Option<String>,
1007 pub body: Value,
1008}
1009
1010#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
1011pub struct PayloadEnvelope {
1012 pub codec: String,
1013 pub blob: String,
1014}
1015
1016impl PayloadEnvelope {
1017 pub fn avro<T: Serialize>(value: &T) -> Result<Self> {
1018 encode_payload(value, DEFAULT_CODEC)
1019 }
1020
1021 pub fn json<T: Serialize>(value: &T) -> Result<Self> {
1022 encode_payload(value, JSON_CODEC)
1023 }
1024
1025 pub fn avro_value(value: &AvroValue) -> Result<Self> {
1028 encode_avro_value(value)
1029 }
1030}
1031
1032#[derive(Clone, Debug, PartialEq)]
1034pub enum AvroValue {
1035 Null,
1036 Boolean(bool),
1037 Long(i64),
1038 Double(f64),
1039 Bytes(Vec<u8>),
1040 String(String),
1041 Array(Vec<AvroValue>),
1042 Map(BTreeMap<String, AvroValue>),
1043}
1044
1045impl AvroValue {
1046 fn from_serialize<T: Serialize>(value: &T) -> Result<Self> {
1047 Self::from_serde_value(
1048 serde_value::to_value(value).map_err(|error| {
1049 Error::Codec(format!("could not adapt value for Avro: {error}"))
1050 })?,
1051 )
1052 }
1053
1054 fn from_serde_value(value: serde_value::Value) -> Result<Self> {
1055 use serde_value::Value as SerdeValue;
1056
1057 match value {
1058 SerdeValue::Unit => Ok(Self::Null),
1059 SerdeValue::Bool(value) => Ok(Self::Boolean(value)),
1060 SerdeValue::I8(value) => Ok(Self::Long(i64::from(value))),
1061 SerdeValue::I16(value) => Ok(Self::Long(i64::from(value))),
1062 SerdeValue::I32(value) => Ok(Self::Long(i64::from(value))),
1063 SerdeValue::I64(value) => Ok(Self::Long(value)),
1064 SerdeValue::U8(value) => Ok(Self::Long(i64::from(value))),
1065 SerdeValue::U16(value) => Ok(Self::Long(i64::from(value))),
1066 SerdeValue::U32(value) => Ok(Self::Long(i64::from(value))),
1067 SerdeValue::U64(value) => i64::try_from(value).map(Self::Long).map_err(|_| {
1068 Error::Codec(
1069 "integer_overflow: Avro Value long must be within signed 64-bit range"
1070 .to_string(),
1071 )
1072 }),
1073 SerdeValue::F32(value) => Self::finite_double(f64::from(value)),
1074 SerdeValue::F64(value) => Self::finite_double(value),
1075 SerdeValue::Char(value) => Ok(Self::String(value.to_string())),
1076 SerdeValue::String(value) => Ok(Self::String(value)),
1077 SerdeValue::Bytes(value) => Ok(Self::Bytes(value)),
1078 SerdeValue::Option(None) => Ok(Self::Null),
1079 SerdeValue::Option(Some(value)) | SerdeValue::Newtype(value) => {
1080 Self::from_serde_value(*value)
1081 }
1082 SerdeValue::Seq(values) => values
1083 .into_iter()
1084 .map(Self::from_serde_value)
1085 .collect::<Result<Vec<_>>>()
1086 .map(Self::Array),
1087 SerdeValue::Map(values) => values
1088 .into_iter()
1089 .map(|(key, value)| {
1090 let SerdeValue::String(key) = key else {
1091 return Err(Error::Codec(
1092 "invalid_map_key: Avro Value map keys must be strings".to_string(),
1093 ));
1094 };
1095
1096 Ok((key, Self::from_serde_value(value)?))
1097 })
1098 .collect::<Result<BTreeMap<_, _>>>()
1099 .map(Self::Map),
1100 }
1101 }
1102
1103 fn finite_double(value: f64) -> Result<Self> {
1104 if !value.is_finite() {
1105 return Err(Error::Codec(
1106 "non_finite_float: Avro Value doubles must be finite".to_string(),
1107 ));
1108 }
1109
1110 Ok(Self::Double(value))
1111 }
1112
1113 fn into_json(self) -> Result<Value> {
1114 match self {
1115 Self::Null => Ok(Value::Null),
1116 Self::Boolean(value) => Ok(Value::Bool(value)),
1117 Self::Long(value) => Ok(Value::Number(value.into())),
1118 Self::Double(value) => serde_json::Number::from_f64(value)
1119 .map(Value::Number)
1120 .ok_or_else(|| {
1121 Error::Codec(
1122 "non_finite_float: decoded Avro Value double is not finite".to_string(),
1123 )
1124 }),
1125 Self::Bytes(value) => Ok(json!({
1126 "$type": "bytes",
1127 "base64": BASE64.encode(value),
1128 })),
1129 Self::String(value) => Ok(Value::String(value)),
1130 Self::Array(values) => values
1131 .into_iter()
1132 .map(Self::into_json)
1133 .collect::<Result<Vec<_>>>()
1134 .map(Value::Array),
1135 Self::Map(values) => values
1136 .into_iter()
1137 .map(|(key, value)| Ok((key, value.into_json()?)))
1138 .collect::<Result<serde_json::Map<_, _>>>()
1139 .map(Value::Object),
1140 }
1141 }
1142
1143 fn into_serde_value(self) -> serde_value::Value {
1144 use serde_value::Value as SerdeValue;
1145
1146 match self {
1147 Self::Null => SerdeValue::Unit,
1148 Self::Boolean(value) => SerdeValue::Bool(value),
1149 Self::Long(value) => SerdeValue::I64(value),
1150 Self::Double(value) => SerdeValue::F64(value),
1151 Self::Bytes(value) => SerdeValue::Bytes(value),
1152 Self::String(value) => SerdeValue::String(value),
1153 Self::Array(values) => {
1154 SerdeValue::Seq(values.into_iter().map(Self::into_serde_value).collect())
1155 }
1156 Self::Map(values) => SerdeValue::Map(
1157 values
1158 .into_iter()
1159 .map(|(key, value)| (SerdeValue::String(key), value.into_serde_value()))
1160 .collect(),
1161 ),
1162 }
1163 }
1164
1165 pub fn deserialize<T: DeserializeOwned>(self) -> Result<T> {
1166 self.into_serde_value().deserialize_into().map_err(|error| {
1167 Error::Codec(format!(
1168 "avro_value_type_mismatch: could not adapt decoded value: {error}"
1169 ))
1170 })
1171 }
1172}
1173
1174impl Serialize for AvroValue {
1175 fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
1176 where
1177 S: Serializer,
1178 {
1179 match self {
1180 Self::Null => serializer.serialize_unit(),
1181 Self::Boolean(value) => serializer.serialize_bool(*value),
1182 Self::Long(value) => serializer.serialize_i64(*value),
1183 Self::Double(value) => serializer.serialize_f64(*value),
1184 Self::Bytes(value) => serializer.serialize_bytes(value),
1185 Self::String(value) => serializer.serialize_str(value),
1186 Self::Array(values) => {
1187 let mut sequence = serializer.serialize_seq(Some(values.len()))?;
1188 for value in values {
1189 sequence.serialize_element(value)?;
1190 }
1191 sequence.end()
1192 }
1193 Self::Map(values) => {
1194 let mut map = serializer.serialize_map(Some(values.len()))?;
1195 for (key, value) in values {
1196 map.serialize_entry(key, value)?;
1197 }
1198 map.end()
1199 }
1200 }
1201 }
1202}
1203
1204pub fn encode_avro_value(value: &AvroValue) -> Result<PayloadEnvelope> {
1205 let datum = avro_value_to_datum(value)?;
1206 let datum = to_avro_datum(avro_value_schema()?, datum)
1207 .map_err(|err| Error::Codec(format!("avro_value_encode_failed: {err}")))?;
1208 let mut bytes = Vec::with_capacity(datum.len() + 10);
1209 bytes.extend_from_slice(&AVRO_SINGLE_OBJECT_MAGIC);
1210 bytes.extend_from_slice(&AVRO_VALUE_SCHEMA_FINGERPRINT);
1211 bytes.extend_from_slice(&datum);
1212 Ok(PayloadEnvelope {
1213 codec: DEFAULT_CODEC.to_string(),
1214 blob: BASE64.encode(bytes),
1215 })
1216}
1217
1218pub fn decode_avro_value(envelope: &PayloadEnvelope) -> Result<AvroValue> {
1219 if envelope.codec != DEFAULT_CODEC {
1220 return Err(Error::Codec(format!(
1221 "unsupported payload codec {:?}",
1222 envelope.codec
1223 )));
1224 }
1225 decode_avro_value_blob(&envelope.blob)
1226}
1227
1228pub fn encode_payload<T: Serialize>(value: &T, codec: &str) -> Result<PayloadEnvelope> {
1229 let blob = match codec {
1230 JSON_CODEC => serde_json::to_string(value)?,
1231 DEFAULT_CODEC => encode_avro_value(&AvroValue::from_serialize(value)?)?.blob,
1232 other => return Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1233 };
1234
1235 Ok(PayloadEnvelope {
1236 codec: codec.to_string(),
1237 blob,
1238 })
1239}
1240
1241pub fn decode_payload<T: DeserializeOwned>(envelope: &PayloadEnvelope) -> Result<T> {
1242 match envelope.codec.as_str() {
1243 JSON_CODEC => Ok(serde_json::from_str(&envelope.blob)?),
1244 DEFAULT_CODEC => decode_avro_value(envelope)?.deserialize(),
1245 other => Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1246 }
1247}
1248
1249#[cfg(test)]
1250fn encode_value_envelope(value: &Value, codec: &str) -> Result<Value> {
1251 Ok(serde_json::to_value(encode_payload(value, codec)?)?)
1252}
1253
1254fn decode_wire_value(value: &Value, fallback_codec: &str) -> Result<Value> {
1255 if value.is_null() {
1256 return Ok(Value::Null);
1257 }
1258
1259 if let Some(object) = value.as_object() {
1260 if let (Some(codec), Some(blob)) = (
1261 object.get("codec").and_then(Value::as_str),
1262 object.get("blob").and_then(Value::as_str),
1263 ) {
1264 return decode_blob(blob, codec);
1265 }
1266 }
1267
1268 if let Some(blob) = value.as_str() {
1269 return decode_blob(blob, fallback_codec);
1270 }
1271
1272 Ok(value.clone())
1273}
1274
1275fn encode_typed_envelope(value: &AvroValue, codec: &str) -> Result<Value> {
1276 let envelope = match codec {
1277 DEFAULT_CODEC => encode_avro_value(value)?,
1278 JSON_CODEC => PayloadEnvelope {
1279 codec: JSON_CODEC.to_string(),
1280 blob: serde_json::to_string(&value.clone().into_json()?)?,
1281 },
1282 other => return Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1283 };
1284 Ok(serde_json::to_value(envelope)?)
1285}
1286
1287fn decode_wire_avro_value(value: &Value, fallback_codec: &str) -> Result<AvroValue> {
1288 if value.is_null() {
1289 return Ok(AvroValue::Null);
1290 }
1291
1292 if let Some(object) = value.as_object() {
1293 if let (Some(codec), Some(blob)) = (
1294 object.get("codec").and_then(Value::as_str),
1295 object.get("blob").and_then(Value::as_str),
1296 ) {
1297 return match codec {
1298 DEFAULT_CODEC => decode_avro_value_blob(blob),
1299 JSON_CODEC => AvroValue::from_serialize(&serde_json::from_str::<Value>(blob)?),
1300 other => Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1301 };
1302 }
1303 }
1304
1305 if let Some(blob) = value.as_str() {
1306 return match fallback_codec {
1307 DEFAULT_CODEC => decode_avro_value_blob(blob),
1308 JSON_CODEC => AvroValue::from_serialize(&serde_json::from_str::<Value>(blob)?),
1309 other => Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1310 };
1311 }
1312
1313 AvroValue::from_serialize(value)
1314}
1315
1316fn normalize_avro_arguments(value: AvroValue) -> AvroValue {
1317 match value {
1318 AvroValue::Null => AvroValue::Array(Vec::new()),
1319 AvroValue::Array(_) => value,
1320 other => AvroValue::Array(vec![other]),
1321 }
1322}
1323
1324fn decode_blob(blob: &str, codec: &str) -> Result<Value> {
1325 match codec {
1326 JSON_CODEC => Ok(serde_json::from_str(blob)?),
1327 DEFAULT_CODEC => decode_avro_value_blob(blob)?.into_json(),
1328 other => Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1329 }
1330}
1331
1332fn decode_avro_value_blob(blob: &str) -> Result<AvroValue> {
1333 let bytes = BASE64.decode(blob).map_err(|err| {
1334 Error::Codec(format!(
1335 "invalid_payload_framing: expected strict base64 Avro single-object bytes: {err}"
1336 ))
1337 })?;
1338
1339 if bytes.len() < 10 || bytes[..2] != AVRO_SINGLE_OBJECT_MAGIC {
1340 return Err(Error::Codec(
1341 "invalid_payload_framing: expected Avro single-object magic c301".to_string(),
1342 ));
1343 }
1344
1345 let fingerprint: [u8; 8] = bytes[2..10]
1346 .try_into()
1347 .map_err(|_| Error::Codec("invalid Avro fingerprint length".to_string()))?;
1348 if fingerprint != AVRO_VALUE_SCHEMA_FINGERPRINT {
1349 return Err(Error::Codec(format!(
1350 "unsupported_payload_schema: unknown CRC-64-AVRO fingerprint {}",
1351 fingerprint
1352 .iter()
1353 .map(|byte| format!("{byte:02x}"))
1354 .collect::<String>()
1355 )));
1356 }
1357
1358 let mut datum_reader = StrictAvroDatumReader::new(&bytes[10..]);
1359 let datum = from_avro_datum(avro_value_schema()?, &mut datum_reader, None);
1363 if datum_reader.truncated {
1364 return Err(Error::Codec(
1365 "invalid_payload_framing: truncated Avro Value datum".to_string(),
1366 ));
1367 }
1368 let datum = datum.map_err(|err| {
1369 Error::Codec(format!(
1370 "invalid_payload_framing: malformed Avro Value datum: {err}"
1371 ))
1372 })?;
1373 if datum_reader.remaining() != 0 {
1374 return Err(Error::Codec(format!(
1375 "invalid_payload_framing: {} trailing bytes after Avro Value datum",
1376 datum_reader.remaining()
1377 )));
1378 }
1379 avro_value_from_datum(datum)
1380}
1381
1382struct StrictAvroDatumReader<'a> {
1383 bytes: &'a [u8],
1384 offset: usize,
1385 truncated: bool,
1386}
1387
1388impl<'a> StrictAvroDatumReader<'a> {
1389 fn new(bytes: &'a [u8]) -> Self {
1390 Self {
1391 bytes,
1392 offset: 0,
1393 truncated: false,
1394 }
1395 }
1396
1397 fn remaining(&self) -> usize {
1398 self.bytes.len() - self.offset
1399 }
1400}
1401
1402impl Read for StrictAvroDatumReader<'_> {
1403 fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
1404 let count = buffer.len().min(self.remaining());
1405 buffer[..count].copy_from_slice(&self.bytes[self.offset..self.offset + count]);
1406 self.offset += count;
1407 if count < buffer.len() {
1408 self.truncated = true;
1409 }
1410
1411 Ok(count)
1412 }
1413}
1414
1415fn avro_value_to_datum(value: &AvroValue) -> Result<AvroDatum> {
1416 let branch = match value {
1417 AvroValue::Null => AvroDatum::Union(0, Box::new(AvroDatum::Null)),
1418 AvroValue::Boolean(value) => AvroDatum::Union(
1419 1,
1420 Box::new(AvroDatum::Record(vec![(
1421 "boolean".to_string(),
1422 AvroDatum::Boolean(*value),
1423 )])),
1424 ),
1425 AvroValue::Long(value) => AvroDatum::Union(
1426 2,
1427 Box::new(AvroDatum::Record(vec![(
1428 "long".to_string(),
1429 AvroDatum::Long(*value),
1430 )])),
1431 ),
1432 AvroValue::Double(value) => {
1433 if !value.is_finite() {
1434 return Err(Error::Codec(
1435 "non_finite_float: Avro Value doubles must be finite".to_string(),
1436 ));
1437 }
1438 AvroDatum::Union(
1439 3,
1440 Box::new(AvroDatum::Record(vec![(
1441 "double".to_string(),
1442 AvroDatum::Double(*value),
1443 )])),
1444 )
1445 }
1446 AvroValue::Bytes(value) => AvroDatum::Union(
1447 4,
1448 Box::new(AvroDatum::Record(vec![(
1449 "bytes".to_string(),
1450 AvroDatum::Bytes(value.clone()),
1451 )])),
1452 ),
1453 AvroValue::String(value) => AvroDatum::Union(
1454 5,
1455 Box::new(AvroDatum::Record(vec![(
1456 "string".to_string(),
1457 AvroDatum::String(value.clone()),
1458 )])),
1459 ),
1460 AvroValue::Array(values) => AvroDatum::Union(
1461 6,
1462 Box::new(AvroDatum::Record(vec![(
1463 "items".to_string(),
1464 AvroDatum::Array(
1465 values
1466 .iter()
1467 .map(avro_value_to_datum)
1468 .collect::<Result<Vec<_>>>()?,
1469 ),
1470 )])),
1471 ),
1472 AvroValue::Map(values) => AvroDatum::Union(
1473 7,
1474 Box::new(AvroDatum::Record(vec![(
1475 "entries".to_string(),
1476 AvroDatum::Map(
1477 values
1478 .iter()
1479 .map(|(key, value)| Ok((key.clone(), avro_value_to_datum(value)?)))
1480 .collect::<Result<HashMap<_, _>>>()?,
1481 ),
1482 )])),
1483 ),
1484 };
1485 Ok(AvroDatum::Record(vec![("value".to_string(), branch)]))
1486}
1487
1488fn avro_value_from_datum(datum: AvroDatum) -> Result<AvroValue> {
1489 let AvroDatum::Record(mut outer) = datum else {
1490 return Err(Error::Codec(
1491 "invalid_payload_framing: datum is not a Value record".to_string(),
1492 ));
1493 };
1494 let (_, branch) = outer
1495 .pop()
1496 .filter(|(name, _)| name == "value")
1497 .ok_or_else(|| Error::Codec("invalid_payload_framing: Value field missing".to_string()))?;
1498 let AvroDatum::Union(_, branch) = branch else {
1499 return Err(Error::Codec(
1500 "invalid_payload_framing: invalid Value union".to_string(),
1501 ));
1502 };
1503 match *branch {
1504 AvroDatum::Null => Ok(AvroValue::Null),
1505 AvroDatum::Record(mut fields) => {
1506 let (name, value) = fields.pop().ok_or_else(|| {
1507 Error::Codec("invalid_payload_framing: empty Value branch".to_string())
1508 })?;
1509 match (name.as_str(), value) {
1510 ("boolean", AvroDatum::Boolean(value)) => Ok(AvroValue::Boolean(value)),
1511 ("long", AvroDatum::Long(value)) => Ok(AvroValue::Long(value)),
1512 ("double", AvroDatum::Double(value)) if value.is_finite() => {
1513 Ok(AvroValue::Double(value))
1514 }
1515 ("bytes", AvroDatum::Bytes(value)) => Ok(AvroValue::Bytes(value)),
1516 ("string", AvroDatum::String(value)) => Ok(AvroValue::String(value)),
1517 ("items", AvroDatum::Array(values)) => values
1518 .into_iter()
1519 .map(avro_value_from_datum)
1520 .collect::<Result<Vec<_>>>()
1521 .map(AvroValue::Array),
1522 ("entries", AvroDatum::Map(values)) => values
1523 .into_iter()
1524 .map(|(key, value)| Ok((key, avro_value_from_datum(value)?)))
1525 .collect::<Result<BTreeMap<_, _>>>()
1526 .map(AvroValue::Map),
1527 _ => Err(Error::Codec(
1528 "invalid_payload_framing: unknown Value branch".to_string(),
1529 )),
1530 }
1531 }
1532 _ => Err(Error::Codec(
1533 "invalid_payload_framing: invalid Value branch".to_string(),
1534 )),
1535 }
1536}
1537
1538fn avro_value_schema() -> Result<&'static Schema> {
1539 match AVRO_VALUE_SCHEMA.get_or_init(|| {
1540 Schema::parse_str(AVRO_VALUE_SCHEMA_JSON)
1541 .map_err(|err| format!("could not parse Avro Value schema: {err}"))
1542 }) {
1543 Ok(schema) => Ok(schema),
1544 Err(message) => Err(Error::Codec(message.clone())),
1545 }
1546}
1547
1548#[derive(Clone, Debug)]
1549pub struct Client {
1550 http: reqwest::Client,
1551 base_url: String,
1552 token: Option<String>,
1553 control_token: Option<String>,
1554 worker_token: Option<String>,
1555 namespace: String,
1556}
1557
1558impl Client {
1559 pub fn new(base_url: impl Into<String>) -> Result<Self> {
1560 Self::builder(base_url).build()
1561 }
1562
1563 pub fn builder(base_url: impl Into<String>) -> ClientBuilder {
1564 ClientBuilder {
1565 base_url: base_url.into(),
1566 token: None,
1567 control_token: None,
1568 worker_token: None,
1569 namespace: "default".to_string(),
1570 timeout: Duration::from_secs(60),
1571 }
1572 }
1573
1574 pub async fn health(&self) -> Result<Value> {
1575 self.request_json(
1576 reqwest::Method::GET,
1577 "/health",
1578 RequestProtocol::ControlPlane,
1579 Option::<&Value>::None,
1580 )
1581 .await
1582 }
1583
1584 pub async fn cluster_info(&self) -> Result<Value> {
1585 self.request_json(
1586 reqwest::Method::GET,
1587 "/cluster/info",
1588 RequestProtocol::ControlPlane,
1589 Option::<&Value>::None,
1590 )
1591 .await
1592 }
1593
1594 pub async fn start_workflow<T: Serialize>(
1595 &self,
1596 workflow_type: &str,
1597 task_queue: &str,
1598 workflow_id: &str,
1599 input: T,
1600 ) -> Result<WorkflowHandle> {
1601 self.start_workflow_with_options(
1602 workflow_type,
1603 task_queue,
1604 workflow_id,
1605 WorkflowStartOptions::default(),
1606 input,
1607 )
1608 .await
1609 }
1610
1611 pub async fn start_workflow_with_options<T: Serialize>(
1614 &self,
1615 workflow_type: &str,
1616 task_queue: &str,
1617 workflow_id: &str,
1618 options: WorkflowStartOptions,
1619 input: T,
1620 ) -> Result<WorkflowHandle> {
1621 options.validate()?;
1622 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1623 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
1624 let body = json!({
1625 "workflow_id": workflow_id,
1626 "workflow_type": workflow_type,
1627 "task_queue": task_queue,
1628 "input": input_envelope,
1629 "execution_timeout_seconds": options.execution_timeout_seconds,
1630 "run_timeout_seconds": options.run_timeout_seconds
1631 });
1632
1633 let data: Value = self
1634 .request_json(
1635 reqwest::Method::POST,
1636 "/workflows",
1637 RequestProtocol::ControlPlane,
1638 Some(&body),
1639 )
1640 .await?;
1641
1642 Ok(WorkflowHandle {
1643 client: self.clone(),
1644 workflow_id: data
1645 .get("workflow_id")
1646 .and_then(Value::as_str)
1647 .unwrap_or(workflow_id)
1648 .to_string(),
1649 run_id: data
1650 .get("run_id")
1651 .and_then(Value::as_str)
1652 .map(str::to_string),
1653 workflow_type: data
1654 .get("workflow_type")
1655 .and_then(Value::as_str)
1656 .unwrap_or(workflow_type)
1657 .to_string(),
1658 })
1659 }
1660
1661 pub async fn signal_workflow<T: Serialize>(
1662 &self,
1663 workflow_id: &str,
1664 signal_name: &str,
1665 input: T,
1666 ) -> Result<Value> {
1667 self.signal_workflow_target(workflow_id, None, signal_name, input)
1668 .await
1669 }
1670
1671 pub async fn signal_workflow_run<T: Serialize>(
1673 &self,
1674 workflow_id: &str,
1675 run_id: &str,
1676 signal_name: &str,
1677 input: T,
1678 ) -> Result<Value> {
1679 self.signal_workflow_target(workflow_id, Some(run_id), signal_name, input)
1680 .await
1681 }
1682
1683 async fn signal_workflow_target<T: Serialize>(
1684 &self,
1685 workflow_id: &str,
1686 run_id: Option<&str>,
1687 signal_name: &str,
1688 input: T,
1689 ) -> Result<Value> {
1690 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1691 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
1692 let body = json!({
1693 "input": input_envelope
1694 });
1695 let path = match run_id {
1696 Some(run_id) => {
1697 format!("/workflows/{workflow_id}/runs/{run_id}/signal/{signal_name}")
1698 }
1699 None => format!("/workflows/{workflow_id}/signal/{signal_name}"),
1700 };
1701 self.request_json(
1702 reqwest::Method::POST,
1703 &path,
1704 RequestProtocol::ControlPlane,
1705 Some(&body),
1706 )
1707 .await
1708 }
1709
1710 pub async fn cancel_workflow(
1712 &self,
1713 workflow_id: &str,
1714 options: WorkflowCommandOptions,
1715 ) -> Result<WorkflowCommandResult> {
1716 self.workflow_command(workflow_id, None, WorkflowCommandKind::Cancel, options)
1717 .await
1718 }
1719
1720 pub async fn cancel_workflow_run(
1722 &self,
1723 workflow_id: &str,
1724 run_id: &str,
1725 options: WorkflowCommandOptions,
1726 ) -> Result<WorkflowCommandResult> {
1727 self.workflow_command(
1728 workflow_id,
1729 Some(run_id),
1730 WorkflowCommandKind::Cancel,
1731 options,
1732 )
1733 .await
1734 }
1735
1736 pub async fn terminate_workflow(
1738 &self,
1739 workflow_id: &str,
1740 options: WorkflowCommandOptions,
1741 ) -> Result<WorkflowCommandResult> {
1742 self.workflow_command(workflow_id, None, WorkflowCommandKind::Terminate, options)
1743 .await
1744 }
1745
1746 pub async fn terminate_workflow_run(
1748 &self,
1749 workflow_id: &str,
1750 run_id: &str,
1751 options: WorkflowCommandOptions,
1752 ) -> Result<WorkflowCommandResult> {
1753 self.workflow_command(
1754 workflow_id,
1755 Some(run_id),
1756 WorkflowCommandKind::Terminate,
1757 options,
1758 )
1759 .await
1760 }
1761
1762 async fn workflow_command(
1763 &self,
1764 workflow_id: &str,
1765 run_id: Option<&str>,
1766 command: WorkflowCommandKind,
1767 options: WorkflowCommandOptions,
1768 ) -> Result<WorkflowCommandResult> {
1769 let path = match run_id {
1770 Some(run_id) => format!(
1771 "/workflows/{workflow_id}/runs/{run_id}/{}",
1772 command.as_str()
1773 ),
1774 None => format!("/workflows/{workflow_id}/{}", command.as_str()),
1775 };
1776 let data = match self
1777 .request_json(
1778 reqwest::Method::POST,
1779 &path,
1780 RequestProtocol::ControlPlane,
1781 Some(&options),
1782 )
1783 .await
1784 {
1785 Ok(data) => data,
1786 Err(Error::Http { status, body }) => {
1787 return Err(Error::WorkflowCommandRejected(workflow_command_rejection(
1788 command,
1789 status,
1790 body,
1791 workflow_id,
1792 run_id,
1793 )));
1794 }
1795 Err(error) => return Err(error),
1796 };
1797
1798 Ok(workflow_command_result(command, data, workflow_id, run_id))
1799 }
1800
1801 pub async fn query_workflow<T: Serialize>(
1807 &self,
1808 workflow_id: &str,
1809 query_name: &str,
1810 input: T,
1811 ) -> Result<Value> {
1812 self.query_workflow_target(workflow_id, None, query_name, input)
1813 .await
1814 }
1815
1816 pub async fn query_workflow_run<T: Serialize>(
1818 &self,
1819 workflow_id: &str,
1820 run_id: &str,
1821 query_name: &str,
1822 input: T,
1823 ) -> Result<Value> {
1824 self.query_workflow_target(workflow_id, Some(run_id), query_name, input)
1825 .await
1826 }
1827
1828 pub async fn query_workflow_avro_value<T: Serialize>(
1830 &self,
1831 workflow_id: &str,
1832 query_name: &str,
1833 input: T,
1834 ) -> Result<AvroValue> {
1835 self.query_workflow_avro_value_target(workflow_id, None, query_name, input)
1836 .await
1837 }
1838
1839 pub async fn query_workflow_run_avro_value<T: Serialize>(
1841 &self,
1842 workflow_id: &str,
1843 run_id: &str,
1844 query_name: &str,
1845 input: T,
1846 ) -> Result<AvroValue> {
1847 self.query_workflow_avro_value_target(workflow_id, Some(run_id), query_name, input)
1848 .await
1849 }
1850
1851 async fn query_workflow_avro_value_target<T: Serialize>(
1852 &self,
1853 workflow_id: &str,
1854 run_id: Option<&str>,
1855 query_name: &str,
1856 input: T,
1857 ) -> Result<AvroValue> {
1858 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1859 let body = json!({"input": encode_typed_envelope(&input, DEFAULT_CODEC)?});
1860 let path = match run_id {
1861 Some(run_id) => {
1862 format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
1863 }
1864 None => format!("/workflows/{workflow_id}/query/{query_name}"),
1865 };
1866 let response: Value = match self
1867 .request_json(
1868 reqwest::Method::POST,
1869 &path,
1870 RequestProtocol::ControlPlane,
1871 Some(&body),
1872 )
1873 .await
1874 {
1875 Ok(response) => response,
1876 Err(Error::Http { status, body }) => {
1877 return Err(Error::QueryFailed(query_failure(status, body)));
1878 }
1879 Err(error) => return Err(error),
1880 };
1881
1882 let envelope = response
1883 .get("result_envelope")
1884 .filter(|envelope| !envelope.is_null())
1885 .ok_or_else(|| {
1886 Error::Codec(
1887 "missing_payload_envelope: typed query result requires result_envelope"
1888 .to_string(),
1889 )
1890 })?;
1891 decode_wire_avro_value(envelope, DEFAULT_CODEC)
1892 }
1893
1894 async fn query_workflow_target<T: Serialize>(
1895 &self,
1896 workflow_id: &str,
1897 run_id: Option<&str>,
1898 query_name: &str,
1899 input: T,
1900 ) -> Result<Value> {
1901 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1902 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
1903 let body = json!({
1904 "input": input_envelope
1905 });
1906 let path = match run_id {
1907 Some(run_id) => {
1908 format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
1909 }
1910 None => format!("/workflows/{workflow_id}/query/{query_name}"),
1911 };
1912 let response: Value = match self
1913 .request_json(
1914 reqwest::Method::POST,
1915 &path,
1916 RequestProtocol::ControlPlane,
1917 Some(&body),
1918 )
1919 .await
1920 {
1921 Ok(response) => response,
1922 Err(Error::Http { status, body }) => {
1923 return Err(Error::QueryFailed(query_failure(status, body)));
1924 }
1925 Err(error) => return Err(error),
1926 };
1927
1928 if let Some(envelope) = response
1929 .get("result_envelope")
1930 .filter(|envelope| !envelope.is_null())
1931 {
1932 return decode_wire_value(envelope, DEFAULT_CODEC);
1933 }
1934
1935 Ok(response.get("result").cloned().unwrap_or(Value::Null))
1936 }
1937
1938 pub async fn update_workflow<T: Serialize>(
1940 &self,
1941 workflow_id: &str,
1942 update_name: &str,
1943 input: T,
1944 request_id: Option<&str>,
1945 ) -> Result<Value> {
1946 let response = self
1947 .update_workflow_response(workflow_id, update_name, input, request_id)
1948 .await?;
1949 if let Some(envelope) = response
1950 .get("result_envelope")
1951 .filter(|envelope| !envelope.is_null())
1952 {
1953 return decode_wire_value(envelope, DEFAULT_CODEC);
1954 }
1955 Ok(response.get("result").cloned().unwrap_or(response))
1956 }
1957
1958 pub async fn update_workflow_avro_value<T: Serialize>(
1960 &self,
1961 workflow_id: &str,
1962 update_name: &str,
1963 input: T,
1964 request_id: Option<&str>,
1965 ) -> Result<AvroValue> {
1966 let response = self
1967 .update_workflow_response(workflow_id, update_name, input, request_id)
1968 .await?;
1969 let envelope = response
1970 .get("result_envelope")
1971 .filter(|envelope| !envelope.is_null())
1972 .ok_or_else(|| {
1973 Error::Codec(
1974 "missing_payload_envelope: typed update result requires result_envelope"
1975 .to_string(),
1976 )
1977 })?;
1978 decode_wire_avro_value(envelope, DEFAULT_CODEC)
1979 }
1980
1981 async fn update_workflow_response<T: Serialize>(
1982 &self,
1983 workflow_id: &str,
1984 update_name: &str,
1985 input: T,
1986 request_id: Option<&str>,
1987 ) -> Result<Value> {
1988 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1989 let mut body = json!({
1990 "input": encode_typed_envelope(&input, DEFAULT_CODEC)?,
1991 "wait_for": "completed",
1992 });
1993 if let Some(request_id) = request_id {
1994 body["request_id"] = json!(request_id);
1995 }
1996 self.request_json(
1997 reqwest::Method::POST,
1998 &format!("/workflows/{workflow_id}/update/{update_name}"),
1999 RequestProtocol::ControlPlane,
2000 Some(&body),
2001 )
2002 .await
2003 }
2004
2005 pub async fn describe_workflow(&self, workflow_id: &str) -> Result<WorkflowDescription> {
2006 let path = format!("/workflows/{workflow_id}");
2007 let mut data: WorkflowDescription = self
2008 .request_json(
2009 reqwest::Method::GET,
2010 &path,
2011 RequestProtocol::ControlPlane,
2012 Option::<&Value>::None,
2013 )
2014 .await?;
2015 data.decode_payloads()?;
2016 Ok(data)
2017 }
2018
2019 pub async fn describe_workflow_run(
2021 &self,
2022 workflow_id: &str,
2023 run_id: &str,
2024 ) -> Result<WorkflowDescription> {
2025 let path = format!("/workflows/{workflow_id}/runs/{run_id}");
2026 let mut data: WorkflowDescription = self
2027 .request_json(
2028 reqwest::Method::GET,
2029 &path,
2030 RequestProtocol::ControlPlane,
2031 Option::<&Value>::None,
2032 )
2033 .await?;
2034 data.decode_payloads()?;
2035 Ok(data)
2036 }
2037
2038 pub async fn register_worker(
2039 &self,
2040 worker_id: &str,
2041 task_queue: &str,
2042 supported_workflow_types: Vec<String>,
2043 supported_activity_types: Vec<String>,
2044 max_concurrent_workflow_tasks: usize,
2045 max_concurrent_activity_tasks: usize,
2046 ) -> Result<RegisterWorkerResponse> {
2047 self.register_worker_with_capabilities(
2048 worker_id,
2049 task_queue,
2050 supported_workflow_types,
2051 supported_activity_types,
2052 max_concurrent_workflow_tasks,
2053 max_concurrent_activity_tasks,
2054 Vec::new(),
2055 )
2056 .await
2057 }
2058
2059 pub async fn register_worker_with_capabilities(
2061 &self,
2062 worker_id: &str,
2063 task_queue: &str,
2064 supported_workflow_types: Vec<String>,
2065 supported_activity_types: Vec<String>,
2066 max_concurrent_workflow_tasks: usize,
2067 max_concurrent_activity_tasks: usize,
2068 capabilities: Vec<String>,
2069 ) -> Result<RegisterWorkerResponse> {
2070 self.register_worker_with_command_contracts(
2071 worker_id,
2072 task_queue,
2073 supported_workflow_types,
2074 supported_activity_types,
2075 max_concurrent_workflow_tasks,
2076 max_concurrent_activity_tasks,
2077 capabilities,
2078 Value::Object(serde_json::Map::new()),
2079 )
2080 .await
2081 }
2082
2083 #[allow(clippy::too_many_arguments)]
2085 pub async fn register_worker_with_command_contracts(
2086 &self,
2087 worker_id: &str,
2088 task_queue: &str,
2089 supported_workflow_types: Vec<String>,
2090 supported_activity_types: Vec<String>,
2091 max_concurrent_workflow_tasks: usize,
2092 max_concurrent_activity_tasks: usize,
2093 capabilities: Vec<String>,
2094 workflow_command_contracts: Value,
2095 ) -> Result<RegisterWorkerResponse> {
2096 let mut body = json!({
2097 "worker_id": worker_id,
2098 "task_queue": task_queue,
2099 "runtime": "rust",
2100 "sdk_version": SDK_VERSION,
2101 "supported_workflow_types": supported_workflow_types,
2102 "supported_activity_types": supported_activity_types,
2103 "capabilities": capabilities,
2104 "max_concurrent_workflow_tasks": max_concurrent_workflow_tasks,
2105 "max_concurrent_activity_tasks": max_concurrent_activity_tasks
2106 });
2107 if workflow_command_contracts
2108 .as_object()
2109 .is_some_and(|contracts| !contracts.is_empty())
2110 {
2111 body["workflow_command_contracts"] = workflow_command_contracts;
2112 }
2113
2114 self.request_json(
2115 reqwest::Method::POST,
2116 "/worker/register",
2117 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2118 Some(&body),
2119 )
2120 .await
2121 }
2122
2123 pub async fn deregister_worker_registration(
2129 &self,
2130 worker_id: &str,
2131 ) -> Result<WorkerDeregistrationEnvelope> {
2132 let path = format!(
2133 "/worker/registrations/{}",
2134 percent_encode_path_segment(worker_id)
2135 );
2136 self.request_json(
2137 reqwest::Method::DELETE,
2138 &path,
2139 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2140 Option::<&Value>::None,
2141 )
2142 .await
2143 }
2144
2145 pub async fn poll_query_task(
2147 &self,
2148 worker_id: &str,
2149 task_queue: &str,
2150 timeout: Duration,
2151 ) -> Result<Option<QueryTask>> {
2152 Ok(self
2153 .poll_query_task_response(worker_id, task_queue, timeout)
2154 .await?
2155 .task)
2156 }
2157
2158 pub async fn poll_query_task_response(
2160 &self,
2161 worker_id: &str,
2162 task_queue: &str,
2163 timeout: Duration,
2164 ) -> Result<PollQueryTaskResponse> {
2165 let poll_request_id = unique_request_id("rust-query-poll");
2166 self.poll_query_task_response_with_request_id(
2167 worker_id,
2168 task_queue,
2169 timeout,
2170 &poll_request_id,
2171 1,
2172 )
2173 .await
2174 }
2175
2176 async fn poll_query_task_response_with_request_id(
2177 &self,
2178 worker_id: &str,
2179 task_queue: &str,
2180 timeout: Duration,
2181 poll_request_id: &str,
2182 transport_retries: usize,
2183 ) -> Result<PollQueryTaskResponse> {
2184 let timeout_seconds = long_poll_timeout_seconds(timeout);
2185 let body = json!({
2186 "worker_id": worker_id,
2187 "task_queue": task_queue,
2188 "poll_request_id": poll_request_id,
2189 "timeout_seconds": timeout_seconds,
2190 });
2191 self.poll_request_json(
2192 "/worker/query-tasks/poll",
2193 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
2194 &body,
2195 timeout + Duration::from_secs(5),
2196 transport_retries,
2197 )
2198 .await
2199 }
2200
2201 pub async fn complete_query_task<T: Serialize>(
2203 &self,
2204 query_task_id: &str,
2205 lease_owner: &str,
2206 query_task_attempt: u64,
2207 result: T,
2208 codec: &str,
2209 ) -> Result<Value> {
2210 let typed_result = AvroValue::from_serialize(&result)?;
2211 let result_envelope = encode_typed_envelope(&typed_result, codec)?;
2212 self.complete_query_task_with_envelope(
2213 query_task_id,
2214 lease_owner,
2215 query_task_attempt,
2216 typed_result.into_json()?,
2217 result_envelope,
2218 )
2219 .await
2220 }
2221
2222 async fn complete_query_task_with_envelope(
2223 &self,
2224 query_task_id: &str,
2225 lease_owner: &str,
2226 query_task_attempt: u64,
2227 result: Value,
2228 result_envelope: Value,
2229 ) -> Result<Value> {
2230 let body = json!({
2231 "lease_owner": lease_owner,
2232 "query_task_attempt": query_task_attempt,
2233 "result": result,
2234 "result_envelope": result_envelope,
2235 });
2236 let path = format!("/worker/query-tasks/{query_task_id}/complete");
2237 let response = self
2238 .request_json(
2239 reqwest::Method::POST,
2240 &path,
2241 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
2242 Some(&body),
2243 )
2244 .await;
2245 query_task_response(response)
2246 }
2247
2248 pub async fn fail_query_task(
2250 &self,
2251 query_task_id: &str,
2252 lease_owner: &str,
2253 query_task_attempt: u64,
2254 message: impl Into<String>,
2255 reason: impl Into<String>,
2256 failure_type: impl Into<String>,
2257 ) -> Result<Value> {
2258 let body = json!({
2259 "lease_owner": lease_owner,
2260 "query_task_attempt": query_task_attempt,
2261 "failure": {
2262 "message": message.into(),
2263 "reason": reason.into(),
2264 "type": failure_type.into(),
2265 }
2266 });
2267 let path = format!("/worker/query-tasks/{query_task_id}/fail");
2268 let response = self
2269 .request_json(
2270 reqwest::Method::POST,
2271 &path,
2272 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
2273 Some(&body),
2274 )
2275 .await;
2276 query_task_response(response)
2277 }
2278
2279 pub async fn heartbeat_worker(
2280 &self,
2281 worker_id: &str,
2282 workflow_available: usize,
2283 activity_available: usize,
2284 ) -> Result<Value> {
2285 let body = json!({
2286 "worker_id": worker_id,
2287 "task_slots": {
2288 "workflow_available": workflow_available,
2289 "activity_available": activity_available
2290 },
2291 "process_metrics": {
2292 "process_id": std::process::id(),
2293 "process_uptime_seconds": 0
2294 }
2295 });
2296
2297 self.request_json(
2298 reqwest::Method::POST,
2299 "/worker/heartbeat",
2300 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2301 Some(&body),
2302 )
2303 .await
2304 }
2305
2306 pub async fn poll_workflow_task(
2307 &self,
2308 worker_id: &str,
2309 task_queue: &str,
2310 timeout: Duration,
2311 ) -> Result<Option<WorkflowTask>> {
2312 Ok(self
2313 .poll_workflow_task_response(worker_id, task_queue, timeout)
2314 .await?
2315 .task)
2316 }
2317
2318 pub async fn poll_workflow_task_response(
2319 &self,
2320 worker_id: &str,
2321 task_queue: &str,
2322 timeout: Duration,
2323 ) -> Result<PollWorkflowTaskResponse> {
2324 let poll_request_id = unique_request_id("rust-workflow-poll");
2325 self.poll_workflow_task_response_with_request_id(
2326 worker_id,
2327 task_queue,
2328 timeout,
2329 &poll_request_id,
2330 1,
2331 )
2332 .await
2333 }
2334
2335 async fn poll_workflow_task_response_with_request_id(
2336 &self,
2337 worker_id: &str,
2338 task_queue: &str,
2339 timeout: Duration,
2340 poll_request_id: &str,
2341 transport_retries: usize,
2342 ) -> Result<PollWorkflowTaskResponse> {
2343 let body = json!({
2344 "worker_id": worker_id,
2345 "task_queue": task_queue,
2346 "poll_request_id": poll_request_id,
2347 "timeout_seconds": long_poll_timeout_seconds(timeout),
2348 });
2349 let mut data: PollWorkflowTaskResponse = self
2350 .poll_request_json(
2351 "/worker/workflow-tasks/poll",
2352 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2353 &body,
2354 timeout + Duration::from_secs(5),
2355 transport_retries,
2356 )
2357 .await?;
2358
2359 if let Some(task) = data.task.as_mut() {
2360 self.fetch_remaining_workflow_history(worker_id, task)
2361 .await?;
2362 }
2363
2364 Ok(data)
2365 }
2366
2367 async fn fetch_remaining_workflow_history(
2368 &self,
2369 worker_id: &str,
2370 task: &mut WorkflowTask,
2371 ) -> Result<()> {
2372 let mut next_token = task.next_history_page_token.clone();
2373
2374 while let Some(token) = next_token.take().filter(|token| !token.is_empty()) {
2375 let lease_owner = task
2376 .lease_owner
2377 .clone()
2378 .unwrap_or_else(|| worker_id.to_string());
2379 let page = self
2380 .workflow_task_history_page(
2381 &task.task_id,
2382 &lease_owner,
2383 task.workflow_task_attempt,
2384 &token,
2385 )
2386 .await?;
2387
2388 task.append_history_page(page);
2389
2390 if task.next_history_page_token.as_deref() == Some(token.as_str()) {
2391 return Err(Error::Codec(
2392 "workflow history pagination returned the same page token".to_string(),
2393 ));
2394 }
2395
2396 next_token = task.next_history_page_token.clone();
2397 }
2398
2399 Ok(())
2400 }
2401
2402 async fn workflow_task_history_page(
2403 &self,
2404 task_id: &str,
2405 lease_owner: &str,
2406 workflow_task_attempt: u64,
2407 next_history_page_token: &str,
2408 ) -> Result<WorkflowTaskHistoryPage> {
2409 let body = json!({
2410 "lease_owner": lease_owner,
2411 "workflow_task_attempt": workflow_task_attempt,
2412 "next_history_page_token": next_history_page_token
2413 });
2414 let path = format!("/worker/workflow-tasks/{task_id}/history");
2415
2416 self.request_json(
2417 reqwest::Method::POST,
2418 &path,
2419 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2420 Some(&body),
2421 )
2422 .await
2423 }
2424
2425 pub async fn complete_workflow_task(
2426 &self,
2427 task_id: &str,
2428 lease_owner: &str,
2429 workflow_task_attempt: u64,
2430 commands: Vec<Value>,
2431 ) -> Result<Value> {
2432 let body = json!({
2433 "lease_owner": lease_owner,
2434 "workflow_task_attempt": workflow_task_attempt,
2435 "commands": commands
2436 });
2437 let path = format!("/worker/workflow-tasks/{task_id}/complete");
2438 self.request_json(
2439 reqwest::Method::POST,
2440 &path,
2441 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2442 Some(&body),
2443 )
2444 .await
2445 }
2446
2447 pub async fn fail_workflow_task(
2448 &self,
2449 task_id: &str,
2450 lease_owner: &str,
2451 workflow_task_attempt: u64,
2452 message: impl Into<String>,
2453 ) -> Result<Value> {
2454 self.fail_workflow_task_with_type(
2455 task_id,
2456 lease_owner,
2457 workflow_task_attempt,
2458 message,
2459 "RustWorkflowTaskFailure",
2460 )
2461 .await
2462 }
2463
2464 async fn fail_workflow_task_with_type(
2465 &self,
2466 task_id: &str,
2467 lease_owner: &str,
2468 workflow_task_attempt: u64,
2469 message: impl Into<String>,
2470 failure_type: &str,
2471 ) -> Result<Value> {
2472 let body = json!({
2473 "lease_owner": lease_owner,
2474 "workflow_task_attempt": workflow_task_attempt,
2475 "failure": {
2476 "message": message.into(),
2477 "type": failure_type
2478 }
2479 });
2480 let path = format!("/worker/workflow-tasks/{task_id}/fail");
2481 self.request_json(
2482 reqwest::Method::POST,
2483 &path,
2484 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2485 Some(&body),
2486 )
2487 .await
2488 }
2489
2490 pub async fn poll_activity_task(
2491 &self,
2492 worker_id: &str,
2493 task_queue: &str,
2494 timeout: Duration,
2495 ) -> Result<Option<ActivityTask>> {
2496 Ok(self
2497 .poll_activity_task_response(worker_id, task_queue, timeout)
2498 .await?
2499 .task)
2500 }
2501
2502 pub async fn poll_activity_task_response(
2504 &self,
2505 worker_id: &str,
2506 task_queue: &str,
2507 timeout: Duration,
2508 ) -> Result<PollActivityTaskResponse> {
2509 let poll_request_id = unique_request_id("rust-activity-poll");
2510 self.poll_activity_task_response_with_request_id(
2511 worker_id,
2512 task_queue,
2513 timeout,
2514 &poll_request_id,
2515 1,
2516 )
2517 .await
2518 }
2519
2520 async fn poll_activity_task_response_with_request_id(
2521 &self,
2522 worker_id: &str,
2523 task_queue: &str,
2524 timeout: Duration,
2525 poll_request_id: &str,
2526 transport_retries: usize,
2527 ) -> Result<PollActivityTaskResponse> {
2528 let body = json!({
2529 "worker_id": worker_id,
2530 "task_queue": task_queue,
2531 "poll_request_id": poll_request_id,
2532 "timeout_seconds": long_poll_timeout_seconds(timeout),
2533 });
2534 let data: PollActivityTaskResponse = self
2535 .poll_request_json(
2536 "/worker/activity-tasks/poll",
2537 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2538 &body,
2539 timeout + Duration::from_secs(5),
2540 transport_retries,
2541 )
2542 .await?;
2543 Ok(data)
2544 }
2545
2546 pub async fn complete_activity_task<T: Serialize>(
2547 &self,
2548 task_id: &str,
2549 activity_attempt_id: &str,
2550 lease_owner: &str,
2551 result: T,
2552 codec: &str,
2553 ) -> Result<Value> {
2554 let result = encode_typed_envelope(&AvroValue::from_serialize(&result)?, codec)?;
2555 let body = json!({
2556 "activity_attempt_id": activity_attempt_id,
2557 "lease_owner": lease_owner,
2558 "result": result
2559 });
2560 let path = format!("/worker/activity-tasks/{task_id}/complete");
2561 activity_task_response(
2562 self.request_json(
2563 reqwest::Method::POST,
2564 &path,
2565 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2566 Some(&body),
2567 )
2568 .await,
2569 "complete",
2570 task_id,
2571 activity_attempt_id,
2572 )
2573 }
2574
2575 pub async fn fail_activity_task(
2576 &self,
2577 task_id: &str,
2578 activity_attempt_id: &str,
2579 lease_owner: &str,
2580 message: impl Into<String>,
2581 non_retryable: bool,
2582 ) -> Result<Value> {
2583 let body = json!({
2584 "activity_attempt_id": activity_attempt_id,
2585 "lease_owner": lease_owner,
2586 "failure": {
2587 "message": message.into(),
2588 "type": "RustActivityFailure",
2589 "non_retryable": non_retryable
2590 }
2591 });
2592 let path = format!("/worker/activity-tasks/{task_id}/fail");
2593 activity_task_response(
2594 self.request_json(
2595 reqwest::Method::POST,
2596 &path,
2597 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2598 Some(&body),
2599 )
2600 .await,
2601 "fail",
2602 task_id,
2603 activity_attempt_id,
2604 )
2605 }
2606
2607 pub async fn heartbeat_activity_task<T: Serialize>(
2608 &self,
2609 task_id: &str,
2610 activity_attempt_id: &str,
2611 lease_owner: &str,
2612 details: T,
2613 ) -> Result<ActivityHeartbeatResponse> {
2614 let details = encode_typed_envelope(&AvroValue::from_serialize(&details)?, DEFAULT_CODEC)?;
2615 let body = json!({
2616 "activity_attempt_id": activity_attempt_id,
2617 "lease_owner": lease_owner,
2618 "details": details
2619 });
2620 let path = format!("/worker/activity-tasks/{task_id}/heartbeat");
2621 activity_task_response(
2622 self.request_json(
2623 reqwest::Method::POST,
2624 &path,
2625 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2626 Some(&body),
2627 )
2628 .await,
2629 "heartbeat",
2630 task_id,
2631 activity_attempt_id,
2632 )
2633 }
2634
2635 async fn request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
2636 &self,
2637 method: reqwest::Method,
2638 path: &str,
2639 protocol: RequestProtocol,
2640 body: Option<&B>,
2641 ) -> Result<T> {
2642 self.request_json_with_timeout(method, path, protocol, body, Duration::from_secs(60))
2643 .await
2644 }
2645
2646 async fn request_json_with_timeout<T: DeserializeOwned, B: Serialize + ?Sized>(
2647 &self,
2648 method: reqwest::Method,
2649 path: &str,
2650 protocol: RequestProtocol,
2651 body: Option<&B>,
2652 timeout: Duration,
2653 ) -> Result<T> {
2654 let auth_token = self.auth_token(protocol)?;
2655 let mut request = self
2656 .http
2657 .request(method, format!("{}/api{}", self.base_url, path))
2658 .timeout(timeout)
2659 .header(reqwest::header::ACCEPT, "application/json")
2660 .header(reqwest::header::CONTENT_TYPE, "application/json")
2661 .header("X-Namespace", &self.namespace);
2662
2663 match protocol {
2664 RequestProtocol::Worker(version) => {
2665 request = request.header("X-Durable-Workflow-Protocol-Version", version);
2666 }
2667 RequestProtocol::ControlPlane => {
2668 request = request.header(
2669 "X-Durable-Workflow-Control-Plane-Version",
2670 CONTROL_PLANE_VERSION,
2671 );
2672 }
2673 }
2674
2675 if let Some(token) = auth_token {
2676 request = request.bearer_auth(token);
2677 }
2678
2679 if let Some(body) = body {
2680 request = request.json(body);
2681 }
2682
2683 let response = request.send().await?;
2684 let status = response.status();
2685 let bytes = response.bytes().await?;
2686
2687 if !status.is_success() {
2688 let body = String::from_utf8_lossy(&bytes).to_string();
2689 if let Some(protocol) = protocol_failure(status, &body) {
2690 return Err(Error::Protocol(protocol));
2691 }
2692 return Err(Error::Http { status, body });
2693 }
2694
2695 if bytes.is_empty() {
2696 return Ok(serde_json::from_value(Value::Null)?);
2697 }
2698
2699 Ok(serde_json::from_slice(&bytes)?)
2700 }
2701
2702 async fn poll_request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
2703 &self,
2704 path: &str,
2705 protocol: RequestProtocol,
2706 body: &B,
2707 timeout: Duration,
2708 max_retries: usize,
2709 ) -> Result<T> {
2710 let mut retries = 0;
2711
2712 loop {
2713 let response = self
2714 .request_json_with_timeout(
2715 reqwest::Method::POST,
2716 path,
2717 protocol,
2718 Some(body),
2719 timeout,
2720 )
2721 .await;
2722
2723 match response {
2724 Err(Error::Transport(_)) if retries < max_retries => retries += 1,
2725 response => return worker_poll_response(response),
2726 }
2727 }
2728 }
2729
2730 fn auth_token(&self, protocol: RequestProtocol) -> Result<Option<&str>> {
2731 match protocol {
2732 RequestProtocol::Worker(_) => {
2733 if let Some(token) = self.worker_token.as_deref().or(self.token.as_deref()) {
2734 return Ok(Some(token));
2735 }
2736 if self.control_token.is_some() {
2737 return Err(Error::MissingRoleCredentials {
2738 role: "worker",
2739 opposite_role: "control",
2740 });
2741 }
2742 Ok(None)
2743 }
2744 RequestProtocol::ControlPlane => {
2745 if let Some(token) = self.control_token.as_deref().or(self.token.as_deref()) {
2746 return Ok(Some(token));
2747 }
2748 if self.worker_token.is_some() {
2749 return Err(Error::MissingRoleCredentials {
2750 role: "control",
2751 opposite_role: "worker",
2752 });
2753 }
2754 Ok(None)
2755 }
2756 }
2757 }
2758}
2759
2760fn query_failure(status: reqwest::StatusCode, raw_body: String) -> QueryFailure {
2761 let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
2762 let reason = body
2763 .get("reason")
2764 .and_then(Value::as_str)
2765 .unwrap_or("query_rejected")
2766 .to_string();
2767 let message = body
2768 .get("message")
2769 .or_else(|| body.get("error"))
2770 .and_then(Value::as_str)
2771 .unwrap_or("workflow query was rejected")
2772 .to_string();
2773
2774 QueryFailure {
2775 status: status.as_u16(),
2776 reason,
2777 message,
2778 body,
2779 }
2780}
2781
2782fn workflow_command_result(
2783 command: WorkflowCommandKind,
2784 data: Value,
2785 workflow_id: &str,
2786 run_id: Option<&str>,
2787) -> WorkflowCommandResult {
2788 WorkflowCommandResult {
2789 command,
2790 workflow_id: data
2791 .get("workflow_id")
2792 .and_then(Value::as_str)
2793 .unwrap_or(workflow_id)
2794 .to_string(),
2795 run_id: data
2796 .get("run_id")
2797 .and_then(Value::as_str)
2798 .or(run_id)
2799 .map(str::to_string),
2800 outcome: data
2801 .get("outcome")
2802 .and_then(Value::as_str)
2803 .map(str::to_string),
2804 reason: data
2805 .get("reason")
2806 .and_then(Value::as_str)
2807 .map(str::to_string),
2808 command_status: data
2809 .get("command_status")
2810 .and_then(Value::as_str)
2811 .map(str::to_string),
2812 raw: data,
2813 }
2814}
2815
2816fn workflow_command_rejection(
2817 command: WorkflowCommandKind,
2818 status: reqwest::StatusCode,
2819 raw_body: String,
2820 workflow_id: &str,
2821 run_id: Option<&str>,
2822) -> WorkflowCommandRejection {
2823 let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
2824 WorkflowCommandRejection {
2825 command,
2826 status: status.as_u16(),
2827 reason: body
2828 .get("reason")
2829 .and_then(Value::as_str)
2830 .unwrap_or("workflow_command_rejected")
2831 .to_string(),
2832 message: body
2833 .get("message")
2834 .or_else(|| body.get("error"))
2835 .and_then(Value::as_str)
2836 .unwrap_or("workflow lifecycle command was rejected")
2837 .to_string(),
2838 workflow_id: body
2839 .get("workflow_id")
2840 .and_then(Value::as_str)
2841 .unwrap_or(workflow_id)
2842 .to_string(),
2843 run_id: body
2844 .get("run_id")
2845 .and_then(Value::as_str)
2846 .or(run_id)
2847 .map(str::to_string),
2848 target_scope: body
2849 .get("target_scope")
2850 .and_then(Value::as_str)
2851 .map(str::to_string),
2852 body,
2853 }
2854}
2855
2856fn query_task_response(response: Result<Value>) -> Result<Value> {
2857 match response {
2858 Err(Error::Http { status, body }) => Err(Error::QueryFailed(query_failure(status, body))),
2859 response => response,
2860 }
2861}
2862
2863fn worker_poll_response<T: DeserializeOwned>(response: Result<T>) -> Result<T> {
2864 match response {
2865 Err(Error::Http { status, body })
2866 if status == reqwest::StatusCode::CONFLICT && worker_poll_body_is_stop(&body) =>
2867 {
2868 Ok(serde_json::from_str(&body)?)
2869 }
2870 response => response,
2871 }
2872}
2873
2874fn worker_poll_body_is_stop(body: &str) -> bool {
2875 serde_json::from_str::<Value>(body)
2876 .ok()
2877 .is_some_and(|body| {
2878 worker_poll_is_stop(
2879 body.get("poll_status").and_then(Value::as_str),
2880 body.get("reason").and_then(Value::as_str),
2881 )
2882 })
2883}
2884
2885fn worker_poll_is_stop(poll_status: Option<&str>, reason: Option<&str>) -> bool {
2886 matches!(poll_status, Some("draining" | "stopped"))
2887 || matches!(reason, Some("worker_draining" | "worker_stopped"))
2888}
2889
2890fn query_task_rejection_is_final(error: &Error) -> bool {
2891 matches!(
2892 error,
2893 Error::QueryFailed(failure)
2894 if QUERY_TASK_FINAL_REJECTION_REASONS.contains(&failure.reason.as_str())
2895 )
2896}
2897
2898fn activity_task_response<T>(
2899 response: Result<T>,
2900 operation: &str,
2901 task_id: &str,
2902 activity_attempt_id: &str,
2903) -> Result<T> {
2904 match response {
2905 Err(Error::Http { status, body }) => {
2906 let body = serde_json::from_str(&body).unwrap_or_else(|_| json!({"message": body}));
2907 Err(Error::ActivityTaskRejected(ActivityTaskRejection {
2908 operation: operation.to_string(),
2909 status: status.as_u16(),
2910 reason: body
2911 .get("reason")
2912 .and_then(Value::as_str)
2913 .unwrap_or("activity_task_rejected")
2914 .to_string(),
2915 task_id: body
2916 .get("task_id")
2917 .and_then(Value::as_str)
2918 .unwrap_or(task_id)
2919 .to_string(),
2920 activity_attempt_id: body
2921 .get("activity_attempt_id")
2922 .and_then(Value::as_str)
2923 .unwrap_or(activity_attempt_id)
2924 .to_string(),
2925 cancel_requested: body
2926 .get("cancel_requested")
2927 .and_then(Value::as_bool)
2928 .unwrap_or(false),
2929 can_continue: body.get("can_continue").and_then(Value::as_bool),
2930 run_closed_reason: body
2931 .get("run_closed_reason")
2932 .and_then(Value::as_str)
2933 .map(str::to_string),
2934 body,
2935 }))
2936 }
2937 response => response,
2938 }
2939}
2940
2941fn activity_task_rejection_is_final(error: &Error) -> bool {
2942 matches!(
2943 error,
2944 Error::ActivityTaskRejected(rejection)
2945 if matches!(
2946 rejection.reason.as_str(),
2947 "run_cancelled"
2948 | "run_terminated"
2949 | "attempt_closed"
2950 | "stale_attempt"
2951 | "activity_cancelled"
2952 | "task_cancelled"
2953 | "run_closed"
2954 | "activity_not_running"
2955 | "attempt_not_found"
2956 )
2957 )
2958}
2959
2960fn workflow_task_completion_is_terminal_timeout(
2961 error: &Error,
2962 task_id: &str,
2963 workflow_task_attempt: u64,
2964 run_id: Option<&str>,
2965) -> bool {
2966 let Error::Http { status, body } = error else {
2967 return false;
2968 };
2969 if *status != reqwest::StatusCode::CONFLICT {
2970 return false;
2971 }
2972
2973 let Some(run_id) = run_id else {
2974 return false;
2975 };
2976 let Ok(body) = serde_json::from_str::<Value>(body) else {
2977 return false;
2978 };
2979
2980 body.get("recorded").and_then(Value::as_bool) == Some(false)
2981 && body.get("reason").and_then(Value::as_str) == Some("run_timed_out")
2982 && body.get("run_status").and_then(Value::as_str) == Some("failed")
2983 && body.get("run_id").and_then(Value::as_str) == Some(run_id)
2984 && body.get("task_id").and_then(Value::as_str) == Some(task_id)
2985 && body.get("workflow_task_attempt").and_then(Value::as_u64) == Some(workflow_task_attempt)
2986}
2987
2988fn protocol_failure(status: reqwest::StatusCode, raw_body: &str) -> Option<ProtocolFailure> {
2989 let body: Value = serde_json::from_str(raw_body).ok()?;
2990 let reason = body.get("reason")?.as_str()?;
2991 if !matches!(
2992 reason,
2993 "missing_protocol_version"
2994 | "unsupported_protocol_version"
2995 | "missing_control_plane_version"
2996 | "unsupported_control_plane_version"
2997 ) {
2998 return None;
2999 }
3000
3001 Some(ProtocolFailure {
3002 status: status.as_u16(),
3003 reason: reason.to_string(),
3004 message: body
3005 .get("message")
3006 .or_else(|| body.get("error"))
3007 .and_then(Value::as_str)
3008 .unwrap_or("protocol version rejected")
3009 .to_string(),
3010 supported_version: body
3011 .get("supported_version")
3012 .and_then(Value::as_str)
3013 .map(str::to_string),
3014 requested_version: body
3015 .get("requested_version")
3016 .and_then(Value::as_str)
3017 .map(str::to_string),
3018 body,
3019 })
3020}
3021
3022fn long_poll_timeout_seconds(timeout: Duration) -> u64 {
3023 timeout
3024 .as_secs()
3025 .saturating_add(u64::from(timeout.subsec_nanos() > 0))
3026 .min(MAX_LONG_POLL_TIMEOUT_SECONDS)
3027}
3028
3029fn worker_operation_is_retryable(error: &Error) -> bool {
3030 match error {
3031 Error::Transport(error) => {
3032 error.is_timeout() || error.is_connect() || error.is_request() || error.is_body()
3033 }
3034 Error::Http { status, .. } => {
3035 matches!(
3036 *status,
3037 reqwest::StatusCode::REQUEST_TIMEOUT | reqwest::StatusCode::TOO_MANY_REQUESTS
3038 ) || status.is_server_error()
3039 }
3040 _ => false,
3041 }
3042}
3043
3044fn worker_retry_delay(policy: WorkerRetryPolicy, retry: usize) -> Duration {
3045 let exponent = retry.saturating_sub(1).min(31) as u32;
3046 policy
3047 .initial_backoff
3048 .saturating_mul(1_u32 << exponent)
3049 .min(policy.max_backoff)
3050}
3051
3052#[derive(Debug)]
3053pub struct ClientBuilder {
3054 base_url: String,
3055 token: Option<String>,
3056 control_token: Option<String>,
3057 worker_token: Option<String>,
3058 namespace: String,
3059 timeout: Duration,
3060}
3061
3062impl ClientBuilder {
3063 pub fn token(mut self, token: Option<String>) -> Self {
3064 self.token = token;
3065 self
3066 }
3067
3068 pub fn control_token(mut self, token: Option<String>) -> Self {
3069 self.control_token = token;
3070 self
3071 }
3072
3073 pub fn worker_token(mut self, token: Option<String>) -> Self {
3074 self.worker_token = token;
3075 self
3076 }
3077
3078 pub fn namespace(mut self, namespace: impl Into<String>) -> Self {
3079 self.namespace = namespace.into();
3080 self
3081 }
3082
3083 pub fn timeout(mut self, timeout: Duration) -> Self {
3084 self.timeout = timeout;
3085 self
3086 }
3087
3088 pub fn build(self) -> Result<Client> {
3089 let base_url = self.base_url.trim_end_matches('/').to_string();
3090 let has_sdk_api_suffix = reqwest::Url::parse(&base_url)
3091 .map(|url| url.path().trim_end_matches('/').ends_with("/api"))
3092 .unwrap_or_else(|_| base_url.ends_with("/api"));
3093
3094 if has_sdk_api_suffix {
3095 return Err(Error::InvalidBaseUrl);
3096 }
3097
3098 Ok(Client {
3099 http: reqwest::Client::builder().timeout(self.timeout).build()?,
3100 base_url,
3101 token: self.token,
3102 control_token: self.control_token,
3103 worker_token: self.worker_token,
3104 namespace: self.namespace,
3105 })
3106 }
3107}
3108
3109#[derive(Clone, Debug)]
3110pub struct WorkflowHandle {
3111 client: Client,
3112 pub workflow_id: String,
3113 pub run_id: Option<String>,
3114 pub workflow_type: String,
3115}
3116
3117impl WorkflowHandle {
3118 pub async fn describe(&self) -> Result<WorkflowDescription> {
3120 self.client.describe_workflow(&self.workflow_id).await
3121 }
3122
3123 pub async fn describe_selected_run(&self) -> Result<WorkflowDescription> {
3125 let run_id = self.run_id.as_deref().ok_or_else(|| {
3126 Error::Codec("run_id is required for selected-run description".to_string())
3127 })?;
3128 self.client
3129 .describe_workflow_run(&self.workflow_id, run_id)
3130 .await
3131 }
3132
3133 pub async fn signal<T: Serialize>(&self, signal_name: &str, input: T) -> Result<Value> {
3134 self.client
3135 .signal_workflow(&self.workflow_id, signal_name, input)
3136 .await
3137 }
3138
3139 pub async fn signal_selected_run<T: Serialize>(
3141 &self,
3142 signal_name: &str,
3143 input: T,
3144 ) -> Result<Value> {
3145 let run_id = self.run_id.as_deref().ok_or_else(|| {
3146 Error::Codec("run_id is required for selected-run signaling".to_string())
3147 })?;
3148 self.client
3149 .signal_workflow_run(&self.workflow_id, run_id, signal_name, input)
3150 .await
3151 }
3152
3153 pub async fn cancel(&self, options: WorkflowCommandOptions) -> Result<WorkflowCommandResult> {
3155 self.client
3156 .cancel_workflow(&self.workflow_id, options)
3157 .await
3158 }
3159
3160 pub async fn cancel_selected_run(
3162 &self,
3163 options: WorkflowCommandOptions,
3164 ) -> Result<WorkflowCommandResult> {
3165 let run_id = self.run_id.as_deref().ok_or_else(|| {
3166 Error::Codec("run_id is required for selected-run cancellation".to_string())
3167 })?;
3168 self.client
3169 .cancel_workflow_run(&self.workflow_id, run_id, options)
3170 .await
3171 }
3172
3173 pub async fn terminate(
3175 &self,
3176 options: WorkflowCommandOptions,
3177 ) -> Result<WorkflowCommandResult> {
3178 self.client
3179 .terminate_workflow(&self.workflow_id, options)
3180 .await
3181 }
3182
3183 pub async fn terminate_selected_run(
3185 &self,
3186 options: WorkflowCommandOptions,
3187 ) -> Result<WorkflowCommandResult> {
3188 let run_id = self.run_id.as_deref().ok_or_else(|| {
3189 Error::Codec("run_id is required for selected-run termination".to_string())
3190 })?;
3191 self.client
3192 .terminate_workflow_run(&self.workflow_id, run_id, options)
3193 .await
3194 }
3195
3196 pub async fn query<T: Serialize>(&self, query_name: &str, input: T) -> Result<Value> {
3198 self.client
3199 .query_workflow(&self.workflow_id, query_name, input)
3200 .await
3201 }
3202
3203 pub async fn query_avro_value<T: Serialize>(
3204 &self,
3205 query_name: &str,
3206 input: T,
3207 ) -> Result<AvroValue> {
3208 self.client
3209 .query_workflow_avro_value(&self.workflow_id, query_name, input)
3210 .await
3211 }
3212
3213 pub async fn update<T: Serialize>(
3214 &self,
3215 update_name: &str,
3216 input: T,
3217 request_id: Option<&str>,
3218 ) -> Result<Value> {
3219 self.client
3220 .update_workflow(&self.workflow_id, update_name, input, request_id)
3221 .await
3222 }
3223
3224 pub async fn update_avro_value<T: Serialize>(
3225 &self,
3226 update_name: &str,
3227 input: T,
3228 request_id: Option<&str>,
3229 ) -> Result<AvroValue> {
3230 self.client
3231 .update_workflow_avro_value(&self.workflow_id, update_name, input, request_id)
3232 .await
3233 }
3234
3235 pub async fn query_selected_run<T: Serialize>(
3237 &self,
3238 query_name: &str,
3239 input: T,
3240 ) -> Result<Value> {
3241 let run_id = self
3242 .run_id
3243 .as_deref()
3244 .ok_or_else(|| Error::Codec("run_id is required for selected-run query".to_string()))?;
3245 self.client
3246 .query_workflow_run(&self.workflow_id, run_id, query_name, input)
3247 .await
3248 }
3249
3250 pub async fn result(&self, options: WorkflowResultOptions) -> Result<Value> {
3252 self.result_target(options, None).await
3253 }
3254
3255 pub async fn result_avro_value(&self, options: WorkflowResultOptions) -> Result<AvroValue> {
3257 self.result_avro_value_target(options, None).await
3258 }
3259
3260 pub async fn result_selected_run(&self, options: WorkflowResultOptions) -> Result<Value> {
3262 let run_id = self.run_id.as_deref().ok_or_else(|| {
3263 Error::Codec("run_id is required for selected-run result".to_string())
3264 })?;
3265 self.result_target(options, Some(run_id)).await
3266 }
3267
3268 pub async fn result_selected_run_avro_value(
3270 &self,
3271 options: WorkflowResultOptions,
3272 ) -> Result<AvroValue> {
3273 let run_id = self.run_id.as_deref().ok_or_else(|| {
3274 Error::Codec("run_id is required for selected-run result".to_string())
3275 })?;
3276 self.result_avro_value_target(options, Some(run_id)).await
3277 }
3278
3279 async fn result_avro_value_target(
3280 &self,
3281 options: WorkflowResultOptions,
3282 selected_run_id: Option<&str>,
3283 ) -> Result<AvroValue> {
3284 let started = Instant::now();
3285
3286 loop {
3287 let description = match selected_run_id {
3288 Some(run_id) => {
3289 self.client
3290 .describe_workflow_run(&self.workflow_id, run_id)
3291 .await?
3292 }
3293 None => self.describe().await?,
3294 };
3295 if description.is_completed() {
3296 return description.output_avro_value.ok_or_else(|| {
3297 Error::Codec(
3298 "missing_payload_envelope: typed workflow result requires output_envelope"
3299 .to_string(),
3300 )
3301 });
3302 }
3303 if description.is_terminal() {
3304 let outcome =
3305 workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
3306 return Err(match outcome.kind {
3307 WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
3308 WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
3309 WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
3310 WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
3311 });
3312 }
3313 if started.elapsed() >= options.timeout {
3314 return Err(Error::Timeout);
3315 }
3316 tokio::time::sleep(options.poll_interval).await;
3317 }
3318 }
3319
3320 async fn result_target(
3321 &self,
3322 options: WorkflowResultOptions,
3323 selected_run_id: Option<&str>,
3324 ) -> Result<Value> {
3325 let started = Instant::now();
3326
3327 loop {
3328 let description = match selected_run_id {
3329 Some(run_id) => {
3330 self.client
3331 .describe_workflow_run(&self.workflow_id, run_id)
3332 .await?
3333 }
3334 None => self.describe().await?,
3335 };
3336 if description.is_completed() {
3337 return Ok(description.output.unwrap_or(Value::Null));
3338 }
3339
3340 if description.is_terminal() {
3341 let outcome =
3342 workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
3343 return Err(match outcome.kind {
3344 WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
3345 WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
3346 WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
3347 WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
3348 });
3349 }
3350
3351 if started.elapsed() >= options.timeout {
3352 return Err(Error::WorkflowTimedOut(WorkflowTerminalOutcome {
3353 kind: WorkflowTerminalKind::TimedOut,
3354 workflow_id: description
3355 .workflow_id
3356 .clone()
3357 .unwrap_or_else(|| self.workflow_id.clone()),
3358 run_id: description
3359 .run_id
3360 .clone()
3361 .or_else(|| selected_run_id.map(str::to_string)),
3362 reason: "result_wait_timeout".to_string(),
3363 failure_category: Some("client_timeout".to_string()),
3364 failure_id: None,
3365 exception_type: None,
3366 exception_class: None,
3367 non_retryable: None,
3368 message: Some(format!(
3369 "workflow result was not terminal within {:?}",
3370 options.timeout
3371 )),
3372 exception: None,
3373 raw: description.raw_value(),
3374 }));
3375 }
3376
3377 tokio::time::sleep(options.poll_interval).await;
3378 }
3379 }
3380}
3381
3382#[derive(Clone, Copy, Debug)]
3383pub struct WorkflowResultOptions {
3384 pub poll_interval: Duration,
3385 pub timeout: Duration,
3386}
3387
3388impl Default for WorkflowResultOptions {
3389 fn default() -> Self {
3390 Self {
3391 poll_interval: Duration::from_millis(500),
3392 timeout: Duration::from_secs(30),
3393 }
3394 }
3395}
3396
3397#[derive(Clone, Debug, Deserialize)]
3398pub struct WorkflowDescription {
3399 pub workflow_id: Option<String>,
3400 pub run_id: Option<String>,
3401 pub workflow_type: Option<String>,
3402 pub status: Option<String>,
3403 #[serde(default)]
3404 pub closed_reason: Option<String>,
3405 #[serde(default)]
3406 pub error: Option<String>,
3407 #[serde(default)]
3408 pub failure: Option<Value>,
3409 #[serde(default)]
3410 pub exception: Option<Value>,
3411 #[serde(default)]
3412 pub failures: Vec<Value>,
3413 #[serde(default)]
3414 pub output: Option<Value>,
3415 #[serde(default)]
3416 pub output_envelope: Option<Value>,
3417 #[serde(skip)]
3418 pub output_avro_value: Option<AvroValue>,
3419 #[serde(flatten)]
3420 pub raw: HashMap<String, Value>,
3421}
3422
3423impl WorkflowDescription {
3424 pub fn is_completed(&self) -> bool {
3425 matches!(self.status.as_deref(), Some("completed" | "Completed"))
3426 }
3427
3428 pub fn is_terminal(&self) -> bool {
3429 matches!(
3430 self.status.as_deref(),
3431 Some(
3432 "completed"
3433 | "Completed"
3434 | "failed"
3435 | "Failed"
3436 | "cancelled"
3437 | "Cancelled"
3438 | "terminated"
3439 | "Terminated"
3440 | "timed_out"
3441 | "TimedOut",
3442 )
3443 )
3444 }
3445
3446 fn decode_payloads(&mut self) -> Result<()> {
3447 if let Some(envelope) = &self.output_envelope {
3448 let value = decode_wire_avro_value(envelope, DEFAULT_CODEC)?;
3449 self.output = Some(value.clone().into_json()?);
3450 self.output_avro_value = Some(value);
3451 }
3452
3453 Ok(())
3454 }
3455
3456 fn raw_value(&self) -> Value {
3457 let mut data = self.raw.clone();
3458 data.insert(
3459 "workflow_id".to_string(),
3460 self.workflow_id
3461 .clone()
3462 .map(Value::String)
3463 .unwrap_or(Value::Null),
3464 );
3465 data.insert(
3466 "run_id".to_string(),
3467 self.run_id
3468 .clone()
3469 .map(Value::String)
3470 .unwrap_or(Value::Null),
3471 );
3472 data.insert(
3473 "workflow_type".to_string(),
3474 self.workflow_type
3475 .clone()
3476 .map(Value::String)
3477 .unwrap_or(Value::Null),
3478 );
3479 data.insert(
3480 "status".to_string(),
3481 self.status
3482 .clone()
3483 .map(Value::String)
3484 .unwrap_or(Value::Null),
3485 );
3486 data.insert(
3487 "closed_reason".to_string(),
3488 self.closed_reason
3489 .clone()
3490 .map(Value::String)
3491 .unwrap_or(Value::Null),
3492 );
3493 if let Some(failure) = &self.failure {
3494 data.insert("failure".to_string(), failure.clone());
3495 }
3496 if let Some(exception) = &self.exception {
3497 data.insert("exception".to_string(), exception.clone());
3498 }
3499 Value::Object(data.into_iter().collect())
3500 }
3501}
3502
3503fn workflow_terminal_outcome(
3504 description: &WorkflowDescription,
3505 workflow_id: &str,
3506 run_id: Option<&str>,
3507) -> WorkflowTerminalOutcome {
3508 let terminal_kind = description
3509 .closed_reason
3510 .as_deref()
3511 .or(description.status.as_deref())
3512 .unwrap_or("failed")
3513 .to_ascii_lowercase();
3514 let kind = match terminal_kind.as_str() {
3515 "cancelled" | "canceled" => WorkflowTerminalKind::Cancelled,
3516 "terminated" => WorkflowTerminalKind::Terminated,
3517 "timed_out" | "timedout" => WorkflowTerminalKind::TimedOut,
3518 _ => WorkflowTerminalKind::Failed,
3519 };
3520 let default_reason = match kind {
3521 WorkflowTerminalKind::Failed => "workflow_failed",
3522 WorkflowTerminalKind::Cancelled => "cancelled",
3523 WorkflowTerminalKind::Terminated => "terminated",
3524 WorkflowTerminalKind::TimedOut => "timed_out",
3525 };
3526 let failure = description
3527 .failure
3528 .as_ref()
3529 .filter(|value| value.is_object());
3530 let nested_failure = failure
3531 .and_then(|value| value.get("failures"))
3532 .and_then(Value::as_array)
3533 .and_then(|failures| failures.last())
3534 .or_else(|| description.failures.last());
3535 let exception = description
3536 .exception
3537 .clone()
3538 .or_else(|| failure.and_then(|value| value.get("exception")).cloned())
3539 .or_else(|| {
3540 nested_failure
3541 .and_then(|value| value.get("exception_payload"))
3542 .cloned()
3543 });
3544 let string_field = |name: &str| {
3545 failure
3546 .and_then(|value| value.get(name))
3547 .and_then(Value::as_str)
3548 .or_else(|| {
3549 nested_failure
3550 .and_then(|value| value.get(name))
3551 .and_then(Value::as_str)
3552 })
3553 .map(str::to_string)
3554 };
3555 let exception_field = |name: &str| {
3556 exception
3557 .as_ref()
3558 .and_then(|value| value.get(name))
3559 .and_then(Value::as_str)
3560 .map(str::to_string)
3561 };
3562 let message = description
3563 .error
3564 .clone()
3565 .or_else(|| string_field("message"))
3566 .or_else(|| exception_field("message"));
3567 let reason = description
3568 .raw
3569 .get("reason")
3570 .and_then(Value::as_str)
3571 .map(str::to_string)
3572 .or_else(|| {
3573 failure
3574 .and_then(|value| value.get("reason"))
3575 .and_then(Value::as_str)
3576 .map(str::to_string)
3577 })
3578 .or_else(|| description.closed_reason.clone())
3579 .unwrap_or_else(|| default_reason.to_string());
3580 let failure_id = string_field("failure_id").or_else(|| {
3581 nested_failure
3582 .and_then(|value| value.get("id"))
3583 .and_then(Value::as_str)
3584 .map(str::to_string)
3585 });
3586
3587 WorkflowTerminalOutcome {
3588 kind,
3589 workflow_id: description
3590 .workflow_id
3591 .clone()
3592 .unwrap_or_else(|| workflow_id.to_string()),
3593 run_id: description
3594 .run_id
3595 .clone()
3596 .or_else(|| run_id.map(str::to_string)),
3597 reason,
3598 failure_category: string_field("failure_category")
3599 .or_else(|| Some(default_reason.to_string())),
3600 failure_id,
3601 exception_type: string_field("exception_type").or_else(|| exception_field("type")),
3602 exception_class: string_field("exception_class").or_else(|| exception_field("class")),
3603 non_retryable: failure
3604 .and_then(|value| value.get("non_retryable"))
3605 .and_then(Value::as_bool)
3606 .or_else(|| {
3607 nested_failure
3608 .and_then(|value| value.get("non_retryable"))
3609 .and_then(Value::as_bool)
3610 }),
3611 message,
3612 exception,
3613 raw: description.raw_value(),
3614 }
3615}
3616
3617#[derive(Clone, Debug, Deserialize)]
3618pub struct RegisterWorkerResponse {
3619 pub worker_id: String,
3620 pub registered: bool,
3621 #[serde(default)]
3622 pub heartbeat_interval_seconds: Option<u64>,
3623 #[serde(default)]
3624 pub protocol_version: Option<String>,
3625 #[serde(default)]
3626 pub server_capabilities: Option<Value>,
3627}
3628
3629#[derive(Clone, Debug, Deserialize, PartialEq, Eq)]
3631pub struct WorkerDeregistrationEnvelope {
3632 pub worker_id: String,
3633 pub outcome: String,
3634 pub recovered_workflow_task_count: u64,
3635}
3636
3637#[derive(Clone, Debug, Deserialize)]
3638pub struct PollWorkflowTaskResponse {
3639 #[serde(default)]
3640 pub task: Option<WorkflowTask>,
3641 #[serde(default)]
3642 pub poll_status: Option<String>,
3643 #[serde(default)]
3644 pub reason: Option<String>,
3645 #[serde(default)]
3646 pub protocol_version: Option<String>,
3647 #[serde(default)]
3648 pub server_capabilities: Option<Value>,
3649}
3650
3651impl PollWorkflowTaskResponse {
3652 pub fn outcome(&self) -> WorkerPollOutcome {
3654 worker_poll_outcome(
3655 self.task.is_some(),
3656 self.poll_status.as_deref(),
3657 self.reason.as_deref(),
3658 )
3659 }
3660}
3661
3662#[derive(Clone, Debug, Deserialize)]
3663pub struct PollActivityTaskResponse {
3664 #[serde(default)]
3665 pub task: Option<ActivityTask>,
3666 #[serde(default)]
3667 pub poll_status: Option<String>,
3668 #[serde(default)]
3669 pub reason: Option<String>,
3670}
3671
3672impl PollActivityTaskResponse {
3673 pub fn outcome(&self) -> WorkerPollOutcome {
3675 worker_poll_outcome(
3676 self.task.is_some(),
3677 self.poll_status.as_deref(),
3678 self.reason.as_deref(),
3679 )
3680 }
3681}
3682
3683#[derive(Clone, Debug, Deserialize)]
3684pub struct PollQueryTaskResponse {
3685 #[serde(default)]
3686 pub task: Option<QueryTask>,
3687 #[serde(default)]
3688 pub poll_status: Option<String>,
3689 #[serde(default)]
3690 pub reason: Option<String>,
3691}
3692
3693impl PollQueryTaskResponse {
3694 pub fn outcome(&self) -> WorkerPollOutcome {
3696 worker_poll_outcome(
3697 self.task.is_some(),
3698 self.poll_status.as_deref(),
3699 self.reason.as_deref(),
3700 )
3701 }
3702}
3703
3704#[derive(Clone, Debug, PartialEq, Eq)]
3706pub enum WorkerPollOutcome {
3707 Task,
3709 Idle {
3711 poll_status: Option<String>,
3712 reason: Option<String>,
3713 },
3714 Stop {
3716 poll_status: Option<String>,
3717 reason: Option<String>,
3718 },
3719}
3720
3721impl WorkerPollOutcome {
3722 pub fn should_stop(&self) -> bool {
3723 matches!(self, Self::Stop { .. })
3724 }
3725}
3726
3727fn worker_poll_outcome(
3728 has_task: bool,
3729 poll_status: Option<&str>,
3730 reason: Option<&str>,
3731) -> WorkerPollOutcome {
3732 if worker_poll_is_stop(poll_status, reason) {
3733 return WorkerPollOutcome::Stop {
3734 poll_status: poll_status.map(str::to_string),
3735 reason: reason.map(str::to_string),
3736 };
3737 }
3738
3739 if has_task {
3740 WorkerPollOutcome::Task
3741 } else {
3742 WorkerPollOutcome::Idle {
3743 poll_status: poll_status.map(str::to_string),
3744 reason: reason.map(str::to_string),
3745 }
3746 }
3747}
3748
3749#[derive(Clone, Debug, Deserialize)]
3751pub struct QueryTask {
3752 pub query_task_id: String,
3753 #[serde(default = "default_workflow_task_attempt")]
3754 pub query_task_attempt: u64,
3755 #[serde(default)]
3756 pub lease_owner: Option<String>,
3757 #[serde(default)]
3758 pub workflow_id: Option<String>,
3759 #[serde(default)]
3760 pub run_id: Option<String>,
3761 pub workflow_type: String,
3762 pub query_name: String,
3763 #[serde(default = "default_payload_codec")]
3764 pub payload_codec: String,
3765 #[serde(default)]
3766 pub workflow_arguments: Option<Value>,
3767 #[serde(default)]
3768 pub query_arguments: Option<Value>,
3769 #[serde(default)]
3770 pub history_events: Vec<HistoryEvent>,
3771 #[serde(default)]
3772 pub history_export: Option<Value>,
3773 #[serde(default)]
3774 pub run_status: Option<String>,
3775}
3776
3777#[derive(Clone, Debug, Deserialize)]
3778pub struct WorkflowTask {
3779 pub task_id: String,
3780 #[serde(default)]
3781 pub workflow_id: Option<String>,
3782 #[serde(default)]
3783 pub run_id: Option<String>,
3784 pub workflow_type: String,
3785 #[serde(default = "default_payload_codec")]
3786 pub payload_codec: String,
3787 #[serde(default)]
3788 pub arguments: Option<Value>,
3789 #[serde(default)]
3790 pub history_events: Vec<HistoryEvent>,
3791 #[serde(default)]
3792 pub total_history_events: Option<u64>,
3793 #[serde(default)]
3794 pub history_size_bytes: Option<u64>,
3795 #[serde(default)]
3796 pub continue_as_new_recommended: Option<bool>,
3797 #[serde(default)]
3798 pub history_budget_pressure: Option<String>,
3799 #[serde(default)]
3800 pub next_history_page_token: Option<String>,
3801 #[serde(default = "default_workflow_task_attempt")]
3802 pub workflow_task_attempt: u64,
3803 #[serde(default)]
3804 pub workflow_signal_id: Option<String>,
3805 #[serde(default)]
3806 pub signal_name: Option<String>,
3807 #[serde(default)]
3808 pub signal_arguments: Option<Value>,
3809 #[serde(default)]
3810 pub workflow_update_id: Option<String>,
3811 #[serde(default)]
3812 pub update_name: Option<String>,
3813 #[serde(default)]
3814 pub lease_owner: Option<String>,
3815}
3816
3817impl WorkflowTask {
3818 fn append_history_page(&mut self, page: WorkflowTaskHistoryPage) {
3819 self.history_events.extend(page.history_events);
3820
3821 if page.total_history_events.is_some() {
3822 self.total_history_events = page.total_history_events;
3823 }
3824
3825 self.next_history_page_token = page
3826 .next_history_page_token
3827 .filter(|token| !token.is_empty());
3828 }
3829}
3830
3831#[derive(Clone, Debug, Deserialize)]
3832struct WorkflowTaskHistoryPage {
3833 #[serde(default)]
3834 history_events: Vec<HistoryEvent>,
3835 #[serde(default)]
3836 total_history_events: Option<u64>,
3837 #[serde(default)]
3838 next_history_page_token: Option<String>,
3839}
3840
3841#[derive(Clone, Debug, Deserialize)]
3842pub struct ActivityTask {
3843 pub task_id: String,
3844 #[serde(default)]
3845 pub activity_attempt_id: Option<String>,
3846 #[serde(default)]
3847 pub attempt_id: Option<String>,
3848 pub activity_type: String,
3849 #[serde(default = "default_payload_codec")]
3850 pub payload_codec: String,
3851 #[serde(default)]
3852 pub arguments: Option<Value>,
3853 #[serde(default = "default_attempt_number")]
3854 pub attempt_number: u64,
3855 #[serde(default)]
3856 pub lease_owner: Option<String>,
3857}
3858
3859#[derive(Clone, Debug, Deserialize)]
3860pub struct HistoryEvent {
3861 #[serde(alias = "type")]
3862 pub event_type: String,
3863 #[serde(default)]
3864 pub payload: Value,
3865 #[serde(flatten)]
3866 pub raw: HashMap<String, Value>,
3867}
3868
3869#[derive(Clone, Debug, PartialEq)]
3871pub struct QuerySignal {
3872 pub id: Option<String>,
3873 pub name: String,
3874 pub arguments: Vec<Value>,
3875 avro_arguments: Vec<AvroValue>,
3876 pub workflow_sequence: Option<u64>,
3877}
3878
3879impl QuerySignal {
3880 pub fn arguments_avro_value(&self) -> &[AvroValue] {
3882 &self.avro_arguments
3883 }
3884}
3885
3886#[derive(Clone, Debug)]
3892pub struct QueryContext {
3893 pub workflow_id: Option<String>,
3894 pub run_id: Option<String>,
3895 pub workflow_type: String,
3896 pub run_status: Option<String>,
3897 workflow_input: Value,
3898 workflow_input_avro_value: AvroValue,
3899 history_events: Arc<Vec<HistoryEvent>>,
3900 signal_events: Arc<Vec<QuerySignal>>,
3901}
3902
3903impl QueryContext {
3904 pub fn workflow_input(&self) -> &Value {
3906 &self.workflow_input
3907 }
3908
3909 pub fn workflow_input_avro_value(&self) -> &AvroValue {
3911 &self.workflow_input_avro_value
3912 }
3913
3914 pub fn history_events(&self) -> &[HistoryEvent] {
3916 self.history_events.as_slice()
3917 }
3918
3919 pub fn signal_events(&self) -> &[QuerySignal] {
3921 self.signal_events.as_slice()
3922 }
3923
3924 pub fn signals(&self, signal_name: &str) -> Vec<Vec<Value>> {
3926 self.signal_events
3927 .iter()
3928 .filter(|signal| signal.name == signal_name)
3929 .map(|signal| signal.arguments.clone())
3930 .collect()
3931 }
3932
3933 pub fn signals_avro_value(&self, signal_name: &str) -> Vec<Vec<AvroValue>> {
3935 self.signal_events
3936 .iter()
3937 .filter(|signal| signal.name == signal_name)
3938 .map(|signal| signal.avro_arguments.clone())
3939 .collect()
3940 }
3941}
3942
3943#[derive(Clone, Debug, Deserialize)]
3944pub struct ActivityHeartbeatResponse {
3945 #[serde(default)]
3946 pub cancel_requested: bool,
3947 #[serde(default)]
3948 pub heartbeat_recorded: bool,
3949 #[serde(default)]
3950 pub can_continue: Option<bool>,
3951 #[serde(default)]
3952 pub reason: Option<String>,
3953 #[serde(default)]
3954 pub run_closed_reason: Option<String>,
3955 #[serde(default)]
3956 pub run_closed_at: Option<String>,
3957 #[serde(default)]
3958 pub lease_expires_at: Option<String>,
3959 #[serde(default)]
3960 pub last_heartbeat_at: Option<String>,
3961}
3962
3963impl ActivityHeartbeatResponse {
3964 pub fn should_stop(&self) -> bool {
3966 self.cancel_requested || self.can_continue == Some(false)
3967 }
3968}
3969
3970fn default_payload_codec() -> String {
3971 DEFAULT_CODEC.to_string()
3972}
3973
3974fn default_workflow_task_attempt() -> u64 {
3975 1
3976}
3977
3978fn default_attempt_number() -> u64 {
3979 1
3980}
3981
3982type WorkflowFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
3983type WorkflowHandler = Arc<dyn Fn(WorkflowContext, AvroValue) -> WorkflowFuture + Send + Sync>;
3984type ErasedWorkflowState = Arc<dyn Any + Send + Sync>;
3985type WorkflowStateSnapshot = Arc<dyn Fn() -> Result<ErasedWorkflowState> + Send + Sync>;
3986type ReplayedWorkflowHandler =
3987 Arc<dyn Fn(WorkflowContext, AvroValue) -> ReplayedWorkflowInvocation + Send + Sync>;
3988type ActivityFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
3989type ActivityHandler = Arc<dyn Fn(ActivityContext, AvroValue) -> ActivityFuture + Send + Sync>;
3990type QueryFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
3991type QueryHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
3992type UpdateHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
3993type ReplayedQueryHandler = Arc<
3994 dyn Fn(QueryContext, ErasedWorkflowState, AvroValue) -> std::result::Result<QueryFuture, String>
3995 + Send
3996 + Sync,
3997>;
3998type WorkerHeartbeatObserver = Arc<dyn Fn(&WorkerHeartbeatObservation) + Send + Sync>;
3999
4000struct ReplayedWorkflowInvocation {
4001 future: WorkflowFuture,
4002 snapshot: WorkflowStateSnapshot,
4003}
4004
4005#[derive(Clone)]
4006struct RegisteredWorkflow {
4007 execute: WorkflowHandler,
4008 replay: Option<ReplayedWorkflowHandler>,
4009 state_type: Option<TypeId>,
4010}
4011
4012#[derive(Clone)]
4013enum RegisteredQuery {
4014 Snapshot(QueryHandler),
4015 Replayed {
4016 state_type: TypeId,
4017 handler: ReplayedQueryHandler,
4018 },
4019}
4020
4021#[derive(Clone, Debug)]
4022pub struct WorkerHeartbeatObservation {
4023 pub worker_id: String,
4024 pub task_queue: String,
4025 pub acknowledged_at_unix_millis: u64,
4026 pub acknowledgement: Value,
4027}
4028
4029#[derive(Clone, Copy, Debug)]
4036pub struct WorkerRetryPolicy {
4037 pub max_retries: usize,
4039 pub initial_backoff: Duration,
4041 pub max_backoff: Duration,
4043}
4044
4045impl Default for WorkerRetryPolicy {
4046 fn default() -> Self {
4047 Self {
4048 max_retries: 5,
4049 initial_backoff: Duration::from_millis(100),
4050 max_backoff: Duration::from_secs(5),
4051 }
4052 }
4053}
4054
4055#[derive(Clone, Copy, Debug, PartialEq, Eq)]
4056enum ManagedPollOutcome {
4057 Idle,
4058 Handled,
4059 Stop,
4060}
4061
4062#[derive(Clone)]
4063pub struct Worker {
4064 client: Client,
4065 worker_id: String,
4066 task_queue: String,
4067 workflows: HashMap<String, RegisteredWorkflow>,
4068 activities: HashMap<String, ActivityHandler>,
4069 queries: HashMap<String, HashMap<String, RegisteredQuery>>,
4070 updates: HashMap<String, HashMap<String, UpdateHandler>>,
4071 max_concurrent_workflow_tasks: usize,
4072 max_concurrent_activity_tasks: usize,
4073 poll_timeout: Duration,
4074 heartbeat_interval: Duration,
4075 retry_policy: WorkerRetryPolicy,
4076 heartbeat_observer: Option<WorkerHeartbeatObserver>,
4077}
4078
4079impl Worker {
4080 pub fn new(client: Client, task_queue: impl Into<String>) -> Self {
4081 Self {
4082 client,
4083 worker_id: default_worker_id(),
4084 task_queue: task_queue.into(),
4085 workflows: HashMap::new(),
4086 activities: HashMap::new(),
4087 queries: HashMap::new(),
4088 updates: HashMap::new(),
4089 max_concurrent_workflow_tasks: 10,
4090 max_concurrent_activity_tasks: 10,
4091 poll_timeout: Duration::from_secs(30),
4092 heartbeat_interval: Duration::from_secs(60),
4093 retry_policy: WorkerRetryPolicy::default(),
4094 heartbeat_observer: None,
4095 }
4096 }
4097
4098 pub fn worker_id(mut self, worker_id: impl Into<String>) -> Self {
4099 self.worker_id = worker_id.into();
4100 self
4101 }
4102
4103 pub fn poll_timeout(mut self, timeout: Duration) -> Self {
4104 self.poll_timeout = timeout;
4105 self
4106 }
4107
4108 pub fn heartbeat_interval(mut self, interval: Duration) -> Self {
4109 self.heartbeat_interval = interval;
4110 self
4111 }
4112
4113 pub fn retry_policy(mut self, policy: WorkerRetryPolicy) -> Self {
4115 self.retry_policy = policy;
4116 self
4117 }
4118
4119 pub fn on_worker_heartbeat<F>(mut self, observer: F) -> Self
4120 where
4121 F: Fn(&WorkerHeartbeatObservation) + Send + Sync + 'static,
4122 {
4123 self.heartbeat_observer = Some(Arc::new(observer));
4124 self
4125 }
4126
4127 pub fn max_concurrent_workflow_tasks(mut self, count: usize) -> Self {
4128 self.max_concurrent_workflow_tasks = count.max(1);
4129 self
4130 }
4131
4132 pub fn max_concurrent_activity_tasks(mut self, count: usize) -> Self {
4133 self.max_concurrent_activity_tasks = count.max(1);
4134 self
4135 }
4136
4137 pub fn register_workflow<F, Fut>(&mut self, workflow_type: impl Into<String>, handler: F)
4144 where
4145 F: Fn(WorkflowContext, Value) -> Fut + Send + Sync + 'static,
4146 Fut: Future<Output = Result<Value>> + Send + 'static,
4147 {
4148 let handler = Arc::new(handler);
4149 self.workflows.insert(
4150 workflow_type.into(),
4151 RegisteredWorkflow {
4152 execute: Arc::new(move |ctx, input| {
4153 let handler = Arc::clone(&handler);
4154 Box::pin(async move {
4155 let result = handler(ctx, input.into_json()?).await?;
4156 AvroValue::from_serialize(&result)
4157 })
4158 }),
4159 replay: None,
4160 state_type: None,
4161 },
4162 );
4163 }
4164
4165 pub fn register_workflow_avro_value<F, Fut>(
4167 &mut self,
4168 workflow_type: impl Into<String>,
4169 handler: F,
4170 ) where
4171 F: Fn(WorkflowContext, AvroValue) -> Fut + Send + Sync + 'static,
4172 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4173 {
4174 self.workflows.insert(
4175 workflow_type.into(),
4176 RegisteredWorkflow {
4177 execute: Arc::new(move |ctx, input| Box::pin(handler(ctx, input))),
4178 replay: None,
4179 state_type: None,
4180 },
4181 );
4182 }
4183
4184 pub fn register_replayed_workflow<S, Factory, F, Fut>(
4192 &mut self,
4193 workflow_type: impl Into<String>,
4194 state_factory: Factory,
4195 handler: F,
4196 ) where
4197 S: Clone + Send + Sync + 'static,
4198 Factory: Fn() -> S + Send + Sync + 'static,
4199 F: Fn(WorkflowContext, Value, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
4200 Fut: Future<Output = Result<Value>> + Send + 'static,
4201 {
4202 let state_factory = Arc::new(state_factory);
4203 let handler = Arc::new(handler);
4204
4205 let execute_factory = Arc::clone(&state_factory);
4206 let execute_handler = Arc::clone(&handler);
4207 let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4208 let state = WorkflowInstance::new(execute_factory());
4209 let handler = Arc::clone(&execute_handler);
4210 Box::pin(async move {
4211 let result = handler(ctx, input.into_json()?, state).await?;
4212 AvroValue::from_serialize(&result)
4213 }) as WorkflowFuture
4214 });
4215
4216 let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4217 let state = WorkflowInstance::new(state_factory());
4218 let snapshot_state = state.clone();
4219 let snapshot: WorkflowStateSnapshot =
4220 Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
4221 let replay_handler = Arc::clone(&handler);
4222 let future = async move {
4223 let result = replay_handler(ctx, input.into_json()?, state).await?;
4224 AvroValue::from_serialize(&result)
4225 };
4226 ReplayedWorkflowInvocation {
4227 future: Box::pin(future),
4228 snapshot,
4229 }
4230 });
4231
4232 self.workflows.insert(
4233 workflow_type.into(),
4234 RegisteredWorkflow {
4235 execute,
4236 replay: Some(replay),
4237 state_type: Some(TypeId::of::<S>()),
4238 },
4239 );
4240 }
4241
4242 pub fn register_replayed_workflow_avro_value<S, Factory, F, Fut>(
4244 &mut self,
4245 workflow_type: impl Into<String>,
4246 state_factory: Factory,
4247 handler: F,
4248 ) where
4249 S: Clone + Send + Sync + 'static,
4250 Factory: Fn() -> S + Send + Sync + 'static,
4251 F: Fn(WorkflowContext, AvroValue, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
4252 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4253 {
4254 let state_factory = Arc::new(state_factory);
4255 let handler = Arc::new(handler);
4256
4257 let execute_factory = Arc::clone(&state_factory);
4258 let execute_handler = Arc::clone(&handler);
4259 let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4260 let state = WorkflowInstance::new(execute_factory());
4261 Box::pin(execute_handler(ctx, input, state)) as WorkflowFuture
4262 });
4263
4264 let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4265 let state = WorkflowInstance::new(state_factory());
4266 let snapshot_state = state.clone();
4267 let snapshot: WorkflowStateSnapshot =
4268 Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
4269 ReplayedWorkflowInvocation {
4270 future: Box::pin(handler(ctx, input, state)),
4271 snapshot,
4272 }
4273 });
4274
4275 self.workflows.insert(
4276 workflow_type.into(),
4277 RegisteredWorkflow {
4278 execute,
4279 replay: Some(replay),
4280 state_type: Some(TypeId::of::<S>()),
4281 },
4282 );
4283 }
4284
4285 pub fn register_activity<F, Fut>(&mut self, activity_type: impl Into<String>, handler: F)
4286 where
4287 F: Fn(ActivityContext, Value) -> Fut + Send + Sync + 'static,
4288 Fut: Future<Output = Result<Value>> + Send + 'static,
4289 {
4290 let handler = Arc::new(handler);
4291 self.activities.insert(
4292 activity_type.into(),
4293 Arc::new(move |ctx, args| {
4294 let handler = Arc::clone(&handler);
4295 Box::pin(async move {
4296 let result = handler(ctx, args.into_json()?).await?;
4297 AvroValue::from_serialize(&result)
4298 })
4299 }),
4300 );
4301 }
4302
4303 pub fn register_activity_avro_value<F, Fut>(
4305 &mut self,
4306 activity_type: impl Into<String>,
4307 handler: F,
4308 ) where
4309 F: Fn(ActivityContext, AvroValue) -> Fut + Send + Sync + 'static,
4310 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4311 {
4312 self.activities.insert(
4313 activity_type.into(),
4314 Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
4315 );
4316 }
4317
4318 pub fn register_query<F, Fut>(
4324 &mut self,
4325 workflow_type: impl Into<String>,
4326 query_name: impl Into<String>,
4327 handler: F,
4328 ) where
4329 F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
4330 Fut: Future<Output = Result<Value>> + Send + 'static,
4331 {
4332 let handler = Arc::new(handler);
4333 self.queries
4334 .entry(workflow_type.into())
4335 .or_default()
4336 .insert(
4337 query_name.into(),
4338 RegisteredQuery::Snapshot(Arc::new(move |ctx, args| {
4339 let handler = Arc::clone(&handler);
4340 Box::pin(async move {
4341 let result = handler(ctx, args.into_json()?).await?;
4342 AvroValue::from_serialize(&result)
4343 })
4344 })),
4345 );
4346 }
4347
4348 pub fn register_query_avro_value<F, Fut>(
4350 &mut self,
4351 workflow_type: impl Into<String>,
4352 query_name: impl Into<String>,
4353 handler: F,
4354 ) where
4355 F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
4356 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4357 {
4358 self.queries
4359 .entry(workflow_type.into())
4360 .or_default()
4361 .insert(
4362 query_name.into(),
4363 RegisteredQuery::Snapshot(Arc::new(move |ctx, args| Box::pin(handler(ctx, args)))),
4364 );
4365 }
4366
4367 pub fn register_replayed_query<S, F, Fut>(
4374 &mut self,
4375 workflow_type: impl Into<String>,
4376 query_name: impl Into<String>,
4377 handler: F,
4378 ) where
4379 S: Clone + Send + Sync + 'static,
4380 F: Fn(QueryContext, Arc<S>, Value) -> Fut + Send + Sync + 'static,
4381 Fut: Future<Output = Result<Value>> + Send + 'static,
4382 {
4383 let handler = Arc::new(handler);
4384 let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
4385 let state = state.downcast::<S>().map_err(|_| {
4386 "registered query state type does not match the replayed workflow state".to_string()
4387 })?;
4388 let handler = Arc::clone(&handler);
4389 Ok(Box::pin(async move {
4390 let result = handler(ctx, state, args.into_json()?).await?;
4391 AvroValue::from_serialize(&result)
4392 }))
4393 });
4394
4395 self.queries
4396 .entry(workflow_type.into())
4397 .or_default()
4398 .insert(
4399 query_name.into(),
4400 RegisteredQuery::Replayed {
4401 state_type: TypeId::of::<S>(),
4402 handler: erased_handler,
4403 },
4404 );
4405 }
4406
4407 pub fn register_replayed_query_avro_value<S, F, Fut>(
4409 &mut self,
4410 workflow_type: impl Into<String>,
4411 query_name: impl Into<String>,
4412 handler: F,
4413 ) where
4414 S: Clone + Send + Sync + 'static,
4415 F: Fn(QueryContext, Arc<S>, AvroValue) -> Fut + Send + Sync + 'static,
4416 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4417 {
4418 let handler = Arc::new(handler);
4419 let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
4420 let state = state.downcast::<S>().map_err(|_| {
4421 "registered query state type does not match the replayed workflow state".to_string()
4422 })?;
4423 Ok(Box::pin(handler(ctx, state, args)))
4424 });
4425
4426 self.queries
4427 .entry(workflow_type.into())
4428 .or_default()
4429 .insert(
4430 query_name.into(),
4431 RegisteredQuery::Replayed {
4432 state_type: TypeId::of::<S>(),
4433 handler: erased_handler,
4434 },
4435 );
4436 }
4437
4438 pub fn register_update<F, Fut>(
4440 &mut self,
4441 workflow_type: impl Into<String>,
4442 update_name: impl Into<String>,
4443 handler: F,
4444 ) where
4445 F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
4446 Fut: Future<Output = Result<Value>> + Send + 'static,
4447 {
4448 let handler = Arc::new(handler);
4449 self.updates
4450 .entry(workflow_type.into())
4451 .or_default()
4452 .insert(
4453 update_name.into(),
4454 Arc::new(move |ctx, args| {
4455 let handler = Arc::clone(&handler);
4456 Box::pin(async move {
4457 let result = handler(ctx, args.into_json()?).await?;
4458 AvroValue::from_serialize(&result)
4459 })
4460 }),
4461 );
4462 }
4463
4464 pub fn register_update_avro_value<F, Fut>(
4466 &mut self,
4467 workflow_type: impl Into<String>,
4468 update_name: impl Into<String>,
4469 handler: F,
4470 ) where
4471 F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
4472 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4473 {
4474 self.updates
4475 .entry(workflow_type.into())
4476 .or_default()
4477 .insert(
4478 update_name.into(),
4479 Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
4480 );
4481 }
4482
4483 pub async fn register(&self) -> Result<RegisterWorkerResponse> {
4484 let mut command_contracts = serde_json::Map::new();
4485 for workflow_type in self.workflows.keys() {
4486 let mut queries = self
4487 .queries
4488 .get(workflow_type)
4489 .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
4490 .unwrap_or_default();
4491 queries.sort();
4492 let mut updates = self
4493 .updates
4494 .get(workflow_type)
4495 .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
4496 .unwrap_or_default();
4497 updates.sort();
4498 if !queries.is_empty() || !updates.is_empty() {
4499 command_contracts.insert(
4500 workflow_type.clone(),
4501 json!({
4502 "queries": queries,
4503 "updates": updates,
4504 }),
4505 );
4506 }
4507 }
4508
4509 self.client
4510 .register_worker_with_command_contracts(
4511 &self.worker_id,
4512 &self.task_queue,
4513 self.workflows.keys().cloned().collect(),
4514 self.activities.keys().cloned().collect(),
4515 self.max_concurrent_workflow_tasks,
4516 self.max_concurrent_activity_tasks,
4517 [
4518 (!self.queries.is_empty()).then(|| QUERY_TASKS_CAPABILITY.to_string()),
4519 (!self.updates.is_empty()).then(|| WORKFLOW_UPDATES_CAPABILITY.to_string()),
4520 ]
4521 .into_iter()
4522 .flatten()
4523 .collect(),
4524 Value::Object(command_contracts),
4525 )
4526 .await
4527 }
4528
4529 pub async fn run(&self) -> Result<()> {
4535 self.run_until(std::future::pending::<()>()).await
4536 }
4537
4538 pub async fn run_until<F>(&self, shutdown: F) -> Result<()>
4542 where
4543 F: Future<Output = ()>,
4544 {
4545 let registration = self.register().await?;
4546 if !registration.registered {
4547 return Err(Error::WorkerLoop(format!(
4548 "worker registration for {:?} was not accepted",
4549 self.worker_id
4550 )));
4551 }
4552 let registered_worker_id = registration.worker_id.clone();
4553 let primary = self.run_registered_until(shutdown, registration).await;
4554 let deregistration = self
4555 .client
4556 .deregister_worker_registration(®istered_worker_id)
4557 .await;
4558
4559 match (primary, deregistration) {
4560 (Ok(()), Ok(_)) => Ok(()),
4561 (Ok(()), Err(deregistration)) => Err(deregistration),
4562 (Err(primary), Ok(_)) => Err(primary),
4563 (Err(primary), Err(deregistration)) => Err(Error::WorkerShutdown {
4564 primary: Box::new(primary),
4565 deregistration: Box::new(deregistration),
4566 }),
4567 }
4568 }
4569
4570 async fn run_registered_until<F>(
4571 &self,
4572 shutdown: F,
4573 registration: RegisterWorkerResponse,
4574 ) -> Result<()>
4575 where
4576 F: Future<Output = ()>,
4577 {
4578 let heartbeat_interval = Duration::from_secs(
4579 registration
4580 .heartbeat_interval_seconds
4581 .unwrap_or(self.heartbeat_interval.as_secs().max(1)),
4582 );
4583 let heartbeat = tokio::time::sleep(Duration::ZERO);
4589 tokio::pin!(heartbeat);
4590 tokio::pin!(shutdown);
4591 let stop = Arc::new(AtomicBool::new(false));
4592 let mut workflow_poller = (!self.workflows.is_empty()).then(|| {
4596 let worker = self.clone();
4597 let stop = Arc::clone(&stop);
4598 tokio::spawn(async move { worker.poll_workflows_until_stopped(stop).await })
4599 });
4600 let mut activity_poller = (!self.activities.is_empty()).then(|| {
4601 let worker = self.clone();
4602 let stop = Arc::clone(&stop);
4603 tokio::spawn(async move { worker.poll_activities_until_stopped(stop).await })
4604 });
4605 let mut query_poller = (!self.queries.is_empty()).then(|| {
4606 let worker = self.clone();
4607 let stop = Arc::clone(&stop);
4608 tokio::spawn(async move { worker.poll_queries_until_stopped(stop).await })
4609 });
4610
4611 loop {
4612 tokio::select! {
4613 _ = &mut shutdown => {
4614 stop.store(true, Ordering::SeqCst);
4615 break;
4616 }
4617 _ = &mut heartbeat => {
4618 let result = self.retry_worker_operation(|| {
4619 self.client.heartbeat_worker(
4620 &self.worker_id,
4621 self.max_concurrent_workflow_tasks,
4622 self.max_concurrent_activity_tasks,
4623 )
4624 }).await;
4625 heartbeat
4626 .as_mut()
4627 .reset(tokio::time::Instant::now() + heartbeat_interval);
4628 match result {
4629 Ok(acknowledgement) => {
4630 if let Some(observer) = &self.heartbeat_observer {
4631 observer(&WorkerHeartbeatObservation {
4632 worker_id: self.worker_id.clone(),
4633 task_queue: self.task_queue.clone(),
4634 acknowledged_at_unix_millis: SystemTime::now()
4635 .duration_since(UNIX_EPOCH)
4636 .unwrap_or_default()
4637 .as_millis()
4638 .min(u64::MAX as u128)
4639 as u64,
4640 acknowledgement,
4641 });
4642 }
4643 }
4644 Err(error) => {
4645 stop.store(true, Ordering::SeqCst);
4646 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await?;
4647 return Err(error);
4648 }
4649 }
4650 }
4651 result = OptionFuture::from(workflow_poller.as_mut()), if workflow_poller.is_some() => {
4652 workflow_poller = None;
4653 let stopped_by_server = stop.load(Ordering::SeqCst);
4654 stop.store(true, Ordering::SeqCst);
4655 let poller_result = optional_poller_result("workflow", result);
4656 let join_result =
4657 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
4658 poller_result?;
4659 join_result?;
4660 if stopped_by_server {
4661 return Ok(());
4662 }
4663 return Err(Error::WorkerLoop(
4664 "workflow poller stopped unexpectedly".to_string(),
4665 ));
4666 }
4667 result = OptionFuture::from(activity_poller.as_mut()), if activity_poller.is_some() => {
4668 activity_poller = None;
4669 let stopped_by_server = stop.load(Ordering::SeqCst);
4670 stop.store(true, Ordering::SeqCst);
4671 let poller_result = optional_poller_result("activity", result);
4672 let join_result =
4673 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
4674 poller_result?;
4675 join_result?;
4676 if stopped_by_server {
4677 return Ok(());
4678 }
4679 return Err(Error::WorkerLoop(
4680 "activity poller stopped unexpectedly".to_string(),
4681 ));
4682 }
4683 result = OptionFuture::from(query_poller.as_mut()), if query_poller.is_some() => {
4684 query_poller = None;
4685 let stopped_by_server = stop.load(Ordering::SeqCst);
4686 stop.store(true, Ordering::SeqCst);
4687 let poller_result = optional_poller_result("query", result);
4688 let join_result =
4689 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
4690 poller_result?;
4691 join_result?;
4692 if stopped_by_server {
4693 return Ok(());
4694 }
4695 return Err(Error::WorkerLoop(
4696 "query poller stopped unexpectedly".to_string(),
4697 ));
4698 }
4699 }
4700 }
4701
4702 join_pollers(
4703 workflow_poller.take(),
4704 activity_poller.take(),
4705 query_poller.take(),
4706 )
4707 .await
4708 }
4709
4710 pub async fn run_once(&self) -> Result<usize> {
4724 let mut handled = 0;
4725 match self.poll_workflow_once().await? {
4726 ManagedPollOutcome::Handled => handled += 1,
4727 ManagedPollOutcome::Stop => return Ok(handled),
4728 ManagedPollOutcome::Idle => {}
4729 }
4730 match self.poll_activity_once().await? {
4731 ManagedPollOutcome::Handled => handled += 1,
4732 ManagedPollOutcome::Stop => return Ok(handled),
4733 ManagedPollOutcome::Idle => {}
4734 }
4735 if !self.queries.is_empty() {
4736 match self.poll_query_once().await? {
4737 ManagedPollOutcome::Handled => handled += 1,
4738 ManagedPollOutcome::Stop => return Ok(handled),
4739 ManagedPollOutcome::Idle => {}
4740 }
4741 }
4742 Ok(handled)
4743 }
4744
4745 async fn poll_workflow_once(&self) -> Result<ManagedPollOutcome> {
4746 let poll_request_id = unique_request_id("rust-workflow-poll");
4747 let response = self
4748 .retry_worker_operation(|| {
4749 self.client.poll_workflow_task_response_with_request_id(
4750 &self.worker_id,
4751 &self.task_queue,
4752 self.poll_timeout,
4753 &poll_request_id,
4754 0,
4755 )
4756 })
4757 .await?;
4758 if response.outcome().should_stop() {
4759 return Ok(ManagedPollOutcome::Stop);
4760 }
4761 let Some(task) = response.task else {
4762 return Ok(ManagedPollOutcome::Idle);
4763 };
4764
4765 let task_id = task.task_id.clone();
4766 let attempt = task.workflow_task_attempt;
4767 let run_id = task.run_id.clone();
4768 let lease_owner = task
4769 .lease_owner
4770 .clone()
4771 .unwrap_or_else(|| self.worker_id.clone());
4772
4773 match self.execute_workflow_task(task) {
4774 Ok(commands) if commands.is_empty() => {
4775 self.client
4781 .fail_workflow_task_with_type(
4782 &task_id,
4783 &lease_owner,
4784 attempt,
4785 WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE,
4786 WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE,
4787 )
4788 .await?;
4789 }
4790 Ok(commands) => {
4791 let completion = self
4792 .client
4793 .complete_workflow_task(&task_id, &lease_owner, attempt, commands)
4794 .await;
4795 if let Err(error) = completion {
4796 if !workflow_task_completion_is_terminal_timeout(
4797 &error,
4798 &task_id,
4799 attempt,
4800 run_id.as_deref(),
4801 ) {
4802 return Err(error);
4803 }
4804 }
4805 }
4806 Err(error) => {
4807 self.client
4808 .fail_workflow_task(&task_id, &lease_owner, attempt, error.to_string())
4809 .await?;
4810 }
4811 }
4812
4813 Ok(ManagedPollOutcome::Handled)
4814 }
4815
4816 async fn poll_workflows_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
4817 while !stop.load(Ordering::SeqCst) {
4818 if self.poll_workflow_once().await? == ManagedPollOutcome::Stop {
4819 stop.store(true, Ordering::SeqCst);
4820 break;
4821 }
4822 }
4823
4824 Ok(())
4825 }
4826
4827 async fn poll_activity_once(&self) -> Result<ManagedPollOutcome> {
4828 let poll_request_id = unique_request_id("rust-activity-poll");
4829 let response = self
4830 .retry_worker_operation(|| {
4831 self.client.poll_activity_task_response_with_request_id(
4832 &self.worker_id,
4833 &self.task_queue,
4834 self.poll_timeout,
4835 &poll_request_id,
4836 0,
4837 )
4838 })
4839 .await?;
4840 if response.outcome().should_stop() {
4841 return Ok(ManagedPollOutcome::Stop);
4842 }
4843 let Some(task) = response.task else {
4844 return Ok(ManagedPollOutcome::Idle);
4845 };
4846
4847 let task_id = task.task_id.clone();
4848 let attempt_id = task
4849 .activity_attempt_id
4850 .clone()
4851 .or(task.attempt_id.clone())
4852 .unwrap_or_default();
4853 let lease_owner = task
4854 .lease_owner
4855 .clone()
4856 .unwrap_or_else(|| self.worker_id.clone());
4857 let codec = task.payload_codec.clone();
4858 let result = self.execute_activity_task(task).await;
4859 match result {
4860 Ok(value) => {
4861 let completion = self
4862 .client
4863 .complete_activity_task(&task_id, &attempt_id, &lease_owner, value, &codec)
4864 .await;
4865 if let Err(error) = completion {
4866 if !activity_task_rejection_is_final(&error) {
4867 return Err(error);
4868 }
4869 }
4870 }
4871 Err(error) => {
4872 let failure = self
4873 .client
4874 .fail_activity_task(
4875 &task_id,
4876 &attempt_id,
4877 &lease_owner,
4878 error.to_string(),
4879 false,
4880 )
4881 .await;
4882 if let Err(error) = failure {
4883 if !activity_task_rejection_is_final(&error) {
4884 return Err(error);
4885 }
4886 }
4887 }
4888 }
4889
4890 Ok(ManagedPollOutcome::Handled)
4891 }
4892
4893 async fn poll_activities_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
4894 while !stop.load(Ordering::SeqCst) {
4895 if self.poll_activity_once().await? == ManagedPollOutcome::Stop {
4896 stop.store(true, Ordering::SeqCst);
4897 break;
4898 }
4899 }
4900
4901 Ok(())
4902 }
4903
4904 async fn poll_query_once(&self) -> Result<ManagedPollOutcome> {
4905 let poll_request_id = unique_request_id("rust-query-poll");
4906 let response = self
4907 .retry_worker_operation(|| {
4908 self.client.poll_query_task_response_with_request_id(
4909 &self.worker_id,
4910 &self.task_queue,
4911 self.poll_timeout,
4912 &poll_request_id,
4913 0,
4914 )
4915 })
4916 .await?;
4917 if response.outcome().should_stop() {
4918 return Ok(ManagedPollOutcome::Stop);
4919 }
4920 let Some(task) = response.task else {
4921 return Ok(ManagedPollOutcome::Idle);
4922 };
4923
4924 let query_task_id = task.query_task_id.clone();
4925 let attempt = task.query_task_attempt;
4926 let lease_owner = task
4927 .lease_owner
4928 .clone()
4929 .unwrap_or_else(|| self.worker_id.clone());
4930 let codec = task.payload_codec.clone();
4931
4932 match self.execute_query_task(task).await {
4933 Ok(value) => {
4934 let result_envelope = match encode_typed_envelope(&value, &codec) {
4935 Ok(result_envelope) => result_envelope,
4936 Err(error) => {
4937 let failure = self
4938 .client
4939 .fail_query_task(
4940 &query_task_id,
4941 &lease_owner,
4942 attempt,
4943 error.to_string(),
4944 "query_result_encode_failed",
4945 "QueryResultEncodeFailed",
4946 )
4947 .await;
4948 if let Err(error) = failure {
4949 if !query_task_rejection_is_final(&error) {
4950 return Err(error);
4951 }
4952 }
4953 return Ok(ManagedPollOutcome::Handled);
4954 }
4955 };
4956
4957 if let Err(error) = self
4958 .client
4959 .complete_query_task_with_envelope(
4960 &query_task_id,
4961 &lease_owner,
4962 attempt,
4963 value.clone().into_json()?,
4964 result_envelope,
4965 )
4966 .await
4967 {
4968 if !query_task_rejection_is_final(&error) {
4969 return Err(error);
4970 }
4971 }
4972 }
4973 Err(failure) => {
4974 let result = self
4975 .client
4976 .fail_query_task(
4977 &query_task_id,
4978 &lease_owner,
4979 attempt,
4980 failure.message,
4981 failure.reason,
4982 failure.failure_type,
4983 )
4984 .await;
4985 if let Err(error) = result {
4986 if !query_task_rejection_is_final(&error) {
4987 return Err(error);
4988 }
4989 }
4990 }
4991 }
4992
4993 Ok(ManagedPollOutcome::Handled)
4994 }
4995
4996 async fn poll_queries_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
4997 while !stop.load(Ordering::SeqCst) {
4998 if self.poll_query_once().await? == ManagedPollOutcome::Stop {
4999 stop.store(true, Ordering::SeqCst);
5000 break;
5001 }
5002 }
5003
5004 Ok(())
5005 }
5006
5007 async fn retry_worker_operation<T, F, Fut>(&self, mut operation: F) -> Result<T>
5008 where
5009 F: FnMut() -> Fut,
5010 Fut: Future<Output = Result<T>>,
5011 {
5012 let mut retries = 0;
5013
5014 loop {
5015 match operation().await {
5016 Err(error)
5017 if worker_operation_is_retryable(&error)
5018 && retries < self.retry_policy.max_retries =>
5019 {
5020 retries += 1;
5021 tokio::time::sleep(worker_retry_delay(self.retry_policy, retries)).await;
5022 }
5023 result => return result,
5024 }
5025 }
5026 }
5027
5028 async fn execute_query_task(
5029 &self,
5030 mut task: QueryTask,
5031 ) -> std::result::Result<AvroValue, QueryTaskExecutionFailure> {
5032 if !matches!(task.payload_codec.as_str(), DEFAULT_CODEC | JSON_CODEC) {
5033 return Err(QueryTaskExecutionFailure::new(
5034 "query_payload_decode_failed",
5035 format!(
5036 "cannot decode query payload with unsupported codec {:?}",
5037 task.payload_codec
5038 ),
5039 "QueryPayloadDecodeFailed",
5040 ));
5041 }
5042
5043 if !self.workflows.contains_key(&task.workflow_type) {
5044 return Err(QueryTaskExecutionFailure::new(
5045 "query_workflow_type_not_registered",
5046 format!("no workflow registered for type {:?}", task.workflow_type),
5047 "WorkflowTypeNotRegistered",
5048 ));
5049 }
5050
5051 let Some(handlers) = self.queries.get(&task.workflow_type) else {
5052 return Err(QueryTaskExecutionFailure::new(
5053 "query_handler_unavailable",
5054 format!(
5055 "query handlers are unavailable for workflow type {:?}",
5056 task.workflow_type
5057 ),
5058 "QueryHandlerUnavailable",
5059 ));
5060 };
5061 let Some(query) = handlers.get(&task.query_name) else {
5062 return Err(QueryTaskExecutionFailure::new(
5063 "rejected_unknown_query",
5064 format!("unknown query {:?}", task.query_name),
5065 "QueryFailed",
5066 ));
5067 };
5068
5069 let args = decode_task_avro_arguments(task.query_arguments.as_ref(), &task.payload_codec)
5070 .map_err(|error| {
5071 QueryTaskExecutionFailure::new(
5072 "query_payload_decode_failed",
5073 format!("cannot decode query arguments: {error}"),
5074 "QueryPayloadDecodeFailed",
5075 )
5076 })?;
5077 let workflow_input_typed =
5078 decode_task_avro_arguments(task.workflow_arguments.as_ref(), &task.payload_codec)
5079 .map_err(|error| {
5080 QueryTaskExecutionFailure::new(
5081 "query_workflow_state_unavailable",
5082 format!("cannot decode workflow start input: {error}"),
5083 "QueryWorkflowStateUnavailable",
5084 )
5085 })?;
5086 let workflow_input = workflow_input_typed.clone().into_json().map_err(|error| {
5087 QueryTaskExecutionFailure::new(
5088 "query_workflow_state_unavailable",
5089 format!("cannot project workflow start input: {error}"),
5090 "QueryWorkflowStateUnavailable",
5091 )
5092 })?;
5093 hydrate_query_history_from_export(&mut task).map_err(|error| {
5094 QueryTaskExecutionFailure::new(
5095 "query_workflow_state_unavailable",
5096 format!("cannot restore query history snapshot: {error}"),
5097 "QueryWorkflowStateUnavailable",
5098 )
5099 })?;
5100 enrich_query_history_from_export(&mut task).map_err(|error| {
5101 QueryTaskExecutionFailure::new(
5102 "query_workflow_state_unavailable",
5103 format!("cannot restore compact query history payloads: {error}"),
5104 "QueryWorkflowStateUnavailable",
5105 )
5106 })?;
5107 let signal_events = query_signal_events(&task).map_err(|error| {
5108 QueryTaskExecutionFailure::new(
5109 "query_workflow_state_unavailable",
5110 format!("cannot decode committed workflow signals: {error}"),
5111 "QueryWorkflowStateUnavailable",
5112 )
5113 })?;
5114 let history_events = Arc::new(std::mem::take(&mut task.history_events));
5115 let context = QueryContext {
5116 workflow_id: task.workflow_id,
5117 run_id: task.run_id,
5118 workflow_type: task.workflow_type.clone(),
5119 run_status: task.run_status,
5120 workflow_input,
5121 workflow_input_avro_value: workflow_input_typed.clone(),
5122 history_events: Arc::clone(&history_events),
5123 signal_events: Arc::new(signal_events),
5124 };
5125
5126 let future = match query {
5127 RegisteredQuery::Snapshot(handler) => handler(context, args),
5128 RegisteredQuery::Replayed {
5129 state_type,
5130 handler,
5131 } => {
5132 let workflow = self
5133 .workflows
5134 .get(&task.workflow_type)
5135 .expect("workflow registration was checked above");
5136 if workflow.state_type != Some(*state_type) {
5137 return Err(QueryTaskExecutionFailure::new(
5138 "query_workflow_state_unavailable",
5139 "replayed query state type does not match its workflow registration",
5140 "QueryWorkflowStateUnavailable",
5141 ));
5142 }
5143 let replay = workflow.replay.as_ref().ok_or_else(|| {
5144 QueryTaskExecutionFailure::new(
5145 "query_workflow_state_unavailable",
5146 format!(
5147 "workflow type {:?} is not registered for instance-state replay",
5148 task.workflow_type
5149 ),
5150 "QueryWorkflowStateUnavailable",
5151 )
5152 })?;
5153 let workflow_state = Arc::new(Mutex::new(
5154 WorkflowState::new_with_identity(
5155 history_events.as_ref().clone(),
5156 context.workflow_id.clone(),
5157 context.run_id.clone(),
5158 self.task_queue.clone(),
5159 task.payload_codec,
5160 None,
5161 )
5162 .map_err(|error| {
5163 QueryTaskExecutionFailure::new(
5164 "query_workflow_state_unavailable",
5165 format!("workflow replay failed before query: {error}"),
5166 "QueryWorkflowStateUnavailable",
5167 )
5168 })?,
5169 ));
5170 let workflow_context = WorkflowContext {
5171 state: workflow_state,
5172 };
5173 let mut invocation = replay(workflow_context.clone(), workflow_input_typed.clone());
5174 let mut cx = TaskContext::from_waker(noop_waker_ref());
5175 match invocation.future.as_mut().poll(&mut cx) {
5176 Poll::Ready(Ok(_)) => {
5177 workflow_context
5178 .ensure_history_consumed()
5179 .map_err(|error| {
5180 QueryTaskExecutionFailure::new(
5181 "query_workflow_state_unavailable",
5182 format!("workflow replay failed before query: {error}"),
5183 "QueryWorkflowStateUnavailable",
5184 )
5185 })?;
5186 }
5187 Poll::Ready(Err(error)) => {
5188 return Err(QueryTaskExecutionFailure::new(
5189 "query_workflow_state_unavailable",
5190 format!("workflow replay failed before query: {error}"),
5191 "QueryWorkflowStateUnavailable",
5192 ));
5193 }
5194 Poll::Pending => {
5195 let commands = workflow_context.take_commands().map_err(|error| {
5196 QueryTaskExecutionFailure::new(
5197 "query_workflow_state_unavailable",
5198 format!("workflow replay failed before query: {error}"),
5199 "QueryWorkflowStateUnavailable",
5200 )
5201 })?;
5202 if commands.is_empty()
5203 && !workflow_context
5204 .matched_recorded_pending()
5205 .map_err(|error| {
5206 QueryTaskExecutionFailure::new(
5207 "query_workflow_state_unavailable",
5208 format!("workflow replay failed before query: {error}"),
5209 "QueryWorkflowStateUnavailable",
5210 )
5211 })?
5212 {
5213 return Err(QueryTaskExecutionFailure::new(
5214 "query_workflow_state_unavailable",
5215 "workflow replay yielded without a durable command",
5216 "QueryWorkflowStateUnavailable",
5217 ));
5218 }
5219 }
5220 }
5221 let state = (invocation.snapshot)().map_err(|error| {
5222 QueryTaskExecutionFailure::new(
5223 "query_workflow_state_unavailable",
5224 format!("cannot snapshot replayed workflow state: {error}"),
5225 "QueryWorkflowStateUnavailable",
5226 )
5227 })?;
5228 handler(context, state, args).map_err(|message| {
5229 QueryTaskExecutionFailure::new(
5230 "query_workflow_state_unavailable",
5231 message,
5232 "QueryWorkflowStateUnavailable",
5233 )
5234 })?
5235 }
5236 };
5237
5238 future.await.map_err(|error| {
5239 QueryTaskExecutionFailure::new("query_rejected", error.to_string(), "QueryFailed")
5240 })
5241 }
5242
5243 fn execute_workflow_task(&self, task: WorkflowTask) -> Result<Vec<Value>> {
5244 if let Some(update_id) = task
5245 .workflow_update_id
5246 .as_deref()
5247 .filter(|update_id| !update_id.is_empty())
5248 {
5249 return self.execute_update_task(&task, update_id);
5250 }
5251
5252 let workflow = self
5253 .workflows
5254 .get(&task.workflow_type)
5255 .ok_or_else(|| Error::WorkflowNotRegistered(task.workflow_type.clone()))?;
5256 let input = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
5257 let resume_signal = decode_resume_signal(&task)?;
5258 let history_budget = WorkflowHistoryBudget {
5259 event_count: task
5260 .total_history_events
5261 .unwrap_or_else(|| u64::try_from(task.history_events.len()).unwrap_or(u64::MAX)),
5262 size_bytes: task.history_size_bytes,
5263 continue_as_new_recommended: task.continue_as_new_recommended.unwrap_or(false),
5264 pressure: task.history_budget_pressure.clone(),
5265 };
5266 let mut workflow_state = WorkflowState::new_with_identity(
5267 task.history_events,
5268 task.workflow_id,
5269 task.run_id,
5270 self.task_queue.clone(),
5271 task.payload_codec.clone(),
5272 resume_signal,
5273 )?;
5274 workflow_state.history_budget = history_budget;
5275 let state = Arc::new(Mutex::new(workflow_state));
5276 let ctx = WorkflowContext { state };
5277 let mut future = (workflow.execute)(ctx.clone(), input);
5278 let mut cx = TaskContext::from_waker(noop_waker_ref());
5279
5280 match future.as_mut().poll(&mut cx) {
5281 Poll::Ready(Ok(result)) => {
5282 ctx.ensure_history_consumed()?;
5283 let result = encode_typed_envelope(&result, &task.payload_codec)?;
5284 let mut commands = ctx.take_commands()?;
5285 commands.push(json!({
5286 "type": "complete_workflow",
5287 "result": result
5288 }));
5289 Ok(commands)
5290 }
5291 Poll::Ready(Err(error)) => {
5292 if let Error::ContinueAsNew(request) = error {
5293 let mut commands = ctx.take_commands()?;
5294 if let Some(command) = ctx.continue_as_new_command(request)? {
5295 commands.push(command);
5296 }
5297 ctx.ensure_history_consumed()?;
5298 return Ok(commands);
5299 }
5300 ctx.ensure_history_consumed()?;
5303 if workflow_task_integrity_error(&error) {
5304 return Err(error);
5309 }
5310 let mut commands = ctx.take_commands()?;
5311 commands.push(workflow_failure_command(&error));
5312 Ok(commands)
5313 }
5314 Poll::Pending => {
5315 let commands = ctx.take_commands()?;
5316 if commands.is_empty() && !ctx.matched_recorded_pending()? {
5317 Err(Error::WorkflowYieldedWithoutCommand)
5318 } else {
5319 Ok(commands)
5320 }
5321 }
5322 }
5323 }
5324
5325 fn execute_update_task(&self, task: &WorkflowTask, update_id: &str) -> Result<Vec<Value>> {
5326 if !self.workflows.contains_key(&task.workflow_type) {
5327 return Err(Error::WorkflowNotRegistered(task.workflow_type.clone()));
5328 }
5329
5330 let accepted = task.history_events.iter().rev().find_map(|event| {
5331 (event.event_type == "UpdateAccepted"
5332 && event.payload.get("update_id").and_then(Value::as_str) == Some(update_id))
5333 .then_some(&event.payload)
5334 });
5335 let update_name = accepted
5336 .and_then(|payload| payload.get("update_name"))
5337 .and_then(Value::as_str)
5338 .or(task.update_name.as_deref())
5339 .unwrap_or_default();
5340 let Some(handler) = self
5341 .updates
5342 .get(&task.workflow_type)
5343 .and_then(|handlers| handlers.get(update_name))
5344 else {
5345 return Ok(vec![json!({
5346 "type": "fail_update",
5347 "update_id": update_id,
5348 "message": format!(
5349 "no update handler is registered for {}.{update_name}",
5350 task.workflow_type
5351 ),
5352 "exception_type": "UnknownUpdate",
5353 "non_retryable": true,
5354 })]);
5355 };
5356 let arguments = accepted
5357 .and_then(|payload| payload.get("arguments"))
5358 .or(task.arguments.as_ref());
5359 let arguments = decode_task_avro_arguments(arguments, &task.payload_codec)?;
5360 let context = QueryContext {
5361 workflow_id: task.workflow_id.clone(),
5362 run_id: task.run_id.clone(),
5363 workflow_type: task.workflow_type.clone(),
5364 run_status: Some("running".to_string()),
5365 workflow_input: Value::Null,
5366 workflow_input_avro_value: AvroValue::Null,
5367 history_events: Arc::new(task.history_events.clone()),
5368 signal_events: Arc::new(Vec::new()),
5369 };
5370 let mut future = handler(context, arguments);
5371 let mut cx = TaskContext::from_waker(noop_waker_ref());
5372
5373 match future.as_mut().poll(&mut cx) {
5374 Poll::Ready(Ok(result)) => Ok(vec![json!({
5375 "type": "complete_update",
5376 "update_id": update_id,
5377 "result": encode_typed_envelope(&result, &task.payload_codec)?,
5378 })]),
5379 Poll::Ready(Err(error)) => Ok(vec![json!({
5380 "type": "fail_update",
5381 "update_id": update_id,
5382 "message": error.to_string(),
5383 "exception_type": "UpdateFailed",
5384 "non_retryable": true,
5385 })]),
5386 Poll::Pending => Err(Error::WorkflowYieldedWithoutCommand),
5387 }
5388 }
5389
5390 async fn execute_activity_task(&self, task: ActivityTask) -> Result<AvroValue> {
5391 let handler = self
5392 .activities
5393 .get(&task.activity_type)
5394 .ok_or_else(|| Error::ActivityNotRegistered(task.activity_type.clone()))?;
5395 let args = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
5396 let attempt_id = task
5397 .activity_attempt_id
5398 .clone()
5399 .or(task.attempt_id.clone())
5400 .unwrap_or_default();
5401 let lease_owner = task
5402 .lease_owner
5403 .clone()
5404 .unwrap_or_else(|| self.worker_id.clone());
5405 let ctx = ActivityContext {
5406 client: self.client.clone(),
5407 task_id: task.task_id,
5408 activity_attempt_id: attempt_id,
5409 lease_owner,
5410 activity_type: task.activity_type,
5411 attempt_number: task.attempt_number,
5412 task_queue: self.task_queue.clone(),
5413 worker_id: self.worker_id.clone(),
5414 };
5415
5416 handler(ctx, args).await
5417 }
5418}
5419
5420fn poller_result(
5421 kind: &str,
5422 result: std::result::Result<Result<()>, tokio::task::JoinError>,
5423) -> Result<()> {
5424 match result {
5425 Ok(result) => result,
5426 Err(error) => Err(Error::WorkerLoop(format!(
5427 "{kind} poller join error: {error}"
5428 ))),
5429 }
5430}
5431
5432fn optional_poller_result(
5433 kind: &str,
5434 result: Option<std::result::Result<Result<()>, tokio::task::JoinError>>,
5435) -> Result<()> {
5436 match result {
5437 Some(result) => poller_result(kind, result),
5438 None => Ok(()),
5439 }
5440}
5441
5442async fn join_pollers(
5443 workflow_poller: Option<tokio::task::JoinHandle<Result<()>>>,
5444 activity_poller: Option<tokio::task::JoinHandle<Result<()>>>,
5445 query_poller: Option<tokio::task::JoinHandle<Result<()>>>,
5446) -> Result<()> {
5447 let mut first_error = None;
5448
5449 if let Some(handle) = workflow_poller {
5450 if let Err(error) = poller_result("workflow", handle.await) {
5451 first_error.get_or_insert(error);
5452 }
5453 }
5454
5455 if let Some(handle) = activity_poller {
5456 if let Err(error) = poller_result("activity", handle.await) {
5457 first_error.get_or_insert(error);
5458 }
5459 }
5460
5461 if let Some(handle) = query_poller {
5462 if let Err(error) = poller_result("query", handle.await) {
5463 first_error.get_or_insert(error);
5464 }
5465 }
5466
5467 if let Some(error) = first_error {
5468 Err(error)
5469 } else {
5470 Ok(())
5471 }
5472}
5473
5474fn default_worker_id() -> String {
5475 let millis = SystemTime::now()
5476 .duration_since(UNIX_EPOCH)
5477 .unwrap_or_default()
5478 .as_millis();
5479 format!("rust-worker-{}-{millis}", std::process::id())
5480}
5481
5482fn percent_encode_path_segment(segment: &str) -> String {
5483 const HEX: &[u8; 16] = b"0123456789ABCDEF";
5484 let mut encoded = String::with_capacity(segment.len());
5485
5486 for byte in segment.bytes() {
5487 if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') {
5488 encoded.push(char::from(byte));
5489 } else {
5490 encoded.push('%');
5491 encoded.push(char::from(HEX[(byte >> 4) as usize]));
5492 encoded.push(char::from(HEX[(byte & 0x0f) as usize]));
5493 }
5494 }
5495
5496 encoded
5497}
5498
5499fn unique_request_id(prefix: &str) -> String {
5500 let nanos = SystemTime::now()
5501 .duration_since(UNIX_EPOCH)
5502 .unwrap_or_default()
5503 .as_nanos();
5504 format!("{prefix}-{}-{nanos}", std::process::id())
5505}
5506
5507#[derive(Debug)]
5508struct QueryTaskExecutionFailure {
5509 reason: String,
5510 message: String,
5511 failure_type: String,
5512}
5513
5514impl QueryTaskExecutionFailure {
5515 fn new(
5516 reason: impl Into<String>,
5517 message: impl Into<String>,
5518 failure_type: impl Into<String>,
5519 ) -> Self {
5520 Self {
5521 reason: reason.into(),
5522 message: message.into(),
5523 failure_type: failure_type.into(),
5524 }
5525 }
5526}
5527
5528#[derive(Clone, Debug)]
5534pub struct WorkflowInstance<S> {
5535 state: Arc<Mutex<S>>,
5536}
5537
5538impl<S> WorkflowInstance<S> {
5539 fn new(state: S) -> Self {
5540 Self {
5541 state: Arc::new(Mutex::new(state)),
5542 }
5543 }
5544
5545 pub fn read<R>(&self, reader: impl FnOnce(&S) -> R) -> Result<R> {
5547 let state = self
5548 .state
5549 .lock()
5550 .map_err(|_| Error::WorkflowStatePoisoned)?;
5551 Ok(reader(&state))
5552 }
5553
5554 pub fn update<R>(&self, transition: impl FnOnce(&mut S) -> R) -> Result<R> {
5556 let mut state = self
5557 .state
5558 .lock()
5559 .map_err(|_| Error::WorkflowStatePoisoned)?;
5560 Ok(transition(&mut state))
5561 }
5562}
5563
5564impl<S: Clone> WorkflowInstance<S> {
5565 fn snapshot(&self) -> Result<S> {
5566 self.read(Clone::clone)
5567 }
5568}
5569
5570#[derive(Clone, Debug)]
5571pub struct WorkflowContext {
5572 state: Arc<Mutex<WorkflowState>>,
5573}
5574
5575impl WorkflowContext {
5576 pub fn workflow_identity(&self) -> Result<WorkflowIdentity> {
5578 let state = self
5579 .state
5580 .lock()
5581 .map_err(|_| Error::WorkflowStatePoisoned)?;
5582 Ok(WorkflowIdentity {
5583 workflow_id: state.workflow_id.clone(),
5584 run_id: state.run_id.clone(),
5585 })
5586 }
5587
5588 pub fn history_budget(&self) -> Result<WorkflowHistoryBudget> {
5590 let state = self
5591 .state
5592 .lock()
5593 .map_err(|_| Error::WorkflowStatePoisoned)?;
5594 Ok(state.history_budget.clone())
5595 }
5596
5597 pub fn continue_as_new<T: Serialize>(&self, args: T) -> Result<Value> {
5603 self.continue_as_new_with_options(ContinueAsNewOptions::new(), args)
5604 }
5605
5606 pub fn continue_as_new_with_options<T: Serialize>(
5608 &self,
5609 options: ContinueAsNewOptions,
5610 args: T,
5611 ) -> Result<Value> {
5612 options.validate()?;
5613 Err(Error::ContinueAsNew(ContinueAsNewRequest {
5614 arguments: normalize_avro_arguments(AvroValue::from_serialize(&args)?),
5615 options,
5616 }))
5617 }
5618
5619 pub fn activity<T: Serialize>(
5620 &self,
5621 activity_type: impl Into<String>,
5622 args: T,
5623 ) -> ActivityCall {
5624 self.activity_with_options(activity_type, ActivityOptions::new(), args)
5625 }
5626
5627 pub fn activity_on_queue<T, Q>(
5628 &self,
5629 activity_type: impl Into<String>,
5630 task_queue: Option<Q>,
5631 args: T,
5632 ) -> ActivityCall
5633 where
5634 T: Serialize,
5635 Q: Into<String>,
5636 {
5637 let mut options = ActivityOptions::new();
5638 options.task_queue = task_queue.map(Into::into);
5639 self.activity_with_options(activity_type, options, args)
5640 }
5641
5642 pub fn activity_with_options<T: Serialize>(
5680 &self,
5681 activity_type: impl Into<String>,
5682 options: ActivityOptions,
5683 args: T,
5684 ) -> ActivityCall {
5685 ActivityCall {
5686 ctx: self.clone(),
5687 activity_type: activity_type.into(),
5688 options,
5689 args: Some(AvroValue::from_serialize(&args)),
5690 scheduled: false,
5691 }
5692 }
5693
5694 pub async fn activity_avro_value<T: Serialize>(
5695 &self,
5696 activity_type: impl Into<String>,
5697 args: T,
5698 ) -> Result<AvroValue> {
5699 let mut call = self.activity(activity_type, args);
5700 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
5701 }
5702
5703 pub async fn activity_avro_value_with_options<T: Serialize>(
5704 &self,
5705 activity_type: impl Into<String>,
5706 options: ActivityOptions,
5707 args: T,
5708 ) -> Result<AvroValue> {
5709 let mut call = self.activity_with_options(activity_type, options, args);
5710 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
5711 }
5712
5713 pub fn wait_signal(&self, signal_name: impl Into<String>) -> SignalCall {
5714 SignalCall {
5715 ctx: self.clone(),
5716 signal_name: signal_name.into(),
5717 opened_wait: false,
5718 matched_pending: false,
5719 }
5720 }
5721
5722 pub async fn wait_signal_avro_value(
5723 &self,
5724 signal_name: impl Into<String>,
5725 ) -> Result<Vec<AvroValue>> {
5726 let mut call = self.wait_signal(signal_name);
5727 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
5728 }
5729
5730 pub fn sleep(&self, duration: Duration) -> TimerCall {
5751 let delay_seconds = duration
5752 .as_secs()
5753 .checked_add(u64::from(duration.subsec_nanos() > 0));
5754 TimerCall {
5755 ctx: self.clone(),
5756 delay_seconds,
5757 scheduled: false,
5758 matched_pending: false,
5759 }
5760 }
5761
5762 pub fn start_timer(&self, duration: Duration) -> TimerCall {
5764 self.sleep(duration)
5765 }
5766
5767 pub fn side_effect<T, F>(&self, callback: F) -> Result<T>
5774 where
5775 T: Serialize + DeserializeOwned,
5776 F: FnOnce() -> T,
5777 {
5778 {
5779 let mut state = self
5780 .state
5781 .lock()
5782 .map_err(|_| Error::WorkflowStatePoisoned)?;
5783 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
5784 return match recorded {
5785 RecordedCommand::SideEffect { sequence, value } => {
5786 state.command_cursor += 1;
5787 value.deserialize().map_err(|error| {
5788 Error::NonDeterministicReplay(ReplayFailure::new(
5789 "side_effect_type_mismatch",
5790 Some(sequence),
5791 Some(std::any::type_name::<T>().to_string()),
5792 Some(error.to_string()),
5793 "recorded side-effect value is incompatible with the requested Rust type",
5794 ))
5795 })
5796 }
5797 other => Err(command_mismatch(&other, "side effect")),
5798 };
5799 }
5800 }
5801
5802 let value = callback();
5803 let avro_value = AvroValue::from_serialize(&value)?;
5804 let mut state = self
5805 .state
5806 .lock()
5807 .map_err(|_| Error::WorkflowStatePoisoned)?;
5808 let result = encode_typed_envelope(&avro_value, &state.payload_codec)?;
5809 state.commands.push(json!({
5810 "type": "record_side_effect",
5811 "result": result,
5812 }));
5813 Ok(value)
5814 }
5815
5816 pub fn side_effect_avro_value<F>(&self, callback: F) -> Result<AvroValue>
5818 where
5819 F: FnOnce() -> AvroValue,
5820 {
5821 {
5822 let mut state = self
5823 .state
5824 .lock()
5825 .map_err(|_| Error::WorkflowStatePoisoned)?;
5826 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
5827 return match recorded {
5828 RecordedCommand::SideEffect { value, .. } => {
5829 state.command_cursor += 1;
5830 Ok(value)
5831 }
5832 other => Err(command_mismatch(&other, "side effect")),
5833 };
5834 }
5835 }
5836
5837 let value = callback();
5838 let mut state = self
5839 .state
5840 .lock()
5841 .map_err(|_| Error::WorkflowStatePoisoned)?;
5842 let result = encode_typed_envelope(&value, &state.payload_codec)?;
5843 state.commands.push(json!({
5844 "type": "record_side_effect",
5845 "result": result,
5846 }));
5847 Ok(value)
5848 }
5849
5850 pub fn uuid_v4(&self) -> Result<Uuid> {
5852 self.side_effect(Uuid::new_v4)
5853 }
5854
5855 pub fn get_version(
5858 &self,
5859 change_id: impl Into<String>,
5860 min_supported: i32,
5861 max_supported: i32,
5862 ) -> Result<i32> {
5863 let change_id = change_id.into();
5864 if change_id.trim().is_empty() {
5865 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5866 "version_change_id_invalid",
5867 None,
5868 Some("non-empty change ID".to_string()),
5869 Some(change_id),
5870 "version markers require a stable non-empty change ID",
5871 )));
5872 }
5873 if min_supported > max_supported {
5874 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5875 "version_range_invalid",
5876 None,
5877 Some("min_supported <= max_supported".to_string()),
5878 Some(format!("{min_supported}..={max_supported}")),
5879 "version marker supported range is invalid",
5880 )));
5881 }
5882
5883 let mut state = self
5884 .state
5885 .lock()
5886 .map_err(|_| Error::WorkflowStatePoisoned)?;
5887 if let Some((version, sequence)) = state.version_markers.get(&change_id).copied() {
5888 ensure_version_supported(&change_id, version, min_supported, max_supported, sequence)?;
5889 return Ok(version);
5890 }
5891
5892 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
5893 return match recorded {
5894 RecordedCommand::VersionMarker {
5895 sequence,
5896 change_id: recorded_change_id,
5897 version,
5898 ..
5899 } => {
5900 if recorded_change_id != change_id {
5901 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5902 "version_change_id_mismatch",
5903 Some(sequence),
5904 Some(recorded_change_id),
5905 Some(change_id),
5906 "recorded version marker change ID differs from current workflow code",
5907 )));
5908 }
5909 ensure_version_supported(
5910 &change_id,
5911 version,
5912 min_supported,
5913 max_supported,
5914 sequence,
5915 )?;
5916 state.command_cursor += 1;
5917 state.version_markers.insert(change_id, (version, sequence));
5918 Ok(version)
5919 }
5920 other => Err(command_mismatch(
5921 &other,
5922 format!("version marker:{change_id}"),
5923 )),
5924 };
5925 }
5926
5927 let version = max_supported;
5928 state.commands.push(json!({
5929 "type": "record_version_marker",
5930 "change_id": change_id,
5931 "version": version,
5932 "min_supported": min_supported,
5933 "max_supported": max_supported,
5934 }));
5935 state.version_markers.insert(change_id, (version, 0));
5938 Ok(version)
5939 }
5940
5941 pub fn patched(&self, change_id: impl Into<String>) -> Result<bool> {
5943 Ok(self.get_version(change_id, -1, 1)? == 1)
5944 }
5945
5946 pub fn deprecate_patch(&self, change_id: impl Into<String>) -> Result<()> {
5948 self.get_version(change_id, -1, 1).map(|_| ())
5949 }
5950
5951 pub fn start_child_workflow<T: Serialize>(
5978 &self,
5979 workflow_type: impl Into<String>,
5980 options: ChildWorkflowOptions,
5981 args: T,
5982 ) -> ChildWorkflowCall {
5983 ChildWorkflowCall {
5984 ctx: self.clone(),
5985 workflow_type: workflow_type.into(),
5986 options,
5987 args: Some(AvroValue::from_serialize(&args)),
5988 scheduled: false,
5989 matched_pending: false,
5990 }
5991 }
5992
5993 pub async fn start_child_workflow_avro_value<T: Serialize>(
5994 &self,
5995 workflow_type: impl Into<String>,
5996 options: ChildWorkflowOptions,
5997 args: T,
5998 ) -> Result<ChildWorkflowAvroResult> {
5999 let mut call = self.start_child_workflow(workflow_type, options, args);
6000 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
6001 }
6002
6003 fn take_commands(&self) -> Result<Vec<Value>> {
6004 let mut state = self
6005 .state
6006 .lock()
6007 .map_err(|_| Error::WorkflowStatePoisoned)?;
6008 Ok(std::mem::take(&mut state.commands))
6009 }
6010
6011 fn continue_as_new_command(&self, request: ContinueAsNewRequest) -> Result<Option<Value>> {
6012 let mut state = self
6013 .state
6014 .lock()
6015 .map_err(|_| Error::WorkflowStatePoisoned)?;
6016
6017 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6018 return Err(command_mismatch(&recorded, "continue as new"));
6019 }
6020 if state.recorded_continue_as_new_sequence.is_some() {
6021 state.continue_as_new_consumed = true;
6022 return Ok(None);
6023 }
6024
6025 let arguments = encode_typed_envelope(&request.arguments, &state.payload_codec)?;
6026 let mut command = serde_json::Map::from_iter([
6027 ("type".to_string(), json!("continue_as_new")),
6028 ("arguments".to_string(), arguments),
6029 ("queue".to_string(), json!(state.task_queue.clone())),
6030 ]);
6031 if let Some(workflow_type) = request.options.workflow_type {
6032 command.insert("workflow_type".to_string(), json!(workflow_type));
6033 }
6034 if let Some(task_queue) = request.options.task_queue {
6035 command.insert("queue".to_string(), json!(task_queue));
6036 }
6037 Ok(Some(Value::Object(command)))
6038 }
6039
6040 fn matched_recorded_pending(&self) -> Result<bool> {
6041 let state = self
6042 .state
6043 .lock()
6044 .map_err(|_| Error::WorkflowStatePoisoned)?;
6045 Ok(state.matched_recorded_pending)
6046 }
6047
6048 fn ensure_history_consumed(&self) -> Result<()> {
6049 let state = self
6050 .state
6051 .lock()
6052 .map_err(|_| Error::WorkflowStatePoisoned)?;
6053 if let Some(command) = state.recorded_commands.get(state.command_cursor) {
6054 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
6055 "recorded_commands_unconsumed",
6056 Some(command.sequence()),
6057 Some(command.shape().to_string()),
6058 Some("workflow completion".to_string()),
6059 "workflow completed before consuming all recorded durable commands",
6060 )));
6061 }
6062 if let Some(sequence) = state
6063 .recorded_continue_as_new_sequence
6064 .filter(|_| !state.continue_as_new_consumed)
6065 {
6066 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
6067 "recorded_continue_as_new_unconsumed",
6068 Some(sequence),
6069 Some("continue as new".to_string()),
6070 Some("workflow completion".to_string()),
6071 "workflow completed without consuming its recorded continue-as-new transition",
6072 )));
6073 }
6074 Ok(())
6075 }
6076}
6077
6078#[derive(Debug)]
6079struct WorkflowState {
6080 workflow_id: Option<String>,
6081 run_id: Option<String>,
6082 task_queue: String,
6083 payload_codec: String,
6084 history_budget: WorkflowHistoryBudget,
6085 resume_signal: Option<ResumeSignal>,
6086 recorded_commands: Vec<RecordedCommand>,
6087 recorded_continue_as_new_sequence: Option<u64>,
6088 continue_as_new_consumed: bool,
6089 command_cursor: usize,
6090 matched_recorded_pending: bool,
6091 version_markers: HashMap<String, (i32, u64)>,
6092 commands: Vec<Value>,
6093}
6094
6095impl WorkflowState {
6096 #[cfg(test)]
6097 fn new(
6098 history: Vec<HistoryEvent>,
6099 task_queue: String,
6100 payload_codec: String,
6101 resume_signal: Option<ResumeSignal>,
6102 ) -> Result<Self> {
6103 Self::new_with_identity(
6104 history,
6105 None,
6106 None,
6107 task_queue,
6108 payload_codec,
6109 resume_signal,
6110 )
6111 }
6112
6113 fn new_with_identity(
6114 history: Vec<HistoryEvent>,
6115 workflow_id: Option<String>,
6116 run_id: Option<String>,
6117 task_queue: String,
6118 payload_codec: String,
6119 resume_signal: Option<ResumeSignal>,
6120 ) -> Result<Self> {
6121 let recorded_commands = recorded_commands(
6122 &history,
6123 &payload_codec,
6124 WorkflowIdentity {
6125 workflow_id: workflow_id.clone(),
6126 run_id: run_id.clone(),
6127 },
6128 )?;
6129 let recorded_continue_as_new = history
6130 .iter()
6131 .filter(|event| event.event_type == "WorkflowContinuedAsNew")
6132 .collect::<Vec<_>>();
6133 if recorded_continue_as_new.len() > 1 {
6134 return Err(invalid_recorded_history(
6135 "duplicate_continue_as_new_transition",
6136 recorded_continue_as_new
6137 .last()
6138 .and_then(|event| durable_event_sequence(event))
6139 .unwrap_or(0),
6140 "one WorkflowContinuedAsNew event",
6141 &format!(
6142 "{} WorkflowContinuedAsNew events",
6143 recorded_continue_as_new.len()
6144 ),
6145 "workflow history records one continue-as-new transition more than once",
6146 ));
6147 }
6148 let recorded_continue_as_new_sequence = recorded_continue_as_new
6149 .first()
6150 .map(|event| {
6151 durable_event_sequence(event).ok_or_else(|| {
6152 Error::NonDeterministicReplay(ReplayFailure::new(
6153 "continue_as_new_sequence_missing",
6154 None,
6155 Some("recorded transition sequence".to_string()),
6156 Some("missing sequence".to_string()),
6157 "WorkflowContinuedAsNew history is missing its recorded sequence",
6158 ))
6159 })
6160 })
6161 .transpose()?;
6162 let event_count = u64::try_from(history.len()).unwrap_or(u64::MAX);
6163 Ok(Self {
6164 workflow_id,
6165 run_id,
6166 task_queue,
6167 payload_codec,
6168 history_budget: WorkflowHistoryBudget {
6169 event_count,
6170 ..WorkflowHistoryBudget::default()
6171 },
6172 resume_signal,
6173 recorded_commands,
6174 recorded_continue_as_new_sequence,
6175 continue_as_new_consumed: false,
6176 command_cursor: 0,
6177 matched_recorded_pending: false,
6178 version_markers: HashMap::new(),
6179 commands: Vec::new(),
6180 })
6181 }
6182}
6183
6184#[derive(Clone, Debug)]
6185enum RecordedCommand {
6186 Activity {
6187 sequence: u64,
6188 activity_type: Option<String>,
6189 options: Option<RecordedActivityOptions>,
6190 outcome: Option<ActivityOutcome>,
6191 },
6192 Timer {
6193 sequence: u64,
6194 delay_seconds: u64,
6195 fired: bool,
6196 },
6197 ChildWorkflow {
6198 sequence: u64,
6199 workflow_type: Option<String>,
6200 outcome: Option<ChildWorkflowOutcome>,
6201 },
6202 SignalWait {
6203 sequence: u64,
6204 signal_name: String,
6205 value: Option<Vec<AvroValue>>,
6206 },
6207 SideEffect {
6208 sequence: u64,
6209 value: AvroValue,
6210 },
6211 VersionMarker {
6212 sequence: u64,
6213 change_id: String,
6214 version: i32,
6215 },
6216}
6217
6218#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
6219struct RecordedActivityOptions {
6220 task_queue: RecordedSnapshotValue<Option<String>>,
6221 execution_mode: RecordedSnapshotValue<Option<String>>,
6222 retry_policy: ActivityRetrySnapshot,
6223}
6224
6225#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
6226enum RecordedSnapshotValue<T> {
6227 Unknown,
6229 Known(T),
6230}
6231
6232impl<T: PartialEq> RecordedSnapshotValue<T> {
6233 fn matches_current(&self, current: &Self) -> bool {
6234 match self {
6235 Self::Unknown => true,
6236 Self::Known(recorded) => matches!(current, Self::Known(value) if value == recorded),
6237 }
6238 }
6239}
6240
6241#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
6242struct ActivityRetrySnapshot {
6243 snapshot_version: RecordedSnapshotValue<Option<u64>>,
6244 max_attempts: RecordedSnapshotValue<Option<u64>>,
6245 backoff_seconds: RecordedSnapshotValue<Vec<u64>>,
6246 start_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
6247 schedule_to_start_timeout: RecordedSnapshotValue<Option<u64>>,
6248 schedule_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
6249 heartbeat_timeout: RecordedSnapshotValue<Option<u64>>,
6250 non_retryable_error_types: RecordedSnapshotValue<Vec<String>>,
6251}
6252
6253impl ActivityRetrySnapshot {
6254 fn matches_current(&self, current: &Self) -> bool {
6255 self.snapshot_version
6256 .matches_current(¤t.snapshot_version)
6257 && self.max_attempts.matches_current(¤t.max_attempts)
6258 && self
6259 .backoff_seconds
6260 .matches_current(¤t.backoff_seconds)
6261 && self
6262 .start_to_close_timeout
6263 .matches_current(¤t.start_to_close_timeout)
6264 && self
6265 .schedule_to_start_timeout
6266 .matches_current(¤t.schedule_to_start_timeout)
6267 && self
6268 .schedule_to_close_timeout
6269 .matches_current(¤t.schedule_to_close_timeout)
6270 && self
6271 .heartbeat_timeout
6272 .matches_current(¤t.heartbeat_timeout)
6273 && self
6274 .non_retryable_error_types
6275 .matches_current(¤t.non_retryable_error_types)
6276 }
6277}
6278
6279fn recorded_optional_u64(
6280 object: Option<&serde_json::Map<String, Value>>,
6281 field: &str,
6282) -> RecordedSnapshotValue<Option<u64>> {
6283 match object.and_then(|object| object.get(field)) {
6284 None => RecordedSnapshotValue::Unknown,
6285 Some(Value::Null) => RecordedSnapshotValue::Known(None),
6286 Some(value) => RecordedSnapshotValue::Known(value_as_u64(value)),
6287 }
6288}
6289
6290fn recorded_optional_string(
6291 object: &serde_json::Map<String, Value>,
6292 field: &str,
6293) -> RecordedSnapshotValue<Option<String>> {
6294 match object.get(field) {
6295 None => RecordedSnapshotValue::Unknown,
6296 Some(Value::Null) => RecordedSnapshotValue::Known(None),
6297 Some(value) => RecordedSnapshotValue::Known(value.as_str().map(str::to_string)),
6298 }
6299}
6300
6301fn recorded_activity_retry_snapshot(policy: Option<&Value>) -> ActivityRetrySnapshot {
6302 let policy = policy.and_then(Value::as_object);
6303 let backoff_seconds = policy
6304 .and_then(|policy| policy.get("backoff_seconds"))
6305 .and_then(Value::as_array)
6306 .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
6307 .map_or(RecordedSnapshotValue::Unknown, RecordedSnapshotValue::Known);
6308 let mut non_retryable_error_types = Vec::new();
6309 for error_type in policy
6310 .and_then(|policy| policy.get("non_retryable_error_types"))
6311 .and_then(Value::as_array)
6312 .into_iter()
6313 .flatten()
6314 .filter_map(Value::as_str)
6315 .map(str::trim)
6316 .filter(|error_type| !error_type.is_empty())
6317 {
6318 if !non_retryable_error_types
6319 .iter()
6320 .any(|recorded| recorded == error_type)
6321 {
6322 non_retryable_error_types.push(error_type.to_string());
6323 }
6324 }
6325
6326 ActivityRetrySnapshot {
6327 snapshot_version: recorded_optional_u64(policy, "snapshot_version"),
6328 max_attempts: recorded_optional_u64(policy, "max_attempts"),
6329 backoff_seconds,
6330 start_to_close_timeout: recorded_optional_u64(policy, "start_to_close_timeout"),
6331 schedule_to_start_timeout: recorded_optional_u64(policy, "schedule_to_start_timeout"),
6332 schedule_to_close_timeout: recorded_optional_u64(policy, "schedule_to_close_timeout"),
6333 heartbeat_timeout: recorded_optional_u64(policy, "heartbeat_timeout"),
6334 non_retryable_error_types: if policy
6335 .is_some_and(|policy| policy.contains_key("non_retryable_error_types"))
6336 {
6337 RecordedSnapshotValue::Known(non_retryable_error_types)
6338 } else {
6339 RecordedSnapshotValue::Unknown
6340 },
6341 }
6342}
6343
6344fn current_activity_retry_snapshot(options: &ValidatedActivityOptions) -> ActivityRetrySnapshot {
6345 let policy = options.retry_policy.as_ref();
6346 let max_attempts = match policy.and_then(|policy| policy.get("max_attempts")) {
6347 Some(Value::Null) => None,
6348 Some(value) => value_as_u64(value),
6349 None => Some(1),
6350 };
6351 let backoff_seconds = policy
6352 .and_then(|policy| policy.get("backoff_seconds"))
6353 .and_then(Value::as_array)
6354 .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
6355 .unwrap_or_default();
6356 let non_retryable_error_types = policy
6357 .and_then(|policy| policy.get("non_retryable_error_types"))
6358 .and_then(Value::as_array)
6359 .into_iter()
6360 .flatten()
6361 .filter_map(Value::as_str)
6362 .map(str::to_string)
6363 .collect();
6364
6365 ActivityRetrySnapshot {
6366 snapshot_version: RecordedSnapshotValue::Known(Some(1)),
6367 max_attempts: RecordedSnapshotValue::Known(max_attempts),
6368 backoff_seconds: RecordedSnapshotValue::Known(backoff_seconds),
6369 start_to_close_timeout: RecordedSnapshotValue::Known(options.start_to_close_timeout),
6370 schedule_to_start_timeout: RecordedSnapshotValue::Known(options.schedule_to_start_timeout),
6371 schedule_to_close_timeout: RecordedSnapshotValue::Known(options.schedule_to_close_timeout),
6372 heartbeat_timeout: RecordedSnapshotValue::Known(options.heartbeat_timeout),
6373 non_retryable_error_types: RecordedSnapshotValue::Known(non_retryable_error_types),
6374 }
6375}
6376
6377fn activity_options_description(options: &RecordedActivityOptions) -> String {
6378 serde_json::to_string(options).unwrap_or_else(|_| format!("{options:?}"))
6379}
6380
6381impl RecordedCommand {
6382 fn sequence(&self) -> u64 {
6383 match self {
6384 Self::Activity { sequence, .. }
6385 | Self::Timer { sequence, .. }
6386 | Self::ChildWorkflow { sequence, .. }
6387 | Self::SignalWait { sequence, .. }
6388 | Self::SideEffect { sequence, .. }
6389 | Self::VersionMarker { sequence, .. } => *sequence,
6390 }
6391 }
6392
6393 fn shape(&self) -> &'static str {
6394 match self {
6395 Self::Activity { .. } => "activity",
6396 Self::Timer { .. } => "timer",
6397 Self::ChildWorkflow { .. } => "child workflow",
6398 Self::SignalWait { .. } => "signal wait",
6399 Self::SideEffect { .. } => "side effect",
6400 Self::VersionMarker { .. } => "version marker",
6401 }
6402 }
6403}
6404
6405fn ensure_version_supported(
6406 change_id: &str,
6407 version: i32,
6408 min_supported: i32,
6409 max_supported: i32,
6410 sequence: u64,
6411) -> Result<()> {
6412 if (min_supported..=max_supported).contains(&version) {
6413 return Ok(());
6414 }
6415 Err(Error::NonDeterministicReplay(ReplayFailure::new(
6416 "version_marker_incompatible_range",
6417 (sequence != 0).then_some(sequence),
6418 Some(format!("{min_supported}..={max_supported}")),
6419 Some(format!("{change_id}:{version}")),
6420 "recorded workflow version is outside the range supported by current code",
6421 )))
6422}
6423
6424#[derive(Clone, Debug)]
6425struct ResumeSignal {
6426 signal_name: String,
6427 arguments: Vec<AvroValue>,
6428}
6429
6430pub struct ActivityCall {
6431 ctx: WorkflowContext,
6432 activity_type: String,
6433 options: ActivityOptions,
6434 args: Option<Result<AvroValue>>,
6435 scheduled: bool,
6436}
6437
6438impl ActivityCall {
6439 fn poll_avro_value(
6440 mut self: Pin<&mut Self>,
6441 _cx: &mut TaskContext<'_>,
6442 ) -> Poll<Result<AvroValue>> {
6443 let ctx = self.ctx.clone();
6444 let mut state = match ctx.state.lock() {
6445 Ok(state) => state,
6446 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6447 };
6448
6449 if self.scheduled {
6450 return Poll::Pending;
6451 }
6452
6453 let options = match self.options.validate() {
6454 Ok(options) => options,
6455 Err(error) => {
6456 return Poll::Ready(Err(Error::InvalidActivityOptions(error)));
6457 }
6458 };
6459 let task_queue = options
6460 .task_queue
6461 .clone()
6462 .unwrap_or_else(|| state.task_queue.clone());
6463 let current_recorded_options = RecordedActivityOptions {
6464 task_queue: RecordedSnapshotValue::Known(Some(task_queue.clone())),
6465 execution_mode: RecordedSnapshotValue::Known(None),
6468 retry_policy: current_activity_retry_snapshot(&options),
6469 };
6470
6471 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6472 let sequence = recorded.sequence();
6473 match recorded {
6474 RecordedCommand::Activity {
6475 activity_type,
6476 options: recorded_options,
6477 outcome,
6478 ..
6479 } => {
6480 if let Some(recorded_type) = activity_type {
6481 if recorded_type != self.activity_type {
6482 return Poll::Ready(Err(Error::NonDeterministicReplay(
6483 ReplayFailure::new(
6484 "recorded_command_detail_mismatch",
6485 Some(sequence),
6486 Some(format!("activity:{recorded_type}")),
6487 Some(format!("activity:{}", self.activity_type)),
6488 "recorded activity type differs from the current workflow command",
6489 ),
6490 )));
6491 }
6492 }
6493 if let Some(recorded_options) = recorded_options {
6494 if !recorded_options
6495 .task_queue
6496 .matches_current(¤t_recorded_options.task_queue)
6497 {
6498 return Poll::Ready(Err(Error::NonDeterministicReplay(
6499 ReplayFailure::new(
6500 "activity_task_queue_mismatch",
6501 Some(sequence),
6502 Some(activity_options_description(&recorded_options)),
6503 Some(activity_options_description(¤t_recorded_options)),
6504 "recorded activity task queue differs from the current workflow command",
6505 ),
6506 )));
6507 }
6508 if !recorded_options
6509 .execution_mode
6510 .matches_current(¤t_recorded_options.execution_mode)
6511 {
6512 return Poll::Ready(Err(Error::NonDeterministicReplay(
6513 ReplayFailure::new(
6514 "activity_execution_mode_mismatch",
6515 Some(sequence),
6516 Some(activity_options_description(&recorded_options)),
6517 Some(activity_options_description(¤t_recorded_options)),
6518 "recorded activity execution mode differs from the current workflow command",
6519 ),
6520 )));
6521 }
6522 if !recorded_options
6523 .retry_policy
6524 .matches_current(¤t_recorded_options.retry_policy)
6525 {
6526 return Poll::Ready(Err(Error::NonDeterministicReplay(
6527 ReplayFailure::new(
6528 "activity_retry_policy_mismatch",
6529 Some(sequence),
6530 Some(activity_options_description(&recorded_options)),
6531 Some(activity_options_description(¤t_recorded_options)),
6532 "recorded activity retry policy differs from the current workflow command",
6533 ),
6534 )));
6535 }
6536 }
6537 state.command_cursor += 1;
6538 if let Some(outcome) = outcome {
6539 return Poll::Ready(outcome.map_err(Error::ActivityFailed));
6540 }
6541 state.matched_recorded_pending = true;
6542 self.scheduled = true;
6543 return Poll::Pending;
6544 }
6545 other => {
6546 return Poll::Ready(Err(command_mismatch(
6547 &other,
6548 format!("activity:{}", self.activity_type),
6549 )));
6550 }
6551 }
6552 }
6553
6554 if !self.scheduled {
6555 let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
6556 Ok(args) => args,
6557 Err(error) => return Poll::Ready(Err(error)),
6558 };
6559 let arguments = normalize_avro_arguments(args);
6560 let envelope = match encode_typed_envelope(&arguments, &state.payload_codec) {
6561 Ok(envelope) => envelope,
6562 Err(error) => return Poll::Ready(Err(error)),
6563 };
6564
6565 let mut command = serde_json::Map::from_iter([
6566 ("type".to_string(), json!("schedule_activity")),
6567 (
6568 "activity_type".to_string(),
6569 json!(self.activity_type.clone()),
6570 ),
6571 ("queue".to_string(), json!(task_queue)),
6572 ("arguments".to_string(), envelope),
6573 ]);
6574 for (field, value) in [
6575 ("start_to_close_timeout", options.start_to_close_timeout),
6576 (
6577 "schedule_to_start_timeout",
6578 options.schedule_to_start_timeout,
6579 ),
6580 (
6581 "schedule_to_close_timeout",
6582 options.schedule_to_close_timeout,
6583 ),
6584 ("heartbeat_timeout", options.heartbeat_timeout),
6585 ] {
6586 if let Some(value) = value {
6587 command.insert(field.to_string(), json!(value));
6588 }
6589 }
6590 if let Some(retry_policy) = options.retry_policy {
6591 command.insert("retry_policy".to_string(), retry_policy);
6592 }
6593 state.commands.push(Value::Object(command));
6594 self.scheduled = true;
6595 }
6596
6597 Poll::Pending
6598 }
6599}
6600
6601impl Future for ActivityCall {
6602 type Output = Result<Value>;
6603
6604 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6605 match self.poll_avro_value(cx) {
6606 Poll::Ready(Ok(value)) => Poll::Ready(value.into_json()),
6607 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
6608 Poll::Pending => Poll::Pending,
6609 }
6610 }
6611}
6612
6613pub struct TimerCall {
6615 ctx: WorkflowContext,
6616 delay_seconds: Option<u64>,
6617 scheduled: bool,
6618 matched_pending: bool,
6619}
6620
6621impl Future for TimerCall {
6622 type Output = Result<()>;
6623
6624 fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6625 if self.matched_pending {
6626 return Poll::Pending;
6627 }
6628
6629 let ctx = self.ctx.clone();
6630 let Some(requested_delay) = self.delay_seconds else {
6631 return Poll::Ready(Err(Error::TimerDurationOverflow));
6632 };
6633 let mut state = match ctx.state.lock() {
6634 Ok(state) => state,
6635 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6636 };
6637
6638 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6639 match recorded {
6640 RecordedCommand::Timer {
6641 sequence,
6642 delay_seconds,
6643 fired,
6644 ..
6645 } => {
6646 if delay_seconds != requested_delay {
6647 return Poll::Ready(Err(Error::NonDeterministicReplay(
6648 ReplayFailure::new(
6649 "timer_delay_mismatch",
6650 Some(sequence),
6651 Some(format!("timer:{delay_seconds}s")),
6652 Some(format!("timer:{requested_delay}s")),
6653 "recorded timer delay differs from the current workflow command",
6654 ),
6655 )));
6656 }
6657 state.command_cursor += 1;
6658 if fired {
6659 return Poll::Ready(Ok(()));
6660 }
6661 state.matched_recorded_pending = true;
6662 self.scheduled = true;
6663 self.matched_pending = true;
6664 return Poll::Pending;
6665 }
6666 other => return Poll::Ready(Err(command_mismatch(&other, "timer"))),
6667 }
6668 }
6669
6670 if !self.scheduled {
6671 state.commands.push(json!({
6672 "type": "start_timer",
6673 "delay_seconds": requested_delay,
6674 }));
6675 self.scheduled = true;
6676 }
6677
6678 Poll::Pending
6679 }
6680}
6681
6682pub struct ChildWorkflowCall {
6684 ctx: WorkflowContext,
6685 workflow_type: String,
6686 options: ChildWorkflowOptions,
6687 args: Option<Result<AvroValue>>,
6688 scheduled: bool,
6689 matched_pending: bool,
6690}
6691
6692impl ChildWorkflowCall {
6693 fn poll_avro_value(
6694 mut self: Pin<&mut Self>,
6695 _cx: &mut TaskContext<'_>,
6696 ) -> Poll<Result<ChildWorkflowAvroResult>> {
6697 if self.matched_pending {
6698 return Poll::Pending;
6699 }
6700
6701 let ctx = self.ctx.clone();
6702 let mut state = match ctx.state.lock() {
6703 Ok(state) => state,
6704 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6705 };
6706
6707 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6708 let sequence = recorded.sequence();
6709 match recorded {
6710 RecordedCommand::ChildWorkflow {
6711 workflow_type,
6712 outcome,
6713 ..
6714 } => {
6715 if let Some(recorded_type) = workflow_type {
6716 if recorded_type != self.workflow_type {
6717 return Poll::Ready(Err(Error::NonDeterministicReplay(
6718 ReplayFailure::new(
6719 "recorded_command_detail_mismatch",
6720 Some(sequence),
6721 Some(format!("child workflow:{recorded_type}")),
6722 Some(format!("child workflow:{}", self.workflow_type)),
6723 "recorded child workflow type differs from the current workflow command",
6724 ),
6725 )));
6726 }
6727 }
6728 state.command_cursor += 1;
6729 if let Some(outcome) = outcome {
6730 return Poll::Ready(outcome.map_err(Error::ChildWorkflowFailed));
6731 }
6732 state.matched_recorded_pending = true;
6733 self.scheduled = true;
6734 self.matched_pending = true;
6735 return Poll::Pending;
6736 }
6737 other => {
6738 return Poll::Ready(Err(command_mismatch(
6739 &other,
6740 format!("child workflow:{}", self.workflow_type),
6741 )));
6742 }
6743 }
6744 }
6745
6746 if !self.scheduled {
6747 if self.options.task_queue.trim().is_empty() {
6748 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
6749 "task_queue must not be empty".to_string(),
6750 )));
6751 }
6752 for (name, value) in [
6753 (
6754 "execution_timeout_seconds",
6755 self.options.execution_timeout_seconds,
6756 ),
6757 ("run_timeout_seconds", self.options.run_timeout_seconds),
6758 ] {
6759 if value == Some(0) {
6760 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(format!(
6761 "{name} must be at least 1"
6762 ))));
6763 }
6764 }
6765
6766 let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
6767 Ok(args) => args,
6768 Err(error) => return Poll::Ready(Err(error)),
6769 };
6770 let arguments = match encode_typed_envelope(
6771 &normalize_avro_arguments(args),
6772 &state.payload_codec,
6773 ) {
6774 Ok(arguments) => arguments,
6775 Err(error) => return Poll::Ready(Err(error)),
6776 };
6777 let mut command = json!({
6778 "type": "start_child_workflow",
6779 "workflow_type": self.workflow_type,
6780 "queue": self.options.task_queue,
6781 "parent_close_policy": self.options.parent_close_policy.as_str(),
6782 "arguments": arguments,
6783 });
6784 let object = command
6785 .as_object_mut()
6786 .expect("child workflow command is always an object");
6787 if let Some(policy) = &self.options.retry_policy {
6788 let mut retry_policy = serde_json::Map::new();
6789 if let Some(max_attempts) = policy.max_attempts {
6790 if max_attempts == 0 {
6791 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
6792 "retry_policy.max_attempts must be at least 1".to_string(),
6793 )));
6794 }
6795 retry_policy.insert("max_attempts".to_string(), json!(max_attempts));
6796 }
6797 if !policy.backoff_seconds.is_empty() {
6798 retry_policy
6799 .insert("backoff_seconds".to_string(), json!(policy.backoff_seconds));
6800 }
6801 if !policy.non_retryable_error_types.is_empty() {
6802 retry_policy.insert(
6803 "non_retryable_error_types".to_string(),
6804 json!(policy.non_retryable_error_types),
6805 );
6806 }
6807 if retry_policy.is_empty() {
6808 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
6809 "retry_policy must configure at least one field".to_string(),
6810 )));
6811 }
6812 object.insert("retry_policy".to_string(), Value::Object(retry_policy));
6813 }
6814 if let Some(seconds) = self.options.execution_timeout_seconds {
6815 object.insert("execution_timeout_seconds".to_string(), json!(seconds));
6816 }
6817 if let Some(seconds) = self.options.run_timeout_seconds {
6818 object.insert("run_timeout_seconds".to_string(), json!(seconds));
6819 }
6820 state.commands.push(command);
6821 self.scheduled = true;
6822 }
6823
6824 Poll::Pending
6825 }
6826}
6827
6828impl Future for ChildWorkflowCall {
6829 type Output = Result<ChildWorkflowResult>;
6830
6831 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6832 match self.poll_avro_value(cx) {
6833 Poll::Ready(Ok(result)) => match result.result.into_json() {
6834 Ok(projected) => Poll::Ready(Ok(ChildWorkflowResult {
6835 parent: result.parent,
6836 child: result.child,
6837 child_workflow_type: result.child_workflow_type,
6838 result: projected,
6839 })),
6840 Err(error) => Poll::Ready(Err(error)),
6841 },
6842 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
6843 Poll::Pending => Poll::Pending,
6844 }
6845 }
6846}
6847
6848fn command_mismatch(recorded: &RecordedCommand, actual: impl Into<String>) -> Error {
6849 Error::NonDeterministicReplay(ReplayFailure::new(
6850 "recorded_command_mismatch",
6851 Some(recorded.sequence()),
6852 Some(recorded.shape().to_string()),
6853 Some(actual.into()),
6854 "current workflow command does not match the recorded durable command sequence",
6855 ))
6856}
6857
6858pub struct SignalCall {
6859 ctx: WorkflowContext,
6860 signal_name: String,
6861 opened_wait: bool,
6862 matched_pending: bool,
6863}
6864
6865impl SignalCall {
6866 fn poll_avro_value(
6867 mut self: Pin<&mut Self>,
6868 _cx: &mut TaskContext<'_>,
6869 ) -> Poll<Result<Vec<AvroValue>>> {
6870 if self.matched_pending {
6871 return Poll::Pending;
6872 }
6873
6874 let ctx = self.ctx.clone();
6875 let mut state = match ctx.state.lock() {
6876 Ok(state) => state,
6877 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6878 };
6879
6880 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6881 match recorded {
6882 RecordedCommand::SignalWait {
6883 sequence,
6884 signal_name,
6885 value,
6886 } => {
6887 if signal_name != self.signal_name {
6888 return Poll::Ready(Err(Error::NonDeterministicReplay(
6889 ReplayFailure::new(
6890 "recorded_command_detail_mismatch",
6891 Some(sequence),
6892 Some(format!("signal wait:{signal_name}")),
6893 Some(format!("signal wait:{}", self.signal_name)),
6894 "recorded signal name differs from the current workflow command",
6895 ),
6896 )));
6897 }
6898
6899 state.command_cursor += 1;
6900 if let Some(value) = value {
6901 return Poll::Ready(Ok(value));
6902 }
6903 if state
6904 .resume_signal
6905 .as_ref()
6906 .is_some_and(|signal| signal.signal_name == self.signal_name)
6907 {
6908 let signal = state
6909 .resume_signal
6910 .take()
6911 .expect("matching resume signal is present");
6912 return Poll::Ready(Ok(signal.arguments));
6913 }
6914
6915 state.matched_recorded_pending = true;
6916 self.opened_wait = true;
6917 self.matched_pending = true;
6918 return Poll::Pending;
6919 }
6920 other => {
6921 return Poll::Ready(Err(command_mismatch(
6922 &other,
6923 format!("signal wait:{}", self.signal_name),
6924 )));
6925 }
6926 }
6927 }
6928
6929 if state
6930 .resume_signal
6931 .as_ref()
6932 .is_some_and(|signal| signal.signal_name == self.signal_name)
6933 {
6934 let signal = state
6935 .resume_signal
6936 .take()
6937 .expect("matching resume signal is present");
6938 return Poll::Ready(Ok(signal.arguments));
6939 }
6940
6941 if !self.opened_wait {
6942 state.commands.push(json!({
6943 "type": "open_signal_wait",
6944 "signal_name": self.signal_name
6945 }));
6946 self.opened_wait = true;
6947 }
6948
6949 Poll::Pending
6950 }
6951}
6952
6953impl Future for SignalCall {
6954 type Output = Result<Vec<Value>>;
6955
6956 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6957 match self.poll_avro_value(cx) {
6958 Poll::Ready(Ok(values)) => Poll::Ready(
6959 values
6960 .into_iter()
6961 .map(AvroValue::into_json)
6962 .collect::<Result<Vec<_>>>(),
6963 ),
6964 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
6965 Poll::Pending => Poll::Pending,
6966 }
6967 }
6968}
6969
6970#[derive(Clone, Debug)]
6971pub struct ActivityContext {
6972 client: Client,
6973 pub task_id: String,
6974 pub activity_attempt_id: String,
6975 pub lease_owner: String,
6976 pub activity_type: String,
6977 pub attempt_number: u64,
6978 pub task_queue: String,
6979 pub worker_id: String,
6980}
6981
6982impl ActivityContext {
6983 pub async fn heartbeat<T: Serialize>(&self, details: T) -> Result<ActivityHeartbeatResponse> {
6984 self.client
6985 .heartbeat_activity_task(
6986 &self.task_id,
6987 &self.activity_attempt_id,
6988 &self.lease_owner,
6989 details,
6990 )
6991 .await
6992 }
6993}
6994
6995fn decode_task_avro_arguments(value: Option<&Value>, codec: &str) -> Result<AvroValue> {
6996 match value {
6997 Some(value) => Ok(normalize_avro_arguments(decode_wire_avro_value(
6998 value, codec,
6999 )?)),
7000 None => Ok(AvroValue::Array(Vec::new())),
7001 }
7002}
7003
7004fn decode_resume_signal(task: &WorkflowTask) -> Result<Option<ResumeSignal>> {
7005 let Some(signal_name) = task
7006 .signal_name
7007 .as_deref()
7008 .filter(|value| !value.is_empty())
7009 else {
7010 return Ok(None);
7011 };
7012 let Some(arguments) = task.signal_arguments.as_ref() else {
7013 return Ok(None);
7014 };
7015
7016 let decoded = normalize_avro_arguments(decode_wire_avro_value(arguments, &task.payload_codec)?);
7017 let AvroValue::Array(arguments) = decoded else {
7018 unreachable!("normalize_avro_arguments always returns an array");
7019 };
7020
7021 Ok(Some(ResumeSignal {
7022 signal_name: signal_name.to_string(),
7023 arguments,
7024 }))
7025}
7026
7027fn recorded_commands(
7028 events: &[HistoryEvent],
7029 fallback_codec: &str,
7030 parent: WorkflowIdentity,
7031) -> Result<Vec<RecordedCommand>> {
7032 let mut events_by_sequence: BTreeMap<u64, Vec<&HistoryEvent>> = BTreeMap::new();
7033 let mut last_new_sequence = None;
7034
7035 for event in events {
7036 let is_activity = matches!(
7037 event.event_type.as_str(),
7038 "ActivityScheduled"
7039 | "ActivityStarted"
7040 | "ActivityHeartbeatRecorded"
7041 | "ActivityRetryScheduled"
7042 | "ActivityCompleted"
7043 | "ActivityFailed"
7044 | "ActivityCancelled"
7045 | "ActivityTimedOut"
7046 );
7047 let is_workflow_timer = matches!(
7048 event.event_type.as_str(),
7049 "TimerScheduled" | "TimerCancelled" | "TimerFired"
7050 ) && !is_internal_timer_event(event);
7051 let is_child_workflow = matches!(
7052 event.event_type.as_str(),
7053 "ChildWorkflowScheduled"
7054 | "ChildRunCompleted"
7055 | "ChildRunFailed"
7056 | "ChildRunCancelled"
7057 | "ChildRunTerminated"
7058 );
7059 let is_signal_wait = is_recorded_signal_wait_event(event);
7060 let is_side_effect = event.event_type == "SideEffectRecorded";
7061 let is_version_marker = event.event_type == "VersionMarkerRecorded";
7062 if !is_activity
7063 && !is_workflow_timer
7064 && !is_child_workflow
7065 && !is_signal_wait
7066 && !is_side_effect
7067 && !is_version_marker
7068 {
7069 continue;
7070 }
7071
7072 let sequence = durable_event_sequence(event).ok_or_else(|| {
7073 Error::NonDeterministicReplay(ReplayFailure::new(
7074 "durable_command_sequence_missing",
7075 None,
7076 Some("positive workflow sequence".to_string()),
7077 Some(event.event_type.clone()),
7078 "durable command history event has no workflow sequence",
7079 ))
7080 })?;
7081 if sequence == 0 {
7082 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
7083 "durable_command_sequence_invalid",
7084 Some(sequence),
7085 Some("positive workflow sequence".to_string()),
7086 Some(sequence.to_string()),
7087 "durable command history uses an invalid workflow sequence",
7088 )));
7089 }
7090 if !events_by_sequence.contains_key(&sequence) {
7091 if let Some(previous) = last_new_sequence {
7092 if sequence < previous {
7093 return Err(invalid_recorded_history(
7094 "durable_command_sequence_mismatch",
7095 sequence,
7096 &format!("workflow sequence greater than {previous}"),
7097 &sequence.to_string(),
7098 "durable commands are not strictly ordered by their recorded workflow sequence",
7099 ));
7100 }
7101 }
7102 last_new_sequence = Some(sequence);
7103 }
7104 events_by_sequence.entry(sequence).or_default().push(event);
7105 }
7106
7107 let commands: Vec<RecordedCommand> = events_by_sequence
7108 .into_iter()
7109 .map(|(sequence, sequence_events)| {
7110 let activity_events: Vec<_> = sequence_events
7111 .iter()
7112 .copied()
7113 .filter(|event| event.event_type.starts_with("Activity"))
7114 .collect();
7115 let timer_events: Vec<_> = sequence_events
7116 .iter()
7117 .copied()
7118 .filter(|event| event.event_type.starts_with("Timer"))
7119 .collect();
7120 let child_events: Vec<_> = sequence_events
7121 .iter()
7122 .copied()
7123 .filter(|event| {
7124 event.event_type == "ChildWorkflowScheduled"
7125 || event.event_type.starts_with("ChildRun")
7126 })
7127 .collect();
7128 let signal_wait_events: Vec<_> = sequence_events
7129 .iter()
7130 .copied()
7131 .filter(|event| is_recorded_signal_wait_event(event))
7132 .collect();
7133 let side_effect_events: Vec<_> = sequence_events
7134 .iter()
7135 .copied()
7136 .filter(|event| event.event_type == "SideEffectRecorded")
7137 .collect();
7138 let version_marker_events: Vec<_> = sequence_events
7139 .iter()
7140 .copied()
7141 .filter(|event| event.event_type == "VersionMarkerRecorded")
7142 .collect();
7143
7144 let command_kind_count = usize::from(!activity_events.is_empty())
7145 + usize::from(!timer_events.is_empty())
7146 + usize::from(!child_events.is_empty())
7147 + usize::from(!signal_wait_events.is_empty())
7148 + usize::from(!side_effect_events.is_empty())
7149 + usize::from(!version_marker_events.is_empty());
7150 if command_kind_count > 1 {
7151 let actual = [
7152 (!activity_events.is_empty()).then_some("activity"),
7153 (!timer_events.is_empty()).then_some("timer"),
7154 (!child_events.is_empty()).then_some("child workflow"),
7155 (!signal_wait_events.is_empty()).then_some("signal wait"),
7156 (!side_effect_events.is_empty()).then_some("side effect"),
7157 (!version_marker_events.is_empty()).then_some("version marker"),
7158 ]
7159 .into_iter()
7160 .flatten()
7161 .collect::<Vec<_>>()
7162 .join(" and ");
7163 return Err(invalid_recorded_history(
7164 "durable_command_sequence_collision",
7165 sequence,
7166 "one durable command kind",
7167 &actual,
7168 "one workflow sequence records more than one durable command kind",
7169 ));
7170 }
7171
7172 if !activity_events.is_empty() {
7173 let scheduled_count = activity_events
7174 .iter()
7175 .filter(|event| event.event_type == "ActivityScheduled")
7176 .count();
7177 if scheduled_count > 1 {
7178 return Err(invalid_recorded_history(
7179 "duplicate_activity_schedule",
7180 sequence,
7181 "at most one ActivityScheduled event",
7182 "multiple ActivityScheduled events",
7183 "activity history schedules more than one command at one workflow sequence",
7184 ));
7185 }
7186 let activity_type = activity_events.iter().find_map(|event| {
7187 event
7188 .payload
7189 .get("activity_type")
7190 .or_else(|| event.payload.get("activity_name"))
7191 .and_then(Value::as_str)
7192 .map(str::to_string)
7193 });
7194 if activity_events.iter().filter_map(|event| {
7195 event
7196 .payload
7197 .get("activity_type")
7198 .or_else(|| event.payload.get("activity_name"))
7199 .and_then(Value::as_str)
7200 }).any(|candidate| Some(candidate) != activity_type.as_deref()) {
7201 return Err(invalid_recorded_history(
7202 "activity_identity_mismatch",
7203 sequence,
7204 activity_type.as_deref().unwrap_or("one activity identity"),
7205 "conflicting activity identities",
7206 "activity lifecycle events at one workflow sequence disagree on identity",
7207 ));
7208 }
7209 let terminal: Vec<_> = activity_events
7210 .iter()
7211 .copied()
7212 .filter(|event| {
7213 matches!(
7214 event.event_type.as_str(),
7215 "ActivityCompleted"
7216 | "ActivityFailed"
7217 | "ActivityCancelled"
7218 | "ActivityTimedOut"
7219 )
7220 })
7221 .collect();
7222 if terminal.len() > 1 {
7223 return Err(invalid_recorded_history(
7224 "duplicate_activity_terminal_event",
7225 sequence,
7226 "at most one terminal activity event",
7227 "multiple terminal activity events",
7228 "activity history settles one command more than once",
7229 ));
7230 }
7231 let outcome = terminal
7232 .first()
7233 .map(|event| activity_outcome(event, fallback_codec, activity_type.clone()))
7234 .transpose()?;
7235 let options = activity_events
7236 .iter()
7237 .find(|event| event.event_type == "ActivityScheduled")
7238 .and_then(|event| event.payload.get("activity"))
7239 .and_then(Value::as_object)
7240 .map(|activity| RecordedActivityOptions {
7241 task_queue: recorded_optional_string(activity, "queue"),
7242 execution_mode: recorded_optional_string(activity, "execution_mode"),
7243 retry_policy: recorded_activity_retry_snapshot(
7244 activity.get("retry_policy"),
7245 ),
7246 });
7247 return Ok(RecordedCommand::Activity {
7248 sequence,
7249 activity_type,
7250 options,
7251 outcome,
7252 });
7253 }
7254
7255 if !child_events.is_empty() {
7256 let scheduled: Vec<_> = child_events
7257 .iter()
7258 .copied()
7259 .filter(|event| event.event_type == "ChildWorkflowScheduled")
7260 .collect();
7261 if scheduled.len() != 1 {
7262 return Err(invalid_recorded_history(
7263 "child_workflow_schedule_missing_or_duplicate",
7264 sequence,
7265 "one ChildWorkflowScheduled event",
7266 &format!("{} ChildWorkflowScheduled events", scheduled.len()),
7267 "child workflow replay requires exactly one recorded schedule event",
7268 ));
7269 }
7270 let workflow_type = child_events.iter().find_map(|event| {
7271 event
7272 .payload
7273 .get("child_workflow_type")
7274 .or_else(|| event.payload.get("workflow_type"))
7275 .and_then(Value::as_str)
7276 .filter(|value| !value.is_empty())
7277 .map(str::to_string)
7278 });
7279 if child_events
7280 .iter()
7281 .filter_map(|event| {
7282 event
7283 .payload
7284 .get("child_workflow_type")
7285 .or_else(|| event.payload.get("workflow_type"))
7286 .and_then(Value::as_str)
7287 })
7288 .any(|candidate| Some(candidate) != workflow_type.as_deref())
7289 {
7290 return Err(invalid_recorded_history(
7291 "child_workflow_identity_mismatch",
7292 sequence,
7293 workflow_type
7294 .as_deref()
7295 .unwrap_or("one child workflow type"),
7296 "conflicting child workflow types",
7297 "child workflow lifecycle events at one sequence disagree on type",
7298 ));
7299 }
7300 let mut outcomes = child_workflow_outcomes(
7301 &child_events.iter().map(|event| (*event).clone()).collect::<Vec<_>>(),
7302 fallback_codec,
7303 parent.clone(),
7304 )?;
7305 if outcomes.len() > 1 {
7306 return Err(invalid_recorded_history(
7307 "duplicate_child_workflow_terminal_event",
7308 sequence,
7309 "at most one terminal child event",
7310 "multiple terminal child events",
7311 "child workflow history settles one command more than once",
7312 ));
7313 }
7314 return Ok(RecordedCommand::ChildWorkflow {
7315 sequence,
7316 workflow_type,
7317 outcome: outcomes.pop(),
7318 });
7319 }
7320
7321 if !signal_wait_events.is_empty() {
7322 let opened: Vec<_> = signal_wait_events
7323 .iter()
7324 .copied()
7325 .filter(|event| event.event_type == "SignalWaitOpened")
7326 .collect();
7327 if opened.len() != 1 {
7328 return Err(invalid_recorded_history(
7329 "signal_wait_open_missing_or_duplicate",
7330 sequence,
7331 "one SignalWaitOpened event",
7332 &format!("{} SignalWaitOpened events", opened.len()),
7333 "signal replay requires exactly one canonical wait-open event",
7334 ));
7335 }
7336
7337 let applied: Vec<_> = signal_wait_events
7338 .iter()
7339 .copied()
7340 .filter(|event| event.event_type == "SignalApplied")
7341 .collect();
7342 if applied.len() > 1 {
7343 return Err(invalid_recorded_history(
7344 "duplicate_signal_wait_apply",
7345 sequence,
7346 "at most one SignalApplied event",
7347 "multiple SignalApplied events",
7348 "signal history applies one durable wait more than once",
7349 ));
7350 }
7351
7352 let signal_names = signal_wait_events
7353 .iter()
7354 .map(|event| required_signal_wait_name(event, sequence))
7355 .collect::<Result<Vec<_>>>()?;
7356 let signal_name = signal_names
7357 .first()
7358 .expect("signal wait events are not empty")
7359 .clone();
7360 if signal_names.iter().any(|candidate| candidate != &signal_name) {
7361 return Err(invalid_recorded_history(
7362 "signal_wait_identity_mismatch",
7363 sequence,
7364 &signal_name,
7365 "conflicting signal names",
7366 "signal wait lifecycle events at one workflow sequence disagree on identity",
7367 ));
7368 }
7369 let value = applied
7370 .first()
7371 .map(|event| decode_signal_event_arguments(event, fallback_codec))
7372 .transpose()?;
7373 return Ok(RecordedCommand::SignalWait {
7374 sequence,
7375 signal_name,
7376 value,
7377 });
7378 }
7379
7380 if !side_effect_events.is_empty() {
7381 if side_effect_events.len() != 1 {
7382 return Err(invalid_recorded_history(
7383 "duplicate_side_effect_record",
7384 sequence,
7385 "one SideEffectRecorded event",
7386 &format!("{} SideEffectRecorded events", side_effect_events.len()),
7387 "side-effect history records one workflow command more than once",
7388 ));
7389 }
7390 let event = side_effect_events[0];
7391 let result = event.payload.get("result").ok_or_else(|| {
7392 invalid_recorded_history(
7393 "side_effect_result_missing",
7394 sequence,
7395 "recorded result payload",
7396 "missing result",
7397 "side-effect history is missing its recorded value",
7398 )
7399 })?;
7400 let has_published_envelope = result.as_str().is_some()
7401 || result.as_object().is_some_and(|envelope| {
7402 envelope.get("codec").and_then(Value::as_str).is_some()
7403 && envelope.get("blob").and_then(Value::as_str).is_some()
7404 });
7405 if !has_published_envelope {
7406 return Err(invalid_recorded_history(
7407 "side_effect_payload_malformed",
7408 sequence,
7409 "payload blob or {codec, blob} envelope",
7410 &result.to_string(),
7411 "side-effect history result does not use a published payload envelope",
7412 ));
7413 }
7414 let codec = event
7415 .payload
7416 .get("payload_codec")
7417 .and_then(Value::as_str)
7418 .unwrap_or(fallback_codec);
7419 let value = decode_wire_avro_value(result, codec).map_err(|error| {
7420 invalid_recorded_history(
7421 "side_effect_payload_incompatible",
7422 sequence,
7423 &format!("valid {codec} payload envelope"),
7424 &error.to_string(),
7425 "side-effect history payload cannot be decoded with its recorded codec",
7426 )
7427 })?;
7428 return Ok(RecordedCommand::SideEffect { sequence, value });
7429 }
7430
7431 if !version_marker_events.is_empty() {
7432 if version_marker_events.len() != 1 {
7433 return Err(invalid_recorded_history(
7434 "duplicate_version_marker_record",
7435 sequence,
7436 "one VersionMarkerRecorded event",
7437 &format!("{} VersionMarkerRecorded events", version_marker_events.len()),
7438 "version-marker history records one workflow command more than once",
7439 ));
7440 }
7441 let payload = &version_marker_events[0].payload;
7442 let change_id = payload
7443 .get("change_id")
7444 .and_then(Value::as_str)
7445 .filter(|value| !value.is_empty())
7446 .map(str::to_string)
7447 .ok_or_else(|| {
7448 invalid_recorded_history(
7449 "version_marker_field_missing",
7450 sequence,
7451 "non-empty change_id",
7452 "missing or invalid change_id",
7453 "version-marker history is missing its stable change ID",
7454 )
7455 })?;
7456 let version = required_version_i32(payload, "version", sequence)?;
7457 let min_supported = required_version_i32(payload, "min_supported", sequence)?;
7458 let max_supported = required_version_i32(payload, "max_supported", sequence)?;
7459 if min_supported > max_supported || version < min_supported || version > max_supported {
7460 return Err(invalid_recorded_history(
7461 "version_marker_history_range_invalid",
7462 sequence,
7463 "min_supported <= version <= max_supported",
7464 &format!("{min_supported} <= {version} <= {max_supported}"),
7465 "recorded version marker contains an internally incompatible range",
7466 ));
7467 }
7468 return Ok(RecordedCommand::VersionMarker {
7469 sequence,
7470 change_id,
7471 version,
7472 });
7473 }
7474
7475 let scheduled: Vec<_> = timer_events
7476 .iter()
7477 .copied()
7478 .filter(|event| event.event_type == "TimerScheduled")
7479 .collect();
7480 let fired: Vec<_> = timer_events
7481 .iter()
7482 .copied()
7483 .filter(|event| event.event_type == "TimerFired")
7484 .collect();
7485 if scheduled.len() != 1 {
7486 return Err(invalid_recorded_history(
7487 "timer_schedule_missing_or_duplicate",
7488 sequence,
7489 "one TimerScheduled event",
7490 &format!("{} TimerScheduled events", scheduled.len()),
7491 "timer replay requires exactly one recorded schedule event",
7492 ));
7493 }
7494 if fired.len() > 1 {
7495 return Err(invalid_recorded_history(
7496 "duplicate_timer_fire",
7497 sequence,
7498 "at most one TimerFired event",
7499 "multiple TimerFired events",
7500 "timer history contains more than one fire event for a workflow sequence",
7501 ));
7502 }
7503
7504 let scheduled = scheduled[0];
7505 let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
7506 let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
7507 if let Some(fired) = fired.first() {
7508 let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
7509 if fired_timer_id != timer_id {
7510 return Err(invalid_recorded_history(
7511 "timer_identity_mismatch",
7512 sequence,
7513 &timer_id,
7514 &fired_timer_id,
7515 "TimerFired does not correspond to the recorded TimerScheduled event",
7516 ));
7517 }
7518 let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
7519 if fired_delay != delay_seconds {
7520 return Err(invalid_recorded_history(
7521 "timer_history_delay_mismatch",
7522 sequence,
7523 &delay_seconds.to_string(),
7524 &fired_delay.to_string(),
7525 "TimerScheduled and TimerFired record different delays",
7526 ));
7527 }
7528 }
7529
7530 Ok(RecordedCommand::Timer {
7531 sequence,
7532 delay_seconds,
7533 fired: !fired.is_empty(),
7534 })
7535 })
7536 .collect::<Result<_>>()?;
7537
7538 let mut marker_sequences = HashMap::new();
7539 for command in &commands {
7540 if let RecordedCommand::VersionMarker {
7541 sequence,
7542 change_id,
7543 ..
7544 } = command
7545 {
7546 if let Some(first_sequence) = marker_sequences.insert(change_id.clone(), *sequence) {
7547 return Err(invalid_recorded_history(
7548 "duplicate_version_marker",
7549 *sequence,
7550 &format!("one marker for change ID {change_id:?}"),
7551 &format!("markers at sequences {first_sequence} and {sequence}"),
7552 "workflow history contains duplicate markers for one stable change ID",
7553 ));
7554 }
7555 }
7556 }
7557
7558 Ok(commands)
7559}
7560
7561fn required_version_i32(payload: &Value, field: &str, sequence: u64) -> Result<i32> {
7562 payload
7563 .get(field)
7564 .and_then(Value::as_i64)
7565 .and_then(|value| i32::try_from(value).ok())
7566 .ok_or_else(|| {
7567 invalid_recorded_history(
7568 "version_marker_field_missing",
7569 sequence,
7570 &format!("integer {field}"),
7571 "missing or out-of-range integer",
7572 "version-marker history is missing a required integer field",
7573 )
7574 })
7575}
7576
7577fn durable_event_sequence(event: &HistoryEvent) -> Option<u64> {
7578 event
7579 .payload
7580 .get("sequence")
7581 .or_else(|| event.payload.get("workflow_sequence"))
7582 .or_else(|| event.raw.get("sequence"))
7583 .or_else(|| event.raw.get("workflow_sequence"))
7584 .and_then(value_as_u64)
7585}
7586
7587fn is_internal_timer_event(event: &HistoryEvent) -> bool {
7588 matches!(
7589 event
7590 .payload
7591 .get("timer_kind")
7592 .or_else(|| event.raw.get("timer_kind"))
7593 .and_then(Value::as_str),
7594 Some("condition_timeout" | "signal_timeout")
7595 )
7596}
7597
7598fn required_signal_wait_name(event: &HistoryEvent, sequence: u64) -> Result<String> {
7599 event
7600 .payload
7601 .get("signal_name")
7602 .or_else(|| event.raw.get("signal_name"))
7603 .and_then(Value::as_str)
7604 .filter(|value| !value.is_empty())
7605 .map(str::to_string)
7606 .ok_or_else(|| {
7607 invalid_recorded_history(
7608 "signal_wait_name_missing",
7609 sequence,
7610 "non-empty signal_name",
7611 &event.event_type,
7612 "canonical signal-wait history is missing its signal identity",
7613 )
7614 })
7615}
7616
7617fn is_recorded_signal_wait_event(event: &HistoryEvent) -> bool {
7618 matches!(
7619 event.event_type.as_str(),
7620 "SignalWaitOpened" | "SignalApplied"
7621 )
7622}
7623
7624fn required_history_string(event: &HistoryEvent, field: &str, sequence: u64) -> Result<String> {
7625 event
7626 .payload
7627 .get(field)
7628 .and_then(Value::as_str)
7629 .filter(|value| !value.is_empty())
7630 .map(str::to_string)
7631 .ok_or_else(|| {
7632 invalid_recorded_history(
7633 "timer_history_field_missing",
7634 sequence,
7635 field,
7636 &event.event_type,
7637 "timer history is missing a required identity field",
7638 )
7639 })
7640}
7641
7642fn required_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<u64> {
7643 event
7644 .payload
7645 .get(field)
7646 .and_then(value_as_u64)
7647 .ok_or_else(|| {
7648 invalid_recorded_history(
7649 "timer_history_field_missing",
7650 sequence,
7651 field,
7652 &event.event_type,
7653 "timer history is missing a required numeric field",
7654 )
7655 })
7656}
7657
7658fn invalid_recorded_history(
7659 reason: &str,
7660 sequence: u64,
7661 expected: &str,
7662 actual: &str,
7663 message: &str,
7664) -> Error {
7665 Error::NonDeterministicReplay(ReplayFailure::new(
7666 reason,
7667 Some(sequence),
7668 Some(expected.to_string()),
7669 Some(actual.to_string()),
7670 message,
7671 ))
7672}
7673
7674type ActivityOutcome = std::result::Result<AvroValue, ActivityFailure>;
7675
7676fn activity_outcome(
7677 event: &HistoryEvent,
7678 fallback_codec: &str,
7679 recorded_activity_type: Option<String>,
7680) -> Result<ActivityOutcome> {
7681 if event.event_type == "ActivityCompleted" {
7682 let codec = event
7683 .payload
7684 .get("payload_codec")
7685 .and_then(Value::as_str)
7686 .unwrap_or(fallback_codec);
7687 return Ok(Ok(decode_wire_avro_value(
7688 event.payload.get("result").unwrap_or(&Value::Null),
7689 codec,
7690 )?));
7691 }
7692
7693 let payload = &event.payload;
7694 let (kind, fallback_reason, fallback_message) = match event.event_type.as_str() {
7695 "ActivityFailed" => (ActivityFailureKind::Failed, "activity", "activity failed"),
7696 "ActivityCancelled" => (
7697 ActivityFailureKind::Cancelled,
7698 "cancelled",
7699 "activity was cancelled",
7700 ),
7701 "ActivityTimedOut" => (
7702 ActivityFailureKind::TimedOut,
7703 "timeout",
7704 "activity timed out",
7705 ),
7706 _ => unreachable!("activity_outcome is called only for terminal activity events"),
7707 };
7708 let exception = payload
7709 .get("exception")
7710 .filter(|value| !value.is_null())
7711 .cloned();
7712 let failure_category = payload_string(payload, "failure_category");
7713 let timeout_kind = payload_string(payload, "timeout_kind");
7714 let reason = payload_string(payload, "reason").unwrap_or_else(|| match kind {
7715 ActivityFailureKind::Failed => failure_category
7716 .clone()
7717 .unwrap_or_else(|| fallback_reason.to_string()),
7718 ActivityFailureKind::Cancelled => fallback_reason.to_string(),
7719 ActivityFailureKind::TimedOut => timeout_kind
7720 .clone()
7721 .unwrap_or_else(|| fallback_reason.to_string()),
7722 });
7723 let message = payload_string(payload, "message")
7724 .or_else(|| {
7725 exception
7726 .as_ref()
7727 .and_then(|value| payload_string(value, "message"))
7728 })
7729 .unwrap_or_else(|| fallback_message.to_string());
7730
7731 Ok(Err(ActivityFailure {
7732 kind,
7733 reason,
7734 message,
7735 activity_execution_id: payload_string(payload, "activity_execution_id"),
7736 activity_attempt_id: payload_string(payload, "activity_attempt_id"),
7737 activity_type: payload_string(payload, "activity_type")
7738 .or_else(|| payload_string(payload, "activity_name"))
7739 .or(recorded_activity_type),
7740 activity_class: payload_string(payload, "activity_class"),
7741 attempt_number: payload.get("attempt_number").and_then(value_as_u64),
7742 failure_id: payload_string(payload, "failure_id"),
7743 failure_category,
7744 timeout_kind,
7745 non_retryable: payload
7746 .get("non_retryable")
7747 .and_then(Value::as_bool)
7748 .unwrap_or(false),
7749 exception_type: payload_string(payload, "exception_type").or_else(|| {
7750 exception
7751 .as_ref()
7752 .and_then(|value| payload_string(value, "type"))
7753 }),
7754 exception_class: payload_string(payload, "exception_class").or_else(|| {
7755 exception
7756 .as_ref()
7757 .and_then(|value| payload_string(value, "class"))
7758 }),
7759 code: payload
7760 .get("code")
7761 .filter(|value| !value.is_null())
7762 .cloned(),
7763 exception,
7764 }))
7765}
7766
7767type ChildWorkflowOutcome = std::result::Result<ChildWorkflowAvroResult, ChildWorkflowFailure>;
7768
7769fn child_workflow_outcomes(
7770 events: &[HistoryEvent],
7771 fallback_codec: &str,
7772 parent: WorkflowIdentity,
7773) -> Result<Vec<ChildWorkflowOutcome>> {
7774 let mut outcomes = Vec::new();
7775
7776 for event in events {
7777 let kind = match event.event_type.as_str() {
7778 "ChildRunCompleted" => None,
7779 "ChildRunFailed" => Some((
7780 ChildWorkflowFailureKind::Failed,
7781 "child_workflow",
7782 "child workflow failed",
7783 )),
7784 "ChildRunCancelled" => Some((
7785 ChildWorkflowFailureKind::Cancelled,
7786 "cancelled",
7787 "child workflow was cancelled",
7788 )),
7789 "ChildRunTerminated" => Some((
7790 ChildWorkflowFailureKind::Terminated,
7791 "terminated",
7792 "child workflow was terminated",
7793 )),
7794 _ => continue,
7795 };
7796 let payload = &event.payload;
7797 let child_workflow_id = payload_string(payload, "child_workflow_instance_id");
7798 let child_workflow_run_id = payload_string(payload, "child_workflow_run_id");
7799 let child_workflow_type = payload_string(payload, "child_workflow_type");
7800
7801 if let Some((kind, reason, fallback_message)) = kind {
7802 let exception = payload
7803 .get("exception")
7804 .filter(|value| !value.is_null())
7805 .cloned();
7806 let message = payload_string(payload, "message")
7807 .or_else(|| {
7808 exception
7809 .as_ref()
7810 .and_then(|value| payload_string(value, "message"))
7811 })
7812 .unwrap_or_else(|| fallback_message.to_string());
7813 let exception_type = payload_string(payload, "exception_type").or_else(|| {
7814 exception
7815 .as_ref()
7816 .and_then(|value| payload_string(value, "type"))
7817 });
7818 let exception_class = payload_string(payload, "exception_class").or_else(|| {
7819 exception
7820 .as_ref()
7821 .and_then(|value| payload_string(value, "class"))
7822 });
7823 outcomes.push(Err(ChildWorkflowFailure {
7824 kind,
7825 reason: reason.to_string(),
7826 message,
7827 parent_workflow_id: parent.workflow_id.clone(),
7828 parent_workflow_run_id: parent.run_id.clone(),
7829 child_workflow_id,
7830 child_workflow_run_id,
7831 child_workflow_type,
7832 failure_id: payload_string(payload, "failure_id"),
7833 failure_category: payload_string(payload, "failure_category"),
7834 exception_type,
7835 exception_class,
7836 non_retryable: payload
7837 .get("non_retryable")
7838 .and_then(Value::as_bool)
7839 .unwrap_or(false),
7840 code: payload
7841 .get("code")
7842 .filter(|value| !value.is_null())
7843 .cloned(),
7844 exception,
7845 }));
7846 continue;
7847 }
7848
7849 let codec = payload
7850 .get("payload_codec")
7851 .and_then(Value::as_str)
7852 .unwrap_or(fallback_codec);
7853 let result = payload
7854 .get("result")
7855 .or_else(|| payload.get("output"))
7856 .unwrap_or(&Value::Null);
7857 outcomes.push(Ok(ChildWorkflowAvroResult {
7858 parent: parent.clone(),
7859 child: WorkflowIdentity {
7860 workflow_id: child_workflow_id,
7861 run_id: child_workflow_run_id,
7862 },
7863 child_workflow_type,
7864 result: decode_wire_avro_value(result, codec)?,
7865 }));
7866 }
7867
7868 Ok(outcomes)
7869}
7870
7871fn payload_string(payload: &Value, key: &str) -> Option<String> {
7872 payload
7873 .get(key)
7874 .and_then(Value::as_str)
7875 .filter(|value| !value.is_empty())
7876 .map(str::to_string)
7877}
7878
7879fn workflow_failure_command(error: &Error) -> Value {
7880 let (exception_type, exception_class, properties) = match error {
7881 Error::ActivityFailed(failure) => (
7882 match failure.kind {
7883 ActivityFailureKind::Failed => "ActivityFailed",
7884 ActivityFailureKind::Cancelled => "ActivityCancelled",
7885 ActivityFailureKind::TimedOut => "ActivityTimedOut",
7886 },
7887 "durable_workflow::ActivityFailure",
7888 json!({
7889 "reason": failure.reason,
7890 "activity_execution_id": failure.activity_execution_id,
7891 "activity_attempt_id": failure.activity_attempt_id,
7892 "activity_type": failure.activity_type,
7893 "activity_class": failure.activity_class,
7894 "attempt_number": failure.attempt_number,
7895 "failure_id": failure.failure_id,
7896 "failure_category": failure.failure_category,
7897 "timeout_kind": failure.timeout_kind,
7898 "activity_non_retryable": failure.non_retryable,
7899 "activity_exception_type": failure.exception_type,
7900 "activity_exception_class": failure.exception_class,
7901 "activity_code": failure.code,
7902 "activity_exception": failure.exception,
7903 }),
7904 ),
7905 Error::ChildWorkflowFailed(failure) => (
7906 match failure.kind {
7907 ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
7908 ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
7909 ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
7910 },
7911 "durable_workflow::ChildWorkflowFailure",
7912 json!({
7913 "reason": failure.reason,
7914 "parent_workflow_id": failure.parent_workflow_id,
7915 "parent_workflow_run_id": failure.parent_workflow_run_id,
7916 "child_workflow_id": failure.child_workflow_id,
7917 "child_workflow_run_id": failure.child_workflow_run_id,
7918 "child_workflow_type": failure.child_workflow_type,
7919 "failure_id": failure.failure_id,
7920 "failure_category": failure.failure_category,
7921 "child_exception_type": failure.exception_type,
7922 "child_exception_class": failure.exception_class,
7923 "child_non_retryable": failure.non_retryable,
7924 "child_code": failure.code,
7925 "child_exception": failure.exception,
7926 }),
7927 ),
7928 Error::NonDeterministicReplay(_) => (
7929 "NonDeterministicReplay",
7930 "durable_workflow::Error",
7931 Value::Null,
7932 ),
7933 _ => ("RustWorkflowError", "durable_workflow::Error", Value::Null),
7934 };
7935 let non_retryable = match error {
7936 Error::ActivityFailed(failure) => failure.non_retryable,
7937 Error::ChildWorkflowFailed(failure) => failure.non_retryable,
7938 Error::NonDeterministicReplay(_) => true,
7939 _ => false,
7940 };
7941
7942 json!({
7943 "type": "fail_workflow",
7944 "message": error.to_string(),
7945 "exception_type": exception_type,
7946 "exception_class": exception_class,
7947 "non_retryable": non_retryable,
7948 "exception": {
7949 "type": exception_type,
7950 "class": exception_class,
7951 "message": error.to_string(),
7952 "properties": properties,
7953 }
7954 })
7955}
7956
7957fn workflow_task_integrity_error(error: &Error) -> bool {
7958 matches!(
7959 error,
7960 Error::NonDeterministicReplay(_) | Error::Protocol(_) | Error::WorkflowStatePoisoned
7961 )
7962}
7963
7964fn decode_signal_event_arguments(
7965 event: &HistoryEvent,
7966 fallback_codec: &str,
7967) -> Result<Vec<AvroValue>> {
7968 let codec = event
7969 .payload
7970 .get("payload_codec")
7971 .and_then(Value::as_str)
7972 .unwrap_or(fallback_codec);
7973 let raw = event
7974 .payload
7975 .get("value")
7976 .or_else(|| event.payload.get("input"))
7977 .or_else(|| event.payload.get("arguments"));
7978 let decoded = match raw.filter(|value| !value.is_null()) {
7979 Some(value) => decode_wire_avro_value(value, codec)?,
7980 None => AvroValue::Array(Vec::new()),
7981 };
7982 let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
7983 unreachable!("normalize_avro_arguments always returns an array");
7984 };
7985 Ok(arguments)
7986}
7987
7988fn hydrate_query_history_from_export(task: &mut QueryTask) -> Result<()> {
7989 let Some(export_events) = task
7990 .history_export
7991 .as_ref()
7992 .and_then(|export| export.get("history_events"))
7993 .and_then(Value::as_array)
7994 else {
7995 return Ok(());
7996 };
7997
7998 if export_events.len() > task.history_events.len() {
7999 task.history_events = serde_json::from_value(Value::Array(export_events.clone()))?;
8000 }
8001
8002 Ok(())
8003}
8004
8005fn enrich_query_history_from_export(task: &mut QueryTask) -> Result<()> {
8006 let Some(export) = task.history_export.as_ref() else {
8007 return Ok(());
8008 };
8009 let signals = export
8010 .get("signals")
8011 .and_then(Value::as_array)
8012 .cloned()
8013 .unwrap_or_default();
8014 let activities = export
8015 .get("activities")
8016 .and_then(Value::as_array)
8017 .cloned()
8018 .unwrap_or_default();
8019 let export_codec = export
8020 .get("payloads")
8021 .and_then(|payloads| payloads.get("codec"))
8022 .and_then(Value::as_str)
8023 .unwrap_or(&task.payload_codec)
8024 .to_string();
8025 let mut signal_name_offsets: HashMap<String, usize> = HashMap::new();
8026
8027 for event in &mut task.history_events {
8028 if event.event_type == "ActivityCompleted" {
8029 let sequence = event
8030 .payload
8031 .get("sequence")
8032 .or_else(|| event.payload.get("workflow_sequence"))
8033 .and_then(value_as_u64);
8034 let Some(activity) = sequence.and_then(|sequence| {
8035 activities.iter().find(|activity| {
8036 activity.get("sequence").and_then(value_as_u64) == Some(sequence)
8037 })
8038 }) else {
8039 continue;
8040 };
8041 let Some(payload) = event.payload.as_object_mut() else {
8042 continue;
8043 };
8044 if missing_payload(payload.get("result")) {
8045 if let Some(result) = activity
8046 .get("result")
8047 .filter(|value| !missing_payload(Some(value)))
8048 {
8049 payload.insert("result".to_string(), result.clone());
8050 }
8051 }
8052 for field in ["payload_codec", "activity_type"] {
8053 if payload
8054 .get(field)
8055 .and_then(Value::as_str)
8056 .unwrap_or_default()
8057 .is_empty()
8058 {
8059 if let Some(value) = activity.get(field) {
8060 payload.insert(field.to_string(), value.clone());
8061 }
8062 }
8063 }
8064 continue;
8065 }
8066
8067 if event.event_type != "SignalReceived" && event.event_type != "SignalApplied" {
8068 continue;
8069 }
8070 let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
8071 let command_id = event
8072 .payload
8073 .get("workflow_command_id")
8074 .or_else(|| event.raw.get("workflow_command_id"))
8075 .and_then(Value::as_str);
8076 let signal_name = event
8077 .payload
8078 .get("signal_name")
8079 .and_then(Value::as_str)
8080 .unwrap_or_default()
8081 .to_string();
8082 let matched = signals
8083 .iter()
8084 .find(|signal| {
8085 signal_id.is_some() && signal.get("id").and_then(Value::as_str) == signal_id
8086 })
8087 .or_else(|| {
8088 signals.iter().find(|signal| {
8089 command_id.is_some()
8090 && signal.get("command_id").and_then(Value::as_str) == command_id
8091 })
8092 })
8093 .or_else(|| {
8094 let offset = signal_name_offsets.entry(signal_name.clone()).or_default();
8095 let signal = signals
8096 .iter()
8097 .filter(|signal| {
8098 signal.get("name").and_then(Value::as_str) == Some(signal_name.as_str())
8099 })
8100 .nth(*offset);
8101 if signal.is_some() {
8102 *offset += 1;
8103 }
8104 signal
8105 });
8106 let Some(signal) = matched else {
8107 continue;
8108 };
8109 let signal_codec = signal
8110 .get("payload_codec")
8111 .and_then(Value::as_str)
8112 .unwrap_or(&export_codec);
8113 let Some(payload) = event.payload.as_object_mut() else {
8114 continue;
8115 };
8116 if missing_payload(payload.get("arguments")) {
8117 if let Some(arguments) = signal
8118 .get("arguments")
8119 .filter(|value| !missing_payload(Some(value)))
8120 {
8121 let envelope = match arguments {
8122 Value::String(blob) => json!({"codec": signal_codec, "blob": blob}),
8123 other => other.clone(),
8124 };
8125 payload.insert("arguments".to_string(), envelope);
8126 }
8127 }
8128 if payload
8129 .get("payload_codec")
8130 .and_then(Value::as_str)
8131 .unwrap_or_default()
8132 .is_empty()
8133 {
8134 payload.insert("payload_codec".to_string(), json!(signal_codec));
8135 }
8136 }
8137
8138 Ok(())
8139}
8140
8141fn missing_payload(value: Option<&Value>) -> bool {
8142 match value {
8143 None | Some(Value::Null) => true,
8144 Some(Value::String(value)) => value.is_empty(),
8145 Some(_) => false,
8146 }
8147}
8148
8149fn query_signal_events(task: &QueryTask) -> Result<Vec<QuerySignal>> {
8150 let export_signals = task
8151 .history_export
8152 .as_ref()
8153 .and_then(|export| export.get("signals"))
8154 .and_then(Value::as_array)
8155 .cloned()
8156 .unwrap_or_default();
8157 let export_codec = task
8158 .history_export
8159 .as_ref()
8160 .and_then(|export| export.get("payloads"))
8161 .and_then(|payloads| payloads.get("codec"))
8162 .and_then(Value::as_str)
8163 .unwrap_or(&task.payload_codec);
8164 let mut name_offsets: HashMap<String, usize> = HashMap::new();
8165 let mut signals = Vec::new();
8166
8167 for event in &task.history_events {
8168 if event.event_type != "SignalApplied" && event.event_type != "SignalReceived" {
8169 continue;
8170 }
8171
8172 let name = event
8173 .payload
8174 .get("signal_name")
8175 .and_then(Value::as_str)
8176 .unwrap_or_default();
8177 if name.is_empty() {
8178 continue;
8179 }
8180 let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
8181 let command_id = event
8182 .payload
8183 .get("workflow_command_id")
8184 .or_else(|| event.raw.get("workflow_command_id"))
8185 .and_then(Value::as_str);
8186 let matched_export = export_signals
8187 .iter()
8188 .find(|candidate| {
8189 signal_id.is_some() && candidate.get("id").and_then(Value::as_str) == signal_id
8190 })
8191 .or_else(|| {
8192 export_signals.iter().find(|candidate| {
8193 command_id.is_some()
8194 && candidate.get("command_id").and_then(Value::as_str) == command_id
8195 })
8196 })
8197 .or_else(|| {
8198 let offset = name_offsets.entry(name.to_string()).or_default();
8199 let candidate = export_signals
8200 .iter()
8201 .filter(|candidate| candidate.get("name").and_then(Value::as_str) == Some(name))
8202 .nth(*offset);
8203 if candidate.is_some() {
8204 *offset += 1;
8205 }
8206 candidate
8207 });
8208 let codec = event
8209 .payload
8210 .get("payload_codec")
8211 .and_then(Value::as_str)
8212 .or_else(|| {
8213 matched_export
8214 .and_then(|signal| signal.get("payload_codec"))
8215 .and_then(Value::as_str)
8216 })
8217 .unwrap_or(export_codec);
8218 let raw_arguments = event
8219 .payload
8220 .get("value")
8221 .or_else(|| event.payload.get("input"))
8222 .or_else(|| event.payload.get("arguments"))
8223 .filter(|value| !value.is_null())
8224 .or_else(|| matched_export.and_then(|signal| signal.get("arguments")));
8225 let (arguments, avro_arguments) = decode_query_signal_arguments(raw_arguments, codec)?;
8226 let workflow_sequence = event
8227 .payload
8228 .get("workflow_sequence")
8229 .and_then(value_as_u64)
8230 .or_else(|| {
8231 matched_export
8232 .and_then(|signal| signal.get("workflow_sequence"))
8233 .and_then(value_as_u64)
8234 });
8235
8236 signals.push(QuerySignal {
8237 id: signal_id.map(str::to_string).or_else(|| {
8238 matched_export
8239 .and_then(|signal| signal.get("id"))
8240 .and_then(Value::as_str)
8241 .map(str::to_string)
8242 }),
8243 name: name.to_string(),
8244 arguments,
8245 avro_arguments,
8246 workflow_sequence,
8247 });
8248 }
8249
8250 if signals.is_empty() {
8251 for signal in export_signals {
8252 if signal.get("status").and_then(Value::as_str) == Some("rejected") {
8253 continue;
8254 }
8255 let Some(name) = signal.get("name").and_then(Value::as_str) else {
8256 continue;
8257 };
8258 let codec = signal
8259 .get("payload_codec")
8260 .and_then(Value::as_str)
8261 .unwrap_or(export_codec);
8262 let (arguments, avro_arguments) =
8263 decode_query_signal_arguments(signal.get("arguments"), codec)?;
8264 signals.push(QuerySignal {
8265 id: signal.get("id").and_then(Value::as_str).map(str::to_string),
8266 name: name.to_string(),
8267 arguments,
8268 avro_arguments,
8269 workflow_sequence: signal.get("workflow_sequence").and_then(value_as_u64),
8270 });
8271 }
8272 signals.sort_by_key(|signal| signal.workflow_sequence.unwrap_or(u64::MAX));
8273 }
8274
8275 Ok(signals)
8276}
8277
8278fn decode_query_signal_arguments(
8279 raw: Option<&Value>,
8280 codec: &str,
8281) -> Result<(Vec<Value>, Vec<AvroValue>)> {
8282 let decoded = match raw.filter(|value| !value.is_null()) {
8283 Some(value) => decode_wire_avro_value(value, codec)?,
8284 None => AvroValue::Array(Vec::new()),
8285 };
8286 let AvroValue::Array(avro_arguments) = normalize_avro_arguments(decoded) else {
8287 unreachable!("normalize_avro_arguments always returns an array");
8288 };
8289 let arguments = avro_arguments
8290 .iter()
8291 .cloned()
8292 .map(AvroValue::into_json)
8293 .collect::<Result<Vec<_>>>()?;
8294 Ok((arguments, avro_arguments))
8295}
8296
8297fn value_as_u64(value: &Value) -> Option<u64> {
8298 value
8299 .as_u64()
8300 .or_else(|| value.as_str().and_then(|value| value.parse().ok()))
8301}
8302
8303#[cfg(test)]
8304mod tests {
8305 use super::*;
8306 use std::{
8307 io::{Read, Write},
8308 net::{SocketAddr, TcpListener, TcpStream},
8309 sync::atomic::AtomicUsize,
8310 thread,
8311 };
8312
8313 #[test]
8314 fn client_builder_rejects_the_sdk_owned_api_suffix() {
8315 for base_url in [
8316 "http://127.0.0.1:8080/api",
8317 "http://localhost:8080/api/",
8318 "https://runtime.example.test/namespaces/orders/api",
8319 ] {
8320 let error = Client::builder(base_url)
8321 .build()
8322 .expect_err("SDK-owned /api suffix must be rejected during build");
8323
8324 assert!(matches!(error, Error::InvalidBaseUrl), "{base_url}");
8325 assert!(
8326 error.to_string().contains("SDK appends /api automatically"),
8327 "the validation error must explain how to fix the endpoint"
8328 );
8329 }
8330 }
8331
8332 #[test]
8333 fn client_builder_preserves_self_hosted_and_managed_runtime_prefixes() {
8334 for (base_url, expected) in [
8335 ("http://127.0.0.1:8080", "http://127.0.0.1:8080"),
8336 (
8337 "http://localhost:8080/durable-workflow/",
8338 "http://localhost:8080/durable-workflow",
8339 ),
8340 (
8341 "https://runtime.example.test/namespaces/orders",
8342 "https://runtime.example.test/namespaces/orders",
8343 ),
8344 (
8345 "https://runtime.example.test/gateway/api/namespaces/orders",
8346 "https://runtime.example.test/gateway/api/namespaces/orders",
8347 ),
8348 (
8349 "https://api.example.test/runtime/orders/",
8350 "https://api.example.test/runtime/orders",
8351 ),
8352 ] {
8353 let client = Client::builder(base_url)
8354 .build()
8355 .expect("Server and Cloud runtime base URL must remain valid");
8356
8357 assert_eq!(client.base_url, expected);
8358 }
8359 }
8360
8361 fn typed_fidelity_probe() -> AvroValue {
8362 AvroValue::Map(BTreeMap::from([
8363 ("bytes".to_string(), AvroValue::Bytes(vec![0, 0xff])),
8364 ("empty".to_string(), AvroValue::Map(BTreeMap::new())),
8365 (
8366 "numeric".to_string(),
8367 AvroValue::Map(BTreeMap::from([
8368 ("0".to_string(), AvroValue::String("zero".to_string())),
8369 ("1".to_string(), AvroValue::String("one".to_string())),
8370 ])),
8371 ),
8372 (
8373 "nested".to_string(),
8374 AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([(
8375 "enabled".to_string(),
8376 AvroValue::Boolean(true),
8377 )]))]),
8378 ),
8379 (
8380 "projection_collisions".to_string(),
8381 AvroValue::Array(projection_collision_probe()),
8382 ),
8383 ]))
8384 }
8385
8386 fn projection_collision_probe() -> Vec<AvroValue> {
8387 vec![
8388 AvroValue::Map(BTreeMap::from([
8389 ("$type".to_string(), AvroValue::String("bytes".to_string())),
8390 (
8391 "base64".to_string(),
8392 AvroValue::String("ordinary user text".to_string()),
8393 ),
8394 ])),
8395 AvroValue::Map(BTreeMap::from([
8396 ("$type".to_string(), AvroValue::String("map".to_string())),
8397 (
8398 "entries".to_string(),
8399 AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([
8400 ("key".to_string(), AvroValue::String("ordinary".to_string())),
8401 (
8402 "value".to_string(),
8403 AvroValue::String("user map".to_string()),
8404 ),
8405 ]))]),
8406 ),
8407 ])),
8408 ]
8409 }
8410
8411 #[derive(Clone, Debug, Default, PartialEq)]
8412 struct ReplayCounterState {
8413 loaded: Option<String>,
8414 count: i64,
8415 finished: bool,
8416 }
8417
8418 fn replay_counter_worker() -> Worker {
8419 let client = Client::new("http://127.0.0.1:8080").expect("client");
8420 let mut worker = Worker::new(client, "rust-workers");
8421 worker.register_replayed_workflow(
8422 "replay-counter",
8423 ReplayCounterState::default,
8424 |ctx, _input, state| async move {
8425 let loaded = ctx.activity("load-counter", json!([])).await?;
8426 state.update(|current| {
8427 current.loaded = loaded.as_str().map(str::to_string);
8428 })?;
8429 for _ in 0..2 {
8430 let signal = ctx.wait_signal("increment").await?;
8431 let amount = signal.first().and_then(Value::as_i64).unwrap_or_default();
8432 state.update(|current| current.count += amount)?;
8433 }
8434 state.update(|current| current.finished = true)?;
8435 state.read(|current| Ok(json!(current.count)))?
8436 },
8437 );
8438 worker.register_replayed_query::<ReplayCounterState, _, _>(
8439 "replay-counter",
8440 "current",
8441 |_ctx, state, _args| async move {
8442 Ok(json!({
8443 "loaded": state.loaded,
8444 "count": state.count,
8445 "finished": state.finished,
8446 }))
8447 },
8448 );
8449 worker.register_replayed_query::<ReplayCounterState, _, _>(
8450 "replay-counter",
8451 "detached-mutation",
8452 |_ctx, state, _args| async move {
8453 let mut detached = (*state).clone();
8454 detached.count = 999;
8455 Ok(json!(detached.count))
8456 },
8457 );
8458 worker.register_replayed_query::<ReplayCounterState, _, _>(
8459 "replay-counter",
8460 "failed-mutation",
8461 |_ctx, state, _args| async move {
8462 let mut detached = (*state).clone();
8463 detached.count = 999;
8464 Err(Error::WorkerLoop("query refused".to_string()))
8465 },
8466 );
8467 worker
8468 }
8469
8470 fn replay_counter_query(
8471 query_name: &str,
8472 history_events: Value,
8473 run_status: &str,
8474 ) -> QueryTask {
8475 serde_json::from_value(json!({
8476 "query_task_id": format!("query-{query_name}"),
8477 "workflow_type": "replay-counter",
8478 "query_name": query_name,
8479 "payload_codec": "json",
8480 "workflow_arguments": {"codec": "json", "blob": "[]"},
8481 "query_arguments": {"codec": "json", "blob": "[]"},
8482 "history_events": history_events,
8483 "run_status": run_status,
8484 }))
8485 .expect("query task")
8486 }
8487
8488 fn workflow_context(history: Vec<HistoryEvent>) -> WorkflowContext {
8489 workflow_context_with_codec(history, JSON_CODEC)
8490 }
8491
8492 fn workflow_context_with_codec(
8493 history: Vec<HistoryEvent>,
8494 payload_codec: &str,
8495 ) -> WorkflowContext {
8496 WorkflowContext {
8497 state: Arc::new(Mutex::new(
8498 WorkflowState::new_with_identity(
8499 history,
8500 None,
8501 None,
8502 "rust-workers".to_string(),
8503 payload_codec.to_string(),
8504 None,
8505 )
8506 .expect("valid workflow history"),
8507 )),
8508 }
8509 }
8510
8511 fn history_event(event_type: &str, payload: Value) -> HistoryEvent {
8512 HistoryEvent {
8513 event_type: event_type.to_string(),
8514 payload,
8515 raw: HashMap::new(),
8516 }
8517 }
8518
8519 fn workflow_task(
8520 workflow_type: &str,
8521 history_events: Vec<HistoryEvent>,
8522 payload_codec: &str,
8523 ) -> WorkflowTask {
8524 WorkflowTask {
8525 task_id: format!("wft-{workflow_type}"),
8526 workflow_id: Some(format!("wf-{workflow_type}")),
8527 run_id: Some(format!("run-{workflow_type}")),
8528 workflow_type: workflow_type.to_string(),
8529 payload_codec: payload_codec.to_string(),
8530 arguments: Some(
8531 encode_value_envelope(&json!([]), payload_codec).expect("workflow arguments"),
8532 ),
8533 total_history_events: Some(history_events.len() as u64),
8534 history_size_bytes: None,
8535 continue_as_new_recommended: None,
8536 history_budget_pressure: None,
8537 history_events,
8538 next_history_page_token: None,
8539 workflow_task_attempt: 1,
8540 workflow_signal_id: None,
8541 signal_name: None,
8542 signal_arguments: None,
8543 workflow_update_id: None,
8544 update_name: None,
8545 lease_owner: Some("rust-worker".to_string()),
8546 }
8547 }
8548
8549 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
8550 struct SideEffectProbe {
8551 request_id: String,
8552 attempt: u32,
8553 }
8554
8555 #[test]
8556 fn typed_side_effect_runs_callback_once_and_replay_skips_it() {
8557 let calls = AtomicUsize::new(0);
8558 let ctx = workflow_context(Vec::new());
8559 let value = ctx
8560 .side_effect(|| {
8561 calls.fetch_add(1, Ordering::SeqCst);
8562 SideEffectProbe {
8563 request_id: "request-42".to_string(),
8564 attempt: 3,
8565 }
8566 })
8567 .expect("first side effect");
8568 assert_eq!(value.attempt, 3);
8569 assert_eq!(calls.load(Ordering::SeqCst), 1);
8570 let commands = ctx.take_commands().expect("commands");
8571 assert_eq!(commands.len(), 1);
8572 assert_eq!(commands[0]["type"], "record_side_effect");
8573 assert_eq!(
8574 decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("JSON result"),
8575 serde_json::to_value(&value).expect("value")
8576 );
8577
8578 let replay = workflow_context(vec![history_event(
8579 "SideEffectRecorded",
8580 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
8581 )]);
8582 let replayed: SideEffectProbe = replay
8583 .side_effect(|| {
8584 calls.fetch_add(1, Ordering::SeqCst);
8585 panic!("committed side-effect callbacks must not run during replay")
8586 })
8587 .expect("replayed side effect");
8588 assert_eq!(replayed, value);
8589 assert_eq!(calls.load(Ordering::SeqCst), 1);
8590 assert!(replay.take_commands().expect("commands").is_empty());
8591 replay.ensure_history_consumed().expect("history consumed");
8592 }
8593
8594 #[test]
8595 fn side_effect_uses_avro_envelope_and_uuid_is_replay_stable() {
8596 let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
8597 let value = ctx
8598 .side_effect(|| SideEffectProbe {
8599 request_id: "avro-request".to_string(),
8600 attempt: 1,
8601 })
8602 .expect("Avro side effect");
8603 let uuid = ctx.uuid_v4().expect("deterministic UUID");
8604 let commands = ctx.take_commands().expect("commands");
8605 assert_eq!(commands.len(), 2);
8606 assert_eq!(commands[0]["result"]["codec"], DEFAULT_CODEC);
8607 assert_eq!(commands[1]["result"]["codec"], DEFAULT_CODEC);
8608 assert_eq!(
8609 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
8610 serde_json::to_value(&value).expect("value")
8611 );
8612
8613 let replay = workflow_context_with_codec(
8614 vec![
8615 history_event(
8616 "SideEffectRecorded",
8617 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
8618 ),
8619 history_event(
8620 "SideEffectRecorded",
8621 json!({"sequence": 2, "result": commands[1]["result"].clone()}),
8622 ),
8623 ],
8624 DEFAULT_CODEC,
8625 );
8626 let replayed: SideEffectProbe = replay
8627 .side_effect(|| panic!("Avro callback must not run"))
8628 .expect("replayed Avro value");
8629 let replayed_uuid = replay.uuid_v4().expect("replayed UUID");
8630 assert_eq!(replayed, value);
8631 assert_eq!(replayed_uuid, uuid);
8632 assert!(replay.take_commands().expect("commands").is_empty());
8633 }
8634
8635 #[test]
8636 fn typed_side_effect_replay_preserves_bytes_and_maps() {
8637 let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
8638 let value = ctx
8639 .side_effect_avro_value(typed_fidelity_probe)
8640 .expect("typed side effect");
8641 let commands = ctx.take_commands().expect("side-effect command");
8642 assert_eq!(
8643 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
8644 .expect("recorded side effect"),
8645 value
8646 );
8647
8648 let replay = workflow_context_with_codec(
8649 vec![history_event(
8650 "SideEffectRecorded",
8651 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
8652 )],
8653 DEFAULT_CODEC,
8654 );
8655 assert_eq!(
8656 replay
8657 .side_effect_avro_value(|| panic!("replay must not invoke callback"))
8658 .expect("replayed typed side effect"),
8659 value
8660 );
8661 }
8662
8663 #[test]
8664 fn ordered_side_effects_share_the_durable_command_stream() {
8665 let first = encode_value_envelope(&json!("first"), JSON_CODEC).expect("first");
8666 let second = encode_value_envelope(&json!(29), JSON_CODEC).expect("second");
8667 let ctx = workflow_context(vec![
8668 history_event(
8669 "SideEffectRecorded",
8670 json!({"sequence": 1, "result": first}),
8671 ),
8672 history_event(
8673 "SideEffectRecorded",
8674 json!({"sequence": 2, "result": second}),
8675 ),
8676 ]);
8677 let first: String = ctx
8678 .side_effect(|| panic!("first callback must not run"))
8679 .expect("first replay");
8680 let second: i32 = ctx
8681 .side_effect(|| panic!("second callback must not run"))
8682 .expect("second replay");
8683 assert_eq!(first, "first");
8684 assert_eq!(second, 29);
8685 ctx.ensure_history_consumed().expect("ordered history");
8686
8687 let reordered = workflow_context(vec![history_event(
8688 "VersionMarkerRecorded",
8689 json!({
8690 "sequence": 1,
8691 "change_id": "before-side-effect",
8692 "version": 1,
8693 "min_supported": 1,
8694 "max_supported": 1,
8695 }),
8696 )]);
8697 let error = reordered
8698 .side_effect(|| "new".to_string())
8699 .expect_err("command reordering must fail");
8700 assert!(matches!(
8701 error,
8702 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8703 if reason == "recorded_command_mismatch"
8704 ));
8705 }
8706
8707 #[test]
8708 fn version_markers_replay_across_upgrades_and_do_not_duplicate() {
8709 let ctx = workflow_context(Vec::new());
8710 assert_eq!(ctx.get_version("checkout-v2", 1, 2).expect("version"), 2);
8711 assert_eq!(ctx.get_version("checkout-v2", 1, 3).expect("cached"), 2);
8712 assert!(ctx.patched("new-search").expect("patch"));
8713 ctx.deprecate_patch("new-search").expect("deprecate patch");
8714 let commands = ctx.take_commands().expect("commands");
8715 assert_eq!(commands.len(), 2);
8716 assert_eq!(commands[0]["type"], "record_version_marker");
8717 assert_eq!(commands[0]["version"], 2);
8718 assert_eq!(commands[1]["change_id"], "new-search");
8719
8720 let replay = workflow_context(vec![history_event(
8721 "VersionMarkerRecorded",
8722 json!({
8723 "sequence": 1,
8724 "change_id": "checkout-v2",
8725 "version": 2,
8726 "min_supported": 1,
8727 "max_supported": 2,
8728 }),
8729 )]);
8730 assert_eq!(replay.get_version("checkout-v2", 1, 4).expect("upgrade"), 2);
8731 assert_eq!(replay.get_version("checkout-v2", 2, 5).expect("repeat"), 2);
8732 assert!(replay.take_commands().expect("commands").is_empty());
8733 replay.ensure_history_consumed().expect("history consumed");
8734 }
8735
8736 #[test]
8737 fn version_markers_reject_incompatible_or_malformed_history() {
8738 let incompatible = workflow_context(vec![history_event(
8739 "VersionMarkerRecorded",
8740 json!({
8741 "sequence": 1,
8742 "change_id": "checkout-v2",
8743 "version": 1,
8744 "min_supported": 1,
8745 "max_supported": 2,
8746 }),
8747 )]);
8748 let error = incompatible
8749 .get_version("checkout-v2", 2, 3)
8750 .expect_err("old version is unsupported");
8751 assert!(matches!(
8752 error,
8753 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8754 if reason == "version_marker_incompatible_range"
8755 ));
8756
8757 for (history, reason) in [
8758 (
8759 vec![history_event("SideEffectRecorded", json!({"sequence": 1}))],
8760 "side_effect_result_missing",
8761 ),
8762 (
8763 vec![history_event(
8764 "SideEffectRecorded",
8765 json!({
8766 "sequence": 1,
8767 "result": {"codec": "avro", "blob": "not-base64"},
8768 }),
8769 )],
8770 "side_effect_payload_incompatible",
8771 ),
8772 (
8773 vec![history_event(
8774 "SideEffectRecorded",
8775 json!({"sequence": 1, "result": {"unwrapped": true}}),
8776 )],
8777 "side_effect_payload_malformed",
8778 ),
8779 (
8780 vec![history_event(
8781 "VersionMarkerRecorded",
8782 json!({
8783 "sequence": 1,
8784 "change_id": "change",
8785 "version": 1,
8786 "min_supported": 2,
8787 "max_supported": 1,
8788 }),
8789 )],
8790 "version_marker_history_range_invalid",
8791 ),
8792 ] {
8793 let error = WorkflowState::new(
8794 history,
8795 "rust-workers".to_string(),
8796 JSON_CODEC.to_string(),
8797 None,
8798 )
8799 .expect_err("malformed history must fail");
8800 assert!(matches!(
8801 error,
8802 Error::NonDeterministicReplay(ReplayFailure { reason: actual, .. })
8803 if actual == reason
8804 ));
8805 }
8806 }
8807
8808 #[test]
8809 fn duplicate_side_effects_and_version_markers_are_rejected() {
8810 let duplicate_side_effect = WorkflowState::new(
8811 vec![
8812 history_event(
8813 "SideEffectRecorded",
8814 json!({"sequence": 1, "result": {"codec": "json", "blob": "1"}}),
8815 ),
8816 history_event(
8817 "SideEffectRecorded",
8818 json!({"sequence": 1, "result": {"codec": "json", "blob": "2"}}),
8819 ),
8820 ],
8821 "rust-workers".to_string(),
8822 JSON_CODEC.to_string(),
8823 None,
8824 )
8825 .expect_err("duplicate side effect");
8826 assert!(matches!(
8827 duplicate_side_effect,
8828 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8829 if reason == "duplicate_side_effect_record"
8830 ));
8831
8832 let marker = |sequence| {
8833 history_event(
8834 "VersionMarkerRecorded",
8835 json!({
8836 "sequence": sequence,
8837 "change_id": "same-change",
8838 "version": 1,
8839 "min_supported": 1,
8840 "max_supported": 1,
8841 }),
8842 )
8843 };
8844 let duplicate_marker = WorkflowState::new(
8845 vec![marker(1), marker(3)],
8846 "rust-workers".to_string(),
8847 JSON_CODEC.to_string(),
8848 None,
8849 )
8850 .expect_err("duplicate marker");
8851 assert!(matches!(
8852 duplicate_marker,
8853 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8854 if reason == "duplicate_version_marker"
8855 ));
8856 }
8857
8858 #[test]
8859 fn cold_worker_replay_does_not_repeat_committed_side_effects_or_markers() {
8860 fn worker(calls: Arc<AtomicUsize>) -> Worker {
8861 let client = Client::new("http://127.0.0.1:8080").expect("client");
8862 let mut worker = Worker::new(client, "rust-workers");
8863 worker.register_workflow("rust.side-effect-version", move |ctx, _input| {
8864 let calls = Arc::clone(&calls);
8865 async move {
8866 let captured = ctx.side_effect(|| {
8867 calls.fetch_add(1, Ordering::SeqCst);
8868 "captured-once".to_string()
8869 })?;
8870 let version = ctx.get_version("cold-restart", 1, 2)?;
8871 Ok(json!({"captured": captured, "version": version}))
8872 }
8873 });
8874 worker
8875 }
8876
8877 fn task(history_events: Vec<HistoryEvent>) -> WorkflowTask {
8878 WorkflowTask {
8879 task_id: "wft-side-effect-version".to_string(),
8880 workflow_id: Some("wf-side-effect-version".to_string()),
8881 run_id: Some("run-side-effect-version".to_string()),
8882 workflow_type: "rust.side-effect-version".to_string(),
8883 payload_codec: JSON_CODEC.to_string(),
8884 arguments: Some(encode_value_envelope(&json!([]), JSON_CODEC).expect("arguments")),
8885 history_events,
8886 total_history_events: None,
8887 history_size_bytes: None,
8888 continue_as_new_recommended: None,
8889 history_budget_pressure: None,
8890 next_history_page_token: None,
8891 workflow_task_attempt: 1,
8892 workflow_signal_id: None,
8893 signal_name: None,
8894 signal_arguments: None,
8895 workflow_update_id: None,
8896 update_name: None,
8897 lease_owner: Some("rust-worker".to_string()),
8898 }
8899 }
8900
8901 let calls = Arc::new(AtomicUsize::new(0));
8902 let initial = worker(Arc::clone(&calls))
8903 .execute_workflow_task(task(Vec::new()))
8904 .expect("initial execution");
8905 assert_eq!(
8906 initial
8907 .iter()
8908 .map(|command| &command["type"])
8909 .collect::<Vec<_>>(),
8910 vec![
8911 "record_side_effect",
8912 "record_version_marker",
8913 "complete_workflow"
8914 ]
8915 );
8916 assert_eq!(calls.load(Ordering::SeqCst), 1);
8917
8918 let restarted = worker(Arc::clone(&calls));
8919 let replayed = restarted
8920 .execute_workflow_task(task(vec![
8921 history_event(
8922 "SideEffectRecorded",
8923 json!({"sequence": 1, "result": initial[0]["result"].clone()}),
8924 ),
8925 history_event(
8926 "VersionMarkerRecorded",
8927 json!({
8928 "sequence": 2,
8929 "change_id": "cold-restart",
8930 "version": 2,
8931 "min_supported": 1,
8932 "max_supported": 2,
8933 }),
8934 ),
8935 ]))
8936 .expect("cold replay");
8937 assert_eq!(replayed.len(), 1);
8938 assert_eq!(replayed[0]["type"], "complete_workflow");
8939 assert_eq!(calls.load(Ordering::SeqCst), 1);
8940 }
8941
8942 #[test]
8943 fn side_effect_replay_rejects_changed_rust_value_type() {
8944 let result = encode_value_envelope(&json!({"value": 42}), JSON_CODEC).expect("result");
8945 let ctx = workflow_context(vec![history_event(
8946 "SideEffectRecorded",
8947 json!({"sequence": 1, "result": result}),
8948 )]);
8949 let error = ctx
8950 .side_effect::<Vec<String>, _>(|| panic!("callback must not run"))
8951 .expect_err("changed type must fail replay");
8952 assert!(matches!(
8953 error,
8954 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8955 if reason == "side_effect_type_mismatch"
8956 ));
8957 }
8958
8959 fn completed_retry_activity_history() -> Vec<HistoryEvent> {
8960 vec![
8961 history_event(
8962 "ActivityScheduled",
8963 json!({
8964 "sequence": 1,
8965 "activity_type": "flaky",
8966 "activity_execution_id": "act-1",
8967 "activity": {
8968 "id": "act-1",
8969 "sequence": 1,
8970 "type": "flaky",
8971 "queue": "critical-activities",
8972 "execution_mode": null,
8973 "retry_policy": {
8974 "snapshot_version": 1,
8975 "max_attempts": 3,
8976 "backoff_seconds": [2, 4],
8977 "start_to_close_timeout": 30,
8978 "schedule_to_start_timeout": 5,
8979 "schedule_to_close_timeout": 90,
8980 "heartbeat_timeout": 10,
8981 "non_retryable_error_types": ["PermanentError"]
8982 }
8983 }
8984 }),
8985 ),
8986 history_event(
8987 "ActivityStarted",
8988 json!({
8989 "sequence": 1,
8990 "activity_type": "flaky",
8991 "activity_execution_id": "act-1",
8992 "activity_attempt_id": "attempt-1",
8993 "attempt_number": 1
8994 }),
8995 ),
8996 history_event(
8997 "ActivityRetryScheduled",
8998 json!({
8999 "sequence": 1,
9000 "activity_type": "flaky",
9001 "activity_execution_id": "act-1",
9002 "activity_attempt_id": "attempt-1",
9003 "attempt_number": 1,
9004 "retry_after_attempt": 1,
9005 "retry_backoff_seconds": 2,
9006 "failure_category": "activity",
9007 "exception_type": "TransientError"
9008 }),
9009 ),
9010 history_event(
9011 "ActivityStarted",
9012 json!({
9013 "sequence": 1,
9014 "activity_type": "flaky",
9015 "activity_execution_id": "act-1",
9016 "activity_attempt_id": "attempt-2",
9017 "attempt_number": 2
9018 }),
9019 ),
9020 history_event(
9021 "ActivityCompleted",
9022 json!({
9023 "sequence": 1,
9024 "activity_type": "flaky",
9025 "activity_execution_id": "act-1",
9026 "activity_attempt_id": "attempt-2",
9027 "attempt_number": 2,
9028 "payload_codec": "json",
9029 "result": {"codec": "json", "blob": "{\"status\":\"recovered\"}"}
9030 }),
9031 ),
9032 ]
9033 }
9034
9035 fn retry_activity_options() -> ActivityOptions {
9036 ActivityOptions::new()
9037 .task_queue("critical-activities")
9038 .retry_policy(
9039 ActivityRetryPolicy::new(3)
9040 .backoff_intervals([Duration::from_secs(2), Duration::from_secs(4)])
9041 .non_retryable_error_type("PermanentError"),
9042 )
9043 .start_to_close_timeout(Duration::from_secs(30))
9044 .schedule_to_start_timeout(Duration::from_secs(5))
9045 .schedule_to_close_timeout(Duration::from_secs(90))
9046 .heartbeat_timeout(Duration::from_secs(10))
9047 }
9048
9049 #[test]
9050 fn fixed_avro_value_round_trips_json_values() {
9051 let value = json!({"greeting": "hello", "count": 3, "ok": true});
9052 let envelope = PayloadEnvelope::avro(&value).expect("encode");
9053 assert_eq!(envelope.codec, DEFAULT_CODEC);
9054 assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
9055 }
9056
9057 #[tokio::test]
9058 async fn typed_worker_surfaces_preserve_bytes_and_map_list_identity() {
9059 let client = Client::new("http://127.0.0.1:8080").expect("client");
9060 let mut worker = Worker::new(client, "rust-workers");
9061 worker.register_workflow_avro_value("typed.echo", |_ctx, input| async move { Ok(input) });
9062 worker
9063 .register_activity_avro_value("typed.activity", |_ctx, input| async move { Ok(input) });
9064 worker.register_query_avro_value("typed.echo", "inspect", |_ctx, input| async move {
9065 Ok(input)
9066 });
9067 worker.register_update_avro_value("typed.echo", "replace", |_ctx, input| async move {
9068 Ok(input)
9069 });
9070 worker.register_workflow_avro_value("typed.signal", |ctx, _input| async move {
9071 Ok(AvroValue::Array(
9072 ctx.wait_signal_avro_value("changed").await?,
9073 ))
9074 });
9075
9076 let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
9077 let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed envelope");
9078
9079 let mut workflow = workflow_task("typed.echo", Vec::new(), DEFAULT_CODEC);
9080 workflow.arguments = Some(envelope.clone());
9081 let commands = worker
9082 .execute_workflow_task(workflow)
9083 .expect("typed workflow task");
9084 assert_eq!(commands[0]["type"], "complete_workflow");
9085 assert_eq!(
9086 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
9087 .expect("typed workflow result"),
9088 arguments
9089 );
9090
9091 let activity = ActivityTask {
9092 task_id: "activity-typed".to_string(),
9093 activity_attempt_id: Some("attempt-typed".to_string()),
9094 attempt_id: None,
9095 activity_type: "typed.activity".to_string(),
9096 payload_codec: DEFAULT_CODEC.to_string(),
9097 arguments: Some(envelope.clone()),
9098 attempt_number: 1,
9099 lease_owner: Some("rust-worker".to_string()),
9100 };
9101 assert_eq!(
9102 worker
9103 .execute_activity_task(activity)
9104 .await
9105 .expect("typed activity result"),
9106 arguments
9107 );
9108
9109 let query = QueryTask {
9110 query_task_id: "query-typed".to_string(),
9111 query_task_attempt: 1,
9112 lease_owner: Some("rust-worker".to_string()),
9113 workflow_id: Some("typed-1".to_string()),
9114 run_id: Some("run-typed".to_string()),
9115 workflow_type: "typed.echo".to_string(),
9116 query_name: "inspect".to_string(),
9117 payload_codec: DEFAULT_CODEC.to_string(),
9118 workflow_arguments: Some(
9119 encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
9120 .expect("workflow input"),
9121 ),
9122 query_arguments: Some(envelope.clone()),
9123 history_events: Vec::new(),
9124 history_export: None,
9125 run_status: Some("running".to_string()),
9126 };
9127 assert_eq!(
9128 worker
9129 .execute_query_task(query)
9130 .await
9131 .expect("typed query result"),
9132 arguments
9133 );
9134
9135 let mut update = workflow_task(
9136 "typed.echo",
9137 vec![history_event(
9138 "UpdateAccepted",
9139 json!({
9140 "update_id": "update-typed",
9141 "update_name": "replace",
9142 "arguments": envelope.clone(),
9143 }),
9144 )],
9145 DEFAULT_CODEC,
9146 );
9147 update.workflow_update_id = Some("update-typed".to_string());
9148 update.update_name = Some("replace".to_string());
9149 let commands = worker
9150 .execute_workflow_task(update)
9151 .expect("typed update task");
9152 assert_eq!(commands[0]["type"], "complete_update");
9153 assert_eq!(
9154 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
9155 .expect("typed update result"),
9156 arguments
9157 );
9158
9159 let mut signal = workflow_task(
9160 "typed.signal",
9161 vec![history_event(
9162 "SignalReceived",
9163 json!({
9164 "signal_id": "signal-typed",
9165 "signal_name": "changed",
9166 "arguments": envelope.clone(),
9167 }),
9168 )],
9169 DEFAULT_CODEC,
9170 );
9171 signal.workflow_signal_id = Some("signal-typed".to_string());
9172 signal.signal_name = Some("changed".to_string());
9173 signal.signal_arguments = Some(envelope);
9174 let commands = worker
9175 .execute_workflow_task(signal)
9176 .expect("typed signal resume");
9177 assert_eq!(
9178 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
9179 .expect("typed signal result"),
9180 arguments
9181 );
9182 }
9183
9184 #[tokio::test]
9185 async fn typed_helpers_never_parse_json_inspection_projection() {
9186 let collision_values = projection_collision_probe();
9187 let expected = AvroValue::Array(collision_values.clone());
9188 let envelope = encode_typed_envelope(&expected, DEFAULT_CODEC).expect("collision envelope");
9189
9190 let activity_context = workflow_context_with_codec(
9191 vec![history_event(
9192 "ActivityCompleted",
9193 json!({
9194 "sequence": 1,
9195 "activity_type": "collision.activity",
9196 "payload_codec": DEFAULT_CODEC,
9197 "result": envelope.clone(),
9198 }),
9199 )],
9200 DEFAULT_CODEC,
9201 );
9202 assert_eq!(
9203 activity_context
9204 .activity_avro_value("collision.activity", AvroValue::Array(Vec::new()))
9205 .await
9206 .expect("typed activity collision result"),
9207 expected
9208 );
9209
9210 let signal_context = workflow_context_with_codec(
9211 vec![
9212 history_event(
9213 "SignalWaitOpened",
9214 json!({"sequence": 1, "signal_name": "collision"}),
9215 ),
9216 history_event(
9217 "SignalApplied",
9218 json!({
9219 "sequence": 1,
9220 "signal_name": "collision",
9221 "payload_codec": DEFAULT_CODEC,
9222 "value": envelope.clone(),
9223 }),
9224 ),
9225 ],
9226 DEFAULT_CODEC,
9227 );
9228 assert_eq!(
9229 signal_context
9230 .wait_signal_avro_value("collision")
9231 .await
9232 .expect("typed signal collision arguments"),
9233 collision_values
9234 );
9235
9236 let child_context = workflow_context_with_codec(
9237 vec![
9238 history_event(
9239 "ChildWorkflowScheduled",
9240 json!({
9241 "sequence": 1,
9242 "child_workflow_instance_id": "collision-child",
9243 "child_workflow_run_id": "collision-run",
9244 "child_workflow_type": "collision.child",
9245 }),
9246 ),
9247 history_event(
9248 "ChildRunCompleted",
9249 json!({
9250 "sequence": 1,
9251 "child_workflow_instance_id": "collision-child",
9252 "child_workflow_run_id": "collision-run",
9253 "child_workflow_type": "collision.child",
9254 "payload_codec": DEFAULT_CODEC,
9255 "result": envelope,
9256 }),
9257 ),
9258 ],
9259 DEFAULT_CODEC,
9260 );
9261 let child = child_context
9262 .start_child_workflow_avro_value(
9263 "collision.child",
9264 ChildWorkflowOptions::new("collision-workers"),
9265 AvroValue::Array(Vec::new()),
9266 )
9267 .await
9268 .expect("typed child collision result");
9269 assert_eq!(child.result, expected);
9270 }
9271
9272 #[tokio::test]
9273 async fn replayed_typed_query_keeps_lossless_workflow_and_query_inputs() {
9274 let client = Client::new("http://127.0.0.1:8080").expect("client");
9275 let mut worker = Worker::new(client, "rust-workers");
9276 worker.register_replayed_workflow_avro_value(
9277 "typed.replayed",
9278 || (),
9279 |_ctx, input, _state| async move { Ok(input) },
9280 );
9281 worker.register_replayed_query_avro_value::<(), _, _>(
9282 "typed.replayed",
9283 "inspect",
9284 |ctx, _state, args| async move {
9285 let mut signals = ctx.signals_avro_value("collision");
9286 let signal = signals
9287 .pop()
9288 .map(AvroValue::Array)
9289 .unwrap_or_else(|| AvroValue::Array(Vec::new()));
9290 Ok(AvroValue::Array(vec![
9291 ctx.workflow_input_avro_value().clone(),
9292 signal,
9293 args,
9294 ]))
9295 },
9296 );
9297 let arguments = AvroValue::Array(projection_collision_probe());
9298 let signal_arguments =
9299 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed query signal arguments");
9300 let task = QueryTask {
9301 query_task_id: "query-typed-replay".to_string(),
9302 query_task_attempt: 1,
9303 lease_owner: Some("rust-worker".to_string()),
9304 workflow_id: Some("typed-replay".to_string()),
9305 run_id: Some("run-typed-replay".to_string()),
9306 workflow_type: "typed.replayed".to_string(),
9307 query_name: "inspect".to_string(),
9308 payload_codec: DEFAULT_CODEC.to_string(),
9309 workflow_arguments: Some(
9310 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("workflow arguments"),
9311 ),
9312 query_arguments: Some(
9313 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("query arguments"),
9314 ),
9315 history_events: vec![history_event(
9316 "SignalReceived",
9317 json!({
9318 "signal_id": "collision-signal",
9319 "signal_name": "collision",
9320 "workflow_sequence": 1,
9321 "payload_codec": DEFAULT_CODEC,
9322 "arguments": signal_arguments,
9323 }),
9324 )],
9325 history_export: None,
9326 run_status: Some("completed".to_string()),
9327 };
9328
9329 assert_eq!(
9330 worker
9331 .execute_query_task(task)
9332 .await
9333 .expect("typed replay query"),
9334 AvroValue::Array(vec![arguments.clone(), arguments.clone(), arguments])
9335 );
9336 }
9337
9338 #[test]
9339 fn public_avro_adapter_rejects_non_string_map_keys_before_json_conversion() {
9340 let value = BTreeMap::from([(1_i32, "integer key")]);
9341 let error = PayloadEnvelope::avro(&value)
9342 .expect_err("integer map keys must fail")
9343 .to_string();
9344
9345 assert!(error.contains("invalid_map_key"));
9346 }
9347
9348 #[test]
9349 fn json_codec_remains_plain_json() {
9350 let value = json!({"greeting": "hello", "count": 3, "ok": true});
9351 let envelope = PayloadEnvelope::json(&value).expect("encode");
9352
9353 assert_eq!(envelope.codec, JSON_CODEC);
9354 assert_eq!(envelope.blob, serde_json::to_string(&value).expect("json"));
9355 assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
9356 }
9357
9358 #[test]
9359 fn prerelease_avro_payload_without_single_object_frame_is_rejected() {
9360 let envelope = PayloadEnvelope {
9361 codec: DEFAULT_CODEC.to_string(),
9362 blob: BASE64.encode([0x01]),
9363 };
9364
9365 let error = decode_payload::<Value>(&envelope).expect_err("prerelease payload must fail");
9366 assert!(error.to_string().contains("invalid_payload_framing"));
9367 }
9368
9369 #[test]
9370 fn workflow_context_schedules_activity_until_completion_is_in_history() {
9371 let ctx = WorkflowContext {
9372 state: Arc::new(Mutex::new(
9373 WorkflowState::new_with_identity(
9374 Vec::new(),
9375 Some("wf-parent".to_string()),
9376 Some("run-parent".to_string()),
9377 "rust-workers".to_string(),
9378 DEFAULT_CODEC.to_string(),
9379 None,
9380 )
9381 .expect("workflow state"),
9382 )),
9383 };
9384
9385 let mut call = Box::pin(ctx.activity("hello.activity", json!(["Ada"])));
9386 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9387 assert!(matches!(
9388 call.as_mut().poll(&mut task_context),
9389 Poll::Pending
9390 ));
9391
9392 let commands = ctx.take_commands().expect("commands");
9393 assert_eq!(commands[0]["type"], "schedule_activity");
9394 assert_eq!(commands[0]["activity_type"], "hello.activity");
9395 }
9396
9397 #[test]
9398 fn activity_options_encode_retry_policy_queue_and_every_timeout() {
9399 let ctx = workflow_context(Vec::new());
9400 let options = ActivityOptions::new()
9401 .task_queue("payments")
9402 .retry_policy(
9403 ActivityRetryPolicy::new(4)
9404 .exponential_backoff(Duration::from_secs(1), 3, Some(Duration::from_secs(10)))
9405 .non_retryable_error_type("ValidationError"),
9406 )
9407 .start_to_close_timeout(Duration::from_secs(120))
9408 .schedule_to_start_timeout(Duration::from_secs(10))
9409 .schedule_to_close_timeout(Duration::from_secs(300))
9410 .heartbeat_timeout(Duration::from_secs(15));
9411 let mut call = Box::pin(ctx.activity_with_options(
9412 "charge-card",
9413 options,
9414 json!([{"order_id": "o-1"}]),
9415 ));
9416 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9417
9418 assert!(matches!(
9419 call.as_mut().poll(&mut task_context),
9420 Poll::Pending
9421 ));
9422 assert!(matches!(
9423 call.as_mut().poll(&mut task_context),
9424 Poll::Pending
9425 ));
9426
9427 let commands = ctx.take_commands().expect("activity command");
9428 assert_eq!(commands.len(), 1, "one future emits one logical schedule");
9429 assert_eq!(commands[0]["queue"], "payments");
9430 assert_eq!(
9431 commands[0]["retry_policy"],
9432 json!({
9433 "max_attempts": 4,
9434 "backoff_seconds": [1, 3, 9],
9435 "non_retryable_error_types": ["ValidationError"],
9436 })
9437 );
9438 assert_eq!(commands[0]["start_to_close_timeout"], 120);
9439 assert_eq!(commands[0]["schedule_to_start_timeout"], 10);
9440 assert_eq!(commands[0]["schedule_to_close_timeout"], 300);
9441 assert_eq!(commands[0]["heartbeat_timeout"], 15);
9442 }
9443
9444 #[test]
9445 fn activity_options_encode_explicit_and_rounded_backoff_intervals() {
9446 let ctx = workflow_context(Vec::new());
9447 let options = ActivityOptions::new().retry_policy(
9448 ActivityRetryPolicy::new(3)
9449 .backoff_intervals([Duration::from_millis(1), Duration::from_millis(1_001)]),
9450 );
9451 let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
9452 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9453
9454 assert!(matches!(
9455 call.as_mut().poll(&mut task_context),
9456 Poll::Pending
9457 ));
9458 assert_eq!(
9459 ctx.take_commands().expect("command")[0]["retry_policy"]["backoff_seconds"],
9460 json!([1, 2])
9461 );
9462 }
9463
9464 #[test]
9465 fn invalid_activity_options_return_typed_errors_before_emitting_commands() {
9466 let cases = [
9467 (
9468 ActivityOptions::new().task_queue(" "),
9469 ActivityOptionsErrorKind::EmptyTaskQueue,
9470 ),
9471 (
9472 ActivityOptions::new().retry_policy(ActivityRetryPolicy::default()),
9473 ActivityOptionsErrorKind::EmptyRetryPolicy,
9474 ),
9475 (
9476 ActivityOptions::new().retry_policy(ActivityRetryPolicy::new(0)),
9477 ActivityOptionsErrorKind::InvalidMaxAttempts,
9478 ),
9479 (
9480 ActivityOptions::new().retry_policy(ActivityRetryPolicy {
9481 max_attempts: None,
9482 backoff: Some(ActivityBackoff::Explicit(vec![Duration::from_secs(1)])),
9483 non_retryable_error_types: Vec::new(),
9484 }),
9485 ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
9486 ),
9487 (
9488 ActivityOptions::new().retry_policy(
9489 ActivityRetryPolicy::new(2)
9490 .backoff_intervals([Duration::from_secs(1), Duration::from_secs(2)]),
9491 ),
9492 ActivityOptionsErrorKind::TooManyBackoffIntervals,
9493 ),
9494 (
9495 ActivityOptions::new().retry_policy(
9496 ActivityRetryPolicy::new(2).exponential_backoff(
9497 Duration::from_secs(1),
9498 0,
9499 None,
9500 ),
9501 ),
9502 ActivityOptionsErrorKind::InvalidBackoffCoefficient,
9503 ),
9504 (
9505 ActivityOptions::new()
9506 .retry_policy(ActivityRetryPolicy::new(2).non_retryable_error_type(" ")),
9507 ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
9508 ),
9509 (
9510 ActivityOptions::new().retry_policy(
9511 ActivityRetryPolicy::new(10_002).exponential_backoff(
9512 Duration::from_secs(1),
9513 1,
9514 None,
9515 ),
9516 ),
9517 ActivityOptionsErrorKind::BackoffGenerationTooLarge,
9518 ),
9519 (
9520 ActivityOptions::new().retry_policy(
9521 ActivityRetryPolicy::new(2)
9522 .backoff_intervals([Duration::from_secs(i64::MAX as u64 + 1)]),
9523 ),
9524 ActivityOptionsErrorKind::BackoffOverflow,
9525 ),
9526 ];
9527
9528 for (options, expected_kind) in cases {
9529 let ctx = workflow_context(Vec::new());
9530 let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
9531 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9532 let Poll::Ready(Err(Error::InvalidActivityOptions(error))) =
9533 call.as_mut().poll(&mut task_context)
9534 else {
9535 panic!("expected typed activity validation error");
9536 };
9537 assert_eq!(error.kind, expected_kind);
9538 assert!(ctx.take_commands().expect("commands").is_empty());
9539 }
9540 }
9541
9542 #[test]
9543 fn activity_options_validate_positive_and_ordered_timeouts() {
9544 let zero_timeout_cases = [
9545 ActivityOptions::new().start_to_close_timeout(Duration::ZERO),
9546 ActivityOptions::new().schedule_to_start_timeout(Duration::ZERO),
9547 ActivityOptions::new().schedule_to_close_timeout(Duration::ZERO),
9548 ActivityOptions::new().heartbeat_timeout(Duration::ZERO),
9549 ];
9550 for options in zero_timeout_cases {
9551 assert_eq!(
9552 options.validate().expect_err("zero timeout").kind,
9553 ActivityOptionsErrorKind::TimeoutNotPositive
9554 );
9555 }
9556
9557 let ordering_cases = [
9558 ActivityOptions::new()
9559 .heartbeat_timeout(Duration::from_secs(11))
9560 .start_to_close_timeout(Duration::from_secs(10)),
9561 ActivityOptions::new()
9562 .start_to_close_timeout(Duration::from_secs(31))
9563 .schedule_to_close_timeout(Duration::from_secs(30)),
9564 ActivityOptions::new()
9565 .schedule_to_start_timeout(Duration::from_secs(31))
9566 .schedule_to_close_timeout(Duration::from_secs(30)),
9567 ];
9568 for options in ordering_cases {
9569 assert_eq!(
9570 options.validate().expect_err("timeout order").kind,
9571 ActivityOptionsErrorKind::TimeoutOrder
9572 );
9573 }
9574
9575 assert_eq!(
9576 ActivityOptions::new()
9577 .start_to_close_timeout(Duration::from_secs(i64::MAX as u64 + 1))
9578 .validate()
9579 .expect_err("protocol integer overflow")
9580 .kind,
9581 ActivityOptionsErrorKind::TimeoutOverflow
9582 );
9583 }
9584
9585 #[test]
9586 fn replayed_activity_retry_history_completes_without_duplicate_schedule() {
9587 let ctx = workflow_context(completed_retry_activity_history());
9588 let mut call =
9589 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
9590 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9591
9592 assert!(matches!(
9593 call.as_mut().poll(&mut task_context),
9594 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
9595 ));
9596 assert!(ctx.take_commands().expect("commands").is_empty());
9597 ctx.ensure_history_consumed().expect("history consumed");
9598 }
9599
9600 #[test]
9601 fn duplicate_non_retryable_types_use_one_command_and_replay_representation() {
9602 let mut options = retry_activity_options();
9603 options
9604 .retry_policy
9605 .as_mut()
9606 .expect("retry policy")
9607 .non_retryable_error_types
9608 .extend([" PermanentError ".to_string(), "PermanentError".to_string()]);
9609
9610 let new_ctx = workflow_context(Vec::new());
9611 let mut new_call =
9612 Box::pin(new_ctx.activity_with_options("flaky", options.clone(), json!([])));
9613 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9614 assert!(matches!(
9615 new_call.as_mut().poll(&mut task_context),
9616 Poll::Pending
9617 ));
9618 let commands = new_ctx.take_commands().expect("commands");
9619 assert_eq!(commands.len(), 1);
9620 assert_eq!(
9621 commands[0]["retry_policy"]["non_retryable_error_types"],
9622 json!(["PermanentError"])
9623 );
9624
9625 let replay_ctx = workflow_context(completed_retry_activity_history());
9626 let mut replay_call =
9627 Box::pin(replay_ctx.activity_with_options("flaky", options, json!([])));
9628 assert!(matches!(
9629 replay_call.as_mut().poll(&mut task_context),
9630 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
9631 ));
9632 assert!(replay_ctx.take_commands().expect("commands").is_empty());
9633 replay_ctx
9634 .ensure_history_consumed()
9635 .expect("history consumed");
9636 }
9637
9638 #[test]
9639 fn replayed_intermediate_retry_remains_pending_across_restarts() {
9640 let history = completed_retry_activity_history()
9641 .into_iter()
9642 .take(3)
9643 .collect::<Vec<_>>();
9644
9645 for _restart in 0..2 {
9646 let ctx = workflow_context(history.clone());
9647 let mut call =
9648 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
9649 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9650 assert!(matches!(
9651 call.as_mut().poll(&mut task_context),
9652 Poll::Pending
9653 ));
9654 assert!(ctx.take_commands().expect("commands").is_empty());
9655 }
9656 }
9657
9658 #[test]
9659 fn replayed_activity_rejects_changed_queue_retry_and_every_timeout_field() {
9660 let mut changed_queue = retry_activity_options();
9661 changed_queue.task_queue = Some("different-queue".to_string());
9662
9663 let mut changed_max_attempts = retry_activity_options();
9664 let retry_policy = changed_max_attempts
9665 .retry_policy
9666 .as_mut()
9667 .expect("retry policy");
9668 retry_policy.max_attempts = Some(4);
9669
9670 let mut changed_backoff = retry_activity_options();
9671 let retry_policy = changed_backoff.retry_policy.as_mut().expect("retry policy");
9672 retry_policy.backoff = Some(ActivityBackoff::Explicit(vec![
9673 Duration::from_secs(3),
9674 Duration::from_secs(4),
9675 ]));
9676
9677 let mut changed_non_retryable_types = retry_activity_options();
9678 let retry_policy = changed_non_retryable_types
9679 .retry_policy
9680 .as_mut()
9681 .expect("retry policy");
9682 retry_policy.non_retryable_error_types = vec!["AnotherPermanentError".to_string()];
9683
9684 let mut changed_start_to_close = retry_activity_options();
9685 changed_start_to_close.start_to_close_timeout = Some(Duration::from_secs(31));
9686 let mut changed_schedule_to_start = retry_activity_options();
9687 changed_schedule_to_start.schedule_to_start_timeout = Some(Duration::from_secs(6));
9688 let mut changed_schedule_to_close = retry_activity_options();
9689 changed_schedule_to_close.schedule_to_close_timeout = Some(Duration::from_secs(91));
9690 let mut changed_heartbeat = retry_activity_options();
9691 changed_heartbeat.heartbeat_timeout = Some(Duration::from_secs(11));
9692
9693 let cases = [
9694 (changed_queue, "activity_task_queue_mismatch"),
9695 (changed_max_attempts, "activity_retry_policy_mismatch"),
9696 (changed_backoff, "activity_retry_policy_mismatch"),
9697 (
9698 changed_non_retryable_types,
9699 "activity_retry_policy_mismatch",
9700 ),
9701 (changed_start_to_close, "activity_retry_policy_mismatch"),
9702 (changed_schedule_to_start, "activity_retry_policy_mismatch"),
9703 (changed_schedule_to_close, "activity_retry_policy_mismatch"),
9704 (changed_heartbeat, "activity_retry_policy_mismatch"),
9705 ];
9706
9707 for (options, expected_reason) in cases {
9708 let ctx = workflow_context(completed_retry_activity_history());
9709 let mut call = Box::pin(ctx.activity_with_options("flaky", options, json!([])));
9710 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9711 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
9712 call.as_mut().poll(&mut task_context)
9713 else {
9714 panic!("changed activity options must fail replay");
9715 };
9716 assert_eq!(failure.reason, expected_reason);
9717 assert_eq!(failure.sequence, Some(1));
9718 assert!(ctx.take_commands().expect("commands").is_empty());
9719 }
9720 }
9721
9722 #[test]
9723 fn replayed_activity_rejects_changed_execution_mode_and_snapshot_version() {
9724 let cases = [
9725 (
9726 "execution_mode",
9727 json!("local"),
9728 "activity_execution_mode_mismatch",
9729 ),
9730 (
9731 "snapshot_version",
9732 json!(2),
9733 "activity_retry_policy_mismatch",
9734 ),
9735 ];
9736
9737 for (field, value, expected_reason) in cases {
9738 let mut history = completed_retry_activity_history();
9739 let activity = history[0].payload["activity"]
9740 .as_object_mut()
9741 .expect("activity snapshot");
9742 if field == "execution_mode" {
9743 activity.insert(field.to_string(), value);
9744 } else {
9745 activity["retry_policy"]
9746 .as_object_mut()
9747 .expect("retry snapshot")
9748 .insert(field.to_string(), value);
9749 }
9750
9751 let ctx = workflow_context(history);
9752 let mut call =
9753 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
9754 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9755 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
9756 call.as_mut().poll(&mut task_context)
9757 else {
9758 panic!("changed {field} must fail replay");
9759 };
9760 assert_eq!(failure.reason, expected_reason);
9761 assert_eq!(failure.sequence, Some(1));
9762 assert!(ctx.take_commands().expect("commands").is_empty());
9763 }
9764 }
9765
9766 #[test]
9767 fn replayed_legacy_activity_treats_missing_option_snapshot_as_unknown() {
9768 let mut history = completed_retry_activity_history();
9769 let activity = history[0].payload["activity"]
9770 .as_object_mut()
9771 .expect("activity snapshot");
9772 activity.remove("execution_mode");
9773 activity.remove("retry_policy");
9774
9775 let mut current = retry_activity_options();
9776 current.start_to_close_timeout = Some(Duration::from_secs(45));
9777 current.schedule_to_start_timeout = Some(Duration::from_secs(8));
9778 current.schedule_to_close_timeout = Some(Duration::from_secs(120));
9779 current.heartbeat_timeout = Some(Duration::from_secs(12));
9780
9781 let ctx = workflow_context(history);
9782 let mut call = Box::pin(ctx.activity_with_options("flaky", current, json!([])));
9783 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9784 assert!(matches!(
9785 call.as_mut().poll(&mut task_context),
9786 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
9787 ));
9788 assert!(ctx.take_commands().expect("commands").is_empty());
9789 ctx.ensure_history_consumed().expect("history consumed");
9790 }
9791
9792 #[test]
9793 fn terminal_activity_failed_after_start_returns_typed_failure() {
9794 let history = vec![
9795 history_event(
9796 "ActivityScheduled",
9797 json!({
9798 "sequence": 1,
9799 "activity_type": "flaky",
9800 "activity_execution_id": "act-terminal",
9801 "activity": {
9802 "id": "act-terminal",
9803 "sequence": 1,
9804 "type": "flaky",
9805 "queue": "critical-activities",
9806 "retry_policy": {
9807 "snapshot_version": 1,
9808 "max_attempts": 3,
9809 "backoff_seconds": [2, 4],
9810 "non_retryable_error_types": ["PermanentError"]
9811 }
9812 }
9813 }),
9814 ),
9815 history_event(
9816 "ActivityStarted",
9817 json!({
9818 "sequence": 1,
9819 "activity_type": "flaky",
9820 "activity_execution_id": "act-terminal",
9821 "activity_attempt_id": "attempt-1",
9822 "attempt_number": 1
9823 }),
9824 ),
9825 history_event(
9826 "ActivityFailed",
9827 json!({
9828 "sequence": 1,
9829 "activity_type": "flaky",
9830 "activity_execution_id": "act-terminal",
9831 "activity_attempt_id": "attempt-1",
9832 "attempt_number": 1,
9833 "failure_id": "failure-terminal",
9834 "failure_category": "activity",
9835 "exception_type": "PermanentError",
9836 "message": "cannot retry",
9837 "non_retryable": true
9838 }),
9839 ),
9840 ];
9841 let ctx = workflow_context(history);
9842 let mut call =
9843 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
9844 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9845
9846 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
9847 call.as_mut().poll(&mut task_context)
9848 else {
9849 panic!("terminal ActivityFailed must settle the activity future");
9850 };
9851 assert_eq!(failure.kind, ActivityFailureKind::Failed);
9852 assert_eq!(
9853 failure.activity_execution_id.as_deref(),
9854 Some("act-terminal")
9855 );
9856 assert_eq!(failure.exception_type.as_deref(), Some("PermanentError"));
9857 assert!(failure.non_retryable);
9858 assert!(ctx.take_commands().expect("commands").is_empty());
9859 ctx.ensure_history_consumed().expect("history consumed");
9860 }
9861
9862 #[test]
9863 fn activity_terminal_events_return_machine_readable_failures() {
9864 let cases = [
9865 (
9866 "ActivityFailed",
9867 json!({
9868 "sequence": 1,
9869 "activity_type": "charge-card",
9870 "activity_execution_id": "act-1",
9871 "activity_attempt_id": "attempt-2",
9872 "attempt_number": 2,
9873 "failure_id": "failure-1",
9874 "failure_category": "activity",
9875 "exception_type": "PaymentDeclined",
9876 "exception_class": "payments.PaymentDeclined",
9877 "message": "card declined",
9878 "non_retryable": true
9879 }),
9880 ActivityFailureKind::Failed,
9881 "activity",
9882 ),
9883 (
9884 "ActivityCancelled",
9885 json!({
9886 "sequence": 1,
9887 "activity_type": "charge-card",
9888 "activity_execution_id": "act-1",
9889 "activity_attempt_id": "attempt-1"
9890 }),
9891 ActivityFailureKind::Cancelled,
9892 "cancelled",
9893 ),
9894 ];
9895
9896 for (event_type, payload, expected_kind, expected_reason) in cases {
9897 let ctx = workflow_context(vec![history_event(event_type, payload)]);
9898 let mut call = Box::pin(ctx.activity("charge-card", json!([])));
9899 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9900 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
9901 call.as_mut().poll(&mut task_context)
9902 else {
9903 panic!("expected terminal activity failure");
9904 };
9905 assert_eq!(failure.kind, expected_kind);
9906 assert_eq!(failure.reason, expected_reason);
9907 assert_eq!(failure.activity_execution_id.as_deref(), Some("act-1"));
9908 assert_eq!(failure.activity_type.as_deref(), Some("charge-card"));
9909 }
9910 }
9911
9912 #[test]
9913 fn every_activity_timeout_class_is_typed() {
9914 for timeout_kind in [
9915 "start_to_close",
9916 "schedule_to_start",
9917 "schedule_to_close",
9918 "heartbeat",
9919 ] {
9920 let ctx = workflow_context(vec![history_event(
9921 "ActivityTimedOut",
9922 json!({
9923 "sequence": 1,
9924 "activity_type": "slow",
9925 "activity_execution_id": "act-timeout",
9926 "activity_attempt_id": "attempt-timeout",
9927 "failure_category": "timeout",
9928 "timeout_kind": timeout_kind,
9929 "message": "deadline expired"
9930 }),
9931 )]);
9932 let mut call = Box::pin(ctx.activity("slow", json!([])));
9933 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9934 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
9935 call.as_mut().poll(&mut task_context)
9936 else {
9937 panic!("expected timeout failure");
9938 };
9939 assert_eq!(failure.kind, ActivityFailureKind::TimedOut);
9940 assert_eq!(failure.reason, timeout_kind);
9941 assert_eq!(failure.timeout_kind.as_deref(), Some(timeout_kind));
9942 assert_eq!(failure.failure_category.as_deref(), Some("timeout"));
9943 }
9944 }
9945
9946 #[test]
9947 fn workflow_sleep_emits_one_durable_timer_and_rounds_up() {
9948 let ctx = workflow_context(Vec::new());
9949 let mut sleep = Box::pin(ctx.sleep(Duration::from_millis(1_001)));
9950 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9951
9952 assert!(matches!(
9953 sleep.as_mut().poll(&mut task_context),
9954 Poll::Pending
9955 ));
9956 assert!(matches!(
9957 sleep.as_mut().poll(&mut task_context),
9958 Poll::Pending
9959 ));
9960
9961 let commands = ctx.take_commands().expect("timer command");
9962 assert_eq!(
9963 commands,
9964 vec![json!({
9965 "type": "start_timer",
9966 "delay_seconds": 2,
9967 })]
9968 );
9969 }
9970
9971 #[test]
9972 fn workflow_sleep_replays_matching_schedule_and_fire_without_a_command() {
9973 let history = vec![
9974 history_event(
9975 "TimerScheduled",
9976 json!({
9977 "sequence": 1,
9978 "timer_id": "timer-1",
9979 "delay_seconds": 5,
9980 "fire_at": "2026-07-11T12:00:05Z",
9981 }),
9982 ),
9983 history_event(
9984 "TimerFired",
9985 json!({
9986 "sequence": 1,
9987 "timer_id": "timer-1",
9988 "delay_seconds": 5,
9989 "fire_at": "2026-07-11T12:00:05Z",
9990 "fired_at": "2026-07-11T12:00:05Z",
9991 }),
9992 ),
9993 ];
9994
9995 for _restart in 0..2 {
9996 let ctx = workflow_context(history.clone());
9997 let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(5)));
9998 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9999 assert!(matches!(
10000 sleep.as_mut().poll(&mut task_context),
10001 Poll::Ready(Ok(()))
10002 ));
10003 assert!(ctx.take_commands().expect("commands").is_empty());
10004 ctx.ensure_history_consumed().expect("history consumed");
10005 }
10006 }
10007
10008 #[test]
10009 fn workflow_sleep_rejects_changed_delay_during_replay() {
10010 let ctx = workflow_context(vec![
10011 history_event(
10012 "TimerScheduled",
10013 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10014 ),
10015 history_event(
10016 "TimerFired",
10017 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10018 ),
10019 ]);
10020 let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(500)));
10021 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10022
10023 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10024 sleep.as_mut().poll(&mut task_context)
10025 else {
10026 panic!("changed timer delay must be rejected");
10027 };
10028 assert_eq!(failure.reason, "timer_delay_mismatch");
10029 assert_eq!(failure.sequence, Some(1));
10030 }
10031
10032 #[test]
10033 fn workflow_history_rejects_unpaired_or_mismatched_timer_events() {
10034 let lone_fire = WorkflowState::new(
10035 vec![history_event(
10036 "TimerFired",
10037 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10038 )],
10039 "rust-workers".to_string(),
10040 JSON_CODEC.to_string(),
10041 None,
10042 )
10043 .expect_err("TimerFired requires TimerScheduled");
10044 assert!(matches!(
10045 lone_fire,
10046 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10047 if reason == "timer_schedule_missing_or_duplicate"
10048 ));
10049
10050 let wrong_identity = WorkflowState::new(
10051 vec![
10052 history_event(
10053 "TimerScheduled",
10054 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10055 ),
10056 history_event(
10057 "TimerFired",
10058 json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
10059 ),
10060 ],
10061 "rust-workers".to_string(),
10062 JSON_CODEC.to_string(),
10063 None,
10064 )
10065 .expect_err("fire must match scheduled timer identity");
10066 assert!(matches!(
10067 wrong_identity,
10068 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10069 if reason == "timer_identity_mismatch"
10070 ));
10071
10072 let duplicate_fire = WorkflowState::new(
10073 vec![
10074 history_event(
10075 "TimerScheduled",
10076 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10077 ),
10078 history_event(
10079 "TimerFired",
10080 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10081 ),
10082 history_event(
10083 "TimerFired",
10084 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10085 ),
10086 ],
10087 "rust-workers".to_string(),
10088 JSON_CODEC.to_string(),
10089 None,
10090 )
10091 .expect_err("a durable timer cannot fire twice");
10092 assert!(matches!(
10093 duplicate_fire,
10094 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10095 if reason == "duplicate_timer_fire"
10096 ));
10097
10098 let wrong_fired_delay = WorkflowState::new(
10099 vec![
10100 history_event(
10101 "TimerScheduled",
10102 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10103 ),
10104 history_event(
10105 "TimerFired",
10106 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 6}),
10107 ),
10108 ],
10109 "rust-workers".to_string(),
10110 JSON_CODEC.to_string(),
10111 None,
10112 )
10113 .expect_err("timer schedule and fire delays must agree");
10114 assert!(matches!(
10115 wrong_fired_delay,
10116 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10117 if reason == "timer_history_delay_mismatch"
10118 ));
10119 }
10120
10121 #[test]
10122 fn replay_rejects_activity_moved_before_recorded_timer() {
10123 let ctx = workflow_context(vec![
10124 history_event(
10125 "TimerScheduled",
10126 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10127 ),
10128 history_event(
10129 "TimerFired",
10130 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10131 ),
10132 history_event(
10133 "ActivityCompleted",
10134 json!({
10135 "sequence": 2,
10136 "activity_type": "after-timer",
10137 "payload_codec": "json",
10138 "result": {"codec": "json", "blob": "\"done\""},
10139 }),
10140 ),
10141 ]);
10142 let mut activity = Box::pin(ctx.activity("after-timer", json!([])));
10143 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10144
10145 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10146 activity.as_mut().poll(&mut task_context)
10147 else {
10148 panic!("reordered durable command must be rejected");
10149 };
10150 assert_eq!(failure.reason, "recorded_command_mismatch");
10151 assert_eq!(failure.sequence, Some(1));
10152 assert_eq!(failure.expected.as_deref(), Some("timer"));
10153 assert_eq!(failure.actual.as_deref(), Some("activity:after-timer"));
10154 }
10155
10156 #[test]
10157 fn workflow_context_emits_a_typed_named_signal_wait() {
10158 let ctx = workflow_context(Vec::new());
10159 let mut signal = Box::pin(ctx.wait_signal("finish"));
10160 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10161
10162 assert!(matches!(
10163 signal.as_mut().poll(&mut task_context),
10164 Poll::Pending
10165 ));
10166 assert_eq!(
10167 ctx.take_commands().expect("signal-wait command"),
10168 vec![json!({
10169 "type": "open_signal_wait",
10170 "signal_name": "finish",
10171 })]
10172 );
10173 }
10174
10175 #[test]
10176 fn condition_wait_history_does_not_resolve_a_typed_signal_wait() {
10177 let ctx = workflow_context(vec![
10178 history_event(
10179 "ConditionWaitOpened",
10180 json!({"sequence": 1, "condition_key": "signal:finish"}),
10181 ),
10182 history_event(
10183 "ConditionWaitSatisfied",
10184 json!({"sequence": 1, "condition_key": "signal:finish"}),
10185 ),
10186 history_event(
10187 "SignalReceived",
10188 json!({"signal_name": "finish", "arguments": []}),
10189 ),
10190 ]);
10191 let mut signal = Box::pin(ctx.wait_signal("finish"));
10192 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10193
10194 assert!(matches!(
10195 signal.as_mut().poll(&mut task_context),
10196 Poll::Pending
10197 ));
10198 assert_eq!(
10199 ctx.take_commands().expect("typed signal-wait command"),
10200 vec![json!({
10201 "type": "open_signal_wait",
10202 "signal_name": "finish",
10203 })]
10204 );
10205 }
10206
10207 #[test]
10208 fn replay_orders_signal_waits_and_timers_in_one_command_stream() {
10209 let signal_then_timer = vec![
10210 history_event(
10211 "SignalWaitOpened",
10212 json!({"sequence": 1, "signal_name": "go"}),
10213 ),
10214 history_event(
10215 "SignalApplied",
10216 json!({
10217 "sequence": 1,
10218 "signal_name": "go",
10219 "value": {"codec": "json", "blob": "[\"now\"]"},
10220 }),
10221 ),
10222 history_event(
10223 "TimerScheduled",
10224 json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
10225 ),
10226 history_event(
10227 "TimerFired",
10228 json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
10229 ),
10230 ];
10231
10232 let ctx = workflow_context(signal_then_timer.clone());
10233 let mut signal = Box::pin(ctx.wait_signal("go"));
10234 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10235 assert!(matches!(
10236 signal.as_mut().poll(&mut task_context),
10237 Poll::Ready(Ok(arguments)) if arguments == vec![json!("now")]
10238 ));
10239 let mut timer = Box::pin(ctx.sleep(Duration::from_secs(5)));
10240 assert!(matches!(
10241 timer.as_mut().poll(&mut task_context),
10242 Poll::Ready(Ok(()))
10243 ));
10244 ctx.ensure_history_consumed()
10245 .expect("signal and timer history consumed in order");
10246
10247 let reordered = workflow_context(signal_then_timer);
10248 let mut timer_first = Box::pin(reordered.sleep(Duration::from_secs(5)));
10249 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10250 timer_first.as_mut().poll(&mut task_context)
10251 else {
10252 panic!("timer cannot consume signal-wait-first history");
10253 };
10254 assert_eq!(failure.reason, "recorded_command_mismatch");
10255 assert_eq!(failure.sequence, Some(1));
10256 assert_eq!(failure.expected.as_deref(), Some("signal wait"));
10257
10258 let timer_then_signal = vec![
10259 history_event(
10260 "TimerScheduled",
10261 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10262 ),
10263 history_event(
10264 "TimerFired",
10265 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10266 ),
10267 history_event(
10268 "SignalWaitOpened",
10269 json!({"sequence": 2, "signal_name": "go"}),
10270 ),
10271 history_event(
10272 "SignalApplied",
10273 json!({
10274 "sequence": 2,
10275 "signal_name": "go",
10276 "value": {"codec": "json", "blob": "[]"},
10277 }),
10278 ),
10279 ];
10280 let reordered = workflow_context(timer_then_signal);
10281 let mut signal_first = Box::pin(reordered.wait_signal("go"));
10282 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10283 signal_first.as_mut().poll(&mut task_context)
10284 else {
10285 panic!("signal wait cannot consume timer-first history");
10286 };
10287 assert_eq!(failure.reason, "recorded_command_mismatch");
10288 assert_eq!(failure.sequence, Some(1));
10289 assert_eq!(failure.expected.as_deref(), Some("timer"));
10290 }
10291
10292 #[test]
10293 fn workflow_history_rejects_duplicate_or_colliding_command_sequences() {
10294 let duplicate_timer = WorkflowState::new(
10295 vec![
10296 history_event(
10297 "TimerScheduled",
10298 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10299 ),
10300 history_event(
10301 "TimerScheduled",
10302 json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
10303 ),
10304 ],
10305 "rust-workers".to_string(),
10306 JSON_CODEC.to_string(),
10307 None,
10308 )
10309 .expect_err("one workflow sequence cannot schedule two timers");
10310 assert!(matches!(
10311 duplicate_timer,
10312 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10313 if reason == "timer_schedule_missing_or_duplicate"
10314 ));
10315
10316 let colliding_kinds = WorkflowState::new(
10317 vec![
10318 history_event(
10319 "TimerScheduled",
10320 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10321 ),
10322 history_event(
10323 "ActivityCompleted",
10324 json!({"sequence": 1, "activity_type": "same-sequence"}),
10325 ),
10326 ],
10327 "rust-workers".to_string(),
10328 JSON_CODEC.to_string(),
10329 None,
10330 )
10331 .expect_err("one workflow sequence cannot identify two command kinds");
10332 assert!(matches!(
10333 colliding_kinds,
10334 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10335 if reason == "durable_command_sequence_collision"
10336 ));
10337
10338 let duplicate_signal_wait = WorkflowState::new(
10339 vec![
10340 history_event(
10341 "SignalWaitOpened",
10342 json!({"sequence": 1, "signal_name": "go"}),
10343 ),
10344 history_event(
10345 "SignalWaitOpened",
10346 json!({"sequence": 1, "signal_name": "go"}),
10347 ),
10348 ],
10349 "rust-workers".to_string(),
10350 JSON_CODEC.to_string(),
10351 None,
10352 )
10353 .expect_err("one workflow sequence cannot open two signal waits");
10354 assert!(matches!(
10355 duplicate_signal_wait,
10356 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10357 if reason == "signal_wait_open_missing_or_duplicate"
10358 ));
10359 }
10360
10361 #[test]
10362 fn workflow_history_accepts_a_first_command_after_global_sequence_gaps() {
10363 let result = encode_value_envelope(&json!({"captured": true}), JSON_CODEC)
10364 .expect("side-effect result");
10365 let ctx = workflow_context(vec![history_event(
10366 "SideEffectRecorded",
10367 json!({"sequence": 99, "result": result}),
10368 )]);
10369
10370 let replayed: Value = ctx
10371 .side_effect(|| panic!("recorded side effect must not run"))
10372 .expect("positive global workflow sequence is valid");
10373 assert_eq!(replayed, json!({"captured": true}));
10374 ctx.ensure_history_consumed().expect("history consumed");
10375 }
10376
10377 #[test]
10378 fn workflow_history_rejects_zero_and_descending_command_sequences() {
10379 let result =
10380 encode_value_envelope(&json!("captured"), JSON_CODEC).expect("side-effect result");
10381 let zero = WorkflowState::new(
10382 vec![history_event(
10383 "SideEffectRecorded",
10384 json!({"sequence": 0, "result": result.clone()}),
10385 )],
10386 "rust-workers".to_string(),
10387 JSON_CODEC.to_string(),
10388 None,
10389 )
10390 .expect_err("durable command sequences must be positive");
10391 assert!(matches!(
10392 zero,
10393 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10394 if reason == "durable_command_sequence_invalid"
10395 ));
10396
10397 let descending = WorkflowState::new(
10398 vec![
10399 history_event(
10400 "SideEffectRecorded",
10401 json!({"sequence": 3, "result": result}),
10402 ),
10403 history_event(
10404 "VersionMarkerRecorded",
10405 json!({
10406 "sequence": 2,
10407 "change_id": "descending-marker",
10408 "version": 1,
10409 "min_supported": 1,
10410 "max_supported": 1,
10411 }),
10412 ),
10413 ],
10414 "rust-workers".to_string(),
10415 JSON_CODEC.to_string(),
10416 None,
10417 )
10418 .expect_err("new durable commands must remain strictly ordered");
10419 let Error::NonDeterministicReplay(failure) = descending else {
10420 panic!("expected typed replay failure");
10421 };
10422 assert_eq!(failure.reason, "durable_command_sequence_mismatch");
10423 assert_eq!(failure.sequence, Some(2));
10424 assert_eq!(
10425 failure.expected.as_deref(),
10426 Some("workflow sequence greater than 3")
10427 );
10428 assert_eq!(failure.actual.as_deref(), Some("2"));
10429 }
10430
10431 #[test]
10432 fn workflow_task_replay_completes_after_signals_create_sequence_gaps() {
10433 fn worker() -> Worker {
10434 let client = Client::new("http://127.0.0.1:8080").expect("client");
10435 let mut worker = Worker::new(client, "rust-workers");
10436 worker.register_workflow("rust.finish-after-gaps", |ctx, _input| async move {
10437 ctx.wait_signal("finish").await?;
10438 let marker: String =
10439 ctx.side_effect(|| panic!("recorded side effect must not run"))?;
10440 assert_eq!(marker, "after-finish");
10441 Ok(json!("finished"))
10442 });
10443 worker
10444 }
10445
10446 let marker =
10447 encode_value_envelope(&json!("after-finish"), JSON_CODEC).expect("side-effect result");
10448 let task = workflow_task(
10449 "rust.finish-after-gaps",
10450 vec![
10451 history_event(
10452 "SignalWaitOpened",
10453 json!({"sequence": 1, "signal_name": "finish"}),
10454 ),
10455 history_event(
10456 "SignalReceived",
10457 json!({
10458 "signal_id": "increment-3",
10459 "signal_name": "increment",
10460 "workflow_sequence": 2,
10461 "payload_codec": "json",
10462 "arguments": {"codec": "json", "blob": "[3]"},
10463 }),
10464 ),
10465 history_event(
10466 "SignalReceived",
10467 json!({
10468 "signal_id": "increment-5",
10469 "signal_name": "increment",
10470 "workflow_sequence": 3,
10471 "payload_codec": "json",
10472 "arguments": {"codec": "json", "blob": "[5]"},
10473 }),
10474 ),
10475 history_event(
10476 "SignalReceived",
10477 json!({
10478 "signal_id": "finish",
10479 "signal_name": "finish",
10480 "workflow_sequence": 4,
10481 "payload_codec": "json",
10482 "arguments": {"codec": "json", "blob": "[]"},
10483 }),
10484 ),
10485 history_event(
10486 "SignalApplied",
10487 json!({
10488 "sequence": 1,
10489 "signal_id": "finish",
10490 "signal_name": "finish",
10491 "payload_codec": "json",
10492 "value": {"codec": "json", "blob": "[]"},
10493 }),
10494 ),
10495 history_event(
10496 "SideEffectRecorded",
10497 json!({"sequence": 5, "result": marker}),
10498 ),
10499 ],
10500 JSON_CODEC,
10501 );
10502
10503 for _original_or_cold_worker in 0..2 {
10504 let commands = worker()
10505 .execute_workflow_task(task.clone())
10506 .expect("signal gaps preserve deterministic replay");
10507 assert_eq!(commands.len(), 1, "replay emits only terminal completion");
10508 assert_eq!(commands[0]["type"], "complete_workflow");
10509 assert_eq!(
10510 decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("workflow output"),
10511 json!("finished")
10512 );
10513 }
10514 }
10515
10516 #[test]
10517 fn workflow_sleep_rejects_unrepresentable_rounded_duration() {
10518 let ctx = workflow_context(Vec::new());
10519 let mut sleep = Box::pin(ctx.start_timer(Duration::new(u64::MAX, 1)));
10520 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10521 assert!(matches!(
10522 sleep.as_mut().poll(&mut task_context),
10523 Poll::Ready(Err(Error::TimerDurationOverflow))
10524 ));
10525 assert!(ctx.take_commands().expect("commands").is_empty());
10526 }
10527
10528 #[test]
10529 fn workflow_task_replay_completes_without_rescheduling_recorded_commands() {
10530 let client = Client::new("http://127.0.0.1:8080").expect("client");
10531 let mut worker = Worker::new(client, "rust-workers");
10532 worker.register_workflow("rust.timer", |ctx, _input| async move {
10533 ctx.sleep(Duration::from_secs(5)).await?;
10534 ctx.activity("after-timer", json!([])).await
10535 });
10536
10537 let task = |history_events| WorkflowTask {
10538 task_id: "wft-rust-timer-1".to_string(),
10539 workflow_id: Some("wf-rust-timer".to_string()),
10540 run_id: Some("run-rust-timer".to_string()),
10541 workflow_type: "rust.timer".to_string(),
10542 payload_codec: JSON_CODEC.to_string(),
10543 arguments: Some(json!({"codec": "json", "blob": "[]"})),
10544 history_events,
10545 total_history_events: None,
10546 history_size_bytes: None,
10547 continue_as_new_recommended: None,
10548 history_budget_pressure: None,
10549 next_history_page_token: None,
10550 workflow_task_attempt: 1,
10551 workflow_signal_id: None,
10552 signal_name: None,
10553 signal_arguments: None,
10554 workflow_update_id: None,
10555 update_name: None,
10556 lease_owner: Some("rust-worker".to_string()),
10557 };
10558
10559 let initial = worker
10560 .execute_workflow_task(task(Vec::new()))
10561 .expect("initial timer task");
10562 assert_eq!(
10563 initial,
10564 vec![json!({"type": "start_timer", "delay_seconds": 5})]
10565 );
10566
10567 let replayed = worker
10568 .execute_workflow_task(task(vec![
10569 history_event(
10570 "TimerScheduled",
10571 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10572 ),
10573 history_event(
10574 "TimerFired",
10575 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10576 ),
10577 history_event(
10578 "ActivityCompleted",
10579 json!({
10580 "sequence": 2,
10581 "activity_type": "after-timer",
10582 "payload_codec": "json",
10583 "result": {"codec": "json", "blob": "\"done\""},
10584 }),
10585 ),
10586 ]))
10587 .expect("replayed workflow task");
10588 assert_eq!(replayed.len(), 1);
10589 assert_eq!(replayed[0]["type"], "complete_workflow");
10590 assert_eq!(
10591 decode_wire_value(&replayed[0]["result"], JSON_CODEC).expect("result"),
10592 json!("done")
10593 );
10594 }
10595
10596 #[test]
10597 fn workflow_continue_as_new_emits_arguments_type_and_queue_once() {
10598 let client = Client::new("http://127.0.0.1:8080").expect("client");
10599 let mut worker = Worker::new(client, "rust-workers");
10600 worker.register_workflow("rust.continue", |ctx, _input| async move {
10601 ctx.continue_as_new_with_options(
10602 ContinueAsNewOptions::new()
10603 .workflow_type("rust.next")
10604 .task_queue("next-workers"),
10605 json!([2, {"cursor": "next"}]),
10606 )
10607 });
10608
10609 let commands = worker
10610 .execute_workflow_task(workflow_task("rust.continue", Vec::new(), DEFAULT_CODEC))
10611 .expect("continue-as-new command");
10612
10613 assert_eq!(commands.len(), 1);
10614 assert_eq!(commands[0]["type"], "continue_as_new");
10615 assert_eq!(commands[0]["workflow_type"], "rust.next");
10616 assert_eq!(commands[0]["queue"], "next-workers");
10617 assert_eq!(
10618 decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC)
10619 .expect("continue-as-new arguments"),
10620 json!([2, {"cursor": "next"}])
10621 );
10622 }
10623
10624 #[test]
10625 fn continue_as_new_preserves_typed_arguments() {
10626 let client = Client::new("http://127.0.0.1:8080").expect("client");
10627 let mut worker = Worker::new(client, "rust-workers");
10628 worker.register_workflow_avro_value("rust.typed-continue", |ctx, _input| async move {
10629 ctx.continue_as_new(AvroValue::Array(vec![typed_fidelity_probe()]))?;
10630 unreachable!("continue-as-new returns a control-flow error")
10631 });
10632
10633 let commands = worker
10634 .execute_workflow_task(workflow_task(
10635 "rust.typed-continue",
10636 Vec::new(),
10637 DEFAULT_CODEC,
10638 ))
10639 .expect("typed continue-as-new command");
10640
10641 assert_eq!(commands[0]["type"], "continue_as_new");
10642 assert_eq!(
10643 decode_wire_avro_value(&commands[0]["arguments"], DEFAULT_CODEC)
10644 .expect("typed continue arguments"),
10645 AvroValue::Array(vec![typed_fidelity_probe()])
10646 );
10647 }
10648
10649 #[test]
10650 fn recorded_continue_as_new_is_consumed_without_duplicate_successor_command() {
10651 let client = Client::new("http://127.0.0.1:8080").expect("client");
10652 let mut worker = Worker::new(client, "rust-workers");
10653 worker.register_workflow("rust.continue", |ctx, _input| async move {
10654 ctx.continue_as_new(json!([2]))
10655 });
10656 let task = workflow_task(
10657 "rust.continue",
10658 vec![history_event(
10659 "WorkflowContinuedAsNew",
10660 json!({"sequence": 1, "continued_to_run_id": "run-next"}),
10661 )],
10662 JSON_CODEC,
10663 );
10664
10665 for _worker_restart_or_redelivery in 0..2 {
10666 let commands = worker
10667 .execute_workflow_task(task.clone())
10668 .expect("recorded transition replays");
10669 assert!(
10670 commands.is_empty(),
10671 "replay must not emit another successor"
10672 );
10673 }
10674 }
10675
10676 #[test]
10677 fn continue_as_new_rejects_invalid_overrides_before_emitting_a_command() {
10678 let ctx = workflow_context(Vec::new());
10679 let error = ctx
10680 .continue_as_new_with_options(ContinueAsNewOptions::new().task_queue(" "), json!([1]))
10681 .expect_err("blank queue must be rejected");
10682
10683 let Error::InvalidContinueAsNewOptions(error) = error else {
10684 panic!("expected typed continue-as-new validation error");
10685 };
10686 assert_eq!(error.field, "task_queue");
10687 assert!(ctx.take_commands().expect("commands").is_empty());
10688 }
10689
10690 #[test]
10691 fn workflow_context_exposes_server_history_budget() {
10692 let client = Client::new("http://127.0.0.1:8080").expect("client");
10693 let mut worker = Worker::new(client, "rust-workers");
10694 worker.register_workflow("rust.history-budget", |ctx, _input| async move {
10695 let budget = ctx.history_budget()?;
10696 Ok(json!({
10697 "events": budget.event_count,
10698 "bytes": budget.size_bytes,
10699 "recommended": budget.continue_as_new_recommended,
10700 "pressure": budget.pressure,
10701 }))
10702 });
10703 let task: WorkflowTask = serde_json::from_value(json!({
10704 "task_id": "task-history-budget",
10705 "workflow_type": "rust.history-budget",
10706 "payload_codec": JSON_CODEC,
10707 "history_events": [],
10708 "total_history_events": 480,
10709 "history_size_bytes": 1_048_576,
10710 "continue_as_new_recommended": true,
10711 "history_budget_pressure": "continue_as_new_recommended",
10712 }))
10713 .expect("published workflow task");
10714
10715 let commands = worker
10716 .execute_workflow_task(task)
10717 .expect("history-budget workflow");
10718 let result = decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("result");
10719 assert_eq!(result["events"], 480);
10720 assert_eq!(result["bytes"], 1_048_576);
10721 assert_eq!(result["recommended"], true);
10722 assert_eq!(result["pressure"], "continue_as_new_recommended");
10723 }
10724
10725 #[test]
10726 fn uncaught_workflow_handler_error_emits_terminal_failure_command() {
10727 let client = Client::new("http://127.0.0.1:8080").expect("client");
10728 let mut worker = Worker::new(client, "rust-workers");
10729 worker.register_workflow("rust.failing", |_ctx, _input| async move {
10730 Err(Error::Codec("rust_conformance_failure".to_string()))
10731 });
10732 let task = WorkflowTask {
10733 task_id: "wft-rust-failing-1".to_string(),
10734 workflow_id: Some("wf-rust-failing".to_string()),
10735 run_id: Some("run-rust-failing".to_string()),
10736 workflow_type: "rust.failing".to_string(),
10737 payload_codec: JSON_CODEC.to_string(),
10738 arguments: Some(encode_value_envelope(&json!([]), JSON_CODEC).expect("input")),
10739 history_events: Vec::new(),
10740 total_history_events: Some(0),
10741 history_size_bytes: None,
10742 continue_as_new_recommended: None,
10743 history_budget_pressure: None,
10744 next_history_page_token: None,
10745 workflow_task_attempt: 1,
10746 workflow_signal_id: None,
10747 signal_name: None,
10748 signal_arguments: None,
10749 workflow_update_id: None,
10750 update_name: None,
10751 lease_owner: Some("rust-worker".to_string()),
10752 };
10753
10754 let commands = worker
10755 .execute_workflow_task(task)
10756 .expect("handler failure becomes a workflow command");
10757
10758 assert_eq!(commands.len(), 1);
10759 assert_eq!(commands[0]["type"], "fail_workflow");
10760 assert_eq!(commands[0]["exception_type"], "RustWorkflowError");
10761 assert_eq!(commands[0]["exception_class"], "durable_workflow::Error");
10762 assert_eq!(commands[0]["non_retryable"], false);
10763 assert_eq!(
10764 commands[0]["message"],
10765 "codec error: rust_conformance_failure"
10766 );
10767 assert_eq!(
10768 commands[0]["exception"]["message"],
10769 "codec error: rust_conformance_failure"
10770 );
10771 }
10772
10773 #[test]
10774 fn ordinary_handler_error_preserves_commands_queued_in_the_same_decision() {
10775 let client = Client::new("http://127.0.0.1:8080").expect("client");
10776 let mut worker = Worker::new(client, "rust-workers");
10777 worker.register_workflow("rust.failing-after-side-effect", |ctx, _input| async move {
10778 let _: String = ctx.side_effect(|| "captured".to_string())?;
10779 Err(Error::WorkerLoop("application failure".to_string()))
10780 });
10781
10782 let commands = worker
10783 .execute_workflow_task(workflow_task(
10784 "rust.failing-after-side-effect",
10785 Vec::new(),
10786 JSON_CODEC,
10787 ))
10788 .expect("ordinary failure remains a workflow decision");
10789
10790 assert_eq!(commands.len(), 2);
10791 assert_eq!(commands[0]["type"], "record_side_effect");
10792 assert_eq!(commands[1]["type"], "fail_workflow");
10793 }
10794
10795 #[test]
10796 fn handler_error_cannot_hide_an_unconsumed_committed_side_effect() {
10797 let client = Client::new("http://127.0.0.1:8080").expect("client");
10798 let mut worker = Worker::new(client, "rust-workers");
10799 worker.register_workflow("rust.removed-side-effect", |_ctx, _input| async move {
10800 Err(Error::WorkerLoop("application failure".to_string()))
10801 });
10802 let result =
10803 encode_value_envelope(&json!("committed"), JSON_CODEC).expect("side-effect result");
10804
10805 let error = worker
10806 .execute_workflow_task(workflow_task(
10807 "rust.removed-side-effect",
10808 vec![history_event(
10809 "SideEffectRecorded",
10810 json!({"sequence": 1, "result": result}),
10811 )],
10812 JSON_CODEC,
10813 ))
10814 .expect_err("removed committed history must not become fail_workflow");
10815
10816 let Error::NonDeterministicReplay(failure) = error else {
10817 panic!("expected typed replay failure");
10818 };
10819 assert_eq!(failure.reason, "recorded_commands_unconsumed");
10820 assert_eq!(failure.sequence, Some(1));
10821 assert_eq!(failure.expected.as_deref(), Some("side effect"));
10822 }
10823
10824 #[test]
10825 fn replay_error_discards_side_effect_queued_before_incompatible_marker_check() {
10826 let client = Client::new("http://127.0.0.1:8080").expect("client");
10827 let mut worker = Worker::new(client, "rust-workers");
10828 worker.register_workflow(
10829 "rust.side-effect-before-marker-error",
10830 |ctx, _input| async move {
10831 assert_eq!(ctx.get_version("restart-safe", 1, 1)?, 1);
10832 let _: String = ctx.side_effect(|| "must-not-commit".to_string())?;
10833 ctx.get_version("restart-safe", 2, 2)?;
10834 Ok(Value::Null)
10835 },
10836 );
10837
10838 let error = worker
10839 .execute_workflow_task(workflow_task(
10840 "rust.side-effect-before-marker-error",
10841 vec![history_event(
10842 "VersionMarkerRecorded",
10843 json!({
10844 "sequence": 1,
10845 "change_id": "restart-safe",
10846 "version": 1,
10847 "min_supported": 1,
10848 "max_supported": 1,
10849 }),
10850 )],
10851 JSON_CODEC,
10852 ))
10853 .expect_err("replay error must return no queued workflow commands");
10854
10855 let Error::NonDeterministicReplay(failure) = error else {
10856 panic!("expected typed replay failure");
10857 };
10858 assert_eq!(failure.reason, "version_marker_incompatible_range");
10859 assert_eq!(failure.sequence, Some(1));
10860 }
10861
10862 #[test]
10863 fn workflow_task_replay_keeps_recorded_unfired_timer_pending_without_rescheduling() {
10864 let client = Client::new("http://127.0.0.1:8080").expect("client");
10865 let mut worker = Worker::new(client, "rust-workers");
10866 worker.register_workflow("rust.timer.pending", |ctx, _input| async move {
10867 ctx.sleep(Duration::from_secs(5)).await?;
10868 Ok(json!({"status": "timer fired"}))
10869 });
10870
10871 let task = WorkflowTask {
10872 task_id: "wft-rust-timer-pending".to_string(),
10873 workflow_id: Some("wf-rust-timer".to_string()),
10874 run_id: Some("run-rust-timer".to_string()),
10875 workflow_type: "rust.timer.pending".to_string(),
10876 payload_codec: JSON_CODEC.to_string(),
10877 arguments: Some(json!({"codec": "json", "blob": "[]"})),
10878 history_events: vec![history_event(
10879 "TimerScheduled",
10880 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10881 )],
10882 total_history_events: Some(1),
10883 history_size_bytes: None,
10884 continue_as_new_recommended: None,
10885 history_budget_pressure: None,
10886 next_history_page_token: None,
10887 workflow_task_attempt: 1,
10888 workflow_signal_id: None,
10889 signal_name: None,
10890 signal_arguments: None,
10891 workflow_update_id: None,
10892 update_name: None,
10893 lease_owner: Some("rust-worker".to_string()),
10894 };
10895
10896 for _redelivery_or_restart in 0..2 {
10897 let commands = worker
10898 .execute_workflow_task(task.clone())
10899 .expect("recorded timer remains pending");
10900 assert!(
10901 commands.is_empty(),
10902 "recorded timer must not be rescheduled"
10903 );
10904 }
10905 }
10906
10907 #[test]
10908 fn workflow_task_rejects_recorded_command_removed_from_workflow_code() {
10909 let client = Client::new("http://127.0.0.1:8080").expect("client");
10910 let mut worker = Worker::new(client, "rust-workers");
10911 worker.register_workflow("rust.timer.removed", |_ctx, _input| async move {
10912 Ok(json!({"status": "completed"}))
10913 });
10914 let task = WorkflowTask {
10915 task_id: "wft-rust-timer-removed".to_string(),
10916 workflow_id: Some("wf-rust-timer".to_string()),
10917 run_id: Some("run-rust-timer".to_string()),
10918 workflow_type: "rust.timer.removed".to_string(),
10919 payload_codec: JSON_CODEC.to_string(),
10920 arguments: Some(json!({"codec": "json", "blob": "[]"})),
10921 history_events: vec![
10922 history_event(
10923 "TimerScheduled",
10924 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10925 ),
10926 history_event(
10927 "TimerFired",
10928 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10929 ),
10930 ],
10931 total_history_events: Some(2),
10932 history_size_bytes: None,
10933 continue_as_new_recommended: None,
10934 history_budget_pressure: None,
10935 next_history_page_token: None,
10936 workflow_task_attempt: 1,
10937 workflow_signal_id: None,
10938 signal_name: None,
10939 signal_arguments: None,
10940 workflow_update_id: None,
10941 update_name: None,
10942 lease_owner: Some("rust-worker".to_string()),
10943 };
10944
10945 let Error::NonDeterministicReplay(failure) = worker
10946 .execute_workflow_task(task)
10947 .expect_err("removed timer must fail replay")
10948 else {
10949 panic!("expected typed replay failure");
10950 };
10951 assert_eq!(failure.reason, "recorded_commands_unconsumed");
10952 assert_eq!(failure.sequence, Some(1));
10953 }
10954
10955 #[test]
10956 fn workflow_context_emits_explicit_child_workflow_contract() {
10957 let ctx = WorkflowContext {
10958 state: Arc::new(Mutex::new(
10959 WorkflowState::new_with_identity(
10960 Vec::new(),
10961 Some("wf-parent".to_string()),
10962 Some("run-parent".to_string()),
10963 "parent-workers".to_string(),
10964 JSON_CODEC.to_string(),
10965 None,
10966 )
10967 .expect("workflow state"),
10968 )),
10969 };
10970 let options = ChildWorkflowOptions::new("python-workers")
10971 .parent_close_policy(ParentClosePolicy::RequestCancel)
10972 .retry_policy(ChildWorkflowRetryPolicy {
10973 max_attempts: Some(3),
10974 backoff_seconds: vec![1, 5],
10975 non_retryable_error_types: vec!["ValidationError".to_string()],
10976 })
10977 .execution_timeout_seconds(600)
10978 .run_timeout_seconds(120);
10979 let mut call = Box::pin(ctx.start_child_workflow(
10980 "python.fulfil-order",
10981 options,
10982 json!([{"order_id": "order-42"}]),
10983 ));
10984 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10985
10986 assert!(matches!(
10987 call.as_mut().poll(&mut task_context),
10988 Poll::Pending
10989 ));
10990 let commands = ctx.take_commands().expect("commands");
10991 assert_eq!(commands.len(), 1);
10992 let command = &commands[0];
10993 assert_eq!(command["type"], "start_child_workflow");
10994 assert_eq!(command["workflow_type"], "python.fulfil-order");
10995 assert_eq!(command["queue"], "python-workers");
10996 assert_eq!(command["parent_close_policy"], "request_cancel");
10997 assert_eq!(command["retry_policy"]["max_attempts"], 3);
10998 assert_eq!(command["execution_timeout_seconds"], 600);
10999 assert_eq!(command["run_timeout_seconds"], 120);
11000 assert_eq!(
11001 decode_wire_value(&command["arguments"], JSON_CODEC).expect("child args"),
11002 json!([{"order_id": "order-42"}])
11003 );
11004 }
11005
11006 fn child_parent_worker() -> Worker {
11007 let client = Client::new("http://127.0.0.1:8080").expect("client");
11008 let mut worker = Worker::new(client, "rust-parent-workers");
11009 worker.register_workflow("rust.parent", |ctx, _input| async move {
11010 let child = ctx
11011 .start_child_workflow(
11012 "python.child",
11013 ChildWorkflowOptions::new("python-child-workers")
11014 .parent_close_policy(ParentClosePolicy::Terminate),
11015 json!([{"codec_probe": [1, true, "rust"]}]),
11016 )
11017 .await?;
11018 Ok(json!({
11019 "parent_workflow_id": child.parent.workflow_id,
11020 "parent_run_id": child.parent.run_id,
11021 "child_workflow_id": child.child.workflow_id,
11022 "child_run_id": child.child.run_id,
11023 "child_workflow_type": child.child_workflow_type,
11024 "result": child.result,
11025 }))
11026 });
11027 worker
11028 }
11029
11030 fn child_parent_task(event_type: &str, payload: Value) -> WorkflowTask {
11031 WorkflowTask {
11032 task_id: "wft-child-parent".to_string(),
11033 workflow_id: Some("wf-parent".to_string()),
11034 run_id: Some("run-parent".to_string()),
11035 workflow_type: "rust.parent".to_string(),
11036 payload_codec: JSON_CODEC.to_string(),
11037 arguments: Some(encode_value_envelope(&json!([]), JSON_CODEC).expect("input")),
11038 history_events: vec![
11039 HistoryEvent {
11040 event_type: "ChildWorkflowScheduled".to_string(),
11041 payload: json!({
11042 "sequence": 1,
11043 "child_call_id": "call-child",
11044 "child_workflow_instance_id": "wf-child",
11045 "child_workflow_run_id": "run-child",
11046 "child_workflow_type": "python.child",
11047 }),
11048 raw: HashMap::new(),
11049 },
11050 HistoryEvent {
11051 event_type: event_type.to_string(),
11052 payload,
11053 raw: HashMap::new(),
11054 },
11055 ],
11056 total_history_events: Some(2),
11057 history_size_bytes: None,
11058 continue_as_new_recommended: None,
11059 history_budget_pressure: None,
11060 next_history_page_token: None,
11061 workflow_task_attempt: 1,
11062 workflow_signal_id: None,
11063 signal_name: None,
11064 signal_arguments: None,
11065 workflow_update_id: None,
11066 update_name: None,
11067 lease_owner: Some("rust-worker".to_string()),
11068 }
11069 }
11070
11071 #[test]
11072 fn committed_child_result_replays_without_starting_a_duplicate() {
11073 let worker = child_parent_worker();
11074 let task = child_parent_task(
11075 "ChildRunCompleted",
11076 json!({
11077 "sequence": 1,
11078 "child_call_id": "call-child",
11079 "child_workflow_instance_id": "wf-child",
11080 "child_workflow_run_id": "run-child",
11081 "child_workflow_type": "python.child",
11082 "payload_codec": "json",
11083 "result": {"codec": "json", "blob": "{\"from\":\"python\",\"ok\":true}"},
11084 }),
11085 );
11086
11087 for _restart in 0..2 {
11088 let commands = worker
11089 .execute_workflow_task(task.clone())
11090 .expect("replayed parent task");
11091 assert_eq!(commands.len(), 1);
11092 assert_eq!(commands[0]["type"], "complete_workflow");
11093 assert!(!commands
11094 .iter()
11095 .any(|command| command["type"] == "start_child_workflow"));
11096 let output =
11097 decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("parent output");
11098 assert_eq!(output["parent_workflow_id"], "wf-parent");
11099 assert_eq!(output["parent_run_id"], "run-parent");
11100 assert_eq!(output["child_workflow_id"], "wf-child");
11101 assert_eq!(output["child_run_id"], "run-child");
11102 assert_eq!(output["result"], json!({"from": "python", "ok": true}));
11103 }
11104 }
11105
11106 #[test]
11107 fn typed_child_arguments_and_results_survive_replay() {
11108 let client = Client::new("http://127.0.0.1:8080").expect("client");
11109 let mut worker = Worker::new(client, "rust-parent-workers");
11110 worker.register_workflow_avro_value("rust.typed-parent", |ctx, _input| async move {
11111 let child = ctx
11112 .start_child_workflow_avro_value(
11113 "python.typed-child",
11114 ChildWorkflowOptions::new("python-workers"),
11115 AvroValue::Array(vec![typed_fidelity_probe()]),
11116 )
11117 .await?;
11118 Ok(child.result)
11119 });
11120
11121 let initial = worker
11122 .execute_workflow_task(workflow_task(
11123 "rust.typed-parent",
11124 Vec::new(),
11125 DEFAULT_CODEC,
11126 ))
11127 .expect("typed child start");
11128 assert_eq!(initial[0]["type"], "start_child_workflow");
11129 assert_eq!(
11130 decode_wire_avro_value(&initial[0]["arguments"], DEFAULT_CODEC)
11131 .expect("typed child arguments"),
11132 AvroValue::Array(vec![typed_fidelity_probe()])
11133 );
11134
11135 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
11136 .expect("typed child result");
11137 let task = workflow_task(
11138 "rust.typed-parent",
11139 vec![
11140 history_event(
11141 "ChildWorkflowScheduled",
11142 json!({
11143 "sequence": 1,
11144 "child_call_id": "call-typed",
11145 "child_workflow_instance_id": "wf-child",
11146 "child_workflow_run_id": "run-child",
11147 "child_workflow_type": "python.typed-child",
11148 }),
11149 ),
11150 history_event(
11151 "ChildRunCompleted",
11152 json!({
11153 "sequence": 1,
11154 "child_call_id": "call-typed",
11155 "child_workflow_instance_id": "wf-child",
11156 "child_workflow_run_id": "run-child",
11157 "child_workflow_type": "python.typed-child",
11158 "payload_codec": DEFAULT_CODEC,
11159 "result": result,
11160 }),
11161 ),
11162 ],
11163 DEFAULT_CODEC,
11164 );
11165
11166 let commands = worker
11167 .execute_workflow_task(task)
11168 .expect("typed child replay");
11169 assert_eq!(commands[0]["type"], "complete_workflow");
11170 assert_eq!(
11171 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
11172 .expect("typed parent result"),
11173 typed_fidelity_probe()
11174 );
11175 }
11176
11177 #[test]
11178 fn pending_child_replays_after_restart_without_starting_a_duplicate() {
11179 let worker = child_parent_worker();
11180 let mut task = child_parent_task("unused", Value::Null);
11181 task.history_events.truncate(1);
11182 task.total_history_events = Some(1);
11183
11184 for _redelivery_or_restart in 0..2 {
11185 let commands = worker
11186 .execute_workflow_task(task.clone())
11187 .expect("recorded child remains pending");
11188 assert!(
11189 commands.is_empty(),
11190 "recorded pending child must not be started again"
11191 );
11192 }
11193 }
11194
11195 #[test]
11196 fn child_cancellation_becomes_stable_parent_failure_command() {
11197 let worker = child_parent_worker();
11198 let task = child_parent_task(
11199 "ChildRunCancelled",
11200 json!({
11201 "sequence": 1,
11202 "child_workflow_instance_id": "wf-child",
11203 "child_workflow_run_id": "run-child",
11204 "child_workflow_type": "python.child",
11205 "failure_id": "failure-child",
11206 "failure_category": "cancelled",
11207 "message": "cancelled by parent-close policy",
11208 }),
11209 );
11210
11211 let commands = worker
11212 .execute_workflow_task(task)
11213 .expect("parent settlement");
11214 assert_eq!(commands.len(), 1);
11215 assert_eq!(commands[0]["type"], "fail_workflow");
11216 assert_eq!(commands[0]["exception_type"], "ChildWorkflowCancelled");
11217 assert_eq!(
11218 commands[0]["exception"]["properties"]["reason"],
11219 "cancelled"
11220 );
11221 assert_eq!(
11222 commands[0]["exception"]["properties"]["child_workflow_run_id"],
11223 "run-child"
11224 );
11225 }
11226
11227 #[test]
11228 fn workflow_can_handle_typed_child_failure() {
11229 let client = Client::new("http://127.0.0.1:8080").expect("client");
11230 let mut worker = Worker::new(client, "rust-parent-workers");
11231 worker.register_workflow("rust.handled-parent", |ctx, _input| async move {
11232 match ctx
11233 .start_child_workflow(
11234 "python.child",
11235 ChildWorkflowOptions::new("python-child-workers"),
11236 json!([]),
11237 )
11238 .await
11239 {
11240 Err(Error::ChildWorkflowFailed(failure)) => Ok(json!({
11241 "reason": failure.reason,
11242 "failure_id": failure.failure_id,
11243 "exception_class": failure.exception_class,
11244 "child_run_id": failure.child_workflow_run_id,
11245 })),
11246 Err(error) => Err(error),
11247 Ok(_) => Err(Error::WorkerLoop(
11248 "child unexpectedly succeeded".to_string(),
11249 )),
11250 }
11251 });
11252 let mut task = child_parent_task(
11253 "ChildRunFailed",
11254 json!({
11255 "sequence": 1,
11256 "child_workflow_instance_id": "wf-child",
11257 "child_workflow_run_id": "run-child",
11258 "child_workflow_type": "python.child",
11259 "failure_id": "failure-child",
11260 "failure_category": "child_workflow",
11261 "message": "payment rejected",
11262 "exception": {
11263 "type": "PaymentRejected",
11264 "class": "payments.PaymentRejected",
11265 "message": "payment rejected"
11266 }
11267 }),
11268 );
11269 task.workflow_type = "rust.handled-parent".to_string();
11270
11271 let commands = worker.execute_workflow_task(task).expect("handled failure");
11272 assert_eq!(commands[0]["type"], "complete_workflow");
11273 let output = decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("parent output");
11274 assert_eq!(output["reason"], "child_workflow");
11275 assert_eq!(output["failure_id"], "failure-child");
11276 assert_eq!(output["exception_class"], "payments.PaymentRejected");
11277 assert_eq!(output["child_run_id"], "run-child");
11278 }
11279
11280 #[test]
11281 fn rust_hello_world_uses_signal_arguments_from_resume_payload() {
11282 let client = Client::new("http://127.0.0.1:8080").expect("client");
11283 let mut worker = Worker::new(client, "rust-workers");
11284
11285 worker.register_workflow("rust.hello_workflow", |ctx, _input| async move {
11286 let signal = ctx.wait_signal("start").await?;
11287 let name = signal
11288 .first()
11289 .and_then(|value| value.as_str())
11290 .unwrap_or("world");
11291 let greeting = ctx.activity("rust.hello_activity", json!([name])).await?;
11292 Ok(json!({
11293 "greeting": greeting,
11294 "language": "rust"
11295 }))
11296 });
11297
11298 let signal_arguments =
11299 encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC).expect("signal arguments");
11300 let task = WorkflowTask {
11301 task_id: "wft-rust-signal-1".to_string(),
11302 workflow_id: Some("wf-rust-hello".to_string()),
11303 run_id: Some("run-rust-hello".to_string()),
11304 workflow_type: "rust.hello_workflow".to_string(),
11305 payload_codec: DEFAULT_CODEC.to_string(),
11306 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
11307 history_events: vec![HistoryEvent {
11308 event_type: "SignalReceived".to_string(),
11309 payload: json!({
11310 "signal_id": "sig-rust-1",
11311 "signal_name": "start"
11312 }),
11313 raw: HashMap::new(),
11314 }],
11315 total_history_events: Some(1),
11316 history_size_bytes: None,
11317 continue_as_new_recommended: None,
11318 history_budget_pressure: None,
11319 next_history_page_token: None,
11320 workflow_task_attempt: 1,
11321 workflow_signal_id: Some("sig-rust-1".to_string()),
11322 signal_name: Some("start".to_string()),
11323 signal_arguments: Some(signal_arguments),
11324 workflow_update_id: None,
11325 update_name: None,
11326 lease_owner: Some("rust-worker".to_string()),
11327 };
11328
11329 let commands = worker.execute_workflow_task(task).expect("workflow task");
11330
11331 assert_eq!(commands.len(), 1);
11332 assert_eq!(commands[0]["type"], "schedule_activity");
11333 assert_eq!(commands[0]["activity_type"], "rust.hello_activity");
11334 assert_eq!(
11335 decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC).expect("activity args"),
11336 json!(["Rust"])
11337 );
11338 }
11339
11340 #[test]
11341 fn workflow_task_appends_paginated_history_events() {
11342 let mut task = WorkflowTask {
11343 task_id: "wft-rust-pages-1".to_string(),
11344 workflow_id: Some("wf-rust-pages".to_string()),
11345 run_id: Some("run-rust-pages".to_string()),
11346 workflow_type: "rust.hello_workflow".to_string(),
11347 payload_codec: DEFAULT_CODEC.to_string(),
11348 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
11349 history_events: vec![HistoryEvent {
11350 event_type: "WorkflowStarted".to_string(),
11351 payload: json!({}),
11352 raw: HashMap::new(),
11353 }],
11354 total_history_events: Some(3),
11355 history_size_bytes: None,
11356 continue_as_new_recommended: None,
11357 history_budget_pressure: None,
11358 next_history_page_token: Some("MQ==".to_string()),
11359 workflow_task_attempt: 1,
11360 workflow_signal_id: None,
11361 signal_name: None,
11362 signal_arguments: None,
11363 workflow_update_id: None,
11364 update_name: None,
11365 lease_owner: Some("rust-worker".to_string()),
11366 };
11367
11368 task.append_history_page(WorkflowTaskHistoryPage {
11369 history_events: vec![
11370 HistoryEvent {
11371 event_type: "SignalReceived".to_string(),
11372 payload: json!({
11373 "signal_id": "sig-rust-1",
11374 "signal_name": "start",
11375 "arguments": encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC)
11376 .expect("signal arguments")
11377 }),
11378 raw: HashMap::new(),
11379 },
11380 HistoryEvent {
11381 event_type: "MarkerRecorded".to_string(),
11382 payload: json!({"sequence": 3}),
11383 raw: HashMap::new(),
11384 },
11385 ],
11386 total_history_events: Some(3),
11387 next_history_page_token: None,
11388 });
11389
11390 assert_eq!(task.history_events.len(), 3);
11391 assert_eq!(task.total_history_events, Some(3));
11392 assert_eq!(task.next_history_page_token, None);
11393
11394 let signal = task
11395 .history_events
11396 .iter()
11397 .find(|event| event.event_type == "SignalReceived")
11398 .expect("signal event");
11399 assert_eq!(
11400 decode_signal_event_arguments(signal, DEFAULT_CODEC).expect("signal arguments"),
11401 vec![AvroValue::String("Rust".to_string())]
11402 );
11403 }
11404
11405 #[tokio::test]
11406 async fn query_handler_reads_ordered_cross_codec_signals_without_commands() {
11407 let client = Client::new("http://127.0.0.1:8080").expect("client");
11408 let mut worker = Worker::new(client, "rust-workers");
11409 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
11410 worker.register_query("counter", "current", |ctx, _args| async move {
11411 let mut count = 0_i64;
11412 for signal in ctx.signal_events() {
11413 let value = signal
11414 .arguments
11415 .first()
11416 .and_then(Value::as_i64)
11417 .unwrap_or_default();
11418 match signal.name.as_str() {
11419 "increment" => count += value,
11420 "set" => count = value,
11421 _ => {}
11422 }
11423 }
11424 Ok(json!(count))
11425 });
11426
11427 let task = QueryTask {
11428 query_task_id: "query-rust-counter".to_string(),
11429 query_task_attempt: 1,
11430 lease_owner: Some("rust-worker".to_string()),
11431 workflow_id: Some("counter-1".to_string()),
11432 run_id: Some("run-counter-1".to_string()),
11433 workflow_type: "counter".to_string(),
11434 query_name: "current".to_string(),
11435 payload_codec: DEFAULT_CODEC.to_string(),
11436 workflow_arguments: Some(
11437 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
11438 ),
11439 query_arguments: Some(
11440 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("query arguments"),
11441 ),
11442 history_events: vec![
11443 HistoryEvent {
11444 event_type: "SignalReceived".to_string(),
11445 payload: json!({
11446 "signal_id": "php-signal-1",
11447 "signal_name": "increment",
11448 "workflow_sequence": 1,
11449 "payload_codec": DEFAULT_CODEC,
11450 "arguments": encode_value_envelope(&json!([3]), DEFAULT_CODEC).expect("php avro signal")
11451 }),
11452 raw: HashMap::new(),
11453 },
11454 HistoryEvent {
11455 event_type: "SignalReceived".to_string(),
11456 payload: json!({
11457 "signal_id": "python-signal-2",
11458 "signal_name": "increment",
11459 "workflow_sequence": 2,
11460 "payload_codec": JSON_CODEC,
11461 "arguments": encode_value_envelope(&json!([5]), JSON_CODEC).expect("python json signal")
11462 }),
11463 raw: HashMap::new(),
11464 },
11465 HistoryEvent {
11466 event_type: "SignalReceived".to_string(),
11467 payload: json!({
11468 "signal_id": "rust-signal-3",
11469 "signal_name": "set",
11470 "workflow_sequence": 3,
11471 "payload_codec": DEFAULT_CODEC,
11472 "arguments": encode_value_envelope(&json!([0]), DEFAULT_CODEC).expect("rust avro signal")
11473 }),
11474 raw: HashMap::new(),
11475 },
11476 ],
11477 history_export: None,
11478 run_status: Some("completed".to_string()),
11479 };
11480
11481 let result = worker.execute_query_task(task).await.expect("query result");
11482 assert_eq!(result.into_json().expect("query projection"), json!(0));
11483 }
11484
11485 #[tokio::test]
11486 async fn replayed_queries_read_running_completed_and_cold_restarted_instance_state() {
11487 let worker = replay_counter_worker();
11488 let running_history = json!([
11489 {
11490 "type": "ActivityCompleted",
11491 "payload": {
11492 "sequence": 1,
11493 "activity_type": "load-counter",
11494 "payload_codec": "json",
11495 "result": {"codec": "json", "blob": "\"loaded\""}
11496 }
11497 },
11498 {
11499 "type": "SignalWaitOpened",
11500 "payload": {
11501 "sequence": 3,
11502 "signal_name": "increment"
11503 }
11504 },
11505 {
11506 "type": "SignalReceived",
11507 "payload": {
11508 "signal_id": "signal-3",
11509 "signal_name": "increment",
11510 "workflow_sequence": 2,
11511 "payload_codec": "json",
11512 "arguments": {"codec": "json", "blob": "[3]"}
11513 }
11514 },
11515 {
11516 "type": "SignalApplied",
11517 "payload": {
11518 "sequence": 3,
11519 "signal_id": "signal-3",
11520 "signal_name": "increment",
11521 "payload_codec": "json",
11522 "value": {"codec": "json", "blob": "[3]"}
11523 }
11524 }
11525 ]);
11526
11527 let running = worker
11528 .execute_query_task(replay_counter_query(
11529 "current",
11530 running_history.clone(),
11531 "running",
11532 ))
11533 .await
11534 .expect("running replay query");
11535 assert_eq!(
11536 running.clone().into_json().expect("query projection"),
11537 json!({"loaded": "loaded", "count": 3, "finished": false})
11538 );
11539
11540 let detached = worker
11541 .execute_query_task(replay_counter_query(
11542 "detached-mutation",
11543 running_history.clone(),
11544 "running",
11545 ))
11546 .await
11547 .expect("query mutates only its detached state clone");
11548 assert_eq!(detached.into_json().expect("query projection"), json!(999));
11549 let failed = worker
11550 .execute_query_task(replay_counter_query(
11551 "failed-mutation",
11552 running_history.clone(),
11553 "running",
11554 ))
11555 .await
11556 .expect_err("failed query");
11557 assert_eq!(failed.reason, "query_rejected");
11558 let unchanged = worker
11559 .execute_query_task(replay_counter_query("current", running_history, "running"))
11560 .await
11561 .expect("later query reconstructs unchanged state");
11562 assert_eq!(unchanged, running);
11563
11564 let restarted_worker = replay_counter_worker();
11565 let restarted_task: QueryTask = serde_json::from_value(json!({
11566 "query_task_id": "query-after-restart",
11567 "workflow_id": "counter-1",
11568 "run_id": "run-counter-1",
11569 "workflow_type": "replay-counter",
11570 "query_name": "current",
11571 "payload_codec": "json",
11572 "workflow_arguments": {"codec": "json", "blob": "[]"},
11573 "query_arguments": {"codec": "json", "blob": "[]"},
11574 "history_events": [],
11575 "history_export": {
11576 "payloads": {"codec": "json"},
11577 "history_events": [
11578 {
11579 "type": "ActivityCompleted",
11580 "payload": {
11581 "sequence": 1,
11582 "activity_type": "load-counter",
11583 "payload_codec": "json",
11584 "result": null
11585 }
11586 },
11587 {
11588 "type": "SignalWaitOpened",
11589 "payload": {
11590 "sequence": 3,
11591 "signal_name": "increment"
11592 }
11593 },
11594 {
11595 "type": "SignalReceived",
11596 "payload": {
11597 "signal_id": "signal-3",
11598 "signal_name": "increment",
11599 "workflow_sequence": 2
11600 }
11601 },
11602 {
11603 "type": "SignalApplied",
11604 "payload": {
11605 "sequence": 3,
11606 "signal_id": "signal-3",
11607 "signal_name": "increment"
11608 }
11609 },
11610 {
11611 "type": "SignalWaitOpened",
11612 "payload": {
11613 "sequence": 5,
11614 "signal_name": "increment"
11615 }
11616 },
11617 {
11618 "type": "SignalReceived",
11619 "payload": {
11620 "signal_id": "signal-5",
11621 "signal_name": "increment",
11622 "workflow_sequence": 4
11623 }
11624 },
11625 {
11626 "type": "SignalApplied",
11627 "payload": {
11628 "sequence": 5,
11629 "signal_id": "signal-5",
11630 "signal_name": "increment"
11631 }
11632 }
11633 ],
11634 "activities": [{
11635 "sequence": 1,
11636 "activity_type": "load-counter",
11637 "payload_codec": "json",
11638 "result": {"codec": "json", "blob": "\"loaded\""}
11639 }],
11640 "signals": [
11641 {
11642 "id": "signal-3",
11643 "name": "increment",
11644 "workflow_sequence": 2,
11645 "payload_codec": "json",
11646 "arguments": "[3]"
11647 },
11648 {
11649 "id": "signal-5",
11650 "name": "increment",
11651 "workflow_sequence": 4,
11652 "payload_codec": "json",
11653 "arguments": "[5]"
11654 }
11655 ]
11656 },
11657 "run_status": "completed"
11658 }))
11659 .expect("cold replay query task");
11660 let completed = restarted_worker
11661 .execute_query_task(restarted_task)
11662 .await
11663 .expect("completed cold replay query");
11664 assert_eq!(
11665 completed.into_json().expect("query projection"),
11666 json!({"loaded": "loaded", "count": 8, "finished": true})
11667 );
11668 }
11669
11670 #[tokio::test]
11671 async fn replayed_query_replay_failures_are_machine_readable() {
11672 let worker = replay_counter_worker();
11673 let task = replay_counter_query(
11674 "current",
11675 json!([{
11676 "type": "ActivityCompleted",
11677 "payload": {
11678 "sequence": 1,
11679 "payload_codec": "json",
11680 "result": {"codec": "json", "blob": "{"}
11681 }
11682 }]),
11683 "running",
11684 );
11685 let failure = worker
11686 .execute_query_task(task)
11687 .await
11688 .expect_err("invalid replay history payload");
11689 assert_eq!(failure.reason, "query_workflow_state_unavailable");
11690 assert_eq!(failure.failure_type, "QueryWorkflowStateUnavailable");
11691 }
11692
11693 #[tokio::test]
11694 async fn query_task_restores_compact_history_from_export() {
11695 let client = Client::new("http://127.0.0.1:8080").expect("client");
11696 let mut worker = Worker::new(client, "rust-workers");
11697 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
11698 worker.register_query("counter", "current", |ctx, _args| async move {
11699 Ok(json!(ctx.signals("increment")[0][0]))
11700 });
11701 let task: QueryTask = serde_json::from_value(json!({
11702 "query_task_id": "query-export",
11703 "workflow_type": "counter",
11704 "query_name": "current",
11705 "payload_codec": "json",
11706 "workflow_arguments": {"codec": "json", "blob": "[]"},
11707 "query_arguments": {"codec": "json", "blob": "[]"},
11708 "history_events": [],
11709 "history_export": {
11710 "payloads": {"codec": "json"},
11711 "history_events": [{
11712 "type": "SignalReceived",
11713 "payload": {"signal_id": "signal-export", "signal_name": "increment"}
11714 }],
11715 "signals": [{
11716 "id": "signal-export",
11717 "name": "increment",
11718 "status": "applied",
11719 "workflow_sequence": 1,
11720 "payload_codec": "json",
11721 "arguments": "[9]"
11722 }]
11723 }
11724 }))
11725 .expect("query task");
11726
11727 let result = worker.execute_query_task(task).await.expect("query result");
11728 assert_eq!(result.into_json().expect("query projection"), json!(9));
11729 }
11730
11731 #[tokio::test]
11732 async fn query_task_failures_have_stable_reasons() {
11733 let client = Client::new("http://127.0.0.1:8080").expect("client");
11734 let mut worker = Worker::new(client, "rust-workers");
11735 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
11736 worker.register_query(
11737 "counter",
11738 "current",
11739 |_ctx, _args| async move { Ok(json!(0)) },
11740 );
11741
11742 let base_task = QueryTask {
11743 query_task_id: "query-errors".to_string(),
11744 query_task_attempt: 1,
11745 lease_owner: None,
11746 workflow_id: Some("counter-errors".to_string()),
11747 run_id: Some("run-errors".to_string()),
11748 workflow_type: "counter".to_string(),
11749 query_name: "missing".to_string(),
11750 payload_codec: JSON_CODEC.to_string(),
11751 workflow_arguments: Some(json!({"codec": "json", "blob": "[]"})),
11752 query_arguments: Some(json!({"codec": "json", "blob": "[]"})),
11753 history_events: Vec::new(),
11754 history_export: None,
11755 run_status: Some("running".to_string()),
11756 };
11757
11758 let unknown = worker
11759 .execute_query_task(base_task.clone())
11760 .await
11761 .expect_err("unknown query");
11762 assert_eq!(unknown.reason, "rejected_unknown_query");
11763
11764 let mut malformed = base_task;
11765 malformed.query_name = "current".to_string();
11766 malformed.query_arguments = Some(json!({"codec": "json", "blob": "{"}));
11767 let malformed = worker
11768 .execute_query_task(malformed)
11769 .await
11770 .expect_err("malformed payload");
11771 assert_eq!(malformed.reason, "query_payload_decode_failed");
11772
11773 let client = Client::new("http://127.0.0.1:8080").expect("client");
11774 let mut unavailable_worker = Worker::new(client, "rust-workers");
11775 unavailable_worker
11776 .register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
11777 let unavailable_task: QueryTask = serde_json::from_value(json!({
11778 "query_task_id": "query-unavailable",
11779 "workflow_type": "counter",
11780 "query_name": "current",
11781 "payload_codec": "json",
11782 "workflow_arguments": {"codec": "json", "blob": "[]"},
11783 "query_arguments": {"codec": "json", "blob": "[]"}
11784 }))
11785 .expect("query task");
11786 let unavailable = unavailable_worker
11787 .execute_query_task(unavailable_task)
11788 .await
11789 .expect_err("query handler unavailable");
11790 assert_eq!(unavailable.reason, "query_handler_unavailable");
11791 }
11792
11793 #[tokio::test]
11794 async fn client_query_decodes_result_and_typed_failure() {
11795 let server = MockWorkerServer::start();
11796 let client = Client::builder(server.base_url())
11797 .timeout(Duration::from_secs(2))
11798 .build()
11799 .expect("client");
11800
11801 let result = client
11802 .query_workflow("counter-1", "current", json!([]))
11803 .await
11804 .expect("query result");
11805 assert_eq!(result, json!({"count": 8}));
11806
11807 let error = client
11808 .query_workflow("counter-1", "missing", json!([]))
11809 .await
11810 .expect_err("unknown query");
11811 let Error::QueryFailed(failure) = error else {
11812 panic!("expected typed query failure");
11813 };
11814 assert_eq!(failure.status, 404);
11815 assert_eq!(failure.reason, "rejected_unknown_query");
11816 }
11817
11818 #[tokio::test]
11819 async fn public_client_surfaces_send_and_receive_lossless_avro_values() {
11820 let server = MockWorkerServer::start();
11821 let client = Client::builder(server.base_url())
11822 .timeout(Duration::from_secs(2))
11823 .build()
11824 .expect("client");
11825 let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
11826
11827 client
11828 .start_workflow(
11829 "typed.echo",
11830 "rust-workers",
11831 "typed-start",
11832 arguments.clone(),
11833 )
11834 .await
11835 .expect("typed workflow start");
11836 assert_eq!(
11837 decode_wire_avro_value(
11838 &server.request_body("/api/workflows")["input"],
11839 DEFAULT_CODEC,
11840 )
11841 .expect("typed start input"),
11842 arguments
11843 );
11844
11845 client
11846 .signal_workflow("typed-1", "changed", arguments.clone())
11847 .await
11848 .expect("typed signal");
11849 assert_eq!(
11850 decode_wire_avro_value(
11851 &server.request_body("/api/workflows/typed-1/signal/changed")["input"],
11852 DEFAULT_CODEC,
11853 )
11854 .expect("typed signal input"),
11855 arguments
11856 );
11857
11858 assert_eq!(
11859 client
11860 .query_workflow_avro_value("typed-1", "inspect", arguments.clone())
11861 .await
11862 .expect("typed query"),
11863 typed_fidelity_probe()
11864 );
11865 assert_eq!(
11866 decode_wire_avro_value(
11867 &server.request_body("/api/workflows/typed-1/query/inspect")["input"],
11868 DEFAULT_CODEC,
11869 )
11870 .expect("typed query input"),
11871 arguments
11872 );
11873
11874 assert_eq!(
11875 client
11876 .update_workflow_avro_value(
11877 "typed-1",
11878 "replace",
11879 arguments.clone(),
11880 Some("typed-request"),
11881 )
11882 .await
11883 .expect("typed update"),
11884 typed_fidelity_probe()
11885 );
11886 let update = server.request_body("/api/workflows/typed-1/update/replace");
11887 assert_eq!(update["request_id"], "typed-request");
11888 assert_eq!(
11889 decode_wire_avro_value(&update["input"], DEFAULT_CODEC).expect("typed update input"),
11890 arguments
11891 );
11892
11893 let handle = WorkflowHandle {
11894 client: client.clone(),
11895 workflow_id: "typed-1".to_string(),
11896 run_id: Some("run-typed-1".to_string()),
11897 workflow_type: "typed.echo".to_string(),
11898 };
11899 assert_eq!(
11900 handle
11901 .result_avro_value(WorkflowResultOptions::default())
11902 .await
11903 .expect("typed workflow result"),
11904 typed_fidelity_probe()
11905 );
11906
11907 client
11908 .complete_activity_task(
11909 "activity-typed",
11910 "attempt-typed",
11911 "rust-worker",
11912 typed_fidelity_probe(),
11913 DEFAULT_CODEC,
11914 )
11915 .await
11916 .expect("typed activity completion");
11917 assert_eq!(
11918 decode_wire_avro_value(
11919 &server.request_body("/api/worker/activity-tasks/activity-typed/complete")
11920 ["result"],
11921 DEFAULT_CODEC,
11922 )
11923 .expect("typed activity result"),
11924 typed_fidelity_probe()
11925 );
11926 client
11927 .fail_activity_task(
11928 "activity-typed",
11929 "attempt-typed",
11930 "rust-worker",
11931 "typed failure",
11932 true,
11933 )
11934 .await
11935 .expect("activity failure");
11936 }
11937
11938 #[tokio::test]
11939 async fn lifecycle_commands_support_instance_and_selected_run_targets() {
11940 let server = MockWorkerServer::start();
11941 let client = Client::builder(server.base_url())
11942 .timeout(Duration::from_secs(2))
11943 .build()
11944 .expect("client");
11945
11946 let options = WorkflowCommandOptions::new()
11947 .reason("cleanup requested")
11948 .request_id("cancel-17");
11949 let cancelled = client
11950 .cancel_workflow("wf-lifecycle", options)
11951 .await
11952 .expect("instance cancellation");
11953 assert_eq!(cancelled.command, WorkflowCommandKind::Cancel);
11954 assert_eq!(cancelled.run_id.as_deref(), Some("run-current"));
11955 assert_eq!(cancelled.outcome.as_deref(), Some("cancelled"));
11956 assert_eq!(
11957 server.request_body("/api/workflows/wf-lifecycle/cancel"),
11958 json!({"reason":"cleanup requested","request_id":"cancel-17"})
11959 );
11960
11961 let terminated = client
11962 .terminate_workflow(
11963 "wf-lifecycle",
11964 WorkflowCommandOptions::new().reason("forced stop"),
11965 )
11966 .await
11967 .expect("instance termination");
11968 assert_eq!(terminated.command, WorkflowCommandKind::Terminate);
11969 assert_eq!(terminated.outcome.as_deref(), Some("terminated"));
11970
11971 client
11972 .cancel_workflow_run(
11973 "wf-lifecycle",
11974 "run-current",
11975 WorkflowCommandOptions::default(),
11976 )
11977 .await
11978 .expect("selected run cancellation");
11979 client
11980 .terminate_workflow_run(
11981 "wf-lifecycle",
11982 "run-current",
11983 WorkflowCommandOptions::default(),
11984 )
11985 .await
11986 .expect("selected run termination");
11987
11988 for (command, error) in [
11989 (
11990 WorkflowCommandKind::Cancel,
11991 client
11992 .cancel_workflow_run(
11993 "wf-lifecycle",
11994 "run-stale",
11995 WorkflowCommandOptions::default(),
11996 )
11997 .await
11998 .expect_err("stale cancellation must be rejected"),
11999 ),
12000 (
12001 WorkflowCommandKind::Terminate,
12002 client
12003 .terminate_workflow_run(
12004 "wf-lifecycle",
12005 "run-stale",
12006 WorkflowCommandOptions::default(),
12007 )
12008 .await
12009 .expect_err("stale termination must be rejected"),
12010 ),
12011 ] {
12012 let Error::WorkflowCommandRejected(rejection) = error else {
12013 panic!("expected typed command rejection");
12014 };
12015 assert_eq!(rejection.command, command);
12016 assert_eq!(rejection.status, 409);
12017 assert_eq!(rejection.reason, "historical_run_command_rejected");
12018 assert_eq!(rejection.run_id.as_deref(), Some("run-stale"));
12019 assert_eq!(rejection.target_scope.as_deref(), Some("run"));
12020 }
12021 }
12022
12023 #[tokio::test]
12024 async fn workflow_start_options_send_server_enforced_deadlines() {
12025 let server = MockWorkerServer::start();
12026 let client = Client::builder(server.base_url())
12027 .timeout(Duration::from_secs(2))
12028 .build()
12029 .expect("client");
12030
12031 let handle = client
12032 .start_workflow_with_options(
12033 "rust.timeout",
12034 "rust-timeouts",
12035 "wf-start-options",
12036 WorkflowStartOptions::new()
12037 .execution_timeout_seconds(30)
12038 .run_timeout_seconds(1),
12039 json!([]),
12040 )
12041 .await
12042 .expect("workflow start");
12043
12044 assert_eq!(handle.run_id.as_deref(), Some("run-start-options"));
12045 let body = server.request_body("/api/workflows");
12046 assert_eq!(body["execution_timeout_seconds"], 30);
12047 assert_eq!(body["run_timeout_seconds"], 1);
12048
12049 let invalid = client
12050 .start_workflow_with_options(
12051 "rust.timeout",
12052 "rust-timeouts",
12053 "wf-invalid-options",
12054 WorkflowStartOptions::new()
12055 .execution_timeout_seconds(1)
12056 .run_timeout_seconds(2),
12057 json!([]),
12058 )
12059 .await
12060 .expect_err("invalid deadline ordering");
12061 assert!(invalid
12062 .to_string()
12063 .contains("run_timeout_seconds cannot exceed execution_timeout_seconds"));
12064 }
12065
12066 #[tokio::test]
12067 async fn workflow_result_returns_each_typed_terminal_outcome() {
12068 let server = MockWorkerServer::start();
12069 let client = Client::builder(server.base_url())
12070 .timeout(Duration::from_secs(2))
12071 .build()
12072 .expect("client");
12073 let options = WorkflowResultOptions {
12074 poll_interval: Duration::ZERO,
12075 timeout: Duration::from_secs(1),
12076 };
12077
12078 let failed = WorkflowHandle {
12079 client: client.clone(),
12080 workflow_id: "wf-failed".to_string(),
12081 run_id: Some("run-failed".to_string()),
12082 workflow_type: "failure".to_string(),
12083 }
12084 .result(options)
12085 .await
12086 .expect_err("failed outcome");
12087 let Error::WorkflowFailed(failure) = failed else {
12088 panic!("expected WorkflowFailed");
12089 };
12090 assert_eq!(failure.workflow_id, "wf-failed");
12091 assert_eq!(failure.run_id.as_deref(), Some("run-failed"));
12092 assert_eq!(failure.failure_id.as_deref(), Some("failure-17"));
12093 assert_eq!(failure.failure_category.as_deref(), Some("application"));
12094 assert_eq!(failure.exception_type.as_deref(), Some("PaymentError"));
12095 assert_eq!(
12096 failure.exception_class.as_deref(),
12097 Some("billing::PaymentError")
12098 );
12099 assert_eq!(failure.non_retryable, Some(true));
12100
12101 for (workflow_id, expected_kind, expected_reason) in [
12102 (
12103 "wf-cancelled",
12104 WorkflowTerminalKind::Cancelled,
12105 "cleanup requested",
12106 ),
12107 (
12108 "wf-terminated",
12109 WorkflowTerminalKind::Terminated,
12110 "forced stop",
12111 ),
12112 (
12113 "wf-timed-out",
12114 WorkflowTerminalKind::TimedOut,
12115 "run_timeout",
12116 ),
12117 ] {
12118 let error = WorkflowHandle {
12119 client: client.clone(),
12120 workflow_id: workflow_id.to_string(),
12121 run_id: None,
12122 workflow_type: "terminal".to_string(),
12123 }
12124 .result(options)
12125 .await
12126 .expect_err("typed terminal outcome");
12127 let outcome = match error {
12128 Error::WorkflowCancelled(outcome) => outcome,
12129 Error::WorkflowTerminated(outcome) => outcome,
12130 Error::WorkflowTimedOut(outcome) => outcome,
12131 other => panic!("unexpected terminal error: {other}"),
12132 };
12133 assert_eq!(outcome.kind, expected_kind);
12134 assert_eq!(outcome.workflow_id, workflow_id);
12135 assert_eq!(outcome.reason, expected_reason);
12136 }
12137
12138 let wait_timeout = WorkflowHandle {
12139 client,
12140 workflow_id: "wf-waiting".to_string(),
12141 run_id: Some("run-waiting".to_string()),
12142 workflow_type: "waiting".to_string(),
12143 }
12144 .result(WorkflowResultOptions {
12145 poll_interval: Duration::ZERO,
12146 timeout: Duration::ZERO,
12147 })
12148 .await
12149 .expect_err("client wait timeout");
12150 let Error::WorkflowTimedOut(timeout) = wait_timeout else {
12151 panic!("expected typed client timeout");
12152 };
12153 assert_eq!(timeout.reason, "result_wait_timeout");
12154 assert_eq!(timeout.failure_category.as_deref(), Some("client_timeout"));
12155 assert_eq!(timeout.run_id.as_deref(), Some("run-waiting"));
12156 }
12157
12158 #[tokio::test]
12159 async fn workflow_result_follows_chain_and_selected_result_preserves_history() {
12160 let server = MockWorkerServer::start();
12161 let client = Client::builder(server.base_url())
12162 .timeout(Duration::from_secs(2))
12163 .build()
12164 .expect("client");
12165
12166 let handle = WorkflowHandle {
12167 client,
12168 workflow_id: "wf-selected".to_string(),
12169 run_id: Some("run-selected".to_string()),
12170 workflow_type: "selected".to_string(),
12171 };
12172 let options = WorkflowResultOptions {
12173 poll_interval: Duration::ZERO,
12174 timeout: Duration::from_secs(1),
12175 };
12176
12177 let current = handle
12178 .result(options)
12179 .await
12180 .expect("instance result follows the current run");
12181 assert_eq!(current, json!("current run output"));
12182
12183 let error = handle
12184 .result_selected_run(options)
12185 .await
12186 .expect_err("the selected run is cancelled even though the current run completed");
12187
12188 let Error::WorkflowCancelled(outcome) = error else {
12189 panic!("expected selected run cancellation");
12190 };
12191 assert_eq!(outcome.run_id.as_deref(), Some("run-selected"));
12192 assert_eq!(outcome.reason, "selected run cancelled");
12193 assert_eq!(
12194 server.request_count("/api/workflows/wf-selected/runs/run-selected"),
12195 1
12196 );
12197 assert_eq!(server.request_count("/api/workflows/wf-selected"), 1);
12198 }
12199
12200 #[tokio::test]
12201 async fn poll_responses_decode_http_conflict_drain_as_a_stable_stop() {
12202 let server = MockWorkerServer::draining_polls();
12203 let client = Client::builder(server.base_url())
12204 .timeout(Duration::from_secs(2))
12205 .build()
12206 .expect("client");
12207
12208 let workflow = client
12209 .poll_workflow_task_response("draining-worker", "rust-workers", Duration::ZERO)
12210 .await
12211 .expect("workflow drain response");
12212 let activity = client
12213 .poll_activity_task_response("draining-worker", "rust-workers", Duration::ZERO)
12214 .await
12215 .expect("activity drain response");
12216 let query = client
12217 .poll_query_task_response("draining-worker", "rust-workers", Duration::ZERO)
12218 .await
12219 .expect("query drain response");
12220
12221 for outcome in [workflow.outcome(), activity.outcome(), query.outcome()] {
12222 assert_eq!(
12223 outcome,
12224 WorkerPollOutcome::Stop {
12225 poll_status: Some("draining".to_string()),
12226 reason: Some("worker_draining".to_string()),
12227 }
12228 );
12229 }
12230
12231 assert!(client
12232 .poll_workflow_task("draining-worker", "rust-workers", Duration::ZERO)
12233 .await
12234 .expect("compatibility poll")
12235 .is_none());
12236 }
12237
12238 #[tokio::test]
12239 async fn managed_worker_honors_drain_stop_for_every_task_family() {
12240 let server = MockWorkerServer::draining_polls();
12241 let client = Client::builder(server.base_url())
12242 .timeout(Duration::from_secs(2))
12243 .build()
12244 .expect("client");
12245
12246 let mut workflow_worker = Worker::new(client.clone(), "rust-workers")
12247 .worker_id("draining-workflow-worker")
12248 .poll_timeout(Duration::ZERO);
12249 workflow_worker.register_workflow("counter", |_ctx, _args| async { Ok(Value::Null) });
12250 workflow_worker
12251 .run()
12252 .await
12253 .expect("workflow drain is a clean stop");
12254
12255 let mut activity_worker = Worker::new(client.clone(), "rust-workers")
12256 .worker_id("draining-activity-worker")
12257 .poll_timeout(Duration::ZERO);
12258 activity_worker.register_activity("write", |_ctx, _args| async { Ok(Value::Null) });
12259 activity_worker
12260 .run()
12261 .await
12262 .expect("activity drain is a clean stop");
12263
12264 let mut query_worker = Worker::new(client, "rust-workers")
12265 .worker_id("draining-query-worker")
12266 .poll_timeout(Duration::ZERO);
12267 query_worker.register_query("counter", "current", |_ctx, _args| async {
12268 Ok(Value::Null)
12269 });
12270 query_worker
12271 .run()
12272 .await
12273 .expect("query drain is a clean stop");
12274 }
12275
12276 #[tokio::test]
12277 async fn activity_cancellation_and_late_completion_remain_machine_readable() {
12278 let server = MockWorkerServer::start();
12279 let client = Client::builder(server.base_url())
12280 .timeout(Duration::from_secs(2))
12281 .build()
12282 .expect("client");
12283
12284 let heartbeat = client
12285 .heartbeat_activity_task(
12286 "activity-cancel",
12287 "attempt-cancel",
12288 "rust-worker",
12289 typed_fidelity_probe(),
12290 )
12291 .await
12292 .expect("cancellation heartbeat");
12293 assert!(heartbeat.cancel_requested);
12294 assert!(heartbeat.should_stop());
12295 assert_eq!(heartbeat.reason.as_deref(), Some("run_cancelled"));
12296 assert_eq!(heartbeat.run_closed_reason.as_deref(), Some("cancelled"));
12297 let heartbeat_body =
12298 server.request_body("/api/worker/activity-tasks/activity-cancel/heartbeat");
12299 assert_eq!(heartbeat_body["details"]["codec"], DEFAULT_CODEC);
12300 assert_eq!(
12301 decode_wire_avro_value(&heartbeat_body["details"], DEFAULT_CODEC)
12302 .expect("typed heartbeat details"),
12303 typed_fidelity_probe()
12304 );
12305
12306 let error = client
12307 .complete_activity_task(
12308 "activity-cancel",
12309 "attempt-cancel",
12310 "rust-worker",
12311 json!({"late":true}),
12312 JSON_CODEC,
12313 )
12314 .await
12315 .expect_err("late completion must be refused");
12316 assert!(activity_task_rejection_is_final(&error));
12317 let Error::ActivityTaskRejected(rejection) = error else {
12318 panic!("expected typed activity rejection");
12319 };
12320 assert_eq!(rejection.status, 409);
12321 assert_eq!(rejection.reason, "run_cancelled");
12322 assert!(rejection.cancel_requested);
12323 assert_eq!(rejection.can_continue, Some(false));
12324 }
12325
12326 #[tokio::test]
12327 async fn managed_worker_survives_late_completion_and_restart_during_cancellation() {
12328 let server = MockWorkerServer::cancelled_activity();
12329 let client = Client::builder(server.base_url())
12330 .timeout(Duration::from_secs(2))
12331 .build()
12332 .expect("client");
12333 let cancellation_observed = Arc::new(AtomicBool::new(false));
12334 let observed = Arc::clone(&cancellation_observed);
12335 let mut worker = Worker::new(client.clone(), "rust-workers")
12336 .worker_id("rust-cancel-worker")
12337 .poll_timeout(Duration::from_millis(10));
12338 worker.register_activity("cancel-aware", move |ctx, _args| {
12339 let observed = Arc::clone(&observed);
12340 async move {
12341 let heartbeat = ctx.heartbeat(json!({"stage":"running"})).await?;
12342 observed.store(heartbeat.should_stop(), Ordering::SeqCst);
12343 Ok(json!({"late":"completion"}))
12344 }
12345 });
12346
12347 assert_eq!(
12348 worker.run_once().await.expect("cancelled attempt handled"),
12349 1
12350 );
12351 assert!(cancellation_observed.load(Ordering::SeqCst));
12352 assert_eq!(
12353 server.request_count("/api/worker/activity-tasks/activity-cancel/complete"),
12354 1
12355 );
12356
12357 let mut restarted = Worker::new(client, "rust-workers")
12358 .worker_id("rust-cancel-worker-restarted")
12359 .poll_timeout(Duration::from_millis(10));
12360 restarted.register_activity("cancel-aware", |_ctx, _args| async move { Ok(Value::Null) });
12361 assert_eq!(
12362 restarted
12363 .run_once()
12364 .await
12365 .expect("replacement worker continues polling"),
12366 0
12367 );
12368 }
12369
12370 #[tokio::test]
12371 async fn managed_worker_absorbs_selected_run_terminal_timeout_completion_race() {
12372 let response = r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"outcome":"completed","recorded":false,"run_id":"run-selected-timeout","run_status":"failed","created_task_ids":[],"reason":"run_timed_out"}"#;
12373 let server = MockWorkerServer::workflow_completion("409 Conflict", response);
12374 let client = Client::builder(server.base_url())
12375 .timeout(Duration::from_secs(2))
12376 .build()
12377 .expect("client");
12378
12379 let direct_error = client
12380 .complete_workflow_task(
12381 "workflow-timeout-task",
12382 "timeout-worker",
12383 3,
12384 vec![json!({"type": "complete_workflow", "result": null})],
12385 )
12386 .await
12387 .expect_err("the low-level client preserves the completion rejection");
12388 let Error::Http { status, body } = direct_error else {
12389 panic!("expected the original HTTP completion rejection");
12390 };
12391 assert_eq!(status, reqwest::StatusCode::CONFLICT);
12392 assert_eq!(
12393 serde_json::from_str::<Value>(&body).expect("response body")["reason"],
12394 "run_timed_out"
12395 );
12396
12397 let mut worker = Worker::new(client, "rust-workers")
12398 .worker_id("timeout-worker")
12399 .poll_timeout(Duration::from_millis(10));
12400 worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
12401 Ok(json!({"late": "result"}))
12402 });
12403
12404 assert_eq!(
12405 worker
12406 .run_once()
12407 .await
12408 .expect("authoritative selected-run timeout settles the tick"),
12409 1
12410 );
12411 assert_eq!(
12412 server.request_count("/api/worker/workflow-tasks/workflow-timeout-task/complete"),
12413 2,
12414 "both the direct client proof and managed worker must see the rejection"
12415 );
12416 }
12417
12418 #[tokio::test]
12419 async fn managed_worker_does_not_swallow_nearby_completion_errors() {
12420 for (name, status, response) in [
12421 ("bare conflict", "409 Conflict", r#"{"message":"conflict"}"#),
12422 (
12423 "command was recorded",
12424 "409 Conflict",
12425 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":true,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12426 ),
12427 (
12428 "lease conflict",
12429 "409 Conflict",
12430 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"lease_expired"}"#,
12431 ),
12432 (
12433 "nonterminal run",
12434 "409 Conflict",
12435 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"waiting","reason":"run_timed_out"}"#,
12436 ),
12437 (
12438 "different selected run",
12439 "409 Conflict",
12440 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-reused-workflow-current","run_status":"failed","reason":"run_timed_out"}"#,
12441 ),
12442 (
12443 "different task attempt",
12444 "409 Conflict",
12445 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":4,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12446 ),
12447 (
12448 "authentication failure",
12449 "401 Unauthorized",
12450 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12451 ),
12452 (
12453 "authorization failure",
12454 "403 Forbidden",
12455 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12456 ),
12457 (
12458 "protocol failure",
12459 "400 Bad Request",
12460 r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.2","requested_version":"1.3"}"#,
12461 ),
12462 (
12463 "malformed command",
12464 "422 Unprocessable Entity",
12465 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12466 ),
12467 (
12468 "transient server failure",
12469 "503 Service Unavailable",
12470 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12471 ),
12472 ] {
12473 let server = MockWorkerServer::workflow_completion(status, response);
12474 let client = Client::builder(server.base_url())
12475 .timeout(Duration::from_secs(2))
12476 .build()
12477 .expect("client");
12478 let mut worker = Worker::new(client, "rust-workers")
12479 .worker_id("timeout-worker")
12480 .poll_timeout(Duration::from_millis(10));
12481 worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
12482 Ok(json!({"late": "result"}))
12483 });
12484
12485 let error = worker
12486 .run_once()
12487 .await
12488 .expect_err(&format!("{name} must remain an error"));
12489 assert!(
12490 matches!(error, Error::Http { .. } | Error::Protocol(_)),
12491 "{name} returned an unexpected error variant: {error}"
12492 );
12493 }
12494 }
12495
12496 #[tokio::test]
12497 async fn worker_deregistration_uses_worker_plane_method_path_headers_and_result() {
12498 let server = MockWorkerServer::start();
12499 let client = Client::builder(server.base_url())
12500 .worker_token(Some("worker-secret".to_string()))
12501 .namespace("orders")
12502 .timeout(Duration::from_secs(2))
12503 .build()
12504 .expect("client");
12505 let path = "/api/worker/registrations/worker%2F%CE%B1%20space";
12506
12507 let result = client
12508 .deregister_worker_registration("worker/α space")
12509 .await
12510 .expect("deregister worker registration");
12511
12512 assert_eq!(server.method_for(path).as_deref(), Some("DELETE"));
12513 assert_eq!(
12514 server.worker_protocol_for(path).as_deref(),
12515 Some(WORKER_PROTOCOL_VERSION)
12516 );
12517 assert_eq!(server.control_protocol_for(path), None);
12518 assert_eq!(server.namespace_for(path).as_deref(), Some("orders"));
12519 assert_eq!(
12520 server.authorization_for(path).as_deref(),
12521 Some("Bearer worker-secret")
12522 );
12523 assert_eq!(
12524 result,
12525 WorkerDeregistrationEnvelope {
12526 worker_id: "deregistered-worker".to_string(),
12527 outcome: "deregistered".to_string(),
12528 recovered_workflow_task_count: 2,
12529 }
12530 );
12531 }
12532
12533 #[tokio::test]
12534 async fn role_scoped_tokens_are_never_used_for_the_opposite_plane() {
12535 let server = MockWorkerServer::start();
12536 let control_only = Client::builder(server.base_url())
12537 .control_token(Some("control-secret".to_string()))
12538 .build()
12539 .expect("control client");
12540
12541 let error = control_only
12542 .register_worker("worker", "queue", vec![], vec![], 1, 1)
12543 .await
12544 .expect_err("control token must not authorize a worker request");
12545 assert!(matches!(
12546 error,
12547 Error::MissingRoleCredentials { role: "worker", .. }
12548 ));
12549 assert_eq!(server.request_count("/api/worker/register"), 0);
12550
12551 let worker_only = Client::builder(server.base_url())
12552 .worker_token(Some("worker-secret".to_string()))
12553 .build()
12554 .expect("worker client");
12555 let error = worker_only
12556 .health()
12557 .await
12558 .expect_err("worker token must not authorize a control request");
12559 assert!(matches!(
12560 error,
12561 Error::MissingRoleCredentials {
12562 role: "control",
12563 ..
12564 }
12565 ));
12566 assert_eq!(server.request_count("/api/health"), 0);
12567 }
12568
12569 #[tokio::test]
12570 async fn shared_token_supports_worker_and_control_planes() {
12571 let server = MockWorkerServer::start();
12572 let client = Client::builder(server.base_url())
12573 .token(Some("shared-secret".to_string()))
12574 .build()
12575 .expect("client");
12576
12577 client.health().await.expect("control request");
12578 client
12579 .register_worker("worker", "queue", vec![], vec![], 1, 1)
12580 .await
12581 .expect("worker request");
12582
12583 assert_eq!(
12584 server.authorization_for("/api/health").as_deref(),
12585 Some("Bearer shared-secret")
12586 );
12587 assert_eq!(
12588 server.control_protocol_for("/api/health").as_deref(),
12589 Some(CONTROL_PLANE_VERSION)
12590 );
12591 assert_eq!(
12592 server.authorization_for("/api/worker/register").as_deref(),
12593 Some("Bearer shared-secret")
12594 );
12595 assert_eq!(
12596 server
12597 .worker_protocol_for("/api/worker/register")
12598 .as_deref(),
12599 Some(WORKER_PROTOCOL_VERSION)
12600 );
12601 }
12602
12603 #[tokio::test]
12604 async fn baseline_worker_endpoints_send_the_baseline_protocol() {
12605 let server = MockWorkerServer::start();
12606 let client = Client::builder(server.base_url())
12607 .timeout(Duration::from_secs(2))
12608 .build()
12609 .expect("client");
12610
12611 client
12612 .register_worker("capture-worker", "capture", vec![], vec![], 1, 1)
12613 .await
12614 .expect("register");
12615 client
12616 .heartbeat_worker("capture-worker", 1, 1)
12617 .await
12618 .expect("heartbeat");
12619 client
12620 .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
12621 .await
12622 .expect("workflow poll");
12623 client
12624 .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
12625 .await
12626 .expect("activity poll");
12627
12628 for path in [
12629 "/api/worker/register",
12630 "/api/worker/heartbeat",
12631 "/api/worker/workflow-tasks/poll",
12632 "/api/worker/activity-tasks/poll",
12633 ] {
12634 assert_eq!(
12635 server.worker_protocol_for(path).as_deref(),
12636 Some(WORKER_PROTOCOL_VERSION),
12637 "unexpected protocol for {path}"
12638 );
12639 }
12640
12641 assert_eq!(
12642 server.request_body("/api/worker/workflow-tasks/poll")["timeout_seconds"],
12643 1
12644 );
12645 assert_eq!(
12646 server.request_body("/api/worker/activity-tasks/poll")["timeout_seconds"],
12647 1
12648 );
12649 assert!(
12650 server.request_body("/api/worker/workflow-tasks/poll")["poll_request_id"]
12651 .as_str()
12652 .is_some_and(|id| id.starts_with("rust-workflow-poll-"))
12653 );
12654 assert!(
12655 server.request_body("/api/worker/activity-tasks/poll")["poll_request_id"]
12656 .as_str()
12657 .is_some_and(|id| id.starts_with("rust-activity-poll-"))
12658 );
12659 }
12660
12661 #[tokio::test]
12662 async fn query_task_endpoints_send_the_query_feature_protocol() {
12663 let server = MockWorkerServer::start();
12664 let client = Client::builder(server.base_url())
12665 .timeout(Duration::from_secs(2))
12666 .build()
12667 .expect("client");
12668
12669 client
12670 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
12671 .await
12672 .expect("query poll");
12673 client
12674 .complete_query_task("query-capture", "capture-worker", 1, json!(8), JSON_CODEC)
12675 .await
12676 .expect("query complete");
12677 client
12678 .fail_query_task(
12679 "query-capture",
12680 "capture-worker",
12681 1,
12682 "failed",
12683 "query_rejected",
12684 "QueryFailed",
12685 )
12686 .await
12687 .expect("query fail");
12688
12689 for path in [
12690 "/api/worker/query-tasks/poll",
12691 "/api/worker/query-tasks/query-capture/complete",
12692 "/api/worker/query-tasks/query-capture/fail",
12693 ] {
12694 assert_eq!(
12695 server.worker_protocol_for(path).as_deref(),
12696 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
12697 "unexpected protocol for {path}"
12698 );
12699 }
12700
12701 assert_eq!(
12702 server.request_body("/api/worker/query-tasks/poll")["timeout_seconds"],
12703 1
12704 );
12705 assert!(
12706 server.request_body("/api/worker/query-tasks/poll")["poll_request_id"]
12707 .as_str()
12708 .is_some_and(|id| id.starts_with("rust-query-poll-"))
12709 );
12710 }
12711
12712 #[tokio::test]
12713 async fn disconnected_client_polls_retry_once_with_the_same_request_id() {
12714 let server = MockWorkerServer::transient_worker_failures();
12715 let client = Client::builder(server.base_url())
12716 .timeout(Duration::from_secs(2))
12717 .build()
12718 .expect("client");
12719
12720 client
12721 .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
12722 .await
12723 .expect("workflow poll retry");
12724 client
12725 .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
12726 .await
12727 .expect("activity poll retry");
12728 client
12729 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
12730 .await
12731 .expect("query poll retry");
12732
12733 for path in [
12734 "/api/worker/workflow-tasks/poll",
12735 "/api/worker/activity-tasks/poll",
12736 "/api/worker/query-tasks/poll",
12737 ] {
12738 let bodies = server.request_bodies(path);
12739 assert_eq!(bodies.len(), 2, "{path} must be retried once");
12740 assert_eq!(
12741 bodies[0]["poll_request_id"], bodies[1]["poll_request_id"],
12742 "{path} must preserve the request binding across retry"
12743 );
12744 }
12745 }
12746
12747 #[tokio::test]
12748 async fn worker_poll_retries_preserve_request_id_across_consecutive_failures() {
12749 let server = MockWorkerServer::consecutive_poll_failures(2);
12750 let client = Client::builder(server.base_url())
12751 .timeout(Duration::from_secs(2))
12752 .build()
12753 .expect("client");
12754 let mut worker = Worker::new(client, "capture")
12755 .worker_id("capture-worker")
12756 .poll_timeout(Duration::from_millis(10))
12757 .retry_policy(WorkerRetryPolicy {
12758 max_retries: 2,
12759 initial_backoff: Duration::from_millis(1),
12760 max_backoff: Duration::from_millis(1),
12761 });
12762 worker.register_workflow(
12763 "capture.workflow",
12764 |_ctx, _input| async move { Ok(Value::Null) },
12765 );
12766 worker.register_activity(
12767 "capture.activity",
12768 |_ctx, _input| async move { Ok(Value::Null) },
12769 );
12770 worker.register_query("capture.workflow", "current", |_ctx, _args| async move {
12771 Ok(Value::Null)
12772 });
12773
12774 assert_eq!(worker.run_once().await.expect("poll retries"), 0);
12775
12776 for path in [
12777 "/api/worker/workflow-tasks/poll",
12778 "/api/worker/activity-tasks/poll",
12779 "/api/worker/query-tasks/poll",
12780 ] {
12781 let bodies = server.request_bodies(path);
12782 assert_eq!(bodies.len(), 3, "{path} must use exactly two retries");
12783 assert!(
12784 bodies
12785 .iter()
12786 .all(|body| body["poll_request_id"] == bodies[0]["poll_request_id"]),
12787 "{path} must preserve one request binding across every retry"
12788 );
12789 }
12790 }
12791
12792 #[tokio::test]
12793 async fn query_protocol_rejection_from_older_server_is_typed() {
12794 let server = MockWorkerServer::reject_query_protocol();
12795 let client = Client::builder(server.base_url())
12796 .timeout(Duration::from_secs(2))
12797 .build()
12798 .expect("client");
12799
12800 let error = client
12801 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
12802 .await
12803 .expect_err("server below query protocol floor must reject");
12804 let Error::Protocol(failure) = error else {
12805 panic!("expected typed protocol failure");
12806 };
12807
12808 assert_eq!(failure.status, 400);
12809 assert_eq!(failure.reason, "unsupported_protocol_version");
12810 assert_eq!(failure.supported_version.as_deref(), Some("1.7"));
12811 assert_eq!(
12812 failure.requested_version.as_deref(),
12813 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
12814 );
12815 assert_eq!(
12816 server
12817 .worker_protocol_for("/api/worker/query-tasks/poll")
12818 .as_deref(),
12819 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
12820 );
12821 }
12822
12823 #[tokio::test]
12824 async fn run_once_without_query_handlers_keeps_pre_query_server_compatibility() {
12825 let server = MockWorkerServer::reject_query_protocol();
12826 let client = Client::builder(server.base_url())
12827 .timeout(Duration::from_secs(2))
12828 .build()
12829 .expect("client");
12830 let mut worker = Worker::new(client, "rust-workers")
12831 .worker_id("baseline-worker")
12832 .poll_timeout(Duration::from_millis(10));
12833
12834 worker.register_workflow("baseline.workflow", |_ctx, _input| async move {
12835 Ok(Value::Null)
12836 });
12837
12838 assert_eq!(worker.run_once().await.expect("baseline run once"), 0);
12839 assert_eq!(
12840 server
12841 .worker_protocol_for("/api/worker/workflow-tasks/poll")
12842 .as_deref(),
12843 Some(WORKER_PROTOCOL_VERSION)
12844 );
12845 assert_eq!(
12846 server.worker_protocol_for("/api/worker/query-tasks/poll"),
12847 None,
12848 "a worker without query handlers must not use the query-task endpoint"
12849 );
12850 }
12851
12852 #[tokio::test]
12853 async fn completion_time_query_rejection_is_typed_without_stopping_worker() {
12854 let server = MockWorkerServer::reject_query_completion();
12855 let client = Client::builder(server.base_url())
12856 .timeout(Duration::from_secs(2))
12857 .build()
12858 .expect("client");
12859
12860 let error = client
12861 .complete_query_task("query-late", "late-worker", 1, json!(8), JSON_CODEC)
12862 .await
12863 .expect_err("expired completion must be rejected");
12864 let Error::QueryFailed(failure) = error else {
12865 panic!("expected typed query failure");
12866 };
12867 assert_eq!(failure.status, 409);
12868 assert_eq!(failure.reason, "query_task_timed_out");
12869
12870 let mut worker = Worker::new(client, "rust-workers")
12871 .worker_id("late-worker")
12872 .poll_timeout(Duration::from_millis(10));
12873 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
12874 worker.register_query(
12875 "counter",
12876 "current",
12877 |_ctx, _args| async move { Ok(json!(8)) },
12878 );
12879
12880 assert_eq!(worker.run_once().await.expect("late task is handled"), 1);
12881 assert_eq!(
12882 worker
12883 .run_once()
12884 .await
12885 .expect("worker continues after late completion"),
12886 0
12887 );
12888 assert_eq!(
12889 server.request_count("/api/worker/query-tasks/query-late/complete"),
12890 2
12891 );
12892 assert_eq!(
12893 server.request_count("/api/worker/query-tasks/query-late/fail"),
12894 0,
12895 "a server completion rejection must not be reported as an encoding failure"
12896 );
12897 }
12898
12899 #[tokio::test]
12900 async fn normal_shutdown_joins_pollers_and_deregisters_once() {
12901 let server = MockWorkerServer::start();
12902 let client = Client::builder(server.base_url())
12903 .timeout(Duration::from_secs(2))
12904 .build()
12905 .expect("client");
12906 let mut worker = Worker::new(client, "rust-workers")
12907 .worker_id("joined-worker")
12908 .poll_timeout(Duration::from_millis(10));
12909 worker.register_workflow(
12910 "joined.workflow",
12911 |_ctx, _input| async move { Ok(Value::Null) },
12912 );
12913 worker.register_activity(
12914 "joined.activity",
12915 |_ctx, _input| async move { Ok(Value::Null) },
12916 );
12917 worker.register_query("joined.workflow", "state", |_ctx, _input| async move {
12918 Ok(Value::Null)
12919 });
12920
12921 worker
12922 .run_until(tokio::time::sleep(Duration::from_millis(20)))
12923 .await
12924 .expect("normal shutdown");
12925
12926 let deregistration_path = "/api/worker/registrations/mock-worker";
12927 assert_eq!(server.request_count(deregistration_path), 1);
12928 for poll_path in [
12929 "/api/worker/workflow-tasks/poll",
12930 "/api/worker/activity-tasks/poll",
12931 "/api/worker/query-tasks/poll",
12932 ] {
12933 assert!(server.request_count(poll_path) > 0, "missing {poll_path}");
12934 }
12935 assert_eq!(
12936 server.captured_paths().last().map(String::as_str),
12937 Some(deregistration_path),
12938 "deregistration must start only after every poller has joined"
12939 );
12940 }
12941
12942 #[tokio::test]
12943 async fn registration_failure_does_not_deregister() {
12944 let server = MockWorkerServer::rejected_registration();
12945 let client = Client::builder(server.base_url())
12946 .timeout(Duration::from_secs(2))
12947 .build()
12948 .expect("client");
12949 let worker = Worker::new(client, "rust-workers").worker_id("never-registered");
12950
12951 let error = worker
12952 .run_until(async {})
12953 .await
12954 .expect_err("registration must fail");
12955 assert!(matches!(
12956 error,
12957 Error::Http {
12958 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
12959 ..
12960 }
12961 ));
12962 assert!(server
12963 .captured_paths()
12964 .iter()
12965 .all(|path| !path.starts_with("/api/worker/registrations/")));
12966 }
12967
12968 #[tokio::test]
12969 async fn declined_registration_does_not_deregister() {
12970 let server = MockWorkerServer::declined_registration();
12971 let client = Client::builder(server.base_url())
12972 .timeout(Duration::from_secs(2))
12973 .build()
12974 .expect("client");
12975 let worker = Worker::new(client, "rust-workers").worker_id("declined-worker");
12976
12977 let error = worker
12978 .run_until(async {})
12979 .await
12980 .expect_err("declined registration must fail");
12981 assert!(matches!(error, Error::WorkerLoop(_)));
12982 assert!(error.to_string().contains("was not accepted"));
12983 assert!(server
12984 .captured_paths()
12985 .iter()
12986 .all(|path| !path.starts_with("/api/worker/registrations/")));
12987 }
12988
12989 #[tokio::test]
12990 async fn deregistration_http_failure_is_returned_after_normal_shutdown() {
12991 let server = MockWorkerServer::rejected_deregistration();
12992 let client = Client::builder(server.base_url())
12993 .timeout(Duration::from_secs(2))
12994 .build()
12995 .expect("client");
12996 let worker = Worker::new(client, "rust-workers").worker_id("forbidden-cleanup");
12997
12998 let error = worker
12999 .run_until(async {})
13000 .await
13001 .expect_err("deregistration must fail");
13002 assert!(matches!(
13003 error,
13004 Error::Http {
13005 status: reqwest::StatusCode::FORBIDDEN,
13006 ..
13007 }
13008 ));
13009 assert_eq!(
13010 server.request_count("/api/worker/registrations/mock-worker"),
13011 1
13012 );
13013 }
13014
13015 #[tokio::test]
13016 async fn deregistration_protocol_failure_is_returned_after_normal_shutdown() {
13017 let server = MockWorkerServer::rejected_deregistration_protocol();
13018 let client = Client::builder(server.base_url())
13019 .timeout(Duration::from_secs(2))
13020 .build()
13021 .expect("client");
13022 let worker = Worker::new(client, "rust-workers").worker_id("protocol-cleanup");
13023
13024 let error = worker
13025 .run_until(async {})
13026 .await
13027 .expect_err("protocol rejection must fail shutdown");
13028 let Error::Protocol(failure) = error else {
13029 panic!("expected typed protocol failure");
13030 };
13031 assert_eq!(failure.reason, "unsupported_protocol_version");
13032 assert_eq!(failure.requested_version.as_deref(), Some("1.2"));
13033 assert_eq!(
13034 server.request_count("/api/worker/registrations/mock-worker"),
13035 1
13036 );
13037 }
13038
13039 #[tokio::test]
13040 async fn primary_poller_error_retains_deregistration_failure_context() {
13041 let server = MockWorkerServer::unauthorized_polls_and_rejected_deregistration();
13042 let client = Client::builder(server.base_url())
13043 .timeout(Duration::from_secs(2))
13044 .build()
13045 .expect("client");
13046 let mut worker = Worker::new(client, "rust-workers")
13047 .worker_id("combined-failure")
13048 .poll_timeout(Duration::from_millis(10));
13049 worker.register_workflow("combined.workflow", |_ctx, _input| async move {
13050 Ok(Value::Null)
13051 });
13052
13053 let error = worker
13054 .run()
13055 .await
13056 .expect_err("worker and cleanup must fail");
13057 let summary = error.to_string();
13058 assert!(summary.contains("authentication_failed"));
13059 assert!(summary.contains("worker cannot deregister"));
13060 let Error::WorkerShutdown {
13061 primary,
13062 deregistration,
13063 } = error
13064 else {
13065 panic!("expected combined worker shutdown error");
13066 };
13067 assert!(matches!(
13068 *primary,
13069 Error::Http {
13070 status: reqwest::StatusCode::UNAUTHORIZED,
13071 ..
13072 }
13073 ));
13074 assert!(matches!(
13075 *deregistration,
13076 Error::Http {
13077 status: reqwest::StatusCode::FORBIDDEN,
13078 ..
13079 }
13080 ));
13081 assert_eq!(
13082 server.request_count("/api/worker/registrations/mock-worker"),
13083 1
13084 );
13085 }
13086
13087 #[tokio::test]
13088 async fn activity_only_worker_can_shutdown_without_workflow_poller() {
13089 let server = MockWorkerServer::start();
13090 let client = Client::builder(server.base_url())
13091 .timeout(Duration::from_secs(2))
13092 .build()
13093 .expect("client");
13094 let mut worker = Worker::new(client, "rust-workers")
13095 .worker_id("activity-only-worker")
13096 .poll_timeout(Duration::from_millis(10));
13097
13098 worker.register_activity(
13099 "activity.only",
13100 |_ctx, _args| async move { Ok(Value::Null) },
13101 );
13102
13103 worker.run_until(async {}).await.expect("run worker");
13104 }
13105
13106 #[tokio::test]
13107 async fn workflow_only_worker_can_shutdown_without_activity_poller() {
13108 let server = MockWorkerServer::start();
13109 let client = Client::builder(server.base_url())
13110 .timeout(Duration::from_secs(2))
13111 .build()
13112 .expect("client");
13113 let mut worker = Worker::new(client, "rust-workers")
13114 .worker_id("workflow-only-worker")
13115 .poll_timeout(Duration::from_millis(10));
13116
13117 worker.register_workflow(
13118 "workflow.only",
13119 |_ctx, _input| async move { Ok(Value::Null) },
13120 );
13121
13122 worker.run_until(async {}).await.expect("run worker");
13123 }
13124
13125 #[tokio::test]
13126 async fn worker_heartbeat_observer_receives_server_acknowledgements() {
13127 let server = MockWorkerServer::start();
13128 let client = Client::builder(server.base_url())
13129 .timeout(Duration::from_secs(2))
13130 .build()
13131 .expect("client");
13132 let observations = Arc::new(Mutex::new(Vec::new()));
13133 let observed = Arc::clone(&observations);
13134 let mut worker = Worker::new(client, "rust-workers")
13135 .worker_id("observed-heartbeat-worker")
13136 .poll_timeout(Duration::from_millis(10))
13137 .on_worker_heartbeat(move |observation| {
13138 observed
13139 .lock()
13140 .expect("heartbeat observations")
13141 .push(observation.clone());
13142 });
13143
13144 worker.register_workflow("workflow.observed", |_ctx, _input| async move {
13145 Ok(Value::Null)
13146 });
13147 let acknowledged = Arc::clone(&observations);
13148 worker
13149 .run_until(async move {
13150 tokio::time::timeout(Duration::from_secs(2), async move {
13151 loop {
13152 if !acknowledged
13153 .lock()
13154 .expect("heartbeat observations")
13155 .is_empty()
13156 {
13157 break;
13158 }
13159 tokio::time::sleep(Duration::from_millis(1)).await;
13160 }
13161 })
13162 .await
13163 .expect("heartbeat acknowledgement within timeout");
13164 })
13165 .await
13166 .expect("run worker");
13167
13168 let observations = observations.lock().expect("heartbeat observations");
13169 let first = observations.first().expect("heartbeat acknowledgement");
13170 assert_eq!(first.worker_id, "observed-heartbeat-worker");
13171 assert_eq!(first.task_queue, "rust-workers");
13172 assert!(first.acknowledged_at_unix_millis > 0);
13173 assert_eq!(first.acknowledgement, json!({}));
13174 }
13175
13176 #[tokio::test]
13177 async fn delayed_worker_heartbeat_keeps_cadence_and_pollers_live() {
13178 let server = MockWorkerServer::delayed_heartbeat_worker();
13179 let client = Client::builder(server.base_url())
13180 .timeout(Duration::from_secs(3))
13181 .build()
13182 .expect("client");
13183 let observations = Arc::new(Mutex::new(Vec::new()));
13184 let observed = Arc::clone(&observations);
13185 let mut worker = Worker::new(client, "rust-snapshot-workers")
13186 .worker_id("rust-snapshot-worker")
13187 .poll_timeout(Duration::from_millis(10))
13188 .on_worker_heartbeat(move |observation| {
13189 observed
13190 .lock()
13191 .expect("heartbeat observations")
13192 .push(observation.clone());
13193 });
13194
13195 worker.register_workflow("snapshot", |ctx, _input| async move {
13196 ctx.wait_signal("finish").await?;
13197 Ok(json!({"status": "finished"}))
13198 });
13199 worker.register_query("snapshot", "current", |ctx, _args| async move {
13200 Ok(json!(ctx
13201 .signals("increment")
13202 .iter()
13203 .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
13204 .sum::<i64>()))
13205 });
13206 worker.register_activity("cancel-aware", |_ctx, _args| async move {
13207 Ok(json!({"late": "completion"}))
13208 });
13209
13210 worker
13211 .run_until(tokio::time::sleep(Duration::from_millis(3_800)))
13212 .await
13213 .expect("delayed heartbeat must allow a clean worker shutdown");
13214
13215 let observations = observations.lock().expect("heartbeat observations");
13216 assert!(
13217 observations.len() >= 3,
13218 "the immediate heartbeat, delayed acknowledgement, and next cadence heartbeat must complete"
13219 );
13220 assert!(
13221 observations.windows(2).all(|pair| {
13222 pair[1].acknowledged_at_unix_millis
13223 .saturating_sub(pair[0].acknowledged_at_unix_millis)
13224 >= 850
13225 }),
13226 "successful acknowledgements must not catch up faster than the advertised one-second cadence: {observations:?}"
13227 );
13228 drop(observations);
13229
13230 let heartbeat_times = server.request_times("/api/worker/heartbeat");
13231 let delayed_request_at = *heartbeat_times
13232 .get(1)
13233 .expect("intentionally delayed heartbeat request");
13234 let delay_window_start = delayed_request_at + Duration::from_millis(100);
13235 let delay_window_end = delayed_request_at + Duration::from_millis(1_400);
13236 for path in [
13237 "/api/worker/workflow-tasks/poll",
13238 "/api/worker/activity-tasks/poll",
13239 "/api/worker/query-tasks/poll",
13240 ] {
13241 assert!(
13242 server
13243 .request_times(path)
13244 .iter()
13245 .any(|received_at| *received_at >= delay_window_start
13246 && *received_at <= delay_window_end),
13247 "{path} must keep polling while a heartbeat acknowledgement is delayed"
13248 );
13249 }
13250 assert!(
13251 server.request_count("/api/worker/workflow-tasks/snapshot-wait-3/fail") >= 1,
13252 "workflow work must be settled"
13253 );
13254 assert!(
13255 server.request_count("/api/worker/activity-tasks/activity-cancel/complete") >= 1,
13256 "activity work must be settled"
13257 );
13258 assert!(
13259 server.request_count("/api/worker/query-tasks/snapshot-current/complete") >= 1,
13260 "query work must be settled"
13261 );
13262 }
13263
13264 #[tokio::test]
13265 async fn retried_worker_heartbeat_restarts_the_advertised_cadence() {
13266 let server = MockWorkerServer::heartbeat_retry_worker();
13267 let client = Client::builder(server.base_url())
13268 .timeout(Duration::from_secs(2))
13269 .build()
13270 .expect("client");
13271 let observations = Arc::new(Mutex::new(Vec::new()));
13272 let observed = Arc::clone(&observations);
13273 let worker = Worker::new(client, "rust-workers")
13274 .worker_id("heartbeat-retry-worker")
13275 .retry_policy(WorkerRetryPolicy {
13276 max_retries: 1,
13277 initial_backoff: Duration::from_millis(300),
13278 max_backoff: Duration::from_millis(300),
13279 })
13280 .on_worker_heartbeat(move |observation| {
13281 observed
13282 .lock()
13283 .expect("heartbeat observations")
13284 .push(observation.clone());
13285 });
13286
13287 worker
13288 .run_until(tokio::time::sleep(Duration::from_millis(2_700)))
13289 .await
13290 .expect("retryable heartbeat failure must remain bounded and recover");
13291
13292 let observations = observations.lock().expect("heartbeat observations");
13293 assert!(observations.len() >= 3, "heartbeat retry must recover");
13294 assert!(
13295 observations.windows(2).all(|pair| {
13296 pair[1]
13297 .acknowledged_at_unix_millis
13298 .saturating_sub(pair[0].acknowledged_at_unix_millis)
13299 >= 850
13300 }),
13301 "a successful retry must start a fresh advertised cadence: {observations:?}"
13302 );
13303 assert_eq!(
13304 server.request_count("/api/worker/heartbeat"),
13305 observations.len() + 1,
13306 "one retryable failure must add exactly one bounded request"
13307 );
13308 }
13309
13310 #[tokio::test]
13311 async fn query_enabled_worker_ignores_unmatched_signals_then_completes_once() {
13312 let server = MockWorkerServer::waiting_query_worker();
13313 let client = Client::builder(server.base_url())
13314 .timeout(Duration::from_secs(2))
13315 .build()
13316 .expect("client");
13317 let observations = Arc::new(Mutex::new(Vec::new()));
13318 let observed = Arc::clone(&observations);
13319 let mut worker = Worker::new(client, "rust-snapshot-workers")
13320 .worker_id("rust-snapshot-worker")
13321 .poll_timeout(Duration::from_millis(10))
13322 .on_worker_heartbeat(move |observation| {
13323 observed
13324 .lock()
13325 .expect("heartbeat observations")
13326 .push(observation.clone());
13327 });
13328
13329 worker.register_workflow("snapshot", |ctx, _input| async move {
13330 ctx.wait_signal("finish").await?;
13331 Ok(json!({"status": "finished"}))
13332 });
13333 worker.register_query("snapshot", "current", |ctx, _args| async move {
13334 let current = ctx
13335 .signals("increment")
13336 .iter()
13337 .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
13338 .sum::<i64>();
13339 Ok(json!(current))
13340 });
13341 worker.register_update("snapshot", "replace", |_ctx, args| async move { Ok(args) });
13342
13343 worker
13344 .run_until(tokio::time::sleep(Duration::from_millis(3_200)))
13345 .await
13346 .expect("pending workflow and query poller must remain live until shutdown");
13347
13348 assert!(
13349 observations.lock().expect("heartbeat observations").len() >= 4,
13350 "the immediate heartbeat and at least three advertised one-second intervals must be acknowledged"
13351 );
13352 assert!(
13353 server.request_count("/api/worker/workflow-tasks/poll") >= 3,
13354 "workflow polling must continue after empty replay acknowledgements"
13355 );
13356 assert!(
13357 server.request_count("/api/worker/query-tasks/poll") >= 2,
13358 "query polling must continue after serving the current query"
13359 );
13360 assert_eq!(
13361 server.request_body("/api/worker/register")["capabilities"],
13362 json!([QUERY_TASKS_CAPABILITY, WORKFLOW_UPDATES_CAPABILITY])
13363 );
13364 assert_eq!(
13365 server.request_body("/api/worker/register")["workflow_command_contracts"]["snapshot"],
13366 json!({
13367 "queries": ["current"],
13368 "updates": ["replace"],
13369 })
13370 );
13371
13372 let opened = server.request_body("/api/worker/workflow-tasks/snapshot-open/complete");
13373 assert_eq!(
13374 opened["commands"],
13375 json!([{
13376 "type": "open_signal_wait",
13377 "signal_name": "finish",
13378 }])
13379 );
13380
13381 for task_id in ["snapshot-wait-3", "snapshot-wait-5"] {
13382 let fail_path = format!("/api/worker/workflow-tasks/{task_id}/fail");
13383 let completion_path = format!("/api/worker/workflow-tasks/{task_id}/complete");
13384 let failure = server.request_body(&fail_path);
13385 assert_eq!(
13386 failure["failure"]["type"],
13387 WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE
13388 );
13389 assert_eq!(server.request_count(&completion_path), 0);
13390 }
13391
13392 let query_completion =
13393 server.request_body("/api/worker/query-tasks/snapshot-current/complete");
13394 assert_eq!(query_completion["result"], json!(8));
13395
13396 let terminal_path = "/api/worker/workflow-tasks/snapshot-finish/complete";
13397 assert_eq!(
13398 server.request_count(terminal_path),
13399 1,
13400 "the matching signal must settle the workflow exactly once"
13401 );
13402 let terminal = server.request_body(terminal_path);
13403 assert_eq!(terminal["commands"].as_array().map(Vec::len), Some(1));
13404 assert_eq!(terminal["commands"][0]["type"], "complete_workflow");
13405 assert_eq!(
13406 decode_wire_value(&terminal["commands"][0]["result"], DEFAULT_CODEC)
13407 .expect("terminal workflow result"),
13408 json!({"status": "finished"})
13409 );
13410 }
13411
13412 #[tokio::test]
13413 async fn worker_retries_poll_and_heartbeat_transport_failures_independently() {
13414 let server = MockWorkerServer::transient_worker_failures();
13415 let client = Client::builder(server.base_url())
13416 .timeout(Duration::from_secs(2))
13417 .build()
13418 .expect("client");
13419 let mut worker = Worker::new(client, "rust-workers")
13420 .worker_id("retry-worker")
13421 .poll_timeout(Duration::from_millis(10))
13422 .retry_policy(WorkerRetryPolicy {
13423 max_retries: 2,
13424 initial_backoff: Duration::from_millis(1),
13425 max_backoff: Duration::from_millis(1),
13426 });
13427 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
13428 worker.register_activity(
13429 "counter.activity",
13430 |_ctx, _input| async move { Ok(Value::Null) },
13431 );
13432 worker.register_query(
13433 "counter",
13434 "current",
13435 |_ctx, _args| async move { Ok(json!(8)) },
13436 );
13437
13438 worker
13439 .run_until(tokio::time::sleep(Duration::from_millis(75)))
13440 .await
13441 .expect("transient failures must not stop the worker");
13442
13443 for path in [
13444 "/api/worker/heartbeat",
13445 "/api/worker/workflow-tasks/poll",
13446 "/api/worker/activity-tasks/poll",
13447 "/api/worker/query-tasks/poll",
13448 ] {
13449 assert!(
13450 server.request_count(path) >= 2,
13451 "{path} must continue after its transient failure"
13452 );
13453 }
13454 }
13455
13456 #[tokio::test]
13457 async fn worker_bounds_transport_retries() {
13458 let server = MockWorkerServer::unavailable_polls();
13459 let client = Client::builder(server.base_url())
13460 .timeout(Duration::from_secs(2))
13461 .build()
13462 .expect("client");
13463 let mut worker = Worker::new(client, "rust-workers")
13464 .worker_id("bounded-retry-worker")
13465 .poll_timeout(Duration::from_millis(10))
13466 .retry_policy(WorkerRetryPolicy {
13467 max_retries: 2,
13468 initial_backoff: Duration::from_millis(1),
13469 max_backoff: Duration::from_millis(1),
13470 });
13471 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
13472
13473 let error = worker.run().await.expect_err("retry bound must terminate");
13474 assert!(matches!(error, Error::Transport(_)));
13475 assert_eq!(
13476 server.request_count("/api/worker/workflow-tasks/poll"),
13477 3,
13478 "one initial request plus exactly two retries"
13479 );
13480 }
13481
13482 #[tokio::test]
13483 async fn worker_retry_policy_can_disable_poll_retries() {
13484 let server = MockWorkerServer::unavailable_polls();
13485 let client = Client::builder(server.base_url())
13486 .timeout(Duration::from_secs(2))
13487 .build()
13488 .expect("client");
13489 let mut worker = Worker::new(client, "rust-workers")
13490 .worker_id("no-retry-worker")
13491 .poll_timeout(Duration::from_millis(10))
13492 .retry_policy(WorkerRetryPolicy {
13493 max_retries: 0,
13494 initial_backoff: Duration::from_millis(1),
13495 max_backoff: Duration::from_millis(1),
13496 });
13497 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
13498
13499 let error = worker
13500 .run_once()
13501 .await
13502 .expect_err("disabled retries must return the first transport failure");
13503 assert!(matches!(error, Error::Transport(_)));
13504 assert_eq!(
13505 server.request_count("/api/worker/workflow-tasks/poll"),
13506 1,
13507 "max_retries=0 must send only the initial request"
13508 );
13509 }
13510
13511 #[tokio::test]
13512 async fn worker_does_not_retry_authentication_failures() {
13513 let server = MockWorkerServer::unauthorized_polls();
13514 let client = Client::builder(server.base_url())
13515 .timeout(Duration::from_secs(2))
13516 .build()
13517 .expect("client");
13518 let mut worker = Worker::new(client, "rust-workers")
13519 .worker_id("unauthorized-worker")
13520 .poll_timeout(Duration::from_millis(10));
13521 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
13522
13523 let error = worker
13524 .run()
13525 .await
13526 .expect_err("authentication must terminate");
13527 let Error::Http { status, body } = error else {
13528 panic!("expected stable HTTP authentication error");
13529 };
13530 assert_eq!(status, reqwest::StatusCode::UNAUTHORIZED);
13531 assert!(body.contains("authentication_failed"));
13532 assert_eq!(
13533 server.request_count("/api/worker/workflow-tasks/poll"),
13534 1,
13535 "authentication failures must not be retried"
13536 );
13537 }
13538
13539 #[derive(Clone, Debug)]
13540 struct CapturedRequest {
13541 method: String,
13542 path: String,
13543 authorization: Option<String>,
13544 namespace: Option<String>,
13545 worker_protocol: Option<String>,
13546 control_protocol: Option<String>,
13547 body: String,
13548 received_at: Instant,
13549 }
13550
13551 struct MockWorkerServer {
13552 addr: SocketAddr,
13553 stop: Arc<AtomicBool>,
13554 requests: Arc<Mutex<Vec<CapturedRequest>>>,
13555 thread: Option<thread::JoinHandle<()>>,
13556 }
13557
13558 #[derive(Clone, Copy, Default)]
13559 struct MockWorkerBehavior {
13560 reject_query_protocol: bool,
13561 reject_query_completion: bool,
13562 waiting_query_worker: bool,
13563 decline_registration: bool,
13564 complete_named_signal: bool,
13565 poll_failures_per_path: usize,
13566 heartbeat_failures: usize,
13567 heartbeat_failure_request: Option<usize>,
13568 delayed_heartbeat_request: Option<usize>,
13569 heartbeat_response_delay: Duration,
13570 concurrent_requests: bool,
13571 unauthorized_polls: bool,
13572 reject_registration: bool,
13573 reject_deregistration: bool,
13574 reject_deregistration_protocol: bool,
13575 cancelled_activity: bool,
13576 draining_polls: bool,
13577 workflow_completion_status: Option<&'static str>,
13578 workflow_completion_body: Option<&'static str>,
13579 }
13580
13581 impl MockWorkerServer {
13582 fn start() -> Self {
13583 Self::start_with_behavior(MockWorkerBehavior::default())
13584 }
13585
13586 fn reject_query_protocol() -> Self {
13587 Self::start_with_behavior(MockWorkerBehavior {
13588 reject_query_protocol: true,
13589 ..MockWorkerBehavior::default()
13590 })
13591 }
13592
13593 fn reject_query_completion() -> Self {
13594 Self::start_with_behavior(MockWorkerBehavior {
13595 reject_query_completion: true,
13596 ..MockWorkerBehavior::default()
13597 })
13598 }
13599
13600 fn waiting_query_worker() -> Self {
13601 Self::start_with_behavior(MockWorkerBehavior {
13602 waiting_query_worker: true,
13603 complete_named_signal: true,
13604 ..MockWorkerBehavior::default()
13605 })
13606 }
13607
13608 fn transient_worker_failures() -> Self {
13609 Self::start_with_behavior(MockWorkerBehavior {
13610 poll_failures_per_path: 1,
13611 heartbeat_failures: 1,
13612 ..MockWorkerBehavior::default()
13613 })
13614 }
13615
13616 fn consecutive_poll_failures(count: usize) -> Self {
13617 Self::start_with_behavior(MockWorkerBehavior {
13618 poll_failures_per_path: count,
13619 ..MockWorkerBehavior::default()
13620 })
13621 }
13622
13623 fn delayed_heartbeat_worker() -> Self {
13624 Self::start_with_behavior(MockWorkerBehavior {
13625 waiting_query_worker: true,
13626 delayed_heartbeat_request: Some(2),
13627 heartbeat_response_delay: Duration::from_millis(1_500),
13628 concurrent_requests: true,
13629 cancelled_activity: true,
13630 ..MockWorkerBehavior::default()
13631 })
13632 }
13633
13634 fn heartbeat_retry_worker() -> Self {
13635 Self::start_with_behavior(MockWorkerBehavior {
13636 waiting_query_worker: true,
13637 heartbeat_failure_request: Some(2),
13638 concurrent_requests: true,
13639 ..MockWorkerBehavior::default()
13640 })
13641 }
13642
13643 fn unavailable_polls() -> Self {
13644 Self::start_with_behavior(MockWorkerBehavior {
13645 poll_failures_per_path: usize::MAX,
13646 ..MockWorkerBehavior::default()
13647 })
13648 }
13649
13650 fn unauthorized_polls() -> Self {
13651 Self::start_with_behavior(MockWorkerBehavior {
13652 unauthorized_polls: true,
13653 ..MockWorkerBehavior::default()
13654 })
13655 }
13656
13657 fn rejected_registration() -> Self {
13658 Self::start_with_behavior(MockWorkerBehavior {
13659 reject_registration: true,
13660 ..MockWorkerBehavior::default()
13661 })
13662 }
13663
13664 fn declined_registration() -> Self {
13665 Self::start_with_behavior(MockWorkerBehavior {
13666 decline_registration: true,
13667 ..MockWorkerBehavior::default()
13668 })
13669 }
13670
13671 fn rejected_deregistration() -> Self {
13672 Self::start_with_behavior(MockWorkerBehavior {
13673 reject_deregistration: true,
13674 ..MockWorkerBehavior::default()
13675 })
13676 }
13677
13678 fn rejected_deregistration_protocol() -> Self {
13679 Self::start_with_behavior(MockWorkerBehavior {
13680 reject_deregistration_protocol: true,
13681 ..MockWorkerBehavior::default()
13682 })
13683 }
13684
13685 fn unauthorized_polls_and_rejected_deregistration() -> Self {
13686 Self::start_with_behavior(MockWorkerBehavior {
13687 unauthorized_polls: true,
13688 reject_deregistration: true,
13689 ..MockWorkerBehavior::default()
13690 })
13691 }
13692
13693 fn cancelled_activity() -> Self {
13694 Self::start_with_behavior(MockWorkerBehavior {
13695 cancelled_activity: true,
13696 ..MockWorkerBehavior::default()
13697 })
13698 }
13699
13700 fn draining_polls() -> Self {
13701 Self::start_with_behavior(MockWorkerBehavior {
13702 draining_polls: true,
13703 ..MockWorkerBehavior::default()
13704 })
13705 }
13706
13707 fn workflow_completion(status: &'static str, body: &'static str) -> Self {
13708 Self::start_with_behavior(MockWorkerBehavior {
13709 workflow_completion_status: Some(status),
13710 workflow_completion_body: Some(body),
13711 ..MockWorkerBehavior::default()
13712 })
13713 }
13714
13715 fn start_with_behavior(behavior: MockWorkerBehavior) -> Self {
13716 let listener = TcpListener::bind("127.0.0.1:0").expect("bind mock server");
13717 listener
13718 .set_nonblocking(true)
13719 .expect("configure mock listener");
13720 let addr = listener.local_addr().expect("mock server address");
13721 let stop = Arc::new(AtomicBool::new(false));
13722 let server_stop = Arc::clone(&stop);
13723 let requests = Arc::new(Mutex::new(Vec::new()));
13724 let server_requests = Arc::clone(&requests);
13725 let thread = thread::spawn(move || {
13726 let mut request_threads = Vec::new();
13727 while !server_stop.load(Ordering::SeqCst) {
13728 match listener.accept() {
13729 Ok((mut stream, _)) => {
13730 if behavior.concurrent_requests {
13731 let requests = Arc::clone(&server_requests);
13732 request_threads.push(thread::spawn(move || {
13733 handle_mock_worker_request(&mut stream, &requests, behavior)
13734 }));
13735 } else {
13736 handle_mock_worker_request(&mut stream, &server_requests, behavior);
13737 }
13738 }
13739 Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
13740 let mut index = 0;
13741 while index < request_threads.len() {
13742 if request_threads[index].is_finished() {
13743 request_threads
13744 .swap_remove(index)
13745 .join()
13746 .expect("join mock request");
13747 } else {
13748 index += 1;
13749 }
13750 }
13751 thread::sleep(Duration::from_millis(5));
13752 }
13753 Err(_) => break,
13754 }
13755 }
13756 for request_thread in request_threads {
13757 request_thread.join().expect("join mock request");
13758 }
13759 });
13760
13761 Self {
13762 addr,
13763 stop,
13764 requests,
13765 thread: Some(thread),
13766 }
13767 }
13768
13769 fn base_url(&self) -> String {
13770 format!("http://{}", self.addr)
13771 }
13772
13773 fn worker_protocol_for(&self, path: &str) -> Option<String> {
13774 self.requests
13775 .lock()
13776 .expect("captured requests")
13777 .iter()
13778 .find(|request| request.path == path)
13779 .and_then(|request| request.worker_protocol.clone())
13780 }
13781
13782 fn control_protocol_for(&self, path: &str) -> Option<String> {
13783 self.requests
13784 .lock()
13785 .expect("captured requests")
13786 .iter()
13787 .find(|request| request.path == path)
13788 .and_then(|request| request.control_protocol.clone())
13789 }
13790
13791 fn method_for(&self, path: &str) -> Option<String> {
13792 self.requests
13793 .lock()
13794 .expect("captured requests")
13795 .iter()
13796 .find(|request| request.path == path)
13797 .map(|request| request.method.clone())
13798 }
13799
13800 fn authorization_for(&self, path: &str) -> Option<String> {
13801 self.requests
13802 .lock()
13803 .expect("captured requests")
13804 .iter()
13805 .find(|request| request.path == path)
13806 .and_then(|request| request.authorization.clone())
13807 }
13808
13809 fn namespace_for(&self, path: &str) -> Option<String> {
13810 self.requests
13811 .lock()
13812 .expect("captured requests")
13813 .iter()
13814 .find(|request| request.path == path)
13815 .and_then(|request| request.namespace.clone())
13816 }
13817
13818 fn request_count(&self, path: &str) -> usize {
13819 self.requests
13820 .lock()
13821 .expect("captured requests")
13822 .iter()
13823 .filter(|request| request.path == path)
13824 .count()
13825 }
13826
13827 fn captured_paths(&self) -> Vec<String> {
13828 self.requests
13829 .lock()
13830 .expect("captured requests")
13831 .iter()
13832 .map(|request| request.path.clone())
13833 .collect()
13834 }
13835
13836 fn request_times(&self, path: &str) -> Vec<Instant> {
13837 self.requests
13838 .lock()
13839 .expect("captured requests")
13840 .iter()
13841 .filter(|request| request.path == path)
13842 .map(|request| request.received_at)
13843 .collect()
13844 }
13845
13846 fn request_body(&self, path: &str) -> Value {
13847 let requests = self.requests.lock().expect("captured requests");
13848 let body = &requests
13849 .iter()
13850 .find(|request| request.path == path)
13851 .unwrap_or_else(|| panic!("missing request for {path}"))
13852 .body;
13853 serde_json::from_str(body).unwrap_or_else(|error| {
13854 panic!("invalid JSON request body for {path}: {error}: {body:?}")
13855 })
13856 }
13857
13858 fn request_bodies(&self, path: &str) -> Vec<Value> {
13859 self.requests
13860 .lock()
13861 .expect("captured requests")
13862 .iter()
13863 .filter(|request| request.path == path)
13864 .map(|request| {
13865 serde_json::from_str(&request.body).unwrap_or_else(|error| {
13866 panic!(
13867 "invalid JSON request body for {path}: {error}: {:?}",
13868 request.body
13869 )
13870 })
13871 })
13872 .collect()
13873 }
13874 }
13875
13876 impl Drop for MockWorkerServer {
13877 fn drop(&mut self) {
13878 self.stop.store(true, Ordering::SeqCst);
13879 let _ = TcpStream::connect(self.addr);
13880
13881 if let Some(thread) = self.thread.take() {
13882 thread.join().expect("join mock server");
13883 }
13884 }
13885 }
13886
13887 fn handle_mock_worker_request(
13888 stream: &mut TcpStream,
13889 requests: &Arc<Mutex<Vec<CapturedRequest>>>,
13890 behavior: MockWorkerBehavior,
13891 ) {
13892 let _ = stream.set_read_timeout(Some(Duration::from_millis(200)));
13893 let mut buffer = [0_u8; 8192];
13894 let mut request = Vec::new();
13895
13896 loop {
13897 match stream.read(&mut buffer) {
13898 Ok(0) => break,
13899 Ok(read) => {
13900 request.extend_from_slice(&buffer[..read]);
13901 if mock_request_is_complete(&request) {
13902 break;
13903 }
13904 }
13905 Err(error)
13906 if matches!(
13907 error.kind(),
13908 std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
13909 ) =>
13910 {
13911 break;
13912 }
13913 Err(_) => return,
13914 }
13915 }
13916
13917 let request = String::from_utf8_lossy(&request);
13918 let body = request
13919 .split_once("\r\n\r\n")
13920 .map(|(_, body)| body)
13921 .unwrap_or_default();
13922 let path = request
13923 .lines()
13924 .next()
13925 .and_then(|line| line.split_whitespace().nth(1))
13926 .unwrap_or_default();
13927 let method = request
13928 .lines()
13929 .next()
13930 .and_then(|line| line.split_whitespace().next())
13931 .unwrap_or_default();
13932 let authorization = request.lines().find_map(|line| {
13933 let (name, value) = line.split_once(':')?;
13934 name.eq_ignore_ascii_case("Authorization")
13935 .then(|| value.trim().to_string())
13936 });
13937 let namespace = request.lines().find_map(|line| {
13938 let (name, value) = line.split_once(':')?;
13939 name.eq_ignore_ascii_case("X-Namespace")
13940 .then(|| value.trim().to_string())
13941 });
13942 let worker_protocol = request.lines().find_map(|line| {
13943 let (name, value) = line.split_once(':')?;
13944 name.eq_ignore_ascii_case("X-Durable-Workflow-Protocol-Version")
13945 .then(|| value.trim().to_string())
13946 });
13947 let control_protocol = request.lines().find_map(|line| {
13948 let (name, value) = line.split_once(':')?;
13949 name.eq_ignore_ascii_case("X-Durable-Workflow-Control-Plane-Version")
13950 .then(|| value.trim().to_string())
13951 });
13952 let request_number = {
13953 let mut requests = requests.lock().expect("captured requests");
13954 requests.push(CapturedRequest {
13955 method: method.to_string(),
13956 path: path.to_string(),
13957 authorization,
13958 namespace,
13959 worker_protocol: worker_protocol.clone(),
13960 control_protocol,
13961 body: body.to_string(),
13962 received_at: Instant::now(),
13963 });
13964 requests
13965 .iter()
13966 .filter(|request| request.path == path)
13967 .count()
13968 };
13969
13970 if behavior.reject_registration && path == "/api/worker/register" {
13971 write_mock_response(
13972 stream,
13973 "503 Service Unavailable",
13974 r#"{"reason":"registration_unavailable","message":"registration failed"}"#,
13975 );
13976 return;
13977 }
13978
13979 if path.starts_with("/api/worker/registrations/") {
13980 if behavior.reject_deregistration_protocol {
13981 write_mock_response(
13982 stream,
13983 "400 Bad Request",
13984 r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.1","requested_version":"1.2"}"#,
13985 );
13986 } else if behavior.reject_deregistration {
13987 write_mock_response(
13988 stream,
13989 "403 Forbidden",
13990 r#"{"reason":"authorization_failed","message":"worker cannot deregister"}"#,
13991 );
13992 } else {
13993 write_mock_response(
13994 stream,
13995 "200 OK",
13996 r#"{"worker_id":"deregistered-worker","outcome":"deregistered","recovered_workflow_task_count":2}"#,
13997 );
13998 }
13999 return;
14000 }
14001
14002 let is_poll = matches!(
14003 path,
14004 "/api/worker/workflow-tasks/poll"
14005 | "/api/worker/activity-tasks/poll"
14006 | "/api/worker/query-tasks/poll"
14007 );
14008 if is_poll && request_number <= behavior.poll_failures_per_path {
14009 return;
14010 }
14011 if path == "/api/worker/heartbeat" && request_number <= behavior.heartbeat_failures {
14012 return;
14013 }
14014 if path == "/api/worker/heartbeat"
14015 && behavior.heartbeat_failure_request == Some(request_number)
14016 {
14017 return;
14018 }
14019 if path == "/api/worker/heartbeat"
14020 && behavior.delayed_heartbeat_request == Some(request_number)
14021 {
14022 thread::sleep(behavior.heartbeat_response_delay);
14023 }
14024 if behavior.unauthorized_polls && is_poll {
14025 write_mock_response(
14026 stream,
14027 "401 Unauthorized",
14028 r#"{"reason":"authentication_failed","message":"invalid worker token"}"#,
14029 );
14030 return;
14031 }
14032 if behavior.draining_polls && is_poll {
14033 write_mock_response(
14034 stream,
14035 "409 Conflict",
14036 r#"{"task":null,"poll_status":"draining","reason":"worker_draining","worker_status":"draining","drain_intent":"draining"}"#,
14037 );
14038 return;
14039 }
14040
14041 if behavior.reject_query_protocol && path.starts_with("/api/worker/query-tasks/") {
14042 let requested_version = worker_protocol.as_deref().unwrap_or("missing");
14043 let body = format!(
14044 r#"{{"reason":"unsupported_protocol_version","message":"Query tasks require worker protocol 1.8 or newer.","supported_version":"1.7","requested_version":"{requested_version}"}}"#
14045 );
14046 write_mock_response(stream, "400 Bad Request", &body);
14047 return;
14048 }
14049
14050 if behavior.reject_query_completion && path == "/api/worker/query-tasks/query-late/complete"
14051 {
14052 write_mock_response(
14053 stream,
14054 "409 Conflict",
14055 r#"{"reason":"query_task_timed_out","message":"query task timed out before completion"}"#,
14056 );
14057 return;
14058 }
14059
14060 if behavior.workflow_completion_status.is_some()
14061 && path == "/api/worker/workflow-tasks/poll"
14062 && request_number == 1
14063 {
14064 write_mock_response(
14065 stream,
14066 "200 OK",
14067 r#"{"task":{"task_id":"workflow-timeout-task","workflow_id":"reused-workflow-id","run_id":"run-selected-timeout","workflow_type":"timeout.workflow","payload_codec":"json","arguments":{"codec":"json","blob":"[]"},"history_events":[],"workflow_task_attempt":3,"lease_owner":"timeout-worker"}}"#,
14068 );
14069 return;
14070 }
14071
14072 if path == "/api/worker/workflow-tasks/workflow-timeout-task/complete" {
14073 if let (Some(status), Some(body)) = (
14074 behavior.workflow_completion_status,
14075 behavior.workflow_completion_body,
14076 ) {
14077 write_mock_response(stream, status, body);
14078 return;
14079 }
14080 }
14081
14082 if behavior.waiting_query_worker {
14083 if behavior.complete_named_signal
14084 && path == "/api/worker/workflow-tasks/poll"
14085 && request_number == 1
14086 {
14087 let body = json!({
14088 "task": {
14089 "task_id": "snapshot-open",
14090 "workflow_id": "snapshot-1",
14091 "run_id": "snapshot-run-1",
14092 "workflow_type": "snapshot",
14093 "payload_codec": DEFAULT_CODEC,
14094 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
14095 .expect("Avro workflow arguments"),
14096 "history_events": [],
14097 "workflow_task_attempt": 1,
14098 "lease_owner": "rust-snapshot-worker"
14099 }
14100 })
14101 .to_string();
14102 write_mock_response(stream, "200 OK", &body);
14103 return;
14104 }
14105
14106 let signal_request = request_number - usize::from(behavior.complete_named_signal);
14107 let signal_request_limit = 2 + usize::from(behavior.complete_named_signal);
14108 if path == "/api/worker/workflow-tasks/poll"
14109 && signal_request >= 1
14110 && signal_request <= signal_request_limit
14111 {
14112 let finish = behavior.complete_named_signal && signal_request == 3;
14113 let amounts = if signal_request == 1 {
14114 vec![3]
14115 } else {
14116 vec![3, 5]
14117 };
14118 let task_id = if signal_request == 1 {
14119 "snapshot-wait-3"
14120 } else if finish {
14121 "snapshot-finish"
14122 } else {
14123 "snapshot-wait-5"
14124 };
14125 let mut history_events = std::iter::once(json!({
14126 "event_type": "SignalWaitOpened",
14127 "payload": {"sequence": 1, "signal_name": "finish"}
14128 }))
14129 .chain(amounts.iter().enumerate().map(|(index, amount)| {
14130 json!({
14131 "event_type": "SignalReceived",
14132 "payload": {
14133 "signal_id": format!("increment-{amount}"),
14134 "signal_name": "increment",
14135 "workflow_sequence": index + 2,
14136 "payload_codec": DEFAULT_CODEC,
14137 "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
14138 .expect("Avro signal envelope")
14139 }
14140 })
14141 }))
14142 .collect::<Vec<_>>();
14143 let (resume_id, resume_name, resume_arguments) = if finish {
14144 history_events.push(json!({
14145 "event_type": "SignalReceived",
14146 "payload": {
14147 "signal_id": "finish",
14148 "signal_name": "finish",
14149 "workflow_sequence": 4,
14150 "payload_codec": DEFAULT_CODEC,
14151 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
14152 .expect("Avro finish signal envelope")
14153 }
14154 }));
14155 (
14156 "finish".to_string(),
14157 "finish".to_string(),
14158 encode_value_envelope(&json!([]), DEFAULT_CODEC)
14159 .expect("Avro finish resume signal"),
14160 )
14161 } else {
14162 let amount = amounts.last().expect("amount");
14163 (
14164 format!("increment-{amount}"),
14165 "increment".to_string(),
14166 encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
14167 .expect("Avro increment resume signal"),
14168 )
14169 };
14170 let body = json!({
14171 "task": {
14172 "task_id": task_id,
14173 "workflow_id": "snapshot-1",
14174 "run_id": "snapshot-run-1",
14175 "workflow_type": "snapshot",
14176 "payload_codec": DEFAULT_CODEC,
14177 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
14178 .expect("Avro workflow arguments"),
14179 "history_events": history_events,
14180 "workflow_task_attempt": 1,
14181 "workflow_signal_id": resume_id,
14182 "signal_name": resume_name,
14183 "signal_arguments": resume_arguments,
14184 "lease_owner": "rust-snapshot-worker"
14185 }
14186 })
14187 .to_string();
14188 write_mock_response(stream, "200 OK", &body);
14189 return;
14190 }
14191
14192 if path == "/api/worker/query-tasks/poll" && request_number == 1 {
14193 let history_events = [3, 5]
14194 .into_iter()
14195 .enumerate()
14196 .map(|(index, amount)| {
14197 json!({
14198 "event_type": "SignalReceived",
14199 "payload": {
14200 "signal_id": format!("increment-{amount}"),
14201 "signal_name": "increment",
14202 "workflow_sequence": index + 2,
14203 "payload_codec": DEFAULT_CODEC,
14204 "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
14205 .expect("Avro query signal envelope")
14206 }
14207 })
14208 })
14209 .collect::<Vec<_>>();
14210 let body = json!({
14211 "task": {
14212 "query_task_id": "snapshot-current",
14213 "query_task_attempt": 1,
14214 "lease_owner": "rust-snapshot-worker",
14215 "workflow_id": "snapshot-1",
14216 "run_id": "snapshot-run-1",
14217 "workflow_type": "snapshot",
14218 "query_name": "current",
14219 "payload_codec": DEFAULT_CODEC,
14220 "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
14221 .expect("Avro workflow arguments"),
14222 "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
14223 .expect("Avro query arguments"),
14224 "history_events": history_events,
14225 "run_status": "waiting"
14226 }
14227 })
14228 .to_string();
14229 write_mock_response(stream, "200 OK", &body);
14230 return;
14231 }
14232
14233 if path == "/api/worker/workflow-tasks/snapshot-wait-3/fail"
14234 || path == "/api/worker/workflow-tasks/snapshot-wait-5/fail"
14235 {
14236 write_mock_response(
14237 stream,
14238 "200 OK",
14239 r#"{"outcome":"waiting_for_history","recorded":true}"#,
14240 );
14241 return;
14242 }
14243
14244 if path == "/api/worker/workflow-tasks/snapshot-open/complete" {
14245 write_mock_response(stream, "200 OK", r#"{"outcome":"waiting","recorded":true}"#);
14246 return;
14247 }
14248
14249 if path == "/api/worker/workflow-tasks/snapshot-finish/complete" {
14250 write_mock_response(
14251 stream,
14252 "200 OK",
14253 r#"{"outcome":"completed","run_status":"completed","recorded":true}"#,
14254 );
14255 return;
14256 }
14257
14258 if path == "/api/worker/query-tasks/snapshot-current/complete" {
14259 write_mock_response(stream, "200 OK", r#"{"outcome":"completed"}"#);
14260 return;
14261 }
14262 }
14263
14264 if matches!(
14265 path,
14266 "/api/workflows/typed-1/query/inspect" | "/api/workflows/typed-1/update/replace"
14267 ) {
14268 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
14269 .expect("typed mock result");
14270 let body = json!({
14271 "result": typed_fidelity_probe().into_json().expect("result projection"),
14272 "result_envelope": result,
14273 })
14274 .to_string();
14275 write_mock_response(stream, "200 OK", &body);
14276 return;
14277 }
14278
14279 if path == "/api/workflows/typed-1" {
14280 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
14281 .expect("typed mock result");
14282 let body = json!({
14283 "workflow_id": "typed-1",
14284 "run_id": "run-typed-1",
14285 "workflow_type": "typed.echo",
14286 "status": "completed",
14287 "output": typed_fidelity_probe().into_json().expect("output projection"),
14288 "output_envelope": result,
14289 })
14290 .to_string();
14291 write_mock_response(stream, "200 OK", &body);
14292 return;
14293 }
14294
14295 let (status, body) = match path {
14296 "/api/health" => ("200 OK", r#"{"status":"ok"}"#),
14297 "/api/workflows" => (
14298 "201 Created",
14299 r#"{"workflow_id":"wf-start-options","run_id":"run-start-options","workflow_type":"rust.timeout"}"#,
14300 ),
14301 "/api/worker/register" if behavior.decline_registration => (
14302 "200 OK",
14303 r#"{"worker_id":"declined-worker","registered":false}"#,
14304 ),
14305 "/api/worker/register" if behavior.waiting_query_worker => (
14306 "200 OK",
14307 r#"{"worker_id":"rust-snapshot-worker","registered":true,"heartbeat_interval_seconds":1}"#,
14308 ),
14309 "/api/worker/register" => (
14310 "200 OK",
14311 r#"{"worker_id":"mock-worker","registered":true,"heartbeat_interval_seconds":3600}"#,
14312 ),
14313 "/api/worker/heartbeat" => ("200 OK", "{}"),
14314 "/api/worker/activity-tasks/poll"
14315 if behavior.cancelled_activity && request_number == 1 =>
14316 {
14317 (
14318 "200 OK",
14319 r#"{"task":{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","activity_type":"cancel-aware","payload_codec":"json","arguments":{"codec":"json","blob":"[]"},"attempt_number":1,"lease_owner":"rust-cancel-worker"}}"#,
14320 )
14321 }
14322 "/api/worker/activity-tasks/poll" | "/api/worker/workflow-tasks/poll" => {
14323 ("200 OK", r#"{"task":null}"#)
14324 }
14325 "/api/worker/query-tasks/poll"
14326 if behavior.reject_query_completion && request_number == 1 =>
14327 {
14328 (
14329 "200 OK",
14330 r#"{"task":{"query_task_id":"query-late","query_task_attempt":1,"lease_owner":"late-worker","workflow_id":"counter-late","run_id":"run-late","workflow_type":"counter","query_name":"current","payload_codec":"json","workflow_arguments":{"codec":"json","blob":"[]"},"query_arguments":{"codec":"json","blob":"[]"},"history_events":[],"run_status":"running"}}"#,
14331 )
14332 }
14333 "/api/worker/query-tasks/poll" => ("200 OK", r#"{"task":null}"#),
14334 "/api/worker/query-tasks/query-capture/complete"
14335 | "/api/worker/query-tasks/query-capture/fail" => ("200 OK", "{}"),
14336 "/api/worker/activity-tasks/activity-cancel/heartbeat" => (
14337 "200 OK",
14338 r#"{"activity_attempt_id":"attempt-cancel","cancel_requested":true,"can_continue":false,"reason":"run_cancelled","run_closed_reason":"cancelled","heartbeat_recorded":false}"#,
14339 ),
14340 "/api/worker/activity-tasks/activity-cancel/complete" => (
14341 "409 Conflict",
14342 r#"{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","reason":"run_cancelled","cancel_requested":true,"can_continue":false,"run_closed_reason":"cancelled"}"#,
14343 ),
14344 "/api/worker/activity-tasks/activity-typed/complete"
14345 | "/api/worker/activity-tasks/activity-typed/fail"
14346 | "/api/workflows/typed-1/signal/changed" => ("200 OK", "{}"),
14347 "/api/workflows/counter-1/query/current" => (
14348 "200 OK",
14349 r#"{"workflow_id":"counter-1","query_name":"current","result":{"count":8},"result_envelope":{"codec":"json","blob":"{\"count\":8}"}}"#,
14350 ),
14351 "/api/workflows/counter-1/query/missing" => (
14352 "404 Not Found",
14353 r#"{"workflow_id":"counter-1","query_name":"missing","reason":"rejected_unknown_query","message":"unknown query"}"#,
14354 ),
14355 "/api/workflows/wf-lifecycle/cancel" => (
14356 "200 OK",
14357 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","reason":"cleanup requested","command_status":"accepted"}"#,
14358 ),
14359 "/api/workflows/wf-lifecycle/terminate" => (
14360 "200 OK",
14361 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","reason":"forced stop","command_status":"accepted"}"#,
14362 ),
14363 "/api/workflows/wf-lifecycle/runs/run-current/cancel" => (
14364 "200 OK",
14365 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","command_status":"accepted"}"#,
14366 ),
14367 "/api/workflows/wf-lifecycle/runs/run-current/terminate" => (
14368 "200 OK",
14369 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","command_status":"accepted"}"#,
14370 ),
14371 "/api/workflows/wf-lifecycle/runs/run-stale/cancel"
14372 | "/api/workflows/wf-lifecycle/runs/run-stale/terminate" => (
14373 "409 Conflict",
14374 r#"{"workflow_id":"wf-lifecycle","run_id":"run-stale","reason":"historical_run_command_rejected","target_scope":"run","message":"Commands cannot target historical runs."}"#,
14375 ),
14376 "/api/workflows/wf-failed" | "/api/workflows/wf-failed/runs/run-failed" => (
14377 "200 OK",
14378 r#"{"workflow_id":"wf-failed","run_id":"run-failed","status":"failed","closed_reason":"failed","error":"payment failed","failure":{"message":"payment failed","failure_category":"application","exception_type":"PaymentError","exception_class":"billing::PaymentError","non_retryable":true,"exception":{"type":"PaymentError","class":"billing::PaymentError","message":"payment failed"},"failures":[{"id":"failure-17","failure_category":"application"}]}}"#,
14379 ),
14380 "/api/workflows/wf-cancelled" => (
14381 "200 OK",
14382 r#"{"workflow_id":"wf-cancelled","run_id":"run-cancelled","status":"cancelled","closed_reason":"cancelled","reason":"cleanup requested"}"#,
14383 ),
14384 "/api/workflows/wf-terminated" => (
14385 "200 OK",
14386 r#"{"workflow_id":"wf-terminated","run_id":"run-terminated","status":"terminated","closed_reason":"terminated","reason":"forced stop"}"#,
14387 ),
14388 "/api/workflows/wf-timed-out" => (
14389 "200 OK",
14390 r#"{"workflow_id":"wf-timed-out","run_id":"run-timed-out","status":"failed","closed_reason":"timed_out","reason":"run_timeout"}"#,
14391 ),
14392 "/api/workflows/wf-waiting" | "/api/workflows/wf-waiting/runs/run-waiting" => (
14393 "200 OK",
14394 r#"{"workflow_id":"wf-waiting","run_id":"run-waiting","status":"waiting"}"#,
14395 ),
14396 "/api/workflows/wf-selected" => (
14397 "200 OK",
14398 r#"{"workflow_id":"wf-selected","run_id":"run-current","status":"completed","output":"current run output"}"#,
14399 ),
14400 "/api/workflows/wf-selected/runs/run-selected" => (
14401 "200 OK",
14402 r#"{"workflow_id":"wf-selected","run_id":"run-selected","status":"cancelled","closed_reason":"cancelled","reason":"selected run cancelled"}"#,
14403 ),
14404 _ => ("404 Not Found", r#"{"message":"not found"}"#),
14405 };
14406 write_mock_response(stream, status, body);
14407 }
14408
14409 fn mock_request_is_complete(request: &[u8]) -> bool {
14410 let Some(header_end) = request
14411 .windows(4)
14412 .position(|window| window == b"\r\n\r\n")
14413 .map(|position| position + 4)
14414 else {
14415 return false;
14416 };
14417 let headers = String::from_utf8_lossy(&request[..header_end]);
14418 let content_length = headers.lines().find_map(|line| {
14419 let (name, value) = line.split_once(':')?;
14420 name.eq_ignore_ascii_case("content-length")
14421 .then(|| value.trim().parse::<usize>().ok())
14422 .flatten()
14423 });
14424
14425 request.len() >= header_end + content_length.unwrap_or(0)
14426 }
14427
14428 fn write_mock_response(stream: &mut TcpStream, status: &str, body: &str) {
14429 let response = format!(
14430 "HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
14431 body.len()
14432 );
14433
14434 let _ = stream.write_all(response.as_bytes());
14435 let _ = stream.flush();
14436 }
14437}