1#![doc = include_str!("../README.md")]
2
3use std::{
4 any::{Any, TypeId},
5 collections::{BTreeMap, HashMap},
6 future::Future,
7 io::{self, Read},
8 pin::Pin,
9 sync::{
10 atomic::{AtomicBool, Ordering},
11 Arc, Mutex, OnceLock,
12 },
13 task::{Context as TaskContext, Poll},
14 time::{Duration, Instant, SystemTime, UNIX_EPOCH},
15};
16
17use apache_avro::{from_avro_datum, to_avro_datum, types::Value as AvroDatum, Schema};
18use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
19use futures_util::{future::OptionFuture, task::noop_waker_ref};
20use serde::{
21 de::DeserializeOwned,
22 ser::{SerializeMap, SerializeSeq},
23 Deserialize, Serialize, Serializer,
24};
25pub use serde_json::{json, Value};
26use thiserror::Error;
27pub use uuid::Uuid;
28
29pub const WORKER_PROTOCOL_VERSION: &str = "1.2";
30pub const CONTROL_PLANE_VERSION: &str = "2";
31pub const DEFAULT_CODEC: &str = "avro";
32pub const JSON_CODEC: &str = "json";
33pub const SDK_VERSION: &str = concat!("durable-workflow-rust/", env!("CARGO_PKG_VERSION"));
34pub const QUERY_TASKS_CAPABILITY: &str = "query_tasks";
36pub const WORKFLOW_UPDATES_CAPABILITY: &str = "workflow_updates";
38pub const QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
40
41const MAX_LONG_POLL_TIMEOUT_SECONDS: u64 = 60;
42const WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE: &str =
43 "Workflow task waiting for scheduled history.";
44const WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE: &str = "WorkflowTaskWaitingForHistory";
45
46const QUERY_TASK_FINAL_REJECTION_REASONS: &[&str] = &[
47 "lease_expired",
48 "query_task_not_found",
49 "query_task_not_leased",
50 "query_task_timed_out",
51];
52
53pub const AVRO_VALUE_SCHEMA_JSON: &str =
55 include_str!("../schema/durable_workflow.protocol.Value.v1.avsc");
56pub const AVRO_VALUE_SCHEMA_FINGERPRINT_HEX: &str = "e2a33dff55802237";
57pub const AVRO_VALUE_SCHEMA_FINGERPRINT: [u8; 8] = [0xe2, 0xa3, 0x3d, 0xff, 0x55, 0x80, 0x22, 0x37];
58const AVRO_SINGLE_OBJECT_MAGIC: [u8; 2] = [0xc3, 0x01];
59
60static AVRO_VALUE_SCHEMA: OnceLock<std::result::Result<Schema, String>> = OnceLock::new();
61
62#[derive(Clone, Copy)]
63enum RequestProtocol {
64 ControlPlane,
65 Worker(&'static str),
66}
67
68pub type Result<T> = std::result::Result<T, Error>;
69
70#[derive(Debug, Error)]
71pub enum Error {
72 #[error("transport error: {0}")]
73 Transport(#[from] reqwest::Error),
74 #[error(
75 "invalid Durable Workflow base URL: omit the SDK-owned /api suffix and pass the Server or Cloud runtime base URL; the SDK appends /api automatically"
76 )]
77 InvalidBaseUrl,
78 #[error("json error: {0}")]
79 Json(#[from] serde_json::Error),
80 #[error("http {status}: {body}")]
81 Http {
82 status: reqwest::StatusCode,
83 body: String,
84 },
85 #[error("codec error: {0}")]
86 Codec(String),
87 #[error(transparent)]
88 QueryFailed(QueryFailure),
89 #[error(transparent)]
90 Protocol(ProtocolFailure),
91 #[error(transparent)]
92 NonDeterministicReplay(ReplayFailure),
93 #[error(transparent)]
94 ChildWorkflowFailed(ChildWorkflowFailure),
95 #[error(transparent)]
96 ActivityFailed(ActivityFailure),
97 #[error(transparent)]
98 WorkflowCommandRejected(WorkflowCommandRejection),
99 #[error(transparent)]
100 WorkflowFailed(WorkflowTerminalOutcome),
101 #[error(transparent)]
102 WorkflowCancelled(WorkflowTerminalOutcome),
103 #[error(transparent)]
104 WorkflowTerminated(WorkflowTerminalOutcome),
105 #[error(transparent)]
106 WorkflowTimedOut(WorkflowTerminalOutcome),
107 #[error(transparent)]
108 ActivityTaskRejected(ActivityTaskRejection),
109 #[error("workflow handler {0:?} is not registered")]
110 WorkflowNotRegistered(String),
111 #[error("activity handler {0:?} is not registered")]
112 ActivityNotRegistered(String),
113 #[error("workflow future yielded without emitting a durable command")]
114 WorkflowYieldedWithoutCommand,
115 #[error("workflow state lock is poisoned")]
116 WorkflowStatePoisoned,
117 #[error("timer duration is too large for the worker protocol")]
118 TimerDurationOverflow,
119 #[error("operation timed out")]
120 Timeout,
121 #[error(
122 "missing {role}-plane credentials: configure ClientBuilder::{role}_token or ClientBuilder::token; a {opposite_role}-plane token cannot authorize this request"
123 )]
124 MissingRoleCredentials {
125 role: &'static str,
126 opposite_role: &'static str,
127 },
128 #[error("worker loop error: {0}")]
129 WorkerLoop(String),
130 #[error(
131 "workflow command contract for {workflow_type:?} declares update validators, but this Rust SDK cannot execute synchronous pre-accept update validation"
132 )]
133 UnsupportedUpdateValidators { workflow_type: String },
134 #[error("{primary}; worker deregistration also failed: {deregistration}")]
135 WorkerShutdown {
136 primary: Box<Error>,
137 deregistration: Box<Error>,
138 },
139 #[error("invalid child workflow options: {0}")]
140 InvalidChildWorkflowOptions(String),
141 #[error(transparent)]
142 InvalidActivityOptions(ActivityOptionsError),
143 #[error(transparent)]
144 InvalidContinueAsNewOptions(#[from] ContinueAsNewOptionsError),
145 #[doc(hidden)]
146 #[error("workflow requested continue as new")]
147 ContinueAsNew(ContinueAsNewRequest),
148}
149
150#[derive(Clone, Copy, Debug, PartialEq, Eq)]
152pub enum WorkflowCommandKind {
153 Cancel,
154 Terminate,
155}
156
157impl WorkflowCommandKind {
158 fn as_str(self) -> &'static str {
159 match self {
160 Self::Cancel => "cancel",
161 Self::Terminate => "terminate",
162 }
163 }
164}
165
166#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
168pub struct WorkflowCommandOptions {
169 #[serde(skip_serializing_if = "Option::is_none")]
170 pub reason: Option<String>,
171 #[serde(skip_serializing_if = "Option::is_none")]
172 pub request_id: Option<String>,
173}
174
175#[derive(Clone, Debug, PartialEq, Eq)]
182pub struct WorkflowStartOptions {
183 pub execution_timeout_seconds: u64,
184 pub run_timeout_seconds: u64,
185}
186
187impl Default for WorkflowStartOptions {
188 fn default() -> Self {
189 Self {
190 execution_timeout_seconds: 3600,
191 run_timeout_seconds: 600,
192 }
193 }
194}
195
196impl WorkflowStartOptions {
197 pub fn new() -> Self {
198 Self::default()
199 }
200
201 pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
202 self.execution_timeout_seconds = seconds;
203 self
204 }
205
206 pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
207 self.run_timeout_seconds = seconds;
208 self
209 }
210
211 fn validate(&self) -> Result<()> {
212 if self.execution_timeout_seconds == 0 {
213 return Err(Error::Codec(
214 "execution_timeout_seconds must be at least 1".to_string(),
215 ));
216 }
217 if self.run_timeout_seconds == 0 {
218 return Err(Error::Codec(
219 "run_timeout_seconds must be at least 1".to_string(),
220 ));
221 }
222 if self.run_timeout_seconds > self.execution_timeout_seconds {
223 return Err(Error::Codec(
224 "run_timeout_seconds cannot exceed execution_timeout_seconds".to_string(),
225 ));
226 }
227
228 Ok(())
229 }
230}
231
232#[derive(Clone, Debug, Default, PartialEq, Eq)]
237pub struct ContinueAsNewOptions {
238 pub workflow_type: Option<String>,
239 pub task_queue: Option<String>,
240}
241
242impl ContinueAsNewOptions {
243 pub fn new() -> Self {
244 Self::default()
245 }
246
247 pub fn workflow_type(mut self, workflow_type: impl Into<String>) -> Self {
248 self.workflow_type = Some(workflow_type.into());
249 self
250 }
251
252 pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
253 self.task_queue = Some(task_queue.into());
254 self
255 }
256
257 fn validate(&self) -> std::result::Result<(), ContinueAsNewOptionsError> {
258 for (field, value) in [
259 ("workflow_type", self.workflow_type.as_deref()),
260 ("task_queue", self.task_queue.as_deref()),
261 ] {
262 if value.is_some_and(|value| value.trim().is_empty()) {
263 return Err(ContinueAsNewOptionsError {
264 field,
265 message: format!("{field} must not be empty"),
266 });
267 }
268 }
269 Ok(())
270 }
271}
272
273#[derive(Clone, Debug, Error, PartialEq, Eq)]
275#[error("invalid continue-as-new option {field}: {message}")]
276pub struct ContinueAsNewOptionsError {
277 pub field: &'static str,
278 pub message: String,
279}
280
281#[derive(Clone, Debug, Default, PartialEq, Eq)]
283pub struct WorkflowHistoryBudget {
284 pub event_count: u64,
285 pub size_bytes: Option<u64>,
286 pub continue_as_new_recommended: bool,
287 pub pressure: Option<String>,
288}
289
290#[doc(hidden)]
291#[derive(Clone, Debug)]
292pub struct ContinueAsNewRequest {
293 arguments: AvroValue,
294 options: ContinueAsNewOptions,
295}
296
297impl WorkflowCommandOptions {
298 pub fn new() -> Self {
299 Self::default()
300 }
301
302 pub fn reason(mut self, reason: impl Into<String>) -> Self {
303 self.reason = Some(reason.into());
304 self
305 }
306
307 pub fn request_id(mut self, request_id: impl Into<String>) -> Self {
308 self.request_id = Some(request_id.into());
309 self
310 }
311}
312
313#[derive(Clone, Debug, PartialEq)]
315pub struct WorkflowCommandResult {
316 pub command: WorkflowCommandKind,
317 pub workflow_id: String,
318 pub run_id: Option<String>,
319 pub outcome: Option<String>,
320 pub reason: Option<String>,
321 pub command_status: Option<String>,
322 pub raw: Value,
323}
324
325#[derive(Clone, Debug, Error)]
327#[error("workflow {command:?} rejected ({reason}, HTTP {status}): {message}")]
328pub struct WorkflowCommandRejection {
329 pub command: WorkflowCommandKind,
330 pub status: u16,
331 pub reason: String,
332 pub message: String,
333 pub workflow_id: String,
334 pub run_id: Option<String>,
335 pub target_scope: Option<String>,
336 pub body: Value,
337}
338
339#[derive(Clone, Copy, Debug, PartialEq, Eq)]
341pub enum WorkflowTerminalKind {
342 Failed,
343 Cancelled,
344 Terminated,
345 TimedOut,
346}
347
348#[derive(Clone, Debug, Error)]
354#[error("workflow {workflow_id} run {run_id:?} ended as {kind:?} ({reason})")]
355pub struct WorkflowTerminalOutcome {
356 pub kind: WorkflowTerminalKind,
357 pub workflow_id: String,
358 pub run_id: Option<String>,
359 pub reason: String,
360 pub failure_category: Option<String>,
361 pub failure_id: Option<String>,
362 pub exception_type: Option<String>,
363 pub exception_class: Option<String>,
364 pub non_retryable: Option<bool>,
365 pub message: Option<String>,
366 pub exception: Option<Value>,
367 pub raw: Value,
368}
369
370#[derive(Clone, Debug, Error)]
372#[error("activity task {operation} rejected ({reason}, HTTP {status})")]
373pub struct ActivityTaskRejection {
374 pub operation: String,
375 pub status: u16,
376 pub reason: String,
377 pub task_id: String,
378 pub activity_attempt_id: String,
379 pub cancel_requested: bool,
380 pub can_continue: Option<bool>,
381 pub run_closed_reason: Option<String>,
382 pub body: Value,
383}
384
385#[derive(Clone, Copy, Debug, PartialEq, Eq)]
387pub enum ActivityOptionsErrorKind {
388 EmptyTaskQueue,
389 EmptyRetryPolicy,
390 InvalidMaxAttempts,
391 BackoffWithoutRetryBudget,
392 TooManyBackoffIntervals,
393 InvalidBackoffCoefficient,
394 BackoffGenerationTooLarge,
395 BackoffOverflow,
396 EmptyNonRetryableErrorType,
397 TimeoutNotPositive,
398 TimeoutOverflow,
399 TimeoutOrder,
400}
401
402#[derive(Clone, Debug, Error, PartialEq, Eq)]
404#[error("invalid activity options ({kind:?}, {field:?}): {message}")]
405pub struct ActivityOptionsError {
406 pub kind: ActivityOptionsErrorKind,
407 pub field: Option<&'static str>,
408 pub message: String,
409}
410
411impl ActivityOptionsError {
412 fn new(
413 kind: ActivityOptionsErrorKind,
414 field: Option<&'static str>,
415 message: impl Into<String>,
416 ) -> Self {
417 Self {
418 kind,
419 field,
420 message: message.into(),
421 }
422 }
423}
424
425#[derive(Clone, Copy, Debug, PartialEq, Eq)]
427pub enum ActivityFailureKind {
428 Failed,
429 Cancelled,
430 TimedOut,
431}
432
433#[derive(Clone, Debug, Error)]
438#[error("activity failed ({reason}): {message}")]
439pub struct ActivityFailure {
440 pub kind: ActivityFailureKind,
441 pub reason: String,
442 pub message: String,
443 pub activity_execution_id: Option<String>,
444 pub activity_attempt_id: Option<String>,
445 pub activity_type: Option<String>,
446 pub activity_class: Option<String>,
447 pub attempt_number: Option<u64>,
448 pub failure_id: Option<String>,
449 pub failure_category: Option<String>,
450 pub timeout_kind: Option<String>,
451 pub non_retryable: bool,
452 pub exception_type: Option<String>,
453 pub exception_class: Option<String>,
454 pub code: Option<Value>,
455 pub exception: Option<Value>,
456}
457
458#[derive(Clone, Copy, Debug, PartialEq, Eq)]
460pub enum ChildWorkflowFailureKind {
461 Failed,
462 Cancelled,
463 Terminated,
464}
465
466#[derive(Clone, Debug, Error)]
472#[error("child workflow failed ({reason}): {message}")]
473pub struct ChildWorkflowFailure {
474 pub kind: ChildWorkflowFailureKind,
475 pub reason: String,
476 pub message: String,
477 pub parent_workflow_id: Option<String>,
478 pub parent_workflow_run_id: Option<String>,
479 pub child_workflow_id: Option<String>,
480 pub child_workflow_run_id: Option<String>,
481 pub child_workflow_type: Option<String>,
482 pub failure_id: Option<String>,
483 pub failure_category: Option<String>,
484 pub exception_type: Option<String>,
485 pub exception_class: Option<String>,
486 pub non_retryable: bool,
487 pub code: Option<Value>,
488 pub exception: Option<Value>,
489}
490
491#[derive(Clone, Debug, PartialEq, Eq)]
493pub struct WorkflowIdentity {
494 pub workflow_id: Option<String>,
495 pub run_id: Option<String>,
496}
497
498#[derive(Clone, Debug, PartialEq)]
500pub struct ChildWorkflowResult {
501 pub parent: WorkflowIdentity,
502 pub child: WorkflowIdentity,
503 pub child_workflow_type: Option<String>,
504 pub result: Value,
505}
506
507#[derive(Clone, Debug, PartialEq)]
509pub struct ChildWorkflowAvroResult {
510 pub parent: WorkflowIdentity,
511 pub child: WorkflowIdentity,
512 pub child_workflow_type: Option<String>,
513 pub result: AvroValue,
514}
515
516#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
518pub enum ParentClosePolicy {
519 #[default]
520 Abandon,
521 RequestCancel,
522 Terminate,
523}
524
525impl ParentClosePolicy {
526 fn as_str(self) -> &'static str {
527 match self {
528 Self::Abandon => "abandon",
529 Self::RequestCancel => "request_cancel",
530 Self::Terminate => "terminate",
531 }
532 }
533}
534
535#[derive(Clone, Debug, Default, PartialEq, Eq)]
537pub struct ChildWorkflowRetryPolicy {
538 pub max_attempts: Option<u32>,
539 pub backoff_seconds: Vec<u64>,
540 pub non_retryable_error_types: Vec<String>,
541}
542
543#[derive(Clone, Debug, PartialEq, Eq)]
547pub struct ChildWorkflowOptions {
548 pub task_queue: String,
549 pub parent_close_policy: ParentClosePolicy,
550 pub retry_policy: Option<ChildWorkflowRetryPolicy>,
551 pub execution_timeout_seconds: Option<u64>,
552 pub run_timeout_seconds: Option<u64>,
553}
554
555impl ChildWorkflowOptions {
556 pub fn new(task_queue: impl Into<String>) -> Self {
557 Self {
558 task_queue: task_queue.into(),
559 parent_close_policy: ParentClosePolicy::Abandon,
560 retry_policy: None,
561 execution_timeout_seconds: None,
562 run_timeout_seconds: None,
563 }
564 }
565
566 pub fn parent_close_policy(mut self, policy: ParentClosePolicy) -> Self {
567 self.parent_close_policy = policy;
568 self
569 }
570
571 pub fn retry_policy(mut self, policy: ChildWorkflowRetryPolicy) -> Self {
572 self.retry_policy = Some(policy);
573 self
574 }
575
576 pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
577 self.execution_timeout_seconds = Some(seconds);
578 self
579 }
580
581 pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
582 self.run_timeout_seconds = Some(seconds);
583 self
584 }
585}
586
587#[derive(Clone, Debug, PartialEq, Eq)]
589pub enum ActivityBackoff {
590 Explicit(Vec<Duration>),
593 Exponential {
595 initial_interval: Duration,
596 coefficient: u32,
597 maximum_interval: Option<Duration>,
598 },
599}
600
601#[derive(Clone, Debug, Default, PartialEq, Eq)]
603pub struct ActivityRetryPolicy {
604 pub max_attempts: Option<u32>,
605 pub backoff: Option<ActivityBackoff>,
606 pub non_retryable_error_types: Vec<String>,
607}
608
609impl ActivityRetryPolicy {
610 pub fn new(max_attempts: u32) -> Self {
612 Self {
613 max_attempts: Some(max_attempts),
614 ..Self::default()
615 }
616 }
617
618 pub fn backoff_intervals(mut self, intervals: impl IntoIterator<Item = Duration>) -> Self {
619 self.backoff = Some(ActivityBackoff::Explicit(intervals.into_iter().collect()));
620 self
621 }
622
623 pub fn exponential_backoff(
624 mut self,
625 initial_interval: Duration,
626 coefficient: u32,
627 maximum_interval: Option<Duration>,
628 ) -> Self {
629 self.backoff = Some(ActivityBackoff::Exponential {
630 initial_interval,
631 coefficient,
632 maximum_interval,
633 });
634 self
635 }
636
637 pub fn non_retryable_error_type(mut self, error_type: impl Into<String>) -> Self {
638 self.non_retryable_error_types.push(error_type.into());
639 self
640 }
641
642 pub fn non_retryable_error_types(
643 mut self,
644 error_types: impl IntoIterator<Item = impl Into<String>>,
645 ) -> Self {
646 self.non_retryable_error_types
647 .extend(error_types.into_iter().map(Into::into));
648 self
649 }
650}
651
652#[derive(Clone, Debug, Default, PartialEq, Eq)]
657pub struct ActivityOptions {
658 pub task_queue: Option<String>,
659 pub retry_policy: Option<ActivityRetryPolicy>,
660 pub start_to_close_timeout: Option<Duration>,
661 pub schedule_to_start_timeout: Option<Duration>,
662 pub schedule_to_close_timeout: Option<Duration>,
663 pub heartbeat_timeout: Option<Duration>,
664}
665
666impl ActivityOptions {
667 pub fn new() -> Self {
668 Self::default()
669 }
670
671 pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
672 self.task_queue = Some(task_queue.into());
673 self
674 }
675
676 pub fn retry_policy(mut self, policy: ActivityRetryPolicy) -> Self {
677 self.retry_policy = Some(policy);
678 self
679 }
680
681 pub fn start_to_close_timeout(mut self, timeout: Duration) -> Self {
682 self.start_to_close_timeout = Some(timeout);
683 self
684 }
685
686 pub fn schedule_to_start_timeout(mut self, timeout: Duration) -> Self {
687 self.schedule_to_start_timeout = Some(timeout);
688 self
689 }
690
691 pub fn schedule_to_close_timeout(mut self, timeout: Duration) -> Self {
692 self.schedule_to_close_timeout = Some(timeout);
693 self
694 }
695
696 pub fn heartbeat_timeout(mut self, timeout: Duration) -> Self {
697 self.heartbeat_timeout = Some(timeout);
698 self
699 }
700
701 fn validate(&self) -> std::result::Result<ValidatedActivityOptions, ActivityOptionsError> {
702 if self
703 .task_queue
704 .as_deref()
705 .is_some_and(|queue| queue.trim().is_empty())
706 {
707 return Err(ActivityOptionsError::new(
708 ActivityOptionsErrorKind::EmptyTaskQueue,
709 Some("task_queue"),
710 "task_queue must not be empty",
711 ));
712 }
713
714 for (field, value) in [
715 ("start_to_close_timeout", self.start_to_close_timeout),
716 ("schedule_to_start_timeout", self.schedule_to_start_timeout),
717 ("schedule_to_close_timeout", self.schedule_to_close_timeout),
718 ("heartbeat_timeout", self.heartbeat_timeout),
719 ] {
720 if value.is_some_and(|value| value.is_zero()) {
721 return Err(ActivityOptionsError::new(
722 ActivityOptionsErrorKind::TimeoutNotPositive,
723 Some(field),
724 format!("{field} must be positive"),
725 ));
726 }
727 }
728
729 validate_timeout_order(
730 "heartbeat_timeout",
731 self.heartbeat_timeout,
732 "start_to_close_timeout",
733 self.start_to_close_timeout,
734 )?;
735 validate_timeout_order(
736 "start_to_close_timeout",
737 self.start_to_close_timeout,
738 "schedule_to_close_timeout",
739 self.schedule_to_close_timeout,
740 )?;
741 validate_timeout_order(
742 "schedule_to_start_timeout",
743 self.schedule_to_start_timeout,
744 "schedule_to_close_timeout",
745 self.schedule_to_close_timeout,
746 )?;
747
748 Ok(ValidatedActivityOptions {
749 task_queue: self.task_queue.clone(),
750 retry_policy: self
751 .retry_policy
752 .as_ref()
753 .map(validate_activity_retry_policy)
754 .transpose()?,
755 start_to_close_timeout: timeout_seconds(
756 "start_to_close_timeout",
757 self.start_to_close_timeout,
758 )?,
759 schedule_to_start_timeout: timeout_seconds(
760 "schedule_to_start_timeout",
761 self.schedule_to_start_timeout,
762 )?,
763 schedule_to_close_timeout: timeout_seconds(
764 "schedule_to_close_timeout",
765 self.schedule_to_close_timeout,
766 )?,
767 heartbeat_timeout: timeout_seconds("heartbeat_timeout", self.heartbeat_timeout)?,
768 })
769 }
770}
771
772#[derive(Clone, Debug)]
773struct ValidatedActivityOptions {
774 task_queue: Option<String>,
775 retry_policy: Option<Value>,
776 start_to_close_timeout: Option<u64>,
777 schedule_to_start_timeout: Option<u64>,
778 schedule_to_close_timeout: Option<u64>,
779 heartbeat_timeout: Option<u64>,
780}
781
782fn validate_timeout_order(
783 smaller_name: &'static str,
784 smaller: Option<Duration>,
785 larger_name: &'static str,
786 larger: Option<Duration>,
787) -> std::result::Result<(), ActivityOptionsError> {
788 if matches!((smaller, larger), (Some(smaller), Some(larger)) if smaller > larger) {
789 return Err(ActivityOptionsError::new(
790 ActivityOptionsErrorKind::TimeoutOrder,
791 Some(smaller_name),
792 format!("{smaller_name} must be <= {larger_name}"),
793 ));
794 }
795 Ok(())
796}
797
798fn timeout_seconds(
799 field: &'static str,
800 value: Option<Duration>,
801) -> std::result::Result<Option<u64>, ActivityOptionsError> {
802 value
803 .map(|value| {
804 activity_protocol_seconds(value).ok_or_else(|| {
805 ActivityOptionsError::new(
806 ActivityOptionsErrorKind::TimeoutOverflow,
807 Some(field),
808 format!("{field} is too large for the worker protocol"),
809 )
810 })
811 })
812 .transpose()
813}
814
815fn duration_seconds_ceil(value: Duration) -> Option<u64> {
816 value
817 .as_secs()
818 .checked_add(u64::from(value.subsec_nanos() > 0))
819}
820
821fn activity_protocol_seconds(value: Duration) -> Option<u64> {
822 duration_seconds_ceil(value).filter(|seconds| *seconds <= i64::MAX as u64)
823}
824
825fn validate_activity_retry_policy(
826 policy: &ActivityRetryPolicy,
827) -> std::result::Result<Value, ActivityOptionsError> {
828 if policy.max_attempts.is_none()
829 && policy.backoff.is_none()
830 && policy.non_retryable_error_types.is_empty()
831 {
832 return Err(ActivityOptionsError::new(
833 ActivityOptionsErrorKind::EmptyRetryPolicy,
834 Some("retry_policy"),
835 "retry_policy must configure at least one field",
836 ));
837 }
838 if policy.max_attempts == Some(0) {
839 return Err(ActivityOptionsError::new(
840 ActivityOptionsErrorKind::InvalidMaxAttempts,
841 Some("retry_policy.max_attempts"),
842 "max_attempts must be >= 1",
843 ));
844 }
845 if policy
846 .non_retryable_error_types
847 .iter()
848 .any(|error_type| error_type.trim().is_empty())
849 {
850 return Err(ActivityOptionsError::new(
851 ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
852 Some("retry_policy.non_retryable_error_types"),
853 "non_retryable_error_types must not contain empty values",
854 ));
855 }
856
857 let backoff_seconds = match &policy.backoff {
858 None => None,
859 Some(backoff) => {
860 let max_attempts = policy.max_attempts.ok_or_else(|| {
861 ActivityOptionsError::new(
862 ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
863 Some("retry_policy.backoff"),
864 "backoff requires max_attempts",
865 )
866 })?;
867 let retry_count = max_attempts.saturating_sub(1) as usize;
868 let intervals = match backoff {
869 ActivityBackoff::Explicit(intervals) => {
870 if intervals.len() > retry_count {
871 return Err(ActivityOptionsError::new(
872 ActivityOptionsErrorKind::TooManyBackoffIntervals,
873 Some("retry_policy.backoff"),
874 "backoff interval count must not exceed max_attempts - 1",
875 ));
876 }
877 intervals.clone()
878 }
879 ActivityBackoff::Exponential {
880 initial_interval,
881 coefficient,
882 maximum_interval,
883 } => {
884 if *coefficient < 1 {
885 return Err(ActivityOptionsError::new(
886 ActivityOptionsErrorKind::InvalidBackoffCoefficient,
887 Some("retry_policy.backoff.coefficient"),
888 "backoff coefficient must be >= 1",
889 ));
890 }
891 if retry_count > 10_000 {
892 return Err(ActivityOptionsError::new(
893 ActivityOptionsErrorKind::BackoffGenerationTooLarge,
894 Some("retry_policy.max_attempts"),
895 "generated backoff supports at most 10000 retry intervals",
896 ));
897 }
898 let mut current = *initial_interval;
899 let mut intervals = Vec::with_capacity(retry_count);
900 for _ in 0..retry_count {
901 let interval = maximum_interval
902 .map(|maximum| current.min(maximum))
903 .unwrap_or(current);
904 intervals.push(interval);
905 if maximum_interval.is_some_and(|maximum| interval == maximum) {
906 break;
907 }
908 current = current.checked_mul(*coefficient).ok_or_else(|| {
909 ActivityOptionsError::new(
910 ActivityOptionsErrorKind::BackoffOverflow,
911 Some("retry_policy.backoff"),
912 "generated backoff interval overflowed",
913 )
914 })?;
915 }
916 intervals
917 }
918 };
919 Some(
920 intervals
921 .into_iter()
922 .map(|interval| {
923 activity_protocol_seconds(interval).ok_or_else(|| {
924 ActivityOptionsError::new(
925 ActivityOptionsErrorKind::BackoffOverflow,
926 Some("retry_policy.backoff"),
927 "backoff interval is too large for the worker protocol",
928 )
929 })
930 })
931 .collect::<std::result::Result<Vec<_>, _>>()?,
932 )
933 }
934 };
935
936 let mut encoded = serde_json::Map::new();
937 if let Some(max_attempts) = policy.max_attempts {
938 encoded.insert("max_attempts".to_string(), json!(max_attempts));
939 }
940 if let Some(backoff_seconds) = backoff_seconds {
941 encoded.insert("backoff_seconds".to_string(), json!(backoff_seconds));
942 }
943 if !policy.non_retryable_error_types.is_empty() {
944 let mut canonical_error_types = Vec::new();
945 for error_type in policy
946 .non_retryable_error_types
947 .iter()
948 .map(|error_type| error_type.trim())
949 {
950 if !canonical_error_types.contains(&error_type) {
951 canonical_error_types.push(error_type);
952 }
953 }
954 encoded.insert(
955 "non_retryable_error_types".to_string(),
956 json!(canonical_error_types),
957 );
958 }
959 Ok(Value::Object(encoded))
960}
961
962#[derive(Clone, Debug, Error)]
965#[error("non-deterministic workflow replay ({reason}) at sequence {sequence:?}: {message}")]
966pub struct ReplayFailure {
967 pub reason: String,
968 pub sequence: Option<u64>,
969 pub expected: Option<String>,
970 pub actual: Option<String>,
971 pub message: String,
972}
973
974impl ReplayFailure {
975 fn new(
976 reason: impl Into<String>,
977 sequence: Option<u64>,
978 expected: Option<String>,
979 actual: Option<String>,
980 message: impl Into<String>,
981 ) -> Self {
982 Self {
983 reason: reason.into(),
984 sequence,
985 expected,
986 actual,
987 message: message.into(),
988 }
989 }
990}
991
992#[derive(Clone, Debug, Error)]
994#[error("query failed ({reason}, HTTP {status}): {message}")]
995pub struct QueryFailure {
996 pub status: u16,
997 pub reason: String,
998 pub message: String,
999 pub body: Value,
1000}
1001
1002#[derive(Clone, Debug, Error)]
1004#[error("protocol rejected ({reason}, HTTP {status}): {message}")]
1005pub struct ProtocolFailure {
1006 pub status: u16,
1007 pub reason: String,
1008 pub message: String,
1009 pub supported_version: Option<String>,
1010 pub requested_version: Option<String>,
1011 pub body: Value,
1012}
1013
1014#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
1015pub struct PayloadEnvelope {
1016 pub codec: String,
1017 pub blob: String,
1018}
1019
1020impl PayloadEnvelope {
1021 pub fn avro<T: Serialize>(value: &T) -> Result<Self> {
1022 encode_payload(value, DEFAULT_CODEC)
1023 }
1024
1025 pub fn json<T: Serialize>(value: &T) -> Result<Self> {
1026 encode_payload(value, JSON_CODEC)
1027 }
1028
1029 pub fn avro_value(value: &AvroValue) -> Result<Self> {
1032 encode_avro_value(value)
1033 }
1034}
1035
1036#[derive(Clone, Debug, PartialEq)]
1038pub enum AvroValue {
1039 Null,
1040 Boolean(bool),
1041 Long(i64),
1042 Double(f64),
1043 Bytes(Vec<u8>),
1044 String(String),
1045 Array(Vec<AvroValue>),
1046 Map(BTreeMap<String, AvroValue>),
1047}
1048
1049impl AvroValue {
1050 fn from_serialize<T: Serialize>(value: &T) -> Result<Self> {
1051 Self::from_serde_value(
1052 serde_value::to_value(value).map_err(|error| {
1053 Error::Codec(format!("could not adapt value for Avro: {error}"))
1054 })?,
1055 )
1056 }
1057
1058 fn from_serde_value(value: serde_value::Value) -> Result<Self> {
1059 use serde_value::Value as SerdeValue;
1060
1061 match value {
1062 SerdeValue::Unit => Ok(Self::Null),
1063 SerdeValue::Bool(value) => Ok(Self::Boolean(value)),
1064 SerdeValue::I8(value) => Ok(Self::Long(i64::from(value))),
1065 SerdeValue::I16(value) => Ok(Self::Long(i64::from(value))),
1066 SerdeValue::I32(value) => Ok(Self::Long(i64::from(value))),
1067 SerdeValue::I64(value) => Ok(Self::Long(value)),
1068 SerdeValue::U8(value) => Ok(Self::Long(i64::from(value))),
1069 SerdeValue::U16(value) => Ok(Self::Long(i64::from(value))),
1070 SerdeValue::U32(value) => Ok(Self::Long(i64::from(value))),
1071 SerdeValue::U64(value) => i64::try_from(value).map(Self::Long).map_err(|_| {
1072 Error::Codec(
1073 "integer_overflow: Avro Value long must be within signed 64-bit range"
1074 .to_string(),
1075 )
1076 }),
1077 SerdeValue::F32(value) => Self::finite_double(f64::from(value)),
1078 SerdeValue::F64(value) => Self::finite_double(value),
1079 SerdeValue::Char(value) => Ok(Self::String(value.to_string())),
1080 SerdeValue::String(value) => Ok(Self::String(value)),
1081 SerdeValue::Bytes(value) => Ok(Self::Bytes(value)),
1082 SerdeValue::Option(None) => Ok(Self::Null),
1083 SerdeValue::Option(Some(value)) | SerdeValue::Newtype(value) => {
1084 Self::from_serde_value(*value)
1085 }
1086 SerdeValue::Seq(values) => values
1087 .into_iter()
1088 .map(Self::from_serde_value)
1089 .collect::<Result<Vec<_>>>()
1090 .map(Self::Array),
1091 SerdeValue::Map(values) => values
1092 .into_iter()
1093 .map(|(key, value)| {
1094 let SerdeValue::String(key) = key else {
1095 return Err(Error::Codec(
1096 "invalid_map_key: Avro Value map keys must be strings".to_string(),
1097 ));
1098 };
1099
1100 Ok((key, Self::from_serde_value(value)?))
1101 })
1102 .collect::<Result<BTreeMap<_, _>>>()
1103 .map(Self::Map),
1104 }
1105 }
1106
1107 fn finite_double(value: f64) -> Result<Self> {
1108 if !value.is_finite() {
1109 return Err(Error::Codec(
1110 "non_finite_float: Avro Value doubles must be finite".to_string(),
1111 ));
1112 }
1113
1114 Ok(Self::Double(value))
1115 }
1116
1117 fn into_json(self) -> Result<Value> {
1118 match self {
1119 Self::Null => Ok(Value::Null),
1120 Self::Boolean(value) => Ok(Value::Bool(value)),
1121 Self::Long(value) => Ok(Value::Number(value.into())),
1122 Self::Double(value) => serde_json::Number::from_f64(value)
1123 .map(Value::Number)
1124 .ok_or_else(|| {
1125 Error::Codec(
1126 "non_finite_float: decoded Avro Value double is not finite".to_string(),
1127 )
1128 }),
1129 Self::Bytes(value) => Ok(json!({
1130 "$type": "bytes",
1131 "base64": BASE64.encode(value),
1132 })),
1133 Self::String(value) => Ok(Value::String(value)),
1134 Self::Array(values) => values
1135 .into_iter()
1136 .map(Self::into_json)
1137 .collect::<Result<Vec<_>>>()
1138 .map(Value::Array),
1139 Self::Map(values) => values
1140 .into_iter()
1141 .map(|(key, value)| Ok((key, value.into_json()?)))
1142 .collect::<Result<serde_json::Map<_, _>>>()
1143 .map(Value::Object),
1144 }
1145 }
1146
1147 fn into_serde_value(self) -> serde_value::Value {
1148 use serde_value::Value as SerdeValue;
1149
1150 match self {
1151 Self::Null => SerdeValue::Unit,
1152 Self::Boolean(value) => SerdeValue::Bool(value),
1153 Self::Long(value) => SerdeValue::I64(value),
1154 Self::Double(value) => SerdeValue::F64(value),
1155 Self::Bytes(value) => SerdeValue::Bytes(value),
1156 Self::String(value) => SerdeValue::String(value),
1157 Self::Array(values) => {
1158 SerdeValue::Seq(values.into_iter().map(Self::into_serde_value).collect())
1159 }
1160 Self::Map(values) => SerdeValue::Map(
1161 values
1162 .into_iter()
1163 .map(|(key, value)| (SerdeValue::String(key), value.into_serde_value()))
1164 .collect(),
1165 ),
1166 }
1167 }
1168
1169 pub fn deserialize<T: DeserializeOwned>(self) -> Result<T> {
1170 self.into_serde_value().deserialize_into().map_err(|error| {
1171 Error::Codec(format!(
1172 "avro_value_type_mismatch: could not adapt decoded value: {error}"
1173 ))
1174 })
1175 }
1176}
1177
1178impl Serialize for AvroValue {
1179 fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
1180 where
1181 S: Serializer,
1182 {
1183 match self {
1184 Self::Null => serializer.serialize_unit(),
1185 Self::Boolean(value) => serializer.serialize_bool(*value),
1186 Self::Long(value) => serializer.serialize_i64(*value),
1187 Self::Double(value) => serializer.serialize_f64(*value),
1188 Self::Bytes(value) => serializer.serialize_bytes(value),
1189 Self::String(value) => serializer.serialize_str(value),
1190 Self::Array(values) => {
1191 let mut sequence = serializer.serialize_seq(Some(values.len()))?;
1192 for value in values {
1193 sequence.serialize_element(value)?;
1194 }
1195 sequence.end()
1196 }
1197 Self::Map(values) => {
1198 let mut map = serializer.serialize_map(Some(values.len()))?;
1199 for (key, value) in values {
1200 map.serialize_entry(key, value)?;
1201 }
1202 map.end()
1203 }
1204 }
1205 }
1206}
1207
1208pub fn encode_avro_value(value: &AvroValue) -> Result<PayloadEnvelope> {
1209 let datum = avro_value_to_datum(value)?;
1210 let datum = to_avro_datum(avro_value_schema()?, datum)
1211 .map_err(|err| Error::Codec(format!("avro_value_encode_failed: {err}")))?;
1212 let mut bytes = Vec::with_capacity(datum.len() + 10);
1213 bytes.extend_from_slice(&AVRO_SINGLE_OBJECT_MAGIC);
1214 bytes.extend_from_slice(&AVRO_VALUE_SCHEMA_FINGERPRINT);
1215 bytes.extend_from_slice(&datum);
1216 Ok(PayloadEnvelope {
1217 codec: DEFAULT_CODEC.to_string(),
1218 blob: BASE64.encode(bytes),
1219 })
1220}
1221
1222pub fn decode_avro_value(envelope: &PayloadEnvelope) -> Result<AvroValue> {
1223 if envelope.codec != DEFAULT_CODEC {
1224 return Err(Error::Codec(format!(
1225 "unsupported payload codec {:?}",
1226 envelope.codec
1227 )));
1228 }
1229 decode_avro_value_blob(&envelope.blob)
1230}
1231
1232pub fn encode_payload<T: Serialize>(value: &T, codec: &str) -> Result<PayloadEnvelope> {
1233 let blob = match codec {
1234 JSON_CODEC => serde_json::to_string(value)?,
1235 DEFAULT_CODEC => encode_avro_value(&AvroValue::from_serialize(value)?)?.blob,
1236 other => return Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1237 };
1238
1239 Ok(PayloadEnvelope {
1240 codec: codec.to_string(),
1241 blob,
1242 })
1243}
1244
1245pub fn decode_payload<T: DeserializeOwned>(envelope: &PayloadEnvelope) -> Result<T> {
1246 match envelope.codec.as_str() {
1247 JSON_CODEC => Ok(serde_json::from_str(&envelope.blob)?),
1248 DEFAULT_CODEC => decode_avro_value(envelope)?.deserialize(),
1249 other => Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1250 }
1251}
1252
1253#[cfg(test)]
1254fn encode_value_envelope(value: &Value, codec: &str) -> Result<Value> {
1255 Ok(serde_json::to_value(encode_payload(value, codec)?)?)
1256}
1257
1258fn decode_wire_value(value: &Value, fallback_codec: &str) -> Result<Value> {
1259 if value.is_null() {
1260 return Ok(Value::Null);
1261 }
1262
1263 if let Some(object) = value.as_object() {
1264 if let (Some(codec), Some(blob)) = (
1265 object.get("codec").and_then(Value::as_str),
1266 object.get("blob").and_then(Value::as_str),
1267 ) {
1268 return decode_blob(blob, codec);
1269 }
1270 }
1271
1272 if let Some(blob) = value.as_str() {
1273 return decode_blob(blob, fallback_codec);
1274 }
1275
1276 Ok(value.clone())
1277}
1278
1279fn encode_typed_envelope(value: &AvroValue, codec: &str) -> Result<Value> {
1280 let envelope = match codec {
1281 DEFAULT_CODEC => encode_avro_value(value)?,
1282 JSON_CODEC => PayloadEnvelope {
1283 codec: JSON_CODEC.to_string(),
1284 blob: serde_json::to_string(&value.clone().into_json()?)?,
1285 },
1286 other => return Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1287 };
1288 Ok(serde_json::to_value(envelope)?)
1289}
1290
1291fn decode_wire_avro_value(value: &Value, fallback_codec: &str) -> Result<AvroValue> {
1292 if value.is_null() {
1293 return Ok(AvroValue::Null);
1294 }
1295
1296 if let Some(object) = value.as_object() {
1297 if let (Some(codec), Some(blob)) = (
1298 object.get("codec").and_then(Value::as_str),
1299 object.get("blob").and_then(Value::as_str),
1300 ) {
1301 return match codec {
1302 DEFAULT_CODEC => decode_avro_value_blob(blob),
1303 JSON_CODEC => AvroValue::from_serialize(&serde_json::from_str::<Value>(blob)?),
1304 other => Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1305 };
1306 }
1307 }
1308
1309 if let Some(blob) = value.as_str() {
1310 return match fallback_codec {
1311 DEFAULT_CODEC => decode_avro_value_blob(blob),
1312 JSON_CODEC => AvroValue::from_serialize(&serde_json::from_str::<Value>(blob)?),
1313 other => Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1314 };
1315 }
1316
1317 AvroValue::from_serialize(value)
1318}
1319
1320fn normalize_avro_arguments(value: AvroValue) -> AvroValue {
1321 match value {
1322 AvroValue::Null => AvroValue::Array(Vec::new()),
1323 AvroValue::Array(_) => value,
1324 other => AvroValue::Array(vec![other]),
1325 }
1326}
1327
1328fn decode_blob(blob: &str, codec: &str) -> Result<Value> {
1329 match codec {
1330 JSON_CODEC => Ok(serde_json::from_str(blob)?),
1331 DEFAULT_CODEC => decode_avro_value_blob(blob)?.into_json(),
1332 other => Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1333 }
1334}
1335
1336fn decode_avro_value_blob(blob: &str) -> Result<AvroValue> {
1337 let bytes = BASE64.decode(blob).map_err(|err| {
1338 Error::Codec(format!(
1339 "invalid_payload_framing: expected strict base64 Avro single-object bytes: {err}"
1340 ))
1341 })?;
1342
1343 if bytes.len() < 10 || bytes[..2] != AVRO_SINGLE_OBJECT_MAGIC {
1344 return Err(Error::Codec(
1345 "invalid_payload_framing: expected Avro single-object magic c301".to_string(),
1346 ));
1347 }
1348
1349 let fingerprint: [u8; 8] = bytes[2..10]
1350 .try_into()
1351 .map_err(|_| Error::Codec("invalid Avro fingerprint length".to_string()))?;
1352 if fingerprint != AVRO_VALUE_SCHEMA_FINGERPRINT {
1353 return Err(Error::Codec(format!(
1354 "unsupported_payload_schema: unknown CRC-64-AVRO fingerprint {}",
1355 fingerprint
1356 .iter()
1357 .map(|byte| format!("{byte:02x}"))
1358 .collect::<String>()
1359 )));
1360 }
1361
1362 let mut datum_reader = StrictAvroDatumReader::new(&bytes[10..]);
1363 let datum = from_avro_datum(avro_value_schema()?, &mut datum_reader, None);
1367 if datum_reader.truncated {
1368 return Err(Error::Codec(
1369 "invalid_payload_framing: truncated Avro Value datum".to_string(),
1370 ));
1371 }
1372 let datum = datum.map_err(|err| {
1373 Error::Codec(format!(
1374 "invalid_payload_framing: malformed Avro Value datum: {err}"
1375 ))
1376 })?;
1377 if datum_reader.remaining() != 0 {
1378 return Err(Error::Codec(format!(
1379 "invalid_payload_framing: {} trailing bytes after Avro Value datum",
1380 datum_reader.remaining()
1381 )));
1382 }
1383 avro_value_from_datum(datum)
1384}
1385
1386struct StrictAvroDatumReader<'a> {
1387 bytes: &'a [u8],
1388 offset: usize,
1389 truncated: bool,
1390}
1391
1392impl<'a> StrictAvroDatumReader<'a> {
1393 fn new(bytes: &'a [u8]) -> Self {
1394 Self {
1395 bytes,
1396 offset: 0,
1397 truncated: false,
1398 }
1399 }
1400
1401 fn remaining(&self) -> usize {
1402 self.bytes.len() - self.offset
1403 }
1404}
1405
1406impl Read for StrictAvroDatumReader<'_> {
1407 fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
1408 let count = buffer.len().min(self.remaining());
1409 buffer[..count].copy_from_slice(&self.bytes[self.offset..self.offset + count]);
1410 self.offset += count;
1411 if count < buffer.len() {
1412 self.truncated = true;
1413 }
1414
1415 Ok(count)
1416 }
1417}
1418
1419fn avro_value_to_datum(value: &AvroValue) -> Result<AvroDatum> {
1420 let branch = match value {
1421 AvroValue::Null => AvroDatum::Union(0, Box::new(AvroDatum::Null)),
1422 AvroValue::Boolean(value) => AvroDatum::Union(
1423 1,
1424 Box::new(AvroDatum::Record(vec![(
1425 "boolean".to_string(),
1426 AvroDatum::Boolean(*value),
1427 )])),
1428 ),
1429 AvroValue::Long(value) => AvroDatum::Union(
1430 2,
1431 Box::new(AvroDatum::Record(vec![(
1432 "long".to_string(),
1433 AvroDatum::Long(*value),
1434 )])),
1435 ),
1436 AvroValue::Double(value) => {
1437 if !value.is_finite() {
1438 return Err(Error::Codec(
1439 "non_finite_float: Avro Value doubles must be finite".to_string(),
1440 ));
1441 }
1442 AvroDatum::Union(
1443 3,
1444 Box::new(AvroDatum::Record(vec![(
1445 "double".to_string(),
1446 AvroDatum::Double(*value),
1447 )])),
1448 )
1449 }
1450 AvroValue::Bytes(value) => AvroDatum::Union(
1451 4,
1452 Box::new(AvroDatum::Record(vec![(
1453 "bytes".to_string(),
1454 AvroDatum::Bytes(value.clone()),
1455 )])),
1456 ),
1457 AvroValue::String(value) => AvroDatum::Union(
1458 5,
1459 Box::new(AvroDatum::Record(vec![(
1460 "string".to_string(),
1461 AvroDatum::String(value.clone()),
1462 )])),
1463 ),
1464 AvroValue::Array(values) => AvroDatum::Union(
1465 6,
1466 Box::new(AvroDatum::Record(vec![(
1467 "items".to_string(),
1468 AvroDatum::Array(
1469 values
1470 .iter()
1471 .map(avro_value_to_datum)
1472 .collect::<Result<Vec<_>>>()?,
1473 ),
1474 )])),
1475 ),
1476 AvroValue::Map(values) => AvroDatum::Union(
1477 7,
1478 Box::new(AvroDatum::Record(vec![(
1479 "entries".to_string(),
1480 AvroDatum::Map(
1481 values
1482 .iter()
1483 .map(|(key, value)| Ok((key.clone(), avro_value_to_datum(value)?)))
1484 .collect::<Result<HashMap<_, _>>>()?,
1485 ),
1486 )])),
1487 ),
1488 };
1489 Ok(AvroDatum::Record(vec![("value".to_string(), branch)]))
1490}
1491
1492fn avro_value_from_datum(datum: AvroDatum) -> Result<AvroValue> {
1493 let AvroDatum::Record(mut outer) = datum else {
1494 return Err(Error::Codec(
1495 "invalid_payload_framing: datum is not a Value record".to_string(),
1496 ));
1497 };
1498 let (_, branch) = outer
1499 .pop()
1500 .filter(|(name, _)| name == "value")
1501 .ok_or_else(|| Error::Codec("invalid_payload_framing: Value field missing".to_string()))?;
1502 let AvroDatum::Union(_, branch) = branch else {
1503 return Err(Error::Codec(
1504 "invalid_payload_framing: invalid Value union".to_string(),
1505 ));
1506 };
1507 match *branch {
1508 AvroDatum::Null => Ok(AvroValue::Null),
1509 AvroDatum::Record(mut fields) => {
1510 let (name, value) = fields.pop().ok_or_else(|| {
1511 Error::Codec("invalid_payload_framing: empty Value branch".to_string())
1512 })?;
1513 match (name.as_str(), value) {
1514 ("boolean", AvroDatum::Boolean(value)) => Ok(AvroValue::Boolean(value)),
1515 ("long", AvroDatum::Long(value)) => Ok(AvroValue::Long(value)),
1516 ("double", AvroDatum::Double(value)) if value.is_finite() => {
1517 Ok(AvroValue::Double(value))
1518 }
1519 ("bytes", AvroDatum::Bytes(value)) => Ok(AvroValue::Bytes(value)),
1520 ("string", AvroDatum::String(value)) => Ok(AvroValue::String(value)),
1521 ("items", AvroDatum::Array(values)) => values
1522 .into_iter()
1523 .map(avro_value_from_datum)
1524 .collect::<Result<Vec<_>>>()
1525 .map(AvroValue::Array),
1526 ("entries", AvroDatum::Map(values)) => values
1527 .into_iter()
1528 .map(|(key, value)| Ok((key, avro_value_from_datum(value)?)))
1529 .collect::<Result<BTreeMap<_, _>>>()
1530 .map(AvroValue::Map),
1531 _ => Err(Error::Codec(
1532 "invalid_payload_framing: unknown Value branch".to_string(),
1533 )),
1534 }
1535 }
1536 _ => Err(Error::Codec(
1537 "invalid_payload_framing: invalid Value branch".to_string(),
1538 )),
1539 }
1540}
1541
1542fn avro_value_schema() -> Result<&'static Schema> {
1543 match AVRO_VALUE_SCHEMA.get_or_init(|| {
1544 Schema::parse_str(AVRO_VALUE_SCHEMA_JSON)
1545 .map_err(|err| format!("could not parse Avro Value schema: {err}"))
1546 }) {
1547 Ok(schema) => Ok(schema),
1548 Err(message) => Err(Error::Codec(message.clone())),
1549 }
1550}
1551
1552#[derive(Clone, Debug)]
1553pub struct Client {
1554 http: reqwest::Client,
1555 base_url: String,
1556 token: Option<String>,
1557 control_token: Option<String>,
1558 worker_token: Option<String>,
1559 namespace: String,
1560}
1561
1562impl Client {
1563 pub fn new(base_url: impl Into<String>) -> Result<Self> {
1564 Self::builder(base_url).build()
1565 }
1566
1567 pub fn builder(base_url: impl Into<String>) -> ClientBuilder {
1568 ClientBuilder {
1569 base_url: base_url.into(),
1570 token: None,
1571 control_token: None,
1572 worker_token: None,
1573 namespace: "default".to_string(),
1574 timeout: Duration::from_secs(60),
1575 }
1576 }
1577
1578 pub async fn health(&self) -> Result<Value> {
1579 self.request_json(
1580 reqwest::Method::GET,
1581 "/health",
1582 RequestProtocol::ControlPlane,
1583 Option::<&Value>::None,
1584 )
1585 .await
1586 }
1587
1588 pub async fn cluster_info(&self) -> Result<Value> {
1589 self.request_json(
1590 reqwest::Method::GET,
1591 "/cluster/info",
1592 RequestProtocol::ControlPlane,
1593 Option::<&Value>::None,
1594 )
1595 .await
1596 }
1597
1598 pub async fn start_workflow<T: Serialize>(
1599 &self,
1600 workflow_type: &str,
1601 task_queue: &str,
1602 workflow_id: &str,
1603 input: T,
1604 ) -> Result<WorkflowHandle> {
1605 self.start_workflow_with_options(
1606 workflow_type,
1607 task_queue,
1608 workflow_id,
1609 WorkflowStartOptions::default(),
1610 input,
1611 )
1612 .await
1613 }
1614
1615 pub async fn start_workflow_with_options<T: Serialize>(
1618 &self,
1619 workflow_type: &str,
1620 task_queue: &str,
1621 workflow_id: &str,
1622 options: WorkflowStartOptions,
1623 input: T,
1624 ) -> Result<WorkflowHandle> {
1625 options.validate()?;
1626 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1627 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
1628 let body = json!({
1629 "workflow_id": workflow_id,
1630 "workflow_type": workflow_type,
1631 "task_queue": task_queue,
1632 "input": input_envelope,
1633 "execution_timeout_seconds": options.execution_timeout_seconds,
1634 "run_timeout_seconds": options.run_timeout_seconds
1635 });
1636
1637 let data: Value = self
1638 .request_json(
1639 reqwest::Method::POST,
1640 "/workflows",
1641 RequestProtocol::ControlPlane,
1642 Some(&body),
1643 )
1644 .await?;
1645
1646 Ok(WorkflowHandle {
1647 client: self.clone(),
1648 workflow_id: data
1649 .get("workflow_id")
1650 .and_then(Value::as_str)
1651 .unwrap_or(workflow_id)
1652 .to_string(),
1653 run_id: data
1654 .get("run_id")
1655 .and_then(Value::as_str)
1656 .map(str::to_string),
1657 workflow_type: data
1658 .get("workflow_type")
1659 .and_then(Value::as_str)
1660 .unwrap_or(workflow_type)
1661 .to_string(),
1662 })
1663 }
1664
1665 pub async fn signal_workflow<T: Serialize>(
1666 &self,
1667 workflow_id: &str,
1668 signal_name: &str,
1669 input: T,
1670 ) -> Result<Value> {
1671 self.signal_workflow_target(workflow_id, None, signal_name, input)
1672 .await
1673 }
1674
1675 pub async fn signal_workflow_run<T: Serialize>(
1677 &self,
1678 workflow_id: &str,
1679 run_id: &str,
1680 signal_name: &str,
1681 input: T,
1682 ) -> Result<Value> {
1683 self.signal_workflow_target(workflow_id, Some(run_id), signal_name, input)
1684 .await
1685 }
1686
1687 async fn signal_workflow_target<T: Serialize>(
1688 &self,
1689 workflow_id: &str,
1690 run_id: Option<&str>,
1691 signal_name: &str,
1692 input: T,
1693 ) -> Result<Value> {
1694 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1695 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
1696 let body = json!({
1697 "input": input_envelope
1698 });
1699 let path = match run_id {
1700 Some(run_id) => {
1701 format!("/workflows/{workflow_id}/runs/{run_id}/signal/{signal_name}")
1702 }
1703 None => format!("/workflows/{workflow_id}/signal/{signal_name}"),
1704 };
1705 self.request_json(
1706 reqwest::Method::POST,
1707 &path,
1708 RequestProtocol::ControlPlane,
1709 Some(&body),
1710 )
1711 .await
1712 }
1713
1714 pub async fn cancel_workflow(
1716 &self,
1717 workflow_id: &str,
1718 options: WorkflowCommandOptions,
1719 ) -> Result<WorkflowCommandResult> {
1720 self.workflow_command(workflow_id, None, WorkflowCommandKind::Cancel, options)
1721 .await
1722 }
1723
1724 pub async fn cancel_workflow_run(
1726 &self,
1727 workflow_id: &str,
1728 run_id: &str,
1729 options: WorkflowCommandOptions,
1730 ) -> Result<WorkflowCommandResult> {
1731 self.workflow_command(
1732 workflow_id,
1733 Some(run_id),
1734 WorkflowCommandKind::Cancel,
1735 options,
1736 )
1737 .await
1738 }
1739
1740 pub async fn terminate_workflow(
1742 &self,
1743 workflow_id: &str,
1744 options: WorkflowCommandOptions,
1745 ) -> Result<WorkflowCommandResult> {
1746 self.workflow_command(workflow_id, None, WorkflowCommandKind::Terminate, options)
1747 .await
1748 }
1749
1750 pub async fn terminate_workflow_run(
1752 &self,
1753 workflow_id: &str,
1754 run_id: &str,
1755 options: WorkflowCommandOptions,
1756 ) -> Result<WorkflowCommandResult> {
1757 self.workflow_command(
1758 workflow_id,
1759 Some(run_id),
1760 WorkflowCommandKind::Terminate,
1761 options,
1762 )
1763 .await
1764 }
1765
1766 async fn workflow_command(
1767 &self,
1768 workflow_id: &str,
1769 run_id: Option<&str>,
1770 command: WorkflowCommandKind,
1771 options: WorkflowCommandOptions,
1772 ) -> Result<WorkflowCommandResult> {
1773 let path = match run_id {
1774 Some(run_id) => format!(
1775 "/workflows/{workflow_id}/runs/{run_id}/{}",
1776 command.as_str()
1777 ),
1778 None => format!("/workflows/{workflow_id}/{}", command.as_str()),
1779 };
1780 let data = match self
1781 .request_json(
1782 reqwest::Method::POST,
1783 &path,
1784 RequestProtocol::ControlPlane,
1785 Some(&options),
1786 )
1787 .await
1788 {
1789 Ok(data) => data,
1790 Err(Error::Http { status, body }) => {
1791 return Err(Error::WorkflowCommandRejected(workflow_command_rejection(
1792 command,
1793 status,
1794 body,
1795 workflow_id,
1796 run_id,
1797 )));
1798 }
1799 Err(error) => return Err(error),
1800 };
1801
1802 Ok(workflow_command_result(command, data, workflow_id, run_id))
1803 }
1804
1805 pub async fn query_workflow<T: Serialize>(
1811 &self,
1812 workflow_id: &str,
1813 query_name: &str,
1814 input: T,
1815 ) -> Result<Value> {
1816 self.query_workflow_target(workflow_id, None, query_name, input)
1817 .await
1818 }
1819
1820 pub async fn query_workflow_run<T: Serialize>(
1822 &self,
1823 workflow_id: &str,
1824 run_id: &str,
1825 query_name: &str,
1826 input: T,
1827 ) -> Result<Value> {
1828 self.query_workflow_target(workflow_id, Some(run_id), query_name, input)
1829 .await
1830 }
1831
1832 pub async fn query_workflow_avro_value<T: Serialize>(
1834 &self,
1835 workflow_id: &str,
1836 query_name: &str,
1837 input: T,
1838 ) -> Result<AvroValue> {
1839 self.query_workflow_avro_value_target(workflow_id, None, query_name, input)
1840 .await
1841 }
1842
1843 pub async fn query_workflow_run_avro_value<T: Serialize>(
1845 &self,
1846 workflow_id: &str,
1847 run_id: &str,
1848 query_name: &str,
1849 input: T,
1850 ) -> Result<AvroValue> {
1851 self.query_workflow_avro_value_target(workflow_id, Some(run_id), query_name, input)
1852 .await
1853 }
1854
1855 async fn query_workflow_avro_value_target<T: Serialize>(
1856 &self,
1857 workflow_id: &str,
1858 run_id: Option<&str>,
1859 query_name: &str,
1860 input: T,
1861 ) -> Result<AvroValue> {
1862 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1863 let body = json!({"input": encode_typed_envelope(&input, DEFAULT_CODEC)?});
1864 let path = match run_id {
1865 Some(run_id) => {
1866 format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
1867 }
1868 None => format!("/workflows/{workflow_id}/query/{query_name}"),
1869 };
1870 let response: Value = match self
1871 .request_json(
1872 reqwest::Method::POST,
1873 &path,
1874 RequestProtocol::ControlPlane,
1875 Some(&body),
1876 )
1877 .await
1878 {
1879 Ok(response) => response,
1880 Err(Error::Http { status, body }) => {
1881 return Err(Error::QueryFailed(query_failure(status, body)));
1882 }
1883 Err(error) => return Err(error),
1884 };
1885
1886 let envelope = response
1887 .get("result_envelope")
1888 .filter(|envelope| !envelope.is_null())
1889 .ok_or_else(|| {
1890 Error::Codec(
1891 "missing_payload_envelope: typed query result requires result_envelope"
1892 .to_string(),
1893 )
1894 })?;
1895 decode_wire_avro_value(envelope, DEFAULT_CODEC)
1896 }
1897
1898 async fn query_workflow_target<T: Serialize>(
1899 &self,
1900 workflow_id: &str,
1901 run_id: Option<&str>,
1902 query_name: &str,
1903 input: T,
1904 ) -> Result<Value> {
1905 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1906 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
1907 let body = json!({
1908 "input": input_envelope
1909 });
1910 let path = match run_id {
1911 Some(run_id) => {
1912 format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
1913 }
1914 None => format!("/workflows/{workflow_id}/query/{query_name}"),
1915 };
1916 let response: Value = match self
1917 .request_json(
1918 reqwest::Method::POST,
1919 &path,
1920 RequestProtocol::ControlPlane,
1921 Some(&body),
1922 )
1923 .await
1924 {
1925 Ok(response) => response,
1926 Err(Error::Http { status, body }) => {
1927 return Err(Error::QueryFailed(query_failure(status, body)));
1928 }
1929 Err(error) => return Err(error),
1930 };
1931
1932 if let Some(envelope) = response
1933 .get("result_envelope")
1934 .filter(|envelope| !envelope.is_null())
1935 {
1936 return decode_wire_value(envelope, DEFAULT_CODEC);
1937 }
1938
1939 Ok(response.get("result").cloned().unwrap_or(Value::Null))
1940 }
1941
1942 pub async fn update_workflow<T: Serialize>(
1944 &self,
1945 workflow_id: &str,
1946 update_name: &str,
1947 input: T,
1948 request_id: Option<&str>,
1949 ) -> Result<Value> {
1950 let response = self
1951 .update_workflow_response(workflow_id, update_name, input, request_id)
1952 .await?;
1953 if let Some(envelope) = response
1954 .get("result_envelope")
1955 .filter(|envelope| !envelope.is_null())
1956 {
1957 return decode_wire_value(envelope, DEFAULT_CODEC);
1958 }
1959 Ok(response.get("result").cloned().unwrap_or(response))
1960 }
1961
1962 pub async fn update_workflow_avro_value<T: Serialize>(
1964 &self,
1965 workflow_id: &str,
1966 update_name: &str,
1967 input: T,
1968 request_id: Option<&str>,
1969 ) -> Result<AvroValue> {
1970 let response = self
1971 .update_workflow_response(workflow_id, update_name, input, request_id)
1972 .await?;
1973 let envelope = response
1974 .get("result_envelope")
1975 .filter(|envelope| !envelope.is_null())
1976 .ok_or_else(|| {
1977 Error::Codec(
1978 "missing_payload_envelope: typed update result requires result_envelope"
1979 .to_string(),
1980 )
1981 })?;
1982 decode_wire_avro_value(envelope, DEFAULT_CODEC)
1983 }
1984
1985 async fn update_workflow_response<T: Serialize>(
1986 &self,
1987 workflow_id: &str,
1988 update_name: &str,
1989 input: T,
1990 request_id: Option<&str>,
1991 ) -> Result<Value> {
1992 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1993 let mut body = json!({
1994 "input": encode_typed_envelope(&input, DEFAULT_CODEC)?,
1995 "wait_for": "completed",
1996 });
1997 if let Some(request_id) = request_id {
1998 body["request_id"] = json!(request_id);
1999 }
2000 self.request_json(
2001 reqwest::Method::POST,
2002 &format!("/workflows/{workflow_id}/update/{update_name}"),
2003 RequestProtocol::ControlPlane,
2004 Some(&body),
2005 )
2006 .await
2007 }
2008
2009 pub async fn describe_workflow(&self, workflow_id: &str) -> Result<WorkflowDescription> {
2010 let path = format!("/workflows/{workflow_id}");
2011 let mut data: WorkflowDescription = self
2012 .request_json(
2013 reqwest::Method::GET,
2014 &path,
2015 RequestProtocol::ControlPlane,
2016 Option::<&Value>::None,
2017 )
2018 .await?;
2019 data.decode_payloads()?;
2020 Ok(data)
2021 }
2022
2023 pub async fn describe_workflow_run(
2025 &self,
2026 workflow_id: &str,
2027 run_id: &str,
2028 ) -> Result<WorkflowDescription> {
2029 let path = format!("/workflows/{workflow_id}/runs/{run_id}");
2030 let mut data: WorkflowDescription = self
2031 .request_json(
2032 reqwest::Method::GET,
2033 &path,
2034 RequestProtocol::ControlPlane,
2035 Option::<&Value>::None,
2036 )
2037 .await?;
2038 data.decode_payloads()?;
2039 Ok(data)
2040 }
2041
2042 pub async fn register_worker(
2043 &self,
2044 worker_id: &str,
2045 task_queue: &str,
2046 supported_workflow_types: Vec<String>,
2047 supported_activity_types: Vec<String>,
2048 max_concurrent_workflow_tasks: usize,
2049 max_concurrent_activity_tasks: usize,
2050 ) -> Result<RegisterWorkerResponse> {
2051 self.register_worker_with_capabilities(
2052 worker_id,
2053 task_queue,
2054 supported_workflow_types,
2055 supported_activity_types,
2056 max_concurrent_workflow_tasks,
2057 max_concurrent_activity_tasks,
2058 Vec::new(),
2059 )
2060 .await
2061 }
2062
2063 pub async fn register_worker_with_capabilities(
2065 &self,
2066 worker_id: &str,
2067 task_queue: &str,
2068 supported_workflow_types: Vec<String>,
2069 supported_activity_types: Vec<String>,
2070 max_concurrent_workflow_tasks: usize,
2071 max_concurrent_activity_tasks: usize,
2072 capabilities: Vec<String>,
2073 ) -> Result<RegisterWorkerResponse> {
2074 self.register_worker_with_command_contracts(
2075 worker_id,
2076 task_queue,
2077 supported_workflow_types,
2078 supported_activity_types,
2079 max_concurrent_workflow_tasks,
2080 max_concurrent_activity_tasks,
2081 capabilities,
2082 Value::Object(serde_json::Map::new()),
2083 )
2084 .await
2085 }
2086
2087 #[allow(clippy::too_many_arguments)]
2094 pub async fn register_worker_with_command_contracts(
2095 &self,
2096 worker_id: &str,
2097 task_queue: &str,
2098 supported_workflow_types: Vec<String>,
2099 supported_activity_types: Vec<String>,
2100 max_concurrent_workflow_tasks: usize,
2101 max_concurrent_activity_tasks: usize,
2102 capabilities: Vec<String>,
2103 workflow_command_contracts: Value,
2104 ) -> Result<RegisterWorkerResponse> {
2105 if let Some(contracts) = workflow_command_contracts.as_object() {
2106 for (workflow_type, contract) in contracts {
2107 let Some(update_validators) = contract.get("update_validators") else {
2108 continue;
2109 };
2110 if !update_validators
2111 .as_array()
2112 .is_some_and(|validators| validators.is_empty())
2113 {
2114 return Err(Error::UnsupportedUpdateValidators {
2115 workflow_type: workflow_type.clone(),
2116 });
2117 }
2118 }
2119 }
2120
2121 let mut body = json!({
2122 "worker_id": worker_id,
2123 "task_queue": task_queue,
2124 "runtime": "rust",
2125 "sdk_version": SDK_VERSION,
2126 "supported_workflow_types": supported_workflow_types,
2127 "supported_activity_types": supported_activity_types,
2128 "capabilities": capabilities,
2129 "max_concurrent_workflow_tasks": max_concurrent_workflow_tasks,
2130 "max_concurrent_activity_tasks": max_concurrent_activity_tasks
2131 });
2132 if workflow_command_contracts
2133 .as_object()
2134 .is_some_and(|contracts| !contracts.is_empty())
2135 {
2136 body["workflow_command_contracts"] = workflow_command_contracts;
2137 }
2138
2139 self.request_json(
2140 reqwest::Method::POST,
2141 "/worker/register",
2142 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2143 Some(&body),
2144 )
2145 .await
2146 }
2147
2148 pub async fn deregister_worker_registration(
2154 &self,
2155 worker_id: &str,
2156 ) -> Result<WorkerDeregistrationEnvelope> {
2157 let path = format!(
2158 "/worker/registrations/{}",
2159 percent_encode_path_segment(worker_id)
2160 );
2161 self.request_json(
2162 reqwest::Method::DELETE,
2163 &path,
2164 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2165 Option::<&Value>::None,
2166 )
2167 .await
2168 }
2169
2170 pub async fn poll_query_task(
2172 &self,
2173 worker_id: &str,
2174 task_queue: &str,
2175 timeout: Duration,
2176 ) -> Result<Option<QueryTask>> {
2177 Ok(self
2178 .poll_query_task_response(worker_id, task_queue, timeout)
2179 .await?
2180 .task)
2181 }
2182
2183 pub async fn poll_query_task_response(
2185 &self,
2186 worker_id: &str,
2187 task_queue: &str,
2188 timeout: Duration,
2189 ) -> Result<PollQueryTaskResponse> {
2190 let poll_request_id = unique_request_id("rust-query-poll");
2191 self.poll_query_task_response_with_request_id(
2192 worker_id,
2193 task_queue,
2194 timeout,
2195 &poll_request_id,
2196 1,
2197 )
2198 .await
2199 }
2200
2201 async fn poll_query_task_response_with_request_id(
2202 &self,
2203 worker_id: &str,
2204 task_queue: &str,
2205 timeout: Duration,
2206 poll_request_id: &str,
2207 transport_retries: usize,
2208 ) -> Result<PollQueryTaskResponse> {
2209 let timeout_seconds = long_poll_timeout_seconds(timeout);
2210 let body = json!({
2211 "worker_id": worker_id,
2212 "task_queue": task_queue,
2213 "poll_request_id": poll_request_id,
2214 "timeout_seconds": timeout_seconds,
2215 });
2216 self.poll_request_json(
2217 "/worker/query-tasks/poll",
2218 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
2219 &body,
2220 timeout + Duration::from_secs(5),
2221 transport_retries,
2222 )
2223 .await
2224 }
2225
2226 pub async fn complete_query_task<T: Serialize>(
2228 &self,
2229 query_task_id: &str,
2230 lease_owner: &str,
2231 query_task_attempt: u64,
2232 result: T,
2233 codec: &str,
2234 ) -> Result<Value> {
2235 let typed_result = AvroValue::from_serialize(&result)?;
2236 let result_envelope = encode_typed_envelope(&typed_result, codec)?;
2237 self.complete_query_task_with_envelope(
2238 query_task_id,
2239 lease_owner,
2240 query_task_attempt,
2241 typed_result.into_json()?,
2242 result_envelope,
2243 )
2244 .await
2245 }
2246
2247 async fn complete_query_task_with_envelope(
2248 &self,
2249 query_task_id: &str,
2250 lease_owner: &str,
2251 query_task_attempt: u64,
2252 result: Value,
2253 result_envelope: Value,
2254 ) -> Result<Value> {
2255 let body = json!({
2256 "lease_owner": lease_owner,
2257 "query_task_attempt": query_task_attempt,
2258 "result": result,
2259 "result_envelope": result_envelope,
2260 });
2261 let path = format!("/worker/query-tasks/{query_task_id}/complete");
2262 let response = self
2263 .request_json(
2264 reqwest::Method::POST,
2265 &path,
2266 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
2267 Some(&body),
2268 )
2269 .await;
2270 query_task_response(response)
2271 }
2272
2273 pub async fn fail_query_task(
2275 &self,
2276 query_task_id: &str,
2277 lease_owner: &str,
2278 query_task_attempt: u64,
2279 message: impl Into<String>,
2280 reason: impl Into<String>,
2281 failure_type: impl Into<String>,
2282 ) -> Result<Value> {
2283 let body = json!({
2284 "lease_owner": lease_owner,
2285 "query_task_attempt": query_task_attempt,
2286 "failure": {
2287 "message": message.into(),
2288 "reason": reason.into(),
2289 "type": failure_type.into(),
2290 }
2291 });
2292 let path = format!("/worker/query-tasks/{query_task_id}/fail");
2293 let response = self
2294 .request_json(
2295 reqwest::Method::POST,
2296 &path,
2297 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
2298 Some(&body),
2299 )
2300 .await;
2301 query_task_response(response)
2302 }
2303
2304 pub async fn heartbeat_worker(
2305 &self,
2306 worker_id: &str,
2307 workflow_available: usize,
2308 activity_available: usize,
2309 ) -> Result<Value> {
2310 let body = json!({
2311 "worker_id": worker_id,
2312 "task_slots": {
2313 "workflow_available": workflow_available,
2314 "activity_available": activity_available
2315 },
2316 "process_metrics": {
2317 "process_id": std::process::id(),
2318 "process_uptime_seconds": 0
2319 }
2320 });
2321
2322 self.request_json(
2323 reqwest::Method::POST,
2324 "/worker/heartbeat",
2325 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2326 Some(&body),
2327 )
2328 .await
2329 }
2330
2331 pub async fn poll_workflow_task(
2332 &self,
2333 worker_id: &str,
2334 task_queue: &str,
2335 timeout: Duration,
2336 ) -> Result<Option<WorkflowTask>> {
2337 Ok(self
2338 .poll_workflow_task_response(worker_id, task_queue, timeout)
2339 .await?
2340 .task)
2341 }
2342
2343 pub async fn poll_workflow_task_response(
2344 &self,
2345 worker_id: &str,
2346 task_queue: &str,
2347 timeout: Duration,
2348 ) -> Result<PollWorkflowTaskResponse> {
2349 let poll_request_id = unique_request_id("rust-workflow-poll");
2350 self.poll_workflow_task_response_with_request_id(
2351 worker_id,
2352 task_queue,
2353 timeout,
2354 &poll_request_id,
2355 1,
2356 )
2357 .await
2358 }
2359
2360 async fn poll_workflow_task_response_with_request_id(
2361 &self,
2362 worker_id: &str,
2363 task_queue: &str,
2364 timeout: Duration,
2365 poll_request_id: &str,
2366 transport_retries: usize,
2367 ) -> Result<PollWorkflowTaskResponse> {
2368 let body = json!({
2369 "worker_id": worker_id,
2370 "task_queue": task_queue,
2371 "poll_request_id": poll_request_id,
2372 "timeout_seconds": long_poll_timeout_seconds(timeout),
2373 });
2374 let mut data: PollWorkflowTaskResponse = self
2375 .poll_request_json(
2376 "/worker/workflow-tasks/poll",
2377 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2378 &body,
2379 timeout + Duration::from_secs(5),
2380 transport_retries,
2381 )
2382 .await?;
2383
2384 if let Some(task) = data.task.as_mut() {
2385 self.fetch_remaining_workflow_history(worker_id, task)
2386 .await?;
2387 }
2388
2389 Ok(data)
2390 }
2391
2392 async fn fetch_remaining_workflow_history(
2393 &self,
2394 worker_id: &str,
2395 task: &mut WorkflowTask,
2396 ) -> Result<()> {
2397 let mut next_token = task.next_history_page_token.clone();
2398
2399 while let Some(token) = next_token.take().filter(|token| !token.is_empty()) {
2400 let lease_owner = task
2401 .lease_owner
2402 .clone()
2403 .unwrap_or_else(|| worker_id.to_string());
2404 let page = self
2405 .workflow_task_history_page(
2406 &task.task_id,
2407 &lease_owner,
2408 task.workflow_task_attempt,
2409 &token,
2410 )
2411 .await?;
2412
2413 task.append_history_page(page);
2414
2415 if task.next_history_page_token.as_deref() == Some(token.as_str()) {
2416 return Err(Error::Codec(
2417 "workflow history pagination returned the same page token".to_string(),
2418 ));
2419 }
2420
2421 next_token = task.next_history_page_token.clone();
2422 }
2423
2424 Ok(())
2425 }
2426
2427 async fn workflow_task_history_page(
2428 &self,
2429 task_id: &str,
2430 lease_owner: &str,
2431 workflow_task_attempt: u64,
2432 next_history_page_token: &str,
2433 ) -> Result<WorkflowTaskHistoryPage> {
2434 let body = json!({
2435 "lease_owner": lease_owner,
2436 "workflow_task_attempt": workflow_task_attempt,
2437 "next_history_page_token": next_history_page_token
2438 });
2439 let path = format!("/worker/workflow-tasks/{task_id}/history");
2440
2441 self.request_json(
2442 reqwest::Method::POST,
2443 &path,
2444 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2445 Some(&body),
2446 )
2447 .await
2448 }
2449
2450 pub async fn complete_workflow_task(
2451 &self,
2452 task_id: &str,
2453 lease_owner: &str,
2454 workflow_task_attempt: u64,
2455 commands: Vec<Value>,
2456 ) -> Result<Value> {
2457 let body = json!({
2458 "lease_owner": lease_owner,
2459 "workflow_task_attempt": workflow_task_attempt,
2460 "commands": commands
2461 });
2462 let path = format!("/worker/workflow-tasks/{task_id}/complete");
2463 self.request_json(
2464 reqwest::Method::POST,
2465 &path,
2466 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2467 Some(&body),
2468 )
2469 .await
2470 }
2471
2472 pub async fn fail_workflow_task(
2473 &self,
2474 task_id: &str,
2475 lease_owner: &str,
2476 workflow_task_attempt: u64,
2477 message: impl Into<String>,
2478 ) -> Result<Value> {
2479 self.fail_workflow_task_with_type(
2480 task_id,
2481 lease_owner,
2482 workflow_task_attempt,
2483 message,
2484 "RustWorkflowTaskFailure",
2485 )
2486 .await
2487 }
2488
2489 async fn fail_workflow_task_with_type(
2490 &self,
2491 task_id: &str,
2492 lease_owner: &str,
2493 workflow_task_attempt: u64,
2494 message: impl Into<String>,
2495 failure_type: &str,
2496 ) -> Result<Value> {
2497 let body = json!({
2498 "lease_owner": lease_owner,
2499 "workflow_task_attempt": workflow_task_attempt,
2500 "failure": {
2501 "message": message.into(),
2502 "type": failure_type
2503 }
2504 });
2505 let path = format!("/worker/workflow-tasks/{task_id}/fail");
2506 self.request_json(
2507 reqwest::Method::POST,
2508 &path,
2509 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2510 Some(&body),
2511 )
2512 .await
2513 }
2514
2515 pub async fn poll_activity_task(
2516 &self,
2517 worker_id: &str,
2518 task_queue: &str,
2519 timeout: Duration,
2520 ) -> Result<Option<ActivityTask>> {
2521 Ok(self
2522 .poll_activity_task_response(worker_id, task_queue, timeout)
2523 .await?
2524 .task)
2525 }
2526
2527 pub async fn poll_activity_task_response(
2529 &self,
2530 worker_id: &str,
2531 task_queue: &str,
2532 timeout: Duration,
2533 ) -> Result<PollActivityTaskResponse> {
2534 let poll_request_id = unique_request_id("rust-activity-poll");
2535 self.poll_activity_task_response_with_request_id(
2536 worker_id,
2537 task_queue,
2538 timeout,
2539 &poll_request_id,
2540 1,
2541 )
2542 .await
2543 }
2544
2545 async fn poll_activity_task_response_with_request_id(
2546 &self,
2547 worker_id: &str,
2548 task_queue: &str,
2549 timeout: Duration,
2550 poll_request_id: &str,
2551 transport_retries: usize,
2552 ) -> Result<PollActivityTaskResponse> {
2553 let body = json!({
2554 "worker_id": worker_id,
2555 "task_queue": task_queue,
2556 "poll_request_id": poll_request_id,
2557 "timeout_seconds": long_poll_timeout_seconds(timeout),
2558 });
2559 let data: PollActivityTaskResponse = self
2560 .poll_request_json(
2561 "/worker/activity-tasks/poll",
2562 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2563 &body,
2564 timeout + Duration::from_secs(5),
2565 transport_retries,
2566 )
2567 .await?;
2568 Ok(data)
2569 }
2570
2571 pub async fn complete_activity_task<T: Serialize>(
2572 &self,
2573 task_id: &str,
2574 activity_attempt_id: &str,
2575 lease_owner: &str,
2576 result: T,
2577 codec: &str,
2578 ) -> Result<Value> {
2579 let result = encode_typed_envelope(&AvroValue::from_serialize(&result)?, codec)?;
2580 let body = json!({
2581 "activity_attempt_id": activity_attempt_id,
2582 "lease_owner": lease_owner,
2583 "result": result
2584 });
2585 let path = format!("/worker/activity-tasks/{task_id}/complete");
2586 activity_task_response(
2587 self.request_json(
2588 reqwest::Method::POST,
2589 &path,
2590 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2591 Some(&body),
2592 )
2593 .await,
2594 "complete",
2595 task_id,
2596 activity_attempt_id,
2597 )
2598 }
2599
2600 pub async fn fail_activity_task(
2601 &self,
2602 task_id: &str,
2603 activity_attempt_id: &str,
2604 lease_owner: &str,
2605 message: impl Into<String>,
2606 non_retryable: bool,
2607 ) -> Result<Value> {
2608 let body = json!({
2609 "activity_attempt_id": activity_attempt_id,
2610 "lease_owner": lease_owner,
2611 "failure": {
2612 "message": message.into(),
2613 "type": "RustActivityFailure",
2614 "non_retryable": non_retryable
2615 }
2616 });
2617 let path = format!("/worker/activity-tasks/{task_id}/fail");
2618 activity_task_response(
2619 self.request_json(
2620 reqwest::Method::POST,
2621 &path,
2622 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2623 Some(&body),
2624 )
2625 .await,
2626 "fail",
2627 task_id,
2628 activity_attempt_id,
2629 )
2630 }
2631
2632 pub async fn heartbeat_activity_task<T: Serialize>(
2633 &self,
2634 task_id: &str,
2635 activity_attempt_id: &str,
2636 lease_owner: &str,
2637 details: T,
2638 ) -> Result<ActivityHeartbeatResponse> {
2639 let details = encode_typed_envelope(&AvroValue::from_serialize(&details)?, DEFAULT_CODEC)?;
2640 let body = json!({
2641 "activity_attempt_id": activity_attempt_id,
2642 "lease_owner": lease_owner,
2643 "details": details
2644 });
2645 let path = format!("/worker/activity-tasks/{task_id}/heartbeat");
2646 activity_task_response(
2647 self.request_json(
2648 reqwest::Method::POST,
2649 &path,
2650 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2651 Some(&body),
2652 )
2653 .await,
2654 "heartbeat",
2655 task_id,
2656 activity_attempt_id,
2657 )
2658 }
2659
2660 async fn request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
2661 &self,
2662 method: reqwest::Method,
2663 path: &str,
2664 protocol: RequestProtocol,
2665 body: Option<&B>,
2666 ) -> Result<T> {
2667 self.request_json_with_timeout(method, path, protocol, body, Duration::from_secs(60))
2668 .await
2669 }
2670
2671 async fn request_json_with_timeout<T: DeserializeOwned, B: Serialize + ?Sized>(
2672 &self,
2673 method: reqwest::Method,
2674 path: &str,
2675 protocol: RequestProtocol,
2676 body: Option<&B>,
2677 timeout: Duration,
2678 ) -> Result<T> {
2679 let auth_token = self.auth_token(protocol)?;
2680 let mut request = self
2681 .http
2682 .request(method, format!("{}/api{}", self.base_url, path))
2683 .timeout(timeout)
2684 .header(reqwest::header::ACCEPT, "application/json")
2685 .header(reqwest::header::CONTENT_TYPE, "application/json")
2686 .header("X-Namespace", &self.namespace);
2687
2688 match protocol {
2689 RequestProtocol::Worker(version) => {
2690 request = request.header("X-Durable-Workflow-Protocol-Version", version);
2691 }
2692 RequestProtocol::ControlPlane => {
2693 request = request.header(
2694 "X-Durable-Workflow-Control-Plane-Version",
2695 CONTROL_PLANE_VERSION,
2696 );
2697 }
2698 }
2699
2700 if let Some(token) = auth_token {
2701 request = request.bearer_auth(token);
2702 }
2703
2704 if let Some(body) = body {
2705 request = request.json(body);
2706 }
2707
2708 let response = request.send().await?;
2709 let status = response.status();
2710 let bytes = response.bytes().await?;
2711
2712 if !status.is_success() {
2713 let body = String::from_utf8_lossy(&bytes).to_string();
2714 if let Some(protocol) = protocol_failure(status, &body) {
2715 return Err(Error::Protocol(protocol));
2716 }
2717 return Err(Error::Http { status, body });
2718 }
2719
2720 if bytes.is_empty() {
2721 return Ok(serde_json::from_value(Value::Null)?);
2722 }
2723
2724 Ok(serde_json::from_slice(&bytes)?)
2725 }
2726
2727 async fn poll_request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
2728 &self,
2729 path: &str,
2730 protocol: RequestProtocol,
2731 body: &B,
2732 timeout: Duration,
2733 max_retries: usize,
2734 ) -> Result<T> {
2735 let mut retries = 0;
2736
2737 loop {
2738 let response = self
2739 .request_json_with_timeout(
2740 reqwest::Method::POST,
2741 path,
2742 protocol,
2743 Some(body),
2744 timeout,
2745 )
2746 .await;
2747
2748 match response {
2749 Err(Error::Transport(_)) if retries < max_retries => retries += 1,
2750 response => return worker_poll_response(response),
2751 }
2752 }
2753 }
2754
2755 fn auth_token(&self, protocol: RequestProtocol) -> Result<Option<&str>> {
2756 match protocol {
2757 RequestProtocol::Worker(_) => {
2758 if let Some(token) = self.worker_token.as_deref().or(self.token.as_deref()) {
2759 return Ok(Some(token));
2760 }
2761 if self.control_token.is_some() {
2762 return Err(Error::MissingRoleCredentials {
2763 role: "worker",
2764 opposite_role: "control",
2765 });
2766 }
2767 Ok(None)
2768 }
2769 RequestProtocol::ControlPlane => {
2770 if let Some(token) = self.control_token.as_deref().or(self.token.as_deref()) {
2771 return Ok(Some(token));
2772 }
2773 if self.worker_token.is_some() {
2774 return Err(Error::MissingRoleCredentials {
2775 role: "control",
2776 opposite_role: "worker",
2777 });
2778 }
2779 Ok(None)
2780 }
2781 }
2782 }
2783}
2784
2785fn query_failure(status: reqwest::StatusCode, raw_body: String) -> QueryFailure {
2786 let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
2787 let reason = body
2788 .get("reason")
2789 .and_then(Value::as_str)
2790 .unwrap_or("query_rejected")
2791 .to_string();
2792 let message = body
2793 .get("message")
2794 .or_else(|| body.get("error"))
2795 .and_then(Value::as_str)
2796 .unwrap_or("workflow query was rejected")
2797 .to_string();
2798
2799 QueryFailure {
2800 status: status.as_u16(),
2801 reason,
2802 message,
2803 body,
2804 }
2805}
2806
2807fn workflow_command_result(
2808 command: WorkflowCommandKind,
2809 data: Value,
2810 workflow_id: &str,
2811 run_id: Option<&str>,
2812) -> WorkflowCommandResult {
2813 WorkflowCommandResult {
2814 command,
2815 workflow_id: data
2816 .get("workflow_id")
2817 .and_then(Value::as_str)
2818 .unwrap_or(workflow_id)
2819 .to_string(),
2820 run_id: data
2821 .get("run_id")
2822 .and_then(Value::as_str)
2823 .or(run_id)
2824 .map(str::to_string),
2825 outcome: data
2826 .get("outcome")
2827 .and_then(Value::as_str)
2828 .map(str::to_string),
2829 reason: data
2830 .get("reason")
2831 .and_then(Value::as_str)
2832 .map(str::to_string),
2833 command_status: data
2834 .get("command_status")
2835 .and_then(Value::as_str)
2836 .map(str::to_string),
2837 raw: data,
2838 }
2839}
2840
2841fn workflow_command_rejection(
2842 command: WorkflowCommandKind,
2843 status: reqwest::StatusCode,
2844 raw_body: String,
2845 workflow_id: &str,
2846 run_id: Option<&str>,
2847) -> WorkflowCommandRejection {
2848 let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
2849 WorkflowCommandRejection {
2850 command,
2851 status: status.as_u16(),
2852 reason: body
2853 .get("reason")
2854 .and_then(Value::as_str)
2855 .unwrap_or("workflow_command_rejected")
2856 .to_string(),
2857 message: body
2858 .get("message")
2859 .or_else(|| body.get("error"))
2860 .and_then(Value::as_str)
2861 .unwrap_or("workflow lifecycle command was rejected")
2862 .to_string(),
2863 workflow_id: body
2864 .get("workflow_id")
2865 .and_then(Value::as_str)
2866 .unwrap_or(workflow_id)
2867 .to_string(),
2868 run_id: body
2869 .get("run_id")
2870 .and_then(Value::as_str)
2871 .or(run_id)
2872 .map(str::to_string),
2873 target_scope: body
2874 .get("target_scope")
2875 .and_then(Value::as_str)
2876 .map(str::to_string),
2877 body,
2878 }
2879}
2880
2881fn query_task_response(response: Result<Value>) -> Result<Value> {
2882 match response {
2883 Err(Error::Http { status, body }) => Err(Error::QueryFailed(query_failure(status, body))),
2884 response => response,
2885 }
2886}
2887
2888fn worker_poll_response<T: DeserializeOwned>(response: Result<T>) -> Result<T> {
2889 match response {
2890 Err(Error::Http { status, body })
2891 if status == reqwest::StatusCode::CONFLICT && worker_poll_body_is_stop(&body) =>
2892 {
2893 Ok(serde_json::from_str(&body)?)
2894 }
2895 response => response,
2896 }
2897}
2898
2899fn worker_poll_body_is_stop(body: &str) -> bool {
2900 serde_json::from_str::<Value>(body)
2901 .ok()
2902 .is_some_and(|body| {
2903 worker_poll_is_stop(
2904 body.get("poll_status").and_then(Value::as_str),
2905 body.get("reason").and_then(Value::as_str),
2906 )
2907 })
2908}
2909
2910fn worker_poll_is_stop(poll_status: Option<&str>, reason: Option<&str>) -> bool {
2911 matches!(poll_status, Some("draining" | "stopped"))
2912 || matches!(reason, Some("worker_draining" | "worker_stopped"))
2913}
2914
2915fn query_task_rejection_is_final(error: &Error) -> bool {
2916 matches!(
2917 error,
2918 Error::QueryFailed(failure)
2919 if QUERY_TASK_FINAL_REJECTION_REASONS.contains(&failure.reason.as_str())
2920 )
2921}
2922
2923fn activity_task_response<T>(
2924 response: Result<T>,
2925 operation: &str,
2926 task_id: &str,
2927 activity_attempt_id: &str,
2928) -> Result<T> {
2929 match response {
2930 Err(Error::Http { status, body }) => {
2931 let body = serde_json::from_str(&body).unwrap_or_else(|_| json!({"message": body}));
2932 Err(Error::ActivityTaskRejected(ActivityTaskRejection {
2933 operation: operation.to_string(),
2934 status: status.as_u16(),
2935 reason: body
2936 .get("reason")
2937 .and_then(Value::as_str)
2938 .unwrap_or("activity_task_rejected")
2939 .to_string(),
2940 task_id: body
2941 .get("task_id")
2942 .and_then(Value::as_str)
2943 .unwrap_or(task_id)
2944 .to_string(),
2945 activity_attempt_id: body
2946 .get("activity_attempt_id")
2947 .and_then(Value::as_str)
2948 .unwrap_or(activity_attempt_id)
2949 .to_string(),
2950 cancel_requested: body
2951 .get("cancel_requested")
2952 .and_then(Value::as_bool)
2953 .unwrap_or(false),
2954 can_continue: body.get("can_continue").and_then(Value::as_bool),
2955 run_closed_reason: body
2956 .get("run_closed_reason")
2957 .and_then(Value::as_str)
2958 .map(str::to_string),
2959 body,
2960 }))
2961 }
2962 response => response,
2963 }
2964}
2965
2966fn activity_task_rejection_is_final(error: &Error) -> bool {
2967 matches!(
2968 error,
2969 Error::ActivityTaskRejected(rejection)
2970 if matches!(
2971 rejection.reason.as_str(),
2972 "run_cancelled"
2973 | "run_terminated"
2974 | "attempt_closed"
2975 | "stale_attempt"
2976 | "activity_cancelled"
2977 | "task_cancelled"
2978 | "run_closed"
2979 | "activity_not_running"
2980 | "attempt_not_found"
2981 )
2982 )
2983}
2984
2985fn workflow_task_completion_is_terminal_timeout(
2986 error: &Error,
2987 task_id: &str,
2988 workflow_task_attempt: u64,
2989 run_id: Option<&str>,
2990) -> bool {
2991 let Error::Http { status, body } = error else {
2992 return false;
2993 };
2994 if *status != reqwest::StatusCode::CONFLICT {
2995 return false;
2996 }
2997
2998 let Some(run_id) = run_id else {
2999 return false;
3000 };
3001 let Ok(body) = serde_json::from_str::<Value>(body) else {
3002 return false;
3003 };
3004
3005 body.get("recorded").and_then(Value::as_bool) == Some(false)
3006 && body.get("reason").and_then(Value::as_str) == Some("run_timed_out")
3007 && body.get("run_status").and_then(Value::as_str) == Some("failed")
3008 && body.get("run_id").and_then(Value::as_str) == Some(run_id)
3009 && body.get("task_id").and_then(Value::as_str) == Some(task_id)
3010 && body.get("workflow_task_attempt").and_then(Value::as_u64) == Some(workflow_task_attempt)
3011}
3012
3013fn protocol_failure(status: reqwest::StatusCode, raw_body: &str) -> Option<ProtocolFailure> {
3014 let body: Value = serde_json::from_str(raw_body).ok()?;
3015 let reason = body.get("reason")?.as_str()?;
3016 if !matches!(
3017 reason,
3018 "missing_protocol_version"
3019 | "unsupported_protocol_version"
3020 | "missing_control_plane_version"
3021 | "unsupported_control_plane_version"
3022 ) {
3023 return None;
3024 }
3025
3026 Some(ProtocolFailure {
3027 status: status.as_u16(),
3028 reason: reason.to_string(),
3029 message: body
3030 .get("message")
3031 .or_else(|| body.get("error"))
3032 .and_then(Value::as_str)
3033 .unwrap_or("protocol version rejected")
3034 .to_string(),
3035 supported_version: body
3036 .get("supported_version")
3037 .and_then(Value::as_str)
3038 .map(str::to_string),
3039 requested_version: body
3040 .get("requested_version")
3041 .and_then(Value::as_str)
3042 .map(str::to_string),
3043 body,
3044 })
3045}
3046
3047fn long_poll_timeout_seconds(timeout: Duration) -> u64 {
3048 timeout
3049 .as_secs()
3050 .saturating_add(u64::from(timeout.subsec_nanos() > 0))
3051 .min(MAX_LONG_POLL_TIMEOUT_SECONDS)
3052}
3053
3054fn worker_operation_is_retryable(error: &Error) -> bool {
3055 match error {
3056 Error::Transport(error) => {
3057 error.is_timeout() || error.is_connect() || error.is_request() || error.is_body()
3058 }
3059 Error::Http { status, .. } => {
3060 matches!(
3061 *status,
3062 reqwest::StatusCode::REQUEST_TIMEOUT | reqwest::StatusCode::TOO_MANY_REQUESTS
3063 ) || status.is_server_error()
3064 }
3065 _ => false,
3066 }
3067}
3068
3069fn worker_retry_delay(policy: WorkerRetryPolicy, retry: usize) -> Duration {
3070 let exponent = retry.saturating_sub(1).min(31) as u32;
3071 policy
3072 .initial_backoff
3073 .saturating_mul(1_u32 << exponent)
3074 .min(policy.max_backoff)
3075}
3076
3077#[derive(Debug)]
3078pub struct ClientBuilder {
3079 base_url: String,
3080 token: Option<String>,
3081 control_token: Option<String>,
3082 worker_token: Option<String>,
3083 namespace: String,
3084 timeout: Duration,
3085}
3086
3087impl ClientBuilder {
3088 pub fn token(mut self, token: Option<String>) -> Self {
3089 self.token = token;
3090 self
3091 }
3092
3093 pub fn control_token(mut self, token: Option<String>) -> Self {
3094 self.control_token = token;
3095 self
3096 }
3097
3098 pub fn worker_token(mut self, token: Option<String>) -> Self {
3099 self.worker_token = token;
3100 self
3101 }
3102
3103 pub fn namespace(mut self, namespace: impl Into<String>) -> Self {
3104 self.namespace = namespace.into();
3105 self
3106 }
3107
3108 pub fn timeout(mut self, timeout: Duration) -> Self {
3109 self.timeout = timeout;
3110 self
3111 }
3112
3113 pub fn build(self) -> Result<Client> {
3114 let base_url = self.base_url.trim_end_matches('/').to_string();
3115 let has_sdk_api_suffix = reqwest::Url::parse(&base_url)
3116 .map(|url| url.path().trim_end_matches('/').ends_with("/api"))
3117 .unwrap_or_else(|_| base_url.ends_with("/api"));
3118
3119 if has_sdk_api_suffix {
3120 return Err(Error::InvalidBaseUrl);
3121 }
3122
3123 Ok(Client {
3124 http: reqwest::Client::builder().timeout(self.timeout).build()?,
3125 base_url,
3126 token: self.token,
3127 control_token: self.control_token,
3128 worker_token: self.worker_token,
3129 namespace: self.namespace,
3130 })
3131 }
3132}
3133
3134#[derive(Clone, Debug)]
3135pub struct WorkflowHandle {
3136 client: Client,
3137 pub workflow_id: String,
3138 pub run_id: Option<String>,
3139 pub workflow_type: String,
3140}
3141
3142impl WorkflowHandle {
3143 pub async fn describe(&self) -> Result<WorkflowDescription> {
3145 self.client.describe_workflow(&self.workflow_id).await
3146 }
3147
3148 pub async fn describe_selected_run(&self) -> Result<WorkflowDescription> {
3150 let run_id = self.run_id.as_deref().ok_or_else(|| {
3151 Error::Codec("run_id is required for selected-run description".to_string())
3152 })?;
3153 self.client
3154 .describe_workflow_run(&self.workflow_id, run_id)
3155 .await
3156 }
3157
3158 pub async fn signal<T: Serialize>(&self, signal_name: &str, input: T) -> Result<Value> {
3159 self.client
3160 .signal_workflow(&self.workflow_id, signal_name, input)
3161 .await
3162 }
3163
3164 pub async fn signal_selected_run<T: Serialize>(
3166 &self,
3167 signal_name: &str,
3168 input: T,
3169 ) -> Result<Value> {
3170 let run_id = self.run_id.as_deref().ok_or_else(|| {
3171 Error::Codec("run_id is required for selected-run signaling".to_string())
3172 })?;
3173 self.client
3174 .signal_workflow_run(&self.workflow_id, run_id, signal_name, input)
3175 .await
3176 }
3177
3178 pub async fn cancel(&self, options: WorkflowCommandOptions) -> Result<WorkflowCommandResult> {
3180 self.client
3181 .cancel_workflow(&self.workflow_id, options)
3182 .await
3183 }
3184
3185 pub async fn cancel_selected_run(
3187 &self,
3188 options: WorkflowCommandOptions,
3189 ) -> Result<WorkflowCommandResult> {
3190 let run_id = self.run_id.as_deref().ok_or_else(|| {
3191 Error::Codec("run_id is required for selected-run cancellation".to_string())
3192 })?;
3193 self.client
3194 .cancel_workflow_run(&self.workflow_id, run_id, options)
3195 .await
3196 }
3197
3198 pub async fn terminate(
3200 &self,
3201 options: WorkflowCommandOptions,
3202 ) -> Result<WorkflowCommandResult> {
3203 self.client
3204 .terminate_workflow(&self.workflow_id, options)
3205 .await
3206 }
3207
3208 pub async fn terminate_selected_run(
3210 &self,
3211 options: WorkflowCommandOptions,
3212 ) -> Result<WorkflowCommandResult> {
3213 let run_id = self.run_id.as_deref().ok_or_else(|| {
3214 Error::Codec("run_id is required for selected-run termination".to_string())
3215 })?;
3216 self.client
3217 .terminate_workflow_run(&self.workflow_id, run_id, options)
3218 .await
3219 }
3220
3221 pub async fn query<T: Serialize>(&self, query_name: &str, input: T) -> Result<Value> {
3223 self.client
3224 .query_workflow(&self.workflow_id, query_name, input)
3225 .await
3226 }
3227
3228 pub async fn query_avro_value<T: Serialize>(
3229 &self,
3230 query_name: &str,
3231 input: T,
3232 ) -> Result<AvroValue> {
3233 self.client
3234 .query_workflow_avro_value(&self.workflow_id, query_name, input)
3235 .await
3236 }
3237
3238 pub async fn update<T: Serialize>(
3239 &self,
3240 update_name: &str,
3241 input: T,
3242 request_id: Option<&str>,
3243 ) -> Result<Value> {
3244 self.client
3245 .update_workflow(&self.workflow_id, update_name, input, request_id)
3246 .await
3247 }
3248
3249 pub async fn update_avro_value<T: Serialize>(
3250 &self,
3251 update_name: &str,
3252 input: T,
3253 request_id: Option<&str>,
3254 ) -> Result<AvroValue> {
3255 self.client
3256 .update_workflow_avro_value(&self.workflow_id, update_name, input, request_id)
3257 .await
3258 }
3259
3260 pub async fn query_selected_run<T: Serialize>(
3262 &self,
3263 query_name: &str,
3264 input: T,
3265 ) -> Result<Value> {
3266 let run_id = self
3267 .run_id
3268 .as_deref()
3269 .ok_or_else(|| Error::Codec("run_id is required for selected-run query".to_string()))?;
3270 self.client
3271 .query_workflow_run(&self.workflow_id, run_id, query_name, input)
3272 .await
3273 }
3274
3275 pub async fn result(&self, options: WorkflowResultOptions) -> Result<Value> {
3277 self.result_target(options, None).await
3278 }
3279
3280 pub async fn result_avro_value(&self, options: WorkflowResultOptions) -> Result<AvroValue> {
3282 self.result_avro_value_target(options, None).await
3283 }
3284
3285 pub async fn result_selected_run(&self, options: WorkflowResultOptions) -> Result<Value> {
3287 let run_id = self.run_id.as_deref().ok_or_else(|| {
3288 Error::Codec("run_id is required for selected-run result".to_string())
3289 })?;
3290 self.result_target(options, Some(run_id)).await
3291 }
3292
3293 pub async fn result_selected_run_avro_value(
3295 &self,
3296 options: WorkflowResultOptions,
3297 ) -> Result<AvroValue> {
3298 let run_id = self.run_id.as_deref().ok_or_else(|| {
3299 Error::Codec("run_id is required for selected-run result".to_string())
3300 })?;
3301 self.result_avro_value_target(options, Some(run_id)).await
3302 }
3303
3304 async fn result_avro_value_target(
3305 &self,
3306 options: WorkflowResultOptions,
3307 selected_run_id: Option<&str>,
3308 ) -> Result<AvroValue> {
3309 let started = Instant::now();
3310
3311 loop {
3312 let description = match selected_run_id {
3313 Some(run_id) => {
3314 self.client
3315 .describe_workflow_run(&self.workflow_id, run_id)
3316 .await?
3317 }
3318 None => self.describe().await?,
3319 };
3320 if description.is_completed() {
3321 return description.output_avro_value.ok_or_else(|| {
3322 Error::Codec(
3323 "missing_payload_envelope: typed workflow result requires output_envelope"
3324 .to_string(),
3325 )
3326 });
3327 }
3328 if description.is_terminal() {
3329 let outcome =
3330 workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
3331 return Err(match outcome.kind {
3332 WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
3333 WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
3334 WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
3335 WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
3336 });
3337 }
3338 if started.elapsed() >= options.timeout {
3339 return Err(Error::Timeout);
3340 }
3341 tokio::time::sleep(options.poll_interval).await;
3342 }
3343 }
3344
3345 async fn result_target(
3346 &self,
3347 options: WorkflowResultOptions,
3348 selected_run_id: Option<&str>,
3349 ) -> Result<Value> {
3350 let started = Instant::now();
3351
3352 loop {
3353 let description = match selected_run_id {
3354 Some(run_id) => {
3355 self.client
3356 .describe_workflow_run(&self.workflow_id, run_id)
3357 .await?
3358 }
3359 None => self.describe().await?,
3360 };
3361 if description.is_completed() {
3362 return Ok(description.output.unwrap_or(Value::Null));
3363 }
3364
3365 if description.is_terminal() {
3366 let outcome =
3367 workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
3368 return Err(match outcome.kind {
3369 WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
3370 WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
3371 WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
3372 WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
3373 });
3374 }
3375
3376 if started.elapsed() >= options.timeout {
3377 return Err(Error::WorkflowTimedOut(WorkflowTerminalOutcome {
3378 kind: WorkflowTerminalKind::TimedOut,
3379 workflow_id: description
3380 .workflow_id
3381 .clone()
3382 .unwrap_or_else(|| self.workflow_id.clone()),
3383 run_id: description
3384 .run_id
3385 .clone()
3386 .or_else(|| selected_run_id.map(str::to_string)),
3387 reason: "result_wait_timeout".to_string(),
3388 failure_category: Some("client_timeout".to_string()),
3389 failure_id: None,
3390 exception_type: None,
3391 exception_class: None,
3392 non_retryable: None,
3393 message: Some(format!(
3394 "workflow result was not terminal within {:?}",
3395 options.timeout
3396 )),
3397 exception: None,
3398 raw: description.raw_value(),
3399 }));
3400 }
3401
3402 tokio::time::sleep(options.poll_interval).await;
3403 }
3404 }
3405}
3406
3407#[derive(Clone, Copy, Debug)]
3408pub struct WorkflowResultOptions {
3409 pub poll_interval: Duration,
3410 pub timeout: Duration,
3411}
3412
3413impl Default for WorkflowResultOptions {
3414 fn default() -> Self {
3415 Self {
3416 poll_interval: Duration::from_millis(500),
3417 timeout: Duration::from_secs(30),
3418 }
3419 }
3420}
3421
3422#[derive(Clone, Debug, Deserialize)]
3423pub struct WorkflowDescription {
3424 pub workflow_id: Option<String>,
3425 pub run_id: Option<String>,
3426 pub workflow_type: Option<String>,
3427 pub status: Option<String>,
3428 #[serde(default)]
3429 pub closed_reason: Option<String>,
3430 #[serde(default)]
3431 pub error: Option<String>,
3432 #[serde(default)]
3433 pub failure: Option<Value>,
3434 #[serde(default)]
3435 pub exception: Option<Value>,
3436 #[serde(default)]
3437 pub failures: Vec<Value>,
3438 #[serde(default)]
3439 pub output: Option<Value>,
3440 #[serde(default)]
3441 pub output_envelope: Option<Value>,
3442 #[serde(skip)]
3443 pub output_avro_value: Option<AvroValue>,
3444 #[serde(flatten)]
3445 pub raw: HashMap<String, Value>,
3446}
3447
3448impl WorkflowDescription {
3449 pub fn is_completed(&self) -> bool {
3450 matches!(self.status.as_deref(), Some("completed" | "Completed"))
3451 }
3452
3453 pub fn is_terminal(&self) -> bool {
3454 matches!(
3455 self.status.as_deref(),
3456 Some(
3457 "completed"
3458 | "Completed"
3459 | "failed"
3460 | "Failed"
3461 | "cancelled"
3462 | "Cancelled"
3463 | "terminated"
3464 | "Terminated"
3465 | "timed_out"
3466 | "TimedOut",
3467 )
3468 )
3469 }
3470
3471 fn decode_payloads(&mut self) -> Result<()> {
3472 if let Some(envelope) = &self.output_envelope {
3473 let value = decode_wire_avro_value(envelope, DEFAULT_CODEC)?;
3474 self.output = Some(value.clone().into_json()?);
3475 self.output_avro_value = Some(value);
3476 }
3477
3478 Ok(())
3479 }
3480
3481 fn raw_value(&self) -> Value {
3482 let mut data = self.raw.clone();
3483 data.insert(
3484 "workflow_id".to_string(),
3485 self.workflow_id
3486 .clone()
3487 .map(Value::String)
3488 .unwrap_or(Value::Null),
3489 );
3490 data.insert(
3491 "run_id".to_string(),
3492 self.run_id
3493 .clone()
3494 .map(Value::String)
3495 .unwrap_or(Value::Null),
3496 );
3497 data.insert(
3498 "workflow_type".to_string(),
3499 self.workflow_type
3500 .clone()
3501 .map(Value::String)
3502 .unwrap_or(Value::Null),
3503 );
3504 data.insert(
3505 "status".to_string(),
3506 self.status
3507 .clone()
3508 .map(Value::String)
3509 .unwrap_or(Value::Null),
3510 );
3511 data.insert(
3512 "closed_reason".to_string(),
3513 self.closed_reason
3514 .clone()
3515 .map(Value::String)
3516 .unwrap_or(Value::Null),
3517 );
3518 if let Some(failure) = &self.failure {
3519 data.insert("failure".to_string(), failure.clone());
3520 }
3521 if let Some(exception) = &self.exception {
3522 data.insert("exception".to_string(), exception.clone());
3523 }
3524 Value::Object(data.into_iter().collect())
3525 }
3526}
3527
3528fn workflow_terminal_outcome(
3529 description: &WorkflowDescription,
3530 workflow_id: &str,
3531 run_id: Option<&str>,
3532) -> WorkflowTerminalOutcome {
3533 let terminal_kind = description
3534 .closed_reason
3535 .as_deref()
3536 .or(description.status.as_deref())
3537 .unwrap_or("failed")
3538 .to_ascii_lowercase();
3539 let kind = match terminal_kind.as_str() {
3540 "cancelled" | "canceled" => WorkflowTerminalKind::Cancelled,
3541 "terminated" => WorkflowTerminalKind::Terminated,
3542 "timed_out" | "timedout" => WorkflowTerminalKind::TimedOut,
3543 _ => WorkflowTerminalKind::Failed,
3544 };
3545 let default_reason = match kind {
3546 WorkflowTerminalKind::Failed => "workflow_failed",
3547 WorkflowTerminalKind::Cancelled => "cancelled",
3548 WorkflowTerminalKind::Terminated => "terminated",
3549 WorkflowTerminalKind::TimedOut => "timed_out",
3550 };
3551 let failure = description
3552 .failure
3553 .as_ref()
3554 .filter(|value| value.is_object());
3555 let nested_failure = failure
3556 .and_then(|value| value.get("failures"))
3557 .and_then(Value::as_array)
3558 .and_then(|failures| failures.last())
3559 .or_else(|| description.failures.last());
3560 let exception = description
3561 .exception
3562 .clone()
3563 .or_else(|| failure.and_then(|value| value.get("exception")).cloned())
3564 .or_else(|| {
3565 nested_failure
3566 .and_then(|value| value.get("exception_payload"))
3567 .cloned()
3568 });
3569 let string_field = |name: &str| {
3570 failure
3571 .and_then(|value| value.get(name))
3572 .and_then(Value::as_str)
3573 .or_else(|| {
3574 nested_failure
3575 .and_then(|value| value.get(name))
3576 .and_then(Value::as_str)
3577 })
3578 .map(str::to_string)
3579 };
3580 let exception_field = |name: &str| {
3581 exception
3582 .as_ref()
3583 .and_then(|value| value.get(name))
3584 .and_then(Value::as_str)
3585 .map(str::to_string)
3586 };
3587 let message = description
3588 .error
3589 .clone()
3590 .or_else(|| string_field("message"))
3591 .or_else(|| exception_field("message"));
3592 let reason = description
3593 .raw
3594 .get("reason")
3595 .and_then(Value::as_str)
3596 .map(str::to_string)
3597 .or_else(|| {
3598 failure
3599 .and_then(|value| value.get("reason"))
3600 .and_then(Value::as_str)
3601 .map(str::to_string)
3602 })
3603 .or_else(|| description.closed_reason.clone())
3604 .unwrap_or_else(|| default_reason.to_string());
3605 let failure_id = string_field("failure_id").or_else(|| {
3606 nested_failure
3607 .and_then(|value| value.get("id"))
3608 .and_then(Value::as_str)
3609 .map(str::to_string)
3610 });
3611
3612 WorkflowTerminalOutcome {
3613 kind,
3614 workflow_id: description
3615 .workflow_id
3616 .clone()
3617 .unwrap_or_else(|| workflow_id.to_string()),
3618 run_id: description
3619 .run_id
3620 .clone()
3621 .or_else(|| run_id.map(str::to_string)),
3622 reason,
3623 failure_category: string_field("failure_category")
3624 .or_else(|| Some(default_reason.to_string())),
3625 failure_id,
3626 exception_type: string_field("exception_type").or_else(|| exception_field("type")),
3627 exception_class: string_field("exception_class").or_else(|| exception_field("class")),
3628 non_retryable: failure
3629 .and_then(|value| value.get("non_retryable"))
3630 .and_then(Value::as_bool)
3631 .or_else(|| {
3632 nested_failure
3633 .and_then(|value| value.get("non_retryable"))
3634 .and_then(Value::as_bool)
3635 }),
3636 message,
3637 exception,
3638 raw: description.raw_value(),
3639 }
3640}
3641
3642#[derive(Clone, Debug, Deserialize)]
3643pub struct RegisterWorkerResponse {
3644 pub worker_id: String,
3645 pub registered: bool,
3646 #[serde(default)]
3647 pub heartbeat_interval_seconds: Option<u64>,
3648 #[serde(default)]
3649 pub protocol_version: Option<String>,
3650 #[serde(default)]
3651 pub server_capabilities: Option<Value>,
3652}
3653
3654#[derive(Clone, Debug, Deserialize, PartialEq, Eq)]
3656pub struct WorkerDeregistrationEnvelope {
3657 pub worker_id: String,
3658 pub outcome: String,
3659 pub recovered_workflow_task_count: u64,
3660}
3661
3662#[derive(Clone, Debug, Deserialize)]
3663pub struct PollWorkflowTaskResponse {
3664 #[serde(default)]
3665 pub task: Option<WorkflowTask>,
3666 #[serde(default)]
3667 pub poll_status: Option<String>,
3668 #[serde(default)]
3669 pub reason: Option<String>,
3670 #[serde(default)]
3671 pub protocol_version: Option<String>,
3672 #[serde(default)]
3673 pub server_capabilities: Option<Value>,
3674}
3675
3676impl PollWorkflowTaskResponse {
3677 pub fn outcome(&self) -> WorkerPollOutcome {
3679 worker_poll_outcome(
3680 self.task.is_some(),
3681 self.poll_status.as_deref(),
3682 self.reason.as_deref(),
3683 )
3684 }
3685}
3686
3687#[derive(Clone, Debug, Deserialize)]
3688pub struct PollActivityTaskResponse {
3689 #[serde(default)]
3690 pub task: Option<ActivityTask>,
3691 #[serde(default)]
3692 pub poll_status: Option<String>,
3693 #[serde(default)]
3694 pub reason: Option<String>,
3695}
3696
3697impl PollActivityTaskResponse {
3698 pub fn outcome(&self) -> WorkerPollOutcome {
3700 worker_poll_outcome(
3701 self.task.is_some(),
3702 self.poll_status.as_deref(),
3703 self.reason.as_deref(),
3704 )
3705 }
3706}
3707
3708#[derive(Clone, Debug, Deserialize)]
3709pub struct PollQueryTaskResponse {
3710 #[serde(default)]
3711 pub task: Option<QueryTask>,
3712 #[serde(default)]
3713 pub poll_status: Option<String>,
3714 #[serde(default)]
3715 pub reason: Option<String>,
3716}
3717
3718impl PollQueryTaskResponse {
3719 pub fn outcome(&self) -> WorkerPollOutcome {
3721 worker_poll_outcome(
3722 self.task.is_some(),
3723 self.poll_status.as_deref(),
3724 self.reason.as_deref(),
3725 )
3726 }
3727}
3728
3729#[derive(Clone, Debug, PartialEq, Eq)]
3731pub enum WorkerPollOutcome {
3732 Task,
3734 Idle {
3736 poll_status: Option<String>,
3737 reason: Option<String>,
3738 },
3739 Stop {
3741 poll_status: Option<String>,
3742 reason: Option<String>,
3743 },
3744}
3745
3746impl WorkerPollOutcome {
3747 pub fn should_stop(&self) -> bool {
3748 matches!(self, Self::Stop { .. })
3749 }
3750}
3751
3752fn worker_poll_outcome(
3753 has_task: bool,
3754 poll_status: Option<&str>,
3755 reason: Option<&str>,
3756) -> WorkerPollOutcome {
3757 if worker_poll_is_stop(poll_status, reason) {
3758 return WorkerPollOutcome::Stop {
3759 poll_status: poll_status.map(str::to_string),
3760 reason: reason.map(str::to_string),
3761 };
3762 }
3763
3764 if has_task {
3765 WorkerPollOutcome::Task
3766 } else {
3767 WorkerPollOutcome::Idle {
3768 poll_status: poll_status.map(str::to_string),
3769 reason: reason.map(str::to_string),
3770 }
3771 }
3772}
3773
3774#[derive(Clone, Debug, Deserialize)]
3776pub struct QueryTask {
3777 pub query_task_id: String,
3778 #[serde(default = "default_workflow_task_attempt")]
3779 pub query_task_attempt: u64,
3780 #[serde(default)]
3781 pub lease_owner: Option<String>,
3782 #[serde(default)]
3783 pub workflow_id: Option<String>,
3784 #[serde(default)]
3785 pub run_id: Option<String>,
3786 pub workflow_type: String,
3787 pub query_name: String,
3788 #[serde(default = "default_payload_codec")]
3789 pub payload_codec: String,
3790 #[serde(default)]
3791 pub workflow_arguments: Option<Value>,
3792 #[serde(default)]
3793 pub query_arguments: Option<Value>,
3794 #[serde(default)]
3795 pub history_events: Vec<HistoryEvent>,
3796 #[serde(default)]
3797 pub history_export: Option<Value>,
3798 #[serde(default)]
3799 pub run_status: Option<String>,
3800}
3801
3802#[derive(Clone, Debug, Deserialize)]
3803pub struct WorkflowTask {
3804 pub task_id: String,
3805 #[serde(default)]
3806 pub workflow_id: Option<String>,
3807 #[serde(default)]
3808 pub run_id: Option<String>,
3809 pub workflow_type: String,
3810 #[serde(default = "default_payload_codec")]
3811 pub payload_codec: String,
3812 #[serde(default)]
3813 pub arguments: Option<Value>,
3814 #[serde(default)]
3815 pub history_events: Vec<HistoryEvent>,
3816 #[serde(default)]
3817 pub total_history_events: Option<u64>,
3818 #[serde(default)]
3819 pub history_size_bytes: Option<u64>,
3820 #[serde(default)]
3821 pub continue_as_new_recommended: Option<bool>,
3822 #[serde(default)]
3823 pub history_budget_pressure: Option<String>,
3824 #[serde(default)]
3825 pub next_history_page_token: Option<String>,
3826 #[serde(default = "default_workflow_task_attempt")]
3827 pub workflow_task_attempt: u64,
3828 #[serde(default)]
3829 pub workflow_signal_id: Option<String>,
3830 #[serde(default)]
3831 pub signal_name: Option<String>,
3832 #[serde(default)]
3833 pub signal_arguments: Option<Value>,
3834 #[serde(default)]
3835 pub workflow_update_id: Option<String>,
3836 #[serde(default)]
3837 pub update_name: Option<String>,
3838 #[serde(default)]
3839 pub lease_owner: Option<String>,
3840}
3841
3842impl WorkflowTask {
3843 fn append_history_page(&mut self, page: WorkflowTaskHistoryPage) {
3844 self.history_events.extend(page.history_events);
3845
3846 if page.total_history_events.is_some() {
3847 self.total_history_events = page.total_history_events;
3848 }
3849
3850 self.next_history_page_token = page
3851 .next_history_page_token
3852 .filter(|token| !token.is_empty());
3853 }
3854}
3855
3856#[derive(Clone, Debug, Deserialize)]
3857struct WorkflowTaskHistoryPage {
3858 #[serde(default)]
3859 history_events: Vec<HistoryEvent>,
3860 #[serde(default)]
3861 total_history_events: Option<u64>,
3862 #[serde(default)]
3863 next_history_page_token: Option<String>,
3864}
3865
3866#[derive(Clone, Debug, Deserialize)]
3867pub struct ActivityTask {
3868 pub task_id: String,
3869 #[serde(default)]
3870 pub activity_attempt_id: Option<String>,
3871 #[serde(default)]
3872 pub attempt_id: Option<String>,
3873 pub activity_type: String,
3874 #[serde(default = "default_payload_codec")]
3875 pub payload_codec: String,
3876 #[serde(default)]
3877 pub arguments: Option<Value>,
3878 #[serde(default = "default_attempt_number")]
3879 pub attempt_number: u64,
3880 #[serde(default)]
3881 pub lease_owner: Option<String>,
3882}
3883
3884#[derive(Clone, Debug, Deserialize)]
3885pub struct HistoryEvent {
3886 #[serde(alias = "type")]
3887 pub event_type: String,
3888 #[serde(default)]
3889 pub payload: Value,
3890 #[serde(flatten)]
3891 pub raw: HashMap<String, Value>,
3892}
3893
3894#[derive(Clone, Debug, PartialEq)]
3896pub struct QuerySignal {
3897 pub id: Option<String>,
3898 pub name: String,
3899 pub arguments: Vec<Value>,
3900 avro_arguments: Vec<AvroValue>,
3901 pub workflow_sequence: Option<u64>,
3902}
3903
3904impl QuerySignal {
3905 pub fn arguments_avro_value(&self) -> &[AvroValue] {
3907 &self.avro_arguments
3908 }
3909}
3910
3911#[derive(Clone, Debug)]
3917pub struct QueryContext {
3918 pub workflow_id: Option<String>,
3919 pub run_id: Option<String>,
3920 pub workflow_type: String,
3921 pub run_status: Option<String>,
3922 workflow_input: Value,
3923 workflow_input_avro_value: AvroValue,
3924 history_events: Arc<Vec<HistoryEvent>>,
3925 signal_events: Arc<Vec<QuerySignal>>,
3926}
3927
3928impl QueryContext {
3929 pub fn workflow_input(&self) -> &Value {
3931 &self.workflow_input
3932 }
3933
3934 pub fn workflow_input_avro_value(&self) -> &AvroValue {
3936 &self.workflow_input_avro_value
3937 }
3938
3939 pub fn history_events(&self) -> &[HistoryEvent] {
3941 self.history_events.as_slice()
3942 }
3943
3944 pub fn signal_events(&self) -> &[QuerySignal] {
3946 self.signal_events.as_slice()
3947 }
3948
3949 pub fn signals(&self, signal_name: &str) -> Vec<Vec<Value>> {
3951 self.signal_events
3952 .iter()
3953 .filter(|signal| signal.name == signal_name)
3954 .map(|signal| signal.arguments.clone())
3955 .collect()
3956 }
3957
3958 pub fn signals_avro_value(&self, signal_name: &str) -> Vec<Vec<AvroValue>> {
3960 self.signal_events
3961 .iter()
3962 .filter(|signal| signal.name == signal_name)
3963 .map(|signal| signal.avro_arguments.clone())
3964 .collect()
3965 }
3966}
3967
3968#[derive(Clone, Debug, Deserialize)]
3969pub struct ActivityHeartbeatResponse {
3970 #[serde(default)]
3971 pub cancel_requested: bool,
3972 #[serde(default)]
3973 pub heartbeat_recorded: bool,
3974 #[serde(default)]
3975 pub can_continue: Option<bool>,
3976 #[serde(default)]
3977 pub reason: Option<String>,
3978 #[serde(default)]
3979 pub run_closed_reason: Option<String>,
3980 #[serde(default)]
3981 pub run_closed_at: Option<String>,
3982 #[serde(default)]
3983 pub lease_expires_at: Option<String>,
3984 #[serde(default)]
3985 pub last_heartbeat_at: Option<String>,
3986}
3987
3988impl ActivityHeartbeatResponse {
3989 pub fn should_stop(&self) -> bool {
3991 self.cancel_requested || self.can_continue == Some(false)
3992 }
3993}
3994
3995fn default_payload_codec() -> String {
3996 DEFAULT_CODEC.to_string()
3997}
3998
3999fn default_workflow_task_attempt() -> u64 {
4000 1
4001}
4002
4003fn default_attempt_number() -> u64 {
4004 1
4005}
4006
4007type WorkflowFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
4008type WorkflowHandler = Arc<dyn Fn(WorkflowContext, AvroValue) -> WorkflowFuture + Send + Sync>;
4009type ErasedWorkflowState = Arc<dyn Any + Send + Sync>;
4010type WorkflowStateSnapshot = Arc<dyn Fn() -> Result<ErasedWorkflowState> + Send + Sync>;
4011type ReplayedWorkflowHandler =
4012 Arc<dyn Fn(WorkflowContext, AvroValue) -> ReplayedWorkflowInvocation + Send + Sync>;
4013type ActivityFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
4014type ActivityHandler = Arc<dyn Fn(ActivityContext, AvroValue) -> ActivityFuture + Send + Sync>;
4015type QueryFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
4016type QueryHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
4017type UpdateHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
4018type ReplayedQueryHandler = Arc<
4019 dyn Fn(QueryContext, ErasedWorkflowState, AvroValue) -> std::result::Result<QueryFuture, String>
4020 + Send
4021 + Sync,
4022>;
4023type WorkerHeartbeatObserver = Arc<dyn Fn(&WorkerHeartbeatObservation) + Send + Sync>;
4024
4025struct ReplayedWorkflowInvocation {
4026 future: WorkflowFuture,
4027 snapshot: WorkflowStateSnapshot,
4028}
4029
4030#[derive(Clone)]
4031struct RegisteredWorkflow {
4032 execute: WorkflowHandler,
4033 replay: Option<ReplayedWorkflowHandler>,
4034 state_type: Option<TypeId>,
4035}
4036
4037#[derive(Clone)]
4038enum RegisteredQuery {
4039 Snapshot(QueryHandler),
4040 Replayed {
4041 state_type: TypeId,
4042 handler: ReplayedQueryHandler,
4043 },
4044}
4045
4046#[derive(Clone, Debug)]
4047pub struct WorkerHeartbeatObservation {
4048 pub worker_id: String,
4049 pub task_queue: String,
4050 pub acknowledged_at_unix_millis: u64,
4051 pub acknowledgement: Value,
4052}
4053
4054#[derive(Clone, Copy, Debug)]
4061pub struct WorkerRetryPolicy {
4062 pub max_retries: usize,
4064 pub initial_backoff: Duration,
4066 pub max_backoff: Duration,
4068}
4069
4070impl Default for WorkerRetryPolicy {
4071 fn default() -> Self {
4072 Self {
4073 max_retries: 5,
4074 initial_backoff: Duration::from_millis(100),
4075 max_backoff: Duration::from_secs(5),
4076 }
4077 }
4078}
4079
4080#[derive(Clone, Copy, Debug, PartialEq, Eq)]
4081enum ManagedPollOutcome {
4082 Idle,
4083 Handled,
4084 Stop,
4085}
4086
4087#[derive(Clone)]
4088pub struct Worker {
4089 client: Client,
4090 worker_id: String,
4091 task_queue: String,
4092 workflows: HashMap<String, RegisteredWorkflow>,
4093 activities: HashMap<String, ActivityHandler>,
4094 queries: HashMap<String, HashMap<String, RegisteredQuery>>,
4095 updates: HashMap<String, HashMap<String, UpdateHandler>>,
4096 max_concurrent_workflow_tasks: usize,
4097 max_concurrent_activity_tasks: usize,
4098 poll_timeout: Duration,
4099 heartbeat_interval: Duration,
4100 retry_policy: WorkerRetryPolicy,
4101 heartbeat_observer: Option<WorkerHeartbeatObserver>,
4102}
4103
4104impl Worker {
4105 pub fn new(client: Client, task_queue: impl Into<String>) -> Self {
4106 Self {
4107 client,
4108 worker_id: default_worker_id(),
4109 task_queue: task_queue.into(),
4110 workflows: HashMap::new(),
4111 activities: HashMap::new(),
4112 queries: HashMap::new(),
4113 updates: HashMap::new(),
4114 max_concurrent_workflow_tasks: 10,
4115 max_concurrent_activity_tasks: 10,
4116 poll_timeout: Duration::from_secs(30),
4117 heartbeat_interval: Duration::from_secs(60),
4118 retry_policy: WorkerRetryPolicy::default(),
4119 heartbeat_observer: None,
4120 }
4121 }
4122
4123 pub fn worker_id(mut self, worker_id: impl Into<String>) -> Self {
4124 self.worker_id = worker_id.into();
4125 self
4126 }
4127
4128 pub fn poll_timeout(mut self, timeout: Duration) -> Self {
4129 self.poll_timeout = timeout;
4130 self
4131 }
4132
4133 pub fn heartbeat_interval(mut self, interval: Duration) -> Self {
4134 self.heartbeat_interval = interval;
4135 self
4136 }
4137
4138 pub fn retry_policy(mut self, policy: WorkerRetryPolicy) -> Self {
4140 self.retry_policy = policy;
4141 self
4142 }
4143
4144 pub fn on_worker_heartbeat<F>(mut self, observer: F) -> Self
4145 where
4146 F: Fn(&WorkerHeartbeatObservation) + Send + Sync + 'static,
4147 {
4148 self.heartbeat_observer = Some(Arc::new(observer));
4149 self
4150 }
4151
4152 pub fn max_concurrent_workflow_tasks(mut self, count: usize) -> Self {
4153 self.max_concurrent_workflow_tasks = count.max(1);
4154 self
4155 }
4156
4157 pub fn max_concurrent_activity_tasks(mut self, count: usize) -> Self {
4158 self.max_concurrent_activity_tasks = count.max(1);
4159 self
4160 }
4161
4162 pub fn register_workflow<F, Fut>(&mut self, workflow_type: impl Into<String>, handler: F)
4169 where
4170 F: Fn(WorkflowContext, Value) -> Fut + Send + Sync + 'static,
4171 Fut: Future<Output = Result<Value>> + Send + 'static,
4172 {
4173 let handler = Arc::new(handler);
4174 self.workflows.insert(
4175 workflow_type.into(),
4176 RegisteredWorkflow {
4177 execute: Arc::new(move |ctx, input| {
4178 let handler = Arc::clone(&handler);
4179 Box::pin(async move {
4180 let result = handler(ctx, input.into_json()?).await?;
4181 AvroValue::from_serialize(&result)
4182 })
4183 }),
4184 replay: None,
4185 state_type: None,
4186 },
4187 );
4188 }
4189
4190 pub fn register_workflow_avro_value<F, Fut>(
4192 &mut self,
4193 workflow_type: impl Into<String>,
4194 handler: F,
4195 ) where
4196 F: Fn(WorkflowContext, AvroValue) -> Fut + Send + Sync + 'static,
4197 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4198 {
4199 self.workflows.insert(
4200 workflow_type.into(),
4201 RegisteredWorkflow {
4202 execute: Arc::new(move |ctx, input| Box::pin(handler(ctx, input))),
4203 replay: None,
4204 state_type: None,
4205 },
4206 );
4207 }
4208
4209 pub fn register_replayed_workflow<S, Factory, F, Fut>(
4217 &mut self,
4218 workflow_type: impl Into<String>,
4219 state_factory: Factory,
4220 handler: F,
4221 ) where
4222 S: Clone + Send + Sync + 'static,
4223 Factory: Fn() -> S + Send + Sync + 'static,
4224 F: Fn(WorkflowContext, Value, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
4225 Fut: Future<Output = Result<Value>> + Send + 'static,
4226 {
4227 let state_factory = Arc::new(state_factory);
4228 let handler = Arc::new(handler);
4229
4230 let execute_factory = Arc::clone(&state_factory);
4231 let execute_handler = Arc::clone(&handler);
4232 let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4233 let state = WorkflowInstance::new(execute_factory());
4234 let handler = Arc::clone(&execute_handler);
4235 Box::pin(async move {
4236 let result = handler(ctx, input.into_json()?, state).await?;
4237 AvroValue::from_serialize(&result)
4238 }) as WorkflowFuture
4239 });
4240
4241 let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4242 let state = WorkflowInstance::new(state_factory());
4243 let snapshot_state = state.clone();
4244 let snapshot: WorkflowStateSnapshot =
4245 Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
4246 let replay_handler = Arc::clone(&handler);
4247 let future = async move {
4248 let result = replay_handler(ctx, input.into_json()?, state).await?;
4249 AvroValue::from_serialize(&result)
4250 };
4251 ReplayedWorkflowInvocation {
4252 future: Box::pin(future),
4253 snapshot,
4254 }
4255 });
4256
4257 self.workflows.insert(
4258 workflow_type.into(),
4259 RegisteredWorkflow {
4260 execute,
4261 replay: Some(replay),
4262 state_type: Some(TypeId::of::<S>()),
4263 },
4264 );
4265 }
4266
4267 pub fn register_replayed_workflow_avro_value<S, Factory, F, Fut>(
4269 &mut self,
4270 workflow_type: impl Into<String>,
4271 state_factory: Factory,
4272 handler: F,
4273 ) where
4274 S: Clone + Send + Sync + 'static,
4275 Factory: Fn() -> S + Send + Sync + 'static,
4276 F: Fn(WorkflowContext, AvroValue, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
4277 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4278 {
4279 let state_factory = Arc::new(state_factory);
4280 let handler = Arc::new(handler);
4281
4282 let execute_factory = Arc::clone(&state_factory);
4283 let execute_handler = Arc::clone(&handler);
4284 let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4285 let state = WorkflowInstance::new(execute_factory());
4286 Box::pin(execute_handler(ctx, input, state)) as WorkflowFuture
4287 });
4288
4289 let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4290 let state = WorkflowInstance::new(state_factory());
4291 let snapshot_state = state.clone();
4292 let snapshot: WorkflowStateSnapshot =
4293 Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
4294 ReplayedWorkflowInvocation {
4295 future: Box::pin(handler(ctx, input, state)),
4296 snapshot,
4297 }
4298 });
4299
4300 self.workflows.insert(
4301 workflow_type.into(),
4302 RegisteredWorkflow {
4303 execute,
4304 replay: Some(replay),
4305 state_type: Some(TypeId::of::<S>()),
4306 },
4307 );
4308 }
4309
4310 pub fn register_activity<F, Fut>(&mut self, activity_type: impl Into<String>, handler: F)
4311 where
4312 F: Fn(ActivityContext, Value) -> Fut + Send + Sync + 'static,
4313 Fut: Future<Output = Result<Value>> + Send + 'static,
4314 {
4315 let handler = Arc::new(handler);
4316 self.activities.insert(
4317 activity_type.into(),
4318 Arc::new(move |ctx, args| {
4319 let handler = Arc::clone(&handler);
4320 Box::pin(async move {
4321 let result = handler(ctx, args.into_json()?).await?;
4322 AvroValue::from_serialize(&result)
4323 })
4324 }),
4325 );
4326 }
4327
4328 pub fn register_activity_avro_value<F, Fut>(
4330 &mut self,
4331 activity_type: impl Into<String>,
4332 handler: F,
4333 ) where
4334 F: Fn(ActivityContext, AvroValue) -> Fut + Send + Sync + 'static,
4335 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4336 {
4337 self.activities.insert(
4338 activity_type.into(),
4339 Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
4340 );
4341 }
4342
4343 pub fn register_query<F, Fut>(
4349 &mut self,
4350 workflow_type: impl Into<String>,
4351 query_name: impl Into<String>,
4352 handler: F,
4353 ) where
4354 F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
4355 Fut: Future<Output = Result<Value>> + Send + 'static,
4356 {
4357 let handler = Arc::new(handler);
4358 self.queries
4359 .entry(workflow_type.into())
4360 .or_default()
4361 .insert(
4362 query_name.into(),
4363 RegisteredQuery::Snapshot(Arc::new(move |ctx, args| {
4364 let handler = Arc::clone(&handler);
4365 Box::pin(async move {
4366 let result = handler(ctx, args.into_json()?).await?;
4367 AvroValue::from_serialize(&result)
4368 })
4369 })),
4370 );
4371 }
4372
4373 pub fn register_query_avro_value<F, Fut>(
4375 &mut self,
4376 workflow_type: impl Into<String>,
4377 query_name: impl Into<String>,
4378 handler: F,
4379 ) where
4380 F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
4381 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4382 {
4383 self.queries
4384 .entry(workflow_type.into())
4385 .or_default()
4386 .insert(
4387 query_name.into(),
4388 RegisteredQuery::Snapshot(Arc::new(move |ctx, args| Box::pin(handler(ctx, args)))),
4389 );
4390 }
4391
4392 pub fn register_replayed_query<S, F, Fut>(
4399 &mut self,
4400 workflow_type: impl Into<String>,
4401 query_name: impl Into<String>,
4402 handler: F,
4403 ) where
4404 S: Clone + Send + Sync + 'static,
4405 F: Fn(QueryContext, Arc<S>, Value) -> Fut + Send + Sync + 'static,
4406 Fut: Future<Output = Result<Value>> + Send + 'static,
4407 {
4408 let handler = Arc::new(handler);
4409 let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
4410 let state = state.downcast::<S>().map_err(|_| {
4411 "registered query state type does not match the replayed workflow state".to_string()
4412 })?;
4413 let handler = Arc::clone(&handler);
4414 Ok(Box::pin(async move {
4415 let result = handler(ctx, state, args.into_json()?).await?;
4416 AvroValue::from_serialize(&result)
4417 }))
4418 });
4419
4420 self.queries
4421 .entry(workflow_type.into())
4422 .or_default()
4423 .insert(
4424 query_name.into(),
4425 RegisteredQuery::Replayed {
4426 state_type: TypeId::of::<S>(),
4427 handler: erased_handler,
4428 },
4429 );
4430 }
4431
4432 pub fn register_replayed_query_avro_value<S, F, Fut>(
4434 &mut self,
4435 workflow_type: impl Into<String>,
4436 query_name: impl Into<String>,
4437 handler: F,
4438 ) where
4439 S: Clone + Send + Sync + 'static,
4440 F: Fn(QueryContext, Arc<S>, AvroValue) -> Fut + Send + Sync + 'static,
4441 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4442 {
4443 let handler = Arc::new(handler);
4444 let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
4445 let state = state.downcast::<S>().map_err(|_| {
4446 "registered query state type does not match the replayed workflow state".to_string()
4447 })?;
4448 Ok(Box::pin(handler(ctx, state, args)))
4449 });
4450
4451 self.queries
4452 .entry(workflow_type.into())
4453 .or_default()
4454 .insert(
4455 query_name.into(),
4456 RegisteredQuery::Replayed {
4457 state_type: TypeId::of::<S>(),
4458 handler: erased_handler,
4459 },
4460 );
4461 }
4462
4463 pub fn register_update<F, Fut>(
4465 &mut self,
4466 workflow_type: impl Into<String>,
4467 update_name: impl Into<String>,
4468 handler: F,
4469 ) where
4470 F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
4471 Fut: Future<Output = Result<Value>> + Send + 'static,
4472 {
4473 let handler = Arc::new(handler);
4474 self.updates
4475 .entry(workflow_type.into())
4476 .or_default()
4477 .insert(
4478 update_name.into(),
4479 Arc::new(move |ctx, args| {
4480 let handler = Arc::clone(&handler);
4481 Box::pin(async move {
4482 let result = handler(ctx, args.into_json()?).await?;
4483 AvroValue::from_serialize(&result)
4484 })
4485 }),
4486 );
4487 }
4488
4489 pub fn register_update_avro_value<F, Fut>(
4491 &mut self,
4492 workflow_type: impl Into<String>,
4493 update_name: impl Into<String>,
4494 handler: F,
4495 ) where
4496 F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
4497 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4498 {
4499 self.updates
4500 .entry(workflow_type.into())
4501 .or_default()
4502 .insert(
4503 update_name.into(),
4504 Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
4505 );
4506 }
4507
4508 pub async fn register(&self) -> Result<RegisterWorkerResponse> {
4509 let mut command_contracts = serde_json::Map::new();
4510 for workflow_type in self.workflows.keys() {
4511 let mut queries = self
4512 .queries
4513 .get(workflow_type)
4514 .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
4515 .unwrap_or_default();
4516 queries.sort();
4517 let mut updates = self
4518 .updates
4519 .get(workflow_type)
4520 .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
4521 .unwrap_or_default();
4522 updates.sort();
4523 if !queries.is_empty() || !updates.is_empty() {
4524 command_contracts.insert(
4525 workflow_type.clone(),
4526 json!({
4527 "queries": queries,
4528 "updates": updates,
4529 "update_validators": [],
4530 }),
4531 );
4532 }
4533 }
4534
4535 self.client
4536 .register_worker_with_command_contracts(
4537 &self.worker_id,
4538 &self.task_queue,
4539 self.workflows.keys().cloned().collect(),
4540 self.activities.keys().cloned().collect(),
4541 self.max_concurrent_workflow_tasks,
4542 self.max_concurrent_activity_tasks,
4543 [
4544 (!self.queries.is_empty()).then(|| QUERY_TASKS_CAPABILITY.to_string()),
4545 (!self.updates.is_empty()).then(|| WORKFLOW_UPDATES_CAPABILITY.to_string()),
4546 ]
4547 .into_iter()
4548 .flatten()
4549 .collect(),
4550 Value::Object(command_contracts),
4551 )
4552 .await
4553 }
4554
4555 pub async fn run(&self) -> Result<()> {
4561 self.run_until(std::future::pending::<()>()).await
4562 }
4563
4564 pub async fn run_until<F>(&self, shutdown: F) -> Result<()>
4568 where
4569 F: Future<Output = ()>,
4570 {
4571 let registration = self.register().await?;
4572 if !registration.registered {
4573 return Err(Error::WorkerLoop(format!(
4574 "worker registration for {:?} was not accepted",
4575 self.worker_id
4576 )));
4577 }
4578 let registered_worker_id = registration.worker_id.clone();
4579 let primary = self.run_registered_until(shutdown, registration).await;
4580 let deregistration = self
4581 .client
4582 .deregister_worker_registration(®istered_worker_id)
4583 .await;
4584
4585 match (primary, deregistration) {
4586 (Ok(()), Ok(_)) => Ok(()),
4587 (Ok(()), Err(deregistration)) => Err(deregistration),
4588 (Err(primary), Ok(_)) => Err(primary),
4589 (Err(primary), Err(deregistration)) => Err(Error::WorkerShutdown {
4590 primary: Box::new(primary),
4591 deregistration: Box::new(deregistration),
4592 }),
4593 }
4594 }
4595
4596 async fn run_registered_until<F>(
4597 &self,
4598 shutdown: F,
4599 registration: RegisterWorkerResponse,
4600 ) -> Result<()>
4601 where
4602 F: Future<Output = ()>,
4603 {
4604 let heartbeat_interval = Duration::from_secs(
4605 registration
4606 .heartbeat_interval_seconds
4607 .unwrap_or(self.heartbeat_interval.as_secs().max(1)),
4608 );
4609 let heartbeat = tokio::time::sleep(Duration::ZERO);
4615 tokio::pin!(heartbeat);
4616 tokio::pin!(shutdown);
4617 let stop = Arc::new(AtomicBool::new(false));
4618 let mut workflow_poller = (!self.workflows.is_empty()).then(|| {
4622 let worker = self.clone();
4623 let stop = Arc::clone(&stop);
4624 tokio::spawn(async move { worker.poll_workflows_until_stopped(stop).await })
4625 });
4626 let mut activity_poller = (!self.activities.is_empty()).then(|| {
4627 let worker = self.clone();
4628 let stop = Arc::clone(&stop);
4629 tokio::spawn(async move { worker.poll_activities_until_stopped(stop).await })
4630 });
4631 let mut query_poller = (!self.queries.is_empty()).then(|| {
4632 let worker = self.clone();
4633 let stop = Arc::clone(&stop);
4634 tokio::spawn(async move { worker.poll_queries_until_stopped(stop).await })
4635 });
4636
4637 loop {
4638 tokio::select! {
4639 _ = &mut shutdown => {
4640 stop.store(true, Ordering::SeqCst);
4641 break;
4642 }
4643 _ = &mut heartbeat => {
4644 let result = self.retry_worker_operation(|| {
4645 self.client.heartbeat_worker(
4646 &self.worker_id,
4647 self.max_concurrent_workflow_tasks,
4648 self.max_concurrent_activity_tasks,
4649 )
4650 }).await;
4651 heartbeat
4652 .as_mut()
4653 .reset(tokio::time::Instant::now() + heartbeat_interval);
4654 match result {
4655 Ok(acknowledgement) => {
4656 if let Some(observer) = &self.heartbeat_observer {
4657 observer(&WorkerHeartbeatObservation {
4658 worker_id: self.worker_id.clone(),
4659 task_queue: self.task_queue.clone(),
4660 acknowledged_at_unix_millis: SystemTime::now()
4661 .duration_since(UNIX_EPOCH)
4662 .unwrap_or_default()
4663 .as_millis()
4664 .min(u64::MAX as u128)
4665 as u64,
4666 acknowledgement,
4667 });
4668 }
4669 }
4670 Err(error) => {
4671 stop.store(true, Ordering::SeqCst);
4672 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await?;
4673 return Err(error);
4674 }
4675 }
4676 }
4677 result = OptionFuture::from(workflow_poller.as_mut()), if workflow_poller.is_some() => {
4678 workflow_poller = None;
4679 let stopped_by_server = stop.load(Ordering::SeqCst);
4680 stop.store(true, Ordering::SeqCst);
4681 let poller_result = optional_poller_result("workflow", result);
4682 let join_result =
4683 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
4684 poller_result?;
4685 join_result?;
4686 if stopped_by_server {
4687 return Ok(());
4688 }
4689 return Err(Error::WorkerLoop(
4690 "workflow poller stopped unexpectedly".to_string(),
4691 ));
4692 }
4693 result = OptionFuture::from(activity_poller.as_mut()), if activity_poller.is_some() => {
4694 activity_poller = None;
4695 let stopped_by_server = stop.load(Ordering::SeqCst);
4696 stop.store(true, Ordering::SeqCst);
4697 let poller_result = optional_poller_result("activity", result);
4698 let join_result =
4699 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
4700 poller_result?;
4701 join_result?;
4702 if stopped_by_server {
4703 return Ok(());
4704 }
4705 return Err(Error::WorkerLoop(
4706 "activity poller stopped unexpectedly".to_string(),
4707 ));
4708 }
4709 result = OptionFuture::from(query_poller.as_mut()), if query_poller.is_some() => {
4710 query_poller = None;
4711 let stopped_by_server = stop.load(Ordering::SeqCst);
4712 stop.store(true, Ordering::SeqCst);
4713 let poller_result = optional_poller_result("query", result);
4714 let join_result =
4715 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
4716 poller_result?;
4717 join_result?;
4718 if stopped_by_server {
4719 return Ok(());
4720 }
4721 return Err(Error::WorkerLoop(
4722 "query poller stopped unexpectedly".to_string(),
4723 ));
4724 }
4725 }
4726 }
4727
4728 join_pollers(
4729 workflow_poller.take(),
4730 activity_poller.take(),
4731 query_poller.take(),
4732 )
4733 .await
4734 }
4735
4736 pub async fn run_once(&self) -> Result<usize> {
4750 let mut handled = 0;
4751 match self.poll_workflow_once().await? {
4752 ManagedPollOutcome::Handled => handled += 1,
4753 ManagedPollOutcome::Stop => return Ok(handled),
4754 ManagedPollOutcome::Idle => {}
4755 }
4756 match self.poll_activity_once().await? {
4757 ManagedPollOutcome::Handled => handled += 1,
4758 ManagedPollOutcome::Stop => return Ok(handled),
4759 ManagedPollOutcome::Idle => {}
4760 }
4761 if !self.queries.is_empty() {
4762 match self.poll_query_once().await? {
4763 ManagedPollOutcome::Handled => handled += 1,
4764 ManagedPollOutcome::Stop => return Ok(handled),
4765 ManagedPollOutcome::Idle => {}
4766 }
4767 }
4768 Ok(handled)
4769 }
4770
4771 async fn poll_workflow_once(&self) -> Result<ManagedPollOutcome> {
4772 let poll_request_id = unique_request_id("rust-workflow-poll");
4773 let response = self
4774 .retry_worker_operation(|| {
4775 self.client.poll_workflow_task_response_with_request_id(
4776 &self.worker_id,
4777 &self.task_queue,
4778 self.poll_timeout,
4779 &poll_request_id,
4780 0,
4781 )
4782 })
4783 .await?;
4784 if response.outcome().should_stop() {
4785 return Ok(ManagedPollOutcome::Stop);
4786 }
4787 let Some(task) = response.task else {
4788 return Ok(ManagedPollOutcome::Idle);
4789 };
4790
4791 let task_id = task.task_id.clone();
4792 let attempt = task.workflow_task_attempt;
4793 let run_id = task.run_id.clone();
4794 let lease_owner = task
4795 .lease_owner
4796 .clone()
4797 .unwrap_or_else(|| self.worker_id.clone());
4798
4799 match self.execute_workflow_task(task) {
4800 Ok(commands) if commands.is_empty() => {
4801 self.client
4807 .fail_workflow_task_with_type(
4808 &task_id,
4809 &lease_owner,
4810 attempt,
4811 WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE,
4812 WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE,
4813 )
4814 .await?;
4815 }
4816 Ok(commands) => {
4817 let completion = self
4818 .client
4819 .complete_workflow_task(&task_id, &lease_owner, attempt, commands)
4820 .await;
4821 if let Err(error) = completion {
4822 if !workflow_task_completion_is_terminal_timeout(
4823 &error,
4824 &task_id,
4825 attempt,
4826 run_id.as_deref(),
4827 ) {
4828 return Err(error);
4829 }
4830 }
4831 }
4832 Err(error) => {
4833 self.client
4834 .fail_workflow_task(&task_id, &lease_owner, attempt, error.to_string())
4835 .await?;
4836 }
4837 }
4838
4839 Ok(ManagedPollOutcome::Handled)
4840 }
4841
4842 async fn poll_workflows_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
4843 while !stop.load(Ordering::SeqCst) {
4844 if self.poll_workflow_once().await? == ManagedPollOutcome::Stop {
4845 stop.store(true, Ordering::SeqCst);
4846 break;
4847 }
4848 }
4849
4850 Ok(())
4851 }
4852
4853 async fn poll_activity_once(&self) -> Result<ManagedPollOutcome> {
4854 let poll_request_id = unique_request_id("rust-activity-poll");
4855 let response = self
4856 .retry_worker_operation(|| {
4857 self.client.poll_activity_task_response_with_request_id(
4858 &self.worker_id,
4859 &self.task_queue,
4860 self.poll_timeout,
4861 &poll_request_id,
4862 0,
4863 )
4864 })
4865 .await?;
4866 if response.outcome().should_stop() {
4867 return Ok(ManagedPollOutcome::Stop);
4868 }
4869 let Some(task) = response.task else {
4870 return Ok(ManagedPollOutcome::Idle);
4871 };
4872
4873 let task_id = task.task_id.clone();
4874 let attempt_id = task
4875 .activity_attempt_id
4876 .clone()
4877 .or(task.attempt_id.clone())
4878 .unwrap_or_default();
4879 let lease_owner = task
4880 .lease_owner
4881 .clone()
4882 .unwrap_or_else(|| self.worker_id.clone());
4883 let codec = task.payload_codec.clone();
4884 let result = self.execute_activity_task(task).await;
4885 match result {
4886 Ok(value) => {
4887 let completion = self
4888 .client
4889 .complete_activity_task(&task_id, &attempt_id, &lease_owner, value, &codec)
4890 .await;
4891 if let Err(error) = completion {
4892 if !activity_task_rejection_is_final(&error) {
4893 return Err(error);
4894 }
4895 }
4896 }
4897 Err(error) => {
4898 let failure = self
4899 .client
4900 .fail_activity_task(
4901 &task_id,
4902 &attempt_id,
4903 &lease_owner,
4904 error.to_string(),
4905 false,
4906 )
4907 .await;
4908 if let Err(error) = failure {
4909 if !activity_task_rejection_is_final(&error) {
4910 return Err(error);
4911 }
4912 }
4913 }
4914 }
4915
4916 Ok(ManagedPollOutcome::Handled)
4917 }
4918
4919 async fn poll_activities_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
4920 while !stop.load(Ordering::SeqCst) {
4921 if self.poll_activity_once().await? == ManagedPollOutcome::Stop {
4922 stop.store(true, Ordering::SeqCst);
4923 break;
4924 }
4925 }
4926
4927 Ok(())
4928 }
4929
4930 async fn poll_query_once(&self) -> Result<ManagedPollOutcome> {
4931 let poll_request_id = unique_request_id("rust-query-poll");
4932 let response = self
4933 .retry_worker_operation(|| {
4934 self.client.poll_query_task_response_with_request_id(
4935 &self.worker_id,
4936 &self.task_queue,
4937 self.poll_timeout,
4938 &poll_request_id,
4939 0,
4940 )
4941 })
4942 .await?;
4943 if response.outcome().should_stop() {
4944 return Ok(ManagedPollOutcome::Stop);
4945 }
4946 let Some(task) = response.task else {
4947 return Ok(ManagedPollOutcome::Idle);
4948 };
4949
4950 let query_task_id = task.query_task_id.clone();
4951 let attempt = task.query_task_attempt;
4952 let lease_owner = task
4953 .lease_owner
4954 .clone()
4955 .unwrap_or_else(|| self.worker_id.clone());
4956 let codec = task.payload_codec.clone();
4957
4958 match self.execute_query_task(task).await {
4959 Ok(value) => {
4960 let result_envelope = match encode_typed_envelope(&value, &codec) {
4961 Ok(result_envelope) => result_envelope,
4962 Err(error) => {
4963 let failure = self
4964 .client
4965 .fail_query_task(
4966 &query_task_id,
4967 &lease_owner,
4968 attempt,
4969 error.to_string(),
4970 "query_result_encode_failed",
4971 "QueryResultEncodeFailed",
4972 )
4973 .await;
4974 if let Err(error) = failure {
4975 if !query_task_rejection_is_final(&error) {
4976 return Err(error);
4977 }
4978 }
4979 return Ok(ManagedPollOutcome::Handled);
4980 }
4981 };
4982
4983 if let Err(error) = self
4984 .client
4985 .complete_query_task_with_envelope(
4986 &query_task_id,
4987 &lease_owner,
4988 attempt,
4989 value.clone().into_json()?,
4990 result_envelope,
4991 )
4992 .await
4993 {
4994 if !query_task_rejection_is_final(&error) {
4995 return Err(error);
4996 }
4997 }
4998 }
4999 Err(failure) => {
5000 let result = self
5001 .client
5002 .fail_query_task(
5003 &query_task_id,
5004 &lease_owner,
5005 attempt,
5006 failure.message,
5007 failure.reason,
5008 failure.failure_type,
5009 )
5010 .await;
5011 if let Err(error) = result {
5012 if !query_task_rejection_is_final(&error) {
5013 return Err(error);
5014 }
5015 }
5016 }
5017 }
5018
5019 Ok(ManagedPollOutcome::Handled)
5020 }
5021
5022 async fn poll_queries_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
5023 while !stop.load(Ordering::SeqCst) {
5024 if self.poll_query_once().await? == ManagedPollOutcome::Stop {
5025 stop.store(true, Ordering::SeqCst);
5026 break;
5027 }
5028 }
5029
5030 Ok(())
5031 }
5032
5033 async fn retry_worker_operation<T, F, Fut>(&self, mut operation: F) -> Result<T>
5034 where
5035 F: FnMut() -> Fut,
5036 Fut: Future<Output = Result<T>>,
5037 {
5038 let mut retries = 0;
5039
5040 loop {
5041 match operation().await {
5042 Err(error)
5043 if worker_operation_is_retryable(&error)
5044 && retries < self.retry_policy.max_retries =>
5045 {
5046 retries += 1;
5047 tokio::time::sleep(worker_retry_delay(self.retry_policy, retries)).await;
5048 }
5049 result => return result,
5050 }
5051 }
5052 }
5053
5054 async fn execute_query_task(
5055 &self,
5056 mut task: QueryTask,
5057 ) -> std::result::Result<AvroValue, QueryTaskExecutionFailure> {
5058 if !matches!(task.payload_codec.as_str(), DEFAULT_CODEC | JSON_CODEC) {
5059 return Err(QueryTaskExecutionFailure::new(
5060 "query_payload_decode_failed",
5061 format!(
5062 "cannot decode query payload with unsupported codec {:?}",
5063 task.payload_codec
5064 ),
5065 "QueryPayloadDecodeFailed",
5066 ));
5067 }
5068
5069 if !self.workflows.contains_key(&task.workflow_type) {
5070 return Err(QueryTaskExecutionFailure::new(
5071 "query_workflow_type_not_registered",
5072 format!("no workflow registered for type {:?}", task.workflow_type),
5073 "WorkflowTypeNotRegistered",
5074 ));
5075 }
5076
5077 let Some(handlers) = self.queries.get(&task.workflow_type) else {
5078 return Err(QueryTaskExecutionFailure::new(
5079 "query_handler_unavailable",
5080 format!(
5081 "query handlers are unavailable for workflow type {:?}",
5082 task.workflow_type
5083 ),
5084 "QueryHandlerUnavailable",
5085 ));
5086 };
5087 let Some(query) = handlers.get(&task.query_name) else {
5088 return Err(QueryTaskExecutionFailure::new(
5089 "rejected_unknown_query",
5090 format!("unknown query {:?}", task.query_name),
5091 "QueryFailed",
5092 ));
5093 };
5094
5095 let args = decode_task_avro_arguments(task.query_arguments.as_ref(), &task.payload_codec)
5096 .map_err(|error| {
5097 QueryTaskExecutionFailure::new(
5098 "query_payload_decode_failed",
5099 format!("cannot decode query arguments: {error}"),
5100 "QueryPayloadDecodeFailed",
5101 )
5102 })?;
5103 let workflow_input_typed =
5104 decode_task_avro_arguments(task.workflow_arguments.as_ref(), &task.payload_codec)
5105 .map_err(|error| {
5106 QueryTaskExecutionFailure::new(
5107 "query_workflow_state_unavailable",
5108 format!("cannot decode workflow start input: {error}"),
5109 "QueryWorkflowStateUnavailable",
5110 )
5111 })?;
5112 let workflow_input = workflow_input_typed.clone().into_json().map_err(|error| {
5113 QueryTaskExecutionFailure::new(
5114 "query_workflow_state_unavailable",
5115 format!("cannot project workflow start input: {error}"),
5116 "QueryWorkflowStateUnavailable",
5117 )
5118 })?;
5119 hydrate_query_history_from_export(&mut task).map_err(|error| {
5120 QueryTaskExecutionFailure::new(
5121 "query_workflow_state_unavailable",
5122 format!("cannot restore query history snapshot: {error}"),
5123 "QueryWorkflowStateUnavailable",
5124 )
5125 })?;
5126 enrich_query_history_from_export(&mut task).map_err(|error| {
5127 QueryTaskExecutionFailure::new(
5128 "query_workflow_state_unavailable",
5129 format!("cannot restore compact query history payloads: {error}"),
5130 "QueryWorkflowStateUnavailable",
5131 )
5132 })?;
5133 let signal_events = query_signal_events(&task).map_err(|error| {
5134 QueryTaskExecutionFailure::new(
5135 "query_workflow_state_unavailable",
5136 format!("cannot decode committed workflow signals: {error}"),
5137 "QueryWorkflowStateUnavailable",
5138 )
5139 })?;
5140 let history_events = Arc::new(std::mem::take(&mut task.history_events));
5141 let context = QueryContext {
5142 workflow_id: task.workflow_id,
5143 run_id: task.run_id,
5144 workflow_type: task.workflow_type.clone(),
5145 run_status: task.run_status,
5146 workflow_input,
5147 workflow_input_avro_value: workflow_input_typed.clone(),
5148 history_events: Arc::clone(&history_events),
5149 signal_events: Arc::new(signal_events),
5150 };
5151
5152 let future = match query {
5153 RegisteredQuery::Snapshot(handler) => handler(context, args),
5154 RegisteredQuery::Replayed {
5155 state_type,
5156 handler,
5157 } => {
5158 let workflow = self
5159 .workflows
5160 .get(&task.workflow_type)
5161 .expect("workflow registration was checked above");
5162 if workflow.state_type != Some(*state_type) {
5163 return Err(QueryTaskExecutionFailure::new(
5164 "query_workflow_state_unavailable",
5165 "replayed query state type does not match its workflow registration",
5166 "QueryWorkflowStateUnavailable",
5167 ));
5168 }
5169 let replay = workflow.replay.as_ref().ok_or_else(|| {
5170 QueryTaskExecutionFailure::new(
5171 "query_workflow_state_unavailable",
5172 format!(
5173 "workflow type {:?} is not registered for instance-state replay",
5174 task.workflow_type
5175 ),
5176 "QueryWorkflowStateUnavailable",
5177 )
5178 })?;
5179 let workflow_state = Arc::new(Mutex::new(
5180 WorkflowState::new_with_identity(
5181 history_events.as_ref().clone(),
5182 context.workflow_id.clone(),
5183 context.run_id.clone(),
5184 self.task_queue.clone(),
5185 task.payload_codec,
5186 None,
5187 )
5188 .map_err(|error| {
5189 QueryTaskExecutionFailure::new(
5190 "query_workflow_state_unavailable",
5191 format!("workflow replay failed before query: {error}"),
5192 "QueryWorkflowStateUnavailable",
5193 )
5194 })?,
5195 ));
5196 let workflow_context = WorkflowContext {
5197 state: workflow_state,
5198 };
5199 let mut invocation = replay(workflow_context.clone(), workflow_input_typed.clone());
5200 let mut cx = TaskContext::from_waker(noop_waker_ref());
5201 match invocation.future.as_mut().poll(&mut cx) {
5202 Poll::Ready(Ok(_)) => {
5203 workflow_context
5204 .ensure_history_consumed()
5205 .map_err(|error| {
5206 QueryTaskExecutionFailure::new(
5207 "query_workflow_state_unavailable",
5208 format!("workflow replay failed before query: {error}"),
5209 "QueryWorkflowStateUnavailable",
5210 )
5211 })?;
5212 }
5213 Poll::Ready(Err(error)) => {
5214 return Err(QueryTaskExecutionFailure::new(
5215 "query_workflow_state_unavailable",
5216 format!("workflow replay failed before query: {error}"),
5217 "QueryWorkflowStateUnavailable",
5218 ));
5219 }
5220 Poll::Pending => {
5221 let commands = workflow_context.take_commands().map_err(|error| {
5222 QueryTaskExecutionFailure::new(
5223 "query_workflow_state_unavailable",
5224 format!("workflow replay failed before query: {error}"),
5225 "QueryWorkflowStateUnavailable",
5226 )
5227 })?;
5228 if commands.is_empty()
5229 && !workflow_context
5230 .matched_recorded_pending()
5231 .map_err(|error| {
5232 QueryTaskExecutionFailure::new(
5233 "query_workflow_state_unavailable",
5234 format!("workflow replay failed before query: {error}"),
5235 "QueryWorkflowStateUnavailable",
5236 )
5237 })?
5238 {
5239 return Err(QueryTaskExecutionFailure::new(
5240 "query_workflow_state_unavailable",
5241 "workflow replay yielded without a durable command",
5242 "QueryWorkflowStateUnavailable",
5243 ));
5244 }
5245 }
5246 }
5247 let state = (invocation.snapshot)().map_err(|error| {
5248 QueryTaskExecutionFailure::new(
5249 "query_workflow_state_unavailable",
5250 format!("cannot snapshot replayed workflow state: {error}"),
5251 "QueryWorkflowStateUnavailable",
5252 )
5253 })?;
5254 handler(context, state, args).map_err(|message| {
5255 QueryTaskExecutionFailure::new(
5256 "query_workflow_state_unavailable",
5257 message,
5258 "QueryWorkflowStateUnavailable",
5259 )
5260 })?
5261 }
5262 };
5263
5264 future.await.map_err(|error| {
5265 QueryTaskExecutionFailure::new("query_rejected", error.to_string(), "QueryFailed")
5266 })
5267 }
5268
5269 fn execute_workflow_task(&self, task: WorkflowTask) -> Result<Vec<Value>> {
5270 if let Some(update_id) = task
5271 .workflow_update_id
5272 .as_deref()
5273 .filter(|update_id| !update_id.is_empty())
5274 {
5275 return self.execute_update_task(&task, update_id);
5276 }
5277
5278 let workflow = self
5279 .workflows
5280 .get(&task.workflow_type)
5281 .ok_or_else(|| Error::WorkflowNotRegistered(task.workflow_type.clone()))?;
5282 let input = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
5283 let resume_signal = decode_resume_signal(&task)?;
5284 let history_budget = WorkflowHistoryBudget {
5285 event_count: task
5286 .total_history_events
5287 .unwrap_or_else(|| u64::try_from(task.history_events.len()).unwrap_or(u64::MAX)),
5288 size_bytes: task.history_size_bytes,
5289 continue_as_new_recommended: task.continue_as_new_recommended.unwrap_or(false),
5290 pressure: task.history_budget_pressure.clone(),
5291 };
5292 let mut workflow_state = WorkflowState::new_with_identity(
5293 task.history_events,
5294 task.workflow_id,
5295 task.run_id,
5296 self.task_queue.clone(),
5297 task.payload_codec.clone(),
5298 resume_signal,
5299 )?;
5300 workflow_state.history_budget = history_budget;
5301 let state = Arc::new(Mutex::new(workflow_state));
5302 let ctx = WorkflowContext { state };
5303 let mut future = (workflow.execute)(ctx.clone(), input);
5304 let mut cx = TaskContext::from_waker(noop_waker_ref());
5305
5306 match future.as_mut().poll(&mut cx) {
5307 Poll::Ready(Ok(result)) => {
5308 ctx.ensure_history_consumed()?;
5309 let result = encode_typed_envelope(&result, &task.payload_codec)?;
5310 let mut commands = ctx.take_commands()?;
5311 commands.push(json!({
5312 "type": "complete_workflow",
5313 "result": result
5314 }));
5315 Ok(commands)
5316 }
5317 Poll::Ready(Err(error)) => {
5318 if let Error::ContinueAsNew(request) = error {
5319 let mut commands = ctx.take_commands()?;
5320 if let Some(command) = ctx.continue_as_new_command(request)? {
5321 commands.push(command);
5322 }
5323 ctx.ensure_history_consumed()?;
5324 return Ok(commands);
5325 }
5326 ctx.ensure_history_consumed()?;
5329 if workflow_task_integrity_error(&error) {
5330 return Err(error);
5335 }
5336 let mut commands = ctx.take_commands()?;
5337 commands.push(workflow_failure_command(&error));
5338 Ok(commands)
5339 }
5340 Poll::Pending => {
5341 let commands = ctx.take_commands()?;
5342 if commands.is_empty() && !ctx.matched_recorded_pending()? {
5343 Err(Error::WorkflowYieldedWithoutCommand)
5344 } else {
5345 Ok(commands)
5346 }
5347 }
5348 }
5349 }
5350
5351 fn execute_update_task(&self, task: &WorkflowTask, update_id: &str) -> Result<Vec<Value>> {
5352 if !self.workflows.contains_key(&task.workflow_type) {
5353 return Err(Error::WorkflowNotRegistered(task.workflow_type.clone()));
5354 }
5355
5356 let accepted = task.history_events.iter().rev().find_map(|event| {
5357 (event.event_type == "UpdateAccepted"
5358 && event.payload.get("update_id").and_then(Value::as_str) == Some(update_id))
5359 .then_some(&event.payload)
5360 });
5361 let update_name = accepted
5362 .and_then(|payload| payload.get("update_name"))
5363 .and_then(Value::as_str)
5364 .or(task.update_name.as_deref())
5365 .unwrap_or_default();
5366 let Some(handler) = self
5367 .updates
5368 .get(&task.workflow_type)
5369 .and_then(|handlers| handlers.get(update_name))
5370 else {
5371 return Ok(vec![json!({
5372 "type": "fail_update",
5373 "update_id": update_id,
5374 "message": format!(
5375 "no update handler is registered for {}.{update_name}",
5376 task.workflow_type
5377 ),
5378 "exception_type": "UnknownUpdate",
5379 "non_retryable": true,
5380 })]);
5381 };
5382 let arguments = accepted
5383 .and_then(|payload| payload.get("arguments"))
5384 .or(task.arguments.as_ref());
5385 let arguments = decode_task_avro_arguments(arguments, &task.payload_codec)?;
5386 let context = QueryContext {
5387 workflow_id: task.workflow_id.clone(),
5388 run_id: task.run_id.clone(),
5389 workflow_type: task.workflow_type.clone(),
5390 run_status: Some("running".to_string()),
5391 workflow_input: Value::Null,
5392 workflow_input_avro_value: AvroValue::Null,
5393 history_events: Arc::new(task.history_events.clone()),
5394 signal_events: Arc::new(Vec::new()),
5395 };
5396 let mut future = handler(context, arguments);
5397 let mut cx = TaskContext::from_waker(noop_waker_ref());
5398
5399 match future.as_mut().poll(&mut cx) {
5400 Poll::Ready(Ok(result)) => Ok(vec![json!({
5401 "type": "complete_update",
5402 "update_id": update_id,
5403 "result": encode_typed_envelope(&result, &task.payload_codec)?,
5404 })]),
5405 Poll::Ready(Err(error)) => Ok(vec![json!({
5406 "type": "fail_update",
5407 "update_id": update_id,
5408 "message": error.to_string(),
5409 "exception_type": "UpdateFailed",
5410 "non_retryable": true,
5411 })]),
5412 Poll::Pending => Err(Error::WorkflowYieldedWithoutCommand),
5413 }
5414 }
5415
5416 async fn execute_activity_task(&self, task: ActivityTask) -> Result<AvroValue> {
5417 let handler = self
5418 .activities
5419 .get(&task.activity_type)
5420 .ok_or_else(|| Error::ActivityNotRegistered(task.activity_type.clone()))?;
5421 let args = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
5422 let attempt_id = task
5423 .activity_attempt_id
5424 .clone()
5425 .or(task.attempt_id.clone())
5426 .unwrap_or_default();
5427 let lease_owner = task
5428 .lease_owner
5429 .clone()
5430 .unwrap_or_else(|| self.worker_id.clone());
5431 let ctx = ActivityContext {
5432 client: self.client.clone(),
5433 task_id: task.task_id,
5434 activity_attempt_id: attempt_id,
5435 lease_owner,
5436 activity_type: task.activity_type,
5437 attempt_number: task.attempt_number,
5438 task_queue: self.task_queue.clone(),
5439 worker_id: self.worker_id.clone(),
5440 };
5441
5442 handler(ctx, args).await
5443 }
5444}
5445
5446fn poller_result(
5447 kind: &str,
5448 result: std::result::Result<Result<()>, tokio::task::JoinError>,
5449) -> Result<()> {
5450 match result {
5451 Ok(result) => result,
5452 Err(error) => Err(Error::WorkerLoop(format!(
5453 "{kind} poller join error: {error}"
5454 ))),
5455 }
5456}
5457
5458fn optional_poller_result(
5459 kind: &str,
5460 result: Option<std::result::Result<Result<()>, tokio::task::JoinError>>,
5461) -> Result<()> {
5462 match result {
5463 Some(result) => poller_result(kind, result),
5464 None => Ok(()),
5465 }
5466}
5467
5468async fn join_pollers(
5469 workflow_poller: Option<tokio::task::JoinHandle<Result<()>>>,
5470 activity_poller: Option<tokio::task::JoinHandle<Result<()>>>,
5471 query_poller: Option<tokio::task::JoinHandle<Result<()>>>,
5472) -> Result<()> {
5473 let mut first_error = None;
5474
5475 if let Some(handle) = workflow_poller {
5476 if let Err(error) = poller_result("workflow", handle.await) {
5477 first_error.get_or_insert(error);
5478 }
5479 }
5480
5481 if let Some(handle) = activity_poller {
5482 if let Err(error) = poller_result("activity", handle.await) {
5483 first_error.get_or_insert(error);
5484 }
5485 }
5486
5487 if let Some(handle) = query_poller {
5488 if let Err(error) = poller_result("query", handle.await) {
5489 first_error.get_or_insert(error);
5490 }
5491 }
5492
5493 if let Some(error) = first_error {
5494 Err(error)
5495 } else {
5496 Ok(())
5497 }
5498}
5499
5500fn default_worker_id() -> String {
5501 let millis = SystemTime::now()
5502 .duration_since(UNIX_EPOCH)
5503 .unwrap_or_default()
5504 .as_millis();
5505 format!("rust-worker-{}-{millis}", std::process::id())
5506}
5507
5508fn percent_encode_path_segment(segment: &str) -> String {
5509 const HEX: &[u8; 16] = b"0123456789ABCDEF";
5510 let mut encoded = String::with_capacity(segment.len());
5511
5512 for byte in segment.bytes() {
5513 if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') {
5514 encoded.push(char::from(byte));
5515 } else {
5516 encoded.push('%');
5517 encoded.push(char::from(HEX[(byte >> 4) as usize]));
5518 encoded.push(char::from(HEX[(byte & 0x0f) as usize]));
5519 }
5520 }
5521
5522 encoded
5523}
5524
5525fn unique_request_id(prefix: &str) -> String {
5526 let nanos = SystemTime::now()
5527 .duration_since(UNIX_EPOCH)
5528 .unwrap_or_default()
5529 .as_nanos();
5530 format!("{prefix}-{}-{nanos}", std::process::id())
5531}
5532
5533#[derive(Debug)]
5534struct QueryTaskExecutionFailure {
5535 reason: String,
5536 message: String,
5537 failure_type: String,
5538}
5539
5540impl QueryTaskExecutionFailure {
5541 fn new(
5542 reason: impl Into<String>,
5543 message: impl Into<String>,
5544 failure_type: impl Into<String>,
5545 ) -> Self {
5546 Self {
5547 reason: reason.into(),
5548 message: message.into(),
5549 failure_type: failure_type.into(),
5550 }
5551 }
5552}
5553
5554#[derive(Clone, Debug)]
5560pub struct WorkflowInstance<S> {
5561 state: Arc<Mutex<S>>,
5562}
5563
5564impl<S> WorkflowInstance<S> {
5565 fn new(state: S) -> Self {
5566 Self {
5567 state: Arc::new(Mutex::new(state)),
5568 }
5569 }
5570
5571 pub fn read<R>(&self, reader: impl FnOnce(&S) -> R) -> Result<R> {
5573 let state = self
5574 .state
5575 .lock()
5576 .map_err(|_| Error::WorkflowStatePoisoned)?;
5577 Ok(reader(&state))
5578 }
5579
5580 pub fn update<R>(&self, transition: impl FnOnce(&mut S) -> R) -> Result<R> {
5582 let mut state = self
5583 .state
5584 .lock()
5585 .map_err(|_| Error::WorkflowStatePoisoned)?;
5586 Ok(transition(&mut state))
5587 }
5588}
5589
5590impl<S: Clone> WorkflowInstance<S> {
5591 fn snapshot(&self) -> Result<S> {
5592 self.read(Clone::clone)
5593 }
5594}
5595
5596#[derive(Clone, Debug)]
5597pub struct WorkflowContext {
5598 state: Arc<Mutex<WorkflowState>>,
5599}
5600
5601impl WorkflowContext {
5602 pub fn workflow_identity(&self) -> Result<WorkflowIdentity> {
5604 let state = self
5605 .state
5606 .lock()
5607 .map_err(|_| Error::WorkflowStatePoisoned)?;
5608 Ok(WorkflowIdentity {
5609 workflow_id: state.workflow_id.clone(),
5610 run_id: state.run_id.clone(),
5611 })
5612 }
5613
5614 pub fn history_budget(&self) -> Result<WorkflowHistoryBudget> {
5616 let state = self
5617 .state
5618 .lock()
5619 .map_err(|_| Error::WorkflowStatePoisoned)?;
5620 Ok(state.history_budget.clone())
5621 }
5622
5623 pub fn continue_as_new<T: Serialize>(&self, args: T) -> Result<Value> {
5629 self.continue_as_new_with_options(ContinueAsNewOptions::new(), args)
5630 }
5631
5632 pub fn continue_as_new_with_options<T: Serialize>(
5634 &self,
5635 options: ContinueAsNewOptions,
5636 args: T,
5637 ) -> Result<Value> {
5638 options.validate()?;
5639 Err(Error::ContinueAsNew(ContinueAsNewRequest {
5640 arguments: normalize_avro_arguments(AvroValue::from_serialize(&args)?),
5641 options,
5642 }))
5643 }
5644
5645 pub fn activity<T: Serialize>(
5646 &self,
5647 activity_type: impl Into<String>,
5648 args: T,
5649 ) -> ActivityCall {
5650 self.activity_with_options(activity_type, ActivityOptions::new(), args)
5651 }
5652
5653 pub fn activity_on_queue<T, Q>(
5654 &self,
5655 activity_type: impl Into<String>,
5656 task_queue: Option<Q>,
5657 args: T,
5658 ) -> ActivityCall
5659 where
5660 T: Serialize,
5661 Q: Into<String>,
5662 {
5663 let mut options = ActivityOptions::new();
5664 options.task_queue = task_queue.map(Into::into);
5665 self.activity_with_options(activity_type, options, args)
5666 }
5667
5668 pub fn activity_with_options<T: Serialize>(
5706 &self,
5707 activity_type: impl Into<String>,
5708 options: ActivityOptions,
5709 args: T,
5710 ) -> ActivityCall {
5711 ActivityCall {
5712 ctx: self.clone(),
5713 activity_type: activity_type.into(),
5714 options,
5715 args: Some(AvroValue::from_serialize(&args)),
5716 scheduled: false,
5717 }
5718 }
5719
5720 pub async fn activity_avro_value<T: Serialize>(
5721 &self,
5722 activity_type: impl Into<String>,
5723 args: T,
5724 ) -> Result<AvroValue> {
5725 let mut call = self.activity(activity_type, args);
5726 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
5727 }
5728
5729 pub async fn activity_avro_value_with_options<T: Serialize>(
5730 &self,
5731 activity_type: impl Into<String>,
5732 options: ActivityOptions,
5733 args: T,
5734 ) -> Result<AvroValue> {
5735 let mut call = self.activity_with_options(activity_type, options, args);
5736 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
5737 }
5738
5739 pub fn wait_signal(&self, signal_name: impl Into<String>) -> SignalCall {
5740 SignalCall {
5741 ctx: self.clone(),
5742 signal_name: signal_name.into(),
5743 opened_wait: false,
5744 matched_pending: false,
5745 }
5746 }
5747
5748 pub async fn wait_signal_avro_value(
5749 &self,
5750 signal_name: impl Into<String>,
5751 ) -> Result<Vec<AvroValue>> {
5752 let mut call = self.wait_signal(signal_name);
5753 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
5754 }
5755
5756 pub fn sleep(&self, duration: Duration) -> TimerCall {
5777 let delay_seconds = duration
5778 .as_secs()
5779 .checked_add(u64::from(duration.subsec_nanos() > 0));
5780 TimerCall {
5781 ctx: self.clone(),
5782 delay_seconds,
5783 scheduled: false,
5784 matched_pending: false,
5785 }
5786 }
5787
5788 pub fn start_timer(&self, duration: Duration) -> TimerCall {
5790 self.sleep(duration)
5791 }
5792
5793 pub fn side_effect<T, F>(&self, callback: F) -> Result<T>
5800 where
5801 T: Serialize + DeserializeOwned,
5802 F: FnOnce() -> T,
5803 {
5804 {
5805 let mut state = self
5806 .state
5807 .lock()
5808 .map_err(|_| Error::WorkflowStatePoisoned)?;
5809 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
5810 return match recorded {
5811 RecordedCommand::SideEffect { sequence, value } => {
5812 state.command_cursor += 1;
5813 value.deserialize().map_err(|error| {
5814 Error::NonDeterministicReplay(ReplayFailure::new(
5815 "side_effect_type_mismatch",
5816 Some(sequence),
5817 Some(std::any::type_name::<T>().to_string()),
5818 Some(error.to_string()),
5819 "recorded side-effect value is incompatible with the requested Rust type",
5820 ))
5821 })
5822 }
5823 other => Err(command_mismatch(&other, "side effect")),
5824 };
5825 }
5826 }
5827
5828 let value = callback();
5829 let avro_value = AvroValue::from_serialize(&value)?;
5830 let mut state = self
5831 .state
5832 .lock()
5833 .map_err(|_| Error::WorkflowStatePoisoned)?;
5834 let result = encode_typed_envelope(&avro_value, &state.payload_codec)?;
5835 state.commands.push(json!({
5836 "type": "record_side_effect",
5837 "result": result,
5838 }));
5839 Ok(value)
5840 }
5841
5842 pub fn side_effect_avro_value<F>(&self, callback: F) -> Result<AvroValue>
5844 where
5845 F: FnOnce() -> AvroValue,
5846 {
5847 {
5848 let mut state = self
5849 .state
5850 .lock()
5851 .map_err(|_| Error::WorkflowStatePoisoned)?;
5852 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
5853 return match recorded {
5854 RecordedCommand::SideEffect { value, .. } => {
5855 state.command_cursor += 1;
5856 Ok(value)
5857 }
5858 other => Err(command_mismatch(&other, "side effect")),
5859 };
5860 }
5861 }
5862
5863 let value = callback();
5864 let mut state = self
5865 .state
5866 .lock()
5867 .map_err(|_| Error::WorkflowStatePoisoned)?;
5868 let result = encode_typed_envelope(&value, &state.payload_codec)?;
5869 state.commands.push(json!({
5870 "type": "record_side_effect",
5871 "result": result,
5872 }));
5873 Ok(value)
5874 }
5875
5876 pub fn uuid_v4(&self) -> Result<Uuid> {
5878 self.side_effect(Uuid::new_v4)
5879 }
5880
5881 pub fn get_version(
5884 &self,
5885 change_id: impl Into<String>,
5886 min_supported: i32,
5887 max_supported: i32,
5888 ) -> Result<i32> {
5889 let change_id = change_id.into();
5890 if change_id.trim().is_empty() {
5891 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5892 "version_change_id_invalid",
5893 None,
5894 Some("non-empty change ID".to_string()),
5895 Some(change_id),
5896 "version markers require a stable non-empty change ID",
5897 )));
5898 }
5899 if min_supported > max_supported {
5900 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5901 "version_range_invalid",
5902 None,
5903 Some("min_supported <= max_supported".to_string()),
5904 Some(format!("{min_supported}..={max_supported}")),
5905 "version marker supported range is invalid",
5906 )));
5907 }
5908
5909 let mut state = self
5910 .state
5911 .lock()
5912 .map_err(|_| Error::WorkflowStatePoisoned)?;
5913 if let Some((version, sequence)) = state.version_markers.get(&change_id).copied() {
5914 ensure_version_supported(&change_id, version, min_supported, max_supported, sequence)?;
5915 return Ok(version);
5916 }
5917
5918 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
5919 return match recorded {
5920 RecordedCommand::VersionMarker {
5921 sequence,
5922 change_id: recorded_change_id,
5923 version,
5924 ..
5925 } => {
5926 if recorded_change_id != change_id {
5927 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5928 "version_change_id_mismatch",
5929 Some(sequence),
5930 Some(recorded_change_id),
5931 Some(change_id),
5932 "recorded version marker change ID differs from current workflow code",
5933 )));
5934 }
5935 ensure_version_supported(
5936 &change_id,
5937 version,
5938 min_supported,
5939 max_supported,
5940 sequence,
5941 )?;
5942 state.command_cursor += 1;
5943 state.version_markers.insert(change_id, (version, sequence));
5944 Ok(version)
5945 }
5946 other => Err(command_mismatch(
5947 &other,
5948 format!("version marker:{change_id}"),
5949 )),
5950 };
5951 }
5952
5953 let version = max_supported;
5954 state.commands.push(json!({
5955 "type": "record_version_marker",
5956 "change_id": change_id,
5957 "version": version,
5958 "min_supported": min_supported,
5959 "max_supported": max_supported,
5960 }));
5961 state.version_markers.insert(change_id, (version, 0));
5964 Ok(version)
5965 }
5966
5967 pub fn patched(&self, change_id: impl Into<String>) -> Result<bool> {
5969 Ok(self.get_version(change_id, -1, 1)? == 1)
5970 }
5971
5972 pub fn deprecate_patch(&self, change_id: impl Into<String>) -> Result<()> {
5974 self.get_version(change_id, -1, 1).map(|_| ())
5975 }
5976
5977 pub fn start_child_workflow<T: Serialize>(
6004 &self,
6005 workflow_type: impl Into<String>,
6006 options: ChildWorkflowOptions,
6007 args: T,
6008 ) -> ChildWorkflowCall {
6009 ChildWorkflowCall {
6010 ctx: self.clone(),
6011 workflow_type: workflow_type.into(),
6012 options,
6013 args: Some(AvroValue::from_serialize(&args)),
6014 scheduled: false,
6015 matched_pending: false,
6016 }
6017 }
6018
6019 pub async fn start_child_workflow_avro_value<T: Serialize>(
6020 &self,
6021 workflow_type: impl Into<String>,
6022 options: ChildWorkflowOptions,
6023 args: T,
6024 ) -> Result<ChildWorkflowAvroResult> {
6025 let mut call = self.start_child_workflow(workflow_type, options, args);
6026 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
6027 }
6028
6029 fn take_commands(&self) -> Result<Vec<Value>> {
6030 let mut state = self
6031 .state
6032 .lock()
6033 .map_err(|_| Error::WorkflowStatePoisoned)?;
6034 Ok(std::mem::take(&mut state.commands))
6035 }
6036
6037 fn continue_as_new_command(&self, request: ContinueAsNewRequest) -> Result<Option<Value>> {
6038 let mut state = self
6039 .state
6040 .lock()
6041 .map_err(|_| Error::WorkflowStatePoisoned)?;
6042
6043 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6044 return Err(command_mismatch(&recorded, "continue as new"));
6045 }
6046 if state.recorded_continue_as_new_sequence.is_some() {
6047 state.continue_as_new_consumed = true;
6048 return Ok(None);
6049 }
6050
6051 let arguments = encode_typed_envelope(&request.arguments, &state.payload_codec)?;
6052 let mut command = serde_json::Map::from_iter([
6053 ("type".to_string(), json!("continue_as_new")),
6054 ("arguments".to_string(), arguments),
6055 ("queue".to_string(), json!(state.task_queue.clone())),
6056 ]);
6057 if let Some(workflow_type) = request.options.workflow_type {
6058 command.insert("workflow_type".to_string(), json!(workflow_type));
6059 }
6060 if let Some(task_queue) = request.options.task_queue {
6061 command.insert("queue".to_string(), json!(task_queue));
6062 }
6063 Ok(Some(Value::Object(command)))
6064 }
6065
6066 fn matched_recorded_pending(&self) -> Result<bool> {
6067 let state = self
6068 .state
6069 .lock()
6070 .map_err(|_| Error::WorkflowStatePoisoned)?;
6071 Ok(state.matched_recorded_pending)
6072 }
6073
6074 fn ensure_history_consumed(&self) -> Result<()> {
6075 let state = self
6076 .state
6077 .lock()
6078 .map_err(|_| Error::WorkflowStatePoisoned)?;
6079 if let Some(command) = state.recorded_commands.get(state.command_cursor) {
6080 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
6081 "recorded_commands_unconsumed",
6082 Some(command.sequence()),
6083 Some(command.shape().to_string()),
6084 Some("workflow completion".to_string()),
6085 "workflow completed before consuming all recorded durable commands",
6086 )));
6087 }
6088 if let Some(sequence) = state
6089 .recorded_continue_as_new_sequence
6090 .filter(|_| !state.continue_as_new_consumed)
6091 {
6092 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
6093 "recorded_continue_as_new_unconsumed",
6094 Some(sequence),
6095 Some("continue as new".to_string()),
6096 Some("workflow completion".to_string()),
6097 "workflow completed without consuming its recorded continue-as-new transition",
6098 )));
6099 }
6100 Ok(())
6101 }
6102}
6103
6104#[derive(Debug)]
6105struct WorkflowState {
6106 workflow_id: Option<String>,
6107 run_id: Option<String>,
6108 task_queue: String,
6109 payload_codec: String,
6110 history_budget: WorkflowHistoryBudget,
6111 resume_signal: Option<ResumeSignal>,
6112 recorded_commands: Vec<RecordedCommand>,
6113 recorded_continue_as_new_sequence: Option<u64>,
6114 continue_as_new_consumed: bool,
6115 command_cursor: usize,
6116 matched_recorded_pending: bool,
6117 version_markers: HashMap<String, (i32, u64)>,
6118 commands: Vec<Value>,
6119}
6120
6121impl WorkflowState {
6122 #[cfg(test)]
6123 fn new(
6124 history: Vec<HistoryEvent>,
6125 task_queue: String,
6126 payload_codec: String,
6127 resume_signal: Option<ResumeSignal>,
6128 ) -> Result<Self> {
6129 Self::new_with_identity(
6130 history,
6131 None,
6132 None,
6133 task_queue,
6134 payload_codec,
6135 resume_signal,
6136 )
6137 }
6138
6139 fn new_with_identity(
6140 history: Vec<HistoryEvent>,
6141 workflow_id: Option<String>,
6142 run_id: Option<String>,
6143 task_queue: String,
6144 payload_codec: String,
6145 resume_signal: Option<ResumeSignal>,
6146 ) -> Result<Self> {
6147 let recorded_commands = recorded_commands(
6148 &history,
6149 &payload_codec,
6150 WorkflowIdentity {
6151 workflow_id: workflow_id.clone(),
6152 run_id: run_id.clone(),
6153 },
6154 )?;
6155 let recorded_continue_as_new = history
6156 .iter()
6157 .filter(|event| event.event_type == "WorkflowContinuedAsNew")
6158 .collect::<Vec<_>>();
6159 if recorded_continue_as_new.len() > 1 {
6160 return Err(invalid_recorded_history(
6161 "duplicate_continue_as_new_transition",
6162 recorded_continue_as_new
6163 .last()
6164 .and_then(|event| durable_event_sequence(event))
6165 .unwrap_or(0),
6166 "one WorkflowContinuedAsNew event",
6167 &format!(
6168 "{} WorkflowContinuedAsNew events",
6169 recorded_continue_as_new.len()
6170 ),
6171 "workflow history records one continue-as-new transition more than once",
6172 ));
6173 }
6174 let recorded_continue_as_new_sequence = recorded_continue_as_new
6175 .first()
6176 .map(|event| {
6177 durable_event_sequence(event).ok_or_else(|| {
6178 Error::NonDeterministicReplay(ReplayFailure::new(
6179 "continue_as_new_sequence_missing",
6180 None,
6181 Some("recorded transition sequence".to_string()),
6182 Some("missing sequence".to_string()),
6183 "WorkflowContinuedAsNew history is missing its recorded sequence",
6184 ))
6185 })
6186 })
6187 .transpose()?;
6188 let event_count = u64::try_from(history.len()).unwrap_or(u64::MAX);
6189 Ok(Self {
6190 workflow_id,
6191 run_id,
6192 task_queue,
6193 payload_codec,
6194 history_budget: WorkflowHistoryBudget {
6195 event_count,
6196 ..WorkflowHistoryBudget::default()
6197 },
6198 resume_signal,
6199 recorded_commands,
6200 recorded_continue_as_new_sequence,
6201 continue_as_new_consumed: false,
6202 command_cursor: 0,
6203 matched_recorded_pending: false,
6204 version_markers: HashMap::new(),
6205 commands: Vec::new(),
6206 })
6207 }
6208}
6209
6210#[derive(Clone, Debug)]
6211enum RecordedCommand {
6212 Activity {
6213 sequence: u64,
6214 activity_type: Option<String>,
6215 options: Option<RecordedActivityOptions>,
6216 outcome: Option<ActivityOutcome>,
6217 },
6218 Timer {
6219 sequence: u64,
6220 delay_seconds: u64,
6221 fired: bool,
6222 },
6223 ChildWorkflow {
6224 sequence: u64,
6225 workflow_type: Option<String>,
6226 outcome: Option<ChildWorkflowOutcome>,
6227 },
6228 SignalWait {
6229 sequence: u64,
6230 signal_name: String,
6231 value: Option<Vec<AvroValue>>,
6232 },
6233 SideEffect {
6234 sequence: u64,
6235 value: AvroValue,
6236 },
6237 VersionMarker {
6238 sequence: u64,
6239 change_id: String,
6240 version: i32,
6241 },
6242}
6243
6244#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
6245struct RecordedActivityOptions {
6246 task_queue: RecordedSnapshotValue<Option<String>>,
6247 execution_mode: RecordedSnapshotValue<Option<String>>,
6248 retry_policy: ActivityRetrySnapshot,
6249}
6250
6251#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
6252enum RecordedSnapshotValue<T> {
6253 Unknown,
6255 Known(T),
6256}
6257
6258impl<T: PartialEq> RecordedSnapshotValue<T> {
6259 fn matches_current(&self, current: &Self) -> bool {
6260 match self {
6261 Self::Unknown => true,
6262 Self::Known(recorded) => matches!(current, Self::Known(value) if value == recorded),
6263 }
6264 }
6265}
6266
6267#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
6268struct ActivityRetrySnapshot {
6269 snapshot_version: RecordedSnapshotValue<Option<u64>>,
6270 max_attempts: RecordedSnapshotValue<Option<u64>>,
6271 backoff_seconds: RecordedSnapshotValue<Vec<u64>>,
6272 start_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
6273 schedule_to_start_timeout: RecordedSnapshotValue<Option<u64>>,
6274 schedule_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
6275 heartbeat_timeout: RecordedSnapshotValue<Option<u64>>,
6276 non_retryable_error_types: RecordedSnapshotValue<Vec<String>>,
6277}
6278
6279impl ActivityRetrySnapshot {
6280 fn matches_current(&self, current: &Self) -> bool {
6281 self.snapshot_version
6282 .matches_current(¤t.snapshot_version)
6283 && self.max_attempts.matches_current(¤t.max_attempts)
6284 && self
6285 .backoff_seconds
6286 .matches_current(¤t.backoff_seconds)
6287 && self
6288 .start_to_close_timeout
6289 .matches_current(¤t.start_to_close_timeout)
6290 && self
6291 .schedule_to_start_timeout
6292 .matches_current(¤t.schedule_to_start_timeout)
6293 && self
6294 .schedule_to_close_timeout
6295 .matches_current(¤t.schedule_to_close_timeout)
6296 && self
6297 .heartbeat_timeout
6298 .matches_current(¤t.heartbeat_timeout)
6299 && self
6300 .non_retryable_error_types
6301 .matches_current(¤t.non_retryable_error_types)
6302 }
6303}
6304
6305fn recorded_optional_u64(
6306 object: Option<&serde_json::Map<String, Value>>,
6307 field: &str,
6308) -> RecordedSnapshotValue<Option<u64>> {
6309 match object.and_then(|object| object.get(field)) {
6310 None => RecordedSnapshotValue::Unknown,
6311 Some(Value::Null) => RecordedSnapshotValue::Known(None),
6312 Some(value) => RecordedSnapshotValue::Known(value_as_u64(value)),
6313 }
6314}
6315
6316fn recorded_optional_string(
6317 object: &serde_json::Map<String, Value>,
6318 field: &str,
6319) -> RecordedSnapshotValue<Option<String>> {
6320 match object.get(field) {
6321 None => RecordedSnapshotValue::Unknown,
6322 Some(Value::Null) => RecordedSnapshotValue::Known(None),
6323 Some(value) => RecordedSnapshotValue::Known(value.as_str().map(str::to_string)),
6324 }
6325}
6326
6327fn recorded_activity_retry_snapshot(policy: Option<&Value>) -> ActivityRetrySnapshot {
6328 let policy = policy.and_then(Value::as_object);
6329 let backoff_seconds = policy
6330 .and_then(|policy| policy.get("backoff_seconds"))
6331 .and_then(Value::as_array)
6332 .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
6333 .map_or(RecordedSnapshotValue::Unknown, RecordedSnapshotValue::Known);
6334 let mut non_retryable_error_types = Vec::new();
6335 for error_type in policy
6336 .and_then(|policy| policy.get("non_retryable_error_types"))
6337 .and_then(Value::as_array)
6338 .into_iter()
6339 .flatten()
6340 .filter_map(Value::as_str)
6341 .map(str::trim)
6342 .filter(|error_type| !error_type.is_empty())
6343 {
6344 if !non_retryable_error_types
6345 .iter()
6346 .any(|recorded| recorded == error_type)
6347 {
6348 non_retryable_error_types.push(error_type.to_string());
6349 }
6350 }
6351
6352 ActivityRetrySnapshot {
6353 snapshot_version: recorded_optional_u64(policy, "snapshot_version"),
6354 max_attempts: recorded_optional_u64(policy, "max_attempts"),
6355 backoff_seconds,
6356 start_to_close_timeout: recorded_optional_u64(policy, "start_to_close_timeout"),
6357 schedule_to_start_timeout: recorded_optional_u64(policy, "schedule_to_start_timeout"),
6358 schedule_to_close_timeout: recorded_optional_u64(policy, "schedule_to_close_timeout"),
6359 heartbeat_timeout: recorded_optional_u64(policy, "heartbeat_timeout"),
6360 non_retryable_error_types: if policy
6361 .is_some_and(|policy| policy.contains_key("non_retryable_error_types"))
6362 {
6363 RecordedSnapshotValue::Known(non_retryable_error_types)
6364 } else {
6365 RecordedSnapshotValue::Unknown
6366 },
6367 }
6368}
6369
6370fn current_activity_retry_snapshot(options: &ValidatedActivityOptions) -> ActivityRetrySnapshot {
6371 let policy = options.retry_policy.as_ref();
6372 let max_attempts = match policy.and_then(|policy| policy.get("max_attempts")) {
6373 Some(Value::Null) => None,
6374 Some(value) => value_as_u64(value),
6375 None => Some(1),
6376 };
6377 let backoff_seconds = policy
6378 .and_then(|policy| policy.get("backoff_seconds"))
6379 .and_then(Value::as_array)
6380 .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
6381 .unwrap_or_default();
6382 let non_retryable_error_types = policy
6383 .and_then(|policy| policy.get("non_retryable_error_types"))
6384 .and_then(Value::as_array)
6385 .into_iter()
6386 .flatten()
6387 .filter_map(Value::as_str)
6388 .map(str::to_string)
6389 .collect();
6390
6391 ActivityRetrySnapshot {
6392 snapshot_version: RecordedSnapshotValue::Known(Some(1)),
6393 max_attempts: RecordedSnapshotValue::Known(max_attempts),
6394 backoff_seconds: RecordedSnapshotValue::Known(backoff_seconds),
6395 start_to_close_timeout: RecordedSnapshotValue::Known(options.start_to_close_timeout),
6396 schedule_to_start_timeout: RecordedSnapshotValue::Known(options.schedule_to_start_timeout),
6397 schedule_to_close_timeout: RecordedSnapshotValue::Known(options.schedule_to_close_timeout),
6398 heartbeat_timeout: RecordedSnapshotValue::Known(options.heartbeat_timeout),
6399 non_retryable_error_types: RecordedSnapshotValue::Known(non_retryable_error_types),
6400 }
6401}
6402
6403fn activity_options_description(options: &RecordedActivityOptions) -> String {
6404 serde_json::to_string(options).unwrap_or_else(|_| format!("{options:?}"))
6405}
6406
6407impl RecordedCommand {
6408 fn sequence(&self) -> u64 {
6409 match self {
6410 Self::Activity { sequence, .. }
6411 | Self::Timer { sequence, .. }
6412 | Self::ChildWorkflow { sequence, .. }
6413 | Self::SignalWait { sequence, .. }
6414 | Self::SideEffect { sequence, .. }
6415 | Self::VersionMarker { sequence, .. } => *sequence,
6416 }
6417 }
6418
6419 fn shape(&self) -> &'static str {
6420 match self {
6421 Self::Activity { .. } => "activity",
6422 Self::Timer { .. } => "timer",
6423 Self::ChildWorkflow { .. } => "child workflow",
6424 Self::SignalWait { .. } => "signal wait",
6425 Self::SideEffect { .. } => "side effect",
6426 Self::VersionMarker { .. } => "version marker",
6427 }
6428 }
6429}
6430
6431fn ensure_version_supported(
6432 change_id: &str,
6433 version: i32,
6434 min_supported: i32,
6435 max_supported: i32,
6436 sequence: u64,
6437) -> Result<()> {
6438 if (min_supported..=max_supported).contains(&version) {
6439 return Ok(());
6440 }
6441 Err(Error::NonDeterministicReplay(ReplayFailure::new(
6442 "version_marker_incompatible_range",
6443 (sequence != 0).then_some(sequence),
6444 Some(format!("{min_supported}..={max_supported}")),
6445 Some(format!("{change_id}:{version}")),
6446 "recorded workflow version is outside the range supported by current code",
6447 )))
6448}
6449
6450#[derive(Clone, Debug)]
6451struct ResumeSignal {
6452 signal_name: String,
6453 arguments: Vec<AvroValue>,
6454}
6455
6456pub struct ActivityCall {
6457 ctx: WorkflowContext,
6458 activity_type: String,
6459 options: ActivityOptions,
6460 args: Option<Result<AvroValue>>,
6461 scheduled: bool,
6462}
6463
6464impl ActivityCall {
6465 fn poll_avro_value(
6466 mut self: Pin<&mut Self>,
6467 _cx: &mut TaskContext<'_>,
6468 ) -> Poll<Result<AvroValue>> {
6469 let ctx = self.ctx.clone();
6470 let mut state = match ctx.state.lock() {
6471 Ok(state) => state,
6472 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6473 };
6474
6475 if self.scheduled {
6476 return Poll::Pending;
6477 }
6478
6479 let options = match self.options.validate() {
6480 Ok(options) => options,
6481 Err(error) => {
6482 return Poll::Ready(Err(Error::InvalidActivityOptions(error)));
6483 }
6484 };
6485 let task_queue = options
6486 .task_queue
6487 .clone()
6488 .unwrap_or_else(|| state.task_queue.clone());
6489 let current_recorded_options = RecordedActivityOptions {
6490 task_queue: RecordedSnapshotValue::Known(Some(task_queue.clone())),
6491 execution_mode: RecordedSnapshotValue::Known(None),
6494 retry_policy: current_activity_retry_snapshot(&options),
6495 };
6496
6497 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6498 let sequence = recorded.sequence();
6499 match recorded {
6500 RecordedCommand::Activity {
6501 activity_type,
6502 options: recorded_options,
6503 outcome,
6504 ..
6505 } => {
6506 if let Some(recorded_type) = activity_type {
6507 if recorded_type != self.activity_type {
6508 return Poll::Ready(Err(Error::NonDeterministicReplay(
6509 ReplayFailure::new(
6510 "recorded_command_detail_mismatch",
6511 Some(sequence),
6512 Some(format!("activity:{recorded_type}")),
6513 Some(format!("activity:{}", self.activity_type)),
6514 "recorded activity type differs from the current workflow command",
6515 ),
6516 )));
6517 }
6518 }
6519 if let Some(recorded_options) = recorded_options {
6520 if !recorded_options
6521 .task_queue
6522 .matches_current(¤t_recorded_options.task_queue)
6523 {
6524 return Poll::Ready(Err(Error::NonDeterministicReplay(
6525 ReplayFailure::new(
6526 "activity_task_queue_mismatch",
6527 Some(sequence),
6528 Some(activity_options_description(&recorded_options)),
6529 Some(activity_options_description(¤t_recorded_options)),
6530 "recorded activity task queue differs from the current workflow command",
6531 ),
6532 )));
6533 }
6534 if !recorded_options
6535 .execution_mode
6536 .matches_current(¤t_recorded_options.execution_mode)
6537 {
6538 return Poll::Ready(Err(Error::NonDeterministicReplay(
6539 ReplayFailure::new(
6540 "activity_execution_mode_mismatch",
6541 Some(sequence),
6542 Some(activity_options_description(&recorded_options)),
6543 Some(activity_options_description(¤t_recorded_options)),
6544 "recorded activity execution mode differs from the current workflow command",
6545 ),
6546 )));
6547 }
6548 if !recorded_options
6549 .retry_policy
6550 .matches_current(¤t_recorded_options.retry_policy)
6551 {
6552 return Poll::Ready(Err(Error::NonDeterministicReplay(
6553 ReplayFailure::new(
6554 "activity_retry_policy_mismatch",
6555 Some(sequence),
6556 Some(activity_options_description(&recorded_options)),
6557 Some(activity_options_description(¤t_recorded_options)),
6558 "recorded activity retry policy differs from the current workflow command",
6559 ),
6560 )));
6561 }
6562 }
6563 state.command_cursor += 1;
6564 if let Some(outcome) = outcome {
6565 return Poll::Ready(outcome.map_err(Error::ActivityFailed));
6566 }
6567 state.matched_recorded_pending = true;
6568 self.scheduled = true;
6569 return Poll::Pending;
6570 }
6571 other => {
6572 return Poll::Ready(Err(command_mismatch(
6573 &other,
6574 format!("activity:{}", self.activity_type),
6575 )));
6576 }
6577 }
6578 }
6579
6580 if !self.scheduled {
6581 let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
6582 Ok(args) => args,
6583 Err(error) => return Poll::Ready(Err(error)),
6584 };
6585 let arguments = normalize_avro_arguments(args);
6586 let envelope = match encode_typed_envelope(&arguments, &state.payload_codec) {
6587 Ok(envelope) => envelope,
6588 Err(error) => return Poll::Ready(Err(error)),
6589 };
6590
6591 let mut command = serde_json::Map::from_iter([
6592 ("type".to_string(), json!("schedule_activity")),
6593 (
6594 "activity_type".to_string(),
6595 json!(self.activity_type.clone()),
6596 ),
6597 ("queue".to_string(), json!(task_queue)),
6598 ("arguments".to_string(), envelope),
6599 ]);
6600 for (field, value) in [
6601 ("start_to_close_timeout", options.start_to_close_timeout),
6602 (
6603 "schedule_to_start_timeout",
6604 options.schedule_to_start_timeout,
6605 ),
6606 (
6607 "schedule_to_close_timeout",
6608 options.schedule_to_close_timeout,
6609 ),
6610 ("heartbeat_timeout", options.heartbeat_timeout),
6611 ] {
6612 if let Some(value) = value {
6613 command.insert(field.to_string(), json!(value));
6614 }
6615 }
6616 if let Some(retry_policy) = options.retry_policy {
6617 command.insert("retry_policy".to_string(), retry_policy);
6618 }
6619 state.commands.push(Value::Object(command));
6620 self.scheduled = true;
6621 }
6622
6623 Poll::Pending
6624 }
6625}
6626
6627impl Future for ActivityCall {
6628 type Output = Result<Value>;
6629
6630 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6631 match self.poll_avro_value(cx) {
6632 Poll::Ready(Ok(value)) => Poll::Ready(value.into_json()),
6633 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
6634 Poll::Pending => Poll::Pending,
6635 }
6636 }
6637}
6638
6639pub struct TimerCall {
6641 ctx: WorkflowContext,
6642 delay_seconds: Option<u64>,
6643 scheduled: bool,
6644 matched_pending: bool,
6645}
6646
6647impl Future for TimerCall {
6648 type Output = Result<()>;
6649
6650 fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6651 if self.matched_pending {
6652 return Poll::Pending;
6653 }
6654
6655 let ctx = self.ctx.clone();
6656 let Some(requested_delay) = self.delay_seconds else {
6657 return Poll::Ready(Err(Error::TimerDurationOverflow));
6658 };
6659 let mut state = match ctx.state.lock() {
6660 Ok(state) => state,
6661 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6662 };
6663
6664 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6665 match recorded {
6666 RecordedCommand::Timer {
6667 sequence,
6668 delay_seconds,
6669 fired,
6670 ..
6671 } => {
6672 if delay_seconds != requested_delay {
6673 return Poll::Ready(Err(Error::NonDeterministicReplay(
6674 ReplayFailure::new(
6675 "timer_delay_mismatch",
6676 Some(sequence),
6677 Some(format!("timer:{delay_seconds}s")),
6678 Some(format!("timer:{requested_delay}s")),
6679 "recorded timer delay differs from the current workflow command",
6680 ),
6681 )));
6682 }
6683 state.command_cursor += 1;
6684 if fired {
6685 return Poll::Ready(Ok(()));
6686 }
6687 state.matched_recorded_pending = true;
6688 self.scheduled = true;
6689 self.matched_pending = true;
6690 return Poll::Pending;
6691 }
6692 other => return Poll::Ready(Err(command_mismatch(&other, "timer"))),
6693 }
6694 }
6695
6696 if !self.scheduled {
6697 state.commands.push(json!({
6698 "type": "start_timer",
6699 "delay_seconds": requested_delay,
6700 }));
6701 self.scheduled = true;
6702 }
6703
6704 Poll::Pending
6705 }
6706}
6707
6708pub struct ChildWorkflowCall {
6710 ctx: WorkflowContext,
6711 workflow_type: String,
6712 options: ChildWorkflowOptions,
6713 args: Option<Result<AvroValue>>,
6714 scheduled: bool,
6715 matched_pending: bool,
6716}
6717
6718impl ChildWorkflowCall {
6719 fn poll_avro_value(
6720 mut self: Pin<&mut Self>,
6721 _cx: &mut TaskContext<'_>,
6722 ) -> Poll<Result<ChildWorkflowAvroResult>> {
6723 if self.matched_pending {
6724 return Poll::Pending;
6725 }
6726
6727 let ctx = self.ctx.clone();
6728 let mut state = match ctx.state.lock() {
6729 Ok(state) => state,
6730 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6731 };
6732
6733 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6734 let sequence = recorded.sequence();
6735 match recorded {
6736 RecordedCommand::ChildWorkflow {
6737 workflow_type,
6738 outcome,
6739 ..
6740 } => {
6741 if let Some(recorded_type) = workflow_type {
6742 if recorded_type != self.workflow_type {
6743 return Poll::Ready(Err(Error::NonDeterministicReplay(
6744 ReplayFailure::new(
6745 "recorded_command_detail_mismatch",
6746 Some(sequence),
6747 Some(format!("child workflow:{recorded_type}")),
6748 Some(format!("child workflow:{}", self.workflow_type)),
6749 "recorded child workflow type differs from the current workflow command",
6750 ),
6751 )));
6752 }
6753 }
6754 state.command_cursor += 1;
6755 if let Some(outcome) = outcome {
6756 return Poll::Ready(outcome.map_err(Error::ChildWorkflowFailed));
6757 }
6758 state.matched_recorded_pending = true;
6759 self.scheduled = true;
6760 self.matched_pending = true;
6761 return Poll::Pending;
6762 }
6763 other => {
6764 return Poll::Ready(Err(command_mismatch(
6765 &other,
6766 format!("child workflow:{}", self.workflow_type),
6767 )));
6768 }
6769 }
6770 }
6771
6772 if !self.scheduled {
6773 if self.options.task_queue.trim().is_empty() {
6774 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
6775 "task_queue must not be empty".to_string(),
6776 )));
6777 }
6778 for (name, value) in [
6779 (
6780 "execution_timeout_seconds",
6781 self.options.execution_timeout_seconds,
6782 ),
6783 ("run_timeout_seconds", self.options.run_timeout_seconds),
6784 ] {
6785 if value == Some(0) {
6786 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(format!(
6787 "{name} must be at least 1"
6788 ))));
6789 }
6790 }
6791
6792 let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
6793 Ok(args) => args,
6794 Err(error) => return Poll::Ready(Err(error)),
6795 };
6796 let arguments = match encode_typed_envelope(
6797 &normalize_avro_arguments(args),
6798 &state.payload_codec,
6799 ) {
6800 Ok(arguments) => arguments,
6801 Err(error) => return Poll::Ready(Err(error)),
6802 };
6803 let mut command = json!({
6804 "type": "start_child_workflow",
6805 "workflow_type": self.workflow_type,
6806 "queue": self.options.task_queue,
6807 "parent_close_policy": self.options.parent_close_policy.as_str(),
6808 "arguments": arguments,
6809 });
6810 let object = command
6811 .as_object_mut()
6812 .expect("child workflow command is always an object");
6813 if let Some(policy) = &self.options.retry_policy {
6814 let mut retry_policy = serde_json::Map::new();
6815 if let Some(max_attempts) = policy.max_attempts {
6816 if max_attempts == 0 {
6817 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
6818 "retry_policy.max_attempts must be at least 1".to_string(),
6819 )));
6820 }
6821 retry_policy.insert("max_attempts".to_string(), json!(max_attempts));
6822 }
6823 if !policy.backoff_seconds.is_empty() {
6824 retry_policy
6825 .insert("backoff_seconds".to_string(), json!(policy.backoff_seconds));
6826 }
6827 if !policy.non_retryable_error_types.is_empty() {
6828 retry_policy.insert(
6829 "non_retryable_error_types".to_string(),
6830 json!(policy.non_retryable_error_types),
6831 );
6832 }
6833 if retry_policy.is_empty() {
6834 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
6835 "retry_policy must configure at least one field".to_string(),
6836 )));
6837 }
6838 object.insert("retry_policy".to_string(), Value::Object(retry_policy));
6839 }
6840 if let Some(seconds) = self.options.execution_timeout_seconds {
6841 object.insert("execution_timeout_seconds".to_string(), json!(seconds));
6842 }
6843 if let Some(seconds) = self.options.run_timeout_seconds {
6844 object.insert("run_timeout_seconds".to_string(), json!(seconds));
6845 }
6846 state.commands.push(command);
6847 self.scheduled = true;
6848 }
6849
6850 Poll::Pending
6851 }
6852}
6853
6854impl Future for ChildWorkflowCall {
6855 type Output = Result<ChildWorkflowResult>;
6856
6857 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6858 match self.poll_avro_value(cx) {
6859 Poll::Ready(Ok(result)) => match result.result.into_json() {
6860 Ok(projected) => Poll::Ready(Ok(ChildWorkflowResult {
6861 parent: result.parent,
6862 child: result.child,
6863 child_workflow_type: result.child_workflow_type,
6864 result: projected,
6865 })),
6866 Err(error) => Poll::Ready(Err(error)),
6867 },
6868 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
6869 Poll::Pending => Poll::Pending,
6870 }
6871 }
6872}
6873
6874fn command_mismatch(recorded: &RecordedCommand, actual: impl Into<String>) -> Error {
6875 Error::NonDeterministicReplay(ReplayFailure::new(
6876 "recorded_command_mismatch",
6877 Some(recorded.sequence()),
6878 Some(recorded.shape().to_string()),
6879 Some(actual.into()),
6880 "current workflow command does not match the recorded durable command sequence",
6881 ))
6882}
6883
6884pub struct SignalCall {
6885 ctx: WorkflowContext,
6886 signal_name: String,
6887 opened_wait: bool,
6888 matched_pending: bool,
6889}
6890
6891impl SignalCall {
6892 fn poll_avro_value(
6893 mut self: Pin<&mut Self>,
6894 _cx: &mut TaskContext<'_>,
6895 ) -> Poll<Result<Vec<AvroValue>>> {
6896 if self.matched_pending {
6897 return Poll::Pending;
6898 }
6899
6900 let ctx = self.ctx.clone();
6901 let mut state = match ctx.state.lock() {
6902 Ok(state) => state,
6903 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6904 };
6905
6906 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6907 match recorded {
6908 RecordedCommand::SignalWait {
6909 sequence,
6910 signal_name,
6911 value,
6912 } => {
6913 if signal_name != self.signal_name {
6914 return Poll::Ready(Err(Error::NonDeterministicReplay(
6915 ReplayFailure::new(
6916 "recorded_command_detail_mismatch",
6917 Some(sequence),
6918 Some(format!("signal wait:{signal_name}")),
6919 Some(format!("signal wait:{}", self.signal_name)),
6920 "recorded signal name differs from the current workflow command",
6921 ),
6922 )));
6923 }
6924
6925 state.command_cursor += 1;
6926 if let Some(value) = value {
6927 return Poll::Ready(Ok(value));
6928 }
6929 if state
6930 .resume_signal
6931 .as_ref()
6932 .is_some_and(|signal| signal.signal_name == self.signal_name)
6933 {
6934 let signal = state
6935 .resume_signal
6936 .take()
6937 .expect("matching resume signal is present");
6938 return Poll::Ready(Ok(signal.arguments));
6939 }
6940
6941 state.matched_recorded_pending = true;
6942 self.opened_wait = true;
6943 self.matched_pending = true;
6944 return Poll::Pending;
6945 }
6946 other => {
6947 return Poll::Ready(Err(command_mismatch(
6948 &other,
6949 format!("signal wait:{}", self.signal_name),
6950 )));
6951 }
6952 }
6953 }
6954
6955 if state
6956 .resume_signal
6957 .as_ref()
6958 .is_some_and(|signal| signal.signal_name == self.signal_name)
6959 {
6960 let signal = state
6961 .resume_signal
6962 .take()
6963 .expect("matching resume signal is present");
6964 return Poll::Ready(Ok(signal.arguments));
6965 }
6966
6967 if !self.opened_wait {
6968 state.commands.push(json!({
6969 "type": "open_signal_wait",
6970 "signal_name": self.signal_name
6971 }));
6972 self.opened_wait = true;
6973 }
6974
6975 Poll::Pending
6976 }
6977}
6978
6979impl Future for SignalCall {
6980 type Output = Result<Vec<Value>>;
6981
6982 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6983 match self.poll_avro_value(cx) {
6984 Poll::Ready(Ok(values)) => Poll::Ready(
6985 values
6986 .into_iter()
6987 .map(AvroValue::into_json)
6988 .collect::<Result<Vec<_>>>(),
6989 ),
6990 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
6991 Poll::Pending => Poll::Pending,
6992 }
6993 }
6994}
6995
6996#[derive(Clone, Debug)]
6997pub struct ActivityContext {
6998 client: Client,
6999 pub task_id: String,
7000 pub activity_attempt_id: String,
7001 pub lease_owner: String,
7002 pub activity_type: String,
7003 pub attempt_number: u64,
7004 pub task_queue: String,
7005 pub worker_id: String,
7006}
7007
7008impl ActivityContext {
7009 pub async fn heartbeat<T: Serialize>(&self, details: T) -> Result<ActivityHeartbeatResponse> {
7010 self.client
7011 .heartbeat_activity_task(
7012 &self.task_id,
7013 &self.activity_attempt_id,
7014 &self.lease_owner,
7015 details,
7016 )
7017 .await
7018 }
7019}
7020
7021fn decode_task_avro_arguments(value: Option<&Value>, codec: &str) -> Result<AvroValue> {
7022 match value {
7023 Some(value) => Ok(normalize_avro_arguments(decode_wire_avro_value(
7024 value, codec,
7025 )?)),
7026 None => Ok(AvroValue::Array(Vec::new())),
7027 }
7028}
7029
7030fn decode_resume_signal(task: &WorkflowTask) -> Result<Option<ResumeSignal>> {
7031 let Some(signal_name) = task
7032 .signal_name
7033 .as_deref()
7034 .filter(|value| !value.is_empty())
7035 else {
7036 return Ok(None);
7037 };
7038 let Some(arguments) = task.signal_arguments.as_ref() else {
7039 return Ok(None);
7040 };
7041
7042 let decoded = normalize_avro_arguments(decode_wire_avro_value(arguments, &task.payload_codec)?);
7043 let AvroValue::Array(arguments) = decoded else {
7044 unreachable!("normalize_avro_arguments always returns an array");
7045 };
7046
7047 Ok(Some(ResumeSignal {
7048 signal_name: signal_name.to_string(),
7049 arguments,
7050 }))
7051}
7052
7053fn recorded_commands(
7054 events: &[HistoryEvent],
7055 fallback_codec: &str,
7056 parent: WorkflowIdentity,
7057) -> Result<Vec<RecordedCommand>> {
7058 let mut events_by_sequence: BTreeMap<u64, Vec<&HistoryEvent>> = BTreeMap::new();
7059 let mut last_new_sequence = None;
7060
7061 for event in events {
7062 let is_activity = matches!(
7063 event.event_type.as_str(),
7064 "ActivityScheduled"
7065 | "ActivityStarted"
7066 | "ActivityHeartbeatRecorded"
7067 | "ActivityRetryScheduled"
7068 | "ActivityCompleted"
7069 | "ActivityFailed"
7070 | "ActivityCancelled"
7071 | "ActivityTimedOut"
7072 );
7073 let is_workflow_timer = matches!(
7074 event.event_type.as_str(),
7075 "TimerScheduled" | "TimerCancelled" | "TimerFired"
7076 ) && !is_internal_timer_event(event);
7077 let is_child_workflow = matches!(
7078 event.event_type.as_str(),
7079 "ChildWorkflowScheduled"
7080 | "ChildRunCompleted"
7081 | "ChildRunFailed"
7082 | "ChildRunCancelled"
7083 | "ChildRunTerminated"
7084 );
7085 let is_signal_wait = is_recorded_signal_wait_event(event);
7086 let is_side_effect = event.event_type == "SideEffectRecorded";
7087 let is_version_marker = event.event_type == "VersionMarkerRecorded";
7088 if !is_activity
7089 && !is_workflow_timer
7090 && !is_child_workflow
7091 && !is_signal_wait
7092 && !is_side_effect
7093 && !is_version_marker
7094 {
7095 continue;
7096 }
7097
7098 let sequence = durable_event_sequence(event).ok_or_else(|| {
7099 Error::NonDeterministicReplay(ReplayFailure::new(
7100 "durable_command_sequence_missing",
7101 None,
7102 Some("positive workflow sequence".to_string()),
7103 Some(event.event_type.clone()),
7104 "durable command history event has no workflow sequence",
7105 ))
7106 })?;
7107 if sequence == 0 {
7108 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
7109 "durable_command_sequence_invalid",
7110 Some(sequence),
7111 Some("positive workflow sequence".to_string()),
7112 Some(sequence.to_string()),
7113 "durable command history uses an invalid workflow sequence",
7114 )));
7115 }
7116 if !events_by_sequence.contains_key(&sequence) {
7117 if let Some(previous) = last_new_sequence {
7118 if sequence < previous {
7119 return Err(invalid_recorded_history(
7120 "durable_command_sequence_mismatch",
7121 sequence,
7122 &format!("workflow sequence greater than {previous}"),
7123 &sequence.to_string(),
7124 "durable commands are not strictly ordered by their recorded workflow sequence",
7125 ));
7126 }
7127 }
7128 last_new_sequence = Some(sequence);
7129 }
7130 events_by_sequence.entry(sequence).or_default().push(event);
7131 }
7132
7133 let commands: Vec<RecordedCommand> = events_by_sequence
7134 .into_iter()
7135 .map(|(sequence, sequence_events)| {
7136 let activity_events: Vec<_> = sequence_events
7137 .iter()
7138 .copied()
7139 .filter(|event| event.event_type.starts_with("Activity"))
7140 .collect();
7141 let timer_events: Vec<_> = sequence_events
7142 .iter()
7143 .copied()
7144 .filter(|event| event.event_type.starts_with("Timer"))
7145 .collect();
7146 let child_events: Vec<_> = sequence_events
7147 .iter()
7148 .copied()
7149 .filter(|event| {
7150 event.event_type == "ChildWorkflowScheduled"
7151 || event.event_type.starts_with("ChildRun")
7152 })
7153 .collect();
7154 let signal_wait_events: Vec<_> = sequence_events
7155 .iter()
7156 .copied()
7157 .filter(|event| is_recorded_signal_wait_event(event))
7158 .collect();
7159 let side_effect_events: Vec<_> = sequence_events
7160 .iter()
7161 .copied()
7162 .filter(|event| event.event_type == "SideEffectRecorded")
7163 .collect();
7164 let version_marker_events: Vec<_> = sequence_events
7165 .iter()
7166 .copied()
7167 .filter(|event| event.event_type == "VersionMarkerRecorded")
7168 .collect();
7169
7170 let command_kind_count = usize::from(!activity_events.is_empty())
7171 + usize::from(!timer_events.is_empty())
7172 + usize::from(!child_events.is_empty())
7173 + usize::from(!signal_wait_events.is_empty())
7174 + usize::from(!side_effect_events.is_empty())
7175 + usize::from(!version_marker_events.is_empty());
7176 if command_kind_count > 1 {
7177 let actual = [
7178 (!activity_events.is_empty()).then_some("activity"),
7179 (!timer_events.is_empty()).then_some("timer"),
7180 (!child_events.is_empty()).then_some("child workflow"),
7181 (!signal_wait_events.is_empty()).then_some("signal wait"),
7182 (!side_effect_events.is_empty()).then_some("side effect"),
7183 (!version_marker_events.is_empty()).then_some("version marker"),
7184 ]
7185 .into_iter()
7186 .flatten()
7187 .collect::<Vec<_>>()
7188 .join(" and ");
7189 return Err(invalid_recorded_history(
7190 "durable_command_sequence_collision",
7191 sequence,
7192 "one durable command kind",
7193 &actual,
7194 "one workflow sequence records more than one durable command kind",
7195 ));
7196 }
7197
7198 if !activity_events.is_empty() {
7199 let scheduled_count = activity_events
7200 .iter()
7201 .filter(|event| event.event_type == "ActivityScheduled")
7202 .count();
7203 if scheduled_count > 1 {
7204 return Err(invalid_recorded_history(
7205 "duplicate_activity_schedule",
7206 sequence,
7207 "at most one ActivityScheduled event",
7208 "multiple ActivityScheduled events",
7209 "activity history schedules more than one command at one workflow sequence",
7210 ));
7211 }
7212 let activity_type = activity_events.iter().find_map(|event| {
7213 event
7214 .payload
7215 .get("activity_type")
7216 .or_else(|| event.payload.get("activity_name"))
7217 .and_then(Value::as_str)
7218 .map(str::to_string)
7219 });
7220 if activity_events.iter().filter_map(|event| {
7221 event
7222 .payload
7223 .get("activity_type")
7224 .or_else(|| event.payload.get("activity_name"))
7225 .and_then(Value::as_str)
7226 }).any(|candidate| Some(candidate) != activity_type.as_deref()) {
7227 return Err(invalid_recorded_history(
7228 "activity_identity_mismatch",
7229 sequence,
7230 activity_type.as_deref().unwrap_or("one activity identity"),
7231 "conflicting activity identities",
7232 "activity lifecycle events at one workflow sequence disagree on identity",
7233 ));
7234 }
7235 let terminal: Vec<_> = activity_events
7236 .iter()
7237 .copied()
7238 .filter(|event| {
7239 matches!(
7240 event.event_type.as_str(),
7241 "ActivityCompleted"
7242 | "ActivityFailed"
7243 | "ActivityCancelled"
7244 | "ActivityTimedOut"
7245 )
7246 })
7247 .collect();
7248 if terminal.len() > 1 {
7249 return Err(invalid_recorded_history(
7250 "duplicate_activity_terminal_event",
7251 sequence,
7252 "at most one terminal activity event",
7253 "multiple terminal activity events",
7254 "activity history settles one command more than once",
7255 ));
7256 }
7257 let outcome = terminal
7258 .first()
7259 .map(|event| activity_outcome(event, fallback_codec, activity_type.clone()))
7260 .transpose()?;
7261 let options = activity_events
7262 .iter()
7263 .find(|event| event.event_type == "ActivityScheduled")
7264 .and_then(|event| event.payload.get("activity"))
7265 .and_then(Value::as_object)
7266 .map(|activity| RecordedActivityOptions {
7267 task_queue: recorded_optional_string(activity, "queue"),
7268 execution_mode: recorded_optional_string(activity, "execution_mode"),
7269 retry_policy: recorded_activity_retry_snapshot(
7270 activity.get("retry_policy"),
7271 ),
7272 });
7273 return Ok(RecordedCommand::Activity {
7274 sequence,
7275 activity_type,
7276 options,
7277 outcome,
7278 });
7279 }
7280
7281 if !child_events.is_empty() {
7282 let scheduled: Vec<_> = child_events
7283 .iter()
7284 .copied()
7285 .filter(|event| event.event_type == "ChildWorkflowScheduled")
7286 .collect();
7287 if scheduled.len() != 1 {
7288 return Err(invalid_recorded_history(
7289 "child_workflow_schedule_missing_or_duplicate",
7290 sequence,
7291 "one ChildWorkflowScheduled event",
7292 &format!("{} ChildWorkflowScheduled events", scheduled.len()),
7293 "child workflow replay requires exactly one recorded schedule event",
7294 ));
7295 }
7296 let workflow_type = child_events.iter().find_map(|event| {
7297 event
7298 .payload
7299 .get("child_workflow_type")
7300 .or_else(|| event.payload.get("workflow_type"))
7301 .and_then(Value::as_str)
7302 .filter(|value| !value.is_empty())
7303 .map(str::to_string)
7304 });
7305 if child_events
7306 .iter()
7307 .filter_map(|event| {
7308 event
7309 .payload
7310 .get("child_workflow_type")
7311 .or_else(|| event.payload.get("workflow_type"))
7312 .and_then(Value::as_str)
7313 })
7314 .any(|candidate| Some(candidate) != workflow_type.as_deref())
7315 {
7316 return Err(invalid_recorded_history(
7317 "child_workflow_identity_mismatch",
7318 sequence,
7319 workflow_type
7320 .as_deref()
7321 .unwrap_or("one child workflow type"),
7322 "conflicting child workflow types",
7323 "child workflow lifecycle events at one sequence disagree on type",
7324 ));
7325 }
7326 let mut outcomes = child_workflow_outcomes(
7327 &child_events.iter().map(|event| (*event).clone()).collect::<Vec<_>>(),
7328 fallback_codec,
7329 parent.clone(),
7330 )?;
7331 if outcomes.len() > 1 {
7332 return Err(invalid_recorded_history(
7333 "duplicate_child_workflow_terminal_event",
7334 sequence,
7335 "at most one terminal child event",
7336 "multiple terminal child events",
7337 "child workflow history settles one command more than once",
7338 ));
7339 }
7340 return Ok(RecordedCommand::ChildWorkflow {
7341 sequence,
7342 workflow_type,
7343 outcome: outcomes.pop(),
7344 });
7345 }
7346
7347 if !signal_wait_events.is_empty() {
7348 let opened: Vec<_> = signal_wait_events
7349 .iter()
7350 .copied()
7351 .filter(|event| event.event_type == "SignalWaitOpened")
7352 .collect();
7353 if opened.len() != 1 {
7354 return Err(invalid_recorded_history(
7355 "signal_wait_open_missing_or_duplicate",
7356 sequence,
7357 "one SignalWaitOpened event",
7358 &format!("{} SignalWaitOpened events", opened.len()),
7359 "signal replay requires exactly one canonical wait-open event",
7360 ));
7361 }
7362
7363 let applied: Vec<_> = signal_wait_events
7364 .iter()
7365 .copied()
7366 .filter(|event| event.event_type == "SignalApplied")
7367 .collect();
7368 if applied.len() > 1 {
7369 return Err(invalid_recorded_history(
7370 "duplicate_signal_wait_apply",
7371 sequence,
7372 "at most one SignalApplied event",
7373 "multiple SignalApplied events",
7374 "signal history applies one durable wait more than once",
7375 ));
7376 }
7377
7378 let signal_names = signal_wait_events
7379 .iter()
7380 .map(|event| required_signal_wait_name(event, sequence))
7381 .collect::<Result<Vec<_>>>()?;
7382 let signal_name = signal_names
7383 .first()
7384 .expect("signal wait events are not empty")
7385 .clone();
7386 if signal_names.iter().any(|candidate| candidate != &signal_name) {
7387 return Err(invalid_recorded_history(
7388 "signal_wait_identity_mismatch",
7389 sequence,
7390 &signal_name,
7391 "conflicting signal names",
7392 "signal wait lifecycle events at one workflow sequence disagree on identity",
7393 ));
7394 }
7395 let value = applied
7396 .first()
7397 .map(|event| decode_signal_event_arguments(event, fallback_codec))
7398 .transpose()?;
7399 return Ok(RecordedCommand::SignalWait {
7400 sequence,
7401 signal_name,
7402 value,
7403 });
7404 }
7405
7406 if !side_effect_events.is_empty() {
7407 if side_effect_events.len() != 1 {
7408 return Err(invalid_recorded_history(
7409 "duplicate_side_effect_record",
7410 sequence,
7411 "one SideEffectRecorded event",
7412 &format!("{} SideEffectRecorded events", side_effect_events.len()),
7413 "side-effect history records one workflow command more than once",
7414 ));
7415 }
7416 let event = side_effect_events[0];
7417 let result = event.payload.get("result").ok_or_else(|| {
7418 invalid_recorded_history(
7419 "side_effect_result_missing",
7420 sequence,
7421 "recorded result payload",
7422 "missing result",
7423 "side-effect history is missing its recorded value",
7424 )
7425 })?;
7426 let has_published_envelope = result.as_str().is_some()
7427 || result.as_object().is_some_and(|envelope| {
7428 envelope.get("codec").and_then(Value::as_str).is_some()
7429 && envelope.get("blob").and_then(Value::as_str).is_some()
7430 });
7431 if !has_published_envelope {
7432 return Err(invalid_recorded_history(
7433 "side_effect_payload_malformed",
7434 sequence,
7435 "payload blob or {codec, blob} envelope",
7436 &result.to_string(),
7437 "side-effect history result does not use a published payload envelope",
7438 ));
7439 }
7440 let codec = event
7441 .payload
7442 .get("payload_codec")
7443 .and_then(Value::as_str)
7444 .unwrap_or(fallback_codec);
7445 let value = decode_wire_avro_value(result, codec).map_err(|error| {
7446 invalid_recorded_history(
7447 "side_effect_payload_incompatible",
7448 sequence,
7449 &format!("valid {codec} payload envelope"),
7450 &error.to_string(),
7451 "side-effect history payload cannot be decoded with its recorded codec",
7452 )
7453 })?;
7454 return Ok(RecordedCommand::SideEffect { sequence, value });
7455 }
7456
7457 if !version_marker_events.is_empty() {
7458 if version_marker_events.len() != 1 {
7459 return Err(invalid_recorded_history(
7460 "duplicate_version_marker_record",
7461 sequence,
7462 "one VersionMarkerRecorded event",
7463 &format!("{} VersionMarkerRecorded events", version_marker_events.len()),
7464 "version-marker history records one workflow command more than once",
7465 ));
7466 }
7467 let payload = &version_marker_events[0].payload;
7468 let change_id = payload
7469 .get("change_id")
7470 .and_then(Value::as_str)
7471 .filter(|value| !value.is_empty())
7472 .map(str::to_string)
7473 .ok_or_else(|| {
7474 invalid_recorded_history(
7475 "version_marker_field_missing",
7476 sequence,
7477 "non-empty change_id",
7478 "missing or invalid change_id",
7479 "version-marker history is missing its stable change ID",
7480 )
7481 })?;
7482 let version = required_version_i32(payload, "version", sequence)?;
7483 let min_supported = required_version_i32(payload, "min_supported", sequence)?;
7484 let max_supported = required_version_i32(payload, "max_supported", sequence)?;
7485 if min_supported > max_supported || version < min_supported || version > max_supported {
7486 return Err(invalid_recorded_history(
7487 "version_marker_history_range_invalid",
7488 sequence,
7489 "min_supported <= version <= max_supported",
7490 &format!("{min_supported} <= {version} <= {max_supported}"),
7491 "recorded version marker contains an internally incompatible range",
7492 ));
7493 }
7494 return Ok(RecordedCommand::VersionMarker {
7495 sequence,
7496 change_id,
7497 version,
7498 });
7499 }
7500
7501 let scheduled: Vec<_> = timer_events
7502 .iter()
7503 .copied()
7504 .filter(|event| event.event_type == "TimerScheduled")
7505 .collect();
7506 let fired: Vec<_> = timer_events
7507 .iter()
7508 .copied()
7509 .filter(|event| event.event_type == "TimerFired")
7510 .collect();
7511 if scheduled.len() != 1 {
7512 return Err(invalid_recorded_history(
7513 "timer_schedule_missing_or_duplicate",
7514 sequence,
7515 "one TimerScheduled event",
7516 &format!("{} TimerScheduled events", scheduled.len()),
7517 "timer replay requires exactly one recorded schedule event",
7518 ));
7519 }
7520 if fired.len() > 1 {
7521 return Err(invalid_recorded_history(
7522 "duplicate_timer_fire",
7523 sequence,
7524 "at most one TimerFired event",
7525 "multiple TimerFired events",
7526 "timer history contains more than one fire event for a workflow sequence",
7527 ));
7528 }
7529
7530 let scheduled = scheduled[0];
7531 let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
7532 let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
7533 if let Some(fired) = fired.first() {
7534 let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
7535 if fired_timer_id != timer_id {
7536 return Err(invalid_recorded_history(
7537 "timer_identity_mismatch",
7538 sequence,
7539 &timer_id,
7540 &fired_timer_id,
7541 "TimerFired does not correspond to the recorded TimerScheduled event",
7542 ));
7543 }
7544 let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
7545 if fired_delay != delay_seconds {
7546 return Err(invalid_recorded_history(
7547 "timer_history_delay_mismatch",
7548 sequence,
7549 &delay_seconds.to_string(),
7550 &fired_delay.to_string(),
7551 "TimerScheduled and TimerFired record different delays",
7552 ));
7553 }
7554 }
7555
7556 Ok(RecordedCommand::Timer {
7557 sequence,
7558 delay_seconds,
7559 fired: !fired.is_empty(),
7560 })
7561 })
7562 .collect::<Result<_>>()?;
7563
7564 let mut marker_sequences = HashMap::new();
7565 for command in &commands {
7566 if let RecordedCommand::VersionMarker {
7567 sequence,
7568 change_id,
7569 ..
7570 } = command
7571 {
7572 if let Some(first_sequence) = marker_sequences.insert(change_id.clone(), *sequence) {
7573 return Err(invalid_recorded_history(
7574 "duplicate_version_marker",
7575 *sequence,
7576 &format!("one marker for change ID {change_id:?}"),
7577 &format!("markers at sequences {first_sequence} and {sequence}"),
7578 "workflow history contains duplicate markers for one stable change ID",
7579 ));
7580 }
7581 }
7582 }
7583
7584 Ok(commands)
7585}
7586
7587fn required_version_i32(payload: &Value, field: &str, sequence: u64) -> Result<i32> {
7588 payload
7589 .get(field)
7590 .and_then(Value::as_i64)
7591 .and_then(|value| i32::try_from(value).ok())
7592 .ok_or_else(|| {
7593 invalid_recorded_history(
7594 "version_marker_field_missing",
7595 sequence,
7596 &format!("integer {field}"),
7597 "missing or out-of-range integer",
7598 "version-marker history is missing a required integer field",
7599 )
7600 })
7601}
7602
7603fn durable_event_sequence(event: &HistoryEvent) -> Option<u64> {
7604 event
7605 .payload
7606 .get("sequence")
7607 .or_else(|| event.payload.get("workflow_sequence"))
7608 .or_else(|| event.raw.get("sequence"))
7609 .or_else(|| event.raw.get("workflow_sequence"))
7610 .and_then(value_as_u64)
7611}
7612
7613fn is_internal_timer_event(event: &HistoryEvent) -> bool {
7614 matches!(
7615 event
7616 .payload
7617 .get("timer_kind")
7618 .or_else(|| event.raw.get("timer_kind"))
7619 .and_then(Value::as_str),
7620 Some("condition_timeout" | "signal_timeout")
7621 )
7622}
7623
7624fn required_signal_wait_name(event: &HistoryEvent, sequence: u64) -> Result<String> {
7625 event
7626 .payload
7627 .get("signal_name")
7628 .or_else(|| event.raw.get("signal_name"))
7629 .and_then(Value::as_str)
7630 .filter(|value| !value.is_empty())
7631 .map(str::to_string)
7632 .ok_or_else(|| {
7633 invalid_recorded_history(
7634 "signal_wait_name_missing",
7635 sequence,
7636 "non-empty signal_name",
7637 &event.event_type,
7638 "canonical signal-wait history is missing its signal identity",
7639 )
7640 })
7641}
7642
7643fn is_recorded_signal_wait_event(event: &HistoryEvent) -> bool {
7644 matches!(
7645 event.event_type.as_str(),
7646 "SignalWaitOpened" | "SignalApplied"
7647 )
7648}
7649
7650fn required_history_string(event: &HistoryEvent, field: &str, sequence: u64) -> Result<String> {
7651 event
7652 .payload
7653 .get(field)
7654 .and_then(Value::as_str)
7655 .filter(|value| !value.is_empty())
7656 .map(str::to_string)
7657 .ok_or_else(|| {
7658 invalid_recorded_history(
7659 "timer_history_field_missing",
7660 sequence,
7661 field,
7662 &event.event_type,
7663 "timer history is missing a required identity field",
7664 )
7665 })
7666}
7667
7668fn required_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<u64> {
7669 event
7670 .payload
7671 .get(field)
7672 .and_then(value_as_u64)
7673 .ok_or_else(|| {
7674 invalid_recorded_history(
7675 "timer_history_field_missing",
7676 sequence,
7677 field,
7678 &event.event_type,
7679 "timer history is missing a required numeric field",
7680 )
7681 })
7682}
7683
7684fn invalid_recorded_history(
7685 reason: &str,
7686 sequence: u64,
7687 expected: &str,
7688 actual: &str,
7689 message: &str,
7690) -> Error {
7691 Error::NonDeterministicReplay(ReplayFailure::new(
7692 reason,
7693 Some(sequence),
7694 Some(expected.to_string()),
7695 Some(actual.to_string()),
7696 message,
7697 ))
7698}
7699
7700type ActivityOutcome = std::result::Result<AvroValue, ActivityFailure>;
7701
7702fn activity_outcome(
7703 event: &HistoryEvent,
7704 fallback_codec: &str,
7705 recorded_activity_type: Option<String>,
7706) -> Result<ActivityOutcome> {
7707 if event.event_type == "ActivityCompleted" {
7708 let codec = event
7709 .payload
7710 .get("payload_codec")
7711 .and_then(Value::as_str)
7712 .unwrap_or(fallback_codec);
7713 return Ok(Ok(decode_wire_avro_value(
7714 event.payload.get("result").unwrap_or(&Value::Null),
7715 codec,
7716 )?));
7717 }
7718
7719 let payload = &event.payload;
7720 let (kind, fallback_reason, fallback_message) = match event.event_type.as_str() {
7721 "ActivityFailed" => (ActivityFailureKind::Failed, "activity", "activity failed"),
7722 "ActivityCancelled" => (
7723 ActivityFailureKind::Cancelled,
7724 "cancelled",
7725 "activity was cancelled",
7726 ),
7727 "ActivityTimedOut" => (
7728 ActivityFailureKind::TimedOut,
7729 "timeout",
7730 "activity timed out",
7731 ),
7732 _ => unreachable!("activity_outcome is called only for terminal activity events"),
7733 };
7734 let exception = payload
7735 .get("exception")
7736 .filter(|value| !value.is_null())
7737 .cloned();
7738 let failure_category = payload_string(payload, "failure_category");
7739 let timeout_kind = payload_string(payload, "timeout_kind");
7740 let reason = payload_string(payload, "reason").unwrap_or_else(|| match kind {
7741 ActivityFailureKind::Failed => failure_category
7742 .clone()
7743 .unwrap_or_else(|| fallback_reason.to_string()),
7744 ActivityFailureKind::Cancelled => fallback_reason.to_string(),
7745 ActivityFailureKind::TimedOut => timeout_kind
7746 .clone()
7747 .unwrap_or_else(|| fallback_reason.to_string()),
7748 });
7749 let message = payload_string(payload, "message")
7750 .or_else(|| {
7751 exception
7752 .as_ref()
7753 .and_then(|value| payload_string(value, "message"))
7754 })
7755 .unwrap_or_else(|| fallback_message.to_string());
7756
7757 Ok(Err(ActivityFailure {
7758 kind,
7759 reason,
7760 message,
7761 activity_execution_id: payload_string(payload, "activity_execution_id"),
7762 activity_attempt_id: payload_string(payload, "activity_attempt_id"),
7763 activity_type: payload_string(payload, "activity_type")
7764 .or_else(|| payload_string(payload, "activity_name"))
7765 .or(recorded_activity_type),
7766 activity_class: payload_string(payload, "activity_class"),
7767 attempt_number: payload.get("attempt_number").and_then(value_as_u64),
7768 failure_id: payload_string(payload, "failure_id"),
7769 failure_category,
7770 timeout_kind,
7771 non_retryable: payload
7772 .get("non_retryable")
7773 .and_then(Value::as_bool)
7774 .unwrap_or(false),
7775 exception_type: payload_string(payload, "exception_type").or_else(|| {
7776 exception
7777 .as_ref()
7778 .and_then(|value| payload_string(value, "type"))
7779 }),
7780 exception_class: payload_string(payload, "exception_class").or_else(|| {
7781 exception
7782 .as_ref()
7783 .and_then(|value| payload_string(value, "class"))
7784 }),
7785 code: payload
7786 .get("code")
7787 .filter(|value| !value.is_null())
7788 .cloned(),
7789 exception,
7790 }))
7791}
7792
7793type ChildWorkflowOutcome = std::result::Result<ChildWorkflowAvroResult, ChildWorkflowFailure>;
7794
7795fn child_workflow_outcomes(
7796 events: &[HistoryEvent],
7797 fallback_codec: &str,
7798 parent: WorkflowIdentity,
7799) -> Result<Vec<ChildWorkflowOutcome>> {
7800 let mut outcomes = Vec::new();
7801
7802 for event in events {
7803 let kind = match event.event_type.as_str() {
7804 "ChildRunCompleted" => None,
7805 "ChildRunFailed" => Some((
7806 ChildWorkflowFailureKind::Failed,
7807 "child_workflow",
7808 "child workflow failed",
7809 )),
7810 "ChildRunCancelled" => Some((
7811 ChildWorkflowFailureKind::Cancelled,
7812 "cancelled",
7813 "child workflow was cancelled",
7814 )),
7815 "ChildRunTerminated" => Some((
7816 ChildWorkflowFailureKind::Terminated,
7817 "terminated",
7818 "child workflow was terminated",
7819 )),
7820 _ => continue,
7821 };
7822 let payload = &event.payload;
7823 let child_workflow_id = payload_string(payload, "child_workflow_instance_id");
7824 let child_workflow_run_id = payload_string(payload, "child_workflow_run_id");
7825 let child_workflow_type = payload_string(payload, "child_workflow_type");
7826
7827 if let Some((kind, reason, fallback_message)) = kind {
7828 let exception = payload
7829 .get("exception")
7830 .filter(|value| !value.is_null())
7831 .cloned();
7832 let message = payload_string(payload, "message")
7833 .or_else(|| {
7834 exception
7835 .as_ref()
7836 .and_then(|value| payload_string(value, "message"))
7837 })
7838 .unwrap_or_else(|| fallback_message.to_string());
7839 let exception_type = payload_string(payload, "exception_type").or_else(|| {
7840 exception
7841 .as_ref()
7842 .and_then(|value| payload_string(value, "type"))
7843 });
7844 let exception_class = payload_string(payload, "exception_class").or_else(|| {
7845 exception
7846 .as_ref()
7847 .and_then(|value| payload_string(value, "class"))
7848 });
7849 outcomes.push(Err(ChildWorkflowFailure {
7850 kind,
7851 reason: reason.to_string(),
7852 message,
7853 parent_workflow_id: parent.workflow_id.clone(),
7854 parent_workflow_run_id: parent.run_id.clone(),
7855 child_workflow_id,
7856 child_workflow_run_id,
7857 child_workflow_type,
7858 failure_id: payload_string(payload, "failure_id"),
7859 failure_category: payload_string(payload, "failure_category"),
7860 exception_type,
7861 exception_class,
7862 non_retryable: payload
7863 .get("non_retryable")
7864 .and_then(Value::as_bool)
7865 .unwrap_or(false),
7866 code: payload
7867 .get("code")
7868 .filter(|value| !value.is_null())
7869 .cloned(),
7870 exception,
7871 }));
7872 continue;
7873 }
7874
7875 let codec = payload
7876 .get("payload_codec")
7877 .and_then(Value::as_str)
7878 .unwrap_or(fallback_codec);
7879 let result = payload
7880 .get("result")
7881 .or_else(|| payload.get("output"))
7882 .unwrap_or(&Value::Null);
7883 outcomes.push(Ok(ChildWorkflowAvroResult {
7884 parent: parent.clone(),
7885 child: WorkflowIdentity {
7886 workflow_id: child_workflow_id,
7887 run_id: child_workflow_run_id,
7888 },
7889 child_workflow_type,
7890 result: decode_wire_avro_value(result, codec)?,
7891 }));
7892 }
7893
7894 Ok(outcomes)
7895}
7896
7897fn payload_string(payload: &Value, key: &str) -> Option<String> {
7898 payload
7899 .get(key)
7900 .and_then(Value::as_str)
7901 .filter(|value| !value.is_empty())
7902 .map(str::to_string)
7903}
7904
7905fn workflow_failure_command(error: &Error) -> Value {
7906 let (exception_type, exception_class, properties) = match error {
7907 Error::ActivityFailed(failure) => (
7908 match failure.kind {
7909 ActivityFailureKind::Failed => "ActivityFailed",
7910 ActivityFailureKind::Cancelled => "ActivityCancelled",
7911 ActivityFailureKind::TimedOut => "ActivityTimedOut",
7912 },
7913 "durable_workflow::ActivityFailure",
7914 json!({
7915 "reason": failure.reason,
7916 "activity_execution_id": failure.activity_execution_id,
7917 "activity_attempt_id": failure.activity_attempt_id,
7918 "activity_type": failure.activity_type,
7919 "activity_class": failure.activity_class,
7920 "attempt_number": failure.attempt_number,
7921 "failure_id": failure.failure_id,
7922 "failure_category": failure.failure_category,
7923 "timeout_kind": failure.timeout_kind,
7924 "activity_non_retryable": failure.non_retryable,
7925 "activity_exception_type": failure.exception_type,
7926 "activity_exception_class": failure.exception_class,
7927 "activity_code": failure.code,
7928 "activity_exception": failure.exception,
7929 }),
7930 ),
7931 Error::ChildWorkflowFailed(failure) => (
7932 match failure.kind {
7933 ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
7934 ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
7935 ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
7936 },
7937 "durable_workflow::ChildWorkflowFailure",
7938 json!({
7939 "reason": failure.reason,
7940 "parent_workflow_id": failure.parent_workflow_id,
7941 "parent_workflow_run_id": failure.parent_workflow_run_id,
7942 "child_workflow_id": failure.child_workflow_id,
7943 "child_workflow_run_id": failure.child_workflow_run_id,
7944 "child_workflow_type": failure.child_workflow_type,
7945 "failure_id": failure.failure_id,
7946 "failure_category": failure.failure_category,
7947 "child_exception_type": failure.exception_type,
7948 "child_exception_class": failure.exception_class,
7949 "child_non_retryable": failure.non_retryable,
7950 "child_code": failure.code,
7951 "child_exception": failure.exception,
7952 }),
7953 ),
7954 Error::NonDeterministicReplay(_) => (
7955 "NonDeterministicReplay",
7956 "durable_workflow::Error",
7957 Value::Null,
7958 ),
7959 _ => ("RustWorkflowError", "durable_workflow::Error", Value::Null),
7960 };
7961 let non_retryable = match error {
7962 Error::ActivityFailed(failure) => failure.non_retryable,
7963 Error::ChildWorkflowFailed(failure) => failure.non_retryable,
7964 Error::NonDeterministicReplay(_) => true,
7965 _ => false,
7966 };
7967
7968 json!({
7969 "type": "fail_workflow",
7970 "message": error.to_string(),
7971 "exception_type": exception_type,
7972 "exception_class": exception_class,
7973 "non_retryable": non_retryable,
7974 "exception": {
7975 "type": exception_type,
7976 "class": exception_class,
7977 "message": error.to_string(),
7978 "properties": properties,
7979 }
7980 })
7981}
7982
7983fn workflow_task_integrity_error(error: &Error) -> bool {
7984 matches!(
7985 error,
7986 Error::NonDeterministicReplay(_) | Error::Protocol(_) | Error::WorkflowStatePoisoned
7987 )
7988}
7989
7990fn decode_signal_event_arguments(
7991 event: &HistoryEvent,
7992 fallback_codec: &str,
7993) -> Result<Vec<AvroValue>> {
7994 let codec = event
7995 .payload
7996 .get("payload_codec")
7997 .and_then(Value::as_str)
7998 .unwrap_or(fallback_codec);
7999 let raw = event
8000 .payload
8001 .get("value")
8002 .or_else(|| event.payload.get("input"))
8003 .or_else(|| event.payload.get("arguments"));
8004 let decoded = match raw.filter(|value| !value.is_null()) {
8005 Some(value) => decode_wire_avro_value(value, codec)?,
8006 None => AvroValue::Array(Vec::new()),
8007 };
8008 let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
8009 unreachable!("normalize_avro_arguments always returns an array");
8010 };
8011 Ok(arguments)
8012}
8013
8014fn hydrate_query_history_from_export(task: &mut QueryTask) -> Result<()> {
8015 let Some(export_events) = task
8016 .history_export
8017 .as_ref()
8018 .and_then(|export| export.get("history_events"))
8019 .and_then(Value::as_array)
8020 else {
8021 return Ok(());
8022 };
8023
8024 if export_events.len() > task.history_events.len() {
8025 task.history_events = serde_json::from_value(Value::Array(export_events.clone()))?;
8026 }
8027
8028 Ok(())
8029}
8030
8031fn enrich_query_history_from_export(task: &mut QueryTask) -> Result<()> {
8032 let Some(export) = task.history_export.as_ref() else {
8033 return Ok(());
8034 };
8035 let signals = export
8036 .get("signals")
8037 .and_then(Value::as_array)
8038 .cloned()
8039 .unwrap_or_default();
8040 let activities = export
8041 .get("activities")
8042 .and_then(Value::as_array)
8043 .cloned()
8044 .unwrap_or_default();
8045 let export_codec = export
8046 .get("payloads")
8047 .and_then(|payloads| payloads.get("codec"))
8048 .and_then(Value::as_str)
8049 .unwrap_or(&task.payload_codec)
8050 .to_string();
8051 let mut signal_name_offsets: HashMap<String, usize> = HashMap::new();
8052
8053 for event in &mut task.history_events {
8054 if event.event_type == "ActivityCompleted" {
8055 let sequence = event
8056 .payload
8057 .get("sequence")
8058 .or_else(|| event.payload.get("workflow_sequence"))
8059 .and_then(value_as_u64);
8060 let Some(activity) = sequence.and_then(|sequence| {
8061 activities.iter().find(|activity| {
8062 activity.get("sequence").and_then(value_as_u64) == Some(sequence)
8063 })
8064 }) else {
8065 continue;
8066 };
8067 let Some(payload) = event.payload.as_object_mut() else {
8068 continue;
8069 };
8070 if missing_payload(payload.get("result")) {
8071 if let Some(result) = activity
8072 .get("result")
8073 .filter(|value| !missing_payload(Some(value)))
8074 {
8075 payload.insert("result".to_string(), result.clone());
8076 }
8077 }
8078 for field in ["payload_codec", "activity_type"] {
8079 if payload
8080 .get(field)
8081 .and_then(Value::as_str)
8082 .unwrap_or_default()
8083 .is_empty()
8084 {
8085 if let Some(value) = activity.get(field) {
8086 payload.insert(field.to_string(), value.clone());
8087 }
8088 }
8089 }
8090 continue;
8091 }
8092
8093 if event.event_type != "SignalReceived" && event.event_type != "SignalApplied" {
8094 continue;
8095 }
8096 let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
8097 let command_id = event
8098 .payload
8099 .get("workflow_command_id")
8100 .or_else(|| event.raw.get("workflow_command_id"))
8101 .and_then(Value::as_str);
8102 let signal_name = event
8103 .payload
8104 .get("signal_name")
8105 .and_then(Value::as_str)
8106 .unwrap_or_default()
8107 .to_string();
8108 let matched = signals
8109 .iter()
8110 .find(|signal| {
8111 signal_id.is_some() && signal.get("id").and_then(Value::as_str) == signal_id
8112 })
8113 .or_else(|| {
8114 signals.iter().find(|signal| {
8115 command_id.is_some()
8116 && signal.get("command_id").and_then(Value::as_str) == command_id
8117 })
8118 })
8119 .or_else(|| {
8120 let offset = signal_name_offsets.entry(signal_name.clone()).or_default();
8121 let signal = signals
8122 .iter()
8123 .filter(|signal| {
8124 signal.get("name").and_then(Value::as_str) == Some(signal_name.as_str())
8125 })
8126 .nth(*offset);
8127 if signal.is_some() {
8128 *offset += 1;
8129 }
8130 signal
8131 });
8132 let Some(signal) = matched else {
8133 continue;
8134 };
8135 let signal_codec = signal
8136 .get("payload_codec")
8137 .and_then(Value::as_str)
8138 .unwrap_or(&export_codec);
8139 let Some(payload) = event.payload.as_object_mut() else {
8140 continue;
8141 };
8142 if missing_payload(payload.get("arguments")) {
8143 if let Some(arguments) = signal
8144 .get("arguments")
8145 .filter(|value| !missing_payload(Some(value)))
8146 {
8147 let envelope = match arguments {
8148 Value::String(blob) => json!({"codec": signal_codec, "blob": blob}),
8149 other => other.clone(),
8150 };
8151 payload.insert("arguments".to_string(), envelope);
8152 }
8153 }
8154 if payload
8155 .get("payload_codec")
8156 .and_then(Value::as_str)
8157 .unwrap_or_default()
8158 .is_empty()
8159 {
8160 payload.insert("payload_codec".to_string(), json!(signal_codec));
8161 }
8162 }
8163
8164 Ok(())
8165}
8166
8167fn missing_payload(value: Option<&Value>) -> bool {
8168 match value {
8169 None | Some(Value::Null) => true,
8170 Some(Value::String(value)) => value.is_empty(),
8171 Some(_) => false,
8172 }
8173}
8174
8175fn query_signal_events(task: &QueryTask) -> Result<Vec<QuerySignal>> {
8176 let export_signals = task
8177 .history_export
8178 .as_ref()
8179 .and_then(|export| export.get("signals"))
8180 .and_then(Value::as_array)
8181 .cloned()
8182 .unwrap_or_default();
8183 let export_codec = task
8184 .history_export
8185 .as_ref()
8186 .and_then(|export| export.get("payloads"))
8187 .and_then(|payloads| payloads.get("codec"))
8188 .and_then(Value::as_str)
8189 .unwrap_or(&task.payload_codec);
8190 let mut name_offsets: HashMap<String, usize> = HashMap::new();
8191 let mut signals = Vec::new();
8192
8193 for event in &task.history_events {
8194 if event.event_type != "SignalApplied" && event.event_type != "SignalReceived" {
8195 continue;
8196 }
8197
8198 let name = event
8199 .payload
8200 .get("signal_name")
8201 .and_then(Value::as_str)
8202 .unwrap_or_default();
8203 if name.is_empty() {
8204 continue;
8205 }
8206 let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
8207 let command_id = event
8208 .payload
8209 .get("workflow_command_id")
8210 .or_else(|| event.raw.get("workflow_command_id"))
8211 .and_then(Value::as_str);
8212 let matched_export = export_signals
8213 .iter()
8214 .find(|candidate| {
8215 signal_id.is_some() && candidate.get("id").and_then(Value::as_str) == signal_id
8216 })
8217 .or_else(|| {
8218 export_signals.iter().find(|candidate| {
8219 command_id.is_some()
8220 && candidate.get("command_id").and_then(Value::as_str) == command_id
8221 })
8222 })
8223 .or_else(|| {
8224 let offset = name_offsets.entry(name.to_string()).or_default();
8225 let candidate = export_signals
8226 .iter()
8227 .filter(|candidate| candidate.get("name").and_then(Value::as_str) == Some(name))
8228 .nth(*offset);
8229 if candidate.is_some() {
8230 *offset += 1;
8231 }
8232 candidate
8233 });
8234 let codec = event
8235 .payload
8236 .get("payload_codec")
8237 .and_then(Value::as_str)
8238 .or_else(|| {
8239 matched_export
8240 .and_then(|signal| signal.get("payload_codec"))
8241 .and_then(Value::as_str)
8242 })
8243 .unwrap_or(export_codec);
8244 let raw_arguments = event
8245 .payload
8246 .get("value")
8247 .or_else(|| event.payload.get("input"))
8248 .or_else(|| event.payload.get("arguments"))
8249 .filter(|value| !value.is_null())
8250 .or_else(|| matched_export.and_then(|signal| signal.get("arguments")));
8251 let (arguments, avro_arguments) = decode_query_signal_arguments(raw_arguments, codec)?;
8252 let workflow_sequence = event
8253 .payload
8254 .get("workflow_sequence")
8255 .and_then(value_as_u64)
8256 .or_else(|| {
8257 matched_export
8258 .and_then(|signal| signal.get("workflow_sequence"))
8259 .and_then(value_as_u64)
8260 });
8261
8262 signals.push(QuerySignal {
8263 id: signal_id.map(str::to_string).or_else(|| {
8264 matched_export
8265 .and_then(|signal| signal.get("id"))
8266 .and_then(Value::as_str)
8267 .map(str::to_string)
8268 }),
8269 name: name.to_string(),
8270 arguments,
8271 avro_arguments,
8272 workflow_sequence,
8273 });
8274 }
8275
8276 if signals.is_empty() {
8277 for signal in export_signals {
8278 if signal.get("status").and_then(Value::as_str) == Some("rejected") {
8279 continue;
8280 }
8281 let Some(name) = signal.get("name").and_then(Value::as_str) else {
8282 continue;
8283 };
8284 let codec = signal
8285 .get("payload_codec")
8286 .and_then(Value::as_str)
8287 .unwrap_or(export_codec);
8288 let (arguments, avro_arguments) =
8289 decode_query_signal_arguments(signal.get("arguments"), codec)?;
8290 signals.push(QuerySignal {
8291 id: signal.get("id").and_then(Value::as_str).map(str::to_string),
8292 name: name.to_string(),
8293 arguments,
8294 avro_arguments,
8295 workflow_sequence: signal.get("workflow_sequence").and_then(value_as_u64),
8296 });
8297 }
8298 signals.sort_by_key(|signal| signal.workflow_sequence.unwrap_or(u64::MAX));
8299 }
8300
8301 Ok(signals)
8302}
8303
8304fn decode_query_signal_arguments(
8305 raw: Option<&Value>,
8306 codec: &str,
8307) -> Result<(Vec<Value>, Vec<AvroValue>)> {
8308 let decoded = match raw.filter(|value| !value.is_null()) {
8309 Some(value) => decode_wire_avro_value(value, codec)?,
8310 None => AvroValue::Array(Vec::new()),
8311 };
8312 let AvroValue::Array(avro_arguments) = normalize_avro_arguments(decoded) else {
8313 unreachable!("normalize_avro_arguments always returns an array");
8314 };
8315 let arguments = avro_arguments
8316 .iter()
8317 .cloned()
8318 .map(AvroValue::into_json)
8319 .collect::<Result<Vec<_>>>()?;
8320 Ok((arguments, avro_arguments))
8321}
8322
8323fn value_as_u64(value: &Value) -> Option<u64> {
8324 value
8325 .as_u64()
8326 .or_else(|| value.as_str().and_then(|value| value.parse().ok()))
8327}
8328
8329#[cfg(test)]
8330mod tests {
8331 use super::*;
8332 use std::{
8333 io::{Read, Write},
8334 net::{SocketAddr, TcpListener, TcpStream},
8335 sync::atomic::AtomicUsize,
8336 thread,
8337 };
8338
8339 #[test]
8340 fn client_builder_rejects_the_sdk_owned_api_suffix() {
8341 for base_url in [
8342 "http://127.0.0.1:8080/api",
8343 "http://localhost:8080/api/",
8344 "https://runtime.example.test/namespaces/orders/api",
8345 ] {
8346 let error = Client::builder(base_url)
8347 .build()
8348 .expect_err("SDK-owned /api suffix must be rejected during build");
8349
8350 assert!(matches!(error, Error::InvalidBaseUrl), "{base_url}");
8351 assert!(
8352 error.to_string().contains("SDK appends /api automatically"),
8353 "the validation error must explain how to fix the endpoint"
8354 );
8355 }
8356 }
8357
8358 #[test]
8359 fn client_builder_preserves_self_hosted_and_managed_runtime_prefixes() {
8360 for (base_url, expected) in [
8361 ("http://127.0.0.1:8080", "http://127.0.0.1:8080"),
8362 (
8363 "http://localhost:8080/durable-workflow/",
8364 "http://localhost:8080/durable-workflow",
8365 ),
8366 (
8367 "https://runtime.example.test/namespaces/orders",
8368 "https://runtime.example.test/namespaces/orders",
8369 ),
8370 (
8371 "https://runtime.example.test/gateway/api/namespaces/orders",
8372 "https://runtime.example.test/gateway/api/namespaces/orders",
8373 ),
8374 (
8375 "https://api.example.test/runtime/orders/",
8376 "https://api.example.test/runtime/orders",
8377 ),
8378 ] {
8379 let client = Client::builder(base_url)
8380 .build()
8381 .expect("Server and Cloud runtime base URL must remain valid");
8382
8383 assert_eq!(client.base_url, expected);
8384 }
8385 }
8386
8387 fn typed_fidelity_probe() -> AvroValue {
8388 AvroValue::Map(BTreeMap::from([
8389 ("bytes".to_string(), AvroValue::Bytes(vec![0, 0xff])),
8390 ("empty".to_string(), AvroValue::Map(BTreeMap::new())),
8391 (
8392 "numeric".to_string(),
8393 AvroValue::Map(BTreeMap::from([
8394 ("0".to_string(), AvroValue::String("zero".to_string())),
8395 ("1".to_string(), AvroValue::String("one".to_string())),
8396 ])),
8397 ),
8398 (
8399 "nested".to_string(),
8400 AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([(
8401 "enabled".to_string(),
8402 AvroValue::Boolean(true),
8403 )]))]),
8404 ),
8405 (
8406 "projection_collisions".to_string(),
8407 AvroValue::Array(projection_collision_probe()),
8408 ),
8409 ]))
8410 }
8411
8412 fn projection_collision_probe() -> Vec<AvroValue> {
8413 vec![
8414 AvroValue::Map(BTreeMap::from([
8415 ("$type".to_string(), AvroValue::String("bytes".to_string())),
8416 (
8417 "base64".to_string(),
8418 AvroValue::String("ordinary user text".to_string()),
8419 ),
8420 ])),
8421 AvroValue::Map(BTreeMap::from([
8422 ("$type".to_string(), AvroValue::String("map".to_string())),
8423 (
8424 "entries".to_string(),
8425 AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([
8426 ("key".to_string(), AvroValue::String("ordinary".to_string())),
8427 (
8428 "value".to_string(),
8429 AvroValue::String("user map".to_string()),
8430 ),
8431 ]))]),
8432 ),
8433 ])),
8434 ]
8435 }
8436
8437 #[derive(Clone, Debug, Default, PartialEq)]
8438 struct ReplayCounterState {
8439 loaded: Option<String>,
8440 count: i64,
8441 finished: bool,
8442 }
8443
8444 fn replay_counter_worker() -> Worker {
8445 let client = Client::new("http://127.0.0.1:8080").expect("client");
8446 let mut worker = Worker::new(client, "rust-workers");
8447 worker.register_replayed_workflow(
8448 "replay-counter",
8449 ReplayCounterState::default,
8450 |ctx, _input, state| async move {
8451 let loaded = ctx.activity("load-counter", json!([])).await?;
8452 state.update(|current| {
8453 current.loaded = loaded.as_str().map(str::to_string);
8454 })?;
8455 for _ in 0..2 {
8456 let signal = ctx.wait_signal("increment").await?;
8457 let amount = signal.first().and_then(Value::as_i64).unwrap_or_default();
8458 state.update(|current| current.count += amount)?;
8459 }
8460 state.update(|current| current.finished = true)?;
8461 state.read(|current| Ok(json!(current.count)))?
8462 },
8463 );
8464 worker.register_replayed_query::<ReplayCounterState, _, _>(
8465 "replay-counter",
8466 "current",
8467 |_ctx, state, _args| async move {
8468 Ok(json!({
8469 "loaded": state.loaded,
8470 "count": state.count,
8471 "finished": state.finished,
8472 }))
8473 },
8474 );
8475 worker.register_replayed_query::<ReplayCounterState, _, _>(
8476 "replay-counter",
8477 "detached-mutation",
8478 |_ctx, state, _args| async move {
8479 let mut detached = (*state).clone();
8480 detached.count = 999;
8481 Ok(json!(detached.count))
8482 },
8483 );
8484 worker.register_replayed_query::<ReplayCounterState, _, _>(
8485 "replay-counter",
8486 "failed-mutation",
8487 |_ctx, state, _args| async move {
8488 let mut detached = (*state).clone();
8489 detached.count = 999;
8490 Err(Error::WorkerLoop("query refused".to_string()))
8491 },
8492 );
8493 worker
8494 }
8495
8496 fn replay_counter_query(
8497 query_name: &str,
8498 history_events: Value,
8499 run_status: &str,
8500 ) -> QueryTask {
8501 serde_json::from_value(json!({
8502 "query_task_id": format!("query-{query_name}"),
8503 "workflow_type": "replay-counter",
8504 "query_name": query_name,
8505 "payload_codec": "json",
8506 "workflow_arguments": {"codec": "json", "blob": "[]"},
8507 "query_arguments": {"codec": "json", "blob": "[]"},
8508 "history_events": history_events,
8509 "run_status": run_status,
8510 }))
8511 .expect("query task")
8512 }
8513
8514 fn workflow_context(history: Vec<HistoryEvent>) -> WorkflowContext {
8515 workflow_context_with_codec(history, JSON_CODEC)
8516 }
8517
8518 fn workflow_context_with_codec(
8519 history: Vec<HistoryEvent>,
8520 payload_codec: &str,
8521 ) -> WorkflowContext {
8522 WorkflowContext {
8523 state: Arc::new(Mutex::new(
8524 WorkflowState::new_with_identity(
8525 history,
8526 None,
8527 None,
8528 "rust-workers".to_string(),
8529 payload_codec.to_string(),
8530 None,
8531 )
8532 .expect("valid workflow history"),
8533 )),
8534 }
8535 }
8536
8537 fn history_event(event_type: &str, payload: Value) -> HistoryEvent {
8538 HistoryEvent {
8539 event_type: event_type.to_string(),
8540 payload,
8541 raw: HashMap::new(),
8542 }
8543 }
8544
8545 fn workflow_task(
8546 workflow_type: &str,
8547 history_events: Vec<HistoryEvent>,
8548 payload_codec: &str,
8549 ) -> WorkflowTask {
8550 WorkflowTask {
8551 task_id: format!("wft-{workflow_type}"),
8552 workflow_id: Some(format!("wf-{workflow_type}")),
8553 run_id: Some(format!("run-{workflow_type}")),
8554 workflow_type: workflow_type.to_string(),
8555 payload_codec: payload_codec.to_string(),
8556 arguments: Some(
8557 encode_value_envelope(&json!([]), payload_codec).expect("workflow arguments"),
8558 ),
8559 total_history_events: Some(history_events.len() as u64),
8560 history_size_bytes: None,
8561 continue_as_new_recommended: None,
8562 history_budget_pressure: None,
8563 history_events,
8564 next_history_page_token: None,
8565 workflow_task_attempt: 1,
8566 workflow_signal_id: None,
8567 signal_name: None,
8568 signal_arguments: None,
8569 workflow_update_id: None,
8570 update_name: None,
8571 lease_owner: Some("rust-worker".to_string()),
8572 }
8573 }
8574
8575 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
8576 struct SideEffectProbe {
8577 request_id: String,
8578 attempt: u32,
8579 }
8580
8581 #[test]
8582 fn typed_side_effect_runs_callback_once_and_replay_skips_it() {
8583 let calls = AtomicUsize::new(0);
8584 let ctx = workflow_context(Vec::new());
8585 let value = ctx
8586 .side_effect(|| {
8587 calls.fetch_add(1, Ordering::SeqCst);
8588 SideEffectProbe {
8589 request_id: "request-42".to_string(),
8590 attempt: 3,
8591 }
8592 })
8593 .expect("first side effect");
8594 assert_eq!(value.attempt, 3);
8595 assert_eq!(calls.load(Ordering::SeqCst), 1);
8596 let commands = ctx.take_commands().expect("commands");
8597 assert_eq!(commands.len(), 1);
8598 assert_eq!(commands[0]["type"], "record_side_effect");
8599 assert_eq!(
8600 decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("JSON result"),
8601 serde_json::to_value(&value).expect("value")
8602 );
8603
8604 let replay = workflow_context(vec![history_event(
8605 "SideEffectRecorded",
8606 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
8607 )]);
8608 let replayed: SideEffectProbe = replay
8609 .side_effect(|| {
8610 calls.fetch_add(1, Ordering::SeqCst);
8611 panic!("committed side-effect callbacks must not run during replay")
8612 })
8613 .expect("replayed side effect");
8614 assert_eq!(replayed, value);
8615 assert_eq!(calls.load(Ordering::SeqCst), 1);
8616 assert!(replay.take_commands().expect("commands").is_empty());
8617 replay.ensure_history_consumed().expect("history consumed");
8618 }
8619
8620 #[test]
8621 fn side_effect_uses_avro_envelope_and_uuid_is_replay_stable() {
8622 let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
8623 let value = ctx
8624 .side_effect(|| SideEffectProbe {
8625 request_id: "avro-request".to_string(),
8626 attempt: 1,
8627 })
8628 .expect("Avro side effect");
8629 let uuid = ctx.uuid_v4().expect("deterministic UUID");
8630 let commands = ctx.take_commands().expect("commands");
8631 assert_eq!(commands.len(), 2);
8632 assert_eq!(commands[0]["result"]["codec"], DEFAULT_CODEC);
8633 assert_eq!(commands[1]["result"]["codec"], DEFAULT_CODEC);
8634 assert_eq!(
8635 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
8636 serde_json::to_value(&value).expect("value")
8637 );
8638
8639 let replay = workflow_context_with_codec(
8640 vec![
8641 history_event(
8642 "SideEffectRecorded",
8643 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
8644 ),
8645 history_event(
8646 "SideEffectRecorded",
8647 json!({"sequence": 2, "result": commands[1]["result"].clone()}),
8648 ),
8649 ],
8650 DEFAULT_CODEC,
8651 );
8652 let replayed: SideEffectProbe = replay
8653 .side_effect(|| panic!("Avro callback must not run"))
8654 .expect("replayed Avro value");
8655 let replayed_uuid = replay.uuid_v4().expect("replayed UUID");
8656 assert_eq!(replayed, value);
8657 assert_eq!(replayed_uuid, uuid);
8658 assert!(replay.take_commands().expect("commands").is_empty());
8659 }
8660
8661 #[test]
8662 fn typed_side_effect_replay_preserves_bytes_and_maps() {
8663 let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
8664 let value = ctx
8665 .side_effect_avro_value(typed_fidelity_probe)
8666 .expect("typed side effect");
8667 let commands = ctx.take_commands().expect("side-effect command");
8668 assert_eq!(
8669 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
8670 .expect("recorded side effect"),
8671 value
8672 );
8673
8674 let replay = workflow_context_with_codec(
8675 vec![history_event(
8676 "SideEffectRecorded",
8677 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
8678 )],
8679 DEFAULT_CODEC,
8680 );
8681 assert_eq!(
8682 replay
8683 .side_effect_avro_value(|| panic!("replay must not invoke callback"))
8684 .expect("replayed typed side effect"),
8685 value
8686 );
8687 }
8688
8689 #[test]
8690 fn ordered_side_effects_share_the_durable_command_stream() {
8691 let first = encode_value_envelope(&json!("first"), JSON_CODEC).expect("first");
8692 let second = encode_value_envelope(&json!(29), JSON_CODEC).expect("second");
8693 let ctx = workflow_context(vec![
8694 history_event(
8695 "SideEffectRecorded",
8696 json!({"sequence": 1, "result": first}),
8697 ),
8698 history_event(
8699 "SideEffectRecorded",
8700 json!({"sequence": 2, "result": second}),
8701 ),
8702 ]);
8703 let first: String = ctx
8704 .side_effect(|| panic!("first callback must not run"))
8705 .expect("first replay");
8706 let second: i32 = ctx
8707 .side_effect(|| panic!("second callback must not run"))
8708 .expect("second replay");
8709 assert_eq!(first, "first");
8710 assert_eq!(second, 29);
8711 ctx.ensure_history_consumed().expect("ordered history");
8712
8713 let reordered = workflow_context(vec![history_event(
8714 "VersionMarkerRecorded",
8715 json!({
8716 "sequence": 1,
8717 "change_id": "before-side-effect",
8718 "version": 1,
8719 "min_supported": 1,
8720 "max_supported": 1,
8721 }),
8722 )]);
8723 let error = reordered
8724 .side_effect(|| "new".to_string())
8725 .expect_err("command reordering must fail");
8726 assert!(matches!(
8727 error,
8728 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8729 if reason == "recorded_command_mismatch"
8730 ));
8731 }
8732
8733 #[test]
8734 fn version_markers_replay_across_upgrades_and_do_not_duplicate() {
8735 let ctx = workflow_context(Vec::new());
8736 assert_eq!(ctx.get_version("checkout-v2", 1, 2).expect("version"), 2);
8737 assert_eq!(ctx.get_version("checkout-v2", 1, 3).expect("cached"), 2);
8738 assert!(ctx.patched("new-search").expect("patch"));
8739 ctx.deprecate_patch("new-search").expect("deprecate patch");
8740 let commands = ctx.take_commands().expect("commands");
8741 assert_eq!(commands.len(), 2);
8742 assert_eq!(commands[0]["type"], "record_version_marker");
8743 assert_eq!(commands[0]["version"], 2);
8744 assert_eq!(commands[1]["change_id"], "new-search");
8745
8746 let replay = workflow_context(vec![history_event(
8747 "VersionMarkerRecorded",
8748 json!({
8749 "sequence": 1,
8750 "change_id": "checkout-v2",
8751 "version": 2,
8752 "min_supported": 1,
8753 "max_supported": 2,
8754 }),
8755 )]);
8756 assert_eq!(replay.get_version("checkout-v2", 1, 4).expect("upgrade"), 2);
8757 assert_eq!(replay.get_version("checkout-v2", 2, 5).expect("repeat"), 2);
8758 assert!(replay.take_commands().expect("commands").is_empty());
8759 replay.ensure_history_consumed().expect("history consumed");
8760 }
8761
8762 #[test]
8763 fn version_markers_reject_incompatible_or_malformed_history() {
8764 let incompatible = workflow_context(vec![history_event(
8765 "VersionMarkerRecorded",
8766 json!({
8767 "sequence": 1,
8768 "change_id": "checkout-v2",
8769 "version": 1,
8770 "min_supported": 1,
8771 "max_supported": 2,
8772 }),
8773 )]);
8774 let error = incompatible
8775 .get_version("checkout-v2", 2, 3)
8776 .expect_err("old version is unsupported");
8777 assert!(matches!(
8778 error,
8779 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8780 if reason == "version_marker_incompatible_range"
8781 ));
8782
8783 for (history, reason) in [
8784 (
8785 vec![history_event("SideEffectRecorded", json!({"sequence": 1}))],
8786 "side_effect_result_missing",
8787 ),
8788 (
8789 vec![history_event(
8790 "SideEffectRecorded",
8791 json!({
8792 "sequence": 1,
8793 "result": {"codec": "avro", "blob": "not-base64"},
8794 }),
8795 )],
8796 "side_effect_payload_incompatible",
8797 ),
8798 (
8799 vec![history_event(
8800 "SideEffectRecorded",
8801 json!({"sequence": 1, "result": {"unwrapped": true}}),
8802 )],
8803 "side_effect_payload_malformed",
8804 ),
8805 (
8806 vec![history_event(
8807 "VersionMarkerRecorded",
8808 json!({
8809 "sequence": 1,
8810 "change_id": "change",
8811 "version": 1,
8812 "min_supported": 2,
8813 "max_supported": 1,
8814 }),
8815 )],
8816 "version_marker_history_range_invalid",
8817 ),
8818 ] {
8819 let error = WorkflowState::new(
8820 history,
8821 "rust-workers".to_string(),
8822 JSON_CODEC.to_string(),
8823 None,
8824 )
8825 .expect_err("malformed history must fail");
8826 assert!(matches!(
8827 error,
8828 Error::NonDeterministicReplay(ReplayFailure { reason: actual, .. })
8829 if actual == reason
8830 ));
8831 }
8832 }
8833
8834 #[test]
8835 fn duplicate_side_effects_and_version_markers_are_rejected() {
8836 let duplicate_side_effect = WorkflowState::new(
8837 vec![
8838 history_event(
8839 "SideEffectRecorded",
8840 json!({"sequence": 1, "result": {"codec": "json", "blob": "1"}}),
8841 ),
8842 history_event(
8843 "SideEffectRecorded",
8844 json!({"sequence": 1, "result": {"codec": "json", "blob": "2"}}),
8845 ),
8846 ],
8847 "rust-workers".to_string(),
8848 JSON_CODEC.to_string(),
8849 None,
8850 )
8851 .expect_err("duplicate side effect");
8852 assert!(matches!(
8853 duplicate_side_effect,
8854 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8855 if reason == "duplicate_side_effect_record"
8856 ));
8857
8858 let marker = |sequence| {
8859 history_event(
8860 "VersionMarkerRecorded",
8861 json!({
8862 "sequence": sequence,
8863 "change_id": "same-change",
8864 "version": 1,
8865 "min_supported": 1,
8866 "max_supported": 1,
8867 }),
8868 )
8869 };
8870 let duplicate_marker = WorkflowState::new(
8871 vec![marker(1), marker(3)],
8872 "rust-workers".to_string(),
8873 JSON_CODEC.to_string(),
8874 None,
8875 )
8876 .expect_err("duplicate marker");
8877 assert!(matches!(
8878 duplicate_marker,
8879 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8880 if reason == "duplicate_version_marker"
8881 ));
8882 }
8883
8884 #[test]
8885 fn cold_worker_replay_does_not_repeat_committed_side_effects_or_markers() {
8886 fn worker(calls: Arc<AtomicUsize>) -> Worker {
8887 let client = Client::new("http://127.0.0.1:8080").expect("client");
8888 let mut worker = Worker::new(client, "rust-workers");
8889 worker.register_workflow("rust.side-effect-version", move |ctx, _input| {
8890 let calls = Arc::clone(&calls);
8891 async move {
8892 let captured = ctx.side_effect(|| {
8893 calls.fetch_add(1, Ordering::SeqCst);
8894 "captured-once".to_string()
8895 })?;
8896 let version = ctx.get_version("cold-restart", 1, 2)?;
8897 Ok(json!({"captured": captured, "version": version}))
8898 }
8899 });
8900 worker
8901 }
8902
8903 fn task(history_events: Vec<HistoryEvent>) -> WorkflowTask {
8904 WorkflowTask {
8905 task_id: "wft-side-effect-version".to_string(),
8906 workflow_id: Some("wf-side-effect-version".to_string()),
8907 run_id: Some("run-side-effect-version".to_string()),
8908 workflow_type: "rust.side-effect-version".to_string(),
8909 payload_codec: JSON_CODEC.to_string(),
8910 arguments: Some(encode_value_envelope(&json!([]), JSON_CODEC).expect("arguments")),
8911 history_events,
8912 total_history_events: None,
8913 history_size_bytes: None,
8914 continue_as_new_recommended: None,
8915 history_budget_pressure: None,
8916 next_history_page_token: None,
8917 workflow_task_attempt: 1,
8918 workflow_signal_id: None,
8919 signal_name: None,
8920 signal_arguments: None,
8921 workflow_update_id: None,
8922 update_name: None,
8923 lease_owner: Some("rust-worker".to_string()),
8924 }
8925 }
8926
8927 let calls = Arc::new(AtomicUsize::new(0));
8928 let initial = worker(Arc::clone(&calls))
8929 .execute_workflow_task(task(Vec::new()))
8930 .expect("initial execution");
8931 assert_eq!(
8932 initial
8933 .iter()
8934 .map(|command| &command["type"])
8935 .collect::<Vec<_>>(),
8936 vec![
8937 "record_side_effect",
8938 "record_version_marker",
8939 "complete_workflow"
8940 ]
8941 );
8942 assert_eq!(calls.load(Ordering::SeqCst), 1);
8943
8944 let restarted = worker(Arc::clone(&calls));
8945 let replayed = restarted
8946 .execute_workflow_task(task(vec![
8947 history_event(
8948 "SideEffectRecorded",
8949 json!({"sequence": 1, "result": initial[0]["result"].clone()}),
8950 ),
8951 history_event(
8952 "VersionMarkerRecorded",
8953 json!({
8954 "sequence": 2,
8955 "change_id": "cold-restart",
8956 "version": 2,
8957 "min_supported": 1,
8958 "max_supported": 2,
8959 }),
8960 ),
8961 ]))
8962 .expect("cold replay");
8963 assert_eq!(replayed.len(), 1);
8964 assert_eq!(replayed[0]["type"], "complete_workflow");
8965 assert_eq!(calls.load(Ordering::SeqCst), 1);
8966 }
8967
8968 #[test]
8969 fn side_effect_replay_rejects_changed_rust_value_type() {
8970 let result = encode_value_envelope(&json!({"value": 42}), JSON_CODEC).expect("result");
8971 let ctx = workflow_context(vec![history_event(
8972 "SideEffectRecorded",
8973 json!({"sequence": 1, "result": result}),
8974 )]);
8975 let error = ctx
8976 .side_effect::<Vec<String>, _>(|| panic!("callback must not run"))
8977 .expect_err("changed type must fail replay");
8978 assert!(matches!(
8979 error,
8980 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8981 if reason == "side_effect_type_mismatch"
8982 ));
8983 }
8984
8985 fn completed_retry_activity_history() -> Vec<HistoryEvent> {
8986 vec![
8987 history_event(
8988 "ActivityScheduled",
8989 json!({
8990 "sequence": 1,
8991 "activity_type": "flaky",
8992 "activity_execution_id": "act-1",
8993 "activity": {
8994 "id": "act-1",
8995 "sequence": 1,
8996 "type": "flaky",
8997 "queue": "critical-activities",
8998 "execution_mode": null,
8999 "retry_policy": {
9000 "snapshot_version": 1,
9001 "max_attempts": 3,
9002 "backoff_seconds": [2, 4],
9003 "start_to_close_timeout": 30,
9004 "schedule_to_start_timeout": 5,
9005 "schedule_to_close_timeout": 90,
9006 "heartbeat_timeout": 10,
9007 "non_retryable_error_types": ["PermanentError"]
9008 }
9009 }
9010 }),
9011 ),
9012 history_event(
9013 "ActivityStarted",
9014 json!({
9015 "sequence": 1,
9016 "activity_type": "flaky",
9017 "activity_execution_id": "act-1",
9018 "activity_attempt_id": "attempt-1",
9019 "attempt_number": 1
9020 }),
9021 ),
9022 history_event(
9023 "ActivityRetryScheduled",
9024 json!({
9025 "sequence": 1,
9026 "activity_type": "flaky",
9027 "activity_execution_id": "act-1",
9028 "activity_attempt_id": "attempt-1",
9029 "attempt_number": 1,
9030 "retry_after_attempt": 1,
9031 "retry_backoff_seconds": 2,
9032 "failure_category": "activity",
9033 "exception_type": "TransientError"
9034 }),
9035 ),
9036 history_event(
9037 "ActivityStarted",
9038 json!({
9039 "sequence": 1,
9040 "activity_type": "flaky",
9041 "activity_execution_id": "act-1",
9042 "activity_attempt_id": "attempt-2",
9043 "attempt_number": 2
9044 }),
9045 ),
9046 history_event(
9047 "ActivityCompleted",
9048 json!({
9049 "sequence": 1,
9050 "activity_type": "flaky",
9051 "activity_execution_id": "act-1",
9052 "activity_attempt_id": "attempt-2",
9053 "attempt_number": 2,
9054 "payload_codec": "json",
9055 "result": {"codec": "json", "blob": "{\"status\":\"recovered\"}"}
9056 }),
9057 ),
9058 ]
9059 }
9060
9061 fn retry_activity_options() -> ActivityOptions {
9062 ActivityOptions::new()
9063 .task_queue("critical-activities")
9064 .retry_policy(
9065 ActivityRetryPolicy::new(3)
9066 .backoff_intervals([Duration::from_secs(2), Duration::from_secs(4)])
9067 .non_retryable_error_type("PermanentError"),
9068 )
9069 .start_to_close_timeout(Duration::from_secs(30))
9070 .schedule_to_start_timeout(Duration::from_secs(5))
9071 .schedule_to_close_timeout(Duration::from_secs(90))
9072 .heartbeat_timeout(Duration::from_secs(10))
9073 }
9074
9075 #[test]
9076 fn fixed_avro_value_round_trips_json_values() {
9077 let value = json!({"greeting": "hello", "count": 3, "ok": true});
9078 let envelope = PayloadEnvelope::avro(&value).expect("encode");
9079 assert_eq!(envelope.codec, DEFAULT_CODEC);
9080 assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
9081 }
9082
9083 #[tokio::test]
9084 async fn typed_worker_surfaces_preserve_bytes_and_map_list_identity() {
9085 let client = Client::new("http://127.0.0.1:8080").expect("client");
9086 let mut worker = Worker::new(client, "rust-workers");
9087 worker.register_workflow_avro_value("typed.echo", |_ctx, input| async move { Ok(input) });
9088 worker
9089 .register_activity_avro_value("typed.activity", |_ctx, input| async move { Ok(input) });
9090 worker.register_query_avro_value("typed.echo", "inspect", |_ctx, input| async move {
9091 Ok(input)
9092 });
9093 worker.register_update_avro_value("typed.echo", "replace", |_ctx, input| async move {
9094 Ok(input)
9095 });
9096 worker.register_workflow_avro_value("typed.signal", |ctx, _input| async move {
9097 Ok(AvroValue::Array(
9098 ctx.wait_signal_avro_value("changed").await?,
9099 ))
9100 });
9101
9102 let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
9103 let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed envelope");
9104
9105 let mut workflow = workflow_task("typed.echo", Vec::new(), DEFAULT_CODEC);
9106 workflow.arguments = Some(envelope.clone());
9107 let commands = worker
9108 .execute_workflow_task(workflow)
9109 .expect("typed workflow task");
9110 assert_eq!(commands[0]["type"], "complete_workflow");
9111 assert_eq!(
9112 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
9113 .expect("typed workflow result"),
9114 arguments
9115 );
9116
9117 let activity = ActivityTask {
9118 task_id: "activity-typed".to_string(),
9119 activity_attempt_id: Some("attempt-typed".to_string()),
9120 attempt_id: None,
9121 activity_type: "typed.activity".to_string(),
9122 payload_codec: DEFAULT_CODEC.to_string(),
9123 arguments: Some(envelope.clone()),
9124 attempt_number: 1,
9125 lease_owner: Some("rust-worker".to_string()),
9126 };
9127 assert_eq!(
9128 worker
9129 .execute_activity_task(activity)
9130 .await
9131 .expect("typed activity result"),
9132 arguments
9133 );
9134
9135 let query = QueryTask {
9136 query_task_id: "query-typed".to_string(),
9137 query_task_attempt: 1,
9138 lease_owner: Some("rust-worker".to_string()),
9139 workflow_id: Some("typed-1".to_string()),
9140 run_id: Some("run-typed".to_string()),
9141 workflow_type: "typed.echo".to_string(),
9142 query_name: "inspect".to_string(),
9143 payload_codec: DEFAULT_CODEC.to_string(),
9144 workflow_arguments: Some(
9145 encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
9146 .expect("workflow input"),
9147 ),
9148 query_arguments: Some(envelope.clone()),
9149 history_events: Vec::new(),
9150 history_export: None,
9151 run_status: Some("running".to_string()),
9152 };
9153 assert_eq!(
9154 worker
9155 .execute_query_task(query)
9156 .await
9157 .expect("typed query result"),
9158 arguments
9159 );
9160
9161 let mut update = workflow_task(
9162 "typed.echo",
9163 vec![history_event(
9164 "UpdateAccepted",
9165 json!({
9166 "update_id": "update-typed",
9167 "update_name": "replace",
9168 "arguments": envelope.clone(),
9169 }),
9170 )],
9171 DEFAULT_CODEC,
9172 );
9173 update.workflow_update_id = Some("update-typed".to_string());
9174 update.update_name = Some("replace".to_string());
9175 let commands = worker
9176 .execute_workflow_task(update)
9177 .expect("typed update task");
9178 assert_eq!(commands[0]["type"], "complete_update");
9179 assert_eq!(
9180 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
9181 .expect("typed update result"),
9182 arguments
9183 );
9184
9185 let mut signal = workflow_task(
9186 "typed.signal",
9187 vec![history_event(
9188 "SignalReceived",
9189 json!({
9190 "signal_id": "signal-typed",
9191 "signal_name": "changed",
9192 "arguments": envelope.clone(),
9193 }),
9194 )],
9195 DEFAULT_CODEC,
9196 );
9197 signal.workflow_signal_id = Some("signal-typed".to_string());
9198 signal.signal_name = Some("changed".to_string());
9199 signal.signal_arguments = Some(envelope);
9200 let commands = worker
9201 .execute_workflow_task(signal)
9202 .expect("typed signal resume");
9203 assert_eq!(
9204 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
9205 .expect("typed signal result"),
9206 arguments
9207 );
9208 }
9209
9210 #[tokio::test]
9211 async fn typed_helpers_never_parse_json_inspection_projection() {
9212 let collision_values = projection_collision_probe();
9213 let expected = AvroValue::Array(collision_values.clone());
9214 let envelope = encode_typed_envelope(&expected, DEFAULT_CODEC).expect("collision envelope");
9215
9216 let activity_context = workflow_context_with_codec(
9217 vec![history_event(
9218 "ActivityCompleted",
9219 json!({
9220 "sequence": 1,
9221 "activity_type": "collision.activity",
9222 "payload_codec": DEFAULT_CODEC,
9223 "result": envelope.clone(),
9224 }),
9225 )],
9226 DEFAULT_CODEC,
9227 );
9228 assert_eq!(
9229 activity_context
9230 .activity_avro_value("collision.activity", AvroValue::Array(Vec::new()))
9231 .await
9232 .expect("typed activity collision result"),
9233 expected
9234 );
9235
9236 let signal_context = workflow_context_with_codec(
9237 vec![
9238 history_event(
9239 "SignalWaitOpened",
9240 json!({"sequence": 1, "signal_name": "collision"}),
9241 ),
9242 history_event(
9243 "SignalApplied",
9244 json!({
9245 "sequence": 1,
9246 "signal_name": "collision",
9247 "payload_codec": DEFAULT_CODEC,
9248 "value": envelope.clone(),
9249 }),
9250 ),
9251 ],
9252 DEFAULT_CODEC,
9253 );
9254 assert_eq!(
9255 signal_context
9256 .wait_signal_avro_value("collision")
9257 .await
9258 .expect("typed signal collision arguments"),
9259 collision_values
9260 );
9261
9262 let child_context = workflow_context_with_codec(
9263 vec![
9264 history_event(
9265 "ChildWorkflowScheduled",
9266 json!({
9267 "sequence": 1,
9268 "child_workflow_instance_id": "collision-child",
9269 "child_workflow_run_id": "collision-run",
9270 "child_workflow_type": "collision.child",
9271 }),
9272 ),
9273 history_event(
9274 "ChildRunCompleted",
9275 json!({
9276 "sequence": 1,
9277 "child_workflow_instance_id": "collision-child",
9278 "child_workflow_run_id": "collision-run",
9279 "child_workflow_type": "collision.child",
9280 "payload_codec": DEFAULT_CODEC,
9281 "result": envelope,
9282 }),
9283 ),
9284 ],
9285 DEFAULT_CODEC,
9286 );
9287 let child = child_context
9288 .start_child_workflow_avro_value(
9289 "collision.child",
9290 ChildWorkflowOptions::new("collision-workers"),
9291 AvroValue::Array(Vec::new()),
9292 )
9293 .await
9294 .expect("typed child collision result");
9295 assert_eq!(child.result, expected);
9296 }
9297
9298 #[tokio::test]
9299 async fn replayed_typed_query_keeps_lossless_workflow_and_query_inputs() {
9300 let client = Client::new("http://127.0.0.1:8080").expect("client");
9301 let mut worker = Worker::new(client, "rust-workers");
9302 worker.register_replayed_workflow_avro_value(
9303 "typed.replayed",
9304 || (),
9305 |_ctx, input, _state| async move { Ok(input) },
9306 );
9307 worker.register_replayed_query_avro_value::<(), _, _>(
9308 "typed.replayed",
9309 "inspect",
9310 |ctx, _state, args| async move {
9311 let mut signals = ctx.signals_avro_value("collision");
9312 let signal = signals
9313 .pop()
9314 .map(AvroValue::Array)
9315 .unwrap_or_else(|| AvroValue::Array(Vec::new()));
9316 Ok(AvroValue::Array(vec![
9317 ctx.workflow_input_avro_value().clone(),
9318 signal,
9319 args,
9320 ]))
9321 },
9322 );
9323 let arguments = AvroValue::Array(projection_collision_probe());
9324 let signal_arguments =
9325 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed query signal arguments");
9326 let task = QueryTask {
9327 query_task_id: "query-typed-replay".to_string(),
9328 query_task_attempt: 1,
9329 lease_owner: Some("rust-worker".to_string()),
9330 workflow_id: Some("typed-replay".to_string()),
9331 run_id: Some("run-typed-replay".to_string()),
9332 workflow_type: "typed.replayed".to_string(),
9333 query_name: "inspect".to_string(),
9334 payload_codec: DEFAULT_CODEC.to_string(),
9335 workflow_arguments: Some(
9336 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("workflow arguments"),
9337 ),
9338 query_arguments: Some(
9339 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("query arguments"),
9340 ),
9341 history_events: vec![history_event(
9342 "SignalReceived",
9343 json!({
9344 "signal_id": "collision-signal",
9345 "signal_name": "collision",
9346 "workflow_sequence": 1,
9347 "payload_codec": DEFAULT_CODEC,
9348 "arguments": signal_arguments,
9349 }),
9350 )],
9351 history_export: None,
9352 run_status: Some("completed".to_string()),
9353 };
9354
9355 assert_eq!(
9356 worker
9357 .execute_query_task(task)
9358 .await
9359 .expect("typed replay query"),
9360 AvroValue::Array(vec![arguments.clone(), arguments.clone(), arguments])
9361 );
9362 }
9363
9364 #[test]
9365 fn public_avro_adapter_rejects_non_string_map_keys_before_json_conversion() {
9366 let value = BTreeMap::from([(1_i32, "integer key")]);
9367 let error = PayloadEnvelope::avro(&value)
9368 .expect_err("integer map keys must fail")
9369 .to_string();
9370
9371 assert!(error.contains("invalid_map_key"));
9372 }
9373
9374 #[test]
9375 fn json_codec_remains_plain_json() {
9376 let value = json!({"greeting": "hello", "count": 3, "ok": true});
9377 let envelope = PayloadEnvelope::json(&value).expect("encode");
9378
9379 assert_eq!(envelope.codec, JSON_CODEC);
9380 assert_eq!(envelope.blob, serde_json::to_string(&value).expect("json"));
9381 assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
9382 }
9383
9384 #[test]
9385 fn prerelease_avro_payload_without_single_object_frame_is_rejected() {
9386 let envelope = PayloadEnvelope {
9387 codec: DEFAULT_CODEC.to_string(),
9388 blob: BASE64.encode([0x01]),
9389 };
9390
9391 let error = decode_payload::<Value>(&envelope).expect_err("prerelease payload must fail");
9392 assert!(error.to_string().contains("invalid_payload_framing"));
9393 }
9394
9395 #[test]
9396 fn workflow_context_schedules_activity_until_completion_is_in_history() {
9397 let ctx = WorkflowContext {
9398 state: Arc::new(Mutex::new(
9399 WorkflowState::new_with_identity(
9400 Vec::new(),
9401 Some("wf-parent".to_string()),
9402 Some("run-parent".to_string()),
9403 "rust-workers".to_string(),
9404 DEFAULT_CODEC.to_string(),
9405 None,
9406 )
9407 .expect("workflow state"),
9408 )),
9409 };
9410
9411 let mut call = Box::pin(ctx.activity("hello.activity", json!(["Ada"])));
9412 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9413 assert!(matches!(
9414 call.as_mut().poll(&mut task_context),
9415 Poll::Pending
9416 ));
9417
9418 let commands = ctx.take_commands().expect("commands");
9419 assert_eq!(commands[0]["type"], "schedule_activity");
9420 assert_eq!(commands[0]["activity_type"], "hello.activity");
9421 }
9422
9423 #[test]
9424 fn activity_options_encode_retry_policy_queue_and_every_timeout() {
9425 let ctx = workflow_context(Vec::new());
9426 let options = ActivityOptions::new()
9427 .task_queue("payments")
9428 .retry_policy(
9429 ActivityRetryPolicy::new(4)
9430 .exponential_backoff(Duration::from_secs(1), 3, Some(Duration::from_secs(10)))
9431 .non_retryable_error_type("ValidationError"),
9432 )
9433 .start_to_close_timeout(Duration::from_secs(120))
9434 .schedule_to_start_timeout(Duration::from_secs(10))
9435 .schedule_to_close_timeout(Duration::from_secs(300))
9436 .heartbeat_timeout(Duration::from_secs(15));
9437 let mut call = Box::pin(ctx.activity_with_options(
9438 "charge-card",
9439 options,
9440 json!([{"order_id": "o-1"}]),
9441 ));
9442 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9443
9444 assert!(matches!(
9445 call.as_mut().poll(&mut task_context),
9446 Poll::Pending
9447 ));
9448 assert!(matches!(
9449 call.as_mut().poll(&mut task_context),
9450 Poll::Pending
9451 ));
9452
9453 let commands = ctx.take_commands().expect("activity command");
9454 assert_eq!(commands.len(), 1, "one future emits one logical schedule");
9455 assert_eq!(commands[0]["queue"], "payments");
9456 assert_eq!(
9457 commands[0]["retry_policy"],
9458 json!({
9459 "max_attempts": 4,
9460 "backoff_seconds": [1, 3, 9],
9461 "non_retryable_error_types": ["ValidationError"],
9462 })
9463 );
9464 assert_eq!(commands[0]["start_to_close_timeout"], 120);
9465 assert_eq!(commands[0]["schedule_to_start_timeout"], 10);
9466 assert_eq!(commands[0]["schedule_to_close_timeout"], 300);
9467 assert_eq!(commands[0]["heartbeat_timeout"], 15);
9468 }
9469
9470 #[test]
9471 fn activity_options_encode_explicit_and_rounded_backoff_intervals() {
9472 let ctx = workflow_context(Vec::new());
9473 let options = ActivityOptions::new().retry_policy(
9474 ActivityRetryPolicy::new(3)
9475 .backoff_intervals([Duration::from_millis(1), Duration::from_millis(1_001)]),
9476 );
9477 let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
9478 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9479
9480 assert!(matches!(
9481 call.as_mut().poll(&mut task_context),
9482 Poll::Pending
9483 ));
9484 assert_eq!(
9485 ctx.take_commands().expect("command")[0]["retry_policy"]["backoff_seconds"],
9486 json!([1, 2])
9487 );
9488 }
9489
9490 #[test]
9491 fn invalid_activity_options_return_typed_errors_before_emitting_commands() {
9492 let cases = [
9493 (
9494 ActivityOptions::new().task_queue(" "),
9495 ActivityOptionsErrorKind::EmptyTaskQueue,
9496 ),
9497 (
9498 ActivityOptions::new().retry_policy(ActivityRetryPolicy::default()),
9499 ActivityOptionsErrorKind::EmptyRetryPolicy,
9500 ),
9501 (
9502 ActivityOptions::new().retry_policy(ActivityRetryPolicy::new(0)),
9503 ActivityOptionsErrorKind::InvalidMaxAttempts,
9504 ),
9505 (
9506 ActivityOptions::new().retry_policy(ActivityRetryPolicy {
9507 max_attempts: None,
9508 backoff: Some(ActivityBackoff::Explicit(vec![Duration::from_secs(1)])),
9509 non_retryable_error_types: Vec::new(),
9510 }),
9511 ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
9512 ),
9513 (
9514 ActivityOptions::new().retry_policy(
9515 ActivityRetryPolicy::new(2)
9516 .backoff_intervals([Duration::from_secs(1), Duration::from_secs(2)]),
9517 ),
9518 ActivityOptionsErrorKind::TooManyBackoffIntervals,
9519 ),
9520 (
9521 ActivityOptions::new().retry_policy(
9522 ActivityRetryPolicy::new(2).exponential_backoff(
9523 Duration::from_secs(1),
9524 0,
9525 None,
9526 ),
9527 ),
9528 ActivityOptionsErrorKind::InvalidBackoffCoefficient,
9529 ),
9530 (
9531 ActivityOptions::new()
9532 .retry_policy(ActivityRetryPolicy::new(2).non_retryable_error_type(" ")),
9533 ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
9534 ),
9535 (
9536 ActivityOptions::new().retry_policy(
9537 ActivityRetryPolicy::new(10_002).exponential_backoff(
9538 Duration::from_secs(1),
9539 1,
9540 None,
9541 ),
9542 ),
9543 ActivityOptionsErrorKind::BackoffGenerationTooLarge,
9544 ),
9545 (
9546 ActivityOptions::new().retry_policy(
9547 ActivityRetryPolicy::new(2)
9548 .backoff_intervals([Duration::from_secs(i64::MAX as u64 + 1)]),
9549 ),
9550 ActivityOptionsErrorKind::BackoffOverflow,
9551 ),
9552 ];
9553
9554 for (options, expected_kind) in cases {
9555 let ctx = workflow_context(Vec::new());
9556 let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
9557 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9558 let Poll::Ready(Err(Error::InvalidActivityOptions(error))) =
9559 call.as_mut().poll(&mut task_context)
9560 else {
9561 panic!("expected typed activity validation error");
9562 };
9563 assert_eq!(error.kind, expected_kind);
9564 assert!(ctx.take_commands().expect("commands").is_empty());
9565 }
9566 }
9567
9568 #[test]
9569 fn activity_options_validate_positive_and_ordered_timeouts() {
9570 let zero_timeout_cases = [
9571 ActivityOptions::new().start_to_close_timeout(Duration::ZERO),
9572 ActivityOptions::new().schedule_to_start_timeout(Duration::ZERO),
9573 ActivityOptions::new().schedule_to_close_timeout(Duration::ZERO),
9574 ActivityOptions::new().heartbeat_timeout(Duration::ZERO),
9575 ];
9576 for options in zero_timeout_cases {
9577 assert_eq!(
9578 options.validate().expect_err("zero timeout").kind,
9579 ActivityOptionsErrorKind::TimeoutNotPositive
9580 );
9581 }
9582
9583 let ordering_cases = [
9584 ActivityOptions::new()
9585 .heartbeat_timeout(Duration::from_secs(11))
9586 .start_to_close_timeout(Duration::from_secs(10)),
9587 ActivityOptions::new()
9588 .start_to_close_timeout(Duration::from_secs(31))
9589 .schedule_to_close_timeout(Duration::from_secs(30)),
9590 ActivityOptions::new()
9591 .schedule_to_start_timeout(Duration::from_secs(31))
9592 .schedule_to_close_timeout(Duration::from_secs(30)),
9593 ];
9594 for options in ordering_cases {
9595 assert_eq!(
9596 options.validate().expect_err("timeout order").kind,
9597 ActivityOptionsErrorKind::TimeoutOrder
9598 );
9599 }
9600
9601 assert_eq!(
9602 ActivityOptions::new()
9603 .start_to_close_timeout(Duration::from_secs(i64::MAX as u64 + 1))
9604 .validate()
9605 .expect_err("protocol integer overflow")
9606 .kind,
9607 ActivityOptionsErrorKind::TimeoutOverflow
9608 );
9609 }
9610
9611 #[test]
9612 fn replayed_activity_retry_history_completes_without_duplicate_schedule() {
9613 let ctx = workflow_context(completed_retry_activity_history());
9614 let mut call =
9615 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
9616 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9617
9618 assert!(matches!(
9619 call.as_mut().poll(&mut task_context),
9620 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
9621 ));
9622 assert!(ctx.take_commands().expect("commands").is_empty());
9623 ctx.ensure_history_consumed().expect("history consumed");
9624 }
9625
9626 #[test]
9627 fn duplicate_non_retryable_types_use_one_command_and_replay_representation() {
9628 let mut options = retry_activity_options();
9629 options
9630 .retry_policy
9631 .as_mut()
9632 .expect("retry policy")
9633 .non_retryable_error_types
9634 .extend([" PermanentError ".to_string(), "PermanentError".to_string()]);
9635
9636 let new_ctx = workflow_context(Vec::new());
9637 let mut new_call =
9638 Box::pin(new_ctx.activity_with_options("flaky", options.clone(), json!([])));
9639 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9640 assert!(matches!(
9641 new_call.as_mut().poll(&mut task_context),
9642 Poll::Pending
9643 ));
9644 let commands = new_ctx.take_commands().expect("commands");
9645 assert_eq!(commands.len(), 1);
9646 assert_eq!(
9647 commands[0]["retry_policy"]["non_retryable_error_types"],
9648 json!(["PermanentError"])
9649 );
9650
9651 let replay_ctx = workflow_context(completed_retry_activity_history());
9652 let mut replay_call =
9653 Box::pin(replay_ctx.activity_with_options("flaky", options, json!([])));
9654 assert!(matches!(
9655 replay_call.as_mut().poll(&mut task_context),
9656 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
9657 ));
9658 assert!(replay_ctx.take_commands().expect("commands").is_empty());
9659 replay_ctx
9660 .ensure_history_consumed()
9661 .expect("history consumed");
9662 }
9663
9664 #[test]
9665 fn replayed_intermediate_retry_remains_pending_across_restarts() {
9666 let history = completed_retry_activity_history()
9667 .into_iter()
9668 .take(3)
9669 .collect::<Vec<_>>();
9670
9671 for _restart in 0..2 {
9672 let ctx = workflow_context(history.clone());
9673 let mut call =
9674 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
9675 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9676 assert!(matches!(
9677 call.as_mut().poll(&mut task_context),
9678 Poll::Pending
9679 ));
9680 assert!(ctx.take_commands().expect("commands").is_empty());
9681 }
9682 }
9683
9684 #[test]
9685 fn replayed_activity_rejects_changed_queue_retry_and_every_timeout_field() {
9686 let mut changed_queue = retry_activity_options();
9687 changed_queue.task_queue = Some("different-queue".to_string());
9688
9689 let mut changed_max_attempts = retry_activity_options();
9690 let retry_policy = changed_max_attempts
9691 .retry_policy
9692 .as_mut()
9693 .expect("retry policy");
9694 retry_policy.max_attempts = Some(4);
9695
9696 let mut changed_backoff = retry_activity_options();
9697 let retry_policy = changed_backoff.retry_policy.as_mut().expect("retry policy");
9698 retry_policy.backoff = Some(ActivityBackoff::Explicit(vec![
9699 Duration::from_secs(3),
9700 Duration::from_secs(4),
9701 ]));
9702
9703 let mut changed_non_retryable_types = retry_activity_options();
9704 let retry_policy = changed_non_retryable_types
9705 .retry_policy
9706 .as_mut()
9707 .expect("retry policy");
9708 retry_policy.non_retryable_error_types = vec!["AnotherPermanentError".to_string()];
9709
9710 let mut changed_start_to_close = retry_activity_options();
9711 changed_start_to_close.start_to_close_timeout = Some(Duration::from_secs(31));
9712 let mut changed_schedule_to_start = retry_activity_options();
9713 changed_schedule_to_start.schedule_to_start_timeout = Some(Duration::from_secs(6));
9714 let mut changed_schedule_to_close = retry_activity_options();
9715 changed_schedule_to_close.schedule_to_close_timeout = Some(Duration::from_secs(91));
9716 let mut changed_heartbeat = retry_activity_options();
9717 changed_heartbeat.heartbeat_timeout = Some(Duration::from_secs(11));
9718
9719 let cases = [
9720 (changed_queue, "activity_task_queue_mismatch"),
9721 (changed_max_attempts, "activity_retry_policy_mismatch"),
9722 (changed_backoff, "activity_retry_policy_mismatch"),
9723 (
9724 changed_non_retryable_types,
9725 "activity_retry_policy_mismatch",
9726 ),
9727 (changed_start_to_close, "activity_retry_policy_mismatch"),
9728 (changed_schedule_to_start, "activity_retry_policy_mismatch"),
9729 (changed_schedule_to_close, "activity_retry_policy_mismatch"),
9730 (changed_heartbeat, "activity_retry_policy_mismatch"),
9731 ];
9732
9733 for (options, expected_reason) in cases {
9734 let ctx = workflow_context(completed_retry_activity_history());
9735 let mut call = Box::pin(ctx.activity_with_options("flaky", options, json!([])));
9736 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9737 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
9738 call.as_mut().poll(&mut task_context)
9739 else {
9740 panic!("changed activity options must fail replay");
9741 };
9742 assert_eq!(failure.reason, expected_reason);
9743 assert_eq!(failure.sequence, Some(1));
9744 assert!(ctx.take_commands().expect("commands").is_empty());
9745 }
9746 }
9747
9748 #[test]
9749 fn replayed_activity_rejects_changed_execution_mode_and_snapshot_version() {
9750 let cases = [
9751 (
9752 "execution_mode",
9753 json!("local"),
9754 "activity_execution_mode_mismatch",
9755 ),
9756 (
9757 "snapshot_version",
9758 json!(2),
9759 "activity_retry_policy_mismatch",
9760 ),
9761 ];
9762
9763 for (field, value, expected_reason) in cases {
9764 let mut history = completed_retry_activity_history();
9765 let activity = history[0].payload["activity"]
9766 .as_object_mut()
9767 .expect("activity snapshot");
9768 if field == "execution_mode" {
9769 activity.insert(field.to_string(), value);
9770 } else {
9771 activity["retry_policy"]
9772 .as_object_mut()
9773 .expect("retry snapshot")
9774 .insert(field.to_string(), value);
9775 }
9776
9777 let ctx = workflow_context(history);
9778 let mut call =
9779 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
9780 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9781 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
9782 call.as_mut().poll(&mut task_context)
9783 else {
9784 panic!("changed {field} must fail replay");
9785 };
9786 assert_eq!(failure.reason, expected_reason);
9787 assert_eq!(failure.sequence, Some(1));
9788 assert!(ctx.take_commands().expect("commands").is_empty());
9789 }
9790 }
9791
9792 #[test]
9793 fn replayed_legacy_activity_treats_missing_option_snapshot_as_unknown() {
9794 let mut history = completed_retry_activity_history();
9795 let activity = history[0].payload["activity"]
9796 .as_object_mut()
9797 .expect("activity snapshot");
9798 activity.remove("execution_mode");
9799 activity.remove("retry_policy");
9800
9801 let mut current = retry_activity_options();
9802 current.start_to_close_timeout = Some(Duration::from_secs(45));
9803 current.schedule_to_start_timeout = Some(Duration::from_secs(8));
9804 current.schedule_to_close_timeout = Some(Duration::from_secs(120));
9805 current.heartbeat_timeout = Some(Duration::from_secs(12));
9806
9807 let ctx = workflow_context(history);
9808 let mut call = Box::pin(ctx.activity_with_options("flaky", current, json!([])));
9809 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9810 assert!(matches!(
9811 call.as_mut().poll(&mut task_context),
9812 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
9813 ));
9814 assert!(ctx.take_commands().expect("commands").is_empty());
9815 ctx.ensure_history_consumed().expect("history consumed");
9816 }
9817
9818 #[test]
9819 fn terminal_activity_failed_after_start_returns_typed_failure() {
9820 let history = vec![
9821 history_event(
9822 "ActivityScheduled",
9823 json!({
9824 "sequence": 1,
9825 "activity_type": "flaky",
9826 "activity_execution_id": "act-terminal",
9827 "activity": {
9828 "id": "act-terminal",
9829 "sequence": 1,
9830 "type": "flaky",
9831 "queue": "critical-activities",
9832 "retry_policy": {
9833 "snapshot_version": 1,
9834 "max_attempts": 3,
9835 "backoff_seconds": [2, 4],
9836 "non_retryable_error_types": ["PermanentError"]
9837 }
9838 }
9839 }),
9840 ),
9841 history_event(
9842 "ActivityStarted",
9843 json!({
9844 "sequence": 1,
9845 "activity_type": "flaky",
9846 "activity_execution_id": "act-terminal",
9847 "activity_attempt_id": "attempt-1",
9848 "attempt_number": 1
9849 }),
9850 ),
9851 history_event(
9852 "ActivityFailed",
9853 json!({
9854 "sequence": 1,
9855 "activity_type": "flaky",
9856 "activity_execution_id": "act-terminal",
9857 "activity_attempt_id": "attempt-1",
9858 "attempt_number": 1,
9859 "failure_id": "failure-terminal",
9860 "failure_category": "activity",
9861 "exception_type": "PermanentError",
9862 "message": "cannot retry",
9863 "non_retryable": true
9864 }),
9865 ),
9866 ];
9867 let ctx = workflow_context(history);
9868 let mut call =
9869 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
9870 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9871
9872 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
9873 call.as_mut().poll(&mut task_context)
9874 else {
9875 panic!("terminal ActivityFailed must settle the activity future");
9876 };
9877 assert_eq!(failure.kind, ActivityFailureKind::Failed);
9878 assert_eq!(
9879 failure.activity_execution_id.as_deref(),
9880 Some("act-terminal")
9881 );
9882 assert_eq!(failure.exception_type.as_deref(), Some("PermanentError"));
9883 assert!(failure.non_retryable);
9884 assert!(ctx.take_commands().expect("commands").is_empty());
9885 ctx.ensure_history_consumed().expect("history consumed");
9886 }
9887
9888 #[test]
9889 fn activity_terminal_events_return_machine_readable_failures() {
9890 let cases = [
9891 (
9892 "ActivityFailed",
9893 json!({
9894 "sequence": 1,
9895 "activity_type": "charge-card",
9896 "activity_execution_id": "act-1",
9897 "activity_attempt_id": "attempt-2",
9898 "attempt_number": 2,
9899 "failure_id": "failure-1",
9900 "failure_category": "activity",
9901 "exception_type": "PaymentDeclined",
9902 "exception_class": "payments.PaymentDeclined",
9903 "message": "card declined",
9904 "non_retryable": true
9905 }),
9906 ActivityFailureKind::Failed,
9907 "activity",
9908 ),
9909 (
9910 "ActivityCancelled",
9911 json!({
9912 "sequence": 1,
9913 "activity_type": "charge-card",
9914 "activity_execution_id": "act-1",
9915 "activity_attempt_id": "attempt-1"
9916 }),
9917 ActivityFailureKind::Cancelled,
9918 "cancelled",
9919 ),
9920 ];
9921
9922 for (event_type, payload, expected_kind, expected_reason) in cases {
9923 let ctx = workflow_context(vec![history_event(event_type, payload)]);
9924 let mut call = Box::pin(ctx.activity("charge-card", json!([])));
9925 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9926 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
9927 call.as_mut().poll(&mut task_context)
9928 else {
9929 panic!("expected terminal activity failure");
9930 };
9931 assert_eq!(failure.kind, expected_kind);
9932 assert_eq!(failure.reason, expected_reason);
9933 assert_eq!(failure.activity_execution_id.as_deref(), Some("act-1"));
9934 assert_eq!(failure.activity_type.as_deref(), Some("charge-card"));
9935 }
9936 }
9937
9938 #[test]
9939 fn every_activity_timeout_class_is_typed() {
9940 for timeout_kind in [
9941 "start_to_close",
9942 "schedule_to_start",
9943 "schedule_to_close",
9944 "heartbeat",
9945 ] {
9946 let ctx = workflow_context(vec![history_event(
9947 "ActivityTimedOut",
9948 json!({
9949 "sequence": 1,
9950 "activity_type": "slow",
9951 "activity_execution_id": "act-timeout",
9952 "activity_attempt_id": "attempt-timeout",
9953 "failure_category": "timeout",
9954 "timeout_kind": timeout_kind,
9955 "message": "deadline expired"
9956 }),
9957 )]);
9958 let mut call = Box::pin(ctx.activity("slow", json!([])));
9959 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9960 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
9961 call.as_mut().poll(&mut task_context)
9962 else {
9963 panic!("expected timeout failure");
9964 };
9965 assert_eq!(failure.kind, ActivityFailureKind::TimedOut);
9966 assert_eq!(failure.reason, timeout_kind);
9967 assert_eq!(failure.timeout_kind.as_deref(), Some(timeout_kind));
9968 assert_eq!(failure.failure_category.as_deref(), Some("timeout"));
9969 }
9970 }
9971
9972 #[test]
9973 fn workflow_sleep_emits_one_durable_timer_and_rounds_up() {
9974 let ctx = workflow_context(Vec::new());
9975 let mut sleep = Box::pin(ctx.sleep(Duration::from_millis(1_001)));
9976 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9977
9978 assert!(matches!(
9979 sleep.as_mut().poll(&mut task_context),
9980 Poll::Pending
9981 ));
9982 assert!(matches!(
9983 sleep.as_mut().poll(&mut task_context),
9984 Poll::Pending
9985 ));
9986
9987 let commands = ctx.take_commands().expect("timer command");
9988 assert_eq!(
9989 commands,
9990 vec![json!({
9991 "type": "start_timer",
9992 "delay_seconds": 2,
9993 })]
9994 );
9995 }
9996
9997 #[test]
9998 fn workflow_sleep_replays_matching_schedule_and_fire_without_a_command() {
9999 let history = vec![
10000 history_event(
10001 "TimerScheduled",
10002 json!({
10003 "sequence": 1,
10004 "timer_id": "timer-1",
10005 "delay_seconds": 5,
10006 "fire_at": "2026-07-11T12:00:05Z",
10007 }),
10008 ),
10009 history_event(
10010 "TimerFired",
10011 json!({
10012 "sequence": 1,
10013 "timer_id": "timer-1",
10014 "delay_seconds": 5,
10015 "fire_at": "2026-07-11T12:00:05Z",
10016 "fired_at": "2026-07-11T12:00:05Z",
10017 }),
10018 ),
10019 ];
10020
10021 for _restart in 0..2 {
10022 let ctx = workflow_context(history.clone());
10023 let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(5)));
10024 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10025 assert!(matches!(
10026 sleep.as_mut().poll(&mut task_context),
10027 Poll::Ready(Ok(()))
10028 ));
10029 assert!(ctx.take_commands().expect("commands").is_empty());
10030 ctx.ensure_history_consumed().expect("history consumed");
10031 }
10032 }
10033
10034 #[test]
10035 fn workflow_sleep_rejects_changed_delay_during_replay() {
10036 let ctx = workflow_context(vec![
10037 history_event(
10038 "TimerScheduled",
10039 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10040 ),
10041 history_event(
10042 "TimerFired",
10043 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10044 ),
10045 ]);
10046 let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(500)));
10047 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10048
10049 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10050 sleep.as_mut().poll(&mut task_context)
10051 else {
10052 panic!("changed timer delay must be rejected");
10053 };
10054 assert_eq!(failure.reason, "timer_delay_mismatch");
10055 assert_eq!(failure.sequence, Some(1));
10056 }
10057
10058 #[test]
10059 fn workflow_history_rejects_unpaired_or_mismatched_timer_events() {
10060 let lone_fire = WorkflowState::new(
10061 vec![history_event(
10062 "TimerFired",
10063 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10064 )],
10065 "rust-workers".to_string(),
10066 JSON_CODEC.to_string(),
10067 None,
10068 )
10069 .expect_err("TimerFired requires TimerScheduled");
10070 assert!(matches!(
10071 lone_fire,
10072 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10073 if reason == "timer_schedule_missing_or_duplicate"
10074 ));
10075
10076 let wrong_identity = WorkflowState::new(
10077 vec![
10078 history_event(
10079 "TimerScheduled",
10080 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10081 ),
10082 history_event(
10083 "TimerFired",
10084 json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
10085 ),
10086 ],
10087 "rust-workers".to_string(),
10088 JSON_CODEC.to_string(),
10089 None,
10090 )
10091 .expect_err("fire must match scheduled timer identity");
10092 assert!(matches!(
10093 wrong_identity,
10094 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10095 if reason == "timer_identity_mismatch"
10096 ));
10097
10098 let duplicate_fire = WorkflowState::new(
10099 vec![
10100 history_event(
10101 "TimerScheduled",
10102 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10103 ),
10104 history_event(
10105 "TimerFired",
10106 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10107 ),
10108 history_event(
10109 "TimerFired",
10110 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10111 ),
10112 ],
10113 "rust-workers".to_string(),
10114 JSON_CODEC.to_string(),
10115 None,
10116 )
10117 .expect_err("a durable timer cannot fire twice");
10118 assert!(matches!(
10119 duplicate_fire,
10120 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10121 if reason == "duplicate_timer_fire"
10122 ));
10123
10124 let wrong_fired_delay = WorkflowState::new(
10125 vec![
10126 history_event(
10127 "TimerScheduled",
10128 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10129 ),
10130 history_event(
10131 "TimerFired",
10132 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 6}),
10133 ),
10134 ],
10135 "rust-workers".to_string(),
10136 JSON_CODEC.to_string(),
10137 None,
10138 )
10139 .expect_err("timer schedule and fire delays must agree");
10140 assert!(matches!(
10141 wrong_fired_delay,
10142 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10143 if reason == "timer_history_delay_mismatch"
10144 ));
10145 }
10146
10147 #[test]
10148 fn replay_rejects_activity_moved_before_recorded_timer() {
10149 let ctx = workflow_context(vec![
10150 history_event(
10151 "TimerScheduled",
10152 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10153 ),
10154 history_event(
10155 "TimerFired",
10156 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10157 ),
10158 history_event(
10159 "ActivityCompleted",
10160 json!({
10161 "sequence": 2,
10162 "activity_type": "after-timer",
10163 "payload_codec": "json",
10164 "result": {"codec": "json", "blob": "\"done\""},
10165 }),
10166 ),
10167 ]);
10168 let mut activity = Box::pin(ctx.activity("after-timer", json!([])));
10169 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10170
10171 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10172 activity.as_mut().poll(&mut task_context)
10173 else {
10174 panic!("reordered durable command must be rejected");
10175 };
10176 assert_eq!(failure.reason, "recorded_command_mismatch");
10177 assert_eq!(failure.sequence, Some(1));
10178 assert_eq!(failure.expected.as_deref(), Some("timer"));
10179 assert_eq!(failure.actual.as_deref(), Some("activity:after-timer"));
10180 }
10181
10182 #[test]
10183 fn workflow_context_emits_a_typed_named_signal_wait() {
10184 let ctx = workflow_context(Vec::new());
10185 let mut signal = Box::pin(ctx.wait_signal("finish"));
10186 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10187
10188 assert!(matches!(
10189 signal.as_mut().poll(&mut task_context),
10190 Poll::Pending
10191 ));
10192 assert_eq!(
10193 ctx.take_commands().expect("signal-wait command"),
10194 vec![json!({
10195 "type": "open_signal_wait",
10196 "signal_name": "finish",
10197 })]
10198 );
10199 }
10200
10201 #[test]
10202 fn condition_wait_history_does_not_resolve_a_typed_signal_wait() {
10203 let ctx = workflow_context(vec![
10204 history_event(
10205 "ConditionWaitOpened",
10206 json!({"sequence": 1, "condition_key": "signal:finish"}),
10207 ),
10208 history_event(
10209 "ConditionWaitSatisfied",
10210 json!({"sequence": 1, "condition_key": "signal:finish"}),
10211 ),
10212 history_event(
10213 "SignalReceived",
10214 json!({"signal_name": "finish", "arguments": []}),
10215 ),
10216 ]);
10217 let mut signal = Box::pin(ctx.wait_signal("finish"));
10218 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10219
10220 assert!(matches!(
10221 signal.as_mut().poll(&mut task_context),
10222 Poll::Pending
10223 ));
10224 assert_eq!(
10225 ctx.take_commands().expect("typed signal-wait command"),
10226 vec![json!({
10227 "type": "open_signal_wait",
10228 "signal_name": "finish",
10229 })]
10230 );
10231 }
10232
10233 #[test]
10234 fn replay_orders_signal_waits_and_timers_in_one_command_stream() {
10235 let signal_then_timer = vec![
10236 history_event(
10237 "SignalWaitOpened",
10238 json!({"sequence": 1, "signal_name": "go"}),
10239 ),
10240 history_event(
10241 "SignalApplied",
10242 json!({
10243 "sequence": 1,
10244 "signal_name": "go",
10245 "value": {"codec": "json", "blob": "[\"now\"]"},
10246 }),
10247 ),
10248 history_event(
10249 "TimerScheduled",
10250 json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
10251 ),
10252 history_event(
10253 "TimerFired",
10254 json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
10255 ),
10256 ];
10257
10258 let ctx = workflow_context(signal_then_timer.clone());
10259 let mut signal = Box::pin(ctx.wait_signal("go"));
10260 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10261 assert!(matches!(
10262 signal.as_mut().poll(&mut task_context),
10263 Poll::Ready(Ok(arguments)) if arguments == vec![json!("now")]
10264 ));
10265 let mut timer = Box::pin(ctx.sleep(Duration::from_secs(5)));
10266 assert!(matches!(
10267 timer.as_mut().poll(&mut task_context),
10268 Poll::Ready(Ok(()))
10269 ));
10270 ctx.ensure_history_consumed()
10271 .expect("signal and timer history consumed in order");
10272
10273 let reordered = workflow_context(signal_then_timer);
10274 let mut timer_first = Box::pin(reordered.sleep(Duration::from_secs(5)));
10275 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10276 timer_first.as_mut().poll(&mut task_context)
10277 else {
10278 panic!("timer cannot consume signal-wait-first history");
10279 };
10280 assert_eq!(failure.reason, "recorded_command_mismatch");
10281 assert_eq!(failure.sequence, Some(1));
10282 assert_eq!(failure.expected.as_deref(), Some("signal wait"));
10283
10284 let timer_then_signal = vec![
10285 history_event(
10286 "TimerScheduled",
10287 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10288 ),
10289 history_event(
10290 "TimerFired",
10291 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10292 ),
10293 history_event(
10294 "SignalWaitOpened",
10295 json!({"sequence": 2, "signal_name": "go"}),
10296 ),
10297 history_event(
10298 "SignalApplied",
10299 json!({
10300 "sequence": 2,
10301 "signal_name": "go",
10302 "value": {"codec": "json", "blob": "[]"},
10303 }),
10304 ),
10305 ];
10306 let reordered = workflow_context(timer_then_signal);
10307 let mut signal_first = Box::pin(reordered.wait_signal("go"));
10308 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10309 signal_first.as_mut().poll(&mut task_context)
10310 else {
10311 panic!("signal wait cannot consume timer-first history");
10312 };
10313 assert_eq!(failure.reason, "recorded_command_mismatch");
10314 assert_eq!(failure.sequence, Some(1));
10315 assert_eq!(failure.expected.as_deref(), Some("timer"));
10316 }
10317
10318 #[test]
10319 fn workflow_history_rejects_duplicate_or_colliding_command_sequences() {
10320 let duplicate_timer = WorkflowState::new(
10321 vec![
10322 history_event(
10323 "TimerScheduled",
10324 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10325 ),
10326 history_event(
10327 "TimerScheduled",
10328 json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
10329 ),
10330 ],
10331 "rust-workers".to_string(),
10332 JSON_CODEC.to_string(),
10333 None,
10334 )
10335 .expect_err("one workflow sequence cannot schedule two timers");
10336 assert!(matches!(
10337 duplicate_timer,
10338 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10339 if reason == "timer_schedule_missing_or_duplicate"
10340 ));
10341
10342 let colliding_kinds = WorkflowState::new(
10343 vec![
10344 history_event(
10345 "TimerScheduled",
10346 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10347 ),
10348 history_event(
10349 "ActivityCompleted",
10350 json!({"sequence": 1, "activity_type": "same-sequence"}),
10351 ),
10352 ],
10353 "rust-workers".to_string(),
10354 JSON_CODEC.to_string(),
10355 None,
10356 )
10357 .expect_err("one workflow sequence cannot identify two command kinds");
10358 assert!(matches!(
10359 colliding_kinds,
10360 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10361 if reason == "durable_command_sequence_collision"
10362 ));
10363
10364 let duplicate_signal_wait = WorkflowState::new(
10365 vec![
10366 history_event(
10367 "SignalWaitOpened",
10368 json!({"sequence": 1, "signal_name": "go"}),
10369 ),
10370 history_event(
10371 "SignalWaitOpened",
10372 json!({"sequence": 1, "signal_name": "go"}),
10373 ),
10374 ],
10375 "rust-workers".to_string(),
10376 JSON_CODEC.to_string(),
10377 None,
10378 )
10379 .expect_err("one workflow sequence cannot open two signal waits");
10380 assert!(matches!(
10381 duplicate_signal_wait,
10382 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10383 if reason == "signal_wait_open_missing_or_duplicate"
10384 ));
10385 }
10386
10387 #[test]
10388 fn workflow_history_accepts_a_first_command_after_global_sequence_gaps() {
10389 let result = encode_value_envelope(&json!({"captured": true}), JSON_CODEC)
10390 .expect("side-effect result");
10391 let ctx = workflow_context(vec![history_event(
10392 "SideEffectRecorded",
10393 json!({"sequence": 99, "result": result}),
10394 )]);
10395
10396 let replayed: Value = ctx
10397 .side_effect(|| panic!("recorded side effect must not run"))
10398 .expect("positive global workflow sequence is valid");
10399 assert_eq!(replayed, json!({"captured": true}));
10400 ctx.ensure_history_consumed().expect("history consumed");
10401 }
10402
10403 #[test]
10404 fn workflow_history_rejects_zero_and_descending_command_sequences() {
10405 let result =
10406 encode_value_envelope(&json!("captured"), JSON_CODEC).expect("side-effect result");
10407 let zero = WorkflowState::new(
10408 vec![history_event(
10409 "SideEffectRecorded",
10410 json!({"sequence": 0, "result": result.clone()}),
10411 )],
10412 "rust-workers".to_string(),
10413 JSON_CODEC.to_string(),
10414 None,
10415 )
10416 .expect_err("durable command sequences must be positive");
10417 assert!(matches!(
10418 zero,
10419 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10420 if reason == "durable_command_sequence_invalid"
10421 ));
10422
10423 let descending = WorkflowState::new(
10424 vec![
10425 history_event(
10426 "SideEffectRecorded",
10427 json!({"sequence": 3, "result": result}),
10428 ),
10429 history_event(
10430 "VersionMarkerRecorded",
10431 json!({
10432 "sequence": 2,
10433 "change_id": "descending-marker",
10434 "version": 1,
10435 "min_supported": 1,
10436 "max_supported": 1,
10437 }),
10438 ),
10439 ],
10440 "rust-workers".to_string(),
10441 JSON_CODEC.to_string(),
10442 None,
10443 )
10444 .expect_err("new durable commands must remain strictly ordered");
10445 let Error::NonDeterministicReplay(failure) = descending else {
10446 panic!("expected typed replay failure");
10447 };
10448 assert_eq!(failure.reason, "durable_command_sequence_mismatch");
10449 assert_eq!(failure.sequence, Some(2));
10450 assert_eq!(
10451 failure.expected.as_deref(),
10452 Some("workflow sequence greater than 3")
10453 );
10454 assert_eq!(failure.actual.as_deref(), Some("2"));
10455 }
10456
10457 #[test]
10458 fn workflow_task_replay_completes_after_signals_create_sequence_gaps() {
10459 fn worker() -> Worker {
10460 let client = Client::new("http://127.0.0.1:8080").expect("client");
10461 let mut worker = Worker::new(client, "rust-workers");
10462 worker.register_workflow("rust.finish-after-gaps", |ctx, _input| async move {
10463 ctx.wait_signal("finish").await?;
10464 let marker: String =
10465 ctx.side_effect(|| panic!("recorded side effect must not run"))?;
10466 assert_eq!(marker, "after-finish");
10467 Ok(json!("finished"))
10468 });
10469 worker
10470 }
10471
10472 let marker =
10473 encode_value_envelope(&json!("after-finish"), JSON_CODEC).expect("side-effect result");
10474 let task = workflow_task(
10475 "rust.finish-after-gaps",
10476 vec![
10477 history_event(
10478 "SignalWaitOpened",
10479 json!({"sequence": 1, "signal_name": "finish"}),
10480 ),
10481 history_event(
10482 "SignalReceived",
10483 json!({
10484 "signal_id": "increment-3",
10485 "signal_name": "increment",
10486 "workflow_sequence": 2,
10487 "payload_codec": "json",
10488 "arguments": {"codec": "json", "blob": "[3]"},
10489 }),
10490 ),
10491 history_event(
10492 "SignalReceived",
10493 json!({
10494 "signal_id": "increment-5",
10495 "signal_name": "increment",
10496 "workflow_sequence": 3,
10497 "payload_codec": "json",
10498 "arguments": {"codec": "json", "blob": "[5]"},
10499 }),
10500 ),
10501 history_event(
10502 "SignalReceived",
10503 json!({
10504 "signal_id": "finish",
10505 "signal_name": "finish",
10506 "workflow_sequence": 4,
10507 "payload_codec": "json",
10508 "arguments": {"codec": "json", "blob": "[]"},
10509 }),
10510 ),
10511 history_event(
10512 "SignalApplied",
10513 json!({
10514 "sequence": 1,
10515 "signal_id": "finish",
10516 "signal_name": "finish",
10517 "payload_codec": "json",
10518 "value": {"codec": "json", "blob": "[]"},
10519 }),
10520 ),
10521 history_event(
10522 "SideEffectRecorded",
10523 json!({"sequence": 5, "result": marker}),
10524 ),
10525 ],
10526 JSON_CODEC,
10527 );
10528
10529 for _original_or_cold_worker in 0..2 {
10530 let commands = worker()
10531 .execute_workflow_task(task.clone())
10532 .expect("signal gaps preserve deterministic replay");
10533 assert_eq!(commands.len(), 1, "replay emits only terminal completion");
10534 assert_eq!(commands[0]["type"], "complete_workflow");
10535 assert_eq!(
10536 decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("workflow output"),
10537 json!("finished")
10538 );
10539 }
10540 }
10541
10542 #[test]
10543 fn workflow_sleep_rejects_unrepresentable_rounded_duration() {
10544 let ctx = workflow_context(Vec::new());
10545 let mut sleep = Box::pin(ctx.start_timer(Duration::new(u64::MAX, 1)));
10546 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10547 assert!(matches!(
10548 sleep.as_mut().poll(&mut task_context),
10549 Poll::Ready(Err(Error::TimerDurationOverflow))
10550 ));
10551 assert!(ctx.take_commands().expect("commands").is_empty());
10552 }
10553
10554 #[test]
10555 fn workflow_task_replay_completes_without_rescheduling_recorded_commands() {
10556 let client = Client::new("http://127.0.0.1:8080").expect("client");
10557 let mut worker = Worker::new(client, "rust-workers");
10558 worker.register_workflow("rust.timer", |ctx, _input| async move {
10559 ctx.sleep(Duration::from_secs(5)).await?;
10560 ctx.activity("after-timer", json!([])).await
10561 });
10562
10563 let task = |history_events| WorkflowTask {
10564 task_id: "wft-rust-timer-1".to_string(),
10565 workflow_id: Some("wf-rust-timer".to_string()),
10566 run_id: Some("run-rust-timer".to_string()),
10567 workflow_type: "rust.timer".to_string(),
10568 payload_codec: JSON_CODEC.to_string(),
10569 arguments: Some(json!({"codec": "json", "blob": "[]"})),
10570 history_events,
10571 total_history_events: None,
10572 history_size_bytes: None,
10573 continue_as_new_recommended: None,
10574 history_budget_pressure: None,
10575 next_history_page_token: None,
10576 workflow_task_attempt: 1,
10577 workflow_signal_id: None,
10578 signal_name: None,
10579 signal_arguments: None,
10580 workflow_update_id: None,
10581 update_name: None,
10582 lease_owner: Some("rust-worker".to_string()),
10583 };
10584
10585 let initial = worker
10586 .execute_workflow_task(task(Vec::new()))
10587 .expect("initial timer task");
10588 assert_eq!(
10589 initial,
10590 vec![json!({"type": "start_timer", "delay_seconds": 5})]
10591 );
10592
10593 let replayed = worker
10594 .execute_workflow_task(task(vec![
10595 history_event(
10596 "TimerScheduled",
10597 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10598 ),
10599 history_event(
10600 "TimerFired",
10601 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10602 ),
10603 history_event(
10604 "ActivityCompleted",
10605 json!({
10606 "sequence": 2,
10607 "activity_type": "after-timer",
10608 "payload_codec": "json",
10609 "result": {"codec": "json", "blob": "\"done\""},
10610 }),
10611 ),
10612 ]))
10613 .expect("replayed workflow task");
10614 assert_eq!(replayed.len(), 1);
10615 assert_eq!(replayed[0]["type"], "complete_workflow");
10616 assert_eq!(
10617 decode_wire_value(&replayed[0]["result"], JSON_CODEC).expect("result"),
10618 json!("done")
10619 );
10620 }
10621
10622 #[test]
10623 fn workflow_continue_as_new_emits_arguments_type_and_queue_once() {
10624 let client = Client::new("http://127.0.0.1:8080").expect("client");
10625 let mut worker = Worker::new(client, "rust-workers");
10626 worker.register_workflow("rust.continue", |ctx, _input| async move {
10627 ctx.continue_as_new_with_options(
10628 ContinueAsNewOptions::new()
10629 .workflow_type("rust.next")
10630 .task_queue("next-workers"),
10631 json!([2, {"cursor": "next"}]),
10632 )
10633 });
10634
10635 let commands = worker
10636 .execute_workflow_task(workflow_task("rust.continue", Vec::new(), DEFAULT_CODEC))
10637 .expect("continue-as-new command");
10638
10639 assert_eq!(commands.len(), 1);
10640 assert_eq!(commands[0]["type"], "continue_as_new");
10641 assert_eq!(commands[0]["workflow_type"], "rust.next");
10642 assert_eq!(commands[0]["queue"], "next-workers");
10643 assert_eq!(
10644 decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC)
10645 .expect("continue-as-new arguments"),
10646 json!([2, {"cursor": "next"}])
10647 );
10648 }
10649
10650 #[test]
10651 fn continue_as_new_preserves_typed_arguments() {
10652 let client = Client::new("http://127.0.0.1:8080").expect("client");
10653 let mut worker = Worker::new(client, "rust-workers");
10654 worker.register_workflow_avro_value("rust.typed-continue", |ctx, _input| async move {
10655 ctx.continue_as_new(AvroValue::Array(vec![typed_fidelity_probe()]))?;
10656 unreachable!("continue-as-new returns a control-flow error")
10657 });
10658
10659 let commands = worker
10660 .execute_workflow_task(workflow_task(
10661 "rust.typed-continue",
10662 Vec::new(),
10663 DEFAULT_CODEC,
10664 ))
10665 .expect("typed continue-as-new command");
10666
10667 assert_eq!(commands[0]["type"], "continue_as_new");
10668 assert_eq!(
10669 decode_wire_avro_value(&commands[0]["arguments"], DEFAULT_CODEC)
10670 .expect("typed continue arguments"),
10671 AvroValue::Array(vec![typed_fidelity_probe()])
10672 );
10673 }
10674
10675 #[test]
10676 fn recorded_continue_as_new_is_consumed_without_duplicate_successor_command() {
10677 let client = Client::new("http://127.0.0.1:8080").expect("client");
10678 let mut worker = Worker::new(client, "rust-workers");
10679 worker.register_workflow("rust.continue", |ctx, _input| async move {
10680 ctx.continue_as_new(json!([2]))
10681 });
10682 let task = workflow_task(
10683 "rust.continue",
10684 vec![history_event(
10685 "WorkflowContinuedAsNew",
10686 json!({"sequence": 1, "continued_to_run_id": "run-next"}),
10687 )],
10688 JSON_CODEC,
10689 );
10690
10691 for _worker_restart_or_redelivery in 0..2 {
10692 let commands = worker
10693 .execute_workflow_task(task.clone())
10694 .expect("recorded transition replays");
10695 assert!(
10696 commands.is_empty(),
10697 "replay must not emit another successor"
10698 );
10699 }
10700 }
10701
10702 #[test]
10703 fn continue_as_new_rejects_invalid_overrides_before_emitting_a_command() {
10704 let ctx = workflow_context(Vec::new());
10705 let error = ctx
10706 .continue_as_new_with_options(ContinueAsNewOptions::new().task_queue(" "), json!([1]))
10707 .expect_err("blank queue must be rejected");
10708
10709 let Error::InvalidContinueAsNewOptions(error) = error else {
10710 panic!("expected typed continue-as-new validation error");
10711 };
10712 assert_eq!(error.field, "task_queue");
10713 assert!(ctx.take_commands().expect("commands").is_empty());
10714 }
10715
10716 #[test]
10717 fn workflow_context_exposes_server_history_budget() {
10718 let client = Client::new("http://127.0.0.1:8080").expect("client");
10719 let mut worker = Worker::new(client, "rust-workers");
10720 worker.register_workflow("rust.history-budget", |ctx, _input| async move {
10721 let budget = ctx.history_budget()?;
10722 Ok(json!({
10723 "events": budget.event_count,
10724 "bytes": budget.size_bytes,
10725 "recommended": budget.continue_as_new_recommended,
10726 "pressure": budget.pressure,
10727 }))
10728 });
10729 let task: WorkflowTask = serde_json::from_value(json!({
10730 "task_id": "task-history-budget",
10731 "workflow_type": "rust.history-budget",
10732 "payload_codec": JSON_CODEC,
10733 "history_events": [],
10734 "total_history_events": 480,
10735 "history_size_bytes": 1_048_576,
10736 "continue_as_new_recommended": true,
10737 "history_budget_pressure": "continue_as_new_recommended",
10738 }))
10739 .expect("published workflow task");
10740
10741 let commands = worker
10742 .execute_workflow_task(task)
10743 .expect("history-budget workflow");
10744 let result = decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("result");
10745 assert_eq!(result["events"], 480);
10746 assert_eq!(result["bytes"], 1_048_576);
10747 assert_eq!(result["recommended"], true);
10748 assert_eq!(result["pressure"], "continue_as_new_recommended");
10749 }
10750
10751 #[test]
10752 fn uncaught_workflow_handler_error_emits_terminal_failure_command() {
10753 let client = Client::new("http://127.0.0.1:8080").expect("client");
10754 let mut worker = Worker::new(client, "rust-workers");
10755 worker.register_workflow("rust.failing", |_ctx, _input| async move {
10756 Err(Error::Codec("rust_conformance_failure".to_string()))
10757 });
10758 let task = WorkflowTask {
10759 task_id: "wft-rust-failing-1".to_string(),
10760 workflow_id: Some("wf-rust-failing".to_string()),
10761 run_id: Some("run-rust-failing".to_string()),
10762 workflow_type: "rust.failing".to_string(),
10763 payload_codec: JSON_CODEC.to_string(),
10764 arguments: Some(encode_value_envelope(&json!([]), JSON_CODEC).expect("input")),
10765 history_events: Vec::new(),
10766 total_history_events: Some(0),
10767 history_size_bytes: None,
10768 continue_as_new_recommended: None,
10769 history_budget_pressure: None,
10770 next_history_page_token: None,
10771 workflow_task_attempt: 1,
10772 workflow_signal_id: None,
10773 signal_name: None,
10774 signal_arguments: None,
10775 workflow_update_id: None,
10776 update_name: None,
10777 lease_owner: Some("rust-worker".to_string()),
10778 };
10779
10780 let commands = worker
10781 .execute_workflow_task(task)
10782 .expect("handler failure becomes a workflow command");
10783
10784 assert_eq!(commands.len(), 1);
10785 assert_eq!(commands[0]["type"], "fail_workflow");
10786 assert_eq!(commands[0]["exception_type"], "RustWorkflowError");
10787 assert_eq!(commands[0]["exception_class"], "durable_workflow::Error");
10788 assert_eq!(commands[0]["non_retryable"], false);
10789 assert_eq!(
10790 commands[0]["message"],
10791 "codec error: rust_conformance_failure"
10792 );
10793 assert_eq!(
10794 commands[0]["exception"]["message"],
10795 "codec error: rust_conformance_failure"
10796 );
10797 }
10798
10799 #[test]
10800 fn ordinary_handler_error_preserves_commands_queued_in_the_same_decision() {
10801 let client = Client::new("http://127.0.0.1:8080").expect("client");
10802 let mut worker = Worker::new(client, "rust-workers");
10803 worker.register_workflow("rust.failing-after-side-effect", |ctx, _input| async move {
10804 let _: String = ctx.side_effect(|| "captured".to_string())?;
10805 Err(Error::WorkerLoop("application failure".to_string()))
10806 });
10807
10808 let commands = worker
10809 .execute_workflow_task(workflow_task(
10810 "rust.failing-after-side-effect",
10811 Vec::new(),
10812 JSON_CODEC,
10813 ))
10814 .expect("ordinary failure remains a workflow decision");
10815
10816 assert_eq!(commands.len(), 2);
10817 assert_eq!(commands[0]["type"], "record_side_effect");
10818 assert_eq!(commands[1]["type"], "fail_workflow");
10819 }
10820
10821 #[test]
10822 fn handler_error_cannot_hide_an_unconsumed_committed_side_effect() {
10823 let client = Client::new("http://127.0.0.1:8080").expect("client");
10824 let mut worker = Worker::new(client, "rust-workers");
10825 worker.register_workflow("rust.removed-side-effect", |_ctx, _input| async move {
10826 Err(Error::WorkerLoop("application failure".to_string()))
10827 });
10828 let result =
10829 encode_value_envelope(&json!("committed"), JSON_CODEC).expect("side-effect result");
10830
10831 let error = worker
10832 .execute_workflow_task(workflow_task(
10833 "rust.removed-side-effect",
10834 vec![history_event(
10835 "SideEffectRecorded",
10836 json!({"sequence": 1, "result": result}),
10837 )],
10838 JSON_CODEC,
10839 ))
10840 .expect_err("removed committed history must not become fail_workflow");
10841
10842 let Error::NonDeterministicReplay(failure) = error else {
10843 panic!("expected typed replay failure");
10844 };
10845 assert_eq!(failure.reason, "recorded_commands_unconsumed");
10846 assert_eq!(failure.sequence, Some(1));
10847 assert_eq!(failure.expected.as_deref(), Some("side effect"));
10848 }
10849
10850 #[test]
10851 fn replay_error_discards_side_effect_queued_before_incompatible_marker_check() {
10852 let client = Client::new("http://127.0.0.1:8080").expect("client");
10853 let mut worker = Worker::new(client, "rust-workers");
10854 worker.register_workflow(
10855 "rust.side-effect-before-marker-error",
10856 |ctx, _input| async move {
10857 assert_eq!(ctx.get_version("restart-safe", 1, 1)?, 1);
10858 let _: String = ctx.side_effect(|| "must-not-commit".to_string())?;
10859 ctx.get_version("restart-safe", 2, 2)?;
10860 Ok(Value::Null)
10861 },
10862 );
10863
10864 let error = worker
10865 .execute_workflow_task(workflow_task(
10866 "rust.side-effect-before-marker-error",
10867 vec![history_event(
10868 "VersionMarkerRecorded",
10869 json!({
10870 "sequence": 1,
10871 "change_id": "restart-safe",
10872 "version": 1,
10873 "min_supported": 1,
10874 "max_supported": 1,
10875 }),
10876 )],
10877 JSON_CODEC,
10878 ))
10879 .expect_err("replay error must return no queued workflow commands");
10880
10881 let Error::NonDeterministicReplay(failure) = error else {
10882 panic!("expected typed replay failure");
10883 };
10884 assert_eq!(failure.reason, "version_marker_incompatible_range");
10885 assert_eq!(failure.sequence, Some(1));
10886 }
10887
10888 #[test]
10889 fn workflow_task_replay_keeps_recorded_unfired_timer_pending_without_rescheduling() {
10890 let client = Client::new("http://127.0.0.1:8080").expect("client");
10891 let mut worker = Worker::new(client, "rust-workers");
10892 worker.register_workflow("rust.timer.pending", |ctx, _input| async move {
10893 ctx.sleep(Duration::from_secs(5)).await?;
10894 Ok(json!({"status": "timer fired"}))
10895 });
10896
10897 let task = WorkflowTask {
10898 task_id: "wft-rust-timer-pending".to_string(),
10899 workflow_id: Some("wf-rust-timer".to_string()),
10900 run_id: Some("run-rust-timer".to_string()),
10901 workflow_type: "rust.timer.pending".to_string(),
10902 payload_codec: JSON_CODEC.to_string(),
10903 arguments: Some(json!({"codec": "json", "blob": "[]"})),
10904 history_events: vec![history_event(
10905 "TimerScheduled",
10906 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10907 )],
10908 total_history_events: Some(1),
10909 history_size_bytes: None,
10910 continue_as_new_recommended: None,
10911 history_budget_pressure: None,
10912 next_history_page_token: None,
10913 workflow_task_attempt: 1,
10914 workflow_signal_id: None,
10915 signal_name: None,
10916 signal_arguments: None,
10917 workflow_update_id: None,
10918 update_name: None,
10919 lease_owner: Some("rust-worker".to_string()),
10920 };
10921
10922 for _redelivery_or_restart in 0..2 {
10923 let commands = worker
10924 .execute_workflow_task(task.clone())
10925 .expect("recorded timer remains pending");
10926 assert!(
10927 commands.is_empty(),
10928 "recorded timer must not be rescheduled"
10929 );
10930 }
10931 }
10932
10933 #[test]
10934 fn workflow_task_rejects_recorded_command_removed_from_workflow_code() {
10935 let client = Client::new("http://127.0.0.1:8080").expect("client");
10936 let mut worker = Worker::new(client, "rust-workers");
10937 worker.register_workflow("rust.timer.removed", |_ctx, _input| async move {
10938 Ok(json!({"status": "completed"}))
10939 });
10940 let task = WorkflowTask {
10941 task_id: "wft-rust-timer-removed".to_string(),
10942 workflow_id: Some("wf-rust-timer".to_string()),
10943 run_id: Some("run-rust-timer".to_string()),
10944 workflow_type: "rust.timer.removed".to_string(),
10945 payload_codec: JSON_CODEC.to_string(),
10946 arguments: Some(json!({"codec": "json", "blob": "[]"})),
10947 history_events: vec![
10948 history_event(
10949 "TimerScheduled",
10950 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10951 ),
10952 history_event(
10953 "TimerFired",
10954 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10955 ),
10956 ],
10957 total_history_events: Some(2),
10958 history_size_bytes: None,
10959 continue_as_new_recommended: None,
10960 history_budget_pressure: None,
10961 next_history_page_token: None,
10962 workflow_task_attempt: 1,
10963 workflow_signal_id: None,
10964 signal_name: None,
10965 signal_arguments: None,
10966 workflow_update_id: None,
10967 update_name: None,
10968 lease_owner: Some("rust-worker".to_string()),
10969 };
10970
10971 let Error::NonDeterministicReplay(failure) = worker
10972 .execute_workflow_task(task)
10973 .expect_err("removed timer must fail replay")
10974 else {
10975 panic!("expected typed replay failure");
10976 };
10977 assert_eq!(failure.reason, "recorded_commands_unconsumed");
10978 assert_eq!(failure.sequence, Some(1));
10979 }
10980
10981 #[test]
10982 fn workflow_context_emits_explicit_child_workflow_contract() {
10983 let ctx = WorkflowContext {
10984 state: Arc::new(Mutex::new(
10985 WorkflowState::new_with_identity(
10986 Vec::new(),
10987 Some("wf-parent".to_string()),
10988 Some("run-parent".to_string()),
10989 "parent-workers".to_string(),
10990 JSON_CODEC.to_string(),
10991 None,
10992 )
10993 .expect("workflow state"),
10994 )),
10995 };
10996 let options = ChildWorkflowOptions::new("python-workers")
10997 .parent_close_policy(ParentClosePolicy::RequestCancel)
10998 .retry_policy(ChildWorkflowRetryPolicy {
10999 max_attempts: Some(3),
11000 backoff_seconds: vec![1, 5],
11001 non_retryable_error_types: vec!["ValidationError".to_string()],
11002 })
11003 .execution_timeout_seconds(600)
11004 .run_timeout_seconds(120);
11005 let mut call = Box::pin(ctx.start_child_workflow(
11006 "python.fulfil-order",
11007 options,
11008 json!([{"order_id": "order-42"}]),
11009 ));
11010 let mut task_context = TaskContext::from_waker(noop_waker_ref());
11011
11012 assert!(matches!(
11013 call.as_mut().poll(&mut task_context),
11014 Poll::Pending
11015 ));
11016 let commands = ctx.take_commands().expect("commands");
11017 assert_eq!(commands.len(), 1);
11018 let command = &commands[0];
11019 assert_eq!(command["type"], "start_child_workflow");
11020 assert_eq!(command["workflow_type"], "python.fulfil-order");
11021 assert_eq!(command["queue"], "python-workers");
11022 assert_eq!(command["parent_close_policy"], "request_cancel");
11023 assert_eq!(command["retry_policy"]["max_attempts"], 3);
11024 assert_eq!(command["execution_timeout_seconds"], 600);
11025 assert_eq!(command["run_timeout_seconds"], 120);
11026 assert_eq!(
11027 decode_wire_value(&command["arguments"], JSON_CODEC).expect("child args"),
11028 json!([{"order_id": "order-42"}])
11029 );
11030 }
11031
11032 fn child_parent_worker() -> Worker {
11033 let client = Client::new("http://127.0.0.1:8080").expect("client");
11034 let mut worker = Worker::new(client, "rust-parent-workers");
11035 worker.register_workflow("rust.parent", |ctx, _input| async move {
11036 let child = ctx
11037 .start_child_workflow(
11038 "python.child",
11039 ChildWorkflowOptions::new("python-child-workers")
11040 .parent_close_policy(ParentClosePolicy::Terminate),
11041 json!([{"codec_probe": [1, true, "rust"]}]),
11042 )
11043 .await?;
11044 Ok(json!({
11045 "parent_workflow_id": child.parent.workflow_id,
11046 "parent_run_id": child.parent.run_id,
11047 "child_workflow_id": child.child.workflow_id,
11048 "child_run_id": child.child.run_id,
11049 "child_workflow_type": child.child_workflow_type,
11050 "result": child.result,
11051 }))
11052 });
11053 worker
11054 }
11055
11056 fn child_parent_task(event_type: &str, payload: Value) -> WorkflowTask {
11057 WorkflowTask {
11058 task_id: "wft-child-parent".to_string(),
11059 workflow_id: Some("wf-parent".to_string()),
11060 run_id: Some("run-parent".to_string()),
11061 workflow_type: "rust.parent".to_string(),
11062 payload_codec: JSON_CODEC.to_string(),
11063 arguments: Some(encode_value_envelope(&json!([]), JSON_CODEC).expect("input")),
11064 history_events: vec![
11065 HistoryEvent {
11066 event_type: "ChildWorkflowScheduled".to_string(),
11067 payload: json!({
11068 "sequence": 1,
11069 "child_call_id": "call-child",
11070 "child_workflow_instance_id": "wf-child",
11071 "child_workflow_run_id": "run-child",
11072 "child_workflow_type": "python.child",
11073 }),
11074 raw: HashMap::new(),
11075 },
11076 HistoryEvent {
11077 event_type: event_type.to_string(),
11078 payload,
11079 raw: HashMap::new(),
11080 },
11081 ],
11082 total_history_events: Some(2),
11083 history_size_bytes: None,
11084 continue_as_new_recommended: None,
11085 history_budget_pressure: None,
11086 next_history_page_token: None,
11087 workflow_task_attempt: 1,
11088 workflow_signal_id: None,
11089 signal_name: None,
11090 signal_arguments: None,
11091 workflow_update_id: None,
11092 update_name: None,
11093 lease_owner: Some("rust-worker".to_string()),
11094 }
11095 }
11096
11097 #[test]
11098 fn committed_child_result_replays_without_starting_a_duplicate() {
11099 let worker = child_parent_worker();
11100 let task = child_parent_task(
11101 "ChildRunCompleted",
11102 json!({
11103 "sequence": 1,
11104 "child_call_id": "call-child",
11105 "child_workflow_instance_id": "wf-child",
11106 "child_workflow_run_id": "run-child",
11107 "child_workflow_type": "python.child",
11108 "payload_codec": "json",
11109 "result": {"codec": "json", "blob": "{\"from\":\"python\",\"ok\":true}"},
11110 }),
11111 );
11112
11113 for _restart in 0..2 {
11114 let commands = worker
11115 .execute_workflow_task(task.clone())
11116 .expect("replayed parent task");
11117 assert_eq!(commands.len(), 1);
11118 assert_eq!(commands[0]["type"], "complete_workflow");
11119 assert!(!commands
11120 .iter()
11121 .any(|command| command["type"] == "start_child_workflow"));
11122 let output =
11123 decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("parent output");
11124 assert_eq!(output["parent_workflow_id"], "wf-parent");
11125 assert_eq!(output["parent_run_id"], "run-parent");
11126 assert_eq!(output["child_workflow_id"], "wf-child");
11127 assert_eq!(output["child_run_id"], "run-child");
11128 assert_eq!(output["result"], json!({"from": "python", "ok": true}));
11129 }
11130 }
11131
11132 #[test]
11133 fn typed_child_arguments_and_results_survive_replay() {
11134 let client = Client::new("http://127.0.0.1:8080").expect("client");
11135 let mut worker = Worker::new(client, "rust-parent-workers");
11136 worker.register_workflow_avro_value("rust.typed-parent", |ctx, _input| async move {
11137 let child = ctx
11138 .start_child_workflow_avro_value(
11139 "python.typed-child",
11140 ChildWorkflowOptions::new("python-workers"),
11141 AvroValue::Array(vec![typed_fidelity_probe()]),
11142 )
11143 .await?;
11144 Ok(child.result)
11145 });
11146
11147 let initial = worker
11148 .execute_workflow_task(workflow_task(
11149 "rust.typed-parent",
11150 Vec::new(),
11151 DEFAULT_CODEC,
11152 ))
11153 .expect("typed child start");
11154 assert_eq!(initial[0]["type"], "start_child_workflow");
11155 assert_eq!(
11156 decode_wire_avro_value(&initial[0]["arguments"], DEFAULT_CODEC)
11157 .expect("typed child arguments"),
11158 AvroValue::Array(vec![typed_fidelity_probe()])
11159 );
11160
11161 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
11162 .expect("typed child result");
11163 let task = workflow_task(
11164 "rust.typed-parent",
11165 vec![
11166 history_event(
11167 "ChildWorkflowScheduled",
11168 json!({
11169 "sequence": 1,
11170 "child_call_id": "call-typed",
11171 "child_workflow_instance_id": "wf-child",
11172 "child_workflow_run_id": "run-child",
11173 "child_workflow_type": "python.typed-child",
11174 }),
11175 ),
11176 history_event(
11177 "ChildRunCompleted",
11178 json!({
11179 "sequence": 1,
11180 "child_call_id": "call-typed",
11181 "child_workflow_instance_id": "wf-child",
11182 "child_workflow_run_id": "run-child",
11183 "child_workflow_type": "python.typed-child",
11184 "payload_codec": DEFAULT_CODEC,
11185 "result": result,
11186 }),
11187 ),
11188 ],
11189 DEFAULT_CODEC,
11190 );
11191
11192 let commands = worker
11193 .execute_workflow_task(task)
11194 .expect("typed child replay");
11195 assert_eq!(commands[0]["type"], "complete_workflow");
11196 assert_eq!(
11197 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
11198 .expect("typed parent result"),
11199 typed_fidelity_probe()
11200 );
11201 }
11202
11203 #[test]
11204 fn pending_child_replays_after_restart_without_starting_a_duplicate() {
11205 let worker = child_parent_worker();
11206 let mut task = child_parent_task("unused", Value::Null);
11207 task.history_events.truncate(1);
11208 task.total_history_events = Some(1);
11209
11210 for _redelivery_or_restart in 0..2 {
11211 let commands = worker
11212 .execute_workflow_task(task.clone())
11213 .expect("recorded child remains pending");
11214 assert!(
11215 commands.is_empty(),
11216 "recorded pending child must not be started again"
11217 );
11218 }
11219 }
11220
11221 #[test]
11222 fn child_cancellation_becomes_stable_parent_failure_command() {
11223 let worker = child_parent_worker();
11224 let task = child_parent_task(
11225 "ChildRunCancelled",
11226 json!({
11227 "sequence": 1,
11228 "child_workflow_instance_id": "wf-child",
11229 "child_workflow_run_id": "run-child",
11230 "child_workflow_type": "python.child",
11231 "failure_id": "failure-child",
11232 "failure_category": "cancelled",
11233 "message": "cancelled by parent-close policy",
11234 }),
11235 );
11236
11237 let commands = worker
11238 .execute_workflow_task(task)
11239 .expect("parent settlement");
11240 assert_eq!(commands.len(), 1);
11241 assert_eq!(commands[0]["type"], "fail_workflow");
11242 assert_eq!(commands[0]["exception_type"], "ChildWorkflowCancelled");
11243 assert_eq!(
11244 commands[0]["exception"]["properties"]["reason"],
11245 "cancelled"
11246 );
11247 assert_eq!(
11248 commands[0]["exception"]["properties"]["child_workflow_run_id"],
11249 "run-child"
11250 );
11251 }
11252
11253 #[test]
11254 fn workflow_can_handle_typed_child_failure() {
11255 let client = Client::new("http://127.0.0.1:8080").expect("client");
11256 let mut worker = Worker::new(client, "rust-parent-workers");
11257 worker.register_workflow("rust.handled-parent", |ctx, _input| async move {
11258 match ctx
11259 .start_child_workflow(
11260 "python.child",
11261 ChildWorkflowOptions::new("python-child-workers"),
11262 json!([]),
11263 )
11264 .await
11265 {
11266 Err(Error::ChildWorkflowFailed(failure)) => Ok(json!({
11267 "reason": failure.reason,
11268 "failure_id": failure.failure_id,
11269 "exception_class": failure.exception_class,
11270 "child_run_id": failure.child_workflow_run_id,
11271 })),
11272 Err(error) => Err(error),
11273 Ok(_) => Err(Error::WorkerLoop(
11274 "child unexpectedly succeeded".to_string(),
11275 )),
11276 }
11277 });
11278 let mut task = child_parent_task(
11279 "ChildRunFailed",
11280 json!({
11281 "sequence": 1,
11282 "child_workflow_instance_id": "wf-child",
11283 "child_workflow_run_id": "run-child",
11284 "child_workflow_type": "python.child",
11285 "failure_id": "failure-child",
11286 "failure_category": "child_workflow",
11287 "message": "payment rejected",
11288 "exception": {
11289 "type": "PaymentRejected",
11290 "class": "payments.PaymentRejected",
11291 "message": "payment rejected"
11292 }
11293 }),
11294 );
11295 task.workflow_type = "rust.handled-parent".to_string();
11296
11297 let commands = worker.execute_workflow_task(task).expect("handled failure");
11298 assert_eq!(commands[0]["type"], "complete_workflow");
11299 let output = decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("parent output");
11300 assert_eq!(output["reason"], "child_workflow");
11301 assert_eq!(output["failure_id"], "failure-child");
11302 assert_eq!(output["exception_class"], "payments.PaymentRejected");
11303 assert_eq!(output["child_run_id"], "run-child");
11304 }
11305
11306 #[test]
11307 fn rust_hello_world_uses_signal_arguments_from_resume_payload() {
11308 let client = Client::new("http://127.0.0.1:8080").expect("client");
11309 let mut worker = Worker::new(client, "rust-workers");
11310
11311 worker.register_workflow("rust.hello_workflow", |ctx, _input| async move {
11312 let signal = ctx.wait_signal("start").await?;
11313 let name = signal
11314 .first()
11315 .and_then(|value| value.as_str())
11316 .unwrap_or("world");
11317 let greeting = ctx.activity("rust.hello_activity", json!([name])).await?;
11318 Ok(json!({
11319 "greeting": greeting,
11320 "language": "rust"
11321 }))
11322 });
11323
11324 let signal_arguments =
11325 encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC).expect("signal arguments");
11326 let task = WorkflowTask {
11327 task_id: "wft-rust-signal-1".to_string(),
11328 workflow_id: Some("wf-rust-hello".to_string()),
11329 run_id: Some("run-rust-hello".to_string()),
11330 workflow_type: "rust.hello_workflow".to_string(),
11331 payload_codec: DEFAULT_CODEC.to_string(),
11332 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
11333 history_events: vec![HistoryEvent {
11334 event_type: "SignalReceived".to_string(),
11335 payload: json!({
11336 "signal_id": "sig-rust-1",
11337 "signal_name": "start"
11338 }),
11339 raw: HashMap::new(),
11340 }],
11341 total_history_events: Some(1),
11342 history_size_bytes: None,
11343 continue_as_new_recommended: None,
11344 history_budget_pressure: None,
11345 next_history_page_token: None,
11346 workflow_task_attempt: 1,
11347 workflow_signal_id: Some("sig-rust-1".to_string()),
11348 signal_name: Some("start".to_string()),
11349 signal_arguments: Some(signal_arguments),
11350 workflow_update_id: None,
11351 update_name: None,
11352 lease_owner: Some("rust-worker".to_string()),
11353 };
11354
11355 let commands = worker.execute_workflow_task(task).expect("workflow task");
11356
11357 assert_eq!(commands.len(), 1);
11358 assert_eq!(commands[0]["type"], "schedule_activity");
11359 assert_eq!(commands[0]["activity_type"], "rust.hello_activity");
11360 assert_eq!(
11361 decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC).expect("activity args"),
11362 json!(["Rust"])
11363 );
11364 }
11365
11366 #[test]
11367 fn workflow_task_appends_paginated_history_events() {
11368 let mut task = WorkflowTask {
11369 task_id: "wft-rust-pages-1".to_string(),
11370 workflow_id: Some("wf-rust-pages".to_string()),
11371 run_id: Some("run-rust-pages".to_string()),
11372 workflow_type: "rust.hello_workflow".to_string(),
11373 payload_codec: DEFAULT_CODEC.to_string(),
11374 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
11375 history_events: vec![HistoryEvent {
11376 event_type: "WorkflowStarted".to_string(),
11377 payload: json!({}),
11378 raw: HashMap::new(),
11379 }],
11380 total_history_events: Some(3),
11381 history_size_bytes: None,
11382 continue_as_new_recommended: None,
11383 history_budget_pressure: None,
11384 next_history_page_token: Some("MQ==".to_string()),
11385 workflow_task_attempt: 1,
11386 workflow_signal_id: None,
11387 signal_name: None,
11388 signal_arguments: None,
11389 workflow_update_id: None,
11390 update_name: None,
11391 lease_owner: Some("rust-worker".to_string()),
11392 };
11393
11394 task.append_history_page(WorkflowTaskHistoryPage {
11395 history_events: vec![
11396 HistoryEvent {
11397 event_type: "SignalReceived".to_string(),
11398 payload: json!({
11399 "signal_id": "sig-rust-1",
11400 "signal_name": "start",
11401 "arguments": encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC)
11402 .expect("signal arguments")
11403 }),
11404 raw: HashMap::new(),
11405 },
11406 HistoryEvent {
11407 event_type: "MarkerRecorded".to_string(),
11408 payload: json!({"sequence": 3}),
11409 raw: HashMap::new(),
11410 },
11411 ],
11412 total_history_events: Some(3),
11413 next_history_page_token: None,
11414 });
11415
11416 assert_eq!(task.history_events.len(), 3);
11417 assert_eq!(task.total_history_events, Some(3));
11418 assert_eq!(task.next_history_page_token, None);
11419
11420 let signal = task
11421 .history_events
11422 .iter()
11423 .find(|event| event.event_type == "SignalReceived")
11424 .expect("signal event");
11425 assert_eq!(
11426 decode_signal_event_arguments(signal, DEFAULT_CODEC).expect("signal arguments"),
11427 vec![AvroValue::String("Rust".to_string())]
11428 );
11429 }
11430
11431 #[tokio::test]
11432 async fn query_handler_reads_ordered_cross_codec_signals_without_commands() {
11433 let client = Client::new("http://127.0.0.1:8080").expect("client");
11434 let mut worker = Worker::new(client, "rust-workers");
11435 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
11436 worker.register_query("counter", "current", |ctx, _args| async move {
11437 let mut count = 0_i64;
11438 for signal in ctx.signal_events() {
11439 let value = signal
11440 .arguments
11441 .first()
11442 .and_then(Value::as_i64)
11443 .unwrap_or_default();
11444 match signal.name.as_str() {
11445 "increment" => count += value,
11446 "set" => count = value,
11447 _ => {}
11448 }
11449 }
11450 Ok(json!(count))
11451 });
11452
11453 let task = QueryTask {
11454 query_task_id: "query-rust-counter".to_string(),
11455 query_task_attempt: 1,
11456 lease_owner: Some("rust-worker".to_string()),
11457 workflow_id: Some("counter-1".to_string()),
11458 run_id: Some("run-counter-1".to_string()),
11459 workflow_type: "counter".to_string(),
11460 query_name: "current".to_string(),
11461 payload_codec: DEFAULT_CODEC.to_string(),
11462 workflow_arguments: Some(
11463 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
11464 ),
11465 query_arguments: Some(
11466 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("query arguments"),
11467 ),
11468 history_events: vec![
11469 HistoryEvent {
11470 event_type: "SignalReceived".to_string(),
11471 payload: json!({
11472 "signal_id": "php-signal-1",
11473 "signal_name": "increment",
11474 "workflow_sequence": 1,
11475 "payload_codec": DEFAULT_CODEC,
11476 "arguments": encode_value_envelope(&json!([3]), DEFAULT_CODEC).expect("php avro signal")
11477 }),
11478 raw: HashMap::new(),
11479 },
11480 HistoryEvent {
11481 event_type: "SignalReceived".to_string(),
11482 payload: json!({
11483 "signal_id": "python-signal-2",
11484 "signal_name": "increment",
11485 "workflow_sequence": 2,
11486 "payload_codec": JSON_CODEC,
11487 "arguments": encode_value_envelope(&json!([5]), JSON_CODEC).expect("python json signal")
11488 }),
11489 raw: HashMap::new(),
11490 },
11491 HistoryEvent {
11492 event_type: "SignalReceived".to_string(),
11493 payload: json!({
11494 "signal_id": "rust-signal-3",
11495 "signal_name": "set",
11496 "workflow_sequence": 3,
11497 "payload_codec": DEFAULT_CODEC,
11498 "arguments": encode_value_envelope(&json!([0]), DEFAULT_CODEC).expect("rust avro signal")
11499 }),
11500 raw: HashMap::new(),
11501 },
11502 ],
11503 history_export: None,
11504 run_status: Some("completed".to_string()),
11505 };
11506
11507 let result = worker.execute_query_task(task).await.expect("query result");
11508 assert_eq!(result.into_json().expect("query projection"), json!(0));
11509 }
11510
11511 #[tokio::test]
11512 async fn replayed_queries_read_running_completed_and_cold_restarted_instance_state() {
11513 let worker = replay_counter_worker();
11514 let running_history = json!([
11515 {
11516 "type": "ActivityCompleted",
11517 "payload": {
11518 "sequence": 1,
11519 "activity_type": "load-counter",
11520 "payload_codec": "json",
11521 "result": {"codec": "json", "blob": "\"loaded\""}
11522 }
11523 },
11524 {
11525 "type": "SignalWaitOpened",
11526 "payload": {
11527 "sequence": 3,
11528 "signal_name": "increment"
11529 }
11530 },
11531 {
11532 "type": "SignalReceived",
11533 "payload": {
11534 "signal_id": "signal-3",
11535 "signal_name": "increment",
11536 "workflow_sequence": 2,
11537 "payload_codec": "json",
11538 "arguments": {"codec": "json", "blob": "[3]"}
11539 }
11540 },
11541 {
11542 "type": "SignalApplied",
11543 "payload": {
11544 "sequence": 3,
11545 "signal_id": "signal-3",
11546 "signal_name": "increment",
11547 "payload_codec": "json",
11548 "value": {"codec": "json", "blob": "[3]"}
11549 }
11550 }
11551 ]);
11552
11553 let running = worker
11554 .execute_query_task(replay_counter_query(
11555 "current",
11556 running_history.clone(),
11557 "running",
11558 ))
11559 .await
11560 .expect("running replay query");
11561 assert_eq!(
11562 running.clone().into_json().expect("query projection"),
11563 json!({"loaded": "loaded", "count": 3, "finished": false})
11564 );
11565
11566 let detached = worker
11567 .execute_query_task(replay_counter_query(
11568 "detached-mutation",
11569 running_history.clone(),
11570 "running",
11571 ))
11572 .await
11573 .expect("query mutates only its detached state clone");
11574 assert_eq!(detached.into_json().expect("query projection"), json!(999));
11575 let failed = worker
11576 .execute_query_task(replay_counter_query(
11577 "failed-mutation",
11578 running_history.clone(),
11579 "running",
11580 ))
11581 .await
11582 .expect_err("failed query");
11583 assert_eq!(failed.reason, "query_rejected");
11584 let unchanged = worker
11585 .execute_query_task(replay_counter_query("current", running_history, "running"))
11586 .await
11587 .expect("later query reconstructs unchanged state");
11588 assert_eq!(unchanged, running);
11589
11590 let restarted_worker = replay_counter_worker();
11591 let restarted_task: QueryTask = serde_json::from_value(json!({
11592 "query_task_id": "query-after-restart",
11593 "workflow_id": "counter-1",
11594 "run_id": "run-counter-1",
11595 "workflow_type": "replay-counter",
11596 "query_name": "current",
11597 "payload_codec": "json",
11598 "workflow_arguments": {"codec": "json", "blob": "[]"},
11599 "query_arguments": {"codec": "json", "blob": "[]"},
11600 "history_events": [],
11601 "history_export": {
11602 "payloads": {"codec": "json"},
11603 "history_events": [
11604 {
11605 "type": "ActivityCompleted",
11606 "payload": {
11607 "sequence": 1,
11608 "activity_type": "load-counter",
11609 "payload_codec": "json",
11610 "result": null
11611 }
11612 },
11613 {
11614 "type": "SignalWaitOpened",
11615 "payload": {
11616 "sequence": 3,
11617 "signal_name": "increment"
11618 }
11619 },
11620 {
11621 "type": "SignalReceived",
11622 "payload": {
11623 "signal_id": "signal-3",
11624 "signal_name": "increment",
11625 "workflow_sequence": 2
11626 }
11627 },
11628 {
11629 "type": "SignalApplied",
11630 "payload": {
11631 "sequence": 3,
11632 "signal_id": "signal-3",
11633 "signal_name": "increment"
11634 }
11635 },
11636 {
11637 "type": "SignalWaitOpened",
11638 "payload": {
11639 "sequence": 5,
11640 "signal_name": "increment"
11641 }
11642 },
11643 {
11644 "type": "SignalReceived",
11645 "payload": {
11646 "signal_id": "signal-5",
11647 "signal_name": "increment",
11648 "workflow_sequence": 4
11649 }
11650 },
11651 {
11652 "type": "SignalApplied",
11653 "payload": {
11654 "sequence": 5,
11655 "signal_id": "signal-5",
11656 "signal_name": "increment"
11657 }
11658 }
11659 ],
11660 "activities": [{
11661 "sequence": 1,
11662 "activity_type": "load-counter",
11663 "payload_codec": "json",
11664 "result": {"codec": "json", "blob": "\"loaded\""}
11665 }],
11666 "signals": [
11667 {
11668 "id": "signal-3",
11669 "name": "increment",
11670 "workflow_sequence": 2,
11671 "payload_codec": "json",
11672 "arguments": "[3]"
11673 },
11674 {
11675 "id": "signal-5",
11676 "name": "increment",
11677 "workflow_sequence": 4,
11678 "payload_codec": "json",
11679 "arguments": "[5]"
11680 }
11681 ]
11682 },
11683 "run_status": "completed"
11684 }))
11685 .expect("cold replay query task");
11686 let completed = restarted_worker
11687 .execute_query_task(restarted_task)
11688 .await
11689 .expect("completed cold replay query");
11690 assert_eq!(
11691 completed.into_json().expect("query projection"),
11692 json!({"loaded": "loaded", "count": 8, "finished": true})
11693 );
11694 }
11695
11696 #[tokio::test]
11697 async fn replayed_query_replay_failures_are_machine_readable() {
11698 let worker = replay_counter_worker();
11699 let task = replay_counter_query(
11700 "current",
11701 json!([{
11702 "type": "ActivityCompleted",
11703 "payload": {
11704 "sequence": 1,
11705 "payload_codec": "json",
11706 "result": {"codec": "json", "blob": "{"}
11707 }
11708 }]),
11709 "running",
11710 );
11711 let failure = worker
11712 .execute_query_task(task)
11713 .await
11714 .expect_err("invalid replay history payload");
11715 assert_eq!(failure.reason, "query_workflow_state_unavailable");
11716 assert_eq!(failure.failure_type, "QueryWorkflowStateUnavailable");
11717 }
11718
11719 #[tokio::test]
11720 async fn query_task_restores_compact_history_from_export() {
11721 let client = Client::new("http://127.0.0.1:8080").expect("client");
11722 let mut worker = Worker::new(client, "rust-workers");
11723 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
11724 worker.register_query("counter", "current", |ctx, _args| async move {
11725 Ok(json!(ctx.signals("increment")[0][0]))
11726 });
11727 let task: QueryTask = serde_json::from_value(json!({
11728 "query_task_id": "query-export",
11729 "workflow_type": "counter",
11730 "query_name": "current",
11731 "payload_codec": "json",
11732 "workflow_arguments": {"codec": "json", "blob": "[]"},
11733 "query_arguments": {"codec": "json", "blob": "[]"},
11734 "history_events": [],
11735 "history_export": {
11736 "payloads": {"codec": "json"},
11737 "history_events": [{
11738 "type": "SignalReceived",
11739 "payload": {"signal_id": "signal-export", "signal_name": "increment"}
11740 }],
11741 "signals": [{
11742 "id": "signal-export",
11743 "name": "increment",
11744 "status": "applied",
11745 "workflow_sequence": 1,
11746 "payload_codec": "json",
11747 "arguments": "[9]"
11748 }]
11749 }
11750 }))
11751 .expect("query task");
11752
11753 let result = worker.execute_query_task(task).await.expect("query result");
11754 assert_eq!(result.into_json().expect("query projection"), json!(9));
11755 }
11756
11757 #[tokio::test]
11758 async fn query_task_failures_have_stable_reasons() {
11759 let client = Client::new("http://127.0.0.1:8080").expect("client");
11760 let mut worker = Worker::new(client, "rust-workers");
11761 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
11762 worker.register_query(
11763 "counter",
11764 "current",
11765 |_ctx, _args| async move { Ok(json!(0)) },
11766 );
11767
11768 let base_task = QueryTask {
11769 query_task_id: "query-errors".to_string(),
11770 query_task_attempt: 1,
11771 lease_owner: None,
11772 workflow_id: Some("counter-errors".to_string()),
11773 run_id: Some("run-errors".to_string()),
11774 workflow_type: "counter".to_string(),
11775 query_name: "missing".to_string(),
11776 payload_codec: JSON_CODEC.to_string(),
11777 workflow_arguments: Some(json!({"codec": "json", "blob": "[]"})),
11778 query_arguments: Some(json!({"codec": "json", "blob": "[]"})),
11779 history_events: Vec::new(),
11780 history_export: None,
11781 run_status: Some("running".to_string()),
11782 };
11783
11784 let unknown = worker
11785 .execute_query_task(base_task.clone())
11786 .await
11787 .expect_err("unknown query");
11788 assert_eq!(unknown.reason, "rejected_unknown_query");
11789
11790 let mut malformed = base_task;
11791 malformed.query_name = "current".to_string();
11792 malformed.query_arguments = Some(json!({"codec": "json", "blob": "{"}));
11793 let malformed = worker
11794 .execute_query_task(malformed)
11795 .await
11796 .expect_err("malformed payload");
11797 assert_eq!(malformed.reason, "query_payload_decode_failed");
11798
11799 let client = Client::new("http://127.0.0.1:8080").expect("client");
11800 let mut unavailable_worker = Worker::new(client, "rust-workers");
11801 unavailable_worker
11802 .register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
11803 let unavailable_task: QueryTask = serde_json::from_value(json!({
11804 "query_task_id": "query-unavailable",
11805 "workflow_type": "counter",
11806 "query_name": "current",
11807 "payload_codec": "json",
11808 "workflow_arguments": {"codec": "json", "blob": "[]"},
11809 "query_arguments": {"codec": "json", "blob": "[]"}
11810 }))
11811 .expect("query task");
11812 let unavailable = unavailable_worker
11813 .execute_query_task(unavailable_task)
11814 .await
11815 .expect_err("query handler unavailable");
11816 assert_eq!(unavailable.reason, "query_handler_unavailable");
11817 }
11818
11819 #[tokio::test]
11820 async fn client_query_decodes_result_and_typed_failure() {
11821 let server = MockWorkerServer::start();
11822 let client = Client::builder(server.base_url())
11823 .timeout(Duration::from_secs(2))
11824 .build()
11825 .expect("client");
11826
11827 let result = client
11828 .query_workflow("counter-1", "current", json!([]))
11829 .await
11830 .expect("query result");
11831 assert_eq!(result, json!({"count": 8}));
11832
11833 let error = client
11834 .query_workflow("counter-1", "missing", json!([]))
11835 .await
11836 .expect_err("unknown query");
11837 let Error::QueryFailed(failure) = error else {
11838 panic!("expected typed query failure");
11839 };
11840 assert_eq!(failure.status, 404);
11841 assert_eq!(failure.reason, "rejected_unknown_query");
11842 }
11843
11844 #[tokio::test]
11845 async fn public_client_surfaces_send_and_receive_lossless_avro_values() {
11846 let server = MockWorkerServer::start();
11847 let client = Client::builder(server.base_url())
11848 .timeout(Duration::from_secs(2))
11849 .build()
11850 .expect("client");
11851 let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
11852
11853 client
11854 .start_workflow(
11855 "typed.echo",
11856 "rust-workers",
11857 "typed-start",
11858 arguments.clone(),
11859 )
11860 .await
11861 .expect("typed workflow start");
11862 assert_eq!(
11863 decode_wire_avro_value(
11864 &server.request_body("/api/workflows")["input"],
11865 DEFAULT_CODEC,
11866 )
11867 .expect("typed start input"),
11868 arguments
11869 );
11870
11871 client
11872 .signal_workflow("typed-1", "changed", arguments.clone())
11873 .await
11874 .expect("typed signal");
11875 assert_eq!(
11876 decode_wire_avro_value(
11877 &server.request_body("/api/workflows/typed-1/signal/changed")["input"],
11878 DEFAULT_CODEC,
11879 )
11880 .expect("typed signal input"),
11881 arguments
11882 );
11883
11884 assert_eq!(
11885 client
11886 .query_workflow_avro_value("typed-1", "inspect", arguments.clone())
11887 .await
11888 .expect("typed query"),
11889 typed_fidelity_probe()
11890 );
11891 assert_eq!(
11892 decode_wire_avro_value(
11893 &server.request_body("/api/workflows/typed-1/query/inspect")["input"],
11894 DEFAULT_CODEC,
11895 )
11896 .expect("typed query input"),
11897 arguments
11898 );
11899
11900 assert_eq!(
11901 client
11902 .update_workflow_avro_value(
11903 "typed-1",
11904 "replace",
11905 arguments.clone(),
11906 Some("typed-request"),
11907 )
11908 .await
11909 .expect("typed update"),
11910 typed_fidelity_probe()
11911 );
11912 let update = server.request_body("/api/workflows/typed-1/update/replace");
11913 assert_eq!(update["request_id"], "typed-request");
11914 assert_eq!(
11915 decode_wire_avro_value(&update["input"], DEFAULT_CODEC).expect("typed update input"),
11916 arguments
11917 );
11918
11919 let handle = WorkflowHandle {
11920 client: client.clone(),
11921 workflow_id: "typed-1".to_string(),
11922 run_id: Some("run-typed-1".to_string()),
11923 workflow_type: "typed.echo".to_string(),
11924 };
11925 assert_eq!(
11926 handle
11927 .result_avro_value(WorkflowResultOptions::default())
11928 .await
11929 .expect("typed workflow result"),
11930 typed_fidelity_probe()
11931 );
11932
11933 client
11934 .complete_activity_task(
11935 "activity-typed",
11936 "attempt-typed",
11937 "rust-worker",
11938 typed_fidelity_probe(),
11939 DEFAULT_CODEC,
11940 )
11941 .await
11942 .expect("typed activity completion");
11943 assert_eq!(
11944 decode_wire_avro_value(
11945 &server.request_body("/api/worker/activity-tasks/activity-typed/complete")
11946 ["result"],
11947 DEFAULT_CODEC,
11948 )
11949 .expect("typed activity result"),
11950 typed_fidelity_probe()
11951 );
11952 client
11953 .fail_activity_task(
11954 "activity-typed",
11955 "attempt-typed",
11956 "rust-worker",
11957 "typed failure",
11958 true,
11959 )
11960 .await
11961 .expect("activity failure");
11962 }
11963
11964 #[tokio::test]
11965 async fn lifecycle_commands_support_instance_and_selected_run_targets() {
11966 let server = MockWorkerServer::start();
11967 let client = Client::builder(server.base_url())
11968 .timeout(Duration::from_secs(2))
11969 .build()
11970 .expect("client");
11971
11972 let options = WorkflowCommandOptions::new()
11973 .reason("cleanup requested")
11974 .request_id("cancel-17");
11975 let cancelled = client
11976 .cancel_workflow("wf-lifecycle", options)
11977 .await
11978 .expect("instance cancellation");
11979 assert_eq!(cancelled.command, WorkflowCommandKind::Cancel);
11980 assert_eq!(cancelled.run_id.as_deref(), Some("run-current"));
11981 assert_eq!(cancelled.outcome.as_deref(), Some("cancelled"));
11982 assert_eq!(
11983 server.request_body("/api/workflows/wf-lifecycle/cancel"),
11984 json!({"reason":"cleanup requested","request_id":"cancel-17"})
11985 );
11986
11987 let terminated = client
11988 .terminate_workflow(
11989 "wf-lifecycle",
11990 WorkflowCommandOptions::new().reason("forced stop"),
11991 )
11992 .await
11993 .expect("instance termination");
11994 assert_eq!(terminated.command, WorkflowCommandKind::Terminate);
11995 assert_eq!(terminated.outcome.as_deref(), Some("terminated"));
11996
11997 client
11998 .cancel_workflow_run(
11999 "wf-lifecycle",
12000 "run-current",
12001 WorkflowCommandOptions::default(),
12002 )
12003 .await
12004 .expect("selected run cancellation");
12005 client
12006 .terminate_workflow_run(
12007 "wf-lifecycle",
12008 "run-current",
12009 WorkflowCommandOptions::default(),
12010 )
12011 .await
12012 .expect("selected run termination");
12013
12014 for (command, error) in [
12015 (
12016 WorkflowCommandKind::Cancel,
12017 client
12018 .cancel_workflow_run(
12019 "wf-lifecycle",
12020 "run-stale",
12021 WorkflowCommandOptions::default(),
12022 )
12023 .await
12024 .expect_err("stale cancellation must be rejected"),
12025 ),
12026 (
12027 WorkflowCommandKind::Terminate,
12028 client
12029 .terminate_workflow_run(
12030 "wf-lifecycle",
12031 "run-stale",
12032 WorkflowCommandOptions::default(),
12033 )
12034 .await
12035 .expect_err("stale termination must be rejected"),
12036 ),
12037 ] {
12038 let Error::WorkflowCommandRejected(rejection) = error else {
12039 panic!("expected typed command rejection");
12040 };
12041 assert_eq!(rejection.command, command);
12042 assert_eq!(rejection.status, 409);
12043 assert_eq!(rejection.reason, "historical_run_command_rejected");
12044 assert_eq!(rejection.run_id.as_deref(), Some("run-stale"));
12045 assert_eq!(rejection.target_scope.as_deref(), Some("run"));
12046 }
12047 }
12048
12049 #[tokio::test]
12050 async fn workflow_start_options_send_server_enforced_deadlines() {
12051 let server = MockWorkerServer::start();
12052 let client = Client::builder(server.base_url())
12053 .timeout(Duration::from_secs(2))
12054 .build()
12055 .expect("client");
12056
12057 let handle = client
12058 .start_workflow_with_options(
12059 "rust.timeout",
12060 "rust-timeouts",
12061 "wf-start-options",
12062 WorkflowStartOptions::new()
12063 .execution_timeout_seconds(30)
12064 .run_timeout_seconds(1),
12065 json!([]),
12066 )
12067 .await
12068 .expect("workflow start");
12069
12070 assert_eq!(handle.run_id.as_deref(), Some("run-start-options"));
12071 let body = server.request_body("/api/workflows");
12072 assert_eq!(body["execution_timeout_seconds"], 30);
12073 assert_eq!(body["run_timeout_seconds"], 1);
12074
12075 let invalid = client
12076 .start_workflow_with_options(
12077 "rust.timeout",
12078 "rust-timeouts",
12079 "wf-invalid-options",
12080 WorkflowStartOptions::new()
12081 .execution_timeout_seconds(1)
12082 .run_timeout_seconds(2),
12083 json!([]),
12084 )
12085 .await
12086 .expect_err("invalid deadline ordering");
12087 assert!(invalid
12088 .to_string()
12089 .contains("run_timeout_seconds cannot exceed execution_timeout_seconds"));
12090 }
12091
12092 #[tokio::test]
12093 async fn workflow_result_returns_each_typed_terminal_outcome() {
12094 let server = MockWorkerServer::start();
12095 let client = Client::builder(server.base_url())
12096 .timeout(Duration::from_secs(2))
12097 .build()
12098 .expect("client");
12099 let options = WorkflowResultOptions {
12100 poll_interval: Duration::ZERO,
12101 timeout: Duration::from_secs(1),
12102 };
12103
12104 let failed = WorkflowHandle {
12105 client: client.clone(),
12106 workflow_id: "wf-failed".to_string(),
12107 run_id: Some("run-failed".to_string()),
12108 workflow_type: "failure".to_string(),
12109 }
12110 .result(options)
12111 .await
12112 .expect_err("failed outcome");
12113 let Error::WorkflowFailed(failure) = failed else {
12114 panic!("expected WorkflowFailed");
12115 };
12116 assert_eq!(failure.workflow_id, "wf-failed");
12117 assert_eq!(failure.run_id.as_deref(), Some("run-failed"));
12118 assert_eq!(failure.failure_id.as_deref(), Some("failure-17"));
12119 assert_eq!(failure.failure_category.as_deref(), Some("application"));
12120 assert_eq!(failure.exception_type.as_deref(), Some("PaymentError"));
12121 assert_eq!(
12122 failure.exception_class.as_deref(),
12123 Some("billing::PaymentError")
12124 );
12125 assert_eq!(failure.non_retryable, Some(true));
12126
12127 for (workflow_id, expected_kind, expected_reason) in [
12128 (
12129 "wf-cancelled",
12130 WorkflowTerminalKind::Cancelled,
12131 "cleanup requested",
12132 ),
12133 (
12134 "wf-terminated",
12135 WorkflowTerminalKind::Terminated,
12136 "forced stop",
12137 ),
12138 (
12139 "wf-timed-out",
12140 WorkflowTerminalKind::TimedOut,
12141 "run_timeout",
12142 ),
12143 ] {
12144 let error = WorkflowHandle {
12145 client: client.clone(),
12146 workflow_id: workflow_id.to_string(),
12147 run_id: None,
12148 workflow_type: "terminal".to_string(),
12149 }
12150 .result(options)
12151 .await
12152 .expect_err("typed terminal outcome");
12153 let outcome = match error {
12154 Error::WorkflowCancelled(outcome) => outcome,
12155 Error::WorkflowTerminated(outcome) => outcome,
12156 Error::WorkflowTimedOut(outcome) => outcome,
12157 other => panic!("unexpected terminal error: {other}"),
12158 };
12159 assert_eq!(outcome.kind, expected_kind);
12160 assert_eq!(outcome.workflow_id, workflow_id);
12161 assert_eq!(outcome.reason, expected_reason);
12162 }
12163
12164 let wait_timeout = WorkflowHandle {
12165 client,
12166 workflow_id: "wf-waiting".to_string(),
12167 run_id: Some("run-waiting".to_string()),
12168 workflow_type: "waiting".to_string(),
12169 }
12170 .result(WorkflowResultOptions {
12171 poll_interval: Duration::ZERO,
12172 timeout: Duration::ZERO,
12173 })
12174 .await
12175 .expect_err("client wait timeout");
12176 let Error::WorkflowTimedOut(timeout) = wait_timeout else {
12177 panic!("expected typed client timeout");
12178 };
12179 assert_eq!(timeout.reason, "result_wait_timeout");
12180 assert_eq!(timeout.failure_category.as_deref(), Some("client_timeout"));
12181 assert_eq!(timeout.run_id.as_deref(), Some("run-waiting"));
12182 }
12183
12184 #[tokio::test]
12185 async fn workflow_result_follows_chain_and_selected_result_preserves_history() {
12186 let server = MockWorkerServer::start();
12187 let client = Client::builder(server.base_url())
12188 .timeout(Duration::from_secs(2))
12189 .build()
12190 .expect("client");
12191
12192 let handle = WorkflowHandle {
12193 client,
12194 workflow_id: "wf-selected".to_string(),
12195 run_id: Some("run-selected".to_string()),
12196 workflow_type: "selected".to_string(),
12197 };
12198 let options = WorkflowResultOptions {
12199 poll_interval: Duration::ZERO,
12200 timeout: Duration::from_secs(1),
12201 };
12202
12203 let current = handle
12204 .result(options)
12205 .await
12206 .expect("instance result follows the current run");
12207 assert_eq!(current, json!("current run output"));
12208
12209 let error = handle
12210 .result_selected_run(options)
12211 .await
12212 .expect_err("the selected run is cancelled even though the current run completed");
12213
12214 let Error::WorkflowCancelled(outcome) = error else {
12215 panic!("expected selected run cancellation");
12216 };
12217 assert_eq!(outcome.run_id.as_deref(), Some("run-selected"));
12218 assert_eq!(outcome.reason, "selected run cancelled");
12219 assert_eq!(
12220 server.request_count("/api/workflows/wf-selected/runs/run-selected"),
12221 1
12222 );
12223 assert_eq!(server.request_count("/api/workflows/wf-selected"), 1);
12224 }
12225
12226 #[tokio::test]
12227 async fn poll_responses_decode_http_conflict_drain_as_a_stable_stop() {
12228 let server = MockWorkerServer::draining_polls();
12229 let client = Client::builder(server.base_url())
12230 .timeout(Duration::from_secs(2))
12231 .build()
12232 .expect("client");
12233
12234 let workflow = client
12235 .poll_workflow_task_response("draining-worker", "rust-workers", Duration::ZERO)
12236 .await
12237 .expect("workflow drain response");
12238 let activity = client
12239 .poll_activity_task_response("draining-worker", "rust-workers", Duration::ZERO)
12240 .await
12241 .expect("activity drain response");
12242 let query = client
12243 .poll_query_task_response("draining-worker", "rust-workers", Duration::ZERO)
12244 .await
12245 .expect("query drain response");
12246
12247 for outcome in [workflow.outcome(), activity.outcome(), query.outcome()] {
12248 assert_eq!(
12249 outcome,
12250 WorkerPollOutcome::Stop {
12251 poll_status: Some("draining".to_string()),
12252 reason: Some("worker_draining".to_string()),
12253 }
12254 );
12255 }
12256
12257 assert!(client
12258 .poll_workflow_task("draining-worker", "rust-workers", Duration::ZERO)
12259 .await
12260 .expect("compatibility poll")
12261 .is_none());
12262 }
12263
12264 #[tokio::test]
12265 async fn managed_worker_honors_drain_stop_for_every_task_family() {
12266 let server = MockWorkerServer::draining_polls();
12267 let client = Client::builder(server.base_url())
12268 .timeout(Duration::from_secs(2))
12269 .build()
12270 .expect("client");
12271
12272 let mut workflow_worker = Worker::new(client.clone(), "rust-workers")
12273 .worker_id("draining-workflow-worker")
12274 .poll_timeout(Duration::ZERO);
12275 workflow_worker.register_workflow("counter", |_ctx, _args| async { Ok(Value::Null) });
12276 workflow_worker
12277 .run()
12278 .await
12279 .expect("workflow drain is a clean stop");
12280
12281 let mut activity_worker = Worker::new(client.clone(), "rust-workers")
12282 .worker_id("draining-activity-worker")
12283 .poll_timeout(Duration::ZERO);
12284 activity_worker.register_activity("write", |_ctx, _args| async { Ok(Value::Null) });
12285 activity_worker
12286 .run()
12287 .await
12288 .expect("activity drain is a clean stop");
12289
12290 let mut query_worker = Worker::new(client, "rust-workers")
12291 .worker_id("draining-query-worker")
12292 .poll_timeout(Duration::ZERO);
12293 query_worker.register_query("counter", "current", |_ctx, _args| async {
12294 Ok(Value::Null)
12295 });
12296 query_worker
12297 .run()
12298 .await
12299 .expect("query drain is a clean stop");
12300 }
12301
12302 #[tokio::test]
12303 async fn activity_cancellation_and_late_completion_remain_machine_readable() {
12304 let server = MockWorkerServer::start();
12305 let client = Client::builder(server.base_url())
12306 .timeout(Duration::from_secs(2))
12307 .build()
12308 .expect("client");
12309
12310 let heartbeat = client
12311 .heartbeat_activity_task(
12312 "activity-cancel",
12313 "attempt-cancel",
12314 "rust-worker",
12315 typed_fidelity_probe(),
12316 )
12317 .await
12318 .expect("cancellation heartbeat");
12319 assert!(heartbeat.cancel_requested);
12320 assert!(heartbeat.should_stop());
12321 assert_eq!(heartbeat.reason.as_deref(), Some("run_cancelled"));
12322 assert_eq!(heartbeat.run_closed_reason.as_deref(), Some("cancelled"));
12323 let heartbeat_body =
12324 server.request_body("/api/worker/activity-tasks/activity-cancel/heartbeat");
12325 assert_eq!(heartbeat_body["details"]["codec"], DEFAULT_CODEC);
12326 assert_eq!(
12327 decode_wire_avro_value(&heartbeat_body["details"], DEFAULT_CODEC)
12328 .expect("typed heartbeat details"),
12329 typed_fidelity_probe()
12330 );
12331
12332 let error = client
12333 .complete_activity_task(
12334 "activity-cancel",
12335 "attempt-cancel",
12336 "rust-worker",
12337 json!({"late":true}),
12338 JSON_CODEC,
12339 )
12340 .await
12341 .expect_err("late completion must be refused");
12342 assert!(activity_task_rejection_is_final(&error));
12343 let Error::ActivityTaskRejected(rejection) = error else {
12344 panic!("expected typed activity rejection");
12345 };
12346 assert_eq!(rejection.status, 409);
12347 assert_eq!(rejection.reason, "run_cancelled");
12348 assert!(rejection.cancel_requested);
12349 assert_eq!(rejection.can_continue, Some(false));
12350 }
12351
12352 #[tokio::test]
12353 async fn managed_worker_survives_late_completion_and_restart_during_cancellation() {
12354 let server = MockWorkerServer::cancelled_activity();
12355 let client = Client::builder(server.base_url())
12356 .timeout(Duration::from_secs(2))
12357 .build()
12358 .expect("client");
12359 let cancellation_observed = Arc::new(AtomicBool::new(false));
12360 let observed = Arc::clone(&cancellation_observed);
12361 let mut worker = Worker::new(client.clone(), "rust-workers")
12362 .worker_id("rust-cancel-worker")
12363 .poll_timeout(Duration::from_millis(10));
12364 worker.register_activity("cancel-aware", move |ctx, _args| {
12365 let observed = Arc::clone(&observed);
12366 async move {
12367 let heartbeat = ctx.heartbeat(json!({"stage":"running"})).await?;
12368 observed.store(heartbeat.should_stop(), Ordering::SeqCst);
12369 Ok(json!({"late":"completion"}))
12370 }
12371 });
12372
12373 assert_eq!(
12374 worker.run_once().await.expect("cancelled attempt handled"),
12375 1
12376 );
12377 assert!(cancellation_observed.load(Ordering::SeqCst));
12378 assert_eq!(
12379 server.request_count("/api/worker/activity-tasks/activity-cancel/complete"),
12380 1
12381 );
12382
12383 let mut restarted = Worker::new(client, "rust-workers")
12384 .worker_id("rust-cancel-worker-restarted")
12385 .poll_timeout(Duration::from_millis(10));
12386 restarted.register_activity("cancel-aware", |_ctx, _args| async move { Ok(Value::Null) });
12387 assert_eq!(
12388 restarted
12389 .run_once()
12390 .await
12391 .expect("replacement worker continues polling"),
12392 0
12393 );
12394 }
12395
12396 #[tokio::test]
12397 async fn managed_worker_absorbs_selected_run_terminal_timeout_completion_race() {
12398 let response = r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"outcome":"completed","recorded":false,"run_id":"run-selected-timeout","run_status":"failed","created_task_ids":[],"reason":"run_timed_out"}"#;
12399 let server = MockWorkerServer::workflow_completion("409 Conflict", response);
12400 let client = Client::builder(server.base_url())
12401 .timeout(Duration::from_secs(2))
12402 .build()
12403 .expect("client");
12404
12405 let direct_error = client
12406 .complete_workflow_task(
12407 "workflow-timeout-task",
12408 "timeout-worker",
12409 3,
12410 vec![json!({"type": "complete_workflow", "result": null})],
12411 )
12412 .await
12413 .expect_err("the low-level client preserves the completion rejection");
12414 let Error::Http { status, body } = direct_error else {
12415 panic!("expected the original HTTP completion rejection");
12416 };
12417 assert_eq!(status, reqwest::StatusCode::CONFLICT);
12418 assert_eq!(
12419 serde_json::from_str::<Value>(&body).expect("response body")["reason"],
12420 "run_timed_out"
12421 );
12422
12423 let mut worker = Worker::new(client, "rust-workers")
12424 .worker_id("timeout-worker")
12425 .poll_timeout(Duration::from_millis(10));
12426 worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
12427 Ok(json!({"late": "result"}))
12428 });
12429
12430 assert_eq!(
12431 worker
12432 .run_once()
12433 .await
12434 .expect("authoritative selected-run timeout settles the tick"),
12435 1
12436 );
12437 assert_eq!(
12438 server.request_count("/api/worker/workflow-tasks/workflow-timeout-task/complete"),
12439 2,
12440 "both the direct client proof and managed worker must see the rejection"
12441 );
12442 }
12443
12444 #[tokio::test]
12445 async fn managed_worker_does_not_swallow_nearby_completion_errors() {
12446 for (name, status, response) in [
12447 ("bare conflict", "409 Conflict", r#"{"message":"conflict"}"#),
12448 (
12449 "command was recorded",
12450 "409 Conflict",
12451 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":true,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12452 ),
12453 (
12454 "lease conflict",
12455 "409 Conflict",
12456 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"lease_expired"}"#,
12457 ),
12458 (
12459 "nonterminal run",
12460 "409 Conflict",
12461 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"waiting","reason":"run_timed_out"}"#,
12462 ),
12463 (
12464 "different selected run",
12465 "409 Conflict",
12466 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-reused-workflow-current","run_status":"failed","reason":"run_timed_out"}"#,
12467 ),
12468 (
12469 "different task attempt",
12470 "409 Conflict",
12471 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":4,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12472 ),
12473 (
12474 "authentication failure",
12475 "401 Unauthorized",
12476 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12477 ),
12478 (
12479 "authorization failure",
12480 "403 Forbidden",
12481 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12482 ),
12483 (
12484 "protocol failure",
12485 "400 Bad Request",
12486 r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.2","requested_version":"1.3"}"#,
12487 ),
12488 (
12489 "malformed command",
12490 "422 Unprocessable Entity",
12491 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12492 ),
12493 (
12494 "transient server failure",
12495 "503 Service Unavailable",
12496 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12497 ),
12498 ] {
12499 let server = MockWorkerServer::workflow_completion(status, response);
12500 let client = Client::builder(server.base_url())
12501 .timeout(Duration::from_secs(2))
12502 .build()
12503 .expect("client");
12504 let mut worker = Worker::new(client, "rust-workers")
12505 .worker_id("timeout-worker")
12506 .poll_timeout(Duration::from_millis(10));
12507 worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
12508 Ok(json!({"late": "result"}))
12509 });
12510
12511 let error = worker
12512 .run_once()
12513 .await
12514 .expect_err(&format!("{name} must remain an error"));
12515 assert!(
12516 matches!(error, Error::Http { .. } | Error::Protocol(_)),
12517 "{name} returned an unexpected error variant: {error}"
12518 );
12519 }
12520 }
12521
12522 #[tokio::test]
12523 async fn worker_deregistration_uses_worker_plane_method_path_headers_and_result() {
12524 let server = MockWorkerServer::start();
12525 let client = Client::builder(server.base_url())
12526 .worker_token(Some("worker-secret".to_string()))
12527 .namespace("orders")
12528 .timeout(Duration::from_secs(2))
12529 .build()
12530 .expect("client");
12531 let path = "/api/worker/registrations/worker%2F%CE%B1%20space";
12532
12533 let result = client
12534 .deregister_worker_registration("worker/α space")
12535 .await
12536 .expect("deregister worker registration");
12537
12538 assert_eq!(server.method_for(path).as_deref(), Some("DELETE"));
12539 assert_eq!(
12540 server.worker_protocol_for(path).as_deref(),
12541 Some(WORKER_PROTOCOL_VERSION)
12542 );
12543 assert_eq!(server.control_protocol_for(path), None);
12544 assert_eq!(server.namespace_for(path).as_deref(), Some("orders"));
12545 assert_eq!(
12546 server.authorization_for(path).as_deref(),
12547 Some("Bearer worker-secret")
12548 );
12549 assert_eq!(
12550 result,
12551 WorkerDeregistrationEnvelope {
12552 worker_id: "deregistered-worker".to_string(),
12553 outcome: "deregistered".to_string(),
12554 recovered_workflow_task_count: 2,
12555 }
12556 );
12557 }
12558
12559 #[tokio::test]
12560 async fn low_level_registration_rejects_update_validators_before_transport() {
12561 let server = MockWorkerServer::start();
12562 let client = Client::builder(server.base_url())
12563 .timeout(Duration::from_secs(2))
12564 .build()
12565 .expect("client");
12566
12567 for update_validators in [json!(["approve"]), json!("approve")] {
12568 let error = client
12569 .register_worker_with_command_contracts(
12570 "validator-claiming-worker",
12571 "rust-workers",
12572 vec!["orders".to_string()],
12573 vec![],
12574 1,
12575 1,
12576 vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
12577 json!({
12578 "orders": {
12579 "queries": ["current"],
12580 "updates": ["approve"],
12581 "update_validators": update_validators,
12582 },
12583 }),
12584 )
12585 .await
12586 .expect_err("unsupported validator claims must fail before registration");
12587
12588 let Error::UnsupportedUpdateValidators { workflow_type } = error else {
12589 panic!("expected typed unsupported-validator failure");
12590 };
12591 assert_eq!(workflow_type, "orders");
12592 }
12593 assert_eq!(server.request_count("/api/worker/register"), 0);
12594 }
12595
12596 #[tokio::test]
12597 async fn low_level_registration_preserves_query_and_update_contracts() {
12598 let server = MockWorkerServer::start();
12599 let client = Client::builder(server.base_url())
12600 .timeout(Duration::from_secs(2))
12601 .build()
12602 .expect("client");
12603 let contracts = json!({
12604 "orders": {
12605 "queries": ["current"],
12606 "updates": ["approve"],
12607 "update_validators": [],
12608 },
12609 "payments": {
12610 "queries": ["status"],
12611 "updates": ["capture"],
12612 },
12613 });
12614
12615 client
12616 .register_worker_with_command_contracts(
12617 "command-worker",
12618 "rust-workers",
12619 vec!["orders".to_string(), "payments".to_string()],
12620 vec![],
12621 1,
12622 1,
12623 vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
12624 contracts.clone(),
12625 )
12626 .await
12627 .expect("query and update contracts must remain supported");
12628
12629 assert_eq!(
12630 server.request_body("/api/worker/register")["workflow_command_contracts"],
12631 contracts
12632 );
12633 }
12634
12635 #[tokio::test]
12636 async fn role_scoped_tokens_are_never_used_for_the_opposite_plane() {
12637 let server = MockWorkerServer::start();
12638 let control_only = Client::builder(server.base_url())
12639 .control_token(Some("control-secret".to_string()))
12640 .build()
12641 .expect("control client");
12642
12643 let error = control_only
12644 .register_worker("worker", "queue", vec![], vec![], 1, 1)
12645 .await
12646 .expect_err("control token must not authorize a worker request");
12647 assert!(matches!(
12648 error,
12649 Error::MissingRoleCredentials { role: "worker", .. }
12650 ));
12651 assert_eq!(server.request_count("/api/worker/register"), 0);
12652
12653 let worker_only = Client::builder(server.base_url())
12654 .worker_token(Some("worker-secret".to_string()))
12655 .build()
12656 .expect("worker client");
12657 let error = worker_only
12658 .health()
12659 .await
12660 .expect_err("worker token must not authorize a control request");
12661 assert!(matches!(
12662 error,
12663 Error::MissingRoleCredentials {
12664 role: "control",
12665 ..
12666 }
12667 ));
12668 assert_eq!(server.request_count("/api/health"), 0);
12669 }
12670
12671 #[tokio::test]
12672 async fn shared_token_supports_worker_and_control_planes() {
12673 let server = MockWorkerServer::start();
12674 let client = Client::builder(server.base_url())
12675 .token(Some("shared-secret".to_string()))
12676 .build()
12677 .expect("client");
12678
12679 client.health().await.expect("control request");
12680 client
12681 .register_worker("worker", "queue", vec![], vec![], 1, 1)
12682 .await
12683 .expect("worker request");
12684
12685 assert_eq!(
12686 server.authorization_for("/api/health").as_deref(),
12687 Some("Bearer shared-secret")
12688 );
12689 assert_eq!(
12690 server.control_protocol_for("/api/health").as_deref(),
12691 Some(CONTROL_PLANE_VERSION)
12692 );
12693 assert_eq!(
12694 server.authorization_for("/api/worker/register").as_deref(),
12695 Some("Bearer shared-secret")
12696 );
12697 assert_eq!(
12698 server
12699 .worker_protocol_for("/api/worker/register")
12700 .as_deref(),
12701 Some(WORKER_PROTOCOL_VERSION)
12702 );
12703 }
12704
12705 #[tokio::test]
12706 async fn baseline_worker_endpoints_send_the_baseline_protocol() {
12707 let server = MockWorkerServer::start();
12708 let client = Client::builder(server.base_url())
12709 .timeout(Duration::from_secs(2))
12710 .build()
12711 .expect("client");
12712
12713 client
12714 .register_worker("capture-worker", "capture", vec![], vec![], 1, 1)
12715 .await
12716 .expect("register");
12717 client
12718 .heartbeat_worker("capture-worker", 1, 1)
12719 .await
12720 .expect("heartbeat");
12721 client
12722 .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
12723 .await
12724 .expect("workflow poll");
12725 client
12726 .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
12727 .await
12728 .expect("activity poll");
12729
12730 for path in [
12731 "/api/worker/register",
12732 "/api/worker/heartbeat",
12733 "/api/worker/workflow-tasks/poll",
12734 "/api/worker/activity-tasks/poll",
12735 ] {
12736 assert_eq!(
12737 server.worker_protocol_for(path).as_deref(),
12738 Some(WORKER_PROTOCOL_VERSION),
12739 "unexpected protocol for {path}"
12740 );
12741 }
12742
12743 assert_eq!(
12744 server.request_body("/api/worker/workflow-tasks/poll")["timeout_seconds"],
12745 1
12746 );
12747 assert_eq!(
12748 server.request_body("/api/worker/activity-tasks/poll")["timeout_seconds"],
12749 1
12750 );
12751 assert!(
12752 server.request_body("/api/worker/workflow-tasks/poll")["poll_request_id"]
12753 .as_str()
12754 .is_some_and(|id| id.starts_with("rust-workflow-poll-"))
12755 );
12756 assert!(
12757 server.request_body("/api/worker/activity-tasks/poll")["poll_request_id"]
12758 .as_str()
12759 .is_some_and(|id| id.starts_with("rust-activity-poll-"))
12760 );
12761 }
12762
12763 #[tokio::test]
12764 async fn query_task_endpoints_send_the_query_feature_protocol() {
12765 let server = MockWorkerServer::start();
12766 let client = Client::builder(server.base_url())
12767 .timeout(Duration::from_secs(2))
12768 .build()
12769 .expect("client");
12770
12771 client
12772 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
12773 .await
12774 .expect("query poll");
12775 client
12776 .complete_query_task("query-capture", "capture-worker", 1, json!(8), JSON_CODEC)
12777 .await
12778 .expect("query complete");
12779 client
12780 .fail_query_task(
12781 "query-capture",
12782 "capture-worker",
12783 1,
12784 "failed",
12785 "query_rejected",
12786 "QueryFailed",
12787 )
12788 .await
12789 .expect("query fail");
12790
12791 for path in [
12792 "/api/worker/query-tasks/poll",
12793 "/api/worker/query-tasks/query-capture/complete",
12794 "/api/worker/query-tasks/query-capture/fail",
12795 ] {
12796 assert_eq!(
12797 server.worker_protocol_for(path).as_deref(),
12798 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
12799 "unexpected protocol for {path}"
12800 );
12801 }
12802
12803 assert_eq!(
12804 server.request_body("/api/worker/query-tasks/poll")["timeout_seconds"],
12805 1
12806 );
12807 assert!(
12808 server.request_body("/api/worker/query-tasks/poll")["poll_request_id"]
12809 .as_str()
12810 .is_some_and(|id| id.starts_with("rust-query-poll-"))
12811 );
12812 }
12813
12814 #[tokio::test]
12815 async fn disconnected_client_polls_retry_once_with_the_same_request_id() {
12816 let server = MockWorkerServer::transient_worker_failures();
12817 let client = Client::builder(server.base_url())
12818 .timeout(Duration::from_secs(2))
12819 .build()
12820 .expect("client");
12821
12822 client
12823 .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
12824 .await
12825 .expect("workflow poll retry");
12826 client
12827 .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
12828 .await
12829 .expect("activity poll retry");
12830 client
12831 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
12832 .await
12833 .expect("query poll retry");
12834
12835 for path in [
12836 "/api/worker/workflow-tasks/poll",
12837 "/api/worker/activity-tasks/poll",
12838 "/api/worker/query-tasks/poll",
12839 ] {
12840 let bodies = server.request_bodies(path);
12841 assert_eq!(bodies.len(), 2, "{path} must be retried once");
12842 assert_eq!(
12843 bodies[0]["poll_request_id"], bodies[1]["poll_request_id"],
12844 "{path} must preserve the request binding across retry"
12845 );
12846 }
12847 }
12848
12849 #[tokio::test]
12850 async fn worker_poll_retries_preserve_request_id_across_consecutive_failures() {
12851 let server = MockWorkerServer::consecutive_poll_failures(2);
12852 let client = Client::builder(server.base_url())
12853 .timeout(Duration::from_secs(2))
12854 .build()
12855 .expect("client");
12856 let mut worker = Worker::new(client, "capture")
12857 .worker_id("capture-worker")
12858 .poll_timeout(Duration::from_millis(10))
12859 .retry_policy(WorkerRetryPolicy {
12860 max_retries: 2,
12861 initial_backoff: Duration::from_millis(1),
12862 max_backoff: Duration::from_millis(1),
12863 });
12864 worker.register_workflow(
12865 "capture.workflow",
12866 |_ctx, _input| async move { Ok(Value::Null) },
12867 );
12868 worker.register_activity(
12869 "capture.activity",
12870 |_ctx, _input| async move { Ok(Value::Null) },
12871 );
12872 worker.register_query("capture.workflow", "current", |_ctx, _args| async move {
12873 Ok(Value::Null)
12874 });
12875
12876 assert_eq!(worker.run_once().await.expect("poll retries"), 0);
12877
12878 for path in [
12879 "/api/worker/workflow-tasks/poll",
12880 "/api/worker/activity-tasks/poll",
12881 "/api/worker/query-tasks/poll",
12882 ] {
12883 let bodies = server.request_bodies(path);
12884 assert_eq!(bodies.len(), 3, "{path} must use exactly two retries");
12885 assert!(
12886 bodies
12887 .iter()
12888 .all(|body| body["poll_request_id"] == bodies[0]["poll_request_id"]),
12889 "{path} must preserve one request binding across every retry"
12890 );
12891 }
12892 }
12893
12894 #[tokio::test]
12895 async fn query_protocol_rejection_from_older_server_is_typed() {
12896 let server = MockWorkerServer::reject_query_protocol();
12897 let client = Client::builder(server.base_url())
12898 .timeout(Duration::from_secs(2))
12899 .build()
12900 .expect("client");
12901
12902 let error = client
12903 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
12904 .await
12905 .expect_err("server below query protocol floor must reject");
12906 let Error::Protocol(failure) = error else {
12907 panic!("expected typed protocol failure");
12908 };
12909
12910 assert_eq!(failure.status, 400);
12911 assert_eq!(failure.reason, "unsupported_protocol_version");
12912 assert_eq!(failure.supported_version.as_deref(), Some("1.7"));
12913 assert_eq!(
12914 failure.requested_version.as_deref(),
12915 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
12916 );
12917 assert_eq!(
12918 server
12919 .worker_protocol_for("/api/worker/query-tasks/poll")
12920 .as_deref(),
12921 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
12922 );
12923 }
12924
12925 #[tokio::test]
12926 async fn run_once_without_query_handlers_keeps_pre_query_server_compatibility() {
12927 let server = MockWorkerServer::reject_query_protocol();
12928 let client = Client::builder(server.base_url())
12929 .timeout(Duration::from_secs(2))
12930 .build()
12931 .expect("client");
12932 let mut worker = Worker::new(client, "rust-workers")
12933 .worker_id("baseline-worker")
12934 .poll_timeout(Duration::from_millis(10));
12935
12936 worker.register_workflow("baseline.workflow", |_ctx, _input| async move {
12937 Ok(Value::Null)
12938 });
12939
12940 assert_eq!(worker.run_once().await.expect("baseline run once"), 0);
12941 assert_eq!(
12942 server
12943 .worker_protocol_for("/api/worker/workflow-tasks/poll")
12944 .as_deref(),
12945 Some(WORKER_PROTOCOL_VERSION)
12946 );
12947 assert_eq!(
12948 server.worker_protocol_for("/api/worker/query-tasks/poll"),
12949 None,
12950 "a worker without query handlers must not use the query-task endpoint"
12951 );
12952 }
12953
12954 #[tokio::test]
12955 async fn completion_time_query_rejection_is_typed_without_stopping_worker() {
12956 let server = MockWorkerServer::reject_query_completion();
12957 let client = Client::builder(server.base_url())
12958 .timeout(Duration::from_secs(2))
12959 .build()
12960 .expect("client");
12961
12962 let error = client
12963 .complete_query_task("query-late", "late-worker", 1, json!(8), JSON_CODEC)
12964 .await
12965 .expect_err("expired completion must be rejected");
12966 let Error::QueryFailed(failure) = error else {
12967 panic!("expected typed query failure");
12968 };
12969 assert_eq!(failure.status, 409);
12970 assert_eq!(failure.reason, "query_task_timed_out");
12971
12972 let mut worker = Worker::new(client, "rust-workers")
12973 .worker_id("late-worker")
12974 .poll_timeout(Duration::from_millis(10));
12975 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
12976 worker.register_query(
12977 "counter",
12978 "current",
12979 |_ctx, _args| async move { Ok(json!(8)) },
12980 );
12981
12982 assert_eq!(worker.run_once().await.expect("late task is handled"), 1);
12983 assert_eq!(
12984 worker
12985 .run_once()
12986 .await
12987 .expect("worker continues after late completion"),
12988 0
12989 );
12990 assert_eq!(
12991 server.request_count("/api/worker/query-tasks/query-late/complete"),
12992 2
12993 );
12994 assert_eq!(
12995 server.request_count("/api/worker/query-tasks/query-late/fail"),
12996 0,
12997 "a server completion rejection must not be reported as an encoding failure"
12998 );
12999 }
13000
13001 #[tokio::test]
13002 async fn normal_shutdown_joins_pollers_and_deregisters_once() {
13003 let server = MockWorkerServer::start();
13004 let client = Client::builder(server.base_url())
13005 .timeout(Duration::from_secs(2))
13006 .build()
13007 .expect("client");
13008 let mut worker = Worker::new(client, "rust-workers")
13009 .worker_id("joined-worker")
13010 .poll_timeout(Duration::from_millis(10));
13011 worker.register_workflow(
13012 "joined.workflow",
13013 |_ctx, _input| async move { Ok(Value::Null) },
13014 );
13015 worker.register_activity(
13016 "joined.activity",
13017 |_ctx, _input| async move { Ok(Value::Null) },
13018 );
13019 worker.register_query("joined.workflow", "state", |_ctx, _input| async move {
13020 Ok(Value::Null)
13021 });
13022
13023 worker
13024 .run_until(tokio::time::sleep(Duration::from_millis(20)))
13025 .await
13026 .expect("normal shutdown");
13027
13028 let deregistration_path = "/api/worker/registrations/mock-worker";
13029 assert_eq!(server.request_count(deregistration_path), 1);
13030 for poll_path in [
13031 "/api/worker/workflow-tasks/poll",
13032 "/api/worker/activity-tasks/poll",
13033 "/api/worker/query-tasks/poll",
13034 ] {
13035 assert!(server.request_count(poll_path) > 0, "missing {poll_path}");
13036 }
13037 assert_eq!(
13038 server.captured_paths().last().map(String::as_str),
13039 Some(deregistration_path),
13040 "deregistration must start only after every poller has joined"
13041 );
13042 }
13043
13044 #[tokio::test]
13045 async fn registration_failure_does_not_deregister() {
13046 let server = MockWorkerServer::rejected_registration();
13047 let client = Client::builder(server.base_url())
13048 .timeout(Duration::from_secs(2))
13049 .build()
13050 .expect("client");
13051 let worker = Worker::new(client, "rust-workers").worker_id("never-registered");
13052
13053 let error = worker
13054 .run_until(async {})
13055 .await
13056 .expect_err("registration must fail");
13057 assert!(matches!(
13058 error,
13059 Error::Http {
13060 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
13061 ..
13062 }
13063 ));
13064 assert!(server
13065 .captured_paths()
13066 .iter()
13067 .all(|path| !path.starts_with("/api/worker/registrations/")));
13068 }
13069
13070 #[tokio::test]
13071 async fn declined_registration_does_not_deregister() {
13072 let server = MockWorkerServer::declined_registration();
13073 let client = Client::builder(server.base_url())
13074 .timeout(Duration::from_secs(2))
13075 .build()
13076 .expect("client");
13077 let worker = Worker::new(client, "rust-workers").worker_id("declined-worker");
13078
13079 let error = worker
13080 .run_until(async {})
13081 .await
13082 .expect_err("declined registration must fail");
13083 assert!(matches!(error, Error::WorkerLoop(_)));
13084 assert!(error.to_string().contains("was not accepted"));
13085 assert!(server
13086 .captured_paths()
13087 .iter()
13088 .all(|path| !path.starts_with("/api/worker/registrations/")));
13089 }
13090
13091 #[tokio::test]
13092 async fn deregistration_http_failure_is_returned_after_normal_shutdown() {
13093 let server = MockWorkerServer::rejected_deregistration();
13094 let client = Client::builder(server.base_url())
13095 .timeout(Duration::from_secs(2))
13096 .build()
13097 .expect("client");
13098 let worker = Worker::new(client, "rust-workers").worker_id("forbidden-cleanup");
13099
13100 let error = worker
13101 .run_until(async {})
13102 .await
13103 .expect_err("deregistration must fail");
13104 assert!(matches!(
13105 error,
13106 Error::Http {
13107 status: reqwest::StatusCode::FORBIDDEN,
13108 ..
13109 }
13110 ));
13111 assert_eq!(
13112 server.request_count("/api/worker/registrations/mock-worker"),
13113 1
13114 );
13115 }
13116
13117 #[tokio::test]
13118 async fn deregistration_protocol_failure_is_returned_after_normal_shutdown() {
13119 let server = MockWorkerServer::rejected_deregistration_protocol();
13120 let client = Client::builder(server.base_url())
13121 .timeout(Duration::from_secs(2))
13122 .build()
13123 .expect("client");
13124 let worker = Worker::new(client, "rust-workers").worker_id("protocol-cleanup");
13125
13126 let error = worker
13127 .run_until(async {})
13128 .await
13129 .expect_err("protocol rejection must fail shutdown");
13130 let Error::Protocol(failure) = error else {
13131 panic!("expected typed protocol failure");
13132 };
13133 assert_eq!(failure.reason, "unsupported_protocol_version");
13134 assert_eq!(failure.requested_version.as_deref(), Some("1.2"));
13135 assert_eq!(
13136 server.request_count("/api/worker/registrations/mock-worker"),
13137 1
13138 );
13139 }
13140
13141 #[tokio::test]
13142 async fn primary_poller_error_retains_deregistration_failure_context() {
13143 let server = MockWorkerServer::unauthorized_polls_and_rejected_deregistration();
13144 let client = Client::builder(server.base_url())
13145 .timeout(Duration::from_secs(2))
13146 .build()
13147 .expect("client");
13148 let mut worker = Worker::new(client, "rust-workers")
13149 .worker_id("combined-failure")
13150 .poll_timeout(Duration::from_millis(10));
13151 worker.register_workflow("combined.workflow", |_ctx, _input| async move {
13152 Ok(Value::Null)
13153 });
13154
13155 let error = worker
13156 .run()
13157 .await
13158 .expect_err("worker and cleanup must fail");
13159 let summary = error.to_string();
13160 assert!(summary.contains("authentication_failed"));
13161 assert!(summary.contains("worker cannot deregister"));
13162 let Error::WorkerShutdown {
13163 primary,
13164 deregistration,
13165 } = error
13166 else {
13167 panic!("expected combined worker shutdown error");
13168 };
13169 assert!(matches!(
13170 *primary,
13171 Error::Http {
13172 status: reqwest::StatusCode::UNAUTHORIZED,
13173 ..
13174 }
13175 ));
13176 assert!(matches!(
13177 *deregistration,
13178 Error::Http {
13179 status: reqwest::StatusCode::FORBIDDEN,
13180 ..
13181 }
13182 ));
13183 assert_eq!(
13184 server.request_count("/api/worker/registrations/mock-worker"),
13185 1
13186 );
13187 }
13188
13189 #[tokio::test]
13190 async fn activity_only_worker_can_shutdown_without_workflow_poller() {
13191 let server = MockWorkerServer::start();
13192 let client = Client::builder(server.base_url())
13193 .timeout(Duration::from_secs(2))
13194 .build()
13195 .expect("client");
13196 let mut worker = Worker::new(client, "rust-workers")
13197 .worker_id("activity-only-worker")
13198 .poll_timeout(Duration::from_millis(10));
13199
13200 worker.register_activity(
13201 "activity.only",
13202 |_ctx, _args| async move { Ok(Value::Null) },
13203 );
13204
13205 worker.run_until(async {}).await.expect("run worker");
13206 }
13207
13208 #[tokio::test]
13209 async fn workflow_only_worker_can_shutdown_without_activity_poller() {
13210 let server = MockWorkerServer::start();
13211 let client = Client::builder(server.base_url())
13212 .timeout(Duration::from_secs(2))
13213 .build()
13214 .expect("client");
13215 let mut worker = Worker::new(client, "rust-workers")
13216 .worker_id("workflow-only-worker")
13217 .poll_timeout(Duration::from_millis(10));
13218
13219 worker.register_workflow(
13220 "workflow.only",
13221 |_ctx, _input| async move { Ok(Value::Null) },
13222 );
13223
13224 worker.run_until(async {}).await.expect("run worker");
13225 }
13226
13227 #[tokio::test]
13228 async fn worker_heartbeat_observer_receives_server_acknowledgements() {
13229 let server = MockWorkerServer::start();
13230 let client = Client::builder(server.base_url())
13231 .timeout(Duration::from_secs(2))
13232 .build()
13233 .expect("client");
13234 let observations = Arc::new(Mutex::new(Vec::new()));
13235 let observed = Arc::clone(&observations);
13236 let mut worker = Worker::new(client, "rust-workers")
13237 .worker_id("observed-heartbeat-worker")
13238 .poll_timeout(Duration::from_millis(10))
13239 .on_worker_heartbeat(move |observation| {
13240 observed
13241 .lock()
13242 .expect("heartbeat observations")
13243 .push(observation.clone());
13244 });
13245
13246 worker.register_workflow("workflow.observed", |_ctx, _input| async move {
13247 Ok(Value::Null)
13248 });
13249 let acknowledged = Arc::clone(&observations);
13250 worker
13251 .run_until(async move {
13252 tokio::time::timeout(Duration::from_secs(2), async move {
13253 loop {
13254 if !acknowledged
13255 .lock()
13256 .expect("heartbeat observations")
13257 .is_empty()
13258 {
13259 break;
13260 }
13261 tokio::time::sleep(Duration::from_millis(1)).await;
13262 }
13263 })
13264 .await
13265 .expect("heartbeat acknowledgement within timeout");
13266 })
13267 .await
13268 .expect("run worker");
13269
13270 let observations = observations.lock().expect("heartbeat observations");
13271 let first = observations.first().expect("heartbeat acknowledgement");
13272 assert_eq!(first.worker_id, "observed-heartbeat-worker");
13273 assert_eq!(first.task_queue, "rust-workers");
13274 assert!(first.acknowledged_at_unix_millis > 0);
13275 assert_eq!(first.acknowledgement, json!({}));
13276 }
13277
13278 #[tokio::test]
13279 async fn delayed_worker_heartbeat_keeps_cadence_and_pollers_live() {
13280 let server = MockWorkerServer::delayed_heartbeat_worker();
13281 let client = Client::builder(server.base_url())
13282 .timeout(Duration::from_secs(3))
13283 .build()
13284 .expect("client");
13285 let observations = Arc::new(Mutex::new(Vec::new()));
13286 let observed = Arc::clone(&observations);
13287 let mut worker = Worker::new(client, "rust-snapshot-workers")
13288 .worker_id("rust-snapshot-worker")
13289 .poll_timeout(Duration::from_millis(10))
13290 .on_worker_heartbeat(move |observation| {
13291 observed
13292 .lock()
13293 .expect("heartbeat observations")
13294 .push(observation.clone());
13295 });
13296
13297 worker.register_workflow("snapshot", |ctx, _input| async move {
13298 ctx.wait_signal("finish").await?;
13299 Ok(json!({"status": "finished"}))
13300 });
13301 worker.register_query("snapshot", "current", |ctx, _args| async move {
13302 Ok(json!(ctx
13303 .signals("increment")
13304 .iter()
13305 .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
13306 .sum::<i64>()))
13307 });
13308 worker.register_activity("cancel-aware", |_ctx, _args| async move {
13309 Ok(json!({"late": "completion"}))
13310 });
13311
13312 worker
13313 .run_until(tokio::time::sleep(Duration::from_millis(3_800)))
13314 .await
13315 .expect("delayed heartbeat must allow a clean worker shutdown");
13316
13317 let observations = observations.lock().expect("heartbeat observations");
13318 assert!(
13319 observations.len() >= 3,
13320 "the immediate heartbeat, delayed acknowledgement, and next cadence heartbeat must complete"
13321 );
13322 assert!(
13323 observations.windows(2).all(|pair| {
13324 pair[1].acknowledged_at_unix_millis
13325 .saturating_sub(pair[0].acknowledged_at_unix_millis)
13326 >= 850
13327 }),
13328 "successful acknowledgements must not catch up faster than the advertised one-second cadence: {observations:?}"
13329 );
13330 drop(observations);
13331
13332 let heartbeat_times = server.request_times("/api/worker/heartbeat");
13333 let delayed_request_at = *heartbeat_times
13334 .get(1)
13335 .expect("intentionally delayed heartbeat request");
13336 let delay_window_start = delayed_request_at + Duration::from_millis(100);
13337 let delay_window_end = delayed_request_at + Duration::from_millis(1_400);
13338 for path in [
13339 "/api/worker/workflow-tasks/poll",
13340 "/api/worker/activity-tasks/poll",
13341 "/api/worker/query-tasks/poll",
13342 ] {
13343 assert!(
13344 server
13345 .request_times(path)
13346 .iter()
13347 .any(|received_at| *received_at >= delay_window_start
13348 && *received_at <= delay_window_end),
13349 "{path} must keep polling while a heartbeat acknowledgement is delayed"
13350 );
13351 }
13352 assert!(
13353 server.request_count("/api/worker/workflow-tasks/snapshot-wait-3/fail") >= 1,
13354 "workflow work must be settled"
13355 );
13356 assert!(
13357 server.request_count("/api/worker/activity-tasks/activity-cancel/complete") >= 1,
13358 "activity work must be settled"
13359 );
13360 assert!(
13361 server.request_count("/api/worker/query-tasks/snapshot-current/complete") >= 1,
13362 "query work must be settled"
13363 );
13364 }
13365
13366 #[tokio::test]
13367 async fn retried_worker_heartbeat_restarts_the_advertised_cadence() {
13368 let server = MockWorkerServer::heartbeat_retry_worker();
13369 let client = Client::builder(server.base_url())
13370 .timeout(Duration::from_secs(2))
13371 .build()
13372 .expect("client");
13373 let observations = Arc::new(Mutex::new(Vec::new()));
13374 let observed = Arc::clone(&observations);
13375 let worker = Worker::new(client, "rust-workers")
13376 .worker_id("heartbeat-retry-worker")
13377 .retry_policy(WorkerRetryPolicy {
13378 max_retries: 1,
13379 initial_backoff: Duration::from_millis(300),
13380 max_backoff: Duration::from_millis(300),
13381 })
13382 .on_worker_heartbeat(move |observation| {
13383 observed
13384 .lock()
13385 .expect("heartbeat observations")
13386 .push(observation.clone());
13387 });
13388
13389 worker
13390 .run_until(tokio::time::sleep(Duration::from_millis(2_700)))
13391 .await
13392 .expect("retryable heartbeat failure must remain bounded and recover");
13393
13394 let observations = observations.lock().expect("heartbeat observations");
13395 assert!(observations.len() >= 3, "heartbeat retry must recover");
13396 assert!(
13397 observations.windows(2).all(|pair| {
13398 pair[1]
13399 .acknowledged_at_unix_millis
13400 .saturating_sub(pair[0].acknowledged_at_unix_millis)
13401 >= 850
13402 }),
13403 "a successful retry must start a fresh advertised cadence: {observations:?}"
13404 );
13405 assert_eq!(
13406 server.request_count("/api/worker/heartbeat"),
13407 observations.len() + 1,
13408 "one retryable failure must add exactly one bounded request"
13409 );
13410 }
13411
13412 #[tokio::test]
13413 async fn query_enabled_worker_ignores_unmatched_signals_then_completes_once() {
13414 let server = MockWorkerServer::waiting_query_worker();
13415 let client = Client::builder(server.base_url())
13416 .timeout(Duration::from_secs(2))
13417 .build()
13418 .expect("client");
13419 let observations = Arc::new(Mutex::new(Vec::new()));
13420 let observed = Arc::clone(&observations);
13421 let mut worker = Worker::new(client, "rust-snapshot-workers")
13422 .worker_id("rust-snapshot-worker")
13423 .poll_timeout(Duration::from_millis(10))
13424 .on_worker_heartbeat(move |observation| {
13425 observed
13426 .lock()
13427 .expect("heartbeat observations")
13428 .push(observation.clone());
13429 });
13430
13431 worker.register_workflow("snapshot", |ctx, _input| async move {
13432 ctx.wait_signal("finish").await?;
13433 Ok(json!({"status": "finished"}))
13434 });
13435 worker.register_query("snapshot", "current", |ctx, _args| async move {
13436 let current = ctx
13437 .signals("increment")
13438 .iter()
13439 .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
13440 .sum::<i64>();
13441 Ok(json!(current))
13442 });
13443 worker.register_update("snapshot", "replace", |_ctx, args| async move { Ok(args) });
13444
13445 worker
13446 .run_until(tokio::time::sleep(Duration::from_millis(3_200)))
13447 .await
13448 .expect("pending workflow and query poller must remain live until shutdown");
13449
13450 assert!(
13451 observations.lock().expect("heartbeat observations").len() >= 4,
13452 "the immediate heartbeat and at least three advertised one-second intervals must be acknowledged"
13453 );
13454 assert!(
13455 server.request_count("/api/worker/workflow-tasks/poll") >= 3,
13456 "workflow polling must continue after empty replay acknowledgements"
13457 );
13458 assert!(
13459 server.request_count("/api/worker/query-tasks/poll") >= 2,
13460 "query polling must continue after serving the current query"
13461 );
13462 assert_eq!(
13463 server.request_body("/api/worker/register")["capabilities"],
13464 json!([QUERY_TASKS_CAPABILITY, WORKFLOW_UPDATES_CAPABILITY])
13465 );
13466 assert_eq!(
13467 server.request_body("/api/worker/register")["workflow_command_contracts"]["snapshot"],
13468 json!({
13469 "queries": ["current"],
13470 "updates": ["replace"],
13471 "update_validators": [],
13472 })
13473 );
13474
13475 let opened = server.request_body("/api/worker/workflow-tasks/snapshot-open/complete");
13476 assert_eq!(
13477 opened["commands"],
13478 json!([{
13479 "type": "open_signal_wait",
13480 "signal_name": "finish",
13481 }])
13482 );
13483
13484 for task_id in ["snapshot-wait-3", "snapshot-wait-5"] {
13485 let fail_path = format!("/api/worker/workflow-tasks/{task_id}/fail");
13486 let completion_path = format!("/api/worker/workflow-tasks/{task_id}/complete");
13487 let failure = server.request_body(&fail_path);
13488 assert_eq!(
13489 failure["failure"]["type"],
13490 WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE
13491 );
13492 assert_eq!(server.request_count(&completion_path), 0);
13493 }
13494
13495 let query_completion =
13496 server.request_body("/api/worker/query-tasks/snapshot-current/complete");
13497 assert_eq!(query_completion["result"], json!(8));
13498
13499 let terminal_path = "/api/worker/workflow-tasks/snapshot-finish/complete";
13500 assert_eq!(
13501 server.request_count(terminal_path),
13502 1,
13503 "the matching signal must settle the workflow exactly once"
13504 );
13505 let terminal = server.request_body(terminal_path);
13506 assert_eq!(terminal["commands"].as_array().map(Vec::len), Some(1));
13507 assert_eq!(terminal["commands"][0]["type"], "complete_workflow");
13508 assert_eq!(
13509 decode_wire_value(&terminal["commands"][0]["result"], DEFAULT_CODEC)
13510 .expect("terminal workflow result"),
13511 json!({"status": "finished"})
13512 );
13513 }
13514
13515 #[tokio::test]
13516 async fn worker_retries_poll_and_heartbeat_transport_failures_independently() {
13517 let server = MockWorkerServer::transient_worker_failures();
13518 let client = Client::builder(server.base_url())
13519 .timeout(Duration::from_secs(2))
13520 .build()
13521 .expect("client");
13522 let mut worker = Worker::new(client, "rust-workers")
13523 .worker_id("retry-worker")
13524 .poll_timeout(Duration::from_millis(10))
13525 .retry_policy(WorkerRetryPolicy {
13526 max_retries: 2,
13527 initial_backoff: Duration::from_millis(1),
13528 max_backoff: Duration::from_millis(1),
13529 });
13530 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
13531 worker.register_activity(
13532 "counter.activity",
13533 |_ctx, _input| async move { Ok(Value::Null) },
13534 );
13535 worker.register_query(
13536 "counter",
13537 "current",
13538 |_ctx, _args| async move { Ok(json!(8)) },
13539 );
13540
13541 worker
13542 .run_until(tokio::time::sleep(Duration::from_millis(75)))
13543 .await
13544 .expect("transient failures must not stop the worker");
13545
13546 for path in [
13547 "/api/worker/heartbeat",
13548 "/api/worker/workflow-tasks/poll",
13549 "/api/worker/activity-tasks/poll",
13550 "/api/worker/query-tasks/poll",
13551 ] {
13552 assert!(
13553 server.request_count(path) >= 2,
13554 "{path} must continue after its transient failure"
13555 );
13556 }
13557 }
13558
13559 #[tokio::test]
13560 async fn worker_bounds_transport_retries() {
13561 let server = MockWorkerServer::unavailable_polls();
13562 let client = Client::builder(server.base_url())
13563 .timeout(Duration::from_secs(2))
13564 .build()
13565 .expect("client");
13566 let mut worker = Worker::new(client, "rust-workers")
13567 .worker_id("bounded-retry-worker")
13568 .poll_timeout(Duration::from_millis(10))
13569 .retry_policy(WorkerRetryPolicy {
13570 max_retries: 2,
13571 initial_backoff: Duration::from_millis(1),
13572 max_backoff: Duration::from_millis(1),
13573 });
13574 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
13575
13576 let error = worker.run().await.expect_err("retry bound must terminate");
13577 assert!(matches!(error, Error::Transport(_)));
13578 assert_eq!(
13579 server.request_count("/api/worker/workflow-tasks/poll"),
13580 3,
13581 "one initial request plus exactly two retries"
13582 );
13583 }
13584
13585 #[tokio::test]
13586 async fn worker_retry_policy_can_disable_poll_retries() {
13587 let server = MockWorkerServer::unavailable_polls();
13588 let client = Client::builder(server.base_url())
13589 .timeout(Duration::from_secs(2))
13590 .build()
13591 .expect("client");
13592 let mut worker = Worker::new(client, "rust-workers")
13593 .worker_id("no-retry-worker")
13594 .poll_timeout(Duration::from_millis(10))
13595 .retry_policy(WorkerRetryPolicy {
13596 max_retries: 0,
13597 initial_backoff: Duration::from_millis(1),
13598 max_backoff: Duration::from_millis(1),
13599 });
13600 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
13601
13602 let error = worker
13603 .run_once()
13604 .await
13605 .expect_err("disabled retries must return the first transport failure");
13606 assert!(matches!(error, Error::Transport(_)));
13607 assert_eq!(
13608 server.request_count("/api/worker/workflow-tasks/poll"),
13609 1,
13610 "max_retries=0 must send only the initial request"
13611 );
13612 }
13613
13614 #[tokio::test]
13615 async fn worker_does_not_retry_authentication_failures() {
13616 let server = MockWorkerServer::unauthorized_polls();
13617 let client = Client::builder(server.base_url())
13618 .timeout(Duration::from_secs(2))
13619 .build()
13620 .expect("client");
13621 let mut worker = Worker::new(client, "rust-workers")
13622 .worker_id("unauthorized-worker")
13623 .poll_timeout(Duration::from_millis(10));
13624 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
13625
13626 let error = worker
13627 .run()
13628 .await
13629 .expect_err("authentication must terminate");
13630 let Error::Http { status, body } = error else {
13631 panic!("expected stable HTTP authentication error");
13632 };
13633 assert_eq!(status, reqwest::StatusCode::UNAUTHORIZED);
13634 assert!(body.contains("authentication_failed"));
13635 assert_eq!(
13636 server.request_count("/api/worker/workflow-tasks/poll"),
13637 1,
13638 "authentication failures must not be retried"
13639 );
13640 }
13641
13642 #[derive(Clone, Debug)]
13643 struct CapturedRequest {
13644 method: String,
13645 path: String,
13646 authorization: Option<String>,
13647 namespace: Option<String>,
13648 worker_protocol: Option<String>,
13649 control_protocol: Option<String>,
13650 body: String,
13651 received_at: Instant,
13652 }
13653
13654 struct MockWorkerServer {
13655 addr: SocketAddr,
13656 stop: Arc<AtomicBool>,
13657 requests: Arc<Mutex<Vec<CapturedRequest>>>,
13658 thread: Option<thread::JoinHandle<()>>,
13659 }
13660
13661 #[derive(Clone, Copy, Default)]
13662 struct MockWorkerBehavior {
13663 reject_query_protocol: bool,
13664 reject_query_completion: bool,
13665 waiting_query_worker: bool,
13666 decline_registration: bool,
13667 complete_named_signal: bool,
13668 poll_failures_per_path: usize,
13669 heartbeat_failures: usize,
13670 heartbeat_failure_request: Option<usize>,
13671 delayed_heartbeat_request: Option<usize>,
13672 heartbeat_response_delay: Duration,
13673 concurrent_requests: bool,
13674 unauthorized_polls: bool,
13675 reject_registration: bool,
13676 reject_deregistration: bool,
13677 reject_deregistration_protocol: bool,
13678 cancelled_activity: bool,
13679 draining_polls: bool,
13680 workflow_completion_status: Option<&'static str>,
13681 workflow_completion_body: Option<&'static str>,
13682 }
13683
13684 impl MockWorkerServer {
13685 fn start() -> Self {
13686 Self::start_with_behavior(MockWorkerBehavior::default())
13687 }
13688
13689 fn reject_query_protocol() -> Self {
13690 Self::start_with_behavior(MockWorkerBehavior {
13691 reject_query_protocol: true,
13692 ..MockWorkerBehavior::default()
13693 })
13694 }
13695
13696 fn reject_query_completion() -> Self {
13697 Self::start_with_behavior(MockWorkerBehavior {
13698 reject_query_completion: true,
13699 ..MockWorkerBehavior::default()
13700 })
13701 }
13702
13703 fn waiting_query_worker() -> Self {
13704 Self::start_with_behavior(MockWorkerBehavior {
13705 waiting_query_worker: true,
13706 complete_named_signal: true,
13707 ..MockWorkerBehavior::default()
13708 })
13709 }
13710
13711 fn transient_worker_failures() -> Self {
13712 Self::start_with_behavior(MockWorkerBehavior {
13713 poll_failures_per_path: 1,
13714 heartbeat_failures: 1,
13715 ..MockWorkerBehavior::default()
13716 })
13717 }
13718
13719 fn consecutive_poll_failures(count: usize) -> Self {
13720 Self::start_with_behavior(MockWorkerBehavior {
13721 poll_failures_per_path: count,
13722 ..MockWorkerBehavior::default()
13723 })
13724 }
13725
13726 fn delayed_heartbeat_worker() -> Self {
13727 Self::start_with_behavior(MockWorkerBehavior {
13728 waiting_query_worker: true,
13729 delayed_heartbeat_request: Some(2),
13730 heartbeat_response_delay: Duration::from_millis(1_500),
13731 concurrent_requests: true,
13732 cancelled_activity: true,
13733 ..MockWorkerBehavior::default()
13734 })
13735 }
13736
13737 fn heartbeat_retry_worker() -> Self {
13738 Self::start_with_behavior(MockWorkerBehavior {
13739 waiting_query_worker: true,
13740 heartbeat_failure_request: Some(2),
13741 concurrent_requests: true,
13742 ..MockWorkerBehavior::default()
13743 })
13744 }
13745
13746 fn unavailable_polls() -> Self {
13747 Self::start_with_behavior(MockWorkerBehavior {
13748 poll_failures_per_path: usize::MAX,
13749 ..MockWorkerBehavior::default()
13750 })
13751 }
13752
13753 fn unauthorized_polls() -> Self {
13754 Self::start_with_behavior(MockWorkerBehavior {
13755 unauthorized_polls: true,
13756 ..MockWorkerBehavior::default()
13757 })
13758 }
13759
13760 fn rejected_registration() -> Self {
13761 Self::start_with_behavior(MockWorkerBehavior {
13762 reject_registration: true,
13763 ..MockWorkerBehavior::default()
13764 })
13765 }
13766
13767 fn declined_registration() -> Self {
13768 Self::start_with_behavior(MockWorkerBehavior {
13769 decline_registration: true,
13770 ..MockWorkerBehavior::default()
13771 })
13772 }
13773
13774 fn rejected_deregistration() -> Self {
13775 Self::start_with_behavior(MockWorkerBehavior {
13776 reject_deregistration: true,
13777 ..MockWorkerBehavior::default()
13778 })
13779 }
13780
13781 fn rejected_deregistration_protocol() -> Self {
13782 Self::start_with_behavior(MockWorkerBehavior {
13783 reject_deregistration_protocol: true,
13784 ..MockWorkerBehavior::default()
13785 })
13786 }
13787
13788 fn unauthorized_polls_and_rejected_deregistration() -> Self {
13789 Self::start_with_behavior(MockWorkerBehavior {
13790 unauthorized_polls: true,
13791 reject_deregistration: true,
13792 ..MockWorkerBehavior::default()
13793 })
13794 }
13795
13796 fn cancelled_activity() -> Self {
13797 Self::start_with_behavior(MockWorkerBehavior {
13798 cancelled_activity: true,
13799 ..MockWorkerBehavior::default()
13800 })
13801 }
13802
13803 fn draining_polls() -> Self {
13804 Self::start_with_behavior(MockWorkerBehavior {
13805 draining_polls: true,
13806 ..MockWorkerBehavior::default()
13807 })
13808 }
13809
13810 fn workflow_completion(status: &'static str, body: &'static str) -> Self {
13811 Self::start_with_behavior(MockWorkerBehavior {
13812 workflow_completion_status: Some(status),
13813 workflow_completion_body: Some(body),
13814 ..MockWorkerBehavior::default()
13815 })
13816 }
13817
13818 fn start_with_behavior(behavior: MockWorkerBehavior) -> Self {
13819 let listener = TcpListener::bind("127.0.0.1:0").expect("bind mock server");
13820 listener
13821 .set_nonblocking(true)
13822 .expect("configure mock listener");
13823 let addr = listener.local_addr().expect("mock server address");
13824 let stop = Arc::new(AtomicBool::new(false));
13825 let server_stop = Arc::clone(&stop);
13826 let requests = Arc::new(Mutex::new(Vec::new()));
13827 let server_requests = Arc::clone(&requests);
13828 let thread = thread::spawn(move || {
13829 let mut request_threads = Vec::new();
13830 while !server_stop.load(Ordering::SeqCst) {
13831 match listener.accept() {
13832 Ok((mut stream, _)) => {
13833 if behavior.concurrent_requests {
13834 let requests = Arc::clone(&server_requests);
13835 request_threads.push(thread::spawn(move || {
13836 handle_mock_worker_request(&mut stream, &requests, behavior)
13837 }));
13838 } else {
13839 handle_mock_worker_request(&mut stream, &server_requests, behavior);
13840 }
13841 }
13842 Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
13843 let mut index = 0;
13844 while index < request_threads.len() {
13845 if request_threads[index].is_finished() {
13846 request_threads
13847 .swap_remove(index)
13848 .join()
13849 .expect("join mock request");
13850 } else {
13851 index += 1;
13852 }
13853 }
13854 thread::sleep(Duration::from_millis(5));
13855 }
13856 Err(_) => break,
13857 }
13858 }
13859 for request_thread in request_threads {
13860 request_thread.join().expect("join mock request");
13861 }
13862 });
13863
13864 Self {
13865 addr,
13866 stop,
13867 requests,
13868 thread: Some(thread),
13869 }
13870 }
13871
13872 fn base_url(&self) -> String {
13873 format!("http://{}", self.addr)
13874 }
13875
13876 fn worker_protocol_for(&self, path: &str) -> Option<String> {
13877 self.requests
13878 .lock()
13879 .expect("captured requests")
13880 .iter()
13881 .find(|request| request.path == path)
13882 .and_then(|request| request.worker_protocol.clone())
13883 }
13884
13885 fn control_protocol_for(&self, path: &str) -> Option<String> {
13886 self.requests
13887 .lock()
13888 .expect("captured requests")
13889 .iter()
13890 .find(|request| request.path == path)
13891 .and_then(|request| request.control_protocol.clone())
13892 }
13893
13894 fn method_for(&self, path: &str) -> Option<String> {
13895 self.requests
13896 .lock()
13897 .expect("captured requests")
13898 .iter()
13899 .find(|request| request.path == path)
13900 .map(|request| request.method.clone())
13901 }
13902
13903 fn authorization_for(&self, path: &str) -> Option<String> {
13904 self.requests
13905 .lock()
13906 .expect("captured requests")
13907 .iter()
13908 .find(|request| request.path == path)
13909 .and_then(|request| request.authorization.clone())
13910 }
13911
13912 fn namespace_for(&self, path: &str) -> Option<String> {
13913 self.requests
13914 .lock()
13915 .expect("captured requests")
13916 .iter()
13917 .find(|request| request.path == path)
13918 .and_then(|request| request.namespace.clone())
13919 }
13920
13921 fn request_count(&self, path: &str) -> usize {
13922 self.requests
13923 .lock()
13924 .expect("captured requests")
13925 .iter()
13926 .filter(|request| request.path == path)
13927 .count()
13928 }
13929
13930 fn captured_paths(&self) -> Vec<String> {
13931 self.requests
13932 .lock()
13933 .expect("captured requests")
13934 .iter()
13935 .map(|request| request.path.clone())
13936 .collect()
13937 }
13938
13939 fn request_times(&self, path: &str) -> Vec<Instant> {
13940 self.requests
13941 .lock()
13942 .expect("captured requests")
13943 .iter()
13944 .filter(|request| request.path == path)
13945 .map(|request| request.received_at)
13946 .collect()
13947 }
13948
13949 fn request_body(&self, path: &str) -> Value {
13950 let requests = self.requests.lock().expect("captured requests");
13951 let body = &requests
13952 .iter()
13953 .find(|request| request.path == path)
13954 .unwrap_or_else(|| panic!("missing request for {path}"))
13955 .body;
13956 serde_json::from_str(body).unwrap_or_else(|error| {
13957 panic!("invalid JSON request body for {path}: {error}: {body:?}")
13958 })
13959 }
13960
13961 fn request_bodies(&self, path: &str) -> Vec<Value> {
13962 self.requests
13963 .lock()
13964 .expect("captured requests")
13965 .iter()
13966 .filter(|request| request.path == path)
13967 .map(|request| {
13968 serde_json::from_str(&request.body).unwrap_or_else(|error| {
13969 panic!(
13970 "invalid JSON request body for {path}: {error}: {:?}",
13971 request.body
13972 )
13973 })
13974 })
13975 .collect()
13976 }
13977 }
13978
13979 impl Drop for MockWorkerServer {
13980 fn drop(&mut self) {
13981 self.stop.store(true, Ordering::SeqCst);
13982 let _ = TcpStream::connect(self.addr);
13983
13984 if let Some(thread) = self.thread.take() {
13985 thread.join().expect("join mock server");
13986 }
13987 }
13988 }
13989
13990 fn handle_mock_worker_request(
13991 stream: &mut TcpStream,
13992 requests: &Arc<Mutex<Vec<CapturedRequest>>>,
13993 behavior: MockWorkerBehavior,
13994 ) {
13995 let _ = stream.set_read_timeout(Some(Duration::from_millis(200)));
13996 let mut buffer = [0_u8; 8192];
13997 let mut request = Vec::new();
13998
13999 loop {
14000 match stream.read(&mut buffer) {
14001 Ok(0) => break,
14002 Ok(read) => {
14003 request.extend_from_slice(&buffer[..read]);
14004 if mock_request_is_complete(&request) {
14005 break;
14006 }
14007 }
14008 Err(error)
14009 if matches!(
14010 error.kind(),
14011 std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
14012 ) =>
14013 {
14014 break;
14015 }
14016 Err(_) => return,
14017 }
14018 }
14019
14020 let request = String::from_utf8_lossy(&request);
14021 let body = request
14022 .split_once("\r\n\r\n")
14023 .map(|(_, body)| body)
14024 .unwrap_or_default();
14025 let path = request
14026 .lines()
14027 .next()
14028 .and_then(|line| line.split_whitespace().nth(1))
14029 .unwrap_or_default();
14030 let method = request
14031 .lines()
14032 .next()
14033 .and_then(|line| line.split_whitespace().next())
14034 .unwrap_or_default();
14035 let authorization = request.lines().find_map(|line| {
14036 let (name, value) = line.split_once(':')?;
14037 name.eq_ignore_ascii_case("Authorization")
14038 .then(|| value.trim().to_string())
14039 });
14040 let namespace = request.lines().find_map(|line| {
14041 let (name, value) = line.split_once(':')?;
14042 name.eq_ignore_ascii_case("X-Namespace")
14043 .then(|| value.trim().to_string())
14044 });
14045 let worker_protocol = request.lines().find_map(|line| {
14046 let (name, value) = line.split_once(':')?;
14047 name.eq_ignore_ascii_case("X-Durable-Workflow-Protocol-Version")
14048 .then(|| value.trim().to_string())
14049 });
14050 let control_protocol = request.lines().find_map(|line| {
14051 let (name, value) = line.split_once(':')?;
14052 name.eq_ignore_ascii_case("X-Durable-Workflow-Control-Plane-Version")
14053 .then(|| value.trim().to_string())
14054 });
14055 let request_number = {
14056 let mut requests = requests.lock().expect("captured requests");
14057 requests.push(CapturedRequest {
14058 method: method.to_string(),
14059 path: path.to_string(),
14060 authorization,
14061 namespace,
14062 worker_protocol: worker_protocol.clone(),
14063 control_protocol,
14064 body: body.to_string(),
14065 received_at: Instant::now(),
14066 });
14067 requests
14068 .iter()
14069 .filter(|request| request.path == path)
14070 .count()
14071 };
14072
14073 if behavior.reject_registration && path == "/api/worker/register" {
14074 write_mock_response(
14075 stream,
14076 "503 Service Unavailable",
14077 r#"{"reason":"registration_unavailable","message":"registration failed"}"#,
14078 );
14079 return;
14080 }
14081
14082 if path.starts_with("/api/worker/registrations/") {
14083 if behavior.reject_deregistration_protocol {
14084 write_mock_response(
14085 stream,
14086 "400 Bad Request",
14087 r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.1","requested_version":"1.2"}"#,
14088 );
14089 } else if behavior.reject_deregistration {
14090 write_mock_response(
14091 stream,
14092 "403 Forbidden",
14093 r#"{"reason":"authorization_failed","message":"worker cannot deregister"}"#,
14094 );
14095 } else {
14096 write_mock_response(
14097 stream,
14098 "200 OK",
14099 r#"{"worker_id":"deregistered-worker","outcome":"deregistered","recovered_workflow_task_count":2}"#,
14100 );
14101 }
14102 return;
14103 }
14104
14105 let is_poll = matches!(
14106 path,
14107 "/api/worker/workflow-tasks/poll"
14108 | "/api/worker/activity-tasks/poll"
14109 | "/api/worker/query-tasks/poll"
14110 );
14111 if is_poll && request_number <= behavior.poll_failures_per_path {
14112 return;
14113 }
14114 if path == "/api/worker/heartbeat" && request_number <= behavior.heartbeat_failures {
14115 return;
14116 }
14117 if path == "/api/worker/heartbeat"
14118 && behavior.heartbeat_failure_request == Some(request_number)
14119 {
14120 return;
14121 }
14122 if path == "/api/worker/heartbeat"
14123 && behavior.delayed_heartbeat_request == Some(request_number)
14124 {
14125 thread::sleep(behavior.heartbeat_response_delay);
14126 }
14127 if behavior.unauthorized_polls && is_poll {
14128 write_mock_response(
14129 stream,
14130 "401 Unauthorized",
14131 r#"{"reason":"authentication_failed","message":"invalid worker token"}"#,
14132 );
14133 return;
14134 }
14135 if behavior.draining_polls && is_poll {
14136 write_mock_response(
14137 stream,
14138 "409 Conflict",
14139 r#"{"task":null,"poll_status":"draining","reason":"worker_draining","worker_status":"draining","drain_intent":"draining"}"#,
14140 );
14141 return;
14142 }
14143
14144 if behavior.reject_query_protocol && path.starts_with("/api/worker/query-tasks/") {
14145 let requested_version = worker_protocol.as_deref().unwrap_or("missing");
14146 let body = format!(
14147 r#"{{"reason":"unsupported_protocol_version","message":"Query tasks require worker protocol 1.8 or newer.","supported_version":"1.7","requested_version":"{requested_version}"}}"#
14148 );
14149 write_mock_response(stream, "400 Bad Request", &body);
14150 return;
14151 }
14152
14153 if behavior.reject_query_completion && path == "/api/worker/query-tasks/query-late/complete"
14154 {
14155 write_mock_response(
14156 stream,
14157 "409 Conflict",
14158 r#"{"reason":"query_task_timed_out","message":"query task timed out before completion"}"#,
14159 );
14160 return;
14161 }
14162
14163 if behavior.workflow_completion_status.is_some()
14164 && path == "/api/worker/workflow-tasks/poll"
14165 && request_number == 1
14166 {
14167 write_mock_response(
14168 stream,
14169 "200 OK",
14170 r#"{"task":{"task_id":"workflow-timeout-task","workflow_id":"reused-workflow-id","run_id":"run-selected-timeout","workflow_type":"timeout.workflow","payload_codec":"json","arguments":{"codec":"json","blob":"[]"},"history_events":[],"workflow_task_attempt":3,"lease_owner":"timeout-worker"}}"#,
14171 );
14172 return;
14173 }
14174
14175 if path == "/api/worker/workflow-tasks/workflow-timeout-task/complete" {
14176 if let (Some(status), Some(body)) = (
14177 behavior.workflow_completion_status,
14178 behavior.workflow_completion_body,
14179 ) {
14180 write_mock_response(stream, status, body);
14181 return;
14182 }
14183 }
14184
14185 if behavior.waiting_query_worker {
14186 if behavior.complete_named_signal
14187 && path == "/api/worker/workflow-tasks/poll"
14188 && request_number == 1
14189 {
14190 let body = json!({
14191 "task": {
14192 "task_id": "snapshot-open",
14193 "workflow_id": "snapshot-1",
14194 "run_id": "snapshot-run-1",
14195 "workflow_type": "snapshot",
14196 "payload_codec": DEFAULT_CODEC,
14197 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
14198 .expect("Avro workflow arguments"),
14199 "history_events": [],
14200 "workflow_task_attempt": 1,
14201 "lease_owner": "rust-snapshot-worker"
14202 }
14203 })
14204 .to_string();
14205 write_mock_response(stream, "200 OK", &body);
14206 return;
14207 }
14208
14209 let signal_request = request_number - usize::from(behavior.complete_named_signal);
14210 let signal_request_limit = 2 + usize::from(behavior.complete_named_signal);
14211 if path == "/api/worker/workflow-tasks/poll"
14212 && signal_request >= 1
14213 && signal_request <= signal_request_limit
14214 {
14215 let finish = behavior.complete_named_signal && signal_request == 3;
14216 let amounts = if signal_request == 1 {
14217 vec![3]
14218 } else {
14219 vec![3, 5]
14220 };
14221 let task_id = if signal_request == 1 {
14222 "snapshot-wait-3"
14223 } else if finish {
14224 "snapshot-finish"
14225 } else {
14226 "snapshot-wait-5"
14227 };
14228 let mut history_events = std::iter::once(json!({
14229 "event_type": "SignalWaitOpened",
14230 "payload": {"sequence": 1, "signal_name": "finish"}
14231 }))
14232 .chain(amounts.iter().enumerate().map(|(index, amount)| {
14233 json!({
14234 "event_type": "SignalReceived",
14235 "payload": {
14236 "signal_id": format!("increment-{amount}"),
14237 "signal_name": "increment",
14238 "workflow_sequence": index + 2,
14239 "payload_codec": DEFAULT_CODEC,
14240 "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
14241 .expect("Avro signal envelope")
14242 }
14243 })
14244 }))
14245 .collect::<Vec<_>>();
14246 let (resume_id, resume_name, resume_arguments) = if finish {
14247 history_events.push(json!({
14248 "event_type": "SignalReceived",
14249 "payload": {
14250 "signal_id": "finish",
14251 "signal_name": "finish",
14252 "workflow_sequence": 4,
14253 "payload_codec": DEFAULT_CODEC,
14254 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
14255 .expect("Avro finish signal envelope")
14256 }
14257 }));
14258 (
14259 "finish".to_string(),
14260 "finish".to_string(),
14261 encode_value_envelope(&json!([]), DEFAULT_CODEC)
14262 .expect("Avro finish resume signal"),
14263 )
14264 } else {
14265 let amount = amounts.last().expect("amount");
14266 (
14267 format!("increment-{amount}"),
14268 "increment".to_string(),
14269 encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
14270 .expect("Avro increment resume signal"),
14271 )
14272 };
14273 let body = json!({
14274 "task": {
14275 "task_id": task_id,
14276 "workflow_id": "snapshot-1",
14277 "run_id": "snapshot-run-1",
14278 "workflow_type": "snapshot",
14279 "payload_codec": DEFAULT_CODEC,
14280 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
14281 .expect("Avro workflow arguments"),
14282 "history_events": history_events,
14283 "workflow_task_attempt": 1,
14284 "workflow_signal_id": resume_id,
14285 "signal_name": resume_name,
14286 "signal_arguments": resume_arguments,
14287 "lease_owner": "rust-snapshot-worker"
14288 }
14289 })
14290 .to_string();
14291 write_mock_response(stream, "200 OK", &body);
14292 return;
14293 }
14294
14295 if path == "/api/worker/query-tasks/poll" && request_number == 1 {
14296 let history_events = [3, 5]
14297 .into_iter()
14298 .enumerate()
14299 .map(|(index, amount)| {
14300 json!({
14301 "event_type": "SignalReceived",
14302 "payload": {
14303 "signal_id": format!("increment-{amount}"),
14304 "signal_name": "increment",
14305 "workflow_sequence": index + 2,
14306 "payload_codec": DEFAULT_CODEC,
14307 "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
14308 .expect("Avro query signal envelope")
14309 }
14310 })
14311 })
14312 .collect::<Vec<_>>();
14313 let body = json!({
14314 "task": {
14315 "query_task_id": "snapshot-current",
14316 "query_task_attempt": 1,
14317 "lease_owner": "rust-snapshot-worker",
14318 "workflow_id": "snapshot-1",
14319 "run_id": "snapshot-run-1",
14320 "workflow_type": "snapshot",
14321 "query_name": "current",
14322 "payload_codec": DEFAULT_CODEC,
14323 "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
14324 .expect("Avro workflow arguments"),
14325 "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
14326 .expect("Avro query arguments"),
14327 "history_events": history_events,
14328 "run_status": "waiting"
14329 }
14330 })
14331 .to_string();
14332 write_mock_response(stream, "200 OK", &body);
14333 return;
14334 }
14335
14336 if path == "/api/worker/workflow-tasks/snapshot-wait-3/fail"
14337 || path == "/api/worker/workflow-tasks/snapshot-wait-5/fail"
14338 {
14339 write_mock_response(
14340 stream,
14341 "200 OK",
14342 r#"{"outcome":"waiting_for_history","recorded":true}"#,
14343 );
14344 return;
14345 }
14346
14347 if path == "/api/worker/workflow-tasks/snapshot-open/complete" {
14348 write_mock_response(stream, "200 OK", r#"{"outcome":"waiting","recorded":true}"#);
14349 return;
14350 }
14351
14352 if path == "/api/worker/workflow-tasks/snapshot-finish/complete" {
14353 write_mock_response(
14354 stream,
14355 "200 OK",
14356 r#"{"outcome":"completed","run_status":"completed","recorded":true}"#,
14357 );
14358 return;
14359 }
14360
14361 if path == "/api/worker/query-tasks/snapshot-current/complete" {
14362 write_mock_response(stream, "200 OK", r#"{"outcome":"completed"}"#);
14363 return;
14364 }
14365 }
14366
14367 if matches!(
14368 path,
14369 "/api/workflows/typed-1/query/inspect" | "/api/workflows/typed-1/update/replace"
14370 ) {
14371 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
14372 .expect("typed mock result");
14373 let body = json!({
14374 "result": typed_fidelity_probe().into_json().expect("result projection"),
14375 "result_envelope": result,
14376 })
14377 .to_string();
14378 write_mock_response(stream, "200 OK", &body);
14379 return;
14380 }
14381
14382 if path == "/api/workflows/typed-1" {
14383 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
14384 .expect("typed mock result");
14385 let body = json!({
14386 "workflow_id": "typed-1",
14387 "run_id": "run-typed-1",
14388 "workflow_type": "typed.echo",
14389 "status": "completed",
14390 "output": typed_fidelity_probe().into_json().expect("output projection"),
14391 "output_envelope": result,
14392 })
14393 .to_string();
14394 write_mock_response(stream, "200 OK", &body);
14395 return;
14396 }
14397
14398 let (status, body) = match path {
14399 "/api/health" => ("200 OK", r#"{"status":"ok"}"#),
14400 "/api/workflows" => (
14401 "201 Created",
14402 r#"{"workflow_id":"wf-start-options","run_id":"run-start-options","workflow_type":"rust.timeout"}"#,
14403 ),
14404 "/api/worker/register" if behavior.decline_registration => (
14405 "200 OK",
14406 r#"{"worker_id":"declined-worker","registered":false}"#,
14407 ),
14408 "/api/worker/register" if behavior.waiting_query_worker => (
14409 "200 OK",
14410 r#"{"worker_id":"rust-snapshot-worker","registered":true,"heartbeat_interval_seconds":1}"#,
14411 ),
14412 "/api/worker/register" => (
14413 "200 OK",
14414 r#"{"worker_id":"mock-worker","registered":true,"heartbeat_interval_seconds":3600}"#,
14415 ),
14416 "/api/worker/heartbeat" => ("200 OK", "{}"),
14417 "/api/worker/activity-tasks/poll"
14418 if behavior.cancelled_activity && request_number == 1 =>
14419 {
14420 (
14421 "200 OK",
14422 r#"{"task":{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","activity_type":"cancel-aware","payload_codec":"json","arguments":{"codec":"json","blob":"[]"},"attempt_number":1,"lease_owner":"rust-cancel-worker"}}"#,
14423 )
14424 }
14425 "/api/worker/activity-tasks/poll" | "/api/worker/workflow-tasks/poll" => {
14426 ("200 OK", r#"{"task":null}"#)
14427 }
14428 "/api/worker/query-tasks/poll"
14429 if behavior.reject_query_completion && request_number == 1 =>
14430 {
14431 (
14432 "200 OK",
14433 r#"{"task":{"query_task_id":"query-late","query_task_attempt":1,"lease_owner":"late-worker","workflow_id":"counter-late","run_id":"run-late","workflow_type":"counter","query_name":"current","payload_codec":"json","workflow_arguments":{"codec":"json","blob":"[]"},"query_arguments":{"codec":"json","blob":"[]"},"history_events":[],"run_status":"running"}}"#,
14434 )
14435 }
14436 "/api/worker/query-tasks/poll" => ("200 OK", r#"{"task":null}"#),
14437 "/api/worker/query-tasks/query-capture/complete"
14438 | "/api/worker/query-tasks/query-capture/fail" => ("200 OK", "{}"),
14439 "/api/worker/activity-tasks/activity-cancel/heartbeat" => (
14440 "200 OK",
14441 r#"{"activity_attempt_id":"attempt-cancel","cancel_requested":true,"can_continue":false,"reason":"run_cancelled","run_closed_reason":"cancelled","heartbeat_recorded":false}"#,
14442 ),
14443 "/api/worker/activity-tasks/activity-cancel/complete" => (
14444 "409 Conflict",
14445 r#"{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","reason":"run_cancelled","cancel_requested":true,"can_continue":false,"run_closed_reason":"cancelled"}"#,
14446 ),
14447 "/api/worker/activity-tasks/activity-typed/complete"
14448 | "/api/worker/activity-tasks/activity-typed/fail"
14449 | "/api/workflows/typed-1/signal/changed" => ("200 OK", "{}"),
14450 "/api/workflows/counter-1/query/current" => (
14451 "200 OK",
14452 r#"{"workflow_id":"counter-1","query_name":"current","result":{"count":8},"result_envelope":{"codec":"json","blob":"{\"count\":8}"}}"#,
14453 ),
14454 "/api/workflows/counter-1/query/missing" => (
14455 "404 Not Found",
14456 r#"{"workflow_id":"counter-1","query_name":"missing","reason":"rejected_unknown_query","message":"unknown query"}"#,
14457 ),
14458 "/api/workflows/wf-lifecycle/cancel" => (
14459 "200 OK",
14460 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","reason":"cleanup requested","command_status":"accepted"}"#,
14461 ),
14462 "/api/workflows/wf-lifecycle/terminate" => (
14463 "200 OK",
14464 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","reason":"forced stop","command_status":"accepted"}"#,
14465 ),
14466 "/api/workflows/wf-lifecycle/runs/run-current/cancel" => (
14467 "200 OK",
14468 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","command_status":"accepted"}"#,
14469 ),
14470 "/api/workflows/wf-lifecycle/runs/run-current/terminate" => (
14471 "200 OK",
14472 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","command_status":"accepted"}"#,
14473 ),
14474 "/api/workflows/wf-lifecycle/runs/run-stale/cancel"
14475 | "/api/workflows/wf-lifecycle/runs/run-stale/terminate" => (
14476 "409 Conflict",
14477 r#"{"workflow_id":"wf-lifecycle","run_id":"run-stale","reason":"historical_run_command_rejected","target_scope":"run","message":"Commands cannot target historical runs."}"#,
14478 ),
14479 "/api/workflows/wf-failed" | "/api/workflows/wf-failed/runs/run-failed" => (
14480 "200 OK",
14481 r#"{"workflow_id":"wf-failed","run_id":"run-failed","status":"failed","closed_reason":"failed","error":"payment failed","failure":{"message":"payment failed","failure_category":"application","exception_type":"PaymentError","exception_class":"billing::PaymentError","non_retryable":true,"exception":{"type":"PaymentError","class":"billing::PaymentError","message":"payment failed"},"failures":[{"id":"failure-17","failure_category":"application"}]}}"#,
14482 ),
14483 "/api/workflows/wf-cancelled" => (
14484 "200 OK",
14485 r#"{"workflow_id":"wf-cancelled","run_id":"run-cancelled","status":"cancelled","closed_reason":"cancelled","reason":"cleanup requested"}"#,
14486 ),
14487 "/api/workflows/wf-terminated" => (
14488 "200 OK",
14489 r#"{"workflow_id":"wf-terminated","run_id":"run-terminated","status":"terminated","closed_reason":"terminated","reason":"forced stop"}"#,
14490 ),
14491 "/api/workflows/wf-timed-out" => (
14492 "200 OK",
14493 r#"{"workflow_id":"wf-timed-out","run_id":"run-timed-out","status":"failed","closed_reason":"timed_out","reason":"run_timeout"}"#,
14494 ),
14495 "/api/workflows/wf-waiting" | "/api/workflows/wf-waiting/runs/run-waiting" => (
14496 "200 OK",
14497 r#"{"workflow_id":"wf-waiting","run_id":"run-waiting","status":"waiting"}"#,
14498 ),
14499 "/api/workflows/wf-selected" => (
14500 "200 OK",
14501 r#"{"workflow_id":"wf-selected","run_id":"run-current","status":"completed","output":"current run output"}"#,
14502 ),
14503 "/api/workflows/wf-selected/runs/run-selected" => (
14504 "200 OK",
14505 r#"{"workflow_id":"wf-selected","run_id":"run-selected","status":"cancelled","closed_reason":"cancelled","reason":"selected run cancelled"}"#,
14506 ),
14507 _ => ("404 Not Found", r#"{"message":"not found"}"#),
14508 };
14509 write_mock_response(stream, status, body);
14510 }
14511
14512 fn mock_request_is_complete(request: &[u8]) -> bool {
14513 let Some(header_end) = request
14514 .windows(4)
14515 .position(|window| window == b"\r\n\r\n")
14516 .map(|position| position + 4)
14517 else {
14518 return false;
14519 };
14520 let headers = String::from_utf8_lossy(&request[..header_end]);
14521 let content_length = headers.lines().find_map(|line| {
14522 let (name, value) = line.split_once(':')?;
14523 name.eq_ignore_ascii_case("content-length")
14524 .then(|| value.trim().parse::<usize>().ok())
14525 .flatten()
14526 });
14527
14528 request.len() >= header_end + content_length.unwrap_or(0)
14529 }
14530
14531 fn write_mock_response(stream: &mut TcpStream, status: &str, body: &str) {
14532 let response = format!(
14533 "HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
14534 body.len()
14535 );
14536
14537 let _ = stream.write_all(response.as_bytes());
14538 let _ = stream.flush();
14539 }
14540}