1#![doc = include_str!("../README.md")]
2
3use std::{
4 any::{Any, TypeId},
5 collections::{BTreeMap, HashMap},
6 future::Future,
7 io::{self, Read},
8 pin::Pin,
9 sync::{
10 atomic::{AtomicBool, Ordering},
11 Arc, Mutex, OnceLock,
12 },
13 task::{Context as TaskContext, Poll},
14 time::{Duration, Instant, SystemTime, UNIX_EPOCH},
15};
16
17use apache_avro::{from_avro_datum, to_avro_datum, types::Value as AvroDatum, Schema};
18use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
19use futures_util::{future::OptionFuture, task::noop_waker_ref};
20use serde::{
21 de::DeserializeOwned,
22 ser::{SerializeMap, SerializeSeq},
23 Deserialize, Serialize, Serializer,
24};
25pub use serde_json::{json, Value};
26use thiserror::Error;
27pub use uuid::Uuid;
28
29pub const WORKER_PROTOCOL_VERSION: &str = "1.2";
30pub const CONTROL_PLANE_VERSION: &str = "2";
31pub const DEFAULT_CODEC: &str = "avro";
32pub const JSON_CODEC: &str = "json";
33pub const SDK_VERSION: &str = concat!("durable-workflow-rust/", env!("CARGO_PKG_VERSION"));
34pub const QUERY_TASKS_CAPABILITY: &str = "query_tasks";
36pub const WORKFLOW_UPDATES_CAPABILITY: &str = "workflow_updates";
38pub const QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
40
41const MAX_LONG_POLL_TIMEOUT_SECONDS: u64 = 60;
42const WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE: &str =
43 "Workflow task waiting for scheduled history.";
44const WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE: &str = "WorkflowTaskWaitingForHistory";
45
46const QUERY_TASK_FINAL_REJECTION_REASONS: &[&str] = &[
47 "lease_expired",
48 "query_task_not_found",
49 "query_task_not_leased",
50 "query_task_timed_out",
51];
52
53pub const AVRO_VALUE_SCHEMA_JSON: &str = r#"{"type":"record","name":"Value","namespace":"durable_workflow.protocol","fields":[{"name":"value","type":["null",{"type":"record","name":"BooleanValue","fields":[{"name":"boolean","type":"boolean"}]},{"type":"record","name":"LongValue","fields":[{"name":"long","type":"long"}]},{"type":"record","name":"DoubleValue","fields":[{"name":"double","type":"double"}]},{"type":"record","name":"BytesValue","fields":[{"name":"bytes","type":"bytes"}]},{"type":"record","name":"StringValue","fields":[{"name":"string","type":"string"}]},{"type":"record","name":"ArrayValue","fields":[{"name":"items","type":{"type":"array","items":"Value"}}]},{"type":"record","name":"MapValue","fields":[{"name":"entries","type":{"type":"map","values":"Value"}}]}]}]}"#;
54pub const AVRO_VALUE_SCHEMA_FINGERPRINT_HEX: &str = "e2a33dff55802237";
55pub const AVRO_VALUE_SCHEMA_FINGERPRINT: [u8; 8] = [0xe2, 0xa3, 0x3d, 0xff, 0x55, 0x80, 0x22, 0x37];
56const AVRO_SINGLE_OBJECT_MAGIC: [u8; 2] = [0xc3, 0x01];
57
58static AVRO_VALUE_SCHEMA: OnceLock<std::result::Result<Schema, String>> = OnceLock::new();
59
60#[derive(Clone, Copy)]
61enum RequestProtocol {
62 ControlPlane,
63 Worker(&'static str),
64}
65
66impl RequestProtocol {
67 fn is_worker(self) -> bool {
68 matches!(self, Self::Worker(_))
69 }
70}
71
72pub type Result<T> = std::result::Result<T, Error>;
73
74#[derive(Debug, Error)]
75pub enum Error {
76 #[error("transport error: {0}")]
77 Transport(#[from] reqwest::Error),
78 #[error("json error: {0}")]
79 Json(#[from] serde_json::Error),
80 #[error("http {status}: {body}")]
81 Http {
82 status: reqwest::StatusCode,
83 body: String,
84 },
85 #[error("codec error: {0}")]
86 Codec(String),
87 #[error(transparent)]
88 QueryFailed(QueryFailure),
89 #[error(transparent)]
90 Protocol(ProtocolFailure),
91 #[error(transparent)]
92 NonDeterministicReplay(ReplayFailure),
93 #[error(transparent)]
94 ChildWorkflowFailed(ChildWorkflowFailure),
95 #[error(transparent)]
96 ActivityFailed(ActivityFailure),
97 #[error(transparent)]
98 WorkflowCommandRejected(WorkflowCommandRejection),
99 #[error(transparent)]
100 WorkflowFailed(WorkflowTerminalOutcome),
101 #[error(transparent)]
102 WorkflowCancelled(WorkflowTerminalOutcome),
103 #[error(transparent)]
104 WorkflowTerminated(WorkflowTerminalOutcome),
105 #[error(transparent)]
106 WorkflowTimedOut(WorkflowTerminalOutcome),
107 #[error(transparent)]
108 ActivityTaskRejected(ActivityTaskRejection),
109 #[error("workflow handler {0:?} is not registered")]
110 WorkflowNotRegistered(String),
111 #[error("activity handler {0:?} is not registered")]
112 ActivityNotRegistered(String),
113 #[error("workflow future yielded without emitting a durable command")]
114 WorkflowYieldedWithoutCommand,
115 #[error("workflow state lock is poisoned")]
116 WorkflowStatePoisoned,
117 #[error("timer duration is too large for the worker protocol")]
118 TimerDurationOverflow,
119 #[error("operation timed out")]
120 Timeout,
121 #[error("worker loop error: {0}")]
122 WorkerLoop(String),
123 #[error("invalid child workflow options: {0}")]
124 InvalidChildWorkflowOptions(String),
125 #[error(transparent)]
126 InvalidActivityOptions(ActivityOptionsError),
127 #[error(transparent)]
128 InvalidContinueAsNewOptions(#[from] ContinueAsNewOptionsError),
129 #[doc(hidden)]
130 #[error("workflow requested continue as new")]
131 ContinueAsNew(ContinueAsNewRequest),
132}
133
134#[derive(Clone, Copy, Debug, PartialEq, Eq)]
136pub enum WorkflowCommandKind {
137 Cancel,
138 Terminate,
139}
140
141impl WorkflowCommandKind {
142 fn as_str(self) -> &'static str {
143 match self {
144 Self::Cancel => "cancel",
145 Self::Terminate => "terminate",
146 }
147 }
148}
149
150#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
152pub struct WorkflowCommandOptions {
153 #[serde(skip_serializing_if = "Option::is_none")]
154 pub reason: Option<String>,
155 #[serde(skip_serializing_if = "Option::is_none")]
156 pub request_id: Option<String>,
157}
158
159#[derive(Clone, Debug, PartialEq, Eq)]
166pub struct WorkflowStartOptions {
167 pub execution_timeout_seconds: u64,
168 pub run_timeout_seconds: u64,
169}
170
171impl Default for WorkflowStartOptions {
172 fn default() -> Self {
173 Self {
174 execution_timeout_seconds: 3600,
175 run_timeout_seconds: 600,
176 }
177 }
178}
179
180impl WorkflowStartOptions {
181 pub fn new() -> Self {
182 Self::default()
183 }
184
185 pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
186 self.execution_timeout_seconds = seconds;
187 self
188 }
189
190 pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
191 self.run_timeout_seconds = seconds;
192 self
193 }
194
195 fn validate(&self) -> Result<()> {
196 if self.execution_timeout_seconds == 0 {
197 return Err(Error::Codec(
198 "execution_timeout_seconds must be at least 1".to_string(),
199 ));
200 }
201 if self.run_timeout_seconds == 0 {
202 return Err(Error::Codec(
203 "run_timeout_seconds must be at least 1".to_string(),
204 ));
205 }
206 if self.run_timeout_seconds > self.execution_timeout_seconds {
207 return Err(Error::Codec(
208 "run_timeout_seconds cannot exceed execution_timeout_seconds".to_string(),
209 ));
210 }
211
212 Ok(())
213 }
214}
215
216#[derive(Clone, Debug, Default, PartialEq, Eq)]
221pub struct ContinueAsNewOptions {
222 pub workflow_type: Option<String>,
223 pub task_queue: Option<String>,
224}
225
226impl ContinueAsNewOptions {
227 pub fn new() -> Self {
228 Self::default()
229 }
230
231 pub fn workflow_type(mut self, workflow_type: impl Into<String>) -> Self {
232 self.workflow_type = Some(workflow_type.into());
233 self
234 }
235
236 pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
237 self.task_queue = Some(task_queue.into());
238 self
239 }
240
241 fn validate(&self) -> std::result::Result<(), ContinueAsNewOptionsError> {
242 for (field, value) in [
243 ("workflow_type", self.workflow_type.as_deref()),
244 ("task_queue", self.task_queue.as_deref()),
245 ] {
246 if value.is_some_and(|value| value.trim().is_empty()) {
247 return Err(ContinueAsNewOptionsError {
248 field,
249 message: format!("{field} must not be empty"),
250 });
251 }
252 }
253 Ok(())
254 }
255}
256
257#[derive(Clone, Debug, Error, PartialEq, Eq)]
259#[error("invalid continue-as-new option {field}: {message}")]
260pub struct ContinueAsNewOptionsError {
261 pub field: &'static str,
262 pub message: String,
263}
264
265#[derive(Clone, Debug, Default, PartialEq, Eq)]
267pub struct WorkflowHistoryBudget {
268 pub event_count: u64,
269 pub size_bytes: Option<u64>,
270 pub continue_as_new_recommended: bool,
271 pub pressure: Option<String>,
272}
273
274#[doc(hidden)]
275#[derive(Clone, Debug)]
276pub struct ContinueAsNewRequest {
277 arguments: AvroValue,
278 options: ContinueAsNewOptions,
279}
280
281impl WorkflowCommandOptions {
282 pub fn new() -> Self {
283 Self::default()
284 }
285
286 pub fn reason(mut self, reason: impl Into<String>) -> Self {
287 self.reason = Some(reason.into());
288 self
289 }
290
291 pub fn request_id(mut self, request_id: impl Into<String>) -> Self {
292 self.request_id = Some(request_id.into());
293 self
294 }
295}
296
297#[derive(Clone, Debug, PartialEq)]
299pub struct WorkflowCommandResult {
300 pub command: WorkflowCommandKind,
301 pub workflow_id: String,
302 pub run_id: Option<String>,
303 pub outcome: Option<String>,
304 pub reason: Option<String>,
305 pub command_status: Option<String>,
306 pub raw: Value,
307}
308
309#[derive(Clone, Debug, Error)]
311#[error("workflow {command:?} rejected ({reason}, HTTP {status}): {message}")]
312pub struct WorkflowCommandRejection {
313 pub command: WorkflowCommandKind,
314 pub status: u16,
315 pub reason: String,
316 pub message: String,
317 pub workflow_id: String,
318 pub run_id: Option<String>,
319 pub target_scope: Option<String>,
320 pub body: Value,
321}
322
323#[derive(Clone, Copy, Debug, PartialEq, Eq)]
325pub enum WorkflowTerminalKind {
326 Failed,
327 Cancelled,
328 Terminated,
329 TimedOut,
330}
331
332#[derive(Clone, Debug, Error)]
338#[error("workflow {workflow_id} run {run_id:?} ended as {kind:?} ({reason})")]
339pub struct WorkflowTerminalOutcome {
340 pub kind: WorkflowTerminalKind,
341 pub workflow_id: String,
342 pub run_id: Option<String>,
343 pub reason: String,
344 pub failure_category: Option<String>,
345 pub failure_id: Option<String>,
346 pub exception_type: Option<String>,
347 pub exception_class: Option<String>,
348 pub non_retryable: Option<bool>,
349 pub message: Option<String>,
350 pub exception: Option<Value>,
351 pub raw: Value,
352}
353
354#[derive(Clone, Debug, Error)]
356#[error("activity task {operation} rejected ({reason}, HTTP {status})")]
357pub struct ActivityTaskRejection {
358 pub operation: String,
359 pub status: u16,
360 pub reason: String,
361 pub task_id: String,
362 pub activity_attempt_id: String,
363 pub cancel_requested: bool,
364 pub can_continue: Option<bool>,
365 pub run_closed_reason: Option<String>,
366 pub body: Value,
367}
368
369#[derive(Clone, Copy, Debug, PartialEq, Eq)]
371pub enum ActivityOptionsErrorKind {
372 EmptyTaskQueue,
373 EmptyRetryPolicy,
374 InvalidMaxAttempts,
375 BackoffWithoutRetryBudget,
376 TooManyBackoffIntervals,
377 InvalidBackoffCoefficient,
378 BackoffGenerationTooLarge,
379 BackoffOverflow,
380 EmptyNonRetryableErrorType,
381 TimeoutNotPositive,
382 TimeoutOverflow,
383 TimeoutOrder,
384}
385
386#[derive(Clone, Debug, Error, PartialEq, Eq)]
388#[error("invalid activity options ({kind:?}, {field:?}): {message}")]
389pub struct ActivityOptionsError {
390 pub kind: ActivityOptionsErrorKind,
391 pub field: Option<&'static str>,
392 pub message: String,
393}
394
395impl ActivityOptionsError {
396 fn new(
397 kind: ActivityOptionsErrorKind,
398 field: Option<&'static str>,
399 message: impl Into<String>,
400 ) -> Self {
401 Self {
402 kind,
403 field,
404 message: message.into(),
405 }
406 }
407}
408
409#[derive(Clone, Copy, Debug, PartialEq, Eq)]
411pub enum ActivityFailureKind {
412 Failed,
413 Cancelled,
414 TimedOut,
415}
416
417#[derive(Clone, Debug, Error)]
422#[error("activity failed ({reason}): {message}")]
423pub struct ActivityFailure {
424 pub kind: ActivityFailureKind,
425 pub reason: String,
426 pub message: String,
427 pub activity_execution_id: Option<String>,
428 pub activity_attempt_id: Option<String>,
429 pub activity_type: Option<String>,
430 pub activity_class: Option<String>,
431 pub attempt_number: Option<u64>,
432 pub failure_id: Option<String>,
433 pub failure_category: Option<String>,
434 pub timeout_kind: Option<String>,
435 pub non_retryable: bool,
436 pub exception_type: Option<String>,
437 pub exception_class: Option<String>,
438 pub code: Option<Value>,
439 pub exception: Option<Value>,
440}
441
442#[derive(Clone, Copy, Debug, PartialEq, Eq)]
444pub enum ChildWorkflowFailureKind {
445 Failed,
446 Cancelled,
447 Terminated,
448}
449
450#[derive(Clone, Debug, Error)]
456#[error("child workflow failed ({reason}): {message}")]
457pub struct ChildWorkflowFailure {
458 pub kind: ChildWorkflowFailureKind,
459 pub reason: String,
460 pub message: String,
461 pub parent_workflow_id: Option<String>,
462 pub parent_workflow_run_id: Option<String>,
463 pub child_workflow_id: Option<String>,
464 pub child_workflow_run_id: Option<String>,
465 pub child_workflow_type: Option<String>,
466 pub failure_id: Option<String>,
467 pub failure_category: Option<String>,
468 pub exception_type: Option<String>,
469 pub exception_class: Option<String>,
470 pub non_retryable: bool,
471 pub code: Option<Value>,
472 pub exception: Option<Value>,
473}
474
475#[derive(Clone, Debug, PartialEq, Eq)]
477pub struct WorkflowIdentity {
478 pub workflow_id: Option<String>,
479 pub run_id: Option<String>,
480}
481
482#[derive(Clone, Debug, PartialEq)]
484pub struct ChildWorkflowResult {
485 pub parent: WorkflowIdentity,
486 pub child: WorkflowIdentity,
487 pub child_workflow_type: Option<String>,
488 pub result: Value,
489}
490
491#[derive(Clone, Debug, PartialEq)]
493pub struct ChildWorkflowAvroResult {
494 pub parent: WorkflowIdentity,
495 pub child: WorkflowIdentity,
496 pub child_workflow_type: Option<String>,
497 pub result: AvroValue,
498}
499
500#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
502pub enum ParentClosePolicy {
503 #[default]
504 Abandon,
505 RequestCancel,
506 Terminate,
507}
508
509impl ParentClosePolicy {
510 fn as_str(self) -> &'static str {
511 match self {
512 Self::Abandon => "abandon",
513 Self::RequestCancel => "request_cancel",
514 Self::Terminate => "terminate",
515 }
516 }
517}
518
519#[derive(Clone, Debug, Default, PartialEq, Eq)]
521pub struct ChildWorkflowRetryPolicy {
522 pub max_attempts: Option<u32>,
523 pub backoff_seconds: Vec<u64>,
524 pub non_retryable_error_types: Vec<String>,
525}
526
527#[derive(Clone, Debug, PartialEq, Eq)]
531pub struct ChildWorkflowOptions {
532 pub task_queue: String,
533 pub parent_close_policy: ParentClosePolicy,
534 pub retry_policy: Option<ChildWorkflowRetryPolicy>,
535 pub execution_timeout_seconds: Option<u64>,
536 pub run_timeout_seconds: Option<u64>,
537}
538
539impl ChildWorkflowOptions {
540 pub fn new(task_queue: impl Into<String>) -> Self {
541 Self {
542 task_queue: task_queue.into(),
543 parent_close_policy: ParentClosePolicy::Abandon,
544 retry_policy: None,
545 execution_timeout_seconds: None,
546 run_timeout_seconds: None,
547 }
548 }
549
550 pub fn parent_close_policy(mut self, policy: ParentClosePolicy) -> Self {
551 self.parent_close_policy = policy;
552 self
553 }
554
555 pub fn retry_policy(mut self, policy: ChildWorkflowRetryPolicy) -> Self {
556 self.retry_policy = Some(policy);
557 self
558 }
559
560 pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
561 self.execution_timeout_seconds = Some(seconds);
562 self
563 }
564
565 pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
566 self.run_timeout_seconds = Some(seconds);
567 self
568 }
569}
570
571#[derive(Clone, Debug, PartialEq, Eq)]
573pub enum ActivityBackoff {
574 Explicit(Vec<Duration>),
577 Exponential {
579 initial_interval: Duration,
580 coefficient: u32,
581 maximum_interval: Option<Duration>,
582 },
583}
584
585#[derive(Clone, Debug, Default, PartialEq, Eq)]
587pub struct ActivityRetryPolicy {
588 pub max_attempts: Option<u32>,
589 pub backoff: Option<ActivityBackoff>,
590 pub non_retryable_error_types: Vec<String>,
591}
592
593impl ActivityRetryPolicy {
594 pub fn new(max_attempts: u32) -> Self {
596 Self {
597 max_attempts: Some(max_attempts),
598 ..Self::default()
599 }
600 }
601
602 pub fn backoff_intervals(mut self, intervals: impl IntoIterator<Item = Duration>) -> Self {
603 self.backoff = Some(ActivityBackoff::Explicit(intervals.into_iter().collect()));
604 self
605 }
606
607 pub fn exponential_backoff(
608 mut self,
609 initial_interval: Duration,
610 coefficient: u32,
611 maximum_interval: Option<Duration>,
612 ) -> Self {
613 self.backoff = Some(ActivityBackoff::Exponential {
614 initial_interval,
615 coefficient,
616 maximum_interval,
617 });
618 self
619 }
620
621 pub fn non_retryable_error_type(mut self, error_type: impl Into<String>) -> Self {
622 self.non_retryable_error_types.push(error_type.into());
623 self
624 }
625
626 pub fn non_retryable_error_types(
627 mut self,
628 error_types: impl IntoIterator<Item = impl Into<String>>,
629 ) -> Self {
630 self.non_retryable_error_types
631 .extend(error_types.into_iter().map(Into::into));
632 self
633 }
634}
635
636#[derive(Clone, Debug, Default, PartialEq, Eq)]
641pub struct ActivityOptions {
642 pub task_queue: Option<String>,
643 pub retry_policy: Option<ActivityRetryPolicy>,
644 pub start_to_close_timeout: Option<Duration>,
645 pub schedule_to_start_timeout: Option<Duration>,
646 pub schedule_to_close_timeout: Option<Duration>,
647 pub heartbeat_timeout: Option<Duration>,
648}
649
650impl ActivityOptions {
651 pub fn new() -> Self {
652 Self::default()
653 }
654
655 pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
656 self.task_queue = Some(task_queue.into());
657 self
658 }
659
660 pub fn retry_policy(mut self, policy: ActivityRetryPolicy) -> Self {
661 self.retry_policy = Some(policy);
662 self
663 }
664
665 pub fn start_to_close_timeout(mut self, timeout: Duration) -> Self {
666 self.start_to_close_timeout = Some(timeout);
667 self
668 }
669
670 pub fn schedule_to_start_timeout(mut self, timeout: Duration) -> Self {
671 self.schedule_to_start_timeout = Some(timeout);
672 self
673 }
674
675 pub fn schedule_to_close_timeout(mut self, timeout: Duration) -> Self {
676 self.schedule_to_close_timeout = Some(timeout);
677 self
678 }
679
680 pub fn heartbeat_timeout(mut self, timeout: Duration) -> Self {
681 self.heartbeat_timeout = Some(timeout);
682 self
683 }
684
685 fn validate(&self) -> std::result::Result<ValidatedActivityOptions, ActivityOptionsError> {
686 if self
687 .task_queue
688 .as_deref()
689 .is_some_and(|queue| queue.trim().is_empty())
690 {
691 return Err(ActivityOptionsError::new(
692 ActivityOptionsErrorKind::EmptyTaskQueue,
693 Some("task_queue"),
694 "task_queue must not be empty",
695 ));
696 }
697
698 for (field, value) in [
699 ("start_to_close_timeout", self.start_to_close_timeout),
700 ("schedule_to_start_timeout", self.schedule_to_start_timeout),
701 ("schedule_to_close_timeout", self.schedule_to_close_timeout),
702 ("heartbeat_timeout", self.heartbeat_timeout),
703 ] {
704 if value.is_some_and(|value| value.is_zero()) {
705 return Err(ActivityOptionsError::new(
706 ActivityOptionsErrorKind::TimeoutNotPositive,
707 Some(field),
708 format!("{field} must be positive"),
709 ));
710 }
711 }
712
713 validate_timeout_order(
714 "heartbeat_timeout",
715 self.heartbeat_timeout,
716 "start_to_close_timeout",
717 self.start_to_close_timeout,
718 )?;
719 validate_timeout_order(
720 "start_to_close_timeout",
721 self.start_to_close_timeout,
722 "schedule_to_close_timeout",
723 self.schedule_to_close_timeout,
724 )?;
725 validate_timeout_order(
726 "schedule_to_start_timeout",
727 self.schedule_to_start_timeout,
728 "schedule_to_close_timeout",
729 self.schedule_to_close_timeout,
730 )?;
731
732 Ok(ValidatedActivityOptions {
733 task_queue: self.task_queue.clone(),
734 retry_policy: self
735 .retry_policy
736 .as_ref()
737 .map(validate_activity_retry_policy)
738 .transpose()?,
739 start_to_close_timeout: timeout_seconds(
740 "start_to_close_timeout",
741 self.start_to_close_timeout,
742 )?,
743 schedule_to_start_timeout: timeout_seconds(
744 "schedule_to_start_timeout",
745 self.schedule_to_start_timeout,
746 )?,
747 schedule_to_close_timeout: timeout_seconds(
748 "schedule_to_close_timeout",
749 self.schedule_to_close_timeout,
750 )?,
751 heartbeat_timeout: timeout_seconds("heartbeat_timeout", self.heartbeat_timeout)?,
752 })
753 }
754}
755
756#[derive(Clone, Debug)]
757struct ValidatedActivityOptions {
758 task_queue: Option<String>,
759 retry_policy: Option<Value>,
760 start_to_close_timeout: Option<u64>,
761 schedule_to_start_timeout: Option<u64>,
762 schedule_to_close_timeout: Option<u64>,
763 heartbeat_timeout: Option<u64>,
764}
765
766fn validate_timeout_order(
767 smaller_name: &'static str,
768 smaller: Option<Duration>,
769 larger_name: &'static str,
770 larger: Option<Duration>,
771) -> std::result::Result<(), ActivityOptionsError> {
772 if matches!((smaller, larger), (Some(smaller), Some(larger)) if smaller > larger) {
773 return Err(ActivityOptionsError::new(
774 ActivityOptionsErrorKind::TimeoutOrder,
775 Some(smaller_name),
776 format!("{smaller_name} must be <= {larger_name}"),
777 ));
778 }
779 Ok(())
780}
781
782fn timeout_seconds(
783 field: &'static str,
784 value: Option<Duration>,
785) -> std::result::Result<Option<u64>, ActivityOptionsError> {
786 value
787 .map(|value| {
788 activity_protocol_seconds(value).ok_or_else(|| {
789 ActivityOptionsError::new(
790 ActivityOptionsErrorKind::TimeoutOverflow,
791 Some(field),
792 format!("{field} is too large for the worker protocol"),
793 )
794 })
795 })
796 .transpose()
797}
798
799fn duration_seconds_ceil(value: Duration) -> Option<u64> {
800 value
801 .as_secs()
802 .checked_add(u64::from(value.subsec_nanos() > 0))
803}
804
805fn activity_protocol_seconds(value: Duration) -> Option<u64> {
806 duration_seconds_ceil(value).filter(|seconds| *seconds <= i64::MAX as u64)
807}
808
809fn validate_activity_retry_policy(
810 policy: &ActivityRetryPolicy,
811) -> std::result::Result<Value, ActivityOptionsError> {
812 if policy.max_attempts.is_none()
813 && policy.backoff.is_none()
814 && policy.non_retryable_error_types.is_empty()
815 {
816 return Err(ActivityOptionsError::new(
817 ActivityOptionsErrorKind::EmptyRetryPolicy,
818 Some("retry_policy"),
819 "retry_policy must configure at least one field",
820 ));
821 }
822 if policy.max_attempts == Some(0) {
823 return Err(ActivityOptionsError::new(
824 ActivityOptionsErrorKind::InvalidMaxAttempts,
825 Some("retry_policy.max_attempts"),
826 "max_attempts must be >= 1",
827 ));
828 }
829 if policy
830 .non_retryable_error_types
831 .iter()
832 .any(|error_type| error_type.trim().is_empty())
833 {
834 return Err(ActivityOptionsError::new(
835 ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
836 Some("retry_policy.non_retryable_error_types"),
837 "non_retryable_error_types must not contain empty values",
838 ));
839 }
840
841 let backoff_seconds = match &policy.backoff {
842 None => None,
843 Some(backoff) => {
844 let max_attempts = policy.max_attempts.ok_or_else(|| {
845 ActivityOptionsError::new(
846 ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
847 Some("retry_policy.backoff"),
848 "backoff requires max_attempts",
849 )
850 })?;
851 let retry_count = max_attempts.saturating_sub(1) as usize;
852 let intervals = match backoff {
853 ActivityBackoff::Explicit(intervals) => {
854 if intervals.len() > retry_count {
855 return Err(ActivityOptionsError::new(
856 ActivityOptionsErrorKind::TooManyBackoffIntervals,
857 Some("retry_policy.backoff"),
858 "backoff interval count must not exceed max_attempts - 1",
859 ));
860 }
861 intervals.clone()
862 }
863 ActivityBackoff::Exponential {
864 initial_interval,
865 coefficient,
866 maximum_interval,
867 } => {
868 if *coefficient < 1 {
869 return Err(ActivityOptionsError::new(
870 ActivityOptionsErrorKind::InvalidBackoffCoefficient,
871 Some("retry_policy.backoff.coefficient"),
872 "backoff coefficient must be >= 1",
873 ));
874 }
875 if retry_count > 10_000 {
876 return Err(ActivityOptionsError::new(
877 ActivityOptionsErrorKind::BackoffGenerationTooLarge,
878 Some("retry_policy.max_attempts"),
879 "generated backoff supports at most 10000 retry intervals",
880 ));
881 }
882 let mut current = *initial_interval;
883 let mut intervals = Vec::with_capacity(retry_count);
884 for _ in 0..retry_count {
885 let interval = maximum_interval
886 .map(|maximum| current.min(maximum))
887 .unwrap_or(current);
888 intervals.push(interval);
889 if maximum_interval.is_some_and(|maximum| interval == maximum) {
890 break;
891 }
892 current = current.checked_mul(*coefficient).ok_or_else(|| {
893 ActivityOptionsError::new(
894 ActivityOptionsErrorKind::BackoffOverflow,
895 Some("retry_policy.backoff"),
896 "generated backoff interval overflowed",
897 )
898 })?;
899 }
900 intervals
901 }
902 };
903 Some(
904 intervals
905 .into_iter()
906 .map(|interval| {
907 activity_protocol_seconds(interval).ok_or_else(|| {
908 ActivityOptionsError::new(
909 ActivityOptionsErrorKind::BackoffOverflow,
910 Some("retry_policy.backoff"),
911 "backoff interval is too large for the worker protocol",
912 )
913 })
914 })
915 .collect::<std::result::Result<Vec<_>, _>>()?,
916 )
917 }
918 };
919
920 let mut encoded = serde_json::Map::new();
921 if let Some(max_attempts) = policy.max_attempts {
922 encoded.insert("max_attempts".to_string(), json!(max_attempts));
923 }
924 if let Some(backoff_seconds) = backoff_seconds {
925 encoded.insert("backoff_seconds".to_string(), json!(backoff_seconds));
926 }
927 if !policy.non_retryable_error_types.is_empty() {
928 let mut canonical_error_types = Vec::new();
929 for error_type in policy
930 .non_retryable_error_types
931 .iter()
932 .map(|error_type| error_type.trim())
933 {
934 if !canonical_error_types.contains(&error_type) {
935 canonical_error_types.push(error_type);
936 }
937 }
938 encoded.insert(
939 "non_retryable_error_types".to_string(),
940 json!(canonical_error_types),
941 );
942 }
943 Ok(Value::Object(encoded))
944}
945
946#[derive(Clone, Debug, Error)]
949#[error("non-deterministic workflow replay ({reason}) at sequence {sequence:?}: {message}")]
950pub struct ReplayFailure {
951 pub reason: String,
952 pub sequence: Option<u64>,
953 pub expected: Option<String>,
954 pub actual: Option<String>,
955 pub message: String,
956}
957
958impl ReplayFailure {
959 fn new(
960 reason: impl Into<String>,
961 sequence: Option<u64>,
962 expected: Option<String>,
963 actual: Option<String>,
964 message: impl Into<String>,
965 ) -> Self {
966 Self {
967 reason: reason.into(),
968 sequence,
969 expected,
970 actual,
971 message: message.into(),
972 }
973 }
974}
975
976#[derive(Clone, Debug, Error)]
978#[error("query failed ({reason}, HTTP {status}): {message}")]
979pub struct QueryFailure {
980 pub status: u16,
981 pub reason: String,
982 pub message: String,
983 pub body: Value,
984}
985
986#[derive(Clone, Debug, Error)]
988#[error("protocol rejected ({reason}, HTTP {status}): {message}")]
989pub struct ProtocolFailure {
990 pub status: u16,
991 pub reason: String,
992 pub message: String,
993 pub supported_version: Option<String>,
994 pub requested_version: Option<String>,
995 pub body: Value,
996}
997
998#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
999pub struct PayloadEnvelope {
1000 pub codec: String,
1001 pub blob: String,
1002}
1003
1004impl PayloadEnvelope {
1005 pub fn avro<T: Serialize>(value: &T) -> Result<Self> {
1006 encode_payload(value, DEFAULT_CODEC)
1007 }
1008
1009 pub fn json<T: Serialize>(value: &T) -> Result<Self> {
1010 encode_payload(value, JSON_CODEC)
1011 }
1012
1013 pub fn avro_value(value: &AvroValue) -> Result<Self> {
1016 encode_avro_value(value)
1017 }
1018}
1019
1020#[derive(Clone, Debug, PartialEq)]
1022pub enum AvroValue {
1023 Null,
1024 Boolean(bool),
1025 Long(i64),
1026 Double(f64),
1027 Bytes(Vec<u8>),
1028 String(String),
1029 Array(Vec<AvroValue>),
1030 Map(BTreeMap<String, AvroValue>),
1031}
1032
1033impl AvroValue {
1034 fn from_serialize<T: Serialize>(value: &T) -> Result<Self> {
1035 Self::from_serde_value(
1036 serde_value::to_value(value).map_err(|error| {
1037 Error::Codec(format!("could not adapt value for Avro: {error}"))
1038 })?,
1039 )
1040 }
1041
1042 fn from_serde_value(value: serde_value::Value) -> Result<Self> {
1043 use serde_value::Value as SerdeValue;
1044
1045 match value {
1046 SerdeValue::Unit => Ok(Self::Null),
1047 SerdeValue::Bool(value) => Ok(Self::Boolean(value)),
1048 SerdeValue::I8(value) => Ok(Self::Long(i64::from(value))),
1049 SerdeValue::I16(value) => Ok(Self::Long(i64::from(value))),
1050 SerdeValue::I32(value) => Ok(Self::Long(i64::from(value))),
1051 SerdeValue::I64(value) => Ok(Self::Long(value)),
1052 SerdeValue::U8(value) => Ok(Self::Long(i64::from(value))),
1053 SerdeValue::U16(value) => Ok(Self::Long(i64::from(value))),
1054 SerdeValue::U32(value) => Ok(Self::Long(i64::from(value))),
1055 SerdeValue::U64(value) => i64::try_from(value).map(Self::Long).map_err(|_| {
1056 Error::Codec(
1057 "integer_overflow: Avro Value long must be within signed 64-bit range"
1058 .to_string(),
1059 )
1060 }),
1061 SerdeValue::F32(value) => Self::finite_double(f64::from(value)),
1062 SerdeValue::F64(value) => Self::finite_double(value),
1063 SerdeValue::Char(value) => Ok(Self::String(value.to_string())),
1064 SerdeValue::String(value) => Ok(Self::String(value)),
1065 SerdeValue::Bytes(value) => Ok(Self::Bytes(value)),
1066 SerdeValue::Option(None) => Ok(Self::Null),
1067 SerdeValue::Option(Some(value)) | SerdeValue::Newtype(value) => {
1068 Self::from_serde_value(*value)
1069 }
1070 SerdeValue::Seq(values) => values
1071 .into_iter()
1072 .map(Self::from_serde_value)
1073 .collect::<Result<Vec<_>>>()
1074 .map(Self::Array),
1075 SerdeValue::Map(values) => values
1076 .into_iter()
1077 .map(|(key, value)| {
1078 let SerdeValue::String(key) = key else {
1079 return Err(Error::Codec(
1080 "invalid_map_key: Avro Value map keys must be strings".to_string(),
1081 ));
1082 };
1083
1084 Ok((key, Self::from_serde_value(value)?))
1085 })
1086 .collect::<Result<BTreeMap<_, _>>>()
1087 .map(Self::Map),
1088 }
1089 }
1090
1091 fn finite_double(value: f64) -> Result<Self> {
1092 if !value.is_finite() {
1093 return Err(Error::Codec(
1094 "non_finite_float: Avro Value doubles must be finite".to_string(),
1095 ));
1096 }
1097
1098 Ok(Self::Double(value))
1099 }
1100
1101 fn into_json(self) -> Result<Value> {
1102 match self {
1103 Self::Null => Ok(Value::Null),
1104 Self::Boolean(value) => Ok(Value::Bool(value)),
1105 Self::Long(value) => Ok(Value::Number(value.into())),
1106 Self::Double(value) => serde_json::Number::from_f64(value)
1107 .map(Value::Number)
1108 .ok_or_else(|| {
1109 Error::Codec(
1110 "non_finite_float: decoded Avro Value double is not finite".to_string(),
1111 )
1112 }),
1113 Self::Bytes(value) => Ok(json!({
1114 "$type": "bytes",
1115 "base64": BASE64.encode(value),
1116 })),
1117 Self::String(value) => Ok(Value::String(value)),
1118 Self::Array(values) => values
1119 .into_iter()
1120 .map(Self::into_json)
1121 .collect::<Result<Vec<_>>>()
1122 .map(Value::Array),
1123 Self::Map(values) => values
1124 .into_iter()
1125 .map(|(key, value)| Ok((key, value.into_json()?)))
1126 .collect::<Result<serde_json::Map<_, _>>>()
1127 .map(Value::Object),
1128 }
1129 }
1130
1131 fn into_serde_value(self) -> serde_value::Value {
1132 use serde_value::Value as SerdeValue;
1133
1134 match self {
1135 Self::Null => SerdeValue::Unit,
1136 Self::Boolean(value) => SerdeValue::Bool(value),
1137 Self::Long(value) => SerdeValue::I64(value),
1138 Self::Double(value) => SerdeValue::F64(value),
1139 Self::Bytes(value) => SerdeValue::Bytes(value),
1140 Self::String(value) => SerdeValue::String(value),
1141 Self::Array(values) => {
1142 SerdeValue::Seq(values.into_iter().map(Self::into_serde_value).collect())
1143 }
1144 Self::Map(values) => SerdeValue::Map(
1145 values
1146 .into_iter()
1147 .map(|(key, value)| (SerdeValue::String(key), value.into_serde_value()))
1148 .collect(),
1149 ),
1150 }
1151 }
1152
1153 pub fn deserialize<T: DeserializeOwned>(self) -> Result<T> {
1154 self.into_serde_value().deserialize_into().map_err(|error| {
1155 Error::Codec(format!(
1156 "avro_value_type_mismatch: could not adapt decoded value: {error}"
1157 ))
1158 })
1159 }
1160}
1161
1162impl Serialize for AvroValue {
1163 fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
1164 where
1165 S: Serializer,
1166 {
1167 match self {
1168 Self::Null => serializer.serialize_unit(),
1169 Self::Boolean(value) => serializer.serialize_bool(*value),
1170 Self::Long(value) => serializer.serialize_i64(*value),
1171 Self::Double(value) => serializer.serialize_f64(*value),
1172 Self::Bytes(value) => serializer.serialize_bytes(value),
1173 Self::String(value) => serializer.serialize_str(value),
1174 Self::Array(values) => {
1175 let mut sequence = serializer.serialize_seq(Some(values.len()))?;
1176 for value in values {
1177 sequence.serialize_element(value)?;
1178 }
1179 sequence.end()
1180 }
1181 Self::Map(values) => {
1182 let mut map = serializer.serialize_map(Some(values.len()))?;
1183 for (key, value) in values {
1184 map.serialize_entry(key, value)?;
1185 }
1186 map.end()
1187 }
1188 }
1189 }
1190}
1191
1192pub fn encode_avro_value(value: &AvroValue) -> Result<PayloadEnvelope> {
1193 let datum = avro_value_to_datum(value)?;
1194 let datum = to_avro_datum(avro_value_schema()?, datum)
1195 .map_err(|err| Error::Codec(format!("avro_value_encode_failed: {err}")))?;
1196 let mut bytes = Vec::with_capacity(datum.len() + 10);
1197 bytes.extend_from_slice(&AVRO_SINGLE_OBJECT_MAGIC);
1198 bytes.extend_from_slice(&AVRO_VALUE_SCHEMA_FINGERPRINT);
1199 bytes.extend_from_slice(&datum);
1200 Ok(PayloadEnvelope {
1201 codec: DEFAULT_CODEC.to_string(),
1202 blob: BASE64.encode(bytes),
1203 })
1204}
1205
1206pub fn decode_avro_value(envelope: &PayloadEnvelope) -> Result<AvroValue> {
1207 if envelope.codec != DEFAULT_CODEC {
1208 return Err(Error::Codec(format!(
1209 "unsupported payload codec {:?}",
1210 envelope.codec
1211 )));
1212 }
1213 decode_avro_value_blob(&envelope.blob)
1214}
1215
1216pub fn encode_payload<T: Serialize>(value: &T, codec: &str) -> Result<PayloadEnvelope> {
1217 let blob = match codec {
1218 JSON_CODEC => serde_json::to_string(value)?,
1219 DEFAULT_CODEC => encode_avro_value(&AvroValue::from_serialize(value)?)?.blob,
1220 other => return Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1221 };
1222
1223 Ok(PayloadEnvelope {
1224 codec: codec.to_string(),
1225 blob,
1226 })
1227}
1228
1229pub fn decode_payload<T: DeserializeOwned>(envelope: &PayloadEnvelope) -> Result<T> {
1230 match envelope.codec.as_str() {
1231 JSON_CODEC => Ok(serde_json::from_str(&envelope.blob)?),
1232 DEFAULT_CODEC => decode_avro_value(envelope)?.deserialize(),
1233 other => Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1234 }
1235}
1236
1237#[cfg(test)]
1238fn encode_value_envelope(value: &Value, codec: &str) -> Result<Value> {
1239 Ok(serde_json::to_value(encode_payload(value, codec)?)?)
1240}
1241
1242fn decode_wire_value(value: &Value, fallback_codec: &str) -> Result<Value> {
1243 if value.is_null() {
1244 return Ok(Value::Null);
1245 }
1246
1247 if let Some(object) = value.as_object() {
1248 if let (Some(codec), Some(blob)) = (
1249 object.get("codec").and_then(Value::as_str),
1250 object.get("blob").and_then(Value::as_str),
1251 ) {
1252 return decode_blob(blob, codec);
1253 }
1254 }
1255
1256 if let Some(blob) = value.as_str() {
1257 return decode_blob(blob, fallback_codec);
1258 }
1259
1260 Ok(value.clone())
1261}
1262
1263fn encode_typed_envelope(value: &AvroValue, codec: &str) -> Result<Value> {
1264 let envelope = match codec {
1265 DEFAULT_CODEC => encode_avro_value(value)?,
1266 JSON_CODEC => PayloadEnvelope {
1267 codec: JSON_CODEC.to_string(),
1268 blob: serde_json::to_string(&value.clone().into_json()?)?,
1269 },
1270 other => return Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1271 };
1272 Ok(serde_json::to_value(envelope)?)
1273}
1274
1275fn decode_wire_avro_value(value: &Value, fallback_codec: &str) -> Result<AvroValue> {
1276 if value.is_null() {
1277 return Ok(AvroValue::Null);
1278 }
1279
1280 if let Some(object) = value.as_object() {
1281 if let (Some(codec), Some(blob)) = (
1282 object.get("codec").and_then(Value::as_str),
1283 object.get("blob").and_then(Value::as_str),
1284 ) {
1285 return match codec {
1286 DEFAULT_CODEC => decode_avro_value_blob(blob),
1287 JSON_CODEC => AvroValue::from_serialize(&serde_json::from_str::<Value>(blob)?),
1288 other => Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1289 };
1290 }
1291 }
1292
1293 if let Some(blob) = value.as_str() {
1294 return match fallback_codec {
1295 DEFAULT_CODEC => decode_avro_value_blob(blob),
1296 JSON_CODEC => AvroValue::from_serialize(&serde_json::from_str::<Value>(blob)?),
1297 other => Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1298 };
1299 }
1300
1301 AvroValue::from_serialize(value)
1302}
1303
1304fn normalize_avro_arguments(value: AvroValue) -> AvroValue {
1305 match value {
1306 AvroValue::Null => AvroValue::Array(Vec::new()),
1307 AvroValue::Array(_) => value,
1308 other => AvroValue::Array(vec![other]),
1309 }
1310}
1311
1312fn decode_blob(blob: &str, codec: &str) -> Result<Value> {
1313 match codec {
1314 JSON_CODEC => Ok(serde_json::from_str(blob)?),
1315 DEFAULT_CODEC => decode_avro_value_blob(blob)?.into_json(),
1316 other => Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1317 }
1318}
1319
1320fn decode_avro_value_blob(blob: &str) -> Result<AvroValue> {
1321 let bytes = BASE64.decode(blob).map_err(|err| {
1322 Error::Codec(format!(
1323 "invalid_payload_framing: expected strict base64 Avro single-object bytes: {err}"
1324 ))
1325 })?;
1326
1327 if bytes.len() < 10 || bytes[..2] != AVRO_SINGLE_OBJECT_MAGIC {
1328 return Err(Error::Codec(
1329 "invalid_payload_framing: expected Avro single-object magic c301".to_string(),
1330 ));
1331 }
1332
1333 let fingerprint: [u8; 8] = bytes[2..10]
1334 .try_into()
1335 .map_err(|_| Error::Codec("invalid Avro fingerprint length".to_string()))?;
1336 if fingerprint != AVRO_VALUE_SCHEMA_FINGERPRINT {
1337 return Err(Error::Codec(format!(
1338 "unsupported_payload_schema: unknown CRC-64-AVRO fingerprint {}",
1339 fingerprint
1340 .iter()
1341 .map(|byte| format!("{byte:02x}"))
1342 .collect::<String>()
1343 )));
1344 }
1345
1346 let mut datum_reader = StrictAvroDatumReader::new(&bytes[10..]);
1347 let datum = from_avro_datum(avro_value_schema()?, &mut datum_reader, None);
1351 if datum_reader.truncated {
1352 return Err(Error::Codec(
1353 "invalid_payload_framing: truncated Avro Value datum".to_string(),
1354 ));
1355 }
1356 let datum = datum.map_err(|err| {
1357 Error::Codec(format!(
1358 "invalid_payload_framing: malformed Avro Value datum: {err}"
1359 ))
1360 })?;
1361 if datum_reader.remaining() != 0 {
1362 return Err(Error::Codec(format!(
1363 "invalid_payload_framing: {} trailing bytes after Avro Value datum",
1364 datum_reader.remaining()
1365 )));
1366 }
1367 avro_value_from_datum(datum)
1368}
1369
1370struct StrictAvroDatumReader<'a> {
1371 bytes: &'a [u8],
1372 offset: usize,
1373 truncated: bool,
1374}
1375
1376impl<'a> StrictAvroDatumReader<'a> {
1377 fn new(bytes: &'a [u8]) -> Self {
1378 Self {
1379 bytes,
1380 offset: 0,
1381 truncated: false,
1382 }
1383 }
1384
1385 fn remaining(&self) -> usize {
1386 self.bytes.len() - self.offset
1387 }
1388}
1389
1390impl Read for StrictAvroDatumReader<'_> {
1391 fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
1392 let count = buffer.len().min(self.remaining());
1393 buffer[..count].copy_from_slice(&self.bytes[self.offset..self.offset + count]);
1394 self.offset += count;
1395 if count < buffer.len() {
1396 self.truncated = true;
1397 }
1398
1399 Ok(count)
1400 }
1401}
1402
1403fn avro_value_to_datum(value: &AvroValue) -> Result<AvroDatum> {
1404 let branch = match value {
1405 AvroValue::Null => AvroDatum::Union(0, Box::new(AvroDatum::Null)),
1406 AvroValue::Boolean(value) => AvroDatum::Union(
1407 1,
1408 Box::new(AvroDatum::Record(vec![(
1409 "boolean".to_string(),
1410 AvroDatum::Boolean(*value),
1411 )])),
1412 ),
1413 AvroValue::Long(value) => AvroDatum::Union(
1414 2,
1415 Box::new(AvroDatum::Record(vec![(
1416 "long".to_string(),
1417 AvroDatum::Long(*value),
1418 )])),
1419 ),
1420 AvroValue::Double(value) => {
1421 if !value.is_finite() {
1422 return Err(Error::Codec(
1423 "non_finite_float: Avro Value doubles must be finite".to_string(),
1424 ));
1425 }
1426 AvroDatum::Union(
1427 3,
1428 Box::new(AvroDatum::Record(vec![(
1429 "double".to_string(),
1430 AvroDatum::Double(*value),
1431 )])),
1432 )
1433 }
1434 AvroValue::Bytes(value) => AvroDatum::Union(
1435 4,
1436 Box::new(AvroDatum::Record(vec![(
1437 "bytes".to_string(),
1438 AvroDatum::Bytes(value.clone()),
1439 )])),
1440 ),
1441 AvroValue::String(value) => AvroDatum::Union(
1442 5,
1443 Box::new(AvroDatum::Record(vec![(
1444 "string".to_string(),
1445 AvroDatum::String(value.clone()),
1446 )])),
1447 ),
1448 AvroValue::Array(values) => AvroDatum::Union(
1449 6,
1450 Box::new(AvroDatum::Record(vec![(
1451 "items".to_string(),
1452 AvroDatum::Array(
1453 values
1454 .iter()
1455 .map(avro_value_to_datum)
1456 .collect::<Result<Vec<_>>>()?,
1457 ),
1458 )])),
1459 ),
1460 AvroValue::Map(values) => AvroDatum::Union(
1461 7,
1462 Box::new(AvroDatum::Record(vec![(
1463 "entries".to_string(),
1464 AvroDatum::Map(
1465 values
1466 .iter()
1467 .map(|(key, value)| Ok((key.clone(), avro_value_to_datum(value)?)))
1468 .collect::<Result<HashMap<_, _>>>()?,
1469 ),
1470 )])),
1471 ),
1472 };
1473 Ok(AvroDatum::Record(vec![("value".to_string(), branch)]))
1474}
1475
1476fn avro_value_from_datum(datum: AvroDatum) -> Result<AvroValue> {
1477 let AvroDatum::Record(mut outer) = datum else {
1478 return Err(Error::Codec(
1479 "invalid_payload_framing: datum is not a Value record".to_string(),
1480 ));
1481 };
1482 let (_, branch) = outer
1483 .pop()
1484 .filter(|(name, _)| name == "value")
1485 .ok_or_else(|| Error::Codec("invalid_payload_framing: Value field missing".to_string()))?;
1486 let AvroDatum::Union(_, branch) = branch else {
1487 return Err(Error::Codec(
1488 "invalid_payload_framing: invalid Value union".to_string(),
1489 ));
1490 };
1491 match *branch {
1492 AvroDatum::Null => Ok(AvroValue::Null),
1493 AvroDatum::Record(mut fields) => {
1494 let (name, value) = fields.pop().ok_or_else(|| {
1495 Error::Codec("invalid_payload_framing: empty Value branch".to_string())
1496 })?;
1497 match (name.as_str(), value) {
1498 ("boolean", AvroDatum::Boolean(value)) => Ok(AvroValue::Boolean(value)),
1499 ("long", AvroDatum::Long(value)) => Ok(AvroValue::Long(value)),
1500 ("double", AvroDatum::Double(value)) if value.is_finite() => {
1501 Ok(AvroValue::Double(value))
1502 }
1503 ("bytes", AvroDatum::Bytes(value)) => Ok(AvroValue::Bytes(value)),
1504 ("string", AvroDatum::String(value)) => Ok(AvroValue::String(value)),
1505 ("items", AvroDatum::Array(values)) => values
1506 .into_iter()
1507 .map(avro_value_from_datum)
1508 .collect::<Result<Vec<_>>>()
1509 .map(AvroValue::Array),
1510 ("entries", AvroDatum::Map(values)) => values
1511 .into_iter()
1512 .map(|(key, value)| Ok((key, avro_value_from_datum(value)?)))
1513 .collect::<Result<BTreeMap<_, _>>>()
1514 .map(AvroValue::Map),
1515 _ => Err(Error::Codec(
1516 "invalid_payload_framing: unknown Value branch".to_string(),
1517 )),
1518 }
1519 }
1520 _ => Err(Error::Codec(
1521 "invalid_payload_framing: invalid Value branch".to_string(),
1522 )),
1523 }
1524}
1525
1526fn avro_value_schema() -> Result<&'static Schema> {
1527 match AVRO_VALUE_SCHEMA.get_or_init(|| {
1528 Schema::parse_str(AVRO_VALUE_SCHEMA_JSON)
1529 .map_err(|err| format!("could not parse Avro Value schema: {err}"))
1530 }) {
1531 Ok(schema) => Ok(schema),
1532 Err(message) => Err(Error::Codec(message.clone())),
1533 }
1534}
1535
1536#[derive(Clone, Debug)]
1537pub struct Client {
1538 http: reqwest::Client,
1539 base_url: String,
1540 token: Option<String>,
1541 control_token: Option<String>,
1542 worker_token: Option<String>,
1543 namespace: String,
1544}
1545
1546impl Client {
1547 pub fn new(base_url: impl Into<String>) -> Result<Self> {
1548 Self::builder(base_url).build()
1549 }
1550
1551 pub fn builder(base_url: impl Into<String>) -> ClientBuilder {
1552 ClientBuilder {
1553 base_url: base_url.into(),
1554 token: None,
1555 control_token: None,
1556 worker_token: None,
1557 namespace: "default".to_string(),
1558 timeout: Duration::from_secs(60),
1559 }
1560 }
1561
1562 pub async fn health(&self) -> Result<Value> {
1563 self.request_json(
1564 reqwest::Method::GET,
1565 "/health",
1566 RequestProtocol::ControlPlane,
1567 Option::<&Value>::None,
1568 )
1569 .await
1570 }
1571
1572 pub async fn cluster_info(&self) -> Result<Value> {
1573 self.request_json(
1574 reqwest::Method::GET,
1575 "/cluster/info",
1576 RequestProtocol::ControlPlane,
1577 Option::<&Value>::None,
1578 )
1579 .await
1580 }
1581
1582 pub async fn start_workflow<T: Serialize>(
1583 &self,
1584 workflow_type: &str,
1585 task_queue: &str,
1586 workflow_id: &str,
1587 input: T,
1588 ) -> Result<WorkflowHandle> {
1589 self.start_workflow_with_options(
1590 workflow_type,
1591 task_queue,
1592 workflow_id,
1593 WorkflowStartOptions::default(),
1594 input,
1595 )
1596 .await
1597 }
1598
1599 pub async fn start_workflow_with_options<T: Serialize>(
1602 &self,
1603 workflow_type: &str,
1604 task_queue: &str,
1605 workflow_id: &str,
1606 options: WorkflowStartOptions,
1607 input: T,
1608 ) -> Result<WorkflowHandle> {
1609 options.validate()?;
1610 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1611 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
1612 let body = json!({
1613 "workflow_id": workflow_id,
1614 "workflow_type": workflow_type,
1615 "task_queue": task_queue,
1616 "input": input_envelope,
1617 "execution_timeout_seconds": options.execution_timeout_seconds,
1618 "run_timeout_seconds": options.run_timeout_seconds
1619 });
1620
1621 let data: Value = self
1622 .request_json(
1623 reqwest::Method::POST,
1624 "/workflows",
1625 RequestProtocol::ControlPlane,
1626 Some(&body),
1627 )
1628 .await?;
1629
1630 Ok(WorkflowHandle {
1631 client: self.clone(),
1632 workflow_id: data
1633 .get("workflow_id")
1634 .and_then(Value::as_str)
1635 .unwrap_or(workflow_id)
1636 .to_string(),
1637 run_id: data
1638 .get("run_id")
1639 .and_then(Value::as_str)
1640 .map(str::to_string),
1641 workflow_type: data
1642 .get("workflow_type")
1643 .and_then(Value::as_str)
1644 .unwrap_or(workflow_type)
1645 .to_string(),
1646 })
1647 }
1648
1649 pub async fn signal_workflow<T: Serialize>(
1650 &self,
1651 workflow_id: &str,
1652 signal_name: &str,
1653 input: T,
1654 ) -> Result<Value> {
1655 self.signal_workflow_target(workflow_id, None, signal_name, input)
1656 .await
1657 }
1658
1659 pub async fn signal_workflow_run<T: Serialize>(
1661 &self,
1662 workflow_id: &str,
1663 run_id: &str,
1664 signal_name: &str,
1665 input: T,
1666 ) -> Result<Value> {
1667 self.signal_workflow_target(workflow_id, Some(run_id), signal_name, input)
1668 .await
1669 }
1670
1671 async fn signal_workflow_target<T: Serialize>(
1672 &self,
1673 workflow_id: &str,
1674 run_id: Option<&str>,
1675 signal_name: &str,
1676 input: T,
1677 ) -> Result<Value> {
1678 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1679 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
1680 let body = json!({
1681 "input": input_envelope
1682 });
1683 let path = match run_id {
1684 Some(run_id) => {
1685 format!("/workflows/{workflow_id}/runs/{run_id}/signal/{signal_name}")
1686 }
1687 None => format!("/workflows/{workflow_id}/signal/{signal_name}"),
1688 };
1689 self.request_json(
1690 reqwest::Method::POST,
1691 &path,
1692 RequestProtocol::ControlPlane,
1693 Some(&body),
1694 )
1695 .await
1696 }
1697
1698 pub async fn cancel_workflow(
1700 &self,
1701 workflow_id: &str,
1702 options: WorkflowCommandOptions,
1703 ) -> Result<WorkflowCommandResult> {
1704 self.workflow_command(workflow_id, None, WorkflowCommandKind::Cancel, options)
1705 .await
1706 }
1707
1708 pub async fn cancel_workflow_run(
1710 &self,
1711 workflow_id: &str,
1712 run_id: &str,
1713 options: WorkflowCommandOptions,
1714 ) -> Result<WorkflowCommandResult> {
1715 self.workflow_command(
1716 workflow_id,
1717 Some(run_id),
1718 WorkflowCommandKind::Cancel,
1719 options,
1720 )
1721 .await
1722 }
1723
1724 pub async fn terminate_workflow(
1726 &self,
1727 workflow_id: &str,
1728 options: WorkflowCommandOptions,
1729 ) -> Result<WorkflowCommandResult> {
1730 self.workflow_command(workflow_id, None, WorkflowCommandKind::Terminate, options)
1731 .await
1732 }
1733
1734 pub async fn terminate_workflow_run(
1736 &self,
1737 workflow_id: &str,
1738 run_id: &str,
1739 options: WorkflowCommandOptions,
1740 ) -> Result<WorkflowCommandResult> {
1741 self.workflow_command(
1742 workflow_id,
1743 Some(run_id),
1744 WorkflowCommandKind::Terminate,
1745 options,
1746 )
1747 .await
1748 }
1749
1750 async fn workflow_command(
1751 &self,
1752 workflow_id: &str,
1753 run_id: Option<&str>,
1754 command: WorkflowCommandKind,
1755 options: WorkflowCommandOptions,
1756 ) -> Result<WorkflowCommandResult> {
1757 let path = match run_id {
1758 Some(run_id) => format!(
1759 "/workflows/{workflow_id}/runs/{run_id}/{}",
1760 command.as_str()
1761 ),
1762 None => format!("/workflows/{workflow_id}/{}", command.as_str()),
1763 };
1764 let data = match self
1765 .request_json(
1766 reqwest::Method::POST,
1767 &path,
1768 RequestProtocol::ControlPlane,
1769 Some(&options),
1770 )
1771 .await
1772 {
1773 Ok(data) => data,
1774 Err(Error::Http { status, body }) => {
1775 return Err(Error::WorkflowCommandRejected(workflow_command_rejection(
1776 command,
1777 status,
1778 body,
1779 workflow_id,
1780 run_id,
1781 )));
1782 }
1783 Err(error) => return Err(error),
1784 };
1785
1786 Ok(workflow_command_result(command, data, workflow_id, run_id))
1787 }
1788
1789 pub async fn query_workflow<T: Serialize>(
1795 &self,
1796 workflow_id: &str,
1797 query_name: &str,
1798 input: T,
1799 ) -> Result<Value> {
1800 self.query_workflow_target(workflow_id, None, query_name, input)
1801 .await
1802 }
1803
1804 pub async fn query_workflow_run<T: Serialize>(
1806 &self,
1807 workflow_id: &str,
1808 run_id: &str,
1809 query_name: &str,
1810 input: T,
1811 ) -> Result<Value> {
1812 self.query_workflow_target(workflow_id, Some(run_id), query_name, input)
1813 .await
1814 }
1815
1816 pub async fn query_workflow_avro_value<T: Serialize>(
1818 &self,
1819 workflow_id: &str,
1820 query_name: &str,
1821 input: T,
1822 ) -> Result<AvroValue> {
1823 self.query_workflow_avro_value_target(workflow_id, None, query_name, input)
1824 .await
1825 }
1826
1827 pub async fn query_workflow_run_avro_value<T: Serialize>(
1829 &self,
1830 workflow_id: &str,
1831 run_id: &str,
1832 query_name: &str,
1833 input: T,
1834 ) -> Result<AvroValue> {
1835 self.query_workflow_avro_value_target(workflow_id, Some(run_id), query_name, input)
1836 .await
1837 }
1838
1839 async fn query_workflow_avro_value_target<T: Serialize>(
1840 &self,
1841 workflow_id: &str,
1842 run_id: Option<&str>,
1843 query_name: &str,
1844 input: T,
1845 ) -> Result<AvroValue> {
1846 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1847 let body = json!({"input": encode_typed_envelope(&input, DEFAULT_CODEC)?});
1848 let path = match run_id {
1849 Some(run_id) => {
1850 format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
1851 }
1852 None => format!("/workflows/{workflow_id}/query/{query_name}"),
1853 };
1854 let response: Value = match self
1855 .request_json(
1856 reqwest::Method::POST,
1857 &path,
1858 RequestProtocol::ControlPlane,
1859 Some(&body),
1860 )
1861 .await
1862 {
1863 Ok(response) => response,
1864 Err(Error::Http { status, body }) => {
1865 return Err(Error::QueryFailed(query_failure(status, body)));
1866 }
1867 Err(error) => return Err(error),
1868 };
1869
1870 let envelope = response
1871 .get("result_envelope")
1872 .filter(|envelope| !envelope.is_null())
1873 .ok_or_else(|| {
1874 Error::Codec(
1875 "missing_payload_envelope: typed query result requires result_envelope"
1876 .to_string(),
1877 )
1878 })?;
1879 decode_wire_avro_value(envelope, DEFAULT_CODEC)
1880 }
1881
1882 async fn query_workflow_target<T: Serialize>(
1883 &self,
1884 workflow_id: &str,
1885 run_id: Option<&str>,
1886 query_name: &str,
1887 input: T,
1888 ) -> Result<Value> {
1889 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1890 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
1891 let body = json!({
1892 "input": input_envelope
1893 });
1894 let path = match run_id {
1895 Some(run_id) => {
1896 format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
1897 }
1898 None => format!("/workflows/{workflow_id}/query/{query_name}"),
1899 };
1900 let response: Value = match self
1901 .request_json(
1902 reqwest::Method::POST,
1903 &path,
1904 RequestProtocol::ControlPlane,
1905 Some(&body),
1906 )
1907 .await
1908 {
1909 Ok(response) => response,
1910 Err(Error::Http { status, body }) => {
1911 return Err(Error::QueryFailed(query_failure(status, body)));
1912 }
1913 Err(error) => return Err(error),
1914 };
1915
1916 if let Some(envelope) = response
1917 .get("result_envelope")
1918 .filter(|envelope| !envelope.is_null())
1919 {
1920 return decode_wire_value(envelope, DEFAULT_CODEC);
1921 }
1922
1923 Ok(response.get("result").cloned().unwrap_or(Value::Null))
1924 }
1925
1926 pub async fn update_workflow<T: Serialize>(
1928 &self,
1929 workflow_id: &str,
1930 update_name: &str,
1931 input: T,
1932 request_id: Option<&str>,
1933 ) -> Result<Value> {
1934 let response = self
1935 .update_workflow_response(workflow_id, update_name, input, request_id)
1936 .await?;
1937 if let Some(envelope) = response
1938 .get("result_envelope")
1939 .filter(|envelope| !envelope.is_null())
1940 {
1941 return decode_wire_value(envelope, DEFAULT_CODEC);
1942 }
1943 Ok(response.get("result").cloned().unwrap_or(response))
1944 }
1945
1946 pub async fn update_workflow_avro_value<T: Serialize>(
1948 &self,
1949 workflow_id: &str,
1950 update_name: &str,
1951 input: T,
1952 request_id: Option<&str>,
1953 ) -> Result<AvroValue> {
1954 let response = self
1955 .update_workflow_response(workflow_id, update_name, input, request_id)
1956 .await?;
1957 let envelope = response
1958 .get("result_envelope")
1959 .filter(|envelope| !envelope.is_null())
1960 .ok_or_else(|| {
1961 Error::Codec(
1962 "missing_payload_envelope: typed update result requires result_envelope"
1963 .to_string(),
1964 )
1965 })?;
1966 decode_wire_avro_value(envelope, DEFAULT_CODEC)
1967 }
1968
1969 async fn update_workflow_response<T: Serialize>(
1970 &self,
1971 workflow_id: &str,
1972 update_name: &str,
1973 input: T,
1974 request_id: Option<&str>,
1975 ) -> Result<Value> {
1976 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1977 let mut body = json!({
1978 "input": encode_typed_envelope(&input, DEFAULT_CODEC)?,
1979 "wait_for": "completed",
1980 });
1981 if let Some(request_id) = request_id {
1982 body["request_id"] = json!(request_id);
1983 }
1984 self.request_json(
1985 reqwest::Method::POST,
1986 &format!("/workflows/{workflow_id}/update/{update_name}"),
1987 RequestProtocol::ControlPlane,
1988 Some(&body),
1989 )
1990 .await
1991 }
1992
1993 pub async fn describe_workflow(&self, workflow_id: &str) -> Result<WorkflowDescription> {
1994 let path = format!("/workflows/{workflow_id}");
1995 let mut data: WorkflowDescription = self
1996 .request_json(
1997 reqwest::Method::GET,
1998 &path,
1999 RequestProtocol::ControlPlane,
2000 Option::<&Value>::None,
2001 )
2002 .await?;
2003 data.decode_payloads()?;
2004 Ok(data)
2005 }
2006
2007 pub async fn describe_workflow_run(
2009 &self,
2010 workflow_id: &str,
2011 run_id: &str,
2012 ) -> Result<WorkflowDescription> {
2013 let path = format!("/workflows/{workflow_id}/runs/{run_id}");
2014 let mut data: WorkflowDescription = self
2015 .request_json(
2016 reqwest::Method::GET,
2017 &path,
2018 RequestProtocol::ControlPlane,
2019 Option::<&Value>::None,
2020 )
2021 .await?;
2022 data.decode_payloads()?;
2023 Ok(data)
2024 }
2025
2026 pub async fn register_worker(
2027 &self,
2028 worker_id: &str,
2029 task_queue: &str,
2030 supported_workflow_types: Vec<String>,
2031 supported_activity_types: Vec<String>,
2032 max_concurrent_workflow_tasks: usize,
2033 max_concurrent_activity_tasks: usize,
2034 ) -> Result<RegisterWorkerResponse> {
2035 self.register_worker_with_capabilities(
2036 worker_id,
2037 task_queue,
2038 supported_workflow_types,
2039 supported_activity_types,
2040 max_concurrent_workflow_tasks,
2041 max_concurrent_activity_tasks,
2042 Vec::new(),
2043 )
2044 .await
2045 }
2046
2047 pub async fn register_worker_with_capabilities(
2049 &self,
2050 worker_id: &str,
2051 task_queue: &str,
2052 supported_workflow_types: Vec<String>,
2053 supported_activity_types: Vec<String>,
2054 max_concurrent_workflow_tasks: usize,
2055 max_concurrent_activity_tasks: usize,
2056 capabilities: Vec<String>,
2057 ) -> Result<RegisterWorkerResponse> {
2058 self.register_worker_with_command_contracts(
2059 worker_id,
2060 task_queue,
2061 supported_workflow_types,
2062 supported_activity_types,
2063 max_concurrent_workflow_tasks,
2064 max_concurrent_activity_tasks,
2065 capabilities,
2066 Value::Object(serde_json::Map::new()),
2067 )
2068 .await
2069 }
2070
2071 #[allow(clippy::too_many_arguments)]
2073 pub async fn register_worker_with_command_contracts(
2074 &self,
2075 worker_id: &str,
2076 task_queue: &str,
2077 supported_workflow_types: Vec<String>,
2078 supported_activity_types: Vec<String>,
2079 max_concurrent_workflow_tasks: usize,
2080 max_concurrent_activity_tasks: usize,
2081 capabilities: Vec<String>,
2082 workflow_command_contracts: Value,
2083 ) -> Result<RegisterWorkerResponse> {
2084 let mut body = json!({
2085 "worker_id": worker_id,
2086 "task_queue": task_queue,
2087 "runtime": "rust",
2088 "sdk_version": SDK_VERSION,
2089 "supported_workflow_types": supported_workflow_types,
2090 "supported_activity_types": supported_activity_types,
2091 "capabilities": capabilities,
2092 "max_concurrent_workflow_tasks": max_concurrent_workflow_tasks,
2093 "max_concurrent_activity_tasks": max_concurrent_activity_tasks
2094 });
2095 if workflow_command_contracts
2096 .as_object()
2097 .is_some_and(|contracts| !contracts.is_empty())
2098 {
2099 body["workflow_command_contracts"] = workflow_command_contracts;
2100 }
2101
2102 self.request_json(
2103 reqwest::Method::POST,
2104 "/worker/register",
2105 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2106 Some(&body),
2107 )
2108 .await
2109 }
2110
2111 pub async fn poll_query_task(
2113 &self,
2114 worker_id: &str,
2115 task_queue: &str,
2116 timeout: Duration,
2117 ) -> Result<Option<QueryTask>> {
2118 Ok(self
2119 .poll_query_task_response(worker_id, task_queue, timeout)
2120 .await?
2121 .task)
2122 }
2123
2124 pub async fn poll_query_task_response(
2126 &self,
2127 worker_id: &str,
2128 task_queue: &str,
2129 timeout: Duration,
2130 ) -> Result<PollQueryTaskResponse> {
2131 let poll_request_id = unique_request_id("rust-query-poll");
2132 self.poll_query_task_response_with_request_id(
2133 worker_id,
2134 task_queue,
2135 timeout,
2136 &poll_request_id,
2137 1,
2138 )
2139 .await
2140 }
2141
2142 async fn poll_query_task_response_with_request_id(
2143 &self,
2144 worker_id: &str,
2145 task_queue: &str,
2146 timeout: Duration,
2147 poll_request_id: &str,
2148 transport_retries: usize,
2149 ) -> Result<PollQueryTaskResponse> {
2150 let timeout_seconds = long_poll_timeout_seconds(timeout);
2151 let body = json!({
2152 "worker_id": worker_id,
2153 "task_queue": task_queue,
2154 "poll_request_id": poll_request_id,
2155 "timeout_seconds": timeout_seconds,
2156 });
2157 self.poll_request_json(
2158 "/worker/query-tasks/poll",
2159 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
2160 &body,
2161 timeout + Duration::from_secs(5),
2162 transport_retries,
2163 )
2164 .await
2165 }
2166
2167 pub async fn complete_query_task<T: Serialize>(
2169 &self,
2170 query_task_id: &str,
2171 lease_owner: &str,
2172 query_task_attempt: u64,
2173 result: T,
2174 codec: &str,
2175 ) -> Result<Value> {
2176 let typed_result = AvroValue::from_serialize(&result)?;
2177 let result_envelope = encode_typed_envelope(&typed_result, codec)?;
2178 self.complete_query_task_with_envelope(
2179 query_task_id,
2180 lease_owner,
2181 query_task_attempt,
2182 typed_result.into_json()?,
2183 result_envelope,
2184 )
2185 .await
2186 }
2187
2188 async fn complete_query_task_with_envelope(
2189 &self,
2190 query_task_id: &str,
2191 lease_owner: &str,
2192 query_task_attempt: u64,
2193 result: Value,
2194 result_envelope: Value,
2195 ) -> Result<Value> {
2196 let body = json!({
2197 "lease_owner": lease_owner,
2198 "query_task_attempt": query_task_attempt,
2199 "result": result,
2200 "result_envelope": result_envelope,
2201 });
2202 let path = format!("/worker/query-tasks/{query_task_id}/complete");
2203 let response = self
2204 .request_json(
2205 reqwest::Method::POST,
2206 &path,
2207 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
2208 Some(&body),
2209 )
2210 .await;
2211 query_task_response(response)
2212 }
2213
2214 pub async fn fail_query_task(
2216 &self,
2217 query_task_id: &str,
2218 lease_owner: &str,
2219 query_task_attempt: u64,
2220 message: impl Into<String>,
2221 reason: impl Into<String>,
2222 failure_type: impl Into<String>,
2223 ) -> Result<Value> {
2224 let body = json!({
2225 "lease_owner": lease_owner,
2226 "query_task_attempt": query_task_attempt,
2227 "failure": {
2228 "message": message.into(),
2229 "reason": reason.into(),
2230 "type": failure_type.into(),
2231 }
2232 });
2233 let path = format!("/worker/query-tasks/{query_task_id}/fail");
2234 let response = self
2235 .request_json(
2236 reqwest::Method::POST,
2237 &path,
2238 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
2239 Some(&body),
2240 )
2241 .await;
2242 query_task_response(response)
2243 }
2244
2245 pub async fn heartbeat_worker(
2246 &self,
2247 worker_id: &str,
2248 workflow_available: usize,
2249 activity_available: usize,
2250 ) -> Result<Value> {
2251 let body = json!({
2252 "worker_id": worker_id,
2253 "task_slots": {
2254 "workflow_available": workflow_available,
2255 "activity_available": activity_available
2256 },
2257 "process_metrics": {
2258 "process_id": std::process::id(),
2259 "process_uptime_seconds": 0
2260 }
2261 });
2262
2263 self.request_json(
2264 reqwest::Method::POST,
2265 "/worker/heartbeat",
2266 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2267 Some(&body),
2268 )
2269 .await
2270 }
2271
2272 pub async fn poll_workflow_task(
2273 &self,
2274 worker_id: &str,
2275 task_queue: &str,
2276 timeout: Duration,
2277 ) -> Result<Option<WorkflowTask>> {
2278 Ok(self
2279 .poll_workflow_task_response(worker_id, task_queue, timeout)
2280 .await?
2281 .task)
2282 }
2283
2284 pub async fn poll_workflow_task_response(
2285 &self,
2286 worker_id: &str,
2287 task_queue: &str,
2288 timeout: Duration,
2289 ) -> Result<PollWorkflowTaskResponse> {
2290 let poll_request_id = unique_request_id("rust-workflow-poll");
2291 self.poll_workflow_task_response_with_request_id(
2292 worker_id,
2293 task_queue,
2294 timeout,
2295 &poll_request_id,
2296 1,
2297 )
2298 .await
2299 }
2300
2301 async fn poll_workflow_task_response_with_request_id(
2302 &self,
2303 worker_id: &str,
2304 task_queue: &str,
2305 timeout: Duration,
2306 poll_request_id: &str,
2307 transport_retries: usize,
2308 ) -> Result<PollWorkflowTaskResponse> {
2309 let body = json!({
2310 "worker_id": worker_id,
2311 "task_queue": task_queue,
2312 "poll_request_id": poll_request_id,
2313 "timeout_seconds": long_poll_timeout_seconds(timeout),
2314 });
2315 let mut data: PollWorkflowTaskResponse = self
2316 .poll_request_json(
2317 "/worker/workflow-tasks/poll",
2318 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2319 &body,
2320 timeout + Duration::from_secs(5),
2321 transport_retries,
2322 )
2323 .await?;
2324
2325 if let Some(task) = data.task.as_mut() {
2326 self.fetch_remaining_workflow_history(worker_id, task)
2327 .await?;
2328 }
2329
2330 Ok(data)
2331 }
2332
2333 async fn fetch_remaining_workflow_history(
2334 &self,
2335 worker_id: &str,
2336 task: &mut WorkflowTask,
2337 ) -> Result<()> {
2338 let mut next_token = task.next_history_page_token.clone();
2339
2340 while let Some(token) = next_token.take().filter(|token| !token.is_empty()) {
2341 let lease_owner = task
2342 .lease_owner
2343 .clone()
2344 .unwrap_or_else(|| worker_id.to_string());
2345 let page = self
2346 .workflow_task_history_page(
2347 &task.task_id,
2348 &lease_owner,
2349 task.workflow_task_attempt,
2350 &token,
2351 )
2352 .await?;
2353
2354 task.append_history_page(page);
2355
2356 if task.next_history_page_token.as_deref() == Some(token.as_str()) {
2357 return Err(Error::Codec(
2358 "workflow history pagination returned the same page token".to_string(),
2359 ));
2360 }
2361
2362 next_token = task.next_history_page_token.clone();
2363 }
2364
2365 Ok(())
2366 }
2367
2368 async fn workflow_task_history_page(
2369 &self,
2370 task_id: &str,
2371 lease_owner: &str,
2372 workflow_task_attempt: u64,
2373 next_history_page_token: &str,
2374 ) -> Result<WorkflowTaskHistoryPage> {
2375 let body = json!({
2376 "lease_owner": lease_owner,
2377 "workflow_task_attempt": workflow_task_attempt,
2378 "next_history_page_token": next_history_page_token
2379 });
2380 let path = format!("/worker/workflow-tasks/{task_id}/history");
2381
2382 self.request_json(
2383 reqwest::Method::POST,
2384 &path,
2385 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2386 Some(&body),
2387 )
2388 .await
2389 }
2390
2391 pub async fn complete_workflow_task(
2392 &self,
2393 task_id: &str,
2394 lease_owner: &str,
2395 workflow_task_attempt: u64,
2396 commands: Vec<Value>,
2397 ) -> Result<Value> {
2398 let body = json!({
2399 "lease_owner": lease_owner,
2400 "workflow_task_attempt": workflow_task_attempt,
2401 "commands": commands
2402 });
2403 let path = format!("/worker/workflow-tasks/{task_id}/complete");
2404 self.request_json(
2405 reqwest::Method::POST,
2406 &path,
2407 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2408 Some(&body),
2409 )
2410 .await
2411 }
2412
2413 pub async fn fail_workflow_task(
2414 &self,
2415 task_id: &str,
2416 lease_owner: &str,
2417 workflow_task_attempt: u64,
2418 message: impl Into<String>,
2419 ) -> Result<Value> {
2420 self.fail_workflow_task_with_type(
2421 task_id,
2422 lease_owner,
2423 workflow_task_attempt,
2424 message,
2425 "RustWorkflowTaskFailure",
2426 )
2427 .await
2428 }
2429
2430 async fn fail_workflow_task_with_type(
2431 &self,
2432 task_id: &str,
2433 lease_owner: &str,
2434 workflow_task_attempt: u64,
2435 message: impl Into<String>,
2436 failure_type: &str,
2437 ) -> Result<Value> {
2438 let body = json!({
2439 "lease_owner": lease_owner,
2440 "workflow_task_attempt": workflow_task_attempt,
2441 "failure": {
2442 "message": message.into(),
2443 "type": failure_type
2444 }
2445 });
2446 let path = format!("/worker/workflow-tasks/{task_id}/fail");
2447 self.request_json(
2448 reqwest::Method::POST,
2449 &path,
2450 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2451 Some(&body),
2452 )
2453 .await
2454 }
2455
2456 pub async fn poll_activity_task(
2457 &self,
2458 worker_id: &str,
2459 task_queue: &str,
2460 timeout: Duration,
2461 ) -> Result<Option<ActivityTask>> {
2462 Ok(self
2463 .poll_activity_task_response(worker_id, task_queue, timeout)
2464 .await?
2465 .task)
2466 }
2467
2468 pub async fn poll_activity_task_response(
2470 &self,
2471 worker_id: &str,
2472 task_queue: &str,
2473 timeout: Duration,
2474 ) -> Result<PollActivityTaskResponse> {
2475 let poll_request_id = unique_request_id("rust-activity-poll");
2476 self.poll_activity_task_response_with_request_id(
2477 worker_id,
2478 task_queue,
2479 timeout,
2480 &poll_request_id,
2481 1,
2482 )
2483 .await
2484 }
2485
2486 async fn poll_activity_task_response_with_request_id(
2487 &self,
2488 worker_id: &str,
2489 task_queue: &str,
2490 timeout: Duration,
2491 poll_request_id: &str,
2492 transport_retries: usize,
2493 ) -> Result<PollActivityTaskResponse> {
2494 let body = json!({
2495 "worker_id": worker_id,
2496 "task_queue": task_queue,
2497 "poll_request_id": poll_request_id,
2498 "timeout_seconds": long_poll_timeout_seconds(timeout),
2499 });
2500 let data: PollActivityTaskResponse = self
2501 .poll_request_json(
2502 "/worker/activity-tasks/poll",
2503 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2504 &body,
2505 timeout + Duration::from_secs(5),
2506 transport_retries,
2507 )
2508 .await?;
2509 Ok(data)
2510 }
2511
2512 pub async fn complete_activity_task<T: Serialize>(
2513 &self,
2514 task_id: &str,
2515 activity_attempt_id: &str,
2516 lease_owner: &str,
2517 result: T,
2518 codec: &str,
2519 ) -> Result<Value> {
2520 let result = encode_typed_envelope(&AvroValue::from_serialize(&result)?, codec)?;
2521 let body = json!({
2522 "activity_attempt_id": activity_attempt_id,
2523 "lease_owner": lease_owner,
2524 "result": result
2525 });
2526 let path = format!("/worker/activity-tasks/{task_id}/complete");
2527 activity_task_response(
2528 self.request_json(
2529 reqwest::Method::POST,
2530 &path,
2531 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2532 Some(&body),
2533 )
2534 .await,
2535 "complete",
2536 task_id,
2537 activity_attempt_id,
2538 )
2539 }
2540
2541 pub async fn fail_activity_task(
2542 &self,
2543 task_id: &str,
2544 activity_attempt_id: &str,
2545 lease_owner: &str,
2546 message: impl Into<String>,
2547 non_retryable: bool,
2548 ) -> Result<Value> {
2549 let body = json!({
2550 "activity_attempt_id": activity_attempt_id,
2551 "lease_owner": lease_owner,
2552 "failure": {
2553 "message": message.into(),
2554 "type": "RustActivityFailure",
2555 "non_retryable": non_retryable
2556 }
2557 });
2558 let path = format!("/worker/activity-tasks/{task_id}/fail");
2559 activity_task_response(
2560 self.request_json(
2561 reqwest::Method::POST,
2562 &path,
2563 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2564 Some(&body),
2565 )
2566 .await,
2567 "fail",
2568 task_id,
2569 activity_attempt_id,
2570 )
2571 }
2572
2573 pub async fn heartbeat_activity_task<T: Serialize>(
2574 &self,
2575 task_id: &str,
2576 activity_attempt_id: &str,
2577 lease_owner: &str,
2578 details: T,
2579 ) -> Result<ActivityHeartbeatResponse> {
2580 let details = encode_typed_envelope(&AvroValue::from_serialize(&details)?, DEFAULT_CODEC)?;
2581 let body = json!({
2582 "activity_attempt_id": activity_attempt_id,
2583 "lease_owner": lease_owner,
2584 "details": details
2585 });
2586 let path = format!("/worker/activity-tasks/{task_id}/heartbeat");
2587 activity_task_response(
2588 self.request_json(
2589 reqwest::Method::POST,
2590 &path,
2591 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2592 Some(&body),
2593 )
2594 .await,
2595 "heartbeat",
2596 task_id,
2597 activity_attempt_id,
2598 )
2599 }
2600
2601 async fn request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
2602 &self,
2603 method: reqwest::Method,
2604 path: &str,
2605 protocol: RequestProtocol,
2606 body: Option<&B>,
2607 ) -> Result<T> {
2608 self.request_json_with_timeout(method, path, protocol, body, Duration::from_secs(60))
2609 .await
2610 }
2611
2612 async fn request_json_with_timeout<T: DeserializeOwned, B: Serialize + ?Sized>(
2613 &self,
2614 method: reqwest::Method,
2615 path: &str,
2616 protocol: RequestProtocol,
2617 body: Option<&B>,
2618 timeout: Duration,
2619 ) -> Result<T> {
2620 let mut request = self
2621 .http
2622 .request(method, format!("{}/api{}", self.base_url, path))
2623 .timeout(timeout)
2624 .header(reqwest::header::ACCEPT, "application/json")
2625 .header(reqwest::header::CONTENT_TYPE, "application/json")
2626 .header("X-Namespace", &self.namespace);
2627
2628 match protocol {
2629 RequestProtocol::Worker(version) => {
2630 request = request.header("X-Durable-Workflow-Protocol-Version", version);
2631 }
2632 RequestProtocol::ControlPlane => {
2633 request = request.header(
2634 "X-Durable-Workflow-Control-Plane-Version",
2635 CONTROL_PLANE_VERSION,
2636 );
2637 }
2638 }
2639
2640 if let Some(token) = self.auth_token(protocol.is_worker()) {
2641 request = request.bearer_auth(token);
2642 }
2643
2644 if let Some(body) = body {
2645 request = request.json(body);
2646 }
2647
2648 let response = request.send().await?;
2649 let status = response.status();
2650 let bytes = response.bytes().await?;
2651
2652 if !status.is_success() {
2653 let body = String::from_utf8_lossy(&bytes).to_string();
2654 if let Some(protocol) = protocol_failure(status, &body) {
2655 return Err(Error::Protocol(protocol));
2656 }
2657 return Err(Error::Http { status, body });
2658 }
2659
2660 if bytes.is_empty() {
2661 return Ok(serde_json::from_value(Value::Null)?);
2662 }
2663
2664 Ok(serde_json::from_slice(&bytes)?)
2665 }
2666
2667 async fn poll_request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
2668 &self,
2669 path: &str,
2670 protocol: RequestProtocol,
2671 body: &B,
2672 timeout: Duration,
2673 max_retries: usize,
2674 ) -> Result<T> {
2675 let mut retries = 0;
2676
2677 loop {
2678 let response = self
2679 .request_json_with_timeout(
2680 reqwest::Method::POST,
2681 path,
2682 protocol,
2683 Some(body),
2684 timeout,
2685 )
2686 .await;
2687
2688 match response {
2689 Err(Error::Transport(_)) if retries < max_retries => retries += 1,
2690 response => return worker_poll_response(response),
2691 }
2692 }
2693 }
2694
2695 fn auth_token(&self, worker: bool) -> Option<&str> {
2696 if worker {
2697 self.worker_token
2698 .as_deref()
2699 .or(self.token.as_deref())
2700 .or(self.control_token.as_deref())
2701 } else {
2702 self.control_token
2703 .as_deref()
2704 .or(self.token.as_deref())
2705 .or(self.worker_token.as_deref())
2706 }
2707 }
2708}
2709
2710fn query_failure(status: reqwest::StatusCode, raw_body: String) -> QueryFailure {
2711 let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
2712 let reason = body
2713 .get("reason")
2714 .and_then(Value::as_str)
2715 .unwrap_or("query_rejected")
2716 .to_string();
2717 let message = body
2718 .get("message")
2719 .or_else(|| body.get("error"))
2720 .and_then(Value::as_str)
2721 .unwrap_or("workflow query was rejected")
2722 .to_string();
2723
2724 QueryFailure {
2725 status: status.as_u16(),
2726 reason,
2727 message,
2728 body,
2729 }
2730}
2731
2732fn workflow_command_result(
2733 command: WorkflowCommandKind,
2734 data: Value,
2735 workflow_id: &str,
2736 run_id: Option<&str>,
2737) -> WorkflowCommandResult {
2738 WorkflowCommandResult {
2739 command,
2740 workflow_id: data
2741 .get("workflow_id")
2742 .and_then(Value::as_str)
2743 .unwrap_or(workflow_id)
2744 .to_string(),
2745 run_id: data
2746 .get("run_id")
2747 .and_then(Value::as_str)
2748 .or(run_id)
2749 .map(str::to_string),
2750 outcome: data
2751 .get("outcome")
2752 .and_then(Value::as_str)
2753 .map(str::to_string),
2754 reason: data
2755 .get("reason")
2756 .and_then(Value::as_str)
2757 .map(str::to_string),
2758 command_status: data
2759 .get("command_status")
2760 .and_then(Value::as_str)
2761 .map(str::to_string),
2762 raw: data,
2763 }
2764}
2765
2766fn workflow_command_rejection(
2767 command: WorkflowCommandKind,
2768 status: reqwest::StatusCode,
2769 raw_body: String,
2770 workflow_id: &str,
2771 run_id: Option<&str>,
2772) -> WorkflowCommandRejection {
2773 let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
2774 WorkflowCommandRejection {
2775 command,
2776 status: status.as_u16(),
2777 reason: body
2778 .get("reason")
2779 .and_then(Value::as_str)
2780 .unwrap_or("workflow_command_rejected")
2781 .to_string(),
2782 message: body
2783 .get("message")
2784 .or_else(|| body.get("error"))
2785 .and_then(Value::as_str)
2786 .unwrap_or("workflow lifecycle command was rejected")
2787 .to_string(),
2788 workflow_id: body
2789 .get("workflow_id")
2790 .and_then(Value::as_str)
2791 .unwrap_or(workflow_id)
2792 .to_string(),
2793 run_id: body
2794 .get("run_id")
2795 .and_then(Value::as_str)
2796 .or(run_id)
2797 .map(str::to_string),
2798 target_scope: body
2799 .get("target_scope")
2800 .and_then(Value::as_str)
2801 .map(str::to_string),
2802 body,
2803 }
2804}
2805
2806fn query_task_response(response: Result<Value>) -> Result<Value> {
2807 match response {
2808 Err(Error::Http { status, body }) => Err(Error::QueryFailed(query_failure(status, body))),
2809 response => response,
2810 }
2811}
2812
2813fn worker_poll_response<T: DeserializeOwned>(response: Result<T>) -> Result<T> {
2814 match response {
2815 Err(Error::Http { status, body })
2816 if status == reqwest::StatusCode::CONFLICT && worker_poll_body_is_stop(&body) =>
2817 {
2818 Ok(serde_json::from_str(&body)?)
2819 }
2820 response => response,
2821 }
2822}
2823
2824fn worker_poll_body_is_stop(body: &str) -> bool {
2825 serde_json::from_str::<Value>(body)
2826 .ok()
2827 .is_some_and(|body| {
2828 worker_poll_is_stop(
2829 body.get("poll_status").and_then(Value::as_str),
2830 body.get("reason").and_then(Value::as_str),
2831 )
2832 })
2833}
2834
2835fn worker_poll_is_stop(poll_status: Option<&str>, reason: Option<&str>) -> bool {
2836 matches!(poll_status, Some("draining" | "stopped"))
2837 || matches!(reason, Some("worker_draining" | "worker_stopped"))
2838}
2839
2840fn query_task_rejection_is_final(error: &Error) -> bool {
2841 matches!(
2842 error,
2843 Error::QueryFailed(failure)
2844 if QUERY_TASK_FINAL_REJECTION_REASONS.contains(&failure.reason.as_str())
2845 )
2846}
2847
2848fn activity_task_response<T>(
2849 response: Result<T>,
2850 operation: &str,
2851 task_id: &str,
2852 activity_attempt_id: &str,
2853) -> Result<T> {
2854 match response {
2855 Err(Error::Http { status, body }) => {
2856 let body = serde_json::from_str(&body).unwrap_or_else(|_| json!({"message": body}));
2857 Err(Error::ActivityTaskRejected(ActivityTaskRejection {
2858 operation: operation.to_string(),
2859 status: status.as_u16(),
2860 reason: body
2861 .get("reason")
2862 .and_then(Value::as_str)
2863 .unwrap_or("activity_task_rejected")
2864 .to_string(),
2865 task_id: body
2866 .get("task_id")
2867 .and_then(Value::as_str)
2868 .unwrap_or(task_id)
2869 .to_string(),
2870 activity_attempt_id: body
2871 .get("activity_attempt_id")
2872 .and_then(Value::as_str)
2873 .unwrap_or(activity_attempt_id)
2874 .to_string(),
2875 cancel_requested: body
2876 .get("cancel_requested")
2877 .and_then(Value::as_bool)
2878 .unwrap_or(false),
2879 can_continue: body.get("can_continue").and_then(Value::as_bool),
2880 run_closed_reason: body
2881 .get("run_closed_reason")
2882 .and_then(Value::as_str)
2883 .map(str::to_string),
2884 body,
2885 }))
2886 }
2887 response => response,
2888 }
2889}
2890
2891fn activity_task_rejection_is_final(error: &Error) -> bool {
2892 matches!(
2893 error,
2894 Error::ActivityTaskRejected(rejection)
2895 if matches!(
2896 rejection.reason.as_str(),
2897 "run_cancelled"
2898 | "run_terminated"
2899 | "attempt_closed"
2900 | "stale_attempt"
2901 | "activity_cancelled"
2902 | "task_cancelled"
2903 | "run_closed"
2904 | "activity_not_running"
2905 | "attempt_not_found"
2906 )
2907 )
2908}
2909
2910fn workflow_task_completion_is_terminal_timeout(
2911 error: &Error,
2912 task_id: &str,
2913 workflow_task_attempt: u64,
2914 run_id: Option<&str>,
2915) -> bool {
2916 let Error::Http { status, body } = error else {
2917 return false;
2918 };
2919 if *status != reqwest::StatusCode::CONFLICT {
2920 return false;
2921 }
2922
2923 let Some(run_id) = run_id else {
2924 return false;
2925 };
2926 let Ok(body) = serde_json::from_str::<Value>(body) else {
2927 return false;
2928 };
2929
2930 body.get("recorded").and_then(Value::as_bool) == Some(false)
2931 && body.get("reason").and_then(Value::as_str) == Some("run_timed_out")
2932 && body.get("run_status").and_then(Value::as_str) == Some("failed")
2933 && body.get("run_id").and_then(Value::as_str) == Some(run_id)
2934 && body.get("task_id").and_then(Value::as_str) == Some(task_id)
2935 && body.get("workflow_task_attempt").and_then(Value::as_u64) == Some(workflow_task_attempt)
2936}
2937
2938fn protocol_failure(status: reqwest::StatusCode, raw_body: &str) -> Option<ProtocolFailure> {
2939 let body: Value = serde_json::from_str(raw_body).ok()?;
2940 let reason = body.get("reason")?.as_str()?;
2941 if !matches!(
2942 reason,
2943 "missing_protocol_version"
2944 | "unsupported_protocol_version"
2945 | "missing_control_plane_version"
2946 | "unsupported_control_plane_version"
2947 ) {
2948 return None;
2949 }
2950
2951 Some(ProtocolFailure {
2952 status: status.as_u16(),
2953 reason: reason.to_string(),
2954 message: body
2955 .get("message")
2956 .or_else(|| body.get("error"))
2957 .and_then(Value::as_str)
2958 .unwrap_or("protocol version rejected")
2959 .to_string(),
2960 supported_version: body
2961 .get("supported_version")
2962 .and_then(Value::as_str)
2963 .map(str::to_string),
2964 requested_version: body
2965 .get("requested_version")
2966 .and_then(Value::as_str)
2967 .map(str::to_string),
2968 body,
2969 })
2970}
2971
2972fn long_poll_timeout_seconds(timeout: Duration) -> u64 {
2973 timeout
2974 .as_secs()
2975 .saturating_add(u64::from(timeout.subsec_nanos() > 0))
2976 .min(MAX_LONG_POLL_TIMEOUT_SECONDS)
2977}
2978
2979fn worker_operation_is_retryable(error: &Error) -> bool {
2980 match error {
2981 Error::Transport(error) => {
2982 error.is_timeout() || error.is_connect() || error.is_request() || error.is_body()
2983 }
2984 Error::Http { status, .. } => {
2985 matches!(
2986 *status,
2987 reqwest::StatusCode::REQUEST_TIMEOUT | reqwest::StatusCode::TOO_MANY_REQUESTS
2988 ) || status.is_server_error()
2989 }
2990 _ => false,
2991 }
2992}
2993
2994fn worker_retry_delay(policy: WorkerRetryPolicy, retry: usize) -> Duration {
2995 let exponent = retry.saturating_sub(1).min(31) as u32;
2996 policy
2997 .initial_backoff
2998 .saturating_mul(1_u32 << exponent)
2999 .min(policy.max_backoff)
3000}
3001
3002#[derive(Debug)]
3003pub struct ClientBuilder {
3004 base_url: String,
3005 token: Option<String>,
3006 control_token: Option<String>,
3007 worker_token: Option<String>,
3008 namespace: String,
3009 timeout: Duration,
3010}
3011
3012impl ClientBuilder {
3013 pub fn token(mut self, token: Option<String>) -> Self {
3014 self.token = token;
3015 self
3016 }
3017
3018 pub fn control_token(mut self, token: Option<String>) -> Self {
3019 self.control_token = token;
3020 self
3021 }
3022
3023 pub fn worker_token(mut self, token: Option<String>) -> Self {
3024 self.worker_token = token;
3025 self
3026 }
3027
3028 pub fn namespace(mut self, namespace: impl Into<String>) -> Self {
3029 self.namespace = namespace.into();
3030 self
3031 }
3032
3033 pub fn timeout(mut self, timeout: Duration) -> Self {
3034 self.timeout = timeout;
3035 self
3036 }
3037
3038 pub fn build(self) -> Result<Client> {
3039 Ok(Client {
3040 http: reqwest::Client::builder().timeout(self.timeout).build()?,
3041 base_url: self.base_url.trim_end_matches('/').to_string(),
3042 token: self.token,
3043 control_token: self.control_token,
3044 worker_token: self.worker_token,
3045 namespace: self.namespace,
3046 })
3047 }
3048}
3049
3050#[derive(Clone, Debug)]
3051pub struct WorkflowHandle {
3052 client: Client,
3053 pub workflow_id: String,
3054 pub run_id: Option<String>,
3055 pub workflow_type: String,
3056}
3057
3058impl WorkflowHandle {
3059 pub async fn describe(&self) -> Result<WorkflowDescription> {
3061 self.client.describe_workflow(&self.workflow_id).await
3062 }
3063
3064 pub async fn describe_selected_run(&self) -> Result<WorkflowDescription> {
3066 let run_id = self.run_id.as_deref().ok_or_else(|| {
3067 Error::Codec("run_id is required for selected-run description".to_string())
3068 })?;
3069 self.client
3070 .describe_workflow_run(&self.workflow_id, run_id)
3071 .await
3072 }
3073
3074 pub async fn signal<T: Serialize>(&self, signal_name: &str, input: T) -> Result<Value> {
3075 self.client
3076 .signal_workflow(&self.workflow_id, signal_name, input)
3077 .await
3078 }
3079
3080 pub async fn signal_selected_run<T: Serialize>(
3082 &self,
3083 signal_name: &str,
3084 input: T,
3085 ) -> Result<Value> {
3086 let run_id = self.run_id.as_deref().ok_or_else(|| {
3087 Error::Codec("run_id is required for selected-run signaling".to_string())
3088 })?;
3089 self.client
3090 .signal_workflow_run(&self.workflow_id, run_id, signal_name, input)
3091 .await
3092 }
3093
3094 pub async fn cancel(&self, options: WorkflowCommandOptions) -> Result<WorkflowCommandResult> {
3096 self.client
3097 .cancel_workflow(&self.workflow_id, options)
3098 .await
3099 }
3100
3101 pub async fn cancel_selected_run(
3103 &self,
3104 options: WorkflowCommandOptions,
3105 ) -> Result<WorkflowCommandResult> {
3106 let run_id = self.run_id.as_deref().ok_or_else(|| {
3107 Error::Codec("run_id is required for selected-run cancellation".to_string())
3108 })?;
3109 self.client
3110 .cancel_workflow_run(&self.workflow_id, run_id, options)
3111 .await
3112 }
3113
3114 pub async fn terminate(
3116 &self,
3117 options: WorkflowCommandOptions,
3118 ) -> Result<WorkflowCommandResult> {
3119 self.client
3120 .terminate_workflow(&self.workflow_id, options)
3121 .await
3122 }
3123
3124 pub async fn terminate_selected_run(
3126 &self,
3127 options: WorkflowCommandOptions,
3128 ) -> Result<WorkflowCommandResult> {
3129 let run_id = self.run_id.as_deref().ok_or_else(|| {
3130 Error::Codec("run_id is required for selected-run termination".to_string())
3131 })?;
3132 self.client
3133 .terminate_workflow_run(&self.workflow_id, run_id, options)
3134 .await
3135 }
3136
3137 pub async fn query<T: Serialize>(&self, query_name: &str, input: T) -> Result<Value> {
3139 self.client
3140 .query_workflow(&self.workflow_id, query_name, input)
3141 .await
3142 }
3143
3144 pub async fn query_avro_value<T: Serialize>(
3145 &self,
3146 query_name: &str,
3147 input: T,
3148 ) -> Result<AvroValue> {
3149 self.client
3150 .query_workflow_avro_value(&self.workflow_id, query_name, input)
3151 .await
3152 }
3153
3154 pub async fn update<T: Serialize>(
3155 &self,
3156 update_name: &str,
3157 input: T,
3158 request_id: Option<&str>,
3159 ) -> Result<Value> {
3160 self.client
3161 .update_workflow(&self.workflow_id, update_name, input, request_id)
3162 .await
3163 }
3164
3165 pub async fn update_avro_value<T: Serialize>(
3166 &self,
3167 update_name: &str,
3168 input: T,
3169 request_id: Option<&str>,
3170 ) -> Result<AvroValue> {
3171 self.client
3172 .update_workflow_avro_value(&self.workflow_id, update_name, input, request_id)
3173 .await
3174 }
3175
3176 pub async fn query_selected_run<T: Serialize>(
3178 &self,
3179 query_name: &str,
3180 input: T,
3181 ) -> Result<Value> {
3182 let run_id = self
3183 .run_id
3184 .as_deref()
3185 .ok_or_else(|| Error::Codec("run_id is required for selected-run query".to_string()))?;
3186 self.client
3187 .query_workflow_run(&self.workflow_id, run_id, query_name, input)
3188 .await
3189 }
3190
3191 pub async fn result(&self, options: WorkflowResultOptions) -> Result<Value> {
3193 self.result_target(options, None).await
3194 }
3195
3196 pub async fn result_avro_value(&self, options: WorkflowResultOptions) -> Result<AvroValue> {
3198 self.result_avro_value_target(options, None).await
3199 }
3200
3201 pub async fn result_selected_run(&self, options: WorkflowResultOptions) -> Result<Value> {
3203 let run_id = self.run_id.as_deref().ok_or_else(|| {
3204 Error::Codec("run_id is required for selected-run result".to_string())
3205 })?;
3206 self.result_target(options, Some(run_id)).await
3207 }
3208
3209 pub async fn result_selected_run_avro_value(
3211 &self,
3212 options: WorkflowResultOptions,
3213 ) -> Result<AvroValue> {
3214 let run_id = self.run_id.as_deref().ok_or_else(|| {
3215 Error::Codec("run_id is required for selected-run result".to_string())
3216 })?;
3217 self.result_avro_value_target(options, Some(run_id)).await
3218 }
3219
3220 async fn result_avro_value_target(
3221 &self,
3222 options: WorkflowResultOptions,
3223 selected_run_id: Option<&str>,
3224 ) -> Result<AvroValue> {
3225 let started = Instant::now();
3226
3227 loop {
3228 let description = match selected_run_id {
3229 Some(run_id) => {
3230 self.client
3231 .describe_workflow_run(&self.workflow_id, run_id)
3232 .await?
3233 }
3234 None => self.describe().await?,
3235 };
3236 if description.is_completed() {
3237 return description.output_avro_value.ok_or_else(|| {
3238 Error::Codec(
3239 "missing_payload_envelope: typed workflow result requires output_envelope"
3240 .to_string(),
3241 )
3242 });
3243 }
3244 if description.is_terminal() {
3245 let outcome =
3246 workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
3247 return Err(match outcome.kind {
3248 WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
3249 WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
3250 WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
3251 WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
3252 });
3253 }
3254 if started.elapsed() >= options.timeout {
3255 return Err(Error::Timeout);
3256 }
3257 tokio::time::sleep(options.poll_interval).await;
3258 }
3259 }
3260
3261 async fn result_target(
3262 &self,
3263 options: WorkflowResultOptions,
3264 selected_run_id: Option<&str>,
3265 ) -> Result<Value> {
3266 let started = Instant::now();
3267
3268 loop {
3269 let description = match selected_run_id {
3270 Some(run_id) => {
3271 self.client
3272 .describe_workflow_run(&self.workflow_id, run_id)
3273 .await?
3274 }
3275 None => self.describe().await?,
3276 };
3277 if description.is_completed() {
3278 return Ok(description.output.unwrap_or(Value::Null));
3279 }
3280
3281 if description.is_terminal() {
3282 let outcome =
3283 workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
3284 return Err(match outcome.kind {
3285 WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
3286 WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
3287 WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
3288 WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
3289 });
3290 }
3291
3292 if started.elapsed() >= options.timeout {
3293 return Err(Error::WorkflowTimedOut(WorkflowTerminalOutcome {
3294 kind: WorkflowTerminalKind::TimedOut,
3295 workflow_id: description
3296 .workflow_id
3297 .clone()
3298 .unwrap_or_else(|| self.workflow_id.clone()),
3299 run_id: description
3300 .run_id
3301 .clone()
3302 .or_else(|| selected_run_id.map(str::to_string)),
3303 reason: "result_wait_timeout".to_string(),
3304 failure_category: Some("client_timeout".to_string()),
3305 failure_id: None,
3306 exception_type: None,
3307 exception_class: None,
3308 non_retryable: None,
3309 message: Some(format!(
3310 "workflow result was not terminal within {:?}",
3311 options.timeout
3312 )),
3313 exception: None,
3314 raw: description.raw_value(),
3315 }));
3316 }
3317
3318 tokio::time::sleep(options.poll_interval).await;
3319 }
3320 }
3321}
3322
3323#[derive(Clone, Copy, Debug)]
3324pub struct WorkflowResultOptions {
3325 pub poll_interval: Duration,
3326 pub timeout: Duration,
3327}
3328
3329impl Default for WorkflowResultOptions {
3330 fn default() -> Self {
3331 Self {
3332 poll_interval: Duration::from_millis(500),
3333 timeout: Duration::from_secs(30),
3334 }
3335 }
3336}
3337
3338#[derive(Clone, Debug, Deserialize)]
3339pub struct WorkflowDescription {
3340 pub workflow_id: Option<String>,
3341 pub run_id: Option<String>,
3342 pub workflow_type: Option<String>,
3343 pub status: Option<String>,
3344 #[serde(default)]
3345 pub closed_reason: Option<String>,
3346 #[serde(default)]
3347 pub error: Option<String>,
3348 #[serde(default)]
3349 pub failure: Option<Value>,
3350 #[serde(default)]
3351 pub exception: Option<Value>,
3352 #[serde(default)]
3353 pub failures: Vec<Value>,
3354 #[serde(default)]
3355 pub output: Option<Value>,
3356 #[serde(default)]
3357 pub output_envelope: Option<Value>,
3358 #[serde(skip)]
3359 pub output_avro_value: Option<AvroValue>,
3360 #[serde(flatten)]
3361 pub raw: HashMap<String, Value>,
3362}
3363
3364impl WorkflowDescription {
3365 pub fn is_completed(&self) -> bool {
3366 matches!(self.status.as_deref(), Some("completed" | "Completed"))
3367 }
3368
3369 pub fn is_terminal(&self) -> bool {
3370 matches!(
3371 self.status.as_deref(),
3372 Some(
3373 "completed"
3374 | "Completed"
3375 | "failed"
3376 | "Failed"
3377 | "cancelled"
3378 | "Cancelled"
3379 | "terminated"
3380 | "Terminated"
3381 | "timed_out"
3382 | "TimedOut",
3383 )
3384 )
3385 }
3386
3387 fn decode_payloads(&mut self) -> Result<()> {
3388 if let Some(envelope) = &self.output_envelope {
3389 let value = decode_wire_avro_value(envelope, DEFAULT_CODEC)?;
3390 self.output = Some(value.clone().into_json()?);
3391 self.output_avro_value = Some(value);
3392 }
3393
3394 Ok(())
3395 }
3396
3397 fn raw_value(&self) -> Value {
3398 let mut data = self.raw.clone();
3399 data.insert(
3400 "workflow_id".to_string(),
3401 self.workflow_id
3402 .clone()
3403 .map(Value::String)
3404 .unwrap_or(Value::Null),
3405 );
3406 data.insert(
3407 "run_id".to_string(),
3408 self.run_id
3409 .clone()
3410 .map(Value::String)
3411 .unwrap_or(Value::Null),
3412 );
3413 data.insert(
3414 "workflow_type".to_string(),
3415 self.workflow_type
3416 .clone()
3417 .map(Value::String)
3418 .unwrap_or(Value::Null),
3419 );
3420 data.insert(
3421 "status".to_string(),
3422 self.status
3423 .clone()
3424 .map(Value::String)
3425 .unwrap_or(Value::Null),
3426 );
3427 data.insert(
3428 "closed_reason".to_string(),
3429 self.closed_reason
3430 .clone()
3431 .map(Value::String)
3432 .unwrap_or(Value::Null),
3433 );
3434 if let Some(failure) = &self.failure {
3435 data.insert("failure".to_string(), failure.clone());
3436 }
3437 if let Some(exception) = &self.exception {
3438 data.insert("exception".to_string(), exception.clone());
3439 }
3440 Value::Object(data.into_iter().collect())
3441 }
3442}
3443
3444fn workflow_terminal_outcome(
3445 description: &WorkflowDescription,
3446 workflow_id: &str,
3447 run_id: Option<&str>,
3448) -> WorkflowTerminalOutcome {
3449 let terminal_kind = description
3450 .closed_reason
3451 .as_deref()
3452 .or(description.status.as_deref())
3453 .unwrap_or("failed")
3454 .to_ascii_lowercase();
3455 let kind = match terminal_kind.as_str() {
3456 "cancelled" | "canceled" => WorkflowTerminalKind::Cancelled,
3457 "terminated" => WorkflowTerminalKind::Terminated,
3458 "timed_out" | "timedout" => WorkflowTerminalKind::TimedOut,
3459 _ => WorkflowTerminalKind::Failed,
3460 };
3461 let default_reason = match kind {
3462 WorkflowTerminalKind::Failed => "workflow_failed",
3463 WorkflowTerminalKind::Cancelled => "cancelled",
3464 WorkflowTerminalKind::Terminated => "terminated",
3465 WorkflowTerminalKind::TimedOut => "timed_out",
3466 };
3467 let failure = description
3468 .failure
3469 .as_ref()
3470 .filter(|value| value.is_object());
3471 let nested_failure = failure
3472 .and_then(|value| value.get("failures"))
3473 .and_then(Value::as_array)
3474 .and_then(|failures| failures.last())
3475 .or_else(|| description.failures.last());
3476 let exception = description
3477 .exception
3478 .clone()
3479 .or_else(|| failure.and_then(|value| value.get("exception")).cloned())
3480 .or_else(|| {
3481 nested_failure
3482 .and_then(|value| value.get("exception_payload"))
3483 .cloned()
3484 });
3485 let string_field = |name: &str| {
3486 failure
3487 .and_then(|value| value.get(name))
3488 .and_then(Value::as_str)
3489 .or_else(|| {
3490 nested_failure
3491 .and_then(|value| value.get(name))
3492 .and_then(Value::as_str)
3493 })
3494 .map(str::to_string)
3495 };
3496 let exception_field = |name: &str| {
3497 exception
3498 .as_ref()
3499 .and_then(|value| value.get(name))
3500 .and_then(Value::as_str)
3501 .map(str::to_string)
3502 };
3503 let message = description
3504 .error
3505 .clone()
3506 .or_else(|| string_field("message"))
3507 .or_else(|| exception_field("message"));
3508 let reason = description
3509 .raw
3510 .get("reason")
3511 .and_then(Value::as_str)
3512 .map(str::to_string)
3513 .or_else(|| {
3514 failure
3515 .and_then(|value| value.get("reason"))
3516 .and_then(Value::as_str)
3517 .map(str::to_string)
3518 })
3519 .or_else(|| description.closed_reason.clone())
3520 .unwrap_or_else(|| default_reason.to_string());
3521 let failure_id = string_field("failure_id").or_else(|| {
3522 nested_failure
3523 .and_then(|value| value.get("id"))
3524 .and_then(Value::as_str)
3525 .map(str::to_string)
3526 });
3527
3528 WorkflowTerminalOutcome {
3529 kind,
3530 workflow_id: description
3531 .workflow_id
3532 .clone()
3533 .unwrap_or_else(|| workflow_id.to_string()),
3534 run_id: description
3535 .run_id
3536 .clone()
3537 .or_else(|| run_id.map(str::to_string)),
3538 reason,
3539 failure_category: string_field("failure_category")
3540 .or_else(|| Some(default_reason.to_string())),
3541 failure_id,
3542 exception_type: string_field("exception_type").or_else(|| exception_field("type")),
3543 exception_class: string_field("exception_class").or_else(|| exception_field("class")),
3544 non_retryable: failure
3545 .and_then(|value| value.get("non_retryable"))
3546 .and_then(Value::as_bool)
3547 .or_else(|| {
3548 nested_failure
3549 .and_then(|value| value.get("non_retryable"))
3550 .and_then(Value::as_bool)
3551 }),
3552 message,
3553 exception,
3554 raw: description.raw_value(),
3555 }
3556}
3557
3558#[derive(Clone, Debug, Deserialize)]
3559pub struct RegisterWorkerResponse {
3560 pub worker_id: String,
3561 pub registered: bool,
3562 #[serde(default)]
3563 pub heartbeat_interval_seconds: Option<u64>,
3564 #[serde(default)]
3565 pub protocol_version: Option<String>,
3566 #[serde(default)]
3567 pub server_capabilities: Option<Value>,
3568}
3569
3570#[derive(Clone, Debug, Deserialize)]
3571pub struct PollWorkflowTaskResponse {
3572 #[serde(default)]
3573 pub task: Option<WorkflowTask>,
3574 #[serde(default)]
3575 pub poll_status: Option<String>,
3576 #[serde(default)]
3577 pub reason: Option<String>,
3578 #[serde(default)]
3579 pub protocol_version: Option<String>,
3580 #[serde(default)]
3581 pub server_capabilities: Option<Value>,
3582}
3583
3584impl PollWorkflowTaskResponse {
3585 pub fn outcome(&self) -> WorkerPollOutcome {
3587 worker_poll_outcome(
3588 self.task.is_some(),
3589 self.poll_status.as_deref(),
3590 self.reason.as_deref(),
3591 )
3592 }
3593}
3594
3595#[derive(Clone, Debug, Deserialize)]
3596pub struct PollActivityTaskResponse {
3597 #[serde(default)]
3598 pub task: Option<ActivityTask>,
3599 #[serde(default)]
3600 pub poll_status: Option<String>,
3601 #[serde(default)]
3602 pub reason: Option<String>,
3603}
3604
3605impl PollActivityTaskResponse {
3606 pub fn outcome(&self) -> WorkerPollOutcome {
3608 worker_poll_outcome(
3609 self.task.is_some(),
3610 self.poll_status.as_deref(),
3611 self.reason.as_deref(),
3612 )
3613 }
3614}
3615
3616#[derive(Clone, Debug, Deserialize)]
3617pub struct PollQueryTaskResponse {
3618 #[serde(default)]
3619 pub task: Option<QueryTask>,
3620 #[serde(default)]
3621 pub poll_status: Option<String>,
3622 #[serde(default)]
3623 pub reason: Option<String>,
3624}
3625
3626impl PollQueryTaskResponse {
3627 pub fn outcome(&self) -> WorkerPollOutcome {
3629 worker_poll_outcome(
3630 self.task.is_some(),
3631 self.poll_status.as_deref(),
3632 self.reason.as_deref(),
3633 )
3634 }
3635}
3636
3637#[derive(Clone, Debug, PartialEq, Eq)]
3639pub enum WorkerPollOutcome {
3640 Task,
3642 Idle {
3644 poll_status: Option<String>,
3645 reason: Option<String>,
3646 },
3647 Stop {
3649 poll_status: Option<String>,
3650 reason: Option<String>,
3651 },
3652}
3653
3654impl WorkerPollOutcome {
3655 pub fn should_stop(&self) -> bool {
3656 matches!(self, Self::Stop { .. })
3657 }
3658}
3659
3660fn worker_poll_outcome(
3661 has_task: bool,
3662 poll_status: Option<&str>,
3663 reason: Option<&str>,
3664) -> WorkerPollOutcome {
3665 if worker_poll_is_stop(poll_status, reason) {
3666 return WorkerPollOutcome::Stop {
3667 poll_status: poll_status.map(str::to_string),
3668 reason: reason.map(str::to_string),
3669 };
3670 }
3671
3672 if has_task {
3673 WorkerPollOutcome::Task
3674 } else {
3675 WorkerPollOutcome::Idle {
3676 poll_status: poll_status.map(str::to_string),
3677 reason: reason.map(str::to_string),
3678 }
3679 }
3680}
3681
3682#[derive(Clone, Debug, Deserialize)]
3684pub struct QueryTask {
3685 pub query_task_id: String,
3686 #[serde(default = "default_workflow_task_attempt")]
3687 pub query_task_attempt: u64,
3688 #[serde(default)]
3689 pub lease_owner: Option<String>,
3690 #[serde(default)]
3691 pub workflow_id: Option<String>,
3692 #[serde(default)]
3693 pub run_id: Option<String>,
3694 pub workflow_type: String,
3695 pub query_name: String,
3696 #[serde(default = "default_payload_codec")]
3697 pub payload_codec: String,
3698 #[serde(default)]
3699 pub workflow_arguments: Option<Value>,
3700 #[serde(default)]
3701 pub query_arguments: Option<Value>,
3702 #[serde(default)]
3703 pub history_events: Vec<HistoryEvent>,
3704 #[serde(default)]
3705 pub history_export: Option<Value>,
3706 #[serde(default)]
3707 pub run_status: Option<String>,
3708}
3709
3710#[derive(Clone, Debug, Deserialize)]
3711pub struct WorkflowTask {
3712 pub task_id: String,
3713 #[serde(default)]
3714 pub workflow_id: Option<String>,
3715 #[serde(default)]
3716 pub run_id: Option<String>,
3717 pub workflow_type: String,
3718 #[serde(default = "default_payload_codec")]
3719 pub payload_codec: String,
3720 #[serde(default)]
3721 pub arguments: Option<Value>,
3722 #[serde(default)]
3723 pub history_events: Vec<HistoryEvent>,
3724 #[serde(default)]
3725 pub total_history_events: Option<u64>,
3726 #[serde(default)]
3727 pub history_size_bytes: Option<u64>,
3728 #[serde(default)]
3729 pub continue_as_new_recommended: Option<bool>,
3730 #[serde(default)]
3731 pub history_budget_pressure: Option<String>,
3732 #[serde(default)]
3733 pub next_history_page_token: Option<String>,
3734 #[serde(default = "default_workflow_task_attempt")]
3735 pub workflow_task_attempt: u64,
3736 #[serde(default)]
3737 pub workflow_signal_id: Option<String>,
3738 #[serde(default)]
3739 pub signal_name: Option<String>,
3740 #[serde(default)]
3741 pub signal_arguments: Option<Value>,
3742 #[serde(default)]
3743 pub workflow_update_id: Option<String>,
3744 #[serde(default)]
3745 pub update_name: Option<String>,
3746 #[serde(default)]
3747 pub lease_owner: Option<String>,
3748}
3749
3750impl WorkflowTask {
3751 fn append_history_page(&mut self, page: WorkflowTaskHistoryPage) {
3752 self.history_events.extend(page.history_events);
3753
3754 if page.total_history_events.is_some() {
3755 self.total_history_events = page.total_history_events;
3756 }
3757
3758 self.next_history_page_token = page
3759 .next_history_page_token
3760 .filter(|token| !token.is_empty());
3761 }
3762}
3763
3764#[derive(Clone, Debug, Deserialize)]
3765struct WorkflowTaskHistoryPage {
3766 #[serde(default)]
3767 history_events: Vec<HistoryEvent>,
3768 #[serde(default)]
3769 total_history_events: Option<u64>,
3770 #[serde(default)]
3771 next_history_page_token: Option<String>,
3772}
3773
3774#[derive(Clone, Debug, Deserialize)]
3775pub struct ActivityTask {
3776 pub task_id: String,
3777 #[serde(default)]
3778 pub activity_attempt_id: Option<String>,
3779 #[serde(default)]
3780 pub attempt_id: Option<String>,
3781 pub activity_type: String,
3782 #[serde(default = "default_payload_codec")]
3783 pub payload_codec: String,
3784 #[serde(default)]
3785 pub arguments: Option<Value>,
3786 #[serde(default = "default_attempt_number")]
3787 pub attempt_number: u64,
3788 #[serde(default)]
3789 pub lease_owner: Option<String>,
3790}
3791
3792#[derive(Clone, Debug, Deserialize)]
3793pub struct HistoryEvent {
3794 #[serde(alias = "type")]
3795 pub event_type: String,
3796 #[serde(default)]
3797 pub payload: Value,
3798 #[serde(flatten)]
3799 pub raw: HashMap<String, Value>,
3800}
3801
3802#[derive(Clone, Debug, PartialEq)]
3804pub struct QuerySignal {
3805 pub id: Option<String>,
3806 pub name: String,
3807 pub arguments: Vec<Value>,
3808 avro_arguments: Vec<AvroValue>,
3809 pub workflow_sequence: Option<u64>,
3810}
3811
3812impl QuerySignal {
3813 pub fn arguments_avro_value(&self) -> &[AvroValue] {
3815 &self.avro_arguments
3816 }
3817}
3818
3819#[derive(Clone, Debug)]
3825pub struct QueryContext {
3826 pub workflow_id: Option<String>,
3827 pub run_id: Option<String>,
3828 pub workflow_type: String,
3829 pub run_status: Option<String>,
3830 workflow_input: Value,
3831 workflow_input_avro_value: AvroValue,
3832 history_events: Arc<Vec<HistoryEvent>>,
3833 signal_events: Arc<Vec<QuerySignal>>,
3834}
3835
3836impl QueryContext {
3837 pub fn workflow_input(&self) -> &Value {
3839 &self.workflow_input
3840 }
3841
3842 pub fn workflow_input_avro_value(&self) -> &AvroValue {
3844 &self.workflow_input_avro_value
3845 }
3846
3847 pub fn history_events(&self) -> &[HistoryEvent] {
3849 self.history_events.as_slice()
3850 }
3851
3852 pub fn signal_events(&self) -> &[QuerySignal] {
3854 self.signal_events.as_slice()
3855 }
3856
3857 pub fn signals(&self, signal_name: &str) -> Vec<Vec<Value>> {
3859 self.signal_events
3860 .iter()
3861 .filter(|signal| signal.name == signal_name)
3862 .map(|signal| signal.arguments.clone())
3863 .collect()
3864 }
3865
3866 pub fn signals_avro_value(&self, signal_name: &str) -> Vec<Vec<AvroValue>> {
3868 self.signal_events
3869 .iter()
3870 .filter(|signal| signal.name == signal_name)
3871 .map(|signal| signal.avro_arguments.clone())
3872 .collect()
3873 }
3874}
3875
3876#[derive(Clone, Debug, Deserialize)]
3877pub struct ActivityHeartbeatResponse {
3878 #[serde(default)]
3879 pub cancel_requested: bool,
3880 #[serde(default)]
3881 pub heartbeat_recorded: bool,
3882 #[serde(default)]
3883 pub can_continue: Option<bool>,
3884 #[serde(default)]
3885 pub reason: Option<String>,
3886 #[serde(default)]
3887 pub run_closed_reason: Option<String>,
3888 #[serde(default)]
3889 pub run_closed_at: Option<String>,
3890 #[serde(default)]
3891 pub lease_expires_at: Option<String>,
3892 #[serde(default)]
3893 pub last_heartbeat_at: Option<String>,
3894}
3895
3896impl ActivityHeartbeatResponse {
3897 pub fn should_stop(&self) -> bool {
3899 self.cancel_requested || self.can_continue == Some(false)
3900 }
3901}
3902
3903fn default_payload_codec() -> String {
3904 DEFAULT_CODEC.to_string()
3905}
3906
3907fn default_workflow_task_attempt() -> u64 {
3908 1
3909}
3910
3911fn default_attempt_number() -> u64 {
3912 1
3913}
3914
3915type WorkflowFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
3916type WorkflowHandler = Arc<dyn Fn(WorkflowContext, AvroValue) -> WorkflowFuture + Send + Sync>;
3917type ErasedWorkflowState = Arc<dyn Any + Send + Sync>;
3918type WorkflowStateSnapshot = Arc<dyn Fn() -> Result<ErasedWorkflowState> + Send + Sync>;
3919type ReplayedWorkflowHandler =
3920 Arc<dyn Fn(WorkflowContext, AvroValue) -> ReplayedWorkflowInvocation + Send + Sync>;
3921type ActivityFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
3922type ActivityHandler = Arc<dyn Fn(ActivityContext, AvroValue) -> ActivityFuture + Send + Sync>;
3923type QueryFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
3924type QueryHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
3925type UpdateHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
3926type ReplayedQueryHandler = Arc<
3927 dyn Fn(QueryContext, ErasedWorkflowState, AvroValue) -> std::result::Result<QueryFuture, String>
3928 + Send
3929 + Sync,
3930>;
3931type WorkerHeartbeatObserver = Arc<dyn Fn(&WorkerHeartbeatObservation) + Send + Sync>;
3932
3933struct ReplayedWorkflowInvocation {
3934 future: WorkflowFuture,
3935 snapshot: WorkflowStateSnapshot,
3936}
3937
3938#[derive(Clone)]
3939struct RegisteredWorkflow {
3940 execute: WorkflowHandler,
3941 replay: Option<ReplayedWorkflowHandler>,
3942 state_type: Option<TypeId>,
3943}
3944
3945#[derive(Clone)]
3946enum RegisteredQuery {
3947 Snapshot(QueryHandler),
3948 Replayed {
3949 state_type: TypeId,
3950 handler: ReplayedQueryHandler,
3951 },
3952}
3953
3954#[derive(Clone, Debug)]
3955pub struct WorkerHeartbeatObservation {
3956 pub worker_id: String,
3957 pub task_queue: String,
3958 pub acknowledged_at_unix_millis: u64,
3959 pub acknowledgement: Value,
3960}
3961
3962#[derive(Clone, Copy, Debug)]
3969pub struct WorkerRetryPolicy {
3970 pub max_retries: usize,
3972 pub initial_backoff: Duration,
3974 pub max_backoff: Duration,
3976}
3977
3978impl Default for WorkerRetryPolicy {
3979 fn default() -> Self {
3980 Self {
3981 max_retries: 5,
3982 initial_backoff: Duration::from_millis(100),
3983 max_backoff: Duration::from_secs(5),
3984 }
3985 }
3986}
3987
3988#[derive(Clone, Copy, Debug, PartialEq, Eq)]
3989enum ManagedPollOutcome {
3990 Idle,
3991 Handled,
3992 Stop,
3993}
3994
3995#[derive(Clone)]
3996pub struct Worker {
3997 client: Client,
3998 worker_id: String,
3999 task_queue: String,
4000 workflows: HashMap<String, RegisteredWorkflow>,
4001 activities: HashMap<String, ActivityHandler>,
4002 queries: HashMap<String, HashMap<String, RegisteredQuery>>,
4003 updates: HashMap<String, HashMap<String, UpdateHandler>>,
4004 max_concurrent_workflow_tasks: usize,
4005 max_concurrent_activity_tasks: usize,
4006 poll_timeout: Duration,
4007 heartbeat_interval: Duration,
4008 retry_policy: WorkerRetryPolicy,
4009 heartbeat_observer: Option<WorkerHeartbeatObserver>,
4010}
4011
4012impl Worker {
4013 pub fn new(client: Client, task_queue: impl Into<String>) -> Self {
4014 Self {
4015 client,
4016 worker_id: default_worker_id(),
4017 task_queue: task_queue.into(),
4018 workflows: HashMap::new(),
4019 activities: HashMap::new(),
4020 queries: HashMap::new(),
4021 updates: HashMap::new(),
4022 max_concurrent_workflow_tasks: 10,
4023 max_concurrent_activity_tasks: 10,
4024 poll_timeout: Duration::from_secs(30),
4025 heartbeat_interval: Duration::from_secs(60),
4026 retry_policy: WorkerRetryPolicy::default(),
4027 heartbeat_observer: None,
4028 }
4029 }
4030
4031 pub fn worker_id(mut self, worker_id: impl Into<String>) -> Self {
4032 self.worker_id = worker_id.into();
4033 self
4034 }
4035
4036 pub fn poll_timeout(mut self, timeout: Duration) -> Self {
4037 self.poll_timeout = timeout;
4038 self
4039 }
4040
4041 pub fn heartbeat_interval(mut self, interval: Duration) -> Self {
4042 self.heartbeat_interval = interval;
4043 self
4044 }
4045
4046 pub fn retry_policy(mut self, policy: WorkerRetryPolicy) -> Self {
4048 self.retry_policy = policy;
4049 self
4050 }
4051
4052 pub fn on_worker_heartbeat<F>(mut self, observer: F) -> Self
4053 where
4054 F: Fn(&WorkerHeartbeatObservation) + Send + Sync + 'static,
4055 {
4056 self.heartbeat_observer = Some(Arc::new(observer));
4057 self
4058 }
4059
4060 pub fn max_concurrent_workflow_tasks(mut self, count: usize) -> Self {
4061 self.max_concurrent_workflow_tasks = count.max(1);
4062 self
4063 }
4064
4065 pub fn max_concurrent_activity_tasks(mut self, count: usize) -> Self {
4066 self.max_concurrent_activity_tasks = count.max(1);
4067 self
4068 }
4069
4070 pub fn register_workflow<F, Fut>(&mut self, workflow_type: impl Into<String>, handler: F)
4077 where
4078 F: Fn(WorkflowContext, Value) -> Fut + Send + Sync + 'static,
4079 Fut: Future<Output = Result<Value>> + Send + 'static,
4080 {
4081 let handler = Arc::new(handler);
4082 self.workflows.insert(
4083 workflow_type.into(),
4084 RegisteredWorkflow {
4085 execute: Arc::new(move |ctx, input| {
4086 let handler = Arc::clone(&handler);
4087 Box::pin(async move {
4088 let result = handler(ctx, input.into_json()?).await?;
4089 AvroValue::from_serialize(&result)
4090 })
4091 }),
4092 replay: None,
4093 state_type: None,
4094 },
4095 );
4096 }
4097
4098 pub fn register_workflow_avro_value<F, Fut>(
4100 &mut self,
4101 workflow_type: impl Into<String>,
4102 handler: F,
4103 ) where
4104 F: Fn(WorkflowContext, AvroValue) -> Fut + Send + Sync + 'static,
4105 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4106 {
4107 self.workflows.insert(
4108 workflow_type.into(),
4109 RegisteredWorkflow {
4110 execute: Arc::new(move |ctx, input| Box::pin(handler(ctx, input))),
4111 replay: None,
4112 state_type: None,
4113 },
4114 );
4115 }
4116
4117 pub fn register_replayed_workflow<S, Factory, F, Fut>(
4125 &mut self,
4126 workflow_type: impl Into<String>,
4127 state_factory: Factory,
4128 handler: F,
4129 ) where
4130 S: Clone + Send + Sync + 'static,
4131 Factory: Fn() -> S + Send + Sync + 'static,
4132 F: Fn(WorkflowContext, Value, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
4133 Fut: Future<Output = Result<Value>> + Send + 'static,
4134 {
4135 let state_factory = Arc::new(state_factory);
4136 let handler = Arc::new(handler);
4137
4138 let execute_factory = Arc::clone(&state_factory);
4139 let execute_handler = Arc::clone(&handler);
4140 let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4141 let state = WorkflowInstance::new(execute_factory());
4142 let handler = Arc::clone(&execute_handler);
4143 Box::pin(async move {
4144 let result = handler(ctx, input.into_json()?, state).await?;
4145 AvroValue::from_serialize(&result)
4146 }) as WorkflowFuture
4147 });
4148
4149 let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4150 let state = WorkflowInstance::new(state_factory());
4151 let snapshot_state = state.clone();
4152 let snapshot: WorkflowStateSnapshot =
4153 Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
4154 let replay_handler = Arc::clone(&handler);
4155 let future = async move {
4156 let result = replay_handler(ctx, input.into_json()?, state).await?;
4157 AvroValue::from_serialize(&result)
4158 };
4159 ReplayedWorkflowInvocation {
4160 future: Box::pin(future),
4161 snapshot,
4162 }
4163 });
4164
4165 self.workflows.insert(
4166 workflow_type.into(),
4167 RegisteredWorkflow {
4168 execute,
4169 replay: Some(replay),
4170 state_type: Some(TypeId::of::<S>()),
4171 },
4172 );
4173 }
4174
4175 pub fn register_replayed_workflow_avro_value<S, Factory, F, Fut>(
4177 &mut self,
4178 workflow_type: impl Into<String>,
4179 state_factory: Factory,
4180 handler: F,
4181 ) where
4182 S: Clone + Send + Sync + 'static,
4183 Factory: Fn() -> S + Send + Sync + 'static,
4184 F: Fn(WorkflowContext, AvroValue, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
4185 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4186 {
4187 let state_factory = Arc::new(state_factory);
4188 let handler = Arc::new(handler);
4189
4190 let execute_factory = Arc::clone(&state_factory);
4191 let execute_handler = Arc::clone(&handler);
4192 let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4193 let state = WorkflowInstance::new(execute_factory());
4194 Box::pin(execute_handler(ctx, input, state)) as WorkflowFuture
4195 });
4196
4197 let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4198 let state = WorkflowInstance::new(state_factory());
4199 let snapshot_state = state.clone();
4200 let snapshot: WorkflowStateSnapshot =
4201 Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
4202 ReplayedWorkflowInvocation {
4203 future: Box::pin(handler(ctx, input, state)),
4204 snapshot,
4205 }
4206 });
4207
4208 self.workflows.insert(
4209 workflow_type.into(),
4210 RegisteredWorkflow {
4211 execute,
4212 replay: Some(replay),
4213 state_type: Some(TypeId::of::<S>()),
4214 },
4215 );
4216 }
4217
4218 pub fn register_activity<F, Fut>(&mut self, activity_type: impl Into<String>, handler: F)
4219 where
4220 F: Fn(ActivityContext, Value) -> Fut + Send + Sync + 'static,
4221 Fut: Future<Output = Result<Value>> + Send + 'static,
4222 {
4223 let handler = Arc::new(handler);
4224 self.activities.insert(
4225 activity_type.into(),
4226 Arc::new(move |ctx, args| {
4227 let handler = Arc::clone(&handler);
4228 Box::pin(async move {
4229 let result = handler(ctx, args.into_json()?).await?;
4230 AvroValue::from_serialize(&result)
4231 })
4232 }),
4233 );
4234 }
4235
4236 pub fn register_activity_avro_value<F, Fut>(
4238 &mut self,
4239 activity_type: impl Into<String>,
4240 handler: F,
4241 ) where
4242 F: Fn(ActivityContext, AvroValue) -> Fut + Send + Sync + 'static,
4243 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4244 {
4245 self.activities.insert(
4246 activity_type.into(),
4247 Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
4248 );
4249 }
4250
4251 pub fn register_query<F, Fut>(
4257 &mut self,
4258 workflow_type: impl Into<String>,
4259 query_name: impl Into<String>,
4260 handler: F,
4261 ) where
4262 F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
4263 Fut: Future<Output = Result<Value>> + Send + 'static,
4264 {
4265 let handler = Arc::new(handler);
4266 self.queries
4267 .entry(workflow_type.into())
4268 .or_default()
4269 .insert(
4270 query_name.into(),
4271 RegisteredQuery::Snapshot(Arc::new(move |ctx, args| {
4272 let handler = Arc::clone(&handler);
4273 Box::pin(async move {
4274 let result = handler(ctx, args.into_json()?).await?;
4275 AvroValue::from_serialize(&result)
4276 })
4277 })),
4278 );
4279 }
4280
4281 pub fn register_query_avro_value<F, Fut>(
4283 &mut self,
4284 workflow_type: impl Into<String>,
4285 query_name: impl Into<String>,
4286 handler: F,
4287 ) where
4288 F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
4289 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4290 {
4291 self.queries
4292 .entry(workflow_type.into())
4293 .or_default()
4294 .insert(
4295 query_name.into(),
4296 RegisteredQuery::Snapshot(Arc::new(move |ctx, args| Box::pin(handler(ctx, args)))),
4297 );
4298 }
4299
4300 pub fn register_replayed_query<S, F, Fut>(
4307 &mut self,
4308 workflow_type: impl Into<String>,
4309 query_name: impl Into<String>,
4310 handler: F,
4311 ) where
4312 S: Clone + Send + Sync + 'static,
4313 F: Fn(QueryContext, Arc<S>, Value) -> Fut + Send + Sync + 'static,
4314 Fut: Future<Output = Result<Value>> + Send + 'static,
4315 {
4316 let handler = Arc::new(handler);
4317 let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
4318 let state = state.downcast::<S>().map_err(|_| {
4319 "registered query state type does not match the replayed workflow state".to_string()
4320 })?;
4321 let handler = Arc::clone(&handler);
4322 Ok(Box::pin(async move {
4323 let result = handler(ctx, state, args.into_json()?).await?;
4324 AvroValue::from_serialize(&result)
4325 }))
4326 });
4327
4328 self.queries
4329 .entry(workflow_type.into())
4330 .or_default()
4331 .insert(
4332 query_name.into(),
4333 RegisteredQuery::Replayed {
4334 state_type: TypeId::of::<S>(),
4335 handler: erased_handler,
4336 },
4337 );
4338 }
4339
4340 pub fn register_replayed_query_avro_value<S, F, Fut>(
4342 &mut self,
4343 workflow_type: impl Into<String>,
4344 query_name: impl Into<String>,
4345 handler: F,
4346 ) where
4347 S: Clone + Send + Sync + 'static,
4348 F: Fn(QueryContext, Arc<S>, AvroValue) -> Fut + Send + Sync + 'static,
4349 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4350 {
4351 let handler = Arc::new(handler);
4352 let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
4353 let state = state.downcast::<S>().map_err(|_| {
4354 "registered query state type does not match the replayed workflow state".to_string()
4355 })?;
4356 Ok(Box::pin(handler(ctx, state, args)))
4357 });
4358
4359 self.queries
4360 .entry(workflow_type.into())
4361 .or_default()
4362 .insert(
4363 query_name.into(),
4364 RegisteredQuery::Replayed {
4365 state_type: TypeId::of::<S>(),
4366 handler: erased_handler,
4367 },
4368 );
4369 }
4370
4371 pub fn register_update<F, Fut>(
4373 &mut self,
4374 workflow_type: impl Into<String>,
4375 update_name: impl Into<String>,
4376 handler: F,
4377 ) where
4378 F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
4379 Fut: Future<Output = Result<Value>> + Send + 'static,
4380 {
4381 let handler = Arc::new(handler);
4382 self.updates
4383 .entry(workflow_type.into())
4384 .or_default()
4385 .insert(
4386 update_name.into(),
4387 Arc::new(move |ctx, args| {
4388 let handler = Arc::clone(&handler);
4389 Box::pin(async move {
4390 let result = handler(ctx, args.into_json()?).await?;
4391 AvroValue::from_serialize(&result)
4392 })
4393 }),
4394 );
4395 }
4396
4397 pub fn register_update_avro_value<F, Fut>(
4399 &mut self,
4400 workflow_type: impl Into<String>,
4401 update_name: impl Into<String>,
4402 handler: F,
4403 ) where
4404 F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
4405 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4406 {
4407 self.updates
4408 .entry(workflow_type.into())
4409 .or_default()
4410 .insert(
4411 update_name.into(),
4412 Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
4413 );
4414 }
4415
4416 pub async fn register(&self) -> Result<RegisterWorkerResponse> {
4417 let mut command_contracts = serde_json::Map::new();
4418 for workflow_type in self.workflows.keys() {
4419 let mut queries = self
4420 .queries
4421 .get(workflow_type)
4422 .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
4423 .unwrap_or_default();
4424 queries.sort();
4425 let mut updates = self
4426 .updates
4427 .get(workflow_type)
4428 .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
4429 .unwrap_or_default();
4430 updates.sort();
4431 if !queries.is_empty() || !updates.is_empty() {
4432 command_contracts.insert(
4433 workflow_type.clone(),
4434 json!({
4435 "queries": queries,
4436 "updates": updates,
4437 }),
4438 );
4439 }
4440 }
4441
4442 self.client
4443 .register_worker_with_command_contracts(
4444 &self.worker_id,
4445 &self.task_queue,
4446 self.workflows.keys().cloned().collect(),
4447 self.activities.keys().cloned().collect(),
4448 self.max_concurrent_workflow_tasks,
4449 self.max_concurrent_activity_tasks,
4450 [
4451 (!self.queries.is_empty()).then(|| QUERY_TASKS_CAPABILITY.to_string()),
4452 (!self.updates.is_empty()).then(|| WORKFLOW_UPDATES_CAPABILITY.to_string()),
4453 ]
4454 .into_iter()
4455 .flatten()
4456 .collect(),
4457 Value::Object(command_contracts),
4458 )
4459 .await
4460 }
4461
4462 pub async fn run(&self) -> Result<()> {
4468 self.run_until(std::future::pending::<()>()).await
4469 }
4470
4471 pub async fn run_until<F>(&self, shutdown: F) -> Result<()>
4475 where
4476 F: Future<Output = ()>,
4477 {
4478 let registration = self.register().await?;
4479 let heartbeat_interval = Duration::from_secs(
4480 registration
4481 .heartbeat_interval_seconds
4482 .unwrap_or(self.heartbeat_interval.as_secs().max(1)),
4483 );
4484 let heartbeat = tokio::time::sleep(Duration::ZERO);
4490 tokio::pin!(heartbeat);
4491 tokio::pin!(shutdown);
4492 let stop = Arc::new(AtomicBool::new(false));
4493 let mut workflow_poller = (!self.workflows.is_empty()).then(|| {
4497 let worker = self.clone();
4498 let stop = Arc::clone(&stop);
4499 tokio::spawn(async move { worker.poll_workflows_until_stopped(stop).await })
4500 });
4501 let mut activity_poller = (!self.activities.is_empty()).then(|| {
4502 let worker = self.clone();
4503 let stop = Arc::clone(&stop);
4504 tokio::spawn(async move { worker.poll_activities_until_stopped(stop).await })
4505 });
4506 let mut query_poller = (!self.queries.is_empty()).then(|| {
4507 let worker = self.clone();
4508 let stop = Arc::clone(&stop);
4509 tokio::spawn(async move { worker.poll_queries_until_stopped(stop).await })
4510 });
4511
4512 loop {
4513 tokio::select! {
4514 _ = &mut shutdown => {
4515 stop.store(true, Ordering::SeqCst);
4516 break;
4517 }
4518 _ = &mut heartbeat => {
4519 let result = self.retry_worker_operation(|| {
4520 self.client.heartbeat_worker(
4521 &self.worker_id,
4522 self.max_concurrent_workflow_tasks,
4523 self.max_concurrent_activity_tasks,
4524 )
4525 }).await;
4526 heartbeat
4527 .as_mut()
4528 .reset(tokio::time::Instant::now() + heartbeat_interval);
4529 match result {
4530 Ok(acknowledgement) => {
4531 if let Some(observer) = &self.heartbeat_observer {
4532 observer(&WorkerHeartbeatObservation {
4533 worker_id: self.worker_id.clone(),
4534 task_queue: self.task_queue.clone(),
4535 acknowledged_at_unix_millis: SystemTime::now()
4536 .duration_since(UNIX_EPOCH)
4537 .unwrap_or_default()
4538 .as_millis()
4539 .min(u64::MAX as u128)
4540 as u64,
4541 acknowledgement,
4542 });
4543 }
4544 }
4545 Err(error) => {
4546 stop.store(true, Ordering::SeqCst);
4547 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await?;
4548 return Err(error);
4549 }
4550 }
4551 }
4552 result = OptionFuture::from(workflow_poller.as_mut()), if workflow_poller.is_some() => {
4553 workflow_poller = None;
4554 let stopped_by_server = stop.load(Ordering::SeqCst);
4555 stop.store(true, Ordering::SeqCst);
4556 let poller_result = optional_poller_result("workflow", result);
4557 let join_result =
4558 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
4559 poller_result?;
4560 join_result?;
4561 if stopped_by_server {
4562 return Ok(());
4563 }
4564 return Err(Error::WorkerLoop(
4565 "workflow poller stopped unexpectedly".to_string(),
4566 ));
4567 }
4568 result = OptionFuture::from(activity_poller.as_mut()), if activity_poller.is_some() => {
4569 activity_poller = None;
4570 let stopped_by_server = stop.load(Ordering::SeqCst);
4571 stop.store(true, Ordering::SeqCst);
4572 let poller_result = optional_poller_result("activity", result);
4573 let join_result =
4574 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
4575 poller_result?;
4576 join_result?;
4577 if stopped_by_server {
4578 return Ok(());
4579 }
4580 return Err(Error::WorkerLoop(
4581 "activity poller stopped unexpectedly".to_string(),
4582 ));
4583 }
4584 result = OptionFuture::from(query_poller.as_mut()), if query_poller.is_some() => {
4585 query_poller = None;
4586 let stopped_by_server = stop.load(Ordering::SeqCst);
4587 stop.store(true, Ordering::SeqCst);
4588 let poller_result = optional_poller_result("query", result);
4589 let join_result =
4590 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
4591 poller_result?;
4592 join_result?;
4593 if stopped_by_server {
4594 return Ok(());
4595 }
4596 return Err(Error::WorkerLoop(
4597 "query poller stopped unexpectedly".to_string(),
4598 ));
4599 }
4600 }
4601 }
4602
4603 join_pollers(
4604 workflow_poller.take(),
4605 activity_poller.take(),
4606 query_poller.take(),
4607 )
4608 .await
4609 }
4610
4611 pub async fn run_once(&self) -> Result<usize> {
4625 let mut handled = 0;
4626 match self.poll_workflow_once().await? {
4627 ManagedPollOutcome::Handled => handled += 1,
4628 ManagedPollOutcome::Stop => return Ok(handled),
4629 ManagedPollOutcome::Idle => {}
4630 }
4631 match self.poll_activity_once().await? {
4632 ManagedPollOutcome::Handled => handled += 1,
4633 ManagedPollOutcome::Stop => return Ok(handled),
4634 ManagedPollOutcome::Idle => {}
4635 }
4636 if !self.queries.is_empty() {
4637 match self.poll_query_once().await? {
4638 ManagedPollOutcome::Handled => handled += 1,
4639 ManagedPollOutcome::Stop => return Ok(handled),
4640 ManagedPollOutcome::Idle => {}
4641 }
4642 }
4643 Ok(handled)
4644 }
4645
4646 async fn poll_workflow_once(&self) -> Result<ManagedPollOutcome> {
4647 let poll_request_id = unique_request_id("rust-workflow-poll");
4648 let response = self
4649 .retry_worker_operation(|| {
4650 self.client.poll_workflow_task_response_with_request_id(
4651 &self.worker_id,
4652 &self.task_queue,
4653 self.poll_timeout,
4654 &poll_request_id,
4655 0,
4656 )
4657 })
4658 .await?;
4659 if response.outcome().should_stop() {
4660 return Ok(ManagedPollOutcome::Stop);
4661 }
4662 let Some(task) = response.task else {
4663 return Ok(ManagedPollOutcome::Idle);
4664 };
4665
4666 let task_id = task.task_id.clone();
4667 let attempt = task.workflow_task_attempt;
4668 let run_id = task.run_id.clone();
4669 let lease_owner = task
4670 .lease_owner
4671 .clone()
4672 .unwrap_or_else(|| self.worker_id.clone());
4673
4674 match self.execute_workflow_task(task) {
4675 Ok(commands) if commands.is_empty() => {
4676 self.client
4682 .fail_workflow_task_with_type(
4683 &task_id,
4684 &lease_owner,
4685 attempt,
4686 WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE,
4687 WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE,
4688 )
4689 .await?;
4690 }
4691 Ok(commands) => {
4692 let completion = self
4693 .client
4694 .complete_workflow_task(&task_id, &lease_owner, attempt, commands)
4695 .await;
4696 if let Err(error) = completion {
4697 if !workflow_task_completion_is_terminal_timeout(
4698 &error,
4699 &task_id,
4700 attempt,
4701 run_id.as_deref(),
4702 ) {
4703 return Err(error);
4704 }
4705 }
4706 }
4707 Err(error) => {
4708 self.client
4709 .fail_workflow_task(&task_id, &lease_owner, attempt, error.to_string())
4710 .await?;
4711 }
4712 }
4713
4714 Ok(ManagedPollOutcome::Handled)
4715 }
4716
4717 async fn poll_workflows_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
4718 while !stop.load(Ordering::SeqCst) {
4719 if self.poll_workflow_once().await? == ManagedPollOutcome::Stop {
4720 stop.store(true, Ordering::SeqCst);
4721 break;
4722 }
4723 }
4724
4725 Ok(())
4726 }
4727
4728 async fn poll_activity_once(&self) -> Result<ManagedPollOutcome> {
4729 let poll_request_id = unique_request_id("rust-activity-poll");
4730 let response = self
4731 .retry_worker_operation(|| {
4732 self.client.poll_activity_task_response_with_request_id(
4733 &self.worker_id,
4734 &self.task_queue,
4735 self.poll_timeout,
4736 &poll_request_id,
4737 0,
4738 )
4739 })
4740 .await?;
4741 if response.outcome().should_stop() {
4742 return Ok(ManagedPollOutcome::Stop);
4743 }
4744 let Some(task) = response.task else {
4745 return Ok(ManagedPollOutcome::Idle);
4746 };
4747
4748 let task_id = task.task_id.clone();
4749 let attempt_id = task
4750 .activity_attempt_id
4751 .clone()
4752 .or(task.attempt_id.clone())
4753 .unwrap_or_default();
4754 let lease_owner = task
4755 .lease_owner
4756 .clone()
4757 .unwrap_or_else(|| self.worker_id.clone());
4758 let codec = task.payload_codec.clone();
4759 let result = self.execute_activity_task(task).await;
4760 match result {
4761 Ok(value) => {
4762 let completion = self
4763 .client
4764 .complete_activity_task(&task_id, &attempt_id, &lease_owner, value, &codec)
4765 .await;
4766 if let Err(error) = completion {
4767 if !activity_task_rejection_is_final(&error) {
4768 return Err(error);
4769 }
4770 }
4771 }
4772 Err(error) => {
4773 let failure = self
4774 .client
4775 .fail_activity_task(
4776 &task_id,
4777 &attempt_id,
4778 &lease_owner,
4779 error.to_string(),
4780 false,
4781 )
4782 .await;
4783 if let Err(error) = failure {
4784 if !activity_task_rejection_is_final(&error) {
4785 return Err(error);
4786 }
4787 }
4788 }
4789 }
4790
4791 Ok(ManagedPollOutcome::Handled)
4792 }
4793
4794 async fn poll_activities_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
4795 while !stop.load(Ordering::SeqCst) {
4796 if self.poll_activity_once().await? == ManagedPollOutcome::Stop {
4797 stop.store(true, Ordering::SeqCst);
4798 break;
4799 }
4800 }
4801
4802 Ok(())
4803 }
4804
4805 async fn poll_query_once(&self) -> Result<ManagedPollOutcome> {
4806 let poll_request_id = unique_request_id("rust-query-poll");
4807 let response = self
4808 .retry_worker_operation(|| {
4809 self.client.poll_query_task_response_with_request_id(
4810 &self.worker_id,
4811 &self.task_queue,
4812 self.poll_timeout,
4813 &poll_request_id,
4814 0,
4815 )
4816 })
4817 .await?;
4818 if response.outcome().should_stop() {
4819 return Ok(ManagedPollOutcome::Stop);
4820 }
4821 let Some(task) = response.task else {
4822 return Ok(ManagedPollOutcome::Idle);
4823 };
4824
4825 let query_task_id = task.query_task_id.clone();
4826 let attempt = task.query_task_attempt;
4827 let lease_owner = task
4828 .lease_owner
4829 .clone()
4830 .unwrap_or_else(|| self.worker_id.clone());
4831 let codec = task.payload_codec.clone();
4832
4833 match self.execute_query_task(task).await {
4834 Ok(value) => {
4835 let result_envelope = match encode_typed_envelope(&value, &codec) {
4836 Ok(result_envelope) => result_envelope,
4837 Err(error) => {
4838 let failure = self
4839 .client
4840 .fail_query_task(
4841 &query_task_id,
4842 &lease_owner,
4843 attempt,
4844 error.to_string(),
4845 "query_result_encode_failed",
4846 "QueryResultEncodeFailed",
4847 )
4848 .await;
4849 if let Err(error) = failure {
4850 if !query_task_rejection_is_final(&error) {
4851 return Err(error);
4852 }
4853 }
4854 return Ok(ManagedPollOutcome::Handled);
4855 }
4856 };
4857
4858 if let Err(error) = self
4859 .client
4860 .complete_query_task_with_envelope(
4861 &query_task_id,
4862 &lease_owner,
4863 attempt,
4864 value.clone().into_json()?,
4865 result_envelope,
4866 )
4867 .await
4868 {
4869 if !query_task_rejection_is_final(&error) {
4870 return Err(error);
4871 }
4872 }
4873 }
4874 Err(failure) => {
4875 let result = self
4876 .client
4877 .fail_query_task(
4878 &query_task_id,
4879 &lease_owner,
4880 attempt,
4881 failure.message,
4882 failure.reason,
4883 failure.failure_type,
4884 )
4885 .await;
4886 if let Err(error) = result {
4887 if !query_task_rejection_is_final(&error) {
4888 return Err(error);
4889 }
4890 }
4891 }
4892 }
4893
4894 Ok(ManagedPollOutcome::Handled)
4895 }
4896
4897 async fn poll_queries_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
4898 while !stop.load(Ordering::SeqCst) {
4899 if self.poll_query_once().await? == ManagedPollOutcome::Stop {
4900 stop.store(true, Ordering::SeqCst);
4901 break;
4902 }
4903 }
4904
4905 Ok(())
4906 }
4907
4908 async fn retry_worker_operation<T, F, Fut>(&self, mut operation: F) -> Result<T>
4909 where
4910 F: FnMut() -> Fut,
4911 Fut: Future<Output = Result<T>>,
4912 {
4913 let mut retries = 0;
4914
4915 loop {
4916 match operation().await {
4917 Err(error)
4918 if worker_operation_is_retryable(&error)
4919 && retries < self.retry_policy.max_retries =>
4920 {
4921 retries += 1;
4922 tokio::time::sleep(worker_retry_delay(self.retry_policy, retries)).await;
4923 }
4924 result => return result,
4925 }
4926 }
4927 }
4928
4929 async fn execute_query_task(
4930 &self,
4931 mut task: QueryTask,
4932 ) -> std::result::Result<AvroValue, QueryTaskExecutionFailure> {
4933 if !matches!(task.payload_codec.as_str(), DEFAULT_CODEC | JSON_CODEC) {
4934 return Err(QueryTaskExecutionFailure::new(
4935 "query_payload_decode_failed",
4936 format!(
4937 "cannot decode query payload with unsupported codec {:?}",
4938 task.payload_codec
4939 ),
4940 "QueryPayloadDecodeFailed",
4941 ));
4942 }
4943
4944 if !self.workflows.contains_key(&task.workflow_type) {
4945 return Err(QueryTaskExecutionFailure::new(
4946 "query_workflow_type_not_registered",
4947 format!("no workflow registered for type {:?}", task.workflow_type),
4948 "WorkflowTypeNotRegistered",
4949 ));
4950 }
4951
4952 let Some(handlers) = self.queries.get(&task.workflow_type) else {
4953 return Err(QueryTaskExecutionFailure::new(
4954 "query_handler_unavailable",
4955 format!(
4956 "query handlers are unavailable for workflow type {:?}",
4957 task.workflow_type
4958 ),
4959 "QueryHandlerUnavailable",
4960 ));
4961 };
4962 let Some(query) = handlers.get(&task.query_name) else {
4963 return Err(QueryTaskExecutionFailure::new(
4964 "rejected_unknown_query",
4965 format!("unknown query {:?}", task.query_name),
4966 "QueryFailed",
4967 ));
4968 };
4969
4970 let args = decode_task_avro_arguments(task.query_arguments.as_ref(), &task.payload_codec)
4971 .map_err(|error| {
4972 QueryTaskExecutionFailure::new(
4973 "query_payload_decode_failed",
4974 format!("cannot decode query arguments: {error}"),
4975 "QueryPayloadDecodeFailed",
4976 )
4977 })?;
4978 let workflow_input_typed =
4979 decode_task_avro_arguments(task.workflow_arguments.as_ref(), &task.payload_codec)
4980 .map_err(|error| {
4981 QueryTaskExecutionFailure::new(
4982 "query_workflow_state_unavailable",
4983 format!("cannot decode workflow start input: {error}"),
4984 "QueryWorkflowStateUnavailable",
4985 )
4986 })?;
4987 let workflow_input = workflow_input_typed.clone().into_json().map_err(|error| {
4988 QueryTaskExecutionFailure::new(
4989 "query_workflow_state_unavailable",
4990 format!("cannot project workflow start input: {error}"),
4991 "QueryWorkflowStateUnavailable",
4992 )
4993 })?;
4994 hydrate_query_history_from_export(&mut task).map_err(|error| {
4995 QueryTaskExecutionFailure::new(
4996 "query_workflow_state_unavailable",
4997 format!("cannot restore query history snapshot: {error}"),
4998 "QueryWorkflowStateUnavailable",
4999 )
5000 })?;
5001 enrich_query_history_from_export(&mut task).map_err(|error| {
5002 QueryTaskExecutionFailure::new(
5003 "query_workflow_state_unavailable",
5004 format!("cannot restore compact query history payloads: {error}"),
5005 "QueryWorkflowStateUnavailable",
5006 )
5007 })?;
5008 let signal_events = query_signal_events(&task).map_err(|error| {
5009 QueryTaskExecutionFailure::new(
5010 "query_workflow_state_unavailable",
5011 format!("cannot decode committed workflow signals: {error}"),
5012 "QueryWorkflowStateUnavailable",
5013 )
5014 })?;
5015 let history_events = Arc::new(std::mem::take(&mut task.history_events));
5016 let context = QueryContext {
5017 workflow_id: task.workflow_id,
5018 run_id: task.run_id,
5019 workflow_type: task.workflow_type.clone(),
5020 run_status: task.run_status,
5021 workflow_input,
5022 workflow_input_avro_value: workflow_input_typed.clone(),
5023 history_events: Arc::clone(&history_events),
5024 signal_events: Arc::new(signal_events),
5025 };
5026
5027 let future = match query {
5028 RegisteredQuery::Snapshot(handler) => handler(context, args),
5029 RegisteredQuery::Replayed {
5030 state_type,
5031 handler,
5032 } => {
5033 let workflow = self
5034 .workflows
5035 .get(&task.workflow_type)
5036 .expect("workflow registration was checked above");
5037 if workflow.state_type != Some(*state_type) {
5038 return Err(QueryTaskExecutionFailure::new(
5039 "query_workflow_state_unavailable",
5040 "replayed query state type does not match its workflow registration",
5041 "QueryWorkflowStateUnavailable",
5042 ));
5043 }
5044 let replay = workflow.replay.as_ref().ok_or_else(|| {
5045 QueryTaskExecutionFailure::new(
5046 "query_workflow_state_unavailable",
5047 format!(
5048 "workflow type {:?} is not registered for instance-state replay",
5049 task.workflow_type
5050 ),
5051 "QueryWorkflowStateUnavailable",
5052 )
5053 })?;
5054 let workflow_state = Arc::new(Mutex::new(
5055 WorkflowState::new_with_identity(
5056 history_events.as_ref().clone(),
5057 context.workflow_id.clone(),
5058 context.run_id.clone(),
5059 self.task_queue.clone(),
5060 task.payload_codec,
5061 None,
5062 )
5063 .map_err(|error| {
5064 QueryTaskExecutionFailure::new(
5065 "query_workflow_state_unavailable",
5066 format!("workflow replay failed before query: {error}"),
5067 "QueryWorkflowStateUnavailable",
5068 )
5069 })?,
5070 ));
5071 let workflow_context = WorkflowContext {
5072 state: workflow_state,
5073 };
5074 let mut invocation = replay(workflow_context.clone(), workflow_input_typed.clone());
5075 let mut cx = TaskContext::from_waker(noop_waker_ref());
5076 match invocation.future.as_mut().poll(&mut cx) {
5077 Poll::Ready(Ok(_)) => {
5078 workflow_context
5079 .ensure_history_consumed()
5080 .map_err(|error| {
5081 QueryTaskExecutionFailure::new(
5082 "query_workflow_state_unavailable",
5083 format!("workflow replay failed before query: {error}"),
5084 "QueryWorkflowStateUnavailable",
5085 )
5086 })?;
5087 }
5088 Poll::Ready(Err(error)) => {
5089 return Err(QueryTaskExecutionFailure::new(
5090 "query_workflow_state_unavailable",
5091 format!("workflow replay failed before query: {error}"),
5092 "QueryWorkflowStateUnavailable",
5093 ));
5094 }
5095 Poll::Pending => {
5096 let commands = workflow_context.take_commands().map_err(|error| {
5097 QueryTaskExecutionFailure::new(
5098 "query_workflow_state_unavailable",
5099 format!("workflow replay failed before query: {error}"),
5100 "QueryWorkflowStateUnavailable",
5101 )
5102 })?;
5103 if commands.is_empty()
5104 && !workflow_context
5105 .matched_recorded_pending()
5106 .map_err(|error| {
5107 QueryTaskExecutionFailure::new(
5108 "query_workflow_state_unavailable",
5109 format!("workflow replay failed before query: {error}"),
5110 "QueryWorkflowStateUnavailable",
5111 )
5112 })?
5113 {
5114 return Err(QueryTaskExecutionFailure::new(
5115 "query_workflow_state_unavailable",
5116 "workflow replay yielded without a durable command",
5117 "QueryWorkflowStateUnavailable",
5118 ));
5119 }
5120 }
5121 }
5122 let state = (invocation.snapshot)().map_err(|error| {
5123 QueryTaskExecutionFailure::new(
5124 "query_workflow_state_unavailable",
5125 format!("cannot snapshot replayed workflow state: {error}"),
5126 "QueryWorkflowStateUnavailable",
5127 )
5128 })?;
5129 handler(context, state, args).map_err(|message| {
5130 QueryTaskExecutionFailure::new(
5131 "query_workflow_state_unavailable",
5132 message,
5133 "QueryWorkflowStateUnavailable",
5134 )
5135 })?
5136 }
5137 };
5138
5139 future.await.map_err(|error| {
5140 QueryTaskExecutionFailure::new("query_rejected", error.to_string(), "QueryFailed")
5141 })
5142 }
5143
5144 fn execute_workflow_task(&self, task: WorkflowTask) -> Result<Vec<Value>> {
5145 if let Some(update_id) = task
5146 .workflow_update_id
5147 .as_deref()
5148 .filter(|update_id| !update_id.is_empty())
5149 {
5150 return self.execute_update_task(&task, update_id);
5151 }
5152
5153 let workflow = self
5154 .workflows
5155 .get(&task.workflow_type)
5156 .ok_or_else(|| Error::WorkflowNotRegistered(task.workflow_type.clone()))?;
5157 let input = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
5158 let resume_signal = decode_resume_signal(&task)?;
5159 let history_budget = WorkflowHistoryBudget {
5160 event_count: task
5161 .total_history_events
5162 .unwrap_or_else(|| u64::try_from(task.history_events.len()).unwrap_or(u64::MAX)),
5163 size_bytes: task.history_size_bytes,
5164 continue_as_new_recommended: task.continue_as_new_recommended.unwrap_or(false),
5165 pressure: task.history_budget_pressure.clone(),
5166 };
5167 let mut workflow_state = WorkflowState::new_with_identity(
5168 task.history_events,
5169 task.workflow_id,
5170 task.run_id,
5171 self.task_queue.clone(),
5172 task.payload_codec.clone(),
5173 resume_signal,
5174 )?;
5175 workflow_state.history_budget = history_budget;
5176 let state = Arc::new(Mutex::new(workflow_state));
5177 let ctx = WorkflowContext { state };
5178 let mut future = (workflow.execute)(ctx.clone(), input);
5179 let mut cx = TaskContext::from_waker(noop_waker_ref());
5180
5181 match future.as_mut().poll(&mut cx) {
5182 Poll::Ready(Ok(result)) => {
5183 ctx.ensure_history_consumed()?;
5184 let result = encode_typed_envelope(&result, &task.payload_codec)?;
5185 let mut commands = ctx.take_commands()?;
5186 commands.push(json!({
5187 "type": "complete_workflow",
5188 "result": result
5189 }));
5190 Ok(commands)
5191 }
5192 Poll::Ready(Err(error)) => {
5193 if let Error::ContinueAsNew(request) = error {
5194 let mut commands = ctx.take_commands()?;
5195 if let Some(command) = ctx.continue_as_new_command(request)? {
5196 commands.push(command);
5197 }
5198 ctx.ensure_history_consumed()?;
5199 return Ok(commands);
5200 }
5201 ctx.ensure_history_consumed()?;
5204 if workflow_task_integrity_error(&error) {
5205 return Err(error);
5210 }
5211 let mut commands = ctx.take_commands()?;
5212 commands.push(workflow_failure_command(&error));
5213 Ok(commands)
5214 }
5215 Poll::Pending => {
5216 let commands = ctx.take_commands()?;
5217 if commands.is_empty() && !ctx.matched_recorded_pending()? {
5218 Err(Error::WorkflowYieldedWithoutCommand)
5219 } else {
5220 Ok(commands)
5221 }
5222 }
5223 }
5224 }
5225
5226 fn execute_update_task(&self, task: &WorkflowTask, update_id: &str) -> Result<Vec<Value>> {
5227 if !self.workflows.contains_key(&task.workflow_type) {
5228 return Err(Error::WorkflowNotRegistered(task.workflow_type.clone()));
5229 }
5230
5231 let accepted = task.history_events.iter().rev().find_map(|event| {
5232 (event.event_type == "UpdateAccepted"
5233 && event.payload.get("update_id").and_then(Value::as_str) == Some(update_id))
5234 .then_some(&event.payload)
5235 });
5236 let update_name = accepted
5237 .and_then(|payload| payload.get("update_name"))
5238 .and_then(Value::as_str)
5239 .or(task.update_name.as_deref())
5240 .unwrap_or_default();
5241 let Some(handler) = self
5242 .updates
5243 .get(&task.workflow_type)
5244 .and_then(|handlers| handlers.get(update_name))
5245 else {
5246 return Ok(vec![json!({
5247 "type": "fail_update",
5248 "update_id": update_id,
5249 "message": format!(
5250 "no update handler is registered for {}.{update_name}",
5251 task.workflow_type
5252 ),
5253 "exception_type": "UnknownUpdate",
5254 "non_retryable": true,
5255 })]);
5256 };
5257 let arguments = accepted
5258 .and_then(|payload| payload.get("arguments"))
5259 .or(task.arguments.as_ref());
5260 let arguments = decode_task_avro_arguments(arguments, &task.payload_codec)?;
5261 let context = QueryContext {
5262 workflow_id: task.workflow_id.clone(),
5263 run_id: task.run_id.clone(),
5264 workflow_type: task.workflow_type.clone(),
5265 run_status: Some("running".to_string()),
5266 workflow_input: Value::Null,
5267 workflow_input_avro_value: AvroValue::Null,
5268 history_events: Arc::new(task.history_events.clone()),
5269 signal_events: Arc::new(Vec::new()),
5270 };
5271 let mut future = handler(context, arguments);
5272 let mut cx = TaskContext::from_waker(noop_waker_ref());
5273
5274 match future.as_mut().poll(&mut cx) {
5275 Poll::Ready(Ok(result)) => Ok(vec![json!({
5276 "type": "complete_update",
5277 "update_id": update_id,
5278 "result": encode_typed_envelope(&result, &task.payload_codec)?,
5279 })]),
5280 Poll::Ready(Err(error)) => Ok(vec![json!({
5281 "type": "fail_update",
5282 "update_id": update_id,
5283 "message": error.to_string(),
5284 "exception_type": "UpdateFailed",
5285 "non_retryable": true,
5286 })]),
5287 Poll::Pending => Err(Error::WorkflowYieldedWithoutCommand),
5288 }
5289 }
5290
5291 async fn execute_activity_task(&self, task: ActivityTask) -> Result<AvroValue> {
5292 let handler = self
5293 .activities
5294 .get(&task.activity_type)
5295 .ok_or_else(|| Error::ActivityNotRegistered(task.activity_type.clone()))?;
5296 let args = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
5297 let attempt_id = task
5298 .activity_attempt_id
5299 .clone()
5300 .or(task.attempt_id.clone())
5301 .unwrap_or_default();
5302 let lease_owner = task
5303 .lease_owner
5304 .clone()
5305 .unwrap_or_else(|| self.worker_id.clone());
5306 let ctx = ActivityContext {
5307 client: self.client.clone(),
5308 task_id: task.task_id,
5309 activity_attempt_id: attempt_id,
5310 lease_owner,
5311 activity_type: task.activity_type,
5312 attempt_number: task.attempt_number,
5313 task_queue: self.task_queue.clone(),
5314 worker_id: self.worker_id.clone(),
5315 };
5316
5317 handler(ctx, args).await
5318 }
5319}
5320
5321fn poller_result(
5322 kind: &str,
5323 result: std::result::Result<Result<()>, tokio::task::JoinError>,
5324) -> Result<()> {
5325 match result {
5326 Ok(result) => result,
5327 Err(error) => Err(Error::WorkerLoop(format!(
5328 "{kind} poller join error: {error}"
5329 ))),
5330 }
5331}
5332
5333fn optional_poller_result(
5334 kind: &str,
5335 result: Option<std::result::Result<Result<()>, tokio::task::JoinError>>,
5336) -> Result<()> {
5337 match result {
5338 Some(result) => poller_result(kind, result),
5339 None => Ok(()),
5340 }
5341}
5342
5343async fn join_pollers(
5344 workflow_poller: Option<tokio::task::JoinHandle<Result<()>>>,
5345 activity_poller: Option<tokio::task::JoinHandle<Result<()>>>,
5346 query_poller: Option<tokio::task::JoinHandle<Result<()>>>,
5347) -> Result<()> {
5348 let mut first_error = None;
5349
5350 if let Some(handle) = workflow_poller {
5351 if let Err(error) = poller_result("workflow", handle.await) {
5352 first_error.get_or_insert(error);
5353 }
5354 }
5355
5356 if let Some(handle) = activity_poller {
5357 if let Err(error) = poller_result("activity", handle.await) {
5358 first_error.get_or_insert(error);
5359 }
5360 }
5361
5362 if let Some(handle) = query_poller {
5363 if let Err(error) = poller_result("query", handle.await) {
5364 first_error.get_or_insert(error);
5365 }
5366 }
5367
5368 if let Some(error) = first_error {
5369 Err(error)
5370 } else {
5371 Ok(())
5372 }
5373}
5374
5375fn default_worker_id() -> String {
5376 let millis = SystemTime::now()
5377 .duration_since(UNIX_EPOCH)
5378 .unwrap_or_default()
5379 .as_millis();
5380 format!("rust-worker-{}-{millis}", std::process::id())
5381}
5382
5383fn unique_request_id(prefix: &str) -> String {
5384 let nanos = SystemTime::now()
5385 .duration_since(UNIX_EPOCH)
5386 .unwrap_or_default()
5387 .as_nanos();
5388 format!("{prefix}-{}-{nanos}", std::process::id())
5389}
5390
5391#[derive(Debug)]
5392struct QueryTaskExecutionFailure {
5393 reason: String,
5394 message: String,
5395 failure_type: String,
5396}
5397
5398impl QueryTaskExecutionFailure {
5399 fn new(
5400 reason: impl Into<String>,
5401 message: impl Into<String>,
5402 failure_type: impl Into<String>,
5403 ) -> Self {
5404 Self {
5405 reason: reason.into(),
5406 message: message.into(),
5407 failure_type: failure_type.into(),
5408 }
5409 }
5410}
5411
5412#[derive(Clone, Debug)]
5418pub struct WorkflowInstance<S> {
5419 state: Arc<Mutex<S>>,
5420}
5421
5422impl<S> WorkflowInstance<S> {
5423 fn new(state: S) -> Self {
5424 Self {
5425 state: Arc::new(Mutex::new(state)),
5426 }
5427 }
5428
5429 pub fn read<R>(&self, reader: impl FnOnce(&S) -> R) -> Result<R> {
5431 let state = self
5432 .state
5433 .lock()
5434 .map_err(|_| Error::WorkflowStatePoisoned)?;
5435 Ok(reader(&state))
5436 }
5437
5438 pub fn update<R>(&self, transition: impl FnOnce(&mut S) -> R) -> Result<R> {
5440 let mut state = self
5441 .state
5442 .lock()
5443 .map_err(|_| Error::WorkflowStatePoisoned)?;
5444 Ok(transition(&mut state))
5445 }
5446}
5447
5448impl<S: Clone> WorkflowInstance<S> {
5449 fn snapshot(&self) -> Result<S> {
5450 self.read(Clone::clone)
5451 }
5452}
5453
5454#[derive(Clone, Debug)]
5455pub struct WorkflowContext {
5456 state: Arc<Mutex<WorkflowState>>,
5457}
5458
5459impl WorkflowContext {
5460 pub fn workflow_identity(&self) -> Result<WorkflowIdentity> {
5462 let state = self
5463 .state
5464 .lock()
5465 .map_err(|_| Error::WorkflowStatePoisoned)?;
5466 Ok(WorkflowIdentity {
5467 workflow_id: state.workflow_id.clone(),
5468 run_id: state.run_id.clone(),
5469 })
5470 }
5471
5472 pub fn history_budget(&self) -> Result<WorkflowHistoryBudget> {
5474 let state = self
5475 .state
5476 .lock()
5477 .map_err(|_| Error::WorkflowStatePoisoned)?;
5478 Ok(state.history_budget.clone())
5479 }
5480
5481 pub fn continue_as_new<T: Serialize>(&self, args: T) -> Result<Value> {
5487 self.continue_as_new_with_options(ContinueAsNewOptions::new(), args)
5488 }
5489
5490 pub fn continue_as_new_with_options<T: Serialize>(
5492 &self,
5493 options: ContinueAsNewOptions,
5494 args: T,
5495 ) -> Result<Value> {
5496 options.validate()?;
5497 Err(Error::ContinueAsNew(ContinueAsNewRequest {
5498 arguments: normalize_avro_arguments(AvroValue::from_serialize(&args)?),
5499 options,
5500 }))
5501 }
5502
5503 pub fn activity<T: Serialize>(
5504 &self,
5505 activity_type: impl Into<String>,
5506 args: T,
5507 ) -> ActivityCall {
5508 self.activity_with_options(activity_type, ActivityOptions::new(), args)
5509 }
5510
5511 pub fn activity_on_queue<T, Q>(
5512 &self,
5513 activity_type: impl Into<String>,
5514 task_queue: Option<Q>,
5515 args: T,
5516 ) -> ActivityCall
5517 where
5518 T: Serialize,
5519 Q: Into<String>,
5520 {
5521 let mut options = ActivityOptions::new();
5522 options.task_queue = task_queue.map(Into::into);
5523 self.activity_with_options(activity_type, options, args)
5524 }
5525
5526 pub fn activity_with_options<T: Serialize>(
5564 &self,
5565 activity_type: impl Into<String>,
5566 options: ActivityOptions,
5567 args: T,
5568 ) -> ActivityCall {
5569 ActivityCall {
5570 ctx: self.clone(),
5571 activity_type: activity_type.into(),
5572 options,
5573 args: Some(AvroValue::from_serialize(&args)),
5574 scheduled: false,
5575 }
5576 }
5577
5578 pub async fn activity_avro_value<T: Serialize>(
5579 &self,
5580 activity_type: impl Into<String>,
5581 args: T,
5582 ) -> Result<AvroValue> {
5583 let mut call = self.activity(activity_type, args);
5584 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
5585 }
5586
5587 pub async fn activity_avro_value_with_options<T: Serialize>(
5588 &self,
5589 activity_type: impl Into<String>,
5590 options: ActivityOptions,
5591 args: T,
5592 ) -> Result<AvroValue> {
5593 let mut call = self.activity_with_options(activity_type, options, args);
5594 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
5595 }
5596
5597 pub fn wait_signal(&self, signal_name: impl Into<String>) -> SignalCall {
5598 SignalCall {
5599 ctx: self.clone(),
5600 signal_name: signal_name.into(),
5601 opened_wait: false,
5602 matched_pending: false,
5603 }
5604 }
5605
5606 pub async fn wait_signal_avro_value(
5607 &self,
5608 signal_name: impl Into<String>,
5609 ) -> Result<Vec<AvroValue>> {
5610 let mut call = self.wait_signal(signal_name);
5611 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
5612 }
5613
5614 pub fn sleep(&self, duration: Duration) -> TimerCall {
5635 let delay_seconds = duration
5636 .as_secs()
5637 .checked_add(u64::from(duration.subsec_nanos() > 0));
5638 TimerCall {
5639 ctx: self.clone(),
5640 delay_seconds,
5641 scheduled: false,
5642 matched_pending: false,
5643 }
5644 }
5645
5646 pub fn start_timer(&self, duration: Duration) -> TimerCall {
5648 self.sleep(duration)
5649 }
5650
5651 pub fn side_effect<T, F>(&self, callback: F) -> Result<T>
5658 where
5659 T: Serialize + DeserializeOwned,
5660 F: FnOnce() -> T,
5661 {
5662 {
5663 let mut state = self
5664 .state
5665 .lock()
5666 .map_err(|_| Error::WorkflowStatePoisoned)?;
5667 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
5668 return match recorded {
5669 RecordedCommand::SideEffect { sequence, value } => {
5670 state.command_cursor += 1;
5671 value.deserialize().map_err(|error| {
5672 Error::NonDeterministicReplay(ReplayFailure::new(
5673 "side_effect_type_mismatch",
5674 Some(sequence),
5675 Some(std::any::type_name::<T>().to_string()),
5676 Some(error.to_string()),
5677 "recorded side-effect value is incompatible with the requested Rust type",
5678 ))
5679 })
5680 }
5681 other => Err(command_mismatch(&other, "side effect")),
5682 };
5683 }
5684 }
5685
5686 let value = callback();
5687 let avro_value = AvroValue::from_serialize(&value)?;
5688 let mut state = self
5689 .state
5690 .lock()
5691 .map_err(|_| Error::WorkflowStatePoisoned)?;
5692 let result = encode_typed_envelope(&avro_value, &state.payload_codec)?;
5693 state.commands.push(json!({
5694 "type": "record_side_effect",
5695 "result": result,
5696 }));
5697 Ok(value)
5698 }
5699
5700 pub fn side_effect_avro_value<F>(&self, callback: F) -> Result<AvroValue>
5702 where
5703 F: FnOnce() -> AvroValue,
5704 {
5705 {
5706 let mut state = self
5707 .state
5708 .lock()
5709 .map_err(|_| Error::WorkflowStatePoisoned)?;
5710 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
5711 return match recorded {
5712 RecordedCommand::SideEffect { value, .. } => {
5713 state.command_cursor += 1;
5714 Ok(value)
5715 }
5716 other => Err(command_mismatch(&other, "side effect")),
5717 };
5718 }
5719 }
5720
5721 let value = callback();
5722 let mut state = self
5723 .state
5724 .lock()
5725 .map_err(|_| Error::WorkflowStatePoisoned)?;
5726 let result = encode_typed_envelope(&value, &state.payload_codec)?;
5727 state.commands.push(json!({
5728 "type": "record_side_effect",
5729 "result": result,
5730 }));
5731 Ok(value)
5732 }
5733
5734 pub fn uuid_v4(&self) -> Result<Uuid> {
5736 self.side_effect(Uuid::new_v4)
5737 }
5738
5739 pub fn get_version(
5742 &self,
5743 change_id: impl Into<String>,
5744 min_supported: i32,
5745 max_supported: i32,
5746 ) -> Result<i32> {
5747 let change_id = change_id.into();
5748 if change_id.trim().is_empty() {
5749 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5750 "version_change_id_invalid",
5751 None,
5752 Some("non-empty change ID".to_string()),
5753 Some(change_id),
5754 "version markers require a stable non-empty change ID",
5755 )));
5756 }
5757 if min_supported > max_supported {
5758 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5759 "version_range_invalid",
5760 None,
5761 Some("min_supported <= max_supported".to_string()),
5762 Some(format!("{min_supported}..={max_supported}")),
5763 "version marker supported range is invalid",
5764 )));
5765 }
5766
5767 let mut state = self
5768 .state
5769 .lock()
5770 .map_err(|_| Error::WorkflowStatePoisoned)?;
5771 if let Some((version, sequence)) = state.version_markers.get(&change_id).copied() {
5772 ensure_version_supported(&change_id, version, min_supported, max_supported, sequence)?;
5773 return Ok(version);
5774 }
5775
5776 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
5777 return match recorded {
5778 RecordedCommand::VersionMarker {
5779 sequence,
5780 change_id: recorded_change_id,
5781 version,
5782 ..
5783 } => {
5784 if recorded_change_id != change_id {
5785 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5786 "version_change_id_mismatch",
5787 Some(sequence),
5788 Some(recorded_change_id),
5789 Some(change_id),
5790 "recorded version marker change ID differs from current workflow code",
5791 )));
5792 }
5793 ensure_version_supported(
5794 &change_id,
5795 version,
5796 min_supported,
5797 max_supported,
5798 sequence,
5799 )?;
5800 state.command_cursor += 1;
5801 state.version_markers.insert(change_id, (version, sequence));
5802 Ok(version)
5803 }
5804 other => Err(command_mismatch(
5805 &other,
5806 format!("version marker:{change_id}"),
5807 )),
5808 };
5809 }
5810
5811 let version = max_supported;
5812 state.commands.push(json!({
5813 "type": "record_version_marker",
5814 "change_id": change_id,
5815 "version": version,
5816 "min_supported": min_supported,
5817 "max_supported": max_supported,
5818 }));
5819 state.version_markers.insert(change_id, (version, 0));
5822 Ok(version)
5823 }
5824
5825 pub fn patched(&self, change_id: impl Into<String>) -> Result<bool> {
5827 Ok(self.get_version(change_id, -1, 1)? == 1)
5828 }
5829
5830 pub fn deprecate_patch(&self, change_id: impl Into<String>) -> Result<()> {
5832 self.get_version(change_id, -1, 1).map(|_| ())
5833 }
5834
5835 pub fn start_child_workflow<T: Serialize>(
5862 &self,
5863 workflow_type: impl Into<String>,
5864 options: ChildWorkflowOptions,
5865 args: T,
5866 ) -> ChildWorkflowCall {
5867 ChildWorkflowCall {
5868 ctx: self.clone(),
5869 workflow_type: workflow_type.into(),
5870 options,
5871 args: Some(AvroValue::from_serialize(&args)),
5872 scheduled: false,
5873 matched_pending: false,
5874 }
5875 }
5876
5877 pub async fn start_child_workflow_avro_value<T: Serialize>(
5878 &self,
5879 workflow_type: impl Into<String>,
5880 options: ChildWorkflowOptions,
5881 args: T,
5882 ) -> Result<ChildWorkflowAvroResult> {
5883 let mut call = self.start_child_workflow(workflow_type, options, args);
5884 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
5885 }
5886
5887 fn take_commands(&self) -> Result<Vec<Value>> {
5888 let mut state = self
5889 .state
5890 .lock()
5891 .map_err(|_| Error::WorkflowStatePoisoned)?;
5892 Ok(std::mem::take(&mut state.commands))
5893 }
5894
5895 fn continue_as_new_command(&self, request: ContinueAsNewRequest) -> Result<Option<Value>> {
5896 let mut state = self
5897 .state
5898 .lock()
5899 .map_err(|_| Error::WorkflowStatePoisoned)?;
5900
5901 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
5902 return Err(command_mismatch(&recorded, "continue as new"));
5903 }
5904 if state.recorded_continue_as_new_sequence.is_some() {
5905 state.continue_as_new_consumed = true;
5906 return Ok(None);
5907 }
5908
5909 let arguments = encode_typed_envelope(&request.arguments, &state.payload_codec)?;
5910 let mut command = serde_json::Map::from_iter([
5911 ("type".to_string(), json!("continue_as_new")),
5912 ("arguments".to_string(), arguments),
5913 ("queue".to_string(), json!(state.task_queue.clone())),
5914 ]);
5915 if let Some(workflow_type) = request.options.workflow_type {
5916 command.insert("workflow_type".to_string(), json!(workflow_type));
5917 }
5918 if let Some(task_queue) = request.options.task_queue {
5919 command.insert("queue".to_string(), json!(task_queue));
5920 }
5921 Ok(Some(Value::Object(command)))
5922 }
5923
5924 fn matched_recorded_pending(&self) -> Result<bool> {
5925 let state = self
5926 .state
5927 .lock()
5928 .map_err(|_| Error::WorkflowStatePoisoned)?;
5929 Ok(state.matched_recorded_pending)
5930 }
5931
5932 fn ensure_history_consumed(&self) -> Result<()> {
5933 let state = self
5934 .state
5935 .lock()
5936 .map_err(|_| Error::WorkflowStatePoisoned)?;
5937 if let Some(command) = state.recorded_commands.get(state.command_cursor) {
5938 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5939 "recorded_commands_unconsumed",
5940 Some(command.sequence()),
5941 Some(command.shape().to_string()),
5942 Some("workflow completion".to_string()),
5943 "workflow completed before consuming all recorded durable commands",
5944 )));
5945 }
5946 if let Some(sequence) = state
5947 .recorded_continue_as_new_sequence
5948 .filter(|_| !state.continue_as_new_consumed)
5949 {
5950 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5951 "recorded_continue_as_new_unconsumed",
5952 Some(sequence),
5953 Some("continue as new".to_string()),
5954 Some("workflow completion".to_string()),
5955 "workflow completed without consuming its recorded continue-as-new transition",
5956 )));
5957 }
5958 Ok(())
5959 }
5960}
5961
5962#[derive(Debug)]
5963struct WorkflowState {
5964 workflow_id: Option<String>,
5965 run_id: Option<String>,
5966 task_queue: String,
5967 payload_codec: String,
5968 history_budget: WorkflowHistoryBudget,
5969 resume_signal: Option<ResumeSignal>,
5970 recorded_commands: Vec<RecordedCommand>,
5971 recorded_continue_as_new_sequence: Option<u64>,
5972 continue_as_new_consumed: bool,
5973 command_cursor: usize,
5974 matched_recorded_pending: bool,
5975 version_markers: HashMap<String, (i32, u64)>,
5976 commands: Vec<Value>,
5977}
5978
5979impl WorkflowState {
5980 #[cfg(test)]
5981 fn new(
5982 history: Vec<HistoryEvent>,
5983 task_queue: String,
5984 payload_codec: String,
5985 resume_signal: Option<ResumeSignal>,
5986 ) -> Result<Self> {
5987 Self::new_with_identity(
5988 history,
5989 None,
5990 None,
5991 task_queue,
5992 payload_codec,
5993 resume_signal,
5994 )
5995 }
5996
5997 fn new_with_identity(
5998 history: Vec<HistoryEvent>,
5999 workflow_id: Option<String>,
6000 run_id: Option<String>,
6001 task_queue: String,
6002 payload_codec: String,
6003 resume_signal: Option<ResumeSignal>,
6004 ) -> Result<Self> {
6005 let recorded_commands = recorded_commands(
6006 &history,
6007 &payload_codec,
6008 WorkflowIdentity {
6009 workflow_id: workflow_id.clone(),
6010 run_id: run_id.clone(),
6011 },
6012 )?;
6013 let recorded_continue_as_new = history
6014 .iter()
6015 .filter(|event| event.event_type == "WorkflowContinuedAsNew")
6016 .collect::<Vec<_>>();
6017 if recorded_continue_as_new.len() > 1 {
6018 return Err(invalid_recorded_history(
6019 "duplicate_continue_as_new_transition",
6020 recorded_continue_as_new
6021 .last()
6022 .and_then(|event| durable_event_sequence(event))
6023 .unwrap_or(0),
6024 "one WorkflowContinuedAsNew event",
6025 &format!(
6026 "{} WorkflowContinuedAsNew events",
6027 recorded_continue_as_new.len()
6028 ),
6029 "workflow history records one continue-as-new transition more than once",
6030 ));
6031 }
6032 let recorded_continue_as_new_sequence = recorded_continue_as_new
6033 .first()
6034 .map(|event| {
6035 durable_event_sequence(event).ok_or_else(|| {
6036 Error::NonDeterministicReplay(ReplayFailure::new(
6037 "continue_as_new_sequence_missing",
6038 None,
6039 Some("recorded transition sequence".to_string()),
6040 Some("missing sequence".to_string()),
6041 "WorkflowContinuedAsNew history is missing its recorded sequence",
6042 ))
6043 })
6044 })
6045 .transpose()?;
6046 let event_count = u64::try_from(history.len()).unwrap_or(u64::MAX);
6047 Ok(Self {
6048 workflow_id,
6049 run_id,
6050 task_queue,
6051 payload_codec,
6052 history_budget: WorkflowHistoryBudget {
6053 event_count,
6054 ..WorkflowHistoryBudget::default()
6055 },
6056 resume_signal,
6057 recorded_commands,
6058 recorded_continue_as_new_sequence,
6059 continue_as_new_consumed: false,
6060 command_cursor: 0,
6061 matched_recorded_pending: false,
6062 version_markers: HashMap::new(),
6063 commands: Vec::new(),
6064 })
6065 }
6066}
6067
6068#[derive(Clone, Debug)]
6069enum RecordedCommand {
6070 Activity {
6071 sequence: u64,
6072 activity_type: Option<String>,
6073 options: Option<RecordedActivityOptions>,
6074 outcome: Option<ActivityOutcome>,
6075 },
6076 Timer {
6077 sequence: u64,
6078 delay_seconds: u64,
6079 fired: bool,
6080 },
6081 ChildWorkflow {
6082 sequence: u64,
6083 workflow_type: Option<String>,
6084 outcome: Option<ChildWorkflowOutcome>,
6085 },
6086 SignalWait {
6087 sequence: u64,
6088 signal_name: String,
6089 value: Option<Vec<AvroValue>>,
6090 },
6091 SideEffect {
6092 sequence: u64,
6093 value: AvroValue,
6094 },
6095 VersionMarker {
6096 sequence: u64,
6097 change_id: String,
6098 version: i32,
6099 },
6100}
6101
6102#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
6103struct RecordedActivityOptions {
6104 task_queue: RecordedSnapshotValue<Option<String>>,
6105 execution_mode: RecordedSnapshotValue<Option<String>>,
6106 retry_policy: ActivityRetrySnapshot,
6107}
6108
6109#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
6110enum RecordedSnapshotValue<T> {
6111 Unknown,
6113 Known(T),
6114}
6115
6116impl<T: PartialEq> RecordedSnapshotValue<T> {
6117 fn matches_current(&self, current: &Self) -> bool {
6118 match self {
6119 Self::Unknown => true,
6120 Self::Known(recorded) => matches!(current, Self::Known(value) if value == recorded),
6121 }
6122 }
6123}
6124
6125#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
6126struct ActivityRetrySnapshot {
6127 snapshot_version: RecordedSnapshotValue<Option<u64>>,
6128 max_attempts: RecordedSnapshotValue<Option<u64>>,
6129 backoff_seconds: RecordedSnapshotValue<Vec<u64>>,
6130 start_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
6131 schedule_to_start_timeout: RecordedSnapshotValue<Option<u64>>,
6132 schedule_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
6133 heartbeat_timeout: RecordedSnapshotValue<Option<u64>>,
6134 non_retryable_error_types: RecordedSnapshotValue<Vec<String>>,
6135}
6136
6137impl ActivityRetrySnapshot {
6138 fn matches_current(&self, current: &Self) -> bool {
6139 self.snapshot_version
6140 .matches_current(¤t.snapshot_version)
6141 && self.max_attempts.matches_current(¤t.max_attempts)
6142 && self
6143 .backoff_seconds
6144 .matches_current(¤t.backoff_seconds)
6145 && self
6146 .start_to_close_timeout
6147 .matches_current(¤t.start_to_close_timeout)
6148 && self
6149 .schedule_to_start_timeout
6150 .matches_current(¤t.schedule_to_start_timeout)
6151 && self
6152 .schedule_to_close_timeout
6153 .matches_current(¤t.schedule_to_close_timeout)
6154 && self
6155 .heartbeat_timeout
6156 .matches_current(¤t.heartbeat_timeout)
6157 && self
6158 .non_retryable_error_types
6159 .matches_current(¤t.non_retryable_error_types)
6160 }
6161}
6162
6163fn recorded_optional_u64(
6164 object: Option<&serde_json::Map<String, Value>>,
6165 field: &str,
6166) -> RecordedSnapshotValue<Option<u64>> {
6167 match object.and_then(|object| object.get(field)) {
6168 None => RecordedSnapshotValue::Unknown,
6169 Some(Value::Null) => RecordedSnapshotValue::Known(None),
6170 Some(value) => RecordedSnapshotValue::Known(value_as_u64(value)),
6171 }
6172}
6173
6174fn recorded_optional_string(
6175 object: &serde_json::Map<String, Value>,
6176 field: &str,
6177) -> RecordedSnapshotValue<Option<String>> {
6178 match object.get(field) {
6179 None => RecordedSnapshotValue::Unknown,
6180 Some(Value::Null) => RecordedSnapshotValue::Known(None),
6181 Some(value) => RecordedSnapshotValue::Known(value.as_str().map(str::to_string)),
6182 }
6183}
6184
6185fn recorded_activity_retry_snapshot(policy: Option<&Value>) -> ActivityRetrySnapshot {
6186 let policy = policy.and_then(Value::as_object);
6187 let backoff_seconds = policy
6188 .and_then(|policy| policy.get("backoff_seconds"))
6189 .and_then(Value::as_array)
6190 .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
6191 .map_or(RecordedSnapshotValue::Unknown, RecordedSnapshotValue::Known);
6192 let mut non_retryable_error_types = Vec::new();
6193 for error_type in policy
6194 .and_then(|policy| policy.get("non_retryable_error_types"))
6195 .and_then(Value::as_array)
6196 .into_iter()
6197 .flatten()
6198 .filter_map(Value::as_str)
6199 .map(str::trim)
6200 .filter(|error_type| !error_type.is_empty())
6201 {
6202 if !non_retryable_error_types
6203 .iter()
6204 .any(|recorded| recorded == error_type)
6205 {
6206 non_retryable_error_types.push(error_type.to_string());
6207 }
6208 }
6209
6210 ActivityRetrySnapshot {
6211 snapshot_version: recorded_optional_u64(policy, "snapshot_version"),
6212 max_attempts: recorded_optional_u64(policy, "max_attempts"),
6213 backoff_seconds,
6214 start_to_close_timeout: recorded_optional_u64(policy, "start_to_close_timeout"),
6215 schedule_to_start_timeout: recorded_optional_u64(policy, "schedule_to_start_timeout"),
6216 schedule_to_close_timeout: recorded_optional_u64(policy, "schedule_to_close_timeout"),
6217 heartbeat_timeout: recorded_optional_u64(policy, "heartbeat_timeout"),
6218 non_retryable_error_types: if policy
6219 .is_some_and(|policy| policy.contains_key("non_retryable_error_types"))
6220 {
6221 RecordedSnapshotValue::Known(non_retryable_error_types)
6222 } else {
6223 RecordedSnapshotValue::Unknown
6224 },
6225 }
6226}
6227
6228fn current_activity_retry_snapshot(options: &ValidatedActivityOptions) -> ActivityRetrySnapshot {
6229 let policy = options.retry_policy.as_ref();
6230 let max_attempts = match policy.and_then(|policy| policy.get("max_attempts")) {
6231 Some(Value::Null) => None,
6232 Some(value) => value_as_u64(value),
6233 None => Some(1),
6234 };
6235 let backoff_seconds = policy
6236 .and_then(|policy| policy.get("backoff_seconds"))
6237 .and_then(Value::as_array)
6238 .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
6239 .unwrap_or_default();
6240 let non_retryable_error_types = policy
6241 .and_then(|policy| policy.get("non_retryable_error_types"))
6242 .and_then(Value::as_array)
6243 .into_iter()
6244 .flatten()
6245 .filter_map(Value::as_str)
6246 .map(str::to_string)
6247 .collect();
6248
6249 ActivityRetrySnapshot {
6250 snapshot_version: RecordedSnapshotValue::Known(Some(1)),
6251 max_attempts: RecordedSnapshotValue::Known(max_attempts),
6252 backoff_seconds: RecordedSnapshotValue::Known(backoff_seconds),
6253 start_to_close_timeout: RecordedSnapshotValue::Known(options.start_to_close_timeout),
6254 schedule_to_start_timeout: RecordedSnapshotValue::Known(options.schedule_to_start_timeout),
6255 schedule_to_close_timeout: RecordedSnapshotValue::Known(options.schedule_to_close_timeout),
6256 heartbeat_timeout: RecordedSnapshotValue::Known(options.heartbeat_timeout),
6257 non_retryable_error_types: RecordedSnapshotValue::Known(non_retryable_error_types),
6258 }
6259}
6260
6261fn activity_options_description(options: &RecordedActivityOptions) -> String {
6262 serde_json::to_string(options).unwrap_or_else(|_| format!("{options:?}"))
6263}
6264
6265impl RecordedCommand {
6266 fn sequence(&self) -> u64 {
6267 match self {
6268 Self::Activity { sequence, .. }
6269 | Self::Timer { sequence, .. }
6270 | Self::ChildWorkflow { sequence, .. }
6271 | Self::SignalWait { sequence, .. }
6272 | Self::SideEffect { sequence, .. }
6273 | Self::VersionMarker { sequence, .. } => *sequence,
6274 }
6275 }
6276
6277 fn shape(&self) -> &'static str {
6278 match self {
6279 Self::Activity { .. } => "activity",
6280 Self::Timer { .. } => "timer",
6281 Self::ChildWorkflow { .. } => "child workflow",
6282 Self::SignalWait { .. } => "signal wait",
6283 Self::SideEffect { .. } => "side effect",
6284 Self::VersionMarker { .. } => "version marker",
6285 }
6286 }
6287}
6288
6289fn ensure_version_supported(
6290 change_id: &str,
6291 version: i32,
6292 min_supported: i32,
6293 max_supported: i32,
6294 sequence: u64,
6295) -> Result<()> {
6296 if (min_supported..=max_supported).contains(&version) {
6297 return Ok(());
6298 }
6299 Err(Error::NonDeterministicReplay(ReplayFailure::new(
6300 "version_marker_incompatible_range",
6301 (sequence != 0).then_some(sequence),
6302 Some(format!("{min_supported}..={max_supported}")),
6303 Some(format!("{change_id}:{version}")),
6304 "recorded workflow version is outside the range supported by current code",
6305 )))
6306}
6307
6308#[derive(Clone, Debug)]
6309struct ResumeSignal {
6310 signal_name: String,
6311 arguments: Vec<AvroValue>,
6312}
6313
6314pub struct ActivityCall {
6315 ctx: WorkflowContext,
6316 activity_type: String,
6317 options: ActivityOptions,
6318 args: Option<Result<AvroValue>>,
6319 scheduled: bool,
6320}
6321
6322impl ActivityCall {
6323 fn poll_avro_value(
6324 mut self: Pin<&mut Self>,
6325 _cx: &mut TaskContext<'_>,
6326 ) -> Poll<Result<AvroValue>> {
6327 let ctx = self.ctx.clone();
6328 let mut state = match ctx.state.lock() {
6329 Ok(state) => state,
6330 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6331 };
6332
6333 if self.scheduled {
6334 return Poll::Pending;
6335 }
6336
6337 let options = match self.options.validate() {
6338 Ok(options) => options,
6339 Err(error) => {
6340 return Poll::Ready(Err(Error::InvalidActivityOptions(error)));
6341 }
6342 };
6343 let task_queue = options
6344 .task_queue
6345 .clone()
6346 .unwrap_or_else(|| state.task_queue.clone());
6347 let current_recorded_options = RecordedActivityOptions {
6348 task_queue: RecordedSnapshotValue::Known(Some(task_queue.clone())),
6349 execution_mode: RecordedSnapshotValue::Known(None),
6352 retry_policy: current_activity_retry_snapshot(&options),
6353 };
6354
6355 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6356 let sequence = recorded.sequence();
6357 match recorded {
6358 RecordedCommand::Activity {
6359 activity_type,
6360 options: recorded_options,
6361 outcome,
6362 ..
6363 } => {
6364 if let Some(recorded_type) = activity_type {
6365 if recorded_type != self.activity_type {
6366 return Poll::Ready(Err(Error::NonDeterministicReplay(
6367 ReplayFailure::new(
6368 "recorded_command_detail_mismatch",
6369 Some(sequence),
6370 Some(format!("activity:{recorded_type}")),
6371 Some(format!("activity:{}", self.activity_type)),
6372 "recorded activity type differs from the current workflow command",
6373 ),
6374 )));
6375 }
6376 }
6377 if let Some(recorded_options) = recorded_options {
6378 if !recorded_options
6379 .task_queue
6380 .matches_current(¤t_recorded_options.task_queue)
6381 {
6382 return Poll::Ready(Err(Error::NonDeterministicReplay(
6383 ReplayFailure::new(
6384 "activity_task_queue_mismatch",
6385 Some(sequence),
6386 Some(activity_options_description(&recorded_options)),
6387 Some(activity_options_description(¤t_recorded_options)),
6388 "recorded activity task queue differs from the current workflow command",
6389 ),
6390 )));
6391 }
6392 if !recorded_options
6393 .execution_mode
6394 .matches_current(¤t_recorded_options.execution_mode)
6395 {
6396 return Poll::Ready(Err(Error::NonDeterministicReplay(
6397 ReplayFailure::new(
6398 "activity_execution_mode_mismatch",
6399 Some(sequence),
6400 Some(activity_options_description(&recorded_options)),
6401 Some(activity_options_description(¤t_recorded_options)),
6402 "recorded activity execution mode differs from the current workflow command",
6403 ),
6404 )));
6405 }
6406 if !recorded_options
6407 .retry_policy
6408 .matches_current(¤t_recorded_options.retry_policy)
6409 {
6410 return Poll::Ready(Err(Error::NonDeterministicReplay(
6411 ReplayFailure::new(
6412 "activity_retry_policy_mismatch",
6413 Some(sequence),
6414 Some(activity_options_description(&recorded_options)),
6415 Some(activity_options_description(¤t_recorded_options)),
6416 "recorded activity retry policy differs from the current workflow command",
6417 ),
6418 )));
6419 }
6420 }
6421 state.command_cursor += 1;
6422 if let Some(outcome) = outcome {
6423 return Poll::Ready(outcome.map_err(Error::ActivityFailed));
6424 }
6425 state.matched_recorded_pending = true;
6426 self.scheduled = true;
6427 return Poll::Pending;
6428 }
6429 other => {
6430 return Poll::Ready(Err(command_mismatch(
6431 &other,
6432 format!("activity:{}", self.activity_type),
6433 )));
6434 }
6435 }
6436 }
6437
6438 if !self.scheduled {
6439 let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
6440 Ok(args) => args,
6441 Err(error) => return Poll::Ready(Err(error)),
6442 };
6443 let arguments = normalize_avro_arguments(args);
6444 let envelope = match encode_typed_envelope(&arguments, &state.payload_codec) {
6445 Ok(envelope) => envelope,
6446 Err(error) => return Poll::Ready(Err(error)),
6447 };
6448
6449 let mut command = serde_json::Map::from_iter([
6450 ("type".to_string(), json!("schedule_activity")),
6451 (
6452 "activity_type".to_string(),
6453 json!(self.activity_type.clone()),
6454 ),
6455 ("queue".to_string(), json!(task_queue)),
6456 ("arguments".to_string(), envelope),
6457 ]);
6458 for (field, value) in [
6459 ("start_to_close_timeout", options.start_to_close_timeout),
6460 (
6461 "schedule_to_start_timeout",
6462 options.schedule_to_start_timeout,
6463 ),
6464 (
6465 "schedule_to_close_timeout",
6466 options.schedule_to_close_timeout,
6467 ),
6468 ("heartbeat_timeout", options.heartbeat_timeout),
6469 ] {
6470 if let Some(value) = value {
6471 command.insert(field.to_string(), json!(value));
6472 }
6473 }
6474 if let Some(retry_policy) = options.retry_policy {
6475 command.insert("retry_policy".to_string(), retry_policy);
6476 }
6477 state.commands.push(Value::Object(command));
6478 self.scheduled = true;
6479 }
6480
6481 Poll::Pending
6482 }
6483}
6484
6485impl Future for ActivityCall {
6486 type Output = Result<Value>;
6487
6488 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6489 match self.poll_avro_value(cx) {
6490 Poll::Ready(Ok(value)) => Poll::Ready(value.into_json()),
6491 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
6492 Poll::Pending => Poll::Pending,
6493 }
6494 }
6495}
6496
6497pub struct TimerCall {
6499 ctx: WorkflowContext,
6500 delay_seconds: Option<u64>,
6501 scheduled: bool,
6502 matched_pending: bool,
6503}
6504
6505impl Future for TimerCall {
6506 type Output = Result<()>;
6507
6508 fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6509 if self.matched_pending {
6510 return Poll::Pending;
6511 }
6512
6513 let ctx = self.ctx.clone();
6514 let Some(requested_delay) = self.delay_seconds else {
6515 return Poll::Ready(Err(Error::TimerDurationOverflow));
6516 };
6517 let mut state = match ctx.state.lock() {
6518 Ok(state) => state,
6519 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6520 };
6521
6522 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6523 match recorded {
6524 RecordedCommand::Timer {
6525 sequence,
6526 delay_seconds,
6527 fired,
6528 ..
6529 } => {
6530 if delay_seconds != requested_delay {
6531 return Poll::Ready(Err(Error::NonDeterministicReplay(
6532 ReplayFailure::new(
6533 "timer_delay_mismatch",
6534 Some(sequence),
6535 Some(format!("timer:{delay_seconds}s")),
6536 Some(format!("timer:{requested_delay}s")),
6537 "recorded timer delay differs from the current workflow command",
6538 ),
6539 )));
6540 }
6541 state.command_cursor += 1;
6542 if fired {
6543 return Poll::Ready(Ok(()));
6544 }
6545 state.matched_recorded_pending = true;
6546 self.scheduled = true;
6547 self.matched_pending = true;
6548 return Poll::Pending;
6549 }
6550 other => return Poll::Ready(Err(command_mismatch(&other, "timer"))),
6551 }
6552 }
6553
6554 if !self.scheduled {
6555 state.commands.push(json!({
6556 "type": "start_timer",
6557 "delay_seconds": requested_delay,
6558 }));
6559 self.scheduled = true;
6560 }
6561
6562 Poll::Pending
6563 }
6564}
6565
6566pub struct ChildWorkflowCall {
6568 ctx: WorkflowContext,
6569 workflow_type: String,
6570 options: ChildWorkflowOptions,
6571 args: Option<Result<AvroValue>>,
6572 scheduled: bool,
6573 matched_pending: bool,
6574}
6575
6576impl ChildWorkflowCall {
6577 fn poll_avro_value(
6578 mut self: Pin<&mut Self>,
6579 _cx: &mut TaskContext<'_>,
6580 ) -> Poll<Result<ChildWorkflowAvroResult>> {
6581 if self.matched_pending {
6582 return Poll::Pending;
6583 }
6584
6585 let ctx = self.ctx.clone();
6586 let mut state = match ctx.state.lock() {
6587 Ok(state) => state,
6588 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6589 };
6590
6591 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6592 let sequence = recorded.sequence();
6593 match recorded {
6594 RecordedCommand::ChildWorkflow {
6595 workflow_type,
6596 outcome,
6597 ..
6598 } => {
6599 if let Some(recorded_type) = workflow_type {
6600 if recorded_type != self.workflow_type {
6601 return Poll::Ready(Err(Error::NonDeterministicReplay(
6602 ReplayFailure::new(
6603 "recorded_command_detail_mismatch",
6604 Some(sequence),
6605 Some(format!("child workflow:{recorded_type}")),
6606 Some(format!("child workflow:{}", self.workflow_type)),
6607 "recorded child workflow type differs from the current workflow command",
6608 ),
6609 )));
6610 }
6611 }
6612 state.command_cursor += 1;
6613 if let Some(outcome) = outcome {
6614 return Poll::Ready(outcome.map_err(Error::ChildWorkflowFailed));
6615 }
6616 state.matched_recorded_pending = true;
6617 self.scheduled = true;
6618 self.matched_pending = true;
6619 return Poll::Pending;
6620 }
6621 other => {
6622 return Poll::Ready(Err(command_mismatch(
6623 &other,
6624 format!("child workflow:{}", self.workflow_type),
6625 )));
6626 }
6627 }
6628 }
6629
6630 if !self.scheduled {
6631 if self.options.task_queue.trim().is_empty() {
6632 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
6633 "task_queue must not be empty".to_string(),
6634 )));
6635 }
6636 for (name, value) in [
6637 (
6638 "execution_timeout_seconds",
6639 self.options.execution_timeout_seconds,
6640 ),
6641 ("run_timeout_seconds", self.options.run_timeout_seconds),
6642 ] {
6643 if value == Some(0) {
6644 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(format!(
6645 "{name} must be at least 1"
6646 ))));
6647 }
6648 }
6649
6650 let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
6651 Ok(args) => args,
6652 Err(error) => return Poll::Ready(Err(error)),
6653 };
6654 let arguments = match encode_typed_envelope(
6655 &normalize_avro_arguments(args),
6656 &state.payload_codec,
6657 ) {
6658 Ok(arguments) => arguments,
6659 Err(error) => return Poll::Ready(Err(error)),
6660 };
6661 let mut command = json!({
6662 "type": "start_child_workflow",
6663 "workflow_type": self.workflow_type,
6664 "queue": self.options.task_queue,
6665 "parent_close_policy": self.options.parent_close_policy.as_str(),
6666 "arguments": arguments,
6667 });
6668 let object = command
6669 .as_object_mut()
6670 .expect("child workflow command is always an object");
6671 if let Some(policy) = &self.options.retry_policy {
6672 let mut retry_policy = serde_json::Map::new();
6673 if let Some(max_attempts) = policy.max_attempts {
6674 if max_attempts == 0 {
6675 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
6676 "retry_policy.max_attempts must be at least 1".to_string(),
6677 )));
6678 }
6679 retry_policy.insert("max_attempts".to_string(), json!(max_attempts));
6680 }
6681 if !policy.backoff_seconds.is_empty() {
6682 retry_policy
6683 .insert("backoff_seconds".to_string(), json!(policy.backoff_seconds));
6684 }
6685 if !policy.non_retryable_error_types.is_empty() {
6686 retry_policy.insert(
6687 "non_retryable_error_types".to_string(),
6688 json!(policy.non_retryable_error_types),
6689 );
6690 }
6691 if retry_policy.is_empty() {
6692 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
6693 "retry_policy must configure at least one field".to_string(),
6694 )));
6695 }
6696 object.insert("retry_policy".to_string(), Value::Object(retry_policy));
6697 }
6698 if let Some(seconds) = self.options.execution_timeout_seconds {
6699 object.insert("execution_timeout_seconds".to_string(), json!(seconds));
6700 }
6701 if let Some(seconds) = self.options.run_timeout_seconds {
6702 object.insert("run_timeout_seconds".to_string(), json!(seconds));
6703 }
6704 state.commands.push(command);
6705 self.scheduled = true;
6706 }
6707
6708 Poll::Pending
6709 }
6710}
6711
6712impl Future for ChildWorkflowCall {
6713 type Output = Result<ChildWorkflowResult>;
6714
6715 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6716 match self.poll_avro_value(cx) {
6717 Poll::Ready(Ok(result)) => match result.result.into_json() {
6718 Ok(projected) => Poll::Ready(Ok(ChildWorkflowResult {
6719 parent: result.parent,
6720 child: result.child,
6721 child_workflow_type: result.child_workflow_type,
6722 result: projected,
6723 })),
6724 Err(error) => Poll::Ready(Err(error)),
6725 },
6726 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
6727 Poll::Pending => Poll::Pending,
6728 }
6729 }
6730}
6731
6732fn command_mismatch(recorded: &RecordedCommand, actual: impl Into<String>) -> Error {
6733 Error::NonDeterministicReplay(ReplayFailure::new(
6734 "recorded_command_mismatch",
6735 Some(recorded.sequence()),
6736 Some(recorded.shape().to_string()),
6737 Some(actual.into()),
6738 "current workflow command does not match the recorded durable command sequence",
6739 ))
6740}
6741
6742pub struct SignalCall {
6743 ctx: WorkflowContext,
6744 signal_name: String,
6745 opened_wait: bool,
6746 matched_pending: bool,
6747}
6748
6749impl SignalCall {
6750 fn poll_avro_value(
6751 mut self: Pin<&mut Self>,
6752 _cx: &mut TaskContext<'_>,
6753 ) -> Poll<Result<Vec<AvroValue>>> {
6754 if self.matched_pending {
6755 return Poll::Pending;
6756 }
6757
6758 let ctx = self.ctx.clone();
6759 let mut state = match ctx.state.lock() {
6760 Ok(state) => state,
6761 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6762 };
6763
6764 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6765 match recorded {
6766 RecordedCommand::SignalWait {
6767 sequence,
6768 signal_name,
6769 value,
6770 } => {
6771 if signal_name != self.signal_name {
6772 return Poll::Ready(Err(Error::NonDeterministicReplay(
6773 ReplayFailure::new(
6774 "recorded_command_detail_mismatch",
6775 Some(sequence),
6776 Some(format!("signal wait:{signal_name}")),
6777 Some(format!("signal wait:{}", self.signal_name)),
6778 "recorded signal name differs from the current workflow command",
6779 ),
6780 )));
6781 }
6782
6783 state.command_cursor += 1;
6784 if let Some(value) = value {
6785 return Poll::Ready(Ok(value));
6786 }
6787 if state
6788 .resume_signal
6789 .as_ref()
6790 .is_some_and(|signal| signal.signal_name == self.signal_name)
6791 {
6792 let signal = state
6793 .resume_signal
6794 .take()
6795 .expect("matching resume signal is present");
6796 return Poll::Ready(Ok(signal.arguments));
6797 }
6798
6799 state.matched_recorded_pending = true;
6800 self.opened_wait = true;
6801 self.matched_pending = true;
6802 return Poll::Pending;
6803 }
6804 other => {
6805 return Poll::Ready(Err(command_mismatch(
6806 &other,
6807 format!("signal wait:{}", self.signal_name),
6808 )));
6809 }
6810 }
6811 }
6812
6813 if state
6814 .resume_signal
6815 .as_ref()
6816 .is_some_and(|signal| signal.signal_name == self.signal_name)
6817 {
6818 let signal = state
6819 .resume_signal
6820 .take()
6821 .expect("matching resume signal is present");
6822 return Poll::Ready(Ok(signal.arguments));
6823 }
6824
6825 if !self.opened_wait {
6826 state.commands.push(json!({
6827 "type": "open_signal_wait",
6828 "signal_name": self.signal_name
6829 }));
6830 self.opened_wait = true;
6831 }
6832
6833 Poll::Pending
6834 }
6835}
6836
6837impl Future for SignalCall {
6838 type Output = Result<Vec<Value>>;
6839
6840 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6841 match self.poll_avro_value(cx) {
6842 Poll::Ready(Ok(values)) => Poll::Ready(
6843 values
6844 .into_iter()
6845 .map(AvroValue::into_json)
6846 .collect::<Result<Vec<_>>>(),
6847 ),
6848 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
6849 Poll::Pending => Poll::Pending,
6850 }
6851 }
6852}
6853
6854#[derive(Clone, Debug)]
6855pub struct ActivityContext {
6856 client: Client,
6857 pub task_id: String,
6858 pub activity_attempt_id: String,
6859 pub lease_owner: String,
6860 pub activity_type: String,
6861 pub attempt_number: u64,
6862 pub task_queue: String,
6863 pub worker_id: String,
6864}
6865
6866impl ActivityContext {
6867 pub async fn heartbeat<T: Serialize>(&self, details: T) -> Result<ActivityHeartbeatResponse> {
6868 self.client
6869 .heartbeat_activity_task(
6870 &self.task_id,
6871 &self.activity_attempt_id,
6872 &self.lease_owner,
6873 details,
6874 )
6875 .await
6876 }
6877}
6878
6879fn decode_task_avro_arguments(value: Option<&Value>, codec: &str) -> Result<AvroValue> {
6880 match value {
6881 Some(value) => Ok(normalize_avro_arguments(decode_wire_avro_value(
6882 value, codec,
6883 )?)),
6884 None => Ok(AvroValue::Array(Vec::new())),
6885 }
6886}
6887
6888fn decode_resume_signal(task: &WorkflowTask) -> Result<Option<ResumeSignal>> {
6889 let Some(signal_name) = task
6890 .signal_name
6891 .as_deref()
6892 .filter(|value| !value.is_empty())
6893 else {
6894 return Ok(None);
6895 };
6896 let Some(arguments) = task.signal_arguments.as_ref() else {
6897 return Ok(None);
6898 };
6899
6900 let decoded = normalize_avro_arguments(decode_wire_avro_value(arguments, &task.payload_codec)?);
6901 let AvroValue::Array(arguments) = decoded else {
6902 unreachable!("normalize_avro_arguments always returns an array");
6903 };
6904
6905 Ok(Some(ResumeSignal {
6906 signal_name: signal_name.to_string(),
6907 arguments,
6908 }))
6909}
6910
6911fn recorded_commands(
6912 events: &[HistoryEvent],
6913 fallback_codec: &str,
6914 parent: WorkflowIdentity,
6915) -> Result<Vec<RecordedCommand>> {
6916 let mut events_by_sequence: BTreeMap<u64, Vec<&HistoryEvent>> = BTreeMap::new();
6917 let mut last_new_sequence = None;
6918
6919 for event in events {
6920 let is_activity = matches!(
6921 event.event_type.as_str(),
6922 "ActivityScheduled"
6923 | "ActivityStarted"
6924 | "ActivityHeartbeatRecorded"
6925 | "ActivityRetryScheduled"
6926 | "ActivityCompleted"
6927 | "ActivityFailed"
6928 | "ActivityCancelled"
6929 | "ActivityTimedOut"
6930 );
6931 let is_workflow_timer = matches!(
6932 event.event_type.as_str(),
6933 "TimerScheduled" | "TimerCancelled" | "TimerFired"
6934 ) && !is_internal_timer_event(event);
6935 let is_child_workflow = matches!(
6936 event.event_type.as_str(),
6937 "ChildWorkflowScheduled"
6938 | "ChildRunCompleted"
6939 | "ChildRunFailed"
6940 | "ChildRunCancelled"
6941 | "ChildRunTerminated"
6942 );
6943 let is_signal_wait = is_recorded_signal_wait_event(event);
6944 let is_side_effect = event.event_type == "SideEffectRecorded";
6945 let is_version_marker = event.event_type == "VersionMarkerRecorded";
6946 if !is_activity
6947 && !is_workflow_timer
6948 && !is_child_workflow
6949 && !is_signal_wait
6950 && !is_side_effect
6951 && !is_version_marker
6952 {
6953 continue;
6954 }
6955
6956 let sequence = durable_event_sequence(event).ok_or_else(|| {
6957 Error::NonDeterministicReplay(ReplayFailure::new(
6958 "durable_command_sequence_missing",
6959 None,
6960 Some("positive workflow sequence".to_string()),
6961 Some(event.event_type.clone()),
6962 "durable command history event has no workflow sequence",
6963 ))
6964 })?;
6965 if sequence == 0 {
6966 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
6967 "durable_command_sequence_invalid",
6968 Some(sequence),
6969 Some("positive workflow sequence".to_string()),
6970 Some(sequence.to_string()),
6971 "durable command history uses an invalid workflow sequence",
6972 )));
6973 }
6974 if !events_by_sequence.contains_key(&sequence) {
6975 if let Some(previous) = last_new_sequence {
6976 if sequence < previous {
6977 return Err(invalid_recorded_history(
6978 "durable_command_sequence_mismatch",
6979 sequence,
6980 &format!("workflow sequence greater than {previous}"),
6981 &sequence.to_string(),
6982 "durable commands are not strictly ordered by their recorded workflow sequence",
6983 ));
6984 }
6985 }
6986 last_new_sequence = Some(sequence);
6987 }
6988 events_by_sequence.entry(sequence).or_default().push(event);
6989 }
6990
6991 let commands: Vec<RecordedCommand> = events_by_sequence
6992 .into_iter()
6993 .map(|(sequence, sequence_events)| {
6994 let activity_events: Vec<_> = sequence_events
6995 .iter()
6996 .copied()
6997 .filter(|event| event.event_type.starts_with("Activity"))
6998 .collect();
6999 let timer_events: Vec<_> = sequence_events
7000 .iter()
7001 .copied()
7002 .filter(|event| event.event_type.starts_with("Timer"))
7003 .collect();
7004 let child_events: Vec<_> = sequence_events
7005 .iter()
7006 .copied()
7007 .filter(|event| {
7008 event.event_type == "ChildWorkflowScheduled"
7009 || event.event_type.starts_with("ChildRun")
7010 })
7011 .collect();
7012 let signal_wait_events: Vec<_> = sequence_events
7013 .iter()
7014 .copied()
7015 .filter(|event| is_recorded_signal_wait_event(event))
7016 .collect();
7017 let side_effect_events: Vec<_> = sequence_events
7018 .iter()
7019 .copied()
7020 .filter(|event| event.event_type == "SideEffectRecorded")
7021 .collect();
7022 let version_marker_events: Vec<_> = sequence_events
7023 .iter()
7024 .copied()
7025 .filter(|event| event.event_type == "VersionMarkerRecorded")
7026 .collect();
7027
7028 let command_kind_count = usize::from(!activity_events.is_empty())
7029 + usize::from(!timer_events.is_empty())
7030 + usize::from(!child_events.is_empty())
7031 + usize::from(!signal_wait_events.is_empty())
7032 + usize::from(!side_effect_events.is_empty())
7033 + usize::from(!version_marker_events.is_empty());
7034 if command_kind_count > 1 {
7035 let actual = [
7036 (!activity_events.is_empty()).then_some("activity"),
7037 (!timer_events.is_empty()).then_some("timer"),
7038 (!child_events.is_empty()).then_some("child workflow"),
7039 (!signal_wait_events.is_empty()).then_some("signal wait"),
7040 (!side_effect_events.is_empty()).then_some("side effect"),
7041 (!version_marker_events.is_empty()).then_some("version marker"),
7042 ]
7043 .into_iter()
7044 .flatten()
7045 .collect::<Vec<_>>()
7046 .join(" and ");
7047 return Err(invalid_recorded_history(
7048 "durable_command_sequence_collision",
7049 sequence,
7050 "one durable command kind",
7051 &actual,
7052 "one workflow sequence records more than one durable command kind",
7053 ));
7054 }
7055
7056 if !activity_events.is_empty() {
7057 let scheduled_count = activity_events
7058 .iter()
7059 .filter(|event| event.event_type == "ActivityScheduled")
7060 .count();
7061 if scheduled_count > 1 {
7062 return Err(invalid_recorded_history(
7063 "duplicate_activity_schedule",
7064 sequence,
7065 "at most one ActivityScheduled event",
7066 "multiple ActivityScheduled events",
7067 "activity history schedules more than one command at one workflow sequence",
7068 ));
7069 }
7070 let activity_type = activity_events.iter().find_map(|event| {
7071 event
7072 .payload
7073 .get("activity_type")
7074 .or_else(|| event.payload.get("activity_name"))
7075 .and_then(Value::as_str)
7076 .map(str::to_string)
7077 });
7078 if activity_events.iter().filter_map(|event| {
7079 event
7080 .payload
7081 .get("activity_type")
7082 .or_else(|| event.payload.get("activity_name"))
7083 .and_then(Value::as_str)
7084 }).any(|candidate| Some(candidate) != activity_type.as_deref()) {
7085 return Err(invalid_recorded_history(
7086 "activity_identity_mismatch",
7087 sequence,
7088 activity_type.as_deref().unwrap_or("one activity identity"),
7089 "conflicting activity identities",
7090 "activity lifecycle events at one workflow sequence disagree on identity",
7091 ));
7092 }
7093 let terminal: Vec<_> = activity_events
7094 .iter()
7095 .copied()
7096 .filter(|event| {
7097 matches!(
7098 event.event_type.as_str(),
7099 "ActivityCompleted"
7100 | "ActivityFailed"
7101 | "ActivityCancelled"
7102 | "ActivityTimedOut"
7103 )
7104 })
7105 .collect();
7106 if terminal.len() > 1 {
7107 return Err(invalid_recorded_history(
7108 "duplicate_activity_terminal_event",
7109 sequence,
7110 "at most one terminal activity event",
7111 "multiple terminal activity events",
7112 "activity history settles one command more than once",
7113 ));
7114 }
7115 let outcome = terminal
7116 .first()
7117 .map(|event| activity_outcome(event, fallback_codec, activity_type.clone()))
7118 .transpose()?;
7119 let options = activity_events
7120 .iter()
7121 .find(|event| event.event_type == "ActivityScheduled")
7122 .and_then(|event| event.payload.get("activity"))
7123 .and_then(Value::as_object)
7124 .map(|activity| RecordedActivityOptions {
7125 task_queue: recorded_optional_string(activity, "queue"),
7126 execution_mode: recorded_optional_string(activity, "execution_mode"),
7127 retry_policy: recorded_activity_retry_snapshot(
7128 activity.get("retry_policy"),
7129 ),
7130 });
7131 return Ok(RecordedCommand::Activity {
7132 sequence,
7133 activity_type,
7134 options,
7135 outcome,
7136 });
7137 }
7138
7139 if !child_events.is_empty() {
7140 let scheduled: Vec<_> = child_events
7141 .iter()
7142 .copied()
7143 .filter(|event| event.event_type == "ChildWorkflowScheduled")
7144 .collect();
7145 if scheduled.len() != 1 {
7146 return Err(invalid_recorded_history(
7147 "child_workflow_schedule_missing_or_duplicate",
7148 sequence,
7149 "one ChildWorkflowScheduled event",
7150 &format!("{} ChildWorkflowScheduled events", scheduled.len()),
7151 "child workflow replay requires exactly one recorded schedule event",
7152 ));
7153 }
7154 let workflow_type = child_events.iter().find_map(|event| {
7155 event
7156 .payload
7157 .get("child_workflow_type")
7158 .or_else(|| event.payload.get("workflow_type"))
7159 .and_then(Value::as_str)
7160 .filter(|value| !value.is_empty())
7161 .map(str::to_string)
7162 });
7163 if child_events
7164 .iter()
7165 .filter_map(|event| {
7166 event
7167 .payload
7168 .get("child_workflow_type")
7169 .or_else(|| event.payload.get("workflow_type"))
7170 .and_then(Value::as_str)
7171 })
7172 .any(|candidate| Some(candidate) != workflow_type.as_deref())
7173 {
7174 return Err(invalid_recorded_history(
7175 "child_workflow_identity_mismatch",
7176 sequence,
7177 workflow_type
7178 .as_deref()
7179 .unwrap_or("one child workflow type"),
7180 "conflicting child workflow types",
7181 "child workflow lifecycle events at one sequence disagree on type",
7182 ));
7183 }
7184 let mut outcomes = child_workflow_outcomes(
7185 &child_events.iter().map(|event| (*event).clone()).collect::<Vec<_>>(),
7186 fallback_codec,
7187 parent.clone(),
7188 )?;
7189 if outcomes.len() > 1 {
7190 return Err(invalid_recorded_history(
7191 "duplicate_child_workflow_terminal_event",
7192 sequence,
7193 "at most one terminal child event",
7194 "multiple terminal child events",
7195 "child workflow history settles one command more than once",
7196 ));
7197 }
7198 return Ok(RecordedCommand::ChildWorkflow {
7199 sequence,
7200 workflow_type,
7201 outcome: outcomes.pop(),
7202 });
7203 }
7204
7205 if !signal_wait_events.is_empty() {
7206 let opened: Vec<_> = signal_wait_events
7207 .iter()
7208 .copied()
7209 .filter(|event| event.event_type == "SignalWaitOpened")
7210 .collect();
7211 if opened.len() != 1 {
7212 return Err(invalid_recorded_history(
7213 "signal_wait_open_missing_or_duplicate",
7214 sequence,
7215 "one SignalWaitOpened event",
7216 &format!("{} SignalWaitOpened events", opened.len()),
7217 "signal replay requires exactly one canonical wait-open event",
7218 ));
7219 }
7220
7221 let applied: Vec<_> = signal_wait_events
7222 .iter()
7223 .copied()
7224 .filter(|event| event.event_type == "SignalApplied")
7225 .collect();
7226 if applied.len() > 1 {
7227 return Err(invalid_recorded_history(
7228 "duplicate_signal_wait_apply",
7229 sequence,
7230 "at most one SignalApplied event",
7231 "multiple SignalApplied events",
7232 "signal history applies one durable wait more than once",
7233 ));
7234 }
7235
7236 let signal_names = signal_wait_events
7237 .iter()
7238 .map(|event| required_signal_wait_name(event, sequence))
7239 .collect::<Result<Vec<_>>>()?;
7240 let signal_name = signal_names
7241 .first()
7242 .expect("signal wait events are not empty")
7243 .clone();
7244 if signal_names.iter().any(|candidate| candidate != &signal_name) {
7245 return Err(invalid_recorded_history(
7246 "signal_wait_identity_mismatch",
7247 sequence,
7248 &signal_name,
7249 "conflicting signal names",
7250 "signal wait lifecycle events at one workflow sequence disagree on identity",
7251 ));
7252 }
7253 let value = applied
7254 .first()
7255 .map(|event| decode_signal_event_arguments(event, fallback_codec))
7256 .transpose()?;
7257 return Ok(RecordedCommand::SignalWait {
7258 sequence,
7259 signal_name,
7260 value,
7261 });
7262 }
7263
7264 if !side_effect_events.is_empty() {
7265 if side_effect_events.len() != 1 {
7266 return Err(invalid_recorded_history(
7267 "duplicate_side_effect_record",
7268 sequence,
7269 "one SideEffectRecorded event",
7270 &format!("{} SideEffectRecorded events", side_effect_events.len()),
7271 "side-effect history records one workflow command more than once",
7272 ));
7273 }
7274 let event = side_effect_events[0];
7275 let result = event.payload.get("result").ok_or_else(|| {
7276 invalid_recorded_history(
7277 "side_effect_result_missing",
7278 sequence,
7279 "recorded result payload",
7280 "missing result",
7281 "side-effect history is missing its recorded value",
7282 )
7283 })?;
7284 let has_published_envelope = result.as_str().is_some()
7285 || result.as_object().is_some_and(|envelope| {
7286 envelope.get("codec").and_then(Value::as_str).is_some()
7287 && envelope.get("blob").and_then(Value::as_str).is_some()
7288 });
7289 if !has_published_envelope {
7290 return Err(invalid_recorded_history(
7291 "side_effect_payload_malformed",
7292 sequence,
7293 "payload blob or {codec, blob} envelope",
7294 &result.to_string(),
7295 "side-effect history result does not use a published payload envelope",
7296 ));
7297 }
7298 let codec = event
7299 .payload
7300 .get("payload_codec")
7301 .and_then(Value::as_str)
7302 .unwrap_or(fallback_codec);
7303 let value = decode_wire_avro_value(result, codec).map_err(|error| {
7304 invalid_recorded_history(
7305 "side_effect_payload_incompatible",
7306 sequence,
7307 &format!("valid {codec} payload envelope"),
7308 &error.to_string(),
7309 "side-effect history payload cannot be decoded with its recorded codec",
7310 )
7311 })?;
7312 return Ok(RecordedCommand::SideEffect { sequence, value });
7313 }
7314
7315 if !version_marker_events.is_empty() {
7316 if version_marker_events.len() != 1 {
7317 return Err(invalid_recorded_history(
7318 "duplicate_version_marker_record",
7319 sequence,
7320 "one VersionMarkerRecorded event",
7321 &format!("{} VersionMarkerRecorded events", version_marker_events.len()),
7322 "version-marker history records one workflow command more than once",
7323 ));
7324 }
7325 let payload = &version_marker_events[0].payload;
7326 let change_id = payload
7327 .get("change_id")
7328 .and_then(Value::as_str)
7329 .filter(|value| !value.is_empty())
7330 .map(str::to_string)
7331 .ok_or_else(|| {
7332 invalid_recorded_history(
7333 "version_marker_field_missing",
7334 sequence,
7335 "non-empty change_id",
7336 "missing or invalid change_id",
7337 "version-marker history is missing its stable change ID",
7338 )
7339 })?;
7340 let version = required_version_i32(payload, "version", sequence)?;
7341 let min_supported = required_version_i32(payload, "min_supported", sequence)?;
7342 let max_supported = required_version_i32(payload, "max_supported", sequence)?;
7343 if min_supported > max_supported || version < min_supported || version > max_supported {
7344 return Err(invalid_recorded_history(
7345 "version_marker_history_range_invalid",
7346 sequence,
7347 "min_supported <= version <= max_supported",
7348 &format!("{min_supported} <= {version} <= {max_supported}"),
7349 "recorded version marker contains an internally incompatible range",
7350 ));
7351 }
7352 return Ok(RecordedCommand::VersionMarker {
7353 sequence,
7354 change_id,
7355 version,
7356 });
7357 }
7358
7359 let scheduled: Vec<_> = timer_events
7360 .iter()
7361 .copied()
7362 .filter(|event| event.event_type == "TimerScheduled")
7363 .collect();
7364 let fired: Vec<_> = timer_events
7365 .iter()
7366 .copied()
7367 .filter(|event| event.event_type == "TimerFired")
7368 .collect();
7369 if scheduled.len() != 1 {
7370 return Err(invalid_recorded_history(
7371 "timer_schedule_missing_or_duplicate",
7372 sequence,
7373 "one TimerScheduled event",
7374 &format!("{} TimerScheduled events", scheduled.len()),
7375 "timer replay requires exactly one recorded schedule event",
7376 ));
7377 }
7378 if fired.len() > 1 {
7379 return Err(invalid_recorded_history(
7380 "duplicate_timer_fire",
7381 sequence,
7382 "at most one TimerFired event",
7383 "multiple TimerFired events",
7384 "timer history contains more than one fire event for a workflow sequence",
7385 ));
7386 }
7387
7388 let scheduled = scheduled[0];
7389 let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
7390 let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
7391 if let Some(fired) = fired.first() {
7392 let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
7393 if fired_timer_id != timer_id {
7394 return Err(invalid_recorded_history(
7395 "timer_identity_mismatch",
7396 sequence,
7397 &timer_id,
7398 &fired_timer_id,
7399 "TimerFired does not correspond to the recorded TimerScheduled event",
7400 ));
7401 }
7402 let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
7403 if fired_delay != delay_seconds {
7404 return Err(invalid_recorded_history(
7405 "timer_history_delay_mismatch",
7406 sequence,
7407 &delay_seconds.to_string(),
7408 &fired_delay.to_string(),
7409 "TimerScheduled and TimerFired record different delays",
7410 ));
7411 }
7412 }
7413
7414 Ok(RecordedCommand::Timer {
7415 sequence,
7416 delay_seconds,
7417 fired: !fired.is_empty(),
7418 })
7419 })
7420 .collect::<Result<_>>()?;
7421
7422 let mut marker_sequences = HashMap::new();
7423 for command in &commands {
7424 if let RecordedCommand::VersionMarker {
7425 sequence,
7426 change_id,
7427 ..
7428 } = command
7429 {
7430 if let Some(first_sequence) = marker_sequences.insert(change_id.clone(), *sequence) {
7431 return Err(invalid_recorded_history(
7432 "duplicate_version_marker",
7433 *sequence,
7434 &format!("one marker for change ID {change_id:?}"),
7435 &format!("markers at sequences {first_sequence} and {sequence}"),
7436 "workflow history contains duplicate markers for one stable change ID",
7437 ));
7438 }
7439 }
7440 }
7441
7442 Ok(commands)
7443}
7444
7445fn required_version_i32(payload: &Value, field: &str, sequence: u64) -> Result<i32> {
7446 payload
7447 .get(field)
7448 .and_then(Value::as_i64)
7449 .and_then(|value| i32::try_from(value).ok())
7450 .ok_or_else(|| {
7451 invalid_recorded_history(
7452 "version_marker_field_missing",
7453 sequence,
7454 &format!("integer {field}"),
7455 "missing or out-of-range integer",
7456 "version-marker history is missing a required integer field",
7457 )
7458 })
7459}
7460
7461fn durable_event_sequence(event: &HistoryEvent) -> Option<u64> {
7462 event
7463 .payload
7464 .get("sequence")
7465 .or_else(|| event.payload.get("workflow_sequence"))
7466 .or_else(|| event.raw.get("sequence"))
7467 .or_else(|| event.raw.get("workflow_sequence"))
7468 .and_then(value_as_u64)
7469}
7470
7471fn is_internal_timer_event(event: &HistoryEvent) -> bool {
7472 matches!(
7473 event
7474 .payload
7475 .get("timer_kind")
7476 .or_else(|| event.raw.get("timer_kind"))
7477 .and_then(Value::as_str),
7478 Some("condition_timeout" | "signal_timeout")
7479 )
7480}
7481
7482fn required_signal_wait_name(event: &HistoryEvent, sequence: u64) -> Result<String> {
7483 event
7484 .payload
7485 .get("signal_name")
7486 .or_else(|| event.raw.get("signal_name"))
7487 .and_then(Value::as_str)
7488 .filter(|value| !value.is_empty())
7489 .map(str::to_string)
7490 .ok_or_else(|| {
7491 invalid_recorded_history(
7492 "signal_wait_name_missing",
7493 sequence,
7494 "non-empty signal_name",
7495 &event.event_type,
7496 "canonical signal-wait history is missing its signal identity",
7497 )
7498 })
7499}
7500
7501fn is_recorded_signal_wait_event(event: &HistoryEvent) -> bool {
7502 matches!(
7503 event.event_type.as_str(),
7504 "SignalWaitOpened" | "SignalApplied"
7505 )
7506}
7507
7508fn required_history_string(event: &HistoryEvent, field: &str, sequence: u64) -> Result<String> {
7509 event
7510 .payload
7511 .get(field)
7512 .and_then(Value::as_str)
7513 .filter(|value| !value.is_empty())
7514 .map(str::to_string)
7515 .ok_or_else(|| {
7516 invalid_recorded_history(
7517 "timer_history_field_missing",
7518 sequence,
7519 field,
7520 &event.event_type,
7521 "timer history is missing a required identity field",
7522 )
7523 })
7524}
7525
7526fn required_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<u64> {
7527 event
7528 .payload
7529 .get(field)
7530 .and_then(value_as_u64)
7531 .ok_or_else(|| {
7532 invalid_recorded_history(
7533 "timer_history_field_missing",
7534 sequence,
7535 field,
7536 &event.event_type,
7537 "timer history is missing a required numeric field",
7538 )
7539 })
7540}
7541
7542fn invalid_recorded_history(
7543 reason: &str,
7544 sequence: u64,
7545 expected: &str,
7546 actual: &str,
7547 message: &str,
7548) -> Error {
7549 Error::NonDeterministicReplay(ReplayFailure::new(
7550 reason,
7551 Some(sequence),
7552 Some(expected.to_string()),
7553 Some(actual.to_string()),
7554 message,
7555 ))
7556}
7557
7558type ActivityOutcome = std::result::Result<AvroValue, ActivityFailure>;
7559
7560fn activity_outcome(
7561 event: &HistoryEvent,
7562 fallback_codec: &str,
7563 recorded_activity_type: Option<String>,
7564) -> Result<ActivityOutcome> {
7565 if event.event_type == "ActivityCompleted" {
7566 let codec = event
7567 .payload
7568 .get("payload_codec")
7569 .and_then(Value::as_str)
7570 .unwrap_or(fallback_codec);
7571 return Ok(Ok(decode_wire_avro_value(
7572 event.payload.get("result").unwrap_or(&Value::Null),
7573 codec,
7574 )?));
7575 }
7576
7577 let payload = &event.payload;
7578 let (kind, fallback_reason, fallback_message) = match event.event_type.as_str() {
7579 "ActivityFailed" => (ActivityFailureKind::Failed, "activity", "activity failed"),
7580 "ActivityCancelled" => (
7581 ActivityFailureKind::Cancelled,
7582 "cancelled",
7583 "activity was cancelled",
7584 ),
7585 "ActivityTimedOut" => (
7586 ActivityFailureKind::TimedOut,
7587 "timeout",
7588 "activity timed out",
7589 ),
7590 _ => unreachable!("activity_outcome is called only for terminal activity events"),
7591 };
7592 let exception = payload
7593 .get("exception")
7594 .filter(|value| !value.is_null())
7595 .cloned();
7596 let failure_category = payload_string(payload, "failure_category");
7597 let timeout_kind = payload_string(payload, "timeout_kind");
7598 let reason = payload_string(payload, "reason").unwrap_or_else(|| match kind {
7599 ActivityFailureKind::Failed => failure_category
7600 .clone()
7601 .unwrap_or_else(|| fallback_reason.to_string()),
7602 ActivityFailureKind::Cancelled => fallback_reason.to_string(),
7603 ActivityFailureKind::TimedOut => timeout_kind
7604 .clone()
7605 .unwrap_or_else(|| fallback_reason.to_string()),
7606 });
7607 let message = payload_string(payload, "message")
7608 .or_else(|| {
7609 exception
7610 .as_ref()
7611 .and_then(|value| payload_string(value, "message"))
7612 })
7613 .unwrap_or_else(|| fallback_message.to_string());
7614
7615 Ok(Err(ActivityFailure {
7616 kind,
7617 reason,
7618 message,
7619 activity_execution_id: payload_string(payload, "activity_execution_id"),
7620 activity_attempt_id: payload_string(payload, "activity_attempt_id"),
7621 activity_type: payload_string(payload, "activity_type")
7622 .or_else(|| payload_string(payload, "activity_name"))
7623 .or(recorded_activity_type),
7624 activity_class: payload_string(payload, "activity_class"),
7625 attempt_number: payload.get("attempt_number").and_then(value_as_u64),
7626 failure_id: payload_string(payload, "failure_id"),
7627 failure_category,
7628 timeout_kind,
7629 non_retryable: payload
7630 .get("non_retryable")
7631 .and_then(Value::as_bool)
7632 .unwrap_or(false),
7633 exception_type: payload_string(payload, "exception_type").or_else(|| {
7634 exception
7635 .as_ref()
7636 .and_then(|value| payload_string(value, "type"))
7637 }),
7638 exception_class: payload_string(payload, "exception_class").or_else(|| {
7639 exception
7640 .as_ref()
7641 .and_then(|value| payload_string(value, "class"))
7642 }),
7643 code: payload
7644 .get("code")
7645 .filter(|value| !value.is_null())
7646 .cloned(),
7647 exception,
7648 }))
7649}
7650
7651type ChildWorkflowOutcome = std::result::Result<ChildWorkflowAvroResult, ChildWorkflowFailure>;
7652
7653fn child_workflow_outcomes(
7654 events: &[HistoryEvent],
7655 fallback_codec: &str,
7656 parent: WorkflowIdentity,
7657) -> Result<Vec<ChildWorkflowOutcome>> {
7658 let mut outcomes = Vec::new();
7659
7660 for event in events {
7661 let kind = match event.event_type.as_str() {
7662 "ChildRunCompleted" => None,
7663 "ChildRunFailed" => Some((
7664 ChildWorkflowFailureKind::Failed,
7665 "child_workflow",
7666 "child workflow failed",
7667 )),
7668 "ChildRunCancelled" => Some((
7669 ChildWorkflowFailureKind::Cancelled,
7670 "cancelled",
7671 "child workflow was cancelled",
7672 )),
7673 "ChildRunTerminated" => Some((
7674 ChildWorkflowFailureKind::Terminated,
7675 "terminated",
7676 "child workflow was terminated",
7677 )),
7678 _ => continue,
7679 };
7680 let payload = &event.payload;
7681 let child_workflow_id = payload_string(payload, "child_workflow_instance_id");
7682 let child_workflow_run_id = payload_string(payload, "child_workflow_run_id");
7683 let child_workflow_type = payload_string(payload, "child_workflow_type");
7684
7685 if let Some((kind, reason, fallback_message)) = kind {
7686 let exception = payload
7687 .get("exception")
7688 .filter(|value| !value.is_null())
7689 .cloned();
7690 let message = payload_string(payload, "message")
7691 .or_else(|| {
7692 exception
7693 .as_ref()
7694 .and_then(|value| payload_string(value, "message"))
7695 })
7696 .unwrap_or_else(|| fallback_message.to_string());
7697 let exception_type = payload_string(payload, "exception_type").or_else(|| {
7698 exception
7699 .as_ref()
7700 .and_then(|value| payload_string(value, "type"))
7701 });
7702 let exception_class = payload_string(payload, "exception_class").or_else(|| {
7703 exception
7704 .as_ref()
7705 .and_then(|value| payload_string(value, "class"))
7706 });
7707 outcomes.push(Err(ChildWorkflowFailure {
7708 kind,
7709 reason: reason.to_string(),
7710 message,
7711 parent_workflow_id: parent.workflow_id.clone(),
7712 parent_workflow_run_id: parent.run_id.clone(),
7713 child_workflow_id,
7714 child_workflow_run_id,
7715 child_workflow_type,
7716 failure_id: payload_string(payload, "failure_id"),
7717 failure_category: payload_string(payload, "failure_category"),
7718 exception_type,
7719 exception_class,
7720 non_retryable: payload
7721 .get("non_retryable")
7722 .and_then(Value::as_bool)
7723 .unwrap_or(false),
7724 code: payload
7725 .get("code")
7726 .filter(|value| !value.is_null())
7727 .cloned(),
7728 exception,
7729 }));
7730 continue;
7731 }
7732
7733 let codec = payload
7734 .get("payload_codec")
7735 .and_then(Value::as_str)
7736 .unwrap_or(fallback_codec);
7737 let result = payload
7738 .get("result")
7739 .or_else(|| payload.get("output"))
7740 .unwrap_or(&Value::Null);
7741 outcomes.push(Ok(ChildWorkflowAvroResult {
7742 parent: parent.clone(),
7743 child: WorkflowIdentity {
7744 workflow_id: child_workflow_id,
7745 run_id: child_workflow_run_id,
7746 },
7747 child_workflow_type,
7748 result: decode_wire_avro_value(result, codec)?,
7749 }));
7750 }
7751
7752 Ok(outcomes)
7753}
7754
7755fn payload_string(payload: &Value, key: &str) -> Option<String> {
7756 payload
7757 .get(key)
7758 .and_then(Value::as_str)
7759 .filter(|value| !value.is_empty())
7760 .map(str::to_string)
7761}
7762
7763fn workflow_failure_command(error: &Error) -> Value {
7764 let (exception_type, exception_class, properties) = match error {
7765 Error::ActivityFailed(failure) => (
7766 match failure.kind {
7767 ActivityFailureKind::Failed => "ActivityFailed",
7768 ActivityFailureKind::Cancelled => "ActivityCancelled",
7769 ActivityFailureKind::TimedOut => "ActivityTimedOut",
7770 },
7771 "durable_workflow::ActivityFailure",
7772 json!({
7773 "reason": failure.reason,
7774 "activity_execution_id": failure.activity_execution_id,
7775 "activity_attempt_id": failure.activity_attempt_id,
7776 "activity_type": failure.activity_type,
7777 "activity_class": failure.activity_class,
7778 "attempt_number": failure.attempt_number,
7779 "failure_id": failure.failure_id,
7780 "failure_category": failure.failure_category,
7781 "timeout_kind": failure.timeout_kind,
7782 "activity_non_retryable": failure.non_retryable,
7783 "activity_exception_type": failure.exception_type,
7784 "activity_exception_class": failure.exception_class,
7785 "activity_code": failure.code,
7786 "activity_exception": failure.exception,
7787 }),
7788 ),
7789 Error::ChildWorkflowFailed(failure) => (
7790 match failure.kind {
7791 ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
7792 ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
7793 ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
7794 },
7795 "durable_workflow::ChildWorkflowFailure",
7796 json!({
7797 "reason": failure.reason,
7798 "parent_workflow_id": failure.parent_workflow_id,
7799 "parent_workflow_run_id": failure.parent_workflow_run_id,
7800 "child_workflow_id": failure.child_workflow_id,
7801 "child_workflow_run_id": failure.child_workflow_run_id,
7802 "child_workflow_type": failure.child_workflow_type,
7803 "failure_id": failure.failure_id,
7804 "failure_category": failure.failure_category,
7805 "child_exception_type": failure.exception_type,
7806 "child_exception_class": failure.exception_class,
7807 "child_non_retryable": failure.non_retryable,
7808 "child_code": failure.code,
7809 "child_exception": failure.exception,
7810 }),
7811 ),
7812 Error::NonDeterministicReplay(_) => (
7813 "NonDeterministicReplay",
7814 "durable_workflow::Error",
7815 Value::Null,
7816 ),
7817 _ => ("RustWorkflowError", "durable_workflow::Error", Value::Null),
7818 };
7819 let non_retryable = match error {
7820 Error::ActivityFailed(failure) => failure.non_retryable,
7821 Error::ChildWorkflowFailed(failure) => failure.non_retryable,
7822 Error::NonDeterministicReplay(_) => true,
7823 _ => false,
7824 };
7825
7826 json!({
7827 "type": "fail_workflow",
7828 "message": error.to_string(),
7829 "exception_type": exception_type,
7830 "exception_class": exception_class,
7831 "non_retryable": non_retryable,
7832 "exception": {
7833 "type": exception_type,
7834 "class": exception_class,
7835 "message": error.to_string(),
7836 "properties": properties,
7837 }
7838 })
7839}
7840
7841fn workflow_task_integrity_error(error: &Error) -> bool {
7842 matches!(
7843 error,
7844 Error::NonDeterministicReplay(_) | Error::Protocol(_) | Error::WorkflowStatePoisoned
7845 )
7846}
7847
7848fn decode_signal_event_arguments(
7849 event: &HistoryEvent,
7850 fallback_codec: &str,
7851) -> Result<Vec<AvroValue>> {
7852 let codec = event
7853 .payload
7854 .get("payload_codec")
7855 .and_then(Value::as_str)
7856 .unwrap_or(fallback_codec);
7857 let raw = event
7858 .payload
7859 .get("value")
7860 .or_else(|| event.payload.get("input"))
7861 .or_else(|| event.payload.get("arguments"));
7862 let decoded = match raw.filter(|value| !value.is_null()) {
7863 Some(value) => decode_wire_avro_value(value, codec)?,
7864 None => AvroValue::Array(Vec::new()),
7865 };
7866 let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
7867 unreachable!("normalize_avro_arguments always returns an array");
7868 };
7869 Ok(arguments)
7870}
7871
7872fn hydrate_query_history_from_export(task: &mut QueryTask) -> Result<()> {
7873 let Some(export_events) = task
7874 .history_export
7875 .as_ref()
7876 .and_then(|export| export.get("history_events"))
7877 .and_then(Value::as_array)
7878 else {
7879 return Ok(());
7880 };
7881
7882 if export_events.len() > task.history_events.len() {
7883 task.history_events = serde_json::from_value(Value::Array(export_events.clone()))?;
7884 }
7885
7886 Ok(())
7887}
7888
7889fn enrich_query_history_from_export(task: &mut QueryTask) -> Result<()> {
7890 let Some(export) = task.history_export.as_ref() else {
7891 return Ok(());
7892 };
7893 let signals = export
7894 .get("signals")
7895 .and_then(Value::as_array)
7896 .cloned()
7897 .unwrap_or_default();
7898 let activities = export
7899 .get("activities")
7900 .and_then(Value::as_array)
7901 .cloned()
7902 .unwrap_or_default();
7903 let export_codec = export
7904 .get("payloads")
7905 .and_then(|payloads| payloads.get("codec"))
7906 .and_then(Value::as_str)
7907 .unwrap_or(&task.payload_codec)
7908 .to_string();
7909 let mut signal_name_offsets: HashMap<String, usize> = HashMap::new();
7910
7911 for event in &mut task.history_events {
7912 if event.event_type == "ActivityCompleted" {
7913 let sequence = event
7914 .payload
7915 .get("sequence")
7916 .or_else(|| event.payload.get("workflow_sequence"))
7917 .and_then(value_as_u64);
7918 let Some(activity) = sequence.and_then(|sequence| {
7919 activities.iter().find(|activity| {
7920 activity.get("sequence").and_then(value_as_u64) == Some(sequence)
7921 })
7922 }) else {
7923 continue;
7924 };
7925 let Some(payload) = event.payload.as_object_mut() else {
7926 continue;
7927 };
7928 if missing_payload(payload.get("result")) {
7929 if let Some(result) = activity
7930 .get("result")
7931 .filter(|value| !missing_payload(Some(value)))
7932 {
7933 payload.insert("result".to_string(), result.clone());
7934 }
7935 }
7936 for field in ["payload_codec", "activity_type"] {
7937 if payload
7938 .get(field)
7939 .and_then(Value::as_str)
7940 .unwrap_or_default()
7941 .is_empty()
7942 {
7943 if let Some(value) = activity.get(field) {
7944 payload.insert(field.to_string(), value.clone());
7945 }
7946 }
7947 }
7948 continue;
7949 }
7950
7951 if event.event_type != "SignalReceived" && event.event_type != "SignalApplied" {
7952 continue;
7953 }
7954 let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
7955 let command_id = event
7956 .payload
7957 .get("workflow_command_id")
7958 .or_else(|| event.raw.get("workflow_command_id"))
7959 .and_then(Value::as_str);
7960 let signal_name = event
7961 .payload
7962 .get("signal_name")
7963 .and_then(Value::as_str)
7964 .unwrap_or_default()
7965 .to_string();
7966 let matched = signals
7967 .iter()
7968 .find(|signal| {
7969 signal_id.is_some() && signal.get("id").and_then(Value::as_str) == signal_id
7970 })
7971 .or_else(|| {
7972 signals.iter().find(|signal| {
7973 command_id.is_some()
7974 && signal.get("command_id").and_then(Value::as_str) == command_id
7975 })
7976 })
7977 .or_else(|| {
7978 let offset = signal_name_offsets.entry(signal_name.clone()).or_default();
7979 let signal = signals
7980 .iter()
7981 .filter(|signal| {
7982 signal.get("name").and_then(Value::as_str) == Some(signal_name.as_str())
7983 })
7984 .nth(*offset);
7985 if signal.is_some() {
7986 *offset += 1;
7987 }
7988 signal
7989 });
7990 let Some(signal) = matched else {
7991 continue;
7992 };
7993 let signal_codec = signal
7994 .get("payload_codec")
7995 .and_then(Value::as_str)
7996 .unwrap_or(&export_codec);
7997 let Some(payload) = event.payload.as_object_mut() else {
7998 continue;
7999 };
8000 if missing_payload(payload.get("arguments")) {
8001 if let Some(arguments) = signal
8002 .get("arguments")
8003 .filter(|value| !missing_payload(Some(value)))
8004 {
8005 let envelope = match arguments {
8006 Value::String(blob) => json!({"codec": signal_codec, "blob": blob}),
8007 other => other.clone(),
8008 };
8009 payload.insert("arguments".to_string(), envelope);
8010 }
8011 }
8012 if payload
8013 .get("payload_codec")
8014 .and_then(Value::as_str)
8015 .unwrap_or_default()
8016 .is_empty()
8017 {
8018 payload.insert("payload_codec".to_string(), json!(signal_codec));
8019 }
8020 }
8021
8022 Ok(())
8023}
8024
8025fn missing_payload(value: Option<&Value>) -> bool {
8026 match value {
8027 None | Some(Value::Null) => true,
8028 Some(Value::String(value)) => value.is_empty(),
8029 Some(_) => false,
8030 }
8031}
8032
8033fn query_signal_events(task: &QueryTask) -> Result<Vec<QuerySignal>> {
8034 let export_signals = task
8035 .history_export
8036 .as_ref()
8037 .and_then(|export| export.get("signals"))
8038 .and_then(Value::as_array)
8039 .cloned()
8040 .unwrap_or_default();
8041 let export_codec = task
8042 .history_export
8043 .as_ref()
8044 .and_then(|export| export.get("payloads"))
8045 .and_then(|payloads| payloads.get("codec"))
8046 .and_then(Value::as_str)
8047 .unwrap_or(&task.payload_codec);
8048 let mut name_offsets: HashMap<String, usize> = HashMap::new();
8049 let mut signals = Vec::new();
8050
8051 for event in &task.history_events {
8052 if event.event_type != "SignalApplied" && event.event_type != "SignalReceived" {
8053 continue;
8054 }
8055
8056 let name = event
8057 .payload
8058 .get("signal_name")
8059 .and_then(Value::as_str)
8060 .unwrap_or_default();
8061 if name.is_empty() {
8062 continue;
8063 }
8064 let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
8065 let command_id = event
8066 .payload
8067 .get("workflow_command_id")
8068 .or_else(|| event.raw.get("workflow_command_id"))
8069 .and_then(Value::as_str);
8070 let matched_export = export_signals
8071 .iter()
8072 .find(|candidate| {
8073 signal_id.is_some() && candidate.get("id").and_then(Value::as_str) == signal_id
8074 })
8075 .or_else(|| {
8076 export_signals.iter().find(|candidate| {
8077 command_id.is_some()
8078 && candidate.get("command_id").and_then(Value::as_str) == command_id
8079 })
8080 })
8081 .or_else(|| {
8082 let offset = name_offsets.entry(name.to_string()).or_default();
8083 let candidate = export_signals
8084 .iter()
8085 .filter(|candidate| candidate.get("name").and_then(Value::as_str) == Some(name))
8086 .nth(*offset);
8087 if candidate.is_some() {
8088 *offset += 1;
8089 }
8090 candidate
8091 });
8092 let codec = event
8093 .payload
8094 .get("payload_codec")
8095 .and_then(Value::as_str)
8096 .or_else(|| {
8097 matched_export
8098 .and_then(|signal| signal.get("payload_codec"))
8099 .and_then(Value::as_str)
8100 })
8101 .unwrap_or(export_codec);
8102 let raw_arguments = event
8103 .payload
8104 .get("value")
8105 .or_else(|| event.payload.get("input"))
8106 .or_else(|| event.payload.get("arguments"))
8107 .filter(|value| !value.is_null())
8108 .or_else(|| matched_export.and_then(|signal| signal.get("arguments")));
8109 let (arguments, avro_arguments) = decode_query_signal_arguments(raw_arguments, codec)?;
8110 let workflow_sequence = event
8111 .payload
8112 .get("workflow_sequence")
8113 .and_then(value_as_u64)
8114 .or_else(|| {
8115 matched_export
8116 .and_then(|signal| signal.get("workflow_sequence"))
8117 .and_then(value_as_u64)
8118 });
8119
8120 signals.push(QuerySignal {
8121 id: signal_id.map(str::to_string).or_else(|| {
8122 matched_export
8123 .and_then(|signal| signal.get("id"))
8124 .and_then(Value::as_str)
8125 .map(str::to_string)
8126 }),
8127 name: name.to_string(),
8128 arguments,
8129 avro_arguments,
8130 workflow_sequence,
8131 });
8132 }
8133
8134 if signals.is_empty() {
8135 for signal in export_signals {
8136 if signal.get("status").and_then(Value::as_str) == Some("rejected") {
8137 continue;
8138 }
8139 let Some(name) = signal.get("name").and_then(Value::as_str) else {
8140 continue;
8141 };
8142 let codec = signal
8143 .get("payload_codec")
8144 .and_then(Value::as_str)
8145 .unwrap_or(export_codec);
8146 let (arguments, avro_arguments) =
8147 decode_query_signal_arguments(signal.get("arguments"), codec)?;
8148 signals.push(QuerySignal {
8149 id: signal.get("id").and_then(Value::as_str).map(str::to_string),
8150 name: name.to_string(),
8151 arguments,
8152 avro_arguments,
8153 workflow_sequence: signal.get("workflow_sequence").and_then(value_as_u64),
8154 });
8155 }
8156 signals.sort_by_key(|signal| signal.workflow_sequence.unwrap_or(u64::MAX));
8157 }
8158
8159 Ok(signals)
8160}
8161
8162fn decode_query_signal_arguments(
8163 raw: Option<&Value>,
8164 codec: &str,
8165) -> Result<(Vec<Value>, Vec<AvroValue>)> {
8166 let decoded = match raw.filter(|value| !value.is_null()) {
8167 Some(value) => decode_wire_avro_value(value, codec)?,
8168 None => AvroValue::Array(Vec::new()),
8169 };
8170 let AvroValue::Array(avro_arguments) = normalize_avro_arguments(decoded) else {
8171 unreachable!("normalize_avro_arguments always returns an array");
8172 };
8173 let arguments = avro_arguments
8174 .iter()
8175 .cloned()
8176 .map(AvroValue::into_json)
8177 .collect::<Result<Vec<_>>>()?;
8178 Ok((arguments, avro_arguments))
8179}
8180
8181fn value_as_u64(value: &Value) -> Option<u64> {
8182 value
8183 .as_u64()
8184 .or_else(|| value.as_str().and_then(|value| value.parse().ok()))
8185}
8186
8187#[cfg(test)]
8188mod tests {
8189 use super::*;
8190 use std::{
8191 io::{Read, Write},
8192 net::{SocketAddr, TcpListener, TcpStream},
8193 sync::atomic::AtomicUsize,
8194 thread,
8195 };
8196
8197 fn typed_fidelity_probe() -> AvroValue {
8198 AvroValue::Map(BTreeMap::from([
8199 ("bytes".to_string(), AvroValue::Bytes(vec![0, 0xff])),
8200 ("empty".to_string(), AvroValue::Map(BTreeMap::new())),
8201 (
8202 "numeric".to_string(),
8203 AvroValue::Map(BTreeMap::from([
8204 ("0".to_string(), AvroValue::String("zero".to_string())),
8205 ("1".to_string(), AvroValue::String("one".to_string())),
8206 ])),
8207 ),
8208 (
8209 "nested".to_string(),
8210 AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([(
8211 "enabled".to_string(),
8212 AvroValue::Boolean(true),
8213 )]))]),
8214 ),
8215 (
8216 "projection_collisions".to_string(),
8217 AvroValue::Array(projection_collision_probe()),
8218 ),
8219 ]))
8220 }
8221
8222 fn projection_collision_probe() -> Vec<AvroValue> {
8223 vec![
8224 AvroValue::Map(BTreeMap::from([
8225 ("$type".to_string(), AvroValue::String("bytes".to_string())),
8226 (
8227 "base64".to_string(),
8228 AvroValue::String("ordinary user text".to_string()),
8229 ),
8230 ])),
8231 AvroValue::Map(BTreeMap::from([
8232 ("$type".to_string(), AvroValue::String("map".to_string())),
8233 (
8234 "entries".to_string(),
8235 AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([
8236 ("key".to_string(), AvroValue::String("ordinary".to_string())),
8237 (
8238 "value".to_string(),
8239 AvroValue::String("user map".to_string()),
8240 ),
8241 ]))]),
8242 ),
8243 ])),
8244 ]
8245 }
8246
8247 #[derive(Clone, Debug, Default, PartialEq)]
8248 struct ReplayCounterState {
8249 loaded: Option<String>,
8250 count: i64,
8251 finished: bool,
8252 }
8253
8254 fn replay_counter_worker() -> Worker {
8255 let client = Client::new("http://127.0.0.1:8080").expect("client");
8256 let mut worker = Worker::new(client, "rust-workers");
8257 worker.register_replayed_workflow(
8258 "replay-counter",
8259 ReplayCounterState::default,
8260 |ctx, _input, state| async move {
8261 let loaded = ctx.activity("load-counter", json!([])).await?;
8262 state.update(|current| {
8263 current.loaded = loaded.as_str().map(str::to_string);
8264 })?;
8265 for _ in 0..2 {
8266 let signal = ctx.wait_signal("increment").await?;
8267 let amount = signal.first().and_then(Value::as_i64).unwrap_or_default();
8268 state.update(|current| current.count += amount)?;
8269 }
8270 state.update(|current| current.finished = true)?;
8271 state.read(|current| Ok(json!(current.count)))?
8272 },
8273 );
8274 worker.register_replayed_query::<ReplayCounterState, _, _>(
8275 "replay-counter",
8276 "current",
8277 |_ctx, state, _args| async move {
8278 Ok(json!({
8279 "loaded": state.loaded,
8280 "count": state.count,
8281 "finished": state.finished,
8282 }))
8283 },
8284 );
8285 worker.register_replayed_query::<ReplayCounterState, _, _>(
8286 "replay-counter",
8287 "detached-mutation",
8288 |_ctx, state, _args| async move {
8289 let mut detached = (*state).clone();
8290 detached.count = 999;
8291 Ok(json!(detached.count))
8292 },
8293 );
8294 worker.register_replayed_query::<ReplayCounterState, _, _>(
8295 "replay-counter",
8296 "failed-mutation",
8297 |_ctx, state, _args| async move {
8298 let mut detached = (*state).clone();
8299 detached.count = 999;
8300 Err(Error::WorkerLoop("query refused".to_string()))
8301 },
8302 );
8303 worker
8304 }
8305
8306 fn replay_counter_query(
8307 query_name: &str,
8308 history_events: Value,
8309 run_status: &str,
8310 ) -> QueryTask {
8311 serde_json::from_value(json!({
8312 "query_task_id": format!("query-{query_name}"),
8313 "workflow_type": "replay-counter",
8314 "query_name": query_name,
8315 "payload_codec": "json",
8316 "workflow_arguments": {"codec": "json", "blob": "[]"},
8317 "query_arguments": {"codec": "json", "blob": "[]"},
8318 "history_events": history_events,
8319 "run_status": run_status,
8320 }))
8321 .expect("query task")
8322 }
8323
8324 fn workflow_context(history: Vec<HistoryEvent>) -> WorkflowContext {
8325 workflow_context_with_codec(history, JSON_CODEC)
8326 }
8327
8328 fn workflow_context_with_codec(
8329 history: Vec<HistoryEvent>,
8330 payload_codec: &str,
8331 ) -> WorkflowContext {
8332 WorkflowContext {
8333 state: Arc::new(Mutex::new(
8334 WorkflowState::new_with_identity(
8335 history,
8336 None,
8337 None,
8338 "rust-workers".to_string(),
8339 payload_codec.to_string(),
8340 None,
8341 )
8342 .expect("valid workflow history"),
8343 )),
8344 }
8345 }
8346
8347 fn history_event(event_type: &str, payload: Value) -> HistoryEvent {
8348 HistoryEvent {
8349 event_type: event_type.to_string(),
8350 payload,
8351 raw: HashMap::new(),
8352 }
8353 }
8354
8355 fn workflow_task(
8356 workflow_type: &str,
8357 history_events: Vec<HistoryEvent>,
8358 payload_codec: &str,
8359 ) -> WorkflowTask {
8360 WorkflowTask {
8361 task_id: format!("wft-{workflow_type}"),
8362 workflow_id: Some(format!("wf-{workflow_type}")),
8363 run_id: Some(format!("run-{workflow_type}")),
8364 workflow_type: workflow_type.to_string(),
8365 payload_codec: payload_codec.to_string(),
8366 arguments: Some(
8367 encode_value_envelope(&json!([]), payload_codec).expect("workflow arguments"),
8368 ),
8369 total_history_events: Some(history_events.len() as u64),
8370 history_size_bytes: None,
8371 continue_as_new_recommended: None,
8372 history_budget_pressure: None,
8373 history_events,
8374 next_history_page_token: None,
8375 workflow_task_attempt: 1,
8376 workflow_signal_id: None,
8377 signal_name: None,
8378 signal_arguments: None,
8379 workflow_update_id: None,
8380 update_name: None,
8381 lease_owner: Some("rust-worker".to_string()),
8382 }
8383 }
8384
8385 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
8386 struct SideEffectProbe {
8387 request_id: String,
8388 attempt: u32,
8389 }
8390
8391 #[test]
8392 fn typed_side_effect_runs_callback_once_and_replay_skips_it() {
8393 let calls = AtomicUsize::new(0);
8394 let ctx = workflow_context(Vec::new());
8395 let value = ctx
8396 .side_effect(|| {
8397 calls.fetch_add(1, Ordering::SeqCst);
8398 SideEffectProbe {
8399 request_id: "request-42".to_string(),
8400 attempt: 3,
8401 }
8402 })
8403 .expect("first side effect");
8404 assert_eq!(value.attempt, 3);
8405 assert_eq!(calls.load(Ordering::SeqCst), 1);
8406 let commands = ctx.take_commands().expect("commands");
8407 assert_eq!(commands.len(), 1);
8408 assert_eq!(commands[0]["type"], "record_side_effect");
8409 assert_eq!(
8410 decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("JSON result"),
8411 serde_json::to_value(&value).expect("value")
8412 );
8413
8414 let replay = workflow_context(vec![history_event(
8415 "SideEffectRecorded",
8416 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
8417 )]);
8418 let replayed: SideEffectProbe = replay
8419 .side_effect(|| {
8420 calls.fetch_add(1, Ordering::SeqCst);
8421 panic!("committed side-effect callbacks must not run during replay")
8422 })
8423 .expect("replayed side effect");
8424 assert_eq!(replayed, value);
8425 assert_eq!(calls.load(Ordering::SeqCst), 1);
8426 assert!(replay.take_commands().expect("commands").is_empty());
8427 replay.ensure_history_consumed().expect("history consumed");
8428 }
8429
8430 #[test]
8431 fn side_effect_uses_avro_envelope_and_uuid_is_replay_stable() {
8432 let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
8433 let value = ctx
8434 .side_effect(|| SideEffectProbe {
8435 request_id: "avro-request".to_string(),
8436 attempt: 1,
8437 })
8438 .expect("Avro side effect");
8439 let uuid = ctx.uuid_v4().expect("deterministic UUID");
8440 let commands = ctx.take_commands().expect("commands");
8441 assert_eq!(commands.len(), 2);
8442 assert_eq!(commands[0]["result"]["codec"], DEFAULT_CODEC);
8443 assert_eq!(commands[1]["result"]["codec"], DEFAULT_CODEC);
8444 assert_eq!(
8445 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
8446 serde_json::to_value(&value).expect("value")
8447 );
8448
8449 let replay = workflow_context_with_codec(
8450 vec![
8451 history_event(
8452 "SideEffectRecorded",
8453 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
8454 ),
8455 history_event(
8456 "SideEffectRecorded",
8457 json!({"sequence": 2, "result": commands[1]["result"].clone()}),
8458 ),
8459 ],
8460 DEFAULT_CODEC,
8461 );
8462 let replayed: SideEffectProbe = replay
8463 .side_effect(|| panic!("Avro callback must not run"))
8464 .expect("replayed Avro value");
8465 let replayed_uuid = replay.uuid_v4().expect("replayed UUID");
8466 assert_eq!(replayed, value);
8467 assert_eq!(replayed_uuid, uuid);
8468 assert!(replay.take_commands().expect("commands").is_empty());
8469 }
8470
8471 #[test]
8472 fn typed_side_effect_replay_preserves_bytes_and_maps() {
8473 let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
8474 let value = ctx
8475 .side_effect_avro_value(typed_fidelity_probe)
8476 .expect("typed side effect");
8477 let commands = ctx.take_commands().expect("side-effect command");
8478 assert_eq!(
8479 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
8480 .expect("recorded side effect"),
8481 value
8482 );
8483
8484 let replay = workflow_context_with_codec(
8485 vec![history_event(
8486 "SideEffectRecorded",
8487 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
8488 )],
8489 DEFAULT_CODEC,
8490 );
8491 assert_eq!(
8492 replay
8493 .side_effect_avro_value(|| panic!("replay must not invoke callback"))
8494 .expect("replayed typed side effect"),
8495 value
8496 );
8497 }
8498
8499 #[test]
8500 fn ordered_side_effects_share_the_durable_command_stream() {
8501 let first = encode_value_envelope(&json!("first"), JSON_CODEC).expect("first");
8502 let second = encode_value_envelope(&json!(29), JSON_CODEC).expect("second");
8503 let ctx = workflow_context(vec![
8504 history_event(
8505 "SideEffectRecorded",
8506 json!({"sequence": 1, "result": first}),
8507 ),
8508 history_event(
8509 "SideEffectRecorded",
8510 json!({"sequence": 2, "result": second}),
8511 ),
8512 ]);
8513 let first: String = ctx
8514 .side_effect(|| panic!("first callback must not run"))
8515 .expect("first replay");
8516 let second: i32 = ctx
8517 .side_effect(|| panic!("second callback must not run"))
8518 .expect("second replay");
8519 assert_eq!(first, "first");
8520 assert_eq!(second, 29);
8521 ctx.ensure_history_consumed().expect("ordered history");
8522
8523 let reordered = workflow_context(vec![history_event(
8524 "VersionMarkerRecorded",
8525 json!({
8526 "sequence": 1,
8527 "change_id": "before-side-effect",
8528 "version": 1,
8529 "min_supported": 1,
8530 "max_supported": 1,
8531 }),
8532 )]);
8533 let error = reordered
8534 .side_effect(|| "new".to_string())
8535 .expect_err("command reordering must fail");
8536 assert!(matches!(
8537 error,
8538 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8539 if reason == "recorded_command_mismatch"
8540 ));
8541 }
8542
8543 #[test]
8544 fn version_markers_replay_across_upgrades_and_do_not_duplicate() {
8545 let ctx = workflow_context(Vec::new());
8546 assert_eq!(ctx.get_version("checkout-v2", 1, 2).expect("version"), 2);
8547 assert_eq!(ctx.get_version("checkout-v2", 1, 3).expect("cached"), 2);
8548 assert!(ctx.patched("new-search").expect("patch"));
8549 ctx.deprecate_patch("new-search").expect("deprecate patch");
8550 let commands = ctx.take_commands().expect("commands");
8551 assert_eq!(commands.len(), 2);
8552 assert_eq!(commands[0]["type"], "record_version_marker");
8553 assert_eq!(commands[0]["version"], 2);
8554 assert_eq!(commands[1]["change_id"], "new-search");
8555
8556 let replay = workflow_context(vec![history_event(
8557 "VersionMarkerRecorded",
8558 json!({
8559 "sequence": 1,
8560 "change_id": "checkout-v2",
8561 "version": 2,
8562 "min_supported": 1,
8563 "max_supported": 2,
8564 }),
8565 )]);
8566 assert_eq!(replay.get_version("checkout-v2", 1, 4).expect("upgrade"), 2);
8567 assert_eq!(replay.get_version("checkout-v2", 2, 5).expect("repeat"), 2);
8568 assert!(replay.take_commands().expect("commands").is_empty());
8569 replay.ensure_history_consumed().expect("history consumed");
8570 }
8571
8572 #[test]
8573 fn version_markers_reject_incompatible_or_malformed_history() {
8574 let incompatible = workflow_context(vec![history_event(
8575 "VersionMarkerRecorded",
8576 json!({
8577 "sequence": 1,
8578 "change_id": "checkout-v2",
8579 "version": 1,
8580 "min_supported": 1,
8581 "max_supported": 2,
8582 }),
8583 )]);
8584 let error = incompatible
8585 .get_version("checkout-v2", 2, 3)
8586 .expect_err("old version is unsupported");
8587 assert!(matches!(
8588 error,
8589 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8590 if reason == "version_marker_incompatible_range"
8591 ));
8592
8593 for (history, reason) in [
8594 (
8595 vec![history_event("SideEffectRecorded", json!({"sequence": 1}))],
8596 "side_effect_result_missing",
8597 ),
8598 (
8599 vec![history_event(
8600 "SideEffectRecorded",
8601 json!({
8602 "sequence": 1,
8603 "result": {"codec": "avro", "blob": "not-base64"},
8604 }),
8605 )],
8606 "side_effect_payload_incompatible",
8607 ),
8608 (
8609 vec![history_event(
8610 "SideEffectRecorded",
8611 json!({"sequence": 1, "result": {"unwrapped": true}}),
8612 )],
8613 "side_effect_payload_malformed",
8614 ),
8615 (
8616 vec![history_event(
8617 "VersionMarkerRecorded",
8618 json!({
8619 "sequence": 1,
8620 "change_id": "change",
8621 "version": 1,
8622 "min_supported": 2,
8623 "max_supported": 1,
8624 }),
8625 )],
8626 "version_marker_history_range_invalid",
8627 ),
8628 ] {
8629 let error = WorkflowState::new(
8630 history,
8631 "rust-workers".to_string(),
8632 JSON_CODEC.to_string(),
8633 None,
8634 )
8635 .expect_err("malformed history must fail");
8636 assert!(matches!(
8637 error,
8638 Error::NonDeterministicReplay(ReplayFailure { reason: actual, .. })
8639 if actual == reason
8640 ));
8641 }
8642 }
8643
8644 #[test]
8645 fn duplicate_side_effects_and_version_markers_are_rejected() {
8646 let duplicate_side_effect = WorkflowState::new(
8647 vec![
8648 history_event(
8649 "SideEffectRecorded",
8650 json!({"sequence": 1, "result": {"codec": "json", "blob": "1"}}),
8651 ),
8652 history_event(
8653 "SideEffectRecorded",
8654 json!({"sequence": 1, "result": {"codec": "json", "blob": "2"}}),
8655 ),
8656 ],
8657 "rust-workers".to_string(),
8658 JSON_CODEC.to_string(),
8659 None,
8660 )
8661 .expect_err("duplicate side effect");
8662 assert!(matches!(
8663 duplicate_side_effect,
8664 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8665 if reason == "duplicate_side_effect_record"
8666 ));
8667
8668 let marker = |sequence| {
8669 history_event(
8670 "VersionMarkerRecorded",
8671 json!({
8672 "sequence": sequence,
8673 "change_id": "same-change",
8674 "version": 1,
8675 "min_supported": 1,
8676 "max_supported": 1,
8677 }),
8678 )
8679 };
8680 let duplicate_marker = WorkflowState::new(
8681 vec![marker(1), marker(3)],
8682 "rust-workers".to_string(),
8683 JSON_CODEC.to_string(),
8684 None,
8685 )
8686 .expect_err("duplicate marker");
8687 assert!(matches!(
8688 duplicate_marker,
8689 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8690 if reason == "duplicate_version_marker"
8691 ));
8692 }
8693
8694 #[test]
8695 fn cold_worker_replay_does_not_repeat_committed_side_effects_or_markers() {
8696 fn worker(calls: Arc<AtomicUsize>) -> Worker {
8697 let client = Client::new("http://127.0.0.1:8080").expect("client");
8698 let mut worker = Worker::new(client, "rust-workers");
8699 worker.register_workflow("rust.side-effect-version", move |ctx, _input| {
8700 let calls = Arc::clone(&calls);
8701 async move {
8702 let captured = ctx.side_effect(|| {
8703 calls.fetch_add(1, Ordering::SeqCst);
8704 "captured-once".to_string()
8705 })?;
8706 let version = ctx.get_version("cold-restart", 1, 2)?;
8707 Ok(json!({"captured": captured, "version": version}))
8708 }
8709 });
8710 worker
8711 }
8712
8713 fn task(history_events: Vec<HistoryEvent>) -> WorkflowTask {
8714 WorkflowTask {
8715 task_id: "wft-side-effect-version".to_string(),
8716 workflow_id: Some("wf-side-effect-version".to_string()),
8717 run_id: Some("run-side-effect-version".to_string()),
8718 workflow_type: "rust.side-effect-version".to_string(),
8719 payload_codec: JSON_CODEC.to_string(),
8720 arguments: Some(encode_value_envelope(&json!([]), JSON_CODEC).expect("arguments")),
8721 history_events,
8722 total_history_events: None,
8723 history_size_bytes: None,
8724 continue_as_new_recommended: None,
8725 history_budget_pressure: None,
8726 next_history_page_token: None,
8727 workflow_task_attempt: 1,
8728 workflow_signal_id: None,
8729 signal_name: None,
8730 signal_arguments: None,
8731 workflow_update_id: None,
8732 update_name: None,
8733 lease_owner: Some("rust-worker".to_string()),
8734 }
8735 }
8736
8737 let calls = Arc::new(AtomicUsize::new(0));
8738 let initial = worker(Arc::clone(&calls))
8739 .execute_workflow_task(task(Vec::new()))
8740 .expect("initial execution");
8741 assert_eq!(
8742 initial
8743 .iter()
8744 .map(|command| &command["type"])
8745 .collect::<Vec<_>>(),
8746 vec![
8747 "record_side_effect",
8748 "record_version_marker",
8749 "complete_workflow"
8750 ]
8751 );
8752 assert_eq!(calls.load(Ordering::SeqCst), 1);
8753
8754 let restarted = worker(Arc::clone(&calls));
8755 let replayed = restarted
8756 .execute_workflow_task(task(vec![
8757 history_event(
8758 "SideEffectRecorded",
8759 json!({"sequence": 1, "result": initial[0]["result"].clone()}),
8760 ),
8761 history_event(
8762 "VersionMarkerRecorded",
8763 json!({
8764 "sequence": 2,
8765 "change_id": "cold-restart",
8766 "version": 2,
8767 "min_supported": 1,
8768 "max_supported": 2,
8769 }),
8770 ),
8771 ]))
8772 .expect("cold replay");
8773 assert_eq!(replayed.len(), 1);
8774 assert_eq!(replayed[0]["type"], "complete_workflow");
8775 assert_eq!(calls.load(Ordering::SeqCst), 1);
8776 }
8777
8778 #[test]
8779 fn side_effect_replay_rejects_changed_rust_value_type() {
8780 let result = encode_value_envelope(&json!({"value": 42}), JSON_CODEC).expect("result");
8781 let ctx = workflow_context(vec![history_event(
8782 "SideEffectRecorded",
8783 json!({"sequence": 1, "result": result}),
8784 )]);
8785 let error = ctx
8786 .side_effect::<Vec<String>, _>(|| panic!("callback must not run"))
8787 .expect_err("changed type must fail replay");
8788 assert!(matches!(
8789 error,
8790 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8791 if reason == "side_effect_type_mismatch"
8792 ));
8793 }
8794
8795 fn completed_retry_activity_history() -> Vec<HistoryEvent> {
8796 vec![
8797 history_event(
8798 "ActivityScheduled",
8799 json!({
8800 "sequence": 1,
8801 "activity_type": "flaky",
8802 "activity_execution_id": "act-1",
8803 "activity": {
8804 "id": "act-1",
8805 "sequence": 1,
8806 "type": "flaky",
8807 "queue": "critical-activities",
8808 "execution_mode": null,
8809 "retry_policy": {
8810 "snapshot_version": 1,
8811 "max_attempts": 3,
8812 "backoff_seconds": [2, 4],
8813 "start_to_close_timeout": 30,
8814 "schedule_to_start_timeout": 5,
8815 "schedule_to_close_timeout": 90,
8816 "heartbeat_timeout": 10,
8817 "non_retryable_error_types": ["PermanentError"]
8818 }
8819 }
8820 }),
8821 ),
8822 history_event(
8823 "ActivityStarted",
8824 json!({
8825 "sequence": 1,
8826 "activity_type": "flaky",
8827 "activity_execution_id": "act-1",
8828 "activity_attempt_id": "attempt-1",
8829 "attempt_number": 1
8830 }),
8831 ),
8832 history_event(
8833 "ActivityRetryScheduled",
8834 json!({
8835 "sequence": 1,
8836 "activity_type": "flaky",
8837 "activity_execution_id": "act-1",
8838 "activity_attempt_id": "attempt-1",
8839 "attempt_number": 1,
8840 "retry_after_attempt": 1,
8841 "retry_backoff_seconds": 2,
8842 "failure_category": "activity",
8843 "exception_type": "TransientError"
8844 }),
8845 ),
8846 history_event(
8847 "ActivityStarted",
8848 json!({
8849 "sequence": 1,
8850 "activity_type": "flaky",
8851 "activity_execution_id": "act-1",
8852 "activity_attempt_id": "attempt-2",
8853 "attempt_number": 2
8854 }),
8855 ),
8856 history_event(
8857 "ActivityCompleted",
8858 json!({
8859 "sequence": 1,
8860 "activity_type": "flaky",
8861 "activity_execution_id": "act-1",
8862 "activity_attempt_id": "attempt-2",
8863 "attempt_number": 2,
8864 "payload_codec": "json",
8865 "result": {"codec": "json", "blob": "{\"status\":\"recovered\"}"}
8866 }),
8867 ),
8868 ]
8869 }
8870
8871 fn retry_activity_options() -> ActivityOptions {
8872 ActivityOptions::new()
8873 .task_queue("critical-activities")
8874 .retry_policy(
8875 ActivityRetryPolicy::new(3)
8876 .backoff_intervals([Duration::from_secs(2), Duration::from_secs(4)])
8877 .non_retryable_error_type("PermanentError"),
8878 )
8879 .start_to_close_timeout(Duration::from_secs(30))
8880 .schedule_to_start_timeout(Duration::from_secs(5))
8881 .schedule_to_close_timeout(Duration::from_secs(90))
8882 .heartbeat_timeout(Duration::from_secs(10))
8883 }
8884
8885 #[test]
8886 fn fixed_avro_value_round_trips_json_values() {
8887 let value = json!({"greeting": "hello", "count": 3, "ok": true});
8888 let envelope = PayloadEnvelope::avro(&value).expect("encode");
8889 assert_eq!(envelope.codec, DEFAULT_CODEC);
8890 assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
8891 }
8892
8893 #[tokio::test]
8894 async fn typed_worker_surfaces_preserve_bytes_and_map_list_identity() {
8895 let client = Client::new("http://127.0.0.1:8080").expect("client");
8896 let mut worker = Worker::new(client, "rust-workers");
8897 worker.register_workflow_avro_value("typed.echo", |_ctx, input| async move { Ok(input) });
8898 worker
8899 .register_activity_avro_value("typed.activity", |_ctx, input| async move { Ok(input) });
8900 worker.register_query_avro_value("typed.echo", "inspect", |_ctx, input| async move {
8901 Ok(input)
8902 });
8903 worker.register_update_avro_value("typed.echo", "replace", |_ctx, input| async move {
8904 Ok(input)
8905 });
8906 worker.register_workflow_avro_value("typed.signal", |ctx, _input| async move {
8907 Ok(AvroValue::Array(
8908 ctx.wait_signal_avro_value("changed").await?,
8909 ))
8910 });
8911
8912 let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
8913 let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed envelope");
8914
8915 let mut workflow = workflow_task("typed.echo", Vec::new(), DEFAULT_CODEC);
8916 workflow.arguments = Some(envelope.clone());
8917 let commands = worker
8918 .execute_workflow_task(workflow)
8919 .expect("typed workflow task");
8920 assert_eq!(commands[0]["type"], "complete_workflow");
8921 assert_eq!(
8922 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
8923 .expect("typed workflow result"),
8924 arguments
8925 );
8926
8927 let activity = ActivityTask {
8928 task_id: "activity-typed".to_string(),
8929 activity_attempt_id: Some("attempt-typed".to_string()),
8930 attempt_id: None,
8931 activity_type: "typed.activity".to_string(),
8932 payload_codec: DEFAULT_CODEC.to_string(),
8933 arguments: Some(envelope.clone()),
8934 attempt_number: 1,
8935 lease_owner: Some("rust-worker".to_string()),
8936 };
8937 assert_eq!(
8938 worker
8939 .execute_activity_task(activity)
8940 .await
8941 .expect("typed activity result"),
8942 arguments
8943 );
8944
8945 let query = QueryTask {
8946 query_task_id: "query-typed".to_string(),
8947 query_task_attempt: 1,
8948 lease_owner: Some("rust-worker".to_string()),
8949 workflow_id: Some("typed-1".to_string()),
8950 run_id: Some("run-typed".to_string()),
8951 workflow_type: "typed.echo".to_string(),
8952 query_name: "inspect".to_string(),
8953 payload_codec: DEFAULT_CODEC.to_string(),
8954 workflow_arguments: Some(
8955 encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
8956 .expect("workflow input"),
8957 ),
8958 query_arguments: Some(envelope.clone()),
8959 history_events: Vec::new(),
8960 history_export: None,
8961 run_status: Some("running".to_string()),
8962 };
8963 assert_eq!(
8964 worker
8965 .execute_query_task(query)
8966 .await
8967 .expect("typed query result"),
8968 arguments
8969 );
8970
8971 let mut update = workflow_task(
8972 "typed.echo",
8973 vec![history_event(
8974 "UpdateAccepted",
8975 json!({
8976 "update_id": "update-typed",
8977 "update_name": "replace",
8978 "arguments": envelope.clone(),
8979 }),
8980 )],
8981 DEFAULT_CODEC,
8982 );
8983 update.workflow_update_id = Some("update-typed".to_string());
8984 update.update_name = Some("replace".to_string());
8985 let commands = worker
8986 .execute_workflow_task(update)
8987 .expect("typed update task");
8988 assert_eq!(commands[0]["type"], "complete_update");
8989 assert_eq!(
8990 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
8991 .expect("typed update result"),
8992 arguments
8993 );
8994
8995 let mut signal = workflow_task(
8996 "typed.signal",
8997 vec![history_event(
8998 "SignalReceived",
8999 json!({
9000 "signal_id": "signal-typed",
9001 "signal_name": "changed",
9002 "arguments": envelope.clone(),
9003 }),
9004 )],
9005 DEFAULT_CODEC,
9006 );
9007 signal.workflow_signal_id = Some("signal-typed".to_string());
9008 signal.signal_name = Some("changed".to_string());
9009 signal.signal_arguments = Some(envelope);
9010 let commands = worker
9011 .execute_workflow_task(signal)
9012 .expect("typed signal resume");
9013 assert_eq!(
9014 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
9015 .expect("typed signal result"),
9016 arguments
9017 );
9018 }
9019
9020 #[tokio::test]
9021 async fn typed_helpers_never_parse_json_inspection_projection() {
9022 let collision_values = projection_collision_probe();
9023 let expected = AvroValue::Array(collision_values.clone());
9024 let envelope = encode_typed_envelope(&expected, DEFAULT_CODEC).expect("collision envelope");
9025
9026 let activity_context = workflow_context_with_codec(
9027 vec![history_event(
9028 "ActivityCompleted",
9029 json!({
9030 "sequence": 1,
9031 "activity_type": "collision.activity",
9032 "payload_codec": DEFAULT_CODEC,
9033 "result": envelope.clone(),
9034 }),
9035 )],
9036 DEFAULT_CODEC,
9037 );
9038 assert_eq!(
9039 activity_context
9040 .activity_avro_value("collision.activity", AvroValue::Array(Vec::new()))
9041 .await
9042 .expect("typed activity collision result"),
9043 expected
9044 );
9045
9046 let signal_context = workflow_context_with_codec(
9047 vec![
9048 history_event(
9049 "SignalWaitOpened",
9050 json!({"sequence": 1, "signal_name": "collision"}),
9051 ),
9052 history_event(
9053 "SignalApplied",
9054 json!({
9055 "sequence": 1,
9056 "signal_name": "collision",
9057 "payload_codec": DEFAULT_CODEC,
9058 "value": envelope.clone(),
9059 }),
9060 ),
9061 ],
9062 DEFAULT_CODEC,
9063 );
9064 assert_eq!(
9065 signal_context
9066 .wait_signal_avro_value("collision")
9067 .await
9068 .expect("typed signal collision arguments"),
9069 collision_values
9070 );
9071
9072 let child_context = workflow_context_with_codec(
9073 vec![
9074 history_event(
9075 "ChildWorkflowScheduled",
9076 json!({
9077 "sequence": 1,
9078 "child_workflow_instance_id": "collision-child",
9079 "child_workflow_run_id": "collision-run",
9080 "child_workflow_type": "collision.child",
9081 }),
9082 ),
9083 history_event(
9084 "ChildRunCompleted",
9085 json!({
9086 "sequence": 1,
9087 "child_workflow_instance_id": "collision-child",
9088 "child_workflow_run_id": "collision-run",
9089 "child_workflow_type": "collision.child",
9090 "payload_codec": DEFAULT_CODEC,
9091 "result": envelope,
9092 }),
9093 ),
9094 ],
9095 DEFAULT_CODEC,
9096 );
9097 let child = child_context
9098 .start_child_workflow_avro_value(
9099 "collision.child",
9100 ChildWorkflowOptions::new("collision-workers"),
9101 AvroValue::Array(Vec::new()),
9102 )
9103 .await
9104 .expect("typed child collision result");
9105 assert_eq!(child.result, expected);
9106 }
9107
9108 #[tokio::test]
9109 async fn replayed_typed_query_keeps_lossless_workflow_and_query_inputs() {
9110 let client = Client::new("http://127.0.0.1:8080").expect("client");
9111 let mut worker = Worker::new(client, "rust-workers");
9112 worker.register_replayed_workflow_avro_value(
9113 "typed.replayed",
9114 || (),
9115 |_ctx, input, _state| async move { Ok(input) },
9116 );
9117 worker.register_replayed_query_avro_value::<(), _, _>(
9118 "typed.replayed",
9119 "inspect",
9120 |ctx, _state, args| async move {
9121 let mut signals = ctx.signals_avro_value("collision");
9122 let signal = signals
9123 .pop()
9124 .map(AvroValue::Array)
9125 .unwrap_or_else(|| AvroValue::Array(Vec::new()));
9126 Ok(AvroValue::Array(vec![
9127 ctx.workflow_input_avro_value().clone(),
9128 signal,
9129 args,
9130 ]))
9131 },
9132 );
9133 let arguments = AvroValue::Array(projection_collision_probe());
9134 let signal_arguments =
9135 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed query signal arguments");
9136 let task = QueryTask {
9137 query_task_id: "query-typed-replay".to_string(),
9138 query_task_attempt: 1,
9139 lease_owner: Some("rust-worker".to_string()),
9140 workflow_id: Some("typed-replay".to_string()),
9141 run_id: Some("run-typed-replay".to_string()),
9142 workflow_type: "typed.replayed".to_string(),
9143 query_name: "inspect".to_string(),
9144 payload_codec: DEFAULT_CODEC.to_string(),
9145 workflow_arguments: Some(
9146 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("workflow arguments"),
9147 ),
9148 query_arguments: Some(
9149 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("query arguments"),
9150 ),
9151 history_events: vec![history_event(
9152 "SignalReceived",
9153 json!({
9154 "signal_id": "collision-signal",
9155 "signal_name": "collision",
9156 "workflow_sequence": 1,
9157 "payload_codec": DEFAULT_CODEC,
9158 "arguments": signal_arguments,
9159 }),
9160 )],
9161 history_export: None,
9162 run_status: Some("completed".to_string()),
9163 };
9164
9165 assert_eq!(
9166 worker
9167 .execute_query_task(task)
9168 .await
9169 .expect("typed replay query"),
9170 AvroValue::Array(vec![arguments.clone(), arguments.clone(), arguments])
9171 );
9172 }
9173
9174 #[test]
9175 fn public_avro_adapter_rejects_non_string_map_keys_before_json_conversion() {
9176 let value = BTreeMap::from([(1_i32, "integer key")]);
9177 let error = PayloadEnvelope::avro(&value)
9178 .expect_err("integer map keys must fail")
9179 .to_string();
9180
9181 assert!(error.contains("invalid_map_key"));
9182 }
9183
9184 #[test]
9185 fn json_codec_remains_plain_json() {
9186 let value = json!({"greeting": "hello", "count": 3, "ok": true});
9187 let envelope = PayloadEnvelope::json(&value).expect("encode");
9188
9189 assert_eq!(envelope.codec, JSON_CODEC);
9190 assert_eq!(envelope.blob, serde_json::to_string(&value).expect("json"));
9191 assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
9192 }
9193
9194 #[test]
9195 fn prerelease_avro_payload_without_single_object_frame_is_rejected() {
9196 let envelope = PayloadEnvelope {
9197 codec: DEFAULT_CODEC.to_string(),
9198 blob: BASE64.encode([0x01]),
9199 };
9200
9201 let error = decode_payload::<Value>(&envelope).expect_err("prerelease payload must fail");
9202 assert!(error.to_string().contains("invalid_payload_framing"));
9203 }
9204
9205 #[test]
9206 fn workflow_context_schedules_activity_until_completion_is_in_history() {
9207 let ctx = WorkflowContext {
9208 state: Arc::new(Mutex::new(
9209 WorkflowState::new_with_identity(
9210 Vec::new(),
9211 Some("wf-parent".to_string()),
9212 Some("run-parent".to_string()),
9213 "rust-workers".to_string(),
9214 DEFAULT_CODEC.to_string(),
9215 None,
9216 )
9217 .expect("workflow state"),
9218 )),
9219 };
9220
9221 let mut call = Box::pin(ctx.activity("hello.activity", json!(["Ada"])));
9222 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9223 assert!(matches!(
9224 call.as_mut().poll(&mut task_context),
9225 Poll::Pending
9226 ));
9227
9228 let commands = ctx.take_commands().expect("commands");
9229 assert_eq!(commands[0]["type"], "schedule_activity");
9230 assert_eq!(commands[0]["activity_type"], "hello.activity");
9231 }
9232
9233 #[test]
9234 fn activity_options_encode_retry_policy_queue_and_every_timeout() {
9235 let ctx = workflow_context(Vec::new());
9236 let options = ActivityOptions::new()
9237 .task_queue("payments")
9238 .retry_policy(
9239 ActivityRetryPolicy::new(4)
9240 .exponential_backoff(Duration::from_secs(1), 3, Some(Duration::from_secs(10)))
9241 .non_retryable_error_type("ValidationError"),
9242 )
9243 .start_to_close_timeout(Duration::from_secs(120))
9244 .schedule_to_start_timeout(Duration::from_secs(10))
9245 .schedule_to_close_timeout(Duration::from_secs(300))
9246 .heartbeat_timeout(Duration::from_secs(15));
9247 let mut call = Box::pin(ctx.activity_with_options(
9248 "charge-card",
9249 options,
9250 json!([{"order_id": "o-1"}]),
9251 ));
9252 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9253
9254 assert!(matches!(
9255 call.as_mut().poll(&mut task_context),
9256 Poll::Pending
9257 ));
9258 assert!(matches!(
9259 call.as_mut().poll(&mut task_context),
9260 Poll::Pending
9261 ));
9262
9263 let commands = ctx.take_commands().expect("activity command");
9264 assert_eq!(commands.len(), 1, "one future emits one logical schedule");
9265 assert_eq!(commands[0]["queue"], "payments");
9266 assert_eq!(
9267 commands[0]["retry_policy"],
9268 json!({
9269 "max_attempts": 4,
9270 "backoff_seconds": [1, 3, 9],
9271 "non_retryable_error_types": ["ValidationError"],
9272 })
9273 );
9274 assert_eq!(commands[0]["start_to_close_timeout"], 120);
9275 assert_eq!(commands[0]["schedule_to_start_timeout"], 10);
9276 assert_eq!(commands[0]["schedule_to_close_timeout"], 300);
9277 assert_eq!(commands[0]["heartbeat_timeout"], 15);
9278 }
9279
9280 #[test]
9281 fn activity_options_encode_explicit_and_rounded_backoff_intervals() {
9282 let ctx = workflow_context(Vec::new());
9283 let options = ActivityOptions::new().retry_policy(
9284 ActivityRetryPolicy::new(3)
9285 .backoff_intervals([Duration::from_millis(1), Duration::from_millis(1_001)]),
9286 );
9287 let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
9288 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9289
9290 assert!(matches!(
9291 call.as_mut().poll(&mut task_context),
9292 Poll::Pending
9293 ));
9294 assert_eq!(
9295 ctx.take_commands().expect("command")[0]["retry_policy"]["backoff_seconds"],
9296 json!([1, 2])
9297 );
9298 }
9299
9300 #[test]
9301 fn invalid_activity_options_return_typed_errors_before_emitting_commands() {
9302 let cases = [
9303 (
9304 ActivityOptions::new().task_queue(" "),
9305 ActivityOptionsErrorKind::EmptyTaskQueue,
9306 ),
9307 (
9308 ActivityOptions::new().retry_policy(ActivityRetryPolicy::default()),
9309 ActivityOptionsErrorKind::EmptyRetryPolicy,
9310 ),
9311 (
9312 ActivityOptions::new().retry_policy(ActivityRetryPolicy::new(0)),
9313 ActivityOptionsErrorKind::InvalidMaxAttempts,
9314 ),
9315 (
9316 ActivityOptions::new().retry_policy(ActivityRetryPolicy {
9317 max_attempts: None,
9318 backoff: Some(ActivityBackoff::Explicit(vec![Duration::from_secs(1)])),
9319 non_retryable_error_types: Vec::new(),
9320 }),
9321 ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
9322 ),
9323 (
9324 ActivityOptions::new().retry_policy(
9325 ActivityRetryPolicy::new(2)
9326 .backoff_intervals([Duration::from_secs(1), Duration::from_secs(2)]),
9327 ),
9328 ActivityOptionsErrorKind::TooManyBackoffIntervals,
9329 ),
9330 (
9331 ActivityOptions::new().retry_policy(
9332 ActivityRetryPolicy::new(2).exponential_backoff(
9333 Duration::from_secs(1),
9334 0,
9335 None,
9336 ),
9337 ),
9338 ActivityOptionsErrorKind::InvalidBackoffCoefficient,
9339 ),
9340 (
9341 ActivityOptions::new()
9342 .retry_policy(ActivityRetryPolicy::new(2).non_retryable_error_type(" ")),
9343 ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
9344 ),
9345 (
9346 ActivityOptions::new().retry_policy(
9347 ActivityRetryPolicy::new(10_002).exponential_backoff(
9348 Duration::from_secs(1),
9349 1,
9350 None,
9351 ),
9352 ),
9353 ActivityOptionsErrorKind::BackoffGenerationTooLarge,
9354 ),
9355 (
9356 ActivityOptions::new().retry_policy(
9357 ActivityRetryPolicy::new(2)
9358 .backoff_intervals([Duration::from_secs(i64::MAX as u64 + 1)]),
9359 ),
9360 ActivityOptionsErrorKind::BackoffOverflow,
9361 ),
9362 ];
9363
9364 for (options, expected_kind) in cases {
9365 let ctx = workflow_context(Vec::new());
9366 let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
9367 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9368 let Poll::Ready(Err(Error::InvalidActivityOptions(error))) =
9369 call.as_mut().poll(&mut task_context)
9370 else {
9371 panic!("expected typed activity validation error");
9372 };
9373 assert_eq!(error.kind, expected_kind);
9374 assert!(ctx.take_commands().expect("commands").is_empty());
9375 }
9376 }
9377
9378 #[test]
9379 fn activity_options_validate_positive_and_ordered_timeouts() {
9380 let zero_timeout_cases = [
9381 ActivityOptions::new().start_to_close_timeout(Duration::ZERO),
9382 ActivityOptions::new().schedule_to_start_timeout(Duration::ZERO),
9383 ActivityOptions::new().schedule_to_close_timeout(Duration::ZERO),
9384 ActivityOptions::new().heartbeat_timeout(Duration::ZERO),
9385 ];
9386 for options in zero_timeout_cases {
9387 assert_eq!(
9388 options.validate().expect_err("zero timeout").kind,
9389 ActivityOptionsErrorKind::TimeoutNotPositive
9390 );
9391 }
9392
9393 let ordering_cases = [
9394 ActivityOptions::new()
9395 .heartbeat_timeout(Duration::from_secs(11))
9396 .start_to_close_timeout(Duration::from_secs(10)),
9397 ActivityOptions::new()
9398 .start_to_close_timeout(Duration::from_secs(31))
9399 .schedule_to_close_timeout(Duration::from_secs(30)),
9400 ActivityOptions::new()
9401 .schedule_to_start_timeout(Duration::from_secs(31))
9402 .schedule_to_close_timeout(Duration::from_secs(30)),
9403 ];
9404 for options in ordering_cases {
9405 assert_eq!(
9406 options.validate().expect_err("timeout order").kind,
9407 ActivityOptionsErrorKind::TimeoutOrder
9408 );
9409 }
9410
9411 assert_eq!(
9412 ActivityOptions::new()
9413 .start_to_close_timeout(Duration::from_secs(i64::MAX as u64 + 1))
9414 .validate()
9415 .expect_err("protocol integer overflow")
9416 .kind,
9417 ActivityOptionsErrorKind::TimeoutOverflow
9418 );
9419 }
9420
9421 #[test]
9422 fn replayed_activity_retry_history_completes_without_duplicate_schedule() {
9423 let ctx = workflow_context(completed_retry_activity_history());
9424 let mut call =
9425 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
9426 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9427
9428 assert!(matches!(
9429 call.as_mut().poll(&mut task_context),
9430 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
9431 ));
9432 assert!(ctx.take_commands().expect("commands").is_empty());
9433 ctx.ensure_history_consumed().expect("history consumed");
9434 }
9435
9436 #[test]
9437 fn duplicate_non_retryable_types_use_one_command_and_replay_representation() {
9438 let mut options = retry_activity_options();
9439 options
9440 .retry_policy
9441 .as_mut()
9442 .expect("retry policy")
9443 .non_retryable_error_types
9444 .extend([" PermanentError ".to_string(), "PermanentError".to_string()]);
9445
9446 let new_ctx = workflow_context(Vec::new());
9447 let mut new_call =
9448 Box::pin(new_ctx.activity_with_options("flaky", options.clone(), json!([])));
9449 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9450 assert!(matches!(
9451 new_call.as_mut().poll(&mut task_context),
9452 Poll::Pending
9453 ));
9454 let commands = new_ctx.take_commands().expect("commands");
9455 assert_eq!(commands.len(), 1);
9456 assert_eq!(
9457 commands[0]["retry_policy"]["non_retryable_error_types"],
9458 json!(["PermanentError"])
9459 );
9460
9461 let replay_ctx = workflow_context(completed_retry_activity_history());
9462 let mut replay_call =
9463 Box::pin(replay_ctx.activity_with_options("flaky", options, json!([])));
9464 assert!(matches!(
9465 replay_call.as_mut().poll(&mut task_context),
9466 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
9467 ));
9468 assert!(replay_ctx.take_commands().expect("commands").is_empty());
9469 replay_ctx
9470 .ensure_history_consumed()
9471 .expect("history consumed");
9472 }
9473
9474 #[test]
9475 fn replayed_intermediate_retry_remains_pending_across_restarts() {
9476 let history = completed_retry_activity_history()
9477 .into_iter()
9478 .take(3)
9479 .collect::<Vec<_>>();
9480
9481 for _restart in 0..2 {
9482 let ctx = workflow_context(history.clone());
9483 let mut call =
9484 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
9485 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9486 assert!(matches!(
9487 call.as_mut().poll(&mut task_context),
9488 Poll::Pending
9489 ));
9490 assert!(ctx.take_commands().expect("commands").is_empty());
9491 }
9492 }
9493
9494 #[test]
9495 fn replayed_activity_rejects_changed_queue_retry_and_every_timeout_field() {
9496 let mut changed_queue = retry_activity_options();
9497 changed_queue.task_queue = Some("different-queue".to_string());
9498
9499 let mut changed_max_attempts = retry_activity_options();
9500 let retry_policy = changed_max_attempts
9501 .retry_policy
9502 .as_mut()
9503 .expect("retry policy");
9504 retry_policy.max_attempts = Some(4);
9505
9506 let mut changed_backoff = retry_activity_options();
9507 let retry_policy = changed_backoff.retry_policy.as_mut().expect("retry policy");
9508 retry_policy.backoff = Some(ActivityBackoff::Explicit(vec![
9509 Duration::from_secs(3),
9510 Duration::from_secs(4),
9511 ]));
9512
9513 let mut changed_non_retryable_types = retry_activity_options();
9514 let retry_policy = changed_non_retryable_types
9515 .retry_policy
9516 .as_mut()
9517 .expect("retry policy");
9518 retry_policy.non_retryable_error_types = vec!["AnotherPermanentError".to_string()];
9519
9520 let mut changed_start_to_close = retry_activity_options();
9521 changed_start_to_close.start_to_close_timeout = Some(Duration::from_secs(31));
9522 let mut changed_schedule_to_start = retry_activity_options();
9523 changed_schedule_to_start.schedule_to_start_timeout = Some(Duration::from_secs(6));
9524 let mut changed_schedule_to_close = retry_activity_options();
9525 changed_schedule_to_close.schedule_to_close_timeout = Some(Duration::from_secs(91));
9526 let mut changed_heartbeat = retry_activity_options();
9527 changed_heartbeat.heartbeat_timeout = Some(Duration::from_secs(11));
9528
9529 let cases = [
9530 (changed_queue, "activity_task_queue_mismatch"),
9531 (changed_max_attempts, "activity_retry_policy_mismatch"),
9532 (changed_backoff, "activity_retry_policy_mismatch"),
9533 (
9534 changed_non_retryable_types,
9535 "activity_retry_policy_mismatch",
9536 ),
9537 (changed_start_to_close, "activity_retry_policy_mismatch"),
9538 (changed_schedule_to_start, "activity_retry_policy_mismatch"),
9539 (changed_schedule_to_close, "activity_retry_policy_mismatch"),
9540 (changed_heartbeat, "activity_retry_policy_mismatch"),
9541 ];
9542
9543 for (options, expected_reason) in cases {
9544 let ctx = workflow_context(completed_retry_activity_history());
9545 let mut call = Box::pin(ctx.activity_with_options("flaky", options, json!([])));
9546 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9547 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
9548 call.as_mut().poll(&mut task_context)
9549 else {
9550 panic!("changed activity options must fail replay");
9551 };
9552 assert_eq!(failure.reason, expected_reason);
9553 assert_eq!(failure.sequence, Some(1));
9554 assert!(ctx.take_commands().expect("commands").is_empty());
9555 }
9556 }
9557
9558 #[test]
9559 fn replayed_activity_rejects_changed_execution_mode_and_snapshot_version() {
9560 let cases = [
9561 (
9562 "execution_mode",
9563 json!("local"),
9564 "activity_execution_mode_mismatch",
9565 ),
9566 (
9567 "snapshot_version",
9568 json!(2),
9569 "activity_retry_policy_mismatch",
9570 ),
9571 ];
9572
9573 for (field, value, expected_reason) in cases {
9574 let mut history = completed_retry_activity_history();
9575 let activity = history[0].payload["activity"]
9576 .as_object_mut()
9577 .expect("activity snapshot");
9578 if field == "execution_mode" {
9579 activity.insert(field.to_string(), value);
9580 } else {
9581 activity["retry_policy"]
9582 .as_object_mut()
9583 .expect("retry snapshot")
9584 .insert(field.to_string(), value);
9585 }
9586
9587 let ctx = workflow_context(history);
9588 let mut call =
9589 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
9590 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9591 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
9592 call.as_mut().poll(&mut task_context)
9593 else {
9594 panic!("changed {field} must fail replay");
9595 };
9596 assert_eq!(failure.reason, expected_reason);
9597 assert_eq!(failure.sequence, Some(1));
9598 assert!(ctx.take_commands().expect("commands").is_empty());
9599 }
9600 }
9601
9602 #[test]
9603 fn replayed_legacy_activity_treats_missing_option_snapshot_as_unknown() {
9604 let mut history = completed_retry_activity_history();
9605 let activity = history[0].payload["activity"]
9606 .as_object_mut()
9607 .expect("activity snapshot");
9608 activity.remove("execution_mode");
9609 activity.remove("retry_policy");
9610
9611 let mut current = retry_activity_options();
9612 current.start_to_close_timeout = Some(Duration::from_secs(45));
9613 current.schedule_to_start_timeout = Some(Duration::from_secs(8));
9614 current.schedule_to_close_timeout = Some(Duration::from_secs(120));
9615 current.heartbeat_timeout = Some(Duration::from_secs(12));
9616
9617 let ctx = workflow_context(history);
9618 let mut call = Box::pin(ctx.activity_with_options("flaky", current, json!([])));
9619 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9620 assert!(matches!(
9621 call.as_mut().poll(&mut task_context),
9622 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
9623 ));
9624 assert!(ctx.take_commands().expect("commands").is_empty());
9625 ctx.ensure_history_consumed().expect("history consumed");
9626 }
9627
9628 #[test]
9629 fn terminal_activity_failed_after_start_returns_typed_failure() {
9630 let history = vec![
9631 history_event(
9632 "ActivityScheduled",
9633 json!({
9634 "sequence": 1,
9635 "activity_type": "flaky",
9636 "activity_execution_id": "act-terminal",
9637 "activity": {
9638 "id": "act-terminal",
9639 "sequence": 1,
9640 "type": "flaky",
9641 "queue": "critical-activities",
9642 "retry_policy": {
9643 "snapshot_version": 1,
9644 "max_attempts": 3,
9645 "backoff_seconds": [2, 4],
9646 "non_retryable_error_types": ["PermanentError"]
9647 }
9648 }
9649 }),
9650 ),
9651 history_event(
9652 "ActivityStarted",
9653 json!({
9654 "sequence": 1,
9655 "activity_type": "flaky",
9656 "activity_execution_id": "act-terminal",
9657 "activity_attempt_id": "attempt-1",
9658 "attempt_number": 1
9659 }),
9660 ),
9661 history_event(
9662 "ActivityFailed",
9663 json!({
9664 "sequence": 1,
9665 "activity_type": "flaky",
9666 "activity_execution_id": "act-terminal",
9667 "activity_attempt_id": "attempt-1",
9668 "attempt_number": 1,
9669 "failure_id": "failure-terminal",
9670 "failure_category": "activity",
9671 "exception_type": "PermanentError",
9672 "message": "cannot retry",
9673 "non_retryable": true
9674 }),
9675 ),
9676 ];
9677 let ctx = workflow_context(history);
9678 let mut call =
9679 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
9680 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9681
9682 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
9683 call.as_mut().poll(&mut task_context)
9684 else {
9685 panic!("terminal ActivityFailed must settle the activity future");
9686 };
9687 assert_eq!(failure.kind, ActivityFailureKind::Failed);
9688 assert_eq!(
9689 failure.activity_execution_id.as_deref(),
9690 Some("act-terminal")
9691 );
9692 assert_eq!(failure.exception_type.as_deref(), Some("PermanentError"));
9693 assert!(failure.non_retryable);
9694 assert!(ctx.take_commands().expect("commands").is_empty());
9695 ctx.ensure_history_consumed().expect("history consumed");
9696 }
9697
9698 #[test]
9699 fn activity_terminal_events_return_machine_readable_failures() {
9700 let cases = [
9701 (
9702 "ActivityFailed",
9703 json!({
9704 "sequence": 1,
9705 "activity_type": "charge-card",
9706 "activity_execution_id": "act-1",
9707 "activity_attempt_id": "attempt-2",
9708 "attempt_number": 2,
9709 "failure_id": "failure-1",
9710 "failure_category": "activity",
9711 "exception_type": "PaymentDeclined",
9712 "exception_class": "payments.PaymentDeclined",
9713 "message": "card declined",
9714 "non_retryable": true
9715 }),
9716 ActivityFailureKind::Failed,
9717 "activity",
9718 ),
9719 (
9720 "ActivityCancelled",
9721 json!({
9722 "sequence": 1,
9723 "activity_type": "charge-card",
9724 "activity_execution_id": "act-1",
9725 "activity_attempt_id": "attempt-1"
9726 }),
9727 ActivityFailureKind::Cancelled,
9728 "cancelled",
9729 ),
9730 ];
9731
9732 for (event_type, payload, expected_kind, expected_reason) in cases {
9733 let ctx = workflow_context(vec![history_event(event_type, payload)]);
9734 let mut call = Box::pin(ctx.activity("charge-card", json!([])));
9735 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9736 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
9737 call.as_mut().poll(&mut task_context)
9738 else {
9739 panic!("expected terminal activity failure");
9740 };
9741 assert_eq!(failure.kind, expected_kind);
9742 assert_eq!(failure.reason, expected_reason);
9743 assert_eq!(failure.activity_execution_id.as_deref(), Some("act-1"));
9744 assert_eq!(failure.activity_type.as_deref(), Some("charge-card"));
9745 }
9746 }
9747
9748 #[test]
9749 fn every_activity_timeout_class_is_typed() {
9750 for timeout_kind in [
9751 "start_to_close",
9752 "schedule_to_start",
9753 "schedule_to_close",
9754 "heartbeat",
9755 ] {
9756 let ctx = workflow_context(vec![history_event(
9757 "ActivityTimedOut",
9758 json!({
9759 "sequence": 1,
9760 "activity_type": "slow",
9761 "activity_execution_id": "act-timeout",
9762 "activity_attempt_id": "attempt-timeout",
9763 "failure_category": "timeout",
9764 "timeout_kind": timeout_kind,
9765 "message": "deadline expired"
9766 }),
9767 )]);
9768 let mut call = Box::pin(ctx.activity("slow", json!([])));
9769 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9770 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
9771 call.as_mut().poll(&mut task_context)
9772 else {
9773 panic!("expected timeout failure");
9774 };
9775 assert_eq!(failure.kind, ActivityFailureKind::TimedOut);
9776 assert_eq!(failure.reason, timeout_kind);
9777 assert_eq!(failure.timeout_kind.as_deref(), Some(timeout_kind));
9778 assert_eq!(failure.failure_category.as_deref(), Some("timeout"));
9779 }
9780 }
9781
9782 #[test]
9783 fn workflow_sleep_emits_one_durable_timer_and_rounds_up() {
9784 let ctx = workflow_context(Vec::new());
9785 let mut sleep = Box::pin(ctx.sleep(Duration::from_millis(1_001)));
9786 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9787
9788 assert!(matches!(
9789 sleep.as_mut().poll(&mut task_context),
9790 Poll::Pending
9791 ));
9792 assert!(matches!(
9793 sleep.as_mut().poll(&mut task_context),
9794 Poll::Pending
9795 ));
9796
9797 let commands = ctx.take_commands().expect("timer command");
9798 assert_eq!(
9799 commands,
9800 vec![json!({
9801 "type": "start_timer",
9802 "delay_seconds": 2,
9803 })]
9804 );
9805 }
9806
9807 #[test]
9808 fn workflow_sleep_replays_matching_schedule_and_fire_without_a_command() {
9809 let history = vec![
9810 history_event(
9811 "TimerScheduled",
9812 json!({
9813 "sequence": 1,
9814 "timer_id": "timer-1",
9815 "delay_seconds": 5,
9816 "fire_at": "2026-07-11T12:00:05Z",
9817 }),
9818 ),
9819 history_event(
9820 "TimerFired",
9821 json!({
9822 "sequence": 1,
9823 "timer_id": "timer-1",
9824 "delay_seconds": 5,
9825 "fire_at": "2026-07-11T12:00:05Z",
9826 "fired_at": "2026-07-11T12:00:05Z",
9827 }),
9828 ),
9829 ];
9830
9831 for _restart in 0..2 {
9832 let ctx = workflow_context(history.clone());
9833 let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(5)));
9834 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9835 assert!(matches!(
9836 sleep.as_mut().poll(&mut task_context),
9837 Poll::Ready(Ok(()))
9838 ));
9839 assert!(ctx.take_commands().expect("commands").is_empty());
9840 ctx.ensure_history_consumed().expect("history consumed");
9841 }
9842 }
9843
9844 #[test]
9845 fn workflow_sleep_rejects_changed_delay_during_replay() {
9846 let ctx = workflow_context(vec![
9847 history_event(
9848 "TimerScheduled",
9849 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
9850 ),
9851 history_event(
9852 "TimerFired",
9853 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
9854 ),
9855 ]);
9856 let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(500)));
9857 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9858
9859 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
9860 sleep.as_mut().poll(&mut task_context)
9861 else {
9862 panic!("changed timer delay must be rejected");
9863 };
9864 assert_eq!(failure.reason, "timer_delay_mismatch");
9865 assert_eq!(failure.sequence, Some(1));
9866 }
9867
9868 #[test]
9869 fn workflow_history_rejects_unpaired_or_mismatched_timer_events() {
9870 let lone_fire = WorkflowState::new(
9871 vec![history_event(
9872 "TimerFired",
9873 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
9874 )],
9875 "rust-workers".to_string(),
9876 JSON_CODEC.to_string(),
9877 None,
9878 )
9879 .expect_err("TimerFired requires TimerScheduled");
9880 assert!(matches!(
9881 lone_fire,
9882 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
9883 if reason == "timer_schedule_missing_or_duplicate"
9884 ));
9885
9886 let wrong_identity = WorkflowState::new(
9887 vec![
9888 history_event(
9889 "TimerScheduled",
9890 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
9891 ),
9892 history_event(
9893 "TimerFired",
9894 json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
9895 ),
9896 ],
9897 "rust-workers".to_string(),
9898 JSON_CODEC.to_string(),
9899 None,
9900 )
9901 .expect_err("fire must match scheduled timer identity");
9902 assert!(matches!(
9903 wrong_identity,
9904 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
9905 if reason == "timer_identity_mismatch"
9906 ));
9907
9908 let duplicate_fire = WorkflowState::new(
9909 vec![
9910 history_event(
9911 "TimerScheduled",
9912 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
9913 ),
9914 history_event(
9915 "TimerFired",
9916 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
9917 ),
9918 history_event(
9919 "TimerFired",
9920 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
9921 ),
9922 ],
9923 "rust-workers".to_string(),
9924 JSON_CODEC.to_string(),
9925 None,
9926 )
9927 .expect_err("a durable timer cannot fire twice");
9928 assert!(matches!(
9929 duplicate_fire,
9930 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
9931 if reason == "duplicate_timer_fire"
9932 ));
9933
9934 let wrong_fired_delay = WorkflowState::new(
9935 vec![
9936 history_event(
9937 "TimerScheduled",
9938 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
9939 ),
9940 history_event(
9941 "TimerFired",
9942 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 6}),
9943 ),
9944 ],
9945 "rust-workers".to_string(),
9946 JSON_CODEC.to_string(),
9947 None,
9948 )
9949 .expect_err("timer schedule and fire delays must agree");
9950 assert!(matches!(
9951 wrong_fired_delay,
9952 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
9953 if reason == "timer_history_delay_mismatch"
9954 ));
9955 }
9956
9957 #[test]
9958 fn replay_rejects_activity_moved_before_recorded_timer() {
9959 let ctx = workflow_context(vec![
9960 history_event(
9961 "TimerScheduled",
9962 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
9963 ),
9964 history_event(
9965 "TimerFired",
9966 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
9967 ),
9968 history_event(
9969 "ActivityCompleted",
9970 json!({
9971 "sequence": 2,
9972 "activity_type": "after-timer",
9973 "payload_codec": "json",
9974 "result": {"codec": "json", "blob": "\"done\""},
9975 }),
9976 ),
9977 ]);
9978 let mut activity = Box::pin(ctx.activity("after-timer", json!([])));
9979 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9980
9981 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
9982 activity.as_mut().poll(&mut task_context)
9983 else {
9984 panic!("reordered durable command must be rejected");
9985 };
9986 assert_eq!(failure.reason, "recorded_command_mismatch");
9987 assert_eq!(failure.sequence, Some(1));
9988 assert_eq!(failure.expected.as_deref(), Some("timer"));
9989 assert_eq!(failure.actual.as_deref(), Some("activity:after-timer"));
9990 }
9991
9992 #[test]
9993 fn workflow_context_emits_a_typed_named_signal_wait() {
9994 let ctx = workflow_context(Vec::new());
9995 let mut signal = Box::pin(ctx.wait_signal("finish"));
9996 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9997
9998 assert!(matches!(
9999 signal.as_mut().poll(&mut task_context),
10000 Poll::Pending
10001 ));
10002 assert_eq!(
10003 ctx.take_commands().expect("signal-wait command"),
10004 vec![json!({
10005 "type": "open_signal_wait",
10006 "signal_name": "finish",
10007 })]
10008 );
10009 }
10010
10011 #[test]
10012 fn condition_wait_history_does_not_resolve_a_typed_signal_wait() {
10013 let ctx = workflow_context(vec![
10014 history_event(
10015 "ConditionWaitOpened",
10016 json!({"sequence": 1, "condition_key": "signal:finish"}),
10017 ),
10018 history_event(
10019 "ConditionWaitSatisfied",
10020 json!({"sequence": 1, "condition_key": "signal:finish"}),
10021 ),
10022 history_event(
10023 "SignalReceived",
10024 json!({"signal_name": "finish", "arguments": []}),
10025 ),
10026 ]);
10027 let mut signal = Box::pin(ctx.wait_signal("finish"));
10028 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10029
10030 assert!(matches!(
10031 signal.as_mut().poll(&mut task_context),
10032 Poll::Pending
10033 ));
10034 assert_eq!(
10035 ctx.take_commands().expect("typed signal-wait command"),
10036 vec![json!({
10037 "type": "open_signal_wait",
10038 "signal_name": "finish",
10039 })]
10040 );
10041 }
10042
10043 #[test]
10044 fn replay_orders_signal_waits_and_timers_in_one_command_stream() {
10045 let signal_then_timer = vec![
10046 history_event(
10047 "SignalWaitOpened",
10048 json!({"sequence": 1, "signal_name": "go"}),
10049 ),
10050 history_event(
10051 "SignalApplied",
10052 json!({
10053 "sequence": 1,
10054 "signal_name": "go",
10055 "value": {"codec": "json", "blob": "[\"now\"]"},
10056 }),
10057 ),
10058 history_event(
10059 "TimerScheduled",
10060 json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
10061 ),
10062 history_event(
10063 "TimerFired",
10064 json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
10065 ),
10066 ];
10067
10068 let ctx = workflow_context(signal_then_timer.clone());
10069 let mut signal = Box::pin(ctx.wait_signal("go"));
10070 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10071 assert!(matches!(
10072 signal.as_mut().poll(&mut task_context),
10073 Poll::Ready(Ok(arguments)) if arguments == vec![json!("now")]
10074 ));
10075 let mut timer = Box::pin(ctx.sleep(Duration::from_secs(5)));
10076 assert!(matches!(
10077 timer.as_mut().poll(&mut task_context),
10078 Poll::Ready(Ok(()))
10079 ));
10080 ctx.ensure_history_consumed()
10081 .expect("signal and timer history consumed in order");
10082
10083 let reordered = workflow_context(signal_then_timer);
10084 let mut timer_first = Box::pin(reordered.sleep(Duration::from_secs(5)));
10085 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10086 timer_first.as_mut().poll(&mut task_context)
10087 else {
10088 panic!("timer cannot consume signal-wait-first history");
10089 };
10090 assert_eq!(failure.reason, "recorded_command_mismatch");
10091 assert_eq!(failure.sequence, Some(1));
10092 assert_eq!(failure.expected.as_deref(), Some("signal wait"));
10093
10094 let timer_then_signal = vec![
10095 history_event(
10096 "TimerScheduled",
10097 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10098 ),
10099 history_event(
10100 "TimerFired",
10101 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10102 ),
10103 history_event(
10104 "SignalWaitOpened",
10105 json!({"sequence": 2, "signal_name": "go"}),
10106 ),
10107 history_event(
10108 "SignalApplied",
10109 json!({
10110 "sequence": 2,
10111 "signal_name": "go",
10112 "value": {"codec": "json", "blob": "[]"},
10113 }),
10114 ),
10115 ];
10116 let reordered = workflow_context(timer_then_signal);
10117 let mut signal_first = Box::pin(reordered.wait_signal("go"));
10118 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10119 signal_first.as_mut().poll(&mut task_context)
10120 else {
10121 panic!("signal wait cannot consume timer-first history");
10122 };
10123 assert_eq!(failure.reason, "recorded_command_mismatch");
10124 assert_eq!(failure.sequence, Some(1));
10125 assert_eq!(failure.expected.as_deref(), Some("timer"));
10126 }
10127
10128 #[test]
10129 fn workflow_history_rejects_duplicate_or_colliding_command_sequences() {
10130 let duplicate_timer = WorkflowState::new(
10131 vec![
10132 history_event(
10133 "TimerScheduled",
10134 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10135 ),
10136 history_event(
10137 "TimerScheduled",
10138 json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
10139 ),
10140 ],
10141 "rust-workers".to_string(),
10142 JSON_CODEC.to_string(),
10143 None,
10144 )
10145 .expect_err("one workflow sequence cannot schedule two timers");
10146 assert!(matches!(
10147 duplicate_timer,
10148 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10149 if reason == "timer_schedule_missing_or_duplicate"
10150 ));
10151
10152 let colliding_kinds = WorkflowState::new(
10153 vec![
10154 history_event(
10155 "TimerScheduled",
10156 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10157 ),
10158 history_event(
10159 "ActivityCompleted",
10160 json!({"sequence": 1, "activity_type": "same-sequence"}),
10161 ),
10162 ],
10163 "rust-workers".to_string(),
10164 JSON_CODEC.to_string(),
10165 None,
10166 )
10167 .expect_err("one workflow sequence cannot identify two command kinds");
10168 assert!(matches!(
10169 colliding_kinds,
10170 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10171 if reason == "durable_command_sequence_collision"
10172 ));
10173
10174 let duplicate_signal_wait = WorkflowState::new(
10175 vec![
10176 history_event(
10177 "SignalWaitOpened",
10178 json!({"sequence": 1, "signal_name": "go"}),
10179 ),
10180 history_event(
10181 "SignalWaitOpened",
10182 json!({"sequence": 1, "signal_name": "go"}),
10183 ),
10184 ],
10185 "rust-workers".to_string(),
10186 JSON_CODEC.to_string(),
10187 None,
10188 )
10189 .expect_err("one workflow sequence cannot open two signal waits");
10190 assert!(matches!(
10191 duplicate_signal_wait,
10192 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10193 if reason == "signal_wait_open_missing_or_duplicate"
10194 ));
10195 }
10196
10197 #[test]
10198 fn workflow_history_accepts_a_first_command_after_global_sequence_gaps() {
10199 let result = encode_value_envelope(&json!({"captured": true}), JSON_CODEC)
10200 .expect("side-effect result");
10201 let ctx = workflow_context(vec![history_event(
10202 "SideEffectRecorded",
10203 json!({"sequence": 99, "result": result}),
10204 )]);
10205
10206 let replayed: Value = ctx
10207 .side_effect(|| panic!("recorded side effect must not run"))
10208 .expect("positive global workflow sequence is valid");
10209 assert_eq!(replayed, json!({"captured": true}));
10210 ctx.ensure_history_consumed().expect("history consumed");
10211 }
10212
10213 #[test]
10214 fn workflow_history_rejects_zero_and_descending_command_sequences() {
10215 let result =
10216 encode_value_envelope(&json!("captured"), JSON_CODEC).expect("side-effect result");
10217 let zero = WorkflowState::new(
10218 vec![history_event(
10219 "SideEffectRecorded",
10220 json!({"sequence": 0, "result": result.clone()}),
10221 )],
10222 "rust-workers".to_string(),
10223 JSON_CODEC.to_string(),
10224 None,
10225 )
10226 .expect_err("durable command sequences must be positive");
10227 assert!(matches!(
10228 zero,
10229 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10230 if reason == "durable_command_sequence_invalid"
10231 ));
10232
10233 let descending = WorkflowState::new(
10234 vec![
10235 history_event(
10236 "SideEffectRecorded",
10237 json!({"sequence": 3, "result": result}),
10238 ),
10239 history_event(
10240 "VersionMarkerRecorded",
10241 json!({
10242 "sequence": 2,
10243 "change_id": "descending-marker",
10244 "version": 1,
10245 "min_supported": 1,
10246 "max_supported": 1,
10247 }),
10248 ),
10249 ],
10250 "rust-workers".to_string(),
10251 JSON_CODEC.to_string(),
10252 None,
10253 )
10254 .expect_err("new durable commands must remain strictly ordered");
10255 let Error::NonDeterministicReplay(failure) = descending else {
10256 panic!("expected typed replay failure");
10257 };
10258 assert_eq!(failure.reason, "durable_command_sequence_mismatch");
10259 assert_eq!(failure.sequence, Some(2));
10260 assert_eq!(
10261 failure.expected.as_deref(),
10262 Some("workflow sequence greater than 3")
10263 );
10264 assert_eq!(failure.actual.as_deref(), Some("2"));
10265 }
10266
10267 #[test]
10268 fn workflow_task_replay_completes_after_signals_create_sequence_gaps() {
10269 fn worker() -> Worker {
10270 let client = Client::new("http://127.0.0.1:8080").expect("client");
10271 let mut worker = Worker::new(client, "rust-workers");
10272 worker.register_workflow("rust.finish-after-gaps", |ctx, _input| async move {
10273 ctx.wait_signal("finish").await?;
10274 let marker: String =
10275 ctx.side_effect(|| panic!("recorded side effect must not run"))?;
10276 assert_eq!(marker, "after-finish");
10277 Ok(json!("finished"))
10278 });
10279 worker
10280 }
10281
10282 let marker =
10283 encode_value_envelope(&json!("after-finish"), JSON_CODEC).expect("side-effect result");
10284 let task = workflow_task(
10285 "rust.finish-after-gaps",
10286 vec![
10287 history_event(
10288 "SignalWaitOpened",
10289 json!({"sequence": 1, "signal_name": "finish"}),
10290 ),
10291 history_event(
10292 "SignalReceived",
10293 json!({
10294 "signal_id": "increment-3",
10295 "signal_name": "increment",
10296 "workflow_sequence": 2,
10297 "payload_codec": "json",
10298 "arguments": {"codec": "json", "blob": "[3]"},
10299 }),
10300 ),
10301 history_event(
10302 "SignalReceived",
10303 json!({
10304 "signal_id": "increment-5",
10305 "signal_name": "increment",
10306 "workflow_sequence": 3,
10307 "payload_codec": "json",
10308 "arguments": {"codec": "json", "blob": "[5]"},
10309 }),
10310 ),
10311 history_event(
10312 "SignalReceived",
10313 json!({
10314 "signal_id": "finish",
10315 "signal_name": "finish",
10316 "workflow_sequence": 4,
10317 "payload_codec": "json",
10318 "arguments": {"codec": "json", "blob": "[]"},
10319 }),
10320 ),
10321 history_event(
10322 "SignalApplied",
10323 json!({
10324 "sequence": 1,
10325 "signal_id": "finish",
10326 "signal_name": "finish",
10327 "payload_codec": "json",
10328 "value": {"codec": "json", "blob": "[]"},
10329 }),
10330 ),
10331 history_event(
10332 "SideEffectRecorded",
10333 json!({"sequence": 5, "result": marker}),
10334 ),
10335 ],
10336 JSON_CODEC,
10337 );
10338
10339 for _original_or_cold_worker in 0..2 {
10340 let commands = worker()
10341 .execute_workflow_task(task.clone())
10342 .expect("signal gaps preserve deterministic replay");
10343 assert_eq!(commands.len(), 1, "replay emits only terminal completion");
10344 assert_eq!(commands[0]["type"], "complete_workflow");
10345 assert_eq!(
10346 decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("workflow output"),
10347 json!("finished")
10348 );
10349 }
10350 }
10351
10352 #[test]
10353 fn workflow_sleep_rejects_unrepresentable_rounded_duration() {
10354 let ctx = workflow_context(Vec::new());
10355 let mut sleep = Box::pin(ctx.start_timer(Duration::new(u64::MAX, 1)));
10356 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10357 assert!(matches!(
10358 sleep.as_mut().poll(&mut task_context),
10359 Poll::Ready(Err(Error::TimerDurationOverflow))
10360 ));
10361 assert!(ctx.take_commands().expect("commands").is_empty());
10362 }
10363
10364 #[test]
10365 fn workflow_task_replay_completes_without_rescheduling_recorded_commands() {
10366 let client = Client::new("http://127.0.0.1:8080").expect("client");
10367 let mut worker = Worker::new(client, "rust-workers");
10368 worker.register_workflow("rust.timer", |ctx, _input| async move {
10369 ctx.sleep(Duration::from_secs(5)).await?;
10370 ctx.activity("after-timer", json!([])).await
10371 });
10372
10373 let task = |history_events| WorkflowTask {
10374 task_id: "wft-rust-timer-1".to_string(),
10375 workflow_id: Some("wf-rust-timer".to_string()),
10376 run_id: Some("run-rust-timer".to_string()),
10377 workflow_type: "rust.timer".to_string(),
10378 payload_codec: JSON_CODEC.to_string(),
10379 arguments: Some(json!({"codec": "json", "blob": "[]"})),
10380 history_events,
10381 total_history_events: None,
10382 history_size_bytes: None,
10383 continue_as_new_recommended: None,
10384 history_budget_pressure: None,
10385 next_history_page_token: None,
10386 workflow_task_attempt: 1,
10387 workflow_signal_id: None,
10388 signal_name: None,
10389 signal_arguments: None,
10390 workflow_update_id: None,
10391 update_name: None,
10392 lease_owner: Some("rust-worker".to_string()),
10393 };
10394
10395 let initial = worker
10396 .execute_workflow_task(task(Vec::new()))
10397 .expect("initial timer task");
10398 assert_eq!(
10399 initial,
10400 vec![json!({"type": "start_timer", "delay_seconds": 5})]
10401 );
10402
10403 let replayed = worker
10404 .execute_workflow_task(task(vec![
10405 history_event(
10406 "TimerScheduled",
10407 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10408 ),
10409 history_event(
10410 "TimerFired",
10411 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10412 ),
10413 history_event(
10414 "ActivityCompleted",
10415 json!({
10416 "sequence": 2,
10417 "activity_type": "after-timer",
10418 "payload_codec": "json",
10419 "result": {"codec": "json", "blob": "\"done\""},
10420 }),
10421 ),
10422 ]))
10423 .expect("replayed workflow task");
10424 assert_eq!(replayed.len(), 1);
10425 assert_eq!(replayed[0]["type"], "complete_workflow");
10426 assert_eq!(
10427 decode_wire_value(&replayed[0]["result"], JSON_CODEC).expect("result"),
10428 json!("done")
10429 );
10430 }
10431
10432 #[test]
10433 fn workflow_continue_as_new_emits_arguments_type_and_queue_once() {
10434 let client = Client::new("http://127.0.0.1:8080").expect("client");
10435 let mut worker = Worker::new(client, "rust-workers");
10436 worker.register_workflow("rust.continue", |ctx, _input| async move {
10437 ctx.continue_as_new_with_options(
10438 ContinueAsNewOptions::new()
10439 .workflow_type("rust.next")
10440 .task_queue("next-workers"),
10441 json!([2, {"cursor": "next"}]),
10442 )
10443 });
10444
10445 let commands = worker
10446 .execute_workflow_task(workflow_task("rust.continue", Vec::new(), DEFAULT_CODEC))
10447 .expect("continue-as-new command");
10448
10449 assert_eq!(commands.len(), 1);
10450 assert_eq!(commands[0]["type"], "continue_as_new");
10451 assert_eq!(commands[0]["workflow_type"], "rust.next");
10452 assert_eq!(commands[0]["queue"], "next-workers");
10453 assert_eq!(
10454 decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC)
10455 .expect("continue-as-new arguments"),
10456 json!([2, {"cursor": "next"}])
10457 );
10458 }
10459
10460 #[test]
10461 fn continue_as_new_preserves_typed_arguments() {
10462 let client = Client::new("http://127.0.0.1:8080").expect("client");
10463 let mut worker = Worker::new(client, "rust-workers");
10464 worker.register_workflow_avro_value("rust.typed-continue", |ctx, _input| async move {
10465 ctx.continue_as_new(AvroValue::Array(vec![typed_fidelity_probe()]))?;
10466 unreachable!("continue-as-new returns a control-flow error")
10467 });
10468
10469 let commands = worker
10470 .execute_workflow_task(workflow_task(
10471 "rust.typed-continue",
10472 Vec::new(),
10473 DEFAULT_CODEC,
10474 ))
10475 .expect("typed continue-as-new command");
10476
10477 assert_eq!(commands[0]["type"], "continue_as_new");
10478 assert_eq!(
10479 decode_wire_avro_value(&commands[0]["arguments"], DEFAULT_CODEC)
10480 .expect("typed continue arguments"),
10481 AvroValue::Array(vec![typed_fidelity_probe()])
10482 );
10483 }
10484
10485 #[test]
10486 fn recorded_continue_as_new_is_consumed_without_duplicate_successor_command() {
10487 let client = Client::new("http://127.0.0.1:8080").expect("client");
10488 let mut worker = Worker::new(client, "rust-workers");
10489 worker.register_workflow("rust.continue", |ctx, _input| async move {
10490 ctx.continue_as_new(json!([2]))
10491 });
10492 let task = workflow_task(
10493 "rust.continue",
10494 vec![history_event(
10495 "WorkflowContinuedAsNew",
10496 json!({"sequence": 1, "continued_to_run_id": "run-next"}),
10497 )],
10498 JSON_CODEC,
10499 );
10500
10501 for _worker_restart_or_redelivery in 0..2 {
10502 let commands = worker
10503 .execute_workflow_task(task.clone())
10504 .expect("recorded transition replays");
10505 assert!(
10506 commands.is_empty(),
10507 "replay must not emit another successor"
10508 );
10509 }
10510 }
10511
10512 #[test]
10513 fn continue_as_new_rejects_invalid_overrides_before_emitting_a_command() {
10514 let ctx = workflow_context(Vec::new());
10515 let error = ctx
10516 .continue_as_new_with_options(ContinueAsNewOptions::new().task_queue(" "), json!([1]))
10517 .expect_err("blank queue must be rejected");
10518
10519 let Error::InvalidContinueAsNewOptions(error) = error else {
10520 panic!("expected typed continue-as-new validation error");
10521 };
10522 assert_eq!(error.field, "task_queue");
10523 assert!(ctx.take_commands().expect("commands").is_empty());
10524 }
10525
10526 #[test]
10527 fn workflow_context_exposes_server_history_budget() {
10528 let client = Client::new("http://127.0.0.1:8080").expect("client");
10529 let mut worker = Worker::new(client, "rust-workers");
10530 worker.register_workflow("rust.history-budget", |ctx, _input| async move {
10531 let budget = ctx.history_budget()?;
10532 Ok(json!({
10533 "events": budget.event_count,
10534 "bytes": budget.size_bytes,
10535 "recommended": budget.continue_as_new_recommended,
10536 "pressure": budget.pressure,
10537 }))
10538 });
10539 let task: WorkflowTask = serde_json::from_value(json!({
10540 "task_id": "task-history-budget",
10541 "workflow_type": "rust.history-budget",
10542 "payload_codec": JSON_CODEC,
10543 "history_events": [],
10544 "total_history_events": 480,
10545 "history_size_bytes": 1_048_576,
10546 "continue_as_new_recommended": true,
10547 "history_budget_pressure": "continue_as_new_recommended",
10548 }))
10549 .expect("published workflow task");
10550
10551 let commands = worker
10552 .execute_workflow_task(task)
10553 .expect("history-budget workflow");
10554 let result = decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("result");
10555 assert_eq!(result["events"], 480);
10556 assert_eq!(result["bytes"], 1_048_576);
10557 assert_eq!(result["recommended"], true);
10558 assert_eq!(result["pressure"], "continue_as_new_recommended");
10559 }
10560
10561 #[test]
10562 fn uncaught_workflow_handler_error_emits_terminal_failure_command() {
10563 let client = Client::new("http://127.0.0.1:8080").expect("client");
10564 let mut worker = Worker::new(client, "rust-workers");
10565 worker.register_workflow("rust.failing", |_ctx, _input| async move {
10566 Err(Error::Codec("rust_conformance_failure".to_string()))
10567 });
10568 let task = WorkflowTask {
10569 task_id: "wft-rust-failing-1".to_string(),
10570 workflow_id: Some("wf-rust-failing".to_string()),
10571 run_id: Some("run-rust-failing".to_string()),
10572 workflow_type: "rust.failing".to_string(),
10573 payload_codec: JSON_CODEC.to_string(),
10574 arguments: Some(encode_value_envelope(&json!([]), JSON_CODEC).expect("input")),
10575 history_events: Vec::new(),
10576 total_history_events: Some(0),
10577 history_size_bytes: None,
10578 continue_as_new_recommended: None,
10579 history_budget_pressure: None,
10580 next_history_page_token: None,
10581 workflow_task_attempt: 1,
10582 workflow_signal_id: None,
10583 signal_name: None,
10584 signal_arguments: None,
10585 workflow_update_id: None,
10586 update_name: None,
10587 lease_owner: Some("rust-worker".to_string()),
10588 };
10589
10590 let commands = worker
10591 .execute_workflow_task(task)
10592 .expect("handler failure becomes a workflow command");
10593
10594 assert_eq!(commands.len(), 1);
10595 assert_eq!(commands[0]["type"], "fail_workflow");
10596 assert_eq!(commands[0]["exception_type"], "RustWorkflowError");
10597 assert_eq!(commands[0]["exception_class"], "durable_workflow::Error");
10598 assert_eq!(commands[0]["non_retryable"], false);
10599 assert_eq!(
10600 commands[0]["message"],
10601 "codec error: rust_conformance_failure"
10602 );
10603 assert_eq!(
10604 commands[0]["exception"]["message"],
10605 "codec error: rust_conformance_failure"
10606 );
10607 }
10608
10609 #[test]
10610 fn ordinary_handler_error_preserves_commands_queued_in_the_same_decision() {
10611 let client = Client::new("http://127.0.0.1:8080").expect("client");
10612 let mut worker = Worker::new(client, "rust-workers");
10613 worker.register_workflow("rust.failing-after-side-effect", |ctx, _input| async move {
10614 let _: String = ctx.side_effect(|| "captured".to_string())?;
10615 Err(Error::WorkerLoop("application failure".to_string()))
10616 });
10617
10618 let commands = worker
10619 .execute_workflow_task(workflow_task(
10620 "rust.failing-after-side-effect",
10621 Vec::new(),
10622 JSON_CODEC,
10623 ))
10624 .expect("ordinary failure remains a workflow decision");
10625
10626 assert_eq!(commands.len(), 2);
10627 assert_eq!(commands[0]["type"], "record_side_effect");
10628 assert_eq!(commands[1]["type"], "fail_workflow");
10629 }
10630
10631 #[test]
10632 fn handler_error_cannot_hide_an_unconsumed_committed_side_effect() {
10633 let client = Client::new("http://127.0.0.1:8080").expect("client");
10634 let mut worker = Worker::new(client, "rust-workers");
10635 worker.register_workflow("rust.removed-side-effect", |_ctx, _input| async move {
10636 Err(Error::WorkerLoop("application failure".to_string()))
10637 });
10638 let result =
10639 encode_value_envelope(&json!("committed"), JSON_CODEC).expect("side-effect result");
10640
10641 let error = worker
10642 .execute_workflow_task(workflow_task(
10643 "rust.removed-side-effect",
10644 vec![history_event(
10645 "SideEffectRecorded",
10646 json!({"sequence": 1, "result": result}),
10647 )],
10648 JSON_CODEC,
10649 ))
10650 .expect_err("removed committed history must not become fail_workflow");
10651
10652 let Error::NonDeterministicReplay(failure) = error else {
10653 panic!("expected typed replay failure");
10654 };
10655 assert_eq!(failure.reason, "recorded_commands_unconsumed");
10656 assert_eq!(failure.sequence, Some(1));
10657 assert_eq!(failure.expected.as_deref(), Some("side effect"));
10658 }
10659
10660 #[test]
10661 fn replay_error_discards_side_effect_queued_before_incompatible_marker_check() {
10662 let client = Client::new("http://127.0.0.1:8080").expect("client");
10663 let mut worker = Worker::new(client, "rust-workers");
10664 worker.register_workflow(
10665 "rust.side-effect-before-marker-error",
10666 |ctx, _input| async move {
10667 assert_eq!(ctx.get_version("restart-safe", 1, 1)?, 1);
10668 let _: String = ctx.side_effect(|| "must-not-commit".to_string())?;
10669 ctx.get_version("restart-safe", 2, 2)?;
10670 Ok(Value::Null)
10671 },
10672 );
10673
10674 let error = worker
10675 .execute_workflow_task(workflow_task(
10676 "rust.side-effect-before-marker-error",
10677 vec![history_event(
10678 "VersionMarkerRecorded",
10679 json!({
10680 "sequence": 1,
10681 "change_id": "restart-safe",
10682 "version": 1,
10683 "min_supported": 1,
10684 "max_supported": 1,
10685 }),
10686 )],
10687 JSON_CODEC,
10688 ))
10689 .expect_err("replay error must return no queued workflow commands");
10690
10691 let Error::NonDeterministicReplay(failure) = error else {
10692 panic!("expected typed replay failure");
10693 };
10694 assert_eq!(failure.reason, "version_marker_incompatible_range");
10695 assert_eq!(failure.sequence, Some(1));
10696 }
10697
10698 #[test]
10699 fn workflow_task_replay_keeps_recorded_unfired_timer_pending_without_rescheduling() {
10700 let client = Client::new("http://127.0.0.1:8080").expect("client");
10701 let mut worker = Worker::new(client, "rust-workers");
10702 worker.register_workflow("rust.timer.pending", |ctx, _input| async move {
10703 ctx.sleep(Duration::from_secs(5)).await?;
10704 Ok(json!({"status": "timer fired"}))
10705 });
10706
10707 let task = WorkflowTask {
10708 task_id: "wft-rust-timer-pending".to_string(),
10709 workflow_id: Some("wf-rust-timer".to_string()),
10710 run_id: Some("run-rust-timer".to_string()),
10711 workflow_type: "rust.timer.pending".to_string(),
10712 payload_codec: JSON_CODEC.to_string(),
10713 arguments: Some(json!({"codec": "json", "blob": "[]"})),
10714 history_events: vec![history_event(
10715 "TimerScheduled",
10716 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10717 )],
10718 total_history_events: Some(1),
10719 history_size_bytes: None,
10720 continue_as_new_recommended: None,
10721 history_budget_pressure: None,
10722 next_history_page_token: None,
10723 workflow_task_attempt: 1,
10724 workflow_signal_id: None,
10725 signal_name: None,
10726 signal_arguments: None,
10727 workflow_update_id: None,
10728 update_name: None,
10729 lease_owner: Some("rust-worker".to_string()),
10730 };
10731
10732 for _redelivery_or_restart in 0..2 {
10733 let commands = worker
10734 .execute_workflow_task(task.clone())
10735 .expect("recorded timer remains pending");
10736 assert!(
10737 commands.is_empty(),
10738 "recorded timer must not be rescheduled"
10739 );
10740 }
10741 }
10742
10743 #[test]
10744 fn workflow_task_rejects_recorded_command_removed_from_workflow_code() {
10745 let client = Client::new("http://127.0.0.1:8080").expect("client");
10746 let mut worker = Worker::new(client, "rust-workers");
10747 worker.register_workflow("rust.timer.removed", |_ctx, _input| async move {
10748 Ok(json!({"status": "completed"}))
10749 });
10750 let task = WorkflowTask {
10751 task_id: "wft-rust-timer-removed".to_string(),
10752 workflow_id: Some("wf-rust-timer".to_string()),
10753 run_id: Some("run-rust-timer".to_string()),
10754 workflow_type: "rust.timer.removed".to_string(),
10755 payload_codec: JSON_CODEC.to_string(),
10756 arguments: Some(json!({"codec": "json", "blob": "[]"})),
10757 history_events: vec![
10758 history_event(
10759 "TimerScheduled",
10760 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10761 ),
10762 history_event(
10763 "TimerFired",
10764 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10765 ),
10766 ],
10767 total_history_events: Some(2),
10768 history_size_bytes: None,
10769 continue_as_new_recommended: None,
10770 history_budget_pressure: None,
10771 next_history_page_token: None,
10772 workflow_task_attempt: 1,
10773 workflow_signal_id: None,
10774 signal_name: None,
10775 signal_arguments: None,
10776 workflow_update_id: None,
10777 update_name: None,
10778 lease_owner: Some("rust-worker".to_string()),
10779 };
10780
10781 let Error::NonDeterministicReplay(failure) = worker
10782 .execute_workflow_task(task)
10783 .expect_err("removed timer must fail replay")
10784 else {
10785 panic!("expected typed replay failure");
10786 };
10787 assert_eq!(failure.reason, "recorded_commands_unconsumed");
10788 assert_eq!(failure.sequence, Some(1));
10789 }
10790
10791 #[test]
10792 fn workflow_context_emits_explicit_child_workflow_contract() {
10793 let ctx = WorkflowContext {
10794 state: Arc::new(Mutex::new(
10795 WorkflowState::new_with_identity(
10796 Vec::new(),
10797 Some("wf-parent".to_string()),
10798 Some("run-parent".to_string()),
10799 "parent-workers".to_string(),
10800 JSON_CODEC.to_string(),
10801 None,
10802 )
10803 .expect("workflow state"),
10804 )),
10805 };
10806 let options = ChildWorkflowOptions::new("python-workers")
10807 .parent_close_policy(ParentClosePolicy::RequestCancel)
10808 .retry_policy(ChildWorkflowRetryPolicy {
10809 max_attempts: Some(3),
10810 backoff_seconds: vec![1, 5],
10811 non_retryable_error_types: vec!["ValidationError".to_string()],
10812 })
10813 .execution_timeout_seconds(600)
10814 .run_timeout_seconds(120);
10815 let mut call = Box::pin(ctx.start_child_workflow(
10816 "python.fulfil-order",
10817 options,
10818 json!([{"order_id": "order-42"}]),
10819 ));
10820 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10821
10822 assert!(matches!(
10823 call.as_mut().poll(&mut task_context),
10824 Poll::Pending
10825 ));
10826 let commands = ctx.take_commands().expect("commands");
10827 assert_eq!(commands.len(), 1);
10828 let command = &commands[0];
10829 assert_eq!(command["type"], "start_child_workflow");
10830 assert_eq!(command["workflow_type"], "python.fulfil-order");
10831 assert_eq!(command["queue"], "python-workers");
10832 assert_eq!(command["parent_close_policy"], "request_cancel");
10833 assert_eq!(command["retry_policy"]["max_attempts"], 3);
10834 assert_eq!(command["execution_timeout_seconds"], 600);
10835 assert_eq!(command["run_timeout_seconds"], 120);
10836 assert_eq!(
10837 decode_wire_value(&command["arguments"], JSON_CODEC).expect("child args"),
10838 json!([{"order_id": "order-42"}])
10839 );
10840 }
10841
10842 fn child_parent_worker() -> Worker {
10843 let client = Client::new("http://127.0.0.1:8080").expect("client");
10844 let mut worker = Worker::new(client, "rust-parent-workers");
10845 worker.register_workflow("rust.parent", |ctx, _input| async move {
10846 let child = ctx
10847 .start_child_workflow(
10848 "python.child",
10849 ChildWorkflowOptions::new("python-child-workers")
10850 .parent_close_policy(ParentClosePolicy::Terminate),
10851 json!([{"codec_probe": [1, true, "rust"]}]),
10852 )
10853 .await?;
10854 Ok(json!({
10855 "parent_workflow_id": child.parent.workflow_id,
10856 "parent_run_id": child.parent.run_id,
10857 "child_workflow_id": child.child.workflow_id,
10858 "child_run_id": child.child.run_id,
10859 "child_workflow_type": child.child_workflow_type,
10860 "result": child.result,
10861 }))
10862 });
10863 worker
10864 }
10865
10866 fn child_parent_task(event_type: &str, payload: Value) -> WorkflowTask {
10867 WorkflowTask {
10868 task_id: "wft-child-parent".to_string(),
10869 workflow_id: Some("wf-parent".to_string()),
10870 run_id: Some("run-parent".to_string()),
10871 workflow_type: "rust.parent".to_string(),
10872 payload_codec: JSON_CODEC.to_string(),
10873 arguments: Some(encode_value_envelope(&json!([]), JSON_CODEC).expect("input")),
10874 history_events: vec![
10875 HistoryEvent {
10876 event_type: "ChildWorkflowScheduled".to_string(),
10877 payload: json!({
10878 "sequence": 1,
10879 "child_call_id": "call-child",
10880 "child_workflow_instance_id": "wf-child",
10881 "child_workflow_run_id": "run-child",
10882 "child_workflow_type": "python.child",
10883 }),
10884 raw: HashMap::new(),
10885 },
10886 HistoryEvent {
10887 event_type: event_type.to_string(),
10888 payload,
10889 raw: HashMap::new(),
10890 },
10891 ],
10892 total_history_events: Some(2),
10893 history_size_bytes: None,
10894 continue_as_new_recommended: None,
10895 history_budget_pressure: None,
10896 next_history_page_token: None,
10897 workflow_task_attempt: 1,
10898 workflow_signal_id: None,
10899 signal_name: None,
10900 signal_arguments: None,
10901 workflow_update_id: None,
10902 update_name: None,
10903 lease_owner: Some("rust-worker".to_string()),
10904 }
10905 }
10906
10907 #[test]
10908 fn committed_child_result_replays_without_starting_a_duplicate() {
10909 let worker = child_parent_worker();
10910 let task = child_parent_task(
10911 "ChildRunCompleted",
10912 json!({
10913 "sequence": 1,
10914 "child_call_id": "call-child",
10915 "child_workflow_instance_id": "wf-child",
10916 "child_workflow_run_id": "run-child",
10917 "child_workflow_type": "python.child",
10918 "payload_codec": "json",
10919 "result": {"codec": "json", "blob": "{\"from\":\"python\",\"ok\":true}"},
10920 }),
10921 );
10922
10923 for _restart in 0..2 {
10924 let commands = worker
10925 .execute_workflow_task(task.clone())
10926 .expect("replayed parent task");
10927 assert_eq!(commands.len(), 1);
10928 assert_eq!(commands[0]["type"], "complete_workflow");
10929 assert!(!commands
10930 .iter()
10931 .any(|command| command["type"] == "start_child_workflow"));
10932 let output =
10933 decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("parent output");
10934 assert_eq!(output["parent_workflow_id"], "wf-parent");
10935 assert_eq!(output["parent_run_id"], "run-parent");
10936 assert_eq!(output["child_workflow_id"], "wf-child");
10937 assert_eq!(output["child_run_id"], "run-child");
10938 assert_eq!(output["result"], json!({"from": "python", "ok": true}));
10939 }
10940 }
10941
10942 #[test]
10943 fn typed_child_arguments_and_results_survive_replay() {
10944 let client = Client::new("http://127.0.0.1:8080").expect("client");
10945 let mut worker = Worker::new(client, "rust-parent-workers");
10946 worker.register_workflow_avro_value("rust.typed-parent", |ctx, _input| async move {
10947 let child = ctx
10948 .start_child_workflow_avro_value(
10949 "python.typed-child",
10950 ChildWorkflowOptions::new("python-workers"),
10951 AvroValue::Array(vec![typed_fidelity_probe()]),
10952 )
10953 .await?;
10954 Ok(child.result)
10955 });
10956
10957 let initial = worker
10958 .execute_workflow_task(workflow_task(
10959 "rust.typed-parent",
10960 Vec::new(),
10961 DEFAULT_CODEC,
10962 ))
10963 .expect("typed child start");
10964 assert_eq!(initial[0]["type"], "start_child_workflow");
10965 assert_eq!(
10966 decode_wire_avro_value(&initial[0]["arguments"], DEFAULT_CODEC)
10967 .expect("typed child arguments"),
10968 AvroValue::Array(vec![typed_fidelity_probe()])
10969 );
10970
10971 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
10972 .expect("typed child result");
10973 let task = workflow_task(
10974 "rust.typed-parent",
10975 vec![
10976 history_event(
10977 "ChildWorkflowScheduled",
10978 json!({
10979 "sequence": 1,
10980 "child_call_id": "call-typed",
10981 "child_workflow_instance_id": "wf-child",
10982 "child_workflow_run_id": "run-child",
10983 "child_workflow_type": "python.typed-child",
10984 }),
10985 ),
10986 history_event(
10987 "ChildRunCompleted",
10988 json!({
10989 "sequence": 1,
10990 "child_call_id": "call-typed",
10991 "child_workflow_instance_id": "wf-child",
10992 "child_workflow_run_id": "run-child",
10993 "child_workflow_type": "python.typed-child",
10994 "payload_codec": DEFAULT_CODEC,
10995 "result": result,
10996 }),
10997 ),
10998 ],
10999 DEFAULT_CODEC,
11000 );
11001
11002 let commands = worker
11003 .execute_workflow_task(task)
11004 .expect("typed child replay");
11005 assert_eq!(commands[0]["type"], "complete_workflow");
11006 assert_eq!(
11007 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
11008 .expect("typed parent result"),
11009 typed_fidelity_probe()
11010 );
11011 }
11012
11013 #[test]
11014 fn pending_child_replays_after_restart_without_starting_a_duplicate() {
11015 let worker = child_parent_worker();
11016 let mut task = child_parent_task("unused", Value::Null);
11017 task.history_events.truncate(1);
11018 task.total_history_events = Some(1);
11019
11020 for _redelivery_or_restart in 0..2 {
11021 let commands = worker
11022 .execute_workflow_task(task.clone())
11023 .expect("recorded child remains pending");
11024 assert!(
11025 commands.is_empty(),
11026 "recorded pending child must not be started again"
11027 );
11028 }
11029 }
11030
11031 #[test]
11032 fn child_cancellation_becomes_stable_parent_failure_command() {
11033 let worker = child_parent_worker();
11034 let task = child_parent_task(
11035 "ChildRunCancelled",
11036 json!({
11037 "sequence": 1,
11038 "child_workflow_instance_id": "wf-child",
11039 "child_workflow_run_id": "run-child",
11040 "child_workflow_type": "python.child",
11041 "failure_id": "failure-child",
11042 "failure_category": "cancelled",
11043 "message": "cancelled by parent-close policy",
11044 }),
11045 );
11046
11047 let commands = worker
11048 .execute_workflow_task(task)
11049 .expect("parent settlement");
11050 assert_eq!(commands.len(), 1);
11051 assert_eq!(commands[0]["type"], "fail_workflow");
11052 assert_eq!(commands[0]["exception_type"], "ChildWorkflowCancelled");
11053 assert_eq!(
11054 commands[0]["exception"]["properties"]["reason"],
11055 "cancelled"
11056 );
11057 assert_eq!(
11058 commands[0]["exception"]["properties"]["child_workflow_run_id"],
11059 "run-child"
11060 );
11061 }
11062
11063 #[test]
11064 fn workflow_can_handle_typed_child_failure() {
11065 let client = Client::new("http://127.0.0.1:8080").expect("client");
11066 let mut worker = Worker::new(client, "rust-parent-workers");
11067 worker.register_workflow("rust.handled-parent", |ctx, _input| async move {
11068 match ctx
11069 .start_child_workflow(
11070 "python.child",
11071 ChildWorkflowOptions::new("python-child-workers"),
11072 json!([]),
11073 )
11074 .await
11075 {
11076 Err(Error::ChildWorkflowFailed(failure)) => Ok(json!({
11077 "reason": failure.reason,
11078 "failure_id": failure.failure_id,
11079 "exception_class": failure.exception_class,
11080 "child_run_id": failure.child_workflow_run_id,
11081 })),
11082 Err(error) => Err(error),
11083 Ok(_) => Err(Error::WorkerLoop(
11084 "child unexpectedly succeeded".to_string(),
11085 )),
11086 }
11087 });
11088 let mut task = child_parent_task(
11089 "ChildRunFailed",
11090 json!({
11091 "sequence": 1,
11092 "child_workflow_instance_id": "wf-child",
11093 "child_workflow_run_id": "run-child",
11094 "child_workflow_type": "python.child",
11095 "failure_id": "failure-child",
11096 "failure_category": "child_workflow",
11097 "message": "payment rejected",
11098 "exception": {
11099 "type": "PaymentRejected",
11100 "class": "payments.PaymentRejected",
11101 "message": "payment rejected"
11102 }
11103 }),
11104 );
11105 task.workflow_type = "rust.handled-parent".to_string();
11106
11107 let commands = worker.execute_workflow_task(task).expect("handled failure");
11108 assert_eq!(commands[0]["type"], "complete_workflow");
11109 let output = decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("parent output");
11110 assert_eq!(output["reason"], "child_workflow");
11111 assert_eq!(output["failure_id"], "failure-child");
11112 assert_eq!(output["exception_class"], "payments.PaymentRejected");
11113 assert_eq!(output["child_run_id"], "run-child");
11114 }
11115
11116 #[test]
11117 fn rust_hello_world_uses_signal_arguments_from_resume_payload() {
11118 let client = Client::new("http://127.0.0.1:8080").expect("client");
11119 let mut worker = Worker::new(client, "rust-workers");
11120
11121 worker.register_workflow("rust.hello_workflow", |ctx, _input| async move {
11122 let signal = ctx.wait_signal("start").await?;
11123 let name = signal
11124 .first()
11125 .and_then(|value| value.as_str())
11126 .unwrap_or("world");
11127 let greeting = ctx.activity("rust.hello_activity", json!([name])).await?;
11128 Ok(json!({
11129 "greeting": greeting,
11130 "language": "rust"
11131 }))
11132 });
11133
11134 let signal_arguments =
11135 encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC).expect("signal arguments");
11136 let task = WorkflowTask {
11137 task_id: "wft-rust-signal-1".to_string(),
11138 workflow_id: Some("wf-rust-hello".to_string()),
11139 run_id: Some("run-rust-hello".to_string()),
11140 workflow_type: "rust.hello_workflow".to_string(),
11141 payload_codec: DEFAULT_CODEC.to_string(),
11142 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
11143 history_events: vec![HistoryEvent {
11144 event_type: "SignalReceived".to_string(),
11145 payload: json!({
11146 "signal_id": "sig-rust-1",
11147 "signal_name": "start"
11148 }),
11149 raw: HashMap::new(),
11150 }],
11151 total_history_events: Some(1),
11152 history_size_bytes: None,
11153 continue_as_new_recommended: None,
11154 history_budget_pressure: None,
11155 next_history_page_token: None,
11156 workflow_task_attempt: 1,
11157 workflow_signal_id: Some("sig-rust-1".to_string()),
11158 signal_name: Some("start".to_string()),
11159 signal_arguments: Some(signal_arguments),
11160 workflow_update_id: None,
11161 update_name: None,
11162 lease_owner: Some("rust-worker".to_string()),
11163 };
11164
11165 let commands = worker.execute_workflow_task(task).expect("workflow task");
11166
11167 assert_eq!(commands.len(), 1);
11168 assert_eq!(commands[0]["type"], "schedule_activity");
11169 assert_eq!(commands[0]["activity_type"], "rust.hello_activity");
11170 assert_eq!(
11171 decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC).expect("activity args"),
11172 json!(["Rust"])
11173 );
11174 }
11175
11176 #[test]
11177 fn workflow_task_appends_paginated_history_events() {
11178 let mut task = WorkflowTask {
11179 task_id: "wft-rust-pages-1".to_string(),
11180 workflow_id: Some("wf-rust-pages".to_string()),
11181 run_id: Some("run-rust-pages".to_string()),
11182 workflow_type: "rust.hello_workflow".to_string(),
11183 payload_codec: DEFAULT_CODEC.to_string(),
11184 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
11185 history_events: vec![HistoryEvent {
11186 event_type: "WorkflowStarted".to_string(),
11187 payload: json!({}),
11188 raw: HashMap::new(),
11189 }],
11190 total_history_events: Some(3),
11191 history_size_bytes: None,
11192 continue_as_new_recommended: None,
11193 history_budget_pressure: None,
11194 next_history_page_token: Some("MQ==".to_string()),
11195 workflow_task_attempt: 1,
11196 workflow_signal_id: None,
11197 signal_name: None,
11198 signal_arguments: None,
11199 workflow_update_id: None,
11200 update_name: None,
11201 lease_owner: Some("rust-worker".to_string()),
11202 };
11203
11204 task.append_history_page(WorkflowTaskHistoryPage {
11205 history_events: vec![
11206 HistoryEvent {
11207 event_type: "SignalReceived".to_string(),
11208 payload: json!({
11209 "signal_id": "sig-rust-1",
11210 "signal_name": "start",
11211 "arguments": encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC)
11212 .expect("signal arguments")
11213 }),
11214 raw: HashMap::new(),
11215 },
11216 HistoryEvent {
11217 event_type: "MarkerRecorded".to_string(),
11218 payload: json!({"sequence": 3}),
11219 raw: HashMap::new(),
11220 },
11221 ],
11222 total_history_events: Some(3),
11223 next_history_page_token: None,
11224 });
11225
11226 assert_eq!(task.history_events.len(), 3);
11227 assert_eq!(task.total_history_events, Some(3));
11228 assert_eq!(task.next_history_page_token, None);
11229
11230 let signal = task
11231 .history_events
11232 .iter()
11233 .find(|event| event.event_type == "SignalReceived")
11234 .expect("signal event");
11235 assert_eq!(
11236 decode_signal_event_arguments(signal, DEFAULT_CODEC).expect("signal arguments"),
11237 vec![AvroValue::String("Rust".to_string())]
11238 );
11239 }
11240
11241 #[tokio::test]
11242 async fn query_handler_reads_ordered_cross_codec_signals_without_commands() {
11243 let client = Client::new("http://127.0.0.1:8080").expect("client");
11244 let mut worker = Worker::new(client, "rust-workers");
11245 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
11246 worker.register_query("counter", "current", |ctx, _args| async move {
11247 let mut count = 0_i64;
11248 for signal in ctx.signal_events() {
11249 let value = signal
11250 .arguments
11251 .first()
11252 .and_then(Value::as_i64)
11253 .unwrap_or_default();
11254 match signal.name.as_str() {
11255 "increment" => count += value,
11256 "set" => count = value,
11257 _ => {}
11258 }
11259 }
11260 Ok(json!(count))
11261 });
11262
11263 let task = QueryTask {
11264 query_task_id: "query-rust-counter".to_string(),
11265 query_task_attempt: 1,
11266 lease_owner: Some("rust-worker".to_string()),
11267 workflow_id: Some("counter-1".to_string()),
11268 run_id: Some("run-counter-1".to_string()),
11269 workflow_type: "counter".to_string(),
11270 query_name: "current".to_string(),
11271 payload_codec: DEFAULT_CODEC.to_string(),
11272 workflow_arguments: Some(
11273 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
11274 ),
11275 query_arguments: Some(
11276 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("query arguments"),
11277 ),
11278 history_events: vec![
11279 HistoryEvent {
11280 event_type: "SignalReceived".to_string(),
11281 payload: json!({
11282 "signal_id": "php-signal-1",
11283 "signal_name": "increment",
11284 "workflow_sequence": 1,
11285 "payload_codec": DEFAULT_CODEC,
11286 "arguments": encode_value_envelope(&json!([3]), DEFAULT_CODEC).expect("php avro signal")
11287 }),
11288 raw: HashMap::new(),
11289 },
11290 HistoryEvent {
11291 event_type: "SignalReceived".to_string(),
11292 payload: json!({
11293 "signal_id": "python-signal-2",
11294 "signal_name": "increment",
11295 "workflow_sequence": 2,
11296 "payload_codec": JSON_CODEC,
11297 "arguments": encode_value_envelope(&json!([5]), JSON_CODEC).expect("python json signal")
11298 }),
11299 raw: HashMap::new(),
11300 },
11301 HistoryEvent {
11302 event_type: "SignalReceived".to_string(),
11303 payload: json!({
11304 "signal_id": "rust-signal-3",
11305 "signal_name": "set",
11306 "workflow_sequence": 3,
11307 "payload_codec": DEFAULT_CODEC,
11308 "arguments": encode_value_envelope(&json!([0]), DEFAULT_CODEC).expect("rust avro signal")
11309 }),
11310 raw: HashMap::new(),
11311 },
11312 ],
11313 history_export: None,
11314 run_status: Some("completed".to_string()),
11315 };
11316
11317 let result = worker.execute_query_task(task).await.expect("query result");
11318 assert_eq!(result.into_json().expect("query projection"), json!(0));
11319 }
11320
11321 #[tokio::test]
11322 async fn replayed_queries_read_running_completed_and_cold_restarted_instance_state() {
11323 let worker = replay_counter_worker();
11324 let running_history = json!([
11325 {
11326 "type": "ActivityCompleted",
11327 "payload": {
11328 "sequence": 1,
11329 "activity_type": "load-counter",
11330 "payload_codec": "json",
11331 "result": {"codec": "json", "blob": "\"loaded\""}
11332 }
11333 },
11334 {
11335 "type": "SignalWaitOpened",
11336 "payload": {
11337 "sequence": 3,
11338 "signal_name": "increment"
11339 }
11340 },
11341 {
11342 "type": "SignalReceived",
11343 "payload": {
11344 "signal_id": "signal-3",
11345 "signal_name": "increment",
11346 "workflow_sequence": 2,
11347 "payload_codec": "json",
11348 "arguments": {"codec": "json", "blob": "[3]"}
11349 }
11350 },
11351 {
11352 "type": "SignalApplied",
11353 "payload": {
11354 "sequence": 3,
11355 "signal_id": "signal-3",
11356 "signal_name": "increment",
11357 "payload_codec": "json",
11358 "value": {"codec": "json", "blob": "[3]"}
11359 }
11360 }
11361 ]);
11362
11363 let running = worker
11364 .execute_query_task(replay_counter_query(
11365 "current",
11366 running_history.clone(),
11367 "running",
11368 ))
11369 .await
11370 .expect("running replay query");
11371 assert_eq!(
11372 running.clone().into_json().expect("query projection"),
11373 json!({"loaded": "loaded", "count": 3, "finished": false})
11374 );
11375
11376 let detached = worker
11377 .execute_query_task(replay_counter_query(
11378 "detached-mutation",
11379 running_history.clone(),
11380 "running",
11381 ))
11382 .await
11383 .expect("query mutates only its detached state clone");
11384 assert_eq!(detached.into_json().expect("query projection"), json!(999));
11385 let failed = worker
11386 .execute_query_task(replay_counter_query(
11387 "failed-mutation",
11388 running_history.clone(),
11389 "running",
11390 ))
11391 .await
11392 .expect_err("failed query");
11393 assert_eq!(failed.reason, "query_rejected");
11394 let unchanged = worker
11395 .execute_query_task(replay_counter_query("current", running_history, "running"))
11396 .await
11397 .expect("later query reconstructs unchanged state");
11398 assert_eq!(unchanged, running);
11399
11400 let restarted_worker = replay_counter_worker();
11401 let restarted_task: QueryTask = serde_json::from_value(json!({
11402 "query_task_id": "query-after-restart",
11403 "workflow_id": "counter-1",
11404 "run_id": "run-counter-1",
11405 "workflow_type": "replay-counter",
11406 "query_name": "current",
11407 "payload_codec": "json",
11408 "workflow_arguments": {"codec": "json", "blob": "[]"},
11409 "query_arguments": {"codec": "json", "blob": "[]"},
11410 "history_events": [],
11411 "history_export": {
11412 "payloads": {"codec": "json"},
11413 "history_events": [
11414 {
11415 "type": "ActivityCompleted",
11416 "payload": {
11417 "sequence": 1,
11418 "activity_type": "load-counter",
11419 "payload_codec": "json",
11420 "result": null
11421 }
11422 },
11423 {
11424 "type": "SignalWaitOpened",
11425 "payload": {
11426 "sequence": 3,
11427 "signal_name": "increment"
11428 }
11429 },
11430 {
11431 "type": "SignalReceived",
11432 "payload": {
11433 "signal_id": "signal-3",
11434 "signal_name": "increment",
11435 "workflow_sequence": 2
11436 }
11437 },
11438 {
11439 "type": "SignalApplied",
11440 "payload": {
11441 "sequence": 3,
11442 "signal_id": "signal-3",
11443 "signal_name": "increment"
11444 }
11445 },
11446 {
11447 "type": "SignalWaitOpened",
11448 "payload": {
11449 "sequence": 5,
11450 "signal_name": "increment"
11451 }
11452 },
11453 {
11454 "type": "SignalReceived",
11455 "payload": {
11456 "signal_id": "signal-5",
11457 "signal_name": "increment",
11458 "workflow_sequence": 4
11459 }
11460 },
11461 {
11462 "type": "SignalApplied",
11463 "payload": {
11464 "sequence": 5,
11465 "signal_id": "signal-5",
11466 "signal_name": "increment"
11467 }
11468 }
11469 ],
11470 "activities": [{
11471 "sequence": 1,
11472 "activity_type": "load-counter",
11473 "payload_codec": "json",
11474 "result": {"codec": "json", "blob": "\"loaded\""}
11475 }],
11476 "signals": [
11477 {
11478 "id": "signal-3",
11479 "name": "increment",
11480 "workflow_sequence": 2,
11481 "payload_codec": "json",
11482 "arguments": "[3]"
11483 },
11484 {
11485 "id": "signal-5",
11486 "name": "increment",
11487 "workflow_sequence": 4,
11488 "payload_codec": "json",
11489 "arguments": "[5]"
11490 }
11491 ]
11492 },
11493 "run_status": "completed"
11494 }))
11495 .expect("cold replay query task");
11496 let completed = restarted_worker
11497 .execute_query_task(restarted_task)
11498 .await
11499 .expect("completed cold replay query");
11500 assert_eq!(
11501 completed.into_json().expect("query projection"),
11502 json!({"loaded": "loaded", "count": 8, "finished": true})
11503 );
11504 }
11505
11506 #[tokio::test]
11507 async fn replayed_query_replay_failures_are_machine_readable() {
11508 let worker = replay_counter_worker();
11509 let task = replay_counter_query(
11510 "current",
11511 json!([{
11512 "type": "ActivityCompleted",
11513 "payload": {
11514 "sequence": 1,
11515 "payload_codec": "json",
11516 "result": {"codec": "json", "blob": "{"}
11517 }
11518 }]),
11519 "running",
11520 );
11521 let failure = worker
11522 .execute_query_task(task)
11523 .await
11524 .expect_err("invalid replay history payload");
11525 assert_eq!(failure.reason, "query_workflow_state_unavailable");
11526 assert_eq!(failure.failure_type, "QueryWorkflowStateUnavailable");
11527 }
11528
11529 #[tokio::test]
11530 async fn query_task_restores_compact_history_from_export() {
11531 let client = Client::new("http://127.0.0.1:8080").expect("client");
11532 let mut worker = Worker::new(client, "rust-workers");
11533 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
11534 worker.register_query("counter", "current", |ctx, _args| async move {
11535 Ok(json!(ctx.signals("increment")[0][0]))
11536 });
11537 let task: QueryTask = serde_json::from_value(json!({
11538 "query_task_id": "query-export",
11539 "workflow_type": "counter",
11540 "query_name": "current",
11541 "payload_codec": "json",
11542 "workflow_arguments": {"codec": "json", "blob": "[]"},
11543 "query_arguments": {"codec": "json", "blob": "[]"},
11544 "history_events": [],
11545 "history_export": {
11546 "payloads": {"codec": "json"},
11547 "history_events": [{
11548 "type": "SignalReceived",
11549 "payload": {"signal_id": "signal-export", "signal_name": "increment"}
11550 }],
11551 "signals": [{
11552 "id": "signal-export",
11553 "name": "increment",
11554 "status": "applied",
11555 "workflow_sequence": 1,
11556 "payload_codec": "json",
11557 "arguments": "[9]"
11558 }]
11559 }
11560 }))
11561 .expect("query task");
11562
11563 let result = worker.execute_query_task(task).await.expect("query result");
11564 assert_eq!(result.into_json().expect("query projection"), json!(9));
11565 }
11566
11567 #[tokio::test]
11568 async fn query_task_failures_have_stable_reasons() {
11569 let client = Client::new("http://127.0.0.1:8080").expect("client");
11570 let mut worker = Worker::new(client, "rust-workers");
11571 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
11572 worker.register_query(
11573 "counter",
11574 "current",
11575 |_ctx, _args| async move { Ok(json!(0)) },
11576 );
11577
11578 let base_task = QueryTask {
11579 query_task_id: "query-errors".to_string(),
11580 query_task_attempt: 1,
11581 lease_owner: None,
11582 workflow_id: Some("counter-errors".to_string()),
11583 run_id: Some("run-errors".to_string()),
11584 workflow_type: "counter".to_string(),
11585 query_name: "missing".to_string(),
11586 payload_codec: JSON_CODEC.to_string(),
11587 workflow_arguments: Some(json!({"codec": "json", "blob": "[]"})),
11588 query_arguments: Some(json!({"codec": "json", "blob": "[]"})),
11589 history_events: Vec::new(),
11590 history_export: None,
11591 run_status: Some("running".to_string()),
11592 };
11593
11594 let unknown = worker
11595 .execute_query_task(base_task.clone())
11596 .await
11597 .expect_err("unknown query");
11598 assert_eq!(unknown.reason, "rejected_unknown_query");
11599
11600 let mut malformed = base_task;
11601 malformed.query_name = "current".to_string();
11602 malformed.query_arguments = Some(json!({"codec": "json", "blob": "{"}));
11603 let malformed = worker
11604 .execute_query_task(malformed)
11605 .await
11606 .expect_err("malformed payload");
11607 assert_eq!(malformed.reason, "query_payload_decode_failed");
11608
11609 let client = Client::new("http://127.0.0.1:8080").expect("client");
11610 let mut unavailable_worker = Worker::new(client, "rust-workers");
11611 unavailable_worker
11612 .register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
11613 let unavailable_task: QueryTask = serde_json::from_value(json!({
11614 "query_task_id": "query-unavailable",
11615 "workflow_type": "counter",
11616 "query_name": "current",
11617 "payload_codec": "json",
11618 "workflow_arguments": {"codec": "json", "blob": "[]"},
11619 "query_arguments": {"codec": "json", "blob": "[]"}
11620 }))
11621 .expect("query task");
11622 let unavailable = unavailable_worker
11623 .execute_query_task(unavailable_task)
11624 .await
11625 .expect_err("query handler unavailable");
11626 assert_eq!(unavailable.reason, "query_handler_unavailable");
11627 }
11628
11629 #[tokio::test]
11630 async fn client_query_decodes_result_and_typed_failure() {
11631 let server = MockWorkerServer::start();
11632 let client = Client::builder(server.base_url())
11633 .timeout(Duration::from_secs(2))
11634 .build()
11635 .expect("client");
11636
11637 let result = client
11638 .query_workflow("counter-1", "current", json!([]))
11639 .await
11640 .expect("query result");
11641 assert_eq!(result, json!({"count": 8}));
11642
11643 let error = client
11644 .query_workflow("counter-1", "missing", json!([]))
11645 .await
11646 .expect_err("unknown query");
11647 let Error::QueryFailed(failure) = error else {
11648 panic!("expected typed query failure");
11649 };
11650 assert_eq!(failure.status, 404);
11651 assert_eq!(failure.reason, "rejected_unknown_query");
11652 }
11653
11654 #[tokio::test]
11655 async fn public_client_surfaces_send_and_receive_lossless_avro_values() {
11656 let server = MockWorkerServer::start();
11657 let client = Client::builder(server.base_url())
11658 .timeout(Duration::from_secs(2))
11659 .build()
11660 .expect("client");
11661 let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
11662
11663 client
11664 .start_workflow(
11665 "typed.echo",
11666 "rust-workers",
11667 "typed-start",
11668 arguments.clone(),
11669 )
11670 .await
11671 .expect("typed workflow start");
11672 assert_eq!(
11673 decode_wire_avro_value(
11674 &server.request_body("/api/workflows")["input"],
11675 DEFAULT_CODEC,
11676 )
11677 .expect("typed start input"),
11678 arguments
11679 );
11680
11681 client
11682 .signal_workflow("typed-1", "changed", arguments.clone())
11683 .await
11684 .expect("typed signal");
11685 assert_eq!(
11686 decode_wire_avro_value(
11687 &server.request_body("/api/workflows/typed-1/signal/changed")["input"],
11688 DEFAULT_CODEC,
11689 )
11690 .expect("typed signal input"),
11691 arguments
11692 );
11693
11694 assert_eq!(
11695 client
11696 .query_workflow_avro_value("typed-1", "inspect", arguments.clone())
11697 .await
11698 .expect("typed query"),
11699 typed_fidelity_probe()
11700 );
11701 assert_eq!(
11702 decode_wire_avro_value(
11703 &server.request_body("/api/workflows/typed-1/query/inspect")["input"],
11704 DEFAULT_CODEC,
11705 )
11706 .expect("typed query input"),
11707 arguments
11708 );
11709
11710 assert_eq!(
11711 client
11712 .update_workflow_avro_value(
11713 "typed-1",
11714 "replace",
11715 arguments.clone(),
11716 Some("typed-request"),
11717 )
11718 .await
11719 .expect("typed update"),
11720 typed_fidelity_probe()
11721 );
11722 let update = server.request_body("/api/workflows/typed-1/update/replace");
11723 assert_eq!(update["request_id"], "typed-request");
11724 assert_eq!(
11725 decode_wire_avro_value(&update["input"], DEFAULT_CODEC).expect("typed update input"),
11726 arguments
11727 );
11728
11729 let handle = WorkflowHandle {
11730 client: client.clone(),
11731 workflow_id: "typed-1".to_string(),
11732 run_id: Some("run-typed-1".to_string()),
11733 workflow_type: "typed.echo".to_string(),
11734 };
11735 assert_eq!(
11736 handle
11737 .result_avro_value(WorkflowResultOptions::default())
11738 .await
11739 .expect("typed workflow result"),
11740 typed_fidelity_probe()
11741 );
11742
11743 client
11744 .complete_activity_task(
11745 "activity-typed",
11746 "attempt-typed",
11747 "rust-worker",
11748 typed_fidelity_probe(),
11749 DEFAULT_CODEC,
11750 )
11751 .await
11752 .expect("typed activity completion");
11753 assert_eq!(
11754 decode_wire_avro_value(
11755 &server.request_body("/api/worker/activity-tasks/activity-typed/complete")
11756 ["result"],
11757 DEFAULT_CODEC,
11758 )
11759 .expect("typed activity result"),
11760 typed_fidelity_probe()
11761 );
11762 client
11763 .fail_activity_task(
11764 "activity-typed",
11765 "attempt-typed",
11766 "rust-worker",
11767 "typed failure",
11768 true,
11769 )
11770 .await
11771 .expect("activity failure");
11772 }
11773
11774 #[tokio::test]
11775 async fn lifecycle_commands_support_instance_and_selected_run_targets() {
11776 let server = MockWorkerServer::start();
11777 let client = Client::builder(server.base_url())
11778 .timeout(Duration::from_secs(2))
11779 .build()
11780 .expect("client");
11781
11782 let options = WorkflowCommandOptions::new()
11783 .reason("cleanup requested")
11784 .request_id("cancel-17");
11785 let cancelled = client
11786 .cancel_workflow("wf-lifecycle", options)
11787 .await
11788 .expect("instance cancellation");
11789 assert_eq!(cancelled.command, WorkflowCommandKind::Cancel);
11790 assert_eq!(cancelled.run_id.as_deref(), Some("run-current"));
11791 assert_eq!(cancelled.outcome.as_deref(), Some("cancelled"));
11792 assert_eq!(
11793 server.request_body("/api/workflows/wf-lifecycle/cancel"),
11794 json!({"reason":"cleanup requested","request_id":"cancel-17"})
11795 );
11796
11797 let terminated = client
11798 .terminate_workflow(
11799 "wf-lifecycle",
11800 WorkflowCommandOptions::new().reason("forced stop"),
11801 )
11802 .await
11803 .expect("instance termination");
11804 assert_eq!(terminated.command, WorkflowCommandKind::Terminate);
11805 assert_eq!(terminated.outcome.as_deref(), Some("terminated"));
11806
11807 client
11808 .cancel_workflow_run(
11809 "wf-lifecycle",
11810 "run-current",
11811 WorkflowCommandOptions::default(),
11812 )
11813 .await
11814 .expect("selected run cancellation");
11815 client
11816 .terminate_workflow_run(
11817 "wf-lifecycle",
11818 "run-current",
11819 WorkflowCommandOptions::default(),
11820 )
11821 .await
11822 .expect("selected run termination");
11823
11824 for (command, error) in [
11825 (
11826 WorkflowCommandKind::Cancel,
11827 client
11828 .cancel_workflow_run(
11829 "wf-lifecycle",
11830 "run-stale",
11831 WorkflowCommandOptions::default(),
11832 )
11833 .await
11834 .expect_err("stale cancellation must be rejected"),
11835 ),
11836 (
11837 WorkflowCommandKind::Terminate,
11838 client
11839 .terminate_workflow_run(
11840 "wf-lifecycle",
11841 "run-stale",
11842 WorkflowCommandOptions::default(),
11843 )
11844 .await
11845 .expect_err("stale termination must be rejected"),
11846 ),
11847 ] {
11848 let Error::WorkflowCommandRejected(rejection) = error else {
11849 panic!("expected typed command rejection");
11850 };
11851 assert_eq!(rejection.command, command);
11852 assert_eq!(rejection.status, 409);
11853 assert_eq!(rejection.reason, "historical_run_command_rejected");
11854 assert_eq!(rejection.run_id.as_deref(), Some("run-stale"));
11855 assert_eq!(rejection.target_scope.as_deref(), Some("run"));
11856 }
11857 }
11858
11859 #[tokio::test]
11860 async fn workflow_start_options_send_server_enforced_deadlines() {
11861 let server = MockWorkerServer::start();
11862 let client = Client::builder(server.base_url())
11863 .timeout(Duration::from_secs(2))
11864 .build()
11865 .expect("client");
11866
11867 let handle = client
11868 .start_workflow_with_options(
11869 "rust.timeout",
11870 "rust-timeouts",
11871 "wf-start-options",
11872 WorkflowStartOptions::new()
11873 .execution_timeout_seconds(30)
11874 .run_timeout_seconds(1),
11875 json!([]),
11876 )
11877 .await
11878 .expect("workflow start");
11879
11880 assert_eq!(handle.run_id.as_deref(), Some("run-start-options"));
11881 let body = server.request_body("/api/workflows");
11882 assert_eq!(body["execution_timeout_seconds"], 30);
11883 assert_eq!(body["run_timeout_seconds"], 1);
11884
11885 let invalid = client
11886 .start_workflow_with_options(
11887 "rust.timeout",
11888 "rust-timeouts",
11889 "wf-invalid-options",
11890 WorkflowStartOptions::new()
11891 .execution_timeout_seconds(1)
11892 .run_timeout_seconds(2),
11893 json!([]),
11894 )
11895 .await
11896 .expect_err("invalid deadline ordering");
11897 assert!(invalid
11898 .to_string()
11899 .contains("run_timeout_seconds cannot exceed execution_timeout_seconds"));
11900 }
11901
11902 #[tokio::test]
11903 async fn workflow_result_returns_each_typed_terminal_outcome() {
11904 let server = MockWorkerServer::start();
11905 let client = Client::builder(server.base_url())
11906 .timeout(Duration::from_secs(2))
11907 .build()
11908 .expect("client");
11909 let options = WorkflowResultOptions {
11910 poll_interval: Duration::ZERO,
11911 timeout: Duration::from_secs(1),
11912 };
11913
11914 let failed = WorkflowHandle {
11915 client: client.clone(),
11916 workflow_id: "wf-failed".to_string(),
11917 run_id: Some("run-failed".to_string()),
11918 workflow_type: "failure".to_string(),
11919 }
11920 .result(options)
11921 .await
11922 .expect_err("failed outcome");
11923 let Error::WorkflowFailed(failure) = failed else {
11924 panic!("expected WorkflowFailed");
11925 };
11926 assert_eq!(failure.workflow_id, "wf-failed");
11927 assert_eq!(failure.run_id.as_deref(), Some("run-failed"));
11928 assert_eq!(failure.failure_id.as_deref(), Some("failure-17"));
11929 assert_eq!(failure.failure_category.as_deref(), Some("application"));
11930 assert_eq!(failure.exception_type.as_deref(), Some("PaymentError"));
11931 assert_eq!(
11932 failure.exception_class.as_deref(),
11933 Some("billing::PaymentError")
11934 );
11935 assert_eq!(failure.non_retryable, Some(true));
11936
11937 for (workflow_id, expected_kind, expected_reason) in [
11938 (
11939 "wf-cancelled",
11940 WorkflowTerminalKind::Cancelled,
11941 "cleanup requested",
11942 ),
11943 (
11944 "wf-terminated",
11945 WorkflowTerminalKind::Terminated,
11946 "forced stop",
11947 ),
11948 (
11949 "wf-timed-out",
11950 WorkflowTerminalKind::TimedOut,
11951 "run_timeout",
11952 ),
11953 ] {
11954 let error = WorkflowHandle {
11955 client: client.clone(),
11956 workflow_id: workflow_id.to_string(),
11957 run_id: None,
11958 workflow_type: "terminal".to_string(),
11959 }
11960 .result(options)
11961 .await
11962 .expect_err("typed terminal outcome");
11963 let outcome = match error {
11964 Error::WorkflowCancelled(outcome) => outcome,
11965 Error::WorkflowTerminated(outcome) => outcome,
11966 Error::WorkflowTimedOut(outcome) => outcome,
11967 other => panic!("unexpected terminal error: {other}"),
11968 };
11969 assert_eq!(outcome.kind, expected_kind);
11970 assert_eq!(outcome.workflow_id, workflow_id);
11971 assert_eq!(outcome.reason, expected_reason);
11972 }
11973
11974 let wait_timeout = WorkflowHandle {
11975 client,
11976 workflow_id: "wf-waiting".to_string(),
11977 run_id: Some("run-waiting".to_string()),
11978 workflow_type: "waiting".to_string(),
11979 }
11980 .result(WorkflowResultOptions {
11981 poll_interval: Duration::ZERO,
11982 timeout: Duration::ZERO,
11983 })
11984 .await
11985 .expect_err("client wait timeout");
11986 let Error::WorkflowTimedOut(timeout) = wait_timeout else {
11987 panic!("expected typed client timeout");
11988 };
11989 assert_eq!(timeout.reason, "result_wait_timeout");
11990 assert_eq!(timeout.failure_category.as_deref(), Some("client_timeout"));
11991 assert_eq!(timeout.run_id.as_deref(), Some("run-waiting"));
11992 }
11993
11994 #[tokio::test]
11995 async fn workflow_result_follows_chain_and_selected_result_preserves_history() {
11996 let server = MockWorkerServer::start();
11997 let client = Client::builder(server.base_url())
11998 .timeout(Duration::from_secs(2))
11999 .build()
12000 .expect("client");
12001
12002 let handle = WorkflowHandle {
12003 client,
12004 workflow_id: "wf-selected".to_string(),
12005 run_id: Some("run-selected".to_string()),
12006 workflow_type: "selected".to_string(),
12007 };
12008 let options = WorkflowResultOptions {
12009 poll_interval: Duration::ZERO,
12010 timeout: Duration::from_secs(1),
12011 };
12012
12013 let current = handle
12014 .result(options)
12015 .await
12016 .expect("instance result follows the current run");
12017 assert_eq!(current, json!("current run output"));
12018
12019 let error = handle
12020 .result_selected_run(options)
12021 .await
12022 .expect_err("the selected run is cancelled even though the current run completed");
12023
12024 let Error::WorkflowCancelled(outcome) = error else {
12025 panic!("expected selected run cancellation");
12026 };
12027 assert_eq!(outcome.run_id.as_deref(), Some("run-selected"));
12028 assert_eq!(outcome.reason, "selected run cancelled");
12029 assert_eq!(
12030 server.request_count("/api/workflows/wf-selected/runs/run-selected"),
12031 1
12032 );
12033 assert_eq!(server.request_count("/api/workflows/wf-selected"), 1);
12034 }
12035
12036 #[tokio::test]
12037 async fn poll_responses_decode_http_conflict_drain_as_a_stable_stop() {
12038 let server = MockWorkerServer::draining_polls();
12039 let client = Client::builder(server.base_url())
12040 .timeout(Duration::from_secs(2))
12041 .build()
12042 .expect("client");
12043
12044 let workflow = client
12045 .poll_workflow_task_response("draining-worker", "rust-workers", Duration::ZERO)
12046 .await
12047 .expect("workflow drain response");
12048 let activity = client
12049 .poll_activity_task_response("draining-worker", "rust-workers", Duration::ZERO)
12050 .await
12051 .expect("activity drain response");
12052 let query = client
12053 .poll_query_task_response("draining-worker", "rust-workers", Duration::ZERO)
12054 .await
12055 .expect("query drain response");
12056
12057 for outcome in [workflow.outcome(), activity.outcome(), query.outcome()] {
12058 assert_eq!(
12059 outcome,
12060 WorkerPollOutcome::Stop {
12061 poll_status: Some("draining".to_string()),
12062 reason: Some("worker_draining".to_string()),
12063 }
12064 );
12065 }
12066
12067 assert!(client
12068 .poll_workflow_task("draining-worker", "rust-workers", Duration::ZERO)
12069 .await
12070 .expect("compatibility poll")
12071 .is_none());
12072 }
12073
12074 #[tokio::test]
12075 async fn managed_worker_honors_drain_stop_for_every_task_family() {
12076 let server = MockWorkerServer::draining_polls();
12077 let client = Client::builder(server.base_url())
12078 .timeout(Duration::from_secs(2))
12079 .build()
12080 .expect("client");
12081
12082 let mut workflow_worker = Worker::new(client.clone(), "rust-workers")
12083 .worker_id("draining-workflow-worker")
12084 .poll_timeout(Duration::ZERO);
12085 workflow_worker.register_workflow("counter", |_ctx, _args| async { Ok(Value::Null) });
12086 workflow_worker
12087 .run()
12088 .await
12089 .expect("workflow drain is a clean stop");
12090
12091 let mut activity_worker = Worker::new(client.clone(), "rust-workers")
12092 .worker_id("draining-activity-worker")
12093 .poll_timeout(Duration::ZERO);
12094 activity_worker.register_activity("write", |_ctx, _args| async { Ok(Value::Null) });
12095 activity_worker
12096 .run()
12097 .await
12098 .expect("activity drain is a clean stop");
12099
12100 let mut query_worker = Worker::new(client, "rust-workers")
12101 .worker_id("draining-query-worker")
12102 .poll_timeout(Duration::ZERO);
12103 query_worker.register_query("counter", "current", |_ctx, _args| async {
12104 Ok(Value::Null)
12105 });
12106 query_worker
12107 .run()
12108 .await
12109 .expect("query drain is a clean stop");
12110 }
12111
12112 #[tokio::test]
12113 async fn activity_cancellation_and_late_completion_remain_machine_readable() {
12114 let server = MockWorkerServer::start();
12115 let client = Client::builder(server.base_url())
12116 .timeout(Duration::from_secs(2))
12117 .build()
12118 .expect("client");
12119
12120 let heartbeat = client
12121 .heartbeat_activity_task(
12122 "activity-cancel",
12123 "attempt-cancel",
12124 "rust-worker",
12125 typed_fidelity_probe(),
12126 )
12127 .await
12128 .expect("cancellation heartbeat");
12129 assert!(heartbeat.cancel_requested);
12130 assert!(heartbeat.should_stop());
12131 assert_eq!(heartbeat.reason.as_deref(), Some("run_cancelled"));
12132 assert_eq!(heartbeat.run_closed_reason.as_deref(), Some("cancelled"));
12133 let heartbeat_body =
12134 server.request_body("/api/worker/activity-tasks/activity-cancel/heartbeat");
12135 assert_eq!(heartbeat_body["details"]["codec"], DEFAULT_CODEC);
12136 assert_eq!(
12137 decode_wire_avro_value(&heartbeat_body["details"], DEFAULT_CODEC)
12138 .expect("typed heartbeat details"),
12139 typed_fidelity_probe()
12140 );
12141
12142 let error = client
12143 .complete_activity_task(
12144 "activity-cancel",
12145 "attempt-cancel",
12146 "rust-worker",
12147 json!({"late":true}),
12148 JSON_CODEC,
12149 )
12150 .await
12151 .expect_err("late completion must be refused");
12152 assert!(activity_task_rejection_is_final(&error));
12153 let Error::ActivityTaskRejected(rejection) = error else {
12154 panic!("expected typed activity rejection");
12155 };
12156 assert_eq!(rejection.status, 409);
12157 assert_eq!(rejection.reason, "run_cancelled");
12158 assert!(rejection.cancel_requested);
12159 assert_eq!(rejection.can_continue, Some(false));
12160 }
12161
12162 #[tokio::test]
12163 async fn managed_worker_survives_late_completion_and_restart_during_cancellation() {
12164 let server = MockWorkerServer::cancelled_activity();
12165 let client = Client::builder(server.base_url())
12166 .timeout(Duration::from_secs(2))
12167 .build()
12168 .expect("client");
12169 let cancellation_observed = Arc::new(AtomicBool::new(false));
12170 let observed = Arc::clone(&cancellation_observed);
12171 let mut worker = Worker::new(client.clone(), "rust-workers")
12172 .worker_id("rust-cancel-worker")
12173 .poll_timeout(Duration::from_millis(10));
12174 worker.register_activity("cancel-aware", move |ctx, _args| {
12175 let observed = Arc::clone(&observed);
12176 async move {
12177 let heartbeat = ctx.heartbeat(json!({"stage":"running"})).await?;
12178 observed.store(heartbeat.should_stop(), Ordering::SeqCst);
12179 Ok(json!({"late":"completion"}))
12180 }
12181 });
12182
12183 assert_eq!(
12184 worker.run_once().await.expect("cancelled attempt handled"),
12185 1
12186 );
12187 assert!(cancellation_observed.load(Ordering::SeqCst));
12188 assert_eq!(
12189 server.request_count("/api/worker/activity-tasks/activity-cancel/complete"),
12190 1
12191 );
12192
12193 let mut restarted = Worker::new(client, "rust-workers")
12194 .worker_id("rust-cancel-worker-restarted")
12195 .poll_timeout(Duration::from_millis(10));
12196 restarted.register_activity("cancel-aware", |_ctx, _args| async move { Ok(Value::Null) });
12197 assert_eq!(
12198 restarted
12199 .run_once()
12200 .await
12201 .expect("replacement worker continues polling"),
12202 0
12203 );
12204 }
12205
12206 #[tokio::test]
12207 async fn managed_worker_absorbs_selected_run_terminal_timeout_completion_race() {
12208 let response = r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"outcome":"completed","recorded":false,"run_id":"run-selected-timeout","run_status":"failed","created_task_ids":[],"reason":"run_timed_out"}"#;
12209 let server = MockWorkerServer::workflow_completion("409 Conflict", response);
12210 let client = Client::builder(server.base_url())
12211 .timeout(Duration::from_secs(2))
12212 .build()
12213 .expect("client");
12214
12215 let direct_error = client
12216 .complete_workflow_task(
12217 "workflow-timeout-task",
12218 "timeout-worker",
12219 3,
12220 vec![json!({"type": "complete_workflow", "result": null})],
12221 )
12222 .await
12223 .expect_err("the low-level client preserves the completion rejection");
12224 let Error::Http { status, body } = direct_error else {
12225 panic!("expected the original HTTP completion rejection");
12226 };
12227 assert_eq!(status, reqwest::StatusCode::CONFLICT);
12228 assert_eq!(
12229 serde_json::from_str::<Value>(&body).expect("response body")["reason"],
12230 "run_timed_out"
12231 );
12232
12233 let mut worker = Worker::new(client, "rust-workers")
12234 .worker_id("timeout-worker")
12235 .poll_timeout(Duration::from_millis(10));
12236 worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
12237 Ok(json!({"late": "result"}))
12238 });
12239
12240 assert_eq!(
12241 worker
12242 .run_once()
12243 .await
12244 .expect("authoritative selected-run timeout settles the tick"),
12245 1
12246 );
12247 assert_eq!(
12248 server.request_count("/api/worker/workflow-tasks/workflow-timeout-task/complete"),
12249 2,
12250 "both the direct client proof and managed worker must see the rejection"
12251 );
12252 }
12253
12254 #[tokio::test]
12255 async fn managed_worker_does_not_swallow_nearby_completion_errors() {
12256 for (name, status, response) in [
12257 ("bare conflict", "409 Conflict", r#"{"message":"conflict"}"#),
12258 (
12259 "command was recorded",
12260 "409 Conflict",
12261 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":true,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12262 ),
12263 (
12264 "lease conflict",
12265 "409 Conflict",
12266 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"lease_expired"}"#,
12267 ),
12268 (
12269 "nonterminal run",
12270 "409 Conflict",
12271 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"waiting","reason":"run_timed_out"}"#,
12272 ),
12273 (
12274 "different selected run",
12275 "409 Conflict",
12276 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-reused-workflow-current","run_status":"failed","reason":"run_timed_out"}"#,
12277 ),
12278 (
12279 "different task attempt",
12280 "409 Conflict",
12281 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":4,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12282 ),
12283 (
12284 "authentication failure",
12285 "401 Unauthorized",
12286 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12287 ),
12288 (
12289 "authorization failure",
12290 "403 Forbidden",
12291 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12292 ),
12293 (
12294 "protocol failure",
12295 "400 Bad Request",
12296 r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.2","requested_version":"1.3"}"#,
12297 ),
12298 (
12299 "malformed command",
12300 "422 Unprocessable Entity",
12301 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12302 ),
12303 (
12304 "transient server failure",
12305 "503 Service Unavailable",
12306 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12307 ),
12308 ] {
12309 let server = MockWorkerServer::workflow_completion(status, response);
12310 let client = Client::builder(server.base_url())
12311 .timeout(Duration::from_secs(2))
12312 .build()
12313 .expect("client");
12314 let mut worker = Worker::new(client, "rust-workers")
12315 .worker_id("timeout-worker")
12316 .poll_timeout(Duration::from_millis(10));
12317 worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
12318 Ok(json!({"late": "result"}))
12319 });
12320
12321 let error = worker
12322 .run_once()
12323 .await
12324 .expect_err(&format!("{name} must remain an error"));
12325 assert!(
12326 matches!(error, Error::Http { .. } | Error::Protocol(_)),
12327 "{name} returned an unexpected error variant: {error}"
12328 );
12329 }
12330 }
12331
12332 #[tokio::test]
12333 async fn baseline_worker_endpoints_send_the_baseline_protocol() {
12334 let server = MockWorkerServer::start();
12335 let client = Client::builder(server.base_url())
12336 .timeout(Duration::from_secs(2))
12337 .build()
12338 .expect("client");
12339
12340 client
12341 .register_worker("capture-worker", "capture", vec![], vec![], 1, 1)
12342 .await
12343 .expect("register");
12344 client
12345 .heartbeat_worker("capture-worker", 1, 1)
12346 .await
12347 .expect("heartbeat");
12348 client
12349 .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
12350 .await
12351 .expect("workflow poll");
12352 client
12353 .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
12354 .await
12355 .expect("activity poll");
12356
12357 for path in [
12358 "/api/worker/register",
12359 "/api/worker/heartbeat",
12360 "/api/worker/workflow-tasks/poll",
12361 "/api/worker/activity-tasks/poll",
12362 ] {
12363 assert_eq!(
12364 server.worker_protocol_for(path).as_deref(),
12365 Some(WORKER_PROTOCOL_VERSION),
12366 "unexpected protocol for {path}"
12367 );
12368 }
12369
12370 assert_eq!(
12371 server.request_body("/api/worker/workflow-tasks/poll")["timeout_seconds"],
12372 1
12373 );
12374 assert_eq!(
12375 server.request_body("/api/worker/activity-tasks/poll")["timeout_seconds"],
12376 1
12377 );
12378 assert!(
12379 server.request_body("/api/worker/workflow-tasks/poll")["poll_request_id"]
12380 .as_str()
12381 .is_some_and(|id| id.starts_with("rust-workflow-poll-"))
12382 );
12383 assert!(
12384 server.request_body("/api/worker/activity-tasks/poll")["poll_request_id"]
12385 .as_str()
12386 .is_some_and(|id| id.starts_with("rust-activity-poll-"))
12387 );
12388 }
12389
12390 #[tokio::test]
12391 async fn query_task_endpoints_send_the_query_feature_protocol() {
12392 let server = MockWorkerServer::start();
12393 let client = Client::builder(server.base_url())
12394 .timeout(Duration::from_secs(2))
12395 .build()
12396 .expect("client");
12397
12398 client
12399 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
12400 .await
12401 .expect("query poll");
12402 client
12403 .complete_query_task("query-capture", "capture-worker", 1, json!(8), JSON_CODEC)
12404 .await
12405 .expect("query complete");
12406 client
12407 .fail_query_task(
12408 "query-capture",
12409 "capture-worker",
12410 1,
12411 "failed",
12412 "query_rejected",
12413 "QueryFailed",
12414 )
12415 .await
12416 .expect("query fail");
12417
12418 for path in [
12419 "/api/worker/query-tasks/poll",
12420 "/api/worker/query-tasks/query-capture/complete",
12421 "/api/worker/query-tasks/query-capture/fail",
12422 ] {
12423 assert_eq!(
12424 server.worker_protocol_for(path).as_deref(),
12425 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
12426 "unexpected protocol for {path}"
12427 );
12428 }
12429
12430 assert_eq!(
12431 server.request_body("/api/worker/query-tasks/poll")["timeout_seconds"],
12432 1
12433 );
12434 assert!(
12435 server.request_body("/api/worker/query-tasks/poll")["poll_request_id"]
12436 .as_str()
12437 .is_some_and(|id| id.starts_with("rust-query-poll-"))
12438 );
12439 }
12440
12441 #[tokio::test]
12442 async fn disconnected_client_polls_retry_once_with_the_same_request_id() {
12443 let server = MockWorkerServer::transient_worker_failures();
12444 let client = Client::builder(server.base_url())
12445 .timeout(Duration::from_secs(2))
12446 .build()
12447 .expect("client");
12448
12449 client
12450 .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
12451 .await
12452 .expect("workflow poll retry");
12453 client
12454 .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
12455 .await
12456 .expect("activity poll retry");
12457 client
12458 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
12459 .await
12460 .expect("query poll retry");
12461
12462 for path in [
12463 "/api/worker/workflow-tasks/poll",
12464 "/api/worker/activity-tasks/poll",
12465 "/api/worker/query-tasks/poll",
12466 ] {
12467 let bodies = server.request_bodies(path);
12468 assert_eq!(bodies.len(), 2, "{path} must be retried once");
12469 assert_eq!(
12470 bodies[0]["poll_request_id"], bodies[1]["poll_request_id"],
12471 "{path} must preserve the request binding across retry"
12472 );
12473 }
12474 }
12475
12476 #[tokio::test]
12477 async fn worker_poll_retries_preserve_request_id_across_consecutive_failures() {
12478 let server = MockWorkerServer::consecutive_poll_failures(2);
12479 let client = Client::builder(server.base_url())
12480 .timeout(Duration::from_secs(2))
12481 .build()
12482 .expect("client");
12483 let mut worker = Worker::new(client, "capture")
12484 .worker_id("capture-worker")
12485 .poll_timeout(Duration::from_millis(10))
12486 .retry_policy(WorkerRetryPolicy {
12487 max_retries: 2,
12488 initial_backoff: Duration::from_millis(1),
12489 max_backoff: Duration::from_millis(1),
12490 });
12491 worker.register_workflow(
12492 "capture.workflow",
12493 |_ctx, _input| async move { Ok(Value::Null) },
12494 );
12495 worker.register_activity(
12496 "capture.activity",
12497 |_ctx, _input| async move { Ok(Value::Null) },
12498 );
12499 worker.register_query("capture.workflow", "current", |_ctx, _args| async move {
12500 Ok(Value::Null)
12501 });
12502
12503 assert_eq!(worker.run_once().await.expect("poll retries"), 0);
12504
12505 for path in [
12506 "/api/worker/workflow-tasks/poll",
12507 "/api/worker/activity-tasks/poll",
12508 "/api/worker/query-tasks/poll",
12509 ] {
12510 let bodies = server.request_bodies(path);
12511 assert_eq!(bodies.len(), 3, "{path} must use exactly two retries");
12512 assert!(
12513 bodies
12514 .iter()
12515 .all(|body| body["poll_request_id"] == bodies[0]["poll_request_id"]),
12516 "{path} must preserve one request binding across every retry"
12517 );
12518 }
12519 }
12520
12521 #[tokio::test]
12522 async fn query_protocol_rejection_from_older_server_is_typed() {
12523 let server = MockWorkerServer::reject_query_protocol();
12524 let client = Client::builder(server.base_url())
12525 .timeout(Duration::from_secs(2))
12526 .build()
12527 .expect("client");
12528
12529 let error = client
12530 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
12531 .await
12532 .expect_err("server below query protocol floor must reject");
12533 let Error::Protocol(failure) = error else {
12534 panic!("expected typed protocol failure");
12535 };
12536
12537 assert_eq!(failure.status, 400);
12538 assert_eq!(failure.reason, "unsupported_protocol_version");
12539 assert_eq!(failure.supported_version.as_deref(), Some("1.7"));
12540 assert_eq!(
12541 failure.requested_version.as_deref(),
12542 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
12543 );
12544 assert_eq!(
12545 server
12546 .worker_protocol_for("/api/worker/query-tasks/poll")
12547 .as_deref(),
12548 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
12549 );
12550 }
12551
12552 #[tokio::test]
12553 async fn run_once_without_query_handlers_keeps_pre_query_server_compatibility() {
12554 let server = MockWorkerServer::reject_query_protocol();
12555 let client = Client::builder(server.base_url())
12556 .timeout(Duration::from_secs(2))
12557 .build()
12558 .expect("client");
12559 let mut worker = Worker::new(client, "rust-workers")
12560 .worker_id("baseline-worker")
12561 .poll_timeout(Duration::from_millis(10));
12562
12563 worker.register_workflow("baseline.workflow", |_ctx, _input| async move {
12564 Ok(Value::Null)
12565 });
12566
12567 assert_eq!(worker.run_once().await.expect("baseline run once"), 0);
12568 assert_eq!(
12569 server
12570 .worker_protocol_for("/api/worker/workflow-tasks/poll")
12571 .as_deref(),
12572 Some(WORKER_PROTOCOL_VERSION)
12573 );
12574 assert_eq!(
12575 server.worker_protocol_for("/api/worker/query-tasks/poll"),
12576 None,
12577 "a worker without query handlers must not use the query-task endpoint"
12578 );
12579 }
12580
12581 #[tokio::test]
12582 async fn completion_time_query_rejection_is_typed_without_stopping_worker() {
12583 let server = MockWorkerServer::reject_query_completion();
12584 let client = Client::builder(server.base_url())
12585 .timeout(Duration::from_secs(2))
12586 .build()
12587 .expect("client");
12588
12589 let error = client
12590 .complete_query_task("query-late", "late-worker", 1, json!(8), JSON_CODEC)
12591 .await
12592 .expect_err("expired completion must be rejected");
12593 let Error::QueryFailed(failure) = error else {
12594 panic!("expected typed query failure");
12595 };
12596 assert_eq!(failure.status, 409);
12597 assert_eq!(failure.reason, "query_task_timed_out");
12598
12599 let mut worker = Worker::new(client, "rust-workers")
12600 .worker_id("late-worker")
12601 .poll_timeout(Duration::from_millis(10));
12602 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
12603 worker.register_query(
12604 "counter",
12605 "current",
12606 |_ctx, _args| async move { Ok(json!(8)) },
12607 );
12608
12609 assert_eq!(worker.run_once().await.expect("late task is handled"), 1);
12610 assert_eq!(
12611 worker
12612 .run_once()
12613 .await
12614 .expect("worker continues after late completion"),
12615 0
12616 );
12617 assert_eq!(
12618 server.request_count("/api/worker/query-tasks/query-late/complete"),
12619 2
12620 );
12621 assert_eq!(
12622 server.request_count("/api/worker/query-tasks/query-late/fail"),
12623 0,
12624 "a server completion rejection must not be reported as an encoding failure"
12625 );
12626 }
12627
12628 #[tokio::test]
12629 async fn activity_only_worker_can_shutdown_without_workflow_poller() {
12630 let server = MockWorkerServer::start();
12631 let client = Client::builder(server.base_url())
12632 .timeout(Duration::from_secs(2))
12633 .build()
12634 .expect("client");
12635 let mut worker = Worker::new(client, "rust-workers")
12636 .worker_id("activity-only-worker")
12637 .poll_timeout(Duration::from_millis(10));
12638
12639 worker.register_activity(
12640 "activity.only",
12641 |_ctx, _args| async move { Ok(Value::Null) },
12642 );
12643
12644 worker.run_until(async {}).await.expect("run worker");
12645 }
12646
12647 #[tokio::test]
12648 async fn workflow_only_worker_can_shutdown_without_activity_poller() {
12649 let server = MockWorkerServer::start();
12650 let client = Client::builder(server.base_url())
12651 .timeout(Duration::from_secs(2))
12652 .build()
12653 .expect("client");
12654 let mut worker = Worker::new(client, "rust-workers")
12655 .worker_id("workflow-only-worker")
12656 .poll_timeout(Duration::from_millis(10));
12657
12658 worker.register_workflow(
12659 "workflow.only",
12660 |_ctx, _input| async move { Ok(Value::Null) },
12661 );
12662
12663 worker.run_until(async {}).await.expect("run worker");
12664 }
12665
12666 #[tokio::test]
12667 async fn worker_heartbeat_observer_receives_server_acknowledgements() {
12668 let server = MockWorkerServer::start();
12669 let client = Client::builder(server.base_url())
12670 .timeout(Duration::from_secs(2))
12671 .build()
12672 .expect("client");
12673 let observations = Arc::new(Mutex::new(Vec::new()));
12674 let observed = Arc::clone(&observations);
12675 let mut worker = Worker::new(client, "rust-workers")
12676 .worker_id("observed-heartbeat-worker")
12677 .poll_timeout(Duration::from_millis(10))
12678 .on_worker_heartbeat(move |observation| {
12679 observed
12680 .lock()
12681 .expect("heartbeat observations")
12682 .push(observation.clone());
12683 });
12684
12685 worker.register_workflow("workflow.observed", |_ctx, _input| async move {
12686 Ok(Value::Null)
12687 });
12688 let acknowledged = Arc::clone(&observations);
12689 worker
12690 .run_until(async move {
12691 tokio::time::timeout(Duration::from_secs(2), async move {
12692 loop {
12693 if !acknowledged
12694 .lock()
12695 .expect("heartbeat observations")
12696 .is_empty()
12697 {
12698 break;
12699 }
12700 tokio::time::sleep(Duration::from_millis(1)).await;
12701 }
12702 })
12703 .await
12704 .expect("heartbeat acknowledgement within timeout");
12705 })
12706 .await
12707 .expect("run worker");
12708
12709 let observations = observations.lock().expect("heartbeat observations");
12710 let first = observations.first().expect("heartbeat acknowledgement");
12711 assert_eq!(first.worker_id, "observed-heartbeat-worker");
12712 assert_eq!(first.task_queue, "rust-workers");
12713 assert!(first.acknowledged_at_unix_millis > 0);
12714 assert_eq!(first.acknowledgement, json!({}));
12715 }
12716
12717 #[tokio::test]
12718 async fn delayed_worker_heartbeat_keeps_cadence_and_pollers_live() {
12719 let server = MockWorkerServer::delayed_heartbeat_worker();
12720 let client = Client::builder(server.base_url())
12721 .timeout(Duration::from_secs(3))
12722 .build()
12723 .expect("client");
12724 let observations = Arc::new(Mutex::new(Vec::new()));
12725 let observed = Arc::clone(&observations);
12726 let mut worker = Worker::new(client, "rust-snapshot-workers")
12727 .worker_id("rust-snapshot-worker")
12728 .poll_timeout(Duration::from_millis(10))
12729 .on_worker_heartbeat(move |observation| {
12730 observed
12731 .lock()
12732 .expect("heartbeat observations")
12733 .push(observation.clone());
12734 });
12735
12736 worker.register_workflow("snapshot", |ctx, _input| async move {
12737 ctx.wait_signal("finish").await?;
12738 Ok(json!({"status": "finished"}))
12739 });
12740 worker.register_query("snapshot", "current", |ctx, _args| async move {
12741 Ok(json!(ctx
12742 .signals("increment")
12743 .iter()
12744 .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
12745 .sum::<i64>()))
12746 });
12747 worker.register_activity("cancel-aware", |_ctx, _args| async move {
12748 Ok(json!({"late": "completion"}))
12749 });
12750
12751 worker
12752 .run_until(tokio::time::sleep(Duration::from_millis(3_800)))
12753 .await
12754 .expect("delayed heartbeat must allow a clean worker shutdown");
12755
12756 let observations = observations.lock().expect("heartbeat observations");
12757 assert!(
12758 observations.len() >= 3,
12759 "the immediate heartbeat, delayed acknowledgement, and next cadence heartbeat must complete"
12760 );
12761 assert!(
12762 observations.windows(2).all(|pair| {
12763 pair[1].acknowledged_at_unix_millis
12764 .saturating_sub(pair[0].acknowledged_at_unix_millis)
12765 >= 850
12766 }),
12767 "successful acknowledgements must not catch up faster than the advertised one-second cadence: {observations:?}"
12768 );
12769 drop(observations);
12770
12771 let heartbeat_times = server.request_times("/api/worker/heartbeat");
12772 let delayed_request_at = *heartbeat_times
12773 .get(1)
12774 .expect("intentionally delayed heartbeat request");
12775 let delay_window_start = delayed_request_at + Duration::from_millis(100);
12776 let delay_window_end = delayed_request_at + Duration::from_millis(1_400);
12777 for path in [
12778 "/api/worker/workflow-tasks/poll",
12779 "/api/worker/activity-tasks/poll",
12780 "/api/worker/query-tasks/poll",
12781 ] {
12782 assert!(
12783 server
12784 .request_times(path)
12785 .iter()
12786 .any(|received_at| *received_at >= delay_window_start
12787 && *received_at <= delay_window_end),
12788 "{path} must keep polling while a heartbeat acknowledgement is delayed"
12789 );
12790 }
12791 assert!(
12792 server.request_count("/api/worker/workflow-tasks/snapshot-wait-3/fail") >= 1,
12793 "workflow work must be settled"
12794 );
12795 assert!(
12796 server.request_count("/api/worker/activity-tasks/activity-cancel/complete") >= 1,
12797 "activity work must be settled"
12798 );
12799 assert!(
12800 server.request_count("/api/worker/query-tasks/snapshot-current/complete") >= 1,
12801 "query work must be settled"
12802 );
12803 }
12804
12805 #[tokio::test]
12806 async fn retried_worker_heartbeat_restarts_the_advertised_cadence() {
12807 let server = MockWorkerServer::heartbeat_retry_worker();
12808 let client = Client::builder(server.base_url())
12809 .timeout(Duration::from_secs(2))
12810 .build()
12811 .expect("client");
12812 let observations = Arc::new(Mutex::new(Vec::new()));
12813 let observed = Arc::clone(&observations);
12814 let worker = Worker::new(client, "rust-workers")
12815 .worker_id("heartbeat-retry-worker")
12816 .retry_policy(WorkerRetryPolicy {
12817 max_retries: 1,
12818 initial_backoff: Duration::from_millis(300),
12819 max_backoff: Duration::from_millis(300),
12820 })
12821 .on_worker_heartbeat(move |observation| {
12822 observed
12823 .lock()
12824 .expect("heartbeat observations")
12825 .push(observation.clone());
12826 });
12827
12828 worker
12829 .run_until(tokio::time::sleep(Duration::from_millis(2_700)))
12830 .await
12831 .expect("retryable heartbeat failure must remain bounded and recover");
12832
12833 let observations = observations.lock().expect("heartbeat observations");
12834 assert!(observations.len() >= 3, "heartbeat retry must recover");
12835 assert!(
12836 observations.windows(2).all(|pair| {
12837 pair[1]
12838 .acknowledged_at_unix_millis
12839 .saturating_sub(pair[0].acknowledged_at_unix_millis)
12840 >= 850
12841 }),
12842 "a successful retry must start a fresh advertised cadence: {observations:?}"
12843 );
12844 assert_eq!(
12845 server.request_count("/api/worker/heartbeat"),
12846 observations.len() + 1,
12847 "one retryable failure must add exactly one bounded request"
12848 );
12849 }
12850
12851 #[tokio::test]
12852 async fn query_enabled_worker_ignores_unmatched_signals_then_completes_once() {
12853 let server = MockWorkerServer::waiting_query_worker();
12854 let client = Client::builder(server.base_url())
12855 .timeout(Duration::from_secs(2))
12856 .build()
12857 .expect("client");
12858 let observations = Arc::new(Mutex::new(Vec::new()));
12859 let observed = Arc::clone(&observations);
12860 let mut worker = Worker::new(client, "rust-snapshot-workers")
12861 .worker_id("rust-snapshot-worker")
12862 .poll_timeout(Duration::from_millis(10))
12863 .on_worker_heartbeat(move |observation| {
12864 observed
12865 .lock()
12866 .expect("heartbeat observations")
12867 .push(observation.clone());
12868 });
12869
12870 worker.register_workflow("snapshot", |ctx, _input| async move {
12871 ctx.wait_signal("finish").await?;
12872 Ok(json!({"status": "finished"}))
12873 });
12874 worker.register_query("snapshot", "current", |ctx, _args| async move {
12875 let current = ctx
12876 .signals("increment")
12877 .iter()
12878 .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
12879 .sum::<i64>();
12880 Ok(json!(current))
12881 });
12882 worker.register_update("snapshot", "replace", |_ctx, args| async move { Ok(args) });
12883
12884 worker
12885 .run_until(tokio::time::sleep(Duration::from_millis(3_200)))
12886 .await
12887 .expect("pending workflow and query poller must remain live until shutdown");
12888
12889 assert!(
12890 observations.lock().expect("heartbeat observations").len() >= 4,
12891 "the immediate heartbeat and at least three advertised one-second intervals must be acknowledged"
12892 );
12893 assert!(
12894 server.request_count("/api/worker/workflow-tasks/poll") >= 3,
12895 "workflow polling must continue after empty replay acknowledgements"
12896 );
12897 assert!(
12898 server.request_count("/api/worker/query-tasks/poll") >= 2,
12899 "query polling must continue after serving the current query"
12900 );
12901 assert_eq!(
12902 server.request_body("/api/worker/register")["capabilities"],
12903 json!([QUERY_TASKS_CAPABILITY, WORKFLOW_UPDATES_CAPABILITY])
12904 );
12905 assert_eq!(
12906 server.request_body("/api/worker/register")["workflow_command_contracts"]["snapshot"],
12907 json!({
12908 "queries": ["current"],
12909 "updates": ["replace"],
12910 })
12911 );
12912
12913 let opened = server.request_body("/api/worker/workflow-tasks/snapshot-open/complete");
12914 assert_eq!(
12915 opened["commands"],
12916 json!([{
12917 "type": "open_signal_wait",
12918 "signal_name": "finish",
12919 }])
12920 );
12921
12922 for task_id in ["snapshot-wait-3", "snapshot-wait-5"] {
12923 let fail_path = format!("/api/worker/workflow-tasks/{task_id}/fail");
12924 let completion_path = format!("/api/worker/workflow-tasks/{task_id}/complete");
12925 let failure = server.request_body(&fail_path);
12926 assert_eq!(
12927 failure["failure"]["type"],
12928 WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE
12929 );
12930 assert_eq!(server.request_count(&completion_path), 0);
12931 }
12932
12933 let query_completion =
12934 server.request_body("/api/worker/query-tasks/snapshot-current/complete");
12935 assert_eq!(query_completion["result"], json!(8));
12936
12937 let terminal_path = "/api/worker/workflow-tasks/snapshot-finish/complete";
12938 assert_eq!(
12939 server.request_count(terminal_path),
12940 1,
12941 "the matching signal must settle the workflow exactly once"
12942 );
12943 let terminal = server.request_body(terminal_path);
12944 assert_eq!(terminal["commands"].as_array().map(Vec::len), Some(1));
12945 assert_eq!(terminal["commands"][0]["type"], "complete_workflow");
12946 assert_eq!(
12947 decode_wire_value(&terminal["commands"][0]["result"], DEFAULT_CODEC)
12948 .expect("terminal workflow result"),
12949 json!({"status": "finished"})
12950 );
12951 }
12952
12953 #[tokio::test]
12954 async fn worker_retries_poll_and_heartbeat_transport_failures_independently() {
12955 let server = MockWorkerServer::transient_worker_failures();
12956 let client = Client::builder(server.base_url())
12957 .timeout(Duration::from_secs(2))
12958 .build()
12959 .expect("client");
12960 let mut worker = Worker::new(client, "rust-workers")
12961 .worker_id("retry-worker")
12962 .poll_timeout(Duration::from_millis(10))
12963 .retry_policy(WorkerRetryPolicy {
12964 max_retries: 2,
12965 initial_backoff: Duration::from_millis(1),
12966 max_backoff: Duration::from_millis(1),
12967 });
12968 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
12969 worker.register_activity(
12970 "counter.activity",
12971 |_ctx, _input| async move { Ok(Value::Null) },
12972 );
12973 worker.register_query(
12974 "counter",
12975 "current",
12976 |_ctx, _args| async move { Ok(json!(8)) },
12977 );
12978
12979 worker
12980 .run_until(tokio::time::sleep(Duration::from_millis(75)))
12981 .await
12982 .expect("transient failures must not stop the worker");
12983
12984 for path in [
12985 "/api/worker/heartbeat",
12986 "/api/worker/workflow-tasks/poll",
12987 "/api/worker/activity-tasks/poll",
12988 "/api/worker/query-tasks/poll",
12989 ] {
12990 assert!(
12991 server.request_count(path) >= 2,
12992 "{path} must continue after its transient failure"
12993 );
12994 }
12995 }
12996
12997 #[tokio::test]
12998 async fn worker_bounds_transport_retries() {
12999 let server = MockWorkerServer::unavailable_polls();
13000 let client = Client::builder(server.base_url())
13001 .timeout(Duration::from_secs(2))
13002 .build()
13003 .expect("client");
13004 let mut worker = Worker::new(client, "rust-workers")
13005 .worker_id("bounded-retry-worker")
13006 .poll_timeout(Duration::from_millis(10))
13007 .retry_policy(WorkerRetryPolicy {
13008 max_retries: 2,
13009 initial_backoff: Duration::from_millis(1),
13010 max_backoff: Duration::from_millis(1),
13011 });
13012 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
13013
13014 let error = worker.run().await.expect_err("retry bound must terminate");
13015 assert!(matches!(error, Error::Transport(_)));
13016 assert_eq!(
13017 server.request_count("/api/worker/workflow-tasks/poll"),
13018 3,
13019 "one initial request plus exactly two retries"
13020 );
13021 }
13022
13023 #[tokio::test]
13024 async fn worker_retry_policy_can_disable_poll_retries() {
13025 let server = MockWorkerServer::unavailable_polls();
13026 let client = Client::builder(server.base_url())
13027 .timeout(Duration::from_secs(2))
13028 .build()
13029 .expect("client");
13030 let mut worker = Worker::new(client, "rust-workers")
13031 .worker_id("no-retry-worker")
13032 .poll_timeout(Duration::from_millis(10))
13033 .retry_policy(WorkerRetryPolicy {
13034 max_retries: 0,
13035 initial_backoff: Duration::from_millis(1),
13036 max_backoff: Duration::from_millis(1),
13037 });
13038 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
13039
13040 let error = worker
13041 .run_once()
13042 .await
13043 .expect_err("disabled retries must return the first transport failure");
13044 assert!(matches!(error, Error::Transport(_)));
13045 assert_eq!(
13046 server.request_count("/api/worker/workflow-tasks/poll"),
13047 1,
13048 "max_retries=0 must send only the initial request"
13049 );
13050 }
13051
13052 #[tokio::test]
13053 async fn worker_does_not_retry_authentication_failures() {
13054 let server = MockWorkerServer::unauthorized_polls();
13055 let client = Client::builder(server.base_url())
13056 .timeout(Duration::from_secs(2))
13057 .build()
13058 .expect("client");
13059 let mut worker = Worker::new(client, "rust-workers")
13060 .worker_id("unauthorized-worker")
13061 .poll_timeout(Duration::from_millis(10));
13062 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
13063
13064 let error = worker
13065 .run()
13066 .await
13067 .expect_err("authentication must terminate");
13068 let Error::Http { status, body } = error else {
13069 panic!("expected stable HTTP authentication error");
13070 };
13071 assert_eq!(status, reqwest::StatusCode::UNAUTHORIZED);
13072 assert!(body.contains("authentication_failed"));
13073 assert_eq!(
13074 server.request_count("/api/worker/workflow-tasks/poll"),
13075 1,
13076 "authentication failures must not be retried"
13077 );
13078 }
13079
13080 #[derive(Clone, Debug)]
13081 struct CapturedRequest {
13082 path: String,
13083 worker_protocol: Option<String>,
13084 body: String,
13085 received_at: Instant,
13086 }
13087
13088 struct MockWorkerServer {
13089 addr: SocketAddr,
13090 stop: Arc<AtomicBool>,
13091 requests: Arc<Mutex<Vec<CapturedRequest>>>,
13092 thread: Option<thread::JoinHandle<()>>,
13093 }
13094
13095 #[derive(Clone, Copy, Default)]
13096 struct MockWorkerBehavior {
13097 reject_query_protocol: bool,
13098 reject_query_completion: bool,
13099 waiting_query_worker: bool,
13100 complete_named_signal: bool,
13101 poll_failures_per_path: usize,
13102 heartbeat_failures: usize,
13103 heartbeat_failure_request: Option<usize>,
13104 delayed_heartbeat_request: Option<usize>,
13105 heartbeat_response_delay: Duration,
13106 concurrent_requests: bool,
13107 unauthorized_polls: bool,
13108 cancelled_activity: bool,
13109 draining_polls: bool,
13110 workflow_completion_status: Option<&'static str>,
13111 workflow_completion_body: Option<&'static str>,
13112 }
13113
13114 impl MockWorkerServer {
13115 fn start() -> Self {
13116 Self::start_with_behavior(MockWorkerBehavior::default())
13117 }
13118
13119 fn reject_query_protocol() -> Self {
13120 Self::start_with_behavior(MockWorkerBehavior {
13121 reject_query_protocol: true,
13122 ..MockWorkerBehavior::default()
13123 })
13124 }
13125
13126 fn reject_query_completion() -> Self {
13127 Self::start_with_behavior(MockWorkerBehavior {
13128 reject_query_completion: true,
13129 ..MockWorkerBehavior::default()
13130 })
13131 }
13132
13133 fn waiting_query_worker() -> Self {
13134 Self::start_with_behavior(MockWorkerBehavior {
13135 waiting_query_worker: true,
13136 complete_named_signal: true,
13137 ..MockWorkerBehavior::default()
13138 })
13139 }
13140
13141 fn transient_worker_failures() -> Self {
13142 Self::start_with_behavior(MockWorkerBehavior {
13143 poll_failures_per_path: 1,
13144 heartbeat_failures: 1,
13145 ..MockWorkerBehavior::default()
13146 })
13147 }
13148
13149 fn consecutive_poll_failures(count: usize) -> Self {
13150 Self::start_with_behavior(MockWorkerBehavior {
13151 poll_failures_per_path: count,
13152 ..MockWorkerBehavior::default()
13153 })
13154 }
13155
13156 fn delayed_heartbeat_worker() -> Self {
13157 Self::start_with_behavior(MockWorkerBehavior {
13158 waiting_query_worker: true,
13159 delayed_heartbeat_request: Some(2),
13160 heartbeat_response_delay: Duration::from_millis(1_500),
13161 concurrent_requests: true,
13162 cancelled_activity: true,
13163 ..MockWorkerBehavior::default()
13164 })
13165 }
13166
13167 fn heartbeat_retry_worker() -> Self {
13168 Self::start_with_behavior(MockWorkerBehavior {
13169 waiting_query_worker: true,
13170 heartbeat_failure_request: Some(2),
13171 concurrent_requests: true,
13172 ..MockWorkerBehavior::default()
13173 })
13174 }
13175
13176 fn unavailable_polls() -> Self {
13177 Self::start_with_behavior(MockWorkerBehavior {
13178 poll_failures_per_path: usize::MAX,
13179 ..MockWorkerBehavior::default()
13180 })
13181 }
13182
13183 fn unauthorized_polls() -> Self {
13184 Self::start_with_behavior(MockWorkerBehavior {
13185 unauthorized_polls: true,
13186 ..MockWorkerBehavior::default()
13187 })
13188 }
13189
13190 fn cancelled_activity() -> Self {
13191 Self::start_with_behavior(MockWorkerBehavior {
13192 cancelled_activity: true,
13193 ..MockWorkerBehavior::default()
13194 })
13195 }
13196
13197 fn draining_polls() -> Self {
13198 Self::start_with_behavior(MockWorkerBehavior {
13199 draining_polls: true,
13200 ..MockWorkerBehavior::default()
13201 })
13202 }
13203
13204 fn workflow_completion(status: &'static str, body: &'static str) -> Self {
13205 Self::start_with_behavior(MockWorkerBehavior {
13206 workflow_completion_status: Some(status),
13207 workflow_completion_body: Some(body),
13208 ..MockWorkerBehavior::default()
13209 })
13210 }
13211
13212 fn start_with_behavior(behavior: MockWorkerBehavior) -> Self {
13213 let listener = TcpListener::bind("127.0.0.1:0").expect("bind mock server");
13214 listener
13215 .set_nonblocking(true)
13216 .expect("configure mock listener");
13217 let addr = listener.local_addr().expect("mock server address");
13218 let stop = Arc::new(AtomicBool::new(false));
13219 let server_stop = Arc::clone(&stop);
13220 let requests = Arc::new(Mutex::new(Vec::new()));
13221 let server_requests = Arc::clone(&requests);
13222 let thread = thread::spawn(move || {
13223 let mut request_threads = Vec::new();
13224 while !server_stop.load(Ordering::SeqCst) {
13225 match listener.accept() {
13226 Ok((mut stream, _)) => {
13227 if behavior.concurrent_requests {
13228 let requests = Arc::clone(&server_requests);
13229 request_threads.push(thread::spawn(move || {
13230 handle_mock_worker_request(&mut stream, &requests, behavior)
13231 }));
13232 } else {
13233 handle_mock_worker_request(&mut stream, &server_requests, behavior);
13234 }
13235 }
13236 Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
13237 let mut index = 0;
13238 while index < request_threads.len() {
13239 if request_threads[index].is_finished() {
13240 request_threads
13241 .swap_remove(index)
13242 .join()
13243 .expect("join mock request");
13244 } else {
13245 index += 1;
13246 }
13247 }
13248 thread::sleep(Duration::from_millis(5));
13249 }
13250 Err(_) => break,
13251 }
13252 }
13253 for request_thread in request_threads {
13254 request_thread.join().expect("join mock request");
13255 }
13256 });
13257
13258 Self {
13259 addr,
13260 stop,
13261 requests,
13262 thread: Some(thread),
13263 }
13264 }
13265
13266 fn base_url(&self) -> String {
13267 format!("http://{}", self.addr)
13268 }
13269
13270 fn worker_protocol_for(&self, path: &str) -> Option<String> {
13271 self.requests
13272 .lock()
13273 .expect("captured requests")
13274 .iter()
13275 .find(|request| request.path == path)
13276 .and_then(|request| request.worker_protocol.clone())
13277 }
13278
13279 fn request_count(&self, path: &str) -> usize {
13280 self.requests
13281 .lock()
13282 .expect("captured requests")
13283 .iter()
13284 .filter(|request| request.path == path)
13285 .count()
13286 }
13287
13288 fn request_times(&self, path: &str) -> Vec<Instant> {
13289 self.requests
13290 .lock()
13291 .expect("captured requests")
13292 .iter()
13293 .filter(|request| request.path == path)
13294 .map(|request| request.received_at)
13295 .collect()
13296 }
13297
13298 fn request_body(&self, path: &str) -> Value {
13299 let requests = self.requests.lock().expect("captured requests");
13300 let body = &requests
13301 .iter()
13302 .find(|request| request.path == path)
13303 .unwrap_or_else(|| panic!("missing request for {path}"))
13304 .body;
13305 serde_json::from_str(body).unwrap_or_else(|error| {
13306 panic!("invalid JSON request body for {path}: {error}: {body:?}")
13307 })
13308 }
13309
13310 fn request_bodies(&self, path: &str) -> Vec<Value> {
13311 self.requests
13312 .lock()
13313 .expect("captured requests")
13314 .iter()
13315 .filter(|request| request.path == path)
13316 .map(|request| {
13317 serde_json::from_str(&request.body).unwrap_or_else(|error| {
13318 panic!(
13319 "invalid JSON request body for {path}: {error}: {:?}",
13320 request.body
13321 )
13322 })
13323 })
13324 .collect()
13325 }
13326 }
13327
13328 impl Drop for MockWorkerServer {
13329 fn drop(&mut self) {
13330 self.stop.store(true, Ordering::SeqCst);
13331 let _ = TcpStream::connect(self.addr);
13332
13333 if let Some(thread) = self.thread.take() {
13334 thread.join().expect("join mock server");
13335 }
13336 }
13337 }
13338
13339 fn handle_mock_worker_request(
13340 stream: &mut TcpStream,
13341 requests: &Arc<Mutex<Vec<CapturedRequest>>>,
13342 behavior: MockWorkerBehavior,
13343 ) {
13344 let _ = stream.set_read_timeout(Some(Duration::from_millis(200)));
13345 let mut buffer = [0_u8; 8192];
13346 let mut request = Vec::new();
13347
13348 loop {
13349 match stream.read(&mut buffer) {
13350 Ok(0) => break,
13351 Ok(read) => {
13352 request.extend_from_slice(&buffer[..read]);
13353 if mock_request_is_complete(&request) {
13354 break;
13355 }
13356 }
13357 Err(error)
13358 if matches!(
13359 error.kind(),
13360 std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
13361 ) =>
13362 {
13363 break;
13364 }
13365 Err(_) => return,
13366 }
13367 }
13368
13369 let request = String::from_utf8_lossy(&request);
13370 let body = request
13371 .split_once("\r\n\r\n")
13372 .map(|(_, body)| body)
13373 .unwrap_or_default();
13374 let path = request
13375 .lines()
13376 .next()
13377 .and_then(|line| line.split_whitespace().nth(1))
13378 .unwrap_or_default();
13379 let worker_protocol = request.lines().find_map(|line| {
13380 let (name, value) = line.split_once(':')?;
13381 name.eq_ignore_ascii_case("X-Durable-Workflow-Protocol-Version")
13382 .then(|| value.trim().to_string())
13383 });
13384 let request_number = {
13385 let mut requests = requests.lock().expect("captured requests");
13386 requests.push(CapturedRequest {
13387 path: path.to_string(),
13388 worker_protocol: worker_protocol.clone(),
13389 body: body.to_string(),
13390 received_at: Instant::now(),
13391 });
13392 requests
13393 .iter()
13394 .filter(|request| request.path == path)
13395 .count()
13396 };
13397
13398 let is_poll = matches!(
13399 path,
13400 "/api/worker/workflow-tasks/poll"
13401 | "/api/worker/activity-tasks/poll"
13402 | "/api/worker/query-tasks/poll"
13403 );
13404 if is_poll && request_number <= behavior.poll_failures_per_path {
13405 return;
13406 }
13407 if path == "/api/worker/heartbeat" && request_number <= behavior.heartbeat_failures {
13408 return;
13409 }
13410 if path == "/api/worker/heartbeat"
13411 && behavior.heartbeat_failure_request == Some(request_number)
13412 {
13413 return;
13414 }
13415 if path == "/api/worker/heartbeat"
13416 && behavior.delayed_heartbeat_request == Some(request_number)
13417 {
13418 thread::sleep(behavior.heartbeat_response_delay);
13419 }
13420 if behavior.unauthorized_polls && is_poll {
13421 write_mock_response(
13422 stream,
13423 "401 Unauthorized",
13424 r#"{"reason":"authentication_failed","message":"invalid worker token"}"#,
13425 );
13426 return;
13427 }
13428 if behavior.draining_polls && is_poll {
13429 write_mock_response(
13430 stream,
13431 "409 Conflict",
13432 r#"{"task":null,"poll_status":"draining","reason":"worker_draining","worker_status":"draining","drain_intent":"draining"}"#,
13433 );
13434 return;
13435 }
13436
13437 if behavior.reject_query_protocol && path.starts_with("/api/worker/query-tasks/") {
13438 let requested_version = worker_protocol.as_deref().unwrap_or("missing");
13439 let body = format!(
13440 r#"{{"reason":"unsupported_protocol_version","message":"Query tasks require worker protocol 1.8 or newer.","supported_version":"1.7","requested_version":"{requested_version}"}}"#
13441 );
13442 write_mock_response(stream, "400 Bad Request", &body);
13443 return;
13444 }
13445
13446 if behavior.reject_query_completion && path == "/api/worker/query-tasks/query-late/complete"
13447 {
13448 write_mock_response(
13449 stream,
13450 "409 Conflict",
13451 r#"{"reason":"query_task_timed_out","message":"query task timed out before completion"}"#,
13452 );
13453 return;
13454 }
13455
13456 if behavior.workflow_completion_status.is_some()
13457 && path == "/api/worker/workflow-tasks/poll"
13458 && request_number == 1
13459 {
13460 write_mock_response(
13461 stream,
13462 "200 OK",
13463 r#"{"task":{"task_id":"workflow-timeout-task","workflow_id":"reused-workflow-id","run_id":"run-selected-timeout","workflow_type":"timeout.workflow","payload_codec":"json","arguments":{"codec":"json","blob":"[]"},"history_events":[],"workflow_task_attempt":3,"lease_owner":"timeout-worker"}}"#,
13464 );
13465 return;
13466 }
13467
13468 if path == "/api/worker/workflow-tasks/workflow-timeout-task/complete" {
13469 if let (Some(status), Some(body)) = (
13470 behavior.workflow_completion_status,
13471 behavior.workflow_completion_body,
13472 ) {
13473 write_mock_response(stream, status, body);
13474 return;
13475 }
13476 }
13477
13478 if behavior.waiting_query_worker {
13479 if behavior.complete_named_signal
13480 && path == "/api/worker/workflow-tasks/poll"
13481 && request_number == 1
13482 {
13483 let body = json!({
13484 "task": {
13485 "task_id": "snapshot-open",
13486 "workflow_id": "snapshot-1",
13487 "run_id": "snapshot-run-1",
13488 "workflow_type": "snapshot",
13489 "payload_codec": DEFAULT_CODEC,
13490 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
13491 .expect("Avro workflow arguments"),
13492 "history_events": [],
13493 "workflow_task_attempt": 1,
13494 "lease_owner": "rust-snapshot-worker"
13495 }
13496 })
13497 .to_string();
13498 write_mock_response(stream, "200 OK", &body);
13499 return;
13500 }
13501
13502 let signal_request = request_number - usize::from(behavior.complete_named_signal);
13503 let signal_request_limit = 2 + usize::from(behavior.complete_named_signal);
13504 if path == "/api/worker/workflow-tasks/poll"
13505 && signal_request >= 1
13506 && signal_request <= signal_request_limit
13507 {
13508 let finish = behavior.complete_named_signal && signal_request == 3;
13509 let amounts = if signal_request == 1 {
13510 vec![3]
13511 } else {
13512 vec![3, 5]
13513 };
13514 let task_id = if signal_request == 1 {
13515 "snapshot-wait-3"
13516 } else if finish {
13517 "snapshot-finish"
13518 } else {
13519 "snapshot-wait-5"
13520 };
13521 let mut history_events = std::iter::once(json!({
13522 "event_type": "SignalWaitOpened",
13523 "payload": {"sequence": 1, "signal_name": "finish"}
13524 }))
13525 .chain(amounts.iter().enumerate().map(|(index, amount)| {
13526 json!({
13527 "event_type": "SignalReceived",
13528 "payload": {
13529 "signal_id": format!("increment-{amount}"),
13530 "signal_name": "increment",
13531 "workflow_sequence": index + 2,
13532 "payload_codec": DEFAULT_CODEC,
13533 "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
13534 .expect("Avro signal envelope")
13535 }
13536 })
13537 }))
13538 .collect::<Vec<_>>();
13539 let (resume_id, resume_name, resume_arguments) = if finish {
13540 history_events.push(json!({
13541 "event_type": "SignalReceived",
13542 "payload": {
13543 "signal_id": "finish",
13544 "signal_name": "finish",
13545 "workflow_sequence": 4,
13546 "payload_codec": DEFAULT_CODEC,
13547 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
13548 .expect("Avro finish signal envelope")
13549 }
13550 }));
13551 (
13552 "finish".to_string(),
13553 "finish".to_string(),
13554 encode_value_envelope(&json!([]), DEFAULT_CODEC)
13555 .expect("Avro finish resume signal"),
13556 )
13557 } else {
13558 let amount = amounts.last().expect("amount");
13559 (
13560 format!("increment-{amount}"),
13561 "increment".to_string(),
13562 encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
13563 .expect("Avro increment resume signal"),
13564 )
13565 };
13566 let body = json!({
13567 "task": {
13568 "task_id": task_id,
13569 "workflow_id": "snapshot-1",
13570 "run_id": "snapshot-run-1",
13571 "workflow_type": "snapshot",
13572 "payload_codec": DEFAULT_CODEC,
13573 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
13574 .expect("Avro workflow arguments"),
13575 "history_events": history_events,
13576 "workflow_task_attempt": 1,
13577 "workflow_signal_id": resume_id,
13578 "signal_name": resume_name,
13579 "signal_arguments": resume_arguments,
13580 "lease_owner": "rust-snapshot-worker"
13581 }
13582 })
13583 .to_string();
13584 write_mock_response(stream, "200 OK", &body);
13585 return;
13586 }
13587
13588 if path == "/api/worker/query-tasks/poll" && request_number == 1 {
13589 let history_events = [3, 5]
13590 .into_iter()
13591 .enumerate()
13592 .map(|(index, amount)| {
13593 json!({
13594 "event_type": "SignalReceived",
13595 "payload": {
13596 "signal_id": format!("increment-{amount}"),
13597 "signal_name": "increment",
13598 "workflow_sequence": index + 2,
13599 "payload_codec": DEFAULT_CODEC,
13600 "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
13601 .expect("Avro query signal envelope")
13602 }
13603 })
13604 })
13605 .collect::<Vec<_>>();
13606 let body = json!({
13607 "task": {
13608 "query_task_id": "snapshot-current",
13609 "query_task_attempt": 1,
13610 "lease_owner": "rust-snapshot-worker",
13611 "workflow_id": "snapshot-1",
13612 "run_id": "snapshot-run-1",
13613 "workflow_type": "snapshot",
13614 "query_name": "current",
13615 "payload_codec": DEFAULT_CODEC,
13616 "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
13617 .expect("Avro workflow arguments"),
13618 "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
13619 .expect("Avro query arguments"),
13620 "history_events": history_events,
13621 "run_status": "waiting"
13622 }
13623 })
13624 .to_string();
13625 write_mock_response(stream, "200 OK", &body);
13626 return;
13627 }
13628
13629 if path == "/api/worker/workflow-tasks/snapshot-wait-3/fail"
13630 || path == "/api/worker/workflow-tasks/snapshot-wait-5/fail"
13631 {
13632 write_mock_response(
13633 stream,
13634 "200 OK",
13635 r#"{"outcome":"waiting_for_history","recorded":true}"#,
13636 );
13637 return;
13638 }
13639
13640 if path == "/api/worker/workflow-tasks/snapshot-open/complete" {
13641 write_mock_response(stream, "200 OK", r#"{"outcome":"waiting","recorded":true}"#);
13642 return;
13643 }
13644
13645 if path == "/api/worker/workflow-tasks/snapshot-finish/complete" {
13646 write_mock_response(
13647 stream,
13648 "200 OK",
13649 r#"{"outcome":"completed","run_status":"completed","recorded":true}"#,
13650 );
13651 return;
13652 }
13653
13654 if path == "/api/worker/query-tasks/snapshot-current/complete" {
13655 write_mock_response(stream, "200 OK", r#"{"outcome":"completed"}"#);
13656 return;
13657 }
13658 }
13659
13660 if matches!(
13661 path,
13662 "/api/workflows/typed-1/query/inspect" | "/api/workflows/typed-1/update/replace"
13663 ) {
13664 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
13665 .expect("typed mock result");
13666 let body = json!({
13667 "result": typed_fidelity_probe().into_json().expect("result projection"),
13668 "result_envelope": result,
13669 })
13670 .to_string();
13671 write_mock_response(stream, "200 OK", &body);
13672 return;
13673 }
13674
13675 if path == "/api/workflows/typed-1" {
13676 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
13677 .expect("typed mock result");
13678 let body = json!({
13679 "workflow_id": "typed-1",
13680 "run_id": "run-typed-1",
13681 "workflow_type": "typed.echo",
13682 "status": "completed",
13683 "output": typed_fidelity_probe().into_json().expect("output projection"),
13684 "output_envelope": result,
13685 })
13686 .to_string();
13687 write_mock_response(stream, "200 OK", &body);
13688 return;
13689 }
13690
13691 let (status, body) = match path {
13692 "/api/workflows" => (
13693 "201 Created",
13694 r#"{"workflow_id":"wf-start-options","run_id":"run-start-options","workflow_type":"rust.timeout"}"#,
13695 ),
13696 "/api/worker/register" if behavior.waiting_query_worker => (
13697 "200 OK",
13698 r#"{"worker_id":"rust-snapshot-worker","registered":true,"heartbeat_interval_seconds":1}"#,
13699 ),
13700 "/api/worker/register" => (
13701 "200 OK",
13702 r#"{"worker_id":"mock-worker","registered":true,"heartbeat_interval_seconds":3600}"#,
13703 ),
13704 "/api/worker/heartbeat" => ("200 OK", "{}"),
13705 "/api/worker/activity-tasks/poll"
13706 if behavior.cancelled_activity && request_number == 1 =>
13707 {
13708 (
13709 "200 OK",
13710 r#"{"task":{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","activity_type":"cancel-aware","payload_codec":"json","arguments":{"codec":"json","blob":"[]"},"attempt_number":1,"lease_owner":"rust-cancel-worker"}}"#,
13711 )
13712 }
13713 "/api/worker/activity-tasks/poll" | "/api/worker/workflow-tasks/poll" => {
13714 ("200 OK", r#"{"task":null}"#)
13715 }
13716 "/api/worker/query-tasks/poll"
13717 if behavior.reject_query_completion && request_number == 1 =>
13718 {
13719 (
13720 "200 OK",
13721 r#"{"task":{"query_task_id":"query-late","query_task_attempt":1,"lease_owner":"late-worker","workflow_id":"counter-late","run_id":"run-late","workflow_type":"counter","query_name":"current","payload_codec":"json","workflow_arguments":{"codec":"json","blob":"[]"},"query_arguments":{"codec":"json","blob":"[]"},"history_events":[],"run_status":"running"}}"#,
13722 )
13723 }
13724 "/api/worker/query-tasks/poll" => ("200 OK", r#"{"task":null}"#),
13725 "/api/worker/query-tasks/query-capture/complete"
13726 | "/api/worker/query-tasks/query-capture/fail" => ("200 OK", "{}"),
13727 "/api/worker/activity-tasks/activity-cancel/heartbeat" => (
13728 "200 OK",
13729 r#"{"activity_attempt_id":"attempt-cancel","cancel_requested":true,"can_continue":false,"reason":"run_cancelled","run_closed_reason":"cancelled","heartbeat_recorded":false}"#,
13730 ),
13731 "/api/worker/activity-tasks/activity-cancel/complete" => (
13732 "409 Conflict",
13733 r#"{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","reason":"run_cancelled","cancel_requested":true,"can_continue":false,"run_closed_reason":"cancelled"}"#,
13734 ),
13735 "/api/worker/activity-tasks/activity-typed/complete"
13736 | "/api/worker/activity-tasks/activity-typed/fail"
13737 | "/api/workflows/typed-1/signal/changed" => ("200 OK", "{}"),
13738 "/api/workflows/counter-1/query/current" => (
13739 "200 OK",
13740 r#"{"workflow_id":"counter-1","query_name":"current","result":{"count":8},"result_envelope":{"codec":"json","blob":"{\"count\":8}"}}"#,
13741 ),
13742 "/api/workflows/counter-1/query/missing" => (
13743 "404 Not Found",
13744 r#"{"workflow_id":"counter-1","query_name":"missing","reason":"rejected_unknown_query","message":"unknown query"}"#,
13745 ),
13746 "/api/workflows/wf-lifecycle/cancel" => (
13747 "200 OK",
13748 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","reason":"cleanup requested","command_status":"accepted"}"#,
13749 ),
13750 "/api/workflows/wf-lifecycle/terminate" => (
13751 "200 OK",
13752 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","reason":"forced stop","command_status":"accepted"}"#,
13753 ),
13754 "/api/workflows/wf-lifecycle/runs/run-current/cancel" => (
13755 "200 OK",
13756 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","command_status":"accepted"}"#,
13757 ),
13758 "/api/workflows/wf-lifecycle/runs/run-current/terminate" => (
13759 "200 OK",
13760 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","command_status":"accepted"}"#,
13761 ),
13762 "/api/workflows/wf-lifecycle/runs/run-stale/cancel"
13763 | "/api/workflows/wf-lifecycle/runs/run-stale/terminate" => (
13764 "409 Conflict",
13765 r#"{"workflow_id":"wf-lifecycle","run_id":"run-stale","reason":"historical_run_command_rejected","target_scope":"run","message":"Commands cannot target historical runs."}"#,
13766 ),
13767 "/api/workflows/wf-failed" | "/api/workflows/wf-failed/runs/run-failed" => (
13768 "200 OK",
13769 r#"{"workflow_id":"wf-failed","run_id":"run-failed","status":"failed","closed_reason":"failed","error":"payment failed","failure":{"message":"payment failed","failure_category":"application","exception_type":"PaymentError","exception_class":"billing::PaymentError","non_retryable":true,"exception":{"type":"PaymentError","class":"billing::PaymentError","message":"payment failed"},"failures":[{"id":"failure-17","failure_category":"application"}]}}"#,
13770 ),
13771 "/api/workflows/wf-cancelled" => (
13772 "200 OK",
13773 r#"{"workflow_id":"wf-cancelled","run_id":"run-cancelled","status":"cancelled","closed_reason":"cancelled","reason":"cleanup requested"}"#,
13774 ),
13775 "/api/workflows/wf-terminated" => (
13776 "200 OK",
13777 r#"{"workflow_id":"wf-terminated","run_id":"run-terminated","status":"terminated","closed_reason":"terminated","reason":"forced stop"}"#,
13778 ),
13779 "/api/workflows/wf-timed-out" => (
13780 "200 OK",
13781 r#"{"workflow_id":"wf-timed-out","run_id":"run-timed-out","status":"failed","closed_reason":"timed_out","reason":"run_timeout"}"#,
13782 ),
13783 "/api/workflows/wf-waiting" | "/api/workflows/wf-waiting/runs/run-waiting" => (
13784 "200 OK",
13785 r#"{"workflow_id":"wf-waiting","run_id":"run-waiting","status":"waiting"}"#,
13786 ),
13787 "/api/workflows/wf-selected" => (
13788 "200 OK",
13789 r#"{"workflow_id":"wf-selected","run_id":"run-current","status":"completed","output":"current run output"}"#,
13790 ),
13791 "/api/workflows/wf-selected/runs/run-selected" => (
13792 "200 OK",
13793 r#"{"workflow_id":"wf-selected","run_id":"run-selected","status":"cancelled","closed_reason":"cancelled","reason":"selected run cancelled"}"#,
13794 ),
13795 _ => ("404 Not Found", r#"{"message":"not found"}"#),
13796 };
13797 write_mock_response(stream, status, body);
13798 }
13799
13800 fn mock_request_is_complete(request: &[u8]) -> bool {
13801 let Some(header_end) = request
13802 .windows(4)
13803 .position(|window| window == b"\r\n\r\n")
13804 .map(|position| position + 4)
13805 else {
13806 return false;
13807 };
13808 let headers = String::from_utf8_lossy(&request[..header_end]);
13809 let content_length = headers.lines().find_map(|line| {
13810 let (name, value) = line.split_once(':')?;
13811 name.eq_ignore_ascii_case("content-length")
13812 .then(|| value.trim().parse::<usize>().ok())
13813 .flatten()
13814 });
13815
13816 request.len() >= header_end + content_length.unwrap_or(0)
13817 }
13818
13819 fn write_mock_response(stream: &mut TcpStream, status: &str, body: &str) {
13820 let response = format!(
13821 "HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
13822 body.len()
13823 );
13824
13825 let _ = stream.write_all(response.as_bytes());
13826 let _ = stream.flush();
13827 }
13828}