1use aion_core::{ActivityId, Event, RunId, WorkflowId};
4
5use crate::durability::{
6 correlation::{CorrelationKey, key_for_event},
7 error::DurabilityError,
8};
9
10pub fn current_run_segment(
25 history: Vec<Event>,
26 run_id: &RunId,
27) -> Result<Vec<Event>, DurabilityError> {
28 let start = history
29 .iter()
30 .position(|event| {
31 matches!(
32 event,
33 Event::WorkflowStarted {
34 run_id: event_run_id,
35 ..
36 } if event_run_id == run_id
37 )
38 })
39 .ok_or_else(|| DurabilityError::HistoryShape {
40 reason: format!("history has no WorkflowStarted for run {run_id}"),
41 })?;
42 let mut segment = history;
43 segment.drain(..start);
44 Ok(segment)
45}
46
47#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
49pub enum RecordedEventFamily {
50 Activity,
52 Timer,
54 Signal,
56 Child,
58}
59
60#[derive(Clone, Debug, PartialEq, Eq)]
62pub struct FoundEventDescriptor {
63 pub seq: u64,
65 pub family: Option<RecordedEventFamily>,
67 pub key: Option<CorrelationKey>,
69 pub kind: &'static str,
71}
72
73#[derive(Clone, Debug, PartialEq)]
75pub enum CursorResolveResult {
76 Matched(Vec<Event>),
78 Exhausted,
80 Mismatch {
82 expected_key: CorrelationKey,
84 found: FoundEventDescriptor,
86 },
87}
88
89#[derive(Clone, Debug, PartialEq)]
95pub enum ChildTerminalResolveResult {
96 Matched(Vec<Event>),
98 Exhausted,
100 Mismatch {
102 found: FoundEventDescriptor,
104 },
105}
106
107#[derive(Clone, Debug)]
117pub struct HistoryCursor {
118 events: Vec<Event>,
119 consumed: Vec<bool>,
120 position: usize,
121}
122
123impl HistoryCursor {
124 pub fn new(events: Vec<Event>) -> Result<Self, DurabilityError> {
130 for pair in events.windows(2) {
131 let prior = pair[0].seq();
132 let next = pair[1].seq();
133 if next < prior {
134 return Err(DurabilityError::HistoryShape {
135 reason: format!("history sequence order decreased from {prior} to {next}"),
136 });
137 }
138 }
139
140 let consumed = vec![false; events.len()];
141 Ok(Self {
142 events,
143 consumed,
144 position: 0,
145 })
146 }
147
148 #[must_use]
150 pub fn current_sequence(&self) -> Option<u64> {
151 self.events.get(self.position).map(Event::seq)
152 }
153
154 #[must_use]
156 pub fn events(&self) -> &[Event] {
157 &self.events
158 }
159
160 #[must_use]
162 pub const fn position_index(&self) -> usize {
163 self.position
164 }
165
166 #[must_use]
168 pub fn next_key(&self, family: RecordedEventFamily) -> Option<CorrelationKey> {
169 let index = self.next_matchable_index()?;
170 let descriptor = self.descriptor_at(index, &self.events[index]);
171 if descriptor.family == Some(family) {
172 descriptor.key
173 } else {
174 None
175 }
176 }
177
178 pub fn fast_forward_to_key(&mut self, key: &CorrelationKey) {
190 while let Some(index) = self.next_matchable_index() {
191 let descriptor = self.descriptor_at(index, &self.events[index]);
192 if descriptor.key.as_ref() == Some(key) {
193 return;
194 }
195 self.position = index + 1;
196 }
197 }
198
199 pub fn fast_forward_to_child_terminal(&mut self, child_workflow_id: &WorkflowId) {
211 while let Some(index) = self.next_matchable_index() {
212 if is_child_terminal_for(&self.events[index], child_workflow_id) {
213 return;
214 }
215 self.position = index + 1;
216 }
217 }
218
219 #[must_use]
221 pub fn resolve_child_terminal(
222 &mut self,
223 child_workflow_id: &WorkflowId,
224 ) -> ChildTerminalResolveResult {
225 let Some(found_index) = self.next_matchable_index() else {
226 return ChildTerminalResolveResult::Exhausted;
227 };
228 self.position = found_index;
229 match self.events.get(found_index) {
230 Some(event) if is_child_terminal_for(event, child_workflow_id) => {
231 ChildTerminalResolveResult::Matched(self.take_range(found_index, found_index + 1))
232 }
233 Some(event) => ChildTerminalResolveResult::Mismatch {
234 found: self.descriptor_at(found_index, event),
235 },
236 None => ChildTerminalResolveResult::Exhausted,
237 }
238 }
239
240 #[must_use]
242 pub fn resolve_next(
243 &mut self,
244 family: RecordedEventFamily,
245 expected_key: CorrelationKey,
246 ) -> CursorResolveResult {
247 let Some(found_index) = self.next_matchable_index() else {
248 return CursorResolveResult::Exhausted;
249 };
250 self.position = found_index;
251
252 let found = self.descriptor_at(found_index, &self.events[found_index]);
253 if found.family != Some(family) || found.key.as_ref() != Some(&expected_key) {
254 return CursorResolveResult::Mismatch {
255 expected_key,
256 found,
257 };
258 }
259
260 match family {
261 RecordedEventFamily::Activity => self.resolve_activity(expected_key),
262 RecordedEventFamily::Timer => {
263 self.resolve_started_with_immediate_outcome(&expected_key)
264 }
265 RecordedEventFamily::Child => self.resolve_child(expected_key),
266 RecordedEventFamily::Signal => match expected_key {
267 CorrelationKey::Signal { .. } => self.consume_one(),
268 _ => self.mismatch_at_current(expected_key),
269 },
270 }
271 }
272
273 fn resolve_child(&mut self, expected_key: CorrelationKey) -> CursorResolveResult {
280 match self.events.get(self.position) {
281 Some(Event::ChildWorkflowStarted { .. }) => self.consume_one(),
282 _ => self.mismatch_at_current(expected_key),
283 }
284 }
285
286 fn resolve_activity(&mut self, expected_key: CorrelationKey) -> CursorResolveResult {
298 let Some(Event::ActivityScheduled { activity_id, .. }) = self.events.get(self.position)
299 else {
300 return self.mismatch_at_current(expected_key);
301 };
302 let activity_id = activity_id.clone();
303 let mut matched = vec![self.position];
304 let mut index = self.position + 1;
305
306 while let Some(event) = self.events.get(index) {
307 match event {
308 Event::ActivityStarted {
309 activity_id: event_activity_id,
310 ..
311 } if event_activity_id == &activity_id => {
312 matched.push(index);
313 }
314 Event::ActivityScheduled {
315 activity_id: event_activity_id,
316 ..
317 } if event_activity_id == &activity_id => {
318 matched.push(index);
323 }
324 Event::ActivityFailed {
325 activity_id: event_activity_id,
326 error,
327 ..
328 } if event_activity_id == &activity_id => {
329 matched.push(index);
330 let superseded_by_reopen =
342 self.activity_reopened_after(index + 1, &activity_id);
343 if !error.is_retryable()
344 && !superseded_by_reopen
345 && !self.has_later_activity_attempt_or_outcome(index + 1, &activity_id)
346 {
347 return self.consume_indices(matched);
348 }
349 }
350 Event::ActivityCompleted {
351 activity_id: event_activity_id,
352 ..
353 }
354 | Event::ActivityCancelled {
355 activity_id: event_activity_id,
356 ..
357 } if event_activity_id == &activity_id => {
358 matched.push(index);
359 return self.consume_indices(matched);
360 }
361 _ => {}
362 }
363 index += 1;
364 }
365
366 CursorResolveResult::Exhausted
367 }
368
369 fn resolve_started_with_immediate_outcome(
370 &mut self,
371 expected_key: &CorrelationKey,
372 ) -> CursorResolveResult {
373 let start = self.position;
374 let next = self.position + 1;
375 if self
376 .events
377 .get(next)
378 .is_some_and(|event| self.is_outcome_for_start_key(event, expected_key))
379 {
380 CursorResolveResult::Matched(self.take_range(start, next + 1))
381 } else {
382 self.consume_one()
383 }
384 }
385
386 fn consume_one(&mut self) -> CursorResolveResult {
387 CursorResolveResult::Matched(self.take_range(self.position, self.position + 1))
388 }
389
390 fn take_range(&mut self, start: usize, end: usize) -> Vec<Event> {
391 let consumed = self.events[start..end].to_vec();
392 for slot in &mut self.consumed[start..end] {
393 *slot = true;
394 }
395 self.advance_past_consumed();
396 consumed
397 }
398
399 fn consume_indices(&mut self, indices: Vec<usize>) -> CursorResolveResult {
404 let mut events = Vec::with_capacity(indices.len());
405 for index in indices {
406 if let (Some(event), Some(slot)) =
407 (self.events.get(index), self.consumed.get_mut(index))
408 {
409 *slot = true;
410 events.push(event.clone());
411 }
412 }
413 self.advance_past_consumed();
414 CursorResolveResult::Matched(events)
415 }
416
417 fn advance_past_consumed(&mut self) {
418 while self.consumed.get(self.position).copied().unwrap_or(false) {
419 self.position += 1;
420 }
421 }
422
423 fn next_matchable_index(&self) -> Option<usize> {
424 let events = self.events.get(self.position..)?;
425 let consumed = self.consumed.get(self.position..)?;
426 events
427 .iter()
428 .zip(consumed)
429 .position(|(event, consumed)| !consumed && family_for_event(event).is_some())
430 .map(|offset| self.position + offset)
431 }
432
433 fn mismatch_at_current(&self, expected_key: CorrelationKey) -> CursorResolveResult {
434 match self.events.get(self.position) {
435 Some(event) => CursorResolveResult::Mismatch {
436 expected_key,
437 found: self.descriptor_at(self.position, event),
438 },
439 None => CursorResolveResult::Exhausted,
440 }
441 }
442
443 fn descriptor_at(&self, index: usize, event: &Event) -> FoundEventDescriptor {
444 FoundEventDescriptor {
445 seq: event.seq(),
446 family: family_for_event(event),
447 key: key_for_event(&self.events, index),
448 kind: event_kind(event),
449 }
450 }
451
452 fn has_later_activity_attempt_or_outcome(
453 &self,
454 start: usize,
455 activity_id: &ActivityId,
456 ) -> bool {
457 self.events.iter().skip(start).any(|event| {
458 matches!(
459 event,
460 Event::ActivityStarted {
461 activity_id: event_activity_id,
462 ..
463 } | Event::ActivityFailed {
464 activity_id: event_activity_id,
465 ..
466 } | Event::ActivityCompleted {
467 activity_id: event_activity_id,
468 ..
469 } if event_activity_id == activity_id
470 )
471 })
472 }
473
474 fn activity_reopened_after(&self, start: usize, activity_id: &ActivityId) -> bool {
480 self.events.iter().skip(start).any(|event| {
481 matches!(
482 event,
483 Event::WorkflowReopened { reopened, .. } if reopened.contains(activity_id)
484 )
485 })
486 }
487
488 fn is_outcome_for_start_key(&self, event: &Event, expected_key: &CorrelationKey) -> bool {
489 match (event, expected_key) {
490 (
491 Event::TimerFired { timer_id, .. } | Event::WithTimeoutCompleted { timer_id, .. },
492 CorrelationKey::Timer(expected_timer_id),
493 ) => timer_id == expected_timer_id,
494 (
499 Event::TimerCancelled {
500 timer_id, cause, ..
501 },
502 CorrelationKey::Timer(expected_timer_id),
503 ) => {
504 *cause == aion_core::TimerCancelCause::WorkflowIntent
505 && timer_id == expected_timer_id
506 }
507 (
508 Event::ChildWorkflowCompleted {
509 child_workflow_id, ..
510 }
511 | Event::ChildWorkflowFailed {
512 child_workflow_id, ..
513 }
514 | Event::ChildWorkflowCancelled {
515 child_workflow_id, ..
516 },
517 CorrelationKey::Child(_),
518 ) => self.events.get(self.position).is_some_and(|start| {
519 matches!(
520 start,
521 Event::ChildWorkflowStarted {
522 child_workflow_id: start_child_workflow_id,
523 ..
524 } if start_child_workflow_id == child_workflow_id
525 )
526 }),
527 _ => false,
528 }
529 }
530}
531
532fn is_child_terminal_for(event: &Event, child_workflow_id: &WorkflowId) -> bool {
533 matches!(
534 event,
535 Event::ChildWorkflowCompleted {
536 child_workflow_id: terminal_child,
537 ..
538 } | Event::ChildWorkflowFailed {
539 child_workflow_id: terminal_child,
540 ..
541 } if terminal_child == child_workflow_id
542 )
543}
544
545fn family_for_event(event: &Event) -> Option<RecordedEventFamily> {
546 match event {
547 Event::ActivityScheduled { .. } => Some(RecordedEventFamily::Activity),
548 Event::TimerStarted { .. } | Event::WithTimeoutCompleted { .. } => {
549 Some(RecordedEventFamily::Timer)
550 }
551 Event::SignalReceived { .. } | Event::SignalSent { .. } => {
552 Some(RecordedEventFamily::Signal)
553 }
554 Event::ChildWorkflowStarted { .. }
555 | Event::ChildWorkflowCompleted { .. }
556 | Event::ChildWorkflowFailed { .. } => Some(RecordedEventFamily::Child),
557 _ => None,
558 }
559}
560
561fn event_kind(event: &Event) -> &'static str {
562 match event {
563 Event::WorkflowStarted { .. } => "WorkflowStarted",
564 Event::WorkflowCompleted { .. } => "WorkflowCompleted",
565 Event::WorkflowFailed { .. } => "WorkflowFailed",
566 Event::WorkflowCancelled { .. } => "WorkflowCancelled",
567 Event::WorkflowTimedOut { .. } => "WorkflowTimedOut",
568 Event::WorkflowContinuedAsNew { .. } => "WorkflowContinuedAsNew",
569 Event::WorkflowReopened { .. } => "WorkflowReopened",
570 Event::WorkflowPaused { .. } => "WorkflowPaused",
571 Event::WorkflowResumed { .. } => "WorkflowResumed",
572 Event::SearchAttributesUpdated { .. } => "SearchAttributesUpdated",
573 Event::ActivityScheduled { .. } => "ActivityScheduled",
574 Event::ActivityStarted { .. } => "ActivityStarted",
575 Event::ActivityCompleted { .. } => "ActivityCompleted",
576 Event::ActivityFailed { .. } => "ActivityFailed",
577 Event::ActivityCancelled { .. } => "ActivityCancelled",
578 Event::TimerStarted { .. } => "TimerStarted",
579 Event::TimerFired { .. } => "TimerFired",
580 Event::TimerCancelled { .. } => "TimerCancelled",
581 Event::WithTimeoutCompleted { .. } => "WithTimeoutCompleted",
582 Event::SignalReceived { .. } => "SignalReceived",
583 Event::SignalSent { .. } => "SignalSent",
584 Event::ChildWorkflowStarted { .. } => "ChildWorkflowStarted",
585 Event::ChildWorkflowCompleted { .. } => "ChildWorkflowCompleted",
586 Event::ChildWorkflowFailed { .. } => "ChildWorkflowFailed",
587 Event::ChildWorkflowCancelled { .. } => "ChildWorkflowCancelled",
588 Event::ScheduleCreated { .. } => "ScheduleCreated",
589 Event::ScheduleUpdated { .. } => "ScheduleUpdated",
590 Event::SchedulePaused { .. } => "SchedulePaused",
591 Event::ScheduleResumed { .. } => "ScheduleResumed",
592 Event::ScheduleDeleted { .. } => "ScheduleDeleted",
593 Event::ScheduleTriggered { .. } => "ScheduleTriggered",
594 }
595}
596
597#[cfg(test)]
598mod tests {
599 use aion_core::{
600 ActivityError, ActivityErrorKind, ActivityId, Event, EventEnvelope, Payload, TimerId,
601 WorkflowId,
602 };
603 use chrono::{DateTime, TimeZone, Utc};
604 use serde_json::json;
605 use uuid::Uuid;
606
607 use super::{
608 ChildTerminalResolveResult, CursorResolveResult, HistoryCursor, RecordedEventFamily,
609 };
610 use crate::durability::correlation::CorrelationKey;
611
612 fn timestamp() -> Result<DateTime<Utc>, Box<dyn std::error::Error>> {
613 Utc.timestamp_opt(0, 0)
614 .single()
615 .ok_or_else(|| "invalid timestamp".into())
616 }
617
618 fn envelope(seq: u64) -> Result<EventEnvelope, Box<dyn std::error::Error>> {
619 Ok(EventEnvelope {
620 seq,
621 recorded_at: timestamp()?,
622 workflow_id: WorkflowId::new(Uuid::nil()),
623 })
624 }
625
626 fn payload() -> Result<Payload, Box<dyn std::error::Error>> {
627 Ok(Payload::from_json(&json!(null))?)
628 }
629
630 fn workflow_started(seq: u64) -> Result<Event, Box<dyn std::error::Error>> {
631 Ok(Event::WorkflowStarted {
632 envelope: envelope(seq)?,
633 workflow_type: "workflow".to_owned(),
634 input: payload()?,
635 run_id: aion_core::RunId::new(uuid::Uuid::from_u128(1)),
636 parent_run_id: None,
637 package_version: aion_core::PackageVersion::new("a".repeat(64)),
638 })
639 }
640
641 fn scheduled(seq: u64, ordinal: u64) -> Result<Event, Box<dyn std::error::Error>> {
642 Ok(Event::ActivityScheduled {
643 envelope: envelope(seq)?,
644 activity_id: ActivityId::from_sequence_position(ordinal),
645 activity_type: "activity".to_owned(),
646 input: payload()?,
647 task_queue: String::from("default"),
648 node: None,
649 })
650 }
651
652 fn started(seq: u64, ordinal: u64) -> Result<Event, Box<dyn std::error::Error>> {
653 Ok(Event::ActivityStarted {
654 envelope: envelope(seq)?,
655 activity_id: ActivityId::from_sequence_position(ordinal),
656 attempt: 1,
657 })
658 }
659
660 fn completed(seq: u64, ordinal: u64) -> Result<Event, Box<dyn std::error::Error>> {
661 Ok(Event::ActivityCompleted {
662 envelope: envelope(seq)?,
663 activity_id: ActivityId::from_sequence_position(ordinal),
664 result: payload()?,
665 attempt: 1,
666 })
667 }
668
669 fn failed(
670 seq: u64,
671 ordinal: u64,
672 attempt: u32,
673 kind: ActivityErrorKind,
674 ) -> Result<Event, Box<dyn std::error::Error>> {
675 Ok(Event::ActivityFailed {
676 envelope: envelope(seq)?,
677 activity_id: ActivityId::from_sequence_position(ordinal),
678 error: ActivityError {
679 kind,
680 message: "activity failed".to_owned(),
681 details: None,
682 },
683 attempt,
684 })
685 }
686
687 fn reopened(seq: u64, reopened: &[u64]) -> Result<Event, Box<dyn std::error::Error>> {
688 Ok(Event::WorkflowReopened {
689 envelope: envelope(seq)?,
690 run_id: aion_core::RunId::new(uuid::Uuid::from_u128(1)),
691 reopened: reopened
692 .iter()
693 .map(|ordinal| ActivityId::from_sequence_position(*ordinal))
694 .collect(),
695 })
696 }
697
698 #[test]
699 fn reopen_supersedes_terminal_failure_and_exhausts() -> Result<(), Box<dyn std::error::Error>> {
700 let mut cursor = HistoryCursor::new(vec![
701 scheduled(1, 0)?,
702 failed(2, 0, 1, ActivityErrorKind::Terminal)?,
703 reopened(3, &[0])?,
704 ])?;
705
706 let result =
707 cursor.resolve_next(RecordedEventFamily::Activity, CorrelationKey::Activity(0));
708
709 assert!(
710 matches!(result, CursorResolveResult::Exhausted),
711 "a reopened terminal failure with no later attempt must resolve live, got {result:?}"
712 );
713 Ok(())
714 }
715
716 #[test]
717 fn reopen_then_new_attempt_resolves_to_the_new_outcome()
718 -> Result<(), Box<dyn std::error::Error>> {
719 let mut cursor = HistoryCursor::new(vec![
720 scheduled(1, 0)?,
721 failed(2, 0, 1, ActivityErrorKind::Terminal)?,
722 reopened(3, &[0])?,
723 scheduled(4, 0)?,
724 completed(5, 0)?,
725 ])?;
726
727 let result =
728 cursor.resolve_next(RecordedEventFamily::Activity, CorrelationKey::Activity(0));
729
730 match result {
731 CursorResolveResult::Matched(events) => {
732 assert!(
733 matches!(events.last(), Some(Event::ActivityCompleted { .. })),
734 "reopened activity must resolve to its post-reopen completion"
735 );
736 }
737 other => return Err(format!("expected Matched completion, got {other:?}").into()),
738 }
739 Ok(())
740 }
741
742 #[test]
743 fn terminal_failure_without_reopen_still_matches_unchanged()
744 -> Result<(), Box<dyn std::error::Error>> {
745 let mut cursor = HistoryCursor::new(vec![
746 scheduled(1, 0)?,
747 failed(2, 0, 1, ActivityErrorKind::Terminal)?,
748 ])?;
749
750 let result =
751 cursor.resolve_next(RecordedEventFamily::Activity, CorrelationKey::Activity(0));
752
753 match result {
754 CursorResolveResult::Matched(events) => {
755 assert!(matches!(events.last(), Some(Event::ActivityFailed { .. })));
756 }
757 other => return Err(format!("expected Matched terminal failure, got {other:?}").into()),
758 }
759 Ok(())
760 }
761
762 #[test]
763 fn reopen_with_new_attempt_leaves_no_straggler_for_later_commands()
764 -> Result<(), Box<dyn std::error::Error>> {
765 let mut cursor = HistoryCursor::new(vec![
769 scheduled(1, 0)?,
770 failed(2, 0, 1, ActivityErrorKind::Terminal)?,
771 reopened(3, &[0])?,
772 scheduled(4, 0)?,
773 completed(5, 0)?,
774 scheduled(6, 1)?,
775 completed(7, 1)?,
776 ])?;
777
778 let first = cursor.resolve_next(RecordedEventFamily::Activity, CorrelationKey::Activity(0));
779 assert!(
780 matches!(first, CursorResolveResult::Matched(_)),
781 "reopened activity 0 resolves, got {first:?}"
782 );
783
784 let second =
785 cursor.resolve_next(RecordedEventFamily::Activity, CorrelationKey::Activity(1));
786 match second {
787 CursorResolveResult::Matched(events) => {
788 assert!(
789 matches!(events.last(), Some(Event::ActivityCompleted { .. })),
790 "activity 1 must resolve to its completion after a reopen of activity 0"
791 );
792 }
793 other => {
794 return Err(format!(
795 "activity 1 must resolve cleanly after a reopen, got {other:?}"
796 )
797 .into());
798 }
799 }
800 Ok(())
801 }
802
803 #[test]
804 fn reopen_of_a_different_activity_does_not_supersede_this_failure()
805 -> Result<(), Box<dyn std::error::Error>> {
806 let mut cursor = HistoryCursor::new(vec![
807 scheduled(1, 0)?,
808 failed(2, 0, 1, ActivityErrorKind::Terminal)?,
809 reopened(3, &[5])?,
810 ])?;
811
812 let result =
813 cursor.resolve_next(RecordedEventFamily::Activity, CorrelationKey::Activity(0));
814
815 assert!(
816 matches!(result, CursorResolveResult::Matched(_)),
817 "a reopen naming another activity must not reopen this one, got {result:?}"
818 );
819 Ok(())
820 }
821
822 fn paused(seq: u64) -> Result<Event, Box<dyn std::error::Error>> {
823 Ok(Event::WorkflowPaused {
824 envelope: envelope(seq)?,
825 run_id: aion_core::RunId::new(uuid::Uuid::from_u128(1)),
826 reason: None,
827 operator: None,
828 })
829 }
830
831 fn resumed(seq: u64) -> Result<Event, Box<dyn std::error::Error>> {
832 Ok(Event::WorkflowResumed {
833 envelope: envelope(seq)?,
834 run_id: aion_core::RunId::new(uuid::Uuid::from_u128(1)),
835 operator: None,
836 })
837 }
838
839 #[test]
844 fn pause_resume_markers_are_invisible_to_the_cursor() -> Result<(), Box<dyn std::error::Error>>
845 {
846 let mut cursor = HistoryCursor::new(vec![
847 workflow_started(1)?,
848 paused(2)?,
849 scheduled(3, 0)?,
850 resumed(4)?,
851 completed(5, 0)?,
852 ])?;
853
854 let result =
855 cursor.resolve_next(RecordedEventFamily::Activity, CorrelationKey::Activity(0));
856 match result {
857 CursorResolveResult::Matched(events) => {
858 assert_eq!(
859 events.len(),
860 2,
861 "only the activity's own events are consumed"
862 );
863 assert!(matches!(
864 events.last(),
865 Some(Event::ActivityCompleted { .. })
866 ));
867 }
868 other => {
869 return Err(
870 format!("pause/resume markers must not disturb replay, got {other:?}").into(),
871 );
872 }
873 }
874
875 assert_eq!(
879 cursor.resolve_next(RecordedEventFamily::Activity, CorrelationKey::Activity(1)),
880 CursorResolveResult::Exhausted,
881 "the leftover pause/resume markers are invisible, never matched"
882 );
883 Ok(())
884 }
885
886 #[test]
887 fn new_accepts_in_order_history_and_exposes_starting_sequence()
888 -> Result<(), Box<dyn std::error::Error>> {
889 let cursor = HistoryCursor::new(vec![scheduled(7, 0)?, completed(8, 0)?])?;
890
891 assert_eq!(cursor.current_sequence(), Some(7));
892 assert_eq!(cursor.position_index(), 0);
893 Ok(())
894 }
895
896 #[test]
897 fn new_rejects_decreasing_sequence_order() -> Result<(), Box<dyn std::error::Error>> {
898 let error = HistoryCursor::new(vec![scheduled(9, 0)?, completed(8, 0)?])
899 .map(|_| "unexpected success")
900 .err();
901
902 assert!(error.is_some());
903 Ok(())
904 }
905
906 #[test]
907 fn resolves_activity_match_then_reports_exhaustion() -> Result<(), Box<dyn std::error::Error>> {
908 let mut cursor = HistoryCursor::new(vec![scheduled(1, 0)?, completed(2, 0)?])?;
909
910 let result =
911 cursor.resolve_next(RecordedEventFamily::Activity, CorrelationKey::Activity(0));
912
913 match result {
914 CursorResolveResult::Matched(events) => {
915 assert_eq!(events.len(), 2);
916 assert_eq!(cursor.current_sequence(), None);
917 }
918 CursorResolveResult::Exhausted | CursorResolveResult::Mismatch { .. } => {
919 return Err("activity should match recorded history".into());
920 }
921 }
922
923 assert_eq!(
924 cursor.resolve_next(RecordedEventFamily::Activity, CorrelationKey::Activity(1),),
925 CursorResolveResult::Exhausted
926 );
927 Ok(())
928 }
929
930 #[test]
931 fn skips_non_matchable_lifecycle_events_before_resolving()
932 -> Result<(), Box<dyn std::error::Error>> {
933 let mut cursor = HistoryCursor::new(vec![
934 workflow_started(1)?,
935 scheduled(2, 0)?,
936 completed(3, 0)?,
937 ])?;
938
939 let result =
940 cursor.resolve_next(RecordedEventFamily::Activity, CorrelationKey::Activity(0));
941
942 match result {
943 CursorResolveResult::Matched(events) => {
944 assert_eq!(events.len(), 2);
945 assert!(matches!(
946 events.first(),
947 Some(Event::ActivityScheduled { .. })
948 ));
949 assert_eq!(cursor.current_sequence(), None);
950 }
951 CursorResolveResult::Exhausted | CursorResolveResult::Mismatch { .. } => {
952 return Err("lifecycle events should not block command replay".into());
953 }
954 }
955 Ok(())
956 }
957
958 #[test]
959 fn reports_mismatch_for_different_next_family() -> Result<(), Box<dyn std::error::Error>> {
960 let timer_id = TimerId::anonymous(1);
961 let mut cursor = HistoryCursor::new(vec![Event::TimerStarted {
962 envelope: envelope(1)?,
963 timer_id: timer_id.clone(),
964 fire_at: timestamp()?,
965 }])?;
966
967 let result =
968 cursor.resolve_next(RecordedEventFamily::Activity, CorrelationKey::Activity(0));
969
970 match result {
971 CursorResolveResult::Mismatch {
972 expected_key,
973 found,
974 } => {
975 assert_eq!(expected_key, CorrelationKey::Activity(0));
976 assert_eq!(found.family, Some(RecordedEventFamily::Timer));
977 assert_eq!(found.key, Some(CorrelationKey::Timer(timer_id)));
978 }
979 CursorResolveResult::Matched(_) | CursorResolveResult::Exhausted => {
980 return Err("different next family should be a mismatch".into());
981 }
982 }
983 Ok(())
984 }
985
986 #[test]
987 fn walks_retry_failures_to_eventual_activity_success() -> Result<(), Box<dyn std::error::Error>>
988 {
989 let mut cursor = HistoryCursor::new(vec![
990 scheduled(1, 0)?,
991 failed(2, 0, 1, ActivityErrorKind::Retryable)?,
992 started(3, 0)?,
993 failed(4, 0, 2, ActivityErrorKind::Retryable)?,
994 completed(5, 0)?,
995 ])?;
996
997 let result =
998 cursor.resolve_next(RecordedEventFamily::Activity, CorrelationKey::Activity(0));
999
1000 match result {
1001 CursorResolveResult::Matched(events) => {
1002 assert_eq!(events.len(), 5);
1003 assert!(matches!(
1004 events.last(),
1005 Some(Event::ActivityCompleted { .. })
1006 ));
1007 assert_eq!(cursor.current_sequence(), None);
1008 }
1009 CursorResolveResult::Exhausted | CursorResolveResult::Mismatch { .. } => {
1010 return Err("retry history should resolve to eventual completion".into());
1011 }
1012 }
1013 Ok(())
1014 }
1015
1016 fn child_id(value: u128) -> WorkflowId {
1017 WorkflowId::new(Uuid::from_u128(value))
1018 }
1019
1020 fn child_started(seq: u64, child: u128) -> Result<Event, Box<dyn std::error::Error>> {
1021 Ok(Event::ChildWorkflowStarted {
1022 envelope: envelope(seq)?,
1023 child_workflow_id: child_id(child),
1024 workflow_type: "child".to_owned(),
1025 input: payload()?,
1026 package_version: aion_core::PackageVersion::new("a".repeat(64)),
1027 })
1028 }
1029
1030 fn child_completed(seq: u64, child: u128) -> Result<Event, Box<dyn std::error::Error>> {
1031 Ok(Event::ChildWorkflowCompleted {
1032 envelope: envelope(seq)?,
1033 child_workflow_id: child_id(child),
1034 result: payload()?,
1035 })
1036 }
1037
1038 fn signal_received(seq: u64, name: &str) -> Result<Event, Box<dyn std::error::Error>> {
1039 Ok(Event::SignalReceived {
1040 envelope: envelope(seq)?,
1041 name: name.to_owned(),
1042 payload: payload()?,
1043 })
1044 }
1045
1046 #[test]
1047 fn fast_forward_to_child_terminal_skips_consumed_commands()
1048 -> Result<(), Box<dyn std::error::Error>> {
1049 let mut cursor = HistoryCursor::new(vec![
1050 scheduled(1, 0)?,
1051 completed(2, 0)?,
1052 child_started(3, 1)?,
1053 signal_received(4, "mid")?,
1054 child_started(5, 2)?,
1055 child_completed(6, 1)?,
1056 ])?;
1057
1058 cursor.fast_forward_to_child_terminal(&child_id(1));
1059 let result = cursor.resolve_child_terminal(&child_id(1));
1060
1061 match result {
1062 ChildTerminalResolveResult::Matched(events) => {
1063 assert_eq!(events.len(), 1);
1064 assert!(matches!(
1065 events.first(),
1066 Some(Event::ChildWorkflowCompleted { child_workflow_id, .. })
1067 if *child_workflow_id == child_id(1)
1068 ));
1069 }
1070 ChildTerminalResolveResult::Exhausted | ChildTerminalResolveResult::Mismatch { .. } => {
1071 return Err("await must reach the awaited child's recorded terminal".into());
1072 }
1073 }
1074 Ok(())
1075 }
1076
1077 #[test]
1078 fn fast_forward_to_child_terminal_exhausts_when_no_terminal_recorded()
1079 -> Result<(), Box<dyn std::error::Error>> {
1080 let mut cursor = HistoryCursor::new(vec![
1081 scheduled(1, 0)?,
1082 completed(2, 0)?,
1083 child_started(3, 1)?,
1084 ])?;
1085
1086 cursor.fast_forward_to_child_terminal(&child_id(1));
1087
1088 assert_eq!(
1089 cursor.resolve_child_terminal(&child_id(1)),
1090 ChildTerminalResolveResult::Exhausted
1091 );
1092 Ok(())
1093 }
1094
1095 #[test]
1096 fn resolve_child_terminal_reports_mismatch_without_skipping_in_strict_replay()
1097 -> Result<(), Box<dyn std::error::Error>> {
1098 let mut cursor = HistoryCursor::new(vec![scheduled(1, 0)?, child_completed(2, 1)?])?;
1099
1100 let result = cursor.resolve_child_terminal(&child_id(1));
1101
1102 match result {
1103 ChildTerminalResolveResult::Mismatch { found } => {
1104 assert_eq!(found.seq, 1);
1105 assert_eq!(found.family, Some(RecordedEventFamily::Activity));
1106 }
1107 ChildTerminalResolveResult::Matched(_) | ChildTerminalResolveResult::Exhausted => {
1108 return Err("strict replay must not skip an unconsumed recorded command".into());
1109 }
1110 }
1111 Ok(())
1112 }
1113
1114 fn child_failed(seq: u64, child: u128) -> Result<Event, Box<dyn std::error::Error>> {
1115 Ok(Event::ChildWorkflowFailed {
1116 envelope: envelope(seq)?,
1117 child_workflow_id: child_id(child),
1118 error: aion_core::WorkflowError {
1119 message: "child failed".to_owned(),
1120 details: None,
1121 },
1122 })
1123 }
1124
1125 #[test]
1126 fn resolve_activity_skips_interleaved_signal_and_leaves_it_matchable()
1127 -> Result<(), Box<dyn std::error::Error>> {
1128 let mut cursor = HistoryCursor::new(vec![
1129 scheduled(1, 0)?,
1130 signal_received(2, "mid")?,
1131 completed(3, 0)?,
1132 ])?;
1133
1134 let result =
1135 cursor.resolve_next(RecordedEventFamily::Activity, CorrelationKey::Activity(0));
1136
1137 match result {
1138 CursorResolveResult::Matched(events) => {
1139 assert_eq!(events.len(), 2);
1140 assert!(matches!(
1141 events.first(),
1142 Some(Event::ActivityScheduled { .. })
1143 ));
1144 assert!(matches!(
1145 events.last(),
1146 Some(Event::ActivityCompleted { activity_id, .. })
1147 if activity_id.sequence_position() == 0
1148 ));
1149 }
1150 CursorResolveResult::Exhausted | CursorResolveResult::Mismatch { .. } => {
1151 return Err(
1152 "an async signal arrival inside the activity range must not fail replay".into(),
1153 );
1154 }
1155 }
1156
1157 let signal = cursor.resolve_next(
1158 RecordedEventFamily::Signal,
1159 CorrelationKey::Signal {
1160 name: "mid".to_owned(),
1161 index: 0,
1162 },
1163 );
1164 match signal {
1165 CursorResolveResult::Matched(events) => {
1166 assert_eq!(events.len(), 1);
1167 assert!(matches!(events.first(), Some(Event::SignalReceived { .. })));
1168 }
1169 CursorResolveResult::Exhausted | CursorResolveResult::Mismatch { .. } => {
1170 return Err("the skipped signal must stay matchable for its own command".into());
1171 }
1172 }
1173 Ok(())
1174 }
1175
1176 #[test]
1177 fn resolve_activity_resolves_interleaved_parallel_activity_ranges()
1178 -> Result<(), Box<dyn std::error::Error>> {
1179 let mut cursor = HistoryCursor::new(vec![
1180 scheduled(1, 0)?,
1181 scheduled(2, 1)?,
1182 completed(3, 1)?,
1183 completed(4, 0)?,
1184 ])?;
1185
1186 let first = cursor.resolve_next(RecordedEventFamily::Activity, CorrelationKey::Activity(0));
1187 match first {
1188 CursorResolveResult::Matched(events) => {
1189 assert_eq!(events.len(), 2);
1190 assert!(matches!(
1191 events.last(),
1192 Some(Event::ActivityCompleted { activity_id, .. })
1193 if activity_id.sequence_position() == 0
1194 ));
1195 }
1196 CursorResolveResult::Exhausted | CursorResolveResult::Mismatch { .. } => {
1197 return Err("a parallel activity's events inside the range must be skipped".into());
1198 }
1199 }
1200
1201 let second =
1202 cursor.resolve_next(RecordedEventFamily::Activity, CorrelationKey::Activity(1));
1203 match second {
1204 CursorResolveResult::Matched(events) => {
1205 assert_eq!(events.len(), 2);
1206 assert!(matches!(
1207 events.last(),
1208 Some(Event::ActivityCompleted { activity_id, .. })
1209 if activity_id.sequence_position() == 1
1210 ));
1211 assert_eq!(cursor.current_sequence(), None);
1212 }
1213 CursorResolveResult::Exhausted | CursorResolveResult::Mismatch { .. } => {
1214 return Err("the interleaved activity must remain resolvable afterwards".into());
1215 }
1216 }
1217 Ok(())
1218 }
1219
1220 #[test]
1221 fn resolve_activity_skips_interleaved_child_terminal() -> Result<(), Box<dyn std::error::Error>>
1222 {
1223 let mut cursor = HistoryCursor::new(vec![
1224 child_started(1, 7)?,
1225 scheduled(2, 0)?,
1226 child_completed(3, 7)?,
1227 child_failed(4, 9)?,
1228 completed(5, 0)?,
1229 ])?;
1230
1231 cursor.fast_forward_to_key(&CorrelationKey::Activity(0));
1232 let result =
1233 cursor.resolve_next(RecordedEventFamily::Activity, CorrelationKey::Activity(0));
1234
1235 match result {
1236 CursorResolveResult::Matched(events) => {
1237 assert_eq!(events.len(), 2);
1238 assert!(matches!(
1239 events.last(),
1240 Some(Event::ActivityCompleted { activity_id, .. })
1241 if activity_id.sequence_position() == 0
1242 ));
1243 }
1244 CursorResolveResult::Exhausted | CursorResolveResult::Mismatch { .. } => {
1245 return Err("child terminals inside the activity range must be skipped".into());
1246 }
1247 }
1248 Ok(())
1249 }
1250
1251 #[test]
1252 fn resolve_activity_still_mismatches_on_wrong_anchor_key()
1253 -> Result<(), Box<dyn std::error::Error>> {
1254 let mut cursor = HistoryCursor::new(vec![scheduled(1, 1)?, completed(2, 1)?])?;
1255
1256 let result =
1257 cursor.resolve_next(RecordedEventFamily::Activity, CorrelationKey::Activity(0));
1258
1259 match result {
1260 CursorResolveResult::Mismatch {
1261 expected_key,
1262 found,
1263 } => {
1264 assert_eq!(expected_key, CorrelationKey::Activity(0));
1265 assert_eq!(found.key, Some(CorrelationKey::Activity(1)));
1266 }
1267 CursorResolveResult::Matched(_) | CursorResolveResult::Exhausted => {
1268 return Err("a wrong key at the Scheduled anchor must stay a mismatch".into());
1269 }
1270 }
1271 Ok(())
1272 }
1273
1274 #[test]
1275 fn fast_forward_and_resolution_smoke_over_large_history()
1276 -> Result<(), Box<dyn std::error::Error>> {
1277 let count: u64 = 5_000;
1278 let mut events = Vec::with_capacity(usize::try_from(count * 2)?);
1279 for ordinal in 0..count {
1280 events.push(scheduled(ordinal * 2 + 1, ordinal)?);
1281 events.push(completed(ordinal * 2 + 2, ordinal)?);
1282 }
1283 let mut cursor = HistoryCursor::new(events)?;
1284
1285 for ordinal in 0..count {
1286 let key = CorrelationKey::Activity(ordinal);
1287 cursor.fast_forward_to_key(&key);
1288 let result = cursor.resolve_next(RecordedEventFamily::Activity, key);
1289 assert!(
1290 matches!(result, CursorResolveResult::Matched(ref events) if events.len() == 2),
1291 "ordinal {ordinal} failed to resolve in the large-history smoke"
1292 );
1293 }
1294 assert_eq!(cursor.current_sequence(), None);
1295 Ok(())
1296 }
1297
1298 #[test]
1299 fn returns_terminal_activity_failure_as_recorded_outcome()
1300 -> Result<(), Box<dyn std::error::Error>> {
1301 let mut cursor = HistoryCursor::new(vec![
1302 scheduled(1, 0)?,
1303 failed(2, 0, 1, ActivityErrorKind::Retryable)?,
1304 failed(3, 0, 2, ActivityErrorKind::Terminal)?,
1305 ])?;
1306
1307 let result =
1308 cursor.resolve_next(RecordedEventFamily::Activity, CorrelationKey::Activity(0));
1309
1310 match result {
1311 CursorResolveResult::Matched(events) => {
1312 assert_eq!(events.len(), 3);
1313 assert!(matches!(
1314 events.last(),
1315 Some(Event::ActivityFailed { error, .. }) if error.kind == ActivityErrorKind::Terminal
1316 ));
1317 assert_eq!(cursor.current_sequence(), None);
1318 }
1319 CursorResolveResult::Exhausted | CursorResolveResult::Mismatch { .. } => {
1320 return Err("terminal failure should be the recorded outcome".into());
1321 }
1322 }
1323 Ok(())
1324 }
1325}