Skip to main content

aion/durability/
cursor.rs

1//! `HistoryCursor` over recorded events.
2
3use aion_core::{ActivityId, Event, RunId, WorkflowId};
4
5use crate::durability::{
6    correlation::{CorrelationKey, key_for_event},
7    error::DurabilityError,
8};
9
10/// Slice an ordered multi-run history down to the segment for `run_id`.
11///
12/// continue-as-new appends each replacement run's events to the same
13/// workflow history, but correlation identities (activity and timer
14/// ordinals, per-name signal occurrence indices) are run-scoped: every
15/// run's deterministic counters restart from zero. Replay and live command
16/// resolution must therefore only see events recorded at or after the run's
17/// own `WorkflowStarted`, or a replacement run would match the prior run's
18/// recorded commands.
19///
20/// # Errors
21///
22/// Returns [`DurabilityError::HistoryShape`] when the history holds no
23/// `WorkflowStarted` for `run_id`.
24pub fn current_run_segment(
25    history: Vec<Event>,
26    run_id: &RunId,
27) -> Result<Vec<Event>, DurabilityError> {
28    let start = history
29        .iter()
30        .position(|event| {
31            matches!(
32                event,
33                Event::WorkflowStarted {
34                    run_id: event_run_id,
35                    ..
36                } if event_run_id == run_id
37            )
38        })
39        .ok_or_else(|| DurabilityError::HistoryShape {
40            reason: format!("history has no WorkflowStarted for run {run_id}"),
41        })?;
42    let mut segment = history;
43    segment.drain(..start);
44    Ok(segment)
45}
46
47/// Event families that can satisfy world-touching workflow commands during replay.
48#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
49pub enum RecordedEventFamily {
50    /// Activity scheduling and its recorded outcome.
51    Activity,
52    /// Timer scheduling and its recorded outcome.
53    Timer,
54    /// Signal delivery.
55    Signal,
56    /// Child workflow scheduling and its recorded outcome.
57    Child,
58}
59
60/// Data describing the recorded event found at the cursor's current position.
61#[derive(Clone, Debug, PartialEq, Eq)]
62pub struct FoundEventDescriptor {
63    /// Sequence number of the recorded event.
64    pub seq: u64,
65    /// Replay family for the recorded event, if it is matchable by the cursor.
66    pub family: Option<RecordedEventFamily>,
67    /// Correlation key derived for the recorded event, if it starts a matchable command.
68    pub key: Option<CorrelationKey>,
69    /// Stable event-variant name for diagnostics.
70    pub kind: &'static str,
71}
72
73/// Outcome of asking the cursor to resolve the next recorded command outcome.
74#[derive(Clone, Debug, PartialEq)]
75pub enum CursorResolveResult {
76    /// The recorded command stream matched; contained events were consumed in order.
77    Matched(Vec<Event>),
78    /// The cursor has no remaining recorded event to consider.
79    Exhausted,
80    /// The next recorded event exists, but its family or key differs from the command.
81    Mismatch {
82        /// Correlation key the workflow command expected to replay.
83        expected_key: CorrelationKey,
84        /// Descriptor for the recorded event actually at the cursor position.
85        found: FoundEventDescriptor,
86    },
87}
88
89/// Outcome of asking the cursor for the recorded terminal outcome of one awaited child workflow.
90///
91/// `AwaitChild` is keyed by the child workflow id rather than a positional correlation key, so its
92/// mismatch variant carries only the found-event descriptor; the resolver supplies the awaited
93/// child identity in its diagnostics.
94#[derive(Clone, Debug, PartialEq)]
95pub enum ChildTerminalResolveResult {
96    /// The awaited child's recorded terminal event was consumed.
97    Matched(Vec<Event>),
98    /// The cursor has no remaining recorded event to consider.
99    Exhausted,
100    /// The next matchable recorded event is not the awaited child's terminal outcome.
101    Mismatch {
102        /// Descriptor for the recorded event actually at the cursor position.
103        found: FoundEventDescriptor,
104    },
105}
106
107/// Ordered cursor over a workflow's recorded history.
108///
109/// Commands consume only their own events. Asynchronous arrivals (signals,
110/// child terminals, a parallel activity's events) can be recorded anywhere
111/// inside another command's event range, so consumption is tracked per
112/// event: a resolved command marks exactly its own events consumed and
113/// skipped interior events stay matchable for their own commands.
114/// `position` is the low-water mark — the first index that is neither
115/// consumed nor already skipped past — and scans resume from it.
116#[derive(Clone, Debug)]
117pub struct HistoryCursor {
118    events: Vec<Event>,
119    consumed: Vec<bool>,
120    position: usize,
121}
122
123impl HistoryCursor {
124    /// Builds a cursor from ordered read-history output.
125    ///
126    /// # Errors
127    ///
128    /// Returns [`DurabilityError::HistoryShape`] when event sequence numbers decrease.
129    pub fn new(events: Vec<Event>) -> Result<Self, DurabilityError> {
130        for pair in events.windows(2) {
131            let prior = pair[0].seq();
132            let next = pair[1].seq();
133            if next < prior {
134                return Err(DurabilityError::HistoryShape {
135                    reason: format!("history sequence order decreased from {prior} to {next}"),
136                });
137            }
138        }
139
140        let consumed = vec![false; events.len()];
141        Ok(Self {
142            events,
143            consumed,
144            position: 0,
145        })
146    }
147
148    /// Returns the sequence number at the current cursor position, or `None` when exhausted.
149    #[must_use]
150    pub fn current_sequence(&self) -> Option<u64> {
151        self.events.get(self.position).map(Event::seq)
152    }
153
154    /// Returns the ordered history backing this cursor.
155    #[must_use]
156    pub fn events(&self) -> &[Event] {
157        &self.events
158    }
159
160    /// Returns the current zero-based index into the owned history.
161    #[must_use]
162    pub const fn position_index(&self) -> usize {
163        self.position
164    }
165
166    /// Returns the next matchable correlation key for `family` without consuming history.
167    #[must_use]
168    pub fn next_key(&self, family: RecordedEventFamily) -> Option<CorrelationKey> {
169        let index = self.next_matchable_index()?;
170        let descriptor = self.descriptor_at(index, &self.events[index]);
171        if descriptor.family == Some(family) {
172            descriptor.key
173        } else {
174            None
175        }
176    }
177
178    /// Advance past recorded commands that earlier resolver instances of the
179    /// same live execution already consumed.
180    ///
181    /// NIF calls each build a fresh resolver whose cursor starts at the top
182    /// of history, so commands resolved by earlier calls sit before the one
183    /// being resolved now. Skipping stops at the first matchable entry whose
184    /// correlation key equals `key` — or at history end, leaving live
185    /// resolution to proceed — so a genuinely out-of-order command for an
186    /// already-positioned key still surfaces as a mismatch in
187    /// [`HistoryCursor::resolve_next`]. Strict full-history replay
188    /// (recovery, the replay driver) never calls this.
189    pub fn fast_forward_to_key(&mut self, key: &CorrelationKey) {
190        while let Some(index) = self.next_matchable_index() {
191            let descriptor = self.descriptor_at(index, &self.events[index]);
192            if descriptor.key.as_ref() == Some(key) {
193                return;
194            }
195            self.position = index + 1;
196        }
197    }
198
199    /// Advance to the recorded terminal outcome for `child_workflow_id`.
200    ///
201    /// `AwaitChild` has no positional correlation key — its replay identity
202    /// is the child workflow id returned by the matching spawn — so it gets
203    /// the same skip treatment keyed commands receive from
204    /// [`HistoryCursor::fast_forward_to_key`]: recorded commands consumed by
205    /// earlier resolver instances of the same live execution are skipped
206    /// until the awaited child's `ChildWorkflowCompleted`/`ChildWorkflowFailed`
207    /// is reached. With no recorded terminal for that child the cursor
208    /// exhausts, leaving resolution to hand off live. Strict full-history
209    /// replay (recovery, the replay driver) never calls this.
210    pub fn fast_forward_to_child_terminal(&mut self, child_workflow_id: &WorkflowId) {
211        while let Some(index) = self.next_matchable_index() {
212            if is_child_terminal_for(&self.events[index], child_workflow_id) {
213                return;
214            }
215            self.position = index + 1;
216        }
217    }
218
219    /// Resolves the next recorded child terminal outcome for `child_workflow_id`.
220    #[must_use]
221    pub fn resolve_child_terminal(
222        &mut self,
223        child_workflow_id: &WorkflowId,
224    ) -> ChildTerminalResolveResult {
225        let Some(found_index) = self.next_matchable_index() else {
226            return ChildTerminalResolveResult::Exhausted;
227        };
228        self.position = found_index;
229        match self.events.get(found_index) {
230            Some(event) if is_child_terminal_for(event, child_workflow_id) => {
231                ChildTerminalResolveResult::Matched(self.take_range(found_index, found_index + 1))
232            }
233            Some(event) => ChildTerminalResolveResult::Mismatch {
234                found: self.descriptor_at(found_index, event),
235            },
236            None => ChildTerminalResolveResult::Exhausted,
237        }
238    }
239
240    /// Resolves the next recorded outcome for the expected family and correlation key.
241    #[must_use]
242    pub fn resolve_next(
243        &mut self,
244        family: RecordedEventFamily,
245        expected_key: CorrelationKey,
246    ) -> CursorResolveResult {
247        let Some(found_index) = self.next_matchable_index() else {
248            return CursorResolveResult::Exhausted;
249        };
250        self.position = found_index;
251
252        let found = self.descriptor_at(found_index, &self.events[found_index]);
253        if found.family != Some(family) || found.key.as_ref() != Some(&expected_key) {
254            return CursorResolveResult::Mismatch {
255                expected_key,
256                found,
257            };
258        }
259
260        match family {
261            RecordedEventFamily::Activity => self.resolve_activity(expected_key),
262            RecordedEventFamily::Timer => {
263                self.resolve_started_with_immediate_outcome(&expected_key)
264            }
265            RecordedEventFamily::Child => self.resolve_child(expected_key),
266            RecordedEventFamily::Signal => match expected_key {
267                CorrelationKey::Signal { .. } => self.consume_one(),
268                _ => self.mismatch_at_current(expected_key),
269            },
270        }
271    }
272
273    /// `resolve_next` only dispatches here once the found event's family is
274    /// `Child` and its derived correlation key equals the expected key.
275    /// Child terminal events carry no correlation key (`key_for_event` keys
276    /// only `ChildWorkflowStarted`), so the position is provably at a
277    /// `ChildWorkflowStarted`; the fallback mismatch arm guards the shape
278    /// rather than encoding unreachable terminal handling.
279    fn resolve_child(&mut self, expected_key: CorrelationKey) -> CursorResolveResult {
280        match self.events.get(self.position) {
281            Some(Event::ChildWorkflowStarted { .. }) => self.consume_one(),
282            _ => self.mismatch_at_current(expected_key),
283        }
284    }
285
286    /// Consumes the matched activity's `Scheduled -> terminal` events, keyed
287    /// by activity id.
288    ///
289    /// Asynchronous arrivals (signals, child terminals, a parallel
290    /// activity's events) can be recorded between this activity's
291    /// `Scheduled` anchor and its terminal. They belong to other commands:
292    /// the walk skips them in place — neither consuming them nor failing
293    /// replay — leaving them matchable for their own commands. Determinism
294    /// is enforced at the `Scheduled` anchor by `resolve_next`'s family/key
295    /// equality check; a foreign interior event is an interleaving artifact,
296    /// not a command-stream divergence.
297    fn resolve_activity(&mut self, expected_key: CorrelationKey) -> CursorResolveResult {
298        let Some(Event::ActivityScheduled { activity_id, .. }) = self.events.get(self.position)
299        else {
300            return self.mismatch_at_current(expected_key);
301        };
302        let activity_id = activity_id.clone();
303        let mut matched = vec![self.position];
304        let mut index = self.position + 1;
305
306        while let Some(event) = self.events.get(index) {
307            match event {
308                Event::ActivityStarted {
309                    activity_id: event_activity_id,
310                    ..
311                } if event_activity_id == &activity_id => {
312                    matched.push(index);
313                }
314                Event::ActivityScheduled {
315                    activity_id: event_activity_id,
316                    ..
317                } if event_activity_id == &activity_id => {
318                    // A reopen re-dispatches the activity, recording a fresh
319                    // ActivityScheduled for the same ordinal. Consume it as part
320                    // of this activity's span so no straggler Scheduled is left
321                    // matchable to mis-resolve a later command.
322                    matched.push(index);
323                }
324                Event::ActivityFailed {
325                    activity_id: event_activity_id,
326                    error,
327                    ..
328                } if event_activity_id == &activity_id => {
329                    matched.push(index);
330                    // A RETRYABLE failure is never a terminal (#197): it is one
331                    // attempt in the engine-driven retry trail. Keep walking to
332                    // the later recorded attempt or terminal — or, when the
333                    // trail dangles (a crash mid-retry), exhaust so the
334                    // activity re-dispatches live at the next attempt.
335                    //
336                    // A reopen likewise supersedes a recorded terminal failure:
337                    // treat it as a non-terminal attempt and keep walking, so a
338                    // later recorded attempt resolves the activity, or — with
339                    // none yet — the walk exhausts and the activity
340                    // re-dispatches live.
341                    let superseded_by_reopen =
342                        self.activity_reopened_after(index + 1, &activity_id);
343                    if !error.is_retryable()
344                        && !superseded_by_reopen
345                        && !self.has_later_activity_attempt_or_outcome(index + 1, &activity_id)
346                    {
347                        return self.consume_indices(matched);
348                    }
349                }
350                Event::ActivityCompleted {
351                    activity_id: event_activity_id,
352                    ..
353                }
354                | Event::ActivityCancelled {
355                    activity_id: event_activity_id,
356                    ..
357                } if event_activity_id == &activity_id => {
358                    matched.push(index);
359                    return self.consume_indices(matched);
360                }
361                _ => {}
362            }
363            index += 1;
364        }
365
366        CursorResolveResult::Exhausted
367    }
368
369    fn resolve_started_with_immediate_outcome(
370        &mut self,
371        expected_key: &CorrelationKey,
372    ) -> CursorResolveResult {
373        let start = self.position;
374        let next = self.position + 1;
375        if self
376            .events
377            .get(next)
378            .is_some_and(|event| self.is_outcome_for_start_key(event, expected_key))
379        {
380            CursorResolveResult::Matched(self.take_range(start, next + 1))
381        } else {
382            self.consume_one()
383        }
384    }
385
386    fn consume_one(&mut self) -> CursorResolveResult {
387        CursorResolveResult::Matched(self.take_range(self.position, self.position + 1))
388    }
389
390    fn take_range(&mut self, start: usize, end: usize) -> Vec<Event> {
391        let consumed = self.events[start..end].to_vec();
392        for slot in &mut self.consumed[start..end] {
393            *slot = true;
394        }
395        self.advance_past_consumed();
396        consumed
397    }
398
399    /// Marks exactly `indices` consumed and returns their events in order.
400    ///
401    /// Interior indices left unmarked stay matchable for their own commands;
402    /// the position low-water mark advances only past the consumed prefix.
403    fn consume_indices(&mut self, indices: Vec<usize>) -> CursorResolveResult {
404        let mut events = Vec::with_capacity(indices.len());
405        for index in indices {
406            if let (Some(event), Some(slot)) =
407                (self.events.get(index), self.consumed.get_mut(index))
408            {
409                *slot = true;
410                events.push(event.clone());
411            }
412        }
413        self.advance_past_consumed();
414        CursorResolveResult::Matched(events)
415    }
416
417    fn advance_past_consumed(&mut self) {
418        while self.consumed.get(self.position).copied().unwrap_or(false) {
419            self.position += 1;
420        }
421    }
422
423    fn next_matchable_index(&self) -> Option<usize> {
424        let events = self.events.get(self.position..)?;
425        let consumed = self.consumed.get(self.position..)?;
426        events
427            .iter()
428            .zip(consumed)
429            .position(|(event, consumed)| !consumed && family_for_event(event).is_some())
430            .map(|offset| self.position + offset)
431    }
432
433    fn mismatch_at_current(&self, expected_key: CorrelationKey) -> CursorResolveResult {
434        match self.events.get(self.position) {
435            Some(event) => CursorResolveResult::Mismatch {
436                expected_key,
437                found: self.descriptor_at(self.position, event),
438            },
439            None => CursorResolveResult::Exhausted,
440        }
441    }
442
443    fn descriptor_at(&self, index: usize, event: &Event) -> FoundEventDescriptor {
444        FoundEventDescriptor {
445            seq: event.seq(),
446            family: family_for_event(event),
447            key: key_for_event(&self.events, index),
448            kind: event_kind(event),
449        }
450    }
451
452    fn has_later_activity_attempt_or_outcome(
453        &self,
454        start: usize,
455        activity_id: &ActivityId,
456    ) -> bool {
457        self.events.iter().skip(start).any(|event| {
458            matches!(
459                event,
460                Event::ActivityStarted {
461                    activity_id: event_activity_id,
462                    ..
463                } | Event::ActivityFailed {
464                    activity_id: event_activity_id,
465                    ..
466                } | Event::ActivityCompleted {
467                    activity_id: event_activity_id,
468                    ..
469                } if event_activity_id == activity_id
470            )
471        })
472    }
473
474    /// Whether a later [`Event::WorkflowReopened`] names `activity_id` among its
475    /// reopened activities. A reopen supersedes the activity's recorded terminal
476    /// failure, so the walk treats that failure as a non-terminal attempt and
477    /// continues — to a later recorded attempt, or to exhaustion so the activity
478    /// re-dispatches live.
479    fn activity_reopened_after(&self, start: usize, activity_id: &ActivityId) -> bool {
480        self.events.iter().skip(start).any(|event| {
481            matches!(
482                event,
483                Event::WorkflowReopened { reopened, .. } if reopened.contains(activity_id)
484            )
485        })
486    }
487
488    fn is_outcome_for_start_key(&self, event: &Event, expected_key: &CorrelationKey) -> bool {
489        match (event, expected_key) {
490            (
491                Event::TimerFired { timer_id, .. } | Event::WithTimeoutCompleted { timer_id, .. },
492                CorrelationKey::Timer(expected_timer_id),
493            ) => timer_id == expected_timer_id,
494            // A cancel-teardown TimerCancelled is engine bookkeeping (the run
495            // was cancelled and its timers retired); it is NEVER an outcome a
496            // workflow await observes — reopen re-arms the timer instead
497            // (#222). Only a workflow-intent cancellation resolves the await.
498            (
499                Event::TimerCancelled {
500                    timer_id, cause, ..
501                },
502                CorrelationKey::Timer(expected_timer_id),
503            ) => {
504                *cause == aion_core::TimerCancelCause::WorkflowIntent
505                    && timer_id == expected_timer_id
506            }
507            (
508                Event::ChildWorkflowCompleted {
509                    child_workflow_id, ..
510                }
511                | Event::ChildWorkflowFailed {
512                    child_workflow_id, ..
513                }
514                | Event::ChildWorkflowCancelled {
515                    child_workflow_id, ..
516                },
517                CorrelationKey::Child(_),
518            ) => self.events.get(self.position).is_some_and(|start| {
519                matches!(
520                    start,
521                    Event::ChildWorkflowStarted {
522                        child_workflow_id: start_child_workflow_id,
523                        ..
524                    } if start_child_workflow_id == child_workflow_id
525                )
526            }),
527            _ => false,
528        }
529    }
530}
531
532fn is_child_terminal_for(event: &Event, child_workflow_id: &WorkflowId) -> bool {
533    matches!(
534        event,
535        Event::ChildWorkflowCompleted {
536            child_workflow_id: terminal_child,
537            ..
538        } | Event::ChildWorkflowFailed {
539            child_workflow_id: terminal_child,
540            ..
541        } if terminal_child == child_workflow_id
542    )
543}
544
545fn family_for_event(event: &Event) -> Option<RecordedEventFamily> {
546    match event {
547        Event::ActivityScheduled { .. } => Some(RecordedEventFamily::Activity),
548        Event::TimerStarted { .. } | Event::WithTimeoutCompleted { .. } => {
549            Some(RecordedEventFamily::Timer)
550        }
551        Event::SignalReceived { .. } | Event::SignalSent { .. } => {
552            Some(RecordedEventFamily::Signal)
553        }
554        Event::ChildWorkflowStarted { .. }
555        | Event::ChildWorkflowCompleted { .. }
556        | Event::ChildWorkflowFailed { .. } => Some(RecordedEventFamily::Child),
557        _ => None,
558    }
559}
560
561fn event_kind(event: &Event) -> &'static str {
562    match event {
563        Event::WorkflowStarted { .. } => "WorkflowStarted",
564        Event::WorkflowCompleted { .. } => "WorkflowCompleted",
565        Event::WorkflowFailed { .. } => "WorkflowFailed",
566        Event::WorkflowCancelled { .. } => "WorkflowCancelled",
567        Event::WorkflowTimedOut { .. } => "WorkflowTimedOut",
568        Event::WorkflowContinuedAsNew { .. } => "WorkflowContinuedAsNew",
569        Event::WorkflowReopened { .. } => "WorkflowReopened",
570        Event::WorkflowPaused { .. } => "WorkflowPaused",
571        Event::WorkflowResumed { .. } => "WorkflowResumed",
572        Event::SearchAttributesUpdated { .. } => "SearchAttributesUpdated",
573        Event::ActivityScheduled { .. } => "ActivityScheduled",
574        Event::ActivityStarted { .. } => "ActivityStarted",
575        Event::ActivityCompleted { .. } => "ActivityCompleted",
576        Event::ActivityFailed { .. } => "ActivityFailed",
577        Event::ActivityCancelled { .. } => "ActivityCancelled",
578        Event::TimerStarted { .. } => "TimerStarted",
579        Event::TimerFired { .. } => "TimerFired",
580        Event::TimerCancelled { .. } => "TimerCancelled",
581        Event::WithTimeoutCompleted { .. } => "WithTimeoutCompleted",
582        Event::SignalReceived { .. } => "SignalReceived",
583        Event::SignalSent { .. } => "SignalSent",
584        Event::ChildWorkflowStarted { .. } => "ChildWorkflowStarted",
585        Event::ChildWorkflowCompleted { .. } => "ChildWorkflowCompleted",
586        Event::ChildWorkflowFailed { .. } => "ChildWorkflowFailed",
587        Event::ChildWorkflowCancelled { .. } => "ChildWorkflowCancelled",
588        Event::ScheduleCreated { .. } => "ScheduleCreated",
589        Event::ScheduleUpdated { .. } => "ScheduleUpdated",
590        Event::SchedulePaused { .. } => "SchedulePaused",
591        Event::ScheduleResumed { .. } => "ScheduleResumed",
592        Event::ScheduleDeleted { .. } => "ScheduleDeleted",
593        Event::ScheduleTriggered { .. } => "ScheduleTriggered",
594    }
595}
596
597#[cfg(test)]
598mod tests {
599    use aion_core::{
600        ActivityError, ActivityErrorKind, ActivityId, Event, EventEnvelope, Payload, TimerId,
601        WorkflowId,
602    };
603    use chrono::{DateTime, TimeZone, Utc};
604    use serde_json::json;
605    use uuid::Uuid;
606
607    use super::{
608        ChildTerminalResolveResult, CursorResolveResult, HistoryCursor, RecordedEventFamily,
609    };
610    use crate::durability::correlation::CorrelationKey;
611
612    fn timestamp() -> Result<DateTime<Utc>, Box<dyn std::error::Error>> {
613        Utc.timestamp_opt(0, 0)
614            .single()
615            .ok_or_else(|| "invalid timestamp".into())
616    }
617
618    fn envelope(seq: u64) -> Result<EventEnvelope, Box<dyn std::error::Error>> {
619        Ok(EventEnvelope {
620            seq,
621            recorded_at: timestamp()?,
622            workflow_id: WorkflowId::new(Uuid::nil()),
623        })
624    }
625
626    fn payload() -> Result<Payload, Box<dyn std::error::Error>> {
627        Ok(Payload::from_json(&json!(null))?)
628    }
629
630    fn workflow_started(seq: u64) -> Result<Event, Box<dyn std::error::Error>> {
631        Ok(Event::WorkflowStarted {
632            envelope: envelope(seq)?,
633            workflow_type: "workflow".to_owned(),
634            input: payload()?,
635            run_id: aion_core::RunId::new(uuid::Uuid::from_u128(1)),
636            parent_run_id: None,
637            package_version: aion_core::PackageVersion::new("a".repeat(64)),
638        })
639    }
640
641    fn scheduled(seq: u64, ordinal: u64) -> Result<Event, Box<dyn std::error::Error>> {
642        Ok(Event::ActivityScheduled {
643            envelope: envelope(seq)?,
644            activity_id: ActivityId::from_sequence_position(ordinal),
645            activity_type: "activity".to_owned(),
646            input: payload()?,
647            task_queue: String::from("default"),
648            node: None,
649        })
650    }
651
652    fn started(seq: u64, ordinal: u64) -> Result<Event, Box<dyn std::error::Error>> {
653        Ok(Event::ActivityStarted {
654            envelope: envelope(seq)?,
655            activity_id: ActivityId::from_sequence_position(ordinal),
656            attempt: 1,
657        })
658    }
659
660    fn completed(seq: u64, ordinal: u64) -> Result<Event, Box<dyn std::error::Error>> {
661        Ok(Event::ActivityCompleted {
662            envelope: envelope(seq)?,
663            activity_id: ActivityId::from_sequence_position(ordinal),
664            result: payload()?,
665            attempt: 1,
666        })
667    }
668
669    fn failed(
670        seq: u64,
671        ordinal: u64,
672        attempt: u32,
673        kind: ActivityErrorKind,
674    ) -> Result<Event, Box<dyn std::error::Error>> {
675        Ok(Event::ActivityFailed {
676            envelope: envelope(seq)?,
677            activity_id: ActivityId::from_sequence_position(ordinal),
678            error: ActivityError {
679                kind,
680                message: "activity failed".to_owned(),
681                details: None,
682            },
683            attempt,
684        })
685    }
686
687    fn reopened(seq: u64, reopened: &[u64]) -> Result<Event, Box<dyn std::error::Error>> {
688        Ok(Event::WorkflowReopened {
689            envelope: envelope(seq)?,
690            run_id: aion_core::RunId::new(uuid::Uuid::from_u128(1)),
691            reopened: reopened
692                .iter()
693                .map(|ordinal| ActivityId::from_sequence_position(*ordinal))
694                .collect(),
695        })
696    }
697
698    #[test]
699    fn reopen_supersedes_terminal_failure_and_exhausts() -> Result<(), Box<dyn std::error::Error>> {
700        let mut cursor = HistoryCursor::new(vec![
701            scheduled(1, 0)?,
702            failed(2, 0, 1, ActivityErrorKind::Terminal)?,
703            reopened(3, &[0])?,
704        ])?;
705
706        let result =
707            cursor.resolve_next(RecordedEventFamily::Activity, CorrelationKey::Activity(0));
708
709        assert!(
710            matches!(result, CursorResolveResult::Exhausted),
711            "a reopened terminal failure with no later attempt must resolve live, got {result:?}"
712        );
713        Ok(())
714    }
715
716    #[test]
717    fn reopen_then_new_attempt_resolves_to_the_new_outcome()
718    -> Result<(), Box<dyn std::error::Error>> {
719        let mut cursor = HistoryCursor::new(vec![
720            scheduled(1, 0)?,
721            failed(2, 0, 1, ActivityErrorKind::Terminal)?,
722            reopened(3, &[0])?,
723            scheduled(4, 0)?,
724            completed(5, 0)?,
725        ])?;
726
727        let result =
728            cursor.resolve_next(RecordedEventFamily::Activity, CorrelationKey::Activity(0));
729
730        match result {
731            CursorResolveResult::Matched(events) => {
732                assert!(
733                    matches!(events.last(), Some(Event::ActivityCompleted { .. })),
734                    "reopened activity must resolve to its post-reopen completion"
735                );
736            }
737            other => return Err(format!("expected Matched completion, got {other:?}").into()),
738        }
739        Ok(())
740    }
741
742    #[test]
743    fn terminal_failure_without_reopen_still_matches_unchanged()
744    -> Result<(), Box<dyn std::error::Error>> {
745        let mut cursor = HistoryCursor::new(vec![
746            scheduled(1, 0)?,
747            failed(2, 0, 1, ActivityErrorKind::Terminal)?,
748        ])?;
749
750        let result =
751            cursor.resolve_next(RecordedEventFamily::Activity, CorrelationKey::Activity(0));
752
753        match result {
754            CursorResolveResult::Matched(events) => {
755                assert!(matches!(events.last(), Some(Event::ActivityFailed { .. })));
756            }
757            other => return Err(format!("expected Matched terminal failure, got {other:?}").into()),
758        }
759        Ok(())
760    }
761
762    #[test]
763    fn reopen_with_new_attempt_leaves_no_straggler_for_later_commands()
764    -> Result<(), Box<dyn std::error::Error>> {
765        // After a reopen re-dispatches activity 0 (a second Scheduled is
766        // recorded), the next command (activity 1) must still resolve cleanly:
767        // the post-reopen Scheduled(0) must not be left matchable.
768        let mut cursor = HistoryCursor::new(vec![
769            scheduled(1, 0)?,
770            failed(2, 0, 1, ActivityErrorKind::Terminal)?,
771            reopened(3, &[0])?,
772            scheduled(4, 0)?,
773            completed(5, 0)?,
774            scheduled(6, 1)?,
775            completed(7, 1)?,
776        ])?;
777
778        let first = cursor.resolve_next(RecordedEventFamily::Activity, CorrelationKey::Activity(0));
779        assert!(
780            matches!(first, CursorResolveResult::Matched(_)),
781            "reopened activity 0 resolves, got {first:?}"
782        );
783
784        let second =
785            cursor.resolve_next(RecordedEventFamily::Activity, CorrelationKey::Activity(1));
786        match second {
787            CursorResolveResult::Matched(events) => {
788                assert!(
789                    matches!(events.last(), Some(Event::ActivityCompleted { .. })),
790                    "activity 1 must resolve to its completion after a reopen of activity 0"
791                );
792            }
793            other => {
794                return Err(format!(
795                    "activity 1 must resolve cleanly after a reopen, got {other:?}"
796                )
797                .into());
798            }
799        }
800        Ok(())
801    }
802
803    #[test]
804    fn reopen_of_a_different_activity_does_not_supersede_this_failure()
805    -> Result<(), Box<dyn std::error::Error>> {
806        let mut cursor = HistoryCursor::new(vec![
807            scheduled(1, 0)?,
808            failed(2, 0, 1, ActivityErrorKind::Terminal)?,
809            reopened(3, &[5])?,
810        ])?;
811
812        let result =
813            cursor.resolve_next(RecordedEventFamily::Activity, CorrelationKey::Activity(0));
814
815        assert!(
816            matches!(result, CursorResolveResult::Matched(_)),
817            "a reopen naming another activity must not reopen this one, got {result:?}"
818        );
819        Ok(())
820    }
821
822    fn paused(seq: u64) -> Result<Event, Box<dyn std::error::Error>> {
823        Ok(Event::WorkflowPaused {
824            envelope: envelope(seq)?,
825            run_id: aion_core::RunId::new(uuid::Uuid::from_u128(1)),
826            reason: None,
827            operator: None,
828        })
829    }
830
831    fn resumed(seq: u64) -> Result<Event, Box<dyn std::error::Error>> {
832        Ok(Event::WorkflowResumed {
833            envelope: envelope(seq)?,
834            run_id: aion_core::RunId::new(uuid::Uuid::from_u128(1)),
835            operator: None,
836        })
837    }
838
839    /// GATE-6 replay-invisibility (following the TimerCancelled-cause / reopen
840    /// precedent): WorkflowPaused/WorkflowResumed interleaved mid-history are
841    /// never matched, mismatched, or consumed by the cursor — an activity
842    /// recorded around them resolves exactly as if they were absent.
843    #[test]
844    fn pause_resume_markers_are_invisible_to_the_cursor() -> Result<(), Box<dyn std::error::Error>>
845    {
846        let mut cursor = HistoryCursor::new(vec![
847            workflow_started(1)?,
848            paused(2)?,
849            scheduled(3, 0)?,
850            resumed(4)?,
851            completed(5, 0)?,
852        ])?;
853
854        let result =
855            cursor.resolve_next(RecordedEventFamily::Activity, CorrelationKey::Activity(0));
856        match result {
857            CursorResolveResult::Matched(events) => {
858                assert_eq!(
859                    events.len(),
860                    2,
861                    "only the activity's own events are consumed"
862                );
863                assert!(matches!(
864                    events.last(),
865                    Some(Event::ActivityCompleted { .. })
866                ));
867            }
868            other => {
869                return Err(
870                    format!("pause/resume markers must not disturb replay, got {other:?}").into(),
871                );
872            }
873        }
874
875        // The markers are never matched: a further resolve for any activity
876        // exhausts (the leftover WorkflowResumed marker is invisible, never a
877        // spurious match or mismatch).
878        assert_eq!(
879            cursor.resolve_next(RecordedEventFamily::Activity, CorrelationKey::Activity(1)),
880            CursorResolveResult::Exhausted,
881            "the leftover pause/resume markers are invisible, never matched"
882        );
883        Ok(())
884    }
885
886    #[test]
887    fn new_accepts_in_order_history_and_exposes_starting_sequence()
888    -> Result<(), Box<dyn std::error::Error>> {
889        let cursor = HistoryCursor::new(vec![scheduled(7, 0)?, completed(8, 0)?])?;
890
891        assert_eq!(cursor.current_sequence(), Some(7));
892        assert_eq!(cursor.position_index(), 0);
893        Ok(())
894    }
895
896    #[test]
897    fn new_rejects_decreasing_sequence_order() -> Result<(), Box<dyn std::error::Error>> {
898        let error = HistoryCursor::new(vec![scheduled(9, 0)?, completed(8, 0)?])
899            .map(|_| "unexpected success")
900            .err();
901
902        assert!(error.is_some());
903        Ok(())
904    }
905
906    #[test]
907    fn resolves_activity_match_then_reports_exhaustion() -> Result<(), Box<dyn std::error::Error>> {
908        let mut cursor = HistoryCursor::new(vec![scheduled(1, 0)?, completed(2, 0)?])?;
909
910        let result =
911            cursor.resolve_next(RecordedEventFamily::Activity, CorrelationKey::Activity(0));
912
913        match result {
914            CursorResolveResult::Matched(events) => {
915                assert_eq!(events.len(), 2);
916                assert_eq!(cursor.current_sequence(), None);
917            }
918            CursorResolveResult::Exhausted | CursorResolveResult::Mismatch { .. } => {
919                return Err("activity should match recorded history".into());
920            }
921        }
922
923        assert_eq!(
924            cursor.resolve_next(RecordedEventFamily::Activity, CorrelationKey::Activity(1),),
925            CursorResolveResult::Exhausted
926        );
927        Ok(())
928    }
929
930    #[test]
931    fn skips_non_matchable_lifecycle_events_before_resolving()
932    -> Result<(), Box<dyn std::error::Error>> {
933        let mut cursor = HistoryCursor::new(vec![
934            workflow_started(1)?,
935            scheduled(2, 0)?,
936            completed(3, 0)?,
937        ])?;
938
939        let result =
940            cursor.resolve_next(RecordedEventFamily::Activity, CorrelationKey::Activity(0));
941
942        match result {
943            CursorResolveResult::Matched(events) => {
944                assert_eq!(events.len(), 2);
945                assert!(matches!(
946                    events.first(),
947                    Some(Event::ActivityScheduled { .. })
948                ));
949                assert_eq!(cursor.current_sequence(), None);
950            }
951            CursorResolveResult::Exhausted | CursorResolveResult::Mismatch { .. } => {
952                return Err("lifecycle events should not block command replay".into());
953            }
954        }
955        Ok(())
956    }
957
958    #[test]
959    fn reports_mismatch_for_different_next_family() -> Result<(), Box<dyn std::error::Error>> {
960        let timer_id = TimerId::anonymous(1);
961        let mut cursor = HistoryCursor::new(vec![Event::TimerStarted {
962            envelope: envelope(1)?,
963            timer_id: timer_id.clone(),
964            fire_at: timestamp()?,
965        }])?;
966
967        let result =
968            cursor.resolve_next(RecordedEventFamily::Activity, CorrelationKey::Activity(0));
969
970        match result {
971            CursorResolveResult::Mismatch {
972                expected_key,
973                found,
974            } => {
975                assert_eq!(expected_key, CorrelationKey::Activity(0));
976                assert_eq!(found.family, Some(RecordedEventFamily::Timer));
977                assert_eq!(found.key, Some(CorrelationKey::Timer(timer_id)));
978            }
979            CursorResolveResult::Matched(_) | CursorResolveResult::Exhausted => {
980                return Err("different next family should be a mismatch".into());
981            }
982        }
983        Ok(())
984    }
985
986    #[test]
987    fn walks_retry_failures_to_eventual_activity_success() -> Result<(), Box<dyn std::error::Error>>
988    {
989        let mut cursor = HistoryCursor::new(vec![
990            scheduled(1, 0)?,
991            failed(2, 0, 1, ActivityErrorKind::Retryable)?,
992            started(3, 0)?,
993            failed(4, 0, 2, ActivityErrorKind::Retryable)?,
994            completed(5, 0)?,
995        ])?;
996
997        let result =
998            cursor.resolve_next(RecordedEventFamily::Activity, CorrelationKey::Activity(0));
999
1000        match result {
1001            CursorResolveResult::Matched(events) => {
1002                assert_eq!(events.len(), 5);
1003                assert!(matches!(
1004                    events.last(),
1005                    Some(Event::ActivityCompleted { .. })
1006                ));
1007                assert_eq!(cursor.current_sequence(), None);
1008            }
1009            CursorResolveResult::Exhausted | CursorResolveResult::Mismatch { .. } => {
1010                return Err("retry history should resolve to eventual completion".into());
1011            }
1012        }
1013        Ok(())
1014    }
1015
1016    fn child_id(value: u128) -> WorkflowId {
1017        WorkflowId::new(Uuid::from_u128(value))
1018    }
1019
1020    fn child_started(seq: u64, child: u128) -> Result<Event, Box<dyn std::error::Error>> {
1021        Ok(Event::ChildWorkflowStarted {
1022            envelope: envelope(seq)?,
1023            child_workflow_id: child_id(child),
1024            workflow_type: "child".to_owned(),
1025            input: payload()?,
1026            package_version: aion_core::PackageVersion::new("a".repeat(64)),
1027        })
1028    }
1029
1030    fn child_completed(seq: u64, child: u128) -> Result<Event, Box<dyn std::error::Error>> {
1031        Ok(Event::ChildWorkflowCompleted {
1032            envelope: envelope(seq)?,
1033            child_workflow_id: child_id(child),
1034            result: payload()?,
1035        })
1036    }
1037
1038    fn signal_received(seq: u64, name: &str) -> Result<Event, Box<dyn std::error::Error>> {
1039        Ok(Event::SignalReceived {
1040            envelope: envelope(seq)?,
1041            name: name.to_owned(),
1042            payload: payload()?,
1043        })
1044    }
1045
1046    #[test]
1047    fn fast_forward_to_child_terminal_skips_consumed_commands()
1048    -> Result<(), Box<dyn std::error::Error>> {
1049        let mut cursor = HistoryCursor::new(vec![
1050            scheduled(1, 0)?,
1051            completed(2, 0)?,
1052            child_started(3, 1)?,
1053            signal_received(4, "mid")?,
1054            child_started(5, 2)?,
1055            child_completed(6, 1)?,
1056        ])?;
1057
1058        cursor.fast_forward_to_child_terminal(&child_id(1));
1059        let result = cursor.resolve_child_terminal(&child_id(1));
1060
1061        match result {
1062            ChildTerminalResolveResult::Matched(events) => {
1063                assert_eq!(events.len(), 1);
1064                assert!(matches!(
1065                    events.first(),
1066                    Some(Event::ChildWorkflowCompleted { child_workflow_id, .. })
1067                        if *child_workflow_id == child_id(1)
1068                ));
1069            }
1070            ChildTerminalResolveResult::Exhausted | ChildTerminalResolveResult::Mismatch { .. } => {
1071                return Err("await must reach the awaited child's recorded terminal".into());
1072            }
1073        }
1074        Ok(())
1075    }
1076
1077    #[test]
1078    fn fast_forward_to_child_terminal_exhausts_when_no_terminal_recorded()
1079    -> Result<(), Box<dyn std::error::Error>> {
1080        let mut cursor = HistoryCursor::new(vec![
1081            scheduled(1, 0)?,
1082            completed(2, 0)?,
1083            child_started(3, 1)?,
1084        ])?;
1085
1086        cursor.fast_forward_to_child_terminal(&child_id(1));
1087
1088        assert_eq!(
1089            cursor.resolve_child_terminal(&child_id(1)),
1090            ChildTerminalResolveResult::Exhausted
1091        );
1092        Ok(())
1093    }
1094
1095    #[test]
1096    fn resolve_child_terminal_reports_mismatch_without_skipping_in_strict_replay()
1097    -> Result<(), Box<dyn std::error::Error>> {
1098        let mut cursor = HistoryCursor::new(vec![scheduled(1, 0)?, child_completed(2, 1)?])?;
1099
1100        let result = cursor.resolve_child_terminal(&child_id(1));
1101
1102        match result {
1103            ChildTerminalResolveResult::Mismatch { found } => {
1104                assert_eq!(found.seq, 1);
1105                assert_eq!(found.family, Some(RecordedEventFamily::Activity));
1106            }
1107            ChildTerminalResolveResult::Matched(_) | ChildTerminalResolveResult::Exhausted => {
1108                return Err("strict replay must not skip an unconsumed recorded command".into());
1109            }
1110        }
1111        Ok(())
1112    }
1113
1114    fn child_failed(seq: u64, child: u128) -> Result<Event, Box<dyn std::error::Error>> {
1115        Ok(Event::ChildWorkflowFailed {
1116            envelope: envelope(seq)?,
1117            child_workflow_id: child_id(child),
1118            error: aion_core::WorkflowError {
1119                message: "child failed".to_owned(),
1120                details: None,
1121            },
1122        })
1123    }
1124
1125    #[test]
1126    fn resolve_activity_skips_interleaved_signal_and_leaves_it_matchable()
1127    -> Result<(), Box<dyn std::error::Error>> {
1128        let mut cursor = HistoryCursor::new(vec![
1129            scheduled(1, 0)?,
1130            signal_received(2, "mid")?,
1131            completed(3, 0)?,
1132        ])?;
1133
1134        let result =
1135            cursor.resolve_next(RecordedEventFamily::Activity, CorrelationKey::Activity(0));
1136
1137        match result {
1138            CursorResolveResult::Matched(events) => {
1139                assert_eq!(events.len(), 2);
1140                assert!(matches!(
1141                    events.first(),
1142                    Some(Event::ActivityScheduled { .. })
1143                ));
1144                assert!(matches!(
1145                    events.last(),
1146                    Some(Event::ActivityCompleted { activity_id, .. })
1147                        if activity_id.sequence_position() == 0
1148                ));
1149            }
1150            CursorResolveResult::Exhausted | CursorResolveResult::Mismatch { .. } => {
1151                return Err(
1152                    "an async signal arrival inside the activity range must not fail replay".into(),
1153                );
1154            }
1155        }
1156
1157        let signal = cursor.resolve_next(
1158            RecordedEventFamily::Signal,
1159            CorrelationKey::Signal {
1160                name: "mid".to_owned(),
1161                index: 0,
1162            },
1163        );
1164        match signal {
1165            CursorResolveResult::Matched(events) => {
1166                assert_eq!(events.len(), 1);
1167                assert!(matches!(events.first(), Some(Event::SignalReceived { .. })));
1168            }
1169            CursorResolveResult::Exhausted | CursorResolveResult::Mismatch { .. } => {
1170                return Err("the skipped signal must stay matchable for its own command".into());
1171            }
1172        }
1173        Ok(())
1174    }
1175
1176    #[test]
1177    fn resolve_activity_resolves_interleaved_parallel_activity_ranges()
1178    -> Result<(), Box<dyn std::error::Error>> {
1179        let mut cursor = HistoryCursor::new(vec![
1180            scheduled(1, 0)?,
1181            scheduled(2, 1)?,
1182            completed(3, 1)?,
1183            completed(4, 0)?,
1184        ])?;
1185
1186        let first = cursor.resolve_next(RecordedEventFamily::Activity, CorrelationKey::Activity(0));
1187        match first {
1188            CursorResolveResult::Matched(events) => {
1189                assert_eq!(events.len(), 2);
1190                assert!(matches!(
1191                    events.last(),
1192                    Some(Event::ActivityCompleted { activity_id, .. })
1193                        if activity_id.sequence_position() == 0
1194                ));
1195            }
1196            CursorResolveResult::Exhausted | CursorResolveResult::Mismatch { .. } => {
1197                return Err("a parallel activity's events inside the range must be skipped".into());
1198            }
1199        }
1200
1201        let second =
1202            cursor.resolve_next(RecordedEventFamily::Activity, CorrelationKey::Activity(1));
1203        match second {
1204            CursorResolveResult::Matched(events) => {
1205                assert_eq!(events.len(), 2);
1206                assert!(matches!(
1207                    events.last(),
1208                    Some(Event::ActivityCompleted { activity_id, .. })
1209                        if activity_id.sequence_position() == 1
1210                ));
1211                assert_eq!(cursor.current_sequence(), None);
1212            }
1213            CursorResolveResult::Exhausted | CursorResolveResult::Mismatch { .. } => {
1214                return Err("the interleaved activity must remain resolvable afterwards".into());
1215            }
1216        }
1217        Ok(())
1218    }
1219
1220    #[test]
1221    fn resolve_activity_skips_interleaved_child_terminal() -> Result<(), Box<dyn std::error::Error>>
1222    {
1223        let mut cursor = HistoryCursor::new(vec![
1224            child_started(1, 7)?,
1225            scheduled(2, 0)?,
1226            child_completed(3, 7)?,
1227            child_failed(4, 9)?,
1228            completed(5, 0)?,
1229        ])?;
1230
1231        cursor.fast_forward_to_key(&CorrelationKey::Activity(0));
1232        let result =
1233            cursor.resolve_next(RecordedEventFamily::Activity, CorrelationKey::Activity(0));
1234
1235        match result {
1236            CursorResolveResult::Matched(events) => {
1237                assert_eq!(events.len(), 2);
1238                assert!(matches!(
1239                    events.last(),
1240                    Some(Event::ActivityCompleted { activity_id, .. })
1241                        if activity_id.sequence_position() == 0
1242                ));
1243            }
1244            CursorResolveResult::Exhausted | CursorResolveResult::Mismatch { .. } => {
1245                return Err("child terminals inside the activity range must be skipped".into());
1246            }
1247        }
1248        Ok(())
1249    }
1250
1251    #[test]
1252    fn resolve_activity_still_mismatches_on_wrong_anchor_key()
1253    -> Result<(), Box<dyn std::error::Error>> {
1254        let mut cursor = HistoryCursor::new(vec![scheduled(1, 1)?, completed(2, 1)?])?;
1255
1256        let result =
1257            cursor.resolve_next(RecordedEventFamily::Activity, CorrelationKey::Activity(0));
1258
1259        match result {
1260            CursorResolveResult::Mismatch {
1261                expected_key,
1262                found,
1263            } => {
1264                assert_eq!(expected_key, CorrelationKey::Activity(0));
1265                assert_eq!(found.key, Some(CorrelationKey::Activity(1)));
1266            }
1267            CursorResolveResult::Matched(_) | CursorResolveResult::Exhausted => {
1268                return Err("a wrong key at the Scheduled anchor must stay a mismatch".into());
1269            }
1270        }
1271        Ok(())
1272    }
1273
1274    #[test]
1275    fn fast_forward_and_resolution_smoke_over_large_history()
1276    -> Result<(), Box<dyn std::error::Error>> {
1277        let count: u64 = 5_000;
1278        let mut events = Vec::with_capacity(usize::try_from(count * 2)?);
1279        for ordinal in 0..count {
1280            events.push(scheduled(ordinal * 2 + 1, ordinal)?);
1281            events.push(completed(ordinal * 2 + 2, ordinal)?);
1282        }
1283        let mut cursor = HistoryCursor::new(events)?;
1284
1285        for ordinal in 0..count {
1286            let key = CorrelationKey::Activity(ordinal);
1287            cursor.fast_forward_to_key(&key);
1288            let result = cursor.resolve_next(RecordedEventFamily::Activity, key);
1289            assert!(
1290                matches!(result, CursorResolveResult::Matched(ref events) if events.len() == 2),
1291                "ordinal {ordinal} failed to resolve in the large-history smoke"
1292            );
1293        }
1294        assert_eq!(cursor.current_sequence(), None);
1295        Ok(())
1296    }
1297
1298    #[test]
1299    fn returns_terminal_activity_failure_as_recorded_outcome()
1300    -> Result<(), Box<dyn std::error::Error>> {
1301        let mut cursor = HistoryCursor::new(vec![
1302            scheduled(1, 0)?,
1303            failed(2, 0, 1, ActivityErrorKind::Retryable)?,
1304            failed(3, 0, 2, ActivityErrorKind::Terminal)?,
1305        ])?;
1306
1307        let result =
1308            cursor.resolve_next(RecordedEventFamily::Activity, CorrelationKey::Activity(0));
1309
1310        match result {
1311            CursorResolveResult::Matched(events) => {
1312                assert_eq!(events.len(), 3);
1313                assert!(matches!(
1314                    events.last(),
1315                    Some(Event::ActivityFailed { error, .. }) if error.kind == ActivityErrorKind::Terminal
1316                ));
1317                assert_eq!(cursor.current_sequence(), None);
1318            }
1319            CursorResolveResult::Exhausted | CursorResolveResult::Mismatch { .. } => {
1320                return Err("terminal failure should be the recorded outcome".into());
1321            }
1322        }
1323        Ok(())
1324    }
1325}