Skip to main content

chainfold/
driver.rs

1#[cfg(not(feature = "std"))]
2use alloc::vec::Vec;
3#[cfg(feature = "std")]
4use std::vec::Vec;
5
6use core::time::Duration;
7
8use crate::{
9    batch::Batch,
10    engine::{
11        ApplySummary,
12        Engine,
13        EngineConfig,
14    },
15    error::{
16        ApplyError,
17        ConfigError,
18        DivergenceCause,
19        DurabilityLost,
20        EngineStatus,
21        RollbackError,
22    },
23    fold::Fold,
24    position::{
25        BlockRef,
26        Position,
27    },
28    sink::{
29        NoSink,
30        SnapshotSink,
31    },
32    source::{
33        ReplayHorizon,
34        Source,
35    },
36};
37
38/// Default poll cadence when the source has no error backlog.
39const DEFAULT_POLL_INTERVAL: Duration = Duration::from_secs(1);
40/// Default first backoff step after a source error.
41const DEFAULT_BACKOFF_BASE: Duration = Duration::from_millis(200);
42/// Default ceiling on exponential backoff.
43const DEFAULT_BACKOFF_MAX: Duration = Duration::from_secs(30);
44/// Default blocks between checkpoints.
45const DEFAULT_CHECKPOINT_INTERVAL: u64 = 64;
46
47/// True when `block` has reached the next interval step past the last marked block.
48fn due(last: Option<u64>, block: u64, interval: u64) -> bool {
49    last.is_none_or(|last| block >= last.saturating_add(interval))
50}
51
52/// Outcome of one driver tick.
53#[derive(Debug, Clone, Copy, PartialEq, Eq)]
54pub enum Tick {
55    /// Batch applied; the summary counts what the fold saw.
56    Progressed(ApplySummary),
57    /// Poll returned nothing new.
58    Idle,
59    /// A fork rolled the engine back to a checkpoint.
60    RolledBack {
61        /// Cursor the rollback restored.
62        to: Option<Position>,
63    },
64    /// Engine reset to genesis; the next poll carries no cursor.
65    Resynced,
66    /// Source or its contract failed; the next delay backs off.
67    SourceError,
68    /// Sink refused a snapshot offer; reported once, folding continues unpersisted.
69    DurabilityLost,
70    /// Engine can make no further automated progress.
71    Terminal(EngineStatus),
72}
73
74/// Point-in-time snapshot of driver and engine state for external observers.
75#[derive(Debug, Clone, Copy, PartialEq)]
76pub struct DriverStatus {
77    /// Most recently applied position.
78    pub cursor: Option<Position>,
79    /// Most recent block whose hash the source confirmed.
80    pub last_verified: Option<BlockRef>,
81    /// Engine status behind this driver.
82    pub engine: EngineStatus,
83    /// True once the most recent poll returned no new blocks.
84    pub caught_up: bool,
85    /// Events the fold declared not its own.
86    pub skips: u64,
87    /// Cursor the sink reports a restart would recover; None without a sink or a
88    /// flush. A resync lowers it, so it holds for the instant it was read.
89    pub durable_cursor: Option<Position>,
90    /// True once the sink refused an offer; folding continues unpersisted.
91    pub durability_lost: bool,
92    /// Increments once per tick; a level signal for wait primitives.
93    pub generation: u64,
94}
95
96impl DriverStatus {
97    /// True once the engine can no longer make automated progress.
98    pub fn is_terminal(&self) -> bool {
99        !self.engine.is_active()
100    }
101}
102
103/// Poll cadence, backoff, and recovery tuning for a driver.
104#[derive(Debug, Clone, Copy, PartialEq, Eq)]
105pub struct DriverConfig {
106    /// Fold genesis; resync feasibility is judged against this block.
107    pub start_block: u64,
108    /// Delay between polls while the source is healthy.
109    pub poll_interval: Duration,
110    /// First backoff step after a source error.
111    pub backoff_base: Duration,
112    /// Ceiling on the exponential backoff.
113    pub backoff_max: Duration,
114    /// In-memory rollback points every N blocks of cursor progress; persists
115    /// nothing. None means caller-driven only.
116    pub checkpoint_interval: Option<u64>,
117    /// Blocks of durable-point progress between snapshot offers; None disables
118    /// offers.
119    pub snapshot_interval: Option<u64>,
120}
121
122impl Default for DriverConfig {
123    fn default() -> Self {
124        Self {
125            start_block: 0,
126            poll_interval: DEFAULT_POLL_INTERVAL,
127            backoff_base: DEFAULT_BACKOFF_BASE,
128            backoff_max: DEFAULT_BACKOFF_MAX,
129            checkpoint_interval: Some(DEFAULT_CHECKPOINT_INTERVAL),
130            snapshot_interval: Some(DEFAULT_CHECKPOINT_INTERVAL),
131        }
132    }
133}
134
135impl DriverConfig {
136    /// Defaults, folding from `start_block`.
137    pub fn from_block(start_block: u64) -> Self {
138        Self {
139            start_block,
140            ..Self::default()
141        }
142    }
143}
144
145/// Poll loop over one source: owns cadence, backoff, scanning, and fork recovery.
146pub struct Driver<F, S, K = NoSink>
147where
148    F: Fold,
149    S: Source<Event = F::Event>,
150    K: SnapshotSink<F>,
151{
152    engine: Engine<F>,
153    source: S,
154    sink: K,
155    config: DriverConfig,
156    batch: Batch<F::Event>,
157    scratch: Vec<(BlockRef, u32, F::Event)>,
158    scanned_to: Option<u64>,
159    initial: F,
160    consecutive_errors: u32,
161    caught_up: bool,
162    generation: u64,
163    last_checkpoint_block: Option<u64>,
164    last_snapshot_block: Option<u64>,
165    durability_lost: bool,
166    advanced: bool,
167}
168
169impl<F, S> Driver<F, S>
170where
171    F: Fold + Clone,
172    S: Source<Event = F::Event>,
173{
174    /// Builds a driver that persists nothing.
175    pub fn new(
176        fold: F,
177        source: S,
178        engine: EngineConfig,
179        config: DriverConfig,
180    ) -> Result<Self, ConfigError> {
181        Self::build(fold, source, NoSink, engine, config)
182    }
183
184    /// Resumes from a recovered engine, polling onward from its cursor.
185    ///
186    /// `genesis` is the fold a resync restarts from, so it is empty state rather than
187    /// the recovered state.
188    pub fn resume(
189        engine: Engine<F>,
190        source: S,
191        genesis: F,
192        config: DriverConfig,
193    ) -> Result<Self, ConfigError> {
194        Self::around(engine, source, NoSink, genesis, config)
195    }
196}
197
198impl<F, S, K> Driver<F, S, K>
199where
200    F: Fold + Clone,
201    S: Source<Event = F::Event>,
202    K: SnapshotSink<F>,
203{
204    /// Builds a driver that offers durable snapshots to the sink.
205    pub fn with_sink(
206        fold: F,
207        source: S,
208        sink: K,
209        engine: EngineConfig,
210        config: DriverConfig,
211    ) -> Result<Self, ConfigError> {
212        Self::build(fold, source, sink, engine, config)
213    }
214
215    /// Resumes from a recovered engine, offering durable snapshots to the sink.
216    pub fn resume_with_sink(
217        engine: Engine<F>,
218        source: S,
219        sink: K,
220        genesis: F,
221        config: DriverConfig,
222    ) -> Result<Self, ConfigError> {
223        Self::around(engine, source, sink, genesis, config)
224    }
225}
226
227impl<F, S, K> Driver<F, S, K>
228where
229    F: Fold + Clone,
230    S: Source<Event = F::Event>,
231    K: SnapshotSink<F>,
232{
233    fn build(
234        fold: F,
235        source: S,
236        sink: K,
237        engine_config: EngineConfig,
238        driver_config: DriverConfig,
239    ) -> Result<Self, ConfigError> {
240        let initial = fold.clone();
241        let engine = Engine::new(fold, engine_config)?;
242        Self::around(engine, source, sink, initial, driver_config)
243    }
244
245    /// Wraps an engine, checking the source horizon against the configured start block.
246    fn around(
247        engine: Engine<F>,
248        source: S,
249        sink: K,
250        initial: F,
251        driver_config: DriverConfig,
252    ) -> Result<Self, ConfigError> {
253        if let ReplayHorizon::FromBlock(horizon) = source.horizon()
254            && horizon > driver_config.start_block
255        {
256            return Err(ConfigError::HorizonExceedsStart {
257                start: driver_config.start_block,
258                horizon,
259            });
260        }
261        Ok(Self {
262            engine,
263            source,
264            sink,
265            config: driver_config,
266            batch: Batch::new(),
267            scratch: Vec::new(),
268            scanned_to: None,
269            initial,
270            consecutive_errors: 0,
271            caught_up: false,
272            generation: 0,
273            last_checkpoint_block: None,
274            last_snapshot_block: None,
275            durability_lost: false,
276            advanced: false,
277        })
278    }
279
280    /// Borrows the durability sink.
281    pub fn sink(&self) -> &K {
282        &self.sink
283    }
284
285    /// Consumes the driver, returning the sink for joining or inspection.
286    pub fn into_sink(self) -> K {
287        self.sink
288    }
289
290    /// Borrows the underlying engine.
291    pub fn engine(&self) -> &Engine<F> {
292        &self.engine
293    }
294
295    /// Manual recovery access: rollback out of Halted or Poisoned, then keep ticking.
296    pub fn engine_mut(&mut self) -> &mut Engine<F> {
297        &mut self.engine
298    }
299
300    /// Mutable access to the underlying event source.
301    pub fn source_mut(&mut self) -> &mut S {
302        &mut self.source
303    }
304
305    /// True once the most recent poll returned no new blocks.
306    pub fn is_caught_up(&self) -> bool {
307        self.caught_up
308    }
309
310    /// Runs the interval-based checkpoint rule.
311    fn auto_checkpoint(&mut self) {
312        let Some(interval) = self.config.checkpoint_interval else {
313            return;
314        };
315        let Some(cursor) = self.engine.cursor() else {
316            return;
317        };
318        if due(self.last_checkpoint_block, cursor.block, interval) {
319            self.run_checkpoint();
320        }
321    }
322
323    /// Stores a checkpoint and records the block it was taken at.
324    fn run_checkpoint(&mut self) {
325        self.engine.checkpoint();
326        if let Some(cursor) = self.engine.cursor() {
327            self.last_checkpoint_block = Some(cursor.block);
328        }
329    }
330
331    /// Runs the interval-based snapshot rule; returns the overriding tick, if any.
332    fn offer_snapshot(&mut self) -> Option<Tick> {
333        if self.durability_lost {
334            return None;
335        }
336        let interval = self.config.snapshot_interval?;
337        let point = self.engine.durable_point()?;
338        if !due(self.last_snapshot_block, point.block, interval) {
339            return None;
340        }
341        match self.sink.offer(&self.engine) {
342            Ok(()) => {
343                self.last_snapshot_block = Some(point.block);
344                None
345            }
346            Err(DurabilityLost) => Some(self.lose_durability()),
347        }
348    }
349
350    /// Latches the sink refusal so no further offer runs; folding continues unpersisted.
351    #[cold]
352    fn lose_durability(&mut self) -> Tick {
353        self.durability_lost = true;
354        Tick::DurabilityLost
355    }
356
357    /// Lowers the snapshot mark to the restore point so the next offer is not suppressed.
358    #[cold]
359    fn clamp_snapshot_mark(&mut self, to: Option<Position>) {
360        self.last_snapshot_block = self
361            .last_snapshot_block
362            .zip(to)
363            .map(|(last, point)| last.min(point.block));
364    }
365
366    /// Rolls back to the newest checkpoint at or below the ancestor, else escalates.
367    #[cold]
368    fn roll_back_to(&mut self, ancestor: u64) -> Tick {
369        match self.engine.rollback_at_or_below(ancestor) {
370            Ok(to) => {
371                self.caught_up = false;
372                self.clamp_snapshot_mark(to);
373                Tick::RolledBack { to }
374            }
375            Err(RollbackError::NoCheckpointAtOrBelow { .. }) => self.resync_or_terminal(),
376            Err(RollbackError::Unrecoverable { cause }) => {
377                Tick::Terminal(EngineStatus::Unrecoverable { cause })
378            }
379        }
380    }
381
382    /// Resyncs from genesis when the source horizon still covers the start block,
383    /// otherwise marks the engine unrecoverable with the horizon shortfall.
384    #[cold]
385    fn resync_or_terminal(&mut self) -> Tick {
386        match self.source.horizon() {
387            // The same shortfall `around` rejects at construction, reached at runtime.
388            ReplayHorizon::FromBlock(horizon) if horizon > self.config.start_block => {
389                self.engine
390                    .mark_unrecoverable(DivergenceCause::HorizonExceeded {
391                        needed: self.config.start_block,
392                        horizon,
393                    });
394                Tick::Terminal(self.engine.status())
395            }
396            _ => self.resync(),
397        }
398    }
399
400    #[cold]
401    fn resync(&mut self) -> Tick {
402        self.engine.reset(self.initial.clone());
403        self.caught_up = false;
404        self.consecutive_errors = 0;
405        self.scanned_to = None;
406        self.last_checkpoint_block = None;
407        self.last_snapshot_block = None;
408        Tick::Resynced
409    }
410
411    /// Runs one poll-apply step, then records whether the cursor moved forward.
412    fn step(&mut self) -> Tick {
413        let tick = self.poll_apply();
414        // Only forward cursor movement earns an immediate re-poll; a batch the
415        // engine fully deduped leaves the loop on its poll interval.
416        self.advanced = matches!(
417            tick,
418            Tick::Progressed(summary) if summary.applied > 0 || summary.skipped > 0
419        );
420        tick
421    }
422
423    /// Scans the source and applies the batch, recovering from a fork by bisection.
424    fn poll_apply(&mut self) -> Tick {
425        self.generation = self.generation.wrapping_add(1);
426        if !self.engine.status().is_active() {
427            return Tick::Terminal(self.engine.status());
428        }
429        if self.scan().is_err() {
430            self.consecutive_errors = self.consecutive_errors.saturating_add(1);
431            return Tick::SourceError;
432        }
433        self.consecutive_errors = 0;
434        match self.engine.apply_batch(&self.batch) {
435            Ok(summary) => {
436                self.caught_up = self.batch.is_empty();
437                // A snapshot refusal overrides progress; a checkpoint is silent.
438                self.auto_checkpoint();
439                if let Some(tick) = self.offer_snapshot() {
440                    return tick;
441                }
442                if self.batch.is_empty() {
443                    Tick::Idle
444                } else {
445                    Tick::Progressed(summary)
446                }
447            }
448            Err(
449                ApplyError::ForkSuspected { .. }
450                | ApplyError::MissingBoundary
451                | ApplyError::CursorBlockUnobserved { .. },
452            ) => self.recover_via_bisection(),
453            Err(ApplyError::Halted { .. } | ApplyError::Poisoned { .. }) => {
454                Tick::Terminal(self.engine.status())
455            }
456            Err(ApplyError::Shape(_) | ApplyError::BoundaryNumberMismatch { .. }) => {
457                self.consecutive_errors = self.consecutive_errors.saturating_add(1);
458                Tick::SourceError
459            }
460            Err(ApplyError::NotActive { .. }) => {
461                unreachable!(
462                    "engine status was checked active before this apply_batch call"
463                )
464            }
465        }
466    }
467
468    /// Fills `batch` with the blocks strictly after the cursor, plus the cursor block's
469    /// header as the source reports it now.
470    fn scan(&mut self) -> Result<(), S::Error> {
471        self.batch.clear();
472        let cursor = self.engine.cursor();
473        self.batch.boundary = match cursor {
474            Some(cursor) => self.source.header_at(cursor.block)?,
475            None => None,
476        };
477
478        let head = self.source.head()?;
479        let mut from = match cursor {
480            Some(cursor) => self
481                .scanned_to
482                .map_or(cursor.block + 1, |to| to.saturating_add(1))
483                .min(cursor.block + 1),
484            None => self.config.start_block,
485        };
486
487        let window = self.source.window().max(1);
488        while from <= head {
489            let to = head.min(from.saturating_add(window - 1));
490            self.scratch.clear();
491            self.source.events_in(from, to, &mut self.scratch)?;
492            self.scanned_to = Some(to);
493            from = to.saturating_add(1);
494            if !self.scratch.is_empty() {
495                group_into(&mut self.batch, &mut self.scratch);
496                break;
497            }
498        }
499        Ok(())
500    }
501
502    /// Bisects the observed ring for the deepest still-canonical block, then rolls back.
503    #[cold]
504    fn recover_via_bisection(&mut self) -> Tick {
505        let observed: Vec<BlockRef> = self.engine.observed().collect();
506        let mut lo = 0usize;
507        let mut hi = observed.len();
508        while lo < hi {
509            let mid = lo + (hi - lo) / 2;
510            match self.source.header_at(observed[mid].number) {
511                Ok(Some(header)) if header.hash == observed[mid].hash => lo = mid + 1,
512                Ok(_) => hi = mid,
513                Err(_) => {
514                    self.consecutive_errors = self.consecutive_errors.saturating_add(1);
515                    return Tick::SourceError;
516                }
517            }
518        }
519        if lo == 0 {
520            return self.resync_or_terminal();
521        }
522        self.roll_back_to(observed[lo - 1].number)
523    }
524
525    /// Advances the loop by one poll, apply, and recovery step.
526    pub fn tick(&mut self) -> Tick {
527        self.step()
528    }
529
530    /// Forces a checkpoint now.
531    pub fn checkpoint(&mut self) {
532        self.run_checkpoint();
533    }
534
535    /// Snapshots the current driver and engine state.
536    pub fn status(&self) -> DriverStatus {
537        DriverStatus {
538            cursor: self.engine.cursor(),
539            last_verified: self.engine.last_verified(),
540            engine: self.engine.status(),
541            caught_up: self.caught_up,
542            skips: self.engine.skip_count(),
543            durable_cursor: self.sink.durable_cursor(),
544            durability_lost: self.durability_lost,
545            generation: self.generation,
546        }
547    }
548
549    /// How long to wait before the next tick.
550    pub fn next_delay(&self) -> Duration {
551        if self.consecutive_errors == 0 {
552            // Catch-up polls run back to back; the poll interval paces the tip.
553            return if self.advanced {
554                Duration::ZERO
555            } else {
556                self.config.poll_interval
557            };
558        }
559        let factor = 1u32
560            .checked_shl(self.consecutive_errors - 1)
561            .unwrap_or(u32::MAX);
562        self.config
563            .backoff_base
564            .saturating_mul(factor)
565            .min(self.config.backoff_max)
566    }
567}
568
569/// Drains `entries` into `batch`
570fn group_into<E>(batch: &mut Batch<E>, entries: &mut Vec<(BlockRef, u32, E)>) {
571    entries.sort_by_key(|(block, log_index, _)| (block.number, *log_index));
572
573    let mut span: Vec<(u32, E)> = Vec::new();
574    let mut current: Option<BlockRef> = None;
575    for (block, log_index, event) in entries.drain(..) {
576        match current {
577            Some(open) if open.number != block.number => {
578                batch.push_block(open, span.drain(..));
579                current = Some(block);
580            }
581            None => current = Some(block),
582            _ => {}
583        }
584        span.push((log_index, event));
585    }
586    if let Some(open) = current {
587        batch.push_block(open, span);
588    }
589}
590
591#[cfg(test)]
592mod tests {
593    use super::*;
594    #[cfg(not(feature = "std"))]
595    use alloc::{
596        vec,
597        vec::Vec,
598    };
599    #[cfg(feature = "std")]
600    use std::{
601        vec,
602        vec::Vec,
603    };
604
605    use crate::test_util::{
606        FailKind,
607        PollFailure,
608        RecordingFold,
609        ScriptedChain,
610        WatermarkSink,
611    };
612
613    /// Wraps a scripted chain, counting probes and optionally failing the next few.
614    struct Probe {
615        inner: ScriptedChain,
616        calls: u32,
617        fail_next: u32,
618    }
619
620    impl Probe {
621        fn new(inner: ScriptedChain) -> Self {
622            Self {
623                inner,
624                calls: 0,
625                fail_next: 0,
626            }
627        }
628
629        fn fail_next_probes(&mut self, n: u32) {
630            self.fail_next = n;
631        }
632    }
633
634    impl Source for Probe {
635        type Event = u64;
636        type Error = PollFailure;
637
638        fn head(&mut self) -> Result<u64, PollFailure> {
639            self.inner.head()
640        }
641
642        fn header_at(&mut self, number: u64) -> Result<Option<BlockRef>, PollFailure> {
643            self.calls += 1;
644            if self.fail_next > 0 {
645                self.fail_next -= 1;
646                return Err(PollFailure);
647            }
648            self.inner.header_at(number)
649        }
650
651        fn events_in(
652            &mut self,
653            from: u64,
654            to: u64,
655            out: &mut Vec<(BlockRef, u32, u64)>,
656        ) -> Result<(), PollFailure> {
657            self.inner.events_in(from, to, out)
658        }
659
660        fn horizon(&self) -> ReplayHorizon {
661            self.inner.horizon()
662        }
663
664        fn window(&self) -> u64 {
665            self.inner.window()
666        }
667    }
668
669    /// Source that re-serves the same one-event block whatever range is asked for,
670    /// so the engine dedupes every poll after the first.
671    struct Stuck {
672        block: BlockRef,
673    }
674
675    impl Source for Stuck {
676        type Event = u64;
677        type Error = PollFailure;
678
679        fn head(&mut self) -> Result<u64, PollFailure> {
680            // above the block, so the scan keeps querying and keeps being re-served it
681            Ok(self.block.number + 1)
682        }
683
684        fn header_at(&mut self, _number: u64) -> Result<Option<BlockRef>, PollFailure> {
685            Ok(Some(self.block))
686        }
687
688        fn events_in(
689            &mut self,
690            _from: u64,
691            _to: u64,
692            out: &mut Vec<(BlockRef, u32, u64)>,
693        ) -> Result<(), PollFailure> {
694            out.push((self.block, 0, 1));
695            Ok(())
696        }
697    }
698
699    fn engine_config(checkpoint_slots: usize) -> EngineConfig {
700        EngineConfig {
701            ring_capacity: 8,
702            checkpoint_slots,
703        }
704    }
705
706    fn new_driver(
707        chain: ScriptedChain,
708        engine: EngineConfig,
709        config: DriverConfig,
710    ) -> Driver<RecordingFold, ScriptedChain> {
711        Driver::new(RecordingFold::default(), chain, engine, config).unwrap()
712    }
713
714    fn run_to_idle<F, S, K>(driver: &mut Driver<F, S, K>) -> Tick
715    where
716        F: Fold + Clone,
717        S: Source<Event = F::Event>,
718        K: SnapshotSink<F>,
719    {
720        let mut outcome = driver.tick();
721        while !matches!(outcome, Tick::Idle) {
722            outcome = driver.tick();
723        }
724        outcome
725    }
726
727    fn collect_to_idle<F, S, K>(driver: &mut Driver<F, S, K>) -> Vec<Tick>
728    where
729        F: Fold + Clone,
730        S: Source<Event = F::Event>,
731        K: SnapshotSink<F>,
732    {
733        let mut ticks = vec![driver.tick()];
734        while !matches!(ticks.last(), Some(Tick::Idle)) {
735            ticks.push(driver.tick());
736        }
737        ticks
738    }
739
740    /// Chain of `blocks` one-event blocks served one event-bearing block per poll.
741    fn one_event_chain(blocks: u64) -> ScriptedChain {
742        let mut chain = ScriptedChain::new(1);
743        for value in 1..=blocks {
744            chain.push_block(&[value]);
745        }
746        chain.set_window(1);
747        chain
748    }
749
750    fn cadence_config(checkpoint: u64, snapshot: u64) -> DriverConfig {
751        DriverConfig {
752            checkpoint_interval: Some(checkpoint),
753            snapshot_interval: Some(snapshot),
754            ..DriverConfig::default()
755        }
756    }
757
758    /// Recording fold over a scripted chain, offering snapshots to a watermark sink.
759    type SinkDriver = Driver<RecordingFold, ScriptedChain, WatermarkSink>;
760
761    fn sink_driver(blocks: u64, slots: usize, config: DriverConfig) -> SinkDriver {
762        Driver::with_sink(
763            RecordingFold::default(),
764            one_event_chain(blocks),
765            WatermarkSink::default(),
766            engine_config(slots),
767            config,
768        )
769        .unwrap()
770    }
771
772    fn probed_sink_driver(blocks: u64, slots: usize, config: DriverConfig) -> SinkDriver {
773        Driver::with_sink(
774            RecordingFold::default(),
775            one_event_chain(blocks),
776            WatermarkSink::default(),
777            engine_config(slots),
778            config,
779        )
780        .unwrap()
781    }
782
783    #[test]
784    fn snapshot_interval_offers_on_durable_point_cadence() {
785        // given twelve one-event blocks, 3 slots, checkpoints every 2, snapshots every 4
786        let mut driver = sink_driver(12, 3, cadence_config(2, 4));
787        // when driven to the tip
788        run_to_idle(&mut driver);
789        // then offers land on durable-point progress, not cursor progress
790        assert_eq!(
791            driver.sink().offered,
792            vec![Position::new(1, 0), Position::new(5, 0)]
793        );
794    }
795
796    #[test]
797    fn no_sink_never_offers_and_reports_no_durable_cursor() {
798        // given a NoSink driver over four blocks with both cadences at their tightest
799        let mut driver =
800            new_driver(one_event_chain(4), engine_config(2), cadence_config(1, 1));
801        // when driven to the tip collecting every tick
802        let ticks = collect_to_idle(&mut driver);
803        // then no tick reports lost durability and the status carries no durable cursor
804        assert!(!ticks.contains(&Tick::DurabilityLost));
805        assert_eq!(driver.status().durable_cursor, None);
806        assert!(!driver.status().durability_lost);
807    }
808
809    #[test]
810    fn zero_checkpoint_slots_never_offers() {
811        // given zero checkpoint slots over four blocks with both cadences at 1
812        let mut driver = sink_driver(4, 0, cadence_config(1, 1));
813        // when driven to the tip
814        run_to_idle(&mut driver);
815        // then nothing was ever offered and the durable cursor stays None
816        assert!(driver.sink().offered.is_empty());
817        assert_eq!(driver.status().durable_cursor, None);
818    }
819
820    #[test]
821    fn durable_cursor_trails_the_live_cursor_by_checkpoint_coverage() {
822        // given twelve blocks, 3 slots, checkpoints every 2, snapshots every block
823        let mut driver = sink_driver(12, 3, cadence_config(2, 1));
824        // when driven to the tip
825        run_to_idle(&mut driver);
826        // then the durable cursor trails the live cursor by at least (slots - 1) * interval
827        let status = driver.status();
828        assert_eq!(status.cursor, Some(Position::new(12, 0)));
829        assert_eq!(status.durable_cursor, Some(Position::new(7, 0)));
830        assert!(status.cursor.unwrap().block - status.durable_cursor.unwrap().block >= 4);
831    }
832
833    #[test]
834    fn reorg_across_the_live_cursor_leaves_the_durable_cursor_untouched() {
835        // given eight blocks driven to a durable cursor of (5, 0) with 4 slots
836        let mut driver = probed_sink_driver(8, 4, cadence_config(1, 1));
837        run_to_idle(&mut driver);
838        let before = driver.status().durable_cursor;
839        assert_eq!(before, Some(Position::new(5, 0)));
840        // when a depth-2 reorg rolls the driver back
841        driver.source_mut().reorg(2, &[&[70], &[80]]);
842        let outcome = driver.tick();
843        // then the rollback lands above the durable cursor and leaves it unchanged
844        let status = driver.status();
845        assert_eq!(
846            outcome,
847            Tick::RolledBack {
848                to: Some(Position::new(6, 0)),
849            }
850        );
851        assert_eq!(status.durable_cursor, before);
852        assert!(status.durable_cursor.unwrap() <= Position::new(6, 0));
853    }
854
855    #[test]
856    fn sink_failure_is_reported_once_then_folding_continues() {
857        // given a sink scripted to refuse its first offer, 2 slots, six blocks
858        let mut driver = Driver::with_sink(
859            RecordingFold::default(),
860            one_event_chain(6),
861            WatermarkSink {
862                offered: Vec::new(),
863                fail_next_offers: 1,
864            },
865            engine_config(2),
866            cadence_config(1, 1),
867        )
868        .unwrap();
869        // when driven to the tip collecting every tick
870        let ticks = collect_to_idle(&mut driver);
871        // then exactly one tick reports the loss and folding still reaches every event
872        let lost = ticks
873            .iter()
874            .filter(|tick| **tick == Tick::DurabilityLost)
875            .count();
876        assert_eq!(lost, 1);
877        assert!(driver.status().durability_lost);
878        assert!(driver.sink().offered.is_empty());
879        let expected: Vec<(Position, u64)> = (1..=6u64)
880            .map(|value| (Position::new(value, 0), value))
881            .collect();
882        assert_eq!(driver.engine().fold().applied, expected);
883    }
884
885    #[test]
886    fn rollback_does_not_suppress_the_next_offer() {
887        // given eight blocks driven to the tip with 4 slots and both cadences at 1
888        let mut driver = probed_sink_driver(8, 4, cadence_config(1, 1));
889        run_to_idle(&mut driver);
890        // when a depth-3 reorg rolls back and folding resumes to the new tip
891        driver.source_mut().reorg(3, &[&[60], &[70], &[80], &[90]]);
892        let outcome = driver.tick();
893        let Tick::RolledBack { to } = outcome else {
894            panic!("expected RolledBack, got {outcome:?}");
895        };
896        let restore = to.expect("the rollback restores a cursor").block;
897        run_to_idle(&mut driver);
898        // then offers stayed strictly ascending and resumed above the restore point
899        let offered = &driver.sink().offered;
900        assert!(offered.windows(2).all(|pair| pair[0].block < pair[1].block));
901        assert!(offered.last().expect("offers were made").block > restore);
902    }
903
904    #[test]
905    fn resync_lowers_the_reported_durable_cursor() {
906        // given eight blocks driven to a durable cursor at block 5
907        let mut driver = sink_driver(8, 4, cadence_config(1, 1));
908        run_to_idle(&mut driver);
909        let before = driver.status().durable_cursor.expect("offers were made");
910        assert_eq!(before, Position::new(5, 0));
911        // when a reorg below the ring forces a resync and folding rebuilds from genesis
912        driver.source_mut().reorg(8, &[&[10], &[20], &[30], &[40]]);
913        let outcome = driver.tick();
914        run_to_idle(&mut driver);
915        // then the durable cursor names the rebuilt state, below where it stood
916        assert_eq!(outcome, Tick::Resynced);
917        let after = driver.status().durable_cursor.expect("offers resumed");
918        assert!(after < before);
919    }
920
921    #[test]
922    fn driver_folds_to_tip_and_reports_caught_up() {
923        // given a ten-block chain with one event per block
924        let mut chain = ScriptedChain::new(1);
925        for value in 1..=10u64 {
926            chain.push_block(&[value]);
927        }
928        let mut driver = new_driver(chain, engine_config(0), DriverConfig::default());
929        // when ticking until Idle
930        run_to_idle(&mut driver);
931        // then the view holds every event in order and is_caught_up
932        let expected: Vec<(Position, u64)> = (1..=10u64)
933            .map(|value| (Position::new(value, 0), value))
934            .collect();
935        assert_eq!(driver.engine().fold().applied, expected);
936        assert!(driver.is_caught_up());
937    }
938
939    #[test]
940    fn empty_poll_is_idle() {
941        // given a caught-up driver over a two-block chain
942        let mut chain = ScriptedChain::new(1);
943        chain.push_block(&[1]);
944        chain.push_block(&[2]);
945        let mut driver = new_driver(chain, engine_config(0), DriverConfig::default());
946        driver.tick();
947        // when ticked again
948        let outcome = driver.tick();
949        // then Idle
950        assert_eq!(outcome, Tick::Idle);
951    }
952
953    #[test]
954    fn source_errors_back_off_exponentially() {
955        // given a chain that fails the next three polls
956        let mut chain = ScriptedChain::new(1);
957        chain.push_block(&[1]);
958        chain.fail_next_polls(3);
959        let mut driver = new_driver(chain, engine_config(0), DriverConfig::default());
960        // when ticking
961        let first = driver.tick();
962        let first_delay = driver.next_delay();
963        let second = driver.tick();
964        let second_delay = driver.next_delay();
965        let third = driver.tick();
966        let third_delay = driver.next_delay();
967        let fourth = driver.tick();
968        // then three SourceError ticks with next_delay 200ms, 400ms, 800ms, then a
969        // progressing tick that clears the backoff
970        assert_eq!(first, Tick::SourceError);
971        assert_eq!(first_delay, Duration::from_millis(200));
972        assert_eq!(second, Tick::SourceError);
973        assert_eq!(second_delay, Duration::from_millis(400));
974        assert_eq!(third, Tick::SourceError);
975        assert_eq!(third_delay, Duration::from_millis(800));
976        assert!(matches!(fourth, Tick::Progressed(_)));
977        assert_eq!(driver.next_delay(), Duration::ZERO);
978    }
979
980    #[test]
981    fn backoff_caps_at_max() {
982        // given a chain that fails every poll
983        let mut chain = ScriptedChain::new(1);
984        chain.fail_next_polls(u32::MAX);
985        let mut driver = new_driver(chain, engine_config(0), DriverConfig::default());
986        // when the doubling passes backoff_max
987        for _ in 0..10 {
988            driver.tick();
989        }
990        // then next_delay equals backoff_max
991        assert_eq!(driver.next_delay(), Duration::from_secs(30));
992    }
993
994    #[test]
995    fn catch_up_ticks_ask_for_no_delay_until_the_tip() {
996        // given ten one-event blocks served one per poll
997        let mut driver = new_driver(
998            one_event_chain(10),
999            engine_config(0),
1000            DriverConfig::default(),
1001        );
1002        // when one tick folds a block and the rest run to the tip
1003        driver.tick();
1004        let while_behind = driver.next_delay();
1005        run_to_idle(&mut driver);
1006        let at_tip = driver.next_delay();
1007        // then the catch-up tick asks for no delay and the tip tick asks for the interval
1008        assert_eq!(while_behind, Duration::ZERO);
1009        assert_eq!(at_tip, Duration::from_secs(1));
1010    }
1011
1012    #[test]
1013    fn an_empty_window_does_not_report_caught_up_while_behind_head() {
1014        // given ten blocks where only the last carries events, read one block per query
1015        let mut chain = ScriptedChain::new(1);
1016        for _ in 1..10 {
1017            chain.push_block(&[]);
1018        }
1019        chain.push_block(&[42]);
1020        chain.set_window(1);
1021        let mut driver = new_driver(chain, engine_config(0), DriverConfig::from_block(1));
1022        // when polled once
1023        let tick = driver.tick();
1024        // then the scan walked every empty window instead of stopping at the first
1025        assert!(matches!(tick, Tick::Progressed(_)), "got {tick:?}");
1026        assert!(!driver.is_caught_up());
1027        assert_eq!(
1028            driver.engine().fold().applied,
1029            vec![(Position::new(10, 0), 42)]
1030        );
1031    }
1032
1033    #[test]
1034    fn rollback_replays_from_the_restored_cursor_not_the_scan_mark() {
1035        // given a driver caught up on ten blocks, checkpointing every block
1036        let mut chain = ScriptedChain::new(1);
1037        for value in 1..=10u64 {
1038            chain.push_block(&[value]);
1039        }
1040        chain.set_window(1);
1041        let mut driver = new_driver(
1042            chain,
1043            engine_config(4),
1044            DriverConfig {
1045                checkpoint_interval: Some(1),
1046                ..DriverConfig::from_block(1)
1047            },
1048        );
1049        run_to_idle(&mut driver);
1050        // when the last three blocks are replaced
1051        driver.source_mut().reorg(3, &[&[80], &[90], &[100]]);
1052        run_to_idle(&mut driver);
1053        // then the scan restarted at the restored cursor, so the replacements folded;
1054        // resuming at the high-water mark instead would have skipped them entirely
1055        let applied: Vec<u64> = driver
1056            .engine()
1057            .fold()
1058            .applied
1059            .iter()
1060            .map(|(_, event)| *event)
1061            .collect();
1062        assert_eq!(applied, vec![1, 2, 3, 4, 5, 6, 7, 80, 90, 100]);
1063    }
1064
1065    #[test]
1066    fn a_fully_deduped_batch_keeps_the_poll_interval() {
1067        // given a source that re-serves the same one-event block on every poll
1068        let block = BlockRef {
1069            number: 1,
1070            hash: [7u8; 32],
1071        };
1072        let mut driver = Driver::new(
1073            RecordingFold::default(),
1074            Stuck { block },
1075            engine_config(0),
1076            DriverConfig::default(),
1077        )
1078        .unwrap();
1079        // when the first tick applies the block and the second dedupes it
1080        let applying = driver.tick();
1081        let after_apply = driver.next_delay();
1082        let deduping = driver.tick();
1083        let after_dedup = driver.next_delay();
1084        // then only the applying tick asks for an immediate re-poll
1085        assert_eq!(
1086            applying,
1087            Tick::Progressed(ApplySummary {
1088                applied: 1,
1089                deduped: 0,
1090                skipped: 0,
1091            })
1092        );
1093        assert_eq!(after_apply, Duration::ZERO);
1094        assert_eq!(
1095            deduping,
1096            Tick::Progressed(ApplySummary {
1097                applied: 0,
1098                deduped: 1,
1099                skipped: 0,
1100            })
1101        );
1102        assert_eq!(after_dedup, Duration::from_secs(1));
1103    }
1104
1105    #[test]
1106    fn fork_without_probe_resyncs_from_start() {
1107        // given a chain of five one-event blocks driven to the tip
1108        let mut chain = ScriptedChain::new(1);
1109        for value in 1..=5u64 {
1110            chain.push_block(&[value]);
1111        }
1112        let mut driver = new_driver(chain, engine_config(0), DriverConfig::default());
1113        driver.tick();
1114        // when the chain reorgs below the cursor and the boundary mismatch surfaces
1115        driver.source_mut().reorg(3, &[&[10], &[20], &[30]]);
1116        let outcome = driver.tick();
1117        // then the tick reports Resynced
1118        assert_eq!(outcome, Tick::Resynced);
1119        // and subsequent ticks rebuild the post-reorg view from scratch
1120        run_to_idle(&mut driver);
1121        let expected = vec![
1122            (Position::new(1, 0), 1),
1123            (Position::new(2, 0), 2),
1124            (Position::new(3, 0), 10),
1125            (Position::new(4, 0), 20),
1126            (Position::new(5, 0), 30),
1127        ];
1128        assert_eq!(driver.engine().fold().applied, expected);
1129    }
1130
1131    #[test]
1132    fn resync_with_moved_horizon_is_terminal() {
1133        // given a fork and a horizon raised above start_block
1134        let mut chain = ScriptedChain::new(1);
1135        for value in 1..=5u64 {
1136            chain.push_block(&[value]);
1137        }
1138        let mut driver = new_driver(chain, engine_config(0), DriverConfig::default());
1139        driver.tick();
1140        driver.source_mut().reorg(3, &[&[10], &[20], &[30]]);
1141        driver.source_mut().set_horizon(ReplayHorizon::FromBlock(1));
1142        // when the fork surfaces
1143        let outcome = driver.tick();
1144        // then Terminal with HorizonExceeded { needed, horizon }
1145        assert_eq!(
1146            outcome,
1147            Tick::Terminal(EngineStatus::Unrecoverable {
1148                cause: DivergenceCause::HorizonExceeded {
1149                    needed: 0,
1150                    horizon: 1,
1151                },
1152            })
1153        );
1154    }
1155
1156    #[test]
1157    fn construction_refuses_horizon_above_start() {
1158        // given a chain whose horizon starts at block 100 and default start_block 0
1159        let mut chain = ScriptedChain::new(1);
1160        chain.set_horizon(ReplayHorizon::FromBlock(100));
1161        // when constructing
1162        let result = Driver::new(
1163            RecordingFold::default(),
1164            chain,
1165            engine_config(0),
1166            DriverConfig::default(),
1167        );
1168        // then HorizonExceedsStart
1169        assert_eq!(
1170            result.err(),
1171            Some(ConfigError::HorizonExceedsStart {
1172                start: 0,
1173                horizon: 100,
1174            })
1175        );
1176    }
1177
1178    #[test]
1179    fn auto_checkpoint_follows_interval() {
1180        // given checkpoint_interval 4 over twelve one-event blocks polled one at a time
1181        let mut chain = ScriptedChain::new(1);
1182        for value in 1..=12u64 {
1183            chain.push_block(&[value]);
1184        }
1185        chain.set_window(1);
1186        let config = DriverConfig {
1187            checkpoint_interval: Some(4),
1188            ..DriverConfig::default()
1189        };
1190        let engine = EngineConfig {
1191            ring_capacity: 16,
1192            checkpoint_slots: 8,
1193        };
1194        let mut driver = new_driver(chain, engine, config);
1195        // when driven to the tip
1196        run_to_idle(&mut driver);
1197        // then checkpoint_count is at least 3
1198        assert!(driver.engine().checkpoint_count() >= 3);
1199    }
1200
1201    #[test]
1202    fn checkpoints_expire_once_their_block_leaves_the_ring() {
1203        // given checkpoint_interval 4 over twelve blocks with a ring holding only 8
1204        let mut chain = ScriptedChain::new(1);
1205        for value in 1..=12u64 {
1206            chain.push_block(&[value]);
1207        }
1208        chain.set_window(1);
1209        let config = DriverConfig {
1210            checkpoint_interval: Some(4),
1211            ..DriverConfig::default()
1212        };
1213        let mut driver = new_driver(chain, engine_config(8), config);
1214        // when driven to the tip, leaving the block 4 checkpoint outside the window
1215        run_to_idle(&mut driver);
1216        // then only the checkpoints the ring still observes are retained
1217        assert_eq!(driver.engine().checkpoint_count(), 2);
1218        assert_eq!(driver.engine().durable_point(), Some(Position::new(5, 0)));
1219    }
1220
1221    #[test]
1222    fn halt_is_terminal_and_recoverable_via_engine_mut() {
1223        // given a fold that halts at block 3 after a checkpoint taken at block 2
1224        let mut chain = ScriptedChain::new(1);
1225        chain.push_block(&[1]);
1226        chain.push_block(&[2]);
1227        chain.push_block(&[3]);
1228        chain.push_block(&[4]);
1229        chain.push_block(&[5]);
1230        chain.set_window(1);
1231        let halt_pos = Position::new(3, 0);
1232        let fold = RecordingFold {
1233            applied: Vec::new(),
1234            fail_at: Some((halt_pos, FailKind::Halt)),
1235        };
1236        let mut driver =
1237            Driver::new(fold, chain, engine_config(2), DriverConfig::default()).unwrap();
1238        driver.tick();
1239        driver.tick();
1240        driver.checkpoint();
1241        // when ticked to Terminal
1242        let outcome = driver.tick();
1243        assert_eq!(
1244            outcome,
1245            Tick::Terminal(EngineStatus::Halted { at: halt_pos })
1246        );
1247        driver.source_mut().reorg(3, &[&[], &[40], &[50]]);
1248        // then engine_mut rollback restores Active
1249        let restored = driver.engine_mut().rollback_at_or_below(2).unwrap();
1250        assert_eq!(restored, Some(Position::new(2, 0)));
1251        assert_eq!(driver.engine().status(), EngineStatus::Active);
1252        // and further ticks reach the tip
1253        run_to_idle(&mut driver);
1254        assert!(driver.is_caught_up());
1255        assert_eq!(driver.engine().cursor(), Some(Position::new(5, 0)));
1256    }
1257
1258    #[test]
1259    fn generation_increments_every_tick() {
1260        // given a driver over an empty chain
1261        let chain = ScriptedChain::new(1);
1262        let mut driver = new_driver(chain, engine_config(0), DriverConfig::default());
1263        let start = driver.status().generation;
1264        // when three ticks of any outcome run
1265        driver.tick();
1266        driver.tick();
1267        driver.tick();
1268        // then status generation rose by three
1269        assert_eq!(driver.status().generation, start + 3);
1270    }
1271
1272    #[test]
1273    fn status_snapshot_reflects_engine() {
1274        // given a driven driver over a two-block chain
1275        let mut chain = ScriptedChain::new(1);
1276        chain.push_block(&[1]);
1277        chain.push_block(&[2]);
1278        let mut driver = new_driver(chain, engine_config(0), DriverConfig::default());
1279        driver.tick();
1280        // when reading status
1281        let status = driver.status();
1282        // then cursor, skips, caught_up, engine status all match the engine accessors
1283        assert_eq!(status.cursor, driver.engine().cursor());
1284        assert_eq!(status.last_verified, driver.engine().last_verified());
1285        assert_eq!(status.engine, driver.engine().status());
1286        assert_eq!(status.caught_up, driver.is_caught_up());
1287        assert_eq!(status.skips, driver.engine().skip_count());
1288    }
1289
1290    #[test]
1291    fn reorged_content_produces_typed_fork_then_recovery() {
1292        // given an applied chain with a checkpoint below the fork
1293        let mut chain = ScriptedChain::new(1);
1294        for value in 1..=6u64 {
1295            chain.push_block(&[value]);
1296        }
1297        chain.set_window(1);
1298        let mut driver = Driver::new(
1299            RecordingFold::default(),
1300            chain,
1301            engine_config(4),
1302            DriverConfig::default(),
1303        )
1304        .unwrap();
1305        driver.tick();
1306        driver.tick();
1307        driver.tick();
1308        driver.checkpoint();
1309        driver.tick();
1310        driver.tick();
1311        driver.tick();
1312        // when a reorg redelivers changed content and the tick surfaces the fork
1313        driver.source_mut().reorg(3, &[&[40], &[50], &[60]]);
1314        let outcome = driver.tick();
1315        // then one tick reports RolledBack to the checkpoint, later ticks fold the new branch
1316        assert_eq!(
1317            outcome,
1318            Tick::RolledBack {
1319                to: Some(Position::new(3, 0)),
1320            }
1321        );
1322        run_to_idle(&mut driver);
1323        let expected = vec![
1324            (Position::new(1, 0), 1),
1325            (Position::new(2, 0), 2),
1326            (Position::new(3, 0), 3),
1327            (Position::new(4, 0), 40),
1328            (Position::new(5, 0), 50),
1329            (Position::new(6, 0), 60),
1330        ];
1331        assert_eq!(driver.engine().fold().applied, expected);
1332    }
1333
1334    #[test]
1335    fn eventless_fork_point_is_still_detected() {
1336        // given events only on blocks 2 and 7 with cursor at 7
1337        let mut chain = ScriptedChain::new(1);
1338        chain.push_block(&[]);
1339        chain.push_block(&[2]);
1340        chain.push_block(&[]);
1341        chain.push_block(&[]);
1342        chain.push_block(&[]);
1343        chain.push_block(&[]);
1344        chain.push_block(&[7]);
1345        chain.set_window(1);
1346        let mut driver = Driver::new(
1347            RecordingFold::default(),
1348            chain,
1349            engine_config(2),
1350            DriverConfig::default(),
1351        )
1352        .unwrap();
1353        driver.tick();
1354        driver.checkpoint();
1355        driver.tick();
1356        // when a reorg replaces eventless block 5 upward and the tick surfaces the fork
1357        driver.source_mut().reorg(3, &[&[], &[], &[70]]);
1358        let outcome = driver.tick();
1359        // then the boundary recheck detects it and recovery lands the correct view
1360        assert!(matches!(outcome, Tick::RolledBack { .. }));
1361        run_to_idle(&mut driver);
1362        let expected = vec![(Position::new(2, 0), 2), (Position::new(7, 0), 70)];
1363        assert_eq!(driver.engine().fold().applied, expected);
1364    }
1365
1366    #[test]
1367    fn shorter_chain_fork_is_suspected_not_retried() {
1368        // given a reorg to a chain shorter than the cursor block
1369        let mut chain = ScriptedChain::new(1);
1370        for value in 1..=5u64 {
1371            chain.push_block(&[value]);
1372        }
1373        chain.set_window(1);
1374        let mut driver = Driver::new(
1375            RecordingFold::default(),
1376            chain,
1377            engine_config(2),
1378            DriverConfig::default(),
1379        )
1380        .unwrap();
1381        driver.tick();
1382        driver.checkpoint();
1383        for _ in 0..4 {
1384            driver.tick();
1385        }
1386        // when the chain reorgs to a shorter tip and the tick surfaces the fork
1387        driver.source_mut().reorg(4, &[&[99]]);
1388        let outcome = driver.tick();
1389        // then the fork path runs, never SourceError, and recovery proceeds via bisection
1390        assert_ne!(outcome, Tick::SourceError);
1391        assert!(matches!(outcome, Tick::RolledBack { .. }));
1392        run_to_idle(&mut driver);
1393        let expected = vec![(Position::new(1, 0), 1), (Position::new(2, 0), 99)];
1394        assert_eq!(driver.engine().fold().applied, expected);
1395    }
1396
1397    #[test]
1398    fn bisection_finds_deepest_canonical_block() {
1399        // given a ring of eight observed blocks and a fork at the sixth
1400        let mut chain = ScriptedChain::new(1);
1401        for value in 1..=8u64 {
1402            chain.push_block(&[value]);
1403        }
1404        chain.set_window(1);
1405        let mut driver = Driver::new(
1406            RecordingFold::default(),
1407            Probe::new(chain),
1408            engine_config(2),
1409            DriverConfig::default(),
1410        )
1411        .unwrap();
1412        driver.tick();
1413        driver.checkpoint();
1414        for _ in 0..7 {
1415            driver.tick();
1416        }
1417        // when the chain reorgs at the sixth block and the tick surfaces the fork
1418        driver.source_mut().inner.reorg(3, &[&[60], &[70], &[80]]);
1419        // the scan spends one probe per poll on the boundary; count only the bisection
1420        driver.source_mut().calls = 0;
1421        let outcome = driver.tick();
1422        // then rollback lands at or below the fifth and probe count is at most four
1423        match outcome {
1424            Tick::RolledBack { to } => {
1425                let landed = to.map_or(0, |pos| pos.block);
1426                assert!(landed <= 5);
1427            }
1428            other => panic!("expected RolledBack, got {other:?}"),
1429        }
1430        assert!(driver.source_mut().calls <= 4);
1431    }
1432
1433    #[test]
1434    fn fork_deeper_than_ring_escalates() {
1435        // given ring capacity 4 and a reorg replacing every retained block
1436        fn build(horizon: ReplayHorizon) -> Driver<RecordingFold, ScriptedChain> {
1437            let mut chain = ScriptedChain::new(1);
1438            for value in 1..=6u64 {
1439                chain.push_block(&[value]);
1440            }
1441            chain.set_window(1);
1442            let mut driver = Driver::new(
1443                RecordingFold::default(),
1444                chain,
1445                EngineConfig {
1446                    ring_capacity: 4,
1447                    checkpoint_slots: 0,
1448                },
1449                DriverConfig::default(),
1450            )
1451            .unwrap();
1452            run_to_idle(&mut driver);
1453            driver
1454                .source_mut()
1455                .reorg(6, &[&[10], &[20], &[30], &[40], &[50], &[60]]);
1456            driver.source_mut().set_horizon(horizon);
1457            driver
1458        }
1459        // when the fork surfaces, once with a horizon that still covers start_block
1460        let mut resyncable = build(ReplayHorizon::Genesis);
1461        let resync_outcome = resyncable.tick();
1462        let mut terminal = build(ReplayHorizon::FromBlock(1));
1463        let terminal_outcome = terminal.tick();
1464        // then the resync-capable case reports Resynced, the moved horizon is Terminal
1465        assert_eq!(resync_outcome, Tick::Resynced);
1466        assert_eq!(
1467            terminal_outcome,
1468            Tick::Terminal(EngineStatus::Unrecoverable {
1469                cause: DivergenceCause::HorizonExceeded {
1470                    needed: 0,
1471                    horizon: 1,
1472                },
1473            })
1474        );
1475    }
1476
1477    #[test]
1478    fn no_checkpoint_below_ancestor_escalates() {
1479        // given checkpoints only above the fork ancestor
1480        fn build(horizon: ReplayHorizon) -> Driver<RecordingFold, ScriptedChain> {
1481            let mut chain = ScriptedChain::new(1);
1482            for value in 1..=8u64 {
1483                chain.push_block(&[value]);
1484            }
1485            chain.set_window(1);
1486            let mut driver = Driver::new(
1487                RecordingFold::default(),
1488                chain,
1489                engine_config(1),
1490                DriverConfig::default(),
1491            )
1492            .unwrap();
1493            for _ in 0..7 {
1494                driver.tick();
1495            }
1496            driver.checkpoint();
1497            driver.tick();
1498            driver.source_mut().reorg(3, &[&[60], &[70], &[80]]);
1499            driver.source_mut().set_horizon(horizon);
1500            driver
1501        }
1502        // when recovery runs, once with a horizon that still covers start_block
1503        let mut resyncable = build(ReplayHorizon::Genesis);
1504        let resync_outcome = resyncable.tick();
1505        let mut terminal = build(ReplayHorizon::FromBlock(1));
1506        let terminal_outcome = terminal.tick();
1507        // then resync, or Terminal with the moved horizon, never a wrong-state continue
1508        assert_eq!(resync_outcome, Tick::Resynced);
1509        assert_eq!(
1510            terminal_outcome,
1511            Tick::Terminal(EngineStatus::Unrecoverable {
1512                cause: DivergenceCause::HorizonExceeded {
1513                    needed: 0,
1514                    horizon: 1,
1515                },
1516            })
1517        );
1518    }
1519
1520    #[test]
1521    fn probe_failure_retries_without_state_damage() {
1522        // given header_at failures mid bisection
1523        let mut chain = ScriptedChain::new(1);
1524        for value in 1..=6u64 {
1525            chain.push_block(&[value]);
1526        }
1527        chain.set_window(1);
1528        let mut driver = Driver::new(
1529            RecordingFold::default(),
1530            Probe::new(chain),
1531            engine_config(2),
1532            DriverConfig::default(),
1533        )
1534        .unwrap();
1535        driver.tick();
1536        driver.checkpoint();
1537        for _ in 0..5 {
1538            driver.tick();
1539        }
1540        driver.source_mut().inner.reorg(3, &[&[40], &[50], &[60]]);
1541        driver.source_mut().fail_next_probes(1);
1542        // when ticked
1543        let first = driver.tick();
1544        // then SourceError, and the following tick completes recovery undisturbed
1545        assert_eq!(first, Tick::SourceError);
1546        let second = driver.tick();
1547        assert!(matches!(second, Tick::RolledBack { .. }));
1548        run_to_idle(&mut driver);
1549        let expected = vec![
1550            (Position::new(1, 0), 1),
1551            (Position::new(2, 0), 2),
1552            (Position::new(3, 0), 3),
1553            (Position::new(4, 0), 40),
1554            (Position::new(5, 0), 50),
1555            (Position::new(6, 0), 60),
1556        ];
1557        assert_eq!(driver.engine().fold().applied, expected);
1558    }
1559}