1#[cfg(not(feature = "std"))]
2use alloc::vec::Vec;
3#[cfg(feature = "std")]
4use std::vec::Vec;
5
6use core::time::Duration;
7
8use crate::{
9 batch::Batch,
10 engine::{
11 ApplySummary,
12 Engine,
13 EngineConfig,
14 },
15 error::{
16 ApplyError,
17 ConfigError,
18 DivergenceCause,
19 DurabilityLost,
20 EngineStatus,
21 RollbackError,
22 },
23 fold::Fold,
24 position::{
25 BlockRef,
26 Position,
27 },
28 sink::{
29 NoSink,
30 SnapshotSink,
31 },
32 source::{
33 ReplayHorizon,
34 Source,
35 },
36};
37
38const DEFAULT_POLL_INTERVAL: Duration = Duration::from_secs(1);
40const DEFAULT_BACKOFF_BASE: Duration = Duration::from_millis(200);
42const DEFAULT_BACKOFF_MAX: Duration = Duration::from_secs(30);
44const DEFAULT_CHECKPOINT_INTERVAL: u64 = 64;
46
47fn due(last: Option<u64>, block: u64, interval: u64) -> bool {
49 last.is_none_or(|last| block >= last.saturating_add(interval))
50}
51
52#[derive(Debug, Clone, Copy, PartialEq, Eq)]
54pub enum Tick {
55 Progressed(ApplySummary),
57 Idle,
59 RolledBack {
61 to: Option<Position>,
63 },
64 Resynced,
66 SourceError,
68 DurabilityLost,
70 Terminal(EngineStatus),
72}
73
74#[derive(Debug, Clone, Copy, PartialEq)]
76pub struct DriverStatus {
77 pub cursor: Option<Position>,
79 pub last_verified: Option<BlockRef>,
81 pub engine: EngineStatus,
83 pub caught_up: bool,
85 pub skips: u64,
87 pub durable_cursor: Option<Position>,
90 pub durability_lost: bool,
92 pub generation: u64,
94}
95
96impl DriverStatus {
97 pub fn is_terminal(&self) -> bool {
99 !self.engine.is_active()
100 }
101}
102
103#[derive(Debug, Clone, Copy, PartialEq, Eq)]
105pub struct DriverConfig {
106 pub start_block: u64,
108 pub poll_interval: Duration,
110 pub backoff_base: Duration,
112 pub backoff_max: Duration,
114 pub checkpoint_interval: Option<u64>,
117 pub snapshot_interval: Option<u64>,
120}
121
122impl Default for DriverConfig {
123 fn default() -> Self {
124 Self {
125 start_block: 0,
126 poll_interval: DEFAULT_POLL_INTERVAL,
127 backoff_base: DEFAULT_BACKOFF_BASE,
128 backoff_max: DEFAULT_BACKOFF_MAX,
129 checkpoint_interval: Some(DEFAULT_CHECKPOINT_INTERVAL),
130 snapshot_interval: Some(DEFAULT_CHECKPOINT_INTERVAL),
131 }
132 }
133}
134
135impl DriverConfig {
136 pub fn from_block(start_block: u64) -> Self {
138 Self {
139 start_block,
140 ..Self::default()
141 }
142 }
143}
144
145pub struct Driver<F, S, K = NoSink>
147where
148 F: Fold,
149 S: Source<Event = F::Event>,
150 K: SnapshotSink<F>,
151{
152 engine: Engine<F>,
153 source: S,
154 sink: K,
155 config: DriverConfig,
156 batch: Batch<F::Event>,
157 scratch: Vec<(BlockRef, u32, F::Event)>,
158 scanned_to: Option<u64>,
159 initial: F,
160 consecutive_errors: u32,
161 caught_up: bool,
162 generation: u64,
163 last_checkpoint_block: Option<u64>,
164 last_snapshot_block: Option<u64>,
165 durability_lost: bool,
166 advanced: bool,
167}
168
169impl<F, S> Driver<F, S>
170where
171 F: Fold + Clone,
172 S: Source<Event = F::Event>,
173{
174 pub fn new(
176 fold: F,
177 source: S,
178 engine: EngineConfig,
179 config: DriverConfig,
180 ) -> Result<Self, ConfigError> {
181 Self::build(fold, source, NoSink, engine, config)
182 }
183
184 pub fn resume(
189 engine: Engine<F>,
190 source: S,
191 genesis: F,
192 config: DriverConfig,
193 ) -> Result<Self, ConfigError> {
194 Self::around(engine, source, NoSink, genesis, config)
195 }
196}
197
198impl<F, S, K> Driver<F, S, K>
199where
200 F: Fold + Clone,
201 S: Source<Event = F::Event>,
202 K: SnapshotSink<F>,
203{
204 pub fn with_sink(
206 fold: F,
207 source: S,
208 sink: K,
209 engine: EngineConfig,
210 config: DriverConfig,
211 ) -> Result<Self, ConfigError> {
212 Self::build(fold, source, sink, engine, config)
213 }
214
215 pub fn resume_with_sink(
217 engine: Engine<F>,
218 source: S,
219 sink: K,
220 genesis: F,
221 config: DriverConfig,
222 ) -> Result<Self, ConfigError> {
223 Self::around(engine, source, sink, genesis, config)
224 }
225}
226
227impl<F, S, K> Driver<F, S, K>
228where
229 F: Fold + Clone,
230 S: Source<Event = F::Event>,
231 K: SnapshotSink<F>,
232{
233 fn build(
234 fold: F,
235 source: S,
236 sink: K,
237 engine_config: EngineConfig,
238 driver_config: DriverConfig,
239 ) -> Result<Self, ConfigError> {
240 let initial = fold.clone();
241 let engine = Engine::new(fold, engine_config)?;
242 Self::around(engine, source, sink, initial, driver_config)
243 }
244
245 fn around(
247 engine: Engine<F>,
248 source: S,
249 sink: K,
250 initial: F,
251 driver_config: DriverConfig,
252 ) -> Result<Self, ConfigError> {
253 if let ReplayHorizon::FromBlock(horizon) = source.horizon()
254 && horizon > driver_config.start_block
255 {
256 return Err(ConfigError::HorizonExceedsStart {
257 start: driver_config.start_block,
258 horizon,
259 });
260 }
261 Ok(Self {
262 engine,
263 source,
264 sink,
265 config: driver_config,
266 batch: Batch::new(),
267 scratch: Vec::new(),
268 scanned_to: None,
269 initial,
270 consecutive_errors: 0,
271 caught_up: false,
272 generation: 0,
273 last_checkpoint_block: None,
274 last_snapshot_block: None,
275 durability_lost: false,
276 advanced: false,
277 })
278 }
279
280 pub fn sink(&self) -> &K {
282 &self.sink
283 }
284
285 pub fn into_sink(self) -> K {
287 self.sink
288 }
289
290 pub fn engine(&self) -> &Engine<F> {
292 &self.engine
293 }
294
295 pub fn engine_mut(&mut self) -> &mut Engine<F> {
297 &mut self.engine
298 }
299
300 pub fn source_mut(&mut self) -> &mut S {
302 &mut self.source
303 }
304
305 pub fn is_caught_up(&self) -> bool {
307 self.caught_up
308 }
309
310 fn auto_checkpoint(&mut self) {
312 let Some(interval) = self.config.checkpoint_interval else {
313 return;
314 };
315 let Some(cursor) = self.engine.cursor() else {
316 return;
317 };
318 if due(self.last_checkpoint_block, cursor.block, interval) {
319 self.run_checkpoint();
320 }
321 }
322
323 fn run_checkpoint(&mut self) {
325 self.engine.checkpoint();
326 if let Some(cursor) = self.engine.cursor() {
327 self.last_checkpoint_block = Some(cursor.block);
328 }
329 }
330
331 fn offer_snapshot(&mut self) -> Option<Tick> {
333 if self.durability_lost {
334 return None;
335 }
336 let interval = self.config.snapshot_interval?;
337 let point = self.engine.durable_point()?;
338 if !due(self.last_snapshot_block, point.block, interval) {
339 return None;
340 }
341 match self.sink.offer(&self.engine) {
342 Ok(()) => {
343 self.last_snapshot_block = Some(point.block);
344 None
345 }
346 Err(DurabilityLost) => Some(self.lose_durability()),
347 }
348 }
349
350 #[cold]
352 fn lose_durability(&mut self) -> Tick {
353 self.durability_lost = true;
354 Tick::DurabilityLost
355 }
356
357 #[cold]
359 fn clamp_snapshot_mark(&mut self, to: Option<Position>) {
360 self.last_snapshot_block = self
361 .last_snapshot_block
362 .zip(to)
363 .map(|(last, point)| last.min(point.block));
364 }
365
366 #[cold]
368 fn roll_back_to(&mut self, ancestor: u64) -> Tick {
369 match self.engine.rollback_at_or_below(ancestor) {
370 Ok(to) => {
371 self.caught_up = false;
372 self.clamp_snapshot_mark(to);
373 Tick::RolledBack { to }
374 }
375 Err(RollbackError::NoCheckpointAtOrBelow { .. }) => self.resync_or_terminal(),
376 Err(RollbackError::Unrecoverable { cause }) => {
377 Tick::Terminal(EngineStatus::Unrecoverable { cause })
378 }
379 }
380 }
381
382 #[cold]
385 fn resync_or_terminal(&mut self) -> Tick {
386 match self.source.horizon() {
387 ReplayHorizon::FromBlock(horizon) if horizon > self.config.start_block => {
389 self.engine
390 .mark_unrecoverable(DivergenceCause::HorizonExceeded {
391 needed: self.config.start_block,
392 horizon,
393 });
394 Tick::Terminal(self.engine.status())
395 }
396 _ => self.resync(),
397 }
398 }
399
400 #[cold]
401 fn resync(&mut self) -> Tick {
402 self.engine.reset(self.initial.clone());
403 self.caught_up = false;
404 self.consecutive_errors = 0;
405 self.scanned_to = None;
406 self.last_checkpoint_block = None;
407 self.last_snapshot_block = None;
408 Tick::Resynced
409 }
410
411 fn step(&mut self) -> Tick {
413 let tick = self.poll_apply();
414 self.advanced = matches!(
417 tick,
418 Tick::Progressed(summary) if summary.applied > 0 || summary.skipped > 0
419 );
420 tick
421 }
422
423 fn poll_apply(&mut self) -> Tick {
425 self.generation = self.generation.wrapping_add(1);
426 if !self.engine.status().is_active() {
427 return Tick::Terminal(self.engine.status());
428 }
429 if self.scan().is_err() {
430 self.consecutive_errors = self.consecutive_errors.saturating_add(1);
431 return Tick::SourceError;
432 }
433 self.consecutive_errors = 0;
434 match self.engine.apply_batch(&self.batch) {
435 Ok(summary) => {
436 self.caught_up = self.batch.is_empty();
437 self.auto_checkpoint();
439 if let Some(tick) = self.offer_snapshot() {
440 return tick;
441 }
442 if self.batch.is_empty() {
443 Tick::Idle
444 } else {
445 Tick::Progressed(summary)
446 }
447 }
448 Err(
449 ApplyError::ForkSuspected { .. }
450 | ApplyError::MissingBoundary
451 | ApplyError::CursorBlockUnobserved { .. },
452 ) => self.recover_via_bisection(),
453 Err(ApplyError::Halted { .. } | ApplyError::Poisoned { .. }) => {
454 Tick::Terminal(self.engine.status())
455 }
456 Err(ApplyError::Shape(_) | ApplyError::BoundaryNumberMismatch { .. }) => {
457 self.consecutive_errors = self.consecutive_errors.saturating_add(1);
458 Tick::SourceError
459 }
460 Err(ApplyError::NotActive { .. }) => {
461 unreachable!(
462 "engine status was checked active before this apply_batch call"
463 )
464 }
465 }
466 }
467
468 fn scan(&mut self) -> Result<(), S::Error> {
471 self.batch.clear();
472 let cursor = self.engine.cursor();
473 self.batch.boundary = match cursor {
474 Some(cursor) => self.source.header_at(cursor.block)?,
475 None => None,
476 };
477
478 let head = self.source.head()?;
479 let mut from = match cursor {
480 Some(cursor) => self
481 .scanned_to
482 .map_or(cursor.block + 1, |to| to.saturating_add(1))
483 .min(cursor.block + 1),
484 None => self.config.start_block,
485 };
486
487 let window = self.source.window().max(1);
488 while from <= head {
489 let to = head.min(from.saturating_add(window - 1));
490 self.scratch.clear();
491 self.source.events_in(from, to, &mut self.scratch)?;
492 self.scanned_to = Some(to);
493 from = to.saturating_add(1);
494 if !self.scratch.is_empty() {
495 group_into(&mut self.batch, &mut self.scratch);
496 break;
497 }
498 }
499 Ok(())
500 }
501
502 #[cold]
504 fn recover_via_bisection(&mut self) -> Tick {
505 let observed: Vec<BlockRef> = self.engine.observed().collect();
506 let mut lo = 0usize;
507 let mut hi = observed.len();
508 while lo < hi {
509 let mid = lo + (hi - lo) / 2;
510 match self.source.header_at(observed[mid].number) {
511 Ok(Some(header)) if header.hash == observed[mid].hash => lo = mid + 1,
512 Ok(_) => hi = mid,
513 Err(_) => {
514 self.consecutive_errors = self.consecutive_errors.saturating_add(1);
515 return Tick::SourceError;
516 }
517 }
518 }
519 if lo == 0 {
520 return self.resync_or_terminal();
521 }
522 self.roll_back_to(observed[lo - 1].number)
523 }
524
525 pub fn tick(&mut self) -> Tick {
527 self.step()
528 }
529
530 pub fn checkpoint(&mut self) {
532 self.run_checkpoint();
533 }
534
535 pub fn status(&self) -> DriverStatus {
537 DriverStatus {
538 cursor: self.engine.cursor(),
539 last_verified: self.engine.last_verified(),
540 engine: self.engine.status(),
541 caught_up: self.caught_up,
542 skips: self.engine.skip_count(),
543 durable_cursor: self.sink.durable_cursor(),
544 durability_lost: self.durability_lost,
545 generation: self.generation,
546 }
547 }
548
549 pub fn next_delay(&self) -> Duration {
551 if self.consecutive_errors == 0 {
552 return if self.advanced {
554 Duration::ZERO
555 } else {
556 self.config.poll_interval
557 };
558 }
559 let factor = 1u32
560 .checked_shl(self.consecutive_errors - 1)
561 .unwrap_or(u32::MAX);
562 self.config
563 .backoff_base
564 .saturating_mul(factor)
565 .min(self.config.backoff_max)
566 }
567}
568
569fn group_into<E>(batch: &mut Batch<E>, entries: &mut Vec<(BlockRef, u32, E)>) {
571 entries.sort_by_key(|(block, log_index, _)| (block.number, *log_index));
572
573 let mut span: Vec<(u32, E)> = Vec::new();
574 let mut current: Option<BlockRef> = None;
575 for (block, log_index, event) in entries.drain(..) {
576 match current {
577 Some(open) if open.number != block.number => {
578 batch.push_block(open, span.drain(..));
579 current = Some(block);
580 }
581 None => current = Some(block),
582 _ => {}
583 }
584 span.push((log_index, event));
585 }
586 if let Some(open) = current {
587 batch.push_block(open, span);
588 }
589}
590
591#[cfg(test)]
592mod tests {
593 use super::*;
594 #[cfg(not(feature = "std"))]
595 use alloc::{
596 vec,
597 vec::Vec,
598 };
599 #[cfg(feature = "std")]
600 use std::{
601 vec,
602 vec::Vec,
603 };
604
605 use crate::test_util::{
606 FailKind,
607 PollFailure,
608 RecordingFold,
609 ScriptedChain,
610 WatermarkSink,
611 };
612
613 struct Probe {
615 inner: ScriptedChain,
616 calls: u32,
617 fail_next: u32,
618 }
619
620 impl Probe {
621 fn new(inner: ScriptedChain) -> Self {
622 Self {
623 inner,
624 calls: 0,
625 fail_next: 0,
626 }
627 }
628
629 fn fail_next_probes(&mut self, n: u32) {
630 self.fail_next = n;
631 }
632 }
633
634 impl Source for Probe {
635 type Event = u64;
636 type Error = PollFailure;
637
638 fn head(&mut self) -> Result<u64, PollFailure> {
639 self.inner.head()
640 }
641
642 fn header_at(&mut self, number: u64) -> Result<Option<BlockRef>, PollFailure> {
643 self.calls += 1;
644 if self.fail_next > 0 {
645 self.fail_next -= 1;
646 return Err(PollFailure);
647 }
648 self.inner.header_at(number)
649 }
650
651 fn events_in(
652 &mut self,
653 from: u64,
654 to: u64,
655 out: &mut Vec<(BlockRef, u32, u64)>,
656 ) -> Result<(), PollFailure> {
657 self.inner.events_in(from, to, out)
658 }
659
660 fn horizon(&self) -> ReplayHorizon {
661 self.inner.horizon()
662 }
663
664 fn window(&self) -> u64 {
665 self.inner.window()
666 }
667 }
668
669 struct Stuck {
672 block: BlockRef,
673 }
674
675 impl Source for Stuck {
676 type Event = u64;
677 type Error = PollFailure;
678
679 fn head(&mut self) -> Result<u64, PollFailure> {
680 Ok(self.block.number + 1)
682 }
683
684 fn header_at(&mut self, _number: u64) -> Result<Option<BlockRef>, PollFailure> {
685 Ok(Some(self.block))
686 }
687
688 fn events_in(
689 &mut self,
690 _from: u64,
691 _to: u64,
692 out: &mut Vec<(BlockRef, u32, u64)>,
693 ) -> Result<(), PollFailure> {
694 out.push((self.block, 0, 1));
695 Ok(())
696 }
697 }
698
699 fn engine_config(checkpoint_slots: usize) -> EngineConfig {
700 EngineConfig {
701 ring_capacity: 8,
702 checkpoint_slots,
703 }
704 }
705
706 fn new_driver(
707 chain: ScriptedChain,
708 engine: EngineConfig,
709 config: DriverConfig,
710 ) -> Driver<RecordingFold, ScriptedChain> {
711 Driver::new(RecordingFold::default(), chain, engine, config).unwrap()
712 }
713
714 fn run_to_idle<F, S, K>(driver: &mut Driver<F, S, K>) -> Tick
715 where
716 F: Fold + Clone,
717 S: Source<Event = F::Event>,
718 K: SnapshotSink<F>,
719 {
720 let mut outcome = driver.tick();
721 while !matches!(outcome, Tick::Idle) {
722 outcome = driver.tick();
723 }
724 outcome
725 }
726
727 fn collect_to_idle<F, S, K>(driver: &mut Driver<F, S, K>) -> Vec<Tick>
728 where
729 F: Fold + Clone,
730 S: Source<Event = F::Event>,
731 K: SnapshotSink<F>,
732 {
733 let mut ticks = vec![driver.tick()];
734 while !matches!(ticks.last(), Some(Tick::Idle)) {
735 ticks.push(driver.tick());
736 }
737 ticks
738 }
739
740 fn one_event_chain(blocks: u64) -> ScriptedChain {
742 let mut chain = ScriptedChain::new(1);
743 for value in 1..=blocks {
744 chain.push_block(&[value]);
745 }
746 chain.set_window(1);
747 chain
748 }
749
750 fn cadence_config(checkpoint: u64, snapshot: u64) -> DriverConfig {
751 DriverConfig {
752 checkpoint_interval: Some(checkpoint),
753 snapshot_interval: Some(snapshot),
754 ..DriverConfig::default()
755 }
756 }
757
758 type SinkDriver = Driver<RecordingFold, ScriptedChain, WatermarkSink>;
760
761 fn sink_driver(blocks: u64, slots: usize, config: DriverConfig) -> SinkDriver {
762 Driver::with_sink(
763 RecordingFold::default(),
764 one_event_chain(blocks),
765 WatermarkSink::default(),
766 engine_config(slots),
767 config,
768 )
769 .unwrap()
770 }
771
772 fn probed_sink_driver(blocks: u64, slots: usize, config: DriverConfig) -> SinkDriver {
773 Driver::with_sink(
774 RecordingFold::default(),
775 one_event_chain(blocks),
776 WatermarkSink::default(),
777 engine_config(slots),
778 config,
779 )
780 .unwrap()
781 }
782
783 #[test]
784 fn snapshot_interval_offers_on_durable_point_cadence() {
785 let mut driver = sink_driver(12, 3, cadence_config(2, 4));
787 run_to_idle(&mut driver);
789 assert_eq!(
791 driver.sink().offered,
792 vec![Position::new(1, 0), Position::new(5, 0)]
793 );
794 }
795
796 #[test]
797 fn no_sink_never_offers_and_reports_no_durable_cursor() {
798 let mut driver =
800 new_driver(one_event_chain(4), engine_config(2), cadence_config(1, 1));
801 let ticks = collect_to_idle(&mut driver);
803 assert!(!ticks.contains(&Tick::DurabilityLost));
805 assert_eq!(driver.status().durable_cursor, None);
806 assert!(!driver.status().durability_lost);
807 }
808
809 #[test]
810 fn zero_checkpoint_slots_never_offers() {
811 let mut driver = sink_driver(4, 0, cadence_config(1, 1));
813 run_to_idle(&mut driver);
815 assert!(driver.sink().offered.is_empty());
817 assert_eq!(driver.status().durable_cursor, None);
818 }
819
820 #[test]
821 fn durable_cursor_trails_the_live_cursor_by_checkpoint_coverage() {
822 let mut driver = sink_driver(12, 3, cadence_config(2, 1));
824 run_to_idle(&mut driver);
826 let status = driver.status();
828 assert_eq!(status.cursor, Some(Position::new(12, 0)));
829 assert_eq!(status.durable_cursor, Some(Position::new(7, 0)));
830 assert!(status.cursor.unwrap().block - status.durable_cursor.unwrap().block >= 4);
831 }
832
833 #[test]
834 fn reorg_across_the_live_cursor_leaves_the_durable_cursor_untouched() {
835 let mut driver = probed_sink_driver(8, 4, cadence_config(1, 1));
837 run_to_idle(&mut driver);
838 let before = driver.status().durable_cursor;
839 assert_eq!(before, Some(Position::new(5, 0)));
840 driver.source_mut().reorg(2, &[&[70], &[80]]);
842 let outcome = driver.tick();
843 let status = driver.status();
845 assert_eq!(
846 outcome,
847 Tick::RolledBack {
848 to: Some(Position::new(6, 0)),
849 }
850 );
851 assert_eq!(status.durable_cursor, before);
852 assert!(status.durable_cursor.unwrap() <= Position::new(6, 0));
853 }
854
855 #[test]
856 fn sink_failure_is_reported_once_then_folding_continues() {
857 let mut driver = Driver::with_sink(
859 RecordingFold::default(),
860 one_event_chain(6),
861 WatermarkSink {
862 offered: Vec::new(),
863 fail_next_offers: 1,
864 },
865 engine_config(2),
866 cadence_config(1, 1),
867 )
868 .unwrap();
869 let ticks = collect_to_idle(&mut driver);
871 let lost = ticks
873 .iter()
874 .filter(|tick| **tick == Tick::DurabilityLost)
875 .count();
876 assert_eq!(lost, 1);
877 assert!(driver.status().durability_lost);
878 assert!(driver.sink().offered.is_empty());
879 let expected: Vec<(Position, u64)> = (1..=6u64)
880 .map(|value| (Position::new(value, 0), value))
881 .collect();
882 assert_eq!(driver.engine().fold().applied, expected);
883 }
884
885 #[test]
886 fn rollback_does_not_suppress_the_next_offer() {
887 let mut driver = probed_sink_driver(8, 4, cadence_config(1, 1));
889 run_to_idle(&mut driver);
890 driver.source_mut().reorg(3, &[&[60], &[70], &[80], &[90]]);
892 let outcome = driver.tick();
893 let Tick::RolledBack { to } = outcome else {
894 panic!("expected RolledBack, got {outcome:?}");
895 };
896 let restore = to.expect("the rollback restores a cursor").block;
897 run_to_idle(&mut driver);
898 let offered = &driver.sink().offered;
900 assert!(offered.windows(2).all(|pair| pair[0].block < pair[1].block));
901 assert!(offered.last().expect("offers were made").block > restore);
902 }
903
904 #[test]
905 fn resync_lowers_the_reported_durable_cursor() {
906 let mut driver = sink_driver(8, 4, cadence_config(1, 1));
908 run_to_idle(&mut driver);
909 let before = driver.status().durable_cursor.expect("offers were made");
910 assert_eq!(before, Position::new(5, 0));
911 driver.source_mut().reorg(8, &[&[10], &[20], &[30], &[40]]);
913 let outcome = driver.tick();
914 run_to_idle(&mut driver);
915 assert_eq!(outcome, Tick::Resynced);
917 let after = driver.status().durable_cursor.expect("offers resumed");
918 assert!(after < before);
919 }
920
921 #[test]
922 fn driver_folds_to_tip_and_reports_caught_up() {
923 let mut chain = ScriptedChain::new(1);
925 for value in 1..=10u64 {
926 chain.push_block(&[value]);
927 }
928 let mut driver = new_driver(chain, engine_config(0), DriverConfig::default());
929 run_to_idle(&mut driver);
931 let expected: Vec<(Position, u64)> = (1..=10u64)
933 .map(|value| (Position::new(value, 0), value))
934 .collect();
935 assert_eq!(driver.engine().fold().applied, expected);
936 assert!(driver.is_caught_up());
937 }
938
939 #[test]
940 fn empty_poll_is_idle() {
941 let mut chain = ScriptedChain::new(1);
943 chain.push_block(&[1]);
944 chain.push_block(&[2]);
945 let mut driver = new_driver(chain, engine_config(0), DriverConfig::default());
946 driver.tick();
947 let outcome = driver.tick();
949 assert_eq!(outcome, Tick::Idle);
951 }
952
953 #[test]
954 fn source_errors_back_off_exponentially() {
955 let mut chain = ScriptedChain::new(1);
957 chain.push_block(&[1]);
958 chain.fail_next_polls(3);
959 let mut driver = new_driver(chain, engine_config(0), DriverConfig::default());
960 let first = driver.tick();
962 let first_delay = driver.next_delay();
963 let second = driver.tick();
964 let second_delay = driver.next_delay();
965 let third = driver.tick();
966 let third_delay = driver.next_delay();
967 let fourth = driver.tick();
968 assert_eq!(first, Tick::SourceError);
971 assert_eq!(first_delay, Duration::from_millis(200));
972 assert_eq!(second, Tick::SourceError);
973 assert_eq!(second_delay, Duration::from_millis(400));
974 assert_eq!(third, Tick::SourceError);
975 assert_eq!(third_delay, Duration::from_millis(800));
976 assert!(matches!(fourth, Tick::Progressed(_)));
977 assert_eq!(driver.next_delay(), Duration::ZERO);
978 }
979
980 #[test]
981 fn backoff_caps_at_max() {
982 let mut chain = ScriptedChain::new(1);
984 chain.fail_next_polls(u32::MAX);
985 let mut driver = new_driver(chain, engine_config(0), DriverConfig::default());
986 for _ in 0..10 {
988 driver.tick();
989 }
990 assert_eq!(driver.next_delay(), Duration::from_secs(30));
992 }
993
994 #[test]
995 fn catch_up_ticks_ask_for_no_delay_until_the_tip() {
996 let mut driver = new_driver(
998 one_event_chain(10),
999 engine_config(0),
1000 DriverConfig::default(),
1001 );
1002 driver.tick();
1004 let while_behind = driver.next_delay();
1005 run_to_idle(&mut driver);
1006 let at_tip = driver.next_delay();
1007 assert_eq!(while_behind, Duration::ZERO);
1009 assert_eq!(at_tip, Duration::from_secs(1));
1010 }
1011
1012 #[test]
1013 fn an_empty_window_does_not_report_caught_up_while_behind_head() {
1014 let mut chain = ScriptedChain::new(1);
1016 for _ in 1..10 {
1017 chain.push_block(&[]);
1018 }
1019 chain.push_block(&[42]);
1020 chain.set_window(1);
1021 let mut driver = new_driver(chain, engine_config(0), DriverConfig::from_block(1));
1022 let tick = driver.tick();
1024 assert!(matches!(tick, Tick::Progressed(_)), "got {tick:?}");
1026 assert!(!driver.is_caught_up());
1027 assert_eq!(
1028 driver.engine().fold().applied,
1029 vec![(Position::new(10, 0), 42)]
1030 );
1031 }
1032
1033 #[test]
1034 fn rollback_replays_from_the_restored_cursor_not_the_scan_mark() {
1035 let mut chain = ScriptedChain::new(1);
1037 for value in 1..=10u64 {
1038 chain.push_block(&[value]);
1039 }
1040 chain.set_window(1);
1041 let mut driver = new_driver(
1042 chain,
1043 engine_config(4),
1044 DriverConfig {
1045 checkpoint_interval: Some(1),
1046 ..DriverConfig::from_block(1)
1047 },
1048 );
1049 run_to_idle(&mut driver);
1050 driver.source_mut().reorg(3, &[&[80], &[90], &[100]]);
1052 run_to_idle(&mut driver);
1053 let applied: Vec<u64> = driver
1056 .engine()
1057 .fold()
1058 .applied
1059 .iter()
1060 .map(|(_, event)| *event)
1061 .collect();
1062 assert_eq!(applied, vec![1, 2, 3, 4, 5, 6, 7, 80, 90, 100]);
1063 }
1064
1065 #[test]
1066 fn a_fully_deduped_batch_keeps_the_poll_interval() {
1067 let block = BlockRef {
1069 number: 1,
1070 hash: [7u8; 32],
1071 };
1072 let mut driver = Driver::new(
1073 RecordingFold::default(),
1074 Stuck { block },
1075 engine_config(0),
1076 DriverConfig::default(),
1077 )
1078 .unwrap();
1079 let applying = driver.tick();
1081 let after_apply = driver.next_delay();
1082 let deduping = driver.tick();
1083 let after_dedup = driver.next_delay();
1084 assert_eq!(
1086 applying,
1087 Tick::Progressed(ApplySummary {
1088 applied: 1,
1089 deduped: 0,
1090 skipped: 0,
1091 })
1092 );
1093 assert_eq!(after_apply, Duration::ZERO);
1094 assert_eq!(
1095 deduping,
1096 Tick::Progressed(ApplySummary {
1097 applied: 0,
1098 deduped: 1,
1099 skipped: 0,
1100 })
1101 );
1102 assert_eq!(after_dedup, Duration::from_secs(1));
1103 }
1104
1105 #[test]
1106 fn fork_without_probe_resyncs_from_start() {
1107 let mut chain = ScriptedChain::new(1);
1109 for value in 1..=5u64 {
1110 chain.push_block(&[value]);
1111 }
1112 let mut driver = new_driver(chain, engine_config(0), DriverConfig::default());
1113 driver.tick();
1114 driver.source_mut().reorg(3, &[&[10], &[20], &[30]]);
1116 let outcome = driver.tick();
1117 assert_eq!(outcome, Tick::Resynced);
1119 run_to_idle(&mut driver);
1121 let expected = vec![
1122 (Position::new(1, 0), 1),
1123 (Position::new(2, 0), 2),
1124 (Position::new(3, 0), 10),
1125 (Position::new(4, 0), 20),
1126 (Position::new(5, 0), 30),
1127 ];
1128 assert_eq!(driver.engine().fold().applied, expected);
1129 }
1130
1131 #[test]
1132 fn resync_with_moved_horizon_is_terminal() {
1133 let mut chain = ScriptedChain::new(1);
1135 for value in 1..=5u64 {
1136 chain.push_block(&[value]);
1137 }
1138 let mut driver = new_driver(chain, engine_config(0), DriverConfig::default());
1139 driver.tick();
1140 driver.source_mut().reorg(3, &[&[10], &[20], &[30]]);
1141 driver.source_mut().set_horizon(ReplayHorizon::FromBlock(1));
1142 let outcome = driver.tick();
1144 assert_eq!(
1146 outcome,
1147 Tick::Terminal(EngineStatus::Unrecoverable {
1148 cause: DivergenceCause::HorizonExceeded {
1149 needed: 0,
1150 horizon: 1,
1151 },
1152 })
1153 );
1154 }
1155
1156 #[test]
1157 fn construction_refuses_horizon_above_start() {
1158 let mut chain = ScriptedChain::new(1);
1160 chain.set_horizon(ReplayHorizon::FromBlock(100));
1161 let result = Driver::new(
1163 RecordingFold::default(),
1164 chain,
1165 engine_config(0),
1166 DriverConfig::default(),
1167 );
1168 assert_eq!(
1170 result.err(),
1171 Some(ConfigError::HorizonExceedsStart {
1172 start: 0,
1173 horizon: 100,
1174 })
1175 );
1176 }
1177
1178 #[test]
1179 fn auto_checkpoint_follows_interval() {
1180 let mut chain = ScriptedChain::new(1);
1182 for value in 1..=12u64 {
1183 chain.push_block(&[value]);
1184 }
1185 chain.set_window(1);
1186 let config = DriverConfig {
1187 checkpoint_interval: Some(4),
1188 ..DriverConfig::default()
1189 };
1190 let engine = EngineConfig {
1191 ring_capacity: 16,
1192 checkpoint_slots: 8,
1193 };
1194 let mut driver = new_driver(chain, engine, config);
1195 run_to_idle(&mut driver);
1197 assert!(driver.engine().checkpoint_count() >= 3);
1199 }
1200
1201 #[test]
1202 fn checkpoints_expire_once_their_block_leaves_the_ring() {
1203 let mut chain = ScriptedChain::new(1);
1205 for value in 1..=12u64 {
1206 chain.push_block(&[value]);
1207 }
1208 chain.set_window(1);
1209 let config = DriverConfig {
1210 checkpoint_interval: Some(4),
1211 ..DriverConfig::default()
1212 };
1213 let mut driver = new_driver(chain, engine_config(8), config);
1214 run_to_idle(&mut driver);
1216 assert_eq!(driver.engine().checkpoint_count(), 2);
1218 assert_eq!(driver.engine().durable_point(), Some(Position::new(5, 0)));
1219 }
1220
1221 #[test]
1222 fn halt_is_terminal_and_recoverable_via_engine_mut() {
1223 let mut chain = ScriptedChain::new(1);
1225 chain.push_block(&[1]);
1226 chain.push_block(&[2]);
1227 chain.push_block(&[3]);
1228 chain.push_block(&[4]);
1229 chain.push_block(&[5]);
1230 chain.set_window(1);
1231 let halt_pos = Position::new(3, 0);
1232 let fold = RecordingFold {
1233 applied: Vec::new(),
1234 fail_at: Some((halt_pos, FailKind::Halt)),
1235 };
1236 let mut driver =
1237 Driver::new(fold, chain, engine_config(2), DriverConfig::default()).unwrap();
1238 driver.tick();
1239 driver.tick();
1240 driver.checkpoint();
1241 let outcome = driver.tick();
1243 assert_eq!(
1244 outcome,
1245 Tick::Terminal(EngineStatus::Halted { at: halt_pos })
1246 );
1247 driver.source_mut().reorg(3, &[&[], &[40], &[50]]);
1248 let restored = driver.engine_mut().rollback_at_or_below(2).unwrap();
1250 assert_eq!(restored, Some(Position::new(2, 0)));
1251 assert_eq!(driver.engine().status(), EngineStatus::Active);
1252 run_to_idle(&mut driver);
1254 assert!(driver.is_caught_up());
1255 assert_eq!(driver.engine().cursor(), Some(Position::new(5, 0)));
1256 }
1257
1258 #[test]
1259 fn generation_increments_every_tick() {
1260 let chain = ScriptedChain::new(1);
1262 let mut driver = new_driver(chain, engine_config(0), DriverConfig::default());
1263 let start = driver.status().generation;
1264 driver.tick();
1266 driver.tick();
1267 driver.tick();
1268 assert_eq!(driver.status().generation, start + 3);
1270 }
1271
1272 #[test]
1273 fn status_snapshot_reflects_engine() {
1274 let mut chain = ScriptedChain::new(1);
1276 chain.push_block(&[1]);
1277 chain.push_block(&[2]);
1278 let mut driver = new_driver(chain, engine_config(0), DriverConfig::default());
1279 driver.tick();
1280 let status = driver.status();
1282 assert_eq!(status.cursor, driver.engine().cursor());
1284 assert_eq!(status.last_verified, driver.engine().last_verified());
1285 assert_eq!(status.engine, driver.engine().status());
1286 assert_eq!(status.caught_up, driver.is_caught_up());
1287 assert_eq!(status.skips, driver.engine().skip_count());
1288 }
1289
1290 #[test]
1291 fn reorged_content_produces_typed_fork_then_recovery() {
1292 let mut chain = ScriptedChain::new(1);
1294 for value in 1..=6u64 {
1295 chain.push_block(&[value]);
1296 }
1297 chain.set_window(1);
1298 let mut driver = Driver::new(
1299 RecordingFold::default(),
1300 chain,
1301 engine_config(4),
1302 DriverConfig::default(),
1303 )
1304 .unwrap();
1305 driver.tick();
1306 driver.tick();
1307 driver.tick();
1308 driver.checkpoint();
1309 driver.tick();
1310 driver.tick();
1311 driver.tick();
1312 driver.source_mut().reorg(3, &[&[40], &[50], &[60]]);
1314 let outcome = driver.tick();
1315 assert_eq!(
1317 outcome,
1318 Tick::RolledBack {
1319 to: Some(Position::new(3, 0)),
1320 }
1321 );
1322 run_to_idle(&mut driver);
1323 let expected = vec![
1324 (Position::new(1, 0), 1),
1325 (Position::new(2, 0), 2),
1326 (Position::new(3, 0), 3),
1327 (Position::new(4, 0), 40),
1328 (Position::new(5, 0), 50),
1329 (Position::new(6, 0), 60),
1330 ];
1331 assert_eq!(driver.engine().fold().applied, expected);
1332 }
1333
1334 #[test]
1335 fn eventless_fork_point_is_still_detected() {
1336 let mut chain = ScriptedChain::new(1);
1338 chain.push_block(&[]);
1339 chain.push_block(&[2]);
1340 chain.push_block(&[]);
1341 chain.push_block(&[]);
1342 chain.push_block(&[]);
1343 chain.push_block(&[]);
1344 chain.push_block(&[7]);
1345 chain.set_window(1);
1346 let mut driver = Driver::new(
1347 RecordingFold::default(),
1348 chain,
1349 engine_config(2),
1350 DriverConfig::default(),
1351 )
1352 .unwrap();
1353 driver.tick();
1354 driver.checkpoint();
1355 driver.tick();
1356 driver.source_mut().reorg(3, &[&[], &[], &[70]]);
1358 let outcome = driver.tick();
1359 assert!(matches!(outcome, Tick::RolledBack { .. }));
1361 run_to_idle(&mut driver);
1362 let expected = vec![(Position::new(2, 0), 2), (Position::new(7, 0), 70)];
1363 assert_eq!(driver.engine().fold().applied, expected);
1364 }
1365
1366 #[test]
1367 fn shorter_chain_fork_is_suspected_not_retried() {
1368 let mut chain = ScriptedChain::new(1);
1370 for value in 1..=5u64 {
1371 chain.push_block(&[value]);
1372 }
1373 chain.set_window(1);
1374 let mut driver = Driver::new(
1375 RecordingFold::default(),
1376 chain,
1377 engine_config(2),
1378 DriverConfig::default(),
1379 )
1380 .unwrap();
1381 driver.tick();
1382 driver.checkpoint();
1383 for _ in 0..4 {
1384 driver.tick();
1385 }
1386 driver.source_mut().reorg(4, &[&[99]]);
1388 let outcome = driver.tick();
1389 assert_ne!(outcome, Tick::SourceError);
1391 assert!(matches!(outcome, Tick::RolledBack { .. }));
1392 run_to_idle(&mut driver);
1393 let expected = vec![(Position::new(1, 0), 1), (Position::new(2, 0), 99)];
1394 assert_eq!(driver.engine().fold().applied, expected);
1395 }
1396
1397 #[test]
1398 fn bisection_finds_deepest_canonical_block() {
1399 let mut chain = ScriptedChain::new(1);
1401 for value in 1..=8u64 {
1402 chain.push_block(&[value]);
1403 }
1404 chain.set_window(1);
1405 let mut driver = Driver::new(
1406 RecordingFold::default(),
1407 Probe::new(chain),
1408 engine_config(2),
1409 DriverConfig::default(),
1410 )
1411 .unwrap();
1412 driver.tick();
1413 driver.checkpoint();
1414 for _ in 0..7 {
1415 driver.tick();
1416 }
1417 driver.source_mut().inner.reorg(3, &[&[60], &[70], &[80]]);
1419 driver.source_mut().calls = 0;
1421 let outcome = driver.tick();
1422 match outcome {
1424 Tick::RolledBack { to } => {
1425 let landed = to.map_or(0, |pos| pos.block);
1426 assert!(landed <= 5);
1427 }
1428 other => panic!("expected RolledBack, got {other:?}"),
1429 }
1430 assert!(driver.source_mut().calls <= 4);
1431 }
1432
1433 #[test]
1434 fn fork_deeper_than_ring_escalates() {
1435 fn build(horizon: ReplayHorizon) -> Driver<RecordingFold, ScriptedChain> {
1437 let mut chain = ScriptedChain::new(1);
1438 for value in 1..=6u64 {
1439 chain.push_block(&[value]);
1440 }
1441 chain.set_window(1);
1442 let mut driver = Driver::new(
1443 RecordingFold::default(),
1444 chain,
1445 EngineConfig {
1446 ring_capacity: 4,
1447 checkpoint_slots: 0,
1448 },
1449 DriverConfig::default(),
1450 )
1451 .unwrap();
1452 run_to_idle(&mut driver);
1453 driver
1454 .source_mut()
1455 .reorg(6, &[&[10], &[20], &[30], &[40], &[50], &[60]]);
1456 driver.source_mut().set_horizon(horizon);
1457 driver
1458 }
1459 let mut resyncable = build(ReplayHorizon::Genesis);
1461 let resync_outcome = resyncable.tick();
1462 let mut terminal = build(ReplayHorizon::FromBlock(1));
1463 let terminal_outcome = terminal.tick();
1464 assert_eq!(resync_outcome, Tick::Resynced);
1466 assert_eq!(
1467 terminal_outcome,
1468 Tick::Terminal(EngineStatus::Unrecoverable {
1469 cause: DivergenceCause::HorizonExceeded {
1470 needed: 0,
1471 horizon: 1,
1472 },
1473 })
1474 );
1475 }
1476
1477 #[test]
1478 fn no_checkpoint_below_ancestor_escalates() {
1479 fn build(horizon: ReplayHorizon) -> Driver<RecordingFold, ScriptedChain> {
1481 let mut chain = ScriptedChain::new(1);
1482 for value in 1..=8u64 {
1483 chain.push_block(&[value]);
1484 }
1485 chain.set_window(1);
1486 let mut driver = Driver::new(
1487 RecordingFold::default(),
1488 chain,
1489 engine_config(1),
1490 DriverConfig::default(),
1491 )
1492 .unwrap();
1493 for _ in 0..7 {
1494 driver.tick();
1495 }
1496 driver.checkpoint();
1497 driver.tick();
1498 driver.source_mut().reorg(3, &[&[60], &[70], &[80]]);
1499 driver.source_mut().set_horizon(horizon);
1500 driver
1501 }
1502 let mut resyncable = build(ReplayHorizon::Genesis);
1504 let resync_outcome = resyncable.tick();
1505 let mut terminal = build(ReplayHorizon::FromBlock(1));
1506 let terminal_outcome = terminal.tick();
1507 assert_eq!(resync_outcome, Tick::Resynced);
1509 assert_eq!(
1510 terminal_outcome,
1511 Tick::Terminal(EngineStatus::Unrecoverable {
1512 cause: DivergenceCause::HorizonExceeded {
1513 needed: 0,
1514 horizon: 1,
1515 },
1516 })
1517 );
1518 }
1519
1520 #[test]
1521 fn probe_failure_retries_without_state_damage() {
1522 let mut chain = ScriptedChain::new(1);
1524 for value in 1..=6u64 {
1525 chain.push_block(&[value]);
1526 }
1527 chain.set_window(1);
1528 let mut driver = Driver::new(
1529 RecordingFold::default(),
1530 Probe::new(chain),
1531 engine_config(2),
1532 DriverConfig::default(),
1533 )
1534 .unwrap();
1535 driver.tick();
1536 driver.checkpoint();
1537 for _ in 0..5 {
1538 driver.tick();
1539 }
1540 driver.source_mut().inner.reorg(3, &[&[40], &[50], &[60]]);
1541 driver.source_mut().fail_next_probes(1);
1542 let first = driver.tick();
1544 assert_eq!(first, Tick::SourceError);
1546 let second = driver.tick();
1547 assert!(matches!(second, Tick::RolledBack { .. }));
1548 run_to_idle(&mut driver);
1549 let expected = vec![
1550 (Position::new(1, 0), 1),
1551 (Position::new(2, 0), 2),
1552 (Position::new(3, 0), 3),
1553 (Position::new(4, 0), 40),
1554 (Position::new(5, 0), 50),
1555 (Position::new(6, 0), 60),
1556 ];
1557 assert_eq!(driver.engine().fold().applied, expected);
1558 }
1559}