1use crate::checkpoint::{
23 checkpoint_to_cose, verify_checkpoint_cose_offline, CheckpointError, CheckpointRecord,
24 CheckpointSigner,
25};
26use crate::log::{
27 validate_identifier, CheckpointState, CheckpointStore, Error, ErrorCode, Result, State,
28 Timestamp, WitnessState, DEFAULT_SCAN_LIMIT, MAX_SCAN_LIMIT, MAX_WITNESSES,
29};
30use crate::mmr::{self, Hash, MemoryNodeStore, NodeAppender, NodeReader};
31use std::collections::HashSet;
32use std::sync::atomic::{AtomicBool, Ordering};
33use std::sync::{Arc, Condvar, Mutex};
34use std::time::Duration;
35
36#[derive(Clone, Debug, PartialEq, Eq)]
39pub struct RunnerConfig {
40 pub log_id: String,
42 pub cadence_entries: u64,
44 pub cadence_age: Duration,
46 pub scan_limit: usize,
48 pub poll_interval: Duration,
51 pub witness_ids: Vec<String>,
54}
55
56impl RunnerConfig {
57 pub fn new(log_id: impl Into<String>) -> Self {
61 Self {
62 log_id: log_id.into(),
63 cadence_entries: 100,
64 cadence_age: Duration::from_secs(15 * 60),
65 scan_limit: DEFAULT_SCAN_LIMIT,
66 poll_interval: Duration::from_secs(60),
67 witness_ids: Vec::new(),
68 }
69 }
70
71 fn validate(&self) -> Result<()> {
72 if validate_identifier(&self.log_id).is_err()
73 || !(1..=MAX_SCAN_LIMIT).contains(&self.scan_limit)
74 || self.poll_interval.is_zero()
75 {
76 return Err(Error::invalid("invalid runner configuration"));
77 }
78 if self.cadence_entries == 0 || self.cadence_age.is_zero() {
79 return Err(Error::invalid("invalid checkpoint cadence"));
80 }
81 if self.witness_ids.len() > MAX_WITNESSES {
82 return Err(Error::invalid("too many witnesses"));
83 }
84 let mut seen = HashSet::with_capacity(self.witness_ids.len());
85 for id in &self.witness_ids {
86 if validate_identifier(id).is_err() {
87 return Err(Error::invalid("invalid witness ID"));
88 }
89 if !seen.insert(id.as_str()) {
90 return Err(Error::invalid("duplicate witness ID"));
91 }
92 }
93 Ok(())
94 }
95
96 fn due(&self, entries: u64, first: Timestamp, now: Timestamp) -> bool {
99 if entries == 0 {
100 return false;
101 }
102 if entries >= self.cadence_entries {
103 return true;
104 }
105 let waited = i128::from(now.unix_millis()) - i128::from(first.unix_millis());
106 waited >= 0
107 && Duration::from_millis(u64::try_from(waited).unwrap_or(u64::MAX)) >= self.cadence_age
108 }
109}
110
111pub fn checkpoint_timestamp(time: Timestamp) -> Result<String> {
117 let text = time.to_iso_string()?;
118 if text.starts_with('+') || text.starts_with('-') {
119 return Err(Error::invalid(
120 "checkpoint timestamp year must be 0000..=9999",
121 ));
122 }
123 let (whole, fraction) = text[..text.len() - 1]
125 .split_once('.')
126 .ok_or_else(|| Error::invalid("unexpected time form"))?;
127 let fraction = fraction.trim_end_matches('0');
128 Ok(if fraction.is_empty() {
129 format!("{whole}Z")
130 } else {
131 format!("{whole}.{fraction}Z")
132 })
133}
134
135#[derive(Debug, Default)]
139pub(crate) struct Wake {
140 pending: Mutex<bool>,
141 signal: Condvar,
142}
143
144impl Wake {
145 pub(crate) fn notify(&self) {
146 *self.pending.lock().unwrap_or_else(|e| e.into_inner()) = true;
147 self.signal.notify_all();
148 }
149
150 pub(crate) fn wait(&self, timeout: Duration) {
153 let pending = self.pending.lock().unwrap_or_else(|e| e.into_inner());
154 let (mut pending, _) = self
155 .signal
156 .wait_timeout_while(pending, timeout, |pending| !*pending)
157 .unwrap_or_else(|e| e.into_inner());
158 *pending = false;
159 }
160}
161
162struct Loaded {
164 state: State,
165 nodes: MemoryNodeStore,
166}
167
168pub struct CheckpointRunner<B: ?Sized, S> {
177 config: RunnerConfig,
178 store: Arc<B>,
179 signer: S,
180 lock: Mutex<()>,
181 wake: Wake,
182}
183
184impl<B, S> CheckpointRunner<B, S>
185where
186 B: CheckpointStore + ?Sized,
187 S: CheckpointSigner,
188{
189 pub fn new(config: RunnerConfig, store: Arc<B>, signer: S) -> Result<Self> {
193 config.validate()?;
194 Ok(Self {
195 config,
196 store,
197 signer,
198 lock: Mutex::new(()),
199 wake: Wake::default(),
200 })
201 }
202
203 pub fn config(&self) -> &RunnerConfig {
205 &self.config
206 }
207
208 pub fn notify(&self) {
210 self.wake.notify();
211 }
212
213 pub fn run(&self, stop: &AtomicBool) -> Result<()> {
218 while !stop.load(Ordering::SeqCst) {
219 match self.catch_up_until(Timestamp::now(), Some(stop)) {
220 Err(error) if error.code() != ErrorCode::Contention => return Err(error),
221 _ => {}
222 }
223 if stop.load(Ordering::SeqCst) {
224 break;
225 }
226 self.wake.wait(self.config.poll_interval);
227 }
228 Ok(())
229 }
230
231 pub fn run_once(&self, now: Timestamp) -> Result<bool> {
237 let _guard = self.lock.lock().unwrap_or_else(|e| e.into_inner());
238 let now = runner_time(now)?;
239 let mut loaded = self.load()?;
240 self.advance(now, &mut loaded)
241 }
242
243 pub fn catch_up(&self, now: Timestamp) -> Result<()> {
248 self.catch_up_until(now, None)
249 }
250
251 fn catch_up_until(&self, now: Timestamp, stop: Option<&AtomicBool>) -> Result<()> {
255 let _guard = self.lock.lock().unwrap_or_else(|e| e.into_inner());
256 let now = runner_time(now)?;
257 let mut loaded = self.load()?;
258 let mut reloaded = false;
259 loop {
260 match self.advance(now, &mut loaded) {
261 Err(error) if error.code() == ErrorCode::Contention && !reloaded => {
262 loaded = self.load()?;
263 reloaded = true;
264 }
265 Err(error) => return Err(error),
266 Ok(false) => return Ok(()),
267 Ok(true) if stop.is_some_and(|stop| stop.load(Ordering::SeqCst)) => return Ok(()),
268 Ok(true) => reloaded = false,
269 }
270 }
271 }
272
273 fn load(&self) -> Result<Loaded> {
274 let state = self.store.load_cll()?;
275 if state.size != state.nodes.len() as u64 {
276 return Err(Error::corrupt("MMR size does not match indexed sequence"));
277 }
278 let leaves = mmr::leaf_count(state.size)
279 .map_err(|_| Error::corrupt("MMR size is not a complete MMR"))?;
280 if leaves != state.indexed_seq {
281 return Err(Error::corrupt("MMR size does not match indexed sequence"));
282 }
283 let mut nodes = MemoryNodeStore::new();
284 let hashes = state
285 .nodes
286 .iter()
287 .map(|node| to_hash(node).ok_or_else(|| Error::corrupt("MMR node is not 32 bytes")))
288 .collect::<Result<Vec<_>>>()?;
289 nodes.append_nodes(&hashes);
290 self.verify_checkpoint(&state, &nodes)?;
291 Ok(Loaded { state, nodes })
292 }
293
294 fn verify_checkpoint(&self, state: &State, nodes: &MemoryNodeStore) -> Result<()> {
295 let Some(checkpoint) = &state.checkpoint else {
296 return Ok(());
297 };
298 let position = || Error::corrupt("invalid stored checkpoint position");
299 let leaves = mmr::leaf_count(checkpoint.size).map_err(|_| position())?;
300 if checkpoint.size > state.size
301 || leaves != checkpoint.indexed_seq
302 || checkpoint.indexed_seq > state.indexed_seq
303 {
304 return Err(position());
305 }
306 let expected = peak_hashes(nodes, checkpoint.size)?;
307 if !same_peaks(&expected, &checkpoint.peaks) {
308 return Err(Error::corrupt("stored checkpoint peaks mismatch"));
309 }
310 self.check_statement(&checkpoint.bytes, checkpoint.size, &expected)
311 .map_err(|reason| Error::corrupt(format!("stored checkpoint {reason}")))
312 }
313
314 fn check_statement(
317 &self,
318 statement: &[u8],
319 size: u64,
320 peaks: &[Hash],
321 ) -> std::result::Result<(), &'static str> {
322 let verified = verify_checkpoint_cose_offline(statement);
323 let decoded = match verified.decoded {
324 Some(decoded) if verified.ok => decoded,
325 _ => return Err("signature mismatch"),
326 };
327 if decoded.log_id != self.config.log_id
328 || decoded.key_id != self.signer.key_id()
329 || decoded.mmr_size != size
330 || decoded.new_peak_hashes != peaks
331 || decoded.root != hex::encode(mmr::root_from_peaks(peaks))
332 {
333 return Err("metadata mismatch");
334 }
335 Ok(())
336 }
337
338 fn advance(&self, now: Timestamp, loaded: &mut Loaded) -> Result<bool> {
339 let state = &loaded.state;
340 let nodes = &mut loaded.nodes;
341 let entries = self
342 .store
343 .scan_entries(state.indexed_seq, self.config.scan_limit)?;
344 let mut first_pending = state.first_pending_at;
345 let mut indexed = state.indexed_seq;
346 let mut new_nodes: Vec<Vec<u8>> = Vec::new();
347 for entry in &entries {
348 if Some(entry.seq) != indexed.checked_add(1) || !entry.appended_at.is_portable() {
349 return Err(Error::corrupt("CLL entry sequence or time is invalid"));
350 }
351 let value = to_hash(&entry.value)
352 .ok_or_else(|| Error::corrupt("CLL entry value is not 32 bytes"))?;
353 let (_, appended) = mmr::add_leaf(nodes, mmr::leaf_hash(&value))
354 .map_err(|e| Error::with_source(ErrorCode::Corrupt, "append MMR entry", e))?;
355 new_nodes.extend(appended.iter().map(|node| node.to_vec()));
356 indexed = entry.seq;
357 first_pending.get_or_insert(entry.appended_at);
358 }
359 let last_checkpointed = state.checkpoint.as_ref().map_or(0, |c| c.indexed_seq);
360 let uncheckpointed = indexed - last_checkpointed;
361 let due = first_pending.is_some_and(|first| self.config.due(uncheckpointed, first, now));
362 let size = nodes.size();
363 let mut next = State {
364 size,
365 nodes: state.nodes.iter().cloned().chain(new_nodes).collect(),
366 indexed_seq: indexed,
367 first_pending_at: first_pending,
368 checkpoint: state.checkpoint.clone(),
369 witnesses: state.witnesses.clone(),
370 };
371 if due {
372 let statement = self.sign(now, state.checkpoint.as_ref(), nodes, size)?;
373 let peaks = peak_hashes(nodes, size)?;
374 next.checkpoint = Some(CheckpointState {
375 bytes: statement.clone(),
376 size,
377 indexed_seq: indexed,
378 peaks: peaks.iter().map(|peak| peak.to_vec()).collect(),
379 });
380 next.first_pending_at = None;
381 next.witnesses
382 .extend(self.config.witness_ids.iter().map(|id| WitnessState {
383 witness_id: id.clone(),
384 checkpoint_size: size,
385 checkpoint: statement.clone(),
386 attempts: 0,
387 next_attempt_at: now,
388 receipt: None,
389 permanent: false,
390 last_error: None,
391 }));
392 }
393 if entries.is_empty() && !due {
394 return Ok(false);
395 }
396 let expected = state.checkpoint.as_ref().map(|c| c.bytes.as_slice());
397 self.store.commit_cll(state.size, expected, &next)?;
398 loaded.state = next;
399 Ok(true)
400 }
401
402 fn sign(
404 &self,
405 now: Timestamp,
406 previous: Option<&CheckpointState>,
407 nodes: &MemoryNodeStore,
408 size: u64,
409 ) -> Result<Vec<u8>> {
410 let peaks = peak_hashes(nodes, size)?;
411 let (prev_size, prev_peaks, proof) = match previous {
412 None => (0, None, None),
413 Some(previous) => {
414 let proof = mmr::consistency_proof(nodes, previous.size, size)
415 .map_err(|e| Error::with_source(ErrorCode::Corrupt, "consistency proof", e))?;
416 (
417 previous.size,
418 Some(peak_hashes(nodes, previous.size)?),
419 Some(proof),
420 )
421 }
422 };
423 let record = CheckpointRecord {
424 v: 1,
425 kind: "mmr_checkpoint".to_string(),
426 log_id: self.config.log_id.clone(),
427 mmr_size: size,
428 root: hex::encode(mmr::root_from_peaks(&peaks)),
429 prev_size,
430 prev_root: prev_peaks
431 .as_deref()
432 .map_or_else(String::new, |p| hex::encode(mmr::root_from_peaks(p))),
433 key_id: self.signer.key_id(),
434 timestamp: checkpoint_timestamp(now)?,
435 signature: String::new(),
436 witnesses: Vec::new(),
437 };
438 let statement = checkpoint_to_cose(
439 &record,
440 &self.signer,
441 &peaks,
442 prev_peaks.as_deref(),
443 proof.as_ref(),
444 None,
445 )
446 .map_err(sign_error)?;
447 self.check_statement(&statement, size, &peaks)
449 .map_err(|reason| Error::invalid(format!("newly signed checkpoint {reason}")))?;
450 Ok(statement)
451 }
452}
453
454fn sign_error(error: CheckpointError) -> Error {
458 let code = match error {
459 CheckpointError::Signer(_) => ErrorCode::Storage,
460 _ => ErrorCode::Invalid,
461 };
462 Error::with_source(code, "sign checkpoint", error)
463}
464
465fn runner_time(now: Timestamp) -> Result<Timestamp> {
466 if now.is_portable() {
467 Ok(now)
468 } else {
469 Err(Error::invalid("invalid runner time"))
470 }
471}
472
473fn to_hash(bytes: &[u8]) -> Option<Hash> {
474 bytes.try_into().ok()
475}
476
477fn peak_hashes(nodes: &MemoryNodeStore, size: u64) -> Result<Vec<Hash>> {
478 let positions = mmr::peaks(size).map_err(|_| Error::corrupt("invalid MMR size"))?;
479 if size > nodes.size() {
480 return Err(Error::corrupt("MMR is smaller than the requested size"));
481 }
482 Ok(positions.into_iter().map(|p| nodes.node(p)).collect())
483}
484
485fn same_peaks(expected: &[Hash], stored: &[Vec<u8>]) -> bool {
486 expected.len() == stored.len()
487 && expected
488 .iter()
489 .zip(stored)
490 .all(|(left, right)| left.as_slice() == right.as_slice())
491}
492
493#[cfg(test)]
494mod tests {
495 use super::*;
496 use crate::log::memory::MemoryStore;
497 use crate::log::{
498 AppendInput, CheckpointStateStore, Entry, EntrySource, EntryStore, WitnessStateStore,
499 };
500 use ed25519_dalek::SigningKey;
501 use std::sync::atomic::AtomicUsize;
502
503 fn at(text: &str) -> Timestamp {
504 Timestamp::parse(text).unwrap()
505 }
506
507 fn key() -> SigningKey {
508 SigningKey::from_bytes(&[7; 32])
509 }
510
511 fn config() -> RunnerConfig {
512 RunnerConfig {
513 cadence_entries: 2,
514 witness_ids: vec!["w-a".into()],
515 ..RunnerConfig::new("unit-log")
516 }
517 }
518
519 fn append(store: &MemoryStore, byte: u8, time: &str) {
520 store
521 .append(&AppendInput {
522 value: vec![byte; 32],
523 appended_at: at(time),
524 })
525 .unwrap();
526 }
527
528 #[test]
529 fn checkpoint_timestamps_drop_trailing_fraction_zeros() {
530 for (text, expected) in [
531 ("2026-08-27T12:34:56.000Z", "2026-08-27T12:34:56Z"),
532 ("2026-08-27T12:34:56.500Z", "2026-08-27T12:34:56.5Z"),
533 ("2026-08-27T12:34:56.120Z", "2026-08-27T12:34:56.12Z"),
534 ("2026-08-27T12:34:56.123Z", "2026-08-27T12:34:56.123Z"),
535 ("0000-01-01T00:00:00.000Z", "0000-01-01T00:00:00Z"),
536 ] {
537 assert_eq!(checkpoint_timestamp(at(text)).unwrap(), expected);
538 }
539 for text in ["+010000-01-01T00:00:00.000Z", "-000001-01-01T00:00:00.000Z"] {
540 assert_eq!(
541 checkpoint_timestamp(at(text)).unwrap_err().code(),
542 ErrorCode::Invalid
543 );
544 }
545 }
546
547 #[test]
548 fn configuration_is_validated() {
549 let store = Arc::new(MemoryStore::new());
550 let cases: Vec<(RunnerConfig, &str)> = vec![
551 (RunnerConfig::new("bad id"), "invalid runner configuration"),
552 (
553 RunnerConfig {
554 scan_limit: 0,
555 ..config()
556 },
557 "invalid runner configuration",
558 ),
559 (
560 RunnerConfig {
561 scan_limit: MAX_SCAN_LIMIT + 1,
562 ..config()
563 },
564 "invalid runner configuration",
565 ),
566 (
567 RunnerConfig {
568 poll_interval: Duration::ZERO,
569 ..config()
570 },
571 "invalid runner configuration",
572 ),
573 (
574 RunnerConfig {
575 cadence_entries: 0,
576 ..config()
577 },
578 "invalid checkpoint cadence",
579 ),
580 (
581 RunnerConfig {
582 cadence_age: Duration::ZERO,
583 ..config()
584 },
585 "invalid checkpoint cadence",
586 ),
587 (
588 RunnerConfig {
589 witness_ids: (0..=MAX_WITNESSES).map(|i| format!("w-{i}")).collect(),
590 ..config()
591 },
592 "too many witnesses",
593 ),
594 (
595 RunnerConfig {
596 witness_ids: vec!["bad witness".into()],
597 ..config()
598 },
599 "invalid witness ID",
600 ),
601 (
602 RunnerConfig {
603 witness_ids: vec!["w-a".into(), "w-a".into()],
604 ..config()
605 },
606 "duplicate witness ID",
607 ),
608 ];
609 for (config, message) in cases {
610 let error = CheckpointRunner::new(config, store.clone(), key())
611 .err()
612 .expect("configuration must be rejected");
613 assert_eq!(error.code(), ErrorCode::Invalid);
614 assert_eq!(error.message(), message);
615 }
616 let full = RunnerConfig {
617 witness_ids: (0..MAX_WITNESSES).map(|i| format!("w-{i}")).collect(),
618 scan_limit: MAX_SCAN_LIMIT,
619 ..config()
620 };
621 CheckpointRunner::new(full, store, key()).unwrap();
622 }
623
624 #[test]
625 fn invalid_times_are_rejected() {
626 let runner = CheckpointRunner::new(config(), Arc::new(MemoryStore::new()), key()).unwrap();
627 let late = Timestamp::from_unix_millis(crate::log::MAX_PORTABLE_UNIX_MILLIS + 1);
628 assert_eq!(
629 runner.run_once(late).unwrap_err().code(),
630 ErrorCode::Invalid
631 );
632 assert_eq!(
633 runner.catch_up(late).unwrap_err().code(),
634 ErrorCode::Invalid
635 );
636 }
637
638 #[test]
639 fn indexes_signs_and_queues_witnesses() {
640 let store = Arc::new(MemoryStore::new());
641 let runner = CheckpointRunner::new(config(), store.clone(), key()).unwrap();
642 assert!(!runner.run_once(at("2026-10-01T12:00:00.000Z")).unwrap());
643 append(&store, 1, "2026-10-01T12:00:00.000Z");
644 assert!(runner.run_once(at("2026-10-01T12:00:01.000Z")).unwrap());
645 let state = store.load_cll().unwrap();
646 assert_eq!((state.size, state.indexed_seq), (1, 1));
647 assert!(state.checkpoint.is_none());
648 assert_eq!(state.first_pending_at, Some(at("2026-10-01T12:00:00.000Z")));
649 assert!(!runner.run_once(at("2026-10-01T12:00:02.000Z")).unwrap());
651 assert!(runner.run_once(at("2026-10-01T12:15:00.000Z")).unwrap());
653 let state = store.load_cll().unwrap();
654 let checkpoint = state.checkpoint.clone().unwrap();
655 assert_eq!((checkpoint.size, checkpoint.indexed_seq), (1, 1));
656 assert!(state.first_pending_at.is_none());
657 assert_eq!(state.witnesses.len(), 1);
658 assert_eq!(
659 state.witnesses[0].next_attempt_at,
660 at("2026-10-01T12:15:00.000Z")
661 );
662 append(&store, 2, "2026-10-01T12:16:00.000Z");
664 append(&store, 3, "2026-10-01T12:16:01.000Z");
665 runner.catch_up(at("2026-10-01T12:17:00.000Z")).unwrap();
666 let state = store.load_cll().unwrap();
667 let decoded = verify_checkpoint_cose_offline(&state.checkpoint.unwrap().bytes);
668 assert!(decoded.ok, "{:?}", decoded.errors);
669 let decoded = decoded.decoded.unwrap();
670 assert_eq!((decoded.prev_size, decoded.mmr_size), (1, 4));
671 assert_eq!(decoded.timestamp, "2026-10-01T12:17:00Z");
672 assert!(decoded.consistency_proof.is_some());
673 assert_eq!(state.witnesses.len(), 2);
674 }
675
676 #[test]
677 fn catch_up_runs_one_batch_per_pass_until_idle() {
678 let store = Arc::new(MemoryStore::new());
679 let config = RunnerConfig {
680 scan_limit: 2,
681 cadence_entries: 3,
682 ..config()
683 };
684 let runner = CheckpointRunner::new(config, store.clone(), key()).unwrap();
685 for byte in 1..=5 {
686 append(&store, byte, "2026-10-01T12:00:00.000Z");
687 }
688 assert!(runner.run_once(at("2026-10-01T12:00:01.000Z")).unwrap());
689 assert_eq!(store.load_cll().unwrap().indexed_seq, 2);
690 runner.catch_up(at("2026-10-01T12:00:01.000Z")).unwrap();
691 let state = store.load_cll().unwrap();
692 assert_eq!(state.indexed_seq, 5);
693 assert_eq!(state.checkpoint.unwrap().indexed_seq, 4);
694 }
695
696 fn checkpointed_store() -> Arc<MemoryStore> {
697 let store = Arc::new(MemoryStore::new());
698 let runner = CheckpointRunner::new(config(), store.clone(), key()).unwrap();
699 append(&store, 1, "2026-10-01T12:00:00.000Z");
700 append(&store, 2, "2026-10-01T12:00:01.000Z");
701 assert!(runner.run_once(at("2026-10-01T12:00:02.000Z")).unwrap());
702 assert!(store.load_cll().unwrap().checkpoint.is_some());
703 store
704 }
705
706 struct Tampered {
709 entries: MemoryStore,
710 state: State,
711 }
712
713 impl EntrySource for Tampered {
714 fn scan_entries(&self, after_seq: u64, limit: usize) -> Result<Vec<Entry>> {
715 self.entries.scan_entries(after_seq, limit)
716 }
717 }
718
719 impl CheckpointStateStore for Tampered {
720 fn load_cll(&self) -> Result<State> {
721 Ok(self.state.clone())
722 }
723
724 fn commit_cll(&self, _: u64, _: Option<&[u8]>, _: &State) -> Result<()> {
725 panic!("a corrupt state must not be committed over");
726 }
727 }
728
729 impl WitnessStateStore for Tampered {
730 fn pending_witnesses_for(
731 &self,
732 _: &str,
733 _: Timestamp,
734 _: usize,
735 ) -> Result<Vec<WitnessState>> {
736 Ok(Vec::new())
737 }
738
739 fn pending_witnesses(&self, _: Timestamp, _: usize) -> Result<Vec<WitnessState>> {
740 Ok(Vec::new())
741 }
742
743 fn get_witness(&self, _: &str, _: u64) -> Result<Option<WitnessState>> {
744 Ok(None)
745 }
746
747 fn commit_witness(&self, _: u32, _: &WitnessState) -> Result<()> {
748 Ok(())
749 }
750 }
751
752 fn assert_corrupt(state: State, config: RunnerConfig, signer: SigningKey) {
753 let store = Arc::new(Tampered {
754 entries: MemoryStore::new(),
755 state,
756 });
757 let runner = CheckpointRunner::new(config, store, signer).unwrap();
758 let error = runner.run_once(at("2026-10-01T12:01:00.000Z")).unwrap_err();
759 assert_eq!(error.code(), ErrorCode::Corrupt, "{error}");
760 }
761
762 #[test]
763 fn corrupt_stored_state_is_refused() {
764 let good = checkpointed_store().load_cll().unwrap();
765
766 let mut bad = good.clone();
767 bad.size = 2;
768 assert_corrupt(bad, config(), key());
769
770 let mut bad = good.clone();
771 bad.nodes.pop();
772 assert_corrupt(bad, config(), key());
773
774 let mut bad = good.clone();
775 bad.indexed_seq = 1;
776 assert_corrupt(bad, config(), key());
777
778 let mut bad = good.clone();
780 bad.nodes[2][0] ^= 1;
781 assert_corrupt(bad, config(), key());
782
783 let mut bad = good.clone();
784 bad.checkpoint.as_mut().unwrap().indexed_seq = 1;
785 assert_corrupt(bad, config(), key());
786
787 let mut bad = good.clone();
788 bad.checkpoint.as_mut().unwrap().peaks[0][0] ^= 1;
789 assert_corrupt(bad, config(), key());
790
791 let mut bad = good.clone();
793 let bytes = &mut bad.checkpoint.as_mut().unwrap().bytes;
794 let last = bytes.len() - 1;
795 bytes[last] ^= 1;
796 assert_corrupt(bad, config(), key());
797
798 assert_corrupt(good.clone(), RunnerConfig::new("other-log"), key());
800 assert_corrupt(good, config(), SigningKey::from_bytes(&[8; 32]));
801 }
802
803 #[test]
804 fn a_gap_in_the_entry_log_is_corrupt() {
805 struct Gappy(MemoryStore);
806 impl EntrySource for Gappy {
807 fn scan_entries(&self, after_seq: u64, limit: usize) -> Result<Vec<Entry>> {
808 let mut entries = self.0.scan_entries(after_seq, limit)?;
809 entries.iter_mut().for_each(|e| e.seq += 1);
810 Ok(entries)
811 }
812 }
813 impl CheckpointStateStore for Gappy {
814 fn load_cll(&self) -> Result<State> {
815 self.0.load_cll()
816 }
817 fn commit_cll(&self, s: u64, c: Option<&[u8]>, n: &State) -> Result<()> {
818 self.0.commit_cll(s, c, n)
819 }
820 }
821 impl WitnessStateStore for Gappy {
822 fn pending_witnesses(&self, n: Timestamp, l: usize) -> Result<Vec<WitnessState>> {
823 self.0.pending_witnesses(n, l)
824 }
825 fn pending_witnesses_for(
826 &self,
827 w: &str,
828 n: Timestamp,
829 l: usize,
830 ) -> Result<Vec<WitnessState>> {
831 self.0.pending_witnesses_for(w, n, l)
832 }
833 fn get_witness(&self, w: &str, s: u64) -> Result<Option<WitnessState>> {
834 self.0.get_witness(w, s)
835 }
836 fn commit_witness(&self, a: u32, n: &WitnessState) -> Result<()> {
837 self.0.commit_witness(a, n)
838 }
839 }
840 let inner = MemoryStore::new();
841 append(&inner, 1, "2026-10-01T12:00:00.000Z");
842 let runner = CheckpointRunner::new(config(), Arc::new(Gappy(inner)), key()).unwrap();
843 let error = runner.run_once(at("2026-10-01T12:00:01.000Z")).unwrap_err();
844 assert_eq!(error.code(), ErrorCode::Corrupt);
845 }
846
847 struct Racing {
850 inner: Arc<MemoryStore>,
851 fail: AtomicUsize,
852 loads: AtomicUsize,
853 }
854
855 impl EntrySource for Racing {
856 fn scan_entries(&self, after_seq: u64, limit: usize) -> Result<Vec<Entry>> {
857 self.inner.scan_entries(after_seq, limit)
858 }
859 }
860
861 impl CheckpointStateStore for Racing {
862 fn load_cll(&self) -> Result<State> {
863 self.loads.fetch_add(1, Ordering::SeqCst);
864 self.inner.load_cll()
865 }
866
867 fn commit_cll(&self, size: u64, checkpoint: Option<&[u8]>, next: &State) -> Result<()> {
868 let racing = self.fail.load(Ordering::SeqCst) > 0;
871 if racing {
872 self.fail.fetch_sub(1, Ordering::SeqCst);
873 }
874 if racing {
875 let rival = CheckpointRunner::new(
877 RunnerConfig {
878 scan_limit: 1,
879 ..config()
880 },
881 self.inner.clone(),
882 key(),
883 )
884 .unwrap();
885 rival.run_once(at("2026-10-01T12:00:05.000Z")).unwrap();
886 }
887 self.inner.commit_cll(size, checkpoint, next)
888 }
889 }
890
891 impl WitnessStateStore for Racing {
892 fn pending_witnesses(&self, n: Timestamp, l: usize) -> Result<Vec<WitnessState>> {
893 self.inner.pending_witnesses(n, l)
894 }
895 fn pending_witnesses_for(
896 &self,
897 w: &str,
898 n: Timestamp,
899 l: usize,
900 ) -> Result<Vec<WitnessState>> {
901 self.inner.pending_witnesses_for(w, n, l)
902 }
903 fn get_witness(&self, w: &str, s: u64) -> Result<Option<WitnessState>> {
904 self.inner.get_witness(w, s)
905 }
906 fn commit_witness(&self, a: u32, n: &WitnessState) -> Result<()> {
907 self.inner.commit_witness(a, n)
908 }
909 }
910
911 #[test]
912 fn catch_up_reloads_once_after_contention() {
913 let inner = Arc::new(MemoryStore::new());
914 append(&inner, 1, "2026-10-01T12:00:00.000Z");
915 append(&inner, 2, "2026-10-01T12:00:01.000Z");
916 append(&inner, 3, "2026-10-01T12:00:02.000Z");
917 let store = Arc::new(Racing {
918 inner: inner.clone(),
919 fail: AtomicUsize::new(1),
920 loads: AtomicUsize::new(0),
921 });
922 let runner = CheckpointRunner::new(config(), store.clone(), key()).unwrap();
923 runner.catch_up(at("2026-10-01T12:00:10.000Z")).unwrap();
924 assert_eq!(store.loads.load(Ordering::SeqCst), 2);
925 let state = inner.load_cll().unwrap();
926 assert_eq!(state.indexed_seq, 3);
927 assert_eq!(state.checkpoint.unwrap().indexed_seq, 3);
928
929 let inner = Arc::new(MemoryStore::new());
931 append(&inner, 1, "2026-10-01T12:00:00.000Z");
932 let store = Arc::new(Racing {
933 inner,
934 fail: AtomicUsize::new(1),
935 loads: AtomicUsize::new(0),
936 });
937 let runner = CheckpointRunner::new(config(), store, key()).unwrap();
938 let error = runner.run_once(at("2026-10-01T12:00:10.000Z")).unwrap_err();
939 assert_eq!(error.code(), ErrorCode::Contention);
940 }
941
942 #[test]
943 fn catch_up_returns_contention_right_after_a_reload() {
944 let inner = Arc::new(MemoryStore::new());
945 for byte in 1..=4 {
946 append(&inner, byte, "2026-10-01T12:00:00.000Z");
947 }
948 let store = Arc::new(Racing {
949 inner,
950 fail: AtomicUsize::new(2),
951 loads: AtomicUsize::new(0),
952 });
953 let runner = CheckpointRunner::new(config(), store.clone(), key()).unwrap();
954 let error = runner.catch_up(at("2026-10-01T12:00:10.000Z")).unwrap_err();
955 assert_eq!(error.code(), ErrorCode::Contention);
956 assert_eq!(store.loads.load(Ordering::SeqCst), 2);
957 }
958
959 #[test]
960 fn catch_up_stops_between_batches_once_asked() {
961 let store = Arc::new(MemoryStore::new());
962 for byte in 1..=5 {
963 append(&store, byte, "2026-10-01T12:00:00.000Z");
964 }
965 let config = RunnerConfig {
966 scan_limit: 2,
967 ..config()
968 };
969 let runner = CheckpointRunner::new(config, store.clone(), key()).unwrap();
970 let stop = AtomicBool::new(true);
971 runner
972 .catch_up_until(at("2026-10-01T12:00:01.000Z"), Some(&stop))
973 .unwrap();
974 assert_eq!(store.load_cll().unwrap().indexed_seq, 2);
975 }
976
977 #[test]
978 fn run_stops_when_asked() {
979 let store = Arc::new(MemoryStore::new());
980 append(&store, 1, "2026-10-01T12:00:00.000Z");
981 let config = RunnerConfig {
982 poll_interval: Duration::from_secs(3600),
983 ..config()
984 };
985 let runner = CheckpointRunner::new(config, store.clone(), key()).unwrap();
986 let stop = AtomicBool::new(false);
987 std::thread::scope(|scope| {
988 let handle = scope.spawn(|| runner.run(&stop));
989 while store.load_cll().unwrap().indexed_seq == 0 {
990 std::thread::yield_now();
991 }
992 stop.store(true, Ordering::SeqCst);
993 runner.notify();
994 handle.join().unwrap().unwrap();
995 });
996 }
997}