Skip to main content

cll/
runner.rs

1//! The checkpoint runner: indexes a [`CheckpointStore`]'s entries into its
2//! MMR and signs a COSE checkpoint when the cadence says one is due -- a
3//! port of Go's `checkpoint.Runner` (`go/checkpoint/runner.go`).
4//!
5//! A pass ([`CheckpointRunner::run_once`]) loads the durable state, checks
6//! that its MMR and stored checkpoint are consistent and that the checkpoint
7//! verifies under this runner's log and key, scans one batch of
8//! `scan_limit` entries after the indexed sequence, appends their leaves,
9//! and commits the result with [`CheckpointStateStore::commit_cll`]. When a
10//! checkpoint is due it is signed in the same commit, and one
11//! [`WitnessState`] row per configured witness is queued for a
12//! [`DeliveryRunner`](crate::witness_delivery::DeliveryRunner).
13//!
14//! A checkpoint is due when entries are uncheckpointed and either
15//! `cadence_entries` of them are, or the oldest has waited `cadence_age`.
16//! Checkpoint statements are byte-identical to Go's and TypeScript's for the
17//! same key, log, MMR and time; the shared runner vectors
18//! (`runner-conformance-vectors/`) pin them.
19//!
20//! [`CheckpointStateStore::commit_cll`]: crate::log::CheckpointStateStore::commit_cll
21
22use crate::checkpoint::{
23    checkpoint_to_cose, verify_checkpoint_cose_offline, CheckpointError, CheckpointRecord,
24    CheckpointSigner,
25};
26use crate::log::{
27    validate_identifier, CheckpointState, CheckpointStore, Error, ErrorCode, Result, State,
28    Timestamp, WitnessState, DEFAULT_SCAN_LIMIT, MAX_SCAN_LIMIT, MAX_WITNESSES,
29};
30use crate::mmr::{self, Hash, MemoryNodeStore, NodeAppender, NodeReader};
31use std::collections::HashSet;
32use std::sync::atomic::{AtomicBool, Ordering};
33use std::sync::{Arc, Condvar, Mutex};
34use std::time::Duration;
35
36/// Configuration for a [`CheckpointRunner`] (Go `RunnerConfig` with its
37/// `Cadence`).
38#[derive(Clone, Debug, PartialEq, Eq)]
39pub struct RunnerConfig {
40    /// The checkpoint issuer: the CWT `iss` claim of every statement.
41    pub log_id: String,
42    /// Sign once this many entries are uncheckpointed (at least 1).
43    pub cadence_entries: u64,
44    /// Sign once the oldest uncheckpointed entry is this old (above zero).
45    pub cadence_age: Duration,
46    /// Entries indexed per pass, 1..=[`MAX_SCAN_LIMIT`].
47    pub scan_limit: usize,
48    /// How long [`CheckpointRunner::run`] waits between passes when it is
49    /// not notified (above zero).
50    pub poll_interval: Duration,
51    /// Witnesses that get one delivery row per checkpoint: at most
52    /// [`MAX_WITNESSES`] distinct identifiers.
53    pub witness_ids: Vec<String>,
54}
55
56impl RunnerConfig {
57    /// The interoperable defaults for `log_id`: 100 entries or 15 minutes,
58    /// [`DEFAULT_SCAN_LIMIT`] entries per pass, a one-minute poll and no
59    /// witnesses.
60    pub fn new(log_id: impl Into<String>) -> Self {
61        Self {
62            log_id: log_id.into(),
63            cadence_entries: 100,
64            cadence_age: Duration::from_secs(15 * 60),
65            scan_limit: DEFAULT_SCAN_LIMIT,
66            poll_interval: Duration::from_secs(60),
67            witness_ids: Vec::new(),
68        }
69    }
70
71    fn validate(&self) -> Result<()> {
72        if validate_identifier(&self.log_id).is_err()
73            || !(1..=MAX_SCAN_LIMIT).contains(&self.scan_limit)
74            || self.poll_interval.is_zero()
75        {
76            return Err(Error::invalid("invalid runner configuration"));
77        }
78        if self.cadence_entries == 0 || self.cadence_age.is_zero() {
79            return Err(Error::invalid("invalid checkpoint cadence"));
80        }
81        if self.witness_ids.len() > MAX_WITNESSES {
82            return Err(Error::invalid("too many witnesses"));
83        }
84        let mut seen = HashSet::with_capacity(self.witness_ids.len());
85        for id in &self.witness_ids {
86            if validate_identifier(id).is_err() {
87                return Err(Error::invalid("invalid witness ID"));
88            }
89            if !seen.insert(id.as_str()) {
90                return Err(Error::invalid("duplicate witness ID"));
91            }
92        }
93        Ok(())
94    }
95
96    /// Go `Config.Due`: whether `entries` uncheckpointed entries, the oldest
97    /// appended at `first`, make a checkpoint due at `now`.
98    fn due(&self, entries: u64, first: Timestamp, now: Timestamp) -> bool {
99        if entries == 0 {
100            return false;
101        }
102        if entries >= self.cadence_entries {
103            return true;
104        }
105        let waited = i128::from(now.unix_millis()) - i128::from(first.unix_millis());
106        waited >= 0
107            && Duration::from_millis(u64::try_from(waited).unwrap_or(u64::MAX)) >= self.cadence_age
108    }
109}
110
111/// The checkpoint timestamp form shared with Go and TypeScript: RFC 3339 in
112/// UTC with the fraction's trailing zeros dropped, for example
113/// `2026-08-27T12:34:56Z` or `2026-08-27T12:34:56.5Z` (Go's
114/// `time.RFC3339Nano`). Unlike stored backend times it has no fixed
115/// millisecond field. A year outside 0000..=9999 is `invalid`.
116pub fn checkpoint_timestamp(time: Timestamp) -> Result<String> {
117    let text = time.to_iso_string()?;
118    if text.starts_with('+') || text.starts_with('-') {
119        return Err(Error::invalid(
120            "checkpoint timestamp year must be 0000..=9999",
121        ));
122    }
123    // `to_iso_string` is always `YYYY-MM-DDTHH:MM:SS.mmmZ` here.
124    let (whole, fraction) = text[..text.len() - 1]
125        .split_once('.')
126        .ok_or_else(|| Error::invalid("unexpected time form"))?;
127    let fraction = fraction.trim_end_matches('0');
128    Ok(if fraction.is_empty() {
129        format!("{whole}Z")
130    } else {
131        format!("{whole}.{fraction}Z")
132    })
133}
134
135/// Wakes a host loop: [`notify`](Wake::notify) never blocks, and a
136/// notification sent while no one waits is kept for the next wait, like Go's
137/// one-slot notify channel.
138#[derive(Debug, Default)]
139pub(crate) struct Wake {
140    pending: Mutex<bool>,
141    signal: Condvar,
142}
143
144impl Wake {
145    pub(crate) fn notify(&self) {
146        *self.pending.lock().unwrap_or_else(|e| e.into_inner()) = true;
147        self.signal.notify_all();
148    }
149
150    /// Waits until notified or `timeout` passes, then clears the
151    /// notification.
152    pub(crate) fn wait(&self, timeout: Duration) {
153        let pending = self.pending.lock().unwrap_or_else(|e| e.into_inner());
154        let (mut pending, _) = self
155            .signal
156            .wait_timeout_while(pending, timeout, |pending| !*pending)
157            .unwrap_or_else(|e| e.into_inner());
158        *pending = false;
159    }
160}
161
162/// The state a pass works from: the stored state and its MMR.
163struct Loaded {
164    state: State,
165    nodes: MemoryNodeStore,
166}
167
168/// Advances durable CLL state when its host runs it (Go `checkpoint.Runner`).
169///
170/// `B` is the store, shared in an [`Arc`] so a
171/// [`DeliveryRunner`](crate::witness_delivery::DeliveryRunner) can use the
172/// same one; `S` signs the checkpoints, and its
173/// [`key_id`](CheckpointSigner::key_id) must be the hex raw Ed25519 public
174/// key, which becomes each statement's `kid`. Passes are serialized: one
175/// runner never runs two at once.
176pub struct CheckpointRunner<B: ?Sized, S> {
177    config: RunnerConfig,
178    store: Arc<B>,
179    signer: S,
180    lock: Mutex<()>,
181    wake: Wake,
182}
183
184impl<B, S> CheckpointRunner<B, S>
185where
186    B: CheckpointStore + ?Sized,
187    S: CheckpointSigner,
188{
189    /// Validates `config` without starting background work: an invalid log
190    /// or witness identifier, scan limit, cadence or poll interval, more
191    /// than [`MAX_WITNESSES`] witnesses or a repeated one is `invalid`.
192    pub fn new(config: RunnerConfig, store: Arc<B>, signer: S) -> Result<Self> {
193        config.validate()?;
194        Ok(Self {
195            config,
196            store,
197            signer,
198            lock: Mutex::new(()),
199            wake: Wake::default(),
200        })
201    }
202
203    /// The configuration.
204    pub fn config(&self) -> &RunnerConfig {
205        &self.config
206    }
207
208    /// Wakes a running [`run`](Self::run) loop without blocking the caller.
209    pub fn notify(&self) {
210        self.wake.notify();
211    }
212
213    /// Catches up until `stop` is set, then waits for
214    /// [`notify`](Self::notify) or the poll interval. Contention is retried
215    /// on the next wakeup; any other error ends the loop. To stop promptly,
216    /// set `stop` and then call [`notify`](Self::notify).
217    pub fn run(&self, stop: &AtomicBool) -> Result<()> {
218        while !stop.load(Ordering::SeqCst) {
219            match self.catch_up_until(Timestamp::now(), Some(stop)) {
220                Err(error) if error.code() != ErrorCode::Contention => return Err(error),
221                _ => {}
222            }
223            if stop.load(Ordering::SeqCst) {
224                break;
225            }
226            self.wake.wait(self.config.poll_interval);
227        }
228        Ok(())
229    }
230
231    /// Advances durable state by at most one entry scan batch at `now`.
232    /// Returns `true` when it commits nodes, a checkpoint, or both. A stored
233    /// state whose MMR, indexed sequence or checkpoint is inconsistent, or
234    /// whose checkpoint does not verify under this log and key, is
235    /// `corrupt`; a lost compare-and-set is `contention`.
236    pub fn run_once(&self, now: Timestamp) -> Result<bool> {
237        let _guard = self.lock.lock().unwrap_or_else(|e| e.into_inner());
238        let now = runner_time(now)?;
239        let mut loaded = self.load()?;
240        self.advance(now, &mut loaded)
241    }
242
243    /// Runs passes until one commits nothing, all at the one time `now`, so
244    /// every checkpoint it signs shares one evaluation time. Contention
245    /// reloads the state once; contention again right after the reload is
246    /// returned.
247    pub fn catch_up(&self, now: Timestamp) -> Result<()> {
248        self.catch_up_until(now, None)
249    }
250
251    /// [`catch_up`](Self::catch_up) that also returns once `stop` is set,
252    /// checked between batches, so a log that grows faster than it is
253    /// indexed cannot keep [`run`](Self::run) from stopping.
254    fn catch_up_until(&self, now: Timestamp, stop: Option<&AtomicBool>) -> Result<()> {
255        let _guard = self.lock.lock().unwrap_or_else(|e| e.into_inner());
256        let now = runner_time(now)?;
257        let mut loaded = self.load()?;
258        let mut reloaded = false;
259        loop {
260            match self.advance(now, &mut loaded) {
261                Err(error) if error.code() == ErrorCode::Contention && !reloaded => {
262                    loaded = self.load()?;
263                    reloaded = true;
264                }
265                Err(error) => return Err(error),
266                Ok(false) => return Ok(()),
267                Ok(true) if stop.is_some_and(|stop| stop.load(Ordering::SeqCst)) => return Ok(()),
268                Ok(true) => reloaded = false,
269            }
270        }
271    }
272
273    fn load(&self) -> Result<Loaded> {
274        let state = self.store.load_cll()?;
275        if state.size != state.nodes.len() as u64 {
276            return Err(Error::corrupt("MMR size does not match indexed sequence"));
277        }
278        let leaves = mmr::leaf_count(state.size)
279            .map_err(|_| Error::corrupt("MMR size is not a complete MMR"))?;
280        if leaves != state.indexed_seq {
281            return Err(Error::corrupt("MMR size does not match indexed sequence"));
282        }
283        let mut nodes = MemoryNodeStore::new();
284        let hashes = state
285            .nodes
286            .iter()
287            .map(|node| to_hash(node).ok_or_else(|| Error::corrupt("MMR node is not 32 bytes")))
288            .collect::<Result<Vec<_>>>()?;
289        nodes.append_nodes(&hashes);
290        self.verify_checkpoint(&state, &nodes)?;
291        Ok(Loaded { state, nodes })
292    }
293
294    fn verify_checkpoint(&self, state: &State, nodes: &MemoryNodeStore) -> Result<()> {
295        let Some(checkpoint) = &state.checkpoint else {
296            return Ok(());
297        };
298        let position = || Error::corrupt("invalid stored checkpoint position");
299        let leaves = mmr::leaf_count(checkpoint.size).map_err(|_| position())?;
300        if checkpoint.size > state.size
301            || leaves != checkpoint.indexed_seq
302            || checkpoint.indexed_seq > state.indexed_seq
303        {
304            return Err(position());
305        }
306        let expected = peak_hashes(nodes, checkpoint.size)?;
307        if !same_peaks(&expected, &checkpoint.peaks) {
308            return Err(Error::corrupt("stored checkpoint peaks mismatch"));
309        }
310        self.check_statement(&checkpoint.bytes, checkpoint.size, &expected)
311            .map_err(|reason| Error::corrupt(format!("stored checkpoint {reason}")))
312    }
313
314    /// Checks that `statement` verifies under its own key and is this
315    /// runner's checkpoint of `peaks` at `size`.
316    fn check_statement(
317        &self,
318        statement: &[u8],
319        size: u64,
320        peaks: &[Hash],
321    ) -> std::result::Result<(), &'static str> {
322        let verified = verify_checkpoint_cose_offline(statement);
323        let decoded = match verified.decoded {
324            Some(decoded) if verified.ok => decoded,
325            _ => return Err("signature mismatch"),
326        };
327        if decoded.log_id != self.config.log_id
328            || decoded.key_id != self.signer.key_id()
329            || decoded.mmr_size != size
330            || decoded.new_peak_hashes != peaks
331            || decoded.root != hex::encode(mmr::root_from_peaks(peaks))
332        {
333            return Err("metadata mismatch");
334        }
335        Ok(())
336    }
337
338    fn advance(&self, now: Timestamp, loaded: &mut Loaded) -> Result<bool> {
339        let state = &loaded.state;
340        let nodes = &mut loaded.nodes;
341        let entries = self
342            .store
343            .scan_entries(state.indexed_seq, self.config.scan_limit)?;
344        let mut first_pending = state.first_pending_at;
345        let mut indexed = state.indexed_seq;
346        let mut new_nodes: Vec<Vec<u8>> = Vec::new();
347        for entry in &entries {
348            if Some(entry.seq) != indexed.checked_add(1) || !entry.appended_at.is_portable() {
349                return Err(Error::corrupt("CLL entry sequence or time is invalid"));
350            }
351            let value = to_hash(&entry.value)
352                .ok_or_else(|| Error::corrupt("CLL entry value is not 32 bytes"))?;
353            let (_, appended) = mmr::add_leaf(nodes, mmr::leaf_hash(&value))
354                .map_err(|e| Error::with_source(ErrorCode::Corrupt, "append MMR entry", e))?;
355            new_nodes.extend(appended.iter().map(|node| node.to_vec()));
356            indexed = entry.seq;
357            first_pending.get_or_insert(entry.appended_at);
358        }
359        let last_checkpointed = state.checkpoint.as_ref().map_or(0, |c| c.indexed_seq);
360        let uncheckpointed = indexed - last_checkpointed;
361        let due = first_pending.is_some_and(|first| self.config.due(uncheckpointed, first, now));
362        let size = nodes.size();
363        let mut next = State {
364            size,
365            nodes: state.nodes.iter().cloned().chain(new_nodes).collect(),
366            indexed_seq: indexed,
367            first_pending_at: first_pending,
368            checkpoint: state.checkpoint.clone(),
369            witnesses: state.witnesses.clone(),
370        };
371        if due {
372            let statement = self.sign(now, state.checkpoint.as_ref(), nodes, size)?;
373            let peaks = peak_hashes(nodes, size)?;
374            next.checkpoint = Some(CheckpointState {
375                bytes: statement.clone(),
376                size,
377                indexed_seq: indexed,
378                peaks: peaks.iter().map(|peak| peak.to_vec()).collect(),
379            });
380            next.first_pending_at = None;
381            next.witnesses
382                .extend(self.config.witness_ids.iter().map(|id| WitnessState {
383                    witness_id: id.clone(),
384                    checkpoint_size: size,
385                    checkpoint: statement.clone(),
386                    attempts: 0,
387                    next_attempt_at: now,
388                    receipt: None,
389                    permanent: false,
390                    last_error: None,
391                }));
392        }
393        if entries.is_empty() && !due {
394            return Ok(false);
395        }
396        let expected = state.checkpoint.as_ref().map(|c| c.bytes.as_slice());
397        self.store.commit_cll(state.size, expected, &next)?;
398        loaded.state = next;
399        Ok(true)
400    }
401
402    /// Signs the checkpoint of `nodes` at `size`, following `previous`.
403    fn sign(
404        &self,
405        now: Timestamp,
406        previous: Option<&CheckpointState>,
407        nodes: &MemoryNodeStore,
408        size: u64,
409    ) -> Result<Vec<u8>> {
410        let peaks = peak_hashes(nodes, size)?;
411        let (prev_size, prev_peaks, proof) = match previous {
412            None => (0, None, None),
413            Some(previous) => {
414                let proof = mmr::consistency_proof(nodes, previous.size, size)
415                    .map_err(|e| Error::with_source(ErrorCode::Corrupt, "consistency proof", e))?;
416                (
417                    previous.size,
418                    Some(peak_hashes(nodes, previous.size)?),
419                    Some(proof),
420                )
421            }
422        };
423        let record = CheckpointRecord {
424            v: 1,
425            kind: "mmr_checkpoint".to_string(),
426            log_id: self.config.log_id.clone(),
427            mmr_size: size,
428            root: hex::encode(mmr::root_from_peaks(&peaks)),
429            prev_size,
430            prev_root: prev_peaks
431                .as_deref()
432                .map_or_else(String::new, |p| hex::encode(mmr::root_from_peaks(p))),
433            key_id: self.signer.key_id(),
434            timestamp: checkpoint_timestamp(now)?,
435            signature: String::new(),
436            witnesses: Vec::new(),
437        };
438        let statement = checkpoint_to_cose(
439            &record,
440            &self.signer,
441            &peaks,
442            prev_peaks.as_deref(),
443            proof.as_ref(),
444            None,
445        )
446        .map_err(sign_error)?;
447        // As in Go, a statement that does not verify is never committed.
448        self.check_statement(&statement, size, &peaks)
449            .map_err(|reason| Error::invalid(format!("newly signed checkpoint {reason}")))?;
450        Ok(statement)
451    }
452}
453
454/// A signer failure is outside the contract's codes, so it is `storage`
455/// (Rust's code for unclassified external failures); an encoding failure is
456/// `invalid`.
457fn sign_error(error: CheckpointError) -> Error {
458    let code = match error {
459        CheckpointError::Signer(_) => ErrorCode::Storage,
460        _ => ErrorCode::Invalid,
461    };
462    Error::with_source(code, "sign checkpoint", error)
463}
464
465fn runner_time(now: Timestamp) -> Result<Timestamp> {
466    if now.is_portable() {
467        Ok(now)
468    } else {
469        Err(Error::invalid("invalid runner time"))
470    }
471}
472
473fn to_hash(bytes: &[u8]) -> Option<Hash> {
474    bytes.try_into().ok()
475}
476
477fn peak_hashes(nodes: &MemoryNodeStore, size: u64) -> Result<Vec<Hash>> {
478    let positions = mmr::peaks(size).map_err(|_| Error::corrupt("invalid MMR size"))?;
479    if size > nodes.size() {
480        return Err(Error::corrupt("MMR is smaller than the requested size"));
481    }
482    Ok(positions.into_iter().map(|p| nodes.node(p)).collect())
483}
484
485fn same_peaks(expected: &[Hash], stored: &[Vec<u8>]) -> bool {
486    expected.len() == stored.len()
487        && expected
488            .iter()
489            .zip(stored)
490            .all(|(left, right)| left.as_slice() == right.as_slice())
491}
492
493#[cfg(test)]
494mod tests {
495    use super::*;
496    use crate::log::memory::MemoryStore;
497    use crate::log::{
498        AppendInput, CheckpointStateStore, Entry, EntrySource, EntryStore, WitnessStateStore,
499    };
500    use ed25519_dalek::SigningKey;
501    use std::sync::atomic::AtomicUsize;
502
503    fn at(text: &str) -> Timestamp {
504        Timestamp::parse(text).unwrap()
505    }
506
507    fn key() -> SigningKey {
508        SigningKey::from_bytes(&[7; 32])
509    }
510
511    fn config() -> RunnerConfig {
512        RunnerConfig {
513            cadence_entries: 2,
514            witness_ids: vec!["w-a".into()],
515            ..RunnerConfig::new("unit-log")
516        }
517    }
518
519    fn append(store: &MemoryStore, byte: u8, time: &str) {
520        store
521            .append(&AppendInput {
522                value: vec![byte; 32],
523                appended_at: at(time),
524            })
525            .unwrap();
526    }
527
528    #[test]
529    fn checkpoint_timestamps_drop_trailing_fraction_zeros() {
530        for (text, expected) in [
531            ("2026-08-27T12:34:56.000Z", "2026-08-27T12:34:56Z"),
532            ("2026-08-27T12:34:56.500Z", "2026-08-27T12:34:56.5Z"),
533            ("2026-08-27T12:34:56.120Z", "2026-08-27T12:34:56.12Z"),
534            ("2026-08-27T12:34:56.123Z", "2026-08-27T12:34:56.123Z"),
535            ("0000-01-01T00:00:00.000Z", "0000-01-01T00:00:00Z"),
536        ] {
537            assert_eq!(checkpoint_timestamp(at(text)).unwrap(), expected);
538        }
539        for text in ["+010000-01-01T00:00:00.000Z", "-000001-01-01T00:00:00.000Z"] {
540            assert_eq!(
541                checkpoint_timestamp(at(text)).unwrap_err().code(),
542                ErrorCode::Invalid
543            );
544        }
545    }
546
547    #[test]
548    fn configuration_is_validated() {
549        let store = Arc::new(MemoryStore::new());
550        let cases: Vec<(RunnerConfig, &str)> = vec![
551            (RunnerConfig::new("bad id"), "invalid runner configuration"),
552            (
553                RunnerConfig {
554                    scan_limit: 0,
555                    ..config()
556                },
557                "invalid runner configuration",
558            ),
559            (
560                RunnerConfig {
561                    scan_limit: MAX_SCAN_LIMIT + 1,
562                    ..config()
563                },
564                "invalid runner configuration",
565            ),
566            (
567                RunnerConfig {
568                    poll_interval: Duration::ZERO,
569                    ..config()
570                },
571                "invalid runner configuration",
572            ),
573            (
574                RunnerConfig {
575                    cadence_entries: 0,
576                    ..config()
577                },
578                "invalid checkpoint cadence",
579            ),
580            (
581                RunnerConfig {
582                    cadence_age: Duration::ZERO,
583                    ..config()
584                },
585                "invalid checkpoint cadence",
586            ),
587            (
588                RunnerConfig {
589                    witness_ids: (0..=MAX_WITNESSES).map(|i| format!("w-{i}")).collect(),
590                    ..config()
591                },
592                "too many witnesses",
593            ),
594            (
595                RunnerConfig {
596                    witness_ids: vec!["bad witness".into()],
597                    ..config()
598                },
599                "invalid witness ID",
600            ),
601            (
602                RunnerConfig {
603                    witness_ids: vec!["w-a".into(), "w-a".into()],
604                    ..config()
605                },
606                "duplicate witness ID",
607            ),
608        ];
609        for (config, message) in cases {
610            let error = CheckpointRunner::new(config, store.clone(), key())
611                .err()
612                .expect("configuration must be rejected");
613            assert_eq!(error.code(), ErrorCode::Invalid);
614            assert_eq!(error.message(), message);
615        }
616        let full = RunnerConfig {
617            witness_ids: (0..MAX_WITNESSES).map(|i| format!("w-{i}")).collect(),
618            scan_limit: MAX_SCAN_LIMIT,
619            ..config()
620        };
621        CheckpointRunner::new(full, store, key()).unwrap();
622    }
623
624    #[test]
625    fn invalid_times_are_rejected() {
626        let runner = CheckpointRunner::new(config(), Arc::new(MemoryStore::new()), key()).unwrap();
627        let late = Timestamp::from_unix_millis(crate::log::MAX_PORTABLE_UNIX_MILLIS + 1);
628        assert_eq!(
629            runner.run_once(late).unwrap_err().code(),
630            ErrorCode::Invalid
631        );
632        assert_eq!(
633            runner.catch_up(late).unwrap_err().code(),
634            ErrorCode::Invalid
635        );
636    }
637
638    #[test]
639    fn indexes_signs_and_queues_witnesses() {
640        let store = Arc::new(MemoryStore::new());
641        let runner = CheckpointRunner::new(config(), store.clone(), key()).unwrap();
642        assert!(!runner.run_once(at("2026-10-01T12:00:00.000Z")).unwrap());
643        append(&store, 1, "2026-10-01T12:00:00.000Z");
644        assert!(runner.run_once(at("2026-10-01T12:00:01.000Z")).unwrap());
645        let state = store.load_cll().unwrap();
646        assert_eq!((state.size, state.indexed_seq), (1, 1));
647        assert!(state.checkpoint.is_none());
648        assert_eq!(state.first_pending_at, Some(at("2026-10-01T12:00:00.000Z")));
649        // Nothing new and not due yet: no commit.
650        assert!(!runner.run_once(at("2026-10-01T12:00:02.000Z")).unwrap());
651        // The age cadence makes the single entry due.
652        assert!(runner.run_once(at("2026-10-01T12:15:00.000Z")).unwrap());
653        let state = store.load_cll().unwrap();
654        let checkpoint = state.checkpoint.clone().unwrap();
655        assert_eq!((checkpoint.size, checkpoint.indexed_seq), (1, 1));
656        assert!(state.first_pending_at.is_none());
657        assert_eq!(state.witnesses.len(), 1);
658        assert_eq!(
659            state.witnesses[0].next_attempt_at,
660            at("2026-10-01T12:15:00.000Z")
661        );
662        // The next checkpoint carries a consistency proof from the first.
663        append(&store, 2, "2026-10-01T12:16:00.000Z");
664        append(&store, 3, "2026-10-01T12:16:01.000Z");
665        runner.catch_up(at("2026-10-01T12:17:00.000Z")).unwrap();
666        let state = store.load_cll().unwrap();
667        let decoded = verify_checkpoint_cose_offline(&state.checkpoint.unwrap().bytes);
668        assert!(decoded.ok, "{:?}", decoded.errors);
669        let decoded = decoded.decoded.unwrap();
670        assert_eq!((decoded.prev_size, decoded.mmr_size), (1, 4));
671        assert_eq!(decoded.timestamp, "2026-10-01T12:17:00Z");
672        assert!(decoded.consistency_proof.is_some());
673        assert_eq!(state.witnesses.len(), 2);
674    }
675
676    #[test]
677    fn catch_up_runs_one_batch_per_pass_until_idle() {
678        let store = Arc::new(MemoryStore::new());
679        let config = RunnerConfig {
680            scan_limit: 2,
681            cadence_entries: 3,
682            ..config()
683        };
684        let runner = CheckpointRunner::new(config, store.clone(), key()).unwrap();
685        for byte in 1..=5 {
686            append(&store, byte, "2026-10-01T12:00:00.000Z");
687        }
688        assert!(runner.run_once(at("2026-10-01T12:00:01.000Z")).unwrap());
689        assert_eq!(store.load_cll().unwrap().indexed_seq, 2);
690        runner.catch_up(at("2026-10-01T12:00:01.000Z")).unwrap();
691        let state = store.load_cll().unwrap();
692        assert_eq!(state.indexed_seq, 5);
693        assert_eq!(state.checkpoint.unwrap().indexed_seq, 4);
694    }
695
696    fn checkpointed_store() -> Arc<MemoryStore> {
697        let store = Arc::new(MemoryStore::new());
698        let runner = CheckpointRunner::new(config(), store.clone(), key()).unwrap();
699        append(&store, 1, "2026-10-01T12:00:00.000Z");
700        append(&store, 2, "2026-10-01T12:00:01.000Z");
701        assert!(runner.run_once(at("2026-10-01T12:00:02.000Z")).unwrap());
702        assert!(store.load_cll().unwrap().checkpoint.is_some());
703        store
704    }
705
706    /// A store whose state is whatever the test put there, over a real
707    /// entry log.
708    struct Tampered {
709        entries: MemoryStore,
710        state: State,
711    }
712
713    impl EntrySource for Tampered {
714        fn scan_entries(&self, after_seq: u64, limit: usize) -> Result<Vec<Entry>> {
715            self.entries.scan_entries(after_seq, limit)
716        }
717    }
718
719    impl CheckpointStateStore for Tampered {
720        fn load_cll(&self) -> Result<State> {
721            Ok(self.state.clone())
722        }
723
724        fn commit_cll(&self, _: u64, _: Option<&[u8]>, _: &State) -> Result<()> {
725            panic!("a corrupt state must not be committed over");
726        }
727    }
728
729    impl WitnessStateStore for Tampered {
730        fn pending_witnesses_for(
731            &self,
732            _: &str,
733            _: Timestamp,
734            _: usize,
735        ) -> Result<Vec<WitnessState>> {
736            Ok(Vec::new())
737        }
738
739        fn pending_witnesses(&self, _: Timestamp, _: usize) -> Result<Vec<WitnessState>> {
740            Ok(Vec::new())
741        }
742
743        fn get_witness(&self, _: &str, _: u64) -> Result<Option<WitnessState>> {
744            Ok(None)
745        }
746
747        fn commit_witness(&self, _: u32, _: &WitnessState) -> Result<()> {
748            Ok(())
749        }
750    }
751
752    fn assert_corrupt(state: State, config: RunnerConfig, signer: SigningKey) {
753        let store = Arc::new(Tampered {
754            entries: MemoryStore::new(),
755            state,
756        });
757        let runner = CheckpointRunner::new(config, store, signer).unwrap();
758        let error = runner.run_once(at("2026-10-01T12:01:00.000Z")).unwrap_err();
759        assert_eq!(error.code(), ErrorCode::Corrupt, "{error}");
760    }
761
762    #[test]
763    fn corrupt_stored_state_is_refused() {
764        let good = checkpointed_store().load_cll().unwrap();
765
766        let mut bad = good.clone();
767        bad.size = 2;
768        assert_corrupt(bad, config(), key());
769
770        let mut bad = good.clone();
771        bad.nodes.pop();
772        assert_corrupt(bad, config(), key());
773
774        let mut bad = good.clone();
775        bad.indexed_seq = 1;
776        assert_corrupt(bad, config(), key());
777
778        // The checkpointed peak (node 2 of a 3-node MMR) no longer matches.
779        let mut bad = good.clone();
780        bad.nodes[2][0] ^= 1;
781        assert_corrupt(bad, config(), key());
782
783        let mut bad = good.clone();
784        bad.checkpoint.as_mut().unwrap().indexed_seq = 1;
785        assert_corrupt(bad, config(), key());
786
787        let mut bad = good.clone();
788        bad.checkpoint.as_mut().unwrap().peaks[0][0] ^= 1;
789        assert_corrupt(bad, config(), key());
790
791        // A flipped bit in the signed statement breaks its signature.
792        let mut bad = good.clone();
793        let bytes = &mut bad.checkpoint.as_mut().unwrap().bytes;
794        let last = bytes.len() - 1;
795        bytes[last] ^= 1;
796        assert_corrupt(bad, config(), key());
797
798        // A valid statement from another log or key is not this runner's.
799        assert_corrupt(good.clone(), RunnerConfig::new("other-log"), key());
800        assert_corrupt(good, config(), SigningKey::from_bytes(&[8; 32]));
801    }
802
803    #[test]
804    fn a_gap_in_the_entry_log_is_corrupt() {
805        struct Gappy(MemoryStore);
806        impl EntrySource for Gappy {
807            fn scan_entries(&self, after_seq: u64, limit: usize) -> Result<Vec<Entry>> {
808                let mut entries = self.0.scan_entries(after_seq, limit)?;
809                entries.iter_mut().for_each(|e| e.seq += 1);
810                Ok(entries)
811            }
812        }
813        impl CheckpointStateStore for Gappy {
814            fn load_cll(&self) -> Result<State> {
815                self.0.load_cll()
816            }
817            fn commit_cll(&self, s: u64, c: Option<&[u8]>, n: &State) -> Result<()> {
818                self.0.commit_cll(s, c, n)
819            }
820        }
821        impl WitnessStateStore for Gappy {
822            fn pending_witnesses(&self, n: Timestamp, l: usize) -> Result<Vec<WitnessState>> {
823                self.0.pending_witnesses(n, l)
824            }
825            fn pending_witnesses_for(
826                &self,
827                w: &str,
828                n: Timestamp,
829                l: usize,
830            ) -> Result<Vec<WitnessState>> {
831                self.0.pending_witnesses_for(w, n, l)
832            }
833            fn get_witness(&self, w: &str, s: u64) -> Result<Option<WitnessState>> {
834                self.0.get_witness(w, s)
835            }
836            fn commit_witness(&self, a: u32, n: &WitnessState) -> Result<()> {
837                self.0.commit_witness(a, n)
838            }
839        }
840        let inner = MemoryStore::new();
841        append(&inner, 1, "2026-10-01T12:00:00.000Z");
842        let runner = CheckpointRunner::new(config(), Arc::new(Gappy(inner)), key()).unwrap();
843        let error = runner.run_once(at("2026-10-01T12:00:01.000Z")).unwrap_err();
844        assert_eq!(error.code(), ErrorCode::Corrupt);
845    }
846
847    /// A memory store whose next `fail` commits lose a race: another writer
848    /// indexes the pending entries first, then the commit is `contention`.
849    struct Racing {
850        inner: Arc<MemoryStore>,
851        fail: AtomicUsize,
852        loads: AtomicUsize,
853    }
854
855    impl EntrySource for Racing {
856        fn scan_entries(&self, after_seq: u64, limit: usize) -> Result<Vec<Entry>> {
857            self.inner.scan_entries(after_seq, limit)
858        }
859    }
860
861    impl CheckpointStateStore for Racing {
862        fn load_cll(&self) -> Result<State> {
863            self.loads.fetch_add(1, Ordering::SeqCst);
864            self.inner.load_cll()
865        }
866
867        fn commit_cll(&self, size: u64, checkpoint: Option<&[u8]>, next: &State) -> Result<()> {
868            // Commits are serialized by the runner, so load-then-store is
869            // race-free here.
870            let racing = self.fail.load(Ordering::SeqCst) > 0;
871            if racing {
872                self.fail.fetch_sub(1, Ordering::SeqCst);
873            }
874            if racing {
875                // The other writer: a runner over the same store.
876                let rival = CheckpointRunner::new(
877                    RunnerConfig {
878                        scan_limit: 1,
879                        ..config()
880                    },
881                    self.inner.clone(),
882                    key(),
883                )
884                .unwrap();
885                rival.run_once(at("2026-10-01T12:00:05.000Z")).unwrap();
886            }
887            self.inner.commit_cll(size, checkpoint, next)
888        }
889    }
890
891    impl WitnessStateStore for Racing {
892        fn pending_witnesses(&self, n: Timestamp, l: usize) -> Result<Vec<WitnessState>> {
893            self.inner.pending_witnesses(n, l)
894        }
895        fn pending_witnesses_for(
896            &self,
897            w: &str,
898            n: Timestamp,
899            l: usize,
900        ) -> Result<Vec<WitnessState>> {
901            self.inner.pending_witnesses_for(w, n, l)
902        }
903        fn get_witness(&self, w: &str, s: u64) -> Result<Option<WitnessState>> {
904            self.inner.get_witness(w, s)
905        }
906        fn commit_witness(&self, a: u32, n: &WitnessState) -> Result<()> {
907            self.inner.commit_witness(a, n)
908        }
909    }
910
911    #[test]
912    fn catch_up_reloads_once_after_contention() {
913        let inner = Arc::new(MemoryStore::new());
914        append(&inner, 1, "2026-10-01T12:00:00.000Z");
915        append(&inner, 2, "2026-10-01T12:00:01.000Z");
916        append(&inner, 3, "2026-10-01T12:00:02.000Z");
917        let store = Arc::new(Racing {
918            inner: inner.clone(),
919            fail: AtomicUsize::new(1),
920            loads: AtomicUsize::new(0),
921        });
922        let runner = CheckpointRunner::new(config(), store.clone(), key()).unwrap();
923        runner.catch_up(at("2026-10-01T12:00:10.000Z")).unwrap();
924        assert_eq!(store.loads.load(Ordering::SeqCst), 2);
925        let state = inner.load_cll().unwrap();
926        assert_eq!(state.indexed_seq, 3);
927        assert_eq!(state.checkpoint.unwrap().indexed_seq, 3);
928
929        // run_once never reloads: the lost race is the caller's to retry.
930        let inner = Arc::new(MemoryStore::new());
931        append(&inner, 1, "2026-10-01T12:00:00.000Z");
932        let store = Arc::new(Racing {
933            inner,
934            fail: AtomicUsize::new(1),
935            loads: AtomicUsize::new(0),
936        });
937        let runner = CheckpointRunner::new(config(), store, key()).unwrap();
938        let error = runner.run_once(at("2026-10-01T12:00:10.000Z")).unwrap_err();
939        assert_eq!(error.code(), ErrorCode::Contention);
940    }
941
942    #[test]
943    fn catch_up_returns_contention_right_after_a_reload() {
944        let inner = Arc::new(MemoryStore::new());
945        for byte in 1..=4 {
946            append(&inner, byte, "2026-10-01T12:00:00.000Z");
947        }
948        let store = Arc::new(Racing {
949            inner,
950            fail: AtomicUsize::new(2),
951            loads: AtomicUsize::new(0),
952        });
953        let runner = CheckpointRunner::new(config(), store.clone(), key()).unwrap();
954        let error = runner.catch_up(at("2026-10-01T12:00:10.000Z")).unwrap_err();
955        assert_eq!(error.code(), ErrorCode::Contention);
956        assert_eq!(store.loads.load(Ordering::SeqCst), 2);
957    }
958
959    #[test]
960    fn catch_up_stops_between_batches_once_asked() {
961        let store = Arc::new(MemoryStore::new());
962        for byte in 1..=5 {
963            append(&store, byte, "2026-10-01T12:00:00.000Z");
964        }
965        let config = RunnerConfig {
966            scan_limit: 2,
967            ..config()
968        };
969        let runner = CheckpointRunner::new(config, store.clone(), key()).unwrap();
970        let stop = AtomicBool::new(true);
971        runner
972            .catch_up_until(at("2026-10-01T12:00:01.000Z"), Some(&stop))
973            .unwrap();
974        assert_eq!(store.load_cll().unwrap().indexed_seq, 2);
975    }
976
977    #[test]
978    fn run_stops_when_asked() {
979        let store = Arc::new(MemoryStore::new());
980        append(&store, 1, "2026-10-01T12:00:00.000Z");
981        let config = RunnerConfig {
982            poll_interval: Duration::from_secs(3600),
983            ..config()
984        };
985        let runner = CheckpointRunner::new(config, store.clone(), key()).unwrap();
986        let stop = AtomicBool::new(false);
987        std::thread::scope(|scope| {
988            let handle = scope.spawn(|| runner.run(&stop));
989            while store.load_cll().unwrap().indexed_seq == 0 {
990                std::thread::yield_now();
991            }
992            stop.store(true, Ordering::SeqCst);
993            runner.notify();
994            handle.join().unwrap().unwrap();
995        });
996    }
997}