Skip to main content

second_brain_api/
clock_gate.rs

1//! Holds seq minting until the wall clock can be trusted. A Pi without an
2//! RTC boots on a saved, older time until NTP syncs, and a restored card
3//! boots at roughly its image time; seqs minted then could repeat.
4
5use std::any::Any;
6use std::io::Write;
7use std::path::{Path, PathBuf};
8use std::sync::{Arc, Condvar, Mutex, MutexGuard};
9use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
10
11use chrono::{DateTime, Utc};
12use second_brain_core::sqlite_store::SqliteStore;
13use second_brain_core::sqlite_store::clock::{ClockUntrusted, HLC_CEILING_MICROS, WallClock};
14
15use crate::health::Health;
16
17pub const READING_FILE: &str = "clock-reading";
18const TICK_OPEN: Duration = Duration::from_secs(60);
19const TICK_CLOSED: Duration = Duration::from_secs(1);
20const REEVALUATE_EVERY: Duration = Duration::from_secs(1);
21/// Smaller backward moves are NTP step corrections, which the HLC absorbs
22/// by counting up from its last seq.
23const BACKWARD_STEP_MICROS: i64 = 1_000_000;
24const DAY_MICROS: i64 = 86_400 * 1_000_000;
25const PEER_AHEAD_SYNCED_MICROS: i64 = DAY_MICROS;
26const PEER_AHEAD_UNSYNCED_MICROS: i64 = 365 * DAY_MICROS;
27const BASIS_AHEAD_MICROS: i64 = 60 * 1_000_000;
28
29pub trait TimeSignals: Send + Sync + std::panic::RefUnwindSafe {
30    fn now_micros(&self) -> i64;
31    fn ntp_synced(&self) -> bool;
32    fn rtc_secs(&self) -> Option<i64>;
33}
34
35pub struct SystemSignals;
36
37impl TimeSignals for SystemSignals {
38    fn now_micros(&self) -> i64 {
39        system_micros()
40    }
41
42    #[cfg(target_os = "linux")]
43    fn ntp_synced(&self) -> bool {
44        kernel_synced() || Path::new("/run/systemd/timesync/synchronized").exists()
45    }
46
47    #[cfg(not(target_os = "linux"))]
48    fn ntp_synced(&self) -> bool {
49        false
50    }
51
52    #[cfg(target_os = "linux")]
53    fn rtc_secs(&self) -> Option<i64> {
54        std::fs::read_to_string("/sys/class/rtc/rtc0/since_epoch")
55            .ok()?
56            .trim()
57            .parse()
58            .ok()
59    }
60
61    #[cfg(not(target_os = "linux"))]
62    fn rtc_secs(&self) -> Option<i64> {
63        None
64    }
65}
66
67#[cfg(target_os = "linux")]
68fn kernel_synced() -> bool {
69    let mut tx: libc::timex = unsafe { std::mem::zeroed() };
70    let state = unsafe { libc::adjtimex(&mut tx) };
71    state != -1 && state != libc::TIME_ERROR && tx.status & libc::STA_UNSYNC == 0
72}
73
74fn system_micros() -> i64 {
75    match SystemTime::now().duration_since(UNIX_EPOCH) {
76        Ok(since) => since.as_micros() as i64,
77        Err(before) => -(before.duration().as_micros() as i64),
78    }
79}
80
81struct TrustedSystemTime;
82
83impl TimeSignals for TrustedSystemTime {
84    fn now_micros(&self) -> i64 {
85        system_micros()
86    }
87
88    fn ntp_synced(&self) -> bool {
89        true
90    }
91
92    fn rtc_secs(&self) -> Option<i64> {
93        None
94    }
95}
96
97/// The opening rule used when neither NTP nor a peer vouches for the clock.
98#[derive(Debug, Clone, Copy, PartialEq, Eq)]
99enum Fallback {
100    /// A batteryless or missing RTC never passes, so a Pi waits for NTP.
101    Rtc,
102    /// Every Mac has a battery RTC, so its clock alone is enough.
103    WallClock,
104}
105
106impl Fallback {
107    fn for_target() -> Self {
108        if cfg!(target_os = "linux") {
109            Fallback::Rtc
110        } else {
111            Fallback::WallClock
112        }
113    }
114}
115
116#[derive(Debug, Clone, Copy, PartialEq, Eq)]
117pub struct GateStatus {
118    pub open: bool,
119    pub synced: bool,
120    pub now_micros: i64,
121}
122
123impl GateStatus {
124    pub fn basis_ahead(&self, basis_micros: i64) -> bool {
125        self.synced && basis_micros > self.now_micros.saturating_add(BASIS_AHEAD_MICROS)
126    }
127}
128
129type Observer = Box<dyn Fn(&GateStatus) + Send + Sync>;
130
131struct GateState {
132    open: bool,
133    /// The highest reading seen since the gate last opened.
134    high: i64,
135    persisted: i64,
136    evaluated_at: Option<Instant>,
137    changed: bool,
138    closures: u64,
139}
140
141pub struct ClockGate {
142    signals: Arc<dyn TimeSignals>,
143    build_secs: i64,
144    fallback: Fallback,
145    reading_path: Option<PathBuf>,
146    state: Mutex<GateState>,
147    changed: Condvar,
148    persisting: Mutex<()>,
149    observers: Mutex<Vec<Observer>>,
150}
151
152impl ClockGate {
153    /// Closed until an opening rule holds; `build_secs` is the binary's build
154    /// time, a lower bound on real time.
155    pub fn new(data_dir: &Path, signals: Arc<dyn TimeSignals>, build_secs: i64) -> Arc<Self> {
156        Self::build(
157            Some(data_dir.join(READING_FILE)),
158            signals,
159            build_secs,
160            Fallback::for_target(),
161            false,
162        )
163    }
164
165    pub fn system(data_dir: &Path) -> Arc<Self> {
166        Self::new(data_dir, Arc::new(SystemSignals), build_epoch())
167    }
168
169    pub fn open_for_tests() -> Arc<Self> {
170        Self::build(
171            None,
172            Arc::new(TrustedSystemTime),
173            0,
174            Fallback::for_target(),
175            true,
176        )
177    }
178
179    fn build(
180        reading_path: Option<PathBuf>,
181        signals: Arc<dyn TimeSignals>,
182        build_secs: i64,
183        fallback: Fallback,
184        open: bool,
185    ) -> Arc<Self> {
186        let persisted = reading_path.as_deref().map_or(0, load_reading);
187        let high = signals.now_micros();
188        Arc::new(Self {
189            signals,
190            build_secs,
191            fallback,
192            reading_path,
193            state: Mutex::new(GateState {
194                open,
195                high,
196                persisted,
197                evaluated_at: None,
198                changed: false,
199                closures: 0,
200            }),
201            changed: Condvar::new(),
202            persisting: Mutex::new(()),
203            observers: Mutex::new(Vec::new()),
204        })
205    }
206
207    /// Re-evaluates the opening rules when closed, at most once a second.
208    pub fn is_open(&self) -> bool {
209        self.reading().is_some()
210    }
211
212    /// How many times the gate has gone from open to closed. A unit of work
213    /// that failed while this moved failed on writes the gate held, so it is
214    /// still pending.
215    pub fn closures(&self) -> u64 {
216        self.lock().closures
217    }
218
219    pub fn synced(&self) -> bool {
220        self.signals.ntp_synced()
221    }
222
223    pub fn wall_micros(&self) -> i64 {
224        self.signals.now_micros().min(HLC_CEILING_MICROS)
225    }
226
227    /// For a handshake with a peer that reported its own clock synchronised;
228    /// the caller has already bounded the peer's time.
229    pub fn open_by_peer(&self) {
230        let now = self.signals.now_micros();
231        let mut state = self.lock();
232        if !state.open {
233            tracing::info!("clock gate opened by a synchronised sync peer");
234            self.set_open(&mut state, now);
235        }
236    }
237
238    pub fn check_peer_time(&self, peer_micros: i64) -> Result<(), String> {
239        if peer_micros > HLC_CEILING_MICROS {
240            return Err(format!(
241                "peer time {} is past the clock ceiling {}",
242                show(peer_micros),
243                show(HLC_CEILING_MICROS)
244            ));
245        }
246        let (base, allowed, span, against) = if self.synced() {
247            (
248                self.signals.now_micros(),
249                PEER_AHEAD_SYNCED_MICROS,
250                "a day",
251                "this device's synchronised clock",
252            )
253        } else {
254            (
255                self.lock()
256                    .persisted
257                    .max(self.build_secs.saturating_mul(1_000_000)),
258                PEER_AHEAD_UNSYNCED_MICROS,
259                "a year",
260                "this device's last trusted clock reading",
261            )
262        };
263        if peer_micros > base.saturating_add(allowed) {
264            return Err(format!(
265                "peer time {} is more than {span} ahead of {against} ({})",
266                show(peer_micros),
267                show(base)
268            ));
269        }
270        Ok(())
271    }
272
273    /// Saves the reading while open, re-evaluates while closed, then tells
274    /// every observer where the gate stands.
275    pub fn tick(&self) {
276        if !self.lock().open {
277            self.evaluate();
278        }
279        if let Some(reading) = self.reading() {
280            self.persist(reading);
281        }
282        let status = GateStatus {
283            open: self.lock().open,
284            synced: self.synced(),
285            now_micros: self.signals.now_micros(),
286        };
287        for observer in self.observers().iter() {
288            observer(&status);
289        }
290    }
291
292    /// Ticks every minute while open and every second while closed, and at
293    /// once whenever the gate opens or closes.
294    pub fn spawn_ticker(self: &Arc<Self>) {
295        let gate = self.clone();
296        let spawned = std::thread::Builder::new()
297            .name("clock-gate".to_string())
298            .spawn(move || {
299                loop {
300                    gate.tick();
301                    gate.wait_for_change();
302                }
303            });
304        if let Err(e) = spawned {
305            tracing::error!("cannot start the clock gate ticker: {e}");
306        }
307    }
308
309    /// Observers run on the ticking thread, never inside a store write.
310    pub fn observe(&self, observer: impl Fn(&GateStatus) + Send + Sync + 'static) {
311        self.observers().push(Box::new(observer));
312    }
313
314    fn reading(&self) -> Option<i64> {
315        {
316            let mut state = self.lock();
317            if state.open {
318                let now = self.signals.now_micros();
319                return self.observe_open(&mut state, now);
320            }
321            if state
322                .evaluated_at
323                .is_some_and(|at| at.elapsed() < REEVALUATE_EVERY)
324            {
325                return None;
326            }
327        }
328        if !self.evaluate() {
329            return None;
330        }
331        let mut state = self.lock();
332        if !state.open {
333            return None;
334        }
335        let now = self.signals.now_micros();
336        self.observe_open(&mut state, now)
337    }
338
339    fn observe_open(&self, state: &mut GateState, now: i64) -> Option<i64> {
340        if now < state.high - BACKWARD_STEP_MICROS {
341            tracing::warn!(
342                "the wall clock stepped back from {} to {}; holding writes until it is trusted again",
343                show(state.high),
344                show(now)
345            );
346            state.open = false;
347            state.evaluated_at = None;
348            state.closures += 1;
349            self.mark_changed(state);
350            return None;
351        }
352        state.high = state.high.max(now);
353        Some(now.min(HLC_CEILING_MICROS))
354    }
355
356    fn evaluate(&self) -> bool {
357        let synced = self.signals.ntp_synced();
358        let rtc = match self.fallback {
359            Fallback::Rtc => self.signals.rtc_secs(),
360            Fallback::WallClock => None,
361        };
362        let now = self.signals.now_micros();
363        let mut state = self.lock();
364        state.evaluated_at = Some(Instant::now());
365        if state.open {
366            return true;
367        }
368        let opens = synced
369            || match self.fallback {
370                Fallback::Rtc => {
371                    rtc.is_some_and(|secs| secs >= self.build_secs) && now >= state.persisted
372                }
373                Fallback::WallClock => {
374                    now >= state.persisted && now >= self.build_secs.saturating_mul(1_000_000)
375                }
376            };
377        if opens {
378            tracing::info!(
379                "clock gate opened ({})",
380                if synced {
381                    "time is synchronised"
382                } else {
383                    "the clock is past the last trusted reading"
384                }
385            );
386            self.set_open(&mut state, now);
387        }
388        opens
389    }
390
391    fn set_open(&self, state: &mut GateState, now: i64) {
392        state.open = true;
393        state.high = now;
394        self.mark_changed(state);
395    }
396
397    fn mark_changed(&self, state: &mut GateState) {
398        state.changed = true;
399        self.changed.notify_all();
400    }
401
402    fn persist(&self, reading: i64) {
403        let Some(path) = &self.reading_path else {
404            return;
405        };
406        let _serial = relock(self.persisting.lock());
407        if reading <= self.lock().persisted {
408            return;
409        }
410        match write_reading(path, reading) {
411            Ok(()) => {
412                let mut state = self.lock();
413                state.persisted = state.persisted.max(reading);
414            }
415            Err(e) => tracing::warn!("cannot save the clock reading to {}: {e}", path.display()),
416        }
417    }
418
419    fn wait_for_change(&self) {
420        let state = self.lock();
421        let timeout = if state.open { TICK_OPEN } else { TICK_CLOSED };
422        let (mut state, _) = relock(
423            self.changed
424                .wait_timeout_while(state, timeout, |state| !state.changed),
425        );
426        state.changed = false;
427    }
428
429    fn lock(&self) -> MutexGuard<'_, GateState> {
430        relock(self.state.lock())
431    }
432
433    fn observers(&self) -> MutexGuard<'_, Vec<Observer>> {
434        relock(self.observers.lock())
435    }
436}
437
438impl WallClock for ClockGate {
439    fn now_micros(&self) -> Option<i64> {
440        self.reading()
441    }
442}
443
444/// Reports the gate, and a store basis running ahead of synchronised time,
445/// to `/health` on every tick.
446pub fn report_to(clock: &ClockGate, health: Arc<Health>, sqlite: Arc<SqliteStore>) {
447    let hold = Mutex::new(HoldTimer::default());
448    clock.observe(move |status| {
449        let now = Utc::now();
450        let held_for = relock(hold.lock()).observe(status.open, Instant::now());
451        health.clock_gate(held_for, now);
452        if !status.synced {
453            health.clock_basis_ahead(false, now);
454            return;
455        }
456        match sqlite.clock_state() {
457            Ok(state) => health.clock_basis_ahead(status.basis_ahead(state.basis), now),
458            Err(e) => tracing::warn!("cannot read the store's clock state: {e:#}"),
459        }
460    });
461}
462
463#[derive(Default)]
464struct HoldTimer {
465    closed_at: Option<Instant>,
466}
467
468impl HoldTimer {
469    fn observe(&mut self, open: bool, now: Instant) -> Option<Duration> {
470        if open {
471            self.closed_at = None;
472            return None;
473        }
474        let closed_at = *self.closed_at.get_or_insert(now);
475        Some(now.saturating_duration_since(closed_at))
476    }
477}
478
479/// True for `ClockUntrusted` itself or an `anyhow::Error` carrying it
480/// anywhere in its chain.
481pub fn is_untrusted(error: &dyn Any) -> bool {
482    if error.is::<ClockUntrusted>() {
483        return true;
484    }
485    error
486        .downcast_ref::<anyhow::Error>()
487        .is_some_and(|e| e.chain().any(|cause| cause.is::<ClockUntrusted>()))
488}
489
490fn build_epoch() -> i64 {
491    env!("SB_BUILD_EPOCH").parse().unwrap_or(0)
492}
493
494fn load_reading(path: &Path) -> i64 {
495    match std::fs::read_to_string(path) {
496        Ok(text) => match text.trim().parse::<i64>() {
497            Ok(micros) => micros.min(HLC_CEILING_MICROS),
498            Err(e) => {
499                tracing::warn!("ignoring the clock reading in {}: {e}", path.display());
500                0
501            }
502        },
503        Err(e) if e.kind() == std::io::ErrorKind::NotFound => 0,
504        Err(e) => {
505            tracing::warn!("cannot read the clock reading in {}: {e}", path.display());
506            0
507        }
508    }
509}
510
511fn write_reading(path: &Path, micros: i64) -> std::io::Result<()> {
512    let tmp = path.with_extension("tmp");
513    let mut file = std::fs::File::create(&tmp)?;
514    file.write_all(micros.to_string().as_bytes())?;
515    file.sync_all()?;
516    drop(file);
517    std::fs::rename(&tmp, path)
518}
519
520fn show(micros: i64) -> String {
521    DateTime::<Utc>::from_timestamp_micros(micros)
522        .map_or_else(|| format!("{micros} µs"), |at| at.to_rfc3339())
523}
524
525fn relock<G>(result: Result<G, std::sync::PoisonError<G>>) -> G {
526    match result {
527        Ok(guard) => guard,
528        Err(poisoned) => poisoned.into_inner(),
529    }
530}
531
532#[cfg(test)]
533mod tests {
534    use std::sync::atomic::{AtomicBool, AtomicI64, Ordering};
535
536    use super::*;
537
538    const BUILD_SECS: i64 = 1_790_000_000;
539    const T0: i64 = (BUILD_SECS + 86_400) * 1_000_000;
540    const SECOND: i64 = 1_000_000;
541
542    #[derive(Default)]
543    struct FakeSignals {
544        now: AtomicI64,
545        synced: AtomicBool,
546        rtc: Mutex<Option<i64>>,
547    }
548
549    impl FakeSignals {
550        fn at(now: i64) -> Arc<Self> {
551            let fake = Arc::new(Self::default());
552            fake.set_now(now);
553            fake
554        }
555
556        fn set_now(&self, now: i64) {
557            self.now.store(now, Ordering::SeqCst);
558        }
559
560        fn set_synced(&self, synced: bool) {
561            self.synced.store(synced, Ordering::SeqCst);
562        }
563
564        fn set_rtc(&self, secs: Option<i64>) {
565            *self.rtc.lock().unwrap() = secs;
566        }
567    }
568
569    impl TimeSignals for FakeSignals {
570        fn now_micros(&self) -> i64 {
571            self.now.load(Ordering::SeqCst)
572        }
573
574        fn ntp_synced(&self) -> bool {
575            self.synced.load(Ordering::SeqCst)
576        }
577
578        fn rtc_secs(&self) -> Option<i64> {
579            *self.rtc.lock().unwrap()
580        }
581    }
582
583    fn gate(dir: &Path, fake: &Arc<FakeSignals>, fallback: Fallback) -> Arc<ClockGate> {
584        ClockGate::build(
585            Some(dir.join(READING_FILE)),
586            fake.clone(),
587            BUILD_SECS,
588            fallback,
589            false,
590        )
591    }
592
593    fn saved(dir: &Path) -> Option<i64> {
594        std::fs::read_to_string(dir.join(READING_FILE))
595            .ok()
596            .map(|s| s.trim().parse().unwrap())
597    }
598
599    fn mints(gate: &ClockGate) -> Option<i64> {
600        WallClock::now_micros(gate)
601    }
602
603    #[test]
604    fn without_any_signal_the_gate_stays_closed_and_mints_nothing() {
605        let dir = tempfile::tempdir().unwrap();
606        let fake = FakeSignals::at(T0);
607        for fallback in [Fallback::Rtc, Fallback::WallClock] {
608            let gate = gate(dir.path(), &fake, fallback);
609            if fallback == Fallback::WallClock {
610                fake.set_now(BUILD_SECS * SECOND - SECOND);
611            }
612            gate.tick();
613            assert!(!gate.is_open(), "{fallback:?}");
614            assert_eq!(mints(&gate), None);
615            assert_eq!(saved(dir.path()), None, "a closed gate saves nothing");
616        }
617    }
618
619    #[test]
620    fn ntp_sync_opens_the_gate() {
621        let dir = tempfile::tempdir().unwrap();
622        let fake = FakeSignals::at(T0);
623        let gate = gate(dir.path(), &fake, Fallback::Rtc);
624        gate.tick();
625        assert!(!gate.is_open());
626        fake.set_synced(true);
627        gate.tick();
628        assert!(gate.is_open());
629        assert!(gate.synced());
630        assert_eq!(mints(&gate), Some(T0));
631    }
632
633    #[test]
634    fn a_synced_peer_opens_the_gate() {
635        let dir = tempfile::tempdir().unwrap();
636        let fake = FakeSignals::at(T0);
637        let gate = gate(dir.path(), &fake, Fallback::Rtc);
638        gate.tick();
639        gate.open_by_peer();
640        assert!(gate.is_open());
641        assert!(!gate.synced());
642        assert_eq!(mints(&gate), Some(T0));
643    }
644
645    #[test]
646    fn an_rtc_at_or_after_the_build_opens_once_the_clock_passes_the_saved_reading() {
647        let dir = tempfile::tempdir().unwrap();
648        std::fs::write(
649            dir.path().join(READING_FILE),
650            (T0 + 10 * SECOND).to_string(),
651        )
652        .unwrap();
653        let fake = FakeSignals::at(T0);
654
655        fake.set_rtc(Some(BUILD_SECS - 1));
656        let gate = gate(dir.path(), &fake, Fallback::Rtc);
657        fake.set_now(T0 + 20 * SECOND);
658        gate.tick();
659        assert!(!gate.is_open(), "an RTC before the build time");
660
661        fake.set_rtc(None);
662        gate.tick();
663        assert!(!gate.is_open(), "no RTC");
664
665        fake.set_rtc(Some(BUILD_SECS));
666        fake.set_now(T0);
667        gate.tick();
668        assert!(!gate.is_open(), "the clock is behind the saved reading");
669
670        fake.set_now(T0 + 10 * SECOND);
671        gate.tick();
672        assert!(gate.is_open());
673    }
674
675    #[test]
676    fn without_the_rtc_rule_the_clock_opens_past_both_the_reading_and_the_build() {
677        let dir = tempfile::tempdir().unwrap();
678        std::fs::write(dir.path().join(READING_FILE), T0.to_string()).unwrap();
679        let fake = FakeSignals::at(T0 - SECOND);
680        fake.set_rtc(Some(BUILD_SECS + 1));
681        let gate = gate(dir.path(), &fake, Fallback::WallClock);
682        gate.tick();
683        assert!(!gate.is_open(), "behind the saved reading");
684
685        let fresh = tempfile::tempdir().unwrap();
686        let early = gate_at(fresh.path(), BUILD_SECS * SECOND - SECOND);
687        early.tick();
688        assert!(!early.is_open(), "behind the build time");
689
690        fake.set_now(T0);
691        gate.tick();
692        assert!(gate.is_open());
693    }
694
695    fn gate_at(dir: &Path, now: i64) -> Arc<ClockGate> {
696        gate(dir, &FakeSignals::at(now), Fallback::WallClock)
697    }
698
699    #[test]
700    fn a_backward_step_while_open_closes_the_gate() {
701        let dir = tempfile::tempdir().unwrap();
702        let fake = FakeSignals::at(T0);
703        let gate = gate(dir.path(), &fake, Fallback::Rtc);
704        gate.open_by_peer();
705        fake.set_now(T0 + 30 * SECOND);
706        assert_eq!(mints(&gate), Some(T0 + 30 * SECOND));
707
708        fake.set_now(T0 + 30 * SECOND - SECOND / 2);
709        assert!(gate.is_open(), "a sub-second correction keeps it open");
710
711        fake.set_now(T0 + 10 * SECOND);
712        assert_eq!(mints(&gate), None);
713        assert!(!gate.is_open());
714        gate.tick();
715        assert!(
716            !gate.is_open(),
717            "the peer's opening does not outlast a step"
718        );
719
720        fake.set_synced(true);
721        gate.tick();
722        assert!(gate.is_open());
723    }
724
725    #[test]
726    fn peer_time_is_bounded_by_a_day_when_synced_and_a_year_when_not() {
727        let dir = tempfile::tempdir().unwrap();
728        let fake = FakeSignals::at(T0);
729        fake.set_synced(true);
730        let gate = gate(dir.path(), &fake, Fallback::Rtc);
731        assert_eq!(gate.check_peer_time(T0 + DAY_MICROS), Ok(()));
732        let err = gate.check_peer_time(T0 + DAY_MICROS + 1).unwrap_err();
733        assert!(err.contains("more than a day ahead"), "{err}");
734
735        fake.set_synced(false);
736        let build = BUILD_SECS * SECOND;
737        assert_eq!(gate.check_peer_time(build + 365 * DAY_MICROS), Ok(()));
738        assert!(gate.check_peer_time(build + 365 * DAY_MICROS + 1).is_err());
739
740        let saved_dir = tempfile::tempdir().unwrap();
741        let reading = T0 + 100 * DAY_MICROS;
742        std::fs::write(saved_dir.path().join(READING_FILE), reading.to_string()).unwrap();
743        let unsynced = gate_at(saved_dir.path(), T0);
744        assert_eq!(unsynced.check_peer_time(reading + 365 * DAY_MICROS), Ok(()));
745        let err = unsynced
746            .check_peer_time(reading + 365 * DAY_MICROS + 1)
747            .unwrap_err();
748        assert!(err.contains("more than a year ahead"), "{err}");
749    }
750
751    #[test]
752    fn peer_time_past_the_ceiling_is_rejected_whatever_the_own_clock_reads() {
753        let dir = tempfile::tempdir().unwrap();
754        let fake = FakeSignals::at(HLC_CEILING_MICROS);
755        fake.set_synced(true);
756        let gate = gate(dir.path(), &fake, Fallback::Rtc);
757        assert_eq!(gate.check_peer_time(HLC_CEILING_MICROS), Ok(()));
758        let err = gate.check_peer_time(HLC_CEILING_MICROS + 1).unwrap_err();
759        assert!(err.contains("ceiling"), "{err}");
760    }
761
762    #[test]
763    fn the_saved_reading_only_rises_and_is_capped_at_the_ceiling() {
764        let dir = tempfile::tempdir().unwrap();
765        let fake = FakeSignals::at(T0);
766        let gate = gate(dir.path(), &fake, Fallback::Rtc);
767        gate.open_by_peer();
768        gate.tick();
769        assert_eq!(saved(dir.path()), Some(T0));
770
771        fake.set_now(T0 - SECOND / 2);
772        gate.tick();
773        assert!(gate.is_open());
774        assert_eq!(saved(dir.path()), Some(T0));
775
776        fake.set_now(T0 + 5 * SECOND);
777        gate.tick();
778        assert_eq!(saved(dir.path()), Some(T0 + 5 * SECOND));
779
780        let restarted = gate_at(dir.path(), T0);
781        restarted.open_by_peer();
782        restarted.tick();
783        assert_eq!(saved(dir.path()), Some(T0 + 5 * SECOND));
784
785        fake.set_now(HLC_CEILING_MICROS + DAY_MICROS);
786        gate.tick();
787        assert_eq!(mints(&gate), Some(HLC_CEILING_MICROS));
788        assert_eq!(gate.wall_micros(), HLC_CEILING_MICROS);
789        assert_eq!(saved(dir.path()), Some(HLC_CEILING_MICROS));
790        assert!(!dir.path().join("clock-reading.tmp").exists());
791    }
792
793    #[test]
794    fn observers_hear_the_gate_on_every_tick() {
795        let dir = tempfile::tempdir().unwrap();
796        let fake = FakeSignals::at(T0);
797        let gate = gate(dir.path(), &fake, Fallback::Rtc);
798        let heard = Arc::new(Mutex::new(Vec::new()));
799        let sink = heard.clone();
800        gate.observe(move |status| sink.lock().unwrap().push(*status));
801        gate.tick();
802        fake.set_synced(true);
803        gate.tick();
804        let heard = heard.lock().unwrap();
805        assert_eq!(
806            *heard,
807            [
808                GateStatus {
809                    open: false,
810                    synced: false,
811                    now_micros: T0
812                },
813                GateStatus {
814                    open: true,
815                    synced: true,
816                    now_micros: T0
817                },
818            ]
819        );
820    }
821
822    #[test]
823    fn the_ticker_reports_a_closing_without_waiting_out_its_minute() {
824        let dir = tempfile::tempdir().unwrap();
825        let fake = FakeSignals::at(T0);
826        let gate = gate(dir.path(), &fake, Fallback::Rtc);
827        gate.open_by_peer();
828        let (tx, rx) = std::sync::mpsc::channel();
829        gate.observe(move |status| {
830            let _ = tx.send(status.open);
831        });
832        gate.spawn_ticker();
833        assert_eq!(rx.recv_timeout(Duration::from_secs(10)), Ok(true));
834        fake.set_now(T0 - 10 * SECOND);
835        assert_eq!(mints(&gate), None);
836        let closed = std::iter::from_fn(|| rx.recv_timeout(Duration::from_secs(10)).ok())
837            .find(|open| !*open);
838        assert_eq!(closed, Some(false));
839    }
840
841    #[test]
842    fn the_hold_counts_from_the_first_closed_tick_and_restarts_on_reopening() {
843        let start = Instant::now();
844        let at = |secs: u64| start + Duration::from_secs(secs);
845        let mut hold = HoldTimer::default();
846        assert_eq!(hold.observe(false, at(0)), Some(Duration::ZERO));
847        assert_eq!(hold.observe(false, at(90)), Some(Duration::from_secs(90)));
848        assert_eq!(hold.observe(true, at(120)), None);
849        assert_eq!(hold.observe(false, at(180)), Some(Duration::ZERO));
850        assert_eq!(hold.observe(false, at(240)), Some(Duration::from_secs(60)));
851    }
852
853    #[test]
854    fn a_basis_is_ahead_only_past_a_minute_of_synchronised_time() {
855        let synced = GateStatus {
856            open: true,
857            synced: true,
858            now_micros: T0,
859        };
860        assert!(!synced.basis_ahead(T0 + 60 * SECOND));
861        assert!(synced.basis_ahead(T0 + 60 * SECOND + 1));
862        let unsynced = GateStatus {
863            synced: false,
864            ..synced
865        };
866        assert!(!unsynced.basis_ahead(T0 + DAY_MICROS));
867    }
868
869    #[test]
870    fn untrusted_is_found_through_context_and_nowhere_else() {
871        let wrapped = anyhow::Error::new(ClockUntrusted).context("storing a memory");
872        assert!(is_untrusted(&wrapped));
873        assert!(is_untrusted(&ClockUntrusted));
874        assert!(!is_untrusted(&anyhow::anyhow!("database is locked")));
875        assert!(!is_untrusted(&"wall clock not yet trusted".to_string()));
876    }
877
878    #[test]
879    fn the_test_gate_is_open_and_saves_nothing() {
880        let gate = ClockGate::open_for_tests();
881        assert!(gate.is_open());
882        assert!(mints(&gate).is_some());
883        gate.tick();
884        assert!(gate.reading_path.is_none());
885    }
886}