Skip to main content

exo_core/
hlc.rs

1// Copyright 2026 Exochain Foundation
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at:
6//
7//     https://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14//
15// SPDX-License-Identifier: Apache-2.0
16
17//! Hybrid Logical Clock (HLC) for causal ordering.
18//!
19//! The HLC combines a caller-supplied physical component with a logical
20//! counter so that:
21//!
22//! 1. Timestamps are **monotonically increasing** even when the physical source
23//!    is stale or drifts backward.
24//! 2. Causally-related events are always ordered correctly.
25//! 3. No floating-point arithmetic is involved.
26
27use crate::{
28    error::{ExoError, Result},
29    types::Timestamp,
30};
31
32/// Default maximum tolerable forward drift in milliseconds.
33/// If a remote timestamp is more than this far ahead of our local physical
34/// source, we reject it as drift.
35const MAX_DRIFT_MS: u64 = 5_000; // 5 seconds
36/// Deterministic non-zero epoch for default clocks. This intentionally keeps
37/// zero-epoch records stale without reading host wall-clock time.
38const DEFAULT_DETERMINISTIC_PHYSICAL_MS: u64 = 1_000_000;
39
40/// A Hybrid Logical Clock instance.
41///
42/// Each node in the EXOCHAIN network maintains its own `HybridClock`.
43/// The clock is driven by a deterministic physical source and a logical
44/// counter. The default constructor does not read host time; runtime adapters
45/// that need deployment-specific physical metadata must inject an explicit HLC
46/// source.
47pub struct HybridClock {
48    /// Last-known HLC physical milliseconds.
49    physical: u64,
50    /// Logical counter within the same physical millisecond.
51    logical: u32,
52    /// Maximum tolerated remote forward drift for this clock.
53    max_drift_ms: u64,
54    /// Physical source — returns current HLC physical milliseconds.
55    physical_source: Box<dyn Fn() -> Result<u64> + Send>,
56}
57
58impl HybridClock {
59    /// Create a new clock driven by EXOCHAIN's deterministic default source.
60    #[must_use]
61    pub fn new() -> Self {
62        Self {
63            physical: 0,
64            logical: 0,
65            max_drift_ms: MAX_DRIFT_MS,
66            physical_source: Box::new(|| Ok(DEFAULT_DETERMINISTIC_PHYSICAL_MS)),
67        }
68    }
69
70    /// Create a clock with a custom physical source.
71    #[must_use]
72    pub fn with_wall_clock(physical_source: impl Fn() -> u64 + Send + 'static) -> Self {
73        Self::with_wall_clock_and_max_drift(physical_source, MAX_DRIFT_MS)
74    }
75
76    /// Create a clock with a custom physical source and drift tolerance.
77    #[must_use]
78    pub fn with_wall_clock_and_max_drift(
79        physical_source: impl Fn() -> u64 + Send + 'static,
80        max_drift_ms: u64,
81    ) -> Self {
82        Self::with_fallible_wall_clock_and_max_drift(move || Ok(physical_source()), max_drift_ms)
83    }
84
85    /// Create a clock with a fallible physical source.
86    #[must_use]
87    pub fn with_fallible_wall_clock(
88        physical_source: impl Fn() -> Result<u64> + Send + 'static,
89    ) -> Self {
90        Self::with_fallible_wall_clock_and_max_drift(physical_source, MAX_DRIFT_MS)
91    }
92
93    /// Create a clock with a fallible physical source and drift tolerance.
94    #[must_use]
95    pub fn with_fallible_wall_clock_and_max_drift(
96        physical_source: impl Fn() -> Result<u64> + Send + 'static,
97        max_drift_ms: u64,
98    ) -> Self {
99        Self {
100            physical: 0,
101            logical: 0,
102            max_drift_ms,
103            physical_source: Box::new(physical_source),
104        }
105    }
106
107    /// Return this clock's configured maximum forward drift in milliseconds.
108    #[must_use]
109    pub fn max_drift_ms(&self) -> u64 {
110        self.max_drift_ms
111    }
112
113    /// Reconcile a partition-recovery peer set to the quorum **median** of
114    /// their last-known timestamps.
115    ///
116    /// Per ratified decision D6 (2026-07-02): on reconnect after a network
117    /// partition, the recovering node converges its causal-ordering view to
118    /// the median of its peers' latest known HLC timestamps — never the
119    /// maximum. Silent accept-max would let a single drifted or malicious
120    /// peer steer history ordering for the whole network; the median is
121    /// resilient to any single outlier as long as a majority of the peer set
122    /// is honest.
123    ///
124    /// # Errors
125    ///
126    /// Returns `ExoError::ClockUnavailable` if `peer_timestamps` is empty —
127    /// there is nothing to reconcile against.
128    pub fn reconcile_partition_recovery(peer_timestamps: &[Timestamp]) -> Result<Timestamp> {
129        Ok(quorum_median(peer_timestamps)?.0)
130    }
131
132    /// Reconcile a partition-recovery peer set to the quorum median, and
133    /// additionally report any peer whose timestamp is a wide outlier
134    /// relative to that median.
135    ///
136    /// Per D6, time anomalies detected during partition recovery are
137    /// constitutional events, not silent log lines — the caller is expected
138    /// to record `anomalous_peers` as DAG evidence rather than folding the
139    /// outlier silently into the reconciled median.
140    ///
141    /// # Errors
142    ///
143    /// Returns `ExoError::ClockUnavailable` if `peer_timestamps` is empty.
144    pub fn reconcile_partition_recovery_with_anomaly_report(
145        peer_timestamps: &[Timestamp],
146    ) -> Result<PartitionRecoveryOutcome> {
147        let (median, max_drift_ms) = quorum_median(peer_timestamps)?;
148
149        let anomalous_peers: Vec<Timestamp> = peer_timestamps
150            .iter()
151            .copied()
152            .filter(|peer| is_wide_outlier(peer, &median, max_drift_ms))
153            .collect();
154
155        Ok(PartitionRecoveryOutcome {
156            median,
157            anomalous_peers,
158        })
159    }
160
161    /// Generate the next timestamp.
162    ///
163    /// Guarantees: the returned timestamp is strictly greater than any
164    /// previously returned by this clock.
165    pub fn now(&mut self) -> Result<Timestamp> {
166        let physical_now = (self.physical_source)()?;
167        if physical_now > self.physical {
168            self.physical = physical_now;
169            self.logical = 0;
170        } else {
171            advance_logical_or_carry_physical(&mut self.physical, &mut self.logical)?;
172        }
173        Ok(Timestamp::new(self.physical, self.logical))
174    }
175
176    /// Merge a remote timestamp and advance the local clock.
177    ///
178    /// The returned timestamp is guaranteed to be greater than both the
179    /// local state and the remote timestamp.
180    ///
181    /// # Errors
182    ///
183    /// Returns `ExoError::ClockDrift` if the remote timestamp is
184    /// unreasonably far ahead of the local physical source.
185    pub fn update(&mut self, remote: &Timestamp) -> Result<Timestamp> {
186        let physical_now = (self.physical_source)()?;
187
188        // Drift guard
189        if remote.physical_ms > physical_now.saturating_add(self.max_drift_ms) {
190            return Err(ExoError::ClockDrift {
191                physical_ms: remote.physical_ms,
192                tolerance_ms: self.max_drift_ms,
193            });
194        }
195
196        if physical_now > self.physical && physical_now > remote.physical_ms {
197            // Local physical source is ahead of both — reset logical
198            self.physical = physical_now;
199            self.logical = 0;
200        } else if self.physical == remote.physical_ms {
201            // Same physical — advance logical past both
202            self.logical = self.logical.max(remote.logical);
203            advance_logical_or_carry_physical(&mut self.physical, &mut self.logical)?;
204        } else if remote.physical_ms > self.physical {
205            // Remote is ahead — adopt remote physical, advance logical
206            self.physical = remote.physical_ms;
207            self.logical = remote.logical;
208            advance_logical_or_carry_physical(&mut self.physical, &mut self.logical)?;
209        } else {
210            // Local is ahead — advance own logical
211            advance_logical_or_carry_physical(&mut self.physical, &mut self.logical)?;
212        }
213
214        Ok(Timestamp::new(self.physical, self.logical))
215    }
216
217    /// Causal ordering check: returns `true` if `a` happened-before `b`.
218    #[must_use]
219    pub fn is_before(a: &Timestamp, b: &Timestamp) -> bool {
220        a < b
221    }
222
223    /// Return the current state as a `Timestamp` without advancing.
224    #[must_use]
225    pub fn current(&self) -> Timestamp {
226        Timestamp::new(self.physical, self.logical)
227    }
228}
229
230/// Outcome of [`HybridClock::reconcile_partition_recovery_with_anomaly_report`].
231#[derive(Debug, Clone, PartialEq, Eq)]
232pub struct PartitionRecoveryOutcome {
233    /// The quorum-median timestamp the recovering node should adopt.
234    pub median: Timestamp,
235    /// Peers whose last-known timestamp was a wide outlier relative to the
236    /// median (a candidate constitutional-event / DAG-evidence anomaly).
237    pub anomalous_peers: Vec<Timestamp>,
238}
239
240/// A wide outlier is a peer timestamp whose physical component differs from
241/// the reconciled median by more than the quorum's own drift tolerance. This
242/// reuses the same `MAX_DRIFT_MS` semantics as `HybridClock::update` so
243/// "anomalous" has one consistent meaning across the sync protocol.
244fn is_wide_outlier(peer: &Timestamp, median: &Timestamp, max_drift_ms: u64) -> bool {
245    peer.physical_ms.abs_diff(median.physical_ms) > max_drift_ms
246}
247
248/// Compute the quorum median of a peer timestamp set, ordering by the total
249/// `Timestamp` order (physical, then logical) so ties are broken
250/// deterministically. Returns the median timestamp plus the drift tolerance
251/// to apply against it.
252///
253/// For an even-sized peer set the lower of the two middle elements is used —
254/// deterministic and reproducible across nodes without floating-point
255/// averaging (which `Timestamp`'s integer fields do not support losslessly).
256fn quorum_median(peer_timestamps: &[Timestamp]) -> Result<(Timestamp, u64)> {
257    if peer_timestamps.is_empty() {
258        return Err(ExoError::ClockUnavailable {
259            reason: "cannot reconcile partition recovery: empty peer timestamp set".to_string(),
260        });
261    }
262
263    let mut sorted: Vec<Timestamp> = peer_timestamps.to_vec();
264    sorted.sort_unstable();
265
266    let mid = (sorted.len() - 1) / 2;
267    Ok((sorted[mid], MAX_DRIFT_MS))
268}
269
270fn advance_logical_or_carry_physical(physical: &mut u64, logical: &mut u32) -> Result<()> {
271    if *logical == u32::MAX {
272        if let Some(next_physical) = physical.checked_add(1) {
273            *physical = next_physical;
274            *logical = 0;
275            Ok(())
276        } else {
277            Err(ExoError::ClockOverflow {
278                physical_ms: *physical,
279                logical: *logical,
280            })
281        }
282    } else {
283        *logical += 1;
284        Ok(())
285    }
286}
287
288impl Default for HybridClock {
289    fn default() -> Self {
290        Self::new()
291    }
292}
293
294impl core::fmt::Debug for HybridClock {
295    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
296        f.debug_struct("HybridClock")
297            .field("physical", &self.physical)
298            .field("logical", &self.logical)
299            .field("max_drift_ms", &self.max_drift_ms)
300            .finish()
301    }
302}
303
304// ===========================================================================
305// Tests
306// ===========================================================================
307
308#[cfg(test)]
309mod tests {
310    use std::sync::{
311        Arc,
312        atomic::{AtomicU64, Ordering},
313    };
314
315    use super::*;
316
317    /// Helper: create a clock with a controllable wall time.
318    fn test_clock(initial: u64) -> (HybridClock, Arc<AtomicU64>) {
319        let time = Arc::new(AtomicU64::new(initial));
320        let t = Arc::clone(&time);
321        let clock = HybridClock::with_wall_clock(move || t.load(Ordering::Relaxed));
322        (clock, time)
323    }
324
325    #[test]
326    fn now_monotonic_same_wall_time() {
327        let (mut clock, _wall) = test_clock(1000);
328        let t1 = clock.now().expect("HLC timestamp");
329        let t2 = clock.now().expect("HLC timestamp");
330        let t3 = clock.now().expect("HLC timestamp");
331        assert!(t1 < t2);
332        assert!(t2 < t3);
333        // All share the same physical
334        assert_eq!(t1.physical_ms, 1000);
335        assert_eq!(t2.physical_ms, 1000);
336        assert_eq!(t3.physical_ms, 1000);
337        // Logical increments
338        assert_eq!(t1.logical, 0);
339        assert_eq!(t2.logical, 1);
340        assert_eq!(t3.logical, 2);
341    }
342
343    #[test]
344    fn now_advances_with_wall_clock() {
345        let (mut clock, wall) = test_clock(1000);
346        let t1 = clock.now().expect("HLC timestamp");
347        wall.store(2000, Ordering::Relaxed);
348        let t2 = clock.now().expect("HLC timestamp");
349        assert!(t1 < t2);
350        assert_eq!(t2.physical_ms, 2000);
351        assert_eq!(t2.logical, 0);
352    }
353
354    #[test]
355    fn now_handles_backward_wall_clock() {
356        let (mut clock, wall) = test_clock(2000);
357        let t1 = clock.now().expect("HLC timestamp");
358        wall.store(1000, Ordering::Relaxed); // wall goes backward
359        let t2 = clock.now().expect("HLC timestamp");
360        assert!(t1 < t2);
361        // Physical stays at 2000, logical increments
362        assert_eq!(t2.physical_ms, 2000);
363        assert_eq!(t2.logical, 1);
364    }
365
366    #[test]
367    fn update_wall_ahead_of_both() {
368        let (mut clock, wall) = test_clock(1000);
369        let _ = clock.now().expect("HLC timestamp");
370        wall.store(5000, Ordering::Relaxed);
371        let remote = Timestamp::new(3000, 5);
372        let result = clock.update(&remote).expect("ok");
373        assert_eq!(result.physical_ms, 5000);
374        assert_eq!(result.logical, 0);
375    }
376
377    #[test]
378    fn update_remote_ahead() {
379        let (mut clock, _wall) = test_clock(1000);
380        let _ = clock.now().expect("HLC timestamp");
381        let remote = Timestamp::new(2000, 10);
382        let result = clock.update(&remote).expect("ok");
383        assert_eq!(result.physical_ms, 2000);
384        assert_eq!(result.logical, 11);
385    }
386
387    #[test]
388    fn update_same_physical() {
389        let (mut clock, _wall) = test_clock(1000);
390        let _ = clock.now().expect("HLC timestamp"); // physical=1000, logical=0
391        let remote = Timestamp::new(1000, 5);
392        let result = clock.update(&remote).expect("ok");
393        assert_eq!(result.physical_ms, 1000);
394        // max(local_logical=0, remote_logical=5) + 1 = 6
395        assert_eq!(result.logical, 6);
396    }
397
398    #[test]
399    fn update_local_ahead() {
400        let (mut clock, wall) = test_clock(3000);
401        let _ = clock.now().expect("HLC timestamp"); // physical=3000, logical=0
402        wall.store(1000, Ordering::Relaxed); // wall backward
403        let remote = Timestamp::new(2000, 0);
404        let result = clock.update(&remote).expect("ok");
405        // Local physical (3000) > remote (2000), local advances logical
406        assert_eq!(result.physical_ms, 3000);
407        assert_eq!(result.logical, 1);
408    }
409
410    #[test]
411    fn update_rejects_excessive_drift() {
412        let (mut clock, _wall) = test_clock(1000);
413        let remote = Timestamp::new(1000 + MAX_DRIFT_MS + 1, 0);
414        let err = clock.update(&remote).unwrap_err();
415        assert!(matches!(err, ExoError::ClockDrift { .. }));
416    }
417
418    #[test]
419    fn update_rejects_remote_more_than_default_five_seconds_ahead() {
420        let (mut clock, _wall) = test_clock(1000);
421        let remote = Timestamp::new(1000 + 5_001, 0);
422
423        let err = clock
424            .update(&remote)
425            .expect_err("default HLC drift tolerance must be no more than five seconds");
426
427        assert!(matches!(
428            err,
429            ExoError::ClockDrift {
430                physical_ms: 6001,
431                tolerance_ms: 5000
432            }
433        ));
434    }
435
436    #[test]
437    fn update_accepts_at_drift_boundary() {
438        let (mut clock, _wall) = test_clock(1000);
439        let remote = Timestamp::new(1000 + MAX_DRIFT_MS, 0);
440        let result = clock.update(&remote);
441        assert!(result.is_ok());
442    }
443
444    #[test]
445    fn update_uses_deployment_configured_drift_tolerance() {
446        let mut boundary_clock = HybridClock::with_wall_clock_and_max_drift(|| 1000, 12_000);
447        let boundary = boundary_clock
448            .update(&Timestamp::new(13_000, 0))
449            .expect("configured drift boundary should be accepted");
450        assert_eq!(boundary, Timestamp::new(13_000, 1));
451
452        let mut over_boundary_clock = HybridClock::with_wall_clock_and_max_drift(|| 1000, 12_000);
453        let err = over_boundary_clock
454            .update(&Timestamp::new(13_001, 0))
455            .expect_err("remote timestamp beyond configured drift must be rejected");
456
457        assert!(matches!(
458            err,
459            ExoError::ClockDrift {
460                physical_ms: 13001,
461                tolerance_ms: 12000
462            }
463        ));
464    }
465
466    #[test]
467    fn is_before_ordering() {
468        let a = Timestamp::new(1, 0);
469        let b = Timestamp::new(1, 1);
470        let c = Timestamp::new(2, 0);
471        assert!(HybridClock::is_before(&a, &b));
472        assert!(HybridClock::is_before(&b, &c));
473        assert!(HybridClock::is_before(&a, &c));
474        assert!(!HybridClock::is_before(&b, &a));
475        assert!(!HybridClock::is_before(&a, &a));
476    }
477
478    #[test]
479    fn current_does_not_advance() {
480        let (mut clock, _wall) = test_clock(1000);
481        let _ = clock.now().expect("HLC timestamp");
482        let c1 = clock.current();
483        let c2 = clock.current();
484        assert_eq!(c1, c2);
485    }
486
487    #[test]
488    fn debug_format() {
489        let (clock, _wall) = test_clock(42);
490        let dbg = format!("{clock:?}");
491        assert!(dbg.contains("HybridClock"));
492    }
493
494    #[test]
495    fn default_clock() {
496        let mut clock = HybridClock::default();
497        let t = clock.now().expect("HLC timestamp");
498        assert_eq!(t, Timestamp::new(DEFAULT_DETERMINISTIC_PHYSICAL_MS, 0));
499    }
500
501    #[test]
502    fn default_clock_advances_logical_time_at_fixed_physical_epoch() {
503        let mut clock = HybridClock::default();
504
505        let first = clock.now().expect("first HLC timestamp");
506        let second = clock.now().expect("second HLC timestamp");
507        let third = clock.now().expect("third HLC timestamp");
508
509        assert_eq!(first, Timestamp::new(DEFAULT_DETERMINISTIC_PHYSICAL_MS, 0));
510        assert_eq!(second, Timestamp::new(DEFAULT_DETERMINISTIC_PHYSICAL_MS, 1));
511        assert_eq!(third, Timestamp::new(DEFAULT_DETERMINISTIC_PHYSICAL_MS, 2));
512    }
513
514    #[test]
515    fn production_hlc_source_does_not_read_host_wall_clock() {
516        let production = include_str!("hlc.rs")
517            .split("// ===========================================================================")
518            .next()
519            .expect("production section");
520        let system_time_now = format!("{}{}", "SystemTime::", "now()");
521        let date_now = format!("{}{}", "Date::", "now()");
522
523        assert!(
524            !production.contains(&system_time_now),
525            "production HLC must not read host SystemTime; callers must use deterministic HLC sources"
526        );
527        assert!(
528            !production.contains(&date_now),
529            "production HLC must not read browser Date.now; callers must use deterministic HLC sources"
530        );
531        assert!(
532            !production.contains("std::time"),
533            "production HLC must not import host wall-clock APIs"
534        );
535        assert!(
536            !production.contains("js_sys::Date"),
537            "production HLC must not import browser wall-clock APIs"
538        );
539        assert!(
540            !production.contains("fetch_update"),
541            "default HLC must not fabricate elapsed physical milliseconds from call count"
542        );
543    }
544
545    #[test]
546    fn concurrent_updates_maintain_monotonicity() {
547        let (mut clock, wall) = test_clock(100);
548        let _ = clock.now().expect("HLC timestamp");
549
550        // Simulate multiple rapid remote updates
551        let remotes = [
552            Timestamp::new(100, 3),
553            Timestamp::new(100, 1),
554            Timestamp::new(100, 7),
555            Timestamp::new(100, 2),
556        ];
557
558        let mut last = clock.current();
559        for r in &remotes {
560            let ts = clock.update(r).expect("ok");
561            assert!(ts > last, "monotonicity violated: {ts:?} <= {last:?}");
562            last = ts;
563        }
564
565        // Then advance wall clock
566        wall.store(200, Ordering::Relaxed);
567        let ts = clock.now().expect("HLC timestamp");
568        assert!(ts > last);
569        assert_eq!(ts.physical_ms, 200);
570        assert_eq!(ts.logical, 0);
571    }
572
573    #[test]
574    fn now_remains_monotonic_when_logical_counter_is_exhausted() {
575        let (mut clock, _wall) = test_clock(1000);
576        clock.physical = 1000;
577        clock.logical = u32::MAX;
578
579        let ts = clock.now().expect("HLC timestamp");
580
581        assert!(ts > Timestamp::new(1000, u32::MAX));
582        assert_eq!(ts.physical_ms, 1001);
583        assert_eq!(ts.logical, 0);
584    }
585
586    #[test]
587    fn update_remains_monotonic_when_logical_counter_is_exhausted() {
588        let (mut clock, _wall) = test_clock(1000);
589        clock.physical = 1000;
590        clock.logical = u32::MAX;
591
592        let ts = clock.update(&Timestamp::new(1000, u32::MAX)).expect("ok");
593
594        assert!(ts > Timestamp::new(1000, u32::MAX));
595        assert_eq!(ts.physical_ms, 1001);
596        assert_eq!(ts.logical, 0);
597    }
598
599    #[test]
600    fn now_rejects_terminal_clock_exhaustion_without_reusing_timestamp() {
601        let (mut clock, _wall) = test_clock(u64::MAX);
602        clock.physical = u64::MAX;
603        clock.logical = u32::MAX;
604
605        let err = clock
606            .now()
607            .expect_err("terminal HLC state must fail closed");
608
609        assert!(matches!(
610            err,
611            ExoError::ClockOverflow {
612                physical_ms: u64::MAX,
613                logical: u32::MAX
614            }
615        ));
616        assert_eq!(clock.current(), Timestamp::new(u64::MAX, u32::MAX));
617    }
618
619    #[test]
620    fn update_rejects_terminal_clock_exhaustion_without_reusing_timestamp() {
621        let (mut clock, _wall) = test_clock(u64::MAX);
622        clock.physical = u64::MAX;
623        clock.logical = u32::MAX;
624
625        let err = clock
626            .update(&Timestamp::new(u64::MAX, u32::MAX))
627            .expect_err("terminal HLC update must fail closed");
628
629        assert!(matches!(
630            err,
631            ExoError::ClockOverflow {
632                physical_ms: u64::MAX,
633                logical: u32::MAX
634            }
635        ));
636        assert_eq!(clock.current(), Timestamp::new(u64::MAX, u32::MAX));
637    }
638
639    #[test]
640    fn now_propagates_wall_clock_error_without_mutating_state() {
641        let mut clock = HybridClock::with_fallible_wall_clock(|| {
642            Err(ExoError::ClockUnavailable {
643                reason: "injected wall-clock failure".into(),
644            })
645        });
646
647        let err = clock
648            .now()
649            .expect_err("wall-clock failures must fail closed");
650
651        assert!(matches!(err, ExoError::ClockUnavailable { .. }));
652        assert_eq!(clock.current(), Timestamp::new(0, 0));
653    }
654
655    #[test]
656    fn update_propagates_wall_clock_error_without_mutating_state() {
657        let calls = Arc::new(AtomicU64::new(0));
658        let calls_for_clock = Arc::clone(&calls);
659        let mut clock = HybridClock::with_fallible_wall_clock(move || {
660            if calls_for_clock.fetch_add(1, Ordering::Relaxed) == 0 {
661                Ok(1000)
662            } else {
663                Err(ExoError::ClockUnavailable {
664                    reason: "injected wall-clock failure".into(),
665                })
666            }
667        });
668        let first = clock.now().expect("first timestamp");
669
670        let err = clock
671            .update(&Timestamp::new(1000, 0))
672            .expect_err("wall-clock failures must fail closed");
673
674        assert!(matches!(err, ExoError::ClockUnavailable { .. }));
675        assert_eq!(clock.current(), first);
676    }
677
678    #[test]
679    fn default_source_has_no_epoch_zero_fallback() {
680        let production = include_str!("hlc.rs")
681            .split("// ===========================================================================")
682            .next()
683            .expect("production section");
684
685        assert!(
686            !production.contains(".unwrap_or(0)"),
687            "HLC wall-clock failures must propagate instead of silently using epoch zero"
688        );
689    }
690
691    // -----------------------------------------------------------------
692    // VCG-012 RED — partition-recovery peer-set reconciliation (D6).
693    //
694    // D6 (ratified 2026-07-02): partition recovery converges to the
695    // quorum-MEDIAN of peers' latest known timestamps, never accept-max.
696    // One bad/drifted clock must not steer history ordering. No such
697    // reconciliation policy exists yet on `HybridClock` — this is expected
698    // compile-red until the D6 peer-set reconciliation API lands.
699    // -----------------------------------------------------------------
700
701    #[test]
702    fn partition_recovery_converges_to_quorum_median_not_accept_max() {
703        // Five peers' last-known timestamps before reconnect. Constructed so
704        // the median (100_500) is neither the max (999_999) nor the min
705        // (100_000) — this proves the reconciliation is genuinely
706        // median-based and not a disguised accept-max.
707        let peer_timestamps = vec![
708            Timestamp::new(100_000, 0),
709            Timestamp::new(100_200, 0),
710            Timestamp::new(100_500, 0),
711            Timestamp::new(100_800, 0),
712            Timestamp::new(999_999, 0), // one wildly drifted / malicious peer
713        ];
714
715        let reconciled = HybridClock::reconcile_partition_recovery(&peer_timestamps)
716            .expect("quorum-median reconciliation must succeed with an odd-sized peer set");
717
718        assert_eq!(
719            reconciled,
720            Timestamp::new(100_500, 0),
721            "partition recovery must converge to the quorum MEDIAN, not the max"
722        );
723        assert_ne!(
724            reconciled,
725            Timestamp::new(999_999, 0),
726            "silent accept-max is forbidden by D6: one bad clock must not steer history ordering"
727        );
728    }
729
730    #[test]
731    fn partition_recovery_flags_anomaly_when_a_peer_is_a_wide_outlier() {
732        let peer_timestamps = vec![
733            Timestamp::new(100_000, 0),
734            Timestamp::new(100_100, 0),
735            Timestamp::new(100_200, 0),
736            Timestamp::new(100_300, 0),
737            Timestamp::new(999_999, 0), // wide outlier vs. the quorum
738        ];
739
740        let outcome =
741            HybridClock::reconcile_partition_recovery_with_anomaly_report(&peer_timestamps)
742                .expect("reconciliation with anomaly reporting must succeed");
743
744        assert_eq!(outcome.median, Timestamp::new(100_200, 0));
745        assert!(
746            !outcome.anomalous_peers.is_empty(),
747            "a wide-outlier peer must be flagged, not silently folded into the median"
748        );
749    }
750}