1use crate::{
28 error::{ExoError, Result},
29 types::Timestamp,
30};
31
32const MAX_DRIFT_MS: u64 = 5_000; const DEFAULT_DETERMINISTIC_PHYSICAL_MS: u64 = 1_000_000;
39
40pub struct HybridClock {
48 physical: u64,
50 logical: u32,
52 max_drift_ms: u64,
54 physical_source: Box<dyn Fn() -> Result<u64> + Send>,
56}
57
58impl HybridClock {
59 #[must_use]
61 pub fn new() -> Self {
62 Self {
63 physical: 0,
64 logical: 0,
65 max_drift_ms: MAX_DRIFT_MS,
66 physical_source: Box::new(|| Ok(DEFAULT_DETERMINISTIC_PHYSICAL_MS)),
67 }
68 }
69
70 #[must_use]
72 pub fn with_wall_clock(physical_source: impl Fn() -> u64 + Send + 'static) -> Self {
73 Self::with_wall_clock_and_max_drift(physical_source, MAX_DRIFT_MS)
74 }
75
76 #[must_use]
78 pub fn with_wall_clock_and_max_drift(
79 physical_source: impl Fn() -> u64 + Send + 'static,
80 max_drift_ms: u64,
81 ) -> Self {
82 Self::with_fallible_wall_clock_and_max_drift(move || Ok(physical_source()), max_drift_ms)
83 }
84
85 #[must_use]
87 pub fn with_fallible_wall_clock(
88 physical_source: impl Fn() -> Result<u64> + Send + 'static,
89 ) -> Self {
90 Self::with_fallible_wall_clock_and_max_drift(physical_source, MAX_DRIFT_MS)
91 }
92
93 #[must_use]
95 pub fn with_fallible_wall_clock_and_max_drift(
96 physical_source: impl Fn() -> Result<u64> + Send + 'static,
97 max_drift_ms: u64,
98 ) -> Self {
99 Self {
100 physical: 0,
101 logical: 0,
102 max_drift_ms,
103 physical_source: Box::new(physical_source),
104 }
105 }
106
107 #[must_use]
109 pub fn max_drift_ms(&self) -> u64 {
110 self.max_drift_ms
111 }
112
113 pub fn reconcile_partition_recovery(peer_timestamps: &[Timestamp]) -> Result<Timestamp> {
129 Ok(quorum_median(peer_timestamps)?.0)
130 }
131
132 pub fn reconcile_partition_recovery_with_anomaly_report(
145 peer_timestamps: &[Timestamp],
146 ) -> Result<PartitionRecoveryOutcome> {
147 let (median, max_drift_ms) = quorum_median(peer_timestamps)?;
148
149 let anomalous_peers: Vec<Timestamp> = peer_timestamps
150 .iter()
151 .copied()
152 .filter(|peer| is_wide_outlier(peer, &median, max_drift_ms))
153 .collect();
154
155 Ok(PartitionRecoveryOutcome {
156 median,
157 anomalous_peers,
158 })
159 }
160
161 pub fn now(&mut self) -> Result<Timestamp> {
166 let physical_now = (self.physical_source)()?;
167 if physical_now > self.physical {
168 self.physical = physical_now;
169 self.logical = 0;
170 } else {
171 advance_logical_or_carry_physical(&mut self.physical, &mut self.logical)?;
172 }
173 Ok(Timestamp::new(self.physical, self.logical))
174 }
175
176 pub fn update(&mut self, remote: &Timestamp) -> Result<Timestamp> {
186 let physical_now = (self.physical_source)()?;
187
188 if remote.physical_ms > physical_now.saturating_add(self.max_drift_ms) {
190 return Err(ExoError::ClockDrift {
191 physical_ms: remote.physical_ms,
192 tolerance_ms: self.max_drift_ms,
193 });
194 }
195
196 if physical_now > self.physical && physical_now > remote.physical_ms {
197 self.physical = physical_now;
199 self.logical = 0;
200 } else if self.physical == remote.physical_ms {
201 self.logical = self.logical.max(remote.logical);
203 advance_logical_or_carry_physical(&mut self.physical, &mut self.logical)?;
204 } else if remote.physical_ms > self.physical {
205 self.physical = remote.physical_ms;
207 self.logical = remote.logical;
208 advance_logical_or_carry_physical(&mut self.physical, &mut self.logical)?;
209 } else {
210 advance_logical_or_carry_physical(&mut self.physical, &mut self.logical)?;
212 }
213
214 Ok(Timestamp::new(self.physical, self.logical))
215 }
216
217 #[must_use]
219 pub fn is_before(a: &Timestamp, b: &Timestamp) -> bool {
220 a < b
221 }
222
223 #[must_use]
225 pub fn current(&self) -> Timestamp {
226 Timestamp::new(self.physical, self.logical)
227 }
228}
229
230#[derive(Debug, Clone, PartialEq, Eq)]
232pub struct PartitionRecoveryOutcome {
233 pub median: Timestamp,
235 pub anomalous_peers: Vec<Timestamp>,
238}
239
240fn is_wide_outlier(peer: &Timestamp, median: &Timestamp, max_drift_ms: u64) -> bool {
245 peer.physical_ms.abs_diff(median.physical_ms) > max_drift_ms
246}
247
248fn quorum_median(peer_timestamps: &[Timestamp]) -> Result<(Timestamp, u64)> {
257 if peer_timestamps.is_empty() {
258 return Err(ExoError::ClockUnavailable {
259 reason: "cannot reconcile partition recovery: empty peer timestamp set".to_string(),
260 });
261 }
262
263 let mut sorted: Vec<Timestamp> = peer_timestamps.to_vec();
264 sorted.sort_unstable();
265
266 let mid = (sorted.len() - 1) / 2;
267 Ok((sorted[mid], MAX_DRIFT_MS))
268}
269
270fn advance_logical_or_carry_physical(physical: &mut u64, logical: &mut u32) -> Result<()> {
271 if *logical == u32::MAX {
272 if let Some(next_physical) = physical.checked_add(1) {
273 *physical = next_physical;
274 *logical = 0;
275 Ok(())
276 } else {
277 Err(ExoError::ClockOverflow {
278 physical_ms: *physical,
279 logical: *logical,
280 })
281 }
282 } else {
283 *logical += 1;
284 Ok(())
285 }
286}
287
288impl Default for HybridClock {
289 fn default() -> Self {
290 Self::new()
291 }
292}
293
294impl core::fmt::Debug for HybridClock {
295 fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
296 f.debug_struct("HybridClock")
297 .field("physical", &self.physical)
298 .field("logical", &self.logical)
299 .field("max_drift_ms", &self.max_drift_ms)
300 .finish()
301 }
302}
303
304#[cfg(test)]
309mod tests {
310 use std::sync::{
311 Arc,
312 atomic::{AtomicU64, Ordering},
313 };
314
315 use super::*;
316
317 fn test_clock(initial: u64) -> (HybridClock, Arc<AtomicU64>) {
319 let time = Arc::new(AtomicU64::new(initial));
320 let t = Arc::clone(&time);
321 let clock = HybridClock::with_wall_clock(move || t.load(Ordering::Relaxed));
322 (clock, time)
323 }
324
325 #[test]
326 fn now_monotonic_same_wall_time() {
327 let (mut clock, _wall) = test_clock(1000);
328 let t1 = clock.now().expect("HLC timestamp");
329 let t2 = clock.now().expect("HLC timestamp");
330 let t3 = clock.now().expect("HLC timestamp");
331 assert!(t1 < t2);
332 assert!(t2 < t3);
333 assert_eq!(t1.physical_ms, 1000);
335 assert_eq!(t2.physical_ms, 1000);
336 assert_eq!(t3.physical_ms, 1000);
337 assert_eq!(t1.logical, 0);
339 assert_eq!(t2.logical, 1);
340 assert_eq!(t3.logical, 2);
341 }
342
343 #[test]
344 fn now_advances_with_wall_clock() {
345 let (mut clock, wall) = test_clock(1000);
346 let t1 = clock.now().expect("HLC timestamp");
347 wall.store(2000, Ordering::Relaxed);
348 let t2 = clock.now().expect("HLC timestamp");
349 assert!(t1 < t2);
350 assert_eq!(t2.physical_ms, 2000);
351 assert_eq!(t2.logical, 0);
352 }
353
354 #[test]
355 fn now_handles_backward_wall_clock() {
356 let (mut clock, wall) = test_clock(2000);
357 let t1 = clock.now().expect("HLC timestamp");
358 wall.store(1000, Ordering::Relaxed); let t2 = clock.now().expect("HLC timestamp");
360 assert!(t1 < t2);
361 assert_eq!(t2.physical_ms, 2000);
363 assert_eq!(t2.logical, 1);
364 }
365
366 #[test]
367 fn update_wall_ahead_of_both() {
368 let (mut clock, wall) = test_clock(1000);
369 let _ = clock.now().expect("HLC timestamp");
370 wall.store(5000, Ordering::Relaxed);
371 let remote = Timestamp::new(3000, 5);
372 let result = clock.update(&remote).expect("ok");
373 assert_eq!(result.physical_ms, 5000);
374 assert_eq!(result.logical, 0);
375 }
376
377 #[test]
378 fn update_remote_ahead() {
379 let (mut clock, _wall) = test_clock(1000);
380 let _ = clock.now().expect("HLC timestamp");
381 let remote = Timestamp::new(2000, 10);
382 let result = clock.update(&remote).expect("ok");
383 assert_eq!(result.physical_ms, 2000);
384 assert_eq!(result.logical, 11);
385 }
386
387 #[test]
388 fn update_same_physical() {
389 let (mut clock, _wall) = test_clock(1000);
390 let _ = clock.now().expect("HLC timestamp"); let remote = Timestamp::new(1000, 5);
392 let result = clock.update(&remote).expect("ok");
393 assert_eq!(result.physical_ms, 1000);
394 assert_eq!(result.logical, 6);
396 }
397
398 #[test]
399 fn update_local_ahead() {
400 let (mut clock, wall) = test_clock(3000);
401 let _ = clock.now().expect("HLC timestamp"); wall.store(1000, Ordering::Relaxed); let remote = Timestamp::new(2000, 0);
404 let result = clock.update(&remote).expect("ok");
405 assert_eq!(result.physical_ms, 3000);
407 assert_eq!(result.logical, 1);
408 }
409
410 #[test]
411 fn update_rejects_excessive_drift() {
412 let (mut clock, _wall) = test_clock(1000);
413 let remote = Timestamp::new(1000 + MAX_DRIFT_MS + 1, 0);
414 let err = clock.update(&remote).unwrap_err();
415 assert!(matches!(err, ExoError::ClockDrift { .. }));
416 }
417
418 #[test]
419 fn update_rejects_remote_more_than_default_five_seconds_ahead() {
420 let (mut clock, _wall) = test_clock(1000);
421 let remote = Timestamp::new(1000 + 5_001, 0);
422
423 let err = clock
424 .update(&remote)
425 .expect_err("default HLC drift tolerance must be no more than five seconds");
426
427 assert!(matches!(
428 err,
429 ExoError::ClockDrift {
430 physical_ms: 6001,
431 tolerance_ms: 5000
432 }
433 ));
434 }
435
436 #[test]
437 fn update_accepts_at_drift_boundary() {
438 let (mut clock, _wall) = test_clock(1000);
439 let remote = Timestamp::new(1000 + MAX_DRIFT_MS, 0);
440 let result = clock.update(&remote);
441 assert!(result.is_ok());
442 }
443
444 #[test]
445 fn update_uses_deployment_configured_drift_tolerance() {
446 let mut boundary_clock = HybridClock::with_wall_clock_and_max_drift(|| 1000, 12_000);
447 let boundary = boundary_clock
448 .update(&Timestamp::new(13_000, 0))
449 .expect("configured drift boundary should be accepted");
450 assert_eq!(boundary, Timestamp::new(13_000, 1));
451
452 let mut over_boundary_clock = HybridClock::with_wall_clock_and_max_drift(|| 1000, 12_000);
453 let err = over_boundary_clock
454 .update(&Timestamp::new(13_001, 0))
455 .expect_err("remote timestamp beyond configured drift must be rejected");
456
457 assert!(matches!(
458 err,
459 ExoError::ClockDrift {
460 physical_ms: 13001,
461 tolerance_ms: 12000
462 }
463 ));
464 }
465
466 #[test]
467 fn is_before_ordering() {
468 let a = Timestamp::new(1, 0);
469 let b = Timestamp::new(1, 1);
470 let c = Timestamp::new(2, 0);
471 assert!(HybridClock::is_before(&a, &b));
472 assert!(HybridClock::is_before(&b, &c));
473 assert!(HybridClock::is_before(&a, &c));
474 assert!(!HybridClock::is_before(&b, &a));
475 assert!(!HybridClock::is_before(&a, &a));
476 }
477
478 #[test]
479 fn current_does_not_advance() {
480 let (mut clock, _wall) = test_clock(1000);
481 let _ = clock.now().expect("HLC timestamp");
482 let c1 = clock.current();
483 let c2 = clock.current();
484 assert_eq!(c1, c2);
485 }
486
487 #[test]
488 fn debug_format() {
489 let (clock, _wall) = test_clock(42);
490 let dbg = format!("{clock:?}");
491 assert!(dbg.contains("HybridClock"));
492 }
493
494 #[test]
495 fn default_clock() {
496 let mut clock = HybridClock::default();
497 let t = clock.now().expect("HLC timestamp");
498 assert_eq!(t, Timestamp::new(DEFAULT_DETERMINISTIC_PHYSICAL_MS, 0));
499 }
500
501 #[test]
502 fn default_clock_advances_logical_time_at_fixed_physical_epoch() {
503 let mut clock = HybridClock::default();
504
505 let first = clock.now().expect("first HLC timestamp");
506 let second = clock.now().expect("second HLC timestamp");
507 let third = clock.now().expect("third HLC timestamp");
508
509 assert_eq!(first, Timestamp::new(DEFAULT_DETERMINISTIC_PHYSICAL_MS, 0));
510 assert_eq!(second, Timestamp::new(DEFAULT_DETERMINISTIC_PHYSICAL_MS, 1));
511 assert_eq!(third, Timestamp::new(DEFAULT_DETERMINISTIC_PHYSICAL_MS, 2));
512 }
513
514 #[test]
515 fn production_hlc_source_does_not_read_host_wall_clock() {
516 let production = include_str!("hlc.rs")
517 .split("// ===========================================================================")
518 .next()
519 .expect("production section");
520 let system_time_now = format!("{}{}", "SystemTime::", "now()");
521 let date_now = format!("{}{}", "Date::", "now()");
522
523 assert!(
524 !production.contains(&system_time_now),
525 "production HLC must not read host SystemTime; callers must use deterministic HLC sources"
526 );
527 assert!(
528 !production.contains(&date_now),
529 "production HLC must not read browser Date.now; callers must use deterministic HLC sources"
530 );
531 assert!(
532 !production.contains("std::time"),
533 "production HLC must not import host wall-clock APIs"
534 );
535 assert!(
536 !production.contains("js_sys::Date"),
537 "production HLC must not import browser wall-clock APIs"
538 );
539 assert!(
540 !production.contains("fetch_update"),
541 "default HLC must not fabricate elapsed physical milliseconds from call count"
542 );
543 }
544
545 #[test]
546 fn concurrent_updates_maintain_monotonicity() {
547 let (mut clock, wall) = test_clock(100);
548 let _ = clock.now().expect("HLC timestamp");
549
550 let remotes = [
552 Timestamp::new(100, 3),
553 Timestamp::new(100, 1),
554 Timestamp::new(100, 7),
555 Timestamp::new(100, 2),
556 ];
557
558 let mut last = clock.current();
559 for r in &remotes {
560 let ts = clock.update(r).expect("ok");
561 assert!(ts > last, "monotonicity violated: {ts:?} <= {last:?}");
562 last = ts;
563 }
564
565 wall.store(200, Ordering::Relaxed);
567 let ts = clock.now().expect("HLC timestamp");
568 assert!(ts > last);
569 assert_eq!(ts.physical_ms, 200);
570 assert_eq!(ts.logical, 0);
571 }
572
573 #[test]
574 fn now_remains_monotonic_when_logical_counter_is_exhausted() {
575 let (mut clock, _wall) = test_clock(1000);
576 clock.physical = 1000;
577 clock.logical = u32::MAX;
578
579 let ts = clock.now().expect("HLC timestamp");
580
581 assert!(ts > Timestamp::new(1000, u32::MAX));
582 assert_eq!(ts.physical_ms, 1001);
583 assert_eq!(ts.logical, 0);
584 }
585
586 #[test]
587 fn update_remains_monotonic_when_logical_counter_is_exhausted() {
588 let (mut clock, _wall) = test_clock(1000);
589 clock.physical = 1000;
590 clock.logical = u32::MAX;
591
592 let ts = clock.update(&Timestamp::new(1000, u32::MAX)).expect("ok");
593
594 assert!(ts > Timestamp::new(1000, u32::MAX));
595 assert_eq!(ts.physical_ms, 1001);
596 assert_eq!(ts.logical, 0);
597 }
598
599 #[test]
600 fn now_rejects_terminal_clock_exhaustion_without_reusing_timestamp() {
601 let (mut clock, _wall) = test_clock(u64::MAX);
602 clock.physical = u64::MAX;
603 clock.logical = u32::MAX;
604
605 let err = clock
606 .now()
607 .expect_err("terminal HLC state must fail closed");
608
609 assert!(matches!(
610 err,
611 ExoError::ClockOverflow {
612 physical_ms: u64::MAX,
613 logical: u32::MAX
614 }
615 ));
616 assert_eq!(clock.current(), Timestamp::new(u64::MAX, u32::MAX));
617 }
618
619 #[test]
620 fn update_rejects_terminal_clock_exhaustion_without_reusing_timestamp() {
621 let (mut clock, _wall) = test_clock(u64::MAX);
622 clock.physical = u64::MAX;
623 clock.logical = u32::MAX;
624
625 let err = clock
626 .update(&Timestamp::new(u64::MAX, u32::MAX))
627 .expect_err("terminal HLC update must fail closed");
628
629 assert!(matches!(
630 err,
631 ExoError::ClockOverflow {
632 physical_ms: u64::MAX,
633 logical: u32::MAX
634 }
635 ));
636 assert_eq!(clock.current(), Timestamp::new(u64::MAX, u32::MAX));
637 }
638
639 #[test]
640 fn now_propagates_wall_clock_error_without_mutating_state() {
641 let mut clock = HybridClock::with_fallible_wall_clock(|| {
642 Err(ExoError::ClockUnavailable {
643 reason: "injected wall-clock failure".into(),
644 })
645 });
646
647 let err = clock
648 .now()
649 .expect_err("wall-clock failures must fail closed");
650
651 assert!(matches!(err, ExoError::ClockUnavailable { .. }));
652 assert_eq!(clock.current(), Timestamp::new(0, 0));
653 }
654
655 #[test]
656 fn update_propagates_wall_clock_error_without_mutating_state() {
657 let calls = Arc::new(AtomicU64::new(0));
658 let calls_for_clock = Arc::clone(&calls);
659 let mut clock = HybridClock::with_fallible_wall_clock(move || {
660 if calls_for_clock.fetch_add(1, Ordering::Relaxed) == 0 {
661 Ok(1000)
662 } else {
663 Err(ExoError::ClockUnavailable {
664 reason: "injected wall-clock failure".into(),
665 })
666 }
667 });
668 let first = clock.now().expect("first timestamp");
669
670 let err = clock
671 .update(&Timestamp::new(1000, 0))
672 .expect_err("wall-clock failures must fail closed");
673
674 assert!(matches!(err, ExoError::ClockUnavailable { .. }));
675 assert_eq!(clock.current(), first);
676 }
677
678 #[test]
679 fn default_source_has_no_epoch_zero_fallback() {
680 let production = include_str!("hlc.rs")
681 .split("// ===========================================================================")
682 .next()
683 .expect("production section");
684
685 assert!(
686 !production.contains(".unwrap_or(0)"),
687 "HLC wall-clock failures must propagate instead of silently using epoch zero"
688 );
689 }
690
691 #[test]
702 fn partition_recovery_converges_to_quorum_median_not_accept_max() {
703 let peer_timestamps = vec![
708 Timestamp::new(100_000, 0),
709 Timestamp::new(100_200, 0),
710 Timestamp::new(100_500, 0),
711 Timestamp::new(100_800, 0),
712 Timestamp::new(999_999, 0), ];
714
715 let reconciled = HybridClock::reconcile_partition_recovery(&peer_timestamps)
716 .expect("quorum-median reconciliation must succeed with an odd-sized peer set");
717
718 assert_eq!(
719 reconciled,
720 Timestamp::new(100_500, 0),
721 "partition recovery must converge to the quorum MEDIAN, not the max"
722 );
723 assert_ne!(
724 reconciled,
725 Timestamp::new(999_999, 0),
726 "silent accept-max is forbidden by D6: one bad clock must not steer history ordering"
727 );
728 }
729
730 #[test]
731 fn partition_recovery_flags_anomaly_when_a_peer_is_a_wide_outlier() {
732 let peer_timestamps = vec![
733 Timestamp::new(100_000, 0),
734 Timestamp::new(100_100, 0),
735 Timestamp::new(100_200, 0),
736 Timestamp::new(100_300, 0),
737 Timestamp::new(999_999, 0), ];
739
740 let outcome =
741 HybridClock::reconcile_partition_recovery_with_anomaly_report(&peer_timestamps)
742 .expect("reconciliation with anomaly reporting must succeed");
743
744 assert_eq!(outcome.median, Timestamp::new(100_200, 0));
745 assert!(
746 !outcome.anomalous_peers.is_empty(),
747 "a wide-outlier peer must be flagged, not silently folded into the median"
748 );
749 }
750}