use std::future::Future;
use chrono::{DateTime, Duration, FixedOffset, Timelike, Utc};
pub const MAX_WAIT_SECONDS: i64 = 5;
fn plan_version_time(
previous: Option<DateTime<FixedOffset>>,
now: DateTime<FixedOffset>,
) -> (DateTime<FixedOffset>, Duration) {
let now = truncate_to_second(now);
match previous.map(truncate_to_second) {
Some(prev) if prev < now => (now, Duration::zero()),
Some(prev) => {
let target = prev + Duration::seconds(1);
(target, target - now)
}
None => (now, Duration::zero()),
}
}
pub async fn next_version_time(previous: Option<DateTime<FixedOffset>>) -> DateTime<FixedOffset> {
let now = Utc::now().fixed_offset();
let (target, wait) = plan_version_time(previous, now);
if wait > Duration::zero()
&& let Err(outstanding) = wait_until_not_future(
target,
Duration::seconds(MAX_WAIT_SECONDS),
|| Utc::now().fixed_offset(),
tokio::time::sleep,
)
.await
{
tracing::warn!(
version_time = %target,
outstanding_seconds = outstanding.num_seconds(),
"the next valid versionTime is more than {MAX_WAIT_SECONDS}s ahead of this \
host's clock; stamping it rather than parking the request. The entry \
resolves once clocks agree. Check NTP on this host and on whatever minted \
the previous log entry."
);
}
target
}
async fn wait_until_not_future<Now, Sleep, SleepFuture>(
target: DateTime<FixedOffset>,
max_wait: Duration,
mut now: Now,
mut sleep: Sleep,
) -> Result<(), Duration>
where
Now: FnMut() -> DateTime<FixedOffset>,
Sleep: FnMut(std::time::Duration) -> SleepFuture,
SleepFuture: Future<Output = ()>,
{
let mut slept = Duration::zero();
loop {
let remaining = target - now();
if remaining <= Duration::zero() {
return Ok(());
}
if slept + remaining > max_wait {
return Err(remaining);
}
if let Ok(std_remaining) = remaining.to_std() {
sleep(std_remaining).await;
}
slept += remaining;
}
}
fn truncate_to_second(t: DateTime<FixedOffset>) -> DateTime<FixedOffset> {
t.with_nanosecond(0)
.expect("zero is always a valid nanosecond")
}
#[cfg(test)]
mod tests {
use super::{
MAX_WAIT_SECONDS, next_version_time, plan_version_time, truncate_to_second,
wait_until_not_future,
};
use chrono::{DateTime, Duration, FixedOffset, Utc};
use std::cell::Cell;
use std::rc::Rc;
fn now() -> DateTime<FixedOffset> {
Utc::now().fixed_offset()
}
#[test]
fn genesis_is_now_with_no_wait() {
let n = now();
let (target, wait) = plan_version_time(None, n);
assert_eq!(target.timestamp(), n.timestamp());
assert_eq!(wait, Duration::zero());
}
#[test]
fn previous_in_the_past_needs_no_wait() {
let n = now();
let prev = n - Duration::hours(1);
let (target, wait) = plan_version_time(Some(prev), n);
assert_eq!(target.timestamp(), n.timestamp());
assert_eq!(wait, Duration::zero());
}
#[test]
fn same_second_as_previous_waits_exactly_one_second() {
let n = now();
let prev = n;
let (target, wait) = plan_version_time(Some(prev), n);
assert!(target > prev, "target must be strictly after previous");
assert_eq!(
target.timestamp(),
prev.timestamp() + 1,
"target must be exactly the next second"
);
assert_eq!(wait, Duration::seconds(1));
}
#[test]
fn previous_ahead_of_now_waits_past_it() {
let n = now();
let prev = n + Duration::seconds(3);
let (target, wait) = plan_version_time(Some(prev), n);
assert!(target > prev);
assert_eq!(target.timestamp(), prev.timestamp() + 1);
assert_eq!(wait, Duration::seconds(4));
}
#[tokio::test(start_paused = true)]
async fn waits_for_rapid_back_to_back_calls() {
let genesis = next_version_time(None).await;
let started = tokio::time::Instant::now();
let update = next_version_time(Some(genesis)).await;
let elapsed = started.elapsed();
assert!(update > genesis, "update must be strictly after genesis");
assert_eq!(update.timestamp(), genesis.timestamp() + 1);
assert!(
elapsed >= std::time::Duration::from_secs(1),
"must have waited out the collision, elapsed={elapsed:?}"
);
}
#[tokio::test]
async fn rapid_update_is_not_future_dated_when_returned() {
let genesis = next_version_time(None).await;
let update = next_version_time(Some(genesis)).await;
assert!(update > genesis, "update must be strictly after genesis");
assert!(
update <= Utc::now().fixed_offset(),
"update must not be future-dated when returned: update={update}"
);
}
#[tokio::test]
async fn waits_again_after_a_backward_clock_adjustment() {
let start = truncate_to_second(now());
let target = start + Duration::seconds(1);
let current = Rc::new(Cell::new(start));
let sleeps = Rc::new(std::cell::RefCell::new(Vec::new()));
let current_for_clock = Rc::clone(¤t);
let current_for_sleep = Rc::clone(¤t);
let sleeps_for_sleep = Rc::clone(&sleeps);
let outcome = wait_until_not_future(
target,
Duration::seconds(10),
move || current_for_clock.get(),
move |duration| {
sleeps_for_sleep.borrow_mut().push(duration);
let next = if sleeps_for_sleep.borrow().len() == 1 {
start - Duration::seconds(2)
} else {
target
};
current_for_sleep.set(next);
std::future::ready(())
},
)
.await;
assert_eq!(outcome, Ok(()), "the wait must complete within its budget");
assert_eq!(
sleeps.borrow().as_slice(),
[
std::time::Duration::from_secs(1),
std::time::Duration::from_secs(3)
],
"a backward clock adjustment must trigger another wait"
);
}
#[tokio::test]
async fn gives_up_rather_than_parking_on_a_far_future_target() {
let start = truncate_to_second(now());
let target = start + Duration::hours(1);
let sleeps = Rc::new(std::cell::RefCell::new(Vec::new()));
let sleeps_for_sleep = Rc::clone(&sleeps);
let outcome = wait_until_not_future(
target,
Duration::seconds(MAX_WAIT_SECONDS),
move || start,
move |duration| {
sleeps_for_sleep.borrow_mut().push(duration);
std::future::ready(())
},
)
.await;
assert_eq!(outcome, Err(Duration::hours(1)));
assert!(
sleeps.borrow().is_empty(),
"an over-budget wait must not sleep at all, slept {:?}",
sleeps.borrow()
);
}
#[tokio::test]
async fn next_version_time_returns_promptly_despite_a_far_future_previous() {
let previous = Utc::now().fixed_offset() + Duration::hours(1);
let started = std::time::Instant::now();
let next = next_version_time(Some(previous)).await;
let elapsed = started.elapsed();
assert!(next > previous, "strict increase holds regardless");
assert_eq!(next.timestamp(), previous.timestamp() + 1);
assert!(
elapsed < std::time::Duration::from_secs(MAX_WAIT_SECONDS as u64),
"must not have waited out the skew, elapsed={elapsed:?}"
);
}
#[tokio::test(start_paused = true)]
async fn clamps_against_a_legacy_previous_entry() {
let legacy_genesis = Utc::now().fixed_offset() - Duration::seconds(30);
let next = next_version_time(Some(legacy_genesis)).await;
assert!(next > legacy_genesis);
assert_ne!(next.timestamp(), legacy_genesis.timestamp());
}
#[tokio::test(start_paused = true)]
async fn stays_increasing_across_a_run() {
let mut prev = Utc::now().fixed_offset() - Duration::seconds(5);
for index in 1..=5 {
let next = next_version_time(Some(prev)).await;
assert!(
next > prev,
"entry {index} must be strictly after its predecessor"
);
assert_ne!(next.timestamp(), prev.timestamp());
prev = next;
}
}
#[tokio::test(start_paused = true)]
async fn concurrent_calls_against_the_same_previous_agree() {
let genesis = next_version_time(None).await;
let (a, b) = tokio::join!(
next_version_time(Some(genesis)),
next_version_time(Some(genesis)),
);
assert_eq!(a.timestamp(), b.timestamp());
assert!(a > genesis);
}
}