use std::{fmt, str::FromStr};
use schemars::{JsonSchema, Schema, SchemaGenerator, json_schema};
use serde::{Deserialize, Deserializer, Serialize, Serializer, de};
use thiserror::Error;
use crate::decimal::{UnsignedDecimalError, parse_bounded_u64};
use crate::{AttemptId, RunId, TenantId, Timestamp};
const MAX_DATABASE_ORDINAL: u64 = i64::MAX as u64;
const POSITIVE_I64_PATTERN: &str = "^[1-9][0-9]{0,18}$";
#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
pub struct FencingEpoch(u64);
impl FencingEpoch {
pub const FIRST: Self = Self(1);
pub const MAX: Self = Self(MAX_DATABASE_ORDINAL);
pub const fn new(value: u64) -> Result<Self, FencingEpochError> {
if value == 0 {
return Err(FencingEpochError::Zero);
}
if value > MAX_DATABASE_ORDINAL {
return Err(FencingEpochError::AboveMaximum);
}
Ok(Self(value))
}
#[must_use]
pub const fn get(self) -> u64 {
self.0
}
#[must_use]
pub const fn checked_next(self) -> Option<Self> {
if self.0 == MAX_DATABASE_ORDINAL {
None
} else {
Some(Self(self.0 + 1))
}
}
}
impl fmt::Display for FencingEpoch {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
self.0.fmt(formatter)
}
}
impl FromStr for FencingEpoch {
type Err = FencingEpochError;
fn from_str(value: &str) -> Result<Self, Self::Err> {
let value = parse_bounded_u64(value, MAX_DATABASE_ORDINAL)
.map_err(FencingEpochError::from_decimal_error)?;
Self::new(value)
}
}
impl Serialize for FencingEpoch {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
serializer.collect_str(self)
}
}
impl<'de> Deserialize<'de> for FencingEpoch {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
deserializer.deserialize_str(FencingEpochVisitor)
}
}
impl JsonSchema for FencingEpoch {
fn schema_name() -> std::borrow::Cow<'static, str> {
"FencingEpoch".into()
}
fn schema_id() -> std::borrow::Cow<'static, str> {
concat!(module_path!(), "::FencingEpoch").into()
}
fn json_schema(_: &mut SchemaGenerator) -> Schema {
json_schema!({
"type": "string",
"minLength": 1,
"maxLength": 19,
"pattern": POSITIVE_I64_PATTERN
})
}
fn inline_schema() -> bool {
true
}
}
struct FencingEpochVisitor;
impl de::Visitor<'_> for FencingEpochVisitor {
type Value = FencingEpoch;
fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str("a canonical positive decimal PostgreSQL BIGINT fencing epoch")
}
fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>
where
E: de::Error,
{
value.parse().map_err(E::custom)
}
}
#[derive(Clone, Debug, Eq, Error, PartialEq)]
#[non_exhaustive]
pub enum FencingEpochError {
#[error("fencing epoch must be positive")]
Zero,
#[error("fencing epoch exceeds the PostgreSQL BIGINT maximum")]
AboveMaximum,
#[error("fencing epoch must contain only unsigned ASCII decimal digits")]
InvalidFormat,
#[error("fencing epoch must use canonical decimal text")]
NonCanonical,
}
impl FencingEpochError {
const fn from_decimal_error(error: UnsignedDecimalError) -> Self {
match error {
UnsignedDecimalError::Empty | UnsignedDecimalError::InvalidCharacter { .. } => {
Self::InvalidFormat
}
UnsignedDecimalError::LeadingZero => Self::NonCanonical,
UnsignedDecimalError::TooLong { .. } | UnsignedDecimalError::Overflow => {
Self::AboveMaximum
}
}
}
}
#[derive(Clone, Debug, Deserialize, Eq, Hash, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct RunFence {
tenant_id: TenantId,
run_id: RunId,
attempt_id: AttemptId,
epoch: FencingEpoch,
}
impl RunFence {
#[must_use]
pub const fn new(
tenant_id: TenantId,
run_id: RunId,
attempt_id: AttemptId,
epoch: FencingEpoch,
) -> Self {
Self {
tenant_id,
run_id,
attempt_id,
epoch,
}
}
#[must_use]
pub const fn tenant_id(&self) -> &TenantId {
&self.tenant_id
}
#[must_use]
pub const fn run_id(&self) -> RunId {
self.run_id
}
#[must_use]
pub const fn attempt_id(&self) -> AttemptId {
self.attempt_id
}
#[must_use]
pub const fn epoch(&self) -> FencingEpoch {
self.epoch
}
}
#[derive(Clone, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct RunLease {
fence: RunFence,
acquired_at: Timestamp,
renewed_at: Timestamp,
expires_at: Timestamp,
}
impl RunLease {
pub fn restore(
fence: RunFence,
acquired_at: Timestamp,
renewed_at: Timestamp,
expires_at: Timestamp,
) -> Result<Self, RunLeaseError> {
Self::from_observations(fence, acquired_at, renewed_at, expires_at)
}
pub fn new(
fence: RunFence,
acquired_at: Timestamp,
expires_at: Timestamp,
) -> Result<Self, RunLeaseError> {
Self::from_observations(fence, acquired_at, acquired_at, expires_at)
}
pub fn first(
tenant_id: TenantId,
run_id: RunId,
attempt_id: AttemptId,
acquired_at: Timestamp,
expires_at: Timestamp,
) -> Result<Self, RunLeaseError> {
Self::new(
RunFence::new(tenant_id, run_id, attempt_id, FencingEpoch::FIRST),
acquired_at,
expires_at,
)
}
#[must_use]
pub const fn fence(&self) -> &RunFence {
&self.fence
}
#[must_use]
pub const fn acquired_at(&self) -> Timestamp {
self.acquired_at
}
#[must_use]
pub const fn renewed_at(&self) -> Timestamp {
self.renewed_at
}
#[must_use]
pub const fn expires_at(&self) -> Timestamp {
self.expires_at
}
pub fn validate_write(
&self,
fence: &RunFence,
observed_at: Timestamp,
) -> Result<(), RunLeaseValidationError> {
self.validate_fence(fence)?;
if observed_at < self.acquired_at {
return Err(RunLeaseValidationError::ObservationBeforeAcquisition {
acquired_at: self.acquired_at,
observed_at,
});
}
if observed_at < self.renewed_at {
return Err(RunLeaseValidationError::ObservationBeforeRenewal {
renewed_at: self.renewed_at,
observed_at,
});
}
if observed_at >= self.expires_at {
return Err(RunLeaseValidationError::Expired {
expires_at: self.expires_at,
observed_at,
});
}
Ok(())
}
pub fn renewed(
&self,
fence: &RunFence,
renewed_at: Timestamp,
expires_at: Timestamp,
) -> Result<Self, RunLeaseError> {
self.validate_fence(fence).map_err(RunLeaseError::fence)?;
if renewed_at < self.renewed_at {
return Err(RunLeaseError::RenewalClockRegression {
previous: self.renewed_at,
actual: renewed_at,
});
}
if renewed_at >= self.expires_at {
return Err(RunLeaseError::RenewalTooLate {
expires_at: self.expires_at,
renewed_at,
});
}
if expires_at <= self.expires_at {
return Err(RunLeaseError::ExpiryNotExtended {
previous: self.expires_at,
actual: expires_at,
});
}
Self::from_observations(self.fence.clone(), self.acquired_at, renewed_at, expires_at)
}
pub fn superseded(
&self,
attempt_id: AttemptId,
acquired_at: Timestamp,
expires_at: Timestamp,
) -> Result<Self, RunLeaseError> {
if attempt_id == self.fence.attempt_id {
return Err(RunLeaseError::AttemptReused { attempt_id });
}
if acquired_at < self.renewed_at {
return Err(RunLeaseError::SupersessionClockRegression {
previous: self.renewed_at,
actual: acquired_at,
});
}
let epoch = self
.fence
.epoch
.checked_next()
.ok_or(RunLeaseError::EpochOverflow)?;
let fence = RunFence::new(
self.fence.tenant_id.clone(),
self.fence.run_id,
attempt_id,
epoch,
);
Self::new(fence, acquired_at, expires_at)
}
fn from_observations(
fence: RunFence,
acquired_at: Timestamp,
renewed_at: Timestamp,
expires_at: Timestamp,
) -> Result<Self, RunLeaseError> {
if renewed_at < acquired_at {
return Err(RunLeaseError::RenewalBeforeAcquisition {
acquired_at,
renewed_at,
});
}
if expires_at <= renewed_at {
return Err(RunLeaseError::ExpiryNotAfterObservation {
observed_at: renewed_at,
expires_at,
});
}
Ok(Self {
fence,
acquired_at,
renewed_at,
expires_at,
})
}
fn validate_fence(&self, fence: &RunFence) -> Result<(), RunLeaseValidationError> {
if fence.tenant_id != self.fence.tenant_id {
return Err(RunLeaseValidationError::TenantMismatch);
}
if fence.run_id != self.fence.run_id {
return Err(RunLeaseValidationError::RunMismatch);
}
if fence.attempt_id != self.fence.attempt_id {
return Err(RunLeaseValidationError::AttemptMismatch {
expected: self.fence.attempt_id,
actual: fence.attempt_id,
});
}
if fence.epoch != self.fence.epoch {
return Err(RunLeaseValidationError::EpochMismatch {
expected: self.fence.epoch,
actual: fence.epoch,
});
}
Ok(())
}
}
impl fmt::Debug for RunLease {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("RunLease")
.field("fence", &self.fence)
.field("acquired_at", &self.acquired_at)
.field("renewed_at", &self.renewed_at)
.field("expires_at", &self.expires_at)
.finish_non_exhaustive()
}
}
impl<'de> Deserialize<'de> for RunLease {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct Wire {
fence: RunFence,
acquired_at: Timestamp,
renewed_at: Timestamp,
expires_at: Timestamp,
}
let wire = Wire::deserialize(deserializer)?;
Self::from_observations(
wire.fence,
wire.acquired_at,
wire.renewed_at,
wire.expires_at,
)
.map_err(de::Error::custom)
}
}
#[derive(Clone, Debug, Eq, Error, PartialEq)]
#[non_exhaustive]
pub enum RunLeaseError {
#[error("lease renewal {renewed_at} precedes acquisition {acquired_at}")]
RenewalBeforeAcquisition {
acquired_at: Timestamp,
renewed_at: Timestamp,
},
#[error("lease expiry {expires_at} must be later than observation {observed_at}")]
ExpiryNotAfterObservation {
observed_at: Timestamp,
expires_at: Timestamp,
},
#[error("lease renewal fencing token is stale or belongs to another run")]
Fence {
#[source]
source: RunLeaseValidationError,
},
#[error("lease renewal {actual} precedes previous observation {previous}")]
RenewalClockRegression {
previous: Timestamp,
actual: Timestamp,
},
#[error("lease renewal {renewed_at} is not before expiry {expires_at}")]
RenewalTooLate {
expires_at: Timestamp,
renewed_at: Timestamp,
},
#[error("renewed expiry {actual} must be later than previous expiry {previous}")]
ExpiryNotExtended {
previous: Timestamp,
actual: Timestamp,
},
#[error("successor lease cannot reuse attempt {attempt_id}")]
AttemptReused {
attempt_id: AttemptId,
},
#[error("lease supersession {actual} precedes previous observation {previous}")]
SupersessionClockRegression {
previous: Timestamp,
actual: Timestamp,
},
#[error("fencing epoch overflowed")]
EpochOverflow,
}
impl RunLeaseError {
const fn fence(source: RunLeaseValidationError) -> Self {
Self::Fence { source }
}
}
#[derive(Clone, Debug, Eq, Error, PartialEq)]
#[non_exhaustive]
pub enum RunLeaseValidationError {
#[error("fencing token tenant does not match the current lease")]
TenantMismatch,
#[error("fencing token run does not match the current lease")]
RunMismatch,
#[error("fencing token attempt {actual} does not match current attempt {expected}")]
AttemptMismatch {
expected: AttemptId,
actual: AttemptId,
},
#[error("fencing epoch {actual} does not match current epoch {expected}")]
EpochMismatch {
expected: FencingEpoch,
actual: FencingEpoch,
},
#[error("write observation {observed_at} precedes lease acquisition {acquired_at}")]
ObservationBeforeAcquisition {
acquired_at: Timestamp,
observed_at: Timestamp,
},
#[error("write observation {observed_at} precedes latest lease renewal {renewed_at}")]
ObservationBeforeRenewal {
renewed_at: Timestamp,
observed_at: Timestamp,
},
#[error("lease expired at {expires_at}; write observed at {observed_at}")]
Expired {
expires_at: Timestamp,
observed_at: Timestamp,
},
}
#[cfg(test)]
mod tests {
use super::*;
use proptest::prelude::*;
use serde_json::{Value, from_value, json, to_value};
fn at(offset_micros: i64) -> Timestamp {
let base = "2030-01-01T00:00:00.000000Z".parse::<Timestamp>().unwrap();
Timestamp::from_unix_micros(base.unix_micros() + offset_micros).unwrap()
}
fn tenant() -> TenantId {
TenantId::try_from("tenant-a").unwrap()
}
fn run() -> RunId {
"01912345-6789-7abc-8def-0123456789ab".parse().unwrap()
}
fn attempt(suffix: &str) -> AttemptId {
format!("01912345-6789-7abc-8def-0123456789{suffix}")
.parse()
.unwrap()
}
fn lease() -> RunLease {
RunLease::first(tenant(), run(), attempt("ac"), at(10), at(20)).unwrap()
}
#[test]
fn fencing_epochs_are_positive_canonical_database_ordinals() {
assert_eq!(FencingEpoch::new(1), Ok(FencingEpoch::FIRST));
assert_eq!(FencingEpoch::new(0), Err(FencingEpochError::Zero));
assert_eq!(
FencingEpoch::new(MAX_DATABASE_ORDINAL + 1),
Err(FencingEpochError::AboveMaximum)
);
assert_eq!(FencingEpoch::MAX.checked_next(), None);
assert_eq!(FencingEpoch::FIRST.checked_next().unwrap().get(), 2);
assert_eq!(
to_value(FencingEpoch::MAX).unwrap(),
json!(i64::MAX.to_string())
);
assert_eq!(
from_value::<FencingEpoch>(json!("1")).unwrap(),
FencingEpoch::FIRST
);
for invalid in [json!("0"), json!("01"), json!("x"), json!(1), Value::Null] {
assert!(from_value::<FencingEpoch>(invalid).is_err());
}
}
#[test]
fn fencing_epoch_schema_matches_runtime_shape() {
let schema = to_value(schemars::schema_for!(FencingEpoch)).unwrap();
assert_eq!(schema["type"], "string");
assert_eq!(schema["minLength"], 1);
assert_eq!(schema["maxLength"], 19);
assert_eq!(schema["pattern"], POSITIVE_I64_PATTERN);
}
#[test]
fn lease_expiry_is_exclusive() {
let lease = lease();
assert_eq!(lease.fence().epoch(), FencingEpoch::FIRST);
assert_eq!(lease.acquired_at(), at(10));
assert_eq!(lease.renewed_at(), at(10));
assert_eq!(lease.expires_at(), at(20));
assert_eq!(lease.validate_write(lease.fence(), at(10)), Ok(()));
assert_eq!(lease.validate_write(lease.fence(), at(19)), Ok(()));
assert_eq!(
lease.validate_write(lease.fence(), at(20)),
Err(RunLeaseValidationError::Expired {
expires_at: at(20),
observed_at: at(20),
})
);
assert!(matches!(
lease.validate_write(lease.fence(), at(9)),
Err(RunLeaseValidationError::ObservationBeforeAcquisition { .. })
));
}
#[test]
fn renewal_keeps_the_fence_and_strictly_extends_expiry() {
let lease = lease();
let renewed = lease.renewed(lease.fence(), at(19), at(30)).unwrap();
assert_eq!(renewed.fence(), lease.fence());
assert_eq!(renewed.acquired_at(), at(10));
assert_eq!(renewed.renewed_at(), at(19));
assert_eq!(renewed.expires_at(), at(30));
assert!(matches!(
lease.renewed(lease.fence(), at(20), at(30)),
Err(RunLeaseError::RenewalTooLate { .. })
));
assert!(matches!(
lease.renewed(lease.fence(), at(19), at(20)),
Err(RunLeaseError::ExpiryNotExtended { .. })
));
let twice = renewed.renewed(renewed.fence(), at(18), at(40));
assert!(matches!(
twice,
Err(RunLeaseError::RenewalClockRegression { .. })
));
}
#[test]
fn storage_restore_preserves_renewal_observations_and_revalidates_timing() {
let original = lease().renewed(lease().fence(), at(19), at(30)).unwrap();
let restored = RunLease::restore(
original.fence().clone(),
original.acquired_at(),
original.renewed_at(),
original.expires_at(),
)
.unwrap();
assert_eq!(restored, original);
assert_eq!(
restored.validate_write(restored.fence(), at(18)),
Err(RunLeaseValidationError::ObservationBeforeRenewal {
renewed_at: at(19),
observed_at: at(18),
})
);
assert!(matches!(
RunLease::restore(
original.fence().clone(),
original.acquired_at(),
at(9),
original.expires_at(),
),
Err(RunLeaseError::RenewalBeforeAcquisition { .. })
));
}
#[test]
fn supersession_fences_the_old_attempt_even_before_expiry() {
let old = lease();
let new = old.superseded(attempt("ad"), at(15), at(25)).unwrap();
assert_eq!(new.fence().epoch().get(), 2);
assert_eq!(new.fence().attempt_id(), attempt("ad"));
assert_eq!(
new.validate_write(old.fence(), at(16)),
Err(RunLeaseValidationError::AttemptMismatch {
expected: attempt("ad"),
actual: attempt("ac"),
})
);
assert_eq!(new.validate_write(new.fence(), at(16)), Ok(()));
assert!(matches!(
old.superseded(attempt("ac"), at(15), at(25)),
Err(RunLeaseError::AttemptReused { .. })
));
assert!(matches!(
old.superseded(attempt("ad"), at(9), at(25)),
Err(RunLeaseError::SupersessionClockRegression { .. })
));
}
#[test]
fn every_token_component_is_bound() {
let lease = lease();
let other_tenant = RunFence::new(
TenantId::try_from("tenant-b").unwrap(),
run(),
attempt("ac"),
FencingEpoch::FIRST,
);
assert_eq!(
lease.validate_write(&other_tenant, at(15)),
Err(RunLeaseValidationError::TenantMismatch)
);
let other_run = RunFence::new(
tenant(),
"01912345-6789-7abc-8def-0123456789bb".parse().unwrap(),
attempt("ac"),
FencingEpoch::FIRST,
);
assert_eq!(
lease.validate_write(&other_run, at(15)),
Err(RunLeaseValidationError::RunMismatch)
);
let other_epoch = RunFence::new(
tenant(),
run(),
attempt("ac"),
FencingEpoch::new(2).unwrap(),
);
assert!(matches!(
lease.validate_write(&other_epoch, at(15)),
Err(RunLeaseValidationError::EpochMismatch { .. })
));
}
#[test]
fn snapshot_deserialization_revalidates_timing_and_shape() {
let encoded = to_value(lease()).unwrap();
assert_eq!(from_value::<RunLease>(encoded.clone()).unwrap(), lease());
let mut expiry = encoded.clone();
expiry["expires_at"] = expiry["renewed_at"].clone();
assert!(from_value::<RunLease>(expiry).is_err());
let mut renewal = encoded.clone();
renewal["renewed_at"] = json!(at(9).to_string());
assert!(from_value::<RunLease>(renewal).is_err());
let mut extra = encoded;
extra["owner"] = json!("untrusted-worker");
assert!(from_value::<RunLease>(extra).is_err());
}
#[test]
fn successor_rejects_epoch_exhaustion() {
let fence = RunFence::new(tenant(), run(), attempt("ac"), FencingEpoch::MAX);
let lease = RunLease::new(fence, at(10), at(20)).unwrap();
assert_eq!(
lease.superseded(attempt("ad"), at(20), at(30)),
Err(RunLeaseError::EpochOverflow)
);
}
proptest! {
#[test]
fn accepted_renewals_preserve_token_and_monotonically_extend(
renewal_offsets in proptest::collection::vec(1_i64..10_000, 1..64)
) {
let mut lease = RunLease::first(
tenant(), run(), attempt("ac"), at(0), at(10_000)
).unwrap();
let fence = lease.fence().clone();
let mut observed = 0_i64;
for offset in renewal_offsets {
observed = (observed + offset).min(9_999);
let new_expiry = lease.expires_at().unix_micros().checked_add(10_000).unwrap();
let renewed = lease.renewed(
&fence,
Timestamp::from_unix_micros(at(0).unix_micros() + observed).unwrap(),
Timestamp::from_unix_micros(new_expiry).unwrap(),
).unwrap();
prop_assert_eq!(renewed.fence(), &fence);
prop_assert!(renewed.expires_at() > lease.expires_at());
lease = renewed;
}
}
}
}