use std::{
collections::HashMap,
fmt,
sync::Mutex,
time::{Duration, Instant},
};
use async_trait::async_trait;
use thiserror::Error;
use crate::{
clock::{SharedClock, system_clock},
result::{ClassifyError, ErrorKind},
};
use super::config::{validate_lease_owner, validate_worker_name};
use super::{WorkerConfigError, WorkerLeaseConfig};
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub struct WorkerLease {
worker: String,
owner: String,
token: u64,
expires_at: Instant,
}
impl WorkerLease {
pub fn new(
worker: impl Into<String>,
owner: impl Into<String>,
token: u64,
expires_at: Instant,
) -> Result<Self, WorkerLeaseError> {
let worker = worker.into();
let owner = owner.into();
validate_worker_name(&worker).map_err(WorkerLeaseError::InvalidConfig)?;
validate_lease_owner(&owner).map_err(WorkerLeaseError::InvalidConfig)?;
Ok(Self::from_valid_parts(worker, owner, token, expires_at))
}
fn from_valid_parts(worker: String, owner: String, token: u64, expires_at: Instant) -> Self {
Self {
worker,
owner,
token,
expires_at,
}
}
#[must_use]
pub fn worker(&self) -> &str {
&self.worker
}
#[must_use]
pub fn owner(&self) -> &str {
&self.owner
}
#[must_use]
pub const fn token(&self) -> u64 {
self.token
}
#[must_use]
pub const fn expires_at(&self) -> Instant {
self.expires_at
}
#[must_use]
pub fn should_renew(&self, now: Instant, renew_before: Duration) -> bool {
now.checked_add(renew_before)
.is_none_or(|renew_at| renew_at >= self.expires_at)
}
}
#[derive(Error)]
#[non_exhaustive]
pub enum WorkerLeaseError {
#[error("worker lease store lock is poisoned")]
LockPoisoned,
#[error("invalid worker lease configuration")]
InvalidConfig(#[from] WorkerConfigError),
#[error("worker lease store is unavailable")]
Store(#[source] crate::result::BoxError),
}
impl fmt::Debug for WorkerLeaseError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::LockPoisoned => formatter.write_str("LockPoisoned"),
Self::InvalidConfig(error) => {
formatter.debug_tuple("InvalidConfig").field(error).finish()
}
Self::Store(_) => formatter.debug_tuple("Store").field(&"[REDACTED]").finish(),
}
}
}
impl ClassifyError for WorkerLeaseError {
fn error_kind(&self) -> ErrorKind {
match self {
Self::LockPoisoned => ErrorKind::Internal,
Self::Store(_) => ErrorKind::Unavailable,
Self::InvalidConfig(source) => source.error_kind(),
}
}
fn error_code(&self) -> crate::result::ErrorCode {
match self {
Self::LockPoisoned | Self::Store(_) => {
crate::result::ErrorCode::internal("worker_lease_unavailable")
}
Self::InvalidConfig(source) => source.error_code(),
}
}
}
#[async_trait]
pub trait WorkerLeaseStore: Send + Sync + 'static {
async fn try_acquire(
&self,
worker: &str,
owner: &str,
config: WorkerLeaseConfig,
) -> Result<Option<WorkerLease>, WorkerLeaseError>;
async fn renew(
&self,
lease: &WorkerLease,
config: WorkerLeaseConfig,
) -> Result<Option<WorkerLease>, WorkerLeaseError>;
async fn release(&self, lease: &WorkerLease) -> Result<(), WorkerLeaseError>;
}
#[non_exhaustive]
pub struct LocalWorkerLeaseStore {
state: Mutex<LocalLeaseState>,
clock: SharedClock,
}
struct LocalLeaseState {
leases: HashMap<String, WorkerLease>,
next_token: u64,
}
impl LocalWorkerLeaseStore {
#[must_use]
pub fn new() -> Self {
Self::with_clock(system_clock())
}
#[must_use]
pub fn with_clock(clock: SharedClock) -> Self {
Self {
state: Mutex::new(LocalLeaseState {
leases: HashMap::new(),
next_token: 1,
}),
clock,
}
}
#[must_use]
pub fn clock(&self) -> &SharedClock {
&self.clock
}
}
impl fmt::Debug for LocalWorkerLeaseStore {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
let lease_count = self.state.lock().map(|state| state.leases.len()).ok();
formatter
.debug_struct("LocalWorkerLeaseStore")
.field("lease_count", &lease_count)
.finish_non_exhaustive()
}
}
impl Default for LocalWorkerLeaseStore {
fn default() -> Self {
Self::new()
}
}
#[async_trait]
impl WorkerLeaseStore for LocalWorkerLeaseStore {
async fn try_acquire(
&self,
worker: &str,
owner: &str,
config: WorkerLeaseConfig,
) -> Result<Option<WorkerLease>, WorkerLeaseError> {
validate_worker_name(worker)?;
validate_lease_owner(owner)?;
let mut state = self
.state
.lock()
.map_err(|_| WorkerLeaseError::LockPoisoned)?;
let now = self.clock.now();
let expires_at = worker_lease_expiry_after(now, config.lease_duration)?;
if let Some(existing) = state.leases.get(worker)
&& existing.expires_at > now
{
return Ok(None);
}
let lease = WorkerLease::from_valid_parts(
worker.to_owned(),
owner.to_owned(),
state.next_token,
expires_at,
);
state.next_token = state.next_token.wrapping_add(1);
state.leases.insert(worker.to_owned(), lease.clone());
Ok(Some(lease))
}
async fn renew(
&self,
lease: &WorkerLease,
config: WorkerLeaseConfig,
) -> Result<Option<WorkerLease>, WorkerLeaseError> {
let mut state = self
.state
.lock()
.map_err(|_| WorkerLeaseError::LockPoisoned)?;
let now = self.clock.now();
let expires_at = worker_lease_expiry_after(now, config.lease_duration)?;
let Some(existing) = state.leases.get_mut(lease.worker()) else {
return Ok(None);
};
if existing.owner != lease.owner
|| existing.token != lease.token
|| existing.expires_at <= now
{
return Ok(None);
}
existing.expires_at = expires_at;
Ok(Some(existing.clone()))
}
async fn release(&self, lease: &WorkerLease) -> Result<(), WorkerLeaseError> {
let mut state = self
.state
.lock()
.map_err(|_| WorkerLeaseError::LockPoisoned)?;
if let Some(existing) = state.leases.get(lease.worker())
&& existing.owner == lease.owner
&& existing.token == lease.token
{
state.leases.remove(lease.worker());
}
Ok(())
}
}
fn worker_lease_expiry_after(
now: Instant,
lease_duration: Duration,
) -> Result<Instant, WorkerLeaseError> {
now.checked_add(lease_duration)
.ok_or(WorkerLeaseError::InvalidConfig(
WorkerConfigError::InvalidLeaseDuration,
))
}
#[cfg(test)]
mod tests {
use std::sync::{Arc, TryLockError, mpsc};
use futures_util::FutureExt;
use crate::clock::{Clock, ManualClock};
use super::*;
struct PausingClock {
clock: ManualClock,
pause: Mutex<Option<(mpsc::Sender<()>, mpsc::Receiver<()>)>>,
}
impl Clock for PausingClock {
fn now(&self) -> Instant {
let now = self.clock.now();
let pause = self.pause.lock().unwrap().take();
if let Some((sampled, resume)) = pause {
sampled.send(()).unwrap();
resume.recv().unwrap();
}
now
}
}
fn lease_operation_uses_time_inside_its_state_lock(renew: bool) {
let clock = Arc::new(PausingClock {
clock: ManualClock::new(),
pause: Mutex::new(None),
});
let store = Arc::new(LocalWorkerLeaseStore::with_clock(clock.clone()));
let config = WorkerLeaseConfig::new(
Duration::from_secs(30),
Duration::from_secs(10),
Duration::from_secs(5),
)
.unwrap();
let lease = renew.then(|| {
store
.try_acquire("email", "node-a", config)
.now_or_never()
.unwrap()
.unwrap()
.unwrap()
});
let (sampled, sample_received) = mpsc::channel();
let (resume, resumed) = mpsc::channel();
*clock.pause.lock().unwrap() = Some((sampled, resumed));
let working_store = Arc::clone(&store);
let operation = std::thread::spawn(move || {
match lease {
Some(lease) => working_store.renew(&lease, config).now_or_never(),
None => working_store
.try_acquire("email", "node-a", config)
.now_or_never(),
}
.unwrap()
.unwrap()
});
sample_received.recv().unwrap();
let expired_while_waiting = match store.state.try_lock() {
Ok(guard) => {
clock.clock.advance(Duration::from_secs(60)).unwrap();
resume.send(()).unwrap();
drop(guard);
true
}
Err(TryLockError::WouldBlock) => {
resume.send(()).unwrap();
false
}
Err(TryLockError::Poisoned(_)) => panic!("lease state lock is poisoned"),
};
let result = operation.join().unwrap();
if renew && expired_while_waiting {
assert!(result.is_none(), "an expired lease must not be renewed");
} else {
let result = result.expect("an available lease should be acquired or renewed");
assert!(
result.expires_at() > clock.clock.now(),
"a lease must not expire while waiting to mutate store state"
);
}
}
#[test]
fn acquisition_samples_time_after_lock_contention() {
lease_operation_uses_time_inside_its_state_lock(false);
}
#[test]
fn renewal_does_not_revive_a_lease_expired_during_lock_contention() {
lease_operation_uses_time_inside_its_state_lock(true);
}
}