use std::sync::Arc;
use std::time::Duration;
use crate::cache::keyspace::{Area, KeySpace};
use crate::cache::BoxedCache;
#[derive(Clone)]
pub struct DistributedLock {
cache: BoxedCache,
space: KeySpace,
named: Option<String>,
}
impl DistributedLock {
#[must_use]
pub fn new(cache: BoxedCache) -> Self {
Self::in_space(cache, KeySpace::Global)
}
pub(crate) fn in_space(cache: BoxedCache, space: KeySpace) -> Self {
Self {
cache,
space,
named: None,
}
}
#[cfg(feature = "tenancy")]
#[deprecated(
since = "0.64.0",
note = "use `CacheScope::for_tenant(cache, tenant).locks()`; removed in 0.66.0"
)]
#[must_use]
pub fn for_tenant(self, tenant: &crate::tenancy::TenantSlug) -> Self {
Self::in_space(self.cache, KeySpace::Tenant(tenant.clone()))
}
#[deprecated(
since = "0.64.0",
note = "put the namespace in the lock name, or use `CacheScope::locks()` per tenant; removed in 0.66.0"
)]
#[must_use]
pub fn scoped(self, namespace: impl AsRef<str>) -> Self {
Self {
named: Some(namespace.as_ref().to_owned()),
..Self::new(self.cache)
}
}
fn rest(&self, name: &str) -> String {
match &self.named {
Some(ns) => format!("{ns}:{name}"),
None => name.to_owned(),
}
}
fn key_for(&self, name: &str) -> String {
self.space.key(Area::Lock, &self.rest(name))
}
fn legacy_key_for(&self, name: &str) -> Option<String> {
self.space.legacy_key(Area::Lock, &self.rest(name))
}
pub async fn try_acquire(&self, name: &str, ttl: Duration) -> Option<LockGuard> {
let key = self.key_for(name);
let legacy = self.legacy_key_for(name);
let token = format!(
"{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map_or(0, |d| d.as_nanos())
);
if let Some(old) = &legacy {
if !matches!(self.cache.add(old, &token, Some(ttl)).await, Ok(true)) {
return None;
}
}
let guard = LockGuard {
cache: self.cache.clone(),
key,
legacy,
token: Arc::new(token),
released: Arc::new(std::sync::atomic::AtomicBool::new(false)),
};
match self.cache.add(&guard.key, &guard.token, Some(ttl)).await {
Ok(true) => Some(guard),
_ => {
guard.release_key(guard.legacy.as_deref()).await;
guard.keep_until_ttl();
None
}
}
}
pub async fn with_lock<F, Fut, R>(&self, name: &str, ttl: Duration, body: F) -> Option<R>
where
F: FnOnce() -> Fut,
Fut: std::future::Future<Output = R>,
{
let guard = self.try_acquire(name, ttl).await?;
let result = body().await;
guard.release().await;
Some(result)
}
pub async fn once_per_period<F, Fut, T, E>(
&self,
name: &str,
period: Duration,
body: F,
) -> Option<Result<T, E>>
where
F: FnOnce() -> Fut,
Fut: std::future::Future<Output = Result<T, E>>,
{
let period = period.max(Duration::from_millis(1));
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default();
let window = now.as_millis() / period.as_millis();
self.once_in_window(name, period, window, body).await
}
pub(crate) async fn once_in_window<F, Fut, T, E>(
&self,
name: &str,
period: Duration,
window: u128,
body: F,
) -> Option<Result<T, E>>
where
F: FnOnce() -> Fut,
Fut: std::future::Future<Output = Result<T, E>>,
{
let guard = self
.try_acquire(&format!("{name}@{window}"), period.saturating_mul(2))
.await?;
if let Some(old) = window.checked_sub(2) {
let old = format!("{name}@{old}");
let _ = self.cache.delete(&self.key_for(&old)).await;
if let Some(legacy) = self.legacy_key_for(&old) {
let _ = self.cache.delete(&legacy).await;
}
}
let Some(running) = self.try_acquire(name, period).await else {
guard.release().await;
return None;
};
let mut held = ReleaseOnDrop(Some((guard, running)));
let result = crate::panic_guard::catch_unwind(async move { body().await }).await;
let (guard, running) = held.0.take().expect("held until here");
running.release().await;
match result {
Ok(Ok(v)) => {
guard.keep_until_ttl();
Some(Ok(v))
}
Ok(Err(e)) => {
guard.release().await;
Some(Err(e))
}
Err(panic) => {
guard.release().await;
std::panic::resume_unwind(panic)
}
}
}
}
struct ReleaseOnDrop(Option<(LockGuard, LockGuard)>);
impl Drop for ReleaseOnDrop {
fn drop(&mut self) {
if let (Some((window, running)), Ok(rt)) =
(self.0.take(), tokio::runtime::Handle::try_current())
{
rt.spawn(async move {
running.release().await;
window.release().await;
});
}
}
}
pub struct LockGuard {
cache: BoxedCache,
key: String,
legacy: Option<String>,
token: Arc<String>,
released: Arc<std::sync::atomic::AtomicBool>,
}
impl LockGuard {
pub async fn release(self) {
self.release_inner().await;
}
fn keep_until_ttl(self) {
self.released
.store(true, std::sync::atomic::Ordering::SeqCst);
}
async fn release_inner(&self) {
if self
.released
.swap(true, std::sync::atomic::Ordering::SeqCst)
{
return;
}
self.release_key(Some(&self.key)).await;
self.release_key(self.legacy.as_deref()).await;
}
async fn release_key(&self, key: Option<&str>) {
let Some(key) = key else { return };
let stored = self.cache.get(key).await.ok().flatten();
if stored.as_deref() == Some(self.token.as_str()) {
let _ = self.cache.delete(key).await;
}
}
}
impl Drop for LockGuard {
fn drop(&mut self) {
if !self.released.load(std::sync::atomic::Ordering::SeqCst) {
tracing::debug!(
key = %self.key,
"DistributedLock guard dropped without release(); waiting for TTL"
);
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::cache::InMemoryCache;
use std::sync::Arc as StdArc;
fn lock() -> DistributedLock {
let cache: BoxedCache = StdArc::new(InMemoryCache::new());
DistributedLock::new(cache)
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn only_one_of_many_racing_acquirers_wins() {
let cache: BoxedCache = StdArc::new(InMemoryCache::new());
let l = StdArc::new(DistributedLock::new(cache));
let mut handles = Vec::new();
for _ in 0..50 {
let l = l.clone();
handles.push(tokio::spawn(async move {
l.try_acquire("hot", Duration::from_secs(30))
.await
.map(|g| {
std::mem::forget(g);
})
.is_some()
}));
}
let mut winners = 0;
for h in handles {
if h.await.unwrap() {
winners += 1;
}
}
assert_eq!(winners, 1, "exactly one acquirer must win; got {winners}");
}
#[tokio::test]
async fn lock_is_reacquirable_after_release() {
let l = lock();
for _ in 0..5 {
let g = l.try_acquire("cycle", Duration::from_secs(5)).await;
assert!(g.is_some(), "should re-acquire after each release");
g.unwrap().release().await;
}
}
#[tokio::test]
async fn first_acquirer_succeeds() {
let l = lock();
let g = l.try_acquire("job", Duration::from_secs(5)).await;
assert!(g.is_some());
}
#[tokio::test]
async fn second_acquirer_blocked() {
let l = lock();
let g1 = l.try_acquire("job", Duration::from_secs(5)).await;
assert!(g1.is_some());
let g2 = l.try_acquire("job", Duration::from_secs(5)).await;
assert!(g2.is_none(), "second acquirer should be blocked");
}
#[tokio::test]
async fn release_lets_next_acquirer_in() {
let l = lock();
let g1 = l.try_acquire("job", Duration::from_secs(5)).await.unwrap();
g1.release().await;
let g2 = l.try_acquire("job", Duration::from_secs(5)).await;
assert!(g2.is_some(), "after release the lock is free");
}
#[tokio::test]
async fn different_names_dont_collide() {
let l = lock();
let a = l.try_acquire("a", Duration::from_secs(5)).await;
let b = l.try_acquire("b", Duration::from_secs(5)).await;
assert!(a.is_some());
assert!(b.is_some());
}
#[tokio::test]
async fn with_lock_runs_body_and_releases() {
let l = lock();
let result = l
.with_lock("job", Duration::from_secs(5), || async { 42 })
.await;
assert_eq!(result, Some(42));
let g = l.try_acquire("job", Duration::from_secs(5)).await;
assert!(g.is_some());
}
#[tokio::test]
async fn with_lock_returns_none_when_blocked() {
let l = lock();
let _hold = l.try_acquire("job", Duration::from_secs(5)).await.unwrap();
let result = l
.with_lock("job", Duration::from_secs(5), || async { 42 })
.await;
assert_eq!(result, None);
}
#[tokio::test]
async fn release_is_idempotent_at_least_once() {
let l = lock();
let g = l.try_acquire("job", Duration::from_secs(5)).await.unwrap();
g.release_inner().await;
}
#[tokio::test]
async fn ttl_expiry_frees_lock() {
let l = lock();
let g = l.try_acquire("job", Duration::from_millis(50)).await;
assert!(g.is_some());
std::mem::forget(g);
tokio::time::sleep(Duration::from_millis(120)).await;
let g2 = l.try_acquire("job", Duration::from_millis(50)).await;
assert!(g2.is_some(), "TTL expiry should free the lock");
}
#[tokio::test]
async fn release_after_ttl_does_not_clobber_new_holder() {
let l = lock();
let g1 = l
.try_acquire("job", Duration::from_millis(30))
.await
.unwrap();
tokio::time::sleep(Duration::from_millis(80)).await;
let g2 = l.try_acquire("job", Duration::from_secs(5)).await;
assert!(g2.is_some(), "new acquirer can claim after TTL");
g1.release().await;
let g3 = l.try_acquire("job", Duration::from_secs(5)).await;
assert!(
g3.is_none(),
"g2 still holds the lock — late g1.release() must not clear it"
);
drop(g2);
}
#[tokio::test]
async fn with_lock_releases_even_when_body_returns_unit() {
let l = lock();
let r: Option<()> = l
.with_lock("job", Duration::from_secs(5), || async {})
.await;
assert!(r.is_some());
let g = l.try_acquire("job", Duration::from_secs(5)).await;
assert!(g.is_some());
}
#[tokio::test]
async fn once_per_period_holds_the_window_after_the_body() {
let l = lock();
let day = Duration::from_secs(86_400);
let ok = |n: i32| async move { Ok::<_, ()>(n) };
assert_eq!(l.once_per_period("j", day, || ok(1)).await, Some(Ok(1)));
assert_eq!(l.once_per_period("j", day, || ok(2)).await, None);
}
fn window_now(period: Duration) -> u128 {
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap();
now.as_millis() / period.as_millis()
}
#[tokio::test]
async fn once_per_period_retries_after_err_or_panic() {
let l = lock();
let day = Duration::from_secs(86_400);
let failed = l.once_per_period("j", day, || async { Err::<(), _>("x") });
assert_eq!(failed.await, Some(Err("x")));
let l2 = l.clone();
let panicked = tokio::spawn(async move {
l2.once_per_period("j", day, || async {
if day > Duration::ZERO {
panic!("boom");
}
Ok::<(), ()>(())
})
.await
});
assert!(panicked.await.unwrap_err().is_panic());
let ok = l.once_per_period("j", day, || async { Ok::<_, ()>(1) });
assert_eq!(ok.await, Some(Ok(1)), "the window stayed taken");
assert!(l.try_acquire("j", day).await.is_some());
}
#[tokio::test]
async fn once_per_period_keys_ttl_and_cleanup() {
let cache: BoxedCache = StdArc::new(InMemoryCache::new());
let l = DistributedLock::new(cache.clone());
let period = Duration::from_millis(100);
let mut attempt = 0;
let (name, w) = loop {
attempt += 1;
let name = format!("t{attempt}");
let w = window_now(period);
for old in [w - 2, w - 1] {
cache
.set(&format!("lock:{name}@{old}"), "old", None)
.await
.unwrap();
}
let r = l.once_per_period(&name, period, || async { Ok::<_, ()>(()) });
assert!(r.await.is_some());
if window_now(period) == w {
break (name, w);
}
};
assert!(!cache
.exists(&format!("lock:{name}@{}", w - 2))
.await
.unwrap());
assert!(
cache
.exists(&format!("lock:{name}@{}", w - 1))
.await
.unwrap(),
"the last window's key is kept for skew"
);
tokio::time::sleep(Duration::from_millis(140)).await;
assert!(
cache.exists(&format!("lock:{name}@{w}")).await.unwrap(),
"TTL shorter than twice the period"
);
tokio::time::sleep(Duration::from_millis(120)).await;
assert!(
!cache.exists(&format!("lock:{name}@{w}")).await.unwrap(),
"TTL longer than twice the period"
);
}
#[tokio::test]
async fn once_per_period_clamps_a_zero_period() {
let l = lock();
let r = l.once_per_period("z", Duration::ZERO, || async { Ok::<_, ()>(()) });
assert_eq!(r.await, Some(Ok(())));
}
async fn sleep_to_offset(period: Duration, offset: Duration) {
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_millis();
let into = (now % period.as_millis()) as u64;
let wait = period.as_millis() as u64 - into + offset.as_millis() as u64;
tokio::time::sleep(Duration::from_millis(wait)).await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn once_per_period_does_not_overlap_a_long_run() {
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
let period = Duration::from_millis(200);
let cache: BoxedCache = StdArc::new(InMemoryCache::new());
let (a, b) = (
DistributedLock::new(cache.clone()),
DistributedLock::new(cache),
);
let running = StdArc::new(AtomicBool::new(false));
let (overlaps, b_runs) = (
StdArc::new(AtomicUsize::new(0)),
StdArc::new(AtomicUsize::new(0)),
);
sleep_to_offset(period, Duration::from_millis(100)).await;
let r = running.clone();
let pod_a = tokio::spawn(async move {
a.once_per_period("long", period, || async {
r.store(true, Ordering::SeqCst);
tokio::time::sleep(Duration::from_millis(150)).await;
r.store(false, Ordering::SeqCst);
Ok::<_, ()>(())
})
.await
});
let end = std::time::Instant::now() + Duration::from_millis(240);
while std::time::Instant::now() < end {
tokio::time::sleep(Duration::from_millis(10)).await;
let (r, o, n) = (running.clone(), overlaps.clone(), b_runs.clone());
b.once_per_period("long", period, || async move {
if r.load(Ordering::SeqCst) {
o.fetch_add(1, Ordering::SeqCst);
}
n.fetch_add(1, Ordering::SeqCst);
Ok::<_, ()>(())
})
.await;
}
assert!(pod_a.await.unwrap().is_some(), "pod A never ran");
assert_eq!(overlaps.load(Ordering::SeqCst), 0, "two runs overlapped");
assert_eq!(
b_runs.load(Ordering::SeqCst),
1,
"the next window ran {} times",
b_runs.load(Ordering::SeqCst)
);
}
#[tokio::test]
async fn a_busy_run_lock_leaves_the_window_for_a_later_tick() {
let l = lock();
let day = Duration::from_secs(86_400);
let other = l.try_acquire("j", day).await.expect("another pod's run");
let blocked = l.once_per_period("j", day, || async { Ok::<_, ()>(1) });
assert_eq!(blocked.await, None);
other.release().await;
let later = l.once_per_period("j", day, || async { Ok::<_, ()>(2) });
assert_eq!(later.await, Some(Ok(2)), "the window stayed taken");
}
#[tokio::test]
async fn an_aborted_run_frees_its_window() {
let l = lock();
let day = Duration::from_secs(86_400);
let (started_tx, started) = tokio::sync::oneshot::channel();
let l2 = l.clone();
let task = tokio::spawn(async move {
l2.once_per_period("j", day, || async move {
let _ = started_tx.send(());
tokio::time::sleep(Duration::from_secs(3600)).await;
Ok::<_, ()>(())
})
.await
});
started.await.unwrap();
task.abort();
assert!(task.await.unwrap_err().is_cancelled());
let mut again = None;
for _ in 0..50 {
again = l
.once_per_period("j", day, || async { Ok::<_, ()>(()) })
.await;
if again.is_some() {
break;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
assert_eq!(again, Some(Ok(())), "the aborted run kept the window");
}
#[tokio::test]
async fn a_body_panicking_on_call_frees_the_window() {
let l = lock();
let day = Duration::from_secs(86_400);
let l2 = l.clone();
let r = tokio::spawn(async move {
l2.once_per_period("j", day, || -> std::future::Ready<Result<(), ()>> {
panic!("before the future")
})
.await
});
assert!(r.await.unwrap_err().is_panic());
let again = l.once_per_period("j", day, || async { Ok::<_, ()>(()) });
assert_eq!(again.await, Some(Ok(())));
}
#[tokio::test]
async fn unscoped_lock_is_shared_across_tenants() {
let cache: BoxedCache = StdArc::new(InMemoryCache::new());
let lock = DistributedLock::new(cache);
let acme = lock
.try_acquire("daily_report", Duration::from_secs(30))
.await;
let globex = lock
.try_acquire("daily_report", Duration::from_secs(30))
.await;
assert!(acme.is_some(), "first caller takes the lock");
assert!(
globex.is_none(),
"unscoped, a second tenant contends for the same name and starves"
);
}
#[allow(deprecated)]
#[tokio::test]
async fn scoped_locks_do_not_contend_across_tenants() {
let cache: BoxedCache = StdArc::new(InMemoryCache::new());
let acme_lock = DistributedLock::new(cache.clone()).scoped("tenant:acme");
let globex_lock = DistributedLock::new(cache).scoped("tenant:globex");
let acme = acme_lock
.try_acquire("daily_report", Duration::from_secs(30))
.await;
let globex = globex_lock
.try_acquire("daily_report", Duration::from_secs(30))
.await;
assert!(acme.is_some(), "acme gets its own lock");
assert!(globex.is_some(), "globex gets its own lock, not acme's");
}
#[allow(deprecated)]
#[tokio::test]
async fn scoped_lock_still_excludes_within_a_tenant() {
let cache: BoxedCache = StdArc::new(InMemoryCache::new());
let lock = DistributedLock::new(cache).scoped("tenant:acme");
let first = lock
.try_acquire("daily_report", Duration::from_secs(30))
.await;
let second = lock
.try_acquire("daily_report", Duration::from_secs(30))
.await;
assert!(first.is_some());
assert!(second.is_none(), "one holder at a time, per tenant");
}
#[allow(deprecated)]
#[tokio::test]
async fn tenant_scopes_are_separated_by_slug() {
let cache: BoxedCache = StdArc::new(InMemoryCache::new());
let acme = DistributedLock::new(cache.clone()).scoped("tenant:acme");
let acme_corp = DistributedLock::new(cache).scoped("tenant:acme-corp");
assert!(acme
.try_acquire("j", Duration::from_secs(30))
.await
.is_some());
assert!(
acme_corp
.try_acquire("j", Duration::from_secs(30))
.await
.is_some(),
"`acme-corp` must not be blocked by `acme`'s lock"
);
}
#[cfg(feature = "tenancy")]
#[tokio::test]
async fn a_reused_slug_does_not_contend_with_the_purged_tenant() {
let cache: BoxedCache = StdArc::new(InMemoryCache::new());
let old = tenant_lock(&cache, 1, "acme");
let heir = tenant_lock(&cache, 2, "acme");
let ttl = Duration::from_secs(30);
let _held = old.try_acquire("j", ttl).await.expect("old");
assert!(heir.try_acquire("j", ttl).await.is_some(), "heir blocked");
}
#[cfg(feature = "tenancy")]
#[tokio::test]
async fn tenant_cache_clear_keeps_tenant_locks() {
use crate::cache::{Cache as _, ScopedCache};
let tenant = crate::tenancy::TenantSlug::new(1, "acme");
let cache: BoxedCache = StdArc::new(InMemoryCache::new());
let acme = tenant_lock(&cache, 1, "acme");
let ttl = Duration::from_secs(30);
let _held = acme.try_acquire("j", ttl).await.expect("acme");
ScopedCache::in_space(cache, KeySpace::Tenant(tenant), Area::App)
.clear()
.await
.unwrap();
assert!(acme.try_acquire("j", ttl).await.is_none(), "lock freed");
}
#[cfg(feature = "tenancy")]
fn tenant_lock(cache: &BoxedCache, id: i64, slug: &str) -> DistributedLock {
let t = crate::tenancy::TenantSlug::new(id, slug);
DistributedLock::in_space(cache.clone(), KeySpace::Tenant(t))
}
#[cfg(feature = "tenancy")]
#[tokio::test]
async fn a_tenant_lock_holds_the_new_and_the_old_key() {
let cache: BoxedCache = StdArc::new(InMemoryCache::new());
let ttl = Duration::from_secs(30);
let held = tenant_lock(&cache, 1, "acme").try_acquire("j", ttl).await;
let held = held.expect("free");
assert!(cache.exists("tenant:#org:1:acme:lock:j").await.unwrap());
assert!(!cache
.add("lock:tenant:#org:1:acme:j", "pod-063", Some(ttl))
.await
.unwrap());
held.release().await;
assert!(!cache.exists("tenant:#org:1:acme:lock:j").await.unwrap());
assert!(!cache.exists("lock:tenant:#org:1:acme:j").await.unwrap());
}
#[cfg(feature = "tenancy")]
#[tokio::test]
async fn a_lock_held_under_the_old_key_blocks_a_new_acquire() {
let cache: BoxedCache = StdArc::new(InMemoryCache::new());
let ttl = Duration::from_secs(30);
cache
.add("lock:tenant:#org:1:acme:j", "pod-063", Some(ttl))
.await
.unwrap();
let acme = tenant_lock(&cache, 1, "acme");
assert!(acme.try_acquire("j", ttl).await.is_none(), "ran twice");
assert!(
!cache.exists("tenant:#org:1:acme:lock:j").await.unwrap(),
"a refused acquire left the new key"
);
cache.delete("lock:tenant:#org:1:acme:j").await.unwrap();
cache
.add("tenant:#org:1:acme:lock:j", "other", Some(ttl))
.await
.unwrap();
assert!(acme.try_acquire("j", ttl).await.is_none());
assert!(!cache.exists("lock:tenant:#org:1:acme:j").await.unwrap());
}
#[cfg(feature = "tenancy")]
#[allow(deprecated)]
#[test]
fn deprecated_wrappers_map_onto_the_key_space() {
let cache: BoxedCache = StdArc::new(InMemoryCache::new());
let t = crate::tenancy::TenantSlug::new(1, "acme");
let lock = DistributedLock::new(cache.clone()).for_tenant(&t);
assert_eq!(lock.key_for("j"), "tenant:#org:1:acme:lock:j");
let named = DistributedLock::new(cache.clone())
.for_tenant(&t)
.scoped("r");
assert_eq!(named.key_for("j"), "lock:r:j");
assert_eq!(DistributedLock::new(cache).key_for("j"), "lock:j");
}
#[allow(deprecated)]
#[tokio::test]
async fn scoped_locks_use_their_namespace() {
let cache: BoxedCache = StdArc::new(InMemoryCache::new());
let ttl = Duration::from_secs(30);
let ns = DistributedLock::new(cache.clone()).scoped("reports");
let _held = ns.try_acquire("j", ttl).await.expect("scoped");
assert!(cache.exists("lock:reports:j").await.unwrap());
assert!(
ns.try_acquire("j", ttl).await.is_none(),
"same scope contends"
);
let other = DistributedLock::new(cache.clone()).scoped("billing");
assert!(other.try_acquire("j", ttl).await.is_some());
let unscoped = DistributedLock::new(cache);
assert!(unscoped.try_acquire("j", ttl).await.is_some());
}
}