use super::management::{LOCK_LOST, Management, ManagementError};
use crate::shutdown::CancellationToken;
use fe2o3_amqp::types::primitives::Uuid;
use std::{
collections::HashMap,
sync::{Arc, Mutex},
time::{Duration, SystemTime, UNIX_EPOCH},
};
use tokio::time::Instant;
use tracing::warn;
const BATCH: usize = 100;
const MIN_LOCK: Duration = Duration::from_secs(5);
const MAX_LOCK: Duration = Duration::from_secs(5 * 60);
struct Lock {
token: Uuid,
expires: Instant,
duration: Duration,
revoked: CancellationToken,
}
#[derive(Default)]
pub(super) struct Locks {
entries: Mutex<HashMap<u64, Lock>>,
next: Mutex<u64>,
}
fn remaining(until: i64, now: SystemTime) -> Duration {
let now = now
.duration_since(UNIX_EPOCH)
.map_or(0, |since| since.as_millis());
Duration::from_millis(u64::try_from(i128::from(until) - now as i128).unwrap_or(0))
}
impl Locks {
pub(super) fn add(
&self,
token: Uuid,
locked_until: Option<i64>,
now: (Instant, SystemTime),
) -> Option<(u64, CancellationToken)> {
let left = locked_until.map_or(MIN_LOCK, |until| remaining(until, now.1));
if left.is_zero() {
return None;
}
let id = {
let mut next = self.next.lock().unwrap();
*next += 1;
*next
};
let revoked = CancellationToken::new();
self.entries.lock().unwrap().insert(
id,
Lock {
token,
expires: now.0 + left,
duration: left.clamp(MIN_LOCK, MAX_LOCK),
revoked: revoked.clone(),
},
);
Some((id, revoked))
}
pub(super) fn remove(&self, id: u64) -> bool {
self.entries.lock().unwrap().remove(&id).is_some()
}
pub(super) fn revoke(&self, id: u64) {
if let Some(lock) = self.entries.lock().unwrap().remove(&id) {
lock.revoked.cancel();
}
}
pub(super) fn revoke_all(&self) {
for (_, lock) in self.entries.lock().unwrap().drain() {
lock.revoked.cancel();
}
}
fn due(&self, now: Instant) -> Vec<(u64, Uuid)> {
let mut entries = self.entries.lock().unwrap();
entries.retain(|_, lock| {
let expired = lock.expires <= now;
if expired {
lock.revoked.cancel();
}
!expired
});
entries
.iter()
.filter(|(_, lock)| {
lock.expires.saturating_duration_since(now) <= lock.duration * 2 / 3
})
.map(|(id, lock)| (*id, lock.token.clone()))
.collect()
}
fn renewed(&self, id: u64, expires: Instant, now: Instant) {
if let Some(lock) = self.entries.lock().unwrap().get_mut(&id) {
lock.expires = lock.expires.max(expires);
lock.duration = lock
.duration
.max(expires.saturating_duration_since(now))
.clamp(MIN_LOCK, MAX_LOCK);
}
}
fn interval() -> Duration {
MIN_LOCK / 5
}
}
pub(super) async fn keep(mut management: Management, locks: Arc<Locks>, stop: CancellationToken) {
let mut ticks = tokio::time::interval(Locks::interval());
loop {
tokio::select! {
_ = stop.cancelled() => {
management.close().await;
return;
}
_ = ticks.tick() => {
let due = locks.due(Instant::now());
for chunk in due.chunks(BATCH) {
renew(&mut management, &locks, chunk).await;
}
}
}
}
}
async fn renew(management: &mut Management, locks: &Locks, chunk: &[(u64, Uuid)]) {
let tokens: Vec<Uuid> = chunk.iter().map(|(_, token)| token.clone()).collect();
match management.renew_locks(&tokens).await {
Ok(expirations) => {
for ((id, _), expiration) in chunk.iter().zip(expirations) {
renewed(locks, *id, expiration.milliseconds());
}
}
Err(_) if chunk.len() > 1 => {
for (id, token) in chunk {
match management.renew_locks(std::slice::from_ref(token)).await {
Ok(expirations) => renewed(locks, *id, expirations[0].milliseconds()),
Err(error) => failed(locks, *id, error),
}
}
}
Err(error) => failed(locks, chunk[0].0, error),
}
}
fn renewed(locks: &Locks, id: u64, until: i64) {
let now = (Instant::now(), SystemTime::now());
locks.renewed(id, now.0 + remaining(until, now.1), now.0);
}
fn failed(locks: &Locks, id: u64, error: ManagementError) {
match error {
ManagementError::Status {
code: LOCK_LOST, ..
} => locks.revoke(id),
error => warn!(%error, "renewing a Service Bus message lock failed"),
}
}
#[cfg(test)]
mod tests {
use super::*;
fn at(now: SystemTime, offset: Duration) -> i64 {
(now + offset)
.duration_since(UNIX_EPOCH)
.unwrap()
.as_millis() as i64
}
#[test]
fn locks_are_due_after_a_third_of_their_duration() {
let locks = Locks::default();
let start = (Instant::now(), SystemTime::now());
let (id, revoked) = locks
.add(
Uuid::from([1; 16]),
Some(at(start.1, Duration::from_secs(30))),
start,
)
.unwrap();
assert!(locks.due(start.0 + Duration::from_secs(9)).is_empty());
assert_eq!(locks.due(start.0 + Duration::from_secs(11)).len(), 1);
locks.renewed(
id,
start.0 + Duration::from_secs(41),
start.0 + Duration::from_secs(11),
);
assert!(locks.due(start.0 + Duration::from_secs(20)).is_empty());
assert!(!revoked.is_cancelled());
}
#[test]
fn ended_locks_are_revoked_or_refused() {
let locks = Locks::default();
let start = (Instant::now(), SystemTime::now());
let (id, revoked) = locks
.add(
Uuid::from([1; 16]),
Some(at(start.1, Duration::from_secs(30))),
start,
)
.unwrap();
assert!(locks.due(start.0 + Duration::from_secs(30)).is_empty());
assert!(revoked.is_cancelled());
assert!(!locks.remove(id));
let past = at(start.1, Duration::ZERO) - 1;
assert!(locks.add(Uuid::from([2; 16]), Some(past), start).is_none());
}
#[test]
fn locks_without_an_expiry_assume_the_shortest_duration() {
let locks = Locks::default();
let start = (Instant::now(), SystemTime::now());
locks.add(Uuid::from([1; 16]), None, start).unwrap();
assert!(locks.due(start.0 + Duration::from_millis(1600)).is_empty());
assert_eq!(locks.due(start.0 + Duration::from_millis(1700)).len(), 1);
}
#[test]
fn removed_and_revoked_locks_end() {
let locks = Locks::default();
let start = (Instant::now(), SystemTime::now());
let until = Some(at(start.1, Duration::from_secs(60)));
let (first, first_revoked) = locks.add(Uuid::from([1; 16]), until, start).unwrap();
let (second, second_revoked) = locks.add(Uuid::from([2; 16]), until, start).unwrap();
let (_, third_revoked) = locks.add(Uuid::from([3; 16]), until, start).unwrap();
assert!(locks.remove(first));
assert!(!first_revoked.is_cancelled());
locks.revoke(second);
assert!(second_revoked.is_cancelled());
locks.revoke_all();
assert!(third_revoked.is_cancelled());
assert!(locks.due(start.0).is_empty());
}
}