use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use std::time::Duration;
use tokio::task::JoinHandle;
use tokio_util::sync::CancellationToken;
use crate::config::MAX_REDIS_RUNTIME_DURATION_MS;
use crate::lock::{DistributedLock, LockGuard};
pub struct Leader {
is_leader: Arc<AtomicBool>,
running: Arc<AtomicBool>,
cancel: CancellationToken,
term_lost: Arc<Mutex<CancellationToken>>,
handle: Mutex<Option<JoinHandle<()>>>,
finished: CancellationToken,
}
struct ElectionExit {
is_leader: Arc<AtomicBool>,
running: Arc<AtomicBool>,
term_lost: Arc<Mutex<CancellationToken>>,
finished: CancellationToken,
}
impl Drop for ElectionExit {
fn drop(&mut self) {
self.is_leader.store(false, Ordering::Release);
self.running.store(false, Ordering::Release);
self.term_lost
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.cancel();
self.finished.cancel();
}
}
impl Leader {
pub fn elect(
lock: Arc<DistributedLock>,
key: impl Into<String>,
period: Duration,
) -> Arc<Self> {
let key = key.into();
let period = period.clamp(
Duration::from_millis(1),
Duration::from_millis(MAX_REDIS_RUNTIME_DURATION_MS),
);
let is_leader = Arc::new(AtomicBool::new(false));
let running = Arc::new(AtomicBool::new(false));
let cancel = CancellationToken::new();
let term_lost = Arc::new(Mutex::new({
let t = CancellationToken::new();
t.cancel();
t
}));
let finished = CancellationToken::new();
let completion = ElectionExit {
is_leader: is_leader.clone(),
running: running.clone(),
term_lost: term_lost.clone(),
finished: finished.clone(),
};
let election = election_loop(
lock,
key,
period,
Arc::clone(&is_leader),
Arc::clone(&term_lost),
cancel.clone(),
);
let handle = tokio::spawn(async move {
let _completion = completion;
_completion.running.store(true, Ordering::Release);
election.await;
});
Arc::new(Self {
is_leader,
running,
cancel,
term_lost,
handle: Mutex::new(Some(handle)),
finished,
})
}
pub fn is_leader(&self) -> bool {
self.is_running() && self.is_leader.load(Ordering::Acquire)
}
pub fn is_running(&self) -> bool {
!self.cancel.is_cancelled() && self.running.load(Ordering::Acquire)
}
pub fn abort_handle(&self) -> Option<tokio::task::AbortHandle> {
self.handle
.lock()
.expect("leader handle")
.as_ref()
.map(|h| h.abort_handle())
}
pub async fn run_if_leader<F, Fut, T>(&self, f: F) -> Option<T>
where
F: FnOnce() -> Fut,
Fut: std::future::Future<Output = T>,
{
if self.is_leader() {
Some(f().await)
} else {
None
}
}
pub async fn run_if_leader_cancellable<F, Fut, T>(&self, f: F) -> Option<T>
where
F: FnOnce(CancellationToken) -> Fut,
Fut: std::future::Future<Output = T>,
{
if !self.is_leader() {
return None;
}
let tok = self.term_lost.lock().expect("leader term_lost").clone();
if tok.is_cancelled() {
return None;
}
tokio::select! {
biased;
_ = tok.cancelled() => None,
out = f(tok.clone()) => Some(out),
}
}
pub async fn shutdown(&self) {
self.begin_shutdown();
self.finished.cancelled().await;
let h = self.handle.lock().expect("leader handle").take();
if let Some(h) = h {
let _ = h.await;
}
}
pub fn begin_shutdown(&self) {
self.cancel.cancel();
self.term_lost.lock().expect("leader term_lost").cancel();
self.is_leader.store(false, Ordering::Release);
}
}
impl Drop for Leader {
fn drop(&mut self) {
self.cancel.cancel();
self.term_lost.lock().expect("leader term_lost").cancel();
let handle = self.handle.lock().expect("leader handle").take();
if let Some(mut handle) = handle {
if let Ok(runtime) = tokio::runtime::Handle::try_current() {
runtime.spawn(async move {
if tokio::time::timeout(Duration::from_secs(2), &mut handle)
.await
.is_err()
{
handle.abort();
let _ = handle.await;
}
});
} else {
handle.abort();
}
}
self.is_leader.store(false, Ordering::Release);
}
}
async fn election_loop(
lock: Arc<DistributedLock>,
key: String,
period: Duration,
is_leader: Arc<AtomicBool>,
term_lost: Arc<Mutex<CancellationToken>>,
cancel: CancellationToken,
) {
let begin_term = |term_lost: &Mutex<CancellationToken>| {
*term_lost.lock().expect("term_lost") = CancellationToken::new();
};
let end_term = |term_lost: &Mutex<CancellationToken>| {
term_lost.lock().expect("term_lost").cancel();
};
let mut guard: Option<LockGuard> = None;
let mut lost_rx: Option<tokio::sync::watch::Receiver<bool>> = None;
let mut tick = tokio::time::interval(period.max(Duration::from_millis(1)));
tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
loop {
tokio::select! {
_ = cancel.cancelled() => break,
res = async { lost_rx.as_mut().expect("lost_rx").changed().await }, if lost_rx.is_some() => {
let lost = res.is_err() || lost_rx.as_ref().map(|r| *r.borrow()).unwrap_or(true);
if lost {
tracing::warn!(key = %key, "leader 租约丢失,即时退位改选");
is_leader.store(false, Ordering::Release);
end_term(&term_lost); guard = None; lost_rx = None;
}
}
_ = tick.tick() => {
if guard.is_none() {
match lock.try_lock(&key).await {
Ok(Some(g)) => {
tracing::info!(key = %key, "竞选成功,成为 leader");
begin_term(&term_lost);
lost_rx = Some(g.lost()); is_leader.store(true, Ordering::Release);
guard = Some(g);
}
Ok(None) => is_leader.store(false, Ordering::Release), Err(e) => {
is_leader.store(false, Ordering::Release);
tracing::warn!(key = %key, err = %e, "竞选 try_lock 失败,下轮重试");
}
}
}
}
}
}
is_leader.store(false, Ordering::Release);
end_term(&term_lost);
if let Some(g) = guard.take() {
if let Err(e) = g.unlock().await {
tracing::warn!(key = %key, err = %e, "leader 退位 unlock 失败(锁靠 lease 过期自愈,新 leader 接任延迟 ≤lease)");
}
}
}