use super::{
publisher::{PublisherCoordinator, PublisherSnapshot},
runtime::{engine::RedisPartitionRuntime, GroupRuntime, PartitionSnapshot},
};
use std::{
sync::{
atomic::{AtomicBool, Ordering},
Arc, Mutex,
},
time::Instant,
};
use tokio::sync::Notify;
#[derive(Debug, Clone)]
pub struct PartitionShutdownReport {
pub converged: bool,
pub forced: bool,
pub local_execution_uncertain: bool,
pub remaining: PartitionSnapshot,
pub publishes: PublisherSnapshot,
pub async_delete_pending: usize,
pub background_tasks: usize,
}
pub(super) struct ShutdownOperation {
pub core: Arc<RedisPartitionRuntime>,
pub publisher: Arc<PublisherCoordinator>,
groups: Vec<Arc<GroupRuntime>>,
handle: Mutex<Option<tokio::task::JoinHandle<()>>>,
started: AtomicBool,
done: AtomicBool,
forced: AtomicBool,
changed: Notify,
dependency: Mutex<(bool, Option<Box<dyn Send>>)>,
}
impl ShutdownOperation {
pub(super) fn new(
core: Arc<RedisPartitionRuntime>,
publisher: Arc<PublisherCoordinator>,
groups: Vec<Arc<GroupRuntime>>,
) -> Arc<Self> {
Arc::new(Self {
core,
publisher,
groups,
handle: Mutex::new(None),
started: AtomicBool::new(false),
done: AtomicBool::new(false),
forced: AtomicBool::new(false),
changed: Notify::new(),
dependency: Mutex::new((false, None)),
})
}
pub(super) fn begin(self: &Arc<Self>) {
if self.started.swap(true, Ordering::AcqRel) {
return;
}
self.publisher.close();
self.core.close_roots();
let owner = self.clone();
let handle = tokio::spawn(async move {
owner.publisher.drain().await;
futures::future::join_all(owner.groups.iter().map(|group| group.shutdown())).await;
owner.core.stop().await;
owner.done.store(true, Ordering::Release);
if owner.report().converged {
let dependency = owner
.dependency
.lock()
.expect("shutdown dependency")
.1
.take();
drop(dependency);
}
owner.changed.notify_waiters();
});
*self.handle.lock().expect("shutdown operation") = Some(handle);
}
pub(super) fn retain_dependency<T: Send + 'static>(
&self,
guard: T,
) -> std::result::Result<(), T> {
let mut dependency = self.dependency.lock().expect("shutdown dependency");
if dependency.0 {
return Err(guard);
}
dependency.0 = true;
if self.report().converged {
drop(dependency);
drop(guard);
} else {
dependency.1 = Some(Box::new(guard));
}
Ok(())
}
pub(super) async fn wait(self: &Arc<Self>, deadline: Instant) -> PartitionShutdownReport {
self.begin();
loop {
let changed = self.changed.notified();
if self.done.load(Ordering::Acquire) {
break;
}
if tokio::time::timeout_at(deadline.into(), changed)
.await
.is_err()
{
break;
}
}
self.report()
}
pub(super) async fn force(self: &Arc<Self>, deadline: Instant) -> PartitionShutdownReport {
self.forced.store(true, Ordering::Release);
self.publisher.force();
self.core.revoke_all();
self.begin();
for group in &self.groups {
group.cancel_best_effort();
}
futures::future::join_all(
self.core
.executions
.domains
.iter()
.map(|domain| domain.runner.force_stop(deadline)),
)
.await;
self.wait(deadline).await
}
pub(super) fn report(&self) -> PartitionShutdownReport {
let finished = self.done.load(Ordering::Acquire);
let remaining = self.core.snapshot();
let publishes = self.publisher.snapshot();
let async_delete_pending = self.groups.iter().map(|g| g.async_delete_pending()).sum();
let background_tasks = self
.groups
.iter()
.map(|group| group.unfinished_background_tasks())
.sum();
let converged = finished
&& remaining.inflight_records == 0
&& remaining.inflight_tasks == 0
&& remaining.payload_bytes == 0
&& remaining.active_batches == 0
&& remaining.ordered_keys == 0
&& remaining.source_io == 0
&& remaining.unjoined_readers == 0
&& remaining.unconfirmed_locks == 0
&& remaining
.execution_domains
.iter()
.all(|domain| domain.runner_stopped)
&& publishes.inflight == 0
&& publishes.unjoined_tasks == 0
&& async_delete_pending == 0
&& background_tasks == 0;
PartitionShutdownReport {
converged,
forced: self.forced.load(Ordering::Acquire),
local_execution_uncertain: !converged && self.forced.load(Ordering::Acquire),
remaining,
publishes,
async_delete_pending,
background_tasks,
}
}
}