use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::Arc;
use dashmap::DashMap;
use tokio::sync::{Notify, OwnedSemaphorePermit, Semaphore};
use crate::entry::TaskEnvelope;
use crate::observer::StealRequest;
use crate::queue::{
channel, ConsumerPoll, ProducerBoundary, Reservation, ReserveError, SequencedReceiver,
SequencedSender,
};
use crate::route::{StrictRouteState, TaskType, TypeState};
use crate::runner::RunnerInner;
use crate::tunnel::{NonStrictTunnel, StrictTunnel};
pub(crate) struct SlotConsumers {
main: SequencedReceiver<Arc<TaskEnvelope>>,
control: SequencedReceiver<Arc<StealRequest>>,
deferred: Option<DeferredTask>,
deferred_control: Option<Arc<StealRequest>>,
}
enum DeferredTask {
Main {
sequence: u64,
envelope: Arc<TaskEnvelope>,
},
Tunnel(Arc<TaskEnvelope>),
}
impl DeferredTask {
fn envelope(&self) -> &Arc<TaskEnvelope> {
match self {
Self::Main { envelope, .. } | Self::Tunnel(envelope) => envelope,
}
}
}
pub(crate) struct SlotMetrics {
pub(crate) main_depth: u64,
pub(crate) control_depth: u64,
pub(crate) reserved_heads: u64,
pub(crate) producer_inflight: u64,
pub(crate) stock_depth: u64,
pub(crate) incremental_depth: u64,
pub(crate) staging_depth: u64,
pub(crate) non_strict_open: u64,
pub(crate) non_strict_draining: u64,
pub(crate) strict_routes: [u64; 8],
}
pub(crate) struct PartitionSlot {
runner_id: u64,
generation: u64,
index: u32,
main: SequencedSender<Arc<TaskEnvelope>>,
control: SequencedSender<Arc<StealRequest>>,
pub(crate) type_states: DashMap<TaskType, Arc<TypeState>>,
inbound_non_strict: DashMap<u64, Arc<NonStrictTunnel>>,
inbound_strict: DashMap<u64, Arc<StrictTunnel>>,
max_inbound_tunnels: usize,
inbound_tunnel_count: AtomicUsize,
pub(crate) local_task_count: AtomicUsize,
pub(crate) tunnel_task_count: AtomicUsize,
main_depth: AtomicUsize,
control_depth: AtomicUsize,
main_reserved_head: AtomicBool,
control_reserved_head: AtomicBool,
execution_budget: Arc<Semaphore>,
accepting: AtomicBool,
failed: AtomicBool,
wake: Arc<Notify>,
}
impl PartitionSlot {
pub(crate) fn new(
runner_id: u64,
generation: u64,
index: u32,
max_inbound_tunnels: usize,
) -> (Arc<Self>, SlotConsumers) {
let (main, main_rx) = channel();
let (control, control_rx) = channel();
(
Arc::new(Self {
runner_id,
generation,
index,
main,
control,
type_states: DashMap::new(),
inbound_non_strict: DashMap::new(),
inbound_strict: DashMap::new(),
max_inbound_tunnels,
inbound_tunnel_count: AtomicUsize::new(0),
local_task_count: AtomicUsize::new(0),
tunnel_task_count: AtomicUsize::new(0),
main_depth: AtomicUsize::new(0),
control_depth: AtomicUsize::new(0),
main_reserved_head: AtomicBool::new(false),
control_reserved_head: AtomicBool::new(false),
execution_budget: Arc::new(Semaphore::new(1)),
accepting: AtomicBool::new(true),
failed: AtomicBool::new(false),
wake: Arc::new(Notify::new()),
}),
SlotConsumers {
main: main_rx,
control: control_rx,
deferred: None,
deferred_control: None,
},
)
}
pub(crate) fn runner_id(&self) -> u64 {
self.runner_id
}
pub(crate) fn generation(&self) -> u64 {
self.generation
}
pub(crate) fn index(&self) -> u32 {
self.index
}
pub(crate) fn accepting(&self) -> bool {
self.accepting.load(Ordering::Acquire) && !self.failed.load(Ordering::Acquire)
}
pub(crate) fn reserve_main(&self) -> Result<Reservation<Arc<TaskEnvelope>>, ReserveError> {
if !self.accepting() {
return Err(ReserveError::Closed);
}
let reservation = self.main.reserve()?;
self.main_depth.fetch_add(1, Ordering::AcqRel);
self.wake.notify_one();
Ok(reservation)
}
pub(crate) fn reserve_main_transfer(
&self,
) -> Result<Reservation<Arc<TaskEnvelope>>, ReserveError> {
if self.failed.load(Ordering::Acquire) {
return Err(ReserveError::Closed);
}
let reservation = self.main.reserve()?;
self.main_depth.fetch_add(1, Ordering::AcqRel);
self.wake.notify_one();
Ok(reservation)
}
pub(crate) fn commit_main(&self) {
self.wake.notify_one();
}
pub(crate) fn abort_main_reserved(&self) {
let decremented = self
.main_depth
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |value| {
value.checked_sub(1)
})
.is_ok();
debug_assert!(decremented, "main queue depth authority lost");
self.fail();
}
pub(crate) fn main_producer_inflight(&self) -> usize {
self.main.producer_inflight()
}
pub(crate) fn main_producer_boundary(&self) -> ProducerBoundary {
self.main.boundary()
}
pub(crate) fn offer_control(&self, request: Arc<StealRequest>) -> bool {
if !self.accepting() {
return false;
}
let Ok(reservation) = self.control.reserve() else {
return false;
};
self.control_depth.fetch_add(1, Ordering::AcqRel);
if reservation.publish(request).is_err() {
let _ = self
.control_depth
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |value| {
value.checked_sub(1)
});
self.fail();
return false;
}
self.wake.notify_one();
true
}
pub(crate) fn register_non_strict(&self, tunnel: Arc<NonStrictTunnel>) -> bool {
if !self.accepting()
|| tunnel.runner_id() != self.runner_id
|| tunnel.generation() != self.generation
|| tunnel.target() != self.index
{
return false;
}
if !self.reserve_inbound_tunnel() {
return false;
}
if self
.inbound_non_strict
.insert(tunnel.id(), tunnel)
.is_none()
{
true
} else {
self.release_inbound_tunnel();
false
}
}
pub(crate) fn register_strict(&self, tunnel: Arc<StrictTunnel>) -> bool {
if !self.accepting()
|| tunnel.runner_id() != self.runner_id
|| tunnel.generation() != self.generation
|| tunnel.target() != self.index
{
return false;
}
if !self.reserve_inbound_tunnel() {
return false;
}
if self.inbound_strict.insert(tunnel.id(), tunnel).is_none() {
true
} else {
self.release_inbound_tunnel();
false
}
}
pub(crate) fn unregister_non_strict(&self, tunnel_id: u64) -> bool {
if self.inbound_non_strict.remove(&tunnel_id).is_some() {
self.release_inbound_tunnel();
true
} else {
false
}
}
pub(crate) fn unregister_strict(&self, tunnel_id: u64) -> bool {
if self.inbound_strict.remove(&tunnel_id).is_some() {
self.release_inbound_tunnel();
true
} else {
false
}
}
fn reserve_inbound_tunnel(&self) -> bool {
self.inbound_tunnel_count
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |value| {
(value < self.max_inbound_tunnels).then_some(value + 1)
})
.is_ok()
}
fn release_inbound_tunnel(&self) -> bool {
if self
.inbound_tunnel_count
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |value| {
value.checked_sub(1)
})
.is_ok()
{
true
} else {
self.fail();
false
}
}
pub(crate) fn strict_tunnels(&self) -> Vec<Arc<StrictTunnel>> {
self.inbound_strict
.iter()
.map(|entry| entry.value().clone())
.collect()
}
pub(crate) fn has_inbound_from(&self, source: u32) -> bool {
self.inbound_non_strict
.iter()
.any(|tunnel| tunnel.source() == source)
|| self
.inbound_strict
.iter()
.any(|tunnel| tunnel.source() == source)
}
pub(crate) fn local_task_load(&self) -> usize {
self.local_task_count.load(Ordering::Acquire)
}
pub(crate) fn main_depth(&self) -> usize {
self.main_depth.load(Ordering::Acquire)
}
pub(crate) fn load(&self) -> usize {
self.main_depth()
.saturating_add(self.tunnel_task_count.load(Ordering::Acquire))
.saturating_add(self.local_task_count.load(Ordering::Acquire))
}
pub(crate) fn metrics_snapshot(&self) -> SlotMetrics {
let strict_tunnels = self
.inbound_strict
.iter()
.map(|entry| entry.value().clone())
.collect::<Vec<_>>();
let mut stock_depth = 0_u64;
let mut incremental_depth = 0_u64;
let mut staging_depth = 0_u64;
let mut producer_inflight =
self.main
.producer_inflight()
.saturating_add(self.main.outstanding_reservations())
.saturating_add(self.control.producer_inflight())
.saturating_add(self.control.outstanding_reservations()) as u64;
for tunnel in &strict_tunnels {
let (stock, incremental, staging) = tunnel.queue_depths();
stock_depth = stock_depth.saturating_add(stock as u64);
incremental_depth = incremental_depth.saturating_add(incremental as u64);
staging_depth = staging_depth.saturating_add(staging as u64);
producer_inflight = producer_inflight.saturating_add(tunnel.producer_inflight() as u64);
}
let non_strict_tunnels = self
.inbound_non_strict
.iter()
.map(|entry| entry.value().clone())
.collect::<Vec<_>>();
let mut non_strict_open = 0_u64;
let mut non_strict_draining = 0_u64;
for tunnel in &non_strict_tunnels {
producer_inflight = producer_inflight.saturating_add(tunnel.producer_inflight() as u64);
if tunnel.accepting() {
non_strict_open = non_strict_open.saturating_add(1);
} else if !tunnel.drained() {
non_strict_draining = non_strict_draining.saturating_add(1);
}
}
let mut strict_routes = [0_u64; 8];
for state in &self.type_states {
let Some(route) = state.strict_route() else {
continue;
};
let index = match route.state {
StrictRouteState::Local => 0,
StrictRouteState::Migrating => 1,
StrictRouteState::Stolen => 2,
StrictRouteState::ReturnPrepare => 3,
StrictRouteState::Returning => 4,
StrictRouteState::LocalCatchup => 5,
StrictRouteState::StolenCatchup => 6,
StrictRouteState::Failed => 7,
};
strict_routes[index] = strict_routes[index].saturating_add(1);
}
SlotMetrics {
main_depth: self.main_depth.load(Ordering::Acquire) as u64,
control_depth: self.control_depth.load(Ordering::Acquire) as u64,
reserved_heads: self.main_reserved_head.load(Ordering::Acquire) as u64
+ self.control_reserved_head.load(Ordering::Acquire) as u64,
producer_inflight,
stock_depth,
incremental_depth,
staging_depth,
non_strict_open,
non_strict_draining,
strict_routes,
}
}
pub(crate) fn execution_available(&self) -> bool {
self.execution_budget.available_permits() > 0
}
pub(crate) fn needs_control_tick(&self) -> bool {
if !self.inbound_non_strict.is_empty()
|| !self.inbound_strict.is_empty()
|| self.main_depth.load(Ordering::Acquire) > 0
|| self.control_depth.load(Ordering::Acquire) > 0
|| self.main.producer_inflight() > 0
|| self.main.outstanding_reservations() > 0
|| self.control.producer_inflight() > 0
|| self.control.outstanding_reservations() > 0
{
return true;
}
self.type_states.iter().any(|state| {
state
.strict_route()
.is_some_and(|route| matches!(route.state, StrictRouteState::Migrating))
})
}
pub(crate) fn try_acquire_execution(&self) -> Option<OwnedSemaphorePermit> {
self.execution_budget.clone().try_acquire_owned().ok()
}
pub(crate) async fn acquire_execution(
&self,
) -> Result<OwnedSemaphorePermit, tokio::sync::AcquireError> {
self.execution_budget.clone().acquire_owned().await
}
pub(crate) fn close(&self) {
self.accepting.store(false, Ordering::Release);
self.control.close();
for tunnel in &self.inbound_non_strict {
tunnel.close();
}
for tunnel in &self.inbound_strict {
tunnel.close_all();
}
self.wake.notify_waiters();
self.wake.notify_one();
}
pub(crate) fn fail(&self) -> bool {
let first = !self.failed.swap(true, Ordering::AcqRel);
if first {
self.close();
}
first
}
pub(crate) fn queues_empty(&self) -> bool {
self.main_depth.load(Ordering::Acquire) == 0
&& self.main.producer_inflight() == 0
&& self.main.outstanding_reservations() == 0
&& self.control_depth.load(Ordering::Acquire) == 0
&& self.control.producer_inflight() == 0
&& self.control.outstanding_reservations() == 0
&& self.inbound_non_strict.iter().all(|entry| entry.drained())
&& self.inbound_strict.iter().all(|entry| {
entry.depth() == 0
&& entry.direct_producers_stopped()
&& entry.staging_producers_stopped()
})
}
pub(crate) fn wake_handle(&self) -> Arc<Notify> {
self.wake.clone()
}
}
pub(crate) async fn run(
inner: Arc<RunnerInner>,
slot: Arc<PartitionSlot>,
mut consumers: SlotConsumers,
) {
let mut exit_guard = WorkerExitGuard {
inner: inner.clone(),
slot: slot.clone(),
clean: false,
};
loop {
let runner = inner.clone();
let mut progressed = false;
let batch = runner.config().drain_batch;
if runner.lifecycle_mode() == crate::lifecycle::LifecycleMode::ForceStop {
if let Some(deferred) = consumers.deferred.take() {
runner.fail_deferred(deferred.envelope(), "shutdown_frozen");
progressed = true;
}
}
if let Some(request) = consumers.deferred_control.take() {
if runner.lifecycle_mode() == crate::lifecycle::LifecycleMode::ForceStop {
request.complete();
progressed = true;
} else if slot.execution_available() {
if !runner.handle_steal_request(&slot, request.clone()) {
consumers.deferred_control = Some(request);
}
progressed = true;
} else {
consumers.deferred_control = Some(request);
}
}
for _ in 0..batch.min(8) {
if consumers.deferred_control.is_some() {
break;
}
let control_poll = consumers.control.try_recv();
slot.control_reserved_head.store(
matches!(control_poll, ConsumerPoll::Reserved { .. }),
Ordering::Release,
);
match control_poll {
ConsumerPoll::Item { value, .. } => {
let decremented = slot
.control_depth
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |value| {
value.checked_sub(1)
})
.is_ok();
debug_assert!(decremented, "control queue depth authority lost");
if value.pending() && !runner.handle_steal_request(&slot, value.clone()) {
consumers.deferred_control = Some(value);
}
progressed = true;
}
ConsumerPoll::Poisoned { .. } => {
let decremented = slot
.control_depth
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |value| {
value.checked_sub(1)
})
.is_ok();
debug_assert!(decremented, "control queue depth authority lost");
progressed = true;
}
ConsumerPoll::Failed => {
runner.fail_slot(&slot, "control_queue_failed");
break;
}
ConsumerPoll::Empty | ConsumerPoll::Reserved { .. } | ConsumerPoll::Closed => break,
}
}
if let Some(deferred) = consumers.deferred.take() {
consumers.deferred = match deferred {
DeferredTask::Main { sequence, envelope } => runner
.resume_main_handling(&slot, sequence, envelope)
.map(|envelope| DeferredTask::Main { sequence, envelope }),
DeferredTask::Tunnel(envelope) => runner
.resume_handling(slot.index(), envelope)
.map(DeferredTask::Tunnel),
};
progressed |= consumers.deferred.is_none();
}
for _ in 0..batch {
if consumers.deferred.is_some() {
break;
}
let main_poll = consumers.main.try_recv();
slot.main_reserved_head.store(
matches!(main_poll, ConsumerPoll::Reserved { .. }),
Ordering::Release,
);
match main_poll {
ConsumerPoll::Item { sequence, value } => {
let decremented = slot
.main_depth
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |value| {
value.checked_sub(1)
})
.is_ok();
debug_assert!(decremented, "main queue depth authority lost");
consumers.deferred = runner
.consume_main(&slot, sequence, value)
.map(|envelope| DeferredTask::Main { sequence, envelope });
progressed = true;
}
ConsumerPoll::Poisoned { .. } => {
let decremented = slot
.main_depth
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |value| {
value.checked_sub(1)
})
.is_ok();
debug_assert!(decremented, "main queue depth authority lost");
progressed = true;
}
ConsumerPoll::Failed => {
runner.fail_slot(&slot, "main_queue_failed");
break;
}
ConsumerPoll::Empty | ConsumerPoll::Reserved { .. } | ConsumerPoll::Closed => break,
}
}
runner.progress_migrations(&slot, consumers.main.boundary());
let strict_tunnels = slot
.inbound_strict
.iter()
.map(|entry| entry.value().clone())
.collect::<Vec<_>>();
for tunnel in strict_tunnels {
for _ in 0..batch {
if consumers.deferred.is_some()
|| (runner.lifecycle_mode() != crate::lifecycle::LifecycleMode::ForceStop
&& !slot.execution_available())
{
break;
}
match tunnel.poll_executable() {
ConsumerPoll::Item { sequence, value } => {
consumers.deferred = runner
.consume_tunnel(&slot, sequence, value)
.map(DeferredTask::Tunnel);
progressed = true;
}
ConsumerPoll::Poisoned { .. } => progressed = true,
ConsumerPoll::Failed => {
runner.fail_strict_tunnel(&slot, &tunnel, "strict_tunnel_queue_failed");
break;
}
ConsumerPoll::Empty | ConsumerPoll::Reserved { .. } | ConsumerPoll::Closed => {
break
}
}
}
}
let non_strict_tunnels = slot
.inbound_non_strict
.iter()
.map(|entry| entry.value().clone())
.collect::<Vec<_>>();
for tunnel in non_strict_tunnels {
tunnel.close_if_expired();
for _ in 0..batch {
if consumers.deferred.is_some()
|| (runner.lifecycle_mode() != crate::lifecycle::LifecycleMode::ForceStop
&& !slot.execution_available())
{
break;
}
match tunnel.poll() {
ConsumerPoll::Item { sequence, value } => {
consumers.deferred = runner
.consume_tunnel(&slot, sequence, value)
.map(DeferredTask::Tunnel);
progressed = true;
}
ConsumerPoll::Poisoned { .. } => progressed = true,
ConsumerPoll::Failed => {
tunnel.close();
break;
}
ConsumerPoll::Empty | ConsumerPoll::Reserved { .. } | ConsumerPoll::Closed => {
break
}
}
}
runner.retire_non_strict_tunnel(&slot, &tunnel);
}
runner.progress_returns(&slot);
if runner.worker_should_exit(&slot)
&& slot.queues_empty()
&& consumers.deferred.is_none()
&& consumers.deferred_control.is_none()
{
break;
}
if !progressed {
let wake = slot.wake_handle();
let needs_control_tick =
slot.needs_control_tick() || consumers.deferred_control.is_some();
let control_tick = runner.config().control_tick;
drop(runner);
if needs_control_tick {
let _ = tokio::time::timeout(control_tick, wake.notified()).await;
} else {
wake.notified().await;
}
}
}
exit_guard.clean = true;
}
struct WorkerExitGuard {
inner: Arc<RunnerInner>,
slot: Arc<PartitionSlot>,
clean: bool,
}
impl Drop for WorkerExitGuard {
fn drop(&mut self) {
if !self.clean {
self.inner
.fail_slot(&self.slot, "worker_exited_without_drain_proof");
}
}
}