use super::{fencing, identity::BatchIdentity, GroupRuntime};
use crate::partition::{
plan::{ConsumerPlan, DecodeFailure, Payload, PlanMap},
Envelope, PartitionLimits, RecordIdentity,
};
use crate::{
error::{NasaRedisError, Result},
lock::HoldStatus,
};
use futures::FutureExt;
use std::sync::atomic::{AtomicBool, AtomicU8, AtomicUsize, Ordering};
use std::{
collections::{BTreeMap, HashMap, VecDeque},
sync::{Arc, Mutex, Weak},
time::{Duration, Instant},
};
use tokio::sync::Notify;
#[repr(u8)]
enum SourcePhase {
Active,
Quiescing,
Releasing,
Lost,
}
pub(in crate::partition) struct SourceAuthority {
pub domain: usize,
pub epoch: u64,
pub partition: u32,
pub holder: String,
pub lock_key: String,
pub stamp: Option<fencing::FencingStamp>,
pub group: Weak<GroupRuntime>,
phase: AtomicU8,
pub io: AtomicUsize,
pub blocked: AtomicBool,
pub parked: AtomicBool,
pub uncertain: AtomicBool,
pub sweep_requested: AtomicBool,
pub oversized: AtomicBool,
pub protocol_error: AtomicBool,
pub unreleased: AtomicBool,
pub lost_watch: tokio::sync::watch::Receiver<bool>,
pub lost_cancel: tokio_util::sync::CancellationToken,
}
impl SourceAuthority {
pub(super) fn active(&self) -> bool {
self.phase.load(Ordering::Acquire) == SourcePhase::Active as u8 && self.can_commit()
}
pub(super) fn new(
rt: &Arc<GroupRuntime>,
partition: u32,
identity: BatchIdentity,
guard: &crate::lock::LockGuard,
stamp: Option<fencing::FencingStamp>,
) -> Arc<Self> {
Arc::new(Self {
domain: rt.core.executions.domain(&rt.layout.prefix, partition),
epoch: identity.claim_epoch,
partition,
holder: guard.holder().into(),
lock_key: guard.lock_key().into(),
stamp,
group: Arc::downgrade(rt),
phase: AtomicU8::new(SourcePhase::Active as u8),
io: AtomicUsize::new(0),
blocked: AtomicBool::new(false),
parked: AtomicBool::new(false),
uncertain: AtomicBool::new(false),
sweep_requested: AtomicBool::new(false),
oversized: AtomicBool::new(false),
protocol_error: AtomicBool::new(false),
unreleased: AtomicBool::new(true),
lost_watch: guard.lost(),
lost_cancel: tokio_util::sync::CancellationToken::new(),
})
}
pub(in crate::partition) fn can_start(&self) -> bool {
self.phase.load(Ordering::Acquire) == SourcePhase::Active as u8
&& self.can_commit()
&& !self.parked.load(Ordering::Acquire)
}
pub(in crate::partition) fn can_commit(&self) -> bool {
self.phase.load(Ordering::Acquire) < SourcePhase::Releasing as u8
&& !*self.lost_watch.borrow()
}
pub(in crate::partition) fn quiesce(&self) {
let _ = self.phase.compare_exchange(
SourcePhase::Active as u8,
SourcePhase::Quiescing as u8,
Ordering::AcqRel,
Ordering::Acquire,
);
}
pub(super) fn begin_release(&self) {
let _ = self.phase.compare_exchange(
SourcePhase::Quiescing as u8,
SourcePhase::Releasing as u8,
Ordering::AcqRel,
Ordering::Acquire,
);
}
pub(in crate::partition) fn lose(&self) {
self.phase.store(SourcePhase::Lost as u8, Ordering::Release);
self.lost_cancel.cancel();
}
}
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
pub(in crate::partition) struct Coordinate {
pub group: Arc<str>,
pub identity: RecordIdentity,
}
#[derive(Clone, Copy, PartialEq, Eq, Hash)]
struct GateKey {
plan: u32,
route: napart::RouteHash,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum BusinessState {
Open,
Executing,
Blocked,
Invalidating,
}
struct OrderedGate {
token: u64,
business: BusinessState,
queue: VecDeque<Coordinate>,
executing: Option<u64>,
}
enum RecordOwner {
Ready,
Parked,
Task(u64),
Commit {
unknown: bool,
confirmed: bool,
next: Instant,
},
Retry {
next: Instant,
operation: String,
executed: bool,
},
Unroutable {
malformed: bool,
next: Instant,
},
}
struct Record {
source: Arc<SourceAuthority>,
batch: u64,
revision: u64,
owner: RecordOwner,
plan: Option<Arc<ConsumerPlan>>,
gate: Option<GateKey>,
bytes: usize,
attempts: u32,
retry_intent: Option<crate::partition::retryop::RetryIntent>,
}
type TaskPayloads = Arc<Mutex<Option<Vec<(Coordinate, Payload)>>>>;
struct RetiredPayload<'a> {
_payload: Payload,
_budget: RetiredPayloadBudget<'a>,
}
struct RetiredPayloadBudget<'a> {
core: &'a RedisPartitionRuntime,
source: Arc<SourceAuthority>,
bytes: usize,
}
impl Drop for RetiredPayloadBudget<'_> {
fn drop(&mut self) {
let mut state = self.core.state.lock().expect("ledger");
state.bytes -= self.bytes;
state.reserved_records -= 1;
state.domains[self.source.domain].bytes -= self.bytes;
state.domains[self.source.domain].records -= 1;
self.source.io.fetch_sub(1, Ordering::AcqRel);
self.core.changed.notify_waiters();
}
}
#[derive(Clone)]
struct ConsumeOutcome {
succeeded: Vec<Coordinate>,
failed: Vec<Coordinate>,
deferred: Vec<Coordinate>,
}
struct OutcomeCell {
value: Mutex<Option<ConsumeOutcome>>,
changed: Notify,
}
struct TaskTicket {
source: Arc<SourceAuthority>,
records: Vec<Coordinate>,
gate: Option<(GateKey, u64)>,
submission: Option<napart::Submission>,
outcome: Arc<OutcomeCell>,
}
#[derive(Default)]
struct State {
domains: Vec<DomainUsage>,
sources: HashMap<u64, Weak<SourceAuthority>>,
abandoned: HashMap<u64, Arc<SourceAuthority>>,
abandoned_readers: Vec<tokio::task::JoinHandle<()>>,
records: HashMap<Coordinate, Record>,
prepared: HashMap<Coordinate, Payload>,
tasks: BTreeMap<u64, TaskTicket>,
gates: HashMap<GateKey, OrderedGate>,
waiters: VecDeque<(u64, usize)>,
reads: HashMap<u64, (usize, usize)>,
next_identity: u64,
bytes: usize,
reserved_records: usize,
record_debt: usize,
retained: u64,
committed: u64,
protocol_closed: bool,
}
#[derive(Default, Clone)]
struct DomainUsage {
records: usize,
bytes: usize,
batches: usize,
}
#[derive(Debug, Clone)]
pub struct ExecutionDomainSnapshot {
pub id: usize,
pub group: Option<String>,
pub partition: Option<u32>,
pub record_capacity: usize,
pub payload_capacity: usize,
pub batch_capacity: usize,
pub delete_capacity: usize,
pub inflight_records: usize,
pub payload_bytes: usize,
pub active_batches: usize,
pub inflight_tasks: usize,
pub async_delete_pending: usize,
pub runner_degraded: bool,
pub runner_stopped: bool,
}
#[derive(Debug, Clone, Default)]
pub struct PartitionSnapshot {
pub executor_scope: crate::partition::PartitionExecutorScope,
pub execution_domains: Vec<ExecutionDomainSnapshot>,
pub ready: bool,
pub runner_degraded: bool,
pub inflight_records: usize,
pub payload_bytes: usize,
pub active_batches: usize,
pub read_waiters: usize,
pub inflight_tasks: usize,
pub pending_commits: usize,
pub unknown_commits: usize,
pub retry_tickets: usize,
pub ordered_keys: usize,
pub blocked_keys: usize,
pub deferred_records: usize,
pub unroutable_records: usize,
pub parked_records: usize,
pub blocked_sources: usize,
pub parked_sources: usize,
pub oversized_sources: usize,
pub protocol_error_sources: usize,
pub record_debt: usize,
pub byte_debt: usize,
pub committed_records: u64,
pub retained_records: u64,
pub unconfirmed_locks: usize,
pub running_tasks: usize,
pub source_io: usize,
pub unjoined_readers: usize,
pub delete_retained_records: u64,
}
pub(in crate::partition) struct RedisPartitionRuntime {
state: Mutex<State>,
pub limits: PartitionLimits,
pub executions: crate::partition::execution::ExecutionDomains,
pub changed: Notify,
pub roots_closed: AtomicBool,
pub degraded: AtomicBool,
pub async_delete_records: AtomicUsize,
pub delete_retained_records: std::sync::atomic::AtomicU64,
stopping: AtomicBool,
force_io: tokio_util::sync::CancellationToken,
supervisors: Mutex<Vec<tokio::task::JoinHandle<()>>>,
}
impl RedisPartitionRuntime {
pub(super) fn abandoned_reader(&self, reader: tokio::task::JoinHandle<()>) {
self.state
.lock()
.expect("ledger")
.abandoned_readers
.push(reader);
self.changed.notify_waiters();
}
pub(super) fn abandoned_source(&self, source: &Arc<SourceAuthority>) {
self.state
.lock()
.expect("ledger")
.abandoned
.insert(source.epoch, source.clone());
self.changed.notify_waiters();
}
pub(super) fn resume_source(&self, source: &Arc<SourceAuthority>) {
let mut state = self.state.lock().expect("ledger");
for (coordinate, record) in state.records.iter_mut() {
if record.source.epoch == source.epoch && matches!(record.owner, RecordOwner::Parked) {
record.owner = RecordOwner::Retry {
next: Instant::now(),
operation: format!("park:{}:{}", source.epoch, coordinate.identity.id),
executed: false,
};
record.retry_intent = None;
}
}
let blocked = state.records.values().any(|record| {
record.source.epoch == source.epoch
&& matches!(record.owner, RecordOwner::Unroutable { .. })
&& record.plan.is_none()
});
if !blocked
&& !source.protocol_error.load(Ordering::Acquire)
&& !source.oversized.load(Ordering::Acquire)
{
for (coordinate, record) in state.records.iter_mut() {
if record.source.epoch == source.epoch
&& matches!(
record.owner,
RecordOwner::Unroutable {
malformed: false,
..
}
)
{
record.owner = RecordOwner::Retry {
next: Instant::now(),
operation: format!("resume:{}:{}", source.epoch, coordinate.identity.id),
executed: false,
};
}
}
source.blocked.store(false, Ordering::Release);
}
source.uncertain.store(true, Ordering::Release);
source.parked.store(false, Ordering::Release);
self.changed.notify_waiters();
}
pub(in crate::partition) fn revoke_all(&self) {
self.roots_closed.store(true, Ordering::Release);
self.force_io.cancel();
for source in self
.state
.lock()
.expect("ledger")
.sources
.values()
.filter_map(Weak::upgrade)
{
source.lose();
}
self.changed.notify_waiters();
}
pub(in crate::partition) fn request_recovery(&self, group: &str) {
let state = self.state.lock().expect("ledger");
for source in state.sources.values().filter_map(Weak::upgrade) {
if source
.group
.upgrade()
.is_some_and(|rt| rt.layout.prefix == group)
&& !state
.records
.values()
.any(|r| r.source.epoch == source.epoch)
&& !state.reads.contains_key(&source.epoch)
{
source.sweep_requested.store(true, Ordering::Release);
}
}
}
pub(in crate::partition) fn new(
limits: PartitionLimits,
plan: crate::partition::execution::ExecutionPlan,
) -> Result<Arc<Self>> {
let executions = crate::partition::execution::ExecutionDomains::new(plan)?;
let core = Arc::new(Self {
state: Mutex::new(State {
next_identity: 1,
domains: vec![DomainUsage::default(); executions.domains.len()],
..Default::default()
}),
limits,
executions,
changed: Notify::new(),
roots_closed: AtomicBool::new(false),
degraded: AtomicBool::new(false),
async_delete_records: AtomicUsize::new(0),
delete_retained_records: std::sync::atomic::AtomicU64::new(0),
stopping: AtomicBool::new(false),
force_io: tokio_util::sync::CancellationToken::new(),
supervisors: Mutex::new(Vec::new()),
});
Ok(core)
}
pub(in crate::partition) async fn start(
self: &Arc<Self>,
gate: tokio_util::sync::CancellationToken,
) -> Result<()> {
for domain in &self.executions.domains {
domain
.runner
.start()
.await
.map_err(|e| NasaRedisError::Config(e.to_string()))?;
}
for domain in 0..self.executions.domains.len() {
let owner = self.clone();
let gate = gate.clone();
let handle = tokio::spawn(async move {
loop {
let changed = owner.changed.notified();
tokio::pin!(changed);
changed.as_mut().enable();
if gate.is_cancelled() || owner.stopping.load(Ordering::Acquire) {
break;
}
tokio::select! {
_ = gate.cancelled() => {
}
_ = changed => {
}
}
}
let result = std::panic::AssertUnwindSafe(owner.clone().supervise(domain))
.catch_unwind()
.await;
if result.is_err() {
owner.degraded.store(true, Ordering::Release);
owner.close_roots();
}
});
self.supervisors.lock().expect("supervisors").push(handle);
}
Ok(())
}
pub(in crate::partition) fn register_source(&self, source: &Arc<SourceAuthority>) {
let mut state = self.state.lock().expect("ledger");
if self.roots_closed.load(Ordering::Acquire)
|| self.executions.domains[source.domain]
.degraded
.load(Ordering::Acquire)
{
source.quiesce();
}
state.sources.retain(|_, weak| weak.strong_count() != 0);
state.sources.insert(source.epoch, Arc::downgrade(source));
}
pub(in crate::partition) fn reserve(
self: &Arc<Self>,
source: &Arc<SourceAuthority>,
count: usize,
) -> Option<ReadPermit> {
let mut state = self.state.lock().expect("ledger");
if self.roots_closed.load(Ordering::Acquire)
|| self.degraded.load(Ordering::Acquire)
|| self.executions.domains[source.domain]
.degraded
.load(Ordering::Acquire)
|| state.protocol_closed
|| state.record_debt != 0
|| !source.can_start()
|| source.blocked.load(Ordering::Acquire)
|| state.reads.contains_key(&source.epoch)
{
return None;
}
let group = source.group.upgrade()?;
let group_reads = state
.reads
.keys()
.filter(|epoch| {
state
.sources
.get(epoch)
.and_then(Weak::upgrade)
.and_then(|source| source.group.upgrade())
.is_some_and(|current| current.layout.prefix == group.layout.prefix)
})
.count();
if group_reads >= group.stream_cfg.inflight_max
|| state
.records
.values()
.filter(|record| record.source.epoch == source.epoch)
.count()
.checked_add(count)?
> self.limits.max_blocked_keys_per_source
{
state.waiters.retain(|(epoch, _)| *epoch != source.epoch);
return None;
}
let bytes = count.checked_mul(self.limits.max_record_bytes)?;
let quota = &self.executions.domains[source.domain].spec.quota;
let usage = &state.domains[source.domain];
if usage.records.checked_add(count)? > quota.records
|| usage.bytes.checked_add(bytes)? > quota.bytes
|| usage.batches >= quota.batches
{
state.waiters.retain(|(epoch, _)| *epoch != source.epoch);
return None;
}
let valid: std::collections::HashSet<u64> = state
.sources
.iter()
.filter_map(|(id, weak)| {
weak.upgrade()
.filter(|s| s.can_start() && !s.blocked.load(Ordering::Acquire))
.map(|_| *id)
})
.collect();
let eligible: std::collections::HashSet<u64> = state
.waiters
.iter()
.filter_map(|(epoch, count)| {
let source = state.sources.get(epoch)?.upgrade()?;
let usage = &state.domains[source.domain];
let quota = &self.executions.domains[source.domain].spec.quota;
(usage.records.checked_add(*count)? <= quota.records
&& usage
.bytes
.checked_add(count.checked_mul(self.limits.max_record_bytes)?)?
<= quota.bytes
&& usage.batches < quota.batches)
.then_some(*epoch)
})
.collect();
state
.waiters
.retain(|(id, _)| valid.contains(id) && eligible.contains(id));
if !state.waiters.iter().any(|(id, _)| *id == source.epoch) {
if state.waiters.len() >= self.limits.max_read_waiters {
return None;
}
state.waiters.push_back((source.epoch, count));
}
if state.waiters.front().map(|(id, _)| *id) != Some(source.epoch) {
return None;
}
if state.reads.len() >= self.limits.max_active_batches
|| state
.records
.len()
.checked_add(state.reserved_records)?
.checked_add(count)?
> self.limits.record_limit()
|| state.bytes.checked_add(bytes)? > self.limits.max_inflight_payload_bytes
{
return None;
}
state.waiters.pop_front();
state.reads.insert(source.epoch, (count, bytes));
state.reserved_records += count;
state.bytes += bytes;
state.domains[source.domain].records += count;
state.domains[source.domain].bytes += bytes;
state.domains[source.domain].batches += 1;
source.io.fetch_add(1, Ordering::AcqRel);
Some(ReadPermit {
core: self.clone(),
source: source.clone(),
released: false,
})
}
pub(in crate::partition) fn drained(&self, source: &SourceAuthority) -> bool {
let state = self.state.lock().expect("ledger");
source.io.load(Ordering::Acquire) == 0
&& !state.reads.contains_key(&source.epoch)
&& !state
.records
.values()
.any(|r| r.source.epoch == source.epoch)
&& !state.tasks.values().any(|t| t.source.epoch == source.epoch)
}
pub(in crate::partition) fn close_roots(&self) {
let mut state = self.state.lock().expect("ledger");
self.roots_closed.store(true, Ordering::Release);
state.waiters.clear();
for source in state.sources.values().filter_map(Weak::upgrade) {
source.quiesce();
}
for task in state.tasks.values() {
if let Some(submission) = &task.submission {
submission.cancel();
}
}
self.changed.notify_waiters();
}
pub(in crate::partition) fn snapshot(&self) -> PartitionSnapshot {
let state = self.state.lock().expect("ledger");
let mut result = PartitionSnapshot {
executor_scope: self.executions.scope,
execution_domains: self
.executions
.domains
.iter()
.enumerate()
.map(|(id, domain)| {
let usage = &state.domains[id];
ExecutionDomainSnapshot {
id,
group: domain.spec.group.clone(),
partition: domain.spec.partition,
record_capacity: domain.spec.quota.records,
payload_capacity: domain.spec.quota.bytes,
batch_capacity: domain.spec.quota.batches,
delete_capacity: domain.spec.quota.deletes,
inflight_records: usage.records,
payload_bytes: usage.bytes,
active_batches: usage.batches,
inflight_tasks: state
.tasks
.values()
.filter(|t| t.source.domain == id)
.count(),
async_delete_pending: domain.delete_records.load(Ordering::Acquire),
runner_degraded: domain.degraded.load(Ordering::Acquire),
runner_stopped: domain.runner.health() == napart::RunnerHealth::Stopped,
}
})
.collect(),
ready: !self.roots_closed.load(Ordering::Acquire)
&& !self.degraded.load(Ordering::Acquire)
&& !state.protocol_closed
&& state.record_debt == 0,
runner_degraded: self.degraded.load(Ordering::Acquire),
inflight_records: state.records.len() + state.reserved_records,
payload_bytes: state.bytes,
active_batches: state.reads.len(),
read_waiters: state.waiters.len(),
inflight_tasks: state.tasks.len(),
ordered_keys: state.gates.len(),
record_debt: state.record_debt,
byte_debt: state
.bytes
.saturating_sub(self.limits.max_inflight_payload_bytes),
committed_records: state.committed,
retained_records: state.retained,
unconfirmed_locks: state
.sources
.values()
.filter_map(Weak::upgrade)
.filter(|source| source.unreleased.load(Ordering::Acquire))
.count(),
running_tasks: state
.tasks
.values()
.filter(|t| {
t.submission
.as_ref()
.is_some_and(|s| s.status() == napart::TaskStatus::Running)
})
.count(),
source_io: state
.sources
.values()
.filter_map(Weak::upgrade)
.map(|s| s.io.load(Ordering::Acquire))
.sum(),
unjoined_readers: state.abandoned_readers.len(),
delete_retained_records: self.delete_retained_records.load(Ordering::Acquire),
..Default::default()
};
result.runner_degraded |= result.execution_domains.iter().any(|d| d.runner_degraded);
result.ready &= !result.runner_degraded;
result.blocked_sources = state
.sources
.values()
.filter_map(Weak::upgrade)
.filter(|s| s.blocked.load(Ordering::Acquire))
.count();
result.parked_sources = state
.sources
.values()
.filter_map(Weak::upgrade)
.filter(|source| source.parked.load(Ordering::Acquire))
.count();
result.ready &= result.blocked_sources == 0 && result.parked_sources == 0;
result.oversized_sources = state
.sources
.values()
.filter_map(Weak::upgrade)
.filter(|source| source.oversized.load(Ordering::Acquire))
.count();
result.protocol_error_sources = state
.sources
.values()
.filter_map(Weak::upgrade)
.filter(|source| source.protocol_error.load(Ordering::Acquire))
.count();
for record in state.records.values() {
match record.owner {
RecordOwner::Commit { unknown, .. } => {
result.pending_commits += 1;
result.unknown_commits += usize::from(unknown);
}
RecordOwner::Retry { .. } => result.retry_tickets += 1,
RecordOwner::Parked => result.parked_records += 1,
RecordOwner::Unroutable { .. } => result.unroutable_records += 1,
RecordOwner::Ready if record.gate.is_some() => result.deferred_records += 1,
_ => {}
}
}
result.blocked_keys = state
.gates
.values()
.filter(|g| g.business == BusinessState::Blocked)
.count();
result
}
pub(in crate::partition) fn dispatch(
&self,
mut permit: ReadPermit,
batch: u64,
rows: Vec<(String, Option<Vec<u8>>)>,
plans: &PlanMap,
) {
let source = permit.source.clone();
let Some(rt) = source.group.upgrade() else {
return;
};
let mut prepared = Vec::with_capacity(rows.len());
let mut unroutable = false;
let mut payload_overflow = false;
for (id, body) in rows {
let coordinate = Coordinate {
group: Arc::from(rt.layout.group()),
identity: RecordIdentity {
stream: Arc::from(rt.layout.stream(source.partition)),
id: Arc::from(id),
},
};
let raw_bytes = body.as_ref().map_or(0, Vec::len);
let tombstone = body.is_none();
let mut plan = None;
let mut decoded = None;
let mut failure = None;
if let Some(body) = body {
if body.len() > self.limits.max_record_bytes {
source.oversized.store(true, Ordering::Release);
failure = Some(DecodeFailure::Internal);
} else {
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
let env = serde_json::from_slice::<Envelope>(&body)
.map_err(|_| DecodeFailure::Malformed)?;
let candidate = plans
.get(&(env.topic.clone(), env.event.clone()))
.ok_or(DecodeFailure::Internal)?
.clone();
plan = Some(candidate.clone());
let value = (candidate.decode)(coordinate.identity.clone(), env)?;
Ok::<_, DecodeFailure>((candidate, value))
}));
match result {
Ok(Ok((p, value))) => {
plan = Some(p);
decoded = Some(value);
}
Ok(Err(reason)) => failure = Some(reason),
Err(_) => failure = Some(DecodeFailure::Internal),
}
}
}
if failure.is_some()
&& !(matches!(failure, Some(DecodeFailure::Malformed))
&& plan.as_ref().is_some_and(|p| p.legacy))
{
unroutable = true;
}
let bytes = raw_bytes.checked_add(decoded.as_ref().map_or(0, |d| d.weight));
payload_overflow |= bytes.is_none();
let bytes = bytes.unwrap_or(usize::MAX);
if matches!(failure, Some(DecodeFailure::Internal)) {
plan = None;
}
prepared.push((coordinate, plan, decoded, bytes, tombstone, failure));
}
let mut state = self.state.lock().expect("ledger");
let (reserved, reserved_bytes) =
state.reads.remove(&source.epoch).expect("read reservation");
state.reserved_records -= reserved;
state.bytes -= reserved_bytes;
state.domains[source.domain].records -= reserved;
state.domains[source.domain].bytes -= reserved_bytes;
state.domains[source.domain].batches -= 1;
source.io.fetch_sub(1, Ordering::AcqRel);
permit.released = true;
if prepared.len() > reserved {
state.record_debt = state.record_debt.saturating_add(prepared.len() - reserved);
unroutable = true;
}
let actual = prepared.iter().try_fold(0usize, |n, r| n.checked_add(r.3));
let payload_wait = payload_overflow
|| actual
.and_then(|actual| state.bytes.checked_add(actual))
.is_none_or(|total| total > self.limits.max_inflight_payload_bytes)
|| actual
.and_then(|actual| state.domains[source.domain].bytes.checked_add(actual))
.is_none_or(|total| {
total > self.executions.domains[source.domain].spec.quota.bytes
});
let mut gate_counts: HashMap<GateKey, usize> = HashMap::new();
for (_, plan, decoded, _, _, _) in &prepared {
if let (Some(plan), Some(decoded)) = (plan, decoded) {
let route = if plan.legacy {
Some(napart::RouteHash::from_key(&source.epoch))
} else {
decoded.route
};
if let Some(route) = route {
*gate_counts
.entry(GateKey {
plan: plan.id,
route,
})
.or_default() += 1;
}
}
}
if gate_counts.iter().any(|(key, count)| {
state
.gates
.get(key)
.map_or(0, |g| g.queue.len())
.saturating_add(*count)
> self.limits.max_deferred_records_per_key
}) {
unroutable = true;
}
if state
.next_identity
.checked_add(gate_counts.len() as u64)
.is_none()
{
state.protocol_closed = true;
unroutable = true;
}
if unroutable || payload_wait {
source.blocked.store(true, Ordering::Release);
}
for (coordinate, plan, decoded, bytes, tombstone, failure) in &mut prepared {
if state.records.contains_key(coordinate) {
continue;
}
let gate = if unroutable {
None
} else {
plan.as_ref().and_then(|p| {
decoded.as_ref().and_then(|d| {
let route = if p.legacy {
Some(napart::RouteHash::from_key(&source.epoch))
} else {
d.route
};
route.map(|route| GateKey { plan: p.id, route })
})
})
};
if let Some(key) = gate {
if !state.gates.contains_key(&key) {
let Some(token) = issue(&mut state) else {
source.blocked.store(true, Ordering::Release);
return;
};
state.gates.insert(
key,
OrderedGate {
token,
business: BusinessState::Open,
queue: VecDeque::new(),
executing: None,
},
);
}
state
.gates
.get_mut(&key)
.expect("gate")
.queue
.push_back(coordinate.clone());
}
let owner = if *tombstone {
RecordOwner::Commit {
unknown: false,
confirmed: false,
next: Instant::now(),
}
} else if unroutable || failure.is_some() {
RecordOwner::Unroutable {
malformed: matches!(failure, Some(DecodeFailure::Malformed)),
next: Instant::now(),
}
} else if payload_wait {
RecordOwner::Retry {
next: Instant::now() + Duration::from_millis(200),
operation: format!("capacity:{}:{}", source.epoch, coordinate.identity.id),
executed: false,
}
} else {
RecordOwner::Ready
};
let bytes = if unroutable || payload_wait || *tombstone || failure.is_some() {
0
} else {
*bytes
};
let payload = if unroutable || payload_wait || *tombstone || failure.is_some() {
None
} else {
decoded.take().map(|d| d.payload)
};
if let Some(payload) = payload {
state.prepared.insert(coordinate.clone(), payload);
}
state.bytes = state.bytes.saturating_add(bytes);
state.domains[source.domain].records += 1;
state.domains[source.domain].bytes =
state.domains[source.domain].bytes.saturating_add(bytes);
state.records.insert(
coordinate.clone(),
Record {
source: source.clone(),
batch,
revision: 0,
owner,
plan: plan.clone(),
gate,
bytes,
attempts: 0,
retry_intent: None,
},
);
}
self.changed.notify_waiters();
}
fn settle_tasks(&self, state: &mut State) {
let done: Vec<_> = state
.tasks
.iter()
.filter_map(|(id, task)| {
let status = task
.submission
.as_ref()
.map_or(napart::TaskStatus::Rejected, |s| s.status());
matches!(
status,
napart::TaskStatus::Completed
| napart::TaskStatus::Cancelled
| napart::TaskStatus::Rejected
| napart::TaskStatus::Failed
)
.then_some((*id, status))
})
.collect();
for (id, status) in done {
let task = state.tasks.remove(&id).expect("task ticket");
let outcome = task
.outcome
.value
.lock()
.expect("outcome")
.clone()
.filter(|outcome| {
status == napart::TaskStatus::Completed
&& outcome.succeeded.len() + outcome.failed.len() + outcome.deferred.len()
== task.records.len()
&& task.records.iter().all(|coordinate| {
outcome
.succeeded
.iter()
.chain(&outcome.failed)
.chain(&outcome.deferred)
.filter(|candidate| *candidate == coordinate)
.count()
== 1
})
});
if let Some((key, token)) = task.gate {
if let Some(gate) = state.gates.get_mut(&key) {
if gate.token == token && gate.executing == Some(id) {
gate.executing = None;
}
}
}
for coordinate in &task.records {
let Some(record) = state.records.get_mut(coordinate) else {
continue;
};
if !matches!(record.owner, RecordOwner::Task(current) if current == id) {
continue;
}
state.bytes -= record.bytes;
state.domains[record.source.domain].bytes -= record.bytes;
record.bytes = 0;
let success = outcome
.as_ref()
.is_some_and(|o| o.succeeded.contains(coordinate));
let failed = outcome
.as_ref()
.is_some_and(|o| o.failed.contains(coordinate));
let Some(revision) = record.revision.checked_add(1) else {
record.source.lose();
continue;
};
record.revision = revision;
record.retry_intent = None;
record.owner = if success {
RecordOwner::Commit {
unknown: false,
confirmed: false,
next: Instant::now(),
}
} else {
RecordOwner::Retry {
next: Instant::now() + Duration::from_millis(200),
operation: format!(
"{}:{}:{}:{}",
task.source.epoch,
record.batch,
record.revision,
coordinate.identity.id
),
executed: failed,
}
};
if failed {
record.attempts = record.attempts.saturating_add(1);
}
}
}
}
fn submit_ready(self: &Arc<Self>, state: &mut State, payload_holds: &mut Vec<TaskPayloads>) {
if self.roots_closed.load(Ordering::Acquire) || self.degraded.load(Ordering::Acquire) {
return;
}
let mut candidates: Vec<_> = state
.records
.iter()
.filter(|(_, r)| matches!(r.owner, RecordOwner::Ready) && r.source.can_start())
.map(|(c, r)| (r.batch, c.clone()))
.collect();
candidates.sort_by(|a, b| {
a.0.cmp(&b.0)
.then_with(|| stream_id_cmp(&a.1.identity.id, &b.1.identity.id))
});
for (_, coordinate) in candidates {
let Some(record) = state.records.get(&coordinate) else {
continue;
};
if !matches!(record.owner, RecordOwner::Ready) {
continue;
}
let source = record.source.clone();
let plan = record.plan.as_ref().expect("ready plan").clone();
let batch = record.batch;
if plan.legacy
&& state.records.values().any(|candidate| {
candidate.source.epoch == source.epoch
&& candidate.batch == batch
&& candidate
.plan
.as_ref()
.is_some_and(|candidate_plan| candidate_plan.id == plan.id)
&& matches!(candidate.owner, RecordOwner::Retry { .. })
})
{
continue;
}
let gate_key = record.gate;
let mut records = vec![coordinate.clone()];
let mut gate_token = None;
if let Some(key) = gate_key {
let gate = state.gates.get(&key).expect("record gate");
if gate.executing.is_some()
|| gate.queue.front() != Some(&coordinate)
|| gate.queue.iter().any(|c| {
state
.records
.get(c)
.is_some_and(|r| matches!(r.owner, RecordOwner::Commit { .. }))
})
{
continue;
}
records = gate
.queue
.iter()
.take_while(|c| {
state.records.get(*c).is_some_and(|r| {
matches!(r.owner, RecordOwner::Ready)
&& r.source.epoch == source.epoch
&& r.batch == batch
})
})
.cloned()
.collect();
gate_token = Some((key, gate.token));
}
let Some(id) = issue(state) else {
return;
};
let outcome = Arc::new(OutcomeCell {
value: Mutex::new(None),
changed: Notify::new(),
});
let mut payloads = Vec::with_capacity(records.len());
for coordinate in &records {
let record = state.records.get_mut(coordinate).expect("record");
record.owner = RecordOwner::Task(id);
payloads.push((
coordinate.clone(),
state.prepared.remove(coordinate).expect("ready payload"),
));
}
if let Some((key, _)) = gate_token {
let gate = state.gates.get_mut(&key).expect("gate");
gate.executing = Some(id);
gate.business = BusinessState::Executing;
}
state.tasks.insert(
id,
TaskTicket {
source: source.clone(),
records: records.clone(),
gate: gate_token,
submission: None,
outcome: outcome.clone(),
},
);
let core = self.clone();
let domain = source.domain;
let spec = plan.spec(gate_key.is_some());
let route = gate_key.map_or_else(napart::RouteHash::unordered, |g| g.route);
let payloads = Arc::new(Mutex::new(Some(payloads)));
payload_holds.push(payloads.clone());
let submission = self.executions.domains[source.domain]
.runner
.submit_routed_typed(route, spec, move || async move {
let items = payloads
.lock()
.expect("task payload")
.take()
.expect("payload owner");
core.execute(id, source, plan, items, outcome).await;
});
match submission {
Ok(submission) => {
state
.tasks
.get_mut(&id)
.expect("registered task")
.submission = Some(submission);
}
Err(napart::SubmitRejection::QueueFull | napart::SubmitRejection::Overloaded) => {}
Err(_) => {
self.executions.domains[domain]
.degraded
.store(true, Ordering::Release);
for source in state.sources.values().filter_map(Weak::upgrade) {
if source.domain == domain {
source.quiesce();
}
}
}
}
}
}
fn task_can_start(&self, id: u64, source: &SourceAuthority) -> bool {
let state = self.state.lock().expect("ledger");
source.can_start()
&& state.tasks.get(&id).is_some_and(|task| {
task.source.epoch == source.epoch
&& task.records.iter().all(|coordinate| {
state.records.get(coordinate).is_some_and(|record| {
record.source.epoch == source.epoch
&& matches!(record.owner, RecordOwner::Task(current) if current == id)
})
})
&& task.gate.is_none_or(|(key, token)| {
state
.gates
.get(&key)
.is_some_and(|gate| gate.token == token && gate.executing == Some(id))
})
})
}
async fn execute(
self: Arc<Self>,
id: u64,
source: Arc<SourceAuthority>,
plan: Arc<ConsumerPlan>,
items: Vec<(Coordinate, Payload)>,
cell: Arc<OutcomeCell>,
) {
let mut outcome = ConsumeOutcome {
succeeded: Vec::new(),
failed: Vec::new(),
deferred: Vec::new(),
};
let Some(rt) = source.group.upgrade() else {
return;
};
let all: Vec<_> = items.iter().map(|(c, _)| c.clone()).collect();
let held = source.can_start()
&& rt.lock.holds_status(&source.lock_key, &source.holder).await == HoldStatus::Held;
if !held || !self.task_can_start(id, &source) {
outcome.deferred = all;
} else if plan.legacy {
let args = items
.into_iter()
.map(|(c, payload)| (c.identity.id.to_string(), payload))
.collect();
let result = tokio::time::timeout(
Duration::from_millis(rt.cfg.handler_timeout_ms),
std::panic::AssertUnwindSafe(async { (plan.invoke)(args).await }).catch_unwind(),
)
.await;
match result {
Ok(Ok(failed)) => {
for coordinate in all {
if failed
.iter()
.any(|id| id == coordinate.identity.id.as_ref())
{
outcome.failed.push(coordinate);
} else {
outcome.succeeded.push(coordinate);
}
}
}
_ => outcome.failed = all,
}
} else {
let mut stopped = false;
for (coordinate, payload) in items {
if stopped || !self.task_can_start(id, &source) {
outcome.deferred.push(coordinate);
continue;
}
let entry_id = coordinate.identity.id.to_string();
let result = tokio::time::timeout(
Duration::from_millis(rt.cfg.handler_timeout_ms),
std::panic::AssertUnwindSafe(async {
(plan.invoke)(vec![(entry_id, payload)]).await
})
.catch_unwind(),
)
.await;
if matches!(result, Ok(Ok(ref failed)) if failed.is_empty()) {
outcome.succeeded.push(coordinate);
} else {
stopped = true;
outcome.failed.push(coordinate);
}
}
}
{
let mut state = self.state.lock().expect("ledger");
let gate_ticket = state.tasks.get(&id).and_then(|task| task.gate);
if let Some((key, token)) = gate_ticket {
if let Some(gate) = state.gates.get_mut(&key) {
if gate.token == token && gate.executing == Some(id) {
gate.business = if !source.can_commit() {
BusinessState::Invalidating
} else if !outcome.failed.is_empty() || !outcome.deferred.is_empty() {
BusinessState::Blocked
} else {
BusinessState::Open
};
}
}
}
*cell.value.lock().expect("outcome") = Some(outcome);
}
cell.changed.notify_waiters();
self.changed.notify_waiters();
}
async fn supervise(self: Arc<Self>, domain: usize) {
loop {
let readers = {
let mut state = self.state.lock().expect("ledger");
let mut finished = Vec::new();
let mut index = 0;
while index < state.abandoned_readers.len() {
if state.abandoned_readers[index].is_finished() {
finished.push(state.abandoned_readers.swap_remove(index));
} else {
index += 1;
}
}
finished
};
for reader in readers {
let _ = reader.await;
}
if matches!(
self.executions.domains[domain].runner.health(),
napart::RunnerHealth::Degraded | napart::RunnerHealth::Failed
) {
self.executions.domains[domain]
.degraded
.store(true, Ordering::Release);
for source in self
.state
.lock()
.expect("ledger")
.sources
.values()
.filter_map(Weak::upgrade)
{
if source.domain == domain {
source.quiesce();
}
}
}
let mut retired_payloads = Vec::new();
let mut payload_holds = Vec::new();
let action = {
let mut state = self.state.lock().expect("ledger");
self.settle_tasks(&mut state);
let retained: Vec<_> = state
.records
.iter()
.filter(|(_, r)| {
!matches!(r.owner, RecordOwner::Task(_))
&& (!r.source.can_commit()
|| (!r.source.active()
&& !matches!(r.owner, RecordOwner::Commit { .. })))
})
.map(|(c, _)| c.clone())
.collect();
for coordinate in retained {
if let Some(payload) = retire(&self, &mut state, &coordinate, false) {
retired_payloads.push(payload);
}
}
self.submit_ready(&mut state, &mut payload_holds);
if self.stopping.load(Ordering::Acquire)
&& state.records.is_empty()
&& state.tasks.is_empty()
&& state.reads.is_empty()
&& state.abandoned.is_empty()
&& state.abandoned_readers.is_empty()
{
break;
}
next_io(&state, domain)
};
drop(retired_payloads);
drop(payload_holds);
if let Some(action) = action {
let lost = action.source.lost_cancel.clone();
tokio::select! {
biased;
_ = self.force_io.cancelled() => {
}
_ = lost.cancelled() => {
}
_ = self.run_io(action) => {
}
}
tokio::task::yield_now().await;
} else {
let abandoned = self
.state
.lock()
.expect("ledger")
.abandoned
.values()
.find(|source| source.domain == domain)
.cloned();
if let Some(source) = abandoned {
if let Some(rt) = source.group.upgrade() {
if rt.lock.holds_status(&source.lock_key, &source.holder).await
== HoldStatus::Lost
{
source.unreleased.store(false, Ordering::Release);
self.state
.lock()
.expect("ledger")
.abandoned
.remove(&source.epoch);
}
}
}
tokio::select! {
_ = self.changed.notified() => {
}
_ = tokio::time::sleep(Duration::from_millis(5)) => {
}
}
}
}
}
pub(in crate::partition) async fn stop(&self) {
self.close_roots();
self.stopping.store(true, Ordering::Release);
self.changed.notify_waiters();
let handles = std::mem::take(&mut *self.supervisors.lock().expect("supervisors"));
for handle in handles {
let _ = handle.await;
}
for domain in &self.executions.domains {
domain.runner.request_stop();
}
futures::future::join_all(self.executions.domains.iter().map(|domain| async move {
loop {
if domain
.runner
.stop(Instant::now() + Duration::from_secs(30))
.await
.is_ok()
{
break;
}
}
}))
.await;
}
}
pub(in crate::partition) struct ReadPermit {
core: Arc<RedisPartitionRuntime>,
source: Arc<SourceAuthority>,
released: bool,
}
impl Drop for ReadPermit {
fn drop(&mut self) {
if !self.released {
let mut state = self.core.state.lock().expect("ledger");
if let Some((records, bytes)) = state.reads.remove(&self.source.epoch) {
state.reserved_records -= records;
state.bytes -= bytes;
state.domains[self.source.domain].records -= records;
state.domains[self.source.domain].bytes -= bytes;
state.domains[self.source.domain].batches -= 1;
self.source.io.fetch_sub(1, Ordering::AcqRel);
}
self.core.changed.notify_waiters();
}
}
}
fn issue(state: &mut State) -> Option<u64> {
let id = state.next_identity;
match id.checked_add(1) {
Some(next) => {
state.next_identity = next;
Some(id)
}
None => {
state.protocol_closed = true;
None
}
}
}
fn retire<'a>(
core: &'a RedisPartitionRuntime,
state: &mut State,
coordinate: &Coordinate,
committed: bool,
) -> Option<RetiredPayload<'a>> {
detach_gate(state, coordinate);
if let Some(record) = state.records.remove(coordinate) {
let payload = state.prepared.remove(coordinate);
if committed {
state.committed = state.committed.saturating_add(1);
} else {
state.retained = state.retained.saturating_add(1);
}
if let Some(payload) = payload {
state.reserved_records += 1;
record.source.io.fetch_add(1, Ordering::AcqRel);
return Some(RetiredPayload {
_payload: payload,
_budget: RetiredPayloadBudget {
core,
source: record.source,
bytes: record.bytes,
},
});
}
state.bytes -= record.bytes;
state.domains[record.source.domain].records -= 1;
state.domains[record.source.domain].bytes -= record.bytes;
}
None
}
fn detach_gate(state: &mut State, coordinate: &Coordinate) {
if let Some(key) = state
.records
.get_mut(coordinate)
.and_then(|record| record.gate.take())
{
if let Some(gate) = state.gates.get_mut(&key) {
gate.queue.retain(|candidate| candidate != coordinate);
if gate.queue.is_empty() && gate.executing.is_none() {
state.gates.remove(&key);
}
}
}
}
fn stream_id_cmp(left: &str, right: &str) -> std::cmp::Ordering {
let parse = |value: &str| {
value
.split_once('-')
.and_then(|(a, b)| Some((a.parse::<u64>().ok()?, b.parse::<u64>().ok()?)))
.unwrap_or_default()
};
parse(left).cmp(&parse(right))
}
#[path = "engine_io.rs"]
mod io;
use io::next_io;