use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
use std::sync::Arc;
use futures::FutureExt;
use serde::Serialize;
use tokio::sync::mpsc;
use tokio_util::sync::CancellationToken;
use super::KeyLayout;
use super::{command, disposition, fencing, retryop};
use crate::client::RedisClient;
use crate::config::PartitionCfg;
use crate::error::{NasaRedisError, Result};
use crate::lock::{DistributedLock, LockGuard};
async fn run_when_activated(
gate: CancellationToken,
cancel: CancellationToken,
core: Arc<engine::RedisPartitionRuntime>,
task: impl std::future::Future<Output = ()>,
) {
tokio::select! {
biased;
_ = cancel.cancelled() => return,
_ = gate.cancelled() => {}
}
let result = std::panic::AssertUnwindSafe(task).catch_unwind().await;
if result.is_err() || !cancel.is_cancelled() {
core.degraded.store(true, Ordering::Release);
core.close_roots();
}
}
mod identity;
use identity::BatchIdentity;
pub(super) mod engine;
mod source;
mod wire;
pub use engine::{ExecutionDomainSnapshot, PartitionSnapshot};
struct ClaimSlot {
source: Arc<engine::SourceAuthority>,
guard: Option<LockGuard>,
reader: Option<tokio::task::JoinHandle<()>>,
}
impl Drop for ClaimSlot {
fn drop(&mut self) {
self.source.lose();
if let Some(reader) = self.reader.take() {
reader.abort();
if let Some(rt) = self.source.group.upgrade() {
rt.core.abandoned_reader(reader);
}
}
if let Some(guard) = self.guard.take() {
if let Some(rt) = self.source.group.upgrade() {
rt.core.abandoned_source(&self.source);
}
guard.abandon();
}
}
}
struct AsyncDeleteBatch {
partition: u32,
ids: Vec<String>,
}
enum Event {
ClaimAcquired {
p: u32,
guard: LockGuard,
},
RetainShare {
target: usize,
},
ParkResolved {
p: u32,
},
}
pub struct GroupRuntime {
pub(super) client: Arc<RedisClient>,
pub(super) lock: Arc<DistributedLock>,
pub(super) layout: KeyLayout,
pub(super) count: u32,
pub(super) cfg: PartitionCfg,
pub(super) stream_cfg: crate::config::StreamCfg,
node_id: String,
event_tx: mpsc::Sender<Event>,
cancel: CancellationToken,
bg_cancel: CancellationToken,
async_delete_tx: Option<mpsc::Sender<AsyncDeleteBatch>>,
async_delete_admission: std::sync::Mutex<Vec<usize>>,
async_delete_cancel: CancellationToken,
async_delete_flush: tokio::sync::Notify,
async_delete_handle: std::sync::Mutex<Option<tokio::task::JoinHandle<()>>>,
async_delete_pending: AtomicUsize,
claimed: Arc<std::sync::RwLock<Vec<u32>>>,
rr_counter: AtomicU64,
plans: super::plan::PlanMap,
pub(super) core: Arc<engine::RedisPartitionRuntime>,
pub(super) publisher: Arc<super::publisher::PublisherCoordinator>,
fence_meta: Option<super::fencing::FenceMeta>,
rebalance_lock: tokio::sync::Mutex<()>,
bg_tracker: tokio_util::task::TaskTracker,
bg_aborts: std::sync::Mutex<Vec<tokio::task::AbortHandle>>,
coordinator_handle: std::sync::Mutex<Option<tokio::task::JoinHandle<()>>>,
shutdown_lock: tokio::sync::Mutex<()>,
pub(super) owner_ctx: std::sync::Mutex<std::collections::HashMap<u32, OwnerCtx>>,
}
pub(super) struct OwnerCtx {
pub holder: String,
pub counter: u64,
}
pub(super) struct OwnerFence {
operation_id: String,
expected_park_id: Option<String>,
holder: String,
lock_key: String,
fence_key: String,
round: u64,
nonce: String,
p: u32,
counter: u64,
}
impl OwnerFence {
pub(super) fn lease(&self) -> super::disposition::OwnerLease<'_> {
super::disposition::OwnerLease {
partition: self.p,
operation_id: &self.operation_id,
expected_park_id: self.expected_park_id.as_deref(),
holder: &self.holder,
lock_key: &self.lock_key,
fence_key: &self.fence_key,
round: self.round,
nonce: &self.nonce,
counter: self.counter,
}
}
}
impl GroupRuntime {
pub(super) fn try_enqueue_async_delete(&self, partition: u32, id: &str) -> bool {
let Some(tx) = &self.async_delete_tx else {
return true;
};
let mut counts = self
.async_delete_admission
.lock()
.expect("delete admission");
if self.core.roots_closed.load(Ordering::Acquire) {
self.async_delete_flush.notify_one();
}
if tx.is_closed() {
self.core
.delete_retained_records
.fetch_add(1, Ordering::AcqRel);
return true;
}
let domain = &self.core.executions.domains
[self.core.executions.domain(&self.layout.prefix, partition)];
if domain
.delete_records
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |n| {
n.checked_add(1)
.filter(|next| *next <= domain.spec.quota.deletes)
})
.is_err()
{
return false;
}
counts[partition as usize] += 1;
self.core
.async_delete_records
.fetch_add(1, Ordering::AcqRel);
self.async_delete_pending.fetch_add(1, Ordering::AcqRel);
let result = tx
.try_send(AsyncDeleteBatch {
partition,
ids: vec![id.to_string()],
})
.is_ok();
if !result {
counts[partition as usize] -= 1;
domain.delete_records.fetch_sub(1, Ordering::AcqRel);
self.async_delete_pending.fetch_sub(1, Ordering::AcqRel);
self.core
.async_delete_records
.fetch_sub(1, Ordering::AcqRel);
}
result
}
#[allow(clippy::too_many_arguments)] pub(super) async fn start(
client: Arc<RedisClient>,
lock: Arc<DistributedLock>,
layout: KeyLayout,
count: u32,
plans: super::plan::PlanMap,
core: Arc<engine::RedisPartitionRuntime>,
publisher: Arc<super::publisher::PublisherCoordinator>,
node_id: String,
cfg: PartitionCfg,
stream_cfg: crate::config::StreamCfg,
start_gate: CancellationToken,
) -> Result<Arc<GroupRuntime>> {
let (event_tx, event_rx) = mpsc::channel::<Event>(count as usize * 2 + 16);
let (async_delete_tx, async_delete_rx) = if stream_cfg.async_del_record_period_ms == 0 {
(None, None)
} else {
let capacity = stream_cfg.inflight_max.clamp(1, 4_096);
let (tx, rx) = mpsc::channel(capacity);
(Some(tx), Some(rx))
};
let fence_meta = if matches!(
client.profile(),
crate::config::CompatibilityProfile::RustV2
) {
let tmp_layout = layout.clone();
Some(super::fencing::bootstrap(&client, &tmp_layout, 1).await?)
} else {
None
};
let rt = Arc::new(GroupRuntime {
client,
lock,
layout,
count,
cfg,
stream_cfg,
node_id,
event_tx,
cancel: CancellationToken::new(),
bg_cancel: CancellationToken::new(),
async_delete_tx,
async_delete_admission: std::sync::Mutex::new(vec![0; count as usize]),
async_delete_cancel: CancellationToken::new(),
async_delete_flush: tokio::sync::Notify::new(),
async_delete_handle: std::sync::Mutex::new(None),
async_delete_pending: AtomicUsize::new(0),
claimed: Arc::new(std::sync::RwLock::new(Vec::new())),
rr_counter: AtomicU64::new(0),
plans,
core,
publisher,
fence_meta,
rebalance_lock: tokio::sync::Mutex::new(()),
bg_tracker: tokio_util::task::TaskTracker::new(),
bg_aborts: std::sync::Mutex::new(Vec::new()),
coordinator_handle: std::sync::Mutex::new(None),
shutdown_lock: tokio::sync::Mutex::new(()),
owner_ctx: std::sync::Mutex::new(std::collections::HashMap::new()),
});
let coord = tokio::spawn(run_when_activated(
start_gate.clone(),
rt.cancel.clone(),
rt.core.clone(),
coordinator_loop(Arc::clone(&rt), event_rx),
));
rt.bg_aborts
.lock()
.expect("owner aborts")
.push(coord.abort_handle());
*rt.coordinator_handle.lock().expect("coordinator_handle") = Some(coord);
{
let mut aborts = rt.bg_aborts.lock().expect("bg_aborts");
aborts.push(
rt.bg_tracker
.spawn(run_when_activated(
start_gate.clone(),
rt.bg_cancel.clone(),
rt.core.clone(),
rebalance_loop(Arc::clone(&rt)),
))
.abort_handle(),
); aborts.push(
rt.bg_tracker
.spawn(run_when_activated(
start_gate.clone(),
rt.bg_cancel.clone(),
rt.core.clone(),
wake_loop(Arc::clone(&rt)),
))
.abort_handle(),
); aborts.push(
rt.bg_tracker
.spawn(run_when_activated(
start_gate.clone(),
rt.bg_cancel.clone(),
rt.core.clone(),
control_loop(Arc::clone(&rt)),
))
.abort_handle(),
); aborts.push(
rt.bg_tracker
.spawn(run_when_activated(
start_gate.clone(),
rt.bg_cancel.clone(),
rt.core.clone(),
sweep_loop(Arc::clone(&rt)),
))
.abort_handle(),
);
if rt.stream_cfg.auto_trim_rate_ms != 0 {
aborts.push(
rt.bg_tracker
.spawn(run_when_activated(
start_gate.clone(),
rt.bg_cancel.clone(),
rt.core.clone(),
auto_trim_loop(Arc::clone(&rt)),
))
.abort_handle(),
);
}
}
if let Some(rx) = async_delete_rx {
let handle = tokio::spawn(run_when_activated(
start_gate,
rt.async_delete_cancel.clone(),
rt.core.clone(),
async_delete_loop(Arc::clone(&rt), rx),
));
rt.bg_aborts
.lock()
.expect("owner aborts")
.push(handle.abort_handle());
*rt.async_delete_handle.lock().expect("async_delete_handle") = Some(handle);
}
Ok(rt)
}
pub(super) async fn ensure_stream_contract(&self) -> Result<()> {
for partition in 0..self.count {
let result: redis::RedisResult<String> = redis::cmd("XGROUP")
.arg("CREATE")
.arg(self.layout.stream(partition))
.arg(self.layout.group())
.arg("0")
.arg("MKSTREAM")
.query_async(&mut self.client.conn())
.await;
match result {
Ok(_) => {}
Err(error) if error.code() == Some("BUSYGROUP") => {}
Err(error) => return Err(error.into()),
}
}
Ok(())
}
pub(super) async fn publish_round_robin<T: Serialize>(
&self,
topic: &str,
event: &str,
data: &T,
) -> Result<String> {
let p = (self.rr_counter.fetch_add(1, Ordering::Relaxed) % self.count as u64) as u32;
self.publish(topic, event, p, data, None).await
}
pub(super) async fn publish<T: Serialize>(
&self,
topic: &str,
event: &str,
p: u32,
data: &T,
passthrough: Option<serde_json::Map<String, serde_json::Value>>,
) -> Result<String> {
self.publisher
.publish(
self.client.clone(),
self.layout.stream(p),
topic,
event,
data,
passthrough,
)
.await
}
pub(super) async fn claimed_partitions(&self) -> Vec<u32> {
self.claimed.read().expect("claimed 锁中毒").clone()
}
pub(super) fn async_delete_pending(&self) -> usize {
self.async_delete_pending.load(Ordering::Acquire)
}
pub(super) fn cancel_best_effort(&self) {
self.bg_cancel.cancel();
self.cancel.cancel();
self.async_delete_cancel.cancel();
for abort in self.bg_aborts.lock().expect("bg_aborts").iter() {
abort.abort();
}
if let Some(handle) = self
.coordinator_handle
.lock()
.expect("coordinator_handle")
.as_ref()
{
handle.abort();
}
if let Some(handle) = self
.async_delete_handle
.lock()
.expect("async_delete_handle")
.as_ref()
{
handle.abort();
}
}
pub(super) fn unfinished_background_tasks(&self) -> usize {
self.bg_aborts
.lock()
.expect("bg_aborts")
.iter()
.filter(|task| !task.is_finished())
.count()
}
pub(super) fn shutdown_on_drop(self: &Arc<Self>) {
if self
.coordinator_handle
.lock()
.expect("coordinator_handle")
.is_none()
{
return;
}
self.bg_cancel.cancel();
self.cancel.cancel();
if let Ok(runtime) = tokio::runtime::Handle::try_current() {
let owner = Arc::clone(self);
runtime.spawn(async move {
owner.shutdown().await;
});
} else {
self.cancel_best_effort();
}
}
pub(super) async fn shutdown(&self) {
let _guard = self.shutdown_lock.lock().await;
self.bg_cancel.cancel();
self.bg_tracker.close();
self.bg_tracker.wait().await;
self.cancel.cancel();
let handle = self.coordinator_handle.lock().expect("coordinator").take();
if let Some(handle) = handle {
let _ = handle.await;
}
self.async_delete_cancel.cancel();
let handle = self
.async_delete_handle
.lock()
.expect("async delete")
.take();
if let Some(handle) = handle {
let _ = handle.await;
}
let _ = self
.client
.z_rem(&self.layout.nodes(), self.node_id.as_str())
.await;
let _: redis::RedisResult<i64> = redis::cmd("PUBLISH")
.arg(self.layout.wake())
.arg("")
.query_async(&mut self.client.conn())
.await;
}
}
impl GroupRuntime {
pub(super) fn owner_fence(&self, p: u32, operation_id: String) -> Option<OwnerFence> {
let oc = self.owner_ctx.lock().expect("owner_ctx");
let c = oc.get(&p)?;
let (fence_key, round, nonce) = match &self.fence_meta {
Some(fm) => (
super::fencing::fence_key(&self.layout),
fm.round,
fm.nonce.clone(),
),
None => (String::new(), 0, String::new()),
};
Some(OwnerFence {
operation_id,
expected_park_id: None,
holder: c.holder.clone(),
lock_key: format!(
"{}{}",
self.client.config().lock.prefix,
self.layout.lock_business_key(p)
),
fence_key,
round,
nonce,
p,
counter: c.counter,
})
}
pub(super) fn require_owner_for_op(&self, p: u32, operation_id: &str) -> Result<OwnerFence> {
self.owner_fence(p, operation_id.to_string())
.ok_or_else(|| {
NasaRedisError::OwnershipChanged(format!(
"分区 {p} 非本节点持有(或锁已丢失),管理命令交新 owner"
))
})
}
async fn require_owner_direct(&self, p: u32) -> Result<OwnerFence> {
let park_id: Option<String> = self
.client
.h_get(&super::disposition::marker_key(&self.layout, p), "park_id")
.await?;
let op_id = match park_id.as_deref() {
Some(pk) if !pk.is_empty() => format!("direct:{p}:{pk}"),
_ => new_operation_id(),
};
let mut owner = self.owner_fence(p, op_id).ok_or_else(|| {
NasaRedisError::OwnershipChanged(format!(
"分区 {p} 非本节点持有(或锁已丢失),管理操作交新 owner"
))
})?;
owner.expected_park_id = Some(park_id.unwrap_or_default());
Ok(owner)
}
pub(super) async fn resume_parked(&self, p: u32) -> Result<Vec<String>> {
let of = self.require_owner_direct(p).await?;
let ids = super::disposition::resume(&self.client, &self.layout, p, &of.lease()).await?;
let _ = self
.event_tx
.send(Event::ParkResolved {
p,
})
.await;
Ok(ids)
}
pub(super) async fn drop_parked(&self, p: u32) -> Result<()> {
let of = self.require_owner_direct(p).await?;
super::disposition::drop_parked(&self.client, &self.layout, p, &of.lease()).await?;
let _ = self
.event_tx
.send(Event::ParkResolved {
p,
})
.await;
Ok(())
}
pub(super) async fn dlq_parked(&self, p: u32) -> Result<Vec<String>> {
let of = self.require_owner_direct(p).await?;
let ids =
super::disposition::dlq_from_parked(&self.client, &self.layout, p, &of.lease()).await?;
let _ = self
.event_tx
.send(Event::ParkResolved {
p,
})
.await;
Ok(ids)
}
pub(super) async fn parked_id(&self, p: u32) -> Result<Option<String>> {
super::disposition::parked_id(&self.client, &self.layout, p).await
}
pub(super) async fn force_republish(&self, p: u32) -> Result<Vec<String>> {
let of = self.require_owner_direct(p).await?;
let ids =
super::disposition::force_republish(&self.client, &self.layout, p, &of.lease()).await?;
let _ = self
.event_tx
.send(Event::ParkResolved {
p,
})
.await;
Ok(ids)
}
pub(super) async fn force_takeover(&self, p: u32) -> Result<Vec<String>> {
let of = self.require_owner_direct(p).await?;
let ids =
super::disposition::force_takeover(&self.client, &self.layout, p, &of.lease()).await?;
let _ = self
.event_tx
.send(Event::ParkResolved {
p,
})
.await;
Ok(ids)
}
}
fn new_operation_id() -> String {
format!("op-{}", uuid::Uuid::new_v4().simple())
}
impl GroupRuntime {
pub(super) fn cfg_batch(&self) -> usize {
self.stream_cfg.batch_size
}
}
mod asyncdel;
mod autotrim;
mod control;
#[path = "claim.rs"]
mod coordinator;
mod rebalance;
use asyncdel::*;
use autotrim::*;
use control::*;
use coordinator::*;
use rebalance::*;