use super::*;
use futures::StreamExt;
use std::time::Duration;
pub(super) async fn control_loop(rt: Arc<GroupRuntime>) {
use std::collections::HashSet;
let mut ensured: HashSet<u32> = HashSet::new(); let mut tick = tokio::time::interval(Duration::from_millis(500));
tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
loop {
tokio::select! {
_ = rt.bg_cancel.cancelled() => return, _ = tick.tick() => {}
}
let owned: Vec<u32> = rt.claimed.read().expect("claimed").clone();
for p in owned {
let cstream = super::command::cmd_stream_key(&rt.layout, p);
if !ensured.contains(&p) {
let r: std::result::Result<String, redis::RedisError> = redis::cmd("XGROUP")
.arg("CREATE")
.arg(&cstream)
.arg(super::command::ADMIN_GROUP)
.arg("0")
.arg("MKSTREAM")
.query_async(&mut rt.client.conn())
.await;
match r {
Ok(_) => {
ensured.insert(p);
}
Err(e) if e.to_string().contains("BUSYGROUP") => {
ensured.insert(p);
}
Err(e) => {
tracing::warn!(p, err = %e, "admin group 创建失败");
continue;
}
}
}
let mut entries: Vec<(String, String)> = Vec::new(); let fresh: std::result::Result<redis::Value, _> = redis::cmd("XREADGROUP")
.arg("GROUP")
.arg(super::command::ADMIN_GROUP)
.arg(&rt.node_id)
.arg("COUNT")
.arg(8)
.arg("STREAMS")
.arg(&cstream)
.arg(">")
.query_async(&mut rt.client.conn())
.await;
match fresh {
Ok(v) => collect_cmd_entries(v, &mut entries),
Err(e) => {
if e.to_string().contains("NOGROUP") {
ensured.remove(&p);
tracing::warn!(p, "control admin group 丢失(NOGROUP),下轮重建");
} else {
tracing::warn!(p, err = %e, "control 读新命令(>)失败,下轮重试");
}
}
}
const PAGE_BUDGET: u32 = 4;
let mut cursor = "0".to_string();
let mut pages = 0u32;
loop {
let claimed: std::result::Result<redis::Value, _> = redis::cmd("XAUTOCLAIM")
.arg(&cstream)
.arg(super::command::ADMIN_GROUP)
.arg(&rt.node_id)
.arg(15_000)
.arg(&cursor)
.arg("COUNT")
.arg(8)
.query_async(&mut rt.client.conn())
.await;
let parts = match claimed {
Ok(redis::Value::Array(parts)) => parts,
Ok(_) => break,
Err(e) => {
if e.to_string().contains("NOGROUP") {
ensured.remove(&p); tracing::warn!(p, "control XAUTOCLAIM 遇 NOGROUP,下轮重建 admin group");
} else {
tracing::warn!(p, err = %e, "control XAUTOCLAIM 失败,下轮重试");
}
break;
}
};
let next = match parts.first() {
Some(redis::Value::BulkString(b)) => String::from_utf8_lossy(b).into_owned(),
Some(redis::Value::SimpleString(s)) => s.clone(),
_ => "0".to_string(),
};
if let Some(ev) = parts.into_iter().nth(1) {
collect_cmd_entries(
redis::Value::Array(vec![redis::Value::Array(vec![
redis::Value::BulkString(cstream.clone().into_bytes()),
ev,
])]),
&mut entries,
);
}
pages += 1;
if next == "0" || next == "0-0" || pages >= PAGE_BUDGET {
break;
}
cursor = next;
}
for (entry_id, op_id) in entries {
let should_finalize = run_command(&rt, p, &op_id).await;
if !should_finalize {
continue;
}
let ack: std::result::Result<i64, _> = redis::cmd("XACK")
.arg(&cstream)
.arg(super::command::ADMIN_GROUP)
.arg(&entry_id)
.query_async(&mut rt.client.conn())
.await;
if let Err(e) = &ack {
tracing::warn!(p, entry_id, err = %e, "control XACK 失败,保留 PEL 下轮重 finalize");
continue; }
let del: std::result::Result<i64, _> = redis::cmd("XDEL")
.arg(&cstream)
.arg(&entry_id)
.query_async(&mut rt.client.conn())
.await;
if let Err(e) = &del {
tracing::warn!(p, entry_id, err = %e, "control XDEL 失败(已 XACK,entry 残留 stream 待 trim)");
}
}
}
}
}
pub(super) fn collect_cmd_entries(v: redis::Value, out: &mut Vec<(String, String)>) {
use redis::Value as V;
match v {
V::Nil => {} V::Array(streams) | V::Set(streams) => {
for sv in streams {
let (V::Array(pair) | V::Set(pair)) = sv else {
continue;
};
if let Some(entries_v) = pair.into_iter().nth(1) {
collect_cmd_stream_entries(entries_v, out);
}
}
}
V::Map(pairs) => {
for (_key, entries_v) in pairs {
collect_cmd_stream_entries(entries_v, out);
}
}
_ => {}
}
}
fn collect_cmd_stream_entries(entries_v: redis::Value, out: &mut Vec<(String, String)>) {
use redis::Value as V;
let (V::Array(entries) | V::Set(entries)) = entries_v else {
return;
};
for e in entries {
let (V::Array(ep) | V::Set(ep)) = e else {
continue;
};
let mut it = ep.into_iter();
let (Some(idv), Some(V::Array(fields) | V::Set(fields))) = (it.next(), it.next()) else {
continue;
};
let id = String::from_utf8_lossy(&wire::bytes(idv).unwrap_or_default()).into_owned();
let mut fit = fields.into_iter();
while let (Some(f), Some(val)) = (fit.next(), fit.next()) {
if wire::bytes(f).unwrap_or_default() == b"op" {
out.push((
id.clone(),
String::from_utf8_lossy(&wire::bytes(val).unwrap_or_default()).into_owned(),
));
}
}
}
}
pub(super) async fn run_command(rt: &Arc<GroupRuntime>, p: u32, op_id: &str) -> bool {
use super::command::{execute_one, query, CmdAck, CmdAction};
let pre = match query(&rt.client, &rt.layout, p, op_id).await {
Ok(Some(pre)) => pre,
Ok(None) => {
tracing::warn!(p, op_id, "命令缺 result(producer 违规),清理");
return true;
}
Err(e) => {
tracing::warn!(p, op_id, err = %e, "命令 result 查询失败,保留 PEL 重试");
return false;
}
};
let action = pre.action;
let of = match rt.require_owner_for_op(p, op_id) {
Ok(of) => of,
Err(_) => {
tracing::warn!(
p,
op_id,
"control 执行命令时本节点非 owner(owner_ctx 缺),保留 PEL"
);
return false;
}
};
let fc = of.lease();
let fut = async {
match action {
CmdAction::Resume => super::disposition::resume(&rt.client, &rt.layout, p, &fc).await,
CmdAction::Force => {
super::disposition::force_republish(&rt.client, &rt.layout, p, &fc).await
}
CmdAction::Drop => super::disposition::drop_parked(&rt.client, &rt.layout, p, &fc)
.await
.map(|_| Vec::new()),
CmdAction::Dlq => {
super::disposition::dlq_from_parked(&rt.client, &rt.layout, p, &fc).await
}
}
};
match execute_one(&rt.client, &rt.layout, p, op_id, fut, action).await {
Ok(CmdAck::Finalize) => {
if let Ok(Some(r)) = query(&rt.client, &rt.layout, p, op_id).await {
if matches!(r.state, super::command::CmdState::Succeeded) {
let _ = rt.event_tx.send(Event::ParkResolved { p }).await;
}
}
true
}
Ok(CmdAck::Retry) => false,
Err(e) => {
tracing::warn!(p, op_id, err = %e, "命令执行编排异常(result 留 Executing),保留 PEL 重试");
false
}
}
}
pub(super) async fn sweep_loop(rt: Arc<GroupRuntime>) {
let mut tick = tokio::time::interval(Duration::from_millis(rt.cfg.min_idle_ms.max(1000)));
tick.tick().await;
loop {
tokio::select! {
_ = rt.bg_cancel.cancelled() => return,
_ = tick.tick() => {
}
}
rt.core.request_recovery(&rt.layout.prefix);
}
}
pub(super) async fn wake_loop(rt: Arc<GroupRuntime>) {
loop {
if rt.bg_cancel.is_cancelled() {
return;
}
let Ok(mut pubsub) = rt.client.raw_client().get_async_pubsub().await else {
tokio::time::sleep(Duration::from_secs(1)).await;
continue;
};
if pubsub.subscribe(rt.layout.wake()).await.is_err() {
tokio::time::sleep(Duration::from_secs(1)).await;
continue;
}
let mut msgs = pubsub.on_message();
loop {
tokio::select! {
_ = rt.bg_cancel.cancelled() => return,
m = msgs.next() => {
if m.is_none() {
break;
}
let _ = rebalance_once(&rt).await;
}
}
}
}
}