use super::*;
use futures::StreamExt;
use std::time::Duration;
pub(super) async fn control_loop(rt: Arc<GroupRuntime>) {
use std::collections::HashSet;
let mut ensured: HashSet<u32> = HashSet::new();
let mut tick = tokio::time::interval(Duration::from_millis(500));
tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
loop {
tokio::select! {
_ = rt.bg_cancel.cancelled() => return, _ = tick.tick() => {}
}
let owned: Vec<u32> = rt.claimed.read().expect("claimed").clone();
for p in owned {
let cstream = super::command::cmd_stream_key(&rt.layout, p);
if !ensured.contains(&p) {
let r: std::result::Result<String, redis::RedisError> = redis::cmd("XGROUP")
.arg("CREATE")
.arg(&cstream)
.arg(super::command::ADMIN_GROUP)
.arg("0")
.arg("MKSTREAM")
.query_async(&mut rt.client.conn())
.await;
match r {
Ok(reply) if reply == "OK" => {
ensured.insert(p);
}
Err(e) if e.code() == Some("BUSYGROUP") => {
ensured.insert(p);
}
Ok(_) => {
tracing::warn!(p, "admin group 创建响应非法");
continue;
}
Err(e) => {
tracing::warn!(p, err = %e, "admin group 创建失败");
continue;
}
}
}
let mut entries: Vec<(String, String)> = Vec::new();
let fresh: std::result::Result<redis::Value, _> = redis::cmd("XREADGROUP")
.arg("GROUP")
.arg(super::command::ADMIN_GROUP)
.arg(&rt.node_id)
.arg("COUNT")
.arg(8)
.arg("STREAMS")
.arg(&cstream)
.arg(">")
.query_async(&mut rt.client.conn())
.await;
match fresh {
Ok(v) => {
if let Err(error) = collect_cmd_entries(v, &cstream, &mut entries) {
tracing::warn!(p, err = %error, "control 新命令响应非法,保留待恢复");
}
}
Err(e) => {
if e.code() == Some("NOGROUP") {
ensured.remove(&p);
tracing::warn!(p, "control admin group 丢失(NOGROUP),下轮重建");
} else {
tracing::warn!(p, err = %e, "control 读新命令(>)失败,下轮重试");
}
}
}
const PAGE_BUDGET: u32 = 4;
let mut cursor = "0".to_string();
let mut pages = 0u32;
loop {
let claimed: std::result::Result<redis::Value, _> = redis::cmd("XAUTOCLAIM")
.arg(&cstream)
.arg(super::command::ADMIN_GROUP)
.arg(&rt.node_id)
.arg(15_000)
.arg(&cursor)
.arg("COUNT")
.arg(8)
.query_async(&mut rt.client.conn())
.await;
let value = match claimed {
Ok(value) => value,
Err(e) => {
if e.code() == Some("NOGROUP") {
ensured.remove(&p);
tracing::warn!(p, "control XAUTOCLAIM 遇 NOGROUP,下轮重建 admin group");
} else {
tracing::warn!(p, err = %e, "control XAUTOCLAIM 失败,下轮重试");
}
break;
}
};
let page = (|| -> Result<(String, Vec<(String, String)>)> {
let mut parts = wire::array(value)?;
if !matches!(parts.len(), 2 | 3) {
return Err(NasaRedisError::Parsing("invalid command claim page".into()));
}
let next = wire::id(parts.remove(0))?;
if parts.len() == 2 {
for deleted in wire::array(parts.pop().expect("deleted ids"))? {
wire::id(deleted)?;
}
}
let mut page = Vec::new();
collect_cmd_stream_entries(parts.pop().expect("entries"), &mut page)?;
Ok((next, page))
})();
let next = match page {
Ok((next, page)) => {
entries.extend(page);
next
}
Err(error) => {
tracing::warn!(p, err = %error, "control 接管响应非法,保留待恢复");
break;
}
};
pages += 1;
if next == "0-0" || pages >= PAGE_BUDGET {
break;
}
cursor = next;
}
for (entry_id, op_id) in entries {
let should_finalize = run_command(&rt, p, &op_id).await;
if !should_finalize {
continue;
}
let ack: std::result::Result<i64, _> = redis::cmd("XACK")
.arg(&cstream)
.arg(super::command::ADMIN_GROUP)
.arg(&entry_id)
.query_async(&mut rt.client.conn())
.await;
match ack {
Ok(0 | 1) => {}
Ok(_) => {
tracing::warn!(p, entry_id, "control XACK 数量非法,保留待恢复");
continue;
}
Err(e) => {
tracing::warn!(p, entry_id, err = %e, "control XACK 失败,保留 PEL 下轮重 finalize");
continue;
}
}
let del: std::result::Result<i64, _> = redis::cmd("XDEL")
.arg(&cstream)
.arg(&entry_id)
.query_async(&mut rt.client.conn())
.await;
match del {
Ok(0 | 1) => {}
Ok(_) => {
tracing::warn!(p, entry_id, "control XDEL 数量非法,已确认正文可能残留");
}
Err(e) => {
tracing::warn!(p, entry_id, err = %e, "control XDEL 失败(已 XACK,entry 残留 stream 待 trim)");
}
}
}
}
}
}
pub(super) fn collect_cmd_entries(
v: redis::Value,
stream: &str,
out: &mut Vec<(String, String)>,
) -> Result<()> {
use redis::Value as V;
if matches!(v, V::Nil) {
return Ok(());
}
let pairs = match v {
V::Map(pairs) => pairs,
value => {
let mut pairs = Vec::new();
for item in wire::array(value)? {
let mut pair = wire::array(item)?;
if pair.len() != 2 {
return Err(NasaRedisError::Parsing(
"invalid command stream pair".into(),
));
}
let entries = pair.pop().expect("entries");
pairs.push((pair.pop().expect("stream"), entries));
}
pairs
}
};
if pairs.len() > 1 {
return Err(NasaRedisError::Parsing(
"unexpected command stream count".into(),
));
}
for (key, entries) in pairs {
if wire::bytes(key)? != stream.as_bytes() {
return Err(NasaRedisError::Parsing(
"unexpected command stream coordinate".into(),
));
}
collect_cmd_stream_entries(entries, out)?;
}
Ok(())
}
fn collect_cmd_stream_entries(
entries_v: redis::Value,
out: &mut Vec<(String, String)>,
) -> Result<()> {
use redis::Value as V;
let mut parsed = Vec::new();
let mut seen = std::collections::HashSet::new();
for entry in wire::array(entries_v)? {
let mut pair = wire::array(entry)?;
if pair.len() != 2 {
return Err(NasaRedisError::Parsing("invalid command entry".into()));
}
let fields = pair.pop().expect("fields");
let id = wire::id(pair.pop().expect("id"))?;
if !seen.insert(id.clone()) {
return Err(NasaRedisError::Parsing("duplicate command entry".into()));
}
if matches!(fields, V::Nil) {
continue;
}
let fields = match fields {
V::Map(pairs) => pairs,
value => {
let fields = wire::array(value)?;
if fields.len() % 2 != 0 {
return Err(NasaRedisError::Parsing("odd command fields".into()));
}
let mut pairs = Vec::new();
let mut fields = fields.into_iter();
while let (Some(key), Some(value)) = (fields.next(), fields.next()) {
pairs.push((key, value));
}
pairs
}
};
let mut operation = None;
for (key, value) in fields {
let key = wire::bytes(key)?;
let value = wire::bytes(value)?;
if key == b"op" {
if operation.is_some() {
return Err(NasaRedisError::Parsing(
"duplicate command operation".into(),
));
}
operation =
Some(String::from_utf8(value).map_err(|_| {
NasaRedisError::Parsing("non UTF-8 command operation".into())
})?);
}
}
let operation =
operation.ok_or_else(|| NasaRedisError::Parsing("missing command operation".into()))?;
parsed.push((id, operation));
}
out.extend(parsed);
Ok(())
}
pub(super) async fn run_command(rt: &Arc<GroupRuntime>, p: u32, op_id: &str) -> bool {
use super::command::{execute_one, query, CmdAck, CmdAction};
let pre = match query(&rt.client, &rt.layout, p, op_id).await {
Ok(Some(pre)) => pre,
Ok(None) => {
tracing::warn!(p, op_id, "命令缺 result(producer 违规),清理");
return true;
}
Err(e) => {
tracing::warn!(p, op_id, err = %e, "命令 result 查询失败,保留 PEL 重试");
return false;
}
};
let action = pre.action;
let of = match rt.require_owner_for_op(p, op_id) {
Ok(of) => of,
Err(_) => {
tracing::warn!(
p,
op_id,
"control 执行命令时本节点非 owner(owner_ctx 缺),保留 PEL"
);
return false;
}
};
let fc = of.lease();
let fut = async {
match action {
CmdAction::Resume => super::disposition::resume(&rt.client, &rt.layout, p, &fc).await,
CmdAction::Force => {
super::disposition::force_republish(&rt.client, &rt.layout, p, &fc).await
}
CmdAction::Drop => super::disposition::drop_parked(&rt.client, &rt.layout, p, &fc)
.await
.map(|_| Vec::new()),
CmdAction::Dlq => {
super::disposition::dlq_from_parked(&rt.client, &rt.layout, p, &fc).await
}
}
};
match execute_one(&rt.client, &rt.layout, p, op_id, fut, action).await {
Ok(CmdAck::Finalize) => {
if let Ok(Some(r)) = query(&rt.client, &rt.layout, p, op_id).await {
if matches!(r.state, super::command::CmdState::Succeeded) {
let _ = rt
.event_tx
.send(Event::ParkResolved {
p,
})
.await;
}
}
true
}
Ok(CmdAck::Retry) => false,
Err(e) => {
tracing::warn!(p, op_id, err = %e, "命令执行编排异常(result 留 Executing),保留 PEL 重试");
false
}
}
}
pub(super) async fn sweep_loop(rt: Arc<GroupRuntime>) {
let mut tick = tokio::time::interval(Duration::from_millis(rt.cfg.min_idle_ms.max(1000)));
tick.tick().await;
loop {
tokio::select! {
_ = rt.bg_cancel.cancelled() => return,
_ = tick.tick() => {
}
}
rt.core.request_recovery(&rt.layout.prefix);
}
}
pub(super) async fn wake_loop(rt: Arc<GroupRuntime>) {
loop {
if rt.bg_cancel.is_cancelled() {
return;
}
let Ok(mut pubsub) = rt.client.raw_client().get_async_pubsub().await else {
tokio::time::sleep(Duration::from_secs(1)).await;
continue;
};
if pubsub.subscribe(rt.layout.wake()).await.is_err() {
tokio::time::sleep(Duration::from_secs(1)).await;
continue;
}
let mut msgs = pubsub.on_message();
loop {
tokio::select! {
_ = rt.bg_cancel.cancelled() => return,
m = msgs.next() => {
if m.is_none() {
break;
}
let _ = rebalance_once(&rt).await;
}
}
}
}
}