use std::sync::Arc;
use std::time::Duration;
use super::GroupRuntime;
pub(super) async fn auto_trim_loop(rt: Arc<GroupRuntime>) {
let rate = rt.stream_cfg.auto_trim_rate_ms;
if rate == 0 {
tracing::info!(group = %rt.layout.prefix, "autoTrim 已禁用(auto_trim_rate_ms=0)");
return;
}
let leader = crate::leader::Leader::elect(
Arc::clone(&rt.lock),
format!("{}:leader", rt.layout.prefix),
Duration::from_millis(rt.cfg.rebalance_ms.max(1)),
);
if let Some(ah) = leader.abort_handle() {
rt.bg_aborts.lock().expect("bg_aborts").push(ah);
}
let mut tick = tokio::time::interval(Duration::from_millis(rate));
tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
tick.tick().await;
loop {
tokio::select! {
_ = rt.bg_cancel.cancelled() => break,
_ = tick.tick() => {
let rt2 = Arc::clone(&rt);
leader
.run_if_leader_cancellable(|tok| async move { trim_round(&rt2, &tok).await })
.await;
}
}
}
leader.shutdown().await;
}
async fn trim_round(rt: &Arc<GroupRuntime>, term: &tokio_util::sync::CancellationToken) {
const TRIM_GUARDED_LUA: &str = r#"
local function uint(s)
if type(s) ~= 'string' or not string.match(s, '^%d+$') then return nil end
local normalized = string.gsub(s, '^0+', '')
if normalized == '' then normalized = '0' end
if #normalized > 20 or (#normalized == 20 and normalized > '18446744073709551615') then return nil end
return normalized
end
local function sid(s)
if type(s) ~= 'string' then return nil end
local ms, seq = string.match(s, '^(%d+)%-(%d+)$')
if not ms then return nil end
ms, seq = uint(ms), uint(seq)
if not ms or not seq then return nil end
return {ms, seq}
end
local function less(a, b)
for i = 1, 2 do
if #a[i] ~= #b[i] then return #a[i] < #b[i] end
if a[i] ~= b[i] then return a[i] < b[i] end
end
return false
end
if #KEYS ~= 1 or #ARGV ~= 2 or ARGV[1] == '' then
return redis.error_reply('invalid trim arguments')
end
local boundary = ARGV[2]
local bound = sid(boundary)
if not bound then return redis.error_reply('invalid trim boundary') end
-- 查询后的旧快照不能授权删除;同一原子步骤内重新读取 PEL,先完成全部验证再执行 XTRIM。
local pending = redis.call('XPENDING', KEYS[1], ARGV[1])
if type(pending) ~= 'table' or #pending ~= 4 or type(pending[1]) ~= 'number'
or pending[1] < 0 or pending[1] ~= math.floor(pending[1]) then
return redis.error_reply('invalid pending summary')
end
if pending[1] == 0 then
if pending[2] ~= false or pending[3] ~= false
or (pending[4] ~= false and (type(pending[4]) ~= 'table' or #pending[4] ~= 0)) then
return redis.error_reply('inconsistent empty pending summary')
end
else
local oldest, newest = sid(pending[2]), sid(pending[3])
if not oldest or not newest or less(newest, oldest) or type(pending[4]) ~= 'table' then
return redis.error_reply('invalid pending boundaries')
end
for _, consumer in ipairs(pending[4]) do
if type(consumer) ~= 'table' or #consumer ~= 2 or type(consumer[1]) ~= 'string' then
return redis.error_reply('invalid pending consumer')
end
local count = consumer[2]
if type(count) == 'string' then
count = uint(count)
if not count or count == '0' then return redis.error_reply('invalid pending consumer count') end
elseif type(count) ~= 'number' or count <= 0 or count ~= math.floor(count) then
return redis.error_reply('invalid pending consumer count')
end
end
-- 新增的更早 PEL 降低删除边界;旧边界更低时仍保留它,不因确认并发而扩大本轮删除范围。
if less(oldest, bound) then boundary = pending[2] end
end
return redis.call('XTRIM', KEYS[1], 'MINID', '~', boundary)
"#;
let now = match super::command::redis_now(&rt.client).await {
Ok(n) => n,
Err(e) => {
tracing::warn!(err = %e, "autoTrim 取 Redis TIME 失败,跳过本轮");
return;
}
};
let time_min_id = format!("{}-0", now.saturating_sub(rt.stream_cfg.data_expire_ms));
let group = rt.layout.group().to_string();
let mut trimmed = 0i64;
for p in 0..rt.count {
if rt.bg_cancel.is_cancelled() || term.is_cancelled() {
break;
}
let stream = rt.layout.stream(p);
let min_id = match oldest_pending_id(&rt.client, &stream, &group).await {
Ok(Some(pel_oldest)) => lesser_stream_id(&time_min_id, &pel_oldest),
Ok(None) => time_min_id.clone(), Err(e) => {
tracing::warn!(p, stream = %stream, err = %e, "autoTrim 取 PEL 最老 id 失败,保守跳过本分区");
continue; }
};
if rt.bg_cancel.is_cancelled() || term.is_cancelled() {
break;
}
let r: std::result::Result<i64, redis::RedisError> = redis::Script::new(TRIM_GUARDED_LUA)
.key(&stream)
.arg(&group)
.arg(&min_id)
.invoke_async(&mut rt.client.conn())
.await;
match r {
Ok(n) if n >= 0 => trimmed = trimmed.saturating_add(n),
Ok(_) => tracing::warn!(p, stream = %stream, "autoTrim 裁剪计数为负,无法确认删除数量"),
Err(e) => tracing::warn!(p, stream = %stream, err = %e, "autoTrim XTRIM 失败"),
}
}
if trimmed > 0 {
tracing::debug!(group = %rt.layout.prefix, time_min_id = %time_min_id, trimmed, "autoTrim 裁剪 stream entries(裁剪边界已按各分区 PEL 最老 id 兜底)");
}
}
async fn oldest_pending_id(
client: &crate::client::RedisClient,
stream: &str,
group: &str,
) -> std::result::Result<Option<String>, redis::RedisError> {
fn parse_err(msg: &str) -> redis::RedisError {
redis::RedisError::from((
redis::ErrorKind::Parse,
"XPENDING summary 形态异常",
msg.to_string(),
))
}
let v: redis::Value = redis::cmd("XPENDING")
.arg(stream)
.arg(group)
.query_async(&mut client.conn())
.await?;
let redis::Value::Array(cols) = v else {
return Err(parse_err("顶层非 Array"));
};
if cols.len() != 4 {
return Err(parse_err("摘要字段数不是四"));
}
let count = match cols.first() {
Some(redis::Value::Int(n)) if *n >= 0 => *n,
_ => return Err(parse_err("count 不是非负整数")),
};
if count == 0 {
if !matches!(cols[1], redis::Value::Nil)
|| !matches!(cols[2], redis::Value::Nil)
|| !(matches!(cols[3], redis::Value::Nil)
|| matches!(&cols[3], redis::Value::Array(rows) if rows.is_empty()))
{
return Err(parse_err("零计数与边界或 consumer 列表不一致"));
}
return Ok(None);
}
let min = super::wire::id(cols[1].clone()).map_err(|_| parse_err("min-id 非法"))?;
let max = super::wire::id(cols[2].clone()).map_err(|_| parse_err("max-id 非法"))?;
if lesser_stream_id(&min, &max) != min {
return Err(parse_err("min-id 大于 max-id"));
}
let redis::Value::Array(consumers) = &cols[3] else {
return Err(parse_err("consumer 列表非数组"));
};
for consumer in consumers {
let redis::Value::Array(fields) = consumer else {
return Err(parse_err("consumer 项非数组"));
};
if fields.len() != 2
|| !matches!(
fields[0],
redis::Value::BulkString(_) | redis::Value::SimpleString(_)
)
{
return Err(parse_err("consumer 项结构非法"));
}
let valid_count = match &fields[1] {
redis::Value::Int(value) => *value > 0,
redis::Value::BulkString(value) => {
!value.is_empty()
&& value.iter().all(u8::is_ascii_digit)
&& std::str::from_utf8(value)
.ok()
.and_then(|value| value.parse::<u64>().ok())
.is_some_and(|value| value > 0)
}
redis::Value::SimpleString(value) => {
!value.is_empty()
&& value.bytes().all(|byte| byte.is_ascii_digit())
&& value.parse::<u64>().is_ok_and(|value| value > 0)
}
_ => false,
};
if !valid_count {
return Err(parse_err("consumer 计数非法"));
}
}
Ok(Some(min))
}
fn lesser_stream_id(a: &str, b: &str) -> String {
fn parse(id: &str) -> Option<(u64, u64)> {
let mut it = id.splitn(2, '-');
let ms = it.next()?.parse::<u64>().ok()?;
let seq = it.next().unwrap_or("0").parse::<u64>().ok()?;
Some((ms, seq))
}
let complete = |id: &str| {
id.split_once('-').is_some_and(|(ms, seq)| {
!ms.is_empty()
&& !seq.is_empty()
&& ms.bytes().all(|byte| byte.is_ascii_digit())
&& seq.bytes().all(|byte| byte.is_ascii_digit())
})
};
if !complete(a) || !complete(b) {
return "0-0".into();
}
match (parse(a), parse(b)) {
(Some(pa), Some(pb)) => {
if pa <= pb {
a.to_string()
} else {
b.to_string()
}
}
_ => "0-0".into(),
}
}