use std::collections::HashMap;
use std::sync::Arc;
use std::time::Duration;
use serde::{de::DeserializeOwned, Serialize};
use tokio::task::JoinHandle;
use tokio_util::sync::CancellationToken;
use crate::client::RedisClient;
use crate::config::{MAX_REDIS_NAME_BYTES, MAX_REDIS_RUNTIME_DURATION_MS, MAX_STREAM_BATCH_SIZE};
use crate::error::{NasaRedisError, Result};
use crate::partition::{Envelope, ErasedHandler, HandlerMap, DATA_FIELD};
const MAX_PROXY_CONSUMERS: usize = 256;
#[derive(Debug, Clone, serde::Deserialize)]
#[serde(default, deny_unknown_fields)]
pub struct ProxyCfg {
pub consumers: usize,
pub batch_size: usize,
pub poll_idle_ms: u64,
pub max_redeliver: u32,
pub reclaim_min_idle_ms: u64,
pub poison: ProxyPoison,
pub handler_timeout_ms: u64,
pub max_stream_len: Option<u64>,
pub requeue_unregistered: bool,
pub drain_deadline_ms: u64,
}
impl Default for ProxyCfg {
fn default() -> Self {
Self {
consumers: 1,
batch_size: 100,
poll_idle_ms: 200,
max_redeliver: 5,
reclaim_min_idle_ms: 30_000,
poison: ProxyPoison::Drop,
handler_timeout_ms: 10_000,
max_stream_len: Some(1_000_000),
requeue_unregistered: false,
drain_deadline_ms: 10_000,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ProxyPoison {
Drop,
Dlq,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Deserialize, Default)]
#[serde(rename_all = "snake_case")]
pub enum ProxyStartOffset {
#[default]
New,
History,
}
impl ProxyStartOffset {
fn redis_id(self) -> &'static str {
match self {
ProxyStartOffset::New => "$",
ProxyStartOffset::History => "0-0",
}
}
}
pub struct PreparedProxy {
client: Arc<RedisClient>,
stream: String,
group: String,
handlers: HandlerMap,
cfg: ProxyCfg,
}
impl PreparedProxy {
pub async fn prepare(
client: Arc<RedisClient>,
stream: impl Into<String>,
group: impl Into<String>,
cfg: ProxyCfg,
) -> Result<Self> {
Self::prepare_with_offset(client, stream, group, cfg, ProxyStartOffset::New).await
}
pub async fn prepare_with_offset(
client: Arc<RedisClient>,
stream: impl Into<String>,
group: impl Into<String>,
cfg: ProxyCfg,
start_offset: ProxyStartOffset,
) -> Result<Self> {
if cfg.consumers == 0 || cfg.batch_size == 0 {
return Err(NasaRedisError::Config(
"ProxyCfg.consumers/batch_size 必须 > 0".into(),
));
}
if cfg.consumers > MAX_PROXY_CONSUMERS {
return Err(NasaRedisError::Config(format!(
"ProxyCfg.consumers 过大(上限 {MAX_PROXY_CONSUMERS})"
)));
}
if cfg.batch_size > MAX_STREAM_BATCH_SIZE {
return Err(NasaRedisError::Config(format!(
"ProxyCfg.batch_size 过大(上限 {MAX_STREAM_BATCH_SIZE})"
)));
}
if cfg.reclaim_min_idle_ms == 0 {
return Err(NasaRedisError::Config(
"ProxyCfg.reclaim_min_idle_ms 必须 > 0".into(),
));
}
if cfg.handler_timeout_ms == 0 {
return Err(NasaRedisError::Config(
"ProxyCfg.handler_timeout_ms 必须 > 0".into(),
));
}
if cfg.drain_deadline_ms == 0 {
return Err(NasaRedisError::Config(
"ProxyCfg.drain_deadline_ms 必须 > 0".into(),
));
}
if cfg.max_stream_len == Some(0) {
return Err(NasaRedisError::Config(
"ProxyCfg.max_stream_len 配置后必须 > 0".into(),
));
}
if cfg.poll_idle_ms == 0
|| cfg.poll_idle_ms > MAX_REDIS_RUNTIME_DURATION_MS
|| cfg.reclaim_min_idle_ms > MAX_REDIS_RUNTIME_DURATION_MS
|| cfg.handler_timeout_ms > MAX_REDIS_RUNTIME_DURATION_MS
|| cfg.drain_deadline_ms > MAX_REDIS_RUNTIME_DURATION_MS
{
return Err(NasaRedisError::Config(format!(
"ProxyCfg 的 poll/reclaim/handler/drain 时长必须位于 1..={MAX_REDIS_RUNTIME_DURATION_MS}ms"
)));
}
if cfg.max_redeliver == 0 {
return Err(NasaRedisError::Config(
"ProxyCfg.max_redeliver 必须 >= 1(否则首投即判毒)".into(),
));
}
if cfg.handler_timeout_ms > 0 && cfg.reclaim_min_idle_ms <= cfg.handler_timeout_ms {
return Err(NasaRedisError::Config(format!(
"ProxyCfg.reclaim_min_idle_ms({}) 必须 > handler_timeout_ms({})——否则 reclaim 会抢占在途 handler 致双跑",
cfg.reclaim_min_idle_ms, cfg.handler_timeout_ms
)));
}
let stream = stream.into();
let group = group.into();
if stream.trim().is_empty()
|| group.trim().is_empty()
|| stream != stream.trim()
|| group != group.trim()
|| stream.len() > MAX_REDIS_NAME_BYTES
|| group.len() > MAX_REDIS_NAME_BYTES
{
return Err(NasaRedisError::Config(format!(
"PROXY stream / group 必须无首尾空白、非空且不超过 {MAX_REDIS_NAME_BYTES} 字节"
)));
}
let r: std::result::Result<String, redis::RedisError> = redis::cmd("XGROUP")
.arg("CREATE")
.arg(&stream)
.arg(&group)
.arg(start_offset.redis_id())
.arg("MKSTREAM")
.query_async(&mut client.conn())
.await;
match r {
Ok(_) => {}
Err(e) if e.to_string().contains("BUSYGROUP") => {}
Err(e) => return Err(e.into()),
}
Ok(Self {
client,
stream,
group,
handlers: HashMap::new(),
cfg,
})
}
pub fn register<T, F, Fut>(&mut self, topic: &str, event: &str, f: F) -> &mut Self
where
T: DeserializeOwned + Send + 'static,
F: Fn(Vec<T>) -> Fut + Send + Sync + 'static,
Fut: std::future::Future<Output = std::result::Result<(), String>> + Send + 'static,
{
let f = Arc::new(f);
let h: ErasedHandler = Arc::new(move |items: Vec<(String, serde_json::Value)>| {
let f = Arc::clone(&f);
Box::pin(async move {
let mut typed: Vec<T> = Vec::with_capacity(items.len());
let mut good_ids: Vec<String> = Vec::with_capacity(items.len());
let mut failed: Vec<String> = Vec::new();
for (id, v) in items {
match serde_json::from_value::<T>(v) {
Ok(t) => {
typed.push(t);
good_ids.push(id);
}
Err(e) => {
tracing::error!(id, err = %e, "PROXY 业务类型解码失败,该 ID 进重投/毒");
failed.push(id);
}
}
}
if typed.is_empty() {
return failed;
}
let fut = std::panic::AssertUnwindSafe(f(typed));
match futures::FutureExt::catch_unwind(fut).await {
Ok(Ok(())) => failed,
Ok(Err(e)) => {
tracing::warn!(err = %e, "PROXY handler 失败,本桶 ID 转重投");
failed.extend(good_ids);
failed
}
Err(_) => {
tracing::error!("PROXY handler panic,本桶 ID 转重投");
failed.extend(good_ids);
failed
}
}
})
});
self.handlers
.insert((topic.to_string(), event.to_string()), h);
self
}
pub fn handler_count(&self) -> usize {
self.handlers.len()
}
pub async fn start(self) -> Result<RunningProxy> {
let running = self.start_suspended().await?;
running.activate();
Ok(running)
}
pub async fn start_suspended(self) -> Result<RunningProxy> {
self.start_suspended_with_activation(CancellationToken::new())
.await
}
pub async fn start_suspended_with_activation(
self,
ready: CancellationToken,
) -> Result<RunningProxy> {
let cancel = CancellationToken::new();
let activity = Arc::new(crate::activity::Activity::default());
let shared = Arc::new(ProxyShared {
activity: activity.clone(),
client: self.client.clone(),
stream: self.stream.clone(),
group: self.group.clone(),
handlers: self.handlers,
cfg: self.cfg.clone(),
});
let mut handles = Vec::new();
let mut names = Vec::with_capacity(self.cfg.consumers + 1);
for i in 0..=self.cfg.consumers {
let consumer = format!("c-{}-{i}", uuid::Uuid::new_v4().simple());
names.push(consumer.clone());
let shared = shared.clone();
let cancel = cancel.clone();
let ready = ready.clone();
let reclaim = i == self.cfg.consumers;
let active = activity.task(if reclaim {
crate::activity::TaskKind::Reclaimer
} else {
crate::activity::TaskKind::Consumer
});
handles.push(tokio::spawn(async move {
let _active = active;
tokio::select! {
biased;
_ = cancel.cancelled() => return,
_ = ready.cancelled() => {},
}
if reclaim {
reclaim_loop(shared, consumer, cancel).await;
} else {
consumer_loop(shared, consumer, cancel).await;
}
}));
}
let (deadline, _) = tokio::sync::watch::channel(None);
let (report, outcome) = tokio::sync::watch::channel(ProxyStopReport::default());
let owner = ProxyCloseOwner {
client: self.client.clone(),
stream: self.stream.clone(),
group: self.group.clone(),
names,
aborts: handles.iter().map(JoinHandle::abort_handle).collect(),
handles,
cancel: cancel.clone(),
deadline: deadline.clone(),
budget: Duration::from_millis(self.cfg.drain_deadline_ms),
report: activity.completion(report),
};
tokio::spawn(owner.run());
Ok(RunningProxy {
activity,
client: self.client,
stream: self.stream,
cancel,
ready,
deadline,
outcome,
max_stream_len: self.cfg.max_stream_len,
drain_deadline_ms: self.cfg.drain_deadline_ms,
consumers: self.cfg.consumers,
})
}
}
struct ProxyShared {
activity: Arc<crate::activity::Activity>,
client: Arc<RedisClient>,
stream: String,
group: String,
handlers: HandlerMap,
cfg: ProxyCfg,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum ProxyCleanup {
#[default]
Skipped,
Complete,
Pending,
Unavailable,
Deadline,
}
#[derive(Debug, Clone, Copy, Default)]
pub struct ProxyStopReport {
pub terminated: bool,
pub forced: bool,
pub failed: bool,
pub cleanup: ProxyCleanup,
}
pub struct RunningProxy {
consumers: usize,
activity: Arc<crate::activity::Activity>,
client: Arc<RedisClient>,
stream: String,
cancel: CancellationToken,
ready: CancellationToken,
deadline: tokio::sync::watch::Sender<Option<tokio::time::Instant>>,
outcome: tokio::sync::watch::Receiver<ProxyStopReport>,
max_stream_len: Option<u64>,
drain_deadline_ms: u64,
}
impl RunningProxy {
pub fn with_running<T>(&self, publish: impl FnOnce() -> T) -> Option<T> {
self.activity
.with_tasks(self.consumers, 1, 0, || {
let report = self.stop_report();
if self.cancel.is_cancelled() || report.terminated || report.failed {
None
} else {
Some(publish())
}
})
.flatten()
}
pub fn observation(&self) -> crate::RedisTaskObservation {
self.activity.snapshot()
}
pub async fn publish<T: Serialize>(
&self,
topic: &str,
event: &str,
data: &T,
) -> Result<String> {
if topic.is_empty() || event.is_empty() {
return Err(NasaRedisError::Config(format!(
"InvalidPublish: topic/event 为空(topic={topic:?}, event={event:?})"
)));
}
if self.cancel.is_cancelled() {
return Err(NasaRedisError::NotExecuted("Proxy 已关闭".into()));
}
let env = Envelope {
topic: topic.to_string(),
event: event.to_string(),
data: serde_json::to_value(data).map_err(|e| NasaRedisError::Codec(e.to_string()))?,
passthrough: None,
};
let body = serde_json::to_vec(&env).map_err(|e| NasaRedisError::Codec(e.to_string()))?;
let mut cmd = redis::cmd("XADD");
cmd.arg(&self.stream);
if let Some(n) = self.max_stream_len {
cmd.arg("MAXLEN").arg("~").arg(n);
}
cmd.arg("*").arg(DATA_FIELD).arg(body);
let id: String = cmd.query_async(&mut self.client.conn()).await?;
Ok(id)
}
pub fn activate(&self) -> bool {
if self.cancel.is_cancelled() {
return false;
}
self.ready.cancel();
true
}
pub fn begin_shutdown_until(&self, deadline: tokio::time::Instant) {
self.activity.close();
let local = tokio::time::Instant::now() + Duration::from_millis(self.drain_deadline_ms);
let deadline = deadline.min(local);
self.deadline.send_modify(|current| {
*current = Some(current.map_or(deadline, |old| old.min(deadline)))
});
self.cancel.cancel();
}
pub fn begin_shutdown(&self) {
self.begin_shutdown_until(
tokio::time::Instant::now() + Duration::from_millis(self.drain_deadline_ms),
);
}
pub fn stop_report(&self) -> ProxyStopReport {
let disconnected = self.outcome.has_changed().is_err();
let mut report = *self.outcome.borrow();
if disconnected && !report.terminated {
report.failed = true;
}
report
}
pub async fn wait_closed(&self) -> ProxyStopReport {
let mut outcome = self.outcome.clone();
loop {
let report = *outcome.borrow_and_update();
if report.terminated {
return report;
}
if outcome.changed().await.is_err() {
return ProxyStopReport {
failed: true,
..report
};
}
}
}
pub async fn shutdown(self) {
self.begin_shutdown();
let report = self.wait_closed().await;
if report.forced
|| report.failed
|| matches!(
report.cleanup,
ProxyCleanup::Unavailable | ProxyCleanup::Deadline
)
{
tracing::warn!(
forced = report.forced,
failed = report.failed,
cleanup = ?report.cleanup,
"Proxy 非正常排干"
);
}
}
}
impl Drop for RunningProxy {
fn drop(&mut self) {
self.begin_shutdown();
}
}
struct ProxyCloseOwner {
client: Arc<RedisClient>,
stream: String,
group: String,
names: Vec<String>,
handles: Vec<JoinHandle<()>>,
aborts: Vec<tokio::task::AbortHandle>,
cancel: CancellationToken,
deadline: tokio::sync::watch::Sender<Option<tokio::time::Instant>>,
budget: Duration,
report: crate::activity::CompletionOwner<ProxyStopReport>,
}
impl ProxyCloseOwner {
async fn run(mut self) {
use futures::StreamExt;
let mut tasks: futures::stream::FuturesUnordered<_> = self.handles.drain(..).collect();
let mut report = ProxyStopReport::default();
tokio::select! {
biased;
_ = self.cancel.cancelled() => {},
_ = tasks.next() => {
report.failed = true;
self.cancel.cancel();
},
}
self.report.send_replace(report);
let deadline =
(*self.deadline.borrow()).unwrap_or_else(|| tokio::time::Instant::now() + self.budget);
while !tasks.is_empty() {
match self.within_deadline(deadline, tasks.next()).await {
Ok(Some(Ok(()))) => {}
Ok(Some(Err(_))) => report.failed = true,
Ok(None) => break,
Err(_) => {
report.forced = true;
for abort in &self.aborts {
abort.abort();
}
while let Some(result) = tasks.next().await {
if result.is_err_and(|error| !error.is_cancelled()) {
report.failed = true;
}
}
break;
}
}
}
if !report.forced && !report.failed {
report.cleanup = self.cleanup(deadline).await;
}
report.terminated = true;
self.report.send_replace(report);
}
async fn within_deadline<T>(
&self,
deadline: tokio::time::Instant,
future: impl std::future::Future<Output = T>,
) -> std::result::Result<T, ()> {
let mut changes = self.deadline.subscribe();
tokio::pin!(future);
loop {
let current = changes
.borrow_and_update()
.unwrap_or(deadline)
.min(deadline);
tokio::select! {biased;
_=tokio::time::sleep_until(current)=>return Err(()),
result=&mut future=>return Ok(result),
_=changes.changed()=>{},
}
}
}
async fn cleanup(&self, deadline: tokio::time::Instant) -> ProxyCleanup {
if tokio::time::Instant::now() >= self.deadline.borrow().unwrap_or(deadline).min(deadline) {
return ProxyCleanup::Deadline;
}
let pending = match self
.within_deadline(
deadline,
consumer_pending_counts(&self.client, &self.stream, &self.group),
)
.await
{
Ok(Ok(pending)) => pending,
Ok(Err(_)) => return ProxyCleanup::Unavailable,
Err(_) => return ProxyCleanup::Deadline,
};
let mut outcome = ProxyCleanup::Complete;
for name in &self.names {
if pending.get(name).copied().unwrap_or(0) > 0 {
outcome = ProxyCleanup::Pending;
continue;
}
if tokio::time::Instant::now()
>= self.deadline.borrow().unwrap_or(deadline).min(deadline)
{
return ProxyCleanup::Deadline;
}
let result: std::result::Result<i64, _> = match self
.within_deadline(
deadline,
redis::cmd("XGROUP")
.arg("DELCONSUMER")
.arg(&self.stream)
.arg(&self.group)
.arg(name)
.query_async(&mut self.client.conn()),
)
.await
{
Ok(result) => result,
Err(_) => return ProxyCleanup::Deadline,
};
if result.is_err() {
return ProxyCleanup::Unavailable;
}
}
outcome
}
}
impl Drop for ProxyCloseOwner {
fn drop(&mut self) {
self.cancel.cancel();
for abort in &self.aborts {
abort.abort();
}
}
}
async fn consumer_pending_counts(
client: &RedisClient,
stream: &str,
group: &str,
) -> std::result::Result<HashMap<String, i64>, redis::RedisError> {
let value: redis::Value = redis::cmd("XPENDING")
.arg(stream)
.arg(group)
.query_async(&mut client.conn())
.await?;
let invalid =
|| redis::RedisError::from((redis::ErrorKind::Client, "invalid XPENDING summary"));
let redis::Value::Array(cols) = value else {
return Err(invalid());
};
if cols.len() != 4 {
return Err(invalid());
}
let redis::Value::Int(total) = cols[0] else {
return Err(invalid());
};
if total < 0 {
return Err(invalid());
}
if total == 0 {
if cols[1] != redis::Value::Nil
|| cols[2] != redis::Value::Nil
|| cols[3] != redis::Value::Nil
{
return Err(invalid());
}
return Ok(HashMap::new());
}
if !matches!(
(&cols[1], &cols[2]),
(redis::Value::BulkString(_), redis::Value::BulkString(_))
) {
return Err(invalid());
}
let redis::Value::Array(rows) = &cols[3] else {
return Err(invalid());
};
let mut out = HashMap::new();
let mut sum = 0_i64;
for row in rows {
let redis::Value::Array(kv) = row else {
return Err(invalid());
};
if kv.len() != 2 {
return Err(invalid());
}
let name: String = redis::from_redis_value(kv[0].clone()).map_err(|_| invalid())?;
let count: i64 = redis::from_redis_value(kv[1].clone()).map_err(|_| invalid())?;
if name.is_empty() || count <= 0 || out.insert(name, count).is_some() {
return Err(invalid());
}
sum = sum.checked_add(count).ok_or_else(invalid)?;
}
if sum != total {
return Err(invalid());
}
Ok(out)
}
struct BadEntry {
id: String,
reason: String,
raw: Vec<(Vec<u8>, Vec<u8>)>,
}
struct ParsedEntries {
ok: Vec<(String, Envelope)>,
bad: Vec<BadEntry>,
}
fn parse_entries(entries: &redis::Value) -> ParsedEntries {
let mut ok = Vec::new();
let mut bad = Vec::new();
let (redis::Value::Array(items) | redis::Value::Set(items)) = entries else {
if !matches!(entries, redis::Value::Nil) {
tracing::warn!("PROXY entries 形态非 Array/Set(疑似 RESP3 未适配),本批跳过");
}
return ParsedEntries { ok, bad };
};
for item in items {
let redis::Value::Array(pair) = item else {
continue;
};
let Some(redis::Value::BulkString(id_bytes)) = pair.first() else {
continue;
};
let id = String::from_utf8_lossy(id_bytes).into_owned();
let fields = match pair.get(1) {
Some(redis::Value::Array(f)) => f,
None | Some(redis::Value::Nil) => continue,
Some(_) => {
bad.push(BadEntry {
id,
reason: "entry 字段段形态异常(非 Array/Nil)".to_string(),
raw: Vec::new(),
});
continue;
}
};
let mut data_bytes: Option<&[u8]> = None;
let mut raw: Vec<(Vec<u8>, Vec<u8>)> = Vec::new();
let mut it = fields.iter();
while let (Some(f), Some(v)) = (it.next(), it.next()) {
if let (redis::Value::BulkString(fk), redis::Value::BulkString(vv)) = (f, v) {
if fk == DATA_FIELD.as_bytes() {
data_bytes = Some(vv);
}
raw.push((fk.clone(), vv.clone()));
}
}
let Some(bytes) = data_bytes else {
bad.push(BadEntry {
id,
reason: "缺 data 字段(非标准 Envelope wire)".to_string(),
raw,
});
continue;
};
match serde_json::from_slice::<Envelope>(bytes) {
Ok(env) => ok.push((id, env)),
Err(e) => {
tracing::error!(id, err = %e, "PROXY entry envelope 解析失败,转 DLQ/Drop(不再静默卡 PEL)");
bad.push(BadEntry {
id,
reason: format!("envelope 解析失败: {e}"),
raw,
});
}
}
}
ParsedEntries { ok, bad }
}
async fn dispatch(shared: &ProxyShared, batch: Vec<(String, Envelope)>) -> Vec<String> {
let mut buckets: HashMap<(String, String), Vec<(String, serde_json::Value)>> = HashMap::new();
for (id, env) in batch {
buckets
.entry((env.topic, env.event))
.or_default()
.push((id, env.data));
}
let mut failed = Vec::new();
for (key, items) in buckets {
match shared.handlers.get(&key) {
Some(h) => {
let bucket_ids: Vec<String> = items.iter().map(|(id, _)| id.clone()).collect();
let dur = Duration::from_millis(shared.cfg.handler_timeout_ms);
match tokio::time::timeout(dur, h(items)).await {
Ok(f) => failed.extend(f),
Err(_) => {
tracing::warn!(topic = %key.0, event = %key.1, timeout_ms = shared.cfg.handler_timeout_ms, "PROXY handler 超时,整桶转重投(留 PEL)");
failed.extend(bucket_ids);
}
}
}
None => {
tracing::warn!(topic = %key.0, event = %key.1, requeue = shared.cfg.requeue_unregistered, "PROXY 本节点无注册 handler");
if shared.cfg.requeue_unregistered {
failed.extend(items.into_iter().map(|(id, _)| id));
}
}
}
}
failed
}
async fn ack(shared: &ProxyShared, ids: &[String]) {
if ids.is_empty() {
return;
}
let mut cmd = redis::cmd("XACK");
cmd.arg(&shared.stream).arg(&shared.group);
for id in ids {
cmd.arg(id);
}
let r: std::result::Result<i64, redis::RedisError> =
cmd.query_async(&mut shared.client.conn()).await;
if let Err(e) = r {
tracing::warn!(err = %e, "PROXY XACK 失败(下轮 XAUTOCLAIM 会重投)");
}
}
async fn consumer_loop(shared: Arc<ProxyShared>, consumer: String, cancel: CancellationToken) {
loop {
tokio::select! {
_ = cancel.cancelled() => break,
r = read_new(&shared, &consumer) => {
match r {
Ok(parsed) if !parsed.ok.is_empty() || !parsed.bad.is_empty() => {
let count = parsed.ok.len() + parsed.bad.len();
shared.activity.accept(count, 0);
if !parsed.bad.is_empty() {
dispose_bad(&shared, &parsed.bad).await;
}
if !parsed.ok.is_empty() {
let ids: Vec<String> = parsed.ok.iter().map(|(id, _)| id.clone()).collect();
let failed = dispatch(&shared, parsed.ok).await;
let failed: std::collections::HashSet<String> = failed.into_iter().collect();
let ok: Vec<String> = ids.into_iter().filter(|i| !failed.contains(i)).collect();
ack(&shared, &ok).await;
}
shared.activity.finish(count, false);
}
Ok(_) => { shared.activity.finish(0, false); tokio::time::sleep(Duration::from_millis(shared.cfg.poll_idle_ms.max(1))).await; },
Err(e) => {
tracing::warn!(err = %e, "PROXY XREADGROUP 失败,短暂退避");
tokio::time::sleep(Duration::from_millis(200)).await;
}
}
}
}
}
}
async fn read_new(shared: &ProxyShared, consumer: &str) -> Result<ParsedEntries> {
let v: redis::Value = redis::cmd("XREADGROUP")
.arg("GROUP")
.arg(&shared.group)
.arg(consumer)
.arg("COUNT")
.arg(shared.cfg.batch_size)
.arg("STREAMS")
.arg(&shared.stream)
.arg(">")
.query_async(&mut shared.client.conn())
.await?;
let empty = || ParsedEntries {
ok: Vec::new(),
bad: Vec::new(),
};
match &v {
redis::Value::Nil => Ok(empty()),
redis::Value::Array(streams) | redis::Value::Set(streams) => {
if let Some(redis::Value::Array(pair) | redis::Value::Set(pair)) = streams.first() {
if let Some(entries) = pair.get(1) {
return Ok(parse_entries(entries));
}
}
Ok(empty())
}
redis::Value::Map(pairs) => Ok(pairs
.first()
.map(|(_, entries)| parse_entries(entries))
.unwrap_or_else(empty)),
_ => {
tracing::warn!("PROXY XREADGROUP 顶层形态异常(疑似 RESP3 未适配),本轮空轮");
Ok(empty())
}
}
}
async fn reclaim_loop(shared: Arc<ProxyShared>, consumer: String, cancel: CancellationToken) {
let period = Duration::from_millis((shared.cfg.reclaim_min_idle_ms / 2).clamp(200, 10_000));
let mut tick = tokio::time::interval(period);
tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
loop {
tokio::select! {
_ = cancel.cancelled() => break,
_ = tick.tick() => {
if let Err(e) = reclaim_once(&shared, &consumer).await {
tracing::warn!(err = %e, "PROXY reclaim 失败,下轮重试");
}
}
}
}
}
async fn reclaim_once(shared: &ProxyShared, consumer: &str) -> Result<()> {
let v: redis::Value = redis::cmd("XAUTOCLAIM")
.arg(&shared.stream)
.arg(&shared.group)
.arg(consumer)
.arg(shared.cfg.reclaim_min_idle_ms)
.arg("0")
.arg("COUNT")
.arg(shared.cfg.batch_size)
.query_async(&mut shared.client.conn())
.await?;
let (redis::Value::Array(parts) | redis::Value::Set(parts)) = &v else {
if !matches!(v, redis::Value::Nil) {
tracing::warn!("PROXY XAUTOCLAIM 形态异常(疑似 RESP3 未适配),本轮跳过");
}
return Ok(());
};
let Some(entries) = parts.get(1) else {
return Ok(());
};
let parsed = parse_entries(entries);
let count = parsed.ok.len() + parsed.bad.len();
shared.activity.accept(count, 0);
if !parsed.bad.is_empty() {
dispose_bad(shared, &parsed.bad).await;
}
let batch = parsed.ok;
if batch.is_empty() {
shared.activity.finish(count, false);
return Ok(());
}
let ids: Vec<String> = batch.iter().map(|(id, _)| id.clone()).collect();
let counts = pending_counts(shared, &ids).await?;
let mut to_dispatch = Vec::new();
let mut poison_ids = Vec::new();
for (id, env) in batch {
let n = match counts.get(&id).copied() {
Some(n) => n,
None => {
tracing::warn!(
id,
"PROXY XPENDING 未返回该 id 的重投计数(协议异常/竞态),保守按 1 重投"
);
1
}
};
if n > shared.cfg.max_redeliver {
poison_ids.push(id);
} else {
to_dispatch.push((id, env));
}
}
if !poison_ids.is_empty() {
handle_poison(shared, &poison_ids).await;
}
if !to_dispatch.is_empty() {
let d_ids: Vec<String> = to_dispatch.iter().map(|(id, _)| id.clone()).collect();
let failed = dispatch(shared, to_dispatch).await;
let failed: std::collections::HashSet<String> = failed.into_iter().collect();
let ok: Vec<String> = d_ids.into_iter().filter(|i| !failed.contains(i)).collect();
ack(shared, &ok).await;
}
shared.activity.finish(count, false);
Ok(())
}
async fn pending_counts(shared: &ProxyShared, ids: &[String]) -> Result<HashMap<String, u32>> {
let mut out = HashMap::new();
if ids.is_empty() {
return Ok(out);
}
let mut pipe = redis::pipe();
for id in ids {
pipe.cmd("XPENDING")
.arg(&shared.stream)
.arg(&shared.group)
.arg(id)
.arg(id)
.arg(1);
}
let mut conn = shared.client.conn();
let results: Vec<redis::Value> =
redis::aio::ConnectionLike::req_packed_commands(&mut conn, &pipe, 0, ids.len()).await?;
for (id, res) in ids.iter().zip(results) {
let (redis::Value::Array(rows) | redis::Value::Set(rows)) = res else {
continue;
};
if let Some(redis::Value::Array(cols) | redis::Value::Set(cols)) = rows.into_iter().next() {
if let Some(redis::Value::Int(n)) = cols.get(3) {
out.insert(id.clone(), (*n).max(0) as u32);
}
}
}
Ok(out)
}
async fn handle_poison(shared: &ProxyShared, ids: &[String]) {
match shared.cfg.poison {
ProxyPoison::Drop => {
tracing::error!(?ids, "PROXY 毒消息超重投上限,按 Drop 策略 XACK 丢弃");
ack(shared, ids).await;
}
ProxyPoison::Dlq => {
let dlq = format!("{}:dlq", shared.stream);
let mut conn = shared.client.conn();
let mut rpipe = redis::pipe();
for id in ids {
rpipe.cmd("XRANGE").arg(&shared.stream).arg(id).arg(id);
}
let ranges: Vec<redis::Value> = match redis::aio::ConnectionLike::req_packed_commands(
&mut conn,
&rpipe,
0,
ids.len(),
)
.await
{
Ok(v) => v,
Err(e) => {
tracing::warn!(?ids, err = %e, "PROXY DLQ 批量 XRANGE 失败,本批不 ACK 待重投");
return;
}
};
let mut bodies: Vec<Vec<u8>> = Vec::new();
for r in &ranges {
for (_eid, env) in parse_entries(r).ok {
match serde_json::to_vec(&env) {
Ok(b) => bodies.push(b),
Err(e) => {
tracing::warn!(err = %e, "PROXY DLQ 序列化失败,跳过该条(不写空 body)")
}
}
}
}
if bodies.len() != ids.len() {
tracing::warn!(
?ids,
got = bodies.len(),
want = ids.len(),
"PROXY DLQ body 数量与 ids 不一致(源 entry 可能被外部删/序列化失败),本批不 ACK"
);
return;
}
let mut apipe = redis::pipe();
for b in &bodies {
apipe
.cmd("XADD")
.arg(&dlq)
.arg("*")
.arg(DATA_FIELD)
.arg(b.as_slice());
}
match redis::aio::ConnectionLike::req_packed_commands(
&mut conn,
&apipe,
0,
bodies.len(),
)
.await
{
Ok(v) if all_pipeline_replies_ok(&v, bodies.len()) => {}
Ok(v) => {
tracing::warn!(?ids, replies = ?v, "PROXY DLQ XADD 返回 server error/数量异常,本批不 ACK 待重投");
return;
}
Err(e) => {
tracing::warn!(?ids, err = %e, "PROXY DLQ XADD 传输失败,本批不 ACK 待重投");
return;
}
}
tracing::error!(?ids, dlq = %dlq, "PROXY 毒消息超上限,已转 DLQ 并 XACK 源");
ack(shared, ids).await;
}
}
}
fn all_pipeline_replies_ok(values: &[redis::Value], expected: usize) -> bool {
values.len() == expected
&& !values
.iter()
.any(|v| matches!(v, redis::Value::ServerError(_)))
}
fn base64_encode(bytes: &[u8]) -> String {
const T: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
let mut out = String::with_capacity(bytes.len().div_ceil(3) * 4);
for chunk in bytes.chunks(3) {
let b0 = chunk[0] as usize;
let b1 = chunk.get(1).copied().unwrap_or(0) as usize;
let b2 = chunk.get(2).copied().unwrap_or(0) as usize;
out.push(T[b0 >> 2] as char);
out.push(T[((b0 & 0b11) << 4) | (b1 >> 4)] as char);
out.push(if chunk.len() > 1 {
T[((b1 & 0b1111) << 2) | (b2 >> 6)] as char
} else {
'='
});
out.push(if chunk.len() > 2 {
T[b2 & 0b111111] as char
} else {
'='
});
}
out
}
async fn dispose_bad(shared: &ProxyShared, bad: &[BadEntry]) {
if bad.is_empty() {
return;
}
let ids: Vec<String> = bad.iter().map(|b| b.id.clone()).collect();
match shared.cfg.poison {
ProxyPoison::Drop => {
tracing::error!(
?ids,
"PROXY 不可解析 entry(无 data/非 Envelope),按 Drop 策略 XACK 丢弃"
);
}
ProxyPoison::Dlq => {
let dlq = format!("{}:dlq", shared.stream);
let mut conn = shared.client.conn();
let mut apipe = redis::pipe();
for b in bad {
let raw: Vec<serde_json::Value> = b
.raw
.iter()
.map(|(k, v)| {
serde_json::json!({
"field_utf8": String::from_utf8_lossy(k),
"field_b64": base64_encode(k),
"value_utf8": String::from_utf8_lossy(v),
"value_b64": base64_encode(v),
})
})
.collect();
let body = serde_json::to_vec(&serde_json::json!({
"_proxy_dlq": "unparseable",
"reason": b.reason,
"stream": shared.stream,
"group": shared.group,
"id": b.id,
"raw": raw,
}))
.unwrap_or_default();
apipe
.cmd("XADD")
.arg(&dlq)
.arg("*")
.arg(DATA_FIELD)
.arg(body);
}
match redis::aio::ConnectionLike::req_packed_commands(&mut conn, &apipe, 0, bad.len())
.await
{
Ok(v) if all_pipeline_replies_ok(&v, bad.len()) => {}
Ok(v) => {
tracing::warn!(?ids, replies = ?v, "PROXY 不可解析 entry 转 DLQ 返回 server error/数量异常,本批不 ACK 待重试");
return;
}
Err(e) => {
tracing::warn!(?ids, err = %e, "PROXY 不可解析 entry 转 DLQ 传输失败,本批不 ACK 待重试");
return;
}
}
tracing::error!(?ids, dlq = %dlq, "PROXY 不可解析 entry 已转 DLQ 并 XACK 源");
}
}
ack(shared, &ids).await;
}