use std::collections::HashMap;
use std::sync::Arc;
use std::time::Duration;
use serde::{de::DeserializeOwned, Serialize};
use tokio::task::JoinHandle;
use tokio_util::sync::CancellationToken;
use crate::client::RedisClient;
use crate::config::{MAX_REDIS_NAME_BYTES, MAX_REDIS_RUNTIME_DURATION_MS, MAX_STREAM_BATCH_SIZE};
use crate::error::{NasaRedisError, Result};
use crate::partition::{Envelope, ErasedHandler, HandlerMap, DATA_FIELD};
const MAX_PROXY_CONSUMERS: usize = 256;
#[derive(Debug, Clone, serde::Deserialize)]
#[serde(default, deny_unknown_fields)]
pub struct ProxyCfg {
pub consumers: usize,
pub batch_size: usize,
pub poll_idle_ms: u64,
pub max_redeliver: u32,
pub reclaim_min_idle_ms: u64,
pub poison: ProxyPoison,
pub handler_timeout_ms: u64,
pub max_stream_len: Option<u64>,
pub requeue_unregistered: bool,
pub drain_deadline_ms: u64,
}
impl Default for ProxyCfg {
fn default() -> Self {
Self {
consumers: 1,
batch_size: 100,
poll_idle_ms: 200,
max_redeliver: 5,
reclaim_min_idle_ms: 30_000,
poison: ProxyPoison::Drop,
handler_timeout_ms: 10_000,
max_stream_len: Some(1_000_000),
requeue_unregistered: false,
drain_deadline_ms: 10_000,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ProxyPoison {
Drop,
Dlq,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Deserialize, Default)]
#[serde(rename_all = "snake_case")]
pub enum ProxyStartOffset {
#[default]
New,
History,
}
impl ProxyStartOffset {
fn redis_id(self) -> &'static str {
match self {
ProxyStartOffset::New => "$",
ProxyStartOffset::History => "0-0",
}
}
}
pub struct PreparedProxy {
client: Arc<RedisClient>,
stream: String,
group: String,
handlers: HandlerMap,
cfg: ProxyCfg,
}
impl PreparedProxy {
pub async fn prepare(
client: Arc<RedisClient>,
stream: impl Into<String>,
group: impl Into<String>,
cfg: ProxyCfg,
) -> Result<Self> {
Self::prepare_with_offset(client, stream, group, cfg, ProxyStartOffset::New).await
}
pub async fn prepare_with_offset(
client: Arc<RedisClient>,
stream: impl Into<String>,
group: impl Into<String>,
cfg: ProxyCfg,
start_offset: ProxyStartOffset,
) -> Result<Self> {
if cfg.consumers == 0 || cfg.batch_size == 0 {
return Err(NasaRedisError::Config(
"ProxyCfg.consumers/batch_size 必须 > 0".into(),
));
}
if cfg.consumers > MAX_PROXY_CONSUMERS {
return Err(NasaRedisError::Config(format!(
"ProxyCfg.consumers 过大(上限 {MAX_PROXY_CONSUMERS})"
)));
}
if cfg.batch_size > MAX_STREAM_BATCH_SIZE {
return Err(NasaRedisError::Config(format!(
"ProxyCfg.batch_size 过大(上限 {MAX_STREAM_BATCH_SIZE})"
)));
}
if cfg.reclaim_min_idle_ms == 0 {
return Err(NasaRedisError::Config(
"ProxyCfg.reclaim_min_idle_ms 必须 > 0".into(),
));
}
if cfg.handler_timeout_ms == 0 {
return Err(NasaRedisError::Config(
"ProxyCfg.handler_timeout_ms 必须 > 0".into(),
));
}
if cfg.drain_deadline_ms == 0 {
return Err(NasaRedisError::Config(
"ProxyCfg.drain_deadline_ms 必须 > 0".into(),
));
}
if cfg.max_stream_len == Some(0) {
return Err(NasaRedisError::Config(
"ProxyCfg.max_stream_len 配置后必须 > 0".into(),
));
}
if cfg.poll_idle_ms == 0
|| cfg.poll_idle_ms > MAX_REDIS_RUNTIME_DURATION_MS
|| cfg.reclaim_min_idle_ms > MAX_REDIS_RUNTIME_DURATION_MS
|| cfg.handler_timeout_ms > MAX_REDIS_RUNTIME_DURATION_MS
|| cfg.drain_deadline_ms > MAX_REDIS_RUNTIME_DURATION_MS
{
return Err(NasaRedisError::Config(format!(
"ProxyCfg 的 poll/reclaim/handler/drain 时长必须位于 1..={MAX_REDIS_RUNTIME_DURATION_MS}ms"
)));
}
if cfg.max_redeliver == 0 {
return Err(NasaRedisError::Config(
"ProxyCfg.max_redeliver 必须 >= 1".into(),
));
}
if cfg.handler_timeout_ms > 0 && cfg.reclaim_min_idle_ms <= cfg.handler_timeout_ms {
return Err(NasaRedisError::Config(format!(
"ProxyCfg.reclaim_min_idle_ms({}) 必须 > handler_timeout_ms({}),为本地异步等待预留退出余量",
cfg.reclaim_min_idle_ms, cfg.handler_timeout_ms
)));
}
let stream = stream.into();
let group = group.into();
if stream.trim().is_empty()
|| group.trim().is_empty()
|| stream != stream.trim()
|| group != group.trim()
|| stream.len() > MAX_REDIS_NAME_BYTES
|| group.len() > MAX_REDIS_NAME_BYTES
{
return Err(NasaRedisError::Config(format!(
"PROXY stream / group 必须无首尾空白、非空且不超过 {MAX_REDIS_NAME_BYTES} 字节"
)));
}
let r: std::result::Result<String, redis::RedisError> = redis::cmd("XGROUP")
.arg("CREATE")
.arg(&stream)
.arg(&group)
.arg(start_offset.redis_id())
.arg("MKSTREAM")
.query_async(&mut client.conn())
.await;
match r {
Ok(reply) if reply == "OK" => {}
Err(e) if e.code() == Some("BUSYGROUP") => {}
Ok(_) => {
return Err(NasaRedisError::Parsing(
"PROXY 建组回复不是预期的 OK".into(),
));
}
Err(e) => return Err(e.into()),
}
Ok(Self {
client,
stream,
group,
handlers: HashMap::new(),
cfg,
})
}
pub fn register<T, F, Fut>(&mut self, topic: &str, event: &str, f: F) -> &mut Self
where
T: DeserializeOwned + Send + 'static,
F: Fn(Vec<T>) -> Fut + Send + Sync + 'static,
Fut: std::future::Future<Output = std::result::Result<(), String>> + Send + 'static,
{
let f = Arc::new(f);
let h: ErasedHandler = Arc::new(move |items: Vec<(String, serde_json::Value)>| {
let f = Arc::clone(&f);
Box::pin(async move {
let mut typed: Vec<T> = Vec::with_capacity(items.len());
let mut good_ids: Vec<String> = Vec::with_capacity(items.len());
let mut failed: Vec<String> = Vec::new();
for (id, v) in items {
match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
serde_json::from_value::<T>(v)
})) {
Ok(Ok(t)) => {
typed.push(t);
good_ids.push(id);
}
Ok(Err(e)) => {
tracing::error!(id, category = ?e.classify(), "PROXY 业务类型解码失败,该 ID 进重投/毒");
failed.push(id);
}
Err(_) => {
tracing::error!(id, "PROXY 业务类型解码展开,该 ID 保留重投");
failed.push(id);
}
}
}
if typed.is_empty() {
return failed;
}
let result = match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
f(typed)
})) {
Ok(future) => {
futures::FutureExt::catch_unwind(std::panic::AssertUnwindSafe(future)).await
}
Err(error) => Err(error),
};
match result {
Ok(Ok(())) => failed,
Ok(Err(_)) => {
tracing::warn!("PROXY handler 失败,本桶 ID 转重投");
failed.extend(good_ids);
failed
}
Err(_) => {
tracing::error!("PROXY handler 构造或轮询展开,本桶 ID 转重投");
failed.extend(good_ids);
failed
}
}
})
});
self.handlers
.insert((topic.to_string(), event.to_string()), h);
self
}
pub fn handler_count(&self) -> usize {
self.handlers.len()
}
pub async fn start(self) -> Result<RunningProxy> {
let running = self.start_suspended().await?;
running.activate();
Ok(running)
}
pub async fn start_suspended(self) -> Result<RunningProxy> {
self.start_suspended_with_activation(CancellationToken::new())
.await
}
pub async fn start_suspended_with_activation(
self,
ready: CancellationToken,
) -> Result<RunningProxy> {
let runtime = tokio::runtime::Handle::try_current()
.map_err(|_| NasaRedisError::Config("PROXY 启动需要有效的 Tokio 执行上下文".into()))?;
let cancel = CancellationToken::new();
let activity = Arc::new(crate::activity::Activity::default());
let shared = Arc::new(ProxyShared {
activity: activity.clone(),
client: self.client.clone(),
stream: self.stream.clone(),
group: self.group.clone(),
handlers: self.handlers,
cfg: self.cfg.clone(),
});
let mut handles = Vec::new();
let mut names = Vec::with_capacity(self.cfg.consumers + 1);
for i in 0..=self.cfg.consumers {
let consumer = format!("c-{}-{i}", uuid::Uuid::new_v4().simple());
names.push(consumer.clone());
let shared = shared.clone();
let cancel = cancel.clone();
let ready = ready.clone();
let reclaim = i == self.cfg.consumers;
let active = activity.task(if reclaim {
crate::activity::TaskKind::Reclaimer
} else {
crate::activity::TaskKind::Consumer
});
handles.push(runtime.spawn(async move {
let _active = active;
tokio::select! {
biased;
_ = cancel.cancelled() => return,
_ = ready.cancelled() => {},
}
if reclaim {
reclaim_loop(shared, consumer, cancel).await;
} else {
consumer_loop(shared, consumer, cancel).await;
}
}));
}
let (deadline, _) = tokio::sync::watch::channel(None);
let (report, outcome) = tokio::sync::watch::channel(ProxyStopReport::default());
let owner = ProxyCloseOwner {
client: self.client.clone(),
stream: self.stream.clone(),
group: self.group.clone(),
names,
aborts: handles.iter().map(JoinHandle::abort_handle).collect(),
handles,
cancel: cancel.clone(),
deadline: deadline.clone(),
budget: Duration::from_millis(self.cfg.drain_deadline_ms),
report: activity.completion(report),
};
runtime.spawn(owner.run());
let running = RunningProxy {
activity,
client: self.client,
stream: self.stream,
cancel,
ready,
deadline,
outcome,
max_stream_len: self.cfg.max_stream_len,
drain_deadline_ms: self.cfg.drain_deadline_ms,
consumers: self.cfg.consumers,
};
if running.with_running(|| ()).is_none() {
running.begin_shutdown();
return Err(NasaRedisError::Config(
"PROXY 启动时未取得完整任务责任,已请求关闭".into(),
));
}
Ok(running)
}
}
struct ProxyShared {
activity: Arc<crate::activity::Activity>,
client: Arc<RedisClient>,
stream: String,
group: String,
handlers: HandlerMap,
cfg: ProxyCfg,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum ProxyCleanup {
#[default]
Skipped,
Complete,
Pending,
Unavailable,
Deadline,
}
#[derive(Debug, Clone, Copy, Default)]
pub struct ProxyStopReport {
pub terminated: bool,
pub forced: bool,
pub failed: bool,
pub cleanup: ProxyCleanup,
}
pub struct RunningProxy {
consumers: usize,
activity: Arc<crate::activity::Activity>,
client: Arc<RedisClient>,
stream: String,
cancel: CancellationToken,
ready: CancellationToken,
deadline: tokio::sync::watch::Sender<Option<tokio::time::Instant>>,
outcome: tokio::sync::watch::Receiver<ProxyStopReport>,
max_stream_len: Option<u64>,
drain_deadline_ms: u64,
}
fn parse_stream_id(id: &str) -> Option<(u64, u64)> {
let (milliseconds, sequence) = id.split_once('-')?;
if milliseconds.is_empty()
|| sequence.is_empty()
|| !milliseconds.bytes().all(|byte| byte.is_ascii_digit())
|| !sequence.bytes().all(|byte| byte.is_ascii_digit())
{
return None;
}
Some((milliseconds.parse().ok()?, sequence.parse().ok()?))
}
impl RunningProxy {
pub fn with_running<T>(&self, publish: impl FnOnce() -> T) -> Option<T> {
self.activity
.with_tasks(self.consumers, 1, 0, || {
let report = self.stop_report();
if self.cancel.is_cancelled() || report.terminated || report.failed {
None
} else {
Some(publish())
}
})
.flatten()
}
pub fn observation(&self) -> crate::RedisTaskObservation {
self.activity.snapshot()
}
pub async fn publish<T: Serialize>(
&self,
topic: &str,
event: &str,
data: &T,
) -> Result<String> {
if topic.is_empty() || event.is_empty() {
return Err(NasaRedisError::Config(
"Proxy 发布的 topic/event 不能为空".into(),
));
}
if self.cancel.is_cancelled() {
return Err(NasaRedisError::NotExecuted("Proxy 已关闭".into()));
}
let body = match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
let env = Envelope {
topic: topic.to_string(),
event: event.to_string(),
data: serde_json::to_value(data)?,
passthrough: None,
};
serde_json::to_vec(&env)
})) {
Ok(Ok(body)) => body,
Ok(Err(error)) => {
return Err(NasaRedisError::Codec(format!(
"Proxy 序列化失败: {:?}, line={}, column={}",
error.classify(),
error.line(),
error.column()
)));
}
Err(_) => {
return Err(NasaRedisError::Codec("Proxy 序列化展开".into()));
}
};
let mut cmd = redis::cmd("XADD");
cmd.arg(&self.stream);
if let Some(n) = self.max_stream_len {
cmd.arg("MAXLEN").arg("~").arg(n);
}
cmd.arg("*").arg(DATA_FIELD).arg(body);
let id: String = cmd.query_async(&mut self.client.conn()).await.map_err(
|error: redis::RedisError| {
NasaRedisError::PublishOutcomeUnknown(format!(
"Proxy XADD 未取得有效回包: {:?}",
error.kind()
))
},
)?;
if !matches!(parse_stream_id(&id), Some((milliseconds, sequence)) if milliseconds != 0 || sequence != 0)
{
return Err(NasaRedisError::PublishOutcomeUnknown(
"Proxy XADD 返回非法条目身份".into(),
));
}
Ok(id)
}
pub fn activate(&self) -> bool {
if self.cancel.is_cancelled() {
return false;
}
self.ready.cancel();
true
}
pub fn begin_shutdown_until(&self, deadline: tokio::time::Instant) {
self.activity.close();
let local = tokio::time::Instant::now() + Duration::from_millis(self.drain_deadline_ms);
let deadline = deadline.min(local);
self.deadline.send_modify(|current| {
*current = Some(current.map_or(deadline, |old| old.min(deadline)))
});
self.cancel.cancel();
}
pub fn begin_shutdown(&self) {
self.begin_shutdown_until(
tokio::time::Instant::now() + Duration::from_millis(self.drain_deadline_ms),
);
}
pub fn stop_report(&self) -> ProxyStopReport {
let disconnected = self.outcome.has_changed().is_err();
let mut report = *self.outcome.borrow();
if disconnected && !report.terminated {
report.failed = true;
}
report
}
pub async fn wait_closed(&self) -> ProxyStopReport {
let mut outcome = self.outcome.clone();
loop {
let report = *outcome.borrow_and_update();
if report.terminated {
return report;
}
if outcome.changed().await.is_err() {
return ProxyStopReport {
failed: true,
..report
};
}
}
}
pub async fn shutdown(self) {
self.begin_shutdown();
let report = self.wait_closed().await;
if report.forced
|| report.failed
|| matches!(
report.cleanup,
ProxyCleanup::Unavailable | ProxyCleanup::Deadline
)
{
tracing::warn!(
forced = report.forced,
failed = report.failed,
cleanup = ?report.cleanup,
"Proxy 非正常排干"
);
}
}
}
impl Drop for RunningProxy {
fn drop(&mut self) {
self.begin_shutdown();
}
}
struct ProxyCloseOwner {
client: Arc<RedisClient>,
stream: String,
group: String,
names: Vec<String>,
handles: Vec<JoinHandle<()>>,
aborts: Vec<tokio::task::AbortHandle>,
cancel: CancellationToken,
deadline: tokio::sync::watch::Sender<Option<tokio::time::Instant>>,
budget: Duration,
report: crate::activity::CompletionOwner<ProxyStopReport>,
}
impl ProxyCloseOwner {
async fn run(mut self) {
use futures::StreamExt;
let mut tasks: futures::stream::FuturesUnordered<_> = self.handles.drain(..).collect();
let mut report = ProxyStopReport::default();
tokio::select! {
biased;
_ = self.cancel.cancelled() => {},
_ = tasks.next() => {
report.failed = true;
self.cancel.cancel();
},
}
self.report.send_replace(report);
let deadline =
(*self.deadline.borrow()).unwrap_or_else(|| tokio::time::Instant::now() + self.budget);
while !tasks.is_empty() {
match self.within_deadline(deadline, tasks.next()).await {
Ok(Some(Ok(()))) => {}
Ok(Some(Err(_))) => report.failed = true,
Ok(None) => break,
Err(_) => {
report.forced = true;
for abort in &self.aborts {
abort.abort();
}
while let Some(result) = tasks.next().await {
if result.is_err_and(|error| !error.is_cancelled()) {
report.failed = true;
}
}
break;
}
}
}
if !report.forced && !report.failed {
report.cleanup = self.cleanup(deadline).await;
}
report.terminated = true;
self.report.send_replace(report);
}
async fn within_deadline<T>(
&self,
deadline: tokio::time::Instant,
future: impl std::future::Future<Output = T>,
) -> std::result::Result<T, ()> {
let mut changes = self.deadline.subscribe();
tokio::pin!(future);
loop {
let current = changes
.borrow_and_update()
.unwrap_or(deadline)
.min(deadline);
tokio::select! {
biased;
_ = tokio::time::sleep_until(current) => return Err(()),
result = &mut future => return Ok(result),
_ = changes.changed() => {},
}
}
}
async fn cleanup(&self, deadline: tokio::time::Instant) -> ProxyCleanup {
let script = redis::Script::new(
r#"
local pending = redis.call('XPENDING', KEYS[1], ARGV[1], '-', '+', 1, ARGV[2])
if type(pending) ~= 'table' then
return redis.error_reply('PROXY pending reply is invalid')
end
if next(pending) ~= nil then
return 1
end
local removed = redis.call('XGROUP', 'DELCONSUMER', KEYS[1], ARGV[1], ARGV[2])
if removed ~= 0 then
return redis.error_reply('PROXY consumer deletion reply is invalid')
end
return 0
"#,
);
let mut outcome = ProxyCleanup::Complete;
for name in &self.names {
if tokio::time::Instant::now()
>= self.deadline.borrow().unwrap_or(deadline).min(deadline)
{
return ProxyCleanup::Deadline;
}
let result: std::result::Result<redis::Value, _> = match self
.within_deadline(
deadline,
script
.key(&self.stream)
.arg(&self.group)
.arg(name)
.invoke_async(&mut self.client.conn()),
)
.await
{
Ok(result) => result,
Err(_) => return ProxyCleanup::Deadline,
};
match result {
Ok(redis::Value::Int(0)) => {}
Ok(redis::Value::Int(1)) => outcome = ProxyCleanup::Pending,
_ => return ProxyCleanup::Unavailable,
}
}
outcome
}
}
impl Drop for ProxyCloseOwner {
fn drop(&mut self) {
self.cancel.cancel();
for abort in &self.aborts {
abort.abort();
}
}
}
struct BadEntry {
id: String,
reason: String,
raw: Vec<(Vec<u8>, Vec<u8>)>,
}
struct ParsedEntries {
ok: Vec<(String, Envelope)>,
bad: Vec<BadEntry>,
}
fn parse_entries(entries: &redis::Value, allow_deleted: bool) -> Result<ParsedEntries> {
let mut ok = Vec::new();
let mut bad = Vec::new();
let invalid = || NasaRedisError::Parsing("PROXY 条目回复结构或身份无效".into());
let redis::Value::Array(items) = entries else {
return Err(invalid());
};
let mut previous = (0, 0);
for item in items {
if allow_deleted && matches!(item, redis::Value::Nil) {
continue;
}
let redis::Value::Array(pair) = item else {
return Err(invalid());
};
let [redis::Value::BulkString(id_bytes), redis::Value::Array(fields)] = pair.as_slice()
else {
return Err(invalid());
};
let id = std::str::from_utf8(id_bytes).map_err(|_| invalid())?;
let coordinate = parse_stream_id(id).ok_or_else(invalid)?;
if coordinate <= previous || fields.is_empty() || fields.len() % 2 != 0 {
return Err(invalid());
}
previous = coordinate;
let mut data_bytes: Option<&[u8]> = None;
let mut duplicate_data = false;
let mut raw: Vec<(Vec<u8>, Vec<u8>)> = Vec::new();
for pair in fields.chunks_exact(2) {
let [redis::Value::BulkString(field), redis::Value::BulkString(value)] = pair else {
return Err(invalid());
};
if field == DATA_FIELD.as_bytes() {
duplicate_data |= data_bytes.replace(value).is_some();
}
raw.push((field.clone(), value.clone()));
}
let id = id.to_owned();
if duplicate_data {
bad.push(BadEntry {
id,
reason: "存在多个 data 字段".into(),
raw,
});
continue;
}
let Some(bytes) = data_bytes else {
bad.push(BadEntry {
id,
reason: "缺 data 字段(非标准 Envelope wire)".to_string(),
raw,
});
continue;
};
match serde_json::from_slice::<Envelope>(bytes) {
Ok(env) => ok.push((id, env)),
Err(e) => {
tracing::error!(id, category = ?e.classify(), "PROXY 信封无法解码,进入毒消息策略");
bad.push(BadEntry {
id,
reason: format!(
"envelope 解码失败: {:?}, line={}, column={}",
e.classify(),
e.line(),
e.column()
),
raw,
});
}
}
}
Ok(ParsedEntries {
ok,
bad,
})
}
async fn dispatch(shared: &ProxyShared, batch: Vec<(String, Envelope)>) -> Vec<String> {
let mut buckets: HashMap<(String, String), Vec<(String, serde_json::Value)>> = HashMap::new();
for (id, env) in batch {
buckets
.entry((env.topic, env.event))
.or_default()
.push((id, env.data));
}
let mut failed = Vec::new();
for (key, items) in buckets {
match shared.handlers.get(&key) {
Some(h) => {
let bucket_ids: Vec<String> = items.iter().map(|(id, _)| id.clone()).collect();
let dur = Duration::from_millis(shared.cfg.handler_timeout_ms);
match tokio::time::timeout(dur, h(items)).await {
Ok(f) => failed.extend(f),
Err(_) => {
tracing::warn!(topic = %key.0, event = %key.1, timeout_ms = shared.cfg.handler_timeout_ms, "PROXY handler 超时,整桶转重投(留 PEL)");
failed.extend(bucket_ids);
}
}
}
None => {
tracing::warn!(topic = %key.0, event = %key.1, requeue = shared.cfg.requeue_unregistered, "PROXY 本节点无注册 handler");
if shared.cfg.requeue_unregistered {
failed.extend(items.into_iter().map(|(id, _)| id));
}
}
}
}
failed
}
async fn ack(shared: &ProxyShared, ids: &[String]) {
if ids.is_empty() {
return;
}
let mut cmd = redis::cmd("XACK");
cmd.arg(&shared.stream).arg(&shared.group);
for id in ids {
cmd.arg(id);
}
let r: std::result::Result<redis::Value, redis::RedisError> =
cmd.query_async(&mut shared.client.conn()).await;
match r {
Ok(redis::Value::Int(count))
if usize::try_from(count).is_ok_and(|count| count <= ids.len()) => {}
Ok(_) => tracing::warn!("PROXY XACK 回复计数无效,确认结果未知"),
Err(e) => tracing::warn!(kind = ?e.kind(), "PROXY XACK 未取得有效回复,确认结果未知"),
}
}
async fn consumer_loop(shared: Arc<ProxyShared>, consumer: String, cancel: CancellationToken) {
loop {
if cancel.is_cancelled() {
break;
}
let idle = match read_new(&shared, &consumer).await {
Ok(parsed) if !parsed.ok.is_empty() || !parsed.bad.is_empty() => {
let count = parsed.ok.len() + parsed.bad.len();
shared.activity.accept(count, 0);
if !parsed.bad.is_empty() {
dispose_bad(&shared, &parsed.bad).await;
}
if !parsed.ok.is_empty() {
let ids: Vec<String> = parsed.ok.iter().map(|(id, _)| id.clone()).collect();
let failed = dispatch(&shared, parsed.ok).await;
let failed: std::collections::HashSet<String> = failed.into_iter().collect();
let ok: Vec<String> = ids.into_iter().filter(|i| !failed.contains(i)).collect();
ack(&shared, &ok).await;
}
shared.activity.finish(count, false);
continue;
}
Ok(_) => {
shared.activity.finish(0, false);
Duration::from_millis(shared.cfg.poll_idle_ms.max(1))
}
Err(e) => {
tracing::warn!(err = %e, "PROXY XREADGROUP 失败,短暂退避");
Duration::from_millis(200)
}
};
tokio::select! {
biased;
_ = cancel.cancelled() => break,
_ = tokio::time::sleep(idle) => {},
}
}
}
async fn read_new(shared: &ProxyShared, consumer: &str) -> Result<ParsedEntries> {
let v: redis::Value = redis::cmd("XREADGROUP")
.arg("GROUP")
.arg(&shared.group)
.arg(consumer)
.arg("COUNT")
.arg(shared.cfg.batch_size)
.arg("STREAMS")
.arg(&shared.stream)
.arg(">")
.query_async(&mut shared.client.conn())
.await?;
let empty = || ParsedEntries {
ok: Vec::new(),
bad: Vec::new(),
};
let invalid = || NasaRedisError::Parsing("PROXY XREADGROUP 回复来源或结构无效".into());
let (stream, entries) = match &v {
redis::Value::Nil => return Ok(empty()),
redis::Value::Array(streams) => {
if streams.is_empty() {
return Ok(empty());
}
let [redis::Value::Array(pair)] = streams.as_slice() else {
return Err(invalid());
};
let [stream, entries] = pair.as_slice() else {
return Err(invalid());
};
(stream, entries)
}
redis::Value::Map(pairs) => {
if pairs.is_empty() {
return Ok(empty());
}
let [(stream, entries)] = pairs.as_slice() else {
return Err(invalid());
};
(stream, entries)
}
_ => return Err(invalid()),
};
if !matches!(stream, redis::Value::BulkString(name) if name == shared.stream.as_bytes())
|| !matches!(entries, redis::Value::Array(items) if items.len() <= shared.cfg.batch_size)
{
return Err(invalid());
}
parse_entries(entries, false)
}
async fn reclaim_loop(shared: Arc<ProxyShared>, consumer: String, cancel: CancellationToken) {
let period = Duration::from_millis((shared.cfg.reclaim_min_idle_ms / 2).clamp(200, 10_000));
let mut tick = tokio::time::interval(period);
tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
let mut cursor = "0-0".to_owned();
loop {
tokio::select! {
biased;
_ = cancel.cancelled() => break,
_ = tick.tick() => {
if let Err(e) = reclaim_once(&shared, &consumer, &mut cursor).await {
tracing::warn!(err = %e, "PROXY reclaim 失败,下轮重试");
}
}
}
}
}
async fn reclaim_once(shared: &ProxyShared, consumer: &str, cursor: &mut String) -> Result<()> {
let start = parse_stream_id(cursor)
.ok_or_else(|| NasaRedisError::Config("PROXY 回收扫描起点无效".into()))?;
let v: redis::Value = redis::cmd("XAUTOCLAIM")
.arg(&shared.stream)
.arg(&shared.group)
.arg(consumer)
.arg(shared.cfg.reclaim_min_idle_ms)
.arg(cursor.as_str())
.arg("COUNT")
.arg(shared.cfg.batch_size)
.query_async(&mut shared.client.conn())
.await?;
let invalid = || NasaRedisError::Parsing("PROXY XAUTOCLAIM 回复结构或身份无效".into());
let redis::Value::Array(parts) = &v else {
return Err(invalid());
};
if parts.len() != 2 && parts.len() != 3 {
return Err(invalid());
}
let redis::Value::BulkString(next_cursor) = &parts[0] else {
return Err(invalid());
};
let next_cursor = std::str::from_utf8(next_cursor).map_err(|_| invalid())?;
let next = parse_stream_id(next_cursor).ok_or_else(invalid)?;
if next != (0, 0) && next <= start {
return Err(invalid());
}
let entries = &parts[1];
if !matches!(entries, redis::Value::Array(items) if items.len() <= shared.cfg.batch_size) {
return Err(invalid());
}
let parsed = parse_entries(entries, parts.len() == 2)?;
let delivered: std::collections::HashSet<_> = parsed
.ok
.iter()
.map(|(id, _)| id)
.chain(parsed.bad.iter().map(|entry| &entry.id))
.filter_map(|id| parse_stream_id(id))
.collect();
if delivered
.iter()
.any(|id| *id < start || (next != (0, 0) && *id >= next))
{
return Err(invalid());
}
if let Some(deleted) = parts.get(2) {
let redis::Value::Array(deleted) = deleted else {
return Err(invalid());
};
let mut previous = (0, 0);
for id in deleted {
let redis::Value::BulkString(id) = id else {
return Err(invalid());
};
let coordinate = std::str::from_utf8(id)
.ok()
.and_then(parse_stream_id)
.ok_or_else(invalid)?;
if coordinate <= previous
|| coordinate < start
|| (next != (0, 0) && coordinate >= next)
|| delivered.contains(&coordinate)
{
return Err(invalid());
}
previous = coordinate;
}
}
*cursor = next_cursor.to_owned();
let count = parsed.ok.len() + parsed.bad.len();
shared.activity.accept(count, 0);
if !parsed.bad.is_empty() {
dispose_bad(shared, &parsed.bad).await;
}
let batch = parsed.ok;
if batch.is_empty() {
shared.activity.finish(count, false);
return Ok(());
}
let ids: Vec<String> = batch.iter().map(|(id, _)| id.clone()).collect();
let counts = match pending_counts(shared, &ids).await {
Ok(counts) => counts,
Err(error) => {
shared.activity.finish(count, false);
return Err(error);
}
};
let mut to_dispatch = Vec::new();
let mut poison_ids = Vec::new();
for (id, env) in batch {
let n = match counts.get(&id).copied() {
Some(n) => n,
None => {
tracing::warn!(
id,
"PROXY XPENDING 未返回该 id 的重投计数(协议异常/竞态),保守按 1 重投"
);
1
}
};
if n > u64::from(shared.cfg.max_redeliver) {
poison_ids.push(id);
} else {
to_dispatch.push((id, env));
}
}
if !poison_ids.is_empty() {
handle_poison(shared, &poison_ids).await;
}
if !to_dispatch.is_empty() {
let d_ids: Vec<String> = to_dispatch.iter().map(|(id, _)| id.clone()).collect();
let failed = dispatch(shared, to_dispatch).await;
let failed: std::collections::HashSet<String> = failed.into_iter().collect();
let ok: Vec<String> = d_ids.into_iter().filter(|i| !failed.contains(i)).collect();
ack(shared, &ok).await;
}
shared.activity.finish(count, false);
Ok(())
}
async fn pending_counts(shared: &ProxyShared, ids: &[String]) -> Result<HashMap<String, u64>> {
let mut out = HashMap::new();
if ids.is_empty() {
return Ok(out);
}
let mut pipe = redis::pipe();
for id in ids {
pipe.cmd("XPENDING")
.arg(&shared.stream)
.arg(&shared.group)
.arg(id)
.arg(id)
.arg(1);
}
let mut conn = shared.client.conn();
let results: Vec<redis::Value> =
redis::aio::ConnectionLike::req_packed_commands(&mut conn, &pipe, 0, ids.len()).await?;
let invalid = || NasaRedisError::Parsing("PROXY XPENDING 回复结构、身份或计数无效".into());
if results.len() != ids.len() {
return Err(invalid());
}
for (id, res) in ids.iter().zip(results) {
let redis::Value::Array(rows) = res else {
return Err(invalid());
};
if rows.is_empty() {
continue;
}
let [redis::Value::Array(columns)] = rows.as_slice() else {
return Err(invalid());
};
let [redis::Value::BulkString(reply_id), redis::Value::BulkString(_consumer), redis::Value::Int(idle), redis::Value::Int(deliveries)] =
columns.as_slice()
else {
return Err(invalid());
};
if reply_id != id.as_bytes() || *idle < 0 {
return Err(invalid());
}
let deliveries = u64::try_from(*deliveries).map_err(|_| invalid())?;
out.insert(id.clone(), deliveries);
}
Ok(out)
}
async fn handle_poison(shared: &ProxyShared, ids: &[String]) {
if ids.is_empty() {
return;
}
match shared.cfg.poison {
ProxyPoison::Drop => {
tracing::error!(?ids, "PROXY 毒消息超重投上限,按 Drop 策略 XACK 丢弃");
ack(shared, ids).await;
}
ProxyPoison::Dlq => {
let dlq = format!("{}:dlq", shared.stream);
let mut conn = shared.client.conn();
let mut rpipe = redis::pipe();
for id in ids {
rpipe.cmd("XRANGE").arg(&shared.stream).arg(id).arg(id);
}
let ranges: Vec<redis::Value> = match redis::aio::ConnectionLike::req_packed_commands(
&mut conn,
&rpipe,
0,
ids.len(),
)
.await
{
Ok(v) => v,
Err(e) => {
tracing::warn!(?ids, err = %e, "PROXY DLQ 批量 XRANGE 失败,本批不 ACK 待重投");
return;
}
};
if ranges.len() != ids.len() {
tracing::warn!("PROXY DLQ 重读回复数量异常,保留源 PEL");
return;
}
let mut bodies: Vec<Vec<u8>> = Vec::new();
for (id, range) in ids.iter().zip(&ranges) {
let parsed = match parse_entries(range, false) {
Ok(parsed) => parsed,
Err(_) => {
tracing::warn!(id, "PROXY DLQ 重读协议异常,保留源 PEL");
return;
}
};
let [(entry_id, env)] = parsed.ok.as_slice() else {
tracing::warn!(id, "PROXY DLQ 重读没有唯一信封,保留源 PEL");
return;
};
if !parsed.bad.is_empty() || entry_id != id {
tracing::warn!(id, "PROXY DLQ 重读身份不一致,保留源 PEL");
return;
}
match serde_json::to_vec(env) {
Ok(body) => bodies.push(body),
Err(e) => {
tracing::warn!(id, category = ?e.classify(), "PROXY DLQ 信封编码失败,保留源 PEL");
return;
}
}
}
let mut apipe = redis::pipe();
for b in &bodies {
apipe
.cmd("XADD")
.arg(&dlq)
.arg("*")
.arg(DATA_FIELD)
.arg(b.as_slice());
}
match redis::aio::ConnectionLike::req_packed_commands(
&mut conn,
&apipe,
0,
bodies.len(),
)
.await
{
Ok(v) if all_pipeline_replies_ok(&v, bodies.len()) => {}
Ok(_) => {
tracing::warn!(?ids, "PROXY DLQ XADD 缺少完整有效的写入回复,保留源 PEL");
return;
}
Err(e) => {
tracing::warn!(?ids, kind = ?e.kind(), "PROXY DLQ XADD 未取得完整回复,保留源 PEL");
return;
}
}
tracing::error!(?ids, dlq = %dlq, "PROXY 毒消息转存回复通过,开始确认源 PEL");
ack(shared, ids).await;
}
}
}
fn all_pipeline_replies_ok(values: &[redis::Value], expected: usize) -> bool {
values.len() == expected
&& values.iter().all(|value| {
let id = match value {
redis::Value::BulkString(bytes) => match std::str::from_utf8(bytes) {
Ok(id) => id,
Err(_) => return false,
},
redis::Value::SimpleString(id) => id.as_str(),
_ => return false,
};
matches!(parse_stream_id(id), Some((ms, sequence)) if ms != 0 || sequence != 0)
})
}
fn base64_encode(bytes: &[u8]) -> String {
const T: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
let mut out = String::with_capacity(bytes.len().div_ceil(3) * 4);
for chunk in bytes.chunks(3) {
let b0 = chunk[0] as usize;
let b1 = chunk.get(1).copied().unwrap_or(0) as usize;
let b2 = chunk.get(2).copied().unwrap_or(0) as usize;
out.push(T[b0 >> 2] as char);
out.push(T[((b0 & 0b11) << 4) | (b1 >> 4)] as char);
out.push(if chunk.len() > 1 {
T[((b1 & 0b1111) << 2) | (b2 >> 6)] as char
} else {
'='
});
out.push(if chunk.len() > 2 {
T[b2 & 0b111111] as char
} else {
'='
});
}
out
}
async fn dispose_bad(shared: &ProxyShared, bad: &[BadEntry]) {
if bad.is_empty() {
return;
}
let ids: Vec<String> = bad.iter().map(|b| b.id.clone()).collect();
match shared.cfg.poison {
ProxyPoison::Drop => {
tracing::error!(
?ids,
"PROXY 不可解析 entry(无 data/非 Envelope),按 Drop 策略 XACK 丢弃"
);
}
ProxyPoison::Dlq => {
let dlq = format!("{}:dlq", shared.stream);
let mut conn = shared.client.conn();
let mut apipe = redis::pipe();
for b in bad {
let raw: Vec<serde_json::Value> = b
.raw
.iter()
.map(|(k, v)| {
serde_json::json!({
"field_utf8": String::from_utf8_lossy(k),
"field_b64": base64_encode(k),
"value_utf8": String::from_utf8_lossy(v),
"value_b64": base64_encode(v),
})
})
.collect();
let body = serde_json::to_vec(&serde_json::json!({
"_proxy_dlq": "unparseable",
"reason": b.reason,
"stream": shared.stream,
"group": shared.group,
"id": b.id,
"raw": raw,
}))
.unwrap_or_default();
apipe
.cmd("XADD")
.arg(&dlq)
.arg("*")
.arg(DATA_FIELD)
.arg(body);
}
match redis::aio::ConnectionLike::req_packed_commands(&mut conn, &apipe, 0, bad.len())
.await
{
Ok(v) if all_pipeline_replies_ok(&v, bad.len()) => {}
Ok(_) => {
tracing::warn!(?ids, "PROXY 不可解析条目转存缺少完整有效回复,保留源 PEL");
return;
}
Err(e) => {
tracing::warn!(?ids, kind = ?e.kind(), "PROXY 不可解析条目转存未取得完整回复,保留源 PEL");
return;
}
}
tracing::error!(?ids, dlq = %dlq, "PROXY 不可解析条目转存回复通过,开始确认源 PEL");
}
}
ack(shared, &ids).await;
}