use bytes::Bytes;
use redis_tower_commands::{XAck, XAutoClaim, XGroupCreate, XReadGroup};
use redis_tower_core::{RedisConnection, RedisError};
#[derive(Debug, Clone)]
pub struct StreamMessage {
pub stream: String,
pub id: String,
pub fields: Vec<(String, Bytes)>,
}
#[derive(Debug, Clone)]
pub struct ConsumerConfig {
pub batch_size: u64,
pub block_ms: Option<u64>,
pub auto_ack: bool,
pub claim_idle_ms: Option<u64>,
pub create_group: bool,
}
impl Default for ConsumerConfig {
fn default() -> Self {
Self {
batch_size: 10,
block_ms: Some(5000),
auto_ack: true,
claim_idle_ms: None,
create_group: true,
}
}
}
pub struct StreamConsumer {
group: String,
consumer: String,
streams: Vec<String>,
config: ConsumerConfig,
}
impl StreamConsumer {
pub fn new(
group: impl Into<String>,
consumer: impl Into<String>,
streams: impl IntoIterator<Item = impl Into<String>>,
) -> Self {
Self {
group: group.into(),
consumer: consumer.into(),
streams: streams.into_iter().map(Into::into).collect(),
config: ConsumerConfig::default(),
}
}
pub fn config(mut self, config: ConsumerConfig) -> Self {
self.config = config;
self
}
pub fn into_stream(
self,
mut conn: RedisConnection,
) -> impl futures::Stream<Item = Result<StreamMessage, RedisError>> {
let group = self.group;
let consumer = self.consumer;
let streams = self.streams;
let config = self.config;
async_stream::try_stream! {
if config.create_group {
for stream_key in &streams {
let cmd = XGroupCreate::new(stream_key, &group, "0").mkstream();
match conn.execute(cmd).await {
Ok(()) => {}
Err(RedisError::Redis(ref msg)) if msg.contains("BUSYGROUP") => {
}
Err(e) => Err(e)?,
}
}
}
if let Some(idle_ms) = config.claim_idle_ms {
for stream_key in &streams {
let mut start = "0-0".to_string();
loop {
let cmd = XAutoClaim::new(
stream_key,
&group,
&consumer,
idle_ms,
&start,
)
.count(config.batch_size);
let result = conn.execute(cmd).await?;
for entry in result.entries {
let msg = StreamMessage {
stream: stream_key.clone(),
id: entry.id.clone(),
fields: entry.fields,
};
if config.auto_ack {
let ack = XAck::new(stream_key, &group, &msg.id);
conn.execute(ack).await?;
}
yield msg;
}
if result.next_start_id == "0-0" {
break;
}
start = result.next_start_id;
}
}
}
loop {
let cmd = build_xreadgroup(&group, &consumer, &streams, "0", config.batch_size, None);
let response = conn.execute(cmd).await?;
let mut got_any = false;
for (stream_key, entries) in response {
for entry in entries {
got_any = true;
let msg = StreamMessage {
stream: stream_key.clone(),
id: entry.id.clone(),
fields: entry.fields,
};
if config.auto_ack {
let ack = XAck::new(&stream_key, &group, &msg.id);
conn.execute(ack).await?;
}
yield msg;
}
}
if !got_any {
break;
}
}
loop {
let cmd = build_xreadgroup(
&group,
&consumer,
&streams,
">",
config.batch_size,
config.block_ms,
);
let response = conn.execute(cmd).await?;
for (stream_key, entries) in response {
for entry in entries {
let msg = StreamMessage {
stream: stream_key.clone(),
id: entry.id.clone(),
fields: entry.fields,
};
if config.auto_ack {
let ack = XAck::new(&stream_key, &group, &msg.id);
conn.execute(ack).await?;
}
yield msg;
}
}
}
}
}
}
fn build_xreadgroup(
group: &str,
consumer: &str,
streams: &[String],
id: &str,
count: u64,
block_ms: Option<u64>,
) -> XReadGroup {
assert!(!streams.is_empty(), "at least one stream key is required");
let mut cmd = XReadGroup::new(group, consumer, &streams[0]);
for key in &streams[1..] {
cmd = cmd.stream(key, ">");
}
cmd = cmd.with_id(id).count(count);
if let Some(ms) = block_ms {
cmd = cmd.block(ms);
}
cmd
}