use std::pin::Pin;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::Arc;
use std::task::{Context, Poll};
use std::time::Duration;
use futures_core::Stream;
use tokio::sync::{mpsc, RwLock};
use tracing::warn;
use crate::connection::AeroConnection;
use crate::error::AeroError;
use crate::protocol::Record;
#[derive(Debug, Clone)]
pub struct ConsumerConfig {
pub bootstrap_servers: Vec<String>,
pub topic: String,
pub partition: u32,
pub initial_offset: u64,
pub max_bytes: u32,
pub max_wait_ms: u32,
pub token: Option<String>,
pub connect_timeout: Duration,
pub backoff_on_empty: Duration,
}
impl Default for ConsumerConfig {
fn default() -> Self {
Self {
bootstrap_servers: vec!["127.0.0.1:9091".to_string()],
topic: String::new(),
partition: 0,
initial_offset: 0,
max_bytes: 1024 * 1024,
max_wait_ms: 500,
token: None,
connect_timeout: Duration::from_secs(5),
backoff_on_empty: Duration::from_millis(50),
}
}
}
impl ConsumerConfig {
pub fn builder() -> ConsumerConfigBuilder {
ConsumerConfigBuilder::default()
}
}
#[derive(Default)]
pub struct ConsumerConfigBuilder {
config: ConsumerConfig,
}
impl ConsumerConfigBuilder {
pub fn bootstrap_servers(mut self, servers: Vec<String>) -> Self {
self.config.bootstrap_servers = servers;
self
}
pub fn bootstrap_server(mut self, server: impl Into<String>) -> Self {
self.config.bootstrap_servers = vec![server.into()];
self
}
pub fn topic(mut self, topic: impl Into<String>) -> Self {
self.config.topic = topic.into();
self
}
pub fn partition(mut self, partition: u32) -> Self {
self.config.partition = partition;
self
}
pub fn initial_offset(mut self, offset: u64) -> Self {
self.config.initial_offset = offset;
self
}
pub fn max_bytes(mut self, max_bytes: u32) -> Self {
self.config.max_bytes = max_bytes;
self
}
pub fn max_wait_ms(mut self, max_wait_ms: u32) -> Self {
self.config.max_wait_ms = max_wait_ms;
self
}
pub fn token(mut self, token: impl Into<String>) -> Self {
self.config.token = Some(token.into());
self
}
pub fn connect_timeout(mut self, timeout: Duration) -> Self {
self.config.connect_timeout = timeout;
self
}
pub fn build(self) -> Result<ConsumerConfig, AeroError> {
if self.config.bootstrap_servers.is_empty() {
return Err(AeroError::InvalidConfiguration(
"bootstrap_servers cannot be empty".into(),
));
}
if self.config.topic.is_empty() {
return Err(AeroError::InvalidConfiguration("topic cannot be empty".into()));
}
Ok(self.config)
}
}
pub struct RecordStream {
receiver: mpsc::Receiver<Result<Record, AeroError>>,
worker_handle: tokio::task::JoinHandle<()>,
}
impl Drop for RecordStream {
fn drop(&mut self) {
self.worker_handle.abort();
}
}
impl Stream for RecordStream {
type Item = Result<Record, AeroError>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
self.receiver.poll_recv(cx)
}
}
#[derive(Clone)]
pub struct AeroConsumer {
config: Arc<ConsumerConfig>,
connection: Arc<RwLock<Option<AeroConnection>>>,
current_offset: Arc<AtomicU64>,
}
impl AeroConsumer {
pub fn new(config: ConsumerConfig) -> Self {
let initial_offset = config.initial_offset;
Self {
config: Arc::new(config),
connection: Arc::new(RwLock::new(None)),
current_offset: Arc::new(AtomicU64::new(initial_offset)),
}
}
pub fn offset(&self) -> u64 {
self.current_offset.load(Ordering::Acquire)
}
pub fn seek(&self, offset: u64) {
self.current_offset.store(offset, Ordering::Release);
}
async fn get_or_connect(&self) -> Result<AeroConnection, AeroError> {
{
let read = self.connection.read().await;
if let Some(conn) = &*read {
if conn.is_alive() {
return Ok(conn.clone());
}
}
}
let mut write = self.connection.write().await;
if let Some(conn) = &*write {
if conn.is_alive() {
return Ok(conn.clone());
}
}
let mut last_err = AeroError::Disconnected;
for server in &self.config.bootstrap_servers {
match AeroConnection::connect(
server,
self.config.token.as_deref(),
self.config.connect_timeout,
)
.await
{
Ok(conn) => {
*write = Some(conn.clone());
return Ok(conn);
}
Err(e) => {
warn!(broker = %server, error = %e, "Failed connecting to broker");
last_err = e;
}
}
}
Err(last_err)
}
pub async fn fetch_multi(
&self,
topic: &str,
partition: u32,
start_offset: u64,
max_bytes: u32,
max_wait_ms: u32,
) -> Result<Vec<Record>, AeroError> {
let conn = self.get_or_connect().await?;
match conn
.fetch_multi(topic, partition, start_offset, max_bytes, max_wait_ms)
.await
{
Ok(records) => Ok(records),
Err(e) => {
conn.mark_broken();
Err(e)
}
}
}
pub async fn poll_next_batch(&self) -> Result<Vec<Record>, AeroError> {
let offset = self.offset();
let records = self
.fetch_multi(
&self.config.topic,
self.config.partition,
offset,
self.config.max_bytes,
self.config.max_wait_ms,
)
.await?;
if let Some(last) = records.last() {
self.seek(last.offset + 1);
}
Ok(records)
}
pub fn stream(&self) -> RecordStream {
let (tx, rx) = mpsc::channel(256);
let consumer = self.clone();
let worker_handle = tokio::spawn(async move {
loop {
match consumer.poll_next_batch().await {
Ok(records) => {
if records.is_empty() {
tokio::time::sleep(consumer.config.backoff_on_empty).await;
continue;
}
for record in records {
if tx.send(Ok(record)).await.is_err() {
return; }
}
}
Err(err) => {
let is_fatal = matches!(&err, AeroError::AuthFailed(_));
if tx.send(Err(err)).await.is_err() || is_fatal {
return;
}
tokio::time::sleep(Duration::from_millis(200)).await;
}
}
}
});
RecordStream {
receiver: rx,
worker_handle,
}
}
}