use std::{collections::BTreeMap, fmt, sync::Arc, time::Duration};
use rdkafka::{
ClientConfig,
consumer::Consumer,
producer::{FutureProducer, Producer},
};
use super::{BoxConnectionFuture, Close, Connect, ConnectionError, ConnectionSpec, Verify};
pub use rdkafka::consumer::{Consumer as KafkaConsumerApi, StreamConsumer as KafkaStreamConsumer};
pub use rdkafka::producer::FutureProducer as KafkaProducer;
pub type KafkaConsumer = Arc<KafkaStreamConsumer>;
pub type KafkaProducerConnector = KafkaConnector;
#[derive(Clone)]
#[must_use = "连接器配置需要传给 connect 才会生效"]
pub struct KafkaConnector {
name: String,
enabled: bool,
brokers: String,
client_id: Option<String>,
metadata_timeout: Duration,
flush_timeout: Duration,
properties: BTreeMap<String, String>,
}
impl KafkaConnector {
pub fn new(brokers: impl Into<String>) -> Self {
Self {
name: "kafka".to_owned(),
enabled: true,
brokers: brokers.into(),
client_id: None,
metadata_timeout: Duration::from_secs(5),
flush_timeout: Duration::from_secs(10),
properties: BTreeMap::new(),
}
}
pub fn with_name(mut self, name: impl Into<String>) -> Self {
self.name = name.into();
self
}
pub const fn with_enabled(mut self, enabled: bool) -> Self {
self.enabled = enabled;
self
}
pub fn with_client_id(mut self, client_id: impl Into<String>) -> Self {
self.client_id = Some(client_id.into());
self
}
pub const fn with_metadata_timeout(mut self, timeout: Duration) -> Self {
self.metadata_timeout = timeout;
self
}
pub const fn with_flush_timeout(mut self, timeout: Duration) -> Self {
self.flush_timeout = timeout;
self
}
pub fn with_property(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
self.properties.insert(key.into(), value.into());
self
}
}
impl fmt::Debug for KafkaConnector {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("KafkaConnector")
.field("name", &self.name)
.field("enabled", &self.enabled)
.field("brokers", &"[已隐藏]")
.field("client_id", &self.client_id)
.field("property_keys", &self.properties.keys().collect::<Vec<_>>())
.field("metadata_timeout", &self.metadata_timeout)
.field("flush_timeout", &self.flush_timeout)
.finish()
}
}
impl ConnectionSpec for KafkaConnector {
type Connection = KafkaProducer;
type Error = ConnectionError;
fn name(&self) -> &str {
&self.name
}
fn enabled(&self) -> bool {
self.enabled
}
}
impl Connect for KafkaConnector {
fn connect(&self) -> BoxConnectionFuture<'_, Self::Connection, Self::Error> {
Box::pin(async move {
validate_common_config(&self.brokers, self.metadata_timeout)?;
if self.flush_timeout.is_zero() {
return Err(ConnectionError::InvalidConfig {
connector: "Kafka",
message: "生产者刷新超时必须大于 0",
});
}
let mut config = ClientConfig::new();
config.set("bootstrap.servers", &self.brokers);
if let Some(client_id) = &self.client_id {
config.set("client.id", client_id);
}
for (key, value) in &self.properties {
config.set(key, value);
}
config.create::<FutureProducer>().map_err(Into::into)
})
}
}
impl Verify for KafkaConnector {
fn verify<'a>(
&'a self,
connection: &'a Self::Connection,
) -> BoxConnectionFuture<'a, (), Self::Error> {
Box::pin(async move {
let producer = connection.clone();
let timeout = self.metadata_timeout;
tokio::task::spawn_blocking(move || {
producer.client().fetch_metadata(None, timeout).map(|_| ())
})
.await??;
Ok(())
})
}
}
impl Close for KafkaConnector {
fn close(&self, connection: Self::Connection) -> BoxConnectionFuture<'_, (), Self::Error> {
Box::pin(async move {
let timeout = self.flush_timeout;
tokio::task::spawn_blocking(move || {
connection.flush(timeout)?;
drop(connection);
Ok::<(), rdkafka::error::KafkaError>(())
})
.await??;
Ok(())
})
}
}
#[derive(Clone)]
#[must_use = "连接器配置需要传给 connect 才会生效"]
pub struct KafkaConsumerConnector {
name: String,
enabled: bool,
brokers: String,
group_id: String,
client_id: Option<String>,
metadata_timeout: Duration,
properties: BTreeMap<String, String>,
}
impl KafkaConsumerConnector {
pub fn new(brokers: impl Into<String>, group_id: impl Into<String>) -> Self {
Self {
name: "kafka-consumer".to_owned(),
enabled: true,
brokers: brokers.into(),
group_id: group_id.into(),
client_id: None,
metadata_timeout: Duration::from_secs(5),
properties: BTreeMap::new(),
}
}
pub fn with_name(mut self, name: impl Into<String>) -> Self {
self.name = name.into();
self
}
pub const fn with_enabled(mut self, enabled: bool) -> Self {
self.enabled = enabled;
self
}
pub fn with_client_id(mut self, client_id: impl Into<String>) -> Self {
self.client_id = Some(client_id.into());
self
}
pub const fn with_metadata_timeout(mut self, timeout: Duration) -> Self {
self.metadata_timeout = timeout;
self
}
pub fn with_property(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
self.properties.insert(key.into(), value.into());
self
}
}
impl fmt::Debug for KafkaConsumerConnector {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("KafkaConsumerConnector")
.field("name", &self.name)
.field("enabled", &self.enabled)
.field("brokers", &"[已隐藏]")
.field("group_id", &self.group_id)
.field("client_id", &self.client_id)
.field("property_keys", &self.properties.keys().collect::<Vec<_>>())
.field("metadata_timeout", &self.metadata_timeout)
.finish()
}
}
impl ConnectionSpec for KafkaConsumerConnector {
type Connection = KafkaConsumer;
type Error = ConnectionError;
fn name(&self) -> &str {
&self.name
}
fn enabled(&self) -> bool {
self.enabled
}
}
impl Connect for KafkaConsumerConnector {
fn connect(&self) -> BoxConnectionFuture<'_, Self::Connection, Self::Error> {
Box::pin(async move {
validate_common_config(&self.brokers, self.metadata_timeout)?;
if self.group_id.trim().is_empty() {
return Err(ConnectionError::InvalidConfig {
connector: "Kafka",
message: "消费者组 ID 不能为空",
});
}
let mut config = ClientConfig::new();
config
.set("bootstrap.servers", &self.brokers)
.set("group.id", &self.group_id);
if let Some(client_id) = &self.client_id {
config.set("client.id", client_id);
}
for (key, value) in &self.properties {
config.set(key, value);
}
config
.create::<KafkaStreamConsumer>()
.map(Arc::new)
.map_err(Into::into)
})
}
}
fn validate_common_config(
brokers: &str,
metadata_timeout: Duration,
) -> Result<(), ConnectionError> {
if brokers.trim().is_empty() {
return Err(ConnectionError::InvalidConfig {
connector: "Kafka",
message: "代理服务器地址不能为空",
});
}
if metadata_timeout.is_zero() {
return Err(ConnectionError::InvalidConfig {
connector: "Kafka",
message: "元数据查询超时必须大于 0",
});
}
Ok(())
}
impl Verify for KafkaConsumerConnector {
fn verify<'a>(
&'a self,
connection: &'a Self::Connection,
) -> BoxConnectionFuture<'a, (), Self::Error> {
Box::pin(async move {
let consumer = connection.clone();
let timeout = self.metadata_timeout;
tokio::task::spawn_blocking(move || consumer.fetch_metadata(None, timeout).map(|_| ()))
.await??;
Ok(())
})
}
}
impl Close for KafkaConsumerConnector {
fn close(&self, connection: Self::Connection) -> BoxConnectionFuture<'_, (), Self::Error> {
Box::pin(async move {
connection.unsubscribe();
drop(connection);
Ok(())
})
}
}