use super::{
config::{SinkConfig, SourceConfig, TRANSACTIONAL_ID, validate_properties},
sink::KafkaSink,
source::KafkaSource,
};
use crate::error::{BoxError, Error, ensure};
use rdkafka::{ClientConfig, producer::FutureProducer};
use std::{
collections::HashMap,
fmt,
sync::{Arc, Mutex},
};
#[derive(Clone)]
pub struct Kafka {
shared: Arc<Shared>,
}
#[derive(Clone)]
struct Settings {
brokers: String,
properties: HashMap<String, String>,
producer_properties: HashMap<String, String>,
}
struct Shared {
settings: Settings,
producer: Mutex<Option<FutureProducer>>,
}
impl Kafka {
pub fn new(brokers: impl Into<String>) -> Self {
Self::from_settings(Settings {
brokers: brokers.into(),
properties: HashMap::new(),
producer_properties: HashMap::new(),
})
}
pub fn property(self, name: impl Into<String>, value: impl Into<String>) -> Self {
self.configure(|settings| {
settings.properties.insert(name.into(), value.into());
})
}
pub fn producer_property(self, name: impl Into<String>, value: impl Into<String>) -> Self {
self.configure(|settings| {
settings
.producer_properties
.insert(name.into(), value.into());
})
}
pub fn sasl_plain(self, username: impl Into<String>, password: impl Into<String>) -> Self {
self.configure(|settings| {
let properties = &mut settings.properties;
properties
.entry("security.protocol".into())
.or_insert_with(|| "SASL_SSL".into());
properties.insert("sasl.mechanism".into(), "PLAIN".into());
properties.insert("sasl.username".into(), username.into());
properties.insert("sasl.password".into(), password.into());
})
}
pub fn source<C: Default>(
&self,
group_id: impl Into<String>,
topics: impl IntoIterator<Item = impl Into<String>>,
) -> KafkaSource<C> {
self.source_with_codec(C::default(), group_id, topics)
}
pub fn source_with_codec<C>(
&self,
codec: C,
group_id: impl Into<String>,
topics: impl IntoIterator<Item = impl Into<String>>,
) -> KafkaSource<C> {
let topics = topics.into_iter().map(Into::into).collect();
KafkaSource::new(
self.clone(),
SourceConfig::new(group_id.into(), topics),
codec,
)
}
pub fn sink<C: Default>(&self, topic: impl Into<String>) -> KafkaSink<C> {
self.sink_with_codec(C::default(), topic)
}
pub fn sink_with_codec<C>(&self, codec: C, topic: impl Into<String>) -> KafkaSink<C> {
KafkaSink::new(self.clone(), SinkConfig::new(topic.into()), codec)
}
pub fn validate(&self) -> Result<(), Error> {
let settings = &self.shared.settings;
ensure!(
!settings.brokers.trim().is_empty(),
Error::config,
"Kafka brokers are required"
);
ensure!(
!settings.properties.contains_key(TRANSACTIONAL_ID)
&& !settings.producer_properties.contains_key(TRANSACTIONAL_ID),
Error::config,
"set the Kafka transactional ID with KafkaSink::transactional"
);
validate_properties(&settings.properties)?;
validate_properties(&settings.producer_properties)
}
pub(super) fn client_config(&self) -> ClientConfig {
let settings = &self.shared.settings;
let mut config = ClientConfig::new();
for (key, value) in &settings.properties {
config.set(key, value);
}
config.set("bootstrap.servers", &settings.brokers);
config
}
pub(super) fn producer_config(&self) -> ClientConfig {
let mut config = self.client_config();
for (key, value) in &self.shared.settings.producer_properties {
config.set(key, value);
}
config
}
pub(super) fn producer(&self) -> Result<FutureProducer, BoxError> {
let mut slot = self.shared.producer.lock().unwrap();
if let Some(producer) = &*slot {
return Ok(producer.clone());
}
self.validate()?;
let producer: FutureProducer = self.producer_config().create()?;
*slot = Some(producer.clone());
Ok(producer)
}
pub(super) fn created_producer(&self) -> Option<FutureProducer> {
self.shared.producer.lock().unwrap().clone()
}
fn configure(self, change: impl FnOnce(&mut Settings)) -> Self {
let mut settings = self.shared.settings.clone();
change(&mut settings);
Self::from_settings(settings)
}
fn from_settings(settings: Settings) -> Self {
Self {
shared: Arc::new(Shared {
settings,
producer: Mutex::new(None),
}),
}
}
}
impl fmt::Debug for Kafka {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let settings = &self.shared.settings;
let mut properties: Vec<_> = settings.properties.keys().collect();
properties.sort();
let mut producer_properties: Vec<_> = settings.producer_properties.keys().collect();
producer_properties.sort();
f.debug_struct("Kafka")
.field("brokers", &settings.brokers)
.field("properties", &properties)
.field("producer_properties", &producer_properties)
.finish_non_exhaustive()
}
}