use super::{EventBusConfig, Kafka, ProcessingLimits, TopologyMode};
use crate::{
BusEvent, EventBusError, backends::event_bus_backend::EventBusBackendConfig,
decoder::DecoderRegistry,
};
use bevy::prelude::{App, Message};
use std::any::Any;
use std::collections::{HashMap, HashSet};
use std::time::Duration;
#[derive(Clone, Debug)]
pub struct KafkaChannelCapacities {
pub message: usize,
pub commit: usize,
pub result: usize,
}
impl KafkaChannelCapacities {
pub fn new(message: usize, commit: usize, result: usize) -> Self {
Self {
message,
commit,
result,
}
}
pub fn message_capacity(mut self, capacity: usize) -> Self {
self.message = capacity;
self
}
pub fn commit_capacity(mut self, capacity: usize) -> Self {
self.commit = capacity;
self
}
pub fn result_capacity(mut self, capacity: usize) -> Self {
self.result = capacity;
self
}
}
impl Default for KafkaChannelCapacities {
fn default() -> Self {
Self {
message: 10_000,
commit: 2_048,
result: 1_024,
}
}
}
#[derive(Clone, Debug)]
pub struct KafkaConnectionConfig {
bootstrap_servers: String,
client_id: Option<String>,
timeout_ms: i32,
additional_config: HashMap<String, String>,
}
impl KafkaConnectionConfig {
pub fn new(bootstrap_servers: impl Into<String>) -> Self {
Self {
bootstrap_servers: bootstrap_servers.into(),
client_id: None,
timeout_ms: 10_000,
additional_config: HashMap::new(),
}
}
pub fn set_client_id(mut self, client_id: impl Into<String>) -> Self {
self.client_id = Some(client_id.into());
self
}
pub fn set_timeout_ms(mut self, timeout_ms: i32) -> Self {
self.timeout_ms = timeout_ms;
self
}
pub fn insert_additional_config<K, V>(mut self, key: K, value: V) -> Self
where
K: Into<String>,
V: Into<String>,
{
self.additional_config.insert(key.into(), value.into());
self
}
pub fn bootstrap_servers(&self) -> &str {
&self.bootstrap_servers
}
pub fn client_id(&self) -> Option<&str> {
self.client_id.as_deref()
}
pub fn timeout_ms(&self) -> i32 {
self.timeout_ms
}
pub fn additional_config(&self) -> &HashMap<String, String> {
&self.additional_config
}
}
impl Default for KafkaConnectionConfig {
fn default() -> Self {
Self::new("localhost:9092")
}
}
#[derive(Clone, Debug)]
pub struct KafkaTopologyEventBinding {
topics: Vec<String>,
register_fn: fn(&mut App, &[String]),
}
impl KafkaTopologyEventBinding {
pub fn new<T: BusEvent + Message>(topics: Vec<String>) -> Self {
Self {
topics,
register_fn: register_event_binding::<T>,
}
}
pub fn apply(&self, app: &mut App) {
(self.register_fn)(app, &self.topics);
}
pub fn topics(&self) -> &[String] {
&self.topics
}
}
fn register_event_binding<T: BusEvent + Message>(app: &mut App, topics: &[String]) {
App::add_message::<T>(app);
App::add_message::<EventBusError<T>>(app);
if !app.world().contains_resource::<DecoderRegistry>() {
app.world_mut().insert_resource(DecoderRegistry::new());
}
let mut registry = app.world_mut().resource_mut::<DecoderRegistry>();
for topic in topics {
registry.register_json_decoder::<T>(topic);
}
crate::writers::outbound_bridge::ensure_bridge::<T>(app, topics);
}
#[derive(Clone, Debug, Default)]
pub struct KafkaTopologyConfig {
topics: Vec<KafkaTopicSpec>,
consumer_groups: HashMap<String, KafkaConsumerGroupSpec>,
event_bindings: Vec<KafkaTopologyEventBinding>,
}
impl KafkaTopologyConfig {
pub fn new(
topics: Vec<KafkaTopicSpec>,
consumer_groups: HashMap<String, KafkaConsumerGroupSpec>,
event_bindings: Vec<KafkaTopologyEventBinding>,
) -> Self {
Self {
topics,
consumer_groups,
event_bindings,
}
}
pub fn builder() -> KafkaTopologyBuilder {
KafkaTopologyBuilder::default()
}
pub fn topics(&self) -> &[KafkaTopicSpec] {
&self.topics
}
pub fn consumer_groups(&self) -> &HashMap<String, KafkaConsumerGroupSpec> {
&self.consumer_groups
}
pub fn topic_names(&self) -> HashSet<String> {
self.topics.iter().map(|t| t.name.clone()).collect()
}
pub fn event_bindings(&self) -> &[KafkaTopologyEventBinding] {
&self.event_bindings
}
}
#[derive(Default)]
pub struct KafkaTopologyBuilder {
topics: Vec<KafkaTopicSpec>,
consumer_groups: HashMap<String, KafkaConsumerGroupSpec>,
event_bindings: Vec<KafkaTopologyEventBinding>,
}
impl KafkaTopologyBuilder {
pub fn add_topic(&mut self, topic: KafkaTopicSpec) -> &mut Self {
self.topics.push(topic);
self
}
pub fn add_topics<T: IntoIterator<Item = KafkaTopicSpec>>(&mut self, topics: T) -> &mut Self {
self.topics.extend(topics);
self
}
pub fn add_consumer_group(
&mut self,
id: impl Into<String>,
spec: KafkaConsumerGroupSpec,
) -> &mut Self {
self.consumer_groups.insert(id.into(), spec);
self
}
pub fn add_event<T: BusEvent + Message>(
&mut self,
topics: impl IntoIterator<Item = impl Into<String>>,
) -> &mut Self {
let topics_vec: Vec<String> = topics.into_iter().map(Into::into).collect();
self.event_bindings
.push(KafkaTopologyEventBinding::new::<T>(topics_vec));
self
}
pub fn add_event_single<T: BusEvent + Message>(
&mut self,
topic: impl Into<String>,
) -> &mut Self {
self.add_event::<T>([topic.into()])
}
pub fn build(self) -> KafkaTopologyConfig {
KafkaTopologyConfig::new(self.topics, self.consumer_groups, self.event_bindings)
}
}
#[derive(Clone, Debug)]
pub struct KafkaTopicSpec {
pub name: String,
pub partitions: Option<i32>,
pub replication: Option<i16>,
pub mode: TopologyMode,
}
impl KafkaTopicSpec {
pub fn new(name: impl Into<String>) -> Self {
Self {
name: name.into(),
partitions: None,
replication: None,
mode: TopologyMode::Provision,
}
}
pub fn partitions(mut self, partitions: i32) -> Self {
self.partitions = Some(partitions);
self
}
pub fn replication(mut self, replication: i16) -> Self {
self.replication = Some(replication);
self
}
pub fn mode(mut self, mode: TopologyMode) -> Self {
self.mode = mode;
self
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum KafkaInitialOffset {
Earliest,
Latest,
None,
}
#[derive(Clone, Debug)]
pub struct KafkaConsumerGroupSpec {
pub topics: Vec<String>,
pub manual_commits: bool,
pub initial_offset: KafkaInitialOffset,
pub mode: TopologyMode,
}
impl KafkaConsumerGroupSpec {
pub fn new<T: IntoIterator<Item = impl Into<String>>>(topics: T) -> Self {
Self {
topics: topics.into_iter().map(Into::into).collect(),
manual_commits: false,
initial_offset: KafkaInitialOffset::Latest,
mode: TopologyMode::Provision,
}
}
pub fn manual_commits(mut self, manual: bool) -> Self {
self.manual_commits = manual;
self
}
pub fn initial_offset(mut self, offset: KafkaInitialOffset) -> Self {
self.initial_offset = offset;
self
}
pub fn mode(mut self, mode: TopologyMode) -> Self {
self.mode = mode;
self
}
}
#[derive(Clone, Debug)]
pub struct KafkaBackendConfig {
pub connection: KafkaConnectionConfig,
pub topology: KafkaTopologyConfig,
pub consumer_lag_poll_interval: Duration,
pub channel_capacities: KafkaChannelCapacities,
}
impl KafkaBackendConfig {
pub fn new(
connection: KafkaConnectionConfig,
topology: KafkaTopologyConfig,
consumer_lag_poll_interval: Duration,
) -> Self {
Self {
connection,
topology,
consumer_lag_poll_interval,
channel_capacities: KafkaChannelCapacities::default(),
}
}
pub fn channel_capacities(mut self, capacities: KafkaChannelCapacities) -> Self {
self.channel_capacities = capacities;
self
}
pub fn get_channel_capacities(&self) -> &KafkaChannelCapacities {
&self.channel_capacities
}
}
impl EventBusBackendConfig for KafkaBackendConfig {
fn as_any(&self) -> &dyn Any {
self
}
}
#[derive(Clone, Debug)]
pub struct KafkaConsumerConfig {
consumer_group: String,
topics: Vec<String>,
auto_offset_reset: String,
enable_auto_commit: bool,
session_timeout: Duration,
max_poll_records: u32,
processing_limits: ProcessingLimits,
additional_config: HashMap<String, String>,
}
impl KafkaConsumerConfig {
pub fn new<I, T>(consumer_group: impl Into<String>, topics: I) -> Self
where
I: IntoIterator<Item = T>,
T: Into<String>,
{
Self {
consumer_group: consumer_group.into(),
topics: topics.into_iter().map(Into::into).collect(),
auto_offset_reset: "latest".to_string(),
enable_auto_commit: true,
session_timeout: Duration::from_millis(30_000),
max_poll_records: 500,
processing_limits: ProcessingLimits::default(),
additional_config: HashMap::new(),
}
}
pub fn consumer_group(mut self, group: impl Into<String>) -> Self {
self.consumer_group = group.into();
self
}
pub fn topics<I, T>(mut self, topics: I) -> Self
where
I: IntoIterator<Item = T>,
T: Into<String>,
{
self.topics = topics.into_iter().map(Into::into).collect();
self
}
pub fn auto_offset_reset(mut self, reset: impl Into<String>) -> Self {
self.auto_offset_reset = reset.into();
self
}
pub fn enable_auto_commit(mut self, enable: bool) -> Self {
self.enable_auto_commit = enable;
self
}
pub fn session_timeout(mut self, timeout: Duration) -> Self {
self.session_timeout = timeout;
self
}
pub fn max_poll_records(mut self, records: u32) -> Self {
self.max_poll_records = records;
self
}
pub fn processing_limits(mut self, limits: ProcessingLimits) -> Self {
self.processing_limits = limits;
self
}
pub fn additional_config<K, V>(mut self, key: K, value: V) -> Self
where
K: Into<String>,
V: Into<String>,
{
self.additional_config.insert(key.into(), value.into());
self
}
pub fn is_auto_commit_enabled(&self) -> bool {
self.enable_auto_commit
}
pub fn get_session_timeout(&self) -> Duration {
self.session_timeout
}
pub fn get_max_poll_records(&self) -> u32 {
self.max_poll_records
}
pub fn get_processing_limits(&self) -> &ProcessingLimits {
&self.processing_limits
}
pub fn get_additional_config(&self) -> &HashMap<String, String> {
&self.additional_config
}
pub fn get_auto_offset_reset(&self) -> &str {
&self.auto_offset_reset
}
pub fn get_consumer_group(&self) -> &str {
&self.consumer_group
}
}
impl EventBusConfig for KafkaConsumerConfig {
type Backend = Kafka;
fn topics(&self) -> &[String] {
&self.topics
}
fn config_id(&self) -> String {
format!("kafka_consumer_{}", self.consumer_group)
}
fn as_any(&self) -> &dyn std::any::Any {
self
}
}
#[derive(Clone, Debug)]
pub struct KafkaProducerConfig {
topics: Vec<String>,
acks: String,
retries: u32,
compression_type: String,
batch_size: u32,
linger_ms: u32,
request_timeout_ms: u32,
partition_key: Option<String>,
headers: HashMap<String, String>,
additional_config: HashMap<String, String>,
}
impl KafkaProducerConfig {
pub fn new<I, T>(topics: I) -> Self
where
I: IntoIterator<Item = T>,
T: Into<String>,
{
Self {
topics: topics.into_iter().map(Into::into).collect(),
acks: "1".to_string(), retries: 3,
compression_type: "none".to_string(),
batch_size: 16384,
linger_ms: 0,
request_timeout_ms: 30000,
partition_key: None,
headers: HashMap::new(),
additional_config: HashMap::new(),
}
}
pub fn topics<I, T>(mut self, topics: I) -> Self
where
I: IntoIterator<Item = T>,
T: Into<String>,
{
self.topics = topics.into_iter().map(Into::into).collect();
self
}
pub fn acks(mut self, acks: impl Into<String>) -> Self {
self.acks = acks.into();
self
}
pub fn retries(mut self, retries: u32) -> Self {
self.retries = retries;
self
}
pub fn compression_type(mut self, compression: impl Into<String>) -> Self {
self.compression_type = compression.into();
self
}
pub fn batch_size(mut self, size: u32) -> Self {
self.batch_size = size;
self
}
pub fn linger_ms(mut self, linger: u32) -> Self {
self.linger_ms = linger;
self
}
pub fn request_timeout_ms(mut self, timeout: u32) -> Self {
self.request_timeout_ms = timeout;
self
}
pub fn partition_key(mut self, key: impl Into<String>) -> Self {
self.partition_key = Some(key.into());
self
}
pub fn clear_partition_key(mut self) -> Self {
self.partition_key = None;
self
}
pub fn header(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
self.headers.insert(key.into(), value.into());
self
}
pub fn headers_map<I, K, V>(mut self, headers: I) -> Self
where
I: IntoIterator<Item = (K, V)>,
K: Into<String>,
V: Into<String>,
{
self.headers.clear();
for (key, value) in headers {
self.headers.insert(key.into(), value.into());
}
self
}
pub fn clear_headers(mut self) -> Self {
self.headers.clear();
self
}
pub fn additional_config<K, V>(mut self, key: K, value: V) -> Self
where
K: Into<String>,
V: Into<String>,
{
self.additional_config.insert(key.into(), value.into());
self
}
pub fn get_acks(&self) -> &str {
&self.acks
}
pub fn get_retries(&self) -> u32 {
self.retries
}
pub fn get_compression_type(&self) -> &str {
&self.compression_type
}
pub fn get_batch_size(&self) -> u32 {
self.batch_size
}
pub fn get_linger_ms(&self) -> u32 {
self.linger_ms
}
pub fn get_request_timeout_ms(&self) -> u32 {
self.request_timeout_ms
}
pub fn get_partition_key(&self) -> Option<&str> {
self.partition_key.as_deref()
}
pub fn get_headers(&self) -> &HashMap<String, String> {
&self.headers
}
pub fn get_additional_config(&self) -> &HashMap<String, String> {
&self.additional_config
}
}
impl Default for KafkaProducerConfig {
fn default() -> Self {
Self::new(Vec::<String>::new())
}
}
impl EventBusConfig for KafkaProducerConfig {
type Backend = Kafka;
fn topics(&self) -> &[String] {
&self.topics
}
fn config_id(&self) -> String {
"kafka_producer".to_string()
}
fn as_any(&self) -> &dyn std::any::Any {
self
}
}
#[derive(Debug, Clone)]
pub struct KafkaMessageMetadata {
pub topic: String,
pub partition: i32,
pub offset: i64,
pub timestamp: Option<i64>,
pub key: Option<String>,
pub headers: HashMap<String, String>,
}
pub struct UncommittedEvent<T> {
event: T,
metadata: KafkaMessageMetadata,
commit_fn: Option<Box<dyn FnOnce() -> Result<(), String> + Send + Sync>>,
}
impl<T> std::fmt::Debug for UncommittedEvent<T>
where
T: std::fmt::Debug,
{
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("UncommittedEvent")
.field("event", &self.event)
.field("metadata", &self.metadata)
.field("commit_fn", &self.commit_fn.is_some())
.finish()
}
}
impl<T> UncommittedEvent<T> {
pub fn new(
event: T,
metadata: KafkaMessageMetadata,
commit_fn: impl FnOnce() -> Result<(), String> + Send + Sync + 'static,
) -> Self {
Self {
event,
metadata,
commit_fn: Some(Box::new(commit_fn)),
}
}
pub fn event(&self) -> &T {
&self.event
}
pub fn metadata(&self) -> &KafkaMessageMetadata {
&self.metadata
}
pub fn commit(mut self) -> Result<(), String> {
if let Some(commit_fn) = self.commit_fn.take() {
commit_fn()
} else {
Err("Event already committed".to_string())
}
}
pub fn needs_commit(&self) -> bool {
self.commit_fn.is_some()
}
}
impl<T> std::ops::Deref for UncommittedEvent<T> {
type Target = T;
fn deref(&self) -> &Self::Target {
&self.event
}
}