use super::{
client::Kafka,
config::{TRANSACTIONAL_ID, TransactionalConfig},
progress::{KafkaConsumer, Partition, Progress, TransactionGate},
record::KafkaPublish,
sink::{KafkaPrepared, prepare, send},
source::KafkaMessage,
};
use crate::{
codec::{Decoder, Encoder},
error::{BoxError, Context, Error, causes, ensure},
sink::Sink,
transaction::{TransactionEntry, TransactionalSink},
};
use rdkafka::{
Offset, TopicPartitionList,
consumer::Consumer,
error::{KafkaError, KafkaResult},
message::Message,
producer::{BaseProducer, FutureProducer, Producer},
util::Timeout,
};
use std::{
collections::BTreeMap,
sync::{
Arc, Mutex,
atomic::{AtomicBool, Ordering},
},
time::Duration,
};
const COMMIT_ATTEMPTS: usize = 3;
struct State {
producer: tokio::sync::Mutex<Option<FutureProducer>>,
closed: AtomicBool,
}
pub struct KafkaTransactionalSink<C> {
config: Arc<Config>,
codec: Arc<C>,
state: Arc<State>,
}
struct Config {
kafka: Kafka,
sink: TransactionalConfig,
}
impl<C> Clone for KafkaTransactionalSink<C> {
fn clone(&self) -> Self {
Self {
config: self.config.clone(),
codec: self.codec.clone(),
state: self.state.clone(),
}
}
}
impl<C> KafkaTransactionalSink<C> {
pub(super) fn new(kafka: Kafka, sink: TransactionalConfig, codec: Arc<C>) -> Self {
Self {
config: Arc::new(Config { kafka, sink }),
codec,
state: Arc::new(State {
producer: tokio::sync::Mutex::new(None),
closed: AtomicBool::new(false),
}),
}
}
pub fn transaction_timeout(self, transaction_timeout: Duration) -> Self {
self.configure(|config| config.transaction_timeout = transaction_timeout)
}
pub fn close_timeout(self, close_timeout: Duration) -> Self {
self.configure(|config| config.close_timeout = close_timeout)
}
pub fn validate(&self) -> Result<(), Error> {
self.config.kafka.validate()?;
self.config.sink.validate()?;
Ok(())
}
fn configure(mut self, change: impl FnOnce(&mut TransactionalConfig)) -> Self {
let mut sink = self.config.sink.clone();
change(&mut sink);
self.config = Arc::new(Config {
kafka: self.config.kafka.clone(),
sink,
});
self
}
async fn transact(
&self,
outputs: Vec<KafkaPrepared>,
offsets: Option<SourceOffsets>,
) -> Result<(), BoxError> {
let config = self.config.clone();
let state = self.state.clone();
tokio::spawn(async move {
let mut slot = state.producer.lock().await;
ensure!(
!state.closed.load(Ordering::Acquire),
Error::closed,
"Kafka sink is closed"
);
if slot.is_none() {
*slot = Some(connect(config.clone()).await?);
}
if let Some(offsets) = &offsets {
offsets.ensure_assigned()?;
}
let producer = slot.clone().expect("connected producer");
let result = run(&producer, &config, &outputs, offsets).await;
if let Err(error) = &result
&& !recover(&producer, config.sink.transaction_timeout, error).await
{
let stale = slot.take();
tokio::task::spawn_blocking(move || drop(stale)).await?;
}
result
})
.await?
}
async fn verify_cluster(&self, consumer: Arc<KafkaConsumer>) -> Result<(), BoxError> {
let config = self.config.clone();
tokio::task::spawn_blocking(move || {
config.kafka.validate()?;
config.sink.validate()?;
let timeout = config.sink.transaction_timeout;
let sink: BaseProducer = config.kafka.producer_config().create()?;
let sink = sink
.client()
.fetch_cluster_id(timeout)
.context("Kafka sink brokers did not report their cluster ID")?;
let source = consumer
.client()
.fetch_cluster_id(timeout)
.context("Kafka source did not report its cluster ID")?;
ensure!(
source == sink,
Error::config,
"Kafka source cluster {source} differs from sink cluster {sink}; \
a transactional sink must connect to the source's cluster"
);
Ok(())
})
.await?
}
}
impl<C> Sink<KafkaPublish<C::Value>> for KafkaTransactionalSink<C>
where
C: Encoder<Value: Send + Sync + 'static>,
{
type Prepared = KafkaPrepared;
fn prepare(&self, record: KafkaPublish<C::Value>) -> Result<Self::Prepared, BoxError> {
prepare(&*self.codec, record)
}
async fn publish(&self, output: &Self::Prepared) -> Result<(), BoxError> {
self.transact(vec![output.clone()], None).await
}
async fn close(&self) -> Result<(), BoxError> {
self.state.closed.store(true, Ordering::Release);
let producer = self.state.producer.lock().await.take();
if let Some(producer) = producer {
let timeout = self.config.sink.close_timeout;
tokio::task::spawn_blocking(move || producer.flush(Timeout::After(timeout))).await??;
}
Ok(())
}
}
impl<C, D> TransactionalSink<KafkaMessage<D>, KafkaPublish<C::Value>> for KafkaTransactionalSink<C>
where
C: Encoder<Value: Send + Sync + 'static>,
D: Decoder<Value: Clone + Send + Sync + 'static>,
{
async fn verify_source(&self, delivery: &KafkaMessage<D>) -> Result<(), BoxError> {
self.verify_cluster(delivery.consumer.clone()).await
}
async fn commit(
&self,
batch: &[TransactionEntry<'_, KafkaMessage<D>, Self::Prepared>],
) -> Result<(), BoxError> {
let outputs = batch
.iter()
.flat_map(|entry| entry.outputs.iter().cloned())
.collect();
let offsets = SourceOffsets::new(batch.iter().map(|entry| entry.delivery))?;
self.transact(outputs, offsets).await
}
}
struct SourceOffsets {
consumer: Arc<KafkaConsumer>,
progress: Arc<Mutex<Progress>>,
transactions: TransactionGate,
deliveries: Vec<(u64, Partition)>,
next: BTreeMap<Partition, (u64, i64)>,
}
impl SourceOffsets {
fn new<'a, D: 'a>(
deliveries: impl IntoIterator<Item = &'a KafkaMessage<D>>,
) -> Result<Option<Self>, BoxError> {
let mut offsets: Option<Self> = None;
for delivery in deliveries {
let offsets = offsets.get_or_insert_with(|| Self {
consumer: delivery.consumer.clone(),
progress: delivery.progress.clone(),
transactions: delivery.transactions.clone(),
deliveries: Vec::new(),
next: BTreeMap::new(),
});
ensure!(
Arc::ptr_eq(&offsets.consumer, &delivery.consumer),
Error::msg,
"a Kafka transaction batch contains deliveries of different consumers"
);
let partition = (delivery.raw.topic().to_owned(), delivery.raw.partition());
let next = (delivery.generation, delivery.raw.offset() + 1);
offsets
.next
.entry(partition.clone())
.and_modify(|current| *current = (*current).max(next))
.or_insert(next);
offsets.deliveries.push((delivery.generation, partition));
}
Ok(offsets)
}
fn ensure_assigned(&self) -> Result<(), BoxError> {
if self.consumer.assignment_lost() {
self.progress.lock().unwrap().revoke_all();
return Err(Error::msg("Kafka assignment was lost").into());
}
let progress = self.progress.lock().unwrap();
for (generation, partition) in &self.deliveries {
ensure!(
progress.is_current(*generation, partition),
Error::msg,
"Kafka delivery belongs to a revoked assignment"
);
}
Ok(())
}
fn commit(&self, producer: &FutureProducer, timeout: Duration) -> Result<(), BoxError> {
let _transactions = self.transactions.lock().unwrap();
self.ensure_assigned()?;
let metadata = self
.consumer
.group_metadata()
.context("Kafka consumer has no group metadata")?;
let mut offsets = TopicPartitionList::new();
for ((topic, partition), (_, next)) in &self.next {
offsets.add_partition_offset(topic, *partition, Offset::Offset(*next))?;
}
producer.send_offsets_to_transaction(&offsets, &metadata, timeout)?;
commit(producer, timeout)?;
let mut progress = self.progress.lock().unwrap();
for (partition, (generation, next)) in &self.next {
progress.committed(*generation, partition, *next);
}
Ok(())
}
}
async fn connect(config: Arc<Config>) -> Result<FutureProducer, BoxError> {
config.kafka.validate()?;
config.sink.validate()?;
tokio::task::spawn_blocking(move || {
let producer: FutureProducer = config
.kafka
.producer_config()
.set(TRANSACTIONAL_ID, &config.sink.transactional_id)
.create()?;
producer.init_transactions(config.sink.transaction_timeout)?;
Ok::<_, BoxError>(producer)
})
.await?
}
async fn run(
producer: &FutureProducer,
config: &Config,
outputs: &[KafkaPrepared],
offsets: Option<SourceOffsets>,
) -> Result<(), BoxError> {
producer.begin_transaction()?;
for output in outputs {
send(producer, &config.sink.topic, output).await?;
}
let producer = producer.clone();
let timeout = config.sink.transaction_timeout;
tokio::task::spawn_blocking(move || match offsets {
Some(offsets) => offsets.commit(&producer, timeout),
None => Ok(commit(&producer, timeout)?),
})
.await?
}
fn commit(producer: &FutureProducer, timeout: Duration) -> KafkaResult<()> {
let mut attempt = 1;
loop {
match producer.commit_transaction(timeout) {
Err(KafkaError::Transaction(error))
if error.is_retriable() && attempt < COMMIT_ATTEMPTS =>
{
attempt += 1;
}
result => return result,
}
}
}
async fn recover(producer: &FutureProducer, timeout: Duration, error: &BoxError) -> bool {
let fatal = causes(error.as_ref())
.filter_map(|cause| cause.downcast_ref::<KafkaError>())
.any(|error| matches!(error, KafkaError::Transaction(error) if error.is_fatal()));
if fatal {
return false;
}
let producer = producer.clone();
tokio::task::spawn_blocking(move || producer.abort_transaction(timeout))
.await
.is_ok_and(|result| result.is_ok())
}