use super::{
client::{Pulsar, partition_topics},
config::SinkConfig,
resend::Resender,
sink::PulsarPrepared,
};
use crate::error::BoxError;
use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64};
use magnetar::{
java_string_hash,
proto::{pb::KeyValue, pb::MessageMetadata, producer::OutgoingMessage},
runtime_tokio::Producer,
};
use std::{
future::Future,
sync::atomic::{AtomicUsize, Ordering},
};
pub(super) struct Producers {
_pulsar: Pulsar,
partitions: Vec<Partition>,
next: AtomicUsize,
}
pub(super) struct Partition {
producer: Producer,
resender: Resender<Producer>,
}
impl Producers {
pub(super) async fn connect(pulsar: &Pulsar, config: &SinkConfig) -> Result<Self, BoxError> {
config.validate()?;
let client = pulsar.client().await?;
let mut partitions = Vec::new();
for topic in partition_topics(client, &config.topic).await? {
let mut builder = client.producer(&topic);
if let Some(name) = &config.producer_name {
builder = builder.name(name);
}
let producer = builder.create().await?;
partitions.push(Partition {
resender: Resender::spawn(producer.clone(), config.send_retry.clone()),
producer,
});
}
Ok(Self {
_pulsar: pulsar.clone(),
partitions,
next: AtomicUsize::new(0),
})
}
pub(super) fn route(&self, output: &PulsarPrepared) -> &Partition {
let count = self.partitions.len();
let index = match output.key.as_deref() {
Some(key) if !key.is_empty() => java_string_hash(&BASE64.encode(key)) as usize % count,
_ => self.next.fetch_add(1, Ordering::Relaxed) % count,
};
&self.partitions[index]
}
pub(super) async fn close(self) -> Result<(), BoxError> {
let mut result = Ok(());
for Partition { producer, resender } in self.partitions {
drop(resender);
if let Err(error) = producer.close().await {
result = Err(error.into());
}
}
result
}
}
impl Partition {
pub(super) fn enqueue(
&self,
output: &PulsarPrepared,
) -> impl Future<Output = Result<(), BoxError>> + Send + 'static {
self.resender.send(outgoing(output))
}
}
fn outgoing(output: &PulsarPrepared) -> OutgoingMessage {
let payload = output.value.clone().unwrap_or_default();
let metadata = MessageMetadata {
partition_key: output.key.as_deref().map(|key| BASE64.encode(key)),
partition_key_b64_encoded: output.key.as_ref().map(|_| true),
ordering_key: output.ordering_key.clone().map(Into::into),
event_time: output.event_time,
null_value: output.value.is_none().then_some(true),
properties: output
.properties
.iter()
.map(|(key, value)| KeyValue {
key: key.clone(),
value: value.clone(),
})
.collect(),
..MessageMetadata::default()
};
OutgoingMessage {
uncompressed_size: u32::try_from(payload.len()).unwrap_or(u32::MAX),
payload: payload.into(),
metadata,
num_messages: 1,
txn_id: None,
source_message_id: None,
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn binary_keys_use_pulsars_base64_representation() {
let output = PulsarPrepared {
key: Some(vec![0, 1, 2, 253, 254, 255]),
..PulsarPrepared::default()
};
let message = outgoing(&output);
assert_eq!(message.metadata.partition_key.as_deref(), Some("AAEC/f7/"));
assert_eq!(message.metadata.partition_key_b64_encoded, Some(true));
}
#[test]
fn a_null_value_is_marked_and_sent_as_an_empty_payload() {
let tombstone = outgoing(&PulsarPrepared::default());
assert!(tombstone.payload.is_empty());
assert_eq!(tombstone.metadata.null_value, Some(true));
let empty = outgoing(&PulsarPrepared {
value: Some(Vec::new()),
..PulsarPrepared::default()
});
assert!(empty.payload.is_empty());
assert_eq!(empty.metadata.null_value, None);
}
}