use super::{client::Pulsar, config::SinkConfig, producer::Producers, record::PulsarPublish};
use crate::{
adapters::pending::PendingLimit,
codec::Encoder,
error::{BoxError, Error, ensure},
retry::RetryPolicy,
sink::{Completion, Sink},
};
use std::{
collections::HashMap,
sync::{
Arc,
atomic::{AtomicBool, Ordering},
},
};
use tokio::sync::{OwnedRwLockReadGuard, RwLock};
#[derive(Clone, Debug, Default)]
pub struct PulsarPrepared {
pub value: Option<Vec<u8>>,
pub properties: HashMap<String, String>,
pub key: Option<Vec<u8>>,
pub ordering_key: Option<Vec<u8>>,
pub event_time: Option<u64>,
}
pub(super) struct Connection {
pulsar: Pulsar,
config: SinkConfig,
producers: Arc<RwLock<Option<Producers>>>,
closed: AtomicBool,
}
impl Connection {
pub(super) fn new(pulsar: Pulsar, config: SinkConfig) -> Self {
Self {
pulsar,
config,
producers: Arc::new(RwLock::new(None)),
closed: AtomicBool::new(false),
}
}
pub(super) async fn producers(
&self,
) -> Result<OwnedRwLockReadGuard<Option<Producers>, Producers>, BoxError> {
loop {
let producers = self.producers.clone().read_owned().await;
ensure!(
!self.closed.load(Ordering::Acquire),
Error::closed,
"Pulsar sink is closed"
);
if let Ok(producers) = OwnedRwLockReadGuard::try_map(producers, Option::as_ref) {
return Ok(producers);
}
let mut producers = self.producers.write().await;
ensure!(
!self.closed.load(Ordering::Acquire),
Error::closed,
"Pulsar sink is closed"
);
if producers.is_none() {
*producers = Some(Producers::connect(&self.pulsar, &self.config).await?);
}
}
}
pub(super) async fn close(&self) -> Result<(), BoxError> {
self.closed.store(true, Ordering::Release);
let producers = self.producers.write().await.take();
match producers {
Some(producers) => producers.close().await,
None => Ok(()),
}
}
}
pub struct PulsarSink<C> {
connection: Connection,
codec: Arc<C>,
pending: PendingLimit,
}
impl<C> PulsarSink<C> {
pub(super) fn new(pulsar: Pulsar, config: SinkConfig, codec: C) -> Self {
Self {
pending: PendingLimit::new(config.max_pending, "Pulsar"),
connection: Connection::new(pulsar, config),
codec: Arc::new(codec),
}
}
pub fn producer_name(mut self, name: impl Into<String>) -> Self {
self.connection.config.producer_name = Some(name.into());
self
}
pub fn max_pending(mut self, max_pending: usize) -> Self {
self.connection.config.max_pending = max_pending;
self.pending = PendingLimit::new(max_pending, "Pulsar");
self
}
pub fn send_retry(mut self, send_retry: RetryPolicy) -> Self {
self.connection.config.send_retry = send_retry;
self
}
pub fn validate(&self) -> Result<(), Error> {
self.connection.pulsar.validate()?;
self.connection.config.validate()
}
}
impl<C: Encoder<Value: Send + Sync + 'static>> Sink<PulsarPublish<C::Value>> for PulsarSink<C> {
type Prepared = PulsarPrepared;
fn prepare(&self, output: PulsarPublish<C::Value>) -> Result<Self::Prepared, BoxError> {
prepare(&*self.codec, output)
}
async fn publish(&self, output: &Self::Prepared) -> Result<(), BoxError> {
self.submit(output).await?.wait().await
}
async fn submit(&self, output: &Self::Prepared) -> Result<Completion, BoxError> {
let producers = self.connection.producers().await?;
let permit = self.pending.acquire().await?;
let receipt = producers.route(output).enqueue(output);
Ok(Completion::pending(async move {
let _held = (producers, permit);
receipt.await
}))
}
async fn close(&self) -> Result<(), BoxError> {
self.pending.close();
self.connection.close().await
}
}
pub(super) fn prepare<C: Encoder>(
codec: &C,
output: PulsarPublish<C::Value>,
) -> Result<PulsarPrepared, BoxError> {
Ok(PulsarPrepared {
value: output
.value
.as_ref()
.map(|value| codec.encode(value))
.transpose()?,
properties: output.properties,
key: output.key,
ordering_key: output.ordering_key,
event_time: output.event_time,
})
}