beavers 0.0.2

Typed message processing for Kafka, Pulsar, RabbitMQ, SQS, Azure Service Bus, and HTTP with explicit delivery semantics
Documentation
use super::{
    client::{Pulsar, partition_topics},
    config::SinkConfig,
    resend::Resender,
    sink::PulsarPrepared,
};
use crate::error::BoxError;
use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64};
use magnetar::{
    java_string_hash,
    proto::{pb::KeyValue, pb::MessageMetadata, producer::OutgoingMessage},
    runtime_tokio::Producer,
};
use std::{
    future::Future,
    sync::atomic::{AtomicUsize, Ordering},
};

/// One producer per partition of the sink topic, or a single producer for a
/// non-partitioned topic, on the shared client.
pub(super) struct Producers {
    _pulsar: Pulsar,
    partitions: Vec<Partition>,
    /// Round-robin position for keyless messages.
    next: AtomicUsize,
}

pub(super) struct Partition {
    producer: Producer,
    resender: Resender<Producer>,
}

impl Producers {
    pub(super) async fn connect(pulsar: &Pulsar, config: &SinkConfig) -> Result<Self, BoxError> {
        config.validate()?;
        let client = pulsar.client().await?;
        let mut partitions = Vec::new();
        for topic in partition_topics(client, &config.topic).await? {
            let mut builder = client.producer(&topic);
            if let Some(name) = &config.producer_name {
                builder = builder.name(name);
            }
            let producer = builder.create().await?;
            partitions.push(Partition {
                resender: Resender::spawn(producer.clone(), config.send_retry.clone()),
                producer,
            });
        }
        Ok(Self {
            _pulsar: pulsar.clone(),
            partitions,
            next: AtomicUsize::new(0),
        })
    }

    /// Picks the partition for `output` the way Pulsar's Java client does by
    /// default: a keyed message goes to the Java `String.hashCode` of its wire
    /// key modulo the partition count, and keyless messages rotate.
    pub(super) fn route(&self, output: &PulsarPrepared) -> &Partition {
        let count = self.partitions.len();
        let index = match output.key.as_deref() {
            Some(key) if !key.is_empty() => java_string_hash(&BASE64.encode(key)) as usize % count,
            _ => self.next.fetch_add(1, Ordering::Relaxed) % count,
        };
        &self.partitions[index]
    }

    pub(super) async fn close(self) -> Result<(), BoxError> {
        let mut result = Ok(());
        for Partition { producer, resender } in self.partitions {
            drop(resender);
            if let Err(error) = producer.close().await {
                result = Err(error.into());
            }
        }
        result
    }
}

impl Partition {
    /// Queues `output` on the producer. The returned future resolves with the
    /// broker receipt, after sending the message again if the broker rejects
    /// it.
    pub(super) fn enqueue(
        &self,
        output: &PulsarPrepared,
    ) -> impl Future<Output = Result<(), BoxError>> + Send + 'static {
        self.resender.send(outgoing(output))
    }
}

fn outgoing(output: &PulsarPrepared) -> OutgoingMessage {
    let payload = output.value.clone().unwrap_or_default();
    let metadata = MessageMetadata {
        // Keys are arbitrary bytes, so they are always sent in Pulsar's
        // base64 representation.
        partition_key: output.key.as_deref().map(|key| BASE64.encode(key)),
        partition_key_b64_encoded: output.key.as_ref().map(|_| true),
        ordering_key: output.ordering_key.clone().map(Into::into),
        event_time: output.event_time,
        null_value: output.value.is_none().then_some(true),
        properties: output
            .properties
            .iter()
            .map(|(key, value)| KeyValue {
                key: key.clone(),
                value: value.clone(),
            })
            .collect(),
        ..MessageMetadata::default()
    };
    OutgoingMessage {
        uncompressed_size: u32::try_from(payload.len()).unwrap_or(u32::MAX),
        payload: payload.into(),
        metadata,
        num_messages: 1,
        txn_id: None,
        source_message_id: None,
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn binary_keys_use_pulsars_base64_representation() {
        let output = PulsarPrepared {
            key: Some(vec![0, 1, 2, 253, 254, 255]),
            ..PulsarPrepared::default()
        };
        let message = outgoing(&output);
        assert_eq!(message.metadata.partition_key.as_deref(), Some("AAEC/f7/"));
        assert_eq!(message.metadata.partition_key_b64_encoded, Some(true));
    }

    #[test]
    fn a_null_value_is_marked_and_sent_as_an_empty_payload() {
        let tombstone = outgoing(&PulsarPrepared::default());
        assert!(tombstone.payload.is_empty());
        assert_eq!(tombstone.metadata.null_value, Some(true));

        let empty = outgoing(&PulsarPrepared {
            value: Some(Vec::new()),
            ..PulsarPrepared::default()
        });
        assert!(empty.payload.is_empty());
        assert_eq!(empty.metadata.null_value, None);
    }
}