datum-mq 0.11.1

Native Kafka sources and sinks for Datum streams
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63

#![forbid(unsafe_code)]
//! Kafka sources and sinks for Datum streams.
//!
//! `datum-mq` is an additive Datum satellite crate with Tokio-native Kafka
//! consumer and producer backends. The rustls TLS path uses ring (Rust plus
//! assembly), with no AWS-LC C library or CMake build. Native produce and fetch
//! support uncompressed, gzip, Snappy, LZ4, and Zstd record batches. SASL
//! supports PLAIN, SCRAM-SHA-256, and SCRAM-SHA-512; GSSAPI and OAUTHBEARER are
//! not supported.
//!
//! The consumer's shipped offset-commit guarantee is at-least-once.
//! [`KafkaSource::committable`] emits a [`datum::SourceWithContext`] whose
//! context is a [`KafkaOffset`]; downstream code commits that offset only after
//! its own checkpoint/side effect succeeds.
//! [`KafkaSink`] uses the native producer by default, and its materialized
//! [`KafkaProducerControl`] drains final acknowledgements before shutdown. The
//! native producer enables idempotence by default: it negotiates a producer
//! id/epoch and uses per-partition sequences so the broker deduplicates a
//! retried batch. Set `enable.idempotence=false` to opt out explicitly. The
//! pipeline remains bounded and partition ordered.
//! Kafka transactions and consume-transform-produce EOS are not supported.

mod config;
mod error;
mod metrics;
mod native;
mod offset;
#[doc(hidden)]
pub mod profile;
mod record;
mod sink;
mod source;

pub use config::{
    CommitPolicy, KafkaConfig, KafkaConsumerBackend, KafkaConsumerSettings, KafkaProducerBackend,
    KafkaProducerSettings, Offset, Subscription, TopicPartitionOffset,
};
pub use error::{MqError, MqResult};
pub use metrics::{KafkaMetrics, KafkaMetricsSnapshot};
pub use offset::{KafkaBatchOffset, KafkaOffset, KafkaOffsetBatch, OffsetCommit, TopicPartition};
pub use record::{ConsumerRecord, KafkaHeader, KafkaTimestamp, ProducerRecord};
pub use sink::{KafkaProducerControl, KafkaSink};
pub use source::{KafkaControl, KafkaPayloadBatch, KafkaPayloadRecord, KafkaSource};

/// The `datum-mq` crate version.
pub const VERSION: &str = env!("CARGO_PKG_VERSION");

/// Benchmark-only shims exposed for the criterion harness (a separate crate that
/// otherwise sees only the public API). Gated behind the internal, off-by-default
/// `bench-internals` feature so it is not part of the crates.io API surface.
#[cfg(feature = "bench-internals")]
#[doc(hidden)]
pub mod bench_support {
    pub use crate::native::bench_encode_produce_batch;
}

pub(crate) const fn consumer_backend_hint() -> &'static str {
    "datum-mq supports only the native Kafka consumer backend"
}

pub(crate) const fn producer_backend_hint() -> &'static str {
    "datum-mq supports only the native Kafka producer backend"
}