datum_mq/lib.rs
1#![forbid(unsafe_code)]
2//! Kafka sources and sinks for Datum streams.
3//!
4//! `datum-mq` is an additive Datum satellite crate with Tokio-native Kafka
5//! consumer and producer backends. The rustls TLS path uses ring (Rust plus
6//! assembly), with no AWS-LC C library or CMake build. Native produce and fetch
7//! support uncompressed, gzip, Snappy, LZ4, and Zstd record batches. SASL
8//! supports PLAIN, SCRAM-SHA-256, and SCRAM-SHA-512; GSSAPI and OAUTHBEARER are
9//! not supported.
10//!
11//! The consumer's shipped offset-commit guarantee is at-least-once.
12//! [`KafkaSource::committable`] emits a [`datum::SourceWithContext`] whose
13//! context is a [`KafkaOffset`]; downstream code commits that offset only after
14//! its own checkpoint/side effect succeeds.
15//! [`KafkaSink`] uses the native producer by default, and its materialized
16//! [`KafkaProducerControl`] drains final acknowledgements before shutdown. The
17//! native producer enables idempotence by default: it negotiates a producer
18//! id/epoch and uses per-partition sequences so the broker deduplicates a
19//! retried batch. Set `enable.idempotence=false` to opt out explicitly. The
20//! pipeline remains bounded and partition ordered.
21//! Kafka transactions and consume-transform-produce EOS are not supported.
22
23mod config;
24mod error;
25mod metrics;
26mod native;
27mod offset;
28#[doc(hidden)]
29pub mod profile;
30mod record;
31mod sink;
32mod source;
33
34pub use config::{
35 CommitPolicy, KafkaConfig, KafkaConsumerBackend, KafkaConsumerSettings, KafkaProducerBackend,
36 KafkaProducerSettings, Offset, Subscription, TopicPartitionOffset,
37};
38pub use error::{MqError, MqResult};
39pub use metrics::{KafkaMetrics, KafkaMetricsSnapshot};
40pub use offset::{KafkaBatchOffset, KafkaOffset, KafkaOffsetBatch, OffsetCommit, TopicPartition};
41pub use record::{ConsumerRecord, KafkaHeader, KafkaTimestamp, ProducerRecord};
42pub use sink::{KafkaProducerControl, KafkaSink};
43pub use source::{KafkaControl, KafkaPayloadBatch, KafkaPayloadRecord, KafkaSource};
44
45/// The `datum-mq` crate version.
46pub const VERSION: &str = env!("CARGO_PKG_VERSION");
47
48/// Benchmark-only shims exposed for the criterion harness (a separate crate that
49/// otherwise sees only the public API). Gated behind the internal, off-by-default
50/// `bench-internals` feature so it is not part of the crates.io API surface.
51#[cfg(feature = "bench-internals")]
52#[doc(hidden)]
53pub mod bench_support {
54 pub use crate::native::bench_encode_produce_batch;
55}
56
57pub(crate) const fn consumer_backend_hint() -> &'static str {
58 "datum-mq supports only the native Kafka consumer backend"
59}
60
61pub(crate) const fn producer_backend_hint() -> &'static str {
62 "datum-mq supports only the native Kafka producer backend"
63}