use std::collections::HashMap;
use std::sync::{Arc, OnceLock};
use std::time::Duration;
use crate::auth::AuthConfig;
use crate::dlq::DeadLetterQueue;
use crate::error::{KrafkaError, Result};
use crate::metadata::MetadataRecoveryStrategy;
use crate::protocol::Compression;
#[non_exhaustive]
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum Acks {
None,
Leader,
#[default]
All,
}
impl Acks {
#[inline]
pub fn to_i16(self) -> i16 {
match self {
Acks::None => 0,
Acks::Leader => 1,
Acks::All => -1,
}
}
#[inline]
pub fn from_i16(value: i16) -> Option<Self> {
match value {
0 => Some(Acks::None),
1 => Some(Acks::Leader),
-1 => Some(Acks::All),
_ => None,
}
}
}
#[derive(Debug, Clone)]
pub struct ProducerConfig {
pub(crate) bootstrap_servers: String,
pub(crate) client_id: String,
pub(crate) acks: Acks,
pub(crate) compression: Compression,
pub(crate) compression_level: Option<i32>,
pub(crate) topic_compression: HashMap<String, Compression>,
pub(crate) batch_size: usize,
pub(crate) linger: Duration,
pub(crate) request_timeout: Duration,
pub(crate) connect_timeout: Duration,
pub(crate) delivery_timeout: Duration,
pub(crate) retries: u32,
pub(crate) retry_backoff: Duration,
pub(crate) max_request_size: usize,
pub(crate) idempotent: bool,
pub(crate) max_block: Duration,
pub(crate) buffer_memory: usize,
pub(crate) metadata_max_age: Duration,
pub(crate) metadata_topic_cache_ttl: Option<Duration>,
pub(crate) metadata_recovery_strategy: MetadataRecoveryStrategy,
pub(crate) metadata_recovery_rebootstrap_trigger: Duration,
pub(crate) auth: Option<AuthConfig>,
pub(crate) transport: crate::network::TransportConfig,
pub(crate) dead_letter_queue: Option<Arc<dyn DeadLetterQueue>>,
}
impl Default for ProducerConfig {
fn default() -> Self {
Self {
bootstrap_servers: String::new(),
client_id: "krafka".to_string(),
acks: Acks::All,
compression: Compression::None,
compression_level: None,
topic_compression: HashMap::new(),
batch_size: 16384,
linger: Duration::ZERO,
request_timeout: Duration::from_secs(30),
connect_timeout: crate::network::DEFAULT_CONNECT_TIMEOUT,
delivery_timeout: Duration::from_secs(120),
retries: u32::MAX,
retry_backoff: Duration::from_millis(100),
max_request_size: crate::protocol::MAX_MESSAGE_SIZE,
idempotent: true,
max_block: Duration::from_secs(60),
buffer_memory: 32 * 1024 * 1024, metadata_max_age: Duration::from_secs(300),
metadata_topic_cache_ttl: Some(Duration::from_secs(300)),
metadata_recovery_strategy: MetadataRecoveryStrategy::Rebootstrap,
metadata_recovery_rebootstrap_trigger: Duration::from_secs(300),
auth: None,
transport: crate::network::TransportConfig::default(),
dead_letter_queue: None,
}
}
}
impl ProducerConfig {
#[inline]
pub fn bootstrap_servers(&self) -> &str {
&self.bootstrap_servers
}
#[inline]
pub fn client_id(&self) -> &str {
&self.client_id
}
#[inline]
pub fn acks(&self) -> Acks {
self.acks
}
#[inline]
pub fn compression(&self) -> Compression {
self.compression
}
#[inline]
pub fn compression_level(&self) -> Option<i32> {
self.compression_level
}
#[inline]
pub fn compression_for(&self, topic: &str) -> Compression {
self.topic_compression
.get(topic)
.copied()
.unwrap_or(self.compression)
}
#[inline]
pub fn batch_size(&self) -> usize {
self.batch_size
}
#[inline]
pub fn linger(&self) -> Duration {
self.linger
}
#[inline]
pub fn request_timeout(&self) -> Duration {
self.request_timeout
}
#[inline]
pub fn connect_timeout(&self) -> Duration {
self.connect_timeout
}
#[inline]
pub fn delivery_timeout(&self) -> Duration {
self.delivery_timeout
}
#[inline]
pub fn retries(&self) -> u32 {
self.retries
}
#[inline]
pub fn retry_backoff(&self) -> Duration {
self.retry_backoff
}
#[inline]
pub fn max_request_size(&self) -> usize {
self.max_request_size
}
#[inline]
pub fn idempotent(&self) -> bool {
self.idempotent
}
#[inline]
pub fn max_block(&self) -> Duration {
self.max_block
}
#[inline]
pub fn buffer_memory(&self) -> usize {
self.buffer_memory
}
#[inline]
pub fn metadata_max_age(&self) -> Duration {
self.metadata_max_age
}
#[inline]
pub fn metadata_topic_cache_ttl(&self) -> Option<Duration> {
self.metadata_topic_cache_ttl
}
#[inline]
pub fn metadata_recovery_strategy(&self) -> MetadataRecoveryStrategy {
self.metadata_recovery_strategy
}
#[inline]
pub fn metadata_recovery_rebootstrap_trigger(&self) -> Duration {
self.metadata_recovery_rebootstrap_trigger
}
#[inline]
pub fn auth(&self) -> Option<&AuthConfig> {
self.auth.as_ref()
}
}
pub(crate) fn validate_compression(
compression: Compression,
compression_level: Option<i32>,
topic_compression: &HashMap<String, Compression>,
) -> Result<()> {
if !compression.is_available() {
let feature = compression.required_feature().unwrap_or("unknown");
return Err(KrafkaError::config(format!(
"compression codec {compression:?} requires the `{feature}` Cargo feature; \
either enable the feature or choose a different compression codec"
)));
}
for (topic, codec) in topic_compression {
if !codec.is_available() {
let feature = codec.required_feature().unwrap_or("unknown");
return Err(KrafkaError::config(format!(
"per-topic compression codec {codec:?} for topic {topic:?} requires the \
`{feature}` Cargo feature"
)));
}
}
let Some(level) = compression_level else {
return Ok(());
};
let mut codecs: Vec<(Option<&str>, Compression)> = vec![(None, compression)];
for (topic, codec) in topic_compression {
codecs.push((Some(topic.as_str()), *codec));
}
for (topic, codec) in codecs {
let where_ = topic.map_or_else(String::new, |t| format!(" (topic {t:?})"));
let Some(range) = codec.level_range().filter(|_| codec.supports_level()) else {
return Err(KrafkaError::config(format!(
"compression_level {level} was set but codec {codec:?}{where_} takes no \
level; krafka encodes Snappy with `snap` and LZ4 with `lz4_flex`, neither \
of which exposes one. Remove compression_level or select Gzip or Zstd"
)));
};
if !range.contains(&level) {
return Err(KrafkaError::config(format!(
"compression_level {level} is out of range for codec {codec:?}{where_}; \
valid levels are {}..={}",
range.start(),
range.end()
)));
}
}
Ok(())
}
pub(crate) fn validate(config: &mut ProducerConfig, has_shared_pool: bool) -> Result<()> {
if !has_shared_pool && config.bootstrap_servers.is_empty() {
return Err(KrafkaError::config("bootstrap_servers is required"));
}
const MAX_KAFKA_STRING_LEN: usize = i16::MAX as usize;
if config.client_id.len() > MAX_KAFKA_STRING_LEN {
return Err(KrafkaError::config(format!(
"client_id is {} bytes, exceeding the Kafka wire limit of {MAX_KAFKA_STRING_LEN}",
config.client_id.len()
)));
}
if config.batch_size == 0 {
return Err(KrafkaError::config(format!(
"batch_size must be >= 1 (got {})",
config.batch_size
)));
}
if config.max_request_size == 0 {
return Err(KrafkaError::config("max_request_size must be >= 1"));
}
if config.delivery_timeout.is_zero() {
return Err(KrafkaError::config(
"delivery_timeout must be greater than zero",
));
}
if config.delivery_timeout == Duration::MAX && config.retries == u32::MAX {
return Err(KrafkaError::config(
"delivery_timeout = Duration::MAX combined with retries = u32::MAX creates \
an infinite retry loop; set a finite delivery_timeout or reduce retries",
));
}
validate_compression(
config.compression,
config.compression_level,
&config.topic_compression,
)?;
if config.retries > 0 {
let min_budget = config
.request_timeout
.saturating_mul(config.retries.saturating_add(1));
if config.delivery_timeout < min_budget {
tracing::warn!(
delivery_timeout_secs = config.delivery_timeout.as_secs_f64(),
request_timeout_secs = config.request_timeout.as_secs_f64(),
retries = config.retries,
minimum_budget_secs = min_budget.as_secs_f64(),
"delivery_timeout is shorter than request_timeout × (retries + 1); \
some retry attempts will be cut short by the delivery deadline"
);
}
}
if config.idempotent {
if config.retries == 0 {
return Err(KrafkaError::config(
"idempotent producer requires retries > 0",
));
}
if config.acks != Acks::All {
return Err(KrafkaError::config(format!(
"idempotent producer requires acks = All (got {:?})",
config.acks
)));
}
static IDEMPOTENT_NO_TXN_WARNED: OnceLock<()> = OnceLock::new();
IDEMPOTENT_NO_TXN_WARNED.get_or_init(|| {
tracing::warn!(
"Idempotent producer enabled without a transactional_id. \
This provides per-session duplicate detection (KIP-679) but not zombie \
fencing. Use TransactionalProducer with a stable transactional_id for \
exactly-once end-to-end guarantees across producer restarts (KIP-360)."
);
});
}
if config.buffer_memory > 0 && config.batch_size > config.buffer_memory {
return Err(KrafkaError::config(format!(
"batch_size must not exceed buffer_memory (got batch_size={}, buffer_memory={})",
config.batch_size, config.buffer_memory
)));
}
if config.batch_size > config.max_request_size {
return Err(KrafkaError::config(format!(
"batch_size must not exceed max_request_size (got batch_size={}, max_request_size={})",
config.batch_size, config.max_request_size
)));
}
if config.linger >= config.delivery_timeout {
tracing::warn!(
linger_ms = config.linger.as_millis(),
delivery_timeout_ms = config.delivery_timeout.as_millis(),
"linger >= delivery_timeout: records may expire before they are sent"
);
}
if config.retries == u32::MAX {
tracing::debug!(
"retries = u32::MAX; retry loop is bounded by delivery_timeout ({:?})",
config.delivery_timeout
);
}
Ok(())
}
#[cfg(test)]
#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
mod tests {
use crate::producer::Producer;
#[cfg(feature = "snappy")]
#[test]
fn compression_level_on_a_levelless_codec_is_rejected() {
let err = Producer::builder()
.bootstrap_servers("localhost:9092")
.compression(Compression::Snappy)
.compression_level(Some(9))
.build_config()
.expect_err("Snappy takes no level");
let msg = err.to_string();
assert!(
msg.contains("takes") && msg.contains("no level"),
"the error must say the codec has no level, got: {msg}"
);
}
#[cfg(feature = "gzip")]
#[test]
fn out_of_range_compression_level_is_rejected() {
let err = Producer::builder()
.bootstrap_servers("localhost:9092")
.compression(Compression::Gzip)
.compression_level(Some(42))
.build_config()
.expect_err("gzip tops out at 9");
assert!(
err.to_string().contains("0..=9"),
"the error must name the valid range, got: {err}"
);
}
#[cfg(feature = "zstd")]
#[test]
fn valid_compression_level_reaches_the_config() {
let config = Producer::builder()
.bootstrap_servers("localhost:9092")
.compression(Compression::Zstd)
.compression_level(Some(1))
.build_config()
.expect("level 1 is valid for zstd");
assert_eq!(config.compression_level(), Some(1));
}
#[cfg(all(feature = "zstd", feature = "snappy"))]
#[test]
fn per_topic_codec_is_validated_against_the_level() {
let err = Producer::builder()
.bootstrap_servers("localhost:9092")
.compression(Compression::Zstd)
.compression_level(Some(1))
.topic_compression("events", Compression::Snappy)
.build_config()
.expect_err("the per-topic Snappy override takes no level");
assert!(
err.to_string().contains("events"),
"the error must name the offending topic, got: {err}"
);
}
use super::*;
#[test]
fn build_config_rejects_a_codec_that_is_not_compiled_in() {
let Some(missing) = [
Compression::Gzip,
Compression::Snappy,
Compression::Lz4,
Compression::Zstd,
]
.into_iter()
.find(|c| !c.is_available()) else {
return;
};
let err = crate::producer::Producer::builder()
.bootstrap_servers("localhost:9092")
.compression(missing)
.build_config()
.expect_err("an unavailable codec must be rejected")
.to_string();
assert!(
err.contains("Cargo feature"),
"the error must name the missing feature, got: {err}"
);
}
#[test]
fn build_config_rejects_an_unavailable_per_topic_codec() {
let Some(missing) = [
Compression::Gzip,
Compression::Snappy,
Compression::Lz4,
Compression::Zstd,
]
.into_iter()
.find(|c| !c.is_available()) else {
return;
};
let err = crate::producer::Producer::builder()
.bootstrap_servers("localhost:9092")
.topic_compression("high-volume", missing)
.build_config()
.expect_err("an unavailable per-topic codec must be rejected")
.to_string();
assert!(
err.contains("Cargo feature"),
"the error must name the missing feature, got: {err}"
);
}
#[test]
fn build_config_rejects_the_infinite_retry_loop() {
let err = crate::producer::Producer::builder()
.bootstrap_servers("localhost:9092")
.idempotent(false)
.delivery_timeout(Duration::MAX)
.retries(u32::MAX)
.build_config()
.expect_err("MAX/MAX must be rejected")
.to_string();
assert!(err.contains("infinite retry loop"), "got: {err}");
}
#[test]
fn build_config_rejects_an_oversize_client_id() {
let err = crate::producer::Producer::builder()
.bootstrap_servers("localhost:9092")
.client_id("x".repeat(i16::MAX as usize + 1))
.build_config()
.expect_err("an oversize client_id must be rejected")
.to_string();
assert!(err.contains("client_id"), "got: {err}");
}
#[test]
fn validation_is_idempotent() {
let mut config = crate::producer::Producer::builder()
.bootstrap_servers("localhost:9092")
.build_config()
.expect("a default config is valid");
validate(&mut config, false).expect("re-validation must succeed");
}
#[test]
fn validate_allows_an_empty_bootstrap_list_with_a_shared_pool() {
let mut config = ProducerConfig {
bootstrap_servers: String::new(),
..ProducerConfig::default()
};
assert!(validate(&mut config, true).is_ok());
assert!(validate(&mut config, false).is_err());
}
#[test]
fn test_config_builder_transport_round_trips() {
let transport = crate::network::TransportConfig::builder()
.tcp_keepalive(Some(std::time::Duration::from_secs(11)))
.max_connections(Some(7))
.build()
.expect("valid transport config");
let config = crate::producer::Producer::builder()
.bootstrap_servers("localhost:9092")
.transport(transport)
.build_config()
.expect("config builds");
assert_eq!(
config.transport.tcp_keepalive(),
Some(std::time::Duration::from_secs(11))
);
assert_eq!(config.transport.max_connections(), Some(7));
}
#[test]
fn test_acks_to_i16() {
assert_eq!(Acks::None.to_i16(), 0);
assert_eq!(Acks::Leader.to_i16(), 1);
assert_eq!(Acks::All.to_i16(), -1);
}
#[test]
fn test_acks_from_i16() {
assert_eq!(Acks::from_i16(0), Some(Acks::None));
assert_eq!(Acks::from_i16(1), Some(Acks::Leader));
assert_eq!(Acks::from_i16(-1), Some(Acks::All));
}
#[test]
fn test_config_default() {
let config = ProducerConfig::default();
assert_eq!(config.acks, Acks::All);
assert!(config.idempotent);
assert_eq!(config.compression, Compression::None);
assert_eq!(config.batch_size, 16384);
assert_eq!(config.max_request_size, crate::protocol::MAX_MESSAGE_SIZE);
assert_eq!(config.delivery_timeout, Duration::from_secs(120));
assert_eq!(config.retries, u32::MAX);
assert_eq!(
config.metadata_topic_cache_ttl,
Some(Duration::from_secs(300))
);
}
#[test]
#[cfg(feature = "lz4")]
fn test_config_builder() {
let config = crate::producer::Producer::builder()
.bootstrap_servers("localhost:9092")
.client_id("test")
.acks(Acks::All)
.compression(Compression::Lz4)
.batch_size(32768)
.max_request_size(65536)
.build_config()
.unwrap();
assert_eq!(config.bootstrap_servers, "localhost:9092");
assert_eq!(config.client_id, "test");
assert_eq!(config.acks, Acks::All);
assert_eq!(config.compression, Compression::Lz4);
assert_eq!(config.batch_size, 32768);
assert_eq!(config.max_request_size, 65536);
}
#[test]
fn test_config_builder_request_timeout() {
let config = crate::producer::Producer::builder()
.bootstrap_servers("localhost:9092")
.request_timeout(Duration::from_secs(60))
.build_config()
.unwrap();
assert_eq!(
config.request_timeout,
Duration::from_secs(60),
"request_timeout should be set by builder"
);
}
#[test]
fn test_config_builder_delivery_timeout() {
let config = crate::producer::Producer::builder()
.bootstrap_servers("localhost:9092")
.delivery_timeout(Duration::from_secs(45))
.build_config()
.unwrap();
assert_eq!(config.delivery_timeout(), Duration::from_secs(45));
}
#[test]
fn test_config_builder_infinite_retry_loop_is_err() {
let err = crate::producer::Producer::builder()
.bootstrap_servers("localhost:9092")
.idempotent(false)
.delivery_timeout(Duration::MAX)
.retries(u32::MAX)
.build_config()
.unwrap_err();
let msg = err.to_string();
assert!(
msg.contains("infinite retry loop"),
"expected 'infinite retry loop' in error, got: {msg}"
);
}
#[test]
fn test_config_builder_metadata_max_age() {
let config = crate::producer::Producer::builder()
.bootstrap_servers("localhost:9092")
.metadata_max_age(Duration::from_secs(120))
.build_config()
.unwrap();
assert_eq!(
config.metadata_max_age,
Duration::from_secs(120),
"metadata_max_age should be set by builder"
);
}
#[test]
fn test_config_builder_metadata_topic_cache_ttl() {
let config = crate::producer::Producer::builder()
.bootstrap_servers("localhost:9092")
.metadata_topic_cache_ttl(Duration::from_secs(600))
.build_config()
.unwrap();
assert_eq!(
config.metadata_topic_cache_ttl(),
Some(Duration::from_secs(600))
);
}
#[test]
fn test_config_builder_disable_metadata_topic_cache_ttl() {
let config = crate::producer::Producer::builder()
.bootstrap_servers("localhost:9092")
.disable_metadata_topic_cache_ttl()
.build_config()
.unwrap();
assert_eq!(config.metadata_topic_cache_ttl(), None);
}
#[test]
fn test_acks_from_i16_known_values() {
assert_eq!(Acks::from_i16(0), Some(Acks::None));
assert_eq!(Acks::from_i16(1), Some(Acks::Leader));
assert_eq!(Acks::from_i16(-1), Some(Acks::All));
}
#[test]
fn test_acks_from_i16_unknown_returns_none() {
assert_eq!(Acks::from_i16(2), None);
assert_eq!(Acks::from_i16(99), None);
assert_eq!(Acks::from_i16(-2), None);
}
#[test]
fn test_acks_roundtrip() {
assert_eq!(Acks::from_i16(Acks::None.to_i16()), Some(Acks::None));
assert_eq!(Acks::from_i16(Acks::Leader.to_i16()), Some(Acks::Leader));
assert_eq!(Acks::from_i16(Acks::All.to_i16()), Some(Acks::All));
}
#[cfg(feature = "socks5")]
#[test]
fn test_config_builder_proxy_round_trip() {
let config = crate::producer::Producer::builder()
.bootstrap_servers("localhost:9092")
.proxy(crate::network::ProxyConfig::new("proxy:1080"))
.build_config()
.unwrap();
let proxy = config
.transport
.proxy()
.expect("proxy should reach the transport config");
assert_eq!(proxy.address(), "proxy:1080");
}
#[test]
fn test_config_default_recovery_strategy() {
let config = ProducerConfig::default();
assert_eq!(
config.metadata_recovery_strategy,
MetadataRecoveryStrategy::Rebootstrap,
);
assert_eq!(
config.metadata_recovery_rebootstrap_trigger,
Duration::from_secs(300),
);
}
#[test]
fn test_config_builder_recovery_strategy() {
let config = crate::producer::Producer::builder()
.bootstrap_servers("localhost:9092")
.metadata_recovery_strategy(MetadataRecoveryStrategy::Rebootstrap)
.metadata_recovery_rebootstrap_trigger(Duration::from_secs(120))
.build_config()
.unwrap();
assert_eq!(
config.metadata_recovery_strategy(),
MetadataRecoveryStrategy::Rebootstrap,
);
assert_eq!(
config.metadata_recovery_rebootstrap_trigger(),
Duration::from_secs(120),
);
}
#[test]
fn test_config_builder_rejects_zero_batch_size() {
let err = crate::producer::Producer::builder()
.batch_size(0)
.build_config();
assert!(err.is_err());
}
#[test]
fn test_config_builder_rejects_zero_max_request_size() {
let err = crate::producer::Producer::builder()
.max_request_size(0)
.build_config();
assert!(err.is_err());
}
#[test]
fn test_config_builder_rejects_zero_delivery_timeout() {
let err = crate::producer::Producer::builder()
.delivery_timeout(Duration::ZERO)
.build_config();
assert!(err.is_err());
}
#[test]
fn test_config_builder_rejects_idempotent_without_retries() {
let err = crate::producer::Producer::builder()
.retries(0)
.build_config();
assert!(err.is_err());
}
#[test]
fn test_config_builder_rejects_idempotent_with_acks_leader() {
let err = crate::producer::Producer::builder()
.idempotent(true)
.acks(Acks::Leader)
.build_config();
assert!(err.is_err());
}
#[test]
fn test_config_builder_rejects_batch_exceeding_buffer() {
let err = crate::producer::Producer::builder()
.batch_size(1024)
.buffer_memory(512)
.build_config();
assert!(err.is_err());
}
#[test]
fn test_config_builder_rejects_batch_exceeding_max_request_size() {
let err = crate::producer::Producer::builder()
.batch_size(1024)
.max_request_size(512)
.build_config();
assert!(err.is_err());
}
#[test]
fn test_config_builder_rejects_empty_bootstrap_servers() {
let err = crate::producer::Producer::builder()
.bootstrap_servers("")
.build_config();
assert!(
err.is_err(),
"empty bootstrap_servers should be rejected at build time"
);
assert!(
err.unwrap_err().to_string().contains("bootstrap_servers"),
"error message should mention bootstrap_servers"
);
}
}