use std::time::Duration;
use ahash::AHashMap as HashMap;
use crate::auth::AuthConfig;
use crate::metadata::MetadataRecoveryStrategy;
use crate::{Offset, PartitionId};
#[non_exhaustive]
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum AutoOffsetReset {
Earliest,
#[default]
Latest,
None,
}
impl AutoOffsetReset {
#[inline]
pub fn to_offset(&self) -> Option<i64> {
match self {
AutoOffsetReset::Earliest => Some(-2),
AutoOffsetReset::Latest => Some(-1),
AutoOffsetReset::None => None,
}
}
}
#[non_exhaustive]
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum IsolationLevel {
#[default]
ReadUncommitted,
ReadCommitted,
}
impl IsolationLevel {
#[inline]
pub fn to_i8(self) -> i8 {
match self {
IsolationLevel::ReadUncommitted => 0,
IsolationLevel::ReadCommitted => 1,
}
}
}
#[non_exhaustive]
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum GroupProtocol {
#[default]
Classic,
Consumer,
}
#[non_exhaustive]
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum PartitionAssignmentStrategy {
#[default]
Range,
RoundRobin,
Sticky,
CooperativeSticky,
}
impl PartitionAssignmentStrategy {
#[inline]
pub fn protocol_name(&self) -> &'static str {
match self {
Self::Range => "range",
Self::RoundRobin => "roundrobin",
Self::Sticky => "sticky",
Self::CooperativeSticky => "cooperative-sticky",
}
}
#[inline]
pub fn from_protocol_name(name: &str) -> Option<Self> {
match name {
"range" => Some(Self::Range),
"roundrobin" => Some(Self::RoundRobin),
"sticky" => Some(Self::Sticky),
"cooperative-sticky" => Some(Self::CooperativeSticky),
_ => None,
}
}
#[inline]
pub fn is_cooperative(&self) -> bool {
matches!(self, Self::CooperativeSticky)
}
}
#[derive(Debug, Clone)]
pub struct ConsumerConfig {
pub(crate) bootstrap_servers: String,
pub(crate) group_id: Option<String>,
pub(crate) client_id: String,
pub(crate) auto_offset_reset: AutoOffsetReset,
pub(crate) enable_auto_commit: bool,
pub(crate) auto_commit_interval: Duration,
pub(crate) fetch_min_bytes: i32,
pub(crate) fetch_max_wait: Duration,
pub(crate) fetch_max_bytes: i32,
pub(crate) max_partition_fetch_bytes: i32,
pub(crate) topic_fetch_max_bytes: HashMap<String, i32>,
pub(crate) max_poll_records: i32,
pub(crate) max_buffered_records: i32,
pub(crate) max_poll_interval: Duration,
pub(crate) request_timeout: Duration,
pub(crate) connect_timeout: Duration,
pub(crate) session_timeout: Duration,
pub(crate) heartbeat_interval: Duration,
pub(crate) isolation_level: IsolationLevel,
pub(crate) metadata_max_age: Duration,
pub(crate) partition_assignment_strategies: Vec<PartitionAssignmentStrategy>,
pub(crate) group_protocol: GroupProtocol,
pub(crate) group_instance_id: Option<String>,
pub(crate) client_rack: Option<String>,
pub(crate) metadata_recovery_strategy: MetadataRecoveryStrategy,
pub(crate) metadata_recovery_rebootstrap_trigger: Duration,
pub(crate) metadata_topic_cache_ttl: Option<Duration>,
pub(crate) auth: Option<AuthConfig>,
pub(crate) max_decompressed_size: usize,
#[cfg(feature = "socks5")]
pub(crate) proxy: Option<crate::network::ProxyConfig>,
pub(crate) transport: crate::network::TransportConfig,
pub(crate) initial_offsets: HashMap<(String, PartitionId), Offset>,
pub(crate) max_cooperative_rebalance_rounds: usize,
pub(crate) lag_staleness_threshold: Duration,
pub(crate) idle_poll_backoff: Duration,
pub(crate) revocation_timeout: Duration,
}
impl Default for ConsumerConfig {
fn default() -> Self {
Self {
bootstrap_servers: String::new(),
group_id: None,
client_id: "krafka".to_string(),
auto_offset_reset: AutoOffsetReset::Latest,
enable_auto_commit: true,
auto_commit_interval: Duration::from_secs(5),
fetch_min_bytes: 1,
fetch_max_wait: Duration::from_millis(500),
fetch_max_bytes: 52428800, max_partition_fetch_bytes: 1048576, topic_fetch_max_bytes: HashMap::new(),
max_poll_records: 500,
max_buffered_records: 500,
max_poll_interval: Duration::from_secs(300),
request_timeout: Duration::from_secs(30),
connect_timeout: crate::network::DEFAULT_CONNECT_TIMEOUT,
session_timeout: Duration::from_secs(45),
heartbeat_interval: Duration::from_secs(3),
isolation_level: IsolationLevel::ReadUncommitted,
metadata_max_age: Duration::from_secs(300),
partition_assignment_strategies: vec![
PartitionAssignmentStrategy::Range,
PartitionAssignmentStrategy::CooperativeSticky,
],
group_protocol: GroupProtocol::Classic,
group_instance_id: None,
client_rack: None,
metadata_recovery_strategy: MetadataRecoveryStrategy::Rebootstrap,
metadata_recovery_rebootstrap_trigger: Duration::from_secs(300),
metadata_topic_cache_ttl: Some(Duration::from_secs(300)),
auth: None,
max_decompressed_size: crate::protocol::RecordBatch::MAX_DECOMPRESSED_SIZE,
#[cfg(feature = "socks5")]
proxy: None,
transport: crate::network::TransportConfig::default(),
initial_offsets: HashMap::new(),
max_cooperative_rebalance_rounds: 10,
lag_staleness_threshold: Duration::from_secs(60),
idle_poll_backoff: Duration::from_millis(10),
revocation_timeout: Duration::from_secs(5),
}
}
}
impl ConsumerConfig {
#[inline]
pub fn bootstrap_servers(&self) -> &str {
&self.bootstrap_servers
}
#[inline]
pub fn group_id(&self) -> Option<&str> {
self.group_id.as_deref()
}
#[inline]
pub fn client_id(&self) -> &str {
&self.client_id
}
#[inline]
pub fn auto_offset_reset(&self) -> AutoOffsetReset {
self.auto_offset_reset
}
#[inline]
pub fn enable_auto_commit(&self) -> bool {
self.enable_auto_commit
}
#[inline]
pub fn auto_commit_interval(&self) -> Duration {
self.auto_commit_interval
}
#[inline]
pub fn fetch_min_bytes(&self) -> i32 {
self.fetch_min_bytes
}
#[inline]
pub fn fetch_max_wait(&self) -> Duration {
self.fetch_max_wait
}
#[inline]
pub fn partition_assignment_strategies(&self) -> &[PartitionAssignmentStrategy] {
&self.partition_assignment_strategies
}
#[inline]
pub fn fetch_max_bytes(&self) -> i32 {
self.fetch_max_bytes
}
#[inline]
pub fn max_partition_fetch_bytes(&self) -> i32 {
self.max_partition_fetch_bytes
}
#[inline]
pub fn max_poll_records(&self) -> i32 {
self.max_poll_records
}
#[inline]
pub fn max_buffered_records(&self) -> i32 {
self.max_buffered_records
}
#[inline]
pub fn max_poll_interval(&self) -> Duration {
self.max_poll_interval
}
#[inline]
pub fn request_timeout(&self) -> Duration {
self.request_timeout
}
#[inline]
pub fn connect_timeout(&self) -> Duration {
self.connect_timeout
}
#[inline]
pub fn session_timeout(&self) -> Duration {
self.session_timeout
}
#[inline]
pub fn heartbeat_interval(&self) -> Duration {
self.heartbeat_interval
}
#[inline]
pub fn isolation_level(&self) -> IsolationLevel {
self.isolation_level
}
#[inline]
pub fn metadata_max_age(&self) -> Duration {
self.metadata_max_age
}
#[inline]
pub fn partition_assignment_strategy(&self) -> PartitionAssignmentStrategy {
self.partition_assignment_strategies
.first()
.copied()
.unwrap_or_default()
}
#[inline]
pub fn group_protocol(&self) -> GroupProtocol {
self.group_protocol
}
#[inline]
pub fn group_instance_id(&self) -> Option<&str> {
self.group_instance_id.as_deref()
}
#[inline]
pub fn client_rack(&self) -> Option<&str> {
self.client_rack.as_deref()
}
#[inline]
pub fn metadata_recovery_strategy(&self) -> MetadataRecoveryStrategy {
self.metadata_recovery_strategy
}
#[inline]
pub fn metadata_recovery_rebootstrap_trigger(&self) -> Duration {
self.metadata_recovery_rebootstrap_trigger
}
#[inline]
pub fn auth(&self) -> Option<&AuthConfig> {
self.auth.as_ref()
}
#[inline]
pub fn max_decompressed_size(&self) -> usize {
self.max_decompressed_size
}
#[cfg(feature = "socks5")]
#[inline]
pub fn proxy(&self) -> Option<&crate::network::ProxyConfig> {
self.proxy.as_ref()
}
#[inline]
pub fn idle_poll_backoff(&self) -> Duration {
self.idle_poll_backoff
}
#[inline]
pub fn revocation_timeout(&self) -> Duration {
self.revocation_timeout
}
}
pub(crate) fn validate(config: &ConsumerConfig) -> crate::Result<()> {
if config.bootstrap_servers.is_empty() {
return Err(crate::error::KrafkaError::config(
"bootstrap_servers must not be empty",
));
}
if config.group_id.as_deref() == Some("") {
return Err(crate::error::KrafkaError::config(
"group_id must not be an empty string; omit it entirely to disable group coordination",
));
}
if config.heartbeat_interval >= config.session_timeout {
return Err(crate::error::KrafkaError::config(format!(
"heartbeat_interval ({:?}) must be less than session_timeout ({:?})",
config.heartbeat_interval, config.session_timeout,
)));
}
if config.request_timeout <= config.session_timeout {
tracing::warn!(
request_timeout = ?config.request_timeout,
session_timeout = ?config.session_timeout,
"request_timeout does not exceed session_timeout; long-parked coordinator \
requests may be aborted client-side"
);
}
if config.max_buffered_records < 0 {
return Err(crate::error::KrafkaError::config(format!(
"max_buffered_records ({}) must be >= 0",
config.max_buffered_records,
)));
}
if config.fetch_min_bytes > config.fetch_max_bytes {
return Err(crate::error::KrafkaError::config(format!(
"fetch_min_bytes ({}) must be <= fetch_max_bytes ({})",
config.fetch_min_bytes, config.fetch_max_bytes,
)));
}
if config.max_poll_records == 0 || config.max_poll_records < -1 {
return Err(crate::error::KrafkaError::config(format!(
"max_poll_records ({}) must be -1 (unlimited) or a positive integer",
config.max_poll_records,
)));
}
if config.partition_assignment_strategies.is_empty() {
return Err(crate::error::KrafkaError::config(
"partition_assignment_strategies must not be empty",
));
}
Ok(())
}
#[cfg(test)]
#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
mod tests {
use super::*;
#[test]
fn test_auto_offset_reset_to_offset() {
assert_eq!(AutoOffsetReset::Earliest.to_offset(), Some(-2));
assert_eq!(AutoOffsetReset::Latest.to_offset(), Some(-1));
assert_eq!(AutoOffsetReset::None.to_offset(), None);
}
#[test]
fn test_isolation_level_to_i8() {
assert_eq!(IsolationLevel::ReadUncommitted.to_i8(), 0);
assert_eq!(IsolationLevel::ReadCommitted.to_i8(), 1);
}
#[test]
fn test_config_default() {
let config = ConsumerConfig::default();
assert_eq!(config.auto_offset_reset, AutoOffsetReset::Latest);
assert!(config.enable_auto_commit);
assert_eq!(config.fetch_min_bytes, 1);
assert_eq!(
config.partition_assignment_strategy(),
PartitionAssignmentStrategy::Range
);
assert_eq!(config.group_protocol, GroupProtocol::Classic);
}
#[test]
fn test_config_builder() {
let config = crate::consumer::Consumer::builder()
.bootstrap_servers("localhost:9092")
.group_id("test-group")
.auto_offset_reset(AutoOffsetReset::Earliest)
.enable_auto_commit(false)
.isolation_level(IsolationLevel::ReadCommitted)
.partition_assignment_strategy(PartitionAssignmentStrategy::CooperativeSticky)
.build_config()
.unwrap();
assert_eq!(config.bootstrap_servers, "localhost:9092");
assert_eq!(config.group_id, Some("test-group".to_string()));
assert_eq!(config.auto_offset_reset, AutoOffsetReset::Earliest);
assert!(!config.enable_auto_commit);
assert_eq!(config.isolation_level, IsolationLevel::ReadCommitted);
assert_eq!(
config.partition_assignment_strategy(),
PartitionAssignmentStrategy::CooperativeSticky
);
}
#[test]
fn test_partition_assignment_strategy_protocol_names() {
assert_eq!(PartitionAssignmentStrategy::Range.protocol_name(), "range");
assert_eq!(
PartitionAssignmentStrategy::RoundRobin.protocol_name(),
"roundrobin"
);
assert_eq!(
PartitionAssignmentStrategy::CooperativeSticky.protocol_name(),
"cooperative-sticky"
);
}
#[test]
fn test_config_builder_fetch_min_bytes() {
let config = crate::consumer::Consumer::builder()
.bootstrap_servers("localhost:9092")
.fetch_min_bytes(1024)
.build_config()
.unwrap();
assert_eq!(
config.fetch_min_bytes, 1024,
"fetch_min_bytes should be set by builder"
);
}
#[test]
fn test_config_builder_fetch_max_bytes() {
let config = crate::consumer::Consumer::builder()
.bootstrap_servers("localhost:9092")
.fetch_max_bytes(10 * 1024 * 1024)
.build_config()
.unwrap();
assert_eq!(
config.fetch_max_bytes,
10 * 1024 * 1024,
"fetch_max_bytes should be set by builder"
);
}
#[test]
fn test_config_builder_metadata_max_age() {
let config = crate::consumer::Consumer::builder()
.bootstrap_servers("localhost:9092")
.metadata_max_age(Duration::from_secs(60))
.build_config()
.unwrap();
assert_eq!(
config.metadata_max_age,
Duration::from_secs(60),
"metadata_max_age should be set by builder"
);
}
#[test]
fn test_config_default_group_instance_id() {
let config = ConsumerConfig::default();
assert!(
config.group_instance_id.is_none(),
"group_instance_id should be None by default"
);
}
#[test]
fn test_config_builder_group_instance_id() {
let config = crate::consumer::Consumer::builder()
.bootstrap_servers("localhost:9092")
.group_id("my-group")
.group_instance_id("instance-1")
.build_config()
.unwrap();
assert_eq!(
config.group_instance_id,
Some("instance-1".to_string()),
"group_instance_id should be set by builder"
);
}
#[test]
fn test_config_default_client_rack_is_none() {
let config = ConsumerConfig::default();
assert!(
config.client_rack.is_none(),
"client_rack should be None by default"
);
}
#[test]
fn test_config_builder_client_rack() {
let config = crate::consumer::Consumer::builder()
.bootstrap_servers("localhost:9092")
.client_rack("us-east-1a")
.build_config()
.unwrap();
assert_eq!(
config.client_rack,
Some("us-east-1a".to_string()),
"client_rack should be set by builder"
);
}
#[test]
fn test_config_builder_transport_round_trips() {
let transport = crate::network::TransportConfig::builder()
.tcp_keepalive(Some(std::time::Duration::from_secs(11)))
.max_connections(Some(7))
.build()
.expect("valid transport config");
let config = crate::consumer::Consumer::builder()
.bootstrap_servers("localhost:9092")
.transport(transport)
.build_config()
.expect("config builds");
assert_eq!(
config.transport.tcp_keepalive(),
Some(std::time::Duration::from_secs(11))
);
assert_eq!(config.transport.max_connections(), Some(7));
}
#[test]
fn test_config_default_group_protocol_is_classic() {
let config = ConsumerConfig::default();
assert_eq!(
config.group_protocol(),
GroupProtocol::Classic,
"group_protocol should default to Classic"
);
}
#[test]
fn test_config_builder_group_protocol_consumer() {
let config = crate::consumer::Consumer::builder()
.bootstrap_servers("localhost:9092")
.group_protocol(GroupProtocol::Consumer)
.build_config()
.unwrap();
assert_eq!(
config.group_protocol(),
GroupProtocol::Consumer,
"group_protocol should be Consumer when set"
);
}
#[cfg(feature = "socks5")]
#[test]
fn test_config_builder_proxy_round_trip() {
let config = crate::consumer::Consumer::builder()
.bootstrap_servers("localhost:9092")
.proxy(crate::network::ProxyConfig::new("proxy:1080"))
.build_config()
.unwrap();
let proxy = config.proxy().expect("proxy should be set");
assert_eq!(proxy.address(), "proxy:1080");
}
#[test]
fn test_config_default_recovery_strategy() {
let config = ConsumerConfig::default();
assert_eq!(
config.metadata_recovery_strategy,
MetadataRecoveryStrategy::Rebootstrap,
);
assert_eq!(
config.metadata_recovery_rebootstrap_trigger,
Duration::from_secs(300),
);
}
#[test]
fn test_config_builder_recovery_strategy() {
let config = crate::consumer::Consumer::builder()
.bootstrap_servers("localhost:9092")
.metadata_recovery_strategy(MetadataRecoveryStrategy::Rebootstrap)
.metadata_recovery_rebootstrap_trigger(Duration::from_secs(60))
.build_config()
.unwrap();
assert_eq!(
config.metadata_recovery_strategy(),
MetadataRecoveryStrategy::Rebootstrap,
);
assert_eq!(
config.metadata_recovery_rebootstrap_trigger(),
Duration::from_secs(60),
);
}
#[test]
fn test_config_builder_rejects_negative_max_buffered_records() {
let result = crate::consumer::Consumer::builder()
.bootstrap_servers("localhost:9092")
.max_buffered_records(-1)
.build_config();
assert!(result.is_err());
assert!(
result
.unwrap_err()
.to_string()
.contains("max_buffered_records"),
"error message should mention max_buffered_records"
);
}
#[test]
fn test_config_builder_accepts_zero_max_buffered_records() {
let config = crate::consumer::Consumer::builder()
.bootstrap_servers("localhost:9092")
.max_buffered_records(0)
.build_config()
.unwrap();
assert_eq!(config.max_buffered_records(), 0);
}
#[test]
fn test_config_builder_accepts_minus_one_max_poll_records_as_unlimited() {
let config = crate::consumer::Consumer::builder()
.bootstrap_servers("localhost:9092")
.max_poll_records(-1)
.build_config()
.unwrap();
assert_eq!(
config.max_poll_records(),
-1,
"max_poll_records=-1 should be accepted as unlimited"
);
}
#[test]
fn test_config_builder_rejects_zero_max_poll_records() {
let result = crate::consumer::Consumer::builder()
.bootstrap_servers("localhost:9092")
.max_poll_records(0)
.build_config();
assert!(result.is_err());
assert!(
result.unwrap_err().to_string().contains("max_poll_records"),
"error message should mention max_poll_records"
);
}
#[test]
fn test_config_builder_rejects_negative_max_poll_records() {
let result = crate::consumer::Consumer::builder()
.bootstrap_servers("localhost:9092")
.max_poll_records(-2)
.build_config();
assert!(result.is_err());
assert!(
result.unwrap_err().to_string().contains("max_poll_records"),
"error message should mention max_poll_records"
);
}
#[test]
fn test_config_builder_rejects_empty_bootstrap_servers() {
let result = crate::consumer::Consumer::builder()
.bootstrap_servers("")
.build_config();
assert!(result.is_err());
assert!(
result
.unwrap_err()
.to_string()
.contains("bootstrap_servers"),
"error message should mention bootstrap_servers"
);
}
#[test]
fn test_config_builder_rejects_empty_group_id() {
let result = crate::consumer::Consumer::builder()
.bootstrap_servers("localhost:9092")
.group_id("")
.build_config();
assert!(result.is_err());
assert!(
result.unwrap_err().to_string().contains("group_id"),
"error message should mention group_id"
);
}
}