use std::sync::atomic::{AtomicU32, Ordering};
use krabka_client_core::{
ClientDnsLookup, ClientDnsTimeout, ClientFrameMax, ConnectionDispatchQueueCapacity,
ConnectionOptions, DEFAULT_METADATA_RECOVERY_REBOOTSTRAP_TRIGGER, MetadataRecoveryStrategy,
security::ClientSecurity,
};
use krabka_units::{
ByteSize, Time,
convert::{ByteSizeExt as _, TimeExt as _},
millis, secs,
};
use crate::{AdminError, retry::RetryPolicy};
pub const DEFAULT_ADMIN_REQUEST_TIMEOUT: Time = secs(30);
pub const DEFAULT_API_TIMEOUT: Time = secs(60);
pub const DEFAULT_RETRY_BACKOFF: Time = millis(100);
pub const DEFAULT_RETRY_BACKOFF_MAX: Time = secs(1);
pub const DEFAULT_SOCKET_CONNECTION_SETUP_TIMEOUT: Time = secs(10);
pub const DEFAULT_SOCKET_CONNECTION_SETUP_TIMEOUT_MAX: Time = secs(30);
pub const DEFAULT_ADMIN_CONNECTIONS_MAX_IDLE: Time = krabka_units::minutes(5);
pub const DEFAULT_ENABLE_METRICS_PUSH: bool = true;
const RETRY_BACKOFF_JITTER: f64 = 0.2;
static ADMIN_CLIENT_ID_SEQUENCE: AtomicU32 = AtomicU32::new(1);
#[derive(Clone, Debug)]
pub struct AdminClientConfig {
pub client_id: Option<String>,
pub request_timeout: Time,
pub default_api_timeout: Option<Time>,
pub retries: u32,
pub retry_backoff: Time,
pub retry_backoff_max: Time,
pub socket_connection_setup_timeout: Time,
pub socket_connection_setup_timeout_max: Time,
pub connections_max_idle: Time,
pub dns_timeout: ClientDnsTimeout,
pub client_dns_lookup: ClientDnsLookup,
pub reconnect_backoff: Time,
pub reconnect_backoff_max: Time,
pub send_buffer: Option<ByteSize>,
pub receive_buffer: Option<ByteSize>,
pub dispatch_queue_capacity: ConnectionDispatchQueueCapacity,
pub frame_max: ClientFrameMax,
pub security: Option<ClientSecurity>,
pub metadata_recovery_strategy: MetadataRecoveryStrategy,
pub metadata_recovery_rebootstrap_trigger: Time,
pub enable_metrics_push: bool,
}
impl Default for AdminClientConfig {
fn default() -> Self {
Self {
client_id: None,
request_timeout: DEFAULT_ADMIN_REQUEST_TIMEOUT,
default_api_timeout: None,
retries: u32::MAX,
retry_backoff: DEFAULT_RETRY_BACKOFF,
retry_backoff_max: DEFAULT_RETRY_BACKOFF_MAX,
socket_connection_setup_timeout: DEFAULT_SOCKET_CONNECTION_SETUP_TIMEOUT,
socket_connection_setup_timeout_max: DEFAULT_SOCKET_CONNECTION_SETUP_TIMEOUT_MAX,
connections_max_idle: DEFAULT_ADMIN_CONNECTIONS_MAX_IDLE,
dns_timeout: ClientDnsTimeout::default(),
client_dns_lookup: ClientDnsLookup::default(),
reconnect_backoff: krabka_client_core::DEFAULT_RECONNECT_BACKOFF,
reconnect_backoff_max: krabka_client_core::DEFAULT_RECONNECT_BACKOFF_MAX,
send_buffer: Some(krabka_client_core::DEFAULT_SEND_BUFFER),
receive_buffer: Some(krabka_client_core::DEFAULT_RECEIVE_BUFFER),
dispatch_queue_capacity: ConnectionDispatchQueueCapacity::default(),
frame_max: ClientFrameMax::default(),
security: None,
metadata_recovery_strategy: MetadataRecoveryStrategy::default(),
metadata_recovery_rebootstrap_trigger: DEFAULT_METADATA_RECOVERY_REBOOTSTRAP_TRIGGER,
enable_metrics_push: DEFAULT_ENABLE_METRICS_PUSH,
}
}
}
#[derive(Clone, Debug)]
pub(crate) struct ResolvedAdminConfig {
pub(crate) connection: ConnectionOptions,
pub(crate) retry: RetryPolicy,
pub(crate) metadata_recovery_strategy: MetadataRecoveryStrategy,
pub(crate) metadata_recovery_rebootstrap_trigger: Time,
pub(crate) enable_metrics_push: bool,
}
impl AdminClientConfig {
fn validate_transport(&self) -> Result<(), AdminError> {
for (name, value) in [
("reconnect_backoff", self.reconnect_backoff),
("reconnect_backoff_max", self.reconnect_backoff_max),
] {
if !value.secs_f64().is_finite() || value < Time::ZERO {
return Err(AdminError::InvalidConfig(format!(
"{name} must be finite and nonnegative"
)));
}
}
if self.reconnect_backoff_max < self.reconnect_backoff {
return Err(AdminError::InvalidConfig(
"reconnect_backoff_max is below reconnect_backoff".into(),
));
}
for (name, value) in [
("send_buffer", self.send_buffer),
("receive_buffer", self.receive_buffer),
] {
if let Some(value) = value {
let bytes = value.bytes_f64();
if !bytes.is_finite()
|| bytes < 1.0
|| bytes.fract() != 0.0
|| bytes > f64::from(i32::MAX)
{
return Err(AdminError::InvalidConfig(format!(
"{name} must contain positive whole int32 bytes"
)));
}
}
}
Ok(())
}
pub(crate) fn resolve(self) -> Result<ResolvedAdminConfig, AdminError> {
for (name, value) in [
("request_timeout", self.request_timeout),
(
"socket_connection_setup_timeout",
self.socket_connection_setup_timeout,
),
(
"socket_connection_setup_timeout_max",
self.socket_connection_setup_timeout_max,
),
] {
if !value.secs_f64().is_finite() || value <= Time::ZERO {
return Err(AdminError::InvalidConfig(format!(
"{name} must be positive and finite"
)));
}
}
if let Some(value) = self.default_api_timeout
&& (!value.secs_f64().is_finite() || value <= Time::ZERO)
{
return Err(AdminError::InvalidConfig(
"default_api_timeout must be positive and finite".to_owned(),
));
}
for (name, value) in [
("connections_max_idle", self.connections_max_idle),
("retry_backoff", self.retry_backoff),
("retry_backoff_max", self.retry_backoff_max),
] {
if !value.secs_f64().is_finite() || value < Time::ZERO {
return Err(AdminError::InvalidConfig(format!(
"{name} must be finite and not negative"
)));
}
}
self.validate_transport()?;
if self.socket_connection_setup_timeout_max < self.socket_connection_setup_timeout
|| self.retry_backoff_max < self.retry_backoff
{
return Err(AdminError::InvalidConfig(
"maximum delay is below initial delay".into(),
));
}
let api_timeout = default_api_timeout(self.request_timeout, self.default_api_timeout)?;
let client_id = match self.client_id {
Some(id) if !id.is_empty() => id,
_ => generate_client_id(),
};
let connection = ConnectionOptions {
client_id,
dns_timeout: self.dns_timeout,
client_dns_lookup: self.client_dns_lookup,
reconnect_backoff: self.reconnect_backoff,
reconnect_backoff_max: self.reconnect_backoff_max,
send_buffer: self.send_buffer,
receive_buffer: self.receive_buffer,
dispatch_queue_capacity: self.dispatch_queue_capacity,
frame_max: self.frame_max,
socket_connection_setup_timeout: self.socket_connection_setup_timeout,
socket_connection_setup_timeout_max: self.socket_connection_setup_timeout_max,
connections_max_idle: self.connections_max_idle,
request_timeout: self.request_timeout,
security: self.security.map(Box::new),
..ConnectionOptions::default()
};
connection.validate().map_err(AdminError::InvalidConfig)?;
Ok(ResolvedAdminConfig {
connection,
retry: RetryPolicy {
timeout: api_timeout.to_std(),
initial_backoff: self.retry_backoff.to_std(),
max_backoff: self.retry_backoff_max.to_std(),
jitter: RETRY_BACKOFF_JITTER,
max_retries: self.retries,
},
metadata_recovery_strategy: self.metadata_recovery_strategy,
metadata_recovery_rebootstrap_trigger: self.metadata_recovery_rebootstrap_trigger,
enable_metrics_push: self.enable_metrics_push,
})
}
}
fn generate_client_id() -> String {
format!(
"adminclient-{}",
ADMIN_CLIENT_ID_SEQUENCE.fetch_add(1, Ordering::Relaxed)
)
}
fn default_api_timeout(request_timeout: Time, explicit: Option<Time>) -> Result<Time, AdminError> {
match explicit {
Some(api_timeout) if api_timeout < request_timeout => Err(AdminError::InvalidConfig(
"default_api_timeout must be no smaller than request_timeout".to_owned(),
)),
Some(api_timeout) => Ok(api_timeout),
None if DEFAULT_API_TIMEOUT < request_timeout => {
tracing::warn!(
request_timeout_ms = request_timeout.millis_i64(),
"overriding the default of default.api.timeout.ms (60000) with the configured \
request timeout"
);
Ok(request_timeout)
}
None => Ok(DEFAULT_API_TIMEOUT),
}
}
pub(crate) fn retry_for_connection_options(options: &ConnectionOptions) -> RetryPolicy {
let timeout = if options.request_timeout > DEFAULT_API_TIMEOUT {
options.request_timeout
} else {
DEFAULT_API_TIMEOUT
};
RetryPolicy {
timeout: timeout.to_std(),
initial_backoff: DEFAULT_RETRY_BACKOFF.to_std(),
max_backoff: DEFAULT_RETRY_BACKOFF_MAX.to_std(),
jitter: RETRY_BACKOFF_JITTER,
max_retries: u32::MAX,
}
}
#[cfg(test)]
mod tests {
use std::{
sync::{
Arc, Mutex,
atomic::{AtomicUsize, Ordering},
},
time::Duration,
};
use assert2::{assert, check};
use bytes::{Buf, BytesMut};
use krabka_client_core::MockBroker;
use krabka_protocol::{
Encode,
owned::{
api_versions_request,
api_versions_response::{ApiVersion, ApiVersionsResponse},
unregister_broker_request,
unregister_broker_response::UnregisterBrokerResponse,
},
};
use super::*;
use crate::AdminClient;
#[derive(Debug, PartialEq)]
struct Observed {
client_id: String,
request_timeout: Time,
setup_timeouts: (Time, Time),
connections_max_idle: Time,
retry: RetryPolicy,
}
fn observe(config: AdminClientConfig) -> Result<Observed, String> {
config
.resolve()
.map(|resolved| Observed {
client_id: if resolved.connection.client_id.starts_with("adminclient-") {
"adminclient-<n>".to_owned()
} else {
resolved.connection.client_id
},
request_timeout: resolved.connection.request_timeout,
setup_timeouts: (
resolved.connection.socket_connection_setup_timeout,
resolved.connection.socket_connection_setup_timeout_max,
),
connections_max_idle: resolved.connection.connections_max_idle,
retry: resolved.retry,
})
.map_err(|error| error.to_string())
}
fn retry(timeout: Duration) -> RetryPolicy {
RetryPolicy {
timeout,
initial_backoff: Duration::from_millis(100),
max_backoff: Duration::from_secs(1),
jitter: 0.2,
max_retries: u32::MAX,
}
}
#[test]
fn settings_resolve_as_kafka_admin_client_config() {
let defaults = || Observed {
client_id: "adminclient-<n>".into(),
request_timeout: secs(30),
setup_timeouts: (secs(10), secs(30)),
connections_max_idle: krabka_units::minutes(5),
retry: retry(Duration::from_mins(1)),
};
for (name, config, expected) in [
("defaults", AdminClientConfig::default(), Ok(defaults())),
(
"request timeout 90 s raises the default api timeout",
AdminClientConfig {
request_timeout: secs(90),
..AdminClientConfig::default()
},
Ok(Observed {
request_timeout: secs(90),
retry: retry(Duration::from_secs(90)),
..defaults()
}),
),
(
"api timeout below the request timeout is an error",
AdminClientConfig {
request_timeout: secs(90),
default_api_timeout: Some(secs(60)),
..AdminClientConfig::default()
},
Err(
"invalid admin client configuration: default_api_timeout must be no smaller \
than request_timeout"
.to_owned(),
),
),
(
"api timeout equal to the request timeout",
AdminClientConfig {
default_api_timeout: Some(secs(30)),
..AdminClientConfig::default()
},
Ok(Observed {
retry: retry(Duration::from_secs(30)),
..defaults()
}),
),
(
"explicit client id",
AdminClientConfig {
client_id: Some("tool-a".into()),
..AdminClientConfig::default()
},
Ok(Observed {
client_id: "tool-a".into(),
..defaults()
}),
),
(
"empty client id is generated",
AdminClientConfig {
client_id: Some(String::new()),
..AdminClientConfig::default()
},
Ok(defaults()),
),
(
"retries, backoff and setup timeout",
AdminClientConfig {
retries: 3,
retry_backoff: millis(50),
retry_backoff_max: secs(2),
socket_connection_setup_timeout: secs(5),
socket_connection_setup_timeout_max: secs(20),
connections_max_idle: secs(60),
..AdminClientConfig::default()
},
Ok(Observed {
setup_timeouts: (secs(5), secs(20)),
connections_max_idle: secs(60),
retry: RetryPolicy {
initial_backoff: Duration::from_millis(50),
max_backoff: Duration::from_secs(2),
max_retries: 3,
..retry(Duration::from_mins(1))
},
..defaults()
}),
),
(
"zero request timeout is an error",
AdminClientConfig {
request_timeout: Time::ZERO,
..AdminClientConfig::default()
},
Err(
"invalid admin client configuration: request_timeout must be positive and \
finite"
.to_owned(),
),
),
(
"infinite request timeout is an error",
AdminClientConfig {
request_timeout: Time::from_secs_f64(f64::INFINITY),
..AdminClientConfig::default()
},
Err(
"invalid admin client configuration: request_timeout must be positive and \
finite"
.to_owned(),
),
),
(
"NaN api timeout is an error",
AdminClientConfig {
default_api_timeout: Some(Time::from_secs_f64(f64::NAN)),
..AdminClientConfig::default()
},
Err(
"invalid admin client configuration: default_api_timeout must be positive \
and finite"
.to_owned(),
),
),
(
"NaN retry backoff is an error",
AdminClientConfig {
retry_backoff: Time::from_secs_f64(f64::NAN),
..AdminClientConfig::default()
},
Err(
"invalid admin client configuration: retry_backoff must be finite and not \
negative"
.to_owned(),
),
),
] {
check!(observe(config) == expected, "{name}");
}
}
#[test]
fn generated_client_ids_follow_a_process_wide_sequence() {
let id = || {
AdminClientConfig::default()
.resolve()
.expect("defaults resolve")
.connection
.client_id
};
let number = |id: String| {
id.strip_prefix("adminclient-")
.and_then(|number| number.parse::<u32>().ok())
.expect("adminclient-<n>")
};
let first = number(id());
let second = number(id());
assert!(second > first);
assert!(first >= 1);
}
async fn timing_out_broker(
requests: Arc<AtomicUsize>,
client_ids: Arc<Mutex<Vec<String>>>,
) -> MockBroker {
MockBroker::start(move |api_key, version, _, request| {
let mut header = request;
let client_id_len = usize::try_from(header.get_i16()).expect("client id length");
client_ids
.lock()
.expect("client ids lock")
.push(String::from_utf8_lossy(&header[..client_id_len]).into_owned());
let mut body = BytesMut::new();
match api_key {
api_versions_request::API_KEY => ApiVersionsResponse {
api_keys: vec![
ApiVersion {
api_key: api_versions_request::API_KEY,
..Default::default()
},
ApiVersion {
api_key: unregister_broker_request::API_KEY,
..Default::default()
},
],
..Default::default()
}
.encode(&mut body, 0)
.unwrap(),
unregister_broker_request::API_KEY => {
requests.fetch_add(1, Ordering::SeqCst);
body.extend_from_slice(&[0]);
UnregisterBrokerResponse {
error_code: 7,
..Default::default()
}
.encode(&mut body, version)
.unwrap();
}
_ => return None,
}
Some(body.to_vec())
})
.await
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn a_client_uses_its_client_id_call_deadline_and_retries() {
for (name, config, expected_requests) in [
(
"no retries",
AdminClientConfig {
client_id: Some("tool-a".into()),
retries: 0,
..AdminClientConfig::default()
},
1..=1,
),
(
"two retries",
AdminClientConfig {
client_id: Some("tool-a".into()),
retries: 2,
retry_backoff: millis(1),
retry_backoff_max: millis(1),
..AdminClientConfig::default()
},
3..=3,
),
(
"the call deadline stops the retries",
AdminClientConfig {
client_id: Some("tool-a".into()),
request_timeout: millis(100),
default_api_timeout: Some(millis(400)),
retry_backoff: millis(100),
retry_backoff_max: millis(100),
..AdminClientConfig::default()
},
2..=6,
),
] {
let requests = Arc::new(AtomicUsize::new(0));
let client_ids = Arc::new(Mutex::new(Vec::new()));
let broker = timing_out_broker(Arc::clone(&requests), Arc::clone(&client_ids)).await;
let mut admin = AdminClient::connect_with_config(&[broker.addr.to_string()], config)
.await
.expect("admin connects");
let started = tokio::time::Instant::now();
let result = admin.unregister_broker(1).await;
let elapsed = started.elapsed();
broker.stop();
let code = match result {
Err(crate::AdminError::Broker { code, .. }) => Some(code),
_ => None,
};
let requests = requests.load(Ordering::SeqCst);
let client_ids = client_ids.lock().expect("client ids lock").clone();
check!(code == Some(7), "{name}");
check!(
expected_requests.contains(&requests),
"{name}: {requests} requests"
);
check!(elapsed < Duration::from_secs(5), "{name}");
check!(
client_ids.iter().all(|id| id == "tool-a") && !client_ids.is_empty(),
"{name}: {client_ids:?}"
);
}
}
}
#[cfg(test)]
mod transport_configuration_tests {
use assert2::assert;
use krabka_client_core::{ClientDnsLookup, ClientFrameMax, ConnectionDispatchQueueCapacity};
use krabka_units::{
convert::{ByteSizeExt as _, TimeExt as _},
kibibytes, mebibytes, millis,
};
use super::AdminClientConfig;
#[test]
fn admin_resolves_transport_resources_without_losing_retry_policy() {
let config = AdminClientConfig {
client_dns_lookup: ClientDnsLookup::ResolveCanonicalBootstrapServersOnly,
reconnect_backoff: millis(27),
reconnect_backoff_max: millis(127),
send_buffer: None,
receive_buffer: Some(kibibytes(19)),
dispatch_queue_capacity: ConnectionDispatchQueueCapacity::new(7).expect("capacity"),
frame_max: ClientFrameMax::try_from(mebibytes(2)).expect("frame"),
retry_backoff: millis(37),
..Default::default()
};
let resolved = config.resolve().expect("valid config");
let options = resolved.connection;
assert!(
(
options.client_dns_lookup,
options.reconnect_backoff,
options.reconnect_backoff_max,
options.send_buffer,
options.receive_buffer,
options.dispatch_queue_capacity.get(),
options.frame_max.size()
) == (
ClientDnsLookup::ResolveCanonicalBootstrapServersOnly,
millis(27),
millis(127),
None,
Some(kibibytes(19)),
7,
mebibytes(2)
)
);
assert!(resolved.retry.initial_backoff == std::time::Duration::from_millis(37));
let invalid = AdminClientConfig {
reconnect_backoff: millis(2000),
..Default::default()
};
assert!(invalid.resolve().is_err());
}
#[test]
fn admin_rejects_invalid_transport_bounds_and_socket_sizes() {
let invalid_backoff = AdminClientConfig {
reconnect_backoff: krabka_units::Time::from_secs_f64(f64::NAN),
..Default::default()
};
assert!(invalid_backoff.resolve().is_err());
let fractional_socket_buffer = AdminClientConfig {
send_buffer: Some(krabka_units::ByteSize::from_bytes_f64(0.5)),
..Default::default()
};
assert!(fractional_socket_buffer.resolve().is_err());
let inverted_setup_bounds = AdminClientConfig {
socket_connection_setup_timeout_max: krabka_units::millis(1),
..Default::default()
};
assert!(inverted_setup_bounds.resolve().is_err());
}
}