use std::{
collections::{BTreeMap, BTreeSet},
fmt,
};
use krabka_protocol::owned::{
describe_configs_request::{DescribeConfigsRequest, DescribeConfigsResource},
describe_configs_response::{
DescribeConfigsResourceResult, DescribeConfigsResponse, DescribeConfigsResult,
},
incremental_alter_configs_request::{
AlterConfigsResource, AlterableConfig, IncrementalAlterConfigsRequest,
},
incremental_alter_configs_response::IncrementalAlterConfigsResponse,
};
use crate::{
AdminClient, AdminError, KafkaError, NOT_CONTROLLER,
config_resources::{ConfigResource, ConfigResourceType},
groups::list_groups_kafka_error,
kafka_error_name,
log_dirs::{NodeTarget, call_node},
retry::{
ControllerRetry, CoordinatorRetry, REQUEST_TIMED_OUT, call_timeout_error,
is_connection_failure,
},
};
const UNKNOWN_SERVER_ERROR: i16 = -1;
const NOT_LEADER_OR_FOLLOWER: i16 = 6;
const DESCRIBE_CONFIGS_DOCUMENTATION_VERSION: i16 = 3;
impl ConfigResource {
#[must_use]
pub fn new(resource_type: ConfigResourceType, name: impl Into<String>) -> Self {
Self {
resource_type,
name: name.into(),
}
}
#[must_use]
pub fn topic(name: impl Into<String>) -> Self {
Self::new(ConfigResourceType::Topic, name)
}
#[must_use]
pub fn broker(broker_id: i32) -> Self {
Self::new(ConfigResourceType::Broker, broker_id.to_string())
}
#[must_use]
pub fn broker_default() -> Self {
Self::new(ConfigResourceType::Broker, "")
}
#[must_use]
pub fn broker_logger(broker_id: i32) -> Self {
Self::new(ConfigResourceType::BrokerLogger, broker_id.to_string())
}
#[must_use]
pub fn client_metrics(name: impl Into<String>) -> Self {
Self::new(ConfigResourceType::ClientMetrics, name)
}
#[must_use]
pub fn group(name: impl Into<String>) -> Self {
Self::new(ConfigResourceType::Group, name)
}
#[must_use]
pub fn is_default(&self) -> bool {
self.name.is_empty()
}
#[must_use]
pub fn dynamic_config_source(&self) -> Option<ConfigSource> {
match self.resource_type {
ConfigResourceType::Topic => Some(ConfigSource::DynamicTopicConfig),
ConfigResourceType::Broker if self.is_default() => {
Some(ConfigSource::DynamicDefaultBrokerConfig)
}
ConfigResourceType::Broker => Some(ConfigSource::DynamicBrokerConfig),
ConfigResourceType::BrokerLogger => Some(ConfigSource::DynamicBrokerLoggerConfig),
ConfigResourceType::ClientMetrics => Some(ConfigSource::DynamicClientMetricsConfig),
ConfigResourceType::Group => Some(ConfigSource::DynamicGroupConfig),
ConfigResourceType::Unknown => None,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum ConfigSource {
DynamicTopicConfig,
DynamicBrokerLoggerConfig,
DynamicBrokerConfig,
DynamicDefaultBrokerConfig,
DynamicClientMetricsConfig,
DynamicGroupConfig,
StaticBrokerConfig,
DefaultConfig,
Unknown,
}
impl ConfigSource {
#[must_use]
pub const fn id(self) -> i8 {
match self {
Self::Unknown => 0,
Self::DynamicTopicConfig => 1,
Self::DynamicBrokerConfig => 2,
Self::DynamicDefaultBrokerConfig => 3,
Self::StaticBrokerConfig => 4,
Self::DefaultConfig => 5,
Self::DynamicBrokerLoggerConfig => 6,
Self::DynamicClientMetricsConfig => 7,
Self::DynamicGroupConfig => 8,
}
}
#[must_use]
pub const fn from_id(id: i8) -> Option<Self> {
Some(match id {
i8::MIN..=-1 => return None,
1 => Self::DynamicTopicConfig,
2 => Self::DynamicBrokerConfig,
3 => Self::DynamicDefaultBrokerConfig,
4 => Self::StaticBrokerConfig,
5 => Self::DefaultConfig,
6 => Self::DynamicBrokerLoggerConfig,
7 => Self::DynamicClientMetricsConfig,
8 => Self::DynamicGroupConfig,
_ => Self::Unknown,
})
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum ConfigType {
Unknown,
Boolean,
String,
Int,
Short,
Long,
Double,
List,
Class,
Password,
}
impl ConfigType {
#[must_use]
pub const fn id(self) -> i8 {
match self {
Self::Unknown => 0,
Self::Boolean => 1,
Self::String => 2,
Self::Int => 3,
Self::Short => 4,
Self::Long => 5,
Self::Double => 6,
Self::List => 7,
Self::Class => 8,
Self::Password => 9,
}
}
#[must_use]
pub const fn from_id(id: i8) -> Option<Self> {
Some(match id {
i8::MIN..=-1 => return None,
1 => Self::Boolean,
2 => Self::String,
3 => Self::Int,
4 => Self::Short,
5 => Self::Long,
6 => Self::Double,
7 => Self::List,
8 => Self::Class,
9 => Self::Password,
_ => Self::Unknown,
})
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ConfigSynonym {
pub name: String,
pub value: Option<String>,
pub source: ConfigSource,
}
#[derive(Clone, PartialEq, Eq)]
pub struct ConfigEntry {
pub name: String,
pub value: Option<String>,
pub source: ConfigSource,
pub is_sensitive: bool,
pub is_read_only: bool,
pub synonyms: Vec<ConfigSynonym>,
pub config_type: ConfigType,
pub documentation: Option<String>,
}
impl ConfigEntry {
#[must_use]
pub fn is_default(&self) -> bool {
self.source == ConfigSource::DefaultConfig
}
}
impl fmt::Debug for ConfigEntry {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let value: &dyn fmt::Debug = if self.is_sensitive {
&"Redacted"
} else {
&self.value
};
f.debug_struct("ConfigEntry")
.field("name", &self.name)
.field("value", value)
.field("source", &self.source)
.field("is_sensitive", &self.is_sensitive)
.field("is_read_only", &self.is_read_only)
.field("synonyms", &self.synonyms)
.field("config_type", &self.config_type)
.field("documentation", &self.documentation)
.finish()
}
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct Config {
pub entries: BTreeMap<String, ConfigEntry>,
}
impl Config {
#[must_use]
pub fn get(&self, name: &str) -> Option<&ConfigEntry> {
self.entries.get(name)
}
#[must_use]
pub fn dynamic_overrides(&self, resource: &ConfigResource) -> BTreeMap<String, String> {
let Some(source) = resource.dynamic_config_source() else {
return BTreeMap::new();
};
self.entries
.values()
.filter(|entry| entry.source == source)
.filter_map(|entry| Some((entry.name.clone(), entry.value.clone()?)))
.collect()
}
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct DescribeConfigsOptions {
pub include_synonyms: bool,
pub include_documentation: bool,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum AlterConfigOpType {
Set,
Delete,
Append,
Subtract,
}
impl AlterConfigOpType {
#[must_use]
pub const fn id(self) -> i8 {
match self {
Self::Set => 0,
Self::Delete => 1,
Self::Append => 2,
Self::Subtract => 3,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct AlterConfigOp {
pub name: String,
pub value: Option<String>,
pub op_type: AlterConfigOpType,
}
impl AlterConfigOp {
#[must_use]
pub fn set(name: impl Into<String>, value: impl Into<String>) -> Self {
Self {
name: name.into(),
value: Some(value.into()),
op_type: AlterConfigOpType::Set,
}
}
#[must_use]
pub fn delete(name: impl Into<String>) -> Self {
Self {
name: name.into(),
value: None,
op_type: AlterConfigOpType::Delete,
}
}
#[must_use]
pub fn append(name: impl Into<String>, values: impl Into<String>) -> Self {
Self {
name: name.into(),
value: Some(values.into()),
op_type: AlterConfigOpType::Append,
}
}
#[must_use]
pub fn subtract(name: impl Into<String>, values: impl Into<String>) -> Self {
Self {
name: name.into(),
value: Some(values.into()),
op_type: AlterConfigOpType::Subtract,
}
}
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct IncrementalAlterConfigsOptions {
pub validate_only: bool,
}
pub type DescribeConfigsResults = BTreeMap<ConfigResource, Result<Config, KafkaError>>;
pub type AlterConfigsResults = BTreeMap<ConfigResource, Result<(), KafkaError>>;
fn node_for(resource: &ConfigResource) -> Result<Option<i32>, AdminError> {
let by_id = match resource.resource_type {
ConfigResourceType::Broker => !resource.is_default(),
ConfigResourceType::BrokerLogger => true,
_ => false,
};
if !by_id {
return Ok(None);
}
resource.name.parse().map(Some).map_err(|_| {
AdminError::InvalidArgument(format!(
"the name of a {:?} config resource must be a broker id, found {:?}",
resource.resource_type, resource.name
))
})
}
fn call_error(api: &str, error: &AdminError) -> KafkaError {
if is_connection_failure(error) {
return KafkaError {
code: REQUEST_TIMED_OUT,
name: kafka_error_name(REQUEST_TIMED_OUT),
message: Some(format!("{api} timed out: {error}")),
};
}
list_groups_kafka_error(error)
}
fn unknown_server_error(message: String) -> KafkaError {
KafkaError {
code: UNKNOWN_SERVER_ERROR,
name: kafka_error_name(UNKNOWN_SERVER_ERROR),
message: Some(message),
}
}
fn describe_request(
resources: &BTreeSet<ConfigResource>,
options: DescribeConfigsOptions,
) -> DescribeConfigsRequest {
DescribeConfigsRequest {
resources: resources
.iter()
.map(|resource| DescribeConfigsResource {
resource_type: resource.resource_type.id(),
resource_name: resource.name.clone(),
configuration_keys: None,
..Default::default()
})
.collect(),
include_synonyms: options.include_synonyms,
include_documentation: options.include_documentation,
..Default::default()
}
}
fn describe_min_version(options: DescribeConfigsOptions) -> Option<i16> {
options
.include_documentation
.then_some(DESCRIBE_CONFIGS_DOCUMENTATION_VERSION)
}
fn source_of(id: i8) -> Result<ConfigSource, String> {
ConfigSource::from_id(id).ok_or_else(|| format!("id should be positive, id: {id}"))
}
fn config_of(configs: Vec<DescribeConfigsResourceResult>) -> Result<Config, String> {
let mut entries = BTreeMap::new();
for config in configs {
let synonyms = config
.synonyms
.into_iter()
.map(|synonym| {
Ok(ConfigSynonym {
name: synonym.name,
value: synonym.value,
source: source_of(synonym.source)?,
})
})
.collect::<Result<_, String>>()?;
let entry = ConfigEntry {
source: source_of(config.config_source)?,
config_type: ConfigType::from_id(config.config_type)
.ok_or_else(|| format!("id should be positive, id: {}", config.config_type))?,
name: config.name,
value: config.value,
is_sensitive: config.is_sensitive,
is_read_only: config.read_only,
synonyms,
documentation: config.documentation,
};
entries.insert(entry.name.clone(), entry);
}
Ok(Config { entries })
}
fn describe_results(
asked: &BTreeSet<ConfigResource>,
answer: Result<DescribeConfigsResponse, KafkaError>,
) -> DescribeConfigsResults {
let fail_all = |error: KafkaError| {
asked
.iter()
.map(|resource| (resource.clone(), Err(error.clone())))
.collect()
};
let response = match answer {
Ok(response) => response,
Err(error) => return fail_all(error),
};
let mut answered = BTreeMap::<ConfigResource, DescribeConfigsResult>::new();
for result in response.results {
let resource = ConfigResource::new(
ConfigResourceType::from_id(result.resource_type),
result.resource_name.clone(),
);
if answered.insert(resource.clone(), result).is_some() {
return fail_all(unknown_server_error(format!(
"Duplicate key for config resource {resource:?}"
)));
}
}
let mut out = BTreeMap::new();
for resource in asked {
let result = match answered.remove(resource) {
None => Err(unknown_server_error(format!(
"The node response did not contain a result for config resource {resource:?}"
))),
Some(result) if result.error_code != 0 => Err(KafkaError {
code: result.error_code,
name: kafka_error_name(result.error_code),
message: result.error_message,
}),
Some(result) => match config_of(result.configs) {
Ok(config) => Ok(config),
Err(message) => return fail_all(unknown_server_error(message)),
},
};
out.insert(resource.clone(), result);
}
for resource in answered.into_keys() {
tracing::warn!(
?resource,
"the DescribeConfigs response names a resource that was not asked for"
);
}
out
}
fn alter_request(
resources: &BTreeMap<ConfigResource, &[AlterConfigOp]>,
options: IncrementalAlterConfigsOptions,
) -> IncrementalAlterConfigsRequest {
IncrementalAlterConfigsRequest {
resources: resources
.iter()
.map(|(resource, ops)| AlterConfigsResource {
resource_type: resource.resource_type.id(),
resource_name: resource.name.clone(),
configs: ops
.iter()
.map(|op| AlterableConfig {
name: op.name.clone(),
config_operation: op.op_type.id(),
value: op.value.clone(),
..Default::default()
})
.collect(),
..Default::default()
})
.collect(),
validate_only: options.validate_only,
..Default::default()
}
}
fn names_not_controller(response: &IncrementalAlterConfigsResponse, controllers: bool) -> bool {
response.responses.iter().any(|result| {
result.error_code == NOT_CONTROLLER
|| (controllers && result.error_code == NOT_LEADER_OR_FOLLOWER)
})
}
fn alter_results<'a>(
asked: impl Iterator<Item = &'a ConfigResource>,
answer: Result<IncrementalAlterConfigsResponse, KafkaError>,
) -> AlterConfigsResults {
let response = match answer {
Ok(response) => response,
Err(error) => {
return asked
.map(|resource| (resource.clone(), Err(error.clone())))
.collect();
}
};
let answered = response
.responses
.into_iter()
.map(|result| {
(
ConfigResource::new(
ConfigResourceType::from_id(result.resource_type),
result.resource_name,
),
(result.error_code, result.error_message),
)
})
.collect::<BTreeMap<_, _>>();
asked
.map(|resource| {
let result = match answered.get(resource) {
None => Err(unknown_server_error(format!(
"The node response did not contain a result for config resource {resource:?}"
))),
Some((0, _)) => Ok(()),
Some((code, message)) => Err(KafkaError {
code: *code,
name: kafka_error_name(*code),
message: message.clone(),
}),
};
(resource.clone(), result)
})
.collect()
}
impl AdminClient {
pub async fn describe_configs(
&self,
resources: &[ConfigResource],
options: DescribeConfigsOptions,
) -> Result<DescribeConfigsResults, AdminError> {
let mut by_node = BTreeMap::<Option<i32>, BTreeSet<ConfigResource>>::new();
for resource in resources {
by_node
.entry(node_for(resource)?)
.or_default()
.insert(resource.clone());
}
let start = self.retry.start();
let min_version = describe_min_version(options);
let answers = futures_util::future::join_all(by_node.iter().map(|(node, asked)| {
let request = describe_request(asked, options);
async move {
let answer = if let Some(node_id) = *node {
call_node(
&self.conn,
NodeTarget {
node_id,
supports_controllers: true,
min_version,
},
&self.options,
request,
CoordinatorRetry::from_deadline(start),
)
.await
} else {
let retry = ControllerRetry::new("DescribeConfigs", self.retry);
let sent = match min_version {
Some(min_version) => {
retry
.bounded(self.conn.send_at_least(request, min_version))
.await
}
None => retry.bounded(self.conn.send(request)).await,
};
sent.map_err(|error| call_error("DescribeConfigs", &error))
};
describe_results(asked, answer)
}
}))
.await;
Ok(answers.into_iter().flatten().collect())
}
pub async fn incremental_alter_configs(
&mut self,
configs: &BTreeMap<ConfigResource, Vec<AlterConfigOp>>,
options: IncrementalAlterConfigsOptions,
) -> Result<AlterConfigsResults, AdminError> {
let controllers = self.conn.uses_controller_bootstrap();
let mut unified = BTreeMap::new();
let mut by_node = Vec::new();
for (resource, ops) in configs {
let node = node_for(resource)?.filter(|_| {
!controllers || resource.resource_type == ConfigResourceType::BrokerLogger
});
match node {
Some(node_id) => by_node.push((node_id, resource, ops.as_slice())),
None => {
unified.insert(resource.clone(), ops.as_slice());
}
}
}
let start = self.retry.start();
let (conn, client_options) = (&self.conn, &self.options);
let node_answers =
futures_util::future::join_all(by_node.iter().map(|(node_id, resource, ops)| {
let request = alter_request(&BTreeMap::from([((*resource).clone(), *ops)]), options);
let target = NodeTarget {
node_id: *node_id,
supports_controllers: true,
min_version: None,
};
async move {
let mut deadline = start;
let mut attempts = 1;
let answer = loop {
let answer = call_node(
conn,
target,
client_options,
request.clone(),
CoordinatorRetry::from_deadline(deadline),
)
.await;
if !matches!(&answer, Ok(response) if names_not_controller(response, controllers))
{
break answer;
}
let timed_out = || {
Err(call_timeout_error(
"IncrementalAlterConfigs",
attempts,
"NOT_CONTROLLER",
))
};
if deadline.exhausted() {
break timed_out();
}
deadline.backoff().await;
if deadline.expired() {
break timed_out();
}
attempts += 1;
};
alter_results(std::iter::once(*resource), answer)
}
}))
.await;
let mut out: AlterConfigsResults = node_answers.into_iter().flatten().collect();
if !unified.is_empty() {
let answer = self.alter_unified(&unified, options, controllers).await;
out.extend(alter_results(unified.keys(), answer));
}
Ok(out)
}
async fn alter_unified(
&mut self,
resources: &BTreeMap<ConfigResource, &[AlterConfigOp]>,
options: IncrementalAlterConfigsOptions,
controllers: bool,
) -> Result<IncrementalAlterConfigsResponse, KafkaError> {
const API: &str = "IncrementalAlterConfigs";
let request = alter_request(resources, options);
let mut retry = ControllerRetry::new(API, self.retry);
loop {
let response = retry
.bounded(self.conn.send(request.clone()))
.await
.map_err(|error| call_error(API, &error))?;
if !names_not_controller(&response, controllers) {
return Ok(response);
}
retry
.after_not_controller(self)
.await
.map_err(|error| call_error(API, &error))?;
}
}
}
#[cfg(test)]
mod tests;