use std::collections::{BTreeMap, BTreeSet};
use bytes::BufMut;
use krabka_client_core::{
AuthenticationError, ClientError, Connection, ConnectionOptions, CoordinatorKeyType,
SaslAuthenticationError, build_find_coordinator, coordinator_endpoint,
};
use krabka_protocol::{
Encode, ProtocolError, ProtocolRequest,
owned::{
alter_share_group_offsets_request::{
AlterShareGroupOffsetsRequest, AlterShareGroupOffsetsRequestPartition,
AlterShareGroupOffsetsRequestTopic,
},
alter_share_group_offsets_response::AlterShareGroupOffsetsResponse,
delete_share_group_offsets_request::{
DeleteShareGroupOffsetsRequest, DeleteShareGroupOffsetsRequestTopic,
},
delete_share_group_offsets_response::DeleteShareGroupOffsetsResponse,
leave_group_request::{self, LeaveGroupRequest, MemberIdentity},
leave_group_response::LeaveGroupResponse,
list_groups_request::{self, ListGroupsRequest},
list_groups_response::ListedGroup,
metadata_request::MetadataRequest,
metadata_response::MetadataResponseBroker,
offset_commit_request::{
self, OffsetCommitRequest, OffsetCommitRequestPartition, OffsetCommitRequestTopic,
},
offset_commit_response::OffsetCommitResponse,
offset_delete_request::{
OffsetDeleteRequest, OffsetDeleteRequestPartition, OffsetDeleteRequestTopic,
},
offset_delete_response::OffsetDeleteResponse,
offset_fetch_request::{self, OffsetFetchRequest, OffsetFetchRequestGroup},
offset_fetch_response::OffsetFetchResponse,
},
};
use crate::{
AdminClient, AdminError, KafkaError, format_host_port,
group_descriptions::DescribeGroupsOptions,
kafka_error_if, kafka_error_name,
retry::{CoordinatorRetry, RetryAction, RetryPolicy, connection_failure_action},
send_connection_at_least,
};
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ConsumerGroupOffsetOutcome {
pub topic: String,
pub partition: i32,
pub error: Option<KafkaError>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum GroupState {
Unknown,
PreparingRebalance,
CompletingRebalance,
Stable,
Dead,
Empty,
Assigning,
Reconciling,
NotReady,
}
impl GroupState {
const ALL: [Self; 9] = [
Self::Unknown,
Self::PreparingRebalance,
Self::CompletingRebalance,
Self::Stable,
Self::Dead,
Self::Empty,
Self::Assigning,
Self::Reconciling,
Self::NotReady,
];
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
Self::Unknown => "Unknown",
Self::PreparingRebalance => "PreparingRebalance",
Self::CompletingRebalance => "CompletingRebalance",
Self::Stable => "Stable",
Self::Dead => "Dead",
Self::Empty => "Empty",
Self::Assigning => "Assigning",
Self::Reconciling => "Reconciling",
Self::NotReady => "NotReady",
}
}
#[must_use]
pub fn parse(name: &str) -> Self {
Self::ALL
.into_iter()
.find(|state| state.as_str().eq_ignore_ascii_case(name))
.unwrap_or(Self::Unknown)
}
}
impl std::fmt::Display for GroupState {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(self.as_str())
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum GroupType {
Unknown,
Consumer,
Classic,
Share,
Streams,
}
impl GroupType {
const ALL: [Self; 5] = [
Self::Unknown,
Self::Consumer,
Self::Classic,
Self::Share,
Self::Streams,
];
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
Self::Unknown => "Unknown",
Self::Consumer => "Consumer",
Self::Classic => "Classic",
Self::Share => "Share",
Self::Streams => "Streams",
}
}
#[must_use]
pub fn parse(name: &str) -> Self {
Self::ALL
.into_iter()
.find(|group_type| group_type.as_str().eq_ignore_ascii_case(name))
.unwrap_or(Self::Unknown)
}
}
impl std::fmt::Display for GroupType {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(self.as_str())
}
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct ListGroupsOptions {
pub group_states: BTreeSet<GroupState>,
pub types: BTreeSet<GroupType>,
pub protocol_types: BTreeSet<String>,
}
impl ListGroupsOptions {
#[must_use]
pub fn for_consumer_groups() -> Self {
Self {
group_states: BTreeSet::new(),
types: BTreeSet::from([GroupType::Classic, GroupType::Consumer]),
protocol_types: BTreeSet::from([String::new(), "consumer".to_owned()]),
}
}
#[must_use]
pub fn for_share_groups() -> Self {
Self {
types: BTreeSet::from([GroupType::Share]),
..Self::default()
}
}
#[must_use]
pub fn for_streams_groups() -> Self {
Self {
types: BTreeSet::from([GroupType::Streams]),
..Self::default()
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct GroupListing {
pub group_id: String,
pub group_type: Option<GroupType>,
pub protocol_type: String,
pub group_state: Option<GroupState>,
}
impl GroupListing {
#[must_use]
pub fn is_simple_consumer_group(&self) -> bool {
self.group_type == Some(GroupType::Classic) && self.protocol_type.is_empty()
}
}
impl From<ListedGroup> for GroupListing {
fn from(group: ListedGroup) -> Self {
Self {
group_type: (!group.group_type.is_empty()).then(|| GroupType::parse(&group.group_type)),
group_state: (!group.group_state.is_empty())
.then(|| GroupState::parse(&group.group_state)),
group_id: group.group_id,
protocol_type: group.protocol_type,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ListGroupsError {
pub node_id: i32,
pub host: String,
pub port: i32,
pub error: KafkaError,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ListGroupsResult {
pub valid: Vec<GroupListing>,
pub errors: Vec<ListGroupsError>,
}
impl ListGroupsResult {
pub fn all(self) -> Result<Vec<GroupListing>, ListGroupsError> {
match self.errors.into_iter().next() {
Some(error) => Err(error),
None => Ok(self.valid),
}
}
}
impl AdminClient {
pub async fn alter_consumer_group_offsets(
&mut self,
group: &str,
offsets: &BTreeMap<(String, i32), i64>,
) -> Result<Vec<ConsumerGroupOffsetOutcome>, AdminError> {
self.alter_consumer_group_offsets_with_retry(group, offsets, self.retry)
.await
}
async fn alter_consumer_group_offsets_with_retry(
&mut self,
group: &str,
offsets: &BTreeMap<(String, i32), i64>,
retry: RetryPolicy,
) -> Result<Vec<ConsumerGroupOffsetOutcome>, AdminError> {
let mut retry = CoordinatorRetry::new(retry);
loop {
let action = retry
.run(self.offset_commit_attempt(group, offsets, retry.find_coordinator()))
.await;
if let Some(result) = retry.next(action).await {
return result;
}
}
}
async fn offset_commit_attempt(
&mut self,
group: &str,
offsets: &BTreeMap<(String, i32), i64>,
find_coordinator: bool,
) -> RetryAction<Vec<ConsumerGroupOffsetOutcome>> {
if find_coordinator && let Err(action) = self.find_group_coordinator_attempt(group).await {
return action;
}
match self.conn.send(offset_commit_request(group, offsets)).await {
Ok(response) => offset_commit_retry_action(response),
Err(error) => connection_failure_action(error),
}
}
pub async fn list_groups(
&self,
options: &ListGroupsOptions,
) -> Result<ListGroupsResult, AdminError> {
self.list_groups_with_retry(options, self.retry).await
}
async fn list_groups_with_retry(
&self,
options: &ListGroupsOptions,
retry: RetryPolicy,
) -> Result<ListGroupsResult, AdminError> {
let start = tokio::time::Instant::now();
let brokers = self.list_groups_brokers(start, retry).await?;
let request = ListGroupsRequest {
states_filter: options
.group_states
.iter()
.map(|state| state.as_str().to_owned())
.collect(),
types_filter: options
.types
.iter()
.map(|group_type| group_type.as_str().to_owned())
.collect(),
..Default::default()
};
let min_version = list_groups_min_version(options);
let answers = futures_util::future::join_all(brokers.iter().map(|broker| {
list_groups_on_broker(
broker,
self.options.clone(),
request.clone(),
min_version,
start,
retry,
)
}))
.await;
let mut listings = BTreeMap::new();
let mut errors = Vec::new();
for (broker, answer) in brokers.into_iter().zip(answers) {
match answer {
Ok(groups) => {
for group in groups {
if options.protocol_types.is_empty()
|| options.protocol_types.contains(&group.protocol_type)
{
let listing = GroupListing::from(group);
listings.insert(listing.group_id.clone(), listing);
}
}
}
Err(error) => errors.push(ListGroupsError {
node_id: broker.node_id,
host: broker.host,
port: broker.port,
error,
}),
}
}
Ok(ListGroupsResult {
valid: listings.into_values().collect(),
errors,
})
}
async fn list_groups_brokers(
&self,
start: tokio::time::Instant,
retry: RetryPolicy,
) -> Result<Vec<MetadataResponseBroker>, AdminError> {
let mut backoff = retry.initial_backoff;
loop {
let response = self
.conn
.send(MetadataRequest {
topics: Some(Vec::new()),
allow_auto_topic_creation: true,
..Default::default()
})
.await?;
let mut brokers = response.brokers;
let mut seen = BTreeSet::new();
brokers.retain(|broker| seen.insert(broker.node_id));
if !brokers.is_empty() {
return Ok(brokers);
}
if start.elapsed() >= retry.timeout {
return Err(AdminError::Protocol(
"failed to find brokers to send ListGroups: metadata has no brokers".to_owned(),
));
}
tokio::time::sleep(backoff).await;
backoff = backoff.saturating_mul(2).min(retry.max_backoff);
}
}
pub async fn list_consumer_group_offsets(
&mut self,
group: &str,
) -> Result<BTreeMap<(String, i32), i64>, AdminError> {
self.list_consumer_group_offsets_with_retry(group, self.retry)
.await
}
async fn list_consumer_group_offsets_with_retry(
&mut self,
group: &str,
retry: RetryPolicy,
) -> Result<BTreeMap<(String, i32), i64>, AdminError> {
let mut retry = CoordinatorRetry::new(retry);
loop {
let action = retry
.run(self.offset_fetch_attempt(group, retry.find_coordinator()))
.await;
if let Some(result) = retry.next(action).await {
return result;
}
}
}
async fn offset_fetch_attempt(
&mut self,
group: &str,
find_coordinator: bool,
) -> RetryAction<BTreeMap<(String, i32), i64>> {
if find_coordinator && let Err(action) = self.find_group_coordinator_attempt(group).await {
return action;
}
match self.conn.send(offset_fetch_request(group)).await {
Ok(response) => offset_fetch_retry_action(committed_offsets(group, response)),
Err(error) => connection_failure_action(error),
}
}
async fn find_group_coordinator_attempt<T>(
&mut self,
group: &str,
) -> Result<(), RetryAction<T>> {
match self.reconnect_group_coordinator(group).await {
Ok(()) => Ok(()),
Err(AdminError::Transport(ClientError::Server { error_code }))
if is_retriable_find_coordinator_error(error_code) =>
{
Err(RetryAction::FindCoordinator(Err(AdminError::Transport(
ClientError::Server { error_code },
))))
}
Err(error) => Err(connection_failure_action(error)),
}
}
async fn reconnect_group_coordinator(&mut self, group: &str) -> Result<(), AdminError> {
let response = self
.conn
.send(build_find_coordinator(group, CoordinatorKeyType::Group))
.await?;
let coordinator = coordinator_endpoint(group, response)?;
self.reconnect(&format_host_port(&coordinator.host, coordinator.port))
.await
}
pub async fn delete_consumer_group_offsets(
&mut self,
group: &str,
partitions: &[(String, i32)],
) -> Result<Vec<ConsumerGroupOffsetOutcome>, AdminError> {
self.delete_consumer_group_offsets_with_retry(group, partitions, self.retry)
.await
}
pub async fn delete_streams_group_offsets(
&mut self,
group: &str,
partitions: &[(String, i32)],
) -> Result<Vec<ConsumerGroupOffsetOutcome>, AdminError> {
self.delete_consumer_group_offsets(group, partitions).await
}
async fn delete_consumer_group_offsets_with_retry(
&mut self,
group: &str,
partitions: &[(String, i32)],
retry: RetryPolicy,
) -> Result<Vec<ConsumerGroupOffsetOutcome>, AdminError> {
let mut retry = CoordinatorRetry::new(retry);
loop {
let action = retry
.run(self.offset_delete_attempt(group, partitions, retry.find_coordinator()))
.await;
if let Some(result) = retry.next(action).await {
return result;
}
}
}
async fn offset_delete_attempt(
&mut self,
group: &str,
partitions: &[(String, i32)],
find_coordinator: bool,
) -> RetryAction<Vec<ConsumerGroupOffsetOutcome>> {
if find_coordinator && let Err(action) = self.find_group_coordinator_attempt(group).await {
return action;
}
match self
.conn
.send(offset_delete_request(group, partitions))
.await
{
Ok(response) => offset_delete_retry_action(group, response),
Err(error) => connection_failure_action(error),
}
}
pub async fn remove_members_from_consumer_group(
&mut self,
group: &str,
members: &[MemberToRemove],
options: &RemoveMembersOptions,
) -> Result<RemovedMembers, AdminError> {
let reason = leave_group_reason(options.reason.as_deref());
let identities = if members.is_empty() {
self.all_members_of(group, &reason).await?
} else {
members
.iter()
.map(|member| MemberIdentity {
member_id: String::new(),
group_instance_id: Some(member.group_instance_id.clone()),
reason: Some(reason.clone()),
..Default::default()
})
.collect()
};
self.remove_members_from_consumer_group_with_retry(group, &identities, self.retry)
.await
}
async fn all_members_of(
&self,
group: &str,
reason: &str,
) -> Result<Vec<MemberIdentity>, AdminError> {
let description = self
.describe_consumer_groups(&[group], DescribeGroupsOptions::default())
.await
.remove(group)
.unwrap_or_else(|| {
Err(KafkaError {
code: UNKNOWN_SERVER_ERROR,
name: kafka_error_name(UNKNOWN_SERVER_ERROR),
message: Some(format!("no description for group {group}")),
})
})
.map_err(|error| AdminError::Broker {
api: "describeConsumerGroups",
code: error.code,
name: error.name,
message: Some(format!(
"Encounter exception when trying to get members from group: {group}{}",
error
.message
.map(|message| format!(": {message}"))
.unwrap_or_default()
)),
})?;
Ok(description
.members
.into_iter()
.map(|member| {
let (member_id, group_instance_id) = match member.group_instance_id {
Some(instance_id) => (String::new(), Some(instance_id)),
None => (member.member_id, None),
};
MemberIdentity {
member_id,
group_instance_id,
reason: Some(reason.to_owned()),
..Default::default()
}
})
.collect())
}
async fn remove_members_from_consumer_group_with_retry(
&mut self,
group: &str,
members: &[MemberIdentity],
retry: RetryPolicy,
) -> Result<RemovedMembers, AdminError> {
let mut retry = CoordinatorRetry::new(retry);
loop {
let action = retry
.run(self.leave_group_attempt(group, members, retry.find_coordinator()))
.await;
if let Some(result) = retry.next(action).await {
return result;
}
}
}
async fn leave_group_attempt(
&mut self,
group: &str,
members: &[MemberIdentity],
find_coordinator: bool,
) -> RetryAction<RemovedMembers> {
if find_coordinator && let Err(action) = self.find_group_coordinator_attempt(group).await {
return action;
}
let request = MembersOnlyLeaveGroup(LeaveGroupRequest {
group_id: group.into(),
members: members.to_vec(),
..Default::default()
});
match self.conn.send(request).await {
Ok(response) => leave_group_retry_action(response),
Err(error) => connection_failure_action(error),
}
}
pub async fn alter_share_group_offsets(
&mut self,
group: &str,
offsets: &BTreeMap<(String, i32), i64>,
) -> Result<Vec<ConsumerGroupOffsetOutcome>, AdminError> {
self.alter_share_group_offsets_with_retry(group, offsets, self.retry)
.await
}
pub async fn alter_streams_group_offsets(
&mut self,
group: &str,
offsets: &BTreeMap<(String, i32), i64>,
) -> Result<Vec<ConsumerGroupOffsetOutcome>, AdminError> {
self.alter_consumer_group_offsets(group, offsets).await
}
pub async fn list_streams_group_offsets(
&mut self,
group: &str,
) -> Result<BTreeMap<(String, i32), i64>, AdminError> {
self.list_consumer_group_offsets(group).await
}
async fn alter_share_group_offsets_with_retry(
&mut self,
group: &str,
offsets: &BTreeMap<(String, i32), i64>,
retry: RetryPolicy,
) -> Result<Vec<ConsumerGroupOffsetOutcome>, AdminError> {
let mut retry = CoordinatorRetry::new(retry);
loop {
let action = retry
.run(self.alter_share_group_offsets_attempt(
group,
offsets,
retry.find_coordinator(),
))
.await;
if let Some(result) = retry.next(action).await {
return result;
}
}
}
async fn alter_share_group_offsets_attempt(
&mut self,
group: &str,
offsets: &BTreeMap<(String, i32), i64>,
find_coordinator: bool,
) -> RetryAction<Vec<ConsumerGroupOffsetOutcome>> {
if find_coordinator && let Err(action) = self.find_group_coordinator_attempt(group).await {
return action;
}
match self
.conn
.send(alter_share_group_offsets_request(group, offsets))
.await
{
Ok(response) => alter_share_group_offsets_retry_action(offsets, response),
Err(error) => connection_failure_action(error),
}
}
pub async fn delete_share_group_offsets(
&mut self,
group: &str,
topics: &[String],
) -> Result<Vec<(String, Option<KafkaError>)>, AdminError> {
self.delete_share_group_offsets_with_retry(group, topics, self.retry)
.await
}
async fn delete_share_group_offsets_with_retry(
&mut self,
group: &str,
topics: &[String],
retry: RetryPolicy,
) -> Result<Vec<(String, Option<KafkaError>)>, AdminError> {
let mut retry = CoordinatorRetry::new(retry);
loop {
let action = retry
.run(self.delete_share_group_offsets_attempt(
group,
topics,
retry.find_coordinator(),
))
.await;
if let Some(result) = retry.next(action).await {
return result;
}
}
}
async fn delete_share_group_offsets_attempt(
&mut self,
group: &str,
topics: &[String],
find_coordinator: bool,
) -> RetryAction<Vec<(String, Option<KafkaError>)>> {
if find_coordinator && let Err(action) = self.find_group_coordinator_attempt(group).await {
return action;
}
let request = DeleteShareGroupOffsetsRequest {
group_id: group.into(),
topics: topics
.iter()
.map(|topic_name| DeleteShareGroupOffsetsRequestTopic {
topic_name: topic_name.clone(),
..Default::default()
})
.collect(),
..Default::default()
};
match self.conn.send(request).await {
Ok(response) => delete_share_group_offsets_retry_action(response),
Err(error) => connection_failure_action(error),
}
}
}
const COORDINATOR_LOAD_IN_PROGRESS: i16 = 14;
const COORDINATOR_NOT_AVAILABLE: i16 = 15;
const NOT_COORDINATOR: i16 = 16;
fn is_retriable_find_coordinator_error(code: i16) -> bool {
matches!(
code,
COORDINATOR_LOAD_IN_PROGRESS | COORDINATOR_NOT_AVAILABLE
)
}
fn offset_fetch_retry_action<T>(result: Result<T, AdminError>) -> RetryAction<T> {
match result {
Err(AdminError::Broker {
code: COORDINATOR_LOAD_IN_PROGRESS,
..
}) => RetryAction::SameCoordinator(result),
Err(AdminError::Broker {
code: COORDINATOR_NOT_AVAILABLE | NOT_COORDINATOR,
..
}) => RetryAction::FindCoordinator(result),
result => RetryAction::Done(result),
}
}
const REBALANCE_IN_PROGRESS: i16 = 27;
fn offset_commit_retry_action(
response: OffsetCommitResponse,
) -> RetryAction<Vec<ConsumerGroupOffsetOutcome>> {
let outcomes: Vec<ConsumerGroupOffsetOutcome> = response
.topics
.into_iter()
.flat_map(|topic| {
let name = topic.name;
topic
.partitions
.into_iter()
.map(move |partition| ConsumerGroupOffsetOutcome {
topic: name.clone(),
partition: partition.partition_index,
error: kafka_error_if(partition.error_code, None),
})
})
.collect();
let codes = || {
outcomes
.iter()
.filter_map(|outcome| outcome.error.as_ref().map(|error| error.code))
};
if codes().any(|code| matches!(code, COORDINATOR_NOT_AVAILABLE | NOT_COORDINATOR)) {
RetryAction::FindCoordinator(Ok(outcomes))
} else if codes()
.any(|code| matches!(code, COORDINATOR_LOAD_IN_PROGRESS | REBALANCE_IN_PROGRESS))
{
RetryAction::SameCoordinator(Ok(outcomes))
} else {
RetryAction::Done(Ok(outcomes))
}
}
fn offset_delete_request(group: &str, partitions: &[(String, i32)]) -> OffsetDeleteRequest {
let mut topics = BTreeMap::<String, Vec<OffsetDeleteRequestPartition>>::new();
for (topic, partition) in partitions {
topics
.entry(topic.clone())
.or_default()
.push(OffsetDeleteRequestPartition {
partition_index: *partition,
..Default::default()
});
}
OffsetDeleteRequest {
group_id: group.into(),
topics: topics
.into_iter()
.map(|(name, partitions)| OffsetDeleteRequestTopic {
name,
partitions,
..Default::default()
})
.collect(),
..Default::default()
}
}
fn offset_delete_retry_action(
group: &str,
response: OffsetDeleteResponse,
) -> RetryAction<Vec<ConsumerGroupOffsetOutcome>> {
if response.error_code != 0 {
let result = Err(AdminError::Broker {
api: "OffsetDelete",
code: response.error_code,
name: kafka_error_name(response.error_code),
message: Some(format!("group={group}")),
});
return match response.error_code {
COORDINATOR_LOAD_IN_PROGRESS => RetryAction::SameCoordinator(result),
COORDINATOR_NOT_AVAILABLE | NOT_COORDINATOR => RetryAction::FindCoordinator(result),
_ => RetryAction::Done(result),
};
}
let outcomes = response
.topics
.into_iter()
.flat_map(|topic| {
let name = topic.name;
topic
.partitions
.into_iter()
.map(move |partition| ConsumerGroupOffsetOutcome {
topic: name.clone(),
partition: partition.partition_index,
error: kafka_error_if(partition.error_code, None),
})
})
.collect();
RetryAction::Done(Ok(outcomes))
}
#[derive(Clone, Debug, PartialEq, Eq)]
struct MembersOnlyLeaveGroup(LeaveGroupRequest);
impl Encode for MembersOnlyLeaveGroup {
fn encode<B: BufMut>(&self, buf: &mut B, version: i16) -> Result<(), ProtocolError> {
self.0.encode(buf, version)
}
fn encoded_len(&self, version: i16) -> usize {
self.0.encoded_len(version)
}
}
impl ProtocolRequest for MembersOnlyLeaveGroup {
const API_KEY: i16 = leave_group_request::API_KEY;
const MIN_VERSION: i16 = 3;
const MAX_VERSION: i16 = leave_group_request::MAX_VERSION;
const LATEST_STABLE_VERSION: i16 = leave_group_request::LATEST_STABLE_VERSION;
const FLEXIBLE_MIN: i16 = leave_group_request::FLEXIBLE_MIN;
type Response = LeaveGroupResponse;
}
fn leave_group_retry_action(response: LeaveGroupResponse) -> RetryAction<RemovedMembers> {
if response.error_code != 0 {
let result = Err(AdminError::Broker {
api: "LeaveGroup",
code: response.error_code,
name: kafka_error_name(response.error_code),
message: None,
});
return match response.error_code {
COORDINATOR_LOAD_IN_PROGRESS => RetryAction::SameCoordinator(result),
COORDINATOR_NOT_AVAILABLE | NOT_COORDINATOR => RetryAction::FindCoordinator(result),
_ => RetryAction::Done(result),
};
}
let members = response
.members
.into_iter()
.map(|member| {
(
RemovedMember {
member_id: member.member_id,
group_instance_id: member.group_instance_id,
},
kafka_error_if(member.error_code, None),
)
})
.collect();
RetryAction::Done(Ok(RemovedMembers { members }))
}
const UNKNOWN_SERVER_ERROR: i16 = -1;
const DEFAULT_LEAVE_GROUP_REASON: &str = "member was removed by an admin";
const MAX_REASON_CHARS: usize = 255;
fn leave_group_reason(reason: Option<&str>) -> String {
match reason {
Some(reason) if !reason.is_empty() => reason.chars().take(MAX_REASON_CHARS).collect(),
_ => DEFAULT_LEAVE_GROUP_REASON.to_owned(),
}
}
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct MemberToRemove {
pub group_instance_id: String,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct RemoveMembersOptions {
pub reason: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct RemovedMember {
pub member_id: String,
pub group_instance_id: Option<String>,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct RemovedMembers {
pub members: BTreeMap<RemovedMember, Option<KafkaError>>,
}
impl RemovedMembers {
pub fn member_result(&self, member: &MemberToRemove) -> Result<(), KafkaError> {
let key = RemovedMember {
member_id: String::new(),
group_instance_id: Some(member.group_instance_id.clone()),
};
match self.members.get(&key) {
Some(None) => Ok(()),
Some(Some(error)) => Err(error.clone()),
None => Err(KafkaError {
code: UNKNOWN_SERVER_ERROR,
name: kafka_error_name(UNKNOWN_SERVER_ERROR),
message: Some(format!(
"Member \"{}\" was not included in the removal response",
member.group_instance_id
)),
}),
}
}
}
fn alter_share_group_offsets_request(
group: &str,
offsets: &BTreeMap<(String, i32), i64>,
) -> AlterShareGroupOffsetsRequest {
let mut topics = BTreeMap::<String, Vec<AlterShareGroupOffsetsRequestPartition>>::new();
for ((topic, partition), start_offset) in offsets {
topics
.entry(topic.clone())
.or_default()
.push(AlterShareGroupOffsetsRequestPartition {
partition_index: *partition,
start_offset: *start_offset,
..Default::default()
});
}
AlterShareGroupOffsetsRequest {
group_id: group.into(),
topics: topics
.into_iter()
.map(
|(topic_name, partitions)| AlterShareGroupOffsetsRequestTopic {
topic_name,
partitions,
..Default::default()
},
)
.collect(),
..Default::default()
}
}
fn alter_share_group_offsets_retry_action(
offsets: &BTreeMap<(String, i32), i64>,
response: AlterShareGroupOffsetsResponse,
) -> RetryAction<Vec<ConsumerGroupOffsetOutcome>> {
if response.error_code != 0 {
let result = Err(AdminError::Broker {
api: "AlterShareGroupOffsets",
code: response.error_code,
name: kafka_error_name(response.error_code),
message: response.error_message.clone(),
});
return match response.error_code {
COORDINATOR_LOAD_IN_PROGRESS | REBALANCE_IN_PROGRESS => {
RetryAction::SameCoordinator(result)
}
COORDINATOR_NOT_AVAILABLE | NOT_COORDINATOR => RetryAction::FindCoordinator(result),
code => RetryAction::Done(Ok(offsets
.keys()
.map(|(topic, partition)| ConsumerGroupOffsetOutcome {
topic: topic.clone(),
partition: *partition,
error: kafka_error_if(code, response.error_message.clone()),
})
.collect())),
};
}
let outcomes = response
.responses
.into_iter()
.flat_map(|topic| {
let name = topic.topic_name;
topic
.partitions
.into_iter()
.map(move |partition| ConsumerGroupOffsetOutcome {
topic: name.clone(),
partition: partition.partition_index,
error: kafka_error_if(partition.error_code, partition.error_message),
})
})
.collect();
RetryAction::Done(Ok(outcomes))
}
fn delete_share_group_offsets_retry_action(
response: DeleteShareGroupOffsetsResponse,
) -> RetryAction<Vec<(String, Option<KafkaError>)>> {
if response.error_code != 0 {
let result = Err(AdminError::Broker {
api: "DeleteShareGroupOffsets",
code: response.error_code,
name: kafka_error_name(response.error_code),
message: response.error_message,
});
return match response.error_code {
COORDINATOR_LOAD_IN_PROGRESS | REBALANCE_IN_PROGRESS => {
RetryAction::SameCoordinator(result)
}
COORDINATOR_NOT_AVAILABLE | NOT_COORDINATOR => RetryAction::FindCoordinator(result),
_ => RetryAction::Done(result),
};
}
let outcomes = response
.responses
.into_iter()
.map(|topic| {
(
topic.topic_name,
kafka_error_if(topic.error_code, topic.error_message),
)
})
.collect();
RetryAction::Done(Ok(outcomes))
}
fn list_groups_min_version(options: &ListGroupsOptions) -> i16 {
let types_need_v5 = !options.types.is_empty()
&& (!options.types.contains(&GroupType::Classic)
|| options
.types
.iter()
.any(|group_type| !matches!(group_type, GroupType::Classic | GroupType::Consumer)));
if types_need_v5 {
5
} else if options.group_states.is_empty() {
list_groups_request::MIN_VERSION
} else {
4
}
}
async fn list_groups_on_broker(
broker: &MetadataResponseBroker,
options: ConnectionOptions,
request: ListGroupsRequest,
min_version: i16,
start: tokio::time::Instant,
retry: RetryPolicy,
) -> Result<Vec<ListedGroup>, KafkaError> {
let host_port = format_host_port(&broker.host, broker.port);
let deadline = start + retry.timeout;
let mut backoff = retry.initial_backoff;
let mut connection = None;
let mut attempts = 0_u32;
let mut last = None;
while tokio::time::Instant::now() < deadline {
attempts += 1;
let attempt = list_groups_attempt(
&mut connection,
&host_port,
&options,
request.clone(),
min_version,
);
match tokio::time::timeout_at(deadline, attempt).await {
Ok(RetryAction::Done(result)) => {
return result.map_err(|error| list_groups_kafka_error(&error));
}
Ok(RetryAction::SameCoordinator(result) | RetryAction::FindCoordinator(result)) => {
match result {
Ok(groups) => return Ok(groups),
Err(error) => last = Some(error),
}
}
Err(_) => break,
}
if attempts > retry.max_retries {
break;
}
tracing::debug!(
node_id = broker.node_id,
"ListGroups got a retriable error; retrying"
);
tokio::time::sleep_until(deadline.min(tokio::time::Instant::now() + backoff)).await;
backoff = backoff.saturating_mul(2).min(retry.max_backoff);
}
Err(list_groups_timeout(attempts, last.as_ref()))
}
fn list_groups_timeout(attempts: u32, last: Option<&AdminError>) -> KafkaError {
const REQUEST_TIMED_OUT: i16 = 7;
let cause = last
.map(|error| format!("; last error: {error}"))
.unwrap_or_default();
KafkaError {
code: REQUEST_TIMED_OUT,
name: kafka_error_name(REQUEST_TIMED_OUT),
message: Some(format!(
"ListGroups timed out after {attempts} attempt(s){cause}"
)),
}
}
async fn list_groups_attempt(
connection: &mut Option<Connection>,
host_port: &str,
options: &ConnectionOptions,
request: ListGroupsRequest,
min_version: i16,
) -> RetryAction<Vec<ListedGroup>> {
let current = match connection {
Some(current) => current,
None => match AdminClient::connect_one(host_port, options.clone()).await {
Ok(new) => connection.insert(new),
Err(error) if error.is_authentication_failure() => {
return RetryAction::Done(Err(error));
}
Err(error) => return RetryAction::SameCoordinator(Err(error)),
},
};
match send_connection_at_least(current, request, min_version).await {
Ok(response) => match response.error_code {
0 => RetryAction::Done(Ok(response.groups)),
code => {
let error = Err(AdminError::Broker {
api: "ListGroups",
code,
name: kafka_error_name(code),
message: None,
});
if matches!(
code,
COORDINATOR_LOAD_IN_PROGRESS | COORDINATOR_NOT_AVAILABLE
) {
RetryAction::SameCoordinator(error)
} else {
RetryAction::Done(error)
}
}
},
Err(error) if AdminClient::is_retriable_transport_error(&error) => {
*connection = None;
RetryAction::SameCoordinator(Err(error.into()))
}
Err(error) => RetryAction::Done(Err(error.into())),
}
}
pub(crate) fn list_groups_kafka_error(error: &AdminError) -> KafkaError {
const UNKNOWN_SERVER_ERROR: i16 = -1;
const REQUEST_TIMED_OUT: i16 = 7;
const NETWORK_EXCEPTION: i16 = 13;
const UNSUPPORTED_SASL_MECHANISM: i16 = 33;
const ILLEGAL_SASL_STATE: i16 = 34;
const UNSUPPORTED_VERSION: i16 = 35;
const INVALID_CONFIG: i16 = 40;
const SASL_AUTHENTICATION_FAILED: i16 = 58;
let (code, message) = match error {
AdminError::Broker { code, message, .. } => (*code, message.clone()),
AdminError::Transport(ClientError::IncompatibleVersion { .. }) => {
(UNSUPPORTED_VERSION, Some(error.to_string()))
}
AdminError::Transport(ClientError::Timeout(_)) => {
(REQUEST_TIMED_OUT, Some(error.to_string()))
}
AdminError::Transport(
ClientError::Connect { .. } | ClientError::Disconnected | ClientError::Io(_),
) => (NETWORK_EXCEPTION, Some(error.to_string())),
AdminError::Transport(ClientError::Authentication { source, .. }) => {
let code = match source {
AuthenticationError::Tls(_) => INVALID_CONFIG,
AuthenticationError::Sasl(SaslAuthenticationError::UnsupportedMechanism(_)) => {
UNSUPPORTED_SASL_MECHANISM
}
AuthenticationError::Sasl(SaslAuthenticationError::IllegalState(_)) => {
ILLEGAL_SASL_STATE
}
AuthenticationError::Sasl(SaslAuthenticationError::Failed(_)) => {
SASL_AUTHENTICATION_FAILED
}
};
(code, Some(error.to_string()))
}
_ => (UNKNOWN_SERVER_ERROR, Some(error.to_string())),
};
KafkaError {
code,
name: kafka_error_name(code),
message,
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
struct TopicNameOffsetCommit(OffsetCommitRequest);
impl Encode for TopicNameOffsetCommit {
fn encode<B: BufMut>(&self, buf: &mut B, version: i16) -> Result<(), ProtocolError> {
self.0.encode(buf, version)
}
fn encoded_len(&self, version: i16) -> usize {
self.0.encoded_len(version)
}
}
impl ProtocolRequest for TopicNameOffsetCommit {
const API_KEY: i16 = offset_commit_request::API_KEY;
const MIN_VERSION: i16 = offset_commit_request::MIN_VERSION;
const MAX_VERSION: i16 = 9;
const LATEST_STABLE_VERSION: i16 = Self::MAX_VERSION;
const FLEXIBLE_MIN: i16 = offset_commit_request::FLEXIBLE_MIN;
type Response = OffsetCommitResponse;
}
#[derive(Clone, Debug, PartialEq, Eq)]
struct TopicNameOffsetFetch(OffsetFetchRequest);
impl Encode for TopicNameOffsetFetch {
fn encode<B: BufMut>(&self, buf: &mut B, version: i16) -> Result<(), ProtocolError> {
self.0.encode(buf, version)
}
fn encoded_len(&self, version: i16) -> usize {
self.0.encoded_len(version)
}
}
impl ProtocolRequest for TopicNameOffsetFetch {
const API_KEY: i16 = offset_fetch_request::API_KEY;
const MIN_VERSION: i16 = 2;
const MAX_VERSION: i16 = 9;
const LATEST_STABLE_VERSION: i16 = Self::MAX_VERSION;
const FLEXIBLE_MIN: i16 = offset_fetch_request::FLEXIBLE_MIN;
type Response = OffsetFetchResponse;
}
fn offset_fetch_request(group: &str) -> TopicNameOffsetFetch {
TopicNameOffsetFetch(OffsetFetchRequest {
group_id: group.into(),
topics: None,
groups: vec![OffsetFetchRequestGroup {
group_id: group.into(),
member_id: None,
member_epoch: -1,
topics: None,
..Default::default()
}],
require_stable: false,
..Default::default()
})
}
fn committed_offsets(
group: &str,
response: OffsetFetchResponse,
) -> Result<BTreeMap<(String, i32), i64>, AdminError> {
let (error_code, rows): (i16, Vec<(String, i32, i64, i16)>) = if response.groups.is_empty() {
(
response.error_code,
response
.topics
.into_iter()
.flat_map(|topic| {
let name = topic.name;
topic.partitions.into_iter().map(move |partition| {
(
name.clone(),
partition.partition_index,
partition.committed_offset,
partition.error_code,
)
})
})
.collect(),
)
} else {
let mut error_code = 0;
let mut rows = Vec::new();
for entry in response
.groups
.into_iter()
.filter(|entry| entry.group_id == group)
{
error_code = entry.error_code;
for topic in entry.topics {
let name = topic.name;
rows.extend(topic.partitions.into_iter().map(|partition| {
(
name.clone(),
partition.partition_index,
partition.committed_offset,
partition.error_code,
)
}));
}
}
(error_code, rows)
};
if error_code != 0 {
return Err(AdminError::Broker {
api: "OffsetFetch",
code: error_code,
name: kafka_error_name(error_code),
message: Some(format!("group={group}")),
});
}
Ok(rows
.into_iter()
.filter_map(|(topic, partition, offset, partition_error)| {
if partition_error != 0 {
tracing::warn!(
topic,
partition,
error_code = partition_error,
"skipping the committed offset of a partition with an error"
);
None
} else if offset < 0 {
None
} else {
Some(((topic, partition), offset))
}
})
.collect())
}
fn offset_commit_request(
group: &str,
offsets: &BTreeMap<(String, i32), i64>,
) -> TopicNameOffsetCommit {
let mut topics = BTreeMap::<String, Vec<OffsetCommitRequestPartition>>::new();
for ((topic, partition), offset) in offsets {
topics
.entry(topic.clone())
.or_default()
.push(OffsetCommitRequestPartition {
partition_index: *partition,
committed_offset: *offset,
committed_leader_epoch: -1,
..Default::default()
});
}
TopicNameOffsetCommit(OffsetCommitRequest {
group_id: group.into(),
generation_id_or_member_epoch: -1,
member_id: String::new(),
topics: topics
.into_iter()
.map(|(name, partitions)| OffsetCommitRequestTopic {
name,
partitions,
..Default::default()
})
.collect(),
..Default::default()
})
}
#[cfg(test)]
mod tests {
use std::{
sync::{
Arc, Mutex,
atomic::{AtomicUsize, Ordering},
},
time::Duration,
};
use assert2::assert;
use bytes::{Buf, BytesMut};
use krabka_client_core::{
AuthenticationError, ClientError, MockBroker, MockReply, MockSaslAnswer,
SaslAuthenticationError,
security::{ClientSecurity, SaslCredentials},
};
use krabka_protocol::{
Decode,
owned::{
alter_share_group_offsets_response::{
AlterShareGroupOffsetsResponsePartition, AlterShareGroupOffsetsResponseTopic,
},
api_versions_request,
api_versions_response::{ApiVersion, ApiVersionsResponse},
delete_share_group_offsets_response::DeleteShareGroupOffsetsResponseTopic,
find_coordinator_request,
find_coordinator_response::FindCoordinatorResponse,
leave_group_response::MemberResponse,
list_groups_response::ListGroupsResponse,
metadata_request,
metadata_response::MetadataResponse,
offset_commit_response::{OffsetCommitResponsePartition, OffsetCommitResponseTopic},
offset_delete_response::{OffsetDeleteResponsePartition, OffsetDeleteResponseTopic},
offset_fetch_response::{
OffsetFetchResponseGroup, OffsetFetchResponsePartition,
OffsetFetchResponsePartitions, OffsetFetchResponseTopic, OffsetFetchResponseTopics,
},
sasl_handshake_request,
},
};
use super::*;
#[test]
fn offset_reset_builds_admin_commit() {
let offsets = BTreeMap::from([(("orders".into(), 2), 41)]);
let request = offset_commit_request("worker", &offsets);
let expected = TopicNameOffsetCommit(OffsetCommitRequest {
group_id: "worker".into(),
topics: vec![OffsetCommitRequestTopic {
name: "orders".into(),
partitions: vec![OffsetCommitRequestPartition {
partition_index: 2,
committed_offset: 41,
..Default::default()
}],
..Default::default()
}],
..Default::default()
});
assert!(request == expected);
}
fn encode(response: &impl Encode, version: i16, flexible: bool) -> Vec<u8> {
let mut bytes = BytesMut::new();
if flexible {
bytes.extend_from_slice(&[0]);
}
response.encode(&mut bytes, version).unwrap();
bytes.to_vec()
}
#[derive(Clone, Copy)]
struct GroupRanges {
offset_commit: (i16, i16),
offset_fetch: (i16, i16),
}
fn api_versions(ranges: GroupRanges) -> Vec<u8> {
encode(
&ApiVersionsResponse {
api_keys: vec![
ApiVersion {
api_key: api_versions_request::API_KEY,
min_version: 0,
max_version: 0,
..Default::default()
},
ApiVersion {
api_key: find_coordinator_request::API_KEY,
min_version: 0,
max_version: 0,
..Default::default()
},
ApiVersion {
api_key: offset_commit_request::API_KEY,
min_version: ranges.offset_commit.0,
max_version: ranges.offset_commit.1,
..Default::default()
},
ApiVersion {
api_key: offset_fetch_request::API_KEY,
min_version: ranges.offset_fetch.0,
max_version: ranges.offset_fetch.1,
..Default::default()
},
ApiVersion {
api_key: metadata_request::API_KEY,
min_version: 13,
max_version: 13,
..Default::default()
},
],
..Default::default()
},
0,
false,
)
}
fn decode_request<R: ProtocolRequest + for<'de> Decode<'de>>(
mut body: &[u8],
version: i16,
) -> R {
let client_id_len = body.get_i16();
body.advance(usize::try_from(client_id_len).expect("client id length"));
if version >= R::FLEXIBLE_MIN {
body.advance(1);
}
R::decode(&mut body, version).expect("request decodes")
}
async fn bootstrap_for(
coordinator: &MockBroker,
ranges: GroupRanges,
group_rpcs: Arc<AtomicUsize>,
) -> MockBroker {
let coordinator_addr = coordinator.addr;
MockBroker::start(move |api_key, version, _, _| match api_key {
api_versions_request::API_KEY => Some(api_versions(ranges)),
find_coordinator_request::API_KEY => Some(encode(
&FindCoordinatorResponse {
node_id: 2,
host: coordinator_addr.ip().to_string(),
port: i32::from(coordinator_addr.port()),
..Default::default()
},
version,
false,
)),
offset_commit_request::API_KEY => {
group_rpcs.fetch_add(1, Ordering::SeqCst);
Some(encode(&OffsetCommitResponse::default(), version, true))
}
offset_fetch_request::API_KEY => {
group_rpcs.fetch_add(1, Ordering::SeqCst);
Some(encode(&OffsetFetchResponse::default(), version, true))
}
metadata_request::API_KEY => Some(encode(&MetadataResponse::default(), version, true)),
_ => None,
})
.await
}
type CommitResult = Result<Vec<ConsumerGroupOffsetOutcome>, (i16, i16, i16, i16, i16)>;
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn alter_consumer_group_offsets_sends_offset_commit_by_topic_name() {
let sent_request = |version| {
vec![(
version,
OffsetCommitRequest {
group_id: "workers".into(),
generation_id_or_member_epoch: -1,
topics: vec![OffsetCommitRequestTopic {
name: "orders".into(),
partitions: vec![OffsetCommitRequestPartition {
partition_index: 2,
committed_offset: 41,
committed_leader_epoch: -1,
..Default::default()
}],
..Default::default()
}],
..Default::default()
},
)]
};
let committed = Ok(vec![ConsumerGroupOffsetOutcome {
topic: "orders".into(),
partition: 2,
error: None,
}]);
for (name, offset_commit_range, expected_requests, expected_result) in [
(
"coordinator stops at v7",
(2, 7),
sent_request(7),
committed.clone(),
),
(
"coordinator stops at v9",
(2, 9),
sent_request(9),
committed.clone(),
),
(
"coordinator supports v10",
(2, 10),
sent_request(9),
committed.clone(),
),
(
"coordinator supports only v10",
(10, 10),
Vec::new(),
Err((offset_commit_request::API_KEY, 10, 10, 2, 9)),
),
] {
let requests = Arc::new(Mutex::new(Vec::new()));
let requests_in_mock = Arc::clone(&requests);
let ranges = GroupRanges {
offset_commit: offset_commit_range,
offset_fetch: (2, 10),
};
let coordinator = MockBroker::start(move |api_key, version, _, body| match api_key {
api_versions_request::API_KEY => Some(api_versions(ranges)),
offset_commit_request::API_KEY => {
let request: OffsetCommitRequest = decode_request(body, version);
let response = OffsetCommitResponse {
topics: request
.topics
.iter()
.map(|topic| OffsetCommitResponseTopic {
name: topic.name.clone(),
partitions: topic
.partitions
.iter()
.map(|partition| OffsetCommitResponsePartition {
partition_index: partition.partition_index,
..Default::default()
})
.collect(),
..Default::default()
})
.collect(),
..Default::default()
};
requests_in_mock
.lock()
.expect("requests lock")
.push((version, request));
Some(encode(
&response,
version,
version >= offset_commit_request::FLEXIBLE_MIN,
))
}
_ => None,
})
.await;
let bootstrap = bootstrap_for(&coordinator, ranges, Arc::default()).await;
let mut admin = AdminClient::connect(&[bootstrap.addr.to_string()])
.await
.expect("admin connects");
let result: CommitResult = admin
.alter_consumer_group_offsets(
"workers",
&BTreeMap::from([(("orders".into(), 2), 41)]),
)
.await
.map_err(|error| match error {
AdminError::Transport(ClientError::IncompatibleVersion {
api_key,
broker_min,
broker_max,
client_min,
client_max,
}) => (api_key, broker_min, broker_max, client_min, client_max),
other => panic!("case {name}: unexpected error {other:?}"),
});
bootstrap.stop();
coordinator.stop();
let requests = requests.lock().expect("requests lock").clone();
assert!(
(requests, result) == (expected_requests, expected_result),
"case {name}"
);
}
}
fn offset_fetch_response(version: i16) -> OffsetFetchResponse {
let rows = [
("orders", 2, 41, 0),
("orders", 3, -1, 0),
("payments", 0, 5, 3),
];
if version < 8 {
OffsetFetchResponse {
topics: rows
.iter()
.map(|(name, partition_index, committed_offset, error_code)| {
OffsetFetchResponseTopic {
name: (*name).into(),
partitions: vec![OffsetFetchResponsePartition {
partition_index: *partition_index,
committed_offset: *committed_offset,
error_code: *error_code,
..Default::default()
}],
..Default::default()
}
})
.collect(),
..Default::default()
}
} else {
OffsetFetchResponse {
groups: vec![OffsetFetchResponseGroup {
group_id: "workers".into(),
topics: rows
.iter()
.map(|(name, partition_index, committed_offset, error_code)| {
OffsetFetchResponseTopics {
name: (*name).into(),
partitions: vec![OffsetFetchResponsePartitions {
partition_index: *partition_index,
committed_offset: *committed_offset,
error_code: *error_code,
..Default::default()
}],
..Default::default()
}
})
.collect(),
..Default::default()
}],
..Default::default()
}
}
}
type FetchResult = Result<BTreeMap<(String, i32), i64>, (i16, i16, i16, i16, i16)>;
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn list_consumer_group_offsets_sends_offset_fetch_by_topic_name() {
let legacy_request = |version| {
vec![(
version,
OffsetFetchRequest {
group_id: "workers".into(),
topics: None,
..Default::default()
},
)]
};
let grouped_request = |version| {
vec![(
version,
OffsetFetchRequest {
groups: vec![OffsetFetchRequestGroup {
group_id: "workers".into(),
member_id: None,
member_epoch: -1,
topics: None,
..Default::default()
}],
..Default::default()
},
)]
};
let fetched = Ok(BTreeMap::from([(("orders".into(), 2), 41)]));
for (name, offset_fetch_range, expected_requests, expected_result) in [
(
"coordinator stops at v7",
(2, 7),
legacy_request(7),
fetched.clone(),
),
(
"coordinator stops at v8",
(2, 8),
grouped_request(8),
fetched.clone(),
),
(
"coordinator stops at v9",
(2, 9),
grouped_request(9),
fetched.clone(),
),
(
"coordinator supports v10",
(2, 10),
grouped_request(9),
fetched.clone(),
),
(
"coordinator supports only v10",
(10, 10),
Vec::new(),
Err((offset_fetch_request::API_KEY, 10, 10, 2, 9)),
),
(
"coordinator supports only v1",
(1, 1),
Vec::new(),
Err((offset_fetch_request::API_KEY, 1, 1, 2, 9)),
),
] {
let ranges = GroupRanges {
offset_commit: (2, 9),
offset_fetch: offset_fetch_range,
};
let requests = Arc::new(Mutex::new(Vec::new()));
let requests_in_mock = Arc::clone(&requests);
let metadata_requests = Arc::new(AtomicUsize::new(0));
let metadata_requests_in_mock = Arc::clone(&metadata_requests);
let coordinator = MockBroker::start(move |api_key, version, _, body| match api_key {
api_versions_request::API_KEY => Some(api_versions(ranges)),
offset_fetch_request::API_KEY => {
let request: OffsetFetchRequest = decode_request(body, version);
requests_in_mock
.lock()
.expect("requests lock")
.push((version, request));
Some(encode(
&offset_fetch_response(version),
version,
version >= offset_fetch_request::FLEXIBLE_MIN,
))
}
metadata_request::API_KEY => {
metadata_requests_in_mock.fetch_add(1, Ordering::SeqCst);
Some(encode(&MetadataResponse::default(), version, true))
}
_ => None,
})
.await;
let bootstrap = bootstrap_for(&coordinator, ranges, Arc::default()).await;
let mut admin = AdminClient::connect(&[bootstrap.addr.to_string()])
.await
.expect("admin connects");
let result: FetchResult =
admin
.list_consumer_group_offsets("workers")
.await
.map_err(|error| match error {
AdminError::Transport(ClientError::IncompatibleVersion {
api_key,
broker_min,
broker_max,
client_min,
client_max,
}) => (api_key, broker_min, broker_max, client_min, client_max),
other => panic!("case {name}: unexpected error {other:?}"),
});
bootstrap.stop();
coordinator.stop();
let requests = requests.lock().expect("requests lock").clone();
assert!(
(requests, metadata_requests.load(Ordering::SeqCst), result)
== (expected_requests, 0, expected_result),
"case {name}"
);
}
}
type OffsetsResult =
Result<BTreeMap<(String, i32), i64>, (&'static str, i16, &'static str, Option<String>)>;
#[test]
fn committed_offsets_reads_both_response_shapes() {
let offsets = BTreeMap::from([(("orders".into(), 2), 41)]);
let group_error =
|code, name| Err(("OffsetFetch", code, name, Some("group=workers".into())));
for (name, response, expected) in [
(
"v2 to v7 shape",
offset_fetch_response(7),
Ok(offsets.clone()),
),
("v8 and v9 shape", offset_fetch_response(9), Ok(offsets)),
(
"v2 to v7 group error",
OffsetFetchResponse {
error_code: 15,
..Default::default()
},
group_error(15, "COORDINATOR_NOT_AVAILABLE"),
),
(
"v8 and v9 group error",
OffsetFetchResponse {
groups: vec![OffsetFetchResponseGroup {
group_id: "workers".into(),
error_code: 16,
..Default::default()
}],
..Default::default()
},
group_error(16, "NOT_COORDINATOR"),
),
] {
let result: OffsetsResult =
committed_offsets("workers", response).map_err(|error| match error {
AdminError::Broker {
api,
code,
name,
message,
} => (api, code, name, message),
other => panic!("case {name}: unexpected error {other:?}"),
});
assert!(result == expected, "case {name}");
}
}
#[derive(Debug, PartialEq, Eq)]
enum ListOffsetsError {
OffsetFetch(i16),
FindCoordinator(i16),
Timeout,
}
#[derive(Default)]
struct RetryScript {
find_coordinator: Vec<i16>,
offset_fetch: Vec<i16>,
offset_commit: Vec<i16>,
find_coordinator_requests: usize,
offset_fetch_requests: usize,
offset_commit_requests: usize,
unreachable_answers: usize,
unreachable: Option<std::net::SocketAddr>,
silent_offset_commit: bool,
closed_offset_commits: usize,
}
async fn refused_address() -> std::net::SocketAddr {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
.await
.expect("bind a port");
listener.local_addr().expect("local address")
}
impl RetryScript {
fn next(codes: &[i16], requests: &mut usize) -> i16 {
let code = codes[(*requests).min(codes.len() - 1)];
*requests += 1;
code
}
}
async fn scripted_group_broker(
script: Arc<Mutex<RetryScript>>,
coordinator: Arc<Mutex<Option<std::net::SocketAddr>>>,
) -> MockBroker {
let ranges = GroupRanges {
offset_commit: (2, 9),
offset_fetch: (2, 9),
};
let closing = Arc::clone(&script);
let handler = move |api_key: i16, version: i16, body: &[u8]| match api_key {
api_versions_request::API_KEY => Some(api_versions(ranges)),
offset_commit_request::API_KEY => {
let request: OffsetCommitRequest = decode_request(body, version);
let mut script = script.lock().expect("script lock");
let script = &mut *script;
let error_code =
RetryScript::next(&script.offset_commit, &mut script.offset_commit_requests);
if script.silent_offset_commit {
return None;
}
let response = OffsetCommitResponse {
topics: request
.topics
.iter()
.map(|topic| OffsetCommitResponseTopic {
name: topic.name.clone(),
partitions: topic
.partitions
.iter()
.map(|partition| OffsetCommitResponsePartition {
partition_index: partition.partition_index,
error_code,
..Default::default()
})
.collect(),
..Default::default()
})
.collect(),
..Default::default()
};
Some(encode(
&response,
version,
version >= offset_commit_request::FLEXIBLE_MIN,
))
}
find_coordinator_request::API_KEY => {
let mut script = script.lock().expect("script lock");
let script = &mut *script;
let error_code = RetryScript::next(
&script.find_coordinator,
&mut script.find_coordinator_requests,
);
let addr = match script.unreachable {
Some(unreachable)
if script.find_coordinator_requests <= script.unreachable_answers =>
{
unreachable
}
_ => coordinator
.lock()
.expect("coordinator lock")
.expect("coordinator address"),
};
Some(encode(
&FindCoordinatorResponse {
error_code,
node_id: 2,
host: addr.ip().to_string(),
port: i32::from(addr.port()),
..Default::default()
},
version,
false,
))
}
offset_fetch_request::API_KEY => {
let mut script = script.lock().expect("script lock");
let script = &mut *script;
let error_code =
RetryScript::next(&script.offset_fetch, &mut script.offset_fetch_requests);
let response = if error_code == 0 {
offset_fetch_response(version)
} else {
OffsetFetchResponse {
groups: vec![OffsetFetchResponseGroup {
group_id: "workers".into(),
error_code,
..Default::default()
}],
..Default::default()
}
};
Some(encode(&response, version, true))
}
_ => None,
};
MockBroker::start_with_replies(move |api_key, version, _, body| {
if api_key == offset_commit_request::API_KEY {
let mut script = closing.lock().expect("script lock");
if script.closed_offset_commits > 0 {
script.closed_offset_commits -= 1;
script.offset_commit_requests += 1;
return MockReply::Close;
}
}
handler(api_key, version, body).map_or(MockReply::Silent, MockReply::Respond)
})
.await
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn list_consumer_group_offsets_retries_coordinator_errors_as_kafka_does() {
const LONG: Duration = Duration::from_secs(5);
const NOW: Duration = Duration::from_millis(500);
let backoff = |timeout| {
if timeout == LONG {
Duration::from_millis(1)
} else {
timeout
}
};
let offsets = || Ok(BTreeMap::from([(("orders".into(), 2), 41)]));
let fetch_error = |code| Err(ListOffsetsError::OffsetFetch(code));
let find_error = |code| Err(ListOffsetsError::FindCoordinator(code));
for (name, find_coordinator, offset_fetch, timeout, expected) in [
("no error", vec![0], vec![0], LONG, (offsets(), 1, 1)),
(
"coordinator load in progress retries on the same coordinator",
vec![0],
vec![14, 14, 0],
LONG,
(offsets(), 1, 3),
),
(
"coordinator not available finds the coordinator again",
vec![0],
vec![15, 0],
LONG,
(offsets(), 2, 2),
),
(
"not coordinator finds the coordinator again",
vec![0],
vec![16, 0],
LONG,
(offsets(), 2, 2),
),
(
"coordinator load in progress past the timeout times out",
vec![0],
vec![14],
NOW,
(Err(ListOffsetsError::Timeout), 1, 1),
),
(
"not coordinator past the timeout times out",
vec![0],
vec![16],
NOW,
(Err(ListOffsetsError::Timeout), 1, 1),
),
(
"group authorization failed is final",
vec![0],
vec![30],
LONG,
(fetch_error(30), 1, 1),
),
(
"another retriable group code is final",
vec![0],
vec![7],
LONG,
(fetch_error(7), 1, 1),
),
(
"find coordinator answers coordinator not available, then the coordinator",
vec![15, 0],
vec![0],
LONG,
(offsets(), 2, 1),
),
(
"find coordinator answers coordinator load in progress, then the coordinator",
vec![14, 0],
vec![0],
LONG,
(offsets(), 2, 1),
),
(
"find coordinator not available past the timeout times out",
vec![15],
vec![0],
NOW,
(Err(ListOffsetsError::Timeout), 1, 0),
),
(
"find coordinator group authorization failed is final",
vec![30],
vec![0],
LONG,
(find_error(30), 1, 0),
),
] {
let script = Arc::new(Mutex::new(RetryScript {
find_coordinator,
offset_fetch,
..RetryScript::default()
}));
let coordinator_addr = Arc::new(Mutex::new(None));
let coordinator =
scripted_group_broker(Arc::clone(&script), Arc::clone(&coordinator_addr)).await;
*coordinator_addr.lock().expect("coordinator lock") = Some(coordinator.addr);
let bootstrap =
scripted_group_broker(Arc::clone(&script), Arc::clone(&coordinator_addr)).await;
let mut admin = AdminClient::connect(&[bootstrap.addr.to_string()])
.await
.expect("admin connects");
let result = admin
.list_consumer_group_offsets_with_retry(
"workers",
RetryPolicy {
timeout,
initial_backoff: backoff(timeout),
max_backoff: backoff(timeout),
jitter: 0.0,
max_retries: u32::MAX,
},
)
.await
.map_err(|error| match error {
AdminError::Broker {
api: "OffsetFetch",
code,
..
} => ListOffsetsError::OffsetFetch(code),
AdminError::Transport(ClientError::Server { error_code }) => {
ListOffsetsError::FindCoordinator(error_code)
}
AdminError::Transport(ClientError::Timeout(_)) => ListOffsetsError::Timeout,
other => panic!("case {name}: unexpected error {other:?}"),
});
bootstrap.stop();
coordinator.stop();
let script = script.lock().expect("script lock");
assert!(
(
result,
script.find_coordinator_requests,
script.offset_fetch_requests
) == expected,
"case {name}"
);
}
}
const TIMED_OUT: i16 = 7;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum Coordinator {
Normal,
UnreachableOnce,
Silent,
ClosedOnce,
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn alter_consumer_group_offsets_retries_coordinator_errors_as_kafka_does() {
const LONG: Duration = Duration::from_secs(5);
const NOW: Duration = Duration::from_millis(500);
let backoff = |timeout| {
if timeout == LONG {
Duration::from_millis(1)
} else {
timeout
}
};
let outcome = |code| {
Ok(vec![ConsumerGroupOffsetOutcome {
topic: "orders".into(),
partition: 2,
error: kafka_error_if(code, None),
}])
};
let find_error = |code| Err(code);
let normal = Coordinator::Normal;
for (name, find_coordinator, offset_commit, (timeout, coordinator), expected) in [
(
"no error",
vec![0],
vec![0],
(LONG, normal),
(outcome(0), 1, 1),
),
(
"a closed coordinator connection recovers",
vec![0],
vec![0],
(LONG, Coordinator::ClosedOnce),
(outcome(0), 1, 2),
),
(
"an unreachable coordinator finds the coordinator again",
vec![0],
vec![0],
(LONG, Coordinator::UnreachableOnce),
(outcome(0), 2, 1),
),
(
"a silent coordinator stops the attempt at the deadline",
vec![0],
vec![0],
(Duration::from_millis(300), Coordinator::Silent),
(find_error(TIMED_OUT), 1, 1),
),
(
"coordinator load in progress retries on the same coordinator",
vec![0],
vec![14, 14, 0],
(LONG, normal),
(outcome(0), 1, 3),
),
(
"rebalance in progress retries on the same coordinator",
vec![0],
vec![27, 0],
(LONG, normal),
(outcome(0), 1, 2),
),
(
"coordinator not available finds the coordinator again",
vec![0],
vec![15, 0],
(LONG, normal),
(outcome(0), 2, 2),
),
(
"not coordinator finds the coordinator again",
vec![0],
vec![16, 0],
(LONG, normal),
(outcome(0), 2, 2),
),
(
"rebalance in progress past the timeout times out",
vec![0],
vec![27],
(NOW, normal),
(find_error(TIMED_OUT), 1, 1),
),
(
"not coordinator past the timeout times out",
vec![0],
vec![16],
(NOW, normal),
(find_error(TIMED_OUT), 1, 1),
),
(
"unknown member id is a final outcome",
vec![0],
vec![25],
(LONG, normal),
(outcome(25), 1, 1),
),
(
"find coordinator answers coordinator not available, then the coordinator",
vec![15, 0],
vec![0],
(LONG, normal),
(outcome(0), 2, 1),
),
(
"find coordinator group authorization failed is final",
vec![30],
vec![0],
(LONG, normal),
(find_error(30), 1, 0),
),
] {
let script = Arc::new(Mutex::new(RetryScript {
find_coordinator,
offset_commit,
unreachable_answers: usize::from(coordinator == Coordinator::UnreachableOnce),
unreachable: Some(refused_address().await),
silent_offset_commit: coordinator == Coordinator::Silent,
closed_offset_commits: usize::from(coordinator == Coordinator::ClosedOnce),
..RetryScript::default()
}));
let coordinator_addr = Arc::new(Mutex::new(None));
let coordinator =
scripted_group_broker(Arc::clone(&script), Arc::clone(&coordinator_addr)).await;
*coordinator_addr.lock().expect("coordinator lock") = Some(coordinator.addr);
let bootstrap =
scripted_group_broker(Arc::clone(&script), Arc::clone(&coordinator_addr)).await;
let mut admin = AdminClient::connect(&[bootstrap.addr.to_string()])
.await
.expect("admin connects");
let started = tokio::time::Instant::now();
let result = admin
.alter_consumer_group_offsets_with_retry(
"workers",
&BTreeMap::from([(("orders".into(), 2), 41)]),
RetryPolicy {
timeout,
initial_backoff: backoff(timeout),
max_backoff: backoff(timeout),
jitter: 0.0,
max_retries: u32::MAX,
},
)
.await
.map_err(|error| match error {
AdminError::Transport(ClientError::Server { error_code }) => error_code,
AdminError::Transport(ClientError::Timeout(_)) => TIMED_OUT,
other => panic!("case {name}: unexpected error {other:?}"),
});
bootstrap.stop();
coordinator.stop();
let script = script.lock().expect("script lock");
let within_deadline = started.elapsed() < timeout + Duration::from_secs(2);
assert!(
(
result,
script.find_coordinator_requests,
script.offset_commit_requests,
within_deadline
) == (expected.0, expected.1, expected.2, true),
"case {name}"
);
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn group_offset_rpcs_use_the_group_coordinator() {
let ranges = GroupRanges {
offset_commit: (2, 10),
offset_fetch: (2, 10),
};
let coordinator_group_rpcs = Arc::new(AtomicUsize::new(0));
let seen = Arc::clone(&coordinator_group_rpcs);
let coordinator = MockBroker::start(move |api_key, version, _, _| match api_key {
api_versions_request::API_KEY => Some(api_versions(ranges)),
offset_commit_request::API_KEY => {
seen.fetch_add(1, Ordering::SeqCst);
Some(encode(
&OffsetCommitResponse {
topics: vec![OffsetCommitResponseTopic {
name: "orders".into(),
partitions: vec![OffsetCommitResponsePartition {
partition_index: 2,
..Default::default()
}],
..Default::default()
}],
..Default::default()
},
version,
true,
))
}
offset_fetch_request::API_KEY => {
seen.fetch_add(1, Ordering::SeqCst);
Some(encode(&offset_fetch_response(version), version, true))
}
_ => None,
})
.await;
let bootstrap_group_rpcs = Arc::new(AtomicUsize::new(0));
let bootstrap =
bootstrap_for(&coordinator, ranges, Arc::clone(&bootstrap_group_rpcs)).await;
let mut admin = AdminClient::connect(&[bootstrap.addr.to_string()])
.await
.expect("admin connects");
let committed = admin
.alter_consumer_group_offsets("workers", &BTreeMap::from([(("orders".into(), 2), 41)]))
.await
.expect("offset commit succeeds");
let mut admin = AdminClient::connect(&[bootstrap.addr.to_string()])
.await
.expect("second admin connects");
let fetched = admin
.list_consumer_group_offsets("workers")
.await
.expect("offset fetch succeeds");
assert!(bootstrap_group_rpcs.load(Ordering::SeqCst) == 0);
assert!(coordinator_group_rpcs.load(Ordering::SeqCst) == 2);
assert!(
committed
== vec![ConsumerGroupOffsetOutcome {
topic: "orders".into(),
partition: 2,
error: None,
}]
);
assert!(fetched == BTreeMap::from([(("orders".into(), 2), 41)]));
bootstrap.stop();
coordinator.stop();
}
#[derive(Clone)]
enum ListGroupsAnswer {
Groups(Vec<ListedGroup>),
Error(i16),
}
#[derive(Clone)]
struct ListGroupsBroker {
max_version: i16,
answers: Vec<ListGroupsAnswer>,
}
fn listed(group_id: &str, protocol_type: &str, state: &str, group_type: &str) -> ListedGroup {
ListedGroup {
group_id: group_id.into(),
protocol_type: protocol_type.into(),
group_state: state.into(),
group_type: group_type.into(),
..Default::default()
}
}
fn consumer_group(group_id: &str) -> ListedGroup {
listed(group_id, "consumer", "Stable", "Classic")
}
fn stable_listing(group_id: &str) -> GroupListing {
GroupListing {
group_id: group_id.into(),
group_type: Some(GroupType::Classic),
protocol_type: "consumer".into(),
group_state: Some(GroupState::Stable),
}
}
fn groups(ids: &[&str]) -> ListGroupsBroker {
ListGroupsBroker {
max_version: 5,
answers: vec![ListGroupsAnswer::Groups(
ids.iter().map(|id| consumer_group(id)).collect(),
)],
}
}
async fn list_groups_broker(
script: ListGroupsBroker,
cluster: Arc<Mutex<Vec<std::net::SocketAddr>>>,
requests: Arc<Mutex<Vec<(i16, ListGroupsRequest)>>>,
) -> MockBroker {
MockBroker::start_with_replies(move |api_key, version, _, body| {
MockSaslAnswer::Accept
.reply(api_key, version)
.unwrap_or_else(|| {
list_groups_reply(&script, &cluster, &requests, api_key, version, body)
.map_or(MockReply::Silent, MockReply::Respond)
})
})
.await
}
fn list_groups_reply(
script: &ListGroupsBroker,
cluster: &Mutex<Vec<std::net::SocketAddr>>,
requests: &Mutex<Vec<(i16, ListGroupsRequest)>>,
api_key: i16,
version: i16,
body: &[u8],
) -> Option<Vec<u8>> {
match api_key {
api_versions_request::API_KEY => Some(encode(
&ApiVersionsResponse {
api_keys: vec![
ApiVersion {
api_key: api_versions_request::API_KEY,
..Default::default()
},
ApiVersion {
api_key: metadata_request::API_KEY,
min_version: 12,
max_version: 12,
..Default::default()
},
ApiVersion {
api_key: list_groups_request::API_KEY,
max_version: script.max_version,
..Default::default()
},
],
..Default::default()
},
0,
false,
)),
metadata_request::API_KEY => Some(encode(
&MetadataResponse {
brokers: cluster
.lock()
.expect("cluster lock")
.iter()
.zip(1..)
.map(|(addr, node_id)| MetadataResponseBroker {
node_id,
host: addr.ip().to_string(),
port: i32::from(addr.port()),
..Default::default()
})
.collect(),
..Default::default()
},
version,
true,
)),
list_groups_request::API_KEY => {
let request: ListGroupsRequest = decode_request(body, version);
let mut requests = requests.lock().expect("requests lock");
let answer = &script.answers[requests.len().min(script.answers.len() - 1)];
let response = match answer {
ListGroupsAnswer::Groups(groups) => ListGroupsResponse {
groups: groups
.iter()
.filter(|group| {
(request.states_filter.is_empty()
|| request.states_filter.contains(&group.group_state))
&& (request.types_filter.is_empty()
|| request.types_filter.contains(&group.group_type))
})
.cloned()
.collect(),
..Default::default()
},
ListGroupsAnswer::Error(error_code) => ListGroupsResponse {
error_code: *error_code,
..Default::default()
},
};
requests.push((version, request));
Some(encode(
&response,
version,
version >= list_groups_request::FLEXIBLE_MIN,
))
}
_ => None,
}
}
type ExpectedError = (i32, i16, Option<String>);
struct ListGroupsCase {
name: &'static str,
brokers: [ListGroupsBroker; 3],
options: ListGroupsOptions,
timeout: Duration,
valid: Vec<GroupListing>,
errors: Vec<ExpectedError>,
requests: [Vec<(i16, ListGroupsRequest)>; 3],
}
const LONG: Duration = Duration::from_secs(10);
const SHORT: Duration = Duration::from_millis(500);
const BACKOFF: Duration = Duration::from_secs(1);
fn plain(version: i16) -> (i16, ListGroupsRequest) {
(version, ListGroupsRequest::default())
}
fn stable(version: i16) -> (i16, ListGroupsRequest) {
(
version,
ListGroupsRequest {
states_filter: vec!["Stable".into()],
..Default::default()
},
)
}
fn classic_and_consumer(version: i16) -> (i16, ListGroupsRequest) {
(
version,
ListGroupsRequest {
types_filter: if version >= 5 {
vec!["Consumer".into(), "Classic".into()]
} else {
Vec::new()
},
..Default::default()
},
)
}
fn consumer_type(version: i16) -> (i16, ListGroupsRequest) {
(
version,
ListGroupsRequest {
types_filter: vec!["Consumer".into()],
..Default::default()
},
)
}
fn unsupported(broker_max: i16, client_min: i16) -> String {
AdminError::Transport(ClientError::IncompatibleVersion {
api_key: list_groups_request::API_KEY,
broker_min: 0,
broker_max,
client_min,
client_max: 5,
})
.to_string()
}
fn states(states: &[GroupState]) -> ListGroupsOptions {
ListGroupsOptions {
group_states: states.iter().copied().collect(),
..ListGroupsOptions::default()
}
}
fn types(types: &[GroupType]) -> ListGroupsOptions {
ListGroupsOptions {
types: types.iter().copied().collect(),
..ListGroupsOptions::default()
}
}
fn broker_error(code: i16) -> ListGroupsBroker {
ListGroupsBroker {
max_version: 5,
answers: vec![ListGroupsAnswer::Error(code)],
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn list_groups_stops_a_broker_after_the_retry_limit() {
let cluster = Arc::new(Mutex::new(Vec::new()));
let sent = Arc::new(Mutex::new(Vec::new()));
let broker =
list_groups_broker(broker_error(14), Arc::clone(&cluster), Arc::clone(&sent)).await;
*cluster.lock().expect("cluster lock") = vec![broker.addr];
let admin = AdminClient::connect(&[broker.addr.to_string()])
.await
.expect("admin connects");
let result = admin
.list_groups_with_retry(
&ListGroupsOptions::default(),
RetryPolicy {
timeout: Duration::from_secs(10),
initial_backoff: Duration::from_millis(1),
max_backoff: Duration::from_millis(1),
jitter: 0.0,
max_retries: 2,
},
)
.await
.expect("list_groups succeeds");
broker.stop();
let codes = result
.errors
.iter()
.map(|error| error.error.code)
.collect::<Vec<_>>();
assert!((codes, sent.lock().expect("sent lock").len()) == (vec![7], 3));
}
fn below(max_version: i16, ids: &[&str]) -> ListGroupsBroker {
ListGroupsBroker {
max_version,
..groups(ids)
}
}
async fn run_list_groups_case(case: ListGroupsCase) {
let cluster = Arc::new(Mutex::new(Vec::new()));
let mut brokers = Vec::new();
let mut requests = Vec::new();
for script in case.brokers {
let sent = Arc::new(Mutex::new(Vec::new()));
brokers.push(list_groups_broker(script, Arc::clone(&cluster), Arc::clone(&sent)).await);
requests.push(sent);
}
*cluster.lock().expect("cluster lock") = brokers.iter().map(|broker| broker.addr).collect();
let admin = AdminClient::connect(&[brokers[0].addr.to_string()])
.await
.expect("admin connects");
let result = admin
.list_groups_with_retry(
&case.options,
RetryPolicy {
timeout: case.timeout,
initial_backoff: BACKOFF,
max_backoff: BACKOFF,
jitter: 0.0,
max_retries: u32::MAX,
},
)
.await
.expect("list_groups succeeds");
let expected = ListGroupsResult {
valid: case.valid,
errors: case
.errors
.into_iter()
.map(|(node_id, code, message)| {
let addr = brokers[usize::try_from(node_id - 1).expect("node id")].addr;
ListGroupsError {
node_id,
host: addr.ip().to_string(),
port: i32::from(addr.port()),
error: KafkaError {
code,
name: kafka_error_name(code),
message,
},
}
})
.collect(),
};
for broker in brokers {
broker.stop();
}
let requests = requests
.iter()
.map(|sent| sent.lock().expect("requests lock").clone())
.collect::<Vec<_>>();
let name = case.name;
assert!(
(result, requests) == (expected, case.requests.to_vec()),
"case {name}"
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn list_groups_asks_every_broker_and_merges() {
for case in [
ListGroupsCase {
name: "one group on each broker",
brokers: [groups(&["a"]), groups(&["b"]), groups(&["c"])],
options: ListGroupsOptions::default(),
timeout: LONG,
valid: vec![
stable_listing("a"),
stable_listing("b"),
stable_listing("c"),
],
errors: Vec::new(),
requests: [vec![plain(5)], vec![plain(5)], vec![plain(5)]],
},
ListGroupsCase {
name: "broker 2 has no groups",
brokers: [groups(&["a"]), groups(&[]), groups(&["c"])],
options: ListGroupsOptions::default(),
timeout: LONG,
valid: vec![stable_listing("a"), stable_listing("c")],
errors: Vec::new(),
requests: [vec![plain(5)], vec![plain(5)], vec![plain(5)]],
},
ListGroupsCase {
name: "a group on two brokers is listed once",
brokers: [groups(&["a"]), groups(&["a", "b"]), groups(&["c"])],
options: ListGroupsOptions::default(),
timeout: LONG,
valid: vec![
stable_listing("a"),
stable_listing("b"),
stable_listing("c"),
],
errors: Vec::new(),
requests: [vec![plain(5)], vec![plain(5)], vec![plain(5)]],
},
ListGroupsCase {
name: "broker 2 answers coordinator not available until the timeout",
brokers: [groups(&["a"]), broker_error(15), groups(&["c"])],
options: ListGroupsOptions::default(),
timeout: SHORT,
valid: vec![stable_listing("a"), stable_listing("c")],
errors: vec![(
2,
7,
Some(format!(
"ListGroups timed out after 1 attempt(s); last error: {}",
AdminError::Broker {
api: "ListGroups",
code: 15,
name: "COORDINATOR_NOT_AVAILABLE",
message: None,
}
)),
)],
requests: [vec![plain(5)], vec![plain(5)], vec![plain(5)]],
},
ListGroupsCase {
name: "broker 2 answers coordinator load in progress, then its groups",
brokers: [
groups(&["a"]),
ListGroupsBroker {
max_version: 5,
answers: vec![
ListGroupsAnswer::Error(14),
ListGroupsAnswer::Error(15),
ListGroupsAnswer::Groups(vec![consumer_group("b")]),
],
},
groups(&["c"]),
],
options: ListGroupsOptions::default(),
timeout: LONG,
valid: vec![
stable_listing("a"),
stable_listing("b"),
stable_listing("c"),
],
errors: Vec::new(),
requests: [
vec![plain(5)],
vec![plain(5), plain(5), plain(5)],
vec![plain(5)],
],
},
ListGroupsCase {
name: "broker 2 answers cluster authorization failed",
brokers: [groups(&["a"]), broker_error(31), groups(&["c"])],
options: ListGroupsOptions::default(),
timeout: LONG,
valid: vec![stable_listing("a"), stable_listing("c")],
errors: vec![(2, 31, None)],
requests: [vec![plain(5)], vec![plain(5)], vec![plain(5)]],
},
] {
run_list_groups_case(case).await;
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn list_groups_sends_filters_as_kafka_does() {
for case in [
ListGroupsCase {
name: "state filter Stable is sent to every broker",
brokers: [
ListGroupsBroker {
max_version: 5,
answers: vec![ListGroupsAnswer::Groups(vec![
listed("a", "consumer", "Empty", "Classic"),
consumer_group("b"),
])],
},
groups(&["c"]),
groups(&[]),
],
options: states(&[GroupState::Stable]),
timeout: LONG,
valid: vec![stable_listing("b"), stable_listing("c")],
errors: Vec::new(),
requests: [vec![stable(5)], vec![stable(5)], vec![stable(5)]],
},
ListGroupsCase {
name: "state filter on a broker below v4 is unsupported on that broker",
brokers: [groups(&["a"]), below(3, &["b"]), groups(&["c"])],
options: states(&[GroupState::Stable]),
timeout: LONG,
valid: vec![stable_listing("a"), stable_listing("c")],
errors: vec![(2, 35, Some(unsupported(3, 4)))],
requests: [vec![stable(5)], Vec::new(), vec![stable(5)]],
},
ListGroupsCase {
name: "type filter Consumer on a broker below v5 is unsupported on that broker",
brokers: [groups(&["a"]), below(4, &["b"]), groups(&["c"])],
options: types(&[GroupType::Consumer]),
timeout: LONG,
valid: Vec::new(),
errors: vec![(2, 35, Some(unsupported(4, 5)))],
requests: [vec![consumer_type(5)], Vec::new(), vec![consumer_type(5)]],
},
ListGroupsCase {
name: "type filter Classic and Consumer on a broker below v5 is omitted",
brokers: [groups(&["a"]), below(4, &["b"]), groups(&["c"])],
options: types(&[GroupType::Classic, GroupType::Consumer]),
timeout: LONG,
valid: vec![
stable_listing("a"),
GroupListing {
group_type: None,
..stable_listing("b")
},
stable_listing("c"),
],
errors: Vec::new(),
requests: [
vec![classic_and_consumer(5)],
vec![classic_and_consumer(4)],
vec![classic_and_consumer(5)],
],
},
ListGroupsCase {
name: "protocol type filter is applied by the client",
brokers: [
ListGroupsBroker {
max_version: 5,
answers: vec![ListGroupsAnswer::Groups(vec![
consumer_group("a"),
listed("connect-a", "connect", "Stable", "Classic"),
listed("simple", "", "Empty", "Classic"),
])],
},
groups(&[]),
ListGroupsBroker {
max_version: 5,
answers: vec![ListGroupsAnswer::Groups(vec![listed(
"share", "share", "Stable", "Share",
)])],
},
],
options: ListGroupsOptions::for_consumer_groups(),
timeout: LONG,
valid: vec![
stable_listing("a"),
GroupListing {
group_id: "simple".into(),
group_type: Some(GroupType::Classic),
protocol_type: String::new(),
group_state: Some(GroupState::Empty),
},
],
errors: Vec::new(),
requests: [
vec![classic_and_consumer(5)],
vec![classic_and_consumer(5)],
vec![classic_and_consumer(5)],
],
},
] {
run_list_groups_case(case).await;
}
}
#[derive(Clone, Copy, Debug)]
enum SecuredBroker {
Stalls,
RejectsAuthentication,
}
struct StartedSecuredBroker {
addr: std::net::SocketAddr,
connections: Arc<AtomicUsize>,
mock: Option<MockBroker>,
listener: Option<tokio::task::JoinHandle<()>>,
}
impl SecuredBroker {
async fn start(self) -> StartedSecuredBroker {
let connections = Arc::new(AtomicUsize::new(0));
let counter = Arc::clone(&connections);
match self {
Self::Stalls => {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
.await
.expect("listener binds");
let addr = listener.local_addr().expect("listener address");
let task = tokio::spawn(async move {
let mut open = Vec::new();
while let Ok((stream, _)) = listener.accept().await {
counter.fetch_add(1, Ordering::SeqCst);
open.push(stream);
}
});
StartedSecuredBroker {
addr,
connections,
mock: None,
listener: Some(task),
}
}
Self::RejectsAuthentication => {
let mock = MockBroker::start_with_replies(move |api_key, version, _, _| {
if api_key == sasl_handshake_request::API_KEY {
counter.fetch_add(1, Ordering::SeqCst);
}
MockSaslAnswer::AuthenticateError(SASL_AUTHENTICATION_FAILED)
.reply(api_key, version)
.unwrap_or(MockReply::Silent)
})
.await;
StartedSecuredBroker {
addr: mock.addr,
connections,
mock: Some(mock),
listener: None,
}
}
}
}
fn expected_error(self, addr: std::net::SocketAddr) -> (i16, Option<String>) {
match self {
Self::Stalls => (
7,
Some("ListGroups timed out after 1 attempt(s)".to_owned()),
),
Self::RejectsAuthentication => (
SASL_AUTHENTICATION_FAILED,
Some(
AdminError::Transport(ClientError::Authentication {
addr,
source: AuthenticationError::Sasl(SaslAuthenticationError::Failed(
"SaslAuthenticate(PLAIN) error_code=58 \
error_message=Some(\"rejected by mock broker\")"
.to_owned(),
)),
})
.to_string(),
),
),
}
}
}
impl StartedSecuredBroker {
fn stop(self) {
if let Some(mock) = self.mock {
mock.stop();
}
if let Some(listener) = self.listener {
listener.abort();
}
}
}
const SASL_AUTHENTICATION_FAILED: i16 = 58;
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn list_groups_isolates_a_secured_broker_that_stalls_or_rejects() {
const RETRY: RetryPolicy = RetryPolicy {
timeout: Duration::from_secs(2),
initial_backoff: Duration::from_millis(100),
max_backoff: Duration::from_millis(100),
jitter: 0.0,
max_retries: u32::MAX,
};
for broker_2 in [SecuredBroker::Stalls, SecuredBroker::RejectsAuthentication] {
let cluster = Arc::new(Mutex::new(Vec::new()));
let requests = Arc::new(Mutex::new(Vec::new()));
let broker_1 =
list_groups_broker(groups(&["a"]), Arc::clone(&cluster), Arc::clone(&requests))
.await;
let broker_2_started = broker_2.start().await;
let broker_3 =
list_groups_broker(groups(&["c"]), Arc::clone(&cluster), Arc::clone(&requests))
.await;
let broker_2_addr = broker_2_started.addr;
*cluster.lock().expect("cluster lock") =
vec![broker_1.addr, broker_2_addr, broker_3.addr];
let admin = AdminClient::connect_secured(
&[broker_1.addr.to_string()],
Some(ClientSecurity {
protocol: krabka_security::ListenerProtocol::SaslPlaintext,
tls: None,
sasl: Some(SaslCredentials::Plain {
username: "alice".into(),
password: "secret".into(),
}),
sasl_host: None,
}),
)
.await
.expect("admin connects");
let outcome = tokio::time::timeout(
RETRY.timeout * 5,
admin.list_groups_with_retry(&ListGroupsOptions::default(), RETRY),
)
.await
.map(|result| result.expect("list_groups succeeds"));
let observed = (outcome, broker_2_started.connections.load(Ordering::SeqCst));
let (code, message) = broker_2.expected_error(broker_2_addr);
let expected = (
Ok(ListGroupsResult {
valid: vec![stable_listing("a"), stable_listing("c")],
errors: vec![ListGroupsError {
node_id: 2,
host: broker_2_addr.ip().to_string(),
port: i32::from(broker_2_addr.port()),
error: KafkaError {
code,
name: kafka_error_name(code),
message,
},
}],
}),
1,
);
broker_1.stop();
broker_2_started.stop();
broker_3.stop();
assert!(observed == expected, "case {broker_2:?}");
}
}
#[test]
fn is_simple_consumer_group_matches_kafka() {
for (group_type, protocol_type, expected) in [
(Some(GroupType::Classic), "", true),
(Some(GroupType::Classic), "consumer", false),
(Some(GroupType::Consumer), "", false),
(Some(GroupType::Unknown), "", false),
(None, "", false),
(None, "consumer", false),
] {
let listing = GroupListing {
group_id: "g".into(),
group_type,
protocol_type: protocol_type.into(),
group_state: None,
};
assert!(
listing.is_simple_consumer_group() == expected,
"type {group_type:?}, protocol type {protocol_type:?}"
);
}
}
#[test]
fn group_listing_reads_listed_group() {
for (group, expected) in [
(
listed("g", "consumer", "Stable", "Consumer"),
GroupListing {
group_id: "g".into(),
group_type: Some(GroupType::Consumer),
protocol_type: "consumer".into(),
group_state: Some(GroupState::Stable),
},
),
(
listed("g", "", "", ""),
GroupListing {
group_id: "g".into(),
group_type: None,
protocol_type: String::new(),
group_state: None,
},
),
(
listed("g", "consumer", "preparingrebalance", "STREAMS"),
GroupListing {
group_id: "g".into(),
group_type: Some(GroupType::Streams),
protocol_type: "consumer".into(),
group_state: Some(GroupState::PreparingRebalance),
},
),
(
listed("g", "consumer", "Rebalancing", "Future"),
GroupListing {
group_id: "g".into(),
group_type: Some(GroupType::Unknown),
protocol_type: "consumer".into(),
group_state: Some(GroupState::Unknown),
},
),
] {
assert!(GroupListing::from(group) == expected);
}
}
fn api_versions_all() -> Vec<u8> {
let api = |api_key, max_version| ApiVersion {
api_key,
min_version: 0,
max_version,
..Default::default()
};
encode(
&ApiVersionsResponse {
api_keys: vec![
api(api_versions_request::API_KEY, 0),
api(find_coordinator_request::API_KEY, 0),
api(metadata_request::API_KEY, 13),
api(offset_commit_request::API_KEY, 9),
api(offset_fetch_request::API_KEY, 9),
api(OffsetDeleteRequest::API_KEY, 0),
api(LeaveGroupRequest::API_KEY, 5),
api(AlterShareGroupOffsetsRequest::API_KEY, 1),
api(DeleteShareGroupOffsetsRequest::API_KEY, 0),
],
..Default::default()
},
0,
false,
)
}
async fn connect_self_coordinating(
target_api_key: i16,
codes: Vec<i16>,
respond: impl Fn(i16, i16) -> Vec<u8> + Send + Sync + 'static,
) -> (AdminClient, MockBroker, Arc<AtomicUsize>) {
let coordinator_addr = Arc::new(Mutex::new(None::<std::net::SocketAddr>));
let addr_in_mock = Arc::clone(&coordinator_addr);
let script = Arc::new(Mutex::new((codes, 0_usize)));
let calls = Arc::new(AtomicUsize::new(0));
let calls_in_mock = Arc::clone(&calls);
let broker = MockBroker::start(move |api_key, version, _, _| {
if api_key == target_api_key {
calls_in_mock.fetch_add(1, Ordering::SeqCst);
let mut guard = script.lock().expect("script lock");
let (codes, requests) = &mut *guard;
let code = RetryScript::next(codes, requests);
Some(respond(code, version))
} else {
match api_key {
api_versions_request::API_KEY => Some(api_versions_all()),
find_coordinator_request::API_KEY => {
let addr = addr_in_mock
.lock()
.expect("coordinator lock")
.expect("coordinator address set before use");
Some(encode(
&FindCoordinatorResponse {
node_id: 2,
host: addr.ip().to_string(),
port: i32::from(addr.port()),
..Default::default()
},
version,
false,
))
}
metadata_request::API_KEY => {
Some(encode(&MetadataResponse::default(), version, true))
}
_ => None,
}
}
})
.await;
*coordinator_addr.lock().expect("coordinator lock") = Some(broker.addr);
let admin = AdminClient::connect(&[broker.addr.to_string()])
.await
.expect("admin connects");
(admin, broker, calls)
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn delete_consumer_group_offsets_retries_coordinator_errors_as_kafka_does() {
let respond = |code: i16, version: i16| {
let topics = if code == 0 {
vec![OffsetDeleteResponseTopic {
name: "orders".into(),
partitions: vec![OffsetDeleteResponsePartition {
partition_index: 2,
error_code: 0,
..Default::default()
}],
..Default::default()
}]
} else {
Vec::new()
};
encode(
&OffsetDeleteResponse {
error_code: code,
topics,
..Default::default()
},
version,
false,
)
};
let outcome = Ok(vec![ConsumerGroupOffsetOutcome {
topic: "orders".into(),
partition: 2,
error: None,
}]);
for (name, codes, expected_calls, expected) in [
("no error", vec![0], 1, outcome.clone()),
(
"coordinator load in progress retries on the same coordinator",
vec![14, 0],
2,
outcome.clone(),
),
(
"not coordinator finds the coordinator again",
vec![16, 0],
2,
outcome.clone(),
),
("non-empty group is final", vec![68], 1, Err(68)),
] {
let (mut admin, broker, calls) =
connect_self_coordinating(OffsetDeleteRequest::API_KEY, codes, respond).await;
let result = admin
.delete_consumer_group_offsets("workers", &[("orders".into(), 2)])
.await
.map_err(|error| match error {
AdminError::Broker { code, .. } => code,
other => panic!("case {name}: unexpected error {other:?}"),
});
broker.stop();
assert!(
(result, calls.load(Ordering::SeqCst)) == (expected, expected_calls),
"case {name}"
);
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn delete_streams_group_offsets_delegates_to_offset_delete() {
let respond = |_code: i16, version: i16| {
encode(
&OffsetDeleteResponse {
error_code: 0,
topics: vec![OffsetDeleteResponseTopic {
name: "orders".into(),
partitions: vec![OffsetDeleteResponsePartition {
partition_index: 2,
error_code: 0,
..Default::default()
}],
..Default::default()
}],
..Default::default()
},
version,
false,
)
};
let (mut admin, broker, calls) =
connect_self_coordinating(OffsetDeleteRequest::API_KEY, vec![0], respond).await;
let result = admin
.delete_streams_group_offsets("workers", &[("orders".into(), 2)])
.await
.map_err(|error| error.to_string());
broker.stop();
assert!(
(result, calls.load(Ordering::SeqCst))
== (
Ok(vec![ConsumerGroupOffsetOutcome {
topic: "orders".into(),
partition: 2,
error: None,
}]),
1
)
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn remove_members_from_consumer_group_retries_coordinator_errors_as_kafka_does() {
let answered = |instance_id: &str, error_code| MemberResponse {
member_id: String::new(),
group_instance_id: Some(instance_id.into()),
error_code,
..Default::default()
};
let respond = move |code: i16, version: i16| {
let members = if code == 0 {
vec![answered("b", 25), answered("a", 0)]
} else {
Vec::new()
};
encode(
&LeaveGroupResponse {
error_code: code,
members,
..Default::default()
},
version,
true,
)
};
let removed = |instance_id: &str| RemovedMember {
member_id: String::new(),
group_instance_id: Some(instance_id.into()),
};
let outcome = Ok(RemovedMembers {
members: BTreeMap::from([
(removed("a"), None),
(
removed("b"),
Some(KafkaError {
code: 25,
name: "UNKNOWN_MEMBER_ID",
message: None,
}),
),
]),
});
for (name, codes, expected_calls, expected) in [
("no error", vec![0], 1, outcome.clone()),
(
"coordinator load in progress retries on the same coordinator",
vec![14, 0],
2,
outcome.clone(),
),
(
"not coordinator finds the coordinator again",
vec![16, 0],
2,
outcome.clone(),
),
("group authorization failed is final", vec![30], 1, Err(30)),
] {
let (mut admin, broker, calls) =
connect_self_coordinating(LeaveGroupRequest::API_KEY, codes, respond).await;
let result = admin
.remove_members_from_consumer_group(
"workers",
&[
MemberToRemove {
group_instance_id: "a".into(),
},
MemberToRemove {
group_instance_id: "b".into(),
},
],
&RemoveMembersOptions::default(),
)
.await
.map_err(|error| match error {
AdminError::Broker { code, .. } => code,
other => panic!("case {name}: unexpected error {other:?}"),
});
broker.stop();
assert!(
(result, calls.load(Ordering::SeqCst)) == (expected, expected_calls),
"case {name}"
);
}
}
#[test]
fn leave_group_reason_matches_kafka() {
let long = "r".repeat(300);
for (name, reason, expected) in [
("none", None, DEFAULT_LEAVE_GROUP_REASON.to_owned()),
("empty", Some(""), DEFAULT_LEAVE_GROUP_REASON.to_owned()),
(
"short",
Some("rolling restart"),
"rolling restart".to_owned(),
),
("long", Some(long.as_str()), "r".repeat(255)),
] {
assert!(leave_group_reason(reason) == expected, "case {name}");
}
}
#[test]
fn removed_members_member_result_matches_kafka() {
let member = |instance_id: &str| MemberToRemove {
group_instance_id: instance_id.into(),
};
let fenced = KafkaError {
code: 82,
name: kafka_error_name(82),
message: None,
};
let result = RemovedMembers {
members: BTreeMap::from([
(
RemovedMember {
member_id: String::new(),
group_instance_id: Some("a".into()),
},
None,
),
(
RemovedMember {
member_id: String::new(),
group_instance_id: Some("b".into()),
},
Some(fenced.clone()),
),
]),
};
assert!(
[member("a"), member("b"), member("c")].map(|member| result.member_result(&member))
== [
Ok(()),
Err(fenced),
Err(KafkaError {
code: -1,
name: "UNKNOWN_SERVER_ERROR",
message: Some(
"Member \"c\" was not included in the removal response".into()
),
}),
]
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn alter_share_group_offsets_retries_coordinator_errors_as_kafka_does() {
let respond = |code: i16, version: i16| {
let responses = if code == 0 {
vec![AlterShareGroupOffsetsResponseTopic {
topic_name: "orders".into(),
partitions: vec![AlterShareGroupOffsetsResponsePartition {
partition_index: 2,
error_code: 0,
..Default::default()
}],
..Default::default()
}]
} else {
Vec::new()
};
encode(
&AlterShareGroupOffsetsResponse {
error_code: code,
responses,
..Default::default()
},
version,
true,
)
};
let outcome = Ok(vec![ConsumerGroupOffsetOutcome {
topic: "orders".into(),
partition: 2,
error: None,
}]);
for (name, codes, expected_calls, expected) in [
("no error", vec![0], 1, outcome.clone()),
(
"coordinator load in progress retries on the same coordinator",
vec![14, 0],
2,
outcome.clone(),
),
(
"not coordinator finds the coordinator again",
vec![16, 0],
2,
outcome.clone(),
),
(
"rebalance in progress retries on the same coordinator",
vec![27, 0],
2,
outcome.clone(),
),
(
"a final group error is the error of each partition",
vec![68],
1,
Ok(vec![ConsumerGroupOffsetOutcome {
topic: "orders".into(),
partition: 2,
error: Some(KafkaError {
code: 68,
name: "NON_EMPTY_GROUP",
message: None,
}),
}]),
),
] {
let (mut admin, broker, calls) =
connect_self_coordinating(AlterShareGroupOffsetsRequest::API_KEY, codes, respond)
.await;
let result = admin
.alter_share_group_offsets("workers", &BTreeMap::from([(("orders".into(), 2), 41)]))
.await
.map_err(|error| match error {
AdminError::Broker { code, .. } => code,
other => panic!("case {name}: unexpected error {other:?}"),
});
broker.stop();
assert!(
(result, calls.load(Ordering::SeqCst)) == (expected, expected_calls),
"case {name}"
);
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn alter_streams_group_offsets_delegates_to_offset_commit() {
let respond = |_code: i16, version: i16| {
encode(
&OffsetCommitResponse {
topics: vec![OffsetCommitResponseTopic {
name: "orders".into(),
partitions: vec![OffsetCommitResponsePartition {
partition_index: 2,
error_code: 0,
..Default::default()
}],
..Default::default()
}],
..Default::default()
},
version,
true,
)
};
let (mut admin, broker, calls) =
connect_self_coordinating(offset_commit_request::API_KEY, vec![0], respond).await;
let result = admin
.alter_streams_group_offsets("workers", &BTreeMap::from([(("orders".into(), 2), 41)]))
.await
.map_err(|error| error.to_string());
broker.stop();
assert!(
(result, calls.load(Ordering::SeqCst))
== (
Ok(vec![ConsumerGroupOffsetOutcome {
topic: "orders".into(),
partition: 2,
error: None,
}]),
1
)
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn list_streams_group_offsets_delegates_to_offset_fetch() {
let respond =
|_code: i16, version: i16| encode(&offset_fetch_response(version), version, true);
let (mut admin, broker, calls) =
connect_self_coordinating(offset_fetch_request::API_KEY, vec![0], respond).await;
let result = admin
.list_streams_group_offsets("workers")
.await
.map_err(|error| error.to_string());
broker.stop();
assert!(
(result, calls.load(Ordering::SeqCst))
== (Ok(BTreeMap::from([(("orders".into(), 2), 41)])), 1)
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn delete_share_group_offsets_retries_coordinator_errors_as_kafka_does() {
let respond = |code: i16, version: i16| {
let responses = if code == 0 {
vec![DeleteShareGroupOffsetsResponseTopic {
topic_name: "orders".into(),
error_code: 0,
..Default::default()
}]
} else {
Vec::new()
};
encode(
&DeleteShareGroupOffsetsResponse {
error_code: code,
responses,
..Default::default()
},
version,
true,
)
};
let outcome = Ok(vec![("orders".to_string(), None)]);
for (name, codes, expected_calls, expected) in [
("no error", vec![0], 1, outcome.clone()),
(
"coordinator load in progress retries on the same coordinator",
vec![14, 0],
2,
outcome.clone(),
),
(
"not coordinator finds the coordinator again",
vec![16, 0],
2,
outcome.clone(),
),
(
"rebalance in progress retries on the same coordinator",
vec![27, 0],
2,
outcome.clone(),
),
("group authorization failed is final", vec![30], 1, Err(30)),
] {
let (mut admin, broker, calls) =
connect_self_coordinating(DeleteShareGroupOffsetsRequest::API_KEY, codes, respond)
.await;
let result = admin
.delete_share_group_offsets("workers", &["orders".to_string()])
.await
.map_err(|error| match error {
AdminError::Broker { code, .. } => code,
other => panic!("case {name}: unexpected error {other:?}"),
});
broker.stop();
assert!(
(result, calls.load(Ordering::SeqCst)) == (expected, expected_calls),
"case {name}"
);
}
}
}