Skip to main content

krabka_client_admin/
groups.rs

1//! Consumer-group admin APIs: [`AdminClient::list_groups`] and
2//! [`AdminClient::list_consumer_group_offsets`].
3//!
4//! These wrap the `ListGroups` (`api_key`=16), `OffsetCommit` (`api_key`=8)
5//! and `OffsetFetch` (`api_key`=9) RPCs.
6//!
7//! ## `ListGroups` fan-out
8//!
9//! A broker lists only the groups that it coordinates.
10//! [`AdminClient::list_groups`] sends `ListGroups` to every broker in the
11//! metadata and merges the answers, as Apache Kafka's
12//! `KafkaAdminClient.listGroups` does.
13//!
14//! ## `OffsetCommit` version note
15//!
16//! [`AdminClient::alter_consumer_group_offsets`] sends `OffsetCommit` at v9 or
17//! lower, where the request names each topic. Apache Kafka's
18//! `AlterConsumerGroupOffsetsHandler` builds its request with
19//! `OffsetCommitRequest.Builder.forTopicNames`, which caps the version at 9.
20//!
21//! ## `OffsetFetch` version note
22//!
23//! [`AdminClient::list_consumer_group_offsets`] sends `OffsetFetch` at v2 to
24//! v9, where the response names each topic. Apache Kafka's
25//! `ListConsumerGroupOffsetsHandler` builds its request with
26//! `OffsetFetchRequest.Builder.forTopicNames`, which caps the version at 9.
27//!
28//! ## `OffsetFetch` retries
29//!
30//! [`AdminClient::list_consumer_group_offsets`] retries the coordinator error
31//! codes as Apache Kafka's `ListConsumerGroupOffsetsHandler.handleGroupError`
32//! does, until Kafka's default `default.api.timeout.ms` (60 s) elapses.
33
34use std::collections::{BTreeMap, BTreeSet};
35
36use krabka_client_core::{
37    AllTopicsOffsetFetch, AuthenticationError, ClientError, Connection, ConnectionOptions,
38    CoordinatorKeyType, MembersOnlyLeaveGroup, SaslAuthenticationError, TopicNameOffsetCommit,
39    build_find_coordinator, coordinator_endpoint,
40};
41use krabka_protocol::owned::{
42    alter_share_group_offsets_request::{
43        AlterShareGroupOffsetsRequest, AlterShareGroupOffsetsRequestPartition,
44        AlterShareGroupOffsetsRequestTopic,
45    },
46    alter_share_group_offsets_response::AlterShareGroupOffsetsResponse,
47    delete_share_group_offsets_request::{
48        DeleteShareGroupOffsetsRequest, DeleteShareGroupOffsetsRequestTopic,
49    },
50    delete_share_group_offsets_response::DeleteShareGroupOffsetsResponse,
51    leave_group_request::{LeaveGroupRequest, MemberIdentity},
52    leave_group_response::LeaveGroupResponse,
53    list_groups_request::{self, ListGroupsRequest},
54    list_groups_response::ListedGroup,
55    metadata_request::MetadataRequest,
56    metadata_response::MetadataResponseBroker,
57    offset_commit_request::{
58        OffsetCommitRequest, OffsetCommitRequestPartition, OffsetCommitRequestTopic,
59    },
60    offset_commit_response::{
61        OffsetCommitResponse, OffsetCommitResponsePartition, OffsetCommitResponseTopic,
62    },
63    offset_delete_request::{
64        OffsetDeleteRequest, OffsetDeleteRequestPartition, OffsetDeleteRequestTopic,
65    },
66    offset_delete_response::{
67        OffsetDeleteResponse, OffsetDeleteResponsePartition, OffsetDeleteResponseTopic,
68    },
69    offset_fetch_request::{OffsetFetchRequest, OffsetFetchRequestGroup},
70    offset_fetch_response::{
71        OffsetFetchResponse, OffsetFetchResponsePartition, OffsetFetchResponsePartitions,
72        OffsetFetchResponseTopic, OffsetFetchResponseTopics,
73    },
74};
75
76use crate::{
77    AdminClient, AdminError, KafkaError, format_host_port,
78    group_descriptions::DescribeGroupsOptions,
79    kafka_error_if, kafka_error_name,
80    retry::{CoordinatorRetry, RetryAction, RetryPolicy, connection_failure_action},
81    send_connection_at_least,
82};
83
84macro_rules! group_offset_attempt {
85    ($this:ident, $group:expr, $find_coordinator:expr, $request:expr, $handle:expr) => {{
86        if $find_coordinator && let Err(action) = $this.find_group_coordinator_attempt($group).await
87        {
88            return action;
89        }
90        match $this.conn.send($request).await {
91            Ok(response) => $handle($group, response),
92            Err(error) => connection_failure_action(error),
93        }
94    }};
95}
96
97#[derive(Debug, Clone, PartialEq, Eq)]
98pub struct ConsumerGroupOffsetOutcome {
99    pub topic: String,
100    pub partition: i32,
101    pub error: Option<KafkaError>,
102}
103
104struct GroupOffsetResponseTopic<P> {
105    name: String,
106    partitions: Vec<P>,
107}
108
109trait IntoGroupOffsetResponseTopic {
110    type Partition: GroupOffsetResponsePartition;
111
112    fn into_group_offset_topic(self) -> GroupOffsetResponseTopic<Self::Partition>;
113}
114
115trait GroupOffsetResponsePartition {
116    fn index(&self) -> i32;
117    fn error_code(&self) -> i16;
118}
119
120macro_rules! impl_group_offset_response {
121    ($topic:ty, $partition:ty) => {
122        impl IntoGroupOffsetResponseTopic for $topic {
123            type Partition = $partition;
124
125            fn into_group_offset_topic(self) -> GroupOffsetResponseTopic<Self::Partition> {
126                GroupOffsetResponseTopic {
127                    name: self.name,
128                    partitions: self.partitions,
129                }
130            }
131        }
132
133        impl GroupOffsetResponsePartition for $partition {
134            fn index(&self) -> i32 {
135                self.partition_index
136            }
137            fn error_code(&self) -> i16 {
138                self.error_code
139            }
140        }
141    };
142}
143
144impl_group_offset_response!(OffsetCommitResponseTopic, OffsetCommitResponsePartition);
145impl_group_offset_response!(OffsetDeleteResponseTopic, OffsetDeleteResponsePartition);
146
147fn group_offset_outcomes<T: IntoGroupOffsetResponseTopic>(
148    topics: impl IntoIterator<Item = T>,
149) -> Vec<ConsumerGroupOffsetOutcome> {
150    topics
151        .into_iter()
152        .flat_map(|topic| {
153            let GroupOffsetResponseTopic { name, partitions } = topic.into_group_offset_topic();
154            partitions
155                .into_iter()
156                .map(move |partition| ConsumerGroupOffsetOutcome {
157                    topic: name.clone(),
158                    partition: partition.index(),
159                    error: kafka_error_if(partition.error_code(), None),
160                })
161        })
162        .collect()
163}
164
165/// Offsets keyed by topic name and partition.
166pub type GroupOffsets = BTreeMap<(String, i32), i64>;
167
168#[derive(Clone, Copy)]
169enum GroupOffsetKind {
170    Consumer,
171    Share,
172}
173
174#[derive(Clone, Copy)]
175struct GroupOffsetUpdate<'a> {
176    kind: GroupOffsetKind,
177    group: &'a str,
178    offsets: &'a GroupOffsets,
179}
180
181impl<'a> GroupOffsetUpdate<'a> {
182    fn consumer(group: &'a str, offsets: &'a GroupOffsets) -> Self {
183        Self {
184            kind: GroupOffsetKind::Consumer,
185            group,
186            offsets,
187        }
188    }
189
190    fn share(group: &'a str, offsets: &'a GroupOffsets) -> Self {
191        Self {
192            kind: GroupOffsetKind::Share,
193            group,
194            offsets,
195        }
196    }
197}
198
199struct GroupOffsetAttempt<'a> {
200    update: GroupOffsetUpdate<'a>,
201    find_coordinator: bool,
202}
203
204/// The state of a group, as Apache Kafka's `GroupState` names it.
205///
206/// `ListGroups` v4 (KIP-518) filters groups by these names.
207#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
208pub enum GroupState {
209    /// A state name that the client does not know.
210    Unknown,
211    PreparingRebalance,
212    CompletingRebalance,
213    Stable,
214    Dead,
215    Empty,
216    Assigning,
217    Reconciling,
218    NotReady,
219}
220
221impl GroupState {
222    const ALL: [Self; 9] = [
223        Self::Unknown,
224        Self::PreparingRebalance,
225        Self::CompletingRebalance,
226        Self::Stable,
227        Self::Dead,
228        Self::Empty,
229        Self::Assigning,
230        Self::Reconciling,
231        Self::NotReady,
232    ];
233
234    /// The wire name of the state, for example `"Stable"`.
235    #[must_use]
236    pub const fn as_str(self) -> &'static str {
237        match self {
238            Self::Unknown => "Unknown",
239            Self::PreparingRebalance => "PreparingRebalance",
240            Self::CompletingRebalance => "CompletingRebalance",
241            Self::Stable => "Stable",
242            Self::Dead => "Dead",
243            Self::Empty => "Empty",
244            Self::Assigning => "Assigning",
245            Self::Reconciling => "Reconciling",
246            Self::NotReady => "NotReady",
247        }
248    }
249}
250
251macro_rules! impl_group_name_parse {
252    ($type:ty, $($doc:literal),+ $(,)?) => {
253        impl $type {
254            $(#[doc = $doc])+
255            #[must_use]
256            pub fn parse(name: &str) -> Self {
257                parse_group_name(Self::ALL, name, Self::as_str, Self::Unknown)
258            }
259        }
260    };
261}
262impl_group_name_parse!(
263    GroupState,
264    "Reads a state name without regard to case. A name that the client does not know gives [`GroupState::Unknown`], as Kafka's `GroupState.parse` does."
265);
266
267macro_rules! impl_display_from_as_str {
268    ($type:ty) => {
269        impl std::fmt::Display for $type {
270            fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
271                f.write_str(self.as_str())
272            }
273        }
274    };
275}
276impl_display_from_as_str!(GroupState);
277
278/// The type of a group, as Apache Kafka's `GroupType` names it.
279///
280/// `ListGroups` v5 (KIP-848) filters groups by these names.
281#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
282pub enum GroupType {
283    /// A type name that the client does not know.
284    Unknown,
285    Consumer,
286    Classic,
287    Share,
288    Streams,
289}
290
291impl GroupType {
292    const ALL: [Self; 5] = [
293        Self::Unknown,
294        Self::Consumer,
295        Self::Classic,
296        Self::Share,
297        Self::Streams,
298    ];
299
300    /// The wire name of the type, for example `"Classic"`.
301    #[must_use]
302    pub const fn as_str(self) -> &'static str {
303        match self {
304            Self::Unknown => "Unknown",
305            Self::Consumer => "Consumer",
306            Self::Classic => "Classic",
307            Self::Share => "Share",
308            Self::Streams => "Streams",
309        }
310    }
311}
312impl_group_name_parse!(
313    GroupType,
314    "Reads a type name without regard to case. A name that the client does not know gives [`GroupType::Unknown`], as Kafka's `GroupType.parse` does."
315);
316
317fn parse_group_name<T: Copy>(
318    values: impl IntoIterator<Item = T>,
319    name: &str,
320    as_str: fn(T) -> &'static str,
321    unknown: T,
322) -> T {
323    values
324        .into_iter()
325        .find(|value| as_str(*value).eq_ignore_ascii_case(name))
326        .unwrap_or(unknown)
327}
328
329impl_display_from_as_str!(GroupType);
330
331/// The options of [`AdminClient::list_groups`], as Apache Kafka's
332/// `ListGroupsOptions` holds them. An empty set does not filter.
333#[derive(Debug, Clone, Default, PartialEq, Eq, bon::Builder)]
334pub struct ListGroupsOptions {
335    /// The broker lists only groups in these states (`states_filter`).
336    #[builder(default)]
337    pub group_states: BTreeSet<GroupState>,
338    /// The broker lists only groups of these types (`types_filter`).
339    #[builder(default)]
340    pub types: BTreeSet<GroupType>,
341    /// The client keeps only groups with these protocol types.
342    #[builder(default)]
343    pub protocol_types: BTreeSet<String>,
344}
345
346impl ListGroupsOptions {
347    /// Classic and consumer groups with the `consumer` protocol type or no
348    /// protocol type, as Kafka's `ListGroupsOptions.forConsumerGroups` selects.
349    #[must_use]
350    pub fn for_consumer_groups() -> Self {
351        Self::builder()
352            .types(maplit::btreeset! {
353                GroupType::Classic,
354                GroupType::Consumer,
355            })
356            .protocol_types(maplit::btreeset! {
357                String::new(),
358                "consumer".to_owned(),
359            })
360            .build()
361    }
362
363    /// Share groups, as Kafka's `ListGroupsOptions.forShareGroups` selects.
364    #[must_use]
365    pub fn for_share_groups() -> Self {
366        Self::builder()
367            .types(maplit::btreeset! {
368                GroupType::Share,
369            })
370            .build()
371    }
372
373    /// Streams groups, as Kafka's `ListGroupsOptions.forStreamsGroups`
374    /// selects.
375    #[must_use]
376    pub fn for_streams_groups() -> Self {
377        Self::builder()
378            .types(maplit::btreeset! {
379                GroupType::Streams,
380            })
381            .build()
382    }
383}
384
385/// One group of [`AdminClient::list_groups`], as Apache Kafka's
386/// `GroupListing` holds it.
387#[derive(Debug, Clone, PartialEq, Eq)]
388pub struct GroupListing {
389    pub group_id: String,
390    /// `None` when the broker sends no type (`ListGroups` v4 or lower).
391    pub group_type: Option<GroupType>,
392    /// The protocol type, for example `"consumer"`. Empty for a simple
393    /// consumer group.
394    pub protocol_type: String,
395    /// `None` when the broker sends no state (`ListGroups` v3 or lower).
396    pub group_state: Option<GroupState>,
397}
398
399impl GroupListing {
400    /// Whether the group is a classic group with no protocol type, as Kafka's
401    /// `GroupListing.isSimpleConsumerGroup` decides.
402    ///
403    /// A group with no type (`None`, from `ListGroups` v4 or lower) gives
404    /// `false`, as in Kafka, where `type.filter(gt -> gt == CLASSIC)` is empty
405    /// for `Optional.empty()`. Such a broker lists only classic groups, so a
406    /// caller that wants Kafka's older `listConsumerGroups` answer
407    /// (`ConsumerGroupListing.isSimpleConsumerGroup`, which is
408    /// `protocolType.isEmpty()`) reads [`GroupListing::protocol_type`] alone.
409    #[must_use]
410    pub fn is_simple_consumer_group(&self) -> bool {
411        self.group_type == Some(GroupType::Classic) && self.protocol_type.is_empty()
412    }
413}
414
415impl From<ListedGroup> for GroupListing {
416    fn from(group: ListedGroup) -> Self {
417        Self {
418            group_type: (!group.group_type.is_empty()).then(|| GroupType::parse(&group.group_type)),
419            group_state: (!group.group_state.is_empty())
420                .then(|| GroupState::parse(&group.group_state)),
421            group_id: group.group_id,
422            protocol_type: group.protocol_type,
423        }
424    }
425}
426
427/// The failure of `ListGroups` on one broker.
428///
429/// Kafka's `listGroups` puts one exception for each failed broker in
430/// `ListGroupsResult.errors`. `error` holds the Kafka error code of that
431/// exception, as Kafka's `ApiError.fromThrowable` gives it.
432#[derive(Debug, Clone, PartialEq, Eq)]
433pub struct ListGroupsError {
434    pub node_id: i32,
435    pub host: String,
436    pub port: i32,
437    pub error: KafkaError,
438}
439
440/// The result of [`AdminClient::list_groups`], as Apache Kafka's
441/// `ListGroupsResult` holds it.
442#[derive(Debug, Clone, PartialEq, Eq)]
443pub struct ListGroupsResult {
444    /// The groups of every broker that answered, one entry for each group
445    /// id, in group id order.
446    pub valid: Vec<GroupListing>,
447    /// One entry for each broker that failed, in metadata order.
448    pub errors: Vec<ListGroupsError>,
449}
450
451impl ListGroupsResult {
452    /// The groups when no broker failed, as Kafka's `ListGroupsResult.all`
453    /// gives them.
454    ///
455    /// # Errors
456    /// Returns the first broker failure.
457    pub fn all(self) -> Result<Vec<GroupListing>, ListGroupsError> {
458        match self.errors.into_iter().next() {
459            Some(error) => Err(error),
460            None => Ok(self.valid),
461        }
462    }
463}
464
465macro_rules! group_offsets_method_body {
466    (delete_consumer, $admin:ident, $group:ident, $input:ident) => {
467        $admin
468            .delete_consumer_group_offsets_with_retry($group, $input, $admin.retry)
469            .await
470    };
471    (delete_streams, $admin:ident, $group:ident, $input:ident) => {
472        $admin.delete_consumer_group_offsets($group, $input).await
473    };
474    (alter_consumer, $admin:ident, $group:ident, $input:ident) => {
475        $admin
476            .alter_group_offsets(GroupOffsetUpdate::consumer($group, $input))
477            .await
478    };
479    (alter_share, $admin:ident, $group:ident, $input:ident) => {
480        $admin
481            .alter_group_offsets(GroupOffsetUpdate::share($group, $input))
482            .await
483    };
484}
485
486macro_rules! group_offsets_method {
487    ($(#[$meta:meta])* $name:ident($input:ident: $input_type:ty) => $kind:ident) => {
488        $(#[$meta])*
489        pub async fn $name(
490            &mut self,
491            group: &str,
492            $input: &$input_type,
493        ) -> Result<Vec<ConsumerGroupOffsetOutcome>, AdminError> {
494            group_offsets_method_body!($kind, self, group, $input)
495        }
496    };
497}
498
499macro_rules! retry_group_call {
500    ($retry_policy:expr, |$find:ident| $attempt:expr) => {{
501        let mut retry = CoordinatorRetry::new($retry_policy);
502        loop {
503            let $find = retry.find_coordinator();
504            let action = retry.run($attempt).await;
505            if let Some(result) = retry.next(action).await {
506                return result;
507            }
508        }
509    }};
510}
511
512impl AdminClient {
513    group_offsets_method! {
514        /// Commits explicit offsets for an inactive consumer group.
515        ///
516        /// The request names each topic, so the client negotiates `OffsetCommit`
517        /// v9 or lower, as Apache Kafka's `AlterConsumerGroupOffsetsHandler` does.
518        ///
519        /// The call retries the partition error codes as Apache Kafka's
520        /// `AlterConsumerGroupOffsetsHandler.handleError` does. One retriable code
521        /// in the response makes the call send all the offsets again:
522        ///
523        /// - `COORDINATOR_LOAD_IN_PROGRESS` (14) and `REBALANCE_IN_PROGRESS` (27):
524        ///   send the request again to the same coordinator.
525        /// - `COORDINATOR_NOT_AVAILABLE` (15) and `NOT_COORDINATOR` (16): find the
526        ///   coordinator again, then send the request again.
527        ///
528        /// A `FindCoordinator` answer of 14 or 15 also makes the call find the
529        /// coordinator again. A failed or lost connection to the coordinator,
530        /// including a TLS or SASL handshake that stops before the broker gives a
531        /// verdict, makes the call find the coordinator again, as Kafka's
532        /// `AdminApiDriver.onFailure` does. The call waits between attempts with
533        /// Kafka's backoff, and it stops when Kafka's default
534        /// `default.api.timeout.ms` (60 s) elapses. A call whose retries are
535        /// unresolved at the deadline, or whose attempt is still running, fails
536        /// with [`ClientError::Timeout`], as Kafka gives a `TimeoutException`.
537        ///
538        /// # Errors
539        /// Returns an error when encoding, transport, or response handling fails.
540        /// Returns [`ClientError::Server`] when `FindCoordinator` answers with an
541        /// error code that Kafka does not retry. Returns
542        /// [`ClientError::Timeout`] at the deadline. Returns [`ClientError::IncompatibleVersion`] when the
543        /// coordinator does not support `OffsetCommit` v9 or lower.
544        ///
545        /// [`ClientError::IncompatibleVersion`]: ClientError::IncompatibleVersion
546        /// [`ClientError::Server`]: ClientError::Server
547        /// [`ClientError::Timeout`]: ClientError::Timeout
548        alter_consumer_group_offsets(offsets: GroupOffsets) => alter_consumer
549    }
550
551    async fn alter_group_offsets(
552        &mut self,
553        update: GroupOffsetUpdate<'_>,
554    ) -> Result<Vec<ConsumerGroupOffsetOutcome>, AdminError> {
555        self.alter_group_offsets_with_retry(update, self.retry)
556            .await
557    }
558
559    async fn alter_group_offsets_with_retry(
560        &mut self,
561        update: GroupOffsetUpdate<'_>,
562        retry: RetryPolicy,
563    ) -> Result<Vec<ConsumerGroupOffsetOutcome>, AdminError> {
564        retry_group_call!(retry, |find_coordinator| {
565            self.alter_group_offsets_attempt(GroupOffsetAttempt {
566                update,
567                find_coordinator,
568            })
569        })
570    }
571
572    /// Sends one update attempt, finding the coordinator first when requested.
573    async fn alter_group_offsets_attempt(
574        &mut self,
575        attempt: GroupOffsetAttempt<'_>,
576    ) -> RetryAction<Vec<ConsumerGroupOffsetOutcome>> {
577        let GroupOffsetAttempt {
578            update,
579            find_coordinator,
580        } = attempt;
581        if find_coordinator
582            && let Err(action) = self.find_group_coordinator_attempt(update.group).await
583        {
584            return action;
585        }
586        match update.kind {
587            GroupOffsetKind::Consumer => self
588                .conn
589                .send(offset_commit_request(update.group, update.offsets))
590                .await
591                .map_or_else(connection_failure_action, offset_commit_retry_action),
592            GroupOffsetKind::Share => self
593                .conn
594                .send(alter_share_group_offsets_request(
595                    update.group,
596                    update.offsets,
597                ))
598                .await
599                .map_or_else(connection_failure_action, |response| {
600                    alter_share_group_offsets_retry_action(update.offsets, response)
601                }),
602        }
603    }
604
605    /// Lists the groups of the whole cluster, as Apache Kafka's
606    /// `KafkaAdminClient.listGroups` does.
607    ///
608    /// A broker lists only the groups that it coordinates. The call first
609    /// sends `Metadata` to find every broker. It then sends `ListGroups` to
610    /// each broker at the same time, on a new connection, and merges the
611    /// answers:
612    ///
613    /// - The result has one [`GroupListing`] for each group id. A group that
614    ///   two brokers list appears once.
615    /// - A broker that fails gives one [`ListGroupsError`]. The groups of the
616    ///   other brokers stay in [`ListGroupsResult::valid`].
617    /// - `COORDINATOR_LOAD_IN_PROGRESS` (14) and `COORDINATOR_NOT_AVAILABLE`
618    ///   (15) make the call send `ListGroups` to that broker again. A failed
619    ///   connection or a lost connection also makes the call try again. A
620    ///   rejected TLS or SASL authentication does not. It is an error for that
621    ///   broker at once.
622    /// - Kafka's default `default.api.timeout.ms` (60 s) is the deadline of
623    ///   each broker. Each attempt on a broker (connection, TLS and SASL
624    ///   handshakes, and request) gets the time that remains. A broker that
625    ///   has not answered at the deadline gives `REQUEST_TIMED_OUT` (7), with
626    ///   the last error in the message.
627    /// - A filter that the broker version does not support gives
628    ///   `UNSUPPORTED_VERSION` (35) for that broker. `states_filter` needs
629    ///   `ListGroups` v4 (KIP-518) and `types_filter` needs v5 (KIP-848).
630    ///   Below v5, the call omits a type filter that holds `Classic`, and
631    ///   `Consumer` at most, as Kafka's `ListGroupsRequest.Builder.build`
632    ///   does.
633    /// - [`ListGroupsOptions::protocol_types`] filters on the client.
634    ///
635    /// # Errors
636    /// Returns an error when the `Metadata` request fails, or when the
637    /// metadata names no broker after the timeout. A failure on one broker is
638    /// an entry in [`ListGroupsResult::errors`], not an error of the call.
639    pub async fn list_groups(
640        &self,
641        options: &ListGroupsOptions,
642    ) -> Result<ListGroupsResult, AdminError> {
643        self.list_groups_with_retry(options, self.retry).await
644    }
645
646    async fn list_groups_with_retry(
647        &self,
648        options: &ListGroupsOptions,
649        retry: RetryPolicy,
650    ) -> Result<ListGroupsResult, AdminError> {
651        let start = tokio::time::Instant::now();
652        let brokers = self.list_groups_brokers(start, retry).await?;
653        let request = ListGroupsRequest {
654            states_filter: options
655                .group_states
656                .iter()
657                .map(|state| state.as_str().to_owned())
658                .collect(),
659            types_filter: options
660                .types
661                .iter()
662                .map(|group_type| group_type.as_str().to_owned())
663                .collect(),
664            ..Default::default()
665        };
666        let min_version = list_groups_min_version(options);
667        let answers = futures_util::future::join_all(brokers.iter().map(|broker| {
668            list_groups_on_broker(
669                broker,
670                self.options.clone(),
671                request.clone(),
672                min_version,
673                start,
674                retry,
675            )
676        }))
677        .await;
678
679        let mut listings = BTreeMap::new();
680        let mut errors = Vec::new();
681        for (broker, answer) in brokers.into_iter().zip(answers) {
682            match answer {
683                Ok(groups) => {
684                    for group in groups {
685                        if options.protocol_types.is_empty()
686                            || options.protocol_types.contains(&group.protocol_type)
687                        {
688                            let listing = GroupListing::from(group);
689                            listings.insert(listing.group_id.clone(), listing);
690                        }
691                    }
692                }
693                Err(error) => errors.push(ListGroupsError {
694                    node_id: broker.node_id,
695                    host: broker.host,
696                    port: broker.port,
697                    error,
698                }),
699            }
700        }
701        Ok(ListGroupsResult {
702            valid: listings.into_values().collect(),
703            errors,
704        })
705    }
706
707    /// Sends `Metadata` for no topics and returns each broker once. Kafka's
708    /// `listGroups` retries a broker list that is empty
709    /// (`StaleMetadataException`) until the timeout.
710    async fn list_groups_brokers(
711        &self,
712        start: tokio::time::Instant,
713        retry: RetryPolicy,
714    ) -> Result<Vec<MetadataResponseBroker>, AdminError> {
715        let mut backoff = retry.initial_backoff;
716        loop {
717            let response = self
718                .conn
719                .send(MetadataRequest {
720                    topics: Some(Vec::new()),
721                    allow_auto_topic_creation: true,
722                    ..Default::default()
723                })
724                .await?;
725            let mut brokers = response.brokers;
726            let mut seen = BTreeSet::new();
727            brokers.retain(|broker| seen.insert(broker.node_id));
728            if !brokers.is_empty() {
729                return Ok(brokers);
730            }
731            if start.elapsed() >= retry.timeout {
732                return Err(AdminError::Protocol(
733                    "failed to find brokers to send ListGroups: metadata has no brokers".to_owned(),
734                ));
735            }
736            tokio::time::sleep(backoff).await;
737            backoff = backoff.saturating_mul(2).min(retry.max_backoff);
738        }
739    }
740
741    /// Returns `(topic, partition) → committed_offset` for the named group.
742    ///
743    /// The call requests all topics and partitions (`topics: None`). It skips
744    /// a partition that has an error code, and a partition with a committed
745    /// offset below 0, which means no committed offset.
746    ///
747    /// The response names each topic, so the client negotiates `OffsetFetch`
748    /// v2 to v9, as Apache Kafka's `ListConsumerGroupOffsetsHandler` does.
749    ///
750    /// The call retries the coordinator error codes as Apache Kafka's
751    /// `ListConsumerGroupOffsetsHandler.handleGroupError` does:
752    ///
753    /// - `COORDINATOR_LOAD_IN_PROGRESS` (14): send the request again to the
754    ///   same coordinator.
755    /// - `COORDINATOR_NOT_AVAILABLE` (15) and `NOT_COORDINATOR` (16): find the
756    ///   coordinator again, then send the request again.
757    ///
758    /// A `FindCoordinator` answer of 14 or 15 also makes the call find the
759    /// coordinator again, as Kafka's `CoordinatorStrategy.handleError` does. A
760    /// failed or lost connection to the coordinator does the same, as
761    /// `AdminApiDriver.onFailure` does. The call waits between attempts, and
762    /// it stops when Kafka's default `default.api.timeout.ms` (60 s) elapses.
763    /// A call whose retries are unresolved at the deadline, or whose attempt
764    /// is still running, fails with [`ClientError::Timeout`].
765    ///
766    /// # Errors
767    /// Returns an error when encoding, transport, or response handling fails.
768    /// Returns [`AdminError::Broker`] when the coordinator answers with a
769    /// group error code that Kafka does not retry. Returns
770    /// [`ClientError::Server`] when `FindCoordinator` answers with an error
771    /// code that Kafka does not retry. Returns [`ClientError::Timeout`] at
772    /// the deadline. Returns
773    /// [`ClientError::IncompatibleVersion`] when the coordinator does not
774    /// support `OffsetFetch` v2 to v9.
775    ///
776    /// [`ClientError::IncompatibleVersion`]: ClientError::IncompatibleVersion
777    /// [`ClientError::Server`]: ClientError::Server
778    /// [`ClientError::Timeout`]: ClientError::Timeout
779    pub async fn list_consumer_group_offsets(
780        &mut self,
781        group: &str,
782    ) -> Result<GroupOffsets, AdminError> {
783        self.list_consumer_group_offsets_with_retry(group, self.retry)
784            .await
785    }
786
787    async fn list_consumer_group_offsets_with_retry(
788        &mut self,
789        group: &str,
790        retry: RetryPolicy,
791    ) -> Result<GroupOffsets, AdminError> {
792        retry_group_call!(retry, |find| self.offset_fetch_attempt(group, find))
793    }
794
795    /// One attempt of `list_consumer_group_offsets`. When `find_coordinator`
796    /// is set, the attempt first finds the group coordinator and connects to
797    /// it.
798    async fn offset_fetch_attempt(
799        &mut self,
800        group: &str,
801        find_coordinator: bool,
802    ) -> RetryAction<GroupOffsets> {
803        let request = offset_fetch_request(group);
804        group_offset_attempt!(self, group, find_coordinator, request, |group, response| {
805            offset_fetch_retry_action(committed_offsets(group, response))
806        })
807    }
808
809    /// Finds the group coordinator and connects to it. A `FindCoordinator`
810    /// answer that Kafka's `CoordinatorStrategy.handleError` retries, and a
811    /// connection failure to the coordinator that it names, give
812    /// [`RetryAction::FindCoordinator`], as `AdminApiDriver.onFailure` looks
813    /// the coordinator up again after a disconnect. Another failure gives
814    /// [`RetryAction::Done`].
815    async fn find_group_coordinator_attempt<T>(
816        &mut self,
817        group: &str,
818    ) -> Result<(), RetryAction<T>> {
819        match self.reconnect_group_coordinator(group).await {
820            Ok(()) => Ok(()),
821            Err(AdminError::Transport(ClientError::Server { error_code }))
822                if is_retriable_find_coordinator_error(error_code) =>
823            {
824                Err(RetryAction::FindCoordinator(Err(AdminError::Transport(
825                    ClientError::Server { error_code },
826                ))))
827            }
828            Err(error) => Err(connection_failure_action(error)),
829        }
830    }
831
832    async fn reconnect_group_coordinator(&mut self, group: &str) -> Result<(), AdminError> {
833        let response = self
834            .conn
835            .send(build_find_coordinator(group, CoordinatorKeyType::Group))
836            .await?;
837        let coordinator = coordinator_endpoint(group, response)?;
838        self.reconnect(&format_host_port(&coordinator.host, coordinator.port))
839            .await
840    }
841
842    group_offsets_method! {
843        /// Deletes committed offsets of an inactive consumer group for the named
844        /// partitions with `OffsetDelete`, as Apache Kafka's
845        /// `deleteConsumerGroupOffsets` does. Routes to the group's coordinator
846        /// with the same retry policy as
847        /// [`AdminClient::alter_consumer_group_offsets`].
848        ///
849        /// # Errors
850        /// Returns an error when encoding, transport, or response handling fails.
851        /// Returns [`AdminError::Broker`] when the coordinator's top-level answer
852        /// carries a group error code that Kafka does not retry, such as
853        /// `GROUP_ID_NOT_FOUND` or `NON_EMPTY_GROUP`.
854        delete_consumer_group_offsets(partitions: [(String, i32)]) => delete_consumer
855    }
856
857    group_offsets_method! {
858        /// Deletes committed offsets of an inactive streams group. Apache Kafka's
859        /// `KafkaAdminClient.deleteStreamsGroupOffsets` delegates to
860        /// `deleteConsumerGroupOffsets` verbatim, so this does too.
861        ///
862        /// # Errors
863        /// See [`AdminClient::delete_consumer_group_offsets`].
864        delete_streams_group_offsets(partitions: [(String, i32)]) => delete_streams
865    }
866
867    async fn delete_consumer_group_offsets_with_retry(
868        &mut self,
869        group: &str,
870        partitions: &[(String, i32)],
871        retry: RetryPolicy,
872    ) -> Result<Vec<ConsumerGroupOffsetOutcome>, AdminError> {
873        retry_group_call!(retry, |find| self
874            .offset_delete_attempt(group, partitions, find))
875    }
876
877    async fn offset_delete_attempt(
878        &mut self,
879        group: &str,
880        partitions: &[(String, i32)],
881        find_coordinator: bool,
882    ) -> RetryAction<Vec<ConsumerGroupOffsetOutcome>> {
883        let request = offset_delete_request(group, partitions);
884        group_offset_attempt!(self, group, find_coordinator, request, |group, response| {
885            offset_delete_retry_action(group, response)
886        })
887    }
888
889    /// Removes members from a consumer group without waiting for their
890    /// session timeout, with `LeaveGroup`, as Apache Kafka's
891    /// `removeMembersFromConsumerGroup` does. Routes to the group's
892    /// coordinator with the same retry policy as
893    /// [`AdminClient::alter_consumer_group_offsets`].
894    ///
895    /// Each member is named by its `group.instance.id`, as Kafka's
896    /// `MemberToRemove` names it, and is sent with the member id
897    /// `UNKNOWN_MEMBER_ID` (empty). An empty `members` removes every member of
898    /// the group, as Kafka's `removeAll` mode does: the call first describes
899    /// the group with [`AdminClient::describe_consumer_groups`] and names each
900    /// member by its instance id, or by its member id when it has none.
901    ///
902    /// Each member carries `options.reason`, cut to 255 characters, or
903    /// Kafka's default `"member was removed by an admin"` when the reason is
904    /// `None` or empty. `LeaveGroup`'s member list needs v3, so the client
905    /// negotiates v3 or higher, as Kafka's `LeaveGroupRequest.Builder` does.
906    ///
907    /// The result maps each member of the `LeaveGroup` answer, by member id
908    /// and instance id, to its error, as Kafka's
909    /// `RemoveMembersFromConsumerGroupHandler.handleResponse` does.
910    ///
911    /// # Errors
912    /// Returns an error when encoding, transport, or response handling fails.
913    /// Returns [`AdminError::Broker`] when the coordinator's top-level answer
914    /// carries a group error code that Kafka does not retry, or when the
915    /// `removeAll` describe fails for the group. Returns
916    /// [`ClientError::IncompatibleVersion`] when the coordinator does not
917    /// support `LeaveGroup` v3 or higher.
918    ///
919    /// [`ClientError::IncompatibleVersion`]: ClientError::IncompatibleVersion
920    pub async fn remove_members_from_consumer_group(
921        &mut self,
922        group: &str,
923        members: &[MemberToRemove],
924        options: &RemoveMembersOptions,
925    ) -> Result<RemovedMembers, AdminError> {
926        let reason = leave_group_reason(options.reason.as_deref());
927        let identities = if members.is_empty() {
928            self.all_members_of(group, &reason).await?
929        } else {
930            members
931                .iter()
932                .map(|member| MemberIdentity {
933                    member_id: String::new(),
934                    group_instance_id: Some(member.group_instance_id.clone()),
935                    reason: Some(reason.clone()),
936                    ..Default::default()
937                })
938                .collect()
939        };
940        self.remove_members_from_consumer_group_with_retry(group, &identities, self.retry)
941            .await
942    }
943
944    /// The members of `group` for Kafka's `removeAll` mode, as
945    /// `KafkaAdminClient.getMembersFromGroup` names them.
946    async fn all_members_of(
947        &self,
948        group: &str,
949        reason: &str,
950    ) -> Result<Vec<MemberIdentity>, AdminError> {
951        let description = self
952            .describe_consumer_groups(&[group], DescribeGroupsOptions::builder().build())
953            .await
954            .remove(group)
955            .unwrap_or_else(|| {
956                Err(KafkaError {
957                    code: UNKNOWN_SERVER_ERROR,
958                    name: kafka_error_name(UNKNOWN_SERVER_ERROR),
959                    message: Some(format!("no description for group {group}")),
960                })
961            })
962            .map_err(|error| AdminError::Broker {
963                api: "describeConsumerGroups",
964                code: error.code,
965                name: error.name,
966                message: Some(format!(
967                    "Encounter exception when trying to get members from group: {group}{}",
968                    error
969                        .message
970                        .map(|message| format!(": {message}"))
971                        .unwrap_or_default()
972                )),
973            })?;
974        Ok(description
975            .members
976            .into_iter()
977            .map(|member| {
978                let (member_id, group_instance_id) = match member.group_instance_id {
979                    Some(instance_id) => (String::new(), Some(instance_id)),
980                    None => (member.member_id, None),
981                };
982                MemberIdentity {
983                    member_id,
984                    group_instance_id,
985                    reason: Some(reason.to_owned()),
986                    ..Default::default()
987                }
988            })
989            .collect())
990    }
991
992    async fn remove_members_from_consumer_group_with_retry(
993        &mut self,
994        group: &str,
995        members: &[MemberIdentity],
996        retry: RetryPolicy,
997    ) -> Result<RemovedMembers, AdminError> {
998        retry_group_call!(retry, |find| self.leave_group_attempt(group, members, find))
999    }
1000
1001    async fn leave_group_attempt(
1002        &mut self,
1003        group: &str,
1004        members: &[MemberIdentity],
1005        find_coordinator: bool,
1006    ) -> RetryAction<RemovedMembers> {
1007        if find_coordinator && let Err(action) = self.find_group_coordinator_attempt(group).await {
1008            return action;
1009        }
1010        let request = MembersOnlyLeaveGroup::new(LeaveGroupRequest {
1011            group_id: group.into(),
1012            members: members.to_vec(),
1013            ..Default::default()
1014        });
1015        match self.conn.send(request).await {
1016            Ok(response) => leave_group_retry_action(response),
1017            Err(error) => connection_failure_action(error),
1018        }
1019    }
1020
1021    group_offsets_method! {
1022        /// Commits explicit start offsets for a share group's partitions with
1023        /// `AlterShareGroupOffsets`, as Apache Kafka's `alterShareGroupOffsets`
1024        /// does. Routes to the group's coordinator with the same retry policy as
1025        /// [`AdminClient::alter_consumer_group_offsets`].
1026        ///
1027        /// A top-level `COORDINATOR_LOAD_IN_PROGRESS` (14) or
1028        /// `REBALANCE_IN_PROGRESS` (27) sends the request again to the same
1029        /// coordinator, and `COORDINATOR_NOT_AVAILABLE` (15) or `NOT_COORDINATOR`
1030        /// (16) finds the coordinator again. Every other top-level code, such as
1031        /// `NON_EMPTY_GROUP` (68), is the error of each requested partition, as
1032        /// Kafka's `AlterShareGroupOffsetsHandler.handleError` puts it in the
1033        /// partition results.
1034        ///
1035        /// # Errors
1036        /// Returns an error when the coordinator lookup, the connection, or the
1037        /// request fails, or at the deadline.
1038        alter_share_group_offsets(offsets: GroupOffsets) => alter_share
1039    }
1040
1041    /// Commits explicit start offsets for a streams group's partitions.
1042    /// Apache Kafka's `KafkaAdminClient.alterStreamsGroupOffsets` delegates to
1043    /// `alterConsumerGroupOffsets` verbatim, so this does too.
1044    ///
1045    /// # Errors
1046    /// See [`AdminClient::alter_consumer_group_offsets`].
1047    pub async fn alter_streams_group_offsets(
1048        &mut self,
1049        group: &str,
1050        offsets: &GroupOffsets,
1051    ) -> Result<Vec<ConsumerGroupOffsetOutcome>, AdminError> {
1052        self.alter_consumer_group_offsets(group, offsets).await
1053    }
1054
1055    /// Reads committed start offsets for a streams group's partitions.
1056    /// Apache Kafka's `KafkaAdminClient.listStreamsGroupOffsets` delegates to
1057    /// `listConsumerGroupOffsets` verbatim, so this does too.
1058    ///
1059    /// # Errors
1060    /// See [`AdminClient::list_consumer_group_offsets`].
1061    pub async fn list_streams_group_offsets(
1062        &mut self,
1063        group: &str,
1064    ) -> Result<GroupOffsets, AdminError> {
1065        self.list_consumer_group_offsets(group).await
1066    }
1067
1068    /// Deletes a share group's committed offsets for the named topics with
1069    /// `DeleteShareGroupOffsets`, as Apache Kafka's
1070    /// `deleteShareGroupOffsets` does. Routes to the group's coordinator with
1071    /// the same retry policy as [`AdminClient::alter_consumer_group_offsets`]:
1072    /// a top-level `COORDINATOR_LOAD_IN_PROGRESS` (14) or
1073    /// `REBALANCE_IN_PROGRESS` (27) retries on the same coordinator, and
1074    /// `COORDINATOR_NOT_AVAILABLE` (15) or `NOT_COORDINATOR` (16) finds the
1075    /// coordinator again.
1076    ///
1077    /// # Errors
1078    /// Returns an error when encoding, transport, or response handling fails.
1079    /// Returns [`AdminError::Broker`] when the coordinator's top-level answer
1080    /// carries a group error code that Kafka does not retry.
1081    pub async fn delete_share_group_offsets(
1082        &mut self,
1083        group: &str,
1084        topics: &[String],
1085    ) -> Result<Vec<(String, Option<KafkaError>)>, AdminError> {
1086        self.delete_share_group_offsets_with_retry(group, topics, self.retry)
1087            .await
1088    }
1089
1090    async fn delete_share_group_offsets_with_retry(
1091        &mut self,
1092        group: &str,
1093        topics: &[String],
1094        retry: RetryPolicy,
1095    ) -> Result<Vec<(String, Option<KafkaError>)>, AdminError> {
1096        retry_group_call!(retry, |find| {
1097            self.delete_share_group_offsets_attempt(group, topics, find)
1098        })
1099    }
1100
1101    async fn delete_share_group_offsets_attempt(
1102        &mut self,
1103        group: &str,
1104        topics: &[String],
1105        find_coordinator: bool,
1106    ) -> RetryAction<Vec<(String, Option<KafkaError>)>> {
1107        if find_coordinator && let Err(action) = self.find_group_coordinator_attempt(group).await {
1108            return action;
1109        }
1110        let request = DeleteShareGroupOffsetsRequest {
1111            group_id: group.into(),
1112            topics: topics
1113                .iter()
1114                .map(|topic_name| delete_share_offset_topic(topic_name))
1115                .collect(),
1116            ..Default::default()
1117        };
1118        match self.conn.send(request).await {
1119            Ok(response) => delete_share_group_offsets_retry_action(response),
1120            Err(error) => connection_failure_action(error),
1121        }
1122    }
1123}
1124
1125fn delete_share_offset_topic(topic_name: &str) -> DeleteShareGroupOffsetsRequestTopic {
1126    DeleteShareGroupOffsetsRequestTopic {
1127        topic_name: topic_name.into(),
1128        ..Default::default()
1129    }
1130}
1131
1132/// `COORDINATOR_LOAD_IN_PROGRESS`: the coordinator is loading the group.
1133const COORDINATOR_LOAD_IN_PROGRESS: i16 = 14;
1134/// `COORDINATOR_NOT_AVAILABLE`: no broker coordinates the group now.
1135const COORDINATOR_NOT_AVAILABLE: i16 = 15;
1136/// `NOT_COORDINATOR`: the broker does not coordinate the group.
1137const NOT_COORDINATOR: i16 = 16;
1138
1139/// Whether Kafka's `CoordinatorStrategy.handleError` retries this
1140/// `FindCoordinator` error code.
1141fn is_retriable_find_coordinator_error(code: i16) -> bool {
1142    matches!(
1143        code,
1144        COORDINATOR_LOAD_IN_PROGRESS | COORDINATOR_NOT_AVAILABLE
1145    )
1146}
1147
1148/// Map the result of one `OffsetFetch` attempt to a retry action, as Kafka's
1149/// `ListConsumerGroupOffsetsHandler.handleGroupError` does. 14 retries on the
1150/// same coordinator. 15 and 16 unmap the group, so the next attempt finds the
1151/// coordinator again. Every other result is final.
1152fn offset_fetch_retry_action<T>(result: Result<T, AdminError>) -> RetryAction<T> {
1153    match result {
1154        Err(AdminError::Broker {
1155            code: COORDINATOR_LOAD_IN_PROGRESS,
1156            ..
1157        }) => RetryAction::SameCoordinator(result),
1158        Err(AdminError::Broker {
1159            code: COORDINATOR_NOT_AVAILABLE | NOT_COORDINATOR,
1160            ..
1161        }) => RetryAction::FindCoordinator(result),
1162        result => RetryAction::Done(result),
1163    }
1164}
1165
1166/// `REBALANCE_IN_PROGRESS`: the group is rebalancing.
1167const REBALANCE_IN_PROGRESS: i16 = 27;
1168
1169/// Map one `OffsetCommit` response to a retry action, as Kafka's
1170/// `AlterConsumerGroupOffsetsHandler.handleResponse` does. A partition with 14
1171/// or 27 retries the request on the same coordinator. A partition with 15 or
1172/// 16 unmaps the group, so the next attempt finds the coordinator again. The
1173/// unmap wins when both occur, as the handler returns `ApiResult.unmapped`.
1174/// Every other code is a final per-partition outcome.
1175fn offset_commit_retry_action(
1176    response: OffsetCommitResponse,
1177) -> RetryAction<Vec<ConsumerGroupOffsetOutcome>> {
1178    let outcomes = group_offset_outcomes(response.topics);
1179    let codes = || {
1180        outcomes
1181            .iter()
1182            .filter_map(|outcome| outcome.error.as_ref().map(|error| error.code))
1183    };
1184    if codes().any(|code| matches!(code, COORDINATOR_NOT_AVAILABLE | NOT_COORDINATOR)) {
1185        RetryAction::FindCoordinator(Ok(outcomes))
1186    } else if codes()
1187        .any(|code| matches!(code, COORDINATOR_LOAD_IN_PROGRESS | REBALANCE_IN_PROGRESS))
1188    {
1189        RetryAction::SameCoordinator(Ok(outcomes))
1190    } else {
1191        completed_group_offsets(outcomes)
1192    }
1193}
1194
1195/// Builds an `OffsetDelete` request for the named partitions of `group`.
1196macro_rules! offset_request_topics {
1197    ($topics:expr, $topic_type:ident) => {
1198        $topics
1199            .into_iter()
1200            .map(|(name, partitions)| $topic_type {
1201                name,
1202                partitions,
1203                ..Default::default()
1204            })
1205            .collect()
1206    };
1207}
1208
1209fn offset_delete_request(group: &str, partitions: &[(String, i32)]) -> OffsetDeleteRequest {
1210    let mut topics = BTreeMap::<String, Vec<OffsetDeleteRequestPartition>>::new();
1211    for (topic, partition) in partitions {
1212        topics
1213            .entry(topic.clone())
1214            .or_default()
1215            .push(OffsetDeleteRequestPartition {
1216                partition_index: *partition,
1217                ..Default::default()
1218            });
1219    }
1220    OffsetDeleteRequest {
1221        group_id: group.into(),
1222        topics: offset_request_topics!(topics, OffsetDeleteRequestTopic),
1223        ..Default::default()
1224    }
1225}
1226
1227/// Maps one `OffsetDelete` answer to a retry action, as Kafka's
1228/// `DeleteConsumerGroupOffsetsHandler.handleResponse` does: the top-level
1229/// `error_code` names a group-level failure, and `14` retries on the same
1230/// coordinator while `15` and `16` unmap the group. A final answer becomes one
1231/// [`ConsumerGroupOffsetOutcome`] per requested partition.
1232fn offset_delete_retry_action(
1233    group: &str,
1234    response: OffsetDeleteResponse,
1235) -> RetryAction<Vec<ConsumerGroupOffsetOutcome>> {
1236    if response.error_code != 0 {
1237        let result = Err(AdminError::Broker {
1238            api: "OffsetDelete",
1239            code: response.error_code,
1240            name: kafka_error_name(response.error_code),
1241            message: Some(format!("group={group}")),
1242        });
1243        return group_coordinator_error_action(response.error_code, result);
1244    }
1245    let outcomes = group_offset_outcomes(response.topics);
1246    completed_group_offsets(outcomes)
1247}
1248
1249/// A `LeaveGroup` request that always carries its member list, needing v3.
1250///
1251/// Apache Kafka's `RemoveMembersFromConsumerGroupHandler` builds `LeaveGroup`
1252/// with `LeaveGroupRequest.Builder`, which the client negotiates at v3 or
1253/// above so that the `members` array is not silently dropped by a downgrade.
1254/// Maps one `LeaveGroup` answer to a retry action, as Kafka's
1255/// `RemoveMembersFromConsumerGroupHandler.handleResponse` does. The top-level
1256/// `error_code` is a group-level failure: `14` retries on the same
1257/// coordinator, `15` and `16` unmap the group. A final answer maps each
1258/// answered member, by member id and instance id, to its error.
1259fn leave_group_retry_action(response: LeaveGroupResponse) -> RetryAction<RemovedMembers> {
1260    if response.error_code != 0 {
1261        let result = Err(AdminError::Broker {
1262            api: "LeaveGroup",
1263            code: response.error_code,
1264            name: kafka_error_name(response.error_code),
1265            message: None,
1266        });
1267        return group_coordinator_error_action(response.error_code, result);
1268    }
1269    let members = response
1270        .members
1271        .into_iter()
1272        .map(|member| {
1273            (
1274                RemovedMember {
1275                    member_id: member.member_id,
1276                    group_instance_id: member.group_instance_id,
1277                },
1278                kafka_error_if(member.error_code, None),
1279            )
1280        })
1281        .collect();
1282    RetryAction::Done(Ok(RemovedMembers { members }))
1283}
1284
1285fn group_coordinator_error_action<T>(
1286    error_code: i16,
1287    result: Result<T, AdminError>,
1288) -> RetryAction<T> {
1289    match error_code {
1290        COORDINATOR_LOAD_IN_PROGRESS => RetryAction::SameCoordinator(result),
1291        COORDINATOR_NOT_AVAILABLE | NOT_COORDINATOR => RetryAction::FindCoordinator(result),
1292        _ => RetryAction::Done(result),
1293    }
1294}
1295
1296/// `UNKNOWN_SERVER_ERROR`: the code that Kafka's `ApiError.fromThrowable`
1297/// gives an exception with no Kafka error code.
1298const UNKNOWN_SERVER_ERROR: i16 = -1;
1299/// Kafka's `KafkaAdminClient.DEFAULT_LEAVE_GROUP_REASON`.
1300const DEFAULT_LEAVE_GROUP_REASON: &str = "member was removed by an admin";
1301/// The longest reason that Kafka's `JoinGroupRequest.maybeTruncateReason`
1302/// keeps.
1303const MAX_REASON_CHARS: usize = 255;
1304
1305/// The reason of each removed member: `reason` cut to 255 characters, or
1306/// Kafka's default when it is `None` or empty.
1307fn leave_group_reason(reason: Option<&str>) -> String {
1308    match reason {
1309        Some(reason) if !reason.is_empty() => reason.chars().take(MAX_REASON_CHARS).collect(),
1310        _ => DEFAULT_LEAVE_GROUP_REASON.to_owned(),
1311    }
1312}
1313
1314/// One member to remove, as Apache Kafka's `MemberToRemove` names it: by its
1315/// `group.instance.id`.
1316#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
1317pub struct MemberToRemove {
1318    pub group_instance_id: String,
1319}
1320
1321/// The options of [`AdminClient::remove_members_from_consumer_group`], as
1322/// Apache Kafka's `RemoveMembersFromConsumerGroupOptions` holds them.
1323#[derive(Debug, Clone, Default, PartialEq, Eq, bon::Builder)]
1324pub struct RemoveMembersOptions {
1325    /// The reason that the coordinator logs for each member. `None` or empty
1326    /// sends Kafka's default reason.
1327    #[builder(into)]
1328    pub reason: Option<String>,
1329}
1330
1331/// One member of a `LeaveGroup` answer, as the `MemberIdentity` keys of
1332/// Kafka's `RemoveMembersFromConsumerGroupHandler` result name it.
1333#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
1334pub struct RemovedMember {
1335    pub member_id: String,
1336    pub group_instance_id: Option<String>,
1337}
1338
1339/// The result of [`AdminClient::remove_members_from_consumer_group`], as
1340/// Apache Kafka's `RemoveMembersFromConsumerGroupResult` holds it.
1341#[derive(Debug, Clone, Default, PartialEq, Eq)]
1342pub struct RemovedMembers {
1343    /// Each member of the `LeaveGroup` answer and its error.
1344    pub members: BTreeMap<RemovedMember, Option<KafkaError>>,
1345}
1346
1347impl RemovedMembers {
1348    /// The result of one requested member, as Kafka's
1349    /// `RemoveMembersFromConsumerGroupResult.memberResult` gives it.
1350    ///
1351    /// # Errors
1352    /// Returns the member's error, or `UNKNOWN_SERVER_ERROR` when the answer
1353    /// does not name the member, as Kafka's `getSubLevelError` gives an
1354    /// `IllegalArgumentException`.
1355    pub fn member_result(&self, member: &MemberToRemove) -> Result<(), KafkaError> {
1356        let key = RemovedMember {
1357            member_id: String::new(),
1358            group_instance_id: Some(member.group_instance_id.clone()),
1359        };
1360        match self.members.get(&key) {
1361            Some(None) => Ok(()),
1362            Some(Some(error)) => Err(error.clone()),
1363            None => Err(KafkaError {
1364                code: UNKNOWN_SERVER_ERROR,
1365                name: kafka_error_name(UNKNOWN_SERVER_ERROR),
1366                message: Some(format!(
1367                    "Member \"{}\" was not included in the removal response",
1368                    member.group_instance_id
1369                )),
1370            }),
1371        }
1372    }
1373}
1374
1375/// Builds an `AlterShareGroupOffsets` request for `group`'s partitions.
1376fn alter_share_group_offsets_request(
1377    group: &str,
1378    offsets: &GroupOffsets,
1379) -> AlterShareGroupOffsetsRequest {
1380    let mut topics = BTreeMap::<String, Vec<AlterShareGroupOffsetsRequestPartition>>::new();
1381    for ((topic, partition), start_offset) in offsets {
1382        topics
1383            .entry(topic.clone())
1384            .or_default()
1385            .push(AlterShareGroupOffsetsRequestPartition {
1386                partition_index: *partition,
1387                start_offset: *start_offset,
1388                ..Default::default()
1389            });
1390    }
1391    AlterShareGroupOffsetsRequest {
1392        group_id: group.into(),
1393        topics: topics
1394            .into_iter()
1395            .map(
1396                |(topic_name, partitions)| AlterShareGroupOffsetsRequestTopic {
1397                    topic_name,
1398                    partitions,
1399                    ..Default::default()
1400                },
1401            )
1402            .collect(),
1403        ..Default::default()
1404    }
1405}
1406
1407fn share_group_coordinator_error_action<T>(
1408    code: i16,
1409    error: AdminError,
1410    otherwise: impl FnOnce(AdminError) -> Result<T, AdminError>,
1411) -> RetryAction<T> {
1412    match code {
1413        COORDINATOR_LOAD_IN_PROGRESS | REBALANCE_IN_PROGRESS => {
1414            RetryAction::SameCoordinator(Err(error))
1415        }
1416        COORDINATOR_NOT_AVAILABLE | NOT_COORDINATOR => RetryAction::FindCoordinator(Err(error)),
1417        _ => RetryAction::Done(otherwise(error)),
1418    }
1419}
1420
1421fn completed_group_offsets<T>(outcomes: T) -> RetryAction<T> {
1422    RetryAction::Done(Ok(outcomes))
1423}
1424
1425fn map_group_topic_rows<T, Rows, P, O>(
1426    topics: impl IntoIterator<Item = T>,
1427    mut rows: impl FnMut(T) -> (String, Rows),
1428    mut outcome: impl FnMut(String, P) -> O,
1429) -> Vec<O>
1430where
1431    Rows: IntoIterator<Item = P>,
1432{
1433    let mut outcomes = Vec::new();
1434    for topic in topics {
1435        let (name, partitions) = rows(topic);
1436        for partition in partitions {
1437            outcomes.push(outcome(name.clone(), partition));
1438        }
1439    }
1440    outcomes
1441}
1442
1443/// Maps one `AlterShareGroupOffsets` answer to a retry action, as Kafka's
1444/// `AlterShareGroupOffsetsHandler.handleResponse` does. A top-level
1445/// `COORDINATOR_LOAD_IN_PROGRESS` or `REBALANCE_IN_PROGRESS` retries on the
1446/// same coordinator, and `COORDINATOR_NOT_AVAILABLE` or `NOT_COORDINATOR`
1447/// finds the coordinator again. Every other top-level code becomes the error
1448/// of each requested partition. Otherwise the answer gives one
1449/// [`ConsumerGroupOffsetOutcome`] for each answered partition.
1450fn alter_share_group_offsets_retry_action(
1451    offsets: &GroupOffsets,
1452    response: AlterShareGroupOffsetsResponse,
1453) -> RetryAction<Vec<ConsumerGroupOffsetOutcome>> {
1454    if response.error_code != 0 {
1455        let error = AdminError::Broker {
1456            api: "AlterShareGroupOffsets",
1457            code: response.error_code,
1458            name: kafka_error_name(response.error_code),
1459            message: response.error_message.clone(),
1460        };
1461        return share_group_coordinator_error_action(response.error_code, error, |_| {
1462            Ok(offsets
1463                .keys()
1464                .map(|(topic, partition)| ConsumerGroupOffsetOutcome {
1465                    topic: topic.clone(),
1466                    partition: *partition,
1467                    error: kafka_error_if(response.error_code, response.error_message.clone()),
1468                })
1469                .collect())
1470        });
1471    }
1472    let outcomes = map_group_topic_rows(
1473        response.responses,
1474        |topic| (topic.topic_name, topic.partitions),
1475        |topic, partition| ConsumerGroupOffsetOutcome {
1476            topic,
1477            partition: partition.partition_index,
1478            error: kafka_error_if(partition.error_code, partition.error_message),
1479        },
1480    );
1481    completed_group_offsets(outcomes)
1482}
1483
1484/// Maps one `DeleteShareGroupOffsets` answer to a retry action, as Kafka's
1485/// `DeleteShareGroupOffsetsHandler.handleResponse` does. A top-level
1486/// `COORDINATOR_LOAD_IN_PROGRESS` or `REBALANCE_IN_PROGRESS` retries on the
1487/// same coordinator, `COORDINATOR_NOT_AVAILABLE` or `NOT_COORDINATOR` finds
1488/// the coordinator again, and every other top-level code fails the call. A
1489/// final answer gives one `(topic, error)` pair for each answered topic.
1490fn delete_share_group_offsets_retry_action(
1491    response: DeleteShareGroupOffsetsResponse,
1492) -> RetryAction<Vec<(String, Option<KafkaError>)>> {
1493    if response.error_code != 0 {
1494        let error = AdminError::Broker {
1495            api: "DeleteShareGroupOffsets",
1496            code: response.error_code,
1497            name: kafka_error_name(response.error_code),
1498            message: response.error_message,
1499        };
1500        return share_group_coordinator_error_action(response.error_code, error, Err);
1501    }
1502    let outcomes = map_group_topic_rows(
1503        response.responses,
1504        |topic| {
1505            (
1506                topic.topic_name,
1507                std::iter::once((topic.error_code, topic.error_message)),
1508            )
1509        },
1510        |topic, (code, message)| (topic, kafka_error_if(code, message)),
1511    );
1512    completed_group_offsets(outcomes)
1513}
1514
1515/// The lowest `ListGroups` version that carries the filters of `options`, as
1516/// Kafka's `ListGroupsRequest.Builder.build` requires it.
1517///
1518/// `states_filter` needs v4. `types_filter` needs v5, except when it holds
1519/// `Classic` and `Consumer` at most, with `Classic`. A broker below v5 lists
1520/// only classic groups, so the filter can be omitted. The request encoder
1521/// omits `types_filter` below v5.
1522fn list_groups_min_version(options: &ListGroupsOptions) -> i16 {
1523    let types_need_v5 = !options.types.is_empty()
1524        && (!options.types.contains(&GroupType::Classic)
1525            || options
1526                .types
1527                .iter()
1528                .any(|group_type| !matches!(group_type, GroupType::Classic | GroupType::Consumer)));
1529    if types_need_v5 {
1530        5
1531    } else if options.group_states.is_empty() {
1532        list_groups_request::MIN_VERSION
1533    } else {
1534        4
1535    }
1536}
1537
1538/// Sends `ListGroups` to one broker until it answers, as one Kafka
1539/// `listGroups` node call does.
1540///
1541/// `COORDINATOR_LOAD_IN_PROGRESS`, `COORDINATOR_NOT_AVAILABLE`, a failed
1542/// connection and a lost connection make the call try again. A rejected
1543/// authentication does not, because Kafka's `Call.fail` does not retry an
1544/// `AuthenticationException`.
1545///
1546/// `retry.timeout` after `start` is the deadline of the call. Each attempt
1547/// (the TCP connection, the TLS and SASL handshakes, and the `ListGroups`
1548/// request) gets the time that remains. When the deadline passes, the call
1549/// fails with `REQUEST_TIMED_OUT` (7), as Kafka's `Call.fail` gives a
1550/// `TimeoutException` for a call past its deadline.
1551async fn list_groups_on_broker(
1552    broker: &MetadataResponseBroker,
1553    options: ConnectionOptions,
1554    request: ListGroupsRequest,
1555    min_version: i16,
1556    start: tokio::time::Instant,
1557    retry: RetryPolicy,
1558) -> Result<Vec<ListedGroup>, KafkaError> {
1559    let host_port = format_host_port(&broker.host, broker.port);
1560    let deadline = start + retry.timeout;
1561    let mut backoff = retry.initial_backoff;
1562    let mut connection = None;
1563    let mut attempts = 0_u32;
1564    let mut last = None;
1565    while tokio::time::Instant::now() < deadline {
1566        attempts += 1;
1567        let attempt = list_groups_attempt(
1568            &mut connection,
1569            &host_port,
1570            &options,
1571            request.clone(),
1572            min_version,
1573        );
1574        match tokio::time::timeout_at(deadline, attempt).await {
1575            Ok(RetryAction::Done(result)) => {
1576                return result.map_err(|error| list_groups_kafka_error(&error));
1577            }
1578            Ok(RetryAction::SameCoordinator(result) | RetryAction::FindCoordinator(result)) => {
1579                match result {
1580                    Ok(groups) => return Ok(groups),
1581                    Err(error) => last = Some(error),
1582                }
1583            }
1584            Err(_) => break,
1585        }
1586        // Kafka's `Call.fail` stops after `retries` retries with a
1587        // `TimeoutException`.
1588        if attempts > retry.max_retries {
1589            break;
1590        }
1591        tracing::debug!(
1592            node_id = broker.node_id,
1593            "ListGroups got a retriable error; retrying"
1594        );
1595        tokio::time::sleep_until(deadline.min(tokio::time::Instant::now() + backoff)).await;
1596        backoff = backoff.saturating_mul(2).min(retry.max_backoff);
1597    }
1598    Err(list_groups_timeout(attempts, last.as_ref()))
1599}
1600
1601/// The `REQUEST_TIMED_OUT` error of a `ListGroups` node call past its
1602/// deadline. The message names the attempts and the last retriable error, as
1603/// the `TimeoutException` of Kafka's `Call.handleTimeoutFailure` holds the
1604/// attempt count and the cause.
1605fn list_groups_timeout(attempts: u32, last: Option<&AdminError>) -> KafkaError {
1606    const REQUEST_TIMED_OUT: i16 = 7;
1607    let cause = last
1608        .map(|error| format!("; last error: {error}"))
1609        .unwrap_or_default();
1610    KafkaError {
1611        code: REQUEST_TIMED_OUT,
1612        name: kafka_error_name(REQUEST_TIMED_OUT),
1613        message: Some(format!(
1614            "ListGroups timed out after {attempts} attempt(s){cause}"
1615        )),
1616    }
1617}
1618
1619/// One `ListGroups` attempt on one broker. The attempt connects first when
1620/// `connection` is empty, and empties it after a connection error.
1621async fn list_groups_attempt(
1622    connection: &mut Option<Connection>,
1623    host_port: &str,
1624    options: &ConnectionOptions,
1625    request: ListGroupsRequest,
1626    min_version: i16,
1627) -> RetryAction<Vec<ListedGroup>> {
1628    let current = match connection {
1629        Some(current) => current,
1630        None => match AdminClient::connect_one(host_port, options.clone()).await {
1631            Ok(new) => connection.insert(new),
1632            Err(error) if error.is_authentication_failure() => {
1633                return RetryAction::Done(Err(error));
1634            }
1635            Err(error) => return RetryAction::SameCoordinator(Err(error)),
1636        },
1637    };
1638    match send_connection_at_least(current, request, min_version).await {
1639        Ok(response) => match response.error_code {
1640            0 => RetryAction::Done(Ok(response.groups)),
1641            code => {
1642                let error = Err(AdminError::Broker {
1643                    api: "ListGroups",
1644                    code,
1645                    name: kafka_error_name(code),
1646                    message: None,
1647                });
1648                if matches!(
1649                    code,
1650                    COORDINATOR_LOAD_IN_PROGRESS | COORDINATOR_NOT_AVAILABLE
1651                ) {
1652                    RetryAction::SameCoordinator(error)
1653                } else {
1654                    RetryAction::Done(error)
1655                }
1656            }
1657        },
1658        Err(error) if AdminClient::is_retriable_transport_error(&error) => {
1659            *connection = None;
1660            RetryAction::SameCoordinator(Err(error.into()))
1661        }
1662        Err(error) => RetryAction::Done(Err(error.into())),
1663    }
1664}
1665
1666/// The Kafka error code of a `ListGroups` failure on one broker, as Kafka's
1667/// `Errors.forException` maps the exception class.
1668///
1669/// A rejected authentication maps as its Kafka exception class does:
1670/// `UnsupportedSaslMechanismException` to 33, `IllegalSaslStateException` to
1671/// 34, `SaslAuthenticationException` to 58, and `SslAuthenticationException`
1672/// to 40, the code of its superclass `InvalidConfigurationException`.
1673pub(crate) fn list_groups_kafka_error(error: &AdminError) -> KafkaError {
1674    const UNKNOWN_SERVER_ERROR: i16 = -1;
1675    const REQUEST_TIMED_OUT: i16 = 7;
1676    const NETWORK_EXCEPTION: i16 = 13;
1677    const UNSUPPORTED_SASL_MECHANISM: i16 = 33;
1678    const ILLEGAL_SASL_STATE: i16 = 34;
1679    const UNSUPPORTED_VERSION: i16 = 35;
1680    const INVALID_CONFIG: i16 = 40;
1681    const SASL_AUTHENTICATION_FAILED: i16 = 58;
1682    let (code, message) = match error {
1683        AdminError::Broker { code, message, .. } => (*code, message.clone()),
1684        AdminError::Transport(ClientError::IncompatibleVersion { .. }) => {
1685            (UNSUPPORTED_VERSION, Some(error.to_string()))
1686        }
1687        AdminError::Transport(ClientError::Timeout(_)) => {
1688            (REQUEST_TIMED_OUT, Some(error.to_string()))
1689        }
1690        AdminError::Transport(
1691            ClientError::Connect { .. } | ClientError::Disconnected | ClientError::Io(_),
1692        ) => (NETWORK_EXCEPTION, Some(error.to_string())),
1693        AdminError::Transport(ClientError::Authentication { source, .. }) => {
1694            let code = match source {
1695                AuthenticationError::Tls(_) => INVALID_CONFIG,
1696                AuthenticationError::Sasl(SaslAuthenticationError::UnsupportedMechanism(_)) => {
1697                    UNSUPPORTED_SASL_MECHANISM
1698                }
1699                AuthenticationError::Sasl(SaslAuthenticationError::IllegalState(_)) => {
1700                    ILLEGAL_SASL_STATE
1701                }
1702                AuthenticationError::Sasl(SaslAuthenticationError::Failed(_)) => {
1703                    SASL_AUTHENTICATION_FAILED
1704                }
1705            };
1706            (code, Some(error.to_string()))
1707        }
1708        _ => (UNKNOWN_SERVER_ERROR, Some(error.to_string())),
1709    };
1710    KafkaError {
1711        code,
1712        name: kafka_error_name(code),
1713        message,
1714    }
1715}
1716
1717/// Builds the admin `OffsetFetch` request for all topics of `group`.
1718///
1719/// The request fills the v2 to v7 fields (`group_id`, `topics`) and the v8+
1720/// `groups` array, so it is valid at each negotiated version. Kafka's
1721/// `OffsetFetchRequest.Builder.maybeDowngrade` moves the group into the v2 to
1722/// v7 fields in the same way.
1723fn offset_fetch_group(group: &str) -> OffsetFetchRequestGroup {
1724    OffsetFetchRequestGroup {
1725        group_id: group.into(),
1726        member_id: None,
1727        member_epoch: -1,
1728        topics: None,
1729        ..Default::default()
1730    }
1731}
1732
1733fn offset_fetch_request(group: &str) -> AllTopicsOffsetFetch {
1734    AllTopicsOffsetFetch::new(OffsetFetchRequest {
1735        group_id: group.into(),
1736        topics: None,
1737        groups: vec![offset_fetch_group(group)],
1738        require_stable: false,
1739        ..Default::default()
1740    })
1741}
1742
1743/// Reads the committed offsets of `group` from an `OffsetFetch` response.
1744///
1745/// v8 and v9 put the group in `groups`. v2 to v7 put the group error code and
1746/// the topics at the top level, which Kafka's `OffsetFetchResponse.group`
1747/// reads in the same way. As in Kafka's
1748/// `ListConsumerGroupOffsetsHandler.handleResponse`, a partition with an error
1749/// code gives no row.
1750fn committed_offsets(
1751    group: &str,
1752    response: OffsetFetchResponse,
1753) -> Result<GroupOffsets, AdminError> {
1754    let (error_code, rows): (i16, Vec<(String, i32, i64, i16)>) = if response.groups.is_empty() {
1755        (
1756            response.error_code,
1757            response
1758                .topics
1759                .into_iter()
1760                .flat_map(offset_fetch_topic_rows)
1761                .collect(),
1762        )
1763    } else {
1764        let mut error_code = 0;
1765        let mut rows = Vec::new();
1766        for entry in response
1767            .groups
1768            .into_iter()
1769            .filter(|entry| entry.group_id == group)
1770        {
1771            error_code = entry.error_code;
1772            for topic in entry.topics {
1773                rows.extend(offset_fetch_topic_rows(topic));
1774            }
1775        }
1776        (error_code, rows)
1777    };
1778    if error_code != 0 {
1779        return Err(AdminError::Broker {
1780            api: "OffsetFetch",
1781            code: error_code,
1782            name: kafka_error_name(error_code),
1783            message: Some(format!("group={group}")),
1784        });
1785    }
1786    Ok(rows
1787        .into_iter()
1788        .filter_map(|(topic, partition, offset, partition_error)| {
1789            if partition_error != 0 {
1790                tracing::warn!(
1791                    topic,
1792                    partition,
1793                    error_code = partition_error,
1794                    "skipping the committed offset of a partition with an error"
1795                );
1796                None
1797            } else if offset < 0 {
1798                None
1799            } else {
1800                Some(((topic, partition), offset))
1801            }
1802        })
1803        .collect())
1804}
1805
1806trait IntoOffsetFetchRow {
1807    fn into_offset_fetch_row(self, topic: &str) -> (String, i32, i64, i16);
1808}
1809
1810macro_rules! impl_offset_fetch_row {
1811    ($partition:ty) => {
1812        impl IntoOffsetFetchRow for $partition {
1813            fn into_offset_fetch_row(self, topic: &str) -> (String, i32, i64, i16) {
1814                (
1815                    topic.to_owned(),
1816                    self.partition_index,
1817                    self.committed_offset,
1818                    self.error_code,
1819                )
1820            }
1821        }
1822    };
1823}
1824
1825impl_offset_fetch_row!(OffsetFetchResponsePartition);
1826impl_offset_fetch_row!(OffsetFetchResponsePartitions);
1827
1828trait IntoOffsetFetchTopics {
1829    type Partition: IntoOffsetFetchRow;
1830
1831    fn into_offset_fetch_parts(self) -> (String, Vec<Self::Partition>);
1832}
1833
1834macro_rules! impl_offset_fetch_topics {
1835    ($topics:ty, $partition:ty) => {
1836        impl IntoOffsetFetchTopics for $topics {
1837            type Partition = $partition;
1838
1839            fn into_offset_fetch_parts(self) -> (String, Vec<Self::Partition>) {
1840                (self.name, self.partitions)
1841            }
1842        }
1843    };
1844}
1845
1846impl_offset_fetch_topics!(OffsetFetchResponseTopic, OffsetFetchResponsePartition);
1847impl_offset_fetch_topics!(OffsetFetchResponseTopics, OffsetFetchResponsePartitions);
1848
1849fn offset_fetch_topic_rows<T: IntoOffsetFetchTopics>(topic: T) -> Vec<(String, i32, i64, i16)> {
1850    let (name, partitions) = topic.into_offset_fetch_parts();
1851    partitions
1852        .into_iter()
1853        .map(|partition| partition.into_offset_fetch_row(&name))
1854        .collect()
1855}
1856
1857fn offset_commit_request(group: &str, offsets: &GroupOffsets) -> TopicNameOffsetCommit {
1858    let mut topics = BTreeMap::<String, Vec<OffsetCommitRequestPartition>>::new();
1859    for ((topic, partition), offset) in offsets {
1860        topics
1861            .entry(topic.clone())
1862            .or_default()
1863            .push(OffsetCommitRequestPartition {
1864                partition_index: *partition,
1865                committed_offset: *offset,
1866                committed_leader_epoch: -1,
1867                // Kafka's `OffsetAndMetadata` metadata defaults to "", which
1868                // is also the schema default this row takes.
1869                ..Default::default()
1870            });
1871    }
1872    TopicNameOffsetCommit::new(OffsetCommitRequest {
1873        group_id: group.into(),
1874        generation_id_or_member_epoch: -1,
1875        member_id: String::new(),
1876        topics: offset_request_topics!(topics, OffsetCommitRequestTopic),
1877        ..Default::default()
1878    })
1879}
1880
1881#[cfg(test)]
1882mod tests {
1883    use std::{
1884        sync::{
1885            Arc, Mutex,
1886            atomic::{AtomicUsize, Ordering},
1887        },
1888        time::Duration,
1889    };
1890
1891    use assert2::assert;
1892    use bytes::{Buf, BytesMut};
1893    use krabka_client_core::{
1894        AuthenticationError, ClientError, MockBroker, MockReply, MockSaslAnswer,
1895        SaslAuthenticationError,
1896        mock_test_support::offset_commit_response,
1897        security::{ClientSecurity, SaslCredentials},
1898    };
1899    use krabka_protocol::{
1900        Decode, Encode, ProtocolRequest,
1901        owned::{
1902            alter_share_group_offsets_response::{
1903                AlterShareGroupOffsetsResponsePartition, AlterShareGroupOffsetsResponseTopic,
1904            },
1905            api_versions_request,
1906            api_versions_response::{ApiVersion, ApiVersionsResponse},
1907            delete_share_group_offsets_response::DeleteShareGroupOffsetsResponseTopic,
1908            find_coordinator_request,
1909            find_coordinator_response::FindCoordinatorResponse,
1910            leave_group_response::MemberResponse,
1911            list_groups_response::ListGroupsResponse,
1912            metadata_request,
1913            metadata_response::MetadataResponse,
1914            offset_commit_request,
1915            offset_commit_response::{
1916                OffsetCommitResponse, OffsetCommitResponsePartition, OffsetCommitResponseTopic,
1917            },
1918            offset_delete_response::{OffsetDeleteResponsePartition, OffsetDeleteResponseTopic},
1919            offset_fetch_request,
1920            offset_fetch_response::{
1921                OffsetFetchResponseGroup, OffsetFetchResponsePartition,
1922                OffsetFetchResponsePartitions, OffsetFetchResponseTopic, OffsetFetchResponseTopics,
1923            },
1924            sasl_handshake_request,
1925        },
1926    };
1927
1928    use super::*;
1929
1930    fn test_group_offsets() -> GroupOffsets {
1931        maplit::btreemap! { ("orders".into(), 2) => 41 }
1932    }
1933
1934    #[test]
1935    fn offset_reset_builds_admin_commit() {
1936        let offsets = test_group_offsets();
1937        let request = offset_commit_request("worker", &offsets);
1938        let expected = TopicNameOffsetCommit::new(commit_request_for_test("worker", -1, -1));
1939        assert!(request == expected);
1940    }
1941
1942    fn commit_request_for_test(
1943        group_id: &str,
1944        generation_id_or_member_epoch: i32,
1945        committed_leader_epoch: i32,
1946    ) -> OffsetCommitRequest {
1947        OffsetCommitRequest {
1948            group_id: group_id.into(),
1949            generation_id_or_member_epoch,
1950            topics: vec![OffsetCommitRequestTopic {
1951                name: "orders".into(),
1952                partitions: vec![OffsetCommitRequestPartition {
1953                    partition_index: 2,
1954                    committed_offset: 41,
1955                    committed_leader_epoch,
1956                    ..Default::default()
1957                }],
1958                ..Default::default()
1959            }],
1960            ..Default::default()
1961        }
1962    }
1963
1964    fn encode(response: &impl Encode, version: i16, flexible: bool) -> Vec<u8> {
1965        let mut bytes = BytesMut::new();
1966        if flexible {
1967            bytes.extend_from_slice(&[0]);
1968        }
1969        response.encode(&mut bytes, version).unwrap();
1970        bytes.to_vec()
1971    }
1972
1973    fn encode_flexible(response: &impl Encode, version: i16) -> Vec<u8> {
1974        encode(response, version, true)
1975    }
1976
1977    fn broker_port(addr: std::net::SocketAddr) -> i32 {
1978        i32::from(addr.port())
1979    }
1980
1981    struct MockBrokerAddress {
1982        node_id: i32,
1983        host: String,
1984        port: i32,
1985    }
1986
1987    fn mock_broker_address(addr: std::net::SocketAddr, node_id: i32) -> MockBrokerAddress {
1988        MockBrokerAddress {
1989            node_id,
1990            host: addr.ip().to_string(),
1991            port: broker_port(addr),
1992        }
1993    }
1994
1995    macro_rules! broker_fields {
1996        ($type:ident, $broker:expr) => {
1997            $type {
1998                node_id: $broker.node_id,
1999                host: $broker.host,
2000                port: $broker.port,
2001                ..Default::default()
2002            }
2003        };
2004    }
2005
2006    fn mock_metadata_broker(addr: std::net::SocketAddr, node_id: i32) -> MetadataResponseBroker {
2007        let broker = mock_broker_address(addr, node_id);
2008        broker_fields!(MetadataResponseBroker, broker)
2009    }
2010
2011    async fn connect_admin(addr: std::net::SocketAddr, message: &'static str) -> AdminClient {
2012        AdminClient::connect(&[addr.to_string()])
2013            .await
2014            .expect(message)
2015    }
2016
2017    fn mock_coordinator_response(
2018        addr: std::net::SocketAddr,
2019        error_code: i16,
2020    ) -> FindCoordinatorResponse {
2021        let broker = mock_broker_address(addr, 2);
2022        let mut response = broker_fields!(FindCoordinatorResponse, broker);
2023        response.error_code = error_code;
2024        response
2025    }
2026
2027    fn incompatible_version(error: AdminError, case: &str) -> (i16, i16, i16, i16, i16) {
2028        match error {
2029            AdminError::Transport(ClientError::IncompatibleVersion {
2030                api_key,
2031                broker_min,
2032                broker_max,
2033                client_min,
2034                client_max,
2035            }) => (api_key, broker_min, broker_max, client_min, client_max),
2036            other => panic!("case {case}: unexpected error {other:?}"),
2037        }
2038    }
2039
2040    async fn with_incompatible_version<T>(
2041        case: &'static str,
2042        result: impl std::future::Future<Output = Result<T, AdminError>>,
2043    ) -> Result<T, (i16, i16, i16, i16, i16)> {
2044        result
2045            .await
2046            .map_err(|error| incompatible_version(error, case))
2047    }
2048
2049    /// The `OffsetCommit` and `OffsetFetch` version ranges that a mock broker
2050    /// advertises.
2051    #[derive(Clone, Copy)]
2052    struct GroupRanges {
2053        offset_commit: (i16, i16),
2054        offset_fetch: (i16, i16),
2055    }
2056
2057    impl GroupRanges {
2058        fn offset_commit(offset_commit: (i16, i16)) -> Self {
2059            Self {
2060                offset_commit,
2061                offset_fetch: (2, 10),
2062            }
2063        }
2064
2065        fn offset_fetch(offset_fetch: (i16, i16)) -> Self {
2066            Self {
2067                offset_commit: (2, 9),
2068                offset_fetch,
2069            }
2070        }
2071    }
2072
2073    fn encode_api_versions(api_keys: Vec<ApiVersion>) -> Vec<u8> {
2074        encode(
2075            &ApiVersionsResponse {
2076                api_keys,
2077                ..Default::default()
2078            },
2079            0,
2080            false,
2081        )
2082    }
2083
2084    macro_rules! api_version_entry {
2085        ($key:expr, $min:expr, $max:expr) => {
2086            ApiVersion {
2087                api_key: $key,
2088                min_version: $min,
2089                max_version: $max,
2090                ..Default::default()
2091            }
2092        };
2093    }
2094
2095    /// An `ApiVersions` response that advertises `ranges`.
2096    fn api_versions(ranges: GroupRanges) -> Vec<u8> {
2097        encode_api_versions(vec![
2098            api_version_entry!(api_versions_request::API_KEY, 0, 0),
2099            api_version_entry!(find_coordinator_request::API_KEY, 0, 0),
2100            api_version_entry!(
2101                offset_commit_request::API_KEY,
2102                ranges.offset_commit.0,
2103                ranges.offset_commit.1
2104            ),
2105            api_version_entry!(
2106                offset_fetch_request::API_KEY,
2107                ranges.offset_fetch.0,
2108                ranges.offset_fetch.1
2109            ),
2110            api_version_entry!(metadata_request::API_KEY, 13, 13),
2111        ])
2112    }
2113
2114    /// Decodes a request body of type `R` behind its request header.
2115    fn decode_request<R: ProtocolRequest + for<'de> Decode<'de>>(
2116        mut body: &[u8],
2117        version: i16,
2118    ) -> R {
2119        let client_id_len = body.get_i16();
2120        body.advance(usize::try_from(client_id_len).expect("client id length"));
2121        if version >= R::FLEXIBLE_MIN {
2122            body.advance(1);
2123        }
2124        R::decode(&mut body, version).expect("request decodes")
2125    }
2126
2127    /// A bootstrap broker that sends every group RPC to `coordinator`.
2128    async fn bootstrap_for(
2129        coordinator: &MockBroker,
2130        ranges: GroupRanges,
2131        group_rpcs: Arc<AtomicUsize>,
2132    ) -> MockBroker {
2133        let coordinator_addr = coordinator.addr;
2134        MockBroker::start(move |api_key, version, _, _| match api_key {
2135            api_versions_request::API_KEY => Some(api_versions(ranges)),
2136            find_coordinator_request::API_KEY => Some(encode(
2137                &mock_coordinator_response(coordinator_addr, 0),
2138                version,
2139                false,
2140            )),
2141            offset_commit_request::API_KEY => {
2142                group_rpcs.fetch_add(1, Ordering::SeqCst);
2143                Some(encode(&OffsetCommitResponse::default(), version, true))
2144            }
2145            offset_fetch_request::API_KEY => {
2146                group_rpcs.fetch_add(1, Ordering::SeqCst);
2147                Some(encode(&OffsetFetchResponse::default(), version, true))
2148            }
2149            metadata_request::API_KEY => Some(empty_metadata_reply(version)),
2150            _ => None,
2151        })
2152        .await
2153    }
2154
2155    struct GroupAdminConnection {
2156        bootstrap: MockBroker,
2157        admin: AdminClient,
2158    }
2159
2160    async fn connect_group_admin(
2161        coordinator: &MockBroker,
2162        ranges: GroupRanges,
2163    ) -> GroupAdminConnection {
2164        let bootstrap = bootstrap_for(coordinator, ranges, Arc::default()).await;
2165        let admin = connect_admin(bootstrap.addr, "admin connects").await;
2166        GroupAdminConnection { bootstrap, admin }
2167    }
2168
2169    fn stop_group_brokers(bootstrap: MockBroker, coordinator: MockBroker) {
2170        bootstrap.stop();
2171        coordinator.stop();
2172    }
2173
2174    fn record_group_request<Request>(
2175        requests: &Arc<Mutex<Vec<(i16, Request)>>>,
2176        version: i16,
2177        request: Request,
2178    ) {
2179        requests
2180            .lock()
2181            .expect("requests lock")
2182            .push((version, request));
2183    }
2184
2185    struct GroupVersionAssertion<Request, Output> {
2186        name: &'static str,
2187        requests: Vec<(i16, Request)>,
2188        result: Output,
2189        expected_requests: Vec<(i16, Request)>,
2190        expected_result: Output,
2191    }
2192
2193    fn assert_group_version_case<Request, Output>(case: GroupVersionAssertion<Request, Output>)
2194    where
2195        Request: std::fmt::Debug + PartialEq,
2196        Output: std::fmt::Debug + PartialEq,
2197    {
2198        assert!(
2199            (case.requests, case.result) == (case.expected_requests, case.expected_result),
2200            "case {}",
2201            case.name
2202        );
2203    }
2204
2205    struct ExpectedGroupVersionCase<Request, Output> {
2206        name: &'static str,
2207        ranges: GroupRanges,
2208        requests: Vec<(i16, Request)>,
2209        result: Output,
2210    }
2211
2212    fn expected_group_version_case<Request, Output>(
2213        name: &'static str,
2214        ranges: GroupRanges,
2215        requests: Vec<(i16, Request)>,
2216        result: Output,
2217    ) -> ExpectedGroupVersionCase<Request, Output> {
2218        ExpectedGroupVersionCase {
2219            name,
2220            ranges,
2221            requests,
2222            result,
2223        }
2224    }
2225
2226    fn incompatible_group_version<T>(
2227        api_key: i16,
2228        broker_min: i16,
2229        broker_max: i16,
2230    ) -> Result<T, (i16, i16, i16, i16, i16)> {
2231        Err((api_key, broker_min, broker_max, 2, 9))
2232    }
2233
2234    async fn run_group_version_case<Request, Output, Respond, Action, Fut>(
2235        case: ExpectedGroupVersionCase<Request, Output>,
2236        api_key: i16,
2237        respond: Respond,
2238        action: Action,
2239    ) where
2240        Request: ProtocolRequest
2241            + for<'de> Decode<'de>
2242            + Clone
2243            + PartialEq
2244            + std::fmt::Debug
2245            + Send
2246            + 'static,
2247        Output: PartialEq + std::fmt::Debug,
2248        Respond: Fn(&Request, i16) -> Vec<u8> + Copy + Send + Sync + 'static,
2249        Action: FnOnce(&'static str, AdminClient) -> Fut,
2250        Fut: std::future::Future<Output = Output>,
2251    {
2252        let ranges = case.ranges;
2253        let requests = Arc::new(Mutex::new(Vec::new()));
2254        let recorded = Arc::clone(&requests);
2255        let metadata_requests = Arc::new(AtomicUsize::new(0));
2256        let metadata_count = Arc::clone(&metadata_requests);
2257        let coordinator =
2258            MockBroker::start(move |request_api, version, _, body| match request_api {
2259                api_versions_request::API_KEY => Some(api_versions(ranges)),
2260                request_api if request_api == api_key => {
2261                    let request = decode_request(body, version);
2262                    let response = respond(&request, version);
2263                    record_group_request(&recorded, version, request);
2264                    Some(response)
2265                }
2266                metadata_request::API_KEY => {
2267                    counted_reply(&metadata_count, Some(empty_metadata_reply(version)))
2268                }
2269                _ => None,
2270            })
2271            .await;
2272        let GroupAdminConnection { bootstrap, admin } =
2273            connect_group_admin(&coordinator, ranges).await;
2274        let result = action(case.name, admin).await;
2275        stop_group_brokers(bootstrap, coordinator);
2276        let requests = requests.lock().expect("requests lock").clone();
2277        assert!(
2278            metadata_requests.load(Ordering::SeqCst) == 0,
2279            "case {}",
2280            case.name
2281        );
2282        assert_group_version_case(GroupVersionAssertion {
2283            name: case.name,
2284            requests,
2285            result,
2286            expected_requests: case.requests,
2287            expected_result: case.result,
2288        });
2289    }
2290
2291    /// The negotiated `OffsetCommit` version and the result of
2292    /// `alter_consumer_group_offsets`, with a version error as its ranges.
2293    type CommitResult = Result<Vec<ConsumerGroupOffsetOutcome>, (i16, i16, i16, i16, i16)>;
2294
2295    /// Apache Kafka's `AlterConsumerGroupOffsetsHandler` builds `OffsetCommit`
2296    /// with `OffsetCommitRequest.Builder.forTopicNames`, which caps the version
2297    /// at 9. The request names each topic at every negotiated version.
2298    fn commit_group_version_cases()
2299    -> Vec<ExpectedGroupVersionCase<OffsetCommitRequest, CommitResult>> {
2300        let committed = Ok(successful_group_offset_outcome());
2301        let commit_case = |name, version_range, requests, result| {
2302            expected_group_version_case(
2303                name,
2304                GroupRanges::offset_commit(version_range),
2305                requests,
2306                result,
2307            )
2308        };
2309        let sent_request = |version| vec![(version, commit_request_for_test("workers", -1, -1))];
2310        vec![
2311            commit_case(
2312                "coordinator stops at v7",
2313                (2, 7),
2314                sent_request(7),
2315                committed.clone(),
2316            ),
2317            commit_case(
2318                "coordinator stops at v9",
2319                (2, 9),
2320                sent_request(9),
2321                committed.clone(),
2322            ),
2323            commit_case(
2324                "coordinator supports v10",
2325                (2, 10),
2326                sent_request(9),
2327                committed,
2328            ),
2329            commit_case(
2330                "coordinator supports only v10",
2331                (10, 10),
2332                Vec::new(),
2333                incompatible_group_version(offset_commit_request::API_KEY, 10, 10),
2334            ),
2335        ]
2336    }
2337
2338    #[yare::parameterized(
2339        coordinator_stops_at_v7 = { 0 },
2340        coordinator_stops_at_v9 = { 1 },
2341        coordinator_supports_v10 = { 2 },
2342        coordinator_only_supports_v10 = { 3 },
2343    )]
2344    fn alter_consumer_group_offsets_sends_offset_commit_by_topic_name(index: usize) {
2345        let case = commit_group_version_cases().remove(index);
2346        block_on_test(run_group_version_case(
2347            case,
2348            offset_commit_request::API_KEY,
2349            |request, version| {
2350                encode(
2351                    &offset_commit_response(request, |_| 0),
2352                    version,
2353                    version >= offset_commit_request::FLEXIBLE_MIN,
2354                )
2355            },
2356            commit_group_offsets_by_topic,
2357        ));
2358    }
2359
2360    async fn commit_group_offsets_by_topic(
2361        name: &'static str,
2362        mut admin: AdminClient,
2363    ) -> CommitResult {
2364        let offsets = test_group_offsets();
2365        Box::pin(with_incompatible_version(
2366            name,
2367            admin.alter_consumer_group_offsets("workers", &offsets),
2368        ))
2369        .await
2370    }
2371
2372    macro_rules! offset_fetch_topics {
2373        ($topic:ident, $partition:ident, $rows:expr) => {
2374            ($rows)
2375                .iter()
2376                .map(
2377                    |(name, partition_index, committed_offset, error_code)| $topic {
2378                        name: (*name).into(),
2379                        partitions: vec![$partition {
2380                            partition_index: *partition_index,
2381                            committed_offset: *committed_offset,
2382                            error_code: *error_code,
2383                            ..Default::default()
2384                        }],
2385                        ..Default::default()
2386                    },
2387                )
2388                .collect()
2389        };
2390    }
2391
2392    /// The `OffsetFetch` answer of the mock coordinator at `version`: one
2393    /// committed offset, one partition with no committed offset, and one
2394    /// partition with an error code. v2 to v7 put the topics at the top level.
2395    fn offset_fetch_response(version: i16) -> OffsetFetchResponse {
2396        let rows = [
2397            ("orders", 2, 41, 0),
2398            ("orders", 3, -1, 0),
2399            ("payments", 0, 5, 3),
2400        ];
2401        if version < 8 {
2402            OffsetFetchResponse {
2403                topics: offset_fetch_topics!(
2404                    OffsetFetchResponseTopic,
2405                    OffsetFetchResponsePartition,
2406                    &rows
2407                ),
2408                ..Default::default()
2409            }
2410        } else {
2411            OffsetFetchResponse {
2412                groups: vec![OffsetFetchResponseGroup {
2413                    group_id: "workers".into(),
2414                    topics: offset_fetch_topics!(
2415                        OffsetFetchResponseTopics,
2416                        OffsetFetchResponsePartitions,
2417                        &rows
2418                    ),
2419                    ..Default::default()
2420                }],
2421                ..Default::default()
2422            }
2423        }
2424    }
2425
2426    /// The negotiated `OffsetFetch` version and the result of
2427    /// `list_consumer_group_offsets`, with a version error as its ranges.
2428    type FetchResult = Result<GroupOffsets, (i16, i16, i16, i16, i16)>;
2429
2430    /// Apache Kafka's `ListConsumerGroupOffsetsHandler` builds `OffsetFetch`
2431    /// with `OffsetFetchRequest.Builder.forTopicNames`, which caps the version
2432    /// at 9. A request for all topics also needs v2. The response names each
2433    /// topic, so the call sends no `Metadata` request.
2434    fn fetch_group_version_cases() -> Vec<ExpectedGroupVersionCase<OffsetFetchRequest, FetchResult>>
2435    {
2436        let fetched = Ok(test_group_offsets());
2437        let case = |name, ranges, requests, result| {
2438            expected_group_version_case(name, GroupRanges::offset_fetch(ranges), requests, result)
2439        };
2440        let legacy_request = |version| {
2441            vec![(
2442                version,
2443                OffsetFetchRequest {
2444                    group_id: "workers".into(),
2445                    topics: None,
2446                    ..Default::default()
2447                },
2448            )]
2449        };
2450        let grouped_request = |version| {
2451            vec![(
2452                version,
2453                OffsetFetchRequest {
2454                    groups: vec![offset_fetch_group("workers")],
2455                    ..Default::default()
2456                },
2457            )]
2458        };
2459        vec![
2460            case(
2461                "coordinator stops at v7",
2462                (2, 7),
2463                legacy_request(7),
2464                fetched.clone(),
2465            ),
2466            case(
2467                "coordinator stops at v8",
2468                (2, 8),
2469                grouped_request(8),
2470                fetched.clone(),
2471            ),
2472            case(
2473                "coordinator stops at v9",
2474                (2, 9),
2475                grouped_request(9),
2476                fetched.clone(),
2477            ),
2478            case(
2479                "coordinator supports v10",
2480                (2, 10),
2481                grouped_request(9),
2482                fetched,
2483            ),
2484            case(
2485                "coordinator supports only v10",
2486                (10, 10),
2487                Vec::new(),
2488                incompatible_group_version(offset_fetch_request::API_KEY, 10, 10),
2489            ),
2490            case(
2491                "coordinator supports only v1",
2492                (1, 1),
2493                Vec::new(),
2494                incompatible_group_version(offset_fetch_request::API_KEY, 1, 1),
2495            ),
2496        ]
2497    }
2498
2499    async fn fetch_group_offsets_by_topic(
2500        name: &'static str,
2501        mut admin: AdminClient,
2502    ) -> FetchResult {
2503        let offsets = admin.list_consumer_group_offsets("workers");
2504        Box::pin(with_incompatible_version(name, offsets)).await
2505    }
2506
2507    #[yare::parameterized(
2508        coordinator_stops_at_v7 = { 0 },
2509        coordinator_stops_at_v8 = { 1 },
2510        coordinator_stops_at_v9 = { 2 },
2511        coordinator_supports_v10 = { 3 },
2512        coordinator_only_supports_v10 = { 4 },
2513        coordinator_only_supports_v1 = { 5 },
2514    )]
2515    fn list_consumer_group_offsets_sends_offset_fetch_by_topic_name(index: usize) {
2516        let case = fetch_group_version_cases().remove(index);
2517        block_on_test(run_group_version_case(
2518            case,
2519            offset_fetch_request::API_KEY,
2520            |_, version| {
2521                encode(
2522                    &offset_fetch_response(version),
2523                    version,
2524                    version >= offset_fetch_request::FLEXIBLE_MIN,
2525                )
2526            },
2527            fetch_group_offsets_by_topic,
2528        ));
2529    }
2530
2531    /// The result of `committed_offsets`, with a broker error as its fields.
2532    type OffsetsResult = Result<GroupOffsets, (&'static str, i16, &'static str, Option<String>)>;
2533
2534    /// Kafka's `OffsetFetchResponse.group` reads the group error code from the
2535    /// top level below v8 and from the group entry from v8.
2536    #[yare::parameterized(
2537        v2_to_v7_shape = { offset_fetch_response(7), Ok(test_group_offsets()) },
2538        v8_and_v9_shape = { offset_fetch_response(9), Ok(test_group_offsets()) },
2539        v2_to_v7_group_error = { OffsetFetchResponse { error_code: 15, ..Default::default() }, Err(("OffsetFetch", 15, "COORDINATOR_NOT_AVAILABLE", Some("group=workers".into()))) },
2540        v8_and_v9_group_error = { OffsetFetchResponse { groups: vec![OffsetFetchResponseGroup { group_id: "workers".into(), error_code: 16, ..Default::default() }], ..Default::default() }, Err(("OffsetFetch", 16, "NOT_COORDINATOR", Some("group=workers".into()))) },
2541    )]
2542    fn committed_offsets_reads_both_response_shapes(
2543        response: OffsetFetchResponse,
2544        expected: OffsetsResult,
2545    ) {
2546        let result: OffsetsResult =
2547            committed_offsets("workers", response).map_err(|error| match error {
2548                AdminError::Broker {
2549                    api,
2550                    code,
2551                    name,
2552                    message,
2553                } => (api, code, name, message),
2554                other => panic!("unexpected error {other:?}"),
2555            });
2556        assert2::assert!(result == expected);
2557    }
2558
2559    #[derive(Clone, Copy, Debug, PartialEq, Eq)]
2560    enum RetryExpected {
2561        Success,
2562        OffsetFetch(i16),
2563        CommitOutcome(i16),
2564        FindCoordinator(i16),
2565        Timeout,
2566    }
2567
2568    /// The error codes that the mock brokers answer, one per attempt. The
2569    /// last code repeats. A code of 0 answers with the coordinator or with the
2570    /// offsets of `offset_fetch_response`.
2571    #[derive(Clone, Default)]
2572    struct RetryScript {
2573        find_coordinator: Vec<i16>,
2574        offset_fetch: Vec<i16>,
2575        offset_commit: Vec<i16>,
2576        find_coordinator_requests: usize,
2577        offset_fetch_requests: usize,
2578        offset_commit_requests: usize,
2579        /// The first `unreachable_answers` `FindCoordinator` answers name
2580        /// `unreachable`, an address that refuses connections.
2581        unreachable_answers: usize,
2582        unreachable: Option<std::net::SocketAddr>,
2583        /// The coordinator does not answer `OffsetCommit`.
2584        silent_offset_commit: bool,
2585        /// The first `closed_offset_commits` `OffsetCommit` requests close the
2586        /// connection with no answer.
2587        closed_offset_commits: usize,
2588    }
2589
2590    /// An address on which no listener accepts connections.
2591    async fn refused_address() -> std::net::SocketAddr {
2592        let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
2593            .await
2594            .expect("bind a port");
2595        listener.local_addr().expect("local address")
2596    }
2597
2598    impl RetryScript {
2599        fn next(codes: &[i16], requests: &mut usize) -> i16 {
2600            let code = codes[(*requests).min(codes.len() - 1)];
2601            *requests += 1;
2602            code
2603        }
2604    }
2605
2606    /// A mock broker that answers `FindCoordinator` from `script` with
2607    /// `coordinator` as the coordinator, and `OffsetFetch` and `OffsetCommit`
2608    /// from `script`. An `OffsetCommit` answer puts its code on each
2609    /// partition of the request.
2610    async fn scripted_group_broker(
2611        script: Arc<Mutex<RetryScript>>,
2612        coordinator: Arc<Mutex<Option<std::net::SocketAddr>>>,
2613    ) -> MockBroker {
2614        let ranges = GroupRanges {
2615            offset_commit: (2, 9),
2616            offset_fetch: (2, 9),
2617        };
2618        let closing = Arc::clone(&script);
2619        let handler = move |api_key: i16, version: i16, body: &[u8]| match api_key {
2620            api_versions_request::API_KEY => Some(api_versions(ranges)),
2621            offset_commit_request::API_KEY => {
2622                let request: OffsetCommitRequest = decode_request(body, version);
2623                let mut script = script.lock().expect("script lock");
2624                let script = &mut *script;
2625                let error_code =
2626                    RetryScript::next(&script.offset_commit, &mut script.offset_commit_requests);
2627                if script.silent_offset_commit {
2628                    return None;
2629                }
2630                let response = offset_commit_response(&request, |_| error_code);
2631                Some(encode(
2632                    &response,
2633                    version,
2634                    version >= offset_commit_request::FLEXIBLE_MIN,
2635                ))
2636            }
2637            find_coordinator_request::API_KEY => {
2638                let mut script = script.lock().expect("script lock");
2639                let script = &mut *script;
2640                let error_code = RetryScript::next(
2641                    &script.find_coordinator,
2642                    &mut script.find_coordinator_requests,
2643                );
2644                let addr = match script.unreachable {
2645                    Some(unreachable)
2646                        if script.find_coordinator_requests <= script.unreachable_answers =>
2647                    {
2648                        unreachable
2649                    }
2650                    _ => coordinator
2651                        .lock()
2652                        .expect("coordinator lock")
2653                        .expect("coordinator address"),
2654                };
2655                Some(encode(
2656                    &mock_coordinator_response(addr, error_code),
2657                    version,
2658                    false,
2659                ))
2660            }
2661            offset_fetch_request::API_KEY => {
2662                let mut script = script.lock().expect("script lock");
2663                let script = &mut *script;
2664                let error_code =
2665                    RetryScript::next(&script.offset_fetch, &mut script.offset_fetch_requests);
2666                let response = if error_code == 0 {
2667                    offset_fetch_response(version)
2668                } else {
2669                    OffsetFetchResponse {
2670                        groups: vec![OffsetFetchResponseGroup {
2671                            group_id: "workers".into(),
2672                            error_code,
2673                            ..Default::default()
2674                        }],
2675                        ..Default::default()
2676                    }
2677                };
2678                Some(encode(&response, version, true))
2679            }
2680            _ => None,
2681        };
2682        MockBroker::start_with_replies(move |api_key, version, _, body| {
2683            if api_key == offset_commit_request::API_KEY {
2684                let mut script = closing.lock().expect("script lock");
2685                if script.closed_offset_commits > 0 {
2686                    script.closed_offset_commits -= 1;
2687                    script.offset_commit_requests += 1;
2688                    return MockReply::Close;
2689                }
2690            }
2691            handler(api_key, version, body).map_or(MockReply::Silent, MockReply::Respond)
2692        })
2693        .await
2694    }
2695
2696    struct ScriptedGroupCluster {
2697        coordinator: MockBroker,
2698        bootstrap: MockBroker,
2699        admin: AdminClient,
2700    }
2701
2702    impl ScriptedGroupCluster {
2703        async fn start(script: Arc<Mutex<RetryScript>>) -> Self {
2704            let coordinator_addr = Arc::new(Mutex::new(None));
2705            let coordinator =
2706                scripted_group_broker(Arc::clone(&script), Arc::clone(&coordinator_addr)).await;
2707            *coordinator_addr.lock().expect("coordinator lock") = Some(coordinator.addr);
2708            let bootstrap =
2709                scripted_group_broker(Arc::clone(&script), Arc::clone(&coordinator_addr)).await;
2710            let admin = connect_admin(bootstrap.addr, "admin connects").await;
2711            Self {
2712                coordinator,
2713                bootstrap,
2714                admin,
2715            }
2716        }
2717    }
2718
2719    async fn run_scripted_call<T, F, Fut>(script: RetryScript, call: F) -> (T, RetryScript)
2720    where
2721        F: FnOnce(AdminClient) -> Fut,
2722        Fut: std::future::Future<Output = T>,
2723    {
2724        let shared_script = Arc::new(Mutex::new(script));
2725        let ScriptedGroupCluster {
2726            coordinator,
2727            bootstrap,
2728            admin,
2729        } = ScriptedGroupCluster::start(Arc::clone(&shared_script)).await;
2730        let result = call(admin).await;
2731        bootstrap.stop();
2732        coordinator.stop();
2733        let recorded = shared_script.lock().expect("script lock").clone();
2734        (result, recorded)
2735    }
2736
2737    const GROUP_RETRY_LONG: Duration = Duration::from_secs(5);
2738    const GROUP_RETRY_NOW: Duration = Duration::from_millis(500);
2739
2740    fn group_retry_backoff(timeout: Duration) -> Duration {
2741        if timeout == GROUP_RETRY_LONG {
2742            Duration::from_millis(1)
2743        } else {
2744            timeout
2745        }
2746    }
2747
2748    fn test_retry_policy(timeout: Duration, initial_backoff: Duration) -> RetryPolicy {
2749        RetryPolicy {
2750            timeout,
2751            initial_backoff,
2752            max_backoff: initial_backoff,
2753            jitter: 0.0,
2754            max_retries: u32::MAX,
2755        }
2756    }
2757
2758    fn list_groups_retry_policy(
2759        timeout: Duration,
2760        backoff: Duration,
2761        max_retries: u32,
2762    ) -> RetryPolicy {
2763        RetryPolicy {
2764            timeout,
2765            initial_backoff: backoff,
2766            max_backoff: backoff,
2767            jitter: 0.0,
2768            max_retries,
2769        }
2770    }
2771
2772    #[derive(Clone, Copy)]
2773    struct GroupRetryScenario {
2774        name: &'static str,
2775        find_coordinator: &'static [i16],
2776        offset_fetch: &'static [i16],
2777        offset_commit: &'static [i16],
2778        timeout: Duration,
2779        fetch_expected: RetryExpected,
2780        fetch_calls: (usize, usize),
2781        commit_expected: RetryExpected,
2782        commit_calls: (usize, usize),
2783    }
2784
2785    #[derive(Clone, Copy)]
2786    struct RetryCodes {
2787        find_coordinator: &'static [i16],
2788        offset_fetch: &'static [i16],
2789        offset_commit: &'static [i16],
2790    }
2791
2792    #[derive(Clone, Copy)]
2793    struct RetryExpectations {
2794        fetch: RetryExpected,
2795        commit: RetryExpected,
2796    }
2797
2798    fn successful_retry_case(
2799        name: &'static str,
2800        find_coordinator: &'static [i16],
2801        rpc_errors: &'static [i16],
2802        calls: (usize, usize),
2803    ) -> GroupRetryScenario {
2804        retry_scenario(
2805            name,
2806            RetryCodes {
2807                find_coordinator,
2808                offset_fetch: rpc_errors,
2809                offset_commit: rpc_errors,
2810            },
2811            GROUP_RETRY_LONG,
2812            RetryExpectations {
2813                fetch: RetryExpected::Success,
2814                commit: RetryExpected::CommitOutcome(0),
2815            },
2816            calls,
2817        )
2818    }
2819
2820    fn retry_scenario(
2821        name: &'static str,
2822        codes: RetryCodes,
2823        timeout: Duration,
2824        expected: RetryExpectations,
2825        calls: (usize, usize),
2826    ) -> GroupRetryScenario {
2827        GroupRetryScenario {
2828            name,
2829            find_coordinator: codes.find_coordinator,
2830            offset_fetch: codes.offset_fetch,
2831            offset_commit: codes.offset_commit,
2832            timeout,
2833            fetch_expected: expected.fetch,
2834            fetch_calls: calls,
2835            commit_expected: expected.commit,
2836            commit_calls: calls,
2837        }
2838    }
2839
2840    fn coordinator_recovery_case(name: &'static str, codes: &'static [i16]) -> GroupRetryScenario {
2841        successful_retry_case(name, codes, &[0], (2, 1))
2842    }
2843
2844    fn timed_out_retry_case(name: &'static str, codes: RetryCodes) -> GroupRetryScenario {
2845        retry_scenario(
2846            name,
2847            codes,
2848            GROUP_RETRY_NOW,
2849            RetryExpectations {
2850                fetch: RetryExpected::Timeout,
2851                commit: RetryExpected::Timeout,
2852            },
2853            (1, 1),
2854        )
2855    }
2856
2857    fn group_retry_scenarios() -> Vec<GroupRetryScenario> {
2858        use RetryExpected::{FindCoordinator as FetchFindError, Timeout as FetchTimeout};
2859
2860        vec![
2861            successful_retry_case("no error", &[0], &[0], (1, 1)),
2862            successful_retry_case("coordinator load in progress", &[0], &[14, 14, 0], (1, 3)),
2863            successful_retry_case("coordinator not available", &[0], &[15, 0], (2, 2)),
2864            successful_retry_case("not coordinator", &[0], &[16, 0], (2, 2)),
2865            timed_out_retry_case(
2866                "load in progress timeout",
2867                RetryCodes {
2868                    find_coordinator: &[0],
2869                    offset_fetch: &[14],
2870                    offset_commit: &[27],
2871                },
2872            ),
2873            timed_out_retry_case(
2874                "not coordinator timeout",
2875                RetryCodes {
2876                    find_coordinator: &[0],
2877                    offset_fetch: &[16],
2878                    offset_commit: &[16],
2879                },
2880            ),
2881            retry_scenario(
2882                "group error is final",
2883                RetryCodes {
2884                    find_coordinator: &[0],
2885                    offset_fetch: &[30],
2886                    offset_commit: &[25],
2887                },
2888                GROUP_RETRY_LONG,
2889                RetryExpectations {
2890                    fetch: RetryExpected::OffsetFetch(30),
2891                    commit: RetryExpected::CommitOutcome(25),
2892                },
2893                (1, 1),
2894            ),
2895            retry_scenario(
2896                "another group error is final",
2897                RetryCodes {
2898                    find_coordinator: &[0],
2899                    offset_fetch: &[7],
2900                    offset_commit: &[7],
2901                },
2902                GROUP_RETRY_LONG,
2903                RetryExpectations {
2904                    fetch: RetryExpected::OffsetFetch(7),
2905                    commit: RetryExpected::CommitOutcome(7),
2906                },
2907                (1, 1),
2908            ),
2909            coordinator_recovery_case("find coordinator not available then succeeds", &[15, 0]),
2910            coordinator_recovery_case("find coordinator load in progress then succeeds", &[14, 0]),
2911            retry_scenario(
2912                "find coordinator timeout",
2913                RetryCodes {
2914                    find_coordinator: &[15],
2915                    offset_fetch: &[0],
2916                    offset_commit: &[0],
2917                },
2918                GROUP_RETRY_NOW,
2919                RetryExpectations {
2920                    fetch: FetchTimeout,
2921                    commit: FetchTimeout,
2922                },
2923                (1, 0),
2924            ),
2925            retry_scenario(
2926                "find coordinator authorization error",
2927                RetryCodes {
2928                    find_coordinator: &[30],
2929                    offset_fetch: &[0],
2930                    offset_commit: &[0],
2931                },
2932                GROUP_RETRY_LONG,
2933                RetryExpectations {
2934                    fetch: FetchFindError(30),
2935                    commit: FetchFindError(30),
2936                },
2937                (1, 0),
2938            ),
2939        ]
2940    }
2941
2942    const TIMED_OUT: i16 = 7;
2943
2944    #[derive(Clone, Copy)]
2945    enum RetryApi {
2946        Fetch,
2947        Commit,
2948    }
2949
2950    #[derive(Debug, PartialEq)]
2951    enum RetryObservation {
2952        Fetch(Result<GroupOffsets, RetryExpected>),
2953        Commit(Result<Vec<ConsumerGroupOffsetOutcome>, i16>),
2954    }
2955
2956    async fn alter_consumer_offsets_with_retry(
2957        mut admin: AdminClient,
2958        timeout: Duration,
2959    ) -> Result<Vec<ConsumerGroupOffsetOutcome>, AdminError> {
2960        let offsets = test_group_offsets();
2961        admin
2962            .alter_group_offsets_with_retry(
2963                GroupOffsetUpdate {
2964                    kind: GroupOffsetKind::Consumer,
2965                    group: "workers",
2966                    offsets: &offsets,
2967                },
2968                test_retry_policy(timeout, group_retry_backoff(timeout)),
2969            )
2970            .await
2971    }
2972
2973    async fn assert_retry_scenario(case: GroupRetryScenario, api: RetryApi) {
2974        let expected = match (api, case.fetch_expected, case.commit_expected) {
2975            (RetryApi::Fetch, RetryExpected::Success, _) => {
2976                RetryObservation::Fetch(Ok(test_group_offsets()))
2977            }
2978            (RetryApi::Fetch, RetryExpected::OffsetFetch(code), _) => {
2979                RetryObservation::Fetch(Err(RetryExpected::OffsetFetch(code)))
2980            }
2981            (RetryApi::Fetch, RetryExpected::FindCoordinator(code), _) => {
2982                RetryObservation::Fetch(Err(RetryExpected::FindCoordinator(code)))
2983            }
2984            (RetryApi::Fetch, RetryExpected::Timeout, _) => {
2985                RetryObservation::Fetch(Err(RetryExpected::Timeout))
2986            }
2987            (RetryApi::Commit, _, RetryExpected::CommitOutcome(code)) => {
2988                RetryObservation::Commit(Ok(vec![ConsumerGroupOffsetOutcome {
2989                    topic: "orders".into(),
2990                    partition: 2,
2991                    error: kafka_error_if(code, None),
2992                }]))
2993            }
2994            (RetryApi::Commit, _, RetryExpected::FindCoordinator(code)) => {
2995                RetryObservation::Commit(Err(code))
2996            }
2997            (RetryApi::Commit, _, RetryExpected::Timeout) => {
2998                RetryObservation::Commit(Err(TIMED_OUT))
2999            }
3000            _ => unreachable!("retry expectation belongs to another API"),
3001        };
3002        let expected_calls = match api {
3003            RetryApi::Fetch => case.fetch_calls,
3004            RetryApi::Commit => case.commit_calls,
3005        };
3006        let mut script = RetryScript {
3007            find_coordinator: case.find_coordinator.to_vec(),
3008            ..RetryScript::default()
3009        };
3010        match api {
3011            RetryApi::Fetch => script.offset_fetch = case.offset_fetch.to_vec(),
3012            RetryApi::Commit => script.offset_commit = case.offset_commit.to_vec(),
3013        }
3014
3015        let ((result, within_deadline), script) =
3016            run_scripted_call(script, |mut admin| async move {
3017                let started = tokio::time::Instant::now();
3018                let result = match api {
3019                    RetryApi::Fetch => {
3020                        let result = admin
3021                            .list_consumer_group_offsets_with_retry(
3022                                "workers",
3023                                test_retry_policy(case.timeout, group_retry_backoff(case.timeout)),
3024                            )
3025                            .await
3026                            .map_err(|error| match error {
3027                                AdminError::Broker {
3028                                    api: "OffsetFetch",
3029                                    code,
3030                                    ..
3031                                } => RetryExpected::OffsetFetch(code),
3032                                AdminError::Transport(ClientError::Server { error_code }) => {
3033                                    RetryExpected::FindCoordinator(error_code)
3034                                }
3035                                AdminError::Transport(ClientError::Timeout(_)) => {
3036                                    RetryExpected::Timeout
3037                                }
3038                                other => panic!("case {}: unexpected error {other:?}", case.name),
3039                            });
3040                        RetryObservation::Fetch(result)
3041                    }
3042                    RetryApi::Commit => {
3043                        let result = alter_consumer_offsets_with_retry(admin, case.timeout)
3044                            .await
3045                            .map_err(|error| match error {
3046                                AdminError::Transport(ClientError::Server { error_code }) => {
3047                                    error_code
3048                                }
3049                                AdminError::Transport(ClientError::Timeout(_)) => TIMED_OUT,
3050                                other => panic!("case {}: unexpected error {other:?}", case.name),
3051                            });
3052                        RetryObservation::Commit(result)
3053                    }
3054                };
3055                let within_deadline = match api {
3056                    RetryApi::Fetch => true,
3057                    RetryApi::Commit => started.elapsed() < case.timeout + Duration::from_secs(2),
3058                };
3059                (result, within_deadline)
3060            })
3061            .await;
3062        let rpc_calls = match api {
3063            RetryApi::Fetch => script.offset_fetch_requests,
3064            RetryApi::Commit => script.offset_commit_requests,
3065        };
3066        assert!(
3067            (
3068                result,
3069                script.find_coordinator_requests,
3070                rpc_calls,
3071                within_deadline
3072            ) == (expected, expected_calls.0, expected_calls.1, true),
3073            "case {}",
3074            case.name
3075        );
3076    }
3077
3078    /// These named cases exercise the shared retry behavior of `OffsetFetch` and
3079    /// `OffsetCommit`, while preserving each API's result and call-count checks.
3080    #[yare::parameterized(
3081        no_error = { 0 },
3082        coordinator_load_in_progress = { 1 },
3083        coordinator_not_available = { 2 },
3084        not_coordinator = { 3 },
3085        load_timeout = { 4 },
3086        coordinator_timeout = { 5 },
3087        group_error = { 6 },
3088        other_group_error = { 7 },
3089        find_coordinator_not_available = { 8 },
3090        find_coordinator_load = { 9 },
3091        find_coordinator_timeout = { 10 },
3092        find_coordinator_authorization = { 11 },
3093    )]
3094    fn group_offset_retry_scenarios_match_kafka(index: usize) {
3095        block_on_test(async move {
3096            let case = group_retry_scenarios()[index];
3097            assert_retry_scenario(case, RetryApi::Fetch).await;
3098            assert_retry_scenario(case, RetryApi::Commit).await;
3099        });
3100    }
3101
3102    #[derive(Clone, Copy, Debug, PartialEq, Eq)]
3103    enum Coordinator {
3104        UnreachableOnce,
3105        Silent,
3106        ClosedOnce,
3107    }
3108
3109    /// The coordinator retry driver recovers from a connection close, resolves
3110    /// an unreachable coordinator, and times out when the coordinator is silent.
3111    #[yare::parameterized(
3112        closed_connection = { "closed connection", GROUP_RETRY_LONG, Coordinator::ClosedOnce, 1, 2, Ok(0) },
3113        unreachable_coordinator = { "unreachable coordinator", GROUP_RETRY_LONG, Coordinator::UnreachableOnce, 2, 1, Ok(0) },
3114        silent_coordinator = { "silent coordinator", Duration::from_millis(300), Coordinator::Silent, 1, 1, Err(TIMED_OUT) },
3115    )]
3116    #[test_macro(tokio::test(flavor = "multi_thread", worker_threads = 2))]
3117    async fn alter_consumer_group_offsets_recovers_from_connection_failures(
3118        name: &'static str,
3119        timeout: Duration,
3120        coordinator: Coordinator,
3121        find_calls: usize,
3122        commit_calls: usize,
3123        expected: Result<i16, i16>,
3124    ) {
3125        let script = RetryScript {
3126            find_coordinator: vec![0],
3127            offset_commit: vec![0],
3128            unreachable_answers: usize::from(coordinator == Coordinator::UnreachableOnce),
3129            unreachable: Some(refused_address().await),
3130            silent_offset_commit: coordinator == Coordinator::Silent,
3131            closed_offset_commits: usize::from(coordinator == Coordinator::ClosedOnce),
3132            ..RetryScript::default()
3133        };
3134        let ((result, within_deadline), script) = run_scripted_call(script, |admin| async move {
3135            let started = tokio::time::Instant::now();
3136            let result = alter_consumer_offsets_with_retry(admin, timeout)
3137                .await
3138                .map(|_| 0_i16)
3139                .map_err(|error| match error {
3140                    AdminError::Transport(ClientError::Server { error_code }) => error_code,
3141                    AdminError::Transport(ClientError::Timeout(_)) => TIMED_OUT,
3142                    other => panic!("case {name}: unexpected error {other:?}"),
3143                });
3144            (result, started.elapsed() < timeout + Duration::from_secs(2))
3145        })
3146        .await;
3147        assert2::assert!(
3148            (
3149                result,
3150                script.find_coordinator_requests,
3151                script.offset_commit_requests,
3152                within_deadline
3153            ) == (expected, find_calls, commit_calls, true),
3154            "case {name}"
3155        );
3156    }
3157
3158    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3159    async fn group_offset_rpcs_use_the_group_coordinator() {
3160        let ranges = GroupRanges {
3161            offset_commit: (2, 10),
3162            offset_fetch: (2, 10),
3163        };
3164        let coordinator_group_rpcs = Arc::new(AtomicUsize::new(0));
3165        let seen = Arc::clone(&coordinator_group_rpcs);
3166        let coordinator = MockBroker::start(move |api_key, version, _, _| match api_key {
3167            api_versions_request::API_KEY => Some(api_versions(ranges)),
3168            offset_commit_request::API_KEY => {
3169                seen.fetch_add(1, Ordering::SeqCst);
3170                Some(crate::encode_test_struct_field!(
3171                    encode,
3172                    OffsetCommitResponse,
3173                    topics,
3174                    vec![successful_offset_commit_topic()],
3175                    version,
3176                    true,
3177                ))
3178            }
3179            offset_fetch_request::API_KEY => {
3180                counted_reply(&seen, Some(offset_fetch_reply(version)))
3181            }
3182            _ => None,
3183        })
3184        .await;
3185
3186        let bootstrap_group_rpcs = Arc::new(AtomicUsize::new(0));
3187        let bootstrap =
3188            bootstrap_for(&coordinator, ranges, Arc::clone(&bootstrap_group_rpcs)).await;
3189        let mut admin = connect_admin(bootstrap.addr, "admin connects").await;
3190
3191        let committed = admin
3192            .alter_consumer_group_offsets("workers", &test_group_offsets())
3193            .await
3194            .expect("offset commit succeeds");
3195        let mut admin = connect_admin(bootstrap.addr, "second admin connects").await;
3196        let fetched = admin
3197            .list_consumer_group_offsets("workers")
3198            .await
3199            .expect("offset fetch succeeds");
3200
3201        assert!(bootstrap_group_rpcs.load(Ordering::SeqCst) == 0);
3202        assert!(coordinator_group_rpcs.load(Ordering::SeqCst) == 2);
3203        assert!(committed == successful_group_offset_outcome());
3204        assert!(fetched == test_group_offsets());
3205        bootstrap.stop();
3206        coordinator.stop();
3207    }
3208
3209    /// The answer of a mock broker to one `ListGroups` request.
3210    #[derive(Clone)]
3211    enum ListGroupsAnswer {
3212        /// The groups that the broker coordinates. The broker applies the
3213        /// state and type filters of the request.
3214        Groups(Vec<ListedGroup>),
3215        /// A top-level error code.
3216        Error(i16),
3217    }
3218
3219    /// One mock broker of a `ListGroups` case: its highest `ListGroups`
3220    /// version, and its answers, one per request. The last answer repeats.
3221    #[derive(Clone)]
3222    struct ListGroupsBroker {
3223        max_version: i16,
3224        answers: Vec<ListGroupsAnswer>,
3225    }
3226
3227    fn listed(group_id: &str, protocol_type: &str, state: &str, group_type: &str) -> ListedGroup {
3228        ListedGroup {
3229            group_id: group_id.into(),
3230            protocol_type: protocol_type.into(),
3231            group_state: state.into(),
3232            group_type: group_type.into(),
3233            ..Default::default()
3234        }
3235    }
3236
3237    fn consumer_group(group_id: &str) -> ListedGroup {
3238        listed(group_id, "consumer", "Stable", "Classic")
3239    }
3240
3241    fn stable_listing(group_id: &str) -> GroupListing {
3242        GroupListing {
3243            group_id: group_id.into(),
3244            group_type: Some(GroupType::Classic),
3245            protocol_type: "consumer".into(),
3246            group_state: Some(GroupState::Stable),
3247        }
3248    }
3249
3250    fn stable_listings(group_ids: &[&str]) -> Vec<GroupListing> {
3251        group_ids
3252            .iter()
3253            .map(|group_id| stable_listing(group_id))
3254            .collect()
3255    }
3256
3257    fn groups(ids: &[&str]) -> ListGroupsBroker {
3258        ListGroupsBroker {
3259            max_version: 5,
3260            answers: vec![ListGroupsAnswer::Groups(
3261                ids.iter().map(|id| consumer_group(id)).collect(),
3262            )],
3263        }
3264    }
3265
3266    /// A mock broker that answers `Metadata` with every address in `cluster`
3267    /// (node ids from 1) and `ListGroups` from `script`. It records each
3268    /// `ListGroups` request with its version.
3269    async fn list_groups_broker(
3270        script: ListGroupsBroker,
3271        cluster: Arc<Mutex<Vec<std::net::SocketAddr>>>,
3272        requests: Arc<Mutex<Vec<(i16, ListGroupsRequest)>>>,
3273    ) -> MockBroker {
3274        MockBroker::start_with_replies(move |api_key, version, _, body| {
3275            MockSaslAnswer::Accept
3276                .reply(api_key, version)
3277                .unwrap_or_else(|| {
3278                    list_groups_reply(&script, &cluster, &requests, api_key, version, body)
3279                        .map_or(MockReply::Silent, MockReply::Respond)
3280                })
3281        })
3282        .await
3283    }
3284
3285    /// The reply of a `list_groups_broker` to one request other than SASL.
3286    fn list_groups_reply(
3287        script: &ListGroupsBroker,
3288        cluster: &Mutex<Vec<std::net::SocketAddr>>,
3289        requests: &Mutex<Vec<(i16, ListGroupsRequest)>>,
3290        api_key: i16,
3291        version: i16,
3292        body: &[u8],
3293    ) -> Option<Vec<u8>> {
3294        match api_key {
3295            api_versions_request::API_KEY => Some(encode_api_versions(vec![
3296                api_version_entry!(api_versions_request::API_KEY, 0, 0),
3297                api_version_entry!(metadata_request::API_KEY, 12, 12),
3298                api_version_entry!(list_groups_request::API_KEY, 0, script.max_version),
3299            ])),
3300            metadata_request::API_KEY => Some(crate::encode_test_struct_field!(
3301                encode,
3302                MetadataResponse,
3303                brokers,
3304                cluster
3305                    .lock()
3306                    .expect("cluster lock")
3307                    .iter()
3308                    .zip(1..)
3309                    .map(|(addr, node_id)| mock_metadata_broker(*addr, node_id))
3310                    .collect(),
3311                version,
3312                true,
3313            )),
3314            list_groups_request::API_KEY => {
3315                let request: ListGroupsRequest = decode_request(body, version);
3316                let mut requests = requests.lock().expect("requests lock");
3317                let answer = &script.answers[requests.len().min(script.answers.len() - 1)];
3318                let response = match answer {
3319                    ListGroupsAnswer::Groups(groups) => ListGroupsResponse {
3320                        groups: groups
3321                            .iter()
3322                            .filter(|group| {
3323                                (request.states_filter.is_empty()
3324                                    || request.states_filter.contains(&group.group_state))
3325                                    && (request.types_filter.is_empty()
3326                                        || request.types_filter.contains(&group.group_type))
3327                            })
3328                            .cloned()
3329                            .collect(),
3330                        ..Default::default()
3331                    },
3332                    ListGroupsAnswer::Error(error_code) => ListGroupsResponse {
3333                        error_code: *error_code,
3334                        ..Default::default()
3335                    },
3336                };
3337                requests.push((version, request));
3338                Some(encode(
3339                    &response,
3340                    version,
3341                    version >= list_groups_request::FLEXIBLE_MIN,
3342                ))
3343            }
3344            _ => None,
3345        }
3346    }
3347
3348    /// An expected broker failure: node id, error code and message.
3349    type ExpectedError = (i32, i16, Option<String>);
3350
3351    /// One `ListGroups` fan-out case on a cluster of three mock brokers.
3352    struct ListGroupsCase {
3353        name: &'static str,
3354        brokers: [ListGroupsBroker; 3],
3355        options: ListGroupsOptions,
3356        timeout: Duration,
3357        valid: Vec<GroupListing>,
3358        errors: Vec<ExpectedError>,
3359        /// The `ListGroups` requests that each broker receives.
3360        requests: [Vec<(i16, ListGroupsRequest)>; 3],
3361    }
3362
3363    /// The deadline of a case that ends before it.
3364    const LONG: Duration = Duration::from_secs(10);
3365    /// A deadline that ends before the first retry, which waits `BACKOFF`.
3366    const SHORT: Duration = Duration::from_millis(500);
3367    /// The wait between two `ListGroups` attempts on one broker.
3368    const BACKOFF: Duration = Duration::from_secs(1);
3369
3370    #[derive(Default)]
3371    struct ListGroupsCaseBuilder {
3372        name: &'static str,
3373        brokers: Option<[ListGroupsBroker; 3]>,
3374        options: ListGroupsOptions,
3375        timeout: Option<Duration>,
3376        valid: Vec<GroupListing>,
3377        errors: Vec<ExpectedError>,
3378        requests: Option<[Vec<(i16, ListGroupsRequest)>; 3]>,
3379    }
3380
3381    impl ListGroupsCase {
3382        fn builder() -> ListGroupsCaseBuilder {
3383            ListGroupsCaseBuilder::default()
3384        }
3385    }
3386
3387    impl ListGroupsCaseBuilder {
3388        fn name(mut self, name: &'static str) -> Self {
3389            self.name = name;
3390            self
3391        }
3392
3393        fn brokers(mut self, brokers: [ListGroupsBroker; 3]) -> Self {
3394            self.brokers = Some(brokers);
3395            self
3396        }
3397
3398        fn options(mut self, options: ListGroupsOptions) -> Self {
3399            self.options = options;
3400            self
3401        }
3402
3403        fn timeout(mut self, timeout: Duration) -> Self {
3404            self.timeout = Some(timeout);
3405            self
3406        }
3407
3408        fn valid(mut self, valid: Vec<GroupListing>) -> Self {
3409            self.valid = valid;
3410            self
3411        }
3412
3413        fn errors(mut self, errors: Vec<ExpectedError>) -> Self {
3414            self.errors = errors;
3415            self
3416        }
3417
3418        fn requests(mut self, requests: [Vec<(i16, ListGroupsRequest)>; 3]) -> Self {
3419            self.requests = Some(requests);
3420            self
3421        }
3422
3423        fn build(self) -> ListGroupsCase {
3424            ListGroupsCase {
3425                name: self.name,
3426                brokers: self.brokers.expect("brokers must be set"),
3427                options: self.options,
3428                timeout: self.timeout.unwrap_or(LONG),
3429                valid: self.valid,
3430                errors: self.errors,
3431                requests: self
3432                    .requests
3433                    .unwrap_or_else(|| [vec![plain(5)], vec![plain(5)], vec![plain(5)]]),
3434            }
3435        }
3436    }
3437
3438    fn plain(version: i16) -> (i16, ListGroupsRequest) {
3439        (version, ListGroupsRequest::default())
3440    }
3441
3442    fn stable(version: i16) -> (i16, ListGroupsRequest) {
3443        (
3444            version,
3445            ListGroupsRequest {
3446                states_filter: stable_state_filter(),
3447                ..Default::default()
3448            },
3449        )
3450    }
3451
3452    /// The request for the type filter `Classic` and `Consumer`. The encoder
3453    /// omits `types_filter` below v5.
3454    fn classic_and_consumer(version: i16) -> (i16, ListGroupsRequest) {
3455        (
3456            version,
3457            ListGroupsRequest {
3458                types_filter: if version >= 5 {
3459                    vec!["Consumer".into(), "Classic".into()]
3460                } else {
3461                    Vec::new()
3462                },
3463                ..Default::default()
3464            },
3465        )
3466    }
3467
3468    fn consumer_type(version: i16) -> (i16, ListGroupsRequest) {
3469        (
3470            version,
3471            ListGroupsRequest {
3472                types_filter: vec!["Consumer".into()],
3473                ..Default::default()
3474            },
3475        )
3476    }
3477
3478    /// The message of a `ListGroups` version error on a broker with
3479    /// `ListGroups` v0 to `broker_max`.
3480    fn unsupported(broker_max: i16, client_min: i16) -> String {
3481        AdminError::Transport(ClientError::IncompatibleVersion {
3482            api_key: list_groups_request::API_KEY,
3483            broker_min: 0,
3484            broker_max,
3485            client_min,
3486            client_max: 5,
3487        })
3488        .to_string()
3489    }
3490
3491    fn states(states: &[GroupState]) -> ListGroupsOptions {
3492        ListGroupsOptions::builder()
3493            .group_states(states.iter().copied().collect())
3494            .build()
3495    }
3496
3497    fn stable_state_filter() -> Vec<String> {
3498        vec![GroupState::Stable.to_string()]
3499    }
3500
3501    fn single_broker_error() -> Vec<ExpectedError> {
3502        vec![(2, 31, None)]
3503    }
3504
3505    fn expected_kafka_error(code: i16, message: Option<String>) -> KafkaError {
3506        kafka_error_if(code, message).expect("error code is nonzero")
3507    }
3508
3509    fn types(types: &[GroupType]) -> ListGroupsOptions {
3510        ListGroupsOptions::builder()
3511            .types(types.iter().copied().collect())
3512            .build()
3513    }
3514
3515    fn broker_error(code: i16) -> ListGroupsBroker {
3516        ListGroupsBroker {
3517            max_version: 5,
3518            answers: vec![ListGroupsAnswer::Error(code)],
3519        }
3520    }
3521
3522    /// Kafka's `Call.fail` stops a node call after `retries` retries with a
3523    /// `TimeoutException`, before the deadline.
3524    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3525    async fn list_groups_stops_a_broker_after_the_retry_limit() {
3526        let cluster = Arc::new(Mutex::new(Vec::new()));
3527        let sent = Arc::new(Mutex::new(Vec::new()));
3528        let broker =
3529            list_groups_broker(broker_error(14), Arc::clone(&cluster), Arc::clone(&sent)).await;
3530        *cluster.lock().expect("cluster lock") = vec![broker.addr];
3531        let admin = connect_admin(broker.addr, "admin connects").await;
3532
3533        let options = ListGroupsOptions::builder().build();
3534        let result = run_list_groups_with_retry(
3535            &admin,
3536            &options,
3537            list_groups_retry_policy(Duration::from_secs(10), Duration::from_millis(1), 2),
3538        )
3539        .await;
3540
3541        broker.stop();
3542        let codes = result
3543            .errors
3544            .iter()
3545            .map(|error| error.error.code)
3546            .collect::<Vec<_>>();
3547        assert!((codes, sent.lock().expect("sent lock").len()) == (vec![7], 3));
3548    }
3549
3550    fn below(max_version: i16, ids: &[&str]) -> ListGroupsBroker {
3551        ListGroupsBroker {
3552            max_version,
3553            ..groups(ids)
3554        }
3555    }
3556
3557    /// Runs `case` on a new cluster of three mock brokers. Broker 1 is the
3558    /// bootstrap broker.
3559    async fn run_list_groups_case(case: ListGroupsCase) {
3560        let cluster = Arc::new(Mutex::new(Vec::new()));
3561        let mut brokers = Vec::new();
3562        let mut requests = Vec::new();
3563        for script in case.brokers {
3564            let sent = Arc::new(Mutex::new(Vec::new()));
3565            brokers.push(list_groups_broker(script, Arc::clone(&cluster), Arc::clone(&sent)).await);
3566            requests.push(sent);
3567        }
3568        *cluster.lock().expect("cluster lock") = brokers.iter().map(|broker| broker.addr).collect();
3569        let admin = connect_admin(brokers[0].addr, "admin connects").await;
3570
3571        let result = run_list_groups_with_retry(
3572            &admin,
3573            &case.options,
3574            list_groups_retry_policy(case.timeout, BACKOFF, u32::MAX),
3575        )
3576        .await;
3577
3578        let expected = ListGroupsResult {
3579            valid: case.valid,
3580            errors: case
3581                .errors
3582                .into_iter()
3583                .map(|(node_id, code, message)| {
3584                    let addr = brokers[usize::try_from(node_id - 1).expect("node id")].addr;
3585                    ListGroupsError {
3586                        node_id,
3587                        host: addr.ip().to_string(),
3588                        port: broker_port(addr),
3589                        error: expected_kafka_error(code, message),
3590                    }
3591                })
3592                .collect(),
3593        };
3594        for broker in brokers {
3595            broker.stop();
3596        }
3597        let requests = requests
3598            .iter()
3599            .map(|sent| sent.lock().expect("requests lock").clone())
3600            .collect::<Vec<_>>();
3601        let name = case.name;
3602        assert!(
3603            (result, requests) == (expected, case.requests.to_vec()),
3604            "case {name}"
3605        );
3606    }
3607
3608    async fn run_list_groups_with_retry(
3609        admin: &AdminClient,
3610        options: &ListGroupsOptions,
3611        policy: RetryPolicy,
3612    ) -> ListGroupsResult {
3613        admin
3614            .list_groups_with_retry(options, policy)
3615            .await
3616            .expect("list_groups succeeds")
3617    }
3618
3619    fn block_on_test(future: impl std::future::Future<Output = ()> + Send + 'static) {
3620        tokio::runtime::Builder::new_multi_thread()
3621            .worker_threads(2)
3622            .enable_all()
3623            .build()
3624            .expect("tokio runtime")
3625            .block_on(future);
3626    }
3627
3628    fn run_list_groups_case_sync(case: ListGroupsCase) {
3629        block_on_test(run_list_groups_case(case));
3630    }
3631
3632    /// Apache Kafka's `KafkaAdminClient.listGroups` sends `Metadata`, then
3633    /// `ListGroups` to every broker, and merges the answers: one listing per
3634    /// group id, and one error per failed broker. It retries 14 and 15 on the
3635    /// broker until the deadline, and then `Call.fail` gives a
3636    /// `TimeoutException` (7) with the last error as its cause.
3637    #[yare::parameterized(
3638        one_group_each_broker = { 0 },
3639        empty_broker = { 1 },
3640        duplicate_group = { 2 },
3641        broker_timeout = { 3 },
3642        broker_retries = { 4 },
3643        authorization_error = { 5 },
3644    )]
3645    fn list_groups_asks_every_broker_and_merges(index: usize) {
3646        run_list_groups_case_sync(list_groups_merge_cases().remove(index));
3647    }
3648
3649    /// Kafka's `listGroups` sends the state and type filters to every broker,
3650    /// and filters protocol types on the client. `ListGroupsRequest.Builder.build`
3651    /// rejects a state filter below v4 and a type filter below v5, except a
3652    /// type filter of `Classic` and `Consumer` at most, with `Classic`, which
3653    /// it omits. The rejection is an error for that broker only.
3654    #[yare::parameterized(
3655        state_filter = { 0 },
3656        old_state_filter = { 1 },
3657        consumer_type_filter = { 2 },
3658        old_consumer_type_pair = { 3 },
3659        protocol_filter = { 4 },
3660    )]
3661    fn list_groups_sends_filters_as_kafka_does(index: usize) {
3662        run_list_groups_case_sync(list_groups_filter_cases().remove(index));
3663    }
3664
3665    /// Broker 2 of a SASL cluster in
3666    /// `list_groups_isolates_a_secured_broker_that_stalls_or_rejects`.
3667    #[derive(Clone, Copy, Debug)]
3668    enum SecuredBroker {
3669        /// Accepts TCP and never answers the SASL exchange.
3670        Stalls,
3671        /// Answers `SaslAuthenticate` with `SASL_AUTHENTICATION_FAILED` (58).
3672        RejectsAuthentication,
3673    }
3674
3675    /// A started `SecuredBroker`. `connections` counts the TCP connections
3676    /// that it accepts, or the `SaslHandshake` requests that it answers.
3677    struct StartedSecuredBroker {
3678        addr: std::net::SocketAddr,
3679        connections: Arc<AtomicUsize>,
3680        mock: Option<MockBroker>,
3681        listener: Option<tokio::task::JoinHandle<()>>,
3682    }
3683
3684    impl SecuredBroker {
3685        async fn start(self) -> StartedSecuredBroker {
3686            let connections = Arc::new(AtomicUsize::new(0));
3687            let counter = Arc::clone(&connections);
3688            match self {
3689                Self::Stalls => {
3690                    let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
3691                        .await
3692                        .expect("listener binds");
3693                    let addr = listener.local_addr().expect("listener address");
3694                    let task = tokio::spawn(async move {
3695                        let mut open = Vec::new();
3696                        while let Ok((stream, _)) = listener.accept().await {
3697                            counter.fetch_add(1, Ordering::SeqCst);
3698                            open.push(stream);
3699                        }
3700                    });
3701                    StartedSecuredBroker {
3702                        addr,
3703                        connections,
3704                        mock: None,
3705                        listener: Some(task),
3706                    }
3707                }
3708                Self::RejectsAuthentication => {
3709                    let mock = MockBroker::start_with_replies(move |api_key, version, _, _| {
3710                        if api_key == sasl_handshake_request::API_KEY {
3711                            counter.fetch_add(1, Ordering::SeqCst);
3712                        }
3713                        MockSaslAnswer::AuthenticateError(SASL_AUTHENTICATION_FAILED)
3714                            .reply(api_key, version)
3715                            .unwrap_or(MockReply::Silent)
3716                    })
3717                    .await;
3718                    StartedSecuredBroker {
3719                        addr: mock.addr,
3720                        connections,
3721                        mock: Some(mock),
3722                        listener: None,
3723                    }
3724                }
3725            }
3726        }
3727
3728        /// The error code and message that `list_groups` reports for the
3729        /// broker at `addr`.
3730        fn expected_error(self, addr: std::net::SocketAddr) -> (i16, Option<String>) {
3731            match self {
3732                Self::Stalls => (
3733                    7,
3734                    Some("ListGroups timed out after 1 attempt(s)".to_owned()),
3735                ),
3736                Self::RejectsAuthentication => (
3737                    SASL_AUTHENTICATION_FAILED,
3738                    Some(
3739                        AdminError::Transport(ClientError::Authentication {
3740                            addr,
3741                            source: AuthenticationError::Sasl(SaslAuthenticationError::Failed(
3742                                "SaslAuthenticate(PLAIN) error_code=58 \
3743                                 error_message=Some(\"rejected by mock broker\")"
3744                                    .to_owned(),
3745                            )),
3746                        })
3747                        .to_string(),
3748                    ),
3749                ),
3750            }
3751        }
3752    }
3753
3754    impl StartedSecuredBroker {
3755        fn stop(self) {
3756            if let Some(mock) = self.mock {
3757                mock.stop();
3758            }
3759            if let Some(listener) = self.listener {
3760                listener.abort();
3761            }
3762        }
3763    }
3764
3765    const SASL_AUTHENTICATION_FAILED: i16 = 58;
3766
3767    /// A broker that stalls during the SASL exchange, or that rejects
3768    /// authentication, fails on its own. The groups of the other brokers stay
3769    /// in the result.
3770    ///
3771    /// Kafka bounds each node call of `listGroups` by the call deadline
3772    /// (`KafkaAdminClient` `timeoutCallsInFlight`, and `Call.fail` gives a
3773    /// `TimeoutException`). Kafka fails a node call with the
3774    /// `AuthenticationException` of the node and does not retry it
3775    /// (`handleResponses`, `Call.fail`: the exception is not a
3776    /// `RetriableException`). `ApiError.fromThrowable` maps
3777    /// `SaslAuthenticationException` to 58.
3778    ///
3779    /// The test runs in real time. With paused time, Tokio moves the clock
3780    /// forward while loopback I/O is in flight, and the healthy brokers time
3781    /// out too.
3782    #[yare::parameterized(
3783        stalls = { SecuredBroker::Stalls },
3784        rejects_authentication = { SecuredBroker::RejectsAuthentication },
3785    )]
3786    #[test_macro(tokio::test(flavor = "multi_thread", worker_threads = 2))]
3787    async fn list_groups_isolates_a_secured_broker_that_stalls_or_rejects(broker_2: SecuredBroker) {
3788        const RETRY: RetryPolicy = RetryPolicy {
3789            timeout: Duration::from_secs(2),
3790            initial_backoff: Duration::from_millis(100),
3791            max_backoff: Duration::from_millis(100),
3792            jitter: 0.0,
3793            max_retries: u32::MAX,
3794        };
3795        let cluster = Arc::new(Mutex::new(Vec::new()));
3796        let requests = Arc::new(Mutex::new(Vec::new()));
3797        let broker_1 =
3798            list_groups_broker(groups(&["a"]), Arc::clone(&cluster), Arc::clone(&requests)).await;
3799        let broker_2_started = broker_2.start().await;
3800        let broker_3 =
3801            list_groups_broker(groups(&["c"]), Arc::clone(&cluster), Arc::clone(&requests)).await;
3802        let broker_2_addr = broker_2_started.addr;
3803        *cluster.lock().expect("cluster lock") = vec![broker_1.addr, broker_2_addr, broker_3.addr];
3804        let admin = AdminClient::connect_secured(
3805            &[broker_1.addr.to_string()],
3806            Some(ClientSecurity {
3807                protocol: krabka_security::ListenerProtocol::SaslPlaintext,
3808                tls: None,
3809                sasl: Some(SaslCredentials::Plain {
3810                    username: "alice".into(),
3811                    password: "secret".into(),
3812                }),
3813                sasl_host: None,
3814            }),
3815        )
3816        .await
3817        .expect("admin connects");
3818
3819        let outcome = tokio::time::timeout(
3820            RETRY.timeout * 5,
3821            admin.list_groups_with_retry(&ListGroupsOptions::builder().build(), RETRY),
3822        )
3823        .await
3824        .map(|result| result.expect("list_groups succeeds"));
3825        let observed = (outcome, broker_2_started.connections.load(Ordering::SeqCst));
3826
3827        let (code, message) = broker_2.expected_error(broker_2_addr);
3828        let expected = (
3829            Ok(ListGroupsResult {
3830                valid: stable_listings(&["a", "c"]),
3831                errors: vec![ListGroupsError {
3832                    node_id: 2,
3833                    host: broker_2_addr.ip().to_string(),
3834                    port: broker_port(broker_2_addr),
3835                    error: expected_kafka_error(code, message),
3836                }],
3837            }),
3838            1,
3839        );
3840        broker_1.stop();
3841        broker_2_started.stop();
3842        broker_3.stop();
3843        assert2::assert!(observed == expected, "case {broker_2:?}");
3844    }
3845
3846    fn list_groups_merge_cases() -> Vec<ListGroupsCase> {
3847        vec![
3848            ListGroupsCase::builder()
3849                .name("one group on each broker")
3850                .brokers([groups(&["a"]), groups(&["b"]), groups(&["c"])])
3851                .valid(stable_listings(&["a", "b", "c"]))
3852                .build(),
3853            ListGroupsCase::builder()
3854                .name("broker 2 has no groups")
3855                .brokers([groups(&["a"]), groups(&[]), groups(&["c"])])
3856                .valid(stable_listings(&["a", "c"]))
3857                .build(),
3858            ListGroupsCase::builder()
3859                .name("a group on two brokers is listed once")
3860                .brokers([groups(&["a"]), groups(&["a", "b"]), groups(&["c"])])
3861                .valid(stable_listings(&["a", "b", "c"]))
3862                .build(),
3863            ListGroupsCase::builder()
3864                .name("broker 2 answers coordinator not available until the timeout")
3865                .brokers([groups(&["a"]), broker_error(15), groups(&["c"])])
3866                .timeout(SHORT)
3867                .valid(stable_listings(&["a", "c"]))
3868                .errors(vec![(
3869                    2,
3870                    7,
3871                    Some(format!(
3872                        "ListGroups timed out after 1 attempt(s); last error: {}",
3873                        AdminError::Broker {
3874                            api: "ListGroups",
3875                            code: 15,
3876                            name: "COORDINATOR_NOT_AVAILABLE",
3877                            message: None,
3878                        }
3879                    )),
3880                )])
3881                .build(),
3882            ListGroupsCase::builder()
3883                .name("broker 2 answers coordinator load in progress, then its groups")
3884                .brokers([
3885                    groups(&["a"]),
3886                    ListGroupsBroker {
3887                        max_version: 5,
3888                        answers: vec![
3889                            ListGroupsAnswer::Error(14),
3890                            ListGroupsAnswer::Error(15),
3891                            ListGroupsAnswer::Groups(vec![consumer_group("b")]),
3892                        ],
3893                    },
3894                    groups(&["c"]),
3895                ])
3896                .valid(vec![
3897                    stable_listing("a"),
3898                    stable_listing("b"),
3899                    stable_listing("c"),
3900                ])
3901                .requests([
3902                    vec![plain(5)],
3903                    vec![plain(5), plain(5), plain(5)],
3904                    vec![plain(5)],
3905                ])
3906                .build(),
3907            ListGroupsCase::builder()
3908                .name("broker 2 answers cluster authorization failed")
3909                .brokers([groups(&["a"]), broker_error(31), groups(&["c"])])
3910                .valid(stable_listings(&["a", "c"]))
3911                .errors(single_broker_error())
3912                .build(),
3913        ]
3914    }
3915
3916    fn list_groups_filter_cases() -> Vec<ListGroupsCase> {
3917        vec![
3918            ListGroupsCase::builder()
3919                .name("state filter Stable is sent to every broker")
3920                .brokers([
3921                    ListGroupsBroker {
3922                        max_version: 5,
3923                        answers: vec![ListGroupsAnswer::Groups(vec![
3924                            listed("a", "consumer", "Empty", "Classic"),
3925                            consumer_group("b"),
3926                        ])],
3927                    },
3928                    groups(&["c"]),
3929                    groups(&[]),
3930                ])
3931                .options(states(&[GroupState::Stable]))
3932                .valid(stable_listings(&["b", "c"]))
3933                .requests([vec![stable(5)], vec![stable(5)], vec![stable(5)]])
3934                .build(),
3935            ListGroupsCase::builder()
3936                .name("state filter on a broker below v4 is unsupported on that broker")
3937                .brokers([groups(&["a"]), below(3, &["b"]), groups(&["c"])])
3938                .options(states(&[GroupState::Stable]))
3939                .valid(stable_listings(&["a", "c"]))
3940                .errors(vec![(2, 35, Some(unsupported(3, 4)))])
3941                .requests([vec![stable(5)], Vec::new(), vec![stable(5)]])
3942                .build(),
3943            ListGroupsCase::builder()
3944                .name("type filter Consumer on a broker below v5 is unsupported on that broker")
3945                .brokers([groups(&["a"]), below(4, &["b"]), groups(&["c"])])
3946                .options(types(&[GroupType::Consumer]))
3947                .errors(vec![(2, 35, Some(unsupported(4, 5)))])
3948                .requests([vec![consumer_type(5)], Vec::new(), vec![consumer_type(5)]])
3949                .build(),
3950            ListGroupsCase::builder()
3951                .name("type filter Classic and Consumer on a broker below v5 is omitted")
3952                .brokers([groups(&["a"]), below(4, &["b"]), groups(&["c"])])
3953                .options(types(&[GroupType::Classic, GroupType::Consumer]))
3954                .valid(vec![
3955                    stable_listing("a"),
3956                    GroupListing {
3957                        group_type: None,
3958                        ..stable_listing("b")
3959                    },
3960                    stable_listing("c"),
3961                ])
3962                .requests([
3963                    vec![classic_and_consumer(5)],
3964                    vec![classic_and_consumer(4)],
3965                    vec![classic_and_consumer(5)],
3966                ])
3967                .build(),
3968            ListGroupsCase::builder()
3969                .name("protocol type filter is applied by the client")
3970                .brokers([
3971                    ListGroupsBroker {
3972                        max_version: 5,
3973                        answers: vec![ListGroupsAnswer::Groups(vec![
3974                            consumer_group("a"),
3975                            listed("connect-a", "connect", "Stable", "Classic"),
3976                            listed("simple", "", "Empty", "Classic"),
3977                        ])],
3978                    },
3979                    groups(&[]),
3980                    ListGroupsBroker {
3981                        max_version: 5,
3982                        answers: vec![ListGroupsAnswer::Groups(vec![listed(
3983                            "share", "share", "Stable", "Share",
3984                        )])],
3985                    },
3986                ])
3987                .options(ListGroupsOptions::for_consumer_groups())
3988                .valid(vec![
3989                    stable_listing("a"),
3990                    GroupListing {
3991                        group_id: "simple".into(),
3992                        group_type: Some(GroupType::Classic),
3993                        protocol_type: String::new(),
3994                        group_state: Some(GroupState::Empty),
3995                    },
3996                ])
3997                .requests([
3998                    vec![classic_and_consumer(5)],
3999                    vec![classic_and_consumer(5)],
4000                    vec![classic_and_consumer(5)],
4001                ])
4002                .build(),
4003        ]
4004    }
4005
4006    /// A listing with no type is not a simple consumer group.
4007    #[yare::parameterized(
4008        classic_without_protocol = {
4009            Some(GroupType::Classic),
4010            "",
4011            true,
4012        },
4013        classic_with_protocol = {
4014            Some(GroupType::Classic),
4015            "consumer",
4016            false,
4017        },
4018        consumer = { Some(GroupType::Consumer), "", false },
4019        unknown = { Some(GroupType::Unknown), "", false },
4020        no_type = { None, "", false },
4021        no_type_with_protocol = { None, "consumer", false },
4022    )]
4023    fn is_simple_consumer_group_matches_kafka(
4024        group_type: Option<GroupType>,
4025        protocol_type: &str,
4026        expected: bool,
4027    ) {
4028        let listing = GroupListing {
4029            group_id: "g".into(),
4030            group_type,
4031            protocol_type: protocol_type.into(),
4032            group_state: None,
4033        };
4034        assert2::assert!(listing.is_simple_consumer_group() == expected);
4035    }
4036
4037    /// Kafka's `GroupListing` has no type or state when the broker sends an
4038    /// empty name, and `GroupState.parse` and `GroupType.parse` ignore case
4039    /// and map an unknown name to `Unknown`.
4040    #[yare::parameterized(
4041        consumer = {
4042            listed("g", "consumer", "Stable", "Consumer"),
4043            GroupListing {
4044                group_id: "g".into(),
4045                group_type: Some(GroupType::Consumer),
4046                protocol_type: "consumer".into(),
4047                group_state: Some(GroupState::Stable),
4048            },
4049        },
4050        empty = {
4051            listed("g", "", "", ""),
4052            GroupListing {
4053                group_id: "g".into(),
4054                group_type: None,
4055                protocol_type: String::new(),
4056                group_state: None,
4057            },
4058        },
4059        streams = {
4060            listed("g", "consumer", "preparingrebalance", "STREAMS"),
4061            GroupListing {
4062                group_id: "g".into(),
4063                group_type: Some(GroupType::Streams),
4064                protocol_type: "consumer".into(),
4065                group_state: Some(GroupState::PreparingRebalance),
4066            },
4067        },
4068        unknown = {
4069            listed("g", "consumer", "Rebalancing", "Future"),
4070            GroupListing {
4071                group_id: "g".into(),
4072                group_type: Some(GroupType::Unknown),
4073                protocol_type: "consumer".into(),
4074                group_state: Some(GroupState::Unknown),
4075            },
4076        },
4077    )]
4078    fn group_listing_reads_listed_group(group: ListedGroup, expected: GroupListing) {
4079        assert2::assert!(GroupListing::from(group) == expected);
4080    }
4081
4082    /// An `ApiVersions` answer that advertises every group RPC this module's
4083    /// new-call tests use, each at its highest defined version, so a test
4084    /// picks that call's `LATEST_STABLE_VERSION` and exercises the real wire
4085    /// shape rather than a downgrade.
4086    fn api_versions_all() -> Vec<u8> {
4087        let api = |api_key, max_version| ApiVersion {
4088            api_key,
4089            min_version: 0,
4090            max_version,
4091            ..Default::default()
4092        };
4093        encode_api_versions(vec![
4094            api(api_versions_request::API_KEY, 0),
4095            api(find_coordinator_request::API_KEY, 0),
4096            api(metadata_request::API_KEY, 13),
4097            api(offset_commit_request::API_KEY, 9),
4098            api(offset_fetch_request::API_KEY, 9),
4099            api(OffsetDeleteRequest::API_KEY, 0),
4100            api(LeaveGroupRequest::API_KEY, 5),
4101            api(AlterShareGroupOffsetsRequest::API_KEY, 1),
4102            api(DeleteShareGroupOffsetsRequest::API_KEY, 0),
4103        ])
4104    }
4105
4106    fn empty_metadata_response(version: i16) -> Vec<u8> {
4107        encode(&MetadataResponse::default(), version, true)
4108    }
4109
4110    fn offset_fetch_response_bytes(version: i16) -> Vec<u8> {
4111        encode(&offset_fetch_response(version), version, true)
4112    }
4113
4114    fn empty_metadata_reply(version: i16) -> Vec<u8> {
4115        empty_metadata_response(version)
4116    }
4117
4118    fn offset_fetch_reply(version: i16) -> Vec<u8> {
4119        offset_fetch_response_bytes(version)
4120    }
4121
4122    fn counted_reply(counter: &AtomicUsize, response: Option<Vec<u8>>) -> Option<Vec<u8>> {
4123        counter.fetch_add(1, Ordering::SeqCst);
4124        response
4125    }
4126
4127    /// Connects an `AdminClient` to a single mock broker that is its own
4128    /// group coordinator: it answers `ApiVersions` with
4129    /// [`api_versions_all`], `FindCoordinator` by naming itself, and
4130    /// `target_api_key` by calling `respond` with the next code out of
4131    /// `codes` (the last code repeats, as [`RetryScript::next`] gives it) and
4132    /// the negotiated version. `Metadata` answers empty.
4133    ///
4134    /// Returns the connected client, the broker (the caller stops it), and
4135    /// the number of `target_api_key` requests the broker answered.
4136    async fn connect_self_coordinating(
4137        target_api_key: i16,
4138        codes: Vec<i16>,
4139        respond: impl Fn(i16, i16) -> Vec<u8> + Send + Sync + 'static,
4140    ) -> (AdminClient, MockBroker, Arc<AtomicUsize>) {
4141        let coordinator_addr = Arc::new(Mutex::new(None::<std::net::SocketAddr>));
4142        let addr_in_mock = Arc::clone(&coordinator_addr);
4143        let script = Arc::new(Mutex::new((codes, 0_usize)));
4144        let calls = Arc::new(AtomicUsize::new(0));
4145        let calls_in_mock = Arc::clone(&calls);
4146        let broker = MockBroker::start(move |api_key, version, _, _| {
4147            if api_key == target_api_key {
4148                calls_in_mock.fetch_add(1, Ordering::SeqCst);
4149                let mut guard = script.lock().expect("script lock");
4150                let (codes, requests) = &mut *guard;
4151                let code = RetryScript::next(codes, requests);
4152                Some(respond(code, version))
4153            } else {
4154                match api_key {
4155                    api_versions_request::API_KEY => Some(api_versions_all()),
4156                    find_coordinator_request::API_KEY => {
4157                        let addr = addr_in_mock
4158                            .lock()
4159                            .expect("coordinator lock")
4160                            .expect("coordinator address set before use");
4161                        Some(encode(&mock_coordinator_response(addr, 0), version, false))
4162                    }
4163                    metadata_request::API_KEY => Some(empty_metadata_reply(version)),
4164                    _ => None,
4165                }
4166            }
4167        })
4168        .await;
4169        *coordinator_addr.lock().expect("coordinator lock") = Some(broker.addr);
4170        let admin = connect_admin(broker.addr, "admin connects").await;
4171        (admin, broker, calls)
4172    }
4173
4174    fn successful_group_offset_outcome() -> Vec<ConsumerGroupOffsetOutcome> {
4175        vec![ConsumerGroupOffsetOutcome {
4176            topic: "orders".into(),
4177            partition: 2,
4178            error: None,
4179        }]
4180    }
4181
4182    macro_rules! successful_offset_topic {
4183        ($topic:ident, $partition:ident, $name_field:ident) => {
4184            $topic {
4185                $name_field: "orders".into(),
4186                partitions: vec![$partition {
4187                    partition_index: 2,
4188                    error_code: 0,
4189                    ..Default::default()
4190                }],
4191                ..Default::default()
4192            }
4193        };
4194    }
4195
4196    fn successful_offset_delete_topic() -> OffsetDeleteResponseTopic {
4197        successful_offset_topic!(
4198            OffsetDeleteResponseTopic,
4199            OffsetDeleteResponsePartition,
4200            name
4201        )
4202    }
4203
4204    fn offset_delete_response(error_code: i16, version: i16) -> Vec<u8> {
4205        let topics =
4206            successful_response_rows(error_code, || vec![successful_offset_delete_topic()]);
4207        encode(
4208            &OffsetDeleteResponse {
4209                error_code,
4210                topics,
4211                ..Default::default()
4212            },
4213            version,
4214            false,
4215        )
4216    }
4217
4218    fn offset_commit_success_response(version: i16) -> Vec<u8> {
4219        encode(
4220            &OffsetCommitResponse {
4221                topics: vec![successful_offset_commit_topic()],
4222                ..Default::default()
4223            },
4224            version,
4225            true,
4226        )
4227    }
4228
4229    fn delete_share_offsets_response(error_code: i16, version: i16) -> Vec<u8> {
4230        let responses = successful_response_rows(error_code, || {
4231            vec![DeleteShareGroupOffsetsResponseTopic {
4232                topic_name: "orders".into(),
4233                error_code: 0,
4234                ..Default::default()
4235            }]
4236        });
4237        crate::encode_test_struct_fields!(
4238            encode,
4239            DeleteShareGroupOffsetsResponse,
4240            { error_code: error_code, responses: responses },
4241            version,
4242            true,
4243        )
4244    }
4245
4246    fn successful_offset_commit_topic() -> OffsetCommitResponseTopic {
4247        successful_offset_topic!(
4248            OffsetCommitResponseTopic,
4249            OffsetCommitResponsePartition,
4250            name
4251        )
4252    }
4253
4254    fn successful_share_offset_topic() -> AlterShareGroupOffsetsResponseTopic {
4255        successful_offset_topic!(
4256            AlterShareGroupOffsetsResponseTopic,
4257            AlterShareGroupOffsetsResponsePartition,
4258            topic_name
4259        )
4260    }
4261
4262    fn successful_response_rows<T>(error_code: i16, rows: impl FnOnce() -> Vec<T>) -> Vec<T> {
4263        if error_code == 0 { rows() } else { Vec::new() }
4264    }
4265
4266    fn assert_single_group_offset_success(
4267        result: Result<Vec<ConsumerGroupOffsetOutcome>, String>,
4268        calls: usize,
4269    ) {
4270        assert!((result, calls) == (Ok(successful_group_offset_outcome()), 1));
4271    }
4272
4273    struct CoordinatorRetryCase<T> {
4274        name: &'static str,
4275        codes: Vec<i16>,
4276        expected_calls: usize,
4277        expected: Result<T, i16>,
4278    }
4279
4280    #[bon::bon]
4281    impl<T> CoordinatorRetryCase<T> {
4282        #[builder]
4283        fn success(name: &'static str, codes: Vec<i16>, expected_calls: usize, outcome: T) -> Self {
4284            Self {
4285                name,
4286                codes,
4287                expected_calls,
4288                expected: Ok(outcome),
4289            }
4290        }
4291
4292        fn error(name: &'static str, code: i16) -> Self {
4293            Self {
4294                name,
4295                codes: vec![code],
4296                expected_calls: 1,
4297                expected: Err(code),
4298            }
4299        }
4300    }
4301
4302    impl<T: Clone> CoordinatorRetryCase<T> {
4303        fn standard(outcome: &T) -> Vec<Self> {
4304            vec![
4305                Self::success()
4306                    .name("no error")
4307                    .codes(vec![0])
4308                    .expected_calls(1)
4309                    .outcome(outcome.clone())
4310                    .call(),
4311                Self::success()
4312                    .name("coordinator load in progress retries on the same coordinator")
4313                    .codes(vec![14, 0])
4314                    .expected_calls(2)
4315                    .outcome(outcome.clone())
4316                    .call(),
4317                Self::success()
4318                    .name("not coordinator finds the coordinator again")
4319                    .codes(vec![16, 0])
4320                    .expected_calls(2)
4321                    .outcome(outcome.clone())
4322                    .call(),
4323            ]
4324        }
4325
4326        fn with_rebalance_in_progress(outcome: &T) -> Vec<Self> {
4327            let mut cases = Self::standard(outcome);
4328            cases.push(
4329                Self::success()
4330                    .name("rebalance in progress retries on the same coordinator")
4331                    .codes(vec![27, 0])
4332                    .expected_calls(2)
4333                    .outcome(outcome.clone())
4334                    .call(),
4335            );
4336            cases
4337        }
4338
4339        fn with_authorization_failure(mut cases: Vec<Self>) -> Vec<Self> {
4340            cases.push(Self::error("group authorization failed is final", 30));
4341            cases
4342        }
4343    }
4344
4345    #[bon::builder]
4346    async fn run_coordinator_retry_cases<T, F, Fut, Resp>(
4347        api_key: i16,
4348        respond: Resp,
4349        cases: Vec<CoordinatorRetryCase<T>>,
4350        action: F,
4351    ) where
4352        T: PartialEq + std::fmt::Debug,
4353        Resp: Fn(i16, i16) -> Vec<u8> + Copy + Send + Sync + 'static,
4354        F: Fn(AdminClient) -> Fut,
4355        Fut: std::future::Future<Output = Result<T, AdminError>>,
4356    {
4357        for case in cases {
4358            let (admin, broker, calls) =
4359                connect_self_coordinating(api_key, case.codes, respond).await;
4360            let result = action(admin).await.map_err(|error| match error {
4361                AdminError::Broker { code, .. } => code,
4362                other => panic!("case {}: unexpected error {other:?}", case.name),
4363            });
4364            broker.stop();
4365            assert!(
4366                (result, calls.load(Ordering::SeqCst)) == (case.expected, case.expected_calls),
4367                "case {}",
4368                case.name
4369            );
4370        }
4371    }
4372
4373    macro_rules! run_retry_cases {
4374        ($api_key:expr, $respond:expr, $cases:expr, $action:expr) => {
4375            run_coordinator_retry_cases()
4376                .api_key($api_key)
4377                .respond($respond)
4378                .cases($cases)
4379                .action($action)
4380                .call()
4381                .await
4382        };
4383    }
4384
4385    async fn delete_consumer_offsets(
4386        mut admin: AdminClient,
4387    ) -> Result<Vec<ConsumerGroupOffsetOutcome>, AdminError> {
4388        admin
4389            .delete_consumer_group_offsets("workers", &[("orders".into(), 2)])
4390            .await
4391    }
4392
4393    async fn alter_share_offsets(
4394        mut admin: AdminClient,
4395    ) -> Result<Vec<ConsumerGroupOffsetOutcome>, AdminError> {
4396        admin
4397            .alter_share_group_offsets("workers", &test_group_offsets())
4398            .await
4399    }
4400
4401    async fn delete_share_offsets(
4402        mut admin: AdminClient,
4403    ) -> Result<Vec<(String, Option<KafkaError>)>, AdminError> {
4404        admin
4405            .delete_share_group_offsets("workers", &["orders".to_string()])
4406            .await
4407    }
4408
4409    /// Apache Kafka's `DeleteConsumerGroupOffsetsHandler.handleResponse`
4410    /// reads the top-level `error_code` as a group-level failure, retried the
4411    /// same way as `DeleteGroups`, and otherwise returns one outcome per
4412    /// requested partition.
4413    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
4414    async fn delete_consumer_group_offsets_retries_coordinator_errors_as_kafka_does() {
4415        let respond = offset_delete_response;
4416        let outcome = successful_group_offset_outcome();
4417        let mut cases = CoordinatorRetryCase::standard(&outcome);
4418        cases.push(CoordinatorRetryCase::error("non-empty group is final", 68));
4419        run_retry_cases!(
4420            OffsetDeleteRequest::API_KEY,
4421            respond,
4422            cases,
4423            delete_consumer_offsets
4424        );
4425    }
4426
4427    #[derive(Clone, Copy)]
4428    enum StreamsGroupOffsetCall {
4429        Delete,
4430        Alter,
4431    }
4432
4433    impl StreamsGroupOffsetCall {
4434        fn api_key(self) -> i16 {
4435            match self {
4436                Self::Delete => OffsetDeleteRequest::API_KEY,
4437                Self::Alter => offset_commit_request::API_KEY,
4438            }
4439        }
4440
4441        fn response(self, version: i16) -> Vec<u8> {
4442            match self {
4443                Self::Delete => offset_delete_response(0, version),
4444                Self::Alter => offset_commit_success_response(version),
4445            }
4446        }
4447
4448        async fn call(
4449            self,
4450            admin: &mut AdminClient,
4451        ) -> Result<Vec<ConsumerGroupOffsetOutcome>, AdminError> {
4452            match self {
4453                Self::Delete => {
4454                    admin
4455                        .delete_streams_group_offsets("workers", &[("orders".into(), 2)])
4456                        .await
4457                }
4458                Self::Alter => {
4459                    admin
4460                        .alter_streams_group_offsets("workers", &test_group_offsets())
4461                        .await
4462                }
4463            }
4464        }
4465    }
4466
4467    /// Kafka's Streams group offset APIs delegate to their Consumer group
4468    /// counterparts.
4469    #[yare::parameterized(
4470        delete = { StreamsGroupOffsetCall::Delete },
4471        alter = { StreamsGroupOffsetCall::Alter },
4472    )]
4473    #[test_macro(tokio::test(flavor = "multi_thread", worker_threads = 2))]
4474    async fn streams_group_offset_calls_delegate_to_consumer_calls(call: StreamsGroupOffsetCall) {
4475        let respond = move |_code: i16, version: i16| call.response(version);
4476        let (mut admin, broker, calls) =
4477            connect_self_coordinating(call.api_key(), vec![0], respond).await;
4478        let result = call
4479            .call(&mut admin)
4480            .await
4481            .map_err(|error| error.to_string());
4482        broker.stop();
4483        assert_single_group_offset_success(result, calls.load(Ordering::SeqCst));
4484    }
4485
4486    /// Apache Kafka's `RemoveMembersFromConsumerGroupHandler.handleResponse`
4487    /// reads the top-level `error_code` as a group-level failure, retried the
4488    /// same way as `DeleteGroups`, and otherwise maps each answered member to
4489    /// its error. `removeMembersFromConsumerGroup` names each member by its
4490    /// instance id with `UNKNOWN_MEMBER_ID` and sends the default reason.
4491    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
4492    async fn remove_members_from_consumer_group_retries_coordinator_errors_as_kafka_does() {
4493        let answered = |instance_id: &str, error_code| MemberResponse {
4494            member_id: String::new(),
4495            group_instance_id: Some(instance_id.into()),
4496            error_code,
4497            ..Default::default()
4498        };
4499        let respond = move |code: i16, version: i16| {
4500            let members = successful_response_rows(code, || {
4501                // The coordinator answers in its own order.
4502                vec![answered("b", 25), answered("a", 0)]
4503            });
4504            let response = LeaveGroupResponse {
4505                error_code: code,
4506                members,
4507                ..Default::default()
4508            };
4509            encode_flexible(&response, version)
4510        };
4511        let removed = |instance_id: &str| RemovedMember {
4512            member_id: String::new(),
4513            group_instance_id: Some(instance_id.into()),
4514        };
4515        let outcome = RemovedMembers {
4516            members: maplit::btreemap! {
4517                removed("a") => None,
4518                removed("b") => Some(KafkaError {
4519                code: 25,
4520                name: "UNKNOWN_MEMBER_ID",
4521                message: None,
4522            }),
4523            },
4524        };
4525        let cases = CoordinatorRetryCase::with_authorization_failure(
4526            CoordinatorRetryCase::standard(&outcome),
4527        );
4528        run_retry_cases!(
4529            LeaveGroupRequest::API_KEY,
4530            respond,
4531            cases,
4532            |mut admin| async move {
4533                admin
4534                    .remove_members_from_consumer_group(
4535                        "workers",
4536                        &[
4537                            MemberToRemove {
4538                                group_instance_id: "a".into(),
4539                            },
4540                            MemberToRemove {
4541                                group_instance_id: "b".into(),
4542                            },
4543                        ],
4544                        &RemoveMembersOptions::builder().build(),
4545                    )
4546                    .await
4547            }
4548        );
4549    }
4550
4551    /// Kafka's `removeMembersFromConsumerGroup` sends each member with
4552    /// `UNKNOWN_MEMBER_ID`, its instance id and the reason, which is the
4553    /// default for `None` or empty and is cut to 255 characters.
4554    #[yare::parameterized(
4555        none = { None, DEFAULT_LEAVE_GROUP_REASON.to_owned() },
4556        empty = { Some(String::new()), DEFAULT_LEAVE_GROUP_REASON.to_owned() },
4557        short = { Some("rolling restart".to_owned()), "rolling restart".to_owned() },
4558        long = { Some("r".repeat(300)), "r".repeat(255) },
4559    )]
4560    fn leave_group_reason_matches_kafka(reason: Option<String>, expected: String) {
4561        assert2::assert!(leave_group_reason(reason.as_deref()) == expected);
4562    }
4563
4564    /// `RemoveMembersFromConsumerGroupResult.memberResult` looks a member up
4565    /// by `UNKNOWN_MEMBER_ID` and its instance id, and gives an error for a
4566    /// member that the answer does not name.
4567    #[test]
4568    fn removed_members_member_result_matches_kafka() {
4569        let member = |instance_id: &str| MemberToRemove {
4570            group_instance_id: instance_id.into(),
4571        };
4572        let fenced = KafkaError {
4573            code: 82,
4574            name: kafka_error_name(82),
4575            message: None,
4576        };
4577        let result = RemovedMembers {
4578            members: maplit::btreemap! {
4579                RemovedMember {
4580                member_id: String::new(),
4581                group_instance_id: Some("a".into()),
4582            } => None,
4583                RemovedMember {
4584                member_id: String::new(),
4585                group_instance_id: Some("b".into()),
4586            } => Some(fenced.clone()),
4587            },
4588        };
4589        assert!(
4590            [member("a"), member("b"), member("c")].map(|member| result.member_result(&member))
4591                == [
4592                    Ok(()),
4593                    Err(fenced),
4594                    Err(KafkaError {
4595                        code: -1,
4596                        name: "UNKNOWN_SERVER_ERROR",
4597                        message: Some(
4598                            "Member \"c\" was not included in the removal response".into()
4599                        ),
4600                    }),
4601                ]
4602        );
4603    }
4604
4605    /// Apache Kafka's `AlterShareGroupOffsetsHandler.handleResponse` retries
4606    /// a top-level 14 or 27 on the same coordinator, finds the coordinator
4607    /// again on 15 or 16, and puts every other top-level code on each
4608    /// requested partition.
4609    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
4610    async fn alter_share_group_offsets_retries_coordinator_errors_as_kafka_does() {
4611        let respond = |code: i16, version: i16| {
4612            let responses =
4613                successful_response_rows(code, || vec![successful_share_offset_topic()]);
4614            let response = AlterShareGroupOffsetsResponse {
4615                error_code: code,
4616                responses,
4617                ..Default::default()
4618            };
4619            encode_flexible(&response, version)
4620        };
4621        let outcome = successful_group_offset_outcome();
4622        let mut cases = CoordinatorRetryCase::with_rebalance_in_progress(&outcome);
4623        cases.push(
4624            CoordinatorRetryCase::success()
4625                .name("a final group error is the error of each partition")
4626                .codes(vec![68])
4627                .expected_calls(1)
4628                .outcome(vec![ConsumerGroupOffsetOutcome {
4629                    topic: "orders".into(),
4630                    partition: 2,
4631                    error: Some(KafkaError {
4632                        code: 68,
4633                        name: "NON_EMPTY_GROUP",
4634                        message: None,
4635                    }),
4636                }])
4637                .call(),
4638        );
4639        run_retry_cases!(
4640            AlterShareGroupOffsetsRequest::API_KEY,
4641            respond,
4642            cases,
4643            alter_share_offsets
4644        );
4645    }
4646
4647    /// Apache Kafka's `KafkaAdminClient.listStreamsGroupOffsets` delegates to
4648    /// `listConsumerGroupOffsets` verbatim.
4649    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
4650    async fn list_streams_group_offsets_delegates_to_offset_fetch() {
4651        let respond =
4652            |_code: i16, version: i16| encode(&offset_fetch_response(version), version, true);
4653        let (mut admin, broker, calls) =
4654            connect_self_coordinating(offset_fetch_request::API_KEY, vec![0], respond).await;
4655        let result = admin
4656            .list_streams_group_offsets("workers")
4657            .await
4658            .map_err(|error| error.to_string());
4659        broker.stop();
4660        assert!((result, calls.load(Ordering::SeqCst)) == (Ok(test_group_offsets()), 1));
4661    }
4662
4663    /// Apache Kafka's `DeleteShareGroupOffsetsHandler.handleResponse` reads
4664    /// the top-level `error_code` as a group-level failure, retried the same
4665    /// way as `DeleteGroups`, and otherwise returns one `(topic, error)` pair
4666    /// per requested topic.
4667    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
4668    async fn delete_share_group_offsets_retries_coordinator_errors_as_kafka_does() {
4669        let respond = delete_share_offsets_response;
4670        let outcome = vec![("orders".to_string(), None)];
4671        let cases = CoordinatorRetryCase::with_authorization_failure(
4672            CoordinatorRetryCase::with_rebalance_in_progress(&outcome),
4673        );
4674        run_retry_cases!(
4675            DeleteShareGroupOffsetsRequest::API_KEY,
4676            respond,
4677            cases,
4678            delete_share_offsets
4679        );
4680    }
4681}