1use std::collections::{BTreeMap, BTreeSet};
35
36use krabka_client_core::{
37 AllTopicsOffsetFetch, AuthenticationError, ClientError, Connection, ConnectionOptions,
38 CoordinatorKeyType, MembersOnlyLeaveGroup, SaslAuthenticationError, TopicNameOffsetCommit,
39 build_find_coordinator, coordinator_endpoint,
40};
41use krabka_protocol::owned::{
42 alter_share_group_offsets_request::{
43 AlterShareGroupOffsetsRequest, AlterShareGroupOffsetsRequestPartition,
44 AlterShareGroupOffsetsRequestTopic,
45 },
46 alter_share_group_offsets_response::AlterShareGroupOffsetsResponse,
47 delete_share_group_offsets_request::{
48 DeleteShareGroupOffsetsRequest, DeleteShareGroupOffsetsRequestTopic,
49 },
50 delete_share_group_offsets_response::DeleteShareGroupOffsetsResponse,
51 leave_group_request::{LeaveGroupRequest, MemberIdentity},
52 leave_group_response::LeaveGroupResponse,
53 list_groups_request::{self, ListGroupsRequest},
54 list_groups_response::ListedGroup,
55 metadata_request::MetadataRequest,
56 metadata_response::MetadataResponseBroker,
57 offset_commit_request::{
58 OffsetCommitRequest, OffsetCommitRequestPartition, OffsetCommitRequestTopic,
59 },
60 offset_commit_response::{
61 OffsetCommitResponse, OffsetCommitResponsePartition, OffsetCommitResponseTopic,
62 },
63 offset_delete_request::{
64 OffsetDeleteRequest, OffsetDeleteRequestPartition, OffsetDeleteRequestTopic,
65 },
66 offset_delete_response::{
67 OffsetDeleteResponse, OffsetDeleteResponsePartition, OffsetDeleteResponseTopic,
68 },
69 offset_fetch_request::{OffsetFetchRequest, OffsetFetchRequestGroup},
70 offset_fetch_response::{
71 OffsetFetchResponse, OffsetFetchResponsePartition, OffsetFetchResponsePartitions,
72 OffsetFetchResponseTopic, OffsetFetchResponseTopics,
73 },
74};
75
76use crate::{
77 AdminClient, AdminError, KafkaError, format_host_port,
78 group_descriptions::DescribeGroupsOptions,
79 kafka_error_if, kafka_error_name,
80 retry::{CoordinatorRetry, RetryAction, RetryPolicy, connection_failure_action},
81 send_connection_at_least,
82};
83
84macro_rules! group_offset_attempt {
85 ($this:ident, $group:expr, $find_coordinator:expr, $request:expr, $handle:expr) => {{
86 if $find_coordinator && let Err(action) = $this.find_group_coordinator_attempt($group).await
87 {
88 return action;
89 }
90 match $this.conn.send($request).await {
91 Ok(response) => $handle($group, response),
92 Err(error) => connection_failure_action(error),
93 }
94 }};
95}
96
97#[derive(Debug, Clone, PartialEq, Eq)]
98pub struct ConsumerGroupOffsetOutcome {
99 pub topic: String,
100 pub partition: i32,
101 pub error: Option<KafkaError>,
102}
103
104struct GroupOffsetResponseTopic<P> {
105 name: String,
106 partitions: Vec<P>,
107}
108
109trait IntoGroupOffsetResponseTopic {
110 type Partition: GroupOffsetResponsePartition;
111
112 fn into_group_offset_topic(self) -> GroupOffsetResponseTopic<Self::Partition>;
113}
114
115trait GroupOffsetResponsePartition {
116 fn index(&self) -> i32;
117 fn error_code(&self) -> i16;
118}
119
120macro_rules! impl_group_offset_response {
121 ($topic:ty, $partition:ty) => {
122 impl IntoGroupOffsetResponseTopic for $topic {
123 type Partition = $partition;
124
125 fn into_group_offset_topic(self) -> GroupOffsetResponseTopic<Self::Partition> {
126 GroupOffsetResponseTopic {
127 name: self.name,
128 partitions: self.partitions,
129 }
130 }
131 }
132
133 impl GroupOffsetResponsePartition for $partition {
134 fn index(&self) -> i32 {
135 self.partition_index
136 }
137 fn error_code(&self) -> i16 {
138 self.error_code
139 }
140 }
141 };
142}
143
144impl_group_offset_response!(OffsetCommitResponseTopic, OffsetCommitResponsePartition);
145impl_group_offset_response!(OffsetDeleteResponseTopic, OffsetDeleteResponsePartition);
146
147fn group_offset_outcomes<T: IntoGroupOffsetResponseTopic>(
148 topics: impl IntoIterator<Item = T>,
149) -> Vec<ConsumerGroupOffsetOutcome> {
150 topics
151 .into_iter()
152 .flat_map(|topic| {
153 let GroupOffsetResponseTopic { name, partitions } = topic.into_group_offset_topic();
154 partitions
155 .into_iter()
156 .map(move |partition| ConsumerGroupOffsetOutcome {
157 topic: name.clone(),
158 partition: partition.index(),
159 error: kafka_error_if(partition.error_code(), None),
160 })
161 })
162 .collect()
163}
164
165pub type GroupOffsets = BTreeMap<(String, i32), i64>;
167
168#[derive(Clone, Copy)]
169enum GroupOffsetKind {
170 Consumer,
171 Share,
172}
173
174#[derive(Clone, Copy)]
175struct GroupOffsetUpdate<'a> {
176 kind: GroupOffsetKind,
177 group: &'a str,
178 offsets: &'a GroupOffsets,
179}
180
181impl<'a> GroupOffsetUpdate<'a> {
182 fn consumer(group: &'a str, offsets: &'a GroupOffsets) -> Self {
183 Self {
184 kind: GroupOffsetKind::Consumer,
185 group,
186 offsets,
187 }
188 }
189
190 fn share(group: &'a str, offsets: &'a GroupOffsets) -> Self {
191 Self {
192 kind: GroupOffsetKind::Share,
193 group,
194 offsets,
195 }
196 }
197}
198
199struct GroupOffsetAttempt<'a> {
200 update: GroupOffsetUpdate<'a>,
201 find_coordinator: bool,
202}
203
204#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
208pub enum GroupState {
209 Unknown,
211 PreparingRebalance,
212 CompletingRebalance,
213 Stable,
214 Dead,
215 Empty,
216 Assigning,
217 Reconciling,
218 NotReady,
219}
220
221impl GroupState {
222 const ALL: [Self; 9] = [
223 Self::Unknown,
224 Self::PreparingRebalance,
225 Self::CompletingRebalance,
226 Self::Stable,
227 Self::Dead,
228 Self::Empty,
229 Self::Assigning,
230 Self::Reconciling,
231 Self::NotReady,
232 ];
233
234 #[must_use]
236 pub const fn as_str(self) -> &'static str {
237 match self {
238 Self::Unknown => "Unknown",
239 Self::PreparingRebalance => "PreparingRebalance",
240 Self::CompletingRebalance => "CompletingRebalance",
241 Self::Stable => "Stable",
242 Self::Dead => "Dead",
243 Self::Empty => "Empty",
244 Self::Assigning => "Assigning",
245 Self::Reconciling => "Reconciling",
246 Self::NotReady => "NotReady",
247 }
248 }
249}
250
251macro_rules! impl_group_name_parse {
252 ($type:ty, $($doc:literal),+ $(,)?) => {
253 impl $type {
254 $(#[doc = $doc])+
255 #[must_use]
256 pub fn parse(name: &str) -> Self {
257 parse_group_name(Self::ALL, name, Self::as_str, Self::Unknown)
258 }
259 }
260 };
261}
262impl_group_name_parse!(
263 GroupState,
264 "Reads a state name without regard to case. A name that the client does not know gives [`GroupState::Unknown`], as Kafka's `GroupState.parse` does."
265);
266
267macro_rules! impl_display_from_as_str {
268 ($type:ty) => {
269 impl std::fmt::Display for $type {
270 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
271 f.write_str(self.as_str())
272 }
273 }
274 };
275}
276impl_display_from_as_str!(GroupState);
277
278#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
282pub enum GroupType {
283 Unknown,
285 Consumer,
286 Classic,
287 Share,
288 Streams,
289}
290
291impl GroupType {
292 const ALL: [Self; 5] = [
293 Self::Unknown,
294 Self::Consumer,
295 Self::Classic,
296 Self::Share,
297 Self::Streams,
298 ];
299
300 #[must_use]
302 pub const fn as_str(self) -> &'static str {
303 match self {
304 Self::Unknown => "Unknown",
305 Self::Consumer => "Consumer",
306 Self::Classic => "Classic",
307 Self::Share => "Share",
308 Self::Streams => "Streams",
309 }
310 }
311}
312impl_group_name_parse!(
313 GroupType,
314 "Reads a type name without regard to case. A name that the client does not know gives [`GroupType::Unknown`], as Kafka's `GroupType.parse` does."
315);
316
317fn parse_group_name<T: Copy>(
318 values: impl IntoIterator<Item = T>,
319 name: &str,
320 as_str: fn(T) -> &'static str,
321 unknown: T,
322) -> T {
323 values
324 .into_iter()
325 .find(|value| as_str(*value).eq_ignore_ascii_case(name))
326 .unwrap_or(unknown)
327}
328
329impl_display_from_as_str!(GroupType);
330
331#[derive(Debug, Clone, Default, PartialEq, Eq, bon::Builder)]
334pub struct ListGroupsOptions {
335 #[builder(default)]
337 pub group_states: BTreeSet<GroupState>,
338 #[builder(default)]
340 pub types: BTreeSet<GroupType>,
341 #[builder(default)]
343 pub protocol_types: BTreeSet<String>,
344}
345
346impl ListGroupsOptions {
347 #[must_use]
350 pub fn for_consumer_groups() -> Self {
351 Self::builder()
352 .types(maplit::btreeset! {
353 GroupType::Classic,
354 GroupType::Consumer,
355 })
356 .protocol_types(maplit::btreeset! {
357 String::new(),
358 "consumer".to_owned(),
359 })
360 .build()
361 }
362
363 #[must_use]
365 pub fn for_share_groups() -> Self {
366 Self::builder()
367 .types(maplit::btreeset! {
368 GroupType::Share,
369 })
370 .build()
371 }
372
373 #[must_use]
376 pub fn for_streams_groups() -> Self {
377 Self::builder()
378 .types(maplit::btreeset! {
379 GroupType::Streams,
380 })
381 .build()
382 }
383}
384
385#[derive(Debug, Clone, PartialEq, Eq)]
388pub struct GroupListing {
389 pub group_id: String,
390 pub group_type: Option<GroupType>,
392 pub protocol_type: String,
395 pub group_state: Option<GroupState>,
397}
398
399impl GroupListing {
400 #[must_use]
410 pub fn is_simple_consumer_group(&self) -> bool {
411 self.group_type == Some(GroupType::Classic) && self.protocol_type.is_empty()
412 }
413}
414
415impl From<ListedGroup> for GroupListing {
416 fn from(group: ListedGroup) -> Self {
417 Self {
418 group_type: (!group.group_type.is_empty()).then(|| GroupType::parse(&group.group_type)),
419 group_state: (!group.group_state.is_empty())
420 .then(|| GroupState::parse(&group.group_state)),
421 group_id: group.group_id,
422 protocol_type: group.protocol_type,
423 }
424 }
425}
426
427#[derive(Debug, Clone, PartialEq, Eq)]
433pub struct ListGroupsError {
434 pub node_id: i32,
435 pub host: String,
436 pub port: i32,
437 pub error: KafkaError,
438}
439
440#[derive(Debug, Clone, PartialEq, Eq)]
443pub struct ListGroupsResult {
444 pub valid: Vec<GroupListing>,
447 pub errors: Vec<ListGroupsError>,
449}
450
451impl ListGroupsResult {
452 pub fn all(self) -> Result<Vec<GroupListing>, ListGroupsError> {
458 match self.errors.into_iter().next() {
459 Some(error) => Err(error),
460 None => Ok(self.valid),
461 }
462 }
463}
464
465macro_rules! group_offsets_method_body {
466 (delete_consumer, $admin:ident, $group:ident, $input:ident) => {
467 $admin
468 .delete_consumer_group_offsets_with_retry($group, $input, $admin.retry)
469 .await
470 };
471 (delete_streams, $admin:ident, $group:ident, $input:ident) => {
472 $admin.delete_consumer_group_offsets($group, $input).await
473 };
474 (alter_consumer, $admin:ident, $group:ident, $input:ident) => {
475 $admin
476 .alter_group_offsets(GroupOffsetUpdate::consumer($group, $input))
477 .await
478 };
479 (alter_share, $admin:ident, $group:ident, $input:ident) => {
480 $admin
481 .alter_group_offsets(GroupOffsetUpdate::share($group, $input))
482 .await
483 };
484}
485
486macro_rules! group_offsets_method {
487 ($(#[$meta:meta])* $name:ident($input:ident: $input_type:ty) => $kind:ident) => {
488 $(#[$meta])*
489 pub async fn $name(
490 &mut self,
491 group: &str,
492 $input: &$input_type,
493 ) -> Result<Vec<ConsumerGroupOffsetOutcome>, AdminError> {
494 group_offsets_method_body!($kind, self, group, $input)
495 }
496 };
497}
498
499macro_rules! retry_group_call {
500 ($retry_policy:expr, |$find:ident| $attempt:expr) => {{
501 let mut retry = CoordinatorRetry::new($retry_policy);
502 loop {
503 let $find = retry.find_coordinator();
504 let action = retry.run($attempt).await;
505 if let Some(result) = retry.next(action).await {
506 return result;
507 }
508 }
509 }};
510}
511
512impl AdminClient {
513 group_offsets_method! {
514 alter_consumer_group_offsets(offsets: GroupOffsets) => alter_consumer
549 }
550
551 async fn alter_group_offsets(
552 &mut self,
553 update: GroupOffsetUpdate<'_>,
554 ) -> Result<Vec<ConsumerGroupOffsetOutcome>, AdminError> {
555 self.alter_group_offsets_with_retry(update, self.retry)
556 .await
557 }
558
559 async fn alter_group_offsets_with_retry(
560 &mut self,
561 update: GroupOffsetUpdate<'_>,
562 retry: RetryPolicy,
563 ) -> Result<Vec<ConsumerGroupOffsetOutcome>, AdminError> {
564 retry_group_call!(retry, |find_coordinator| {
565 self.alter_group_offsets_attempt(GroupOffsetAttempt {
566 update,
567 find_coordinator,
568 })
569 })
570 }
571
572 async fn alter_group_offsets_attempt(
574 &mut self,
575 attempt: GroupOffsetAttempt<'_>,
576 ) -> RetryAction<Vec<ConsumerGroupOffsetOutcome>> {
577 let GroupOffsetAttempt {
578 update,
579 find_coordinator,
580 } = attempt;
581 if find_coordinator
582 && let Err(action) = self.find_group_coordinator_attempt(update.group).await
583 {
584 return action;
585 }
586 match update.kind {
587 GroupOffsetKind::Consumer => self
588 .conn
589 .send(offset_commit_request(update.group, update.offsets))
590 .await
591 .map_or_else(connection_failure_action, offset_commit_retry_action),
592 GroupOffsetKind::Share => self
593 .conn
594 .send(alter_share_group_offsets_request(
595 update.group,
596 update.offsets,
597 ))
598 .await
599 .map_or_else(connection_failure_action, |response| {
600 alter_share_group_offsets_retry_action(update.offsets, response)
601 }),
602 }
603 }
604
605 pub async fn list_groups(
640 &self,
641 options: &ListGroupsOptions,
642 ) -> Result<ListGroupsResult, AdminError> {
643 self.list_groups_with_retry(options, self.retry).await
644 }
645
646 async fn list_groups_with_retry(
647 &self,
648 options: &ListGroupsOptions,
649 retry: RetryPolicy,
650 ) -> Result<ListGroupsResult, AdminError> {
651 let start = tokio::time::Instant::now();
652 let brokers = self.list_groups_brokers(start, retry).await?;
653 let request = ListGroupsRequest {
654 states_filter: options
655 .group_states
656 .iter()
657 .map(|state| state.as_str().to_owned())
658 .collect(),
659 types_filter: options
660 .types
661 .iter()
662 .map(|group_type| group_type.as_str().to_owned())
663 .collect(),
664 ..Default::default()
665 };
666 let min_version = list_groups_min_version(options);
667 let answers = futures_util::future::join_all(brokers.iter().map(|broker| {
668 list_groups_on_broker(
669 broker,
670 self.options.clone(),
671 request.clone(),
672 min_version,
673 start,
674 retry,
675 )
676 }))
677 .await;
678
679 let mut listings = BTreeMap::new();
680 let mut errors = Vec::new();
681 for (broker, answer) in brokers.into_iter().zip(answers) {
682 match answer {
683 Ok(groups) => {
684 for group in groups {
685 if options.protocol_types.is_empty()
686 || options.protocol_types.contains(&group.protocol_type)
687 {
688 let listing = GroupListing::from(group);
689 listings.insert(listing.group_id.clone(), listing);
690 }
691 }
692 }
693 Err(error) => errors.push(ListGroupsError {
694 node_id: broker.node_id,
695 host: broker.host,
696 port: broker.port,
697 error,
698 }),
699 }
700 }
701 Ok(ListGroupsResult {
702 valid: listings.into_values().collect(),
703 errors,
704 })
705 }
706
707 async fn list_groups_brokers(
711 &self,
712 start: tokio::time::Instant,
713 retry: RetryPolicy,
714 ) -> Result<Vec<MetadataResponseBroker>, AdminError> {
715 let mut backoff = retry.initial_backoff;
716 loop {
717 let response = self
718 .conn
719 .send(MetadataRequest {
720 topics: Some(Vec::new()),
721 allow_auto_topic_creation: true,
722 ..Default::default()
723 })
724 .await?;
725 let mut brokers = response.brokers;
726 let mut seen = BTreeSet::new();
727 brokers.retain(|broker| seen.insert(broker.node_id));
728 if !brokers.is_empty() {
729 return Ok(brokers);
730 }
731 if start.elapsed() >= retry.timeout {
732 return Err(AdminError::Protocol(
733 "failed to find brokers to send ListGroups: metadata has no brokers".to_owned(),
734 ));
735 }
736 tokio::time::sleep(backoff).await;
737 backoff = backoff.saturating_mul(2).min(retry.max_backoff);
738 }
739 }
740
741 pub async fn list_consumer_group_offsets(
780 &mut self,
781 group: &str,
782 ) -> Result<GroupOffsets, AdminError> {
783 self.list_consumer_group_offsets_with_retry(group, self.retry)
784 .await
785 }
786
787 async fn list_consumer_group_offsets_with_retry(
788 &mut self,
789 group: &str,
790 retry: RetryPolicy,
791 ) -> Result<GroupOffsets, AdminError> {
792 retry_group_call!(retry, |find| self.offset_fetch_attempt(group, find))
793 }
794
795 async fn offset_fetch_attempt(
799 &mut self,
800 group: &str,
801 find_coordinator: bool,
802 ) -> RetryAction<GroupOffsets> {
803 let request = offset_fetch_request(group);
804 group_offset_attempt!(self, group, find_coordinator, request, |group, response| {
805 offset_fetch_retry_action(committed_offsets(group, response))
806 })
807 }
808
809 async fn find_group_coordinator_attempt<T>(
816 &mut self,
817 group: &str,
818 ) -> Result<(), RetryAction<T>> {
819 match self.reconnect_group_coordinator(group).await {
820 Ok(()) => Ok(()),
821 Err(AdminError::Transport(ClientError::Server { error_code }))
822 if is_retriable_find_coordinator_error(error_code) =>
823 {
824 Err(RetryAction::FindCoordinator(Err(AdminError::Transport(
825 ClientError::Server { error_code },
826 ))))
827 }
828 Err(error) => Err(connection_failure_action(error)),
829 }
830 }
831
832 async fn reconnect_group_coordinator(&mut self, group: &str) -> Result<(), AdminError> {
833 let response = self
834 .conn
835 .send(build_find_coordinator(group, CoordinatorKeyType::Group))
836 .await?;
837 let coordinator = coordinator_endpoint(group, response)?;
838 self.reconnect(&format_host_port(&coordinator.host, coordinator.port))
839 .await
840 }
841
842 group_offsets_method! {
843 delete_consumer_group_offsets(partitions: [(String, i32)]) => delete_consumer
855 }
856
857 group_offsets_method! {
858 delete_streams_group_offsets(partitions: [(String, i32)]) => delete_streams
865 }
866
867 async fn delete_consumer_group_offsets_with_retry(
868 &mut self,
869 group: &str,
870 partitions: &[(String, i32)],
871 retry: RetryPolicy,
872 ) -> Result<Vec<ConsumerGroupOffsetOutcome>, AdminError> {
873 retry_group_call!(retry, |find| self
874 .offset_delete_attempt(group, partitions, find))
875 }
876
877 async fn offset_delete_attempt(
878 &mut self,
879 group: &str,
880 partitions: &[(String, i32)],
881 find_coordinator: bool,
882 ) -> RetryAction<Vec<ConsumerGroupOffsetOutcome>> {
883 let request = offset_delete_request(group, partitions);
884 group_offset_attempt!(self, group, find_coordinator, request, |group, response| {
885 offset_delete_retry_action(group, response)
886 })
887 }
888
889 pub async fn remove_members_from_consumer_group(
921 &mut self,
922 group: &str,
923 members: &[MemberToRemove],
924 options: &RemoveMembersOptions,
925 ) -> Result<RemovedMembers, AdminError> {
926 let reason = leave_group_reason(options.reason.as_deref());
927 let identities = if members.is_empty() {
928 self.all_members_of(group, &reason).await?
929 } else {
930 members
931 .iter()
932 .map(|member| MemberIdentity {
933 member_id: String::new(),
934 group_instance_id: Some(member.group_instance_id.clone()),
935 reason: Some(reason.clone()),
936 ..Default::default()
937 })
938 .collect()
939 };
940 self.remove_members_from_consumer_group_with_retry(group, &identities, self.retry)
941 .await
942 }
943
944 async fn all_members_of(
947 &self,
948 group: &str,
949 reason: &str,
950 ) -> Result<Vec<MemberIdentity>, AdminError> {
951 let description = self
952 .describe_consumer_groups(&[group], DescribeGroupsOptions::builder().build())
953 .await
954 .remove(group)
955 .unwrap_or_else(|| {
956 Err(KafkaError {
957 code: UNKNOWN_SERVER_ERROR,
958 name: kafka_error_name(UNKNOWN_SERVER_ERROR),
959 message: Some(format!("no description for group {group}")),
960 })
961 })
962 .map_err(|error| AdminError::Broker {
963 api: "describeConsumerGroups",
964 code: error.code,
965 name: error.name,
966 message: Some(format!(
967 "Encounter exception when trying to get members from group: {group}{}",
968 error
969 .message
970 .map(|message| format!(": {message}"))
971 .unwrap_or_default()
972 )),
973 })?;
974 Ok(description
975 .members
976 .into_iter()
977 .map(|member| {
978 let (member_id, group_instance_id) = match member.group_instance_id {
979 Some(instance_id) => (String::new(), Some(instance_id)),
980 None => (member.member_id, None),
981 };
982 MemberIdentity {
983 member_id,
984 group_instance_id,
985 reason: Some(reason.to_owned()),
986 ..Default::default()
987 }
988 })
989 .collect())
990 }
991
992 async fn remove_members_from_consumer_group_with_retry(
993 &mut self,
994 group: &str,
995 members: &[MemberIdentity],
996 retry: RetryPolicy,
997 ) -> Result<RemovedMembers, AdminError> {
998 retry_group_call!(retry, |find| self.leave_group_attempt(group, members, find))
999 }
1000
1001 async fn leave_group_attempt(
1002 &mut self,
1003 group: &str,
1004 members: &[MemberIdentity],
1005 find_coordinator: bool,
1006 ) -> RetryAction<RemovedMembers> {
1007 if find_coordinator && let Err(action) = self.find_group_coordinator_attempt(group).await {
1008 return action;
1009 }
1010 let request = MembersOnlyLeaveGroup::new(LeaveGroupRequest {
1011 group_id: group.into(),
1012 members: members.to_vec(),
1013 ..Default::default()
1014 });
1015 match self.conn.send(request).await {
1016 Ok(response) => leave_group_retry_action(response),
1017 Err(error) => connection_failure_action(error),
1018 }
1019 }
1020
1021 group_offsets_method! {
1022 alter_share_group_offsets(offsets: GroupOffsets) => alter_share
1039 }
1040
1041 pub async fn alter_streams_group_offsets(
1048 &mut self,
1049 group: &str,
1050 offsets: &GroupOffsets,
1051 ) -> Result<Vec<ConsumerGroupOffsetOutcome>, AdminError> {
1052 self.alter_consumer_group_offsets(group, offsets).await
1053 }
1054
1055 pub async fn list_streams_group_offsets(
1062 &mut self,
1063 group: &str,
1064 ) -> Result<GroupOffsets, AdminError> {
1065 self.list_consumer_group_offsets(group).await
1066 }
1067
1068 pub async fn delete_share_group_offsets(
1082 &mut self,
1083 group: &str,
1084 topics: &[String],
1085 ) -> Result<Vec<(String, Option<KafkaError>)>, AdminError> {
1086 self.delete_share_group_offsets_with_retry(group, topics, self.retry)
1087 .await
1088 }
1089
1090 async fn delete_share_group_offsets_with_retry(
1091 &mut self,
1092 group: &str,
1093 topics: &[String],
1094 retry: RetryPolicy,
1095 ) -> Result<Vec<(String, Option<KafkaError>)>, AdminError> {
1096 retry_group_call!(retry, |find| {
1097 self.delete_share_group_offsets_attempt(group, topics, find)
1098 })
1099 }
1100
1101 async fn delete_share_group_offsets_attempt(
1102 &mut self,
1103 group: &str,
1104 topics: &[String],
1105 find_coordinator: bool,
1106 ) -> RetryAction<Vec<(String, Option<KafkaError>)>> {
1107 if find_coordinator && let Err(action) = self.find_group_coordinator_attempt(group).await {
1108 return action;
1109 }
1110 let request = DeleteShareGroupOffsetsRequest {
1111 group_id: group.into(),
1112 topics: topics
1113 .iter()
1114 .map(|topic_name| delete_share_offset_topic(topic_name))
1115 .collect(),
1116 ..Default::default()
1117 };
1118 match self.conn.send(request).await {
1119 Ok(response) => delete_share_group_offsets_retry_action(response),
1120 Err(error) => connection_failure_action(error),
1121 }
1122 }
1123}
1124
1125fn delete_share_offset_topic(topic_name: &str) -> DeleteShareGroupOffsetsRequestTopic {
1126 DeleteShareGroupOffsetsRequestTopic {
1127 topic_name: topic_name.into(),
1128 ..Default::default()
1129 }
1130}
1131
1132const COORDINATOR_LOAD_IN_PROGRESS: i16 = 14;
1134const COORDINATOR_NOT_AVAILABLE: i16 = 15;
1136const NOT_COORDINATOR: i16 = 16;
1138
1139fn is_retriable_find_coordinator_error(code: i16) -> bool {
1142 matches!(
1143 code,
1144 COORDINATOR_LOAD_IN_PROGRESS | COORDINATOR_NOT_AVAILABLE
1145 )
1146}
1147
1148fn offset_fetch_retry_action<T>(result: Result<T, AdminError>) -> RetryAction<T> {
1153 match result {
1154 Err(AdminError::Broker {
1155 code: COORDINATOR_LOAD_IN_PROGRESS,
1156 ..
1157 }) => RetryAction::SameCoordinator(result),
1158 Err(AdminError::Broker {
1159 code: COORDINATOR_NOT_AVAILABLE | NOT_COORDINATOR,
1160 ..
1161 }) => RetryAction::FindCoordinator(result),
1162 result => RetryAction::Done(result),
1163 }
1164}
1165
1166const REBALANCE_IN_PROGRESS: i16 = 27;
1168
1169fn offset_commit_retry_action(
1176 response: OffsetCommitResponse,
1177) -> RetryAction<Vec<ConsumerGroupOffsetOutcome>> {
1178 let outcomes = group_offset_outcomes(response.topics);
1179 let codes = || {
1180 outcomes
1181 .iter()
1182 .filter_map(|outcome| outcome.error.as_ref().map(|error| error.code))
1183 };
1184 if codes().any(|code| matches!(code, COORDINATOR_NOT_AVAILABLE | NOT_COORDINATOR)) {
1185 RetryAction::FindCoordinator(Ok(outcomes))
1186 } else if codes()
1187 .any(|code| matches!(code, COORDINATOR_LOAD_IN_PROGRESS | REBALANCE_IN_PROGRESS))
1188 {
1189 RetryAction::SameCoordinator(Ok(outcomes))
1190 } else {
1191 completed_group_offsets(outcomes)
1192 }
1193}
1194
1195macro_rules! offset_request_topics {
1197 ($topics:expr, $topic_type:ident) => {
1198 $topics
1199 .into_iter()
1200 .map(|(name, partitions)| $topic_type {
1201 name,
1202 partitions,
1203 ..Default::default()
1204 })
1205 .collect()
1206 };
1207}
1208
1209fn offset_delete_request(group: &str, partitions: &[(String, i32)]) -> OffsetDeleteRequest {
1210 let mut topics = BTreeMap::<String, Vec<OffsetDeleteRequestPartition>>::new();
1211 for (topic, partition) in partitions {
1212 topics
1213 .entry(topic.clone())
1214 .or_default()
1215 .push(OffsetDeleteRequestPartition {
1216 partition_index: *partition,
1217 ..Default::default()
1218 });
1219 }
1220 OffsetDeleteRequest {
1221 group_id: group.into(),
1222 topics: offset_request_topics!(topics, OffsetDeleteRequestTopic),
1223 ..Default::default()
1224 }
1225}
1226
1227fn offset_delete_retry_action(
1233 group: &str,
1234 response: OffsetDeleteResponse,
1235) -> RetryAction<Vec<ConsumerGroupOffsetOutcome>> {
1236 if response.error_code != 0 {
1237 let result = Err(AdminError::Broker {
1238 api: "OffsetDelete",
1239 code: response.error_code,
1240 name: kafka_error_name(response.error_code),
1241 message: Some(format!("group={group}")),
1242 });
1243 return group_coordinator_error_action(response.error_code, result);
1244 }
1245 let outcomes = group_offset_outcomes(response.topics);
1246 completed_group_offsets(outcomes)
1247}
1248
1249fn leave_group_retry_action(response: LeaveGroupResponse) -> RetryAction<RemovedMembers> {
1260 if response.error_code != 0 {
1261 let result = Err(AdminError::Broker {
1262 api: "LeaveGroup",
1263 code: response.error_code,
1264 name: kafka_error_name(response.error_code),
1265 message: None,
1266 });
1267 return group_coordinator_error_action(response.error_code, result);
1268 }
1269 let members = response
1270 .members
1271 .into_iter()
1272 .map(|member| {
1273 (
1274 RemovedMember {
1275 member_id: member.member_id,
1276 group_instance_id: member.group_instance_id,
1277 },
1278 kafka_error_if(member.error_code, None),
1279 )
1280 })
1281 .collect();
1282 RetryAction::Done(Ok(RemovedMembers { members }))
1283}
1284
1285fn group_coordinator_error_action<T>(
1286 error_code: i16,
1287 result: Result<T, AdminError>,
1288) -> RetryAction<T> {
1289 match error_code {
1290 COORDINATOR_LOAD_IN_PROGRESS => RetryAction::SameCoordinator(result),
1291 COORDINATOR_NOT_AVAILABLE | NOT_COORDINATOR => RetryAction::FindCoordinator(result),
1292 _ => RetryAction::Done(result),
1293 }
1294}
1295
1296const UNKNOWN_SERVER_ERROR: i16 = -1;
1299const DEFAULT_LEAVE_GROUP_REASON: &str = "member was removed by an admin";
1301const MAX_REASON_CHARS: usize = 255;
1304
1305fn leave_group_reason(reason: Option<&str>) -> String {
1308 match reason {
1309 Some(reason) if !reason.is_empty() => reason.chars().take(MAX_REASON_CHARS).collect(),
1310 _ => DEFAULT_LEAVE_GROUP_REASON.to_owned(),
1311 }
1312}
1313
1314#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
1317pub struct MemberToRemove {
1318 pub group_instance_id: String,
1319}
1320
1321#[derive(Debug, Clone, Default, PartialEq, Eq, bon::Builder)]
1324pub struct RemoveMembersOptions {
1325 #[builder(into)]
1328 pub reason: Option<String>,
1329}
1330
1331#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
1334pub struct RemovedMember {
1335 pub member_id: String,
1336 pub group_instance_id: Option<String>,
1337}
1338
1339#[derive(Debug, Clone, Default, PartialEq, Eq)]
1342pub struct RemovedMembers {
1343 pub members: BTreeMap<RemovedMember, Option<KafkaError>>,
1345}
1346
1347impl RemovedMembers {
1348 pub fn member_result(&self, member: &MemberToRemove) -> Result<(), KafkaError> {
1356 let key = RemovedMember {
1357 member_id: String::new(),
1358 group_instance_id: Some(member.group_instance_id.clone()),
1359 };
1360 match self.members.get(&key) {
1361 Some(None) => Ok(()),
1362 Some(Some(error)) => Err(error.clone()),
1363 None => Err(KafkaError {
1364 code: UNKNOWN_SERVER_ERROR,
1365 name: kafka_error_name(UNKNOWN_SERVER_ERROR),
1366 message: Some(format!(
1367 "Member \"{}\" was not included in the removal response",
1368 member.group_instance_id
1369 )),
1370 }),
1371 }
1372 }
1373}
1374
1375fn alter_share_group_offsets_request(
1377 group: &str,
1378 offsets: &GroupOffsets,
1379) -> AlterShareGroupOffsetsRequest {
1380 let mut topics = BTreeMap::<String, Vec<AlterShareGroupOffsetsRequestPartition>>::new();
1381 for ((topic, partition), start_offset) in offsets {
1382 topics
1383 .entry(topic.clone())
1384 .or_default()
1385 .push(AlterShareGroupOffsetsRequestPartition {
1386 partition_index: *partition,
1387 start_offset: *start_offset,
1388 ..Default::default()
1389 });
1390 }
1391 AlterShareGroupOffsetsRequest {
1392 group_id: group.into(),
1393 topics: topics
1394 .into_iter()
1395 .map(
1396 |(topic_name, partitions)| AlterShareGroupOffsetsRequestTopic {
1397 topic_name,
1398 partitions,
1399 ..Default::default()
1400 },
1401 )
1402 .collect(),
1403 ..Default::default()
1404 }
1405}
1406
1407fn share_group_coordinator_error_action<T>(
1408 code: i16,
1409 error: AdminError,
1410 otherwise: impl FnOnce(AdminError) -> Result<T, AdminError>,
1411) -> RetryAction<T> {
1412 match code {
1413 COORDINATOR_LOAD_IN_PROGRESS | REBALANCE_IN_PROGRESS => {
1414 RetryAction::SameCoordinator(Err(error))
1415 }
1416 COORDINATOR_NOT_AVAILABLE | NOT_COORDINATOR => RetryAction::FindCoordinator(Err(error)),
1417 _ => RetryAction::Done(otherwise(error)),
1418 }
1419}
1420
1421fn completed_group_offsets<T>(outcomes: T) -> RetryAction<T> {
1422 RetryAction::Done(Ok(outcomes))
1423}
1424
1425fn map_group_topic_rows<T, Rows, P, O>(
1426 topics: impl IntoIterator<Item = T>,
1427 mut rows: impl FnMut(T) -> (String, Rows),
1428 mut outcome: impl FnMut(String, P) -> O,
1429) -> Vec<O>
1430where
1431 Rows: IntoIterator<Item = P>,
1432{
1433 let mut outcomes = Vec::new();
1434 for topic in topics {
1435 let (name, partitions) = rows(topic);
1436 for partition in partitions {
1437 outcomes.push(outcome(name.clone(), partition));
1438 }
1439 }
1440 outcomes
1441}
1442
1443fn alter_share_group_offsets_retry_action(
1451 offsets: &GroupOffsets,
1452 response: AlterShareGroupOffsetsResponse,
1453) -> RetryAction<Vec<ConsumerGroupOffsetOutcome>> {
1454 if response.error_code != 0 {
1455 let error = AdminError::Broker {
1456 api: "AlterShareGroupOffsets",
1457 code: response.error_code,
1458 name: kafka_error_name(response.error_code),
1459 message: response.error_message.clone(),
1460 };
1461 return share_group_coordinator_error_action(response.error_code, error, |_| {
1462 Ok(offsets
1463 .keys()
1464 .map(|(topic, partition)| ConsumerGroupOffsetOutcome {
1465 topic: topic.clone(),
1466 partition: *partition,
1467 error: kafka_error_if(response.error_code, response.error_message.clone()),
1468 })
1469 .collect())
1470 });
1471 }
1472 let outcomes = map_group_topic_rows(
1473 response.responses,
1474 |topic| (topic.topic_name, topic.partitions),
1475 |topic, partition| ConsumerGroupOffsetOutcome {
1476 topic,
1477 partition: partition.partition_index,
1478 error: kafka_error_if(partition.error_code, partition.error_message),
1479 },
1480 );
1481 completed_group_offsets(outcomes)
1482}
1483
1484fn delete_share_group_offsets_retry_action(
1491 response: DeleteShareGroupOffsetsResponse,
1492) -> RetryAction<Vec<(String, Option<KafkaError>)>> {
1493 if response.error_code != 0 {
1494 let error = AdminError::Broker {
1495 api: "DeleteShareGroupOffsets",
1496 code: response.error_code,
1497 name: kafka_error_name(response.error_code),
1498 message: response.error_message,
1499 };
1500 return share_group_coordinator_error_action(response.error_code, error, Err);
1501 }
1502 let outcomes = map_group_topic_rows(
1503 response.responses,
1504 |topic| {
1505 (
1506 topic.topic_name,
1507 std::iter::once((topic.error_code, topic.error_message)),
1508 )
1509 },
1510 |topic, (code, message)| (topic, kafka_error_if(code, message)),
1511 );
1512 completed_group_offsets(outcomes)
1513}
1514
1515fn list_groups_min_version(options: &ListGroupsOptions) -> i16 {
1523 let types_need_v5 = !options.types.is_empty()
1524 && (!options.types.contains(&GroupType::Classic)
1525 || options
1526 .types
1527 .iter()
1528 .any(|group_type| !matches!(group_type, GroupType::Classic | GroupType::Consumer)));
1529 if types_need_v5 {
1530 5
1531 } else if options.group_states.is_empty() {
1532 list_groups_request::MIN_VERSION
1533 } else {
1534 4
1535 }
1536}
1537
1538async fn list_groups_on_broker(
1552 broker: &MetadataResponseBroker,
1553 options: ConnectionOptions,
1554 request: ListGroupsRequest,
1555 min_version: i16,
1556 start: tokio::time::Instant,
1557 retry: RetryPolicy,
1558) -> Result<Vec<ListedGroup>, KafkaError> {
1559 let host_port = format_host_port(&broker.host, broker.port);
1560 let deadline = start + retry.timeout;
1561 let mut backoff = retry.initial_backoff;
1562 let mut connection = None;
1563 let mut attempts = 0_u32;
1564 let mut last = None;
1565 while tokio::time::Instant::now() < deadline {
1566 attempts += 1;
1567 let attempt = list_groups_attempt(
1568 &mut connection,
1569 &host_port,
1570 &options,
1571 request.clone(),
1572 min_version,
1573 );
1574 match tokio::time::timeout_at(deadline, attempt).await {
1575 Ok(RetryAction::Done(result)) => {
1576 return result.map_err(|error| list_groups_kafka_error(&error));
1577 }
1578 Ok(RetryAction::SameCoordinator(result) | RetryAction::FindCoordinator(result)) => {
1579 match result {
1580 Ok(groups) => return Ok(groups),
1581 Err(error) => last = Some(error),
1582 }
1583 }
1584 Err(_) => break,
1585 }
1586 if attempts > retry.max_retries {
1589 break;
1590 }
1591 tracing::debug!(
1592 node_id = broker.node_id,
1593 "ListGroups got a retriable error; retrying"
1594 );
1595 tokio::time::sleep_until(deadline.min(tokio::time::Instant::now() + backoff)).await;
1596 backoff = backoff.saturating_mul(2).min(retry.max_backoff);
1597 }
1598 Err(list_groups_timeout(attempts, last.as_ref()))
1599}
1600
1601fn list_groups_timeout(attempts: u32, last: Option<&AdminError>) -> KafkaError {
1606 const REQUEST_TIMED_OUT: i16 = 7;
1607 let cause = last
1608 .map(|error| format!("; last error: {error}"))
1609 .unwrap_or_default();
1610 KafkaError {
1611 code: REQUEST_TIMED_OUT,
1612 name: kafka_error_name(REQUEST_TIMED_OUT),
1613 message: Some(format!(
1614 "ListGroups timed out after {attempts} attempt(s){cause}"
1615 )),
1616 }
1617}
1618
1619async fn list_groups_attempt(
1622 connection: &mut Option<Connection>,
1623 host_port: &str,
1624 options: &ConnectionOptions,
1625 request: ListGroupsRequest,
1626 min_version: i16,
1627) -> RetryAction<Vec<ListedGroup>> {
1628 let current = match connection {
1629 Some(current) => current,
1630 None => match AdminClient::connect_one(host_port, options.clone()).await {
1631 Ok(new) => connection.insert(new),
1632 Err(error) if error.is_authentication_failure() => {
1633 return RetryAction::Done(Err(error));
1634 }
1635 Err(error) => return RetryAction::SameCoordinator(Err(error)),
1636 },
1637 };
1638 match send_connection_at_least(current, request, min_version).await {
1639 Ok(response) => match response.error_code {
1640 0 => RetryAction::Done(Ok(response.groups)),
1641 code => {
1642 let error = Err(AdminError::Broker {
1643 api: "ListGroups",
1644 code,
1645 name: kafka_error_name(code),
1646 message: None,
1647 });
1648 if matches!(
1649 code,
1650 COORDINATOR_LOAD_IN_PROGRESS | COORDINATOR_NOT_AVAILABLE
1651 ) {
1652 RetryAction::SameCoordinator(error)
1653 } else {
1654 RetryAction::Done(error)
1655 }
1656 }
1657 },
1658 Err(error) if AdminClient::is_retriable_transport_error(&error) => {
1659 *connection = None;
1660 RetryAction::SameCoordinator(Err(error.into()))
1661 }
1662 Err(error) => RetryAction::Done(Err(error.into())),
1663 }
1664}
1665
1666pub(crate) fn list_groups_kafka_error(error: &AdminError) -> KafkaError {
1674 const UNKNOWN_SERVER_ERROR: i16 = -1;
1675 const REQUEST_TIMED_OUT: i16 = 7;
1676 const NETWORK_EXCEPTION: i16 = 13;
1677 const UNSUPPORTED_SASL_MECHANISM: i16 = 33;
1678 const ILLEGAL_SASL_STATE: i16 = 34;
1679 const UNSUPPORTED_VERSION: i16 = 35;
1680 const INVALID_CONFIG: i16 = 40;
1681 const SASL_AUTHENTICATION_FAILED: i16 = 58;
1682 let (code, message) = match error {
1683 AdminError::Broker { code, message, .. } => (*code, message.clone()),
1684 AdminError::Transport(ClientError::IncompatibleVersion { .. }) => {
1685 (UNSUPPORTED_VERSION, Some(error.to_string()))
1686 }
1687 AdminError::Transport(ClientError::Timeout(_)) => {
1688 (REQUEST_TIMED_OUT, Some(error.to_string()))
1689 }
1690 AdminError::Transport(
1691 ClientError::Connect { .. } | ClientError::Disconnected | ClientError::Io(_),
1692 ) => (NETWORK_EXCEPTION, Some(error.to_string())),
1693 AdminError::Transport(ClientError::Authentication { source, .. }) => {
1694 let code = match source {
1695 AuthenticationError::Tls(_) => INVALID_CONFIG,
1696 AuthenticationError::Sasl(SaslAuthenticationError::UnsupportedMechanism(_)) => {
1697 UNSUPPORTED_SASL_MECHANISM
1698 }
1699 AuthenticationError::Sasl(SaslAuthenticationError::IllegalState(_)) => {
1700 ILLEGAL_SASL_STATE
1701 }
1702 AuthenticationError::Sasl(SaslAuthenticationError::Failed(_)) => {
1703 SASL_AUTHENTICATION_FAILED
1704 }
1705 };
1706 (code, Some(error.to_string()))
1707 }
1708 _ => (UNKNOWN_SERVER_ERROR, Some(error.to_string())),
1709 };
1710 KafkaError {
1711 code,
1712 name: kafka_error_name(code),
1713 message,
1714 }
1715}
1716
1717fn offset_fetch_group(group: &str) -> OffsetFetchRequestGroup {
1724 OffsetFetchRequestGroup {
1725 group_id: group.into(),
1726 member_id: None,
1727 member_epoch: -1,
1728 topics: None,
1729 ..Default::default()
1730 }
1731}
1732
1733fn offset_fetch_request(group: &str) -> AllTopicsOffsetFetch {
1734 AllTopicsOffsetFetch::new(OffsetFetchRequest {
1735 group_id: group.into(),
1736 topics: None,
1737 groups: vec![offset_fetch_group(group)],
1738 require_stable: false,
1739 ..Default::default()
1740 })
1741}
1742
1743fn committed_offsets(
1751 group: &str,
1752 response: OffsetFetchResponse,
1753) -> Result<GroupOffsets, AdminError> {
1754 let (error_code, rows): (i16, Vec<(String, i32, i64, i16)>) = if response.groups.is_empty() {
1755 (
1756 response.error_code,
1757 response
1758 .topics
1759 .into_iter()
1760 .flat_map(offset_fetch_topic_rows)
1761 .collect(),
1762 )
1763 } else {
1764 let mut error_code = 0;
1765 let mut rows = Vec::new();
1766 for entry in response
1767 .groups
1768 .into_iter()
1769 .filter(|entry| entry.group_id == group)
1770 {
1771 error_code = entry.error_code;
1772 for topic in entry.topics {
1773 rows.extend(offset_fetch_topic_rows(topic));
1774 }
1775 }
1776 (error_code, rows)
1777 };
1778 if error_code != 0 {
1779 return Err(AdminError::Broker {
1780 api: "OffsetFetch",
1781 code: error_code,
1782 name: kafka_error_name(error_code),
1783 message: Some(format!("group={group}")),
1784 });
1785 }
1786 Ok(rows
1787 .into_iter()
1788 .filter_map(|(topic, partition, offset, partition_error)| {
1789 if partition_error != 0 {
1790 tracing::warn!(
1791 topic,
1792 partition,
1793 error_code = partition_error,
1794 "skipping the committed offset of a partition with an error"
1795 );
1796 None
1797 } else if offset < 0 {
1798 None
1799 } else {
1800 Some(((topic, partition), offset))
1801 }
1802 })
1803 .collect())
1804}
1805
1806trait IntoOffsetFetchRow {
1807 fn into_offset_fetch_row(self, topic: &str) -> (String, i32, i64, i16);
1808}
1809
1810macro_rules! impl_offset_fetch_row {
1811 ($partition:ty) => {
1812 impl IntoOffsetFetchRow for $partition {
1813 fn into_offset_fetch_row(self, topic: &str) -> (String, i32, i64, i16) {
1814 (
1815 topic.to_owned(),
1816 self.partition_index,
1817 self.committed_offset,
1818 self.error_code,
1819 )
1820 }
1821 }
1822 };
1823}
1824
1825impl_offset_fetch_row!(OffsetFetchResponsePartition);
1826impl_offset_fetch_row!(OffsetFetchResponsePartitions);
1827
1828trait IntoOffsetFetchTopics {
1829 type Partition: IntoOffsetFetchRow;
1830
1831 fn into_offset_fetch_parts(self) -> (String, Vec<Self::Partition>);
1832}
1833
1834macro_rules! impl_offset_fetch_topics {
1835 ($topics:ty, $partition:ty) => {
1836 impl IntoOffsetFetchTopics for $topics {
1837 type Partition = $partition;
1838
1839 fn into_offset_fetch_parts(self) -> (String, Vec<Self::Partition>) {
1840 (self.name, self.partitions)
1841 }
1842 }
1843 };
1844}
1845
1846impl_offset_fetch_topics!(OffsetFetchResponseTopic, OffsetFetchResponsePartition);
1847impl_offset_fetch_topics!(OffsetFetchResponseTopics, OffsetFetchResponsePartitions);
1848
1849fn offset_fetch_topic_rows<T: IntoOffsetFetchTopics>(topic: T) -> Vec<(String, i32, i64, i16)> {
1850 let (name, partitions) = topic.into_offset_fetch_parts();
1851 partitions
1852 .into_iter()
1853 .map(|partition| partition.into_offset_fetch_row(&name))
1854 .collect()
1855}
1856
1857fn offset_commit_request(group: &str, offsets: &GroupOffsets) -> TopicNameOffsetCommit {
1858 let mut topics = BTreeMap::<String, Vec<OffsetCommitRequestPartition>>::new();
1859 for ((topic, partition), offset) in offsets {
1860 topics
1861 .entry(topic.clone())
1862 .or_default()
1863 .push(OffsetCommitRequestPartition {
1864 partition_index: *partition,
1865 committed_offset: *offset,
1866 committed_leader_epoch: -1,
1867 ..Default::default()
1870 });
1871 }
1872 TopicNameOffsetCommit::new(OffsetCommitRequest {
1873 group_id: group.into(),
1874 generation_id_or_member_epoch: -1,
1875 member_id: String::new(),
1876 topics: offset_request_topics!(topics, OffsetCommitRequestTopic),
1877 ..Default::default()
1878 })
1879}
1880
1881#[cfg(test)]
1882mod tests {
1883 use std::{
1884 sync::{
1885 Arc, Mutex,
1886 atomic::{AtomicUsize, Ordering},
1887 },
1888 time::Duration,
1889 };
1890
1891 use assert2::assert;
1892 use bytes::{Buf, BytesMut};
1893 use krabka_client_core::{
1894 AuthenticationError, ClientError, MockBroker, MockReply, MockSaslAnswer,
1895 SaslAuthenticationError,
1896 mock_test_support::offset_commit_response,
1897 security::{ClientSecurity, SaslCredentials},
1898 };
1899 use krabka_protocol::{
1900 Decode, Encode, ProtocolRequest,
1901 owned::{
1902 alter_share_group_offsets_response::{
1903 AlterShareGroupOffsetsResponsePartition, AlterShareGroupOffsetsResponseTopic,
1904 },
1905 api_versions_request,
1906 api_versions_response::{ApiVersion, ApiVersionsResponse},
1907 delete_share_group_offsets_response::DeleteShareGroupOffsetsResponseTopic,
1908 find_coordinator_request,
1909 find_coordinator_response::FindCoordinatorResponse,
1910 leave_group_response::MemberResponse,
1911 list_groups_response::ListGroupsResponse,
1912 metadata_request,
1913 metadata_response::MetadataResponse,
1914 offset_commit_request,
1915 offset_commit_response::{
1916 OffsetCommitResponse, OffsetCommitResponsePartition, OffsetCommitResponseTopic,
1917 },
1918 offset_delete_response::{OffsetDeleteResponsePartition, OffsetDeleteResponseTopic},
1919 offset_fetch_request,
1920 offset_fetch_response::{
1921 OffsetFetchResponseGroup, OffsetFetchResponsePartition,
1922 OffsetFetchResponsePartitions, OffsetFetchResponseTopic, OffsetFetchResponseTopics,
1923 },
1924 sasl_handshake_request,
1925 },
1926 };
1927
1928 use super::*;
1929
1930 fn test_group_offsets() -> GroupOffsets {
1931 maplit::btreemap! { ("orders".into(), 2) => 41 }
1932 }
1933
1934 #[test]
1935 fn offset_reset_builds_admin_commit() {
1936 let offsets = test_group_offsets();
1937 let request = offset_commit_request("worker", &offsets);
1938 let expected = TopicNameOffsetCommit::new(commit_request_for_test("worker", -1, -1));
1939 assert!(request == expected);
1940 }
1941
1942 fn commit_request_for_test(
1943 group_id: &str,
1944 generation_id_or_member_epoch: i32,
1945 committed_leader_epoch: i32,
1946 ) -> OffsetCommitRequest {
1947 OffsetCommitRequest {
1948 group_id: group_id.into(),
1949 generation_id_or_member_epoch,
1950 topics: vec![OffsetCommitRequestTopic {
1951 name: "orders".into(),
1952 partitions: vec![OffsetCommitRequestPartition {
1953 partition_index: 2,
1954 committed_offset: 41,
1955 committed_leader_epoch,
1956 ..Default::default()
1957 }],
1958 ..Default::default()
1959 }],
1960 ..Default::default()
1961 }
1962 }
1963
1964 fn encode(response: &impl Encode, version: i16, flexible: bool) -> Vec<u8> {
1965 let mut bytes = BytesMut::new();
1966 if flexible {
1967 bytes.extend_from_slice(&[0]);
1968 }
1969 response.encode(&mut bytes, version).unwrap();
1970 bytes.to_vec()
1971 }
1972
1973 fn encode_flexible(response: &impl Encode, version: i16) -> Vec<u8> {
1974 encode(response, version, true)
1975 }
1976
1977 fn broker_port(addr: std::net::SocketAddr) -> i32 {
1978 i32::from(addr.port())
1979 }
1980
1981 struct MockBrokerAddress {
1982 node_id: i32,
1983 host: String,
1984 port: i32,
1985 }
1986
1987 fn mock_broker_address(addr: std::net::SocketAddr, node_id: i32) -> MockBrokerAddress {
1988 MockBrokerAddress {
1989 node_id,
1990 host: addr.ip().to_string(),
1991 port: broker_port(addr),
1992 }
1993 }
1994
1995 macro_rules! broker_fields {
1996 ($type:ident, $broker:expr) => {
1997 $type {
1998 node_id: $broker.node_id,
1999 host: $broker.host,
2000 port: $broker.port,
2001 ..Default::default()
2002 }
2003 };
2004 }
2005
2006 fn mock_metadata_broker(addr: std::net::SocketAddr, node_id: i32) -> MetadataResponseBroker {
2007 let broker = mock_broker_address(addr, node_id);
2008 broker_fields!(MetadataResponseBroker, broker)
2009 }
2010
2011 async fn connect_admin(addr: std::net::SocketAddr, message: &'static str) -> AdminClient {
2012 AdminClient::connect(&[addr.to_string()])
2013 .await
2014 .expect(message)
2015 }
2016
2017 fn mock_coordinator_response(
2018 addr: std::net::SocketAddr,
2019 error_code: i16,
2020 ) -> FindCoordinatorResponse {
2021 let broker = mock_broker_address(addr, 2);
2022 let mut response = broker_fields!(FindCoordinatorResponse, broker);
2023 response.error_code = error_code;
2024 response
2025 }
2026
2027 fn incompatible_version(error: AdminError, case: &str) -> (i16, i16, i16, i16, i16) {
2028 match error {
2029 AdminError::Transport(ClientError::IncompatibleVersion {
2030 api_key,
2031 broker_min,
2032 broker_max,
2033 client_min,
2034 client_max,
2035 }) => (api_key, broker_min, broker_max, client_min, client_max),
2036 other => panic!("case {case}: unexpected error {other:?}"),
2037 }
2038 }
2039
2040 async fn with_incompatible_version<T>(
2041 case: &'static str,
2042 result: impl std::future::Future<Output = Result<T, AdminError>>,
2043 ) -> Result<T, (i16, i16, i16, i16, i16)> {
2044 result
2045 .await
2046 .map_err(|error| incompatible_version(error, case))
2047 }
2048
2049 #[derive(Clone, Copy)]
2052 struct GroupRanges {
2053 offset_commit: (i16, i16),
2054 offset_fetch: (i16, i16),
2055 }
2056
2057 impl GroupRanges {
2058 fn offset_commit(offset_commit: (i16, i16)) -> Self {
2059 Self {
2060 offset_commit,
2061 offset_fetch: (2, 10),
2062 }
2063 }
2064
2065 fn offset_fetch(offset_fetch: (i16, i16)) -> Self {
2066 Self {
2067 offset_commit: (2, 9),
2068 offset_fetch,
2069 }
2070 }
2071 }
2072
2073 fn encode_api_versions(api_keys: Vec<ApiVersion>) -> Vec<u8> {
2074 encode(
2075 &ApiVersionsResponse {
2076 api_keys,
2077 ..Default::default()
2078 },
2079 0,
2080 false,
2081 )
2082 }
2083
2084 macro_rules! api_version_entry {
2085 ($key:expr, $min:expr, $max:expr) => {
2086 ApiVersion {
2087 api_key: $key,
2088 min_version: $min,
2089 max_version: $max,
2090 ..Default::default()
2091 }
2092 };
2093 }
2094
2095 fn api_versions(ranges: GroupRanges) -> Vec<u8> {
2097 encode_api_versions(vec![
2098 api_version_entry!(api_versions_request::API_KEY, 0, 0),
2099 api_version_entry!(find_coordinator_request::API_KEY, 0, 0),
2100 api_version_entry!(
2101 offset_commit_request::API_KEY,
2102 ranges.offset_commit.0,
2103 ranges.offset_commit.1
2104 ),
2105 api_version_entry!(
2106 offset_fetch_request::API_KEY,
2107 ranges.offset_fetch.0,
2108 ranges.offset_fetch.1
2109 ),
2110 api_version_entry!(metadata_request::API_KEY, 13, 13),
2111 ])
2112 }
2113
2114 fn decode_request<R: ProtocolRequest + for<'de> Decode<'de>>(
2116 mut body: &[u8],
2117 version: i16,
2118 ) -> R {
2119 let client_id_len = body.get_i16();
2120 body.advance(usize::try_from(client_id_len).expect("client id length"));
2121 if version >= R::FLEXIBLE_MIN {
2122 body.advance(1);
2123 }
2124 R::decode(&mut body, version).expect("request decodes")
2125 }
2126
2127 async fn bootstrap_for(
2129 coordinator: &MockBroker,
2130 ranges: GroupRanges,
2131 group_rpcs: Arc<AtomicUsize>,
2132 ) -> MockBroker {
2133 let coordinator_addr = coordinator.addr;
2134 MockBroker::start(move |api_key, version, _, _| match api_key {
2135 api_versions_request::API_KEY => Some(api_versions(ranges)),
2136 find_coordinator_request::API_KEY => Some(encode(
2137 &mock_coordinator_response(coordinator_addr, 0),
2138 version,
2139 false,
2140 )),
2141 offset_commit_request::API_KEY => {
2142 group_rpcs.fetch_add(1, Ordering::SeqCst);
2143 Some(encode(&OffsetCommitResponse::default(), version, true))
2144 }
2145 offset_fetch_request::API_KEY => {
2146 group_rpcs.fetch_add(1, Ordering::SeqCst);
2147 Some(encode(&OffsetFetchResponse::default(), version, true))
2148 }
2149 metadata_request::API_KEY => Some(empty_metadata_reply(version)),
2150 _ => None,
2151 })
2152 .await
2153 }
2154
2155 struct GroupAdminConnection {
2156 bootstrap: MockBroker,
2157 admin: AdminClient,
2158 }
2159
2160 async fn connect_group_admin(
2161 coordinator: &MockBroker,
2162 ranges: GroupRanges,
2163 ) -> GroupAdminConnection {
2164 let bootstrap = bootstrap_for(coordinator, ranges, Arc::default()).await;
2165 let admin = connect_admin(bootstrap.addr, "admin connects").await;
2166 GroupAdminConnection { bootstrap, admin }
2167 }
2168
2169 fn stop_group_brokers(bootstrap: MockBroker, coordinator: MockBroker) {
2170 bootstrap.stop();
2171 coordinator.stop();
2172 }
2173
2174 fn record_group_request<Request>(
2175 requests: &Arc<Mutex<Vec<(i16, Request)>>>,
2176 version: i16,
2177 request: Request,
2178 ) {
2179 requests
2180 .lock()
2181 .expect("requests lock")
2182 .push((version, request));
2183 }
2184
2185 struct GroupVersionAssertion<Request, Output> {
2186 name: &'static str,
2187 requests: Vec<(i16, Request)>,
2188 result: Output,
2189 expected_requests: Vec<(i16, Request)>,
2190 expected_result: Output,
2191 }
2192
2193 fn assert_group_version_case<Request, Output>(case: GroupVersionAssertion<Request, Output>)
2194 where
2195 Request: std::fmt::Debug + PartialEq,
2196 Output: std::fmt::Debug + PartialEq,
2197 {
2198 assert!(
2199 (case.requests, case.result) == (case.expected_requests, case.expected_result),
2200 "case {}",
2201 case.name
2202 );
2203 }
2204
2205 struct ExpectedGroupVersionCase<Request, Output> {
2206 name: &'static str,
2207 ranges: GroupRanges,
2208 requests: Vec<(i16, Request)>,
2209 result: Output,
2210 }
2211
2212 fn expected_group_version_case<Request, Output>(
2213 name: &'static str,
2214 ranges: GroupRanges,
2215 requests: Vec<(i16, Request)>,
2216 result: Output,
2217 ) -> ExpectedGroupVersionCase<Request, Output> {
2218 ExpectedGroupVersionCase {
2219 name,
2220 ranges,
2221 requests,
2222 result,
2223 }
2224 }
2225
2226 fn incompatible_group_version<T>(
2227 api_key: i16,
2228 broker_min: i16,
2229 broker_max: i16,
2230 ) -> Result<T, (i16, i16, i16, i16, i16)> {
2231 Err((api_key, broker_min, broker_max, 2, 9))
2232 }
2233
2234 async fn run_group_version_case<Request, Output, Respond, Action, Fut>(
2235 case: ExpectedGroupVersionCase<Request, Output>,
2236 api_key: i16,
2237 respond: Respond,
2238 action: Action,
2239 ) where
2240 Request: ProtocolRequest
2241 + for<'de> Decode<'de>
2242 + Clone
2243 + PartialEq
2244 + std::fmt::Debug
2245 + Send
2246 + 'static,
2247 Output: PartialEq + std::fmt::Debug,
2248 Respond: Fn(&Request, i16) -> Vec<u8> + Copy + Send + Sync + 'static,
2249 Action: FnOnce(&'static str, AdminClient) -> Fut,
2250 Fut: std::future::Future<Output = Output>,
2251 {
2252 let ranges = case.ranges;
2253 let requests = Arc::new(Mutex::new(Vec::new()));
2254 let recorded = Arc::clone(&requests);
2255 let metadata_requests = Arc::new(AtomicUsize::new(0));
2256 let metadata_count = Arc::clone(&metadata_requests);
2257 let coordinator =
2258 MockBroker::start(move |request_api, version, _, body| match request_api {
2259 api_versions_request::API_KEY => Some(api_versions(ranges)),
2260 request_api if request_api == api_key => {
2261 let request = decode_request(body, version);
2262 let response = respond(&request, version);
2263 record_group_request(&recorded, version, request);
2264 Some(response)
2265 }
2266 metadata_request::API_KEY => {
2267 counted_reply(&metadata_count, Some(empty_metadata_reply(version)))
2268 }
2269 _ => None,
2270 })
2271 .await;
2272 let GroupAdminConnection { bootstrap, admin } =
2273 connect_group_admin(&coordinator, ranges).await;
2274 let result = action(case.name, admin).await;
2275 stop_group_brokers(bootstrap, coordinator);
2276 let requests = requests.lock().expect("requests lock").clone();
2277 assert!(
2278 metadata_requests.load(Ordering::SeqCst) == 0,
2279 "case {}",
2280 case.name
2281 );
2282 assert_group_version_case(GroupVersionAssertion {
2283 name: case.name,
2284 requests,
2285 result,
2286 expected_requests: case.requests,
2287 expected_result: case.result,
2288 });
2289 }
2290
2291 type CommitResult = Result<Vec<ConsumerGroupOffsetOutcome>, (i16, i16, i16, i16, i16)>;
2294
2295 fn commit_group_version_cases()
2299 -> Vec<ExpectedGroupVersionCase<OffsetCommitRequest, CommitResult>> {
2300 let committed = Ok(successful_group_offset_outcome());
2301 let commit_case = |name, version_range, requests, result| {
2302 expected_group_version_case(
2303 name,
2304 GroupRanges::offset_commit(version_range),
2305 requests,
2306 result,
2307 )
2308 };
2309 let sent_request = |version| vec![(version, commit_request_for_test("workers", -1, -1))];
2310 vec![
2311 commit_case(
2312 "coordinator stops at v7",
2313 (2, 7),
2314 sent_request(7),
2315 committed.clone(),
2316 ),
2317 commit_case(
2318 "coordinator stops at v9",
2319 (2, 9),
2320 sent_request(9),
2321 committed.clone(),
2322 ),
2323 commit_case(
2324 "coordinator supports v10",
2325 (2, 10),
2326 sent_request(9),
2327 committed,
2328 ),
2329 commit_case(
2330 "coordinator supports only v10",
2331 (10, 10),
2332 Vec::new(),
2333 incompatible_group_version(offset_commit_request::API_KEY, 10, 10),
2334 ),
2335 ]
2336 }
2337
2338 #[yare::parameterized(
2339 coordinator_stops_at_v7 = { 0 },
2340 coordinator_stops_at_v9 = { 1 },
2341 coordinator_supports_v10 = { 2 },
2342 coordinator_only_supports_v10 = { 3 },
2343 )]
2344 fn alter_consumer_group_offsets_sends_offset_commit_by_topic_name(index: usize) {
2345 let case = commit_group_version_cases().remove(index);
2346 block_on_test(run_group_version_case(
2347 case,
2348 offset_commit_request::API_KEY,
2349 |request, version| {
2350 encode(
2351 &offset_commit_response(request, |_| 0),
2352 version,
2353 version >= offset_commit_request::FLEXIBLE_MIN,
2354 )
2355 },
2356 commit_group_offsets_by_topic,
2357 ));
2358 }
2359
2360 async fn commit_group_offsets_by_topic(
2361 name: &'static str,
2362 mut admin: AdminClient,
2363 ) -> CommitResult {
2364 let offsets = test_group_offsets();
2365 Box::pin(with_incompatible_version(
2366 name,
2367 admin.alter_consumer_group_offsets("workers", &offsets),
2368 ))
2369 .await
2370 }
2371
2372 macro_rules! offset_fetch_topics {
2373 ($topic:ident, $partition:ident, $rows:expr) => {
2374 ($rows)
2375 .iter()
2376 .map(
2377 |(name, partition_index, committed_offset, error_code)| $topic {
2378 name: (*name).into(),
2379 partitions: vec![$partition {
2380 partition_index: *partition_index,
2381 committed_offset: *committed_offset,
2382 error_code: *error_code,
2383 ..Default::default()
2384 }],
2385 ..Default::default()
2386 },
2387 )
2388 .collect()
2389 };
2390 }
2391
2392 fn offset_fetch_response(version: i16) -> OffsetFetchResponse {
2396 let rows = [
2397 ("orders", 2, 41, 0),
2398 ("orders", 3, -1, 0),
2399 ("payments", 0, 5, 3),
2400 ];
2401 if version < 8 {
2402 OffsetFetchResponse {
2403 topics: offset_fetch_topics!(
2404 OffsetFetchResponseTopic,
2405 OffsetFetchResponsePartition,
2406 &rows
2407 ),
2408 ..Default::default()
2409 }
2410 } else {
2411 OffsetFetchResponse {
2412 groups: vec![OffsetFetchResponseGroup {
2413 group_id: "workers".into(),
2414 topics: offset_fetch_topics!(
2415 OffsetFetchResponseTopics,
2416 OffsetFetchResponsePartitions,
2417 &rows
2418 ),
2419 ..Default::default()
2420 }],
2421 ..Default::default()
2422 }
2423 }
2424 }
2425
2426 type FetchResult = Result<GroupOffsets, (i16, i16, i16, i16, i16)>;
2429
2430 fn fetch_group_version_cases() -> Vec<ExpectedGroupVersionCase<OffsetFetchRequest, FetchResult>>
2435 {
2436 let fetched = Ok(test_group_offsets());
2437 let case = |name, ranges, requests, result| {
2438 expected_group_version_case(name, GroupRanges::offset_fetch(ranges), requests, result)
2439 };
2440 let legacy_request = |version| {
2441 vec![(
2442 version,
2443 OffsetFetchRequest {
2444 group_id: "workers".into(),
2445 topics: None,
2446 ..Default::default()
2447 },
2448 )]
2449 };
2450 let grouped_request = |version| {
2451 vec![(
2452 version,
2453 OffsetFetchRequest {
2454 groups: vec![offset_fetch_group("workers")],
2455 ..Default::default()
2456 },
2457 )]
2458 };
2459 vec![
2460 case(
2461 "coordinator stops at v7",
2462 (2, 7),
2463 legacy_request(7),
2464 fetched.clone(),
2465 ),
2466 case(
2467 "coordinator stops at v8",
2468 (2, 8),
2469 grouped_request(8),
2470 fetched.clone(),
2471 ),
2472 case(
2473 "coordinator stops at v9",
2474 (2, 9),
2475 grouped_request(9),
2476 fetched.clone(),
2477 ),
2478 case(
2479 "coordinator supports v10",
2480 (2, 10),
2481 grouped_request(9),
2482 fetched,
2483 ),
2484 case(
2485 "coordinator supports only v10",
2486 (10, 10),
2487 Vec::new(),
2488 incompatible_group_version(offset_fetch_request::API_KEY, 10, 10),
2489 ),
2490 case(
2491 "coordinator supports only v1",
2492 (1, 1),
2493 Vec::new(),
2494 incompatible_group_version(offset_fetch_request::API_KEY, 1, 1),
2495 ),
2496 ]
2497 }
2498
2499 async fn fetch_group_offsets_by_topic(
2500 name: &'static str,
2501 mut admin: AdminClient,
2502 ) -> FetchResult {
2503 let offsets = admin.list_consumer_group_offsets("workers");
2504 Box::pin(with_incompatible_version(name, offsets)).await
2505 }
2506
2507 #[yare::parameterized(
2508 coordinator_stops_at_v7 = { 0 },
2509 coordinator_stops_at_v8 = { 1 },
2510 coordinator_stops_at_v9 = { 2 },
2511 coordinator_supports_v10 = { 3 },
2512 coordinator_only_supports_v10 = { 4 },
2513 coordinator_only_supports_v1 = { 5 },
2514 )]
2515 fn list_consumer_group_offsets_sends_offset_fetch_by_topic_name(index: usize) {
2516 let case = fetch_group_version_cases().remove(index);
2517 block_on_test(run_group_version_case(
2518 case,
2519 offset_fetch_request::API_KEY,
2520 |_, version| {
2521 encode(
2522 &offset_fetch_response(version),
2523 version,
2524 version >= offset_fetch_request::FLEXIBLE_MIN,
2525 )
2526 },
2527 fetch_group_offsets_by_topic,
2528 ));
2529 }
2530
2531 type OffsetsResult = Result<GroupOffsets, (&'static str, i16, &'static str, Option<String>)>;
2533
2534 #[yare::parameterized(
2537 v2_to_v7_shape = { offset_fetch_response(7), Ok(test_group_offsets()) },
2538 v8_and_v9_shape = { offset_fetch_response(9), Ok(test_group_offsets()) },
2539 v2_to_v7_group_error = { OffsetFetchResponse { error_code: 15, ..Default::default() }, Err(("OffsetFetch", 15, "COORDINATOR_NOT_AVAILABLE", Some("group=workers".into()))) },
2540 v8_and_v9_group_error = { OffsetFetchResponse { groups: vec![OffsetFetchResponseGroup { group_id: "workers".into(), error_code: 16, ..Default::default() }], ..Default::default() }, Err(("OffsetFetch", 16, "NOT_COORDINATOR", Some("group=workers".into()))) },
2541 )]
2542 fn committed_offsets_reads_both_response_shapes(
2543 response: OffsetFetchResponse,
2544 expected: OffsetsResult,
2545 ) {
2546 let result: OffsetsResult =
2547 committed_offsets("workers", response).map_err(|error| match error {
2548 AdminError::Broker {
2549 api,
2550 code,
2551 name,
2552 message,
2553 } => (api, code, name, message),
2554 other => panic!("unexpected error {other:?}"),
2555 });
2556 assert2::assert!(result == expected);
2557 }
2558
2559 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
2560 enum RetryExpected {
2561 Success,
2562 OffsetFetch(i16),
2563 CommitOutcome(i16),
2564 FindCoordinator(i16),
2565 Timeout,
2566 }
2567
2568 #[derive(Clone, Default)]
2572 struct RetryScript {
2573 find_coordinator: Vec<i16>,
2574 offset_fetch: Vec<i16>,
2575 offset_commit: Vec<i16>,
2576 find_coordinator_requests: usize,
2577 offset_fetch_requests: usize,
2578 offset_commit_requests: usize,
2579 unreachable_answers: usize,
2582 unreachable: Option<std::net::SocketAddr>,
2583 silent_offset_commit: bool,
2585 closed_offset_commits: usize,
2588 }
2589
2590 async fn refused_address() -> std::net::SocketAddr {
2592 let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
2593 .await
2594 .expect("bind a port");
2595 listener.local_addr().expect("local address")
2596 }
2597
2598 impl RetryScript {
2599 fn next(codes: &[i16], requests: &mut usize) -> i16 {
2600 let code = codes[(*requests).min(codes.len() - 1)];
2601 *requests += 1;
2602 code
2603 }
2604 }
2605
2606 async fn scripted_group_broker(
2611 script: Arc<Mutex<RetryScript>>,
2612 coordinator: Arc<Mutex<Option<std::net::SocketAddr>>>,
2613 ) -> MockBroker {
2614 let ranges = GroupRanges {
2615 offset_commit: (2, 9),
2616 offset_fetch: (2, 9),
2617 };
2618 let closing = Arc::clone(&script);
2619 let handler = move |api_key: i16, version: i16, body: &[u8]| match api_key {
2620 api_versions_request::API_KEY => Some(api_versions(ranges)),
2621 offset_commit_request::API_KEY => {
2622 let request: OffsetCommitRequest = decode_request(body, version);
2623 let mut script = script.lock().expect("script lock");
2624 let script = &mut *script;
2625 let error_code =
2626 RetryScript::next(&script.offset_commit, &mut script.offset_commit_requests);
2627 if script.silent_offset_commit {
2628 return None;
2629 }
2630 let response = offset_commit_response(&request, |_| error_code);
2631 Some(encode(
2632 &response,
2633 version,
2634 version >= offset_commit_request::FLEXIBLE_MIN,
2635 ))
2636 }
2637 find_coordinator_request::API_KEY => {
2638 let mut script = script.lock().expect("script lock");
2639 let script = &mut *script;
2640 let error_code = RetryScript::next(
2641 &script.find_coordinator,
2642 &mut script.find_coordinator_requests,
2643 );
2644 let addr = match script.unreachable {
2645 Some(unreachable)
2646 if script.find_coordinator_requests <= script.unreachable_answers =>
2647 {
2648 unreachable
2649 }
2650 _ => coordinator
2651 .lock()
2652 .expect("coordinator lock")
2653 .expect("coordinator address"),
2654 };
2655 Some(encode(
2656 &mock_coordinator_response(addr, error_code),
2657 version,
2658 false,
2659 ))
2660 }
2661 offset_fetch_request::API_KEY => {
2662 let mut script = script.lock().expect("script lock");
2663 let script = &mut *script;
2664 let error_code =
2665 RetryScript::next(&script.offset_fetch, &mut script.offset_fetch_requests);
2666 let response = if error_code == 0 {
2667 offset_fetch_response(version)
2668 } else {
2669 OffsetFetchResponse {
2670 groups: vec![OffsetFetchResponseGroup {
2671 group_id: "workers".into(),
2672 error_code,
2673 ..Default::default()
2674 }],
2675 ..Default::default()
2676 }
2677 };
2678 Some(encode(&response, version, true))
2679 }
2680 _ => None,
2681 };
2682 MockBroker::start_with_replies(move |api_key, version, _, body| {
2683 if api_key == offset_commit_request::API_KEY {
2684 let mut script = closing.lock().expect("script lock");
2685 if script.closed_offset_commits > 0 {
2686 script.closed_offset_commits -= 1;
2687 script.offset_commit_requests += 1;
2688 return MockReply::Close;
2689 }
2690 }
2691 handler(api_key, version, body).map_or(MockReply::Silent, MockReply::Respond)
2692 })
2693 .await
2694 }
2695
2696 struct ScriptedGroupCluster {
2697 coordinator: MockBroker,
2698 bootstrap: MockBroker,
2699 admin: AdminClient,
2700 }
2701
2702 impl ScriptedGroupCluster {
2703 async fn start(script: Arc<Mutex<RetryScript>>) -> Self {
2704 let coordinator_addr = Arc::new(Mutex::new(None));
2705 let coordinator =
2706 scripted_group_broker(Arc::clone(&script), Arc::clone(&coordinator_addr)).await;
2707 *coordinator_addr.lock().expect("coordinator lock") = Some(coordinator.addr);
2708 let bootstrap =
2709 scripted_group_broker(Arc::clone(&script), Arc::clone(&coordinator_addr)).await;
2710 let admin = connect_admin(bootstrap.addr, "admin connects").await;
2711 Self {
2712 coordinator,
2713 bootstrap,
2714 admin,
2715 }
2716 }
2717 }
2718
2719 async fn run_scripted_call<T, F, Fut>(script: RetryScript, call: F) -> (T, RetryScript)
2720 where
2721 F: FnOnce(AdminClient) -> Fut,
2722 Fut: std::future::Future<Output = T>,
2723 {
2724 let shared_script = Arc::new(Mutex::new(script));
2725 let ScriptedGroupCluster {
2726 coordinator,
2727 bootstrap,
2728 admin,
2729 } = ScriptedGroupCluster::start(Arc::clone(&shared_script)).await;
2730 let result = call(admin).await;
2731 bootstrap.stop();
2732 coordinator.stop();
2733 let recorded = shared_script.lock().expect("script lock").clone();
2734 (result, recorded)
2735 }
2736
2737 const GROUP_RETRY_LONG: Duration = Duration::from_secs(5);
2738 const GROUP_RETRY_NOW: Duration = Duration::from_millis(500);
2739
2740 fn group_retry_backoff(timeout: Duration) -> Duration {
2741 if timeout == GROUP_RETRY_LONG {
2742 Duration::from_millis(1)
2743 } else {
2744 timeout
2745 }
2746 }
2747
2748 fn test_retry_policy(timeout: Duration, initial_backoff: Duration) -> RetryPolicy {
2749 RetryPolicy {
2750 timeout,
2751 initial_backoff,
2752 max_backoff: initial_backoff,
2753 jitter: 0.0,
2754 max_retries: u32::MAX,
2755 }
2756 }
2757
2758 fn list_groups_retry_policy(
2759 timeout: Duration,
2760 backoff: Duration,
2761 max_retries: u32,
2762 ) -> RetryPolicy {
2763 RetryPolicy {
2764 timeout,
2765 initial_backoff: backoff,
2766 max_backoff: backoff,
2767 jitter: 0.0,
2768 max_retries,
2769 }
2770 }
2771
2772 #[derive(Clone, Copy)]
2773 struct GroupRetryScenario {
2774 name: &'static str,
2775 find_coordinator: &'static [i16],
2776 offset_fetch: &'static [i16],
2777 offset_commit: &'static [i16],
2778 timeout: Duration,
2779 fetch_expected: RetryExpected,
2780 fetch_calls: (usize, usize),
2781 commit_expected: RetryExpected,
2782 commit_calls: (usize, usize),
2783 }
2784
2785 #[derive(Clone, Copy)]
2786 struct RetryCodes {
2787 find_coordinator: &'static [i16],
2788 offset_fetch: &'static [i16],
2789 offset_commit: &'static [i16],
2790 }
2791
2792 #[derive(Clone, Copy)]
2793 struct RetryExpectations {
2794 fetch: RetryExpected,
2795 commit: RetryExpected,
2796 }
2797
2798 fn successful_retry_case(
2799 name: &'static str,
2800 find_coordinator: &'static [i16],
2801 rpc_errors: &'static [i16],
2802 calls: (usize, usize),
2803 ) -> GroupRetryScenario {
2804 retry_scenario(
2805 name,
2806 RetryCodes {
2807 find_coordinator,
2808 offset_fetch: rpc_errors,
2809 offset_commit: rpc_errors,
2810 },
2811 GROUP_RETRY_LONG,
2812 RetryExpectations {
2813 fetch: RetryExpected::Success,
2814 commit: RetryExpected::CommitOutcome(0),
2815 },
2816 calls,
2817 )
2818 }
2819
2820 fn retry_scenario(
2821 name: &'static str,
2822 codes: RetryCodes,
2823 timeout: Duration,
2824 expected: RetryExpectations,
2825 calls: (usize, usize),
2826 ) -> GroupRetryScenario {
2827 GroupRetryScenario {
2828 name,
2829 find_coordinator: codes.find_coordinator,
2830 offset_fetch: codes.offset_fetch,
2831 offset_commit: codes.offset_commit,
2832 timeout,
2833 fetch_expected: expected.fetch,
2834 fetch_calls: calls,
2835 commit_expected: expected.commit,
2836 commit_calls: calls,
2837 }
2838 }
2839
2840 fn coordinator_recovery_case(name: &'static str, codes: &'static [i16]) -> GroupRetryScenario {
2841 successful_retry_case(name, codes, &[0], (2, 1))
2842 }
2843
2844 fn timed_out_retry_case(name: &'static str, codes: RetryCodes) -> GroupRetryScenario {
2845 retry_scenario(
2846 name,
2847 codes,
2848 GROUP_RETRY_NOW,
2849 RetryExpectations {
2850 fetch: RetryExpected::Timeout,
2851 commit: RetryExpected::Timeout,
2852 },
2853 (1, 1),
2854 )
2855 }
2856
2857 fn group_retry_scenarios() -> Vec<GroupRetryScenario> {
2858 use RetryExpected::{FindCoordinator as FetchFindError, Timeout as FetchTimeout};
2859
2860 vec![
2861 successful_retry_case("no error", &[0], &[0], (1, 1)),
2862 successful_retry_case("coordinator load in progress", &[0], &[14, 14, 0], (1, 3)),
2863 successful_retry_case("coordinator not available", &[0], &[15, 0], (2, 2)),
2864 successful_retry_case("not coordinator", &[0], &[16, 0], (2, 2)),
2865 timed_out_retry_case(
2866 "load in progress timeout",
2867 RetryCodes {
2868 find_coordinator: &[0],
2869 offset_fetch: &[14],
2870 offset_commit: &[27],
2871 },
2872 ),
2873 timed_out_retry_case(
2874 "not coordinator timeout",
2875 RetryCodes {
2876 find_coordinator: &[0],
2877 offset_fetch: &[16],
2878 offset_commit: &[16],
2879 },
2880 ),
2881 retry_scenario(
2882 "group error is final",
2883 RetryCodes {
2884 find_coordinator: &[0],
2885 offset_fetch: &[30],
2886 offset_commit: &[25],
2887 },
2888 GROUP_RETRY_LONG,
2889 RetryExpectations {
2890 fetch: RetryExpected::OffsetFetch(30),
2891 commit: RetryExpected::CommitOutcome(25),
2892 },
2893 (1, 1),
2894 ),
2895 retry_scenario(
2896 "another group error is final",
2897 RetryCodes {
2898 find_coordinator: &[0],
2899 offset_fetch: &[7],
2900 offset_commit: &[7],
2901 },
2902 GROUP_RETRY_LONG,
2903 RetryExpectations {
2904 fetch: RetryExpected::OffsetFetch(7),
2905 commit: RetryExpected::CommitOutcome(7),
2906 },
2907 (1, 1),
2908 ),
2909 coordinator_recovery_case("find coordinator not available then succeeds", &[15, 0]),
2910 coordinator_recovery_case("find coordinator load in progress then succeeds", &[14, 0]),
2911 retry_scenario(
2912 "find coordinator timeout",
2913 RetryCodes {
2914 find_coordinator: &[15],
2915 offset_fetch: &[0],
2916 offset_commit: &[0],
2917 },
2918 GROUP_RETRY_NOW,
2919 RetryExpectations {
2920 fetch: FetchTimeout,
2921 commit: FetchTimeout,
2922 },
2923 (1, 0),
2924 ),
2925 retry_scenario(
2926 "find coordinator authorization error",
2927 RetryCodes {
2928 find_coordinator: &[30],
2929 offset_fetch: &[0],
2930 offset_commit: &[0],
2931 },
2932 GROUP_RETRY_LONG,
2933 RetryExpectations {
2934 fetch: FetchFindError(30),
2935 commit: FetchFindError(30),
2936 },
2937 (1, 0),
2938 ),
2939 ]
2940 }
2941
2942 const TIMED_OUT: i16 = 7;
2943
2944 #[derive(Clone, Copy)]
2945 enum RetryApi {
2946 Fetch,
2947 Commit,
2948 }
2949
2950 #[derive(Debug, PartialEq)]
2951 enum RetryObservation {
2952 Fetch(Result<GroupOffsets, RetryExpected>),
2953 Commit(Result<Vec<ConsumerGroupOffsetOutcome>, i16>),
2954 }
2955
2956 async fn alter_consumer_offsets_with_retry(
2957 mut admin: AdminClient,
2958 timeout: Duration,
2959 ) -> Result<Vec<ConsumerGroupOffsetOutcome>, AdminError> {
2960 let offsets = test_group_offsets();
2961 admin
2962 .alter_group_offsets_with_retry(
2963 GroupOffsetUpdate {
2964 kind: GroupOffsetKind::Consumer,
2965 group: "workers",
2966 offsets: &offsets,
2967 },
2968 test_retry_policy(timeout, group_retry_backoff(timeout)),
2969 )
2970 .await
2971 }
2972
2973 async fn assert_retry_scenario(case: GroupRetryScenario, api: RetryApi) {
2974 let expected = match (api, case.fetch_expected, case.commit_expected) {
2975 (RetryApi::Fetch, RetryExpected::Success, _) => {
2976 RetryObservation::Fetch(Ok(test_group_offsets()))
2977 }
2978 (RetryApi::Fetch, RetryExpected::OffsetFetch(code), _) => {
2979 RetryObservation::Fetch(Err(RetryExpected::OffsetFetch(code)))
2980 }
2981 (RetryApi::Fetch, RetryExpected::FindCoordinator(code), _) => {
2982 RetryObservation::Fetch(Err(RetryExpected::FindCoordinator(code)))
2983 }
2984 (RetryApi::Fetch, RetryExpected::Timeout, _) => {
2985 RetryObservation::Fetch(Err(RetryExpected::Timeout))
2986 }
2987 (RetryApi::Commit, _, RetryExpected::CommitOutcome(code)) => {
2988 RetryObservation::Commit(Ok(vec![ConsumerGroupOffsetOutcome {
2989 topic: "orders".into(),
2990 partition: 2,
2991 error: kafka_error_if(code, None),
2992 }]))
2993 }
2994 (RetryApi::Commit, _, RetryExpected::FindCoordinator(code)) => {
2995 RetryObservation::Commit(Err(code))
2996 }
2997 (RetryApi::Commit, _, RetryExpected::Timeout) => {
2998 RetryObservation::Commit(Err(TIMED_OUT))
2999 }
3000 _ => unreachable!("retry expectation belongs to another API"),
3001 };
3002 let expected_calls = match api {
3003 RetryApi::Fetch => case.fetch_calls,
3004 RetryApi::Commit => case.commit_calls,
3005 };
3006 let mut script = RetryScript {
3007 find_coordinator: case.find_coordinator.to_vec(),
3008 ..RetryScript::default()
3009 };
3010 match api {
3011 RetryApi::Fetch => script.offset_fetch = case.offset_fetch.to_vec(),
3012 RetryApi::Commit => script.offset_commit = case.offset_commit.to_vec(),
3013 }
3014
3015 let ((result, within_deadline), script) =
3016 run_scripted_call(script, |mut admin| async move {
3017 let started = tokio::time::Instant::now();
3018 let result = match api {
3019 RetryApi::Fetch => {
3020 let result = admin
3021 .list_consumer_group_offsets_with_retry(
3022 "workers",
3023 test_retry_policy(case.timeout, group_retry_backoff(case.timeout)),
3024 )
3025 .await
3026 .map_err(|error| match error {
3027 AdminError::Broker {
3028 api: "OffsetFetch",
3029 code,
3030 ..
3031 } => RetryExpected::OffsetFetch(code),
3032 AdminError::Transport(ClientError::Server { error_code }) => {
3033 RetryExpected::FindCoordinator(error_code)
3034 }
3035 AdminError::Transport(ClientError::Timeout(_)) => {
3036 RetryExpected::Timeout
3037 }
3038 other => panic!("case {}: unexpected error {other:?}", case.name),
3039 });
3040 RetryObservation::Fetch(result)
3041 }
3042 RetryApi::Commit => {
3043 let result = alter_consumer_offsets_with_retry(admin, case.timeout)
3044 .await
3045 .map_err(|error| match error {
3046 AdminError::Transport(ClientError::Server { error_code }) => {
3047 error_code
3048 }
3049 AdminError::Transport(ClientError::Timeout(_)) => TIMED_OUT,
3050 other => panic!("case {}: unexpected error {other:?}", case.name),
3051 });
3052 RetryObservation::Commit(result)
3053 }
3054 };
3055 let within_deadline = match api {
3056 RetryApi::Fetch => true,
3057 RetryApi::Commit => started.elapsed() < case.timeout + Duration::from_secs(2),
3058 };
3059 (result, within_deadline)
3060 })
3061 .await;
3062 let rpc_calls = match api {
3063 RetryApi::Fetch => script.offset_fetch_requests,
3064 RetryApi::Commit => script.offset_commit_requests,
3065 };
3066 assert!(
3067 (
3068 result,
3069 script.find_coordinator_requests,
3070 rpc_calls,
3071 within_deadline
3072 ) == (expected, expected_calls.0, expected_calls.1, true),
3073 "case {}",
3074 case.name
3075 );
3076 }
3077
3078 #[yare::parameterized(
3081 no_error = { 0 },
3082 coordinator_load_in_progress = { 1 },
3083 coordinator_not_available = { 2 },
3084 not_coordinator = { 3 },
3085 load_timeout = { 4 },
3086 coordinator_timeout = { 5 },
3087 group_error = { 6 },
3088 other_group_error = { 7 },
3089 find_coordinator_not_available = { 8 },
3090 find_coordinator_load = { 9 },
3091 find_coordinator_timeout = { 10 },
3092 find_coordinator_authorization = { 11 },
3093 )]
3094 fn group_offset_retry_scenarios_match_kafka(index: usize) {
3095 block_on_test(async move {
3096 let case = group_retry_scenarios()[index];
3097 assert_retry_scenario(case, RetryApi::Fetch).await;
3098 assert_retry_scenario(case, RetryApi::Commit).await;
3099 });
3100 }
3101
3102 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
3103 enum Coordinator {
3104 UnreachableOnce,
3105 Silent,
3106 ClosedOnce,
3107 }
3108
3109 #[yare::parameterized(
3112 closed_connection = { "closed connection", GROUP_RETRY_LONG, Coordinator::ClosedOnce, 1, 2, Ok(0) },
3113 unreachable_coordinator = { "unreachable coordinator", GROUP_RETRY_LONG, Coordinator::UnreachableOnce, 2, 1, Ok(0) },
3114 silent_coordinator = { "silent coordinator", Duration::from_millis(300), Coordinator::Silent, 1, 1, Err(TIMED_OUT) },
3115 )]
3116 #[test_macro(tokio::test(flavor = "multi_thread", worker_threads = 2))]
3117 async fn alter_consumer_group_offsets_recovers_from_connection_failures(
3118 name: &'static str,
3119 timeout: Duration,
3120 coordinator: Coordinator,
3121 find_calls: usize,
3122 commit_calls: usize,
3123 expected: Result<i16, i16>,
3124 ) {
3125 let script = RetryScript {
3126 find_coordinator: vec![0],
3127 offset_commit: vec![0],
3128 unreachable_answers: usize::from(coordinator == Coordinator::UnreachableOnce),
3129 unreachable: Some(refused_address().await),
3130 silent_offset_commit: coordinator == Coordinator::Silent,
3131 closed_offset_commits: usize::from(coordinator == Coordinator::ClosedOnce),
3132 ..RetryScript::default()
3133 };
3134 let ((result, within_deadline), script) = run_scripted_call(script, |admin| async move {
3135 let started = tokio::time::Instant::now();
3136 let result = alter_consumer_offsets_with_retry(admin, timeout)
3137 .await
3138 .map(|_| 0_i16)
3139 .map_err(|error| match error {
3140 AdminError::Transport(ClientError::Server { error_code }) => error_code,
3141 AdminError::Transport(ClientError::Timeout(_)) => TIMED_OUT,
3142 other => panic!("case {name}: unexpected error {other:?}"),
3143 });
3144 (result, started.elapsed() < timeout + Duration::from_secs(2))
3145 })
3146 .await;
3147 assert2::assert!(
3148 (
3149 result,
3150 script.find_coordinator_requests,
3151 script.offset_commit_requests,
3152 within_deadline
3153 ) == (expected, find_calls, commit_calls, true),
3154 "case {name}"
3155 );
3156 }
3157
3158 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3159 async fn group_offset_rpcs_use_the_group_coordinator() {
3160 let ranges = GroupRanges {
3161 offset_commit: (2, 10),
3162 offset_fetch: (2, 10),
3163 };
3164 let coordinator_group_rpcs = Arc::new(AtomicUsize::new(0));
3165 let seen = Arc::clone(&coordinator_group_rpcs);
3166 let coordinator = MockBroker::start(move |api_key, version, _, _| match api_key {
3167 api_versions_request::API_KEY => Some(api_versions(ranges)),
3168 offset_commit_request::API_KEY => {
3169 seen.fetch_add(1, Ordering::SeqCst);
3170 Some(crate::encode_test_struct_field!(
3171 encode,
3172 OffsetCommitResponse,
3173 topics,
3174 vec![successful_offset_commit_topic()],
3175 version,
3176 true,
3177 ))
3178 }
3179 offset_fetch_request::API_KEY => {
3180 counted_reply(&seen, Some(offset_fetch_reply(version)))
3181 }
3182 _ => None,
3183 })
3184 .await;
3185
3186 let bootstrap_group_rpcs = Arc::new(AtomicUsize::new(0));
3187 let bootstrap =
3188 bootstrap_for(&coordinator, ranges, Arc::clone(&bootstrap_group_rpcs)).await;
3189 let mut admin = connect_admin(bootstrap.addr, "admin connects").await;
3190
3191 let committed = admin
3192 .alter_consumer_group_offsets("workers", &test_group_offsets())
3193 .await
3194 .expect("offset commit succeeds");
3195 let mut admin = connect_admin(bootstrap.addr, "second admin connects").await;
3196 let fetched = admin
3197 .list_consumer_group_offsets("workers")
3198 .await
3199 .expect("offset fetch succeeds");
3200
3201 assert!(bootstrap_group_rpcs.load(Ordering::SeqCst) == 0);
3202 assert!(coordinator_group_rpcs.load(Ordering::SeqCst) == 2);
3203 assert!(committed == successful_group_offset_outcome());
3204 assert!(fetched == test_group_offsets());
3205 bootstrap.stop();
3206 coordinator.stop();
3207 }
3208
3209 #[derive(Clone)]
3211 enum ListGroupsAnswer {
3212 Groups(Vec<ListedGroup>),
3215 Error(i16),
3217 }
3218
3219 #[derive(Clone)]
3222 struct ListGroupsBroker {
3223 max_version: i16,
3224 answers: Vec<ListGroupsAnswer>,
3225 }
3226
3227 fn listed(group_id: &str, protocol_type: &str, state: &str, group_type: &str) -> ListedGroup {
3228 ListedGroup {
3229 group_id: group_id.into(),
3230 protocol_type: protocol_type.into(),
3231 group_state: state.into(),
3232 group_type: group_type.into(),
3233 ..Default::default()
3234 }
3235 }
3236
3237 fn consumer_group(group_id: &str) -> ListedGroup {
3238 listed(group_id, "consumer", "Stable", "Classic")
3239 }
3240
3241 fn stable_listing(group_id: &str) -> GroupListing {
3242 GroupListing {
3243 group_id: group_id.into(),
3244 group_type: Some(GroupType::Classic),
3245 protocol_type: "consumer".into(),
3246 group_state: Some(GroupState::Stable),
3247 }
3248 }
3249
3250 fn stable_listings(group_ids: &[&str]) -> Vec<GroupListing> {
3251 group_ids
3252 .iter()
3253 .map(|group_id| stable_listing(group_id))
3254 .collect()
3255 }
3256
3257 fn groups(ids: &[&str]) -> ListGroupsBroker {
3258 ListGroupsBroker {
3259 max_version: 5,
3260 answers: vec![ListGroupsAnswer::Groups(
3261 ids.iter().map(|id| consumer_group(id)).collect(),
3262 )],
3263 }
3264 }
3265
3266 async fn list_groups_broker(
3270 script: ListGroupsBroker,
3271 cluster: Arc<Mutex<Vec<std::net::SocketAddr>>>,
3272 requests: Arc<Mutex<Vec<(i16, ListGroupsRequest)>>>,
3273 ) -> MockBroker {
3274 MockBroker::start_with_replies(move |api_key, version, _, body| {
3275 MockSaslAnswer::Accept
3276 .reply(api_key, version)
3277 .unwrap_or_else(|| {
3278 list_groups_reply(&script, &cluster, &requests, api_key, version, body)
3279 .map_or(MockReply::Silent, MockReply::Respond)
3280 })
3281 })
3282 .await
3283 }
3284
3285 fn list_groups_reply(
3287 script: &ListGroupsBroker,
3288 cluster: &Mutex<Vec<std::net::SocketAddr>>,
3289 requests: &Mutex<Vec<(i16, ListGroupsRequest)>>,
3290 api_key: i16,
3291 version: i16,
3292 body: &[u8],
3293 ) -> Option<Vec<u8>> {
3294 match api_key {
3295 api_versions_request::API_KEY => Some(encode_api_versions(vec![
3296 api_version_entry!(api_versions_request::API_KEY, 0, 0),
3297 api_version_entry!(metadata_request::API_KEY, 12, 12),
3298 api_version_entry!(list_groups_request::API_KEY, 0, script.max_version),
3299 ])),
3300 metadata_request::API_KEY => Some(crate::encode_test_struct_field!(
3301 encode,
3302 MetadataResponse,
3303 brokers,
3304 cluster
3305 .lock()
3306 .expect("cluster lock")
3307 .iter()
3308 .zip(1..)
3309 .map(|(addr, node_id)| mock_metadata_broker(*addr, node_id))
3310 .collect(),
3311 version,
3312 true,
3313 )),
3314 list_groups_request::API_KEY => {
3315 let request: ListGroupsRequest = decode_request(body, version);
3316 let mut requests = requests.lock().expect("requests lock");
3317 let answer = &script.answers[requests.len().min(script.answers.len() - 1)];
3318 let response = match answer {
3319 ListGroupsAnswer::Groups(groups) => ListGroupsResponse {
3320 groups: groups
3321 .iter()
3322 .filter(|group| {
3323 (request.states_filter.is_empty()
3324 || request.states_filter.contains(&group.group_state))
3325 && (request.types_filter.is_empty()
3326 || request.types_filter.contains(&group.group_type))
3327 })
3328 .cloned()
3329 .collect(),
3330 ..Default::default()
3331 },
3332 ListGroupsAnswer::Error(error_code) => ListGroupsResponse {
3333 error_code: *error_code,
3334 ..Default::default()
3335 },
3336 };
3337 requests.push((version, request));
3338 Some(encode(
3339 &response,
3340 version,
3341 version >= list_groups_request::FLEXIBLE_MIN,
3342 ))
3343 }
3344 _ => None,
3345 }
3346 }
3347
3348 type ExpectedError = (i32, i16, Option<String>);
3350
3351 struct ListGroupsCase {
3353 name: &'static str,
3354 brokers: [ListGroupsBroker; 3],
3355 options: ListGroupsOptions,
3356 timeout: Duration,
3357 valid: Vec<GroupListing>,
3358 errors: Vec<ExpectedError>,
3359 requests: [Vec<(i16, ListGroupsRequest)>; 3],
3361 }
3362
3363 const LONG: Duration = Duration::from_secs(10);
3365 const SHORT: Duration = Duration::from_millis(500);
3367 const BACKOFF: Duration = Duration::from_secs(1);
3369
3370 #[derive(Default)]
3371 struct ListGroupsCaseBuilder {
3372 name: &'static str,
3373 brokers: Option<[ListGroupsBroker; 3]>,
3374 options: ListGroupsOptions,
3375 timeout: Option<Duration>,
3376 valid: Vec<GroupListing>,
3377 errors: Vec<ExpectedError>,
3378 requests: Option<[Vec<(i16, ListGroupsRequest)>; 3]>,
3379 }
3380
3381 impl ListGroupsCase {
3382 fn builder() -> ListGroupsCaseBuilder {
3383 ListGroupsCaseBuilder::default()
3384 }
3385 }
3386
3387 impl ListGroupsCaseBuilder {
3388 fn name(mut self, name: &'static str) -> Self {
3389 self.name = name;
3390 self
3391 }
3392
3393 fn brokers(mut self, brokers: [ListGroupsBroker; 3]) -> Self {
3394 self.brokers = Some(brokers);
3395 self
3396 }
3397
3398 fn options(mut self, options: ListGroupsOptions) -> Self {
3399 self.options = options;
3400 self
3401 }
3402
3403 fn timeout(mut self, timeout: Duration) -> Self {
3404 self.timeout = Some(timeout);
3405 self
3406 }
3407
3408 fn valid(mut self, valid: Vec<GroupListing>) -> Self {
3409 self.valid = valid;
3410 self
3411 }
3412
3413 fn errors(mut self, errors: Vec<ExpectedError>) -> Self {
3414 self.errors = errors;
3415 self
3416 }
3417
3418 fn requests(mut self, requests: [Vec<(i16, ListGroupsRequest)>; 3]) -> Self {
3419 self.requests = Some(requests);
3420 self
3421 }
3422
3423 fn build(self) -> ListGroupsCase {
3424 ListGroupsCase {
3425 name: self.name,
3426 brokers: self.brokers.expect("brokers must be set"),
3427 options: self.options,
3428 timeout: self.timeout.unwrap_or(LONG),
3429 valid: self.valid,
3430 errors: self.errors,
3431 requests: self
3432 .requests
3433 .unwrap_or_else(|| [vec![plain(5)], vec![plain(5)], vec![plain(5)]]),
3434 }
3435 }
3436 }
3437
3438 fn plain(version: i16) -> (i16, ListGroupsRequest) {
3439 (version, ListGroupsRequest::default())
3440 }
3441
3442 fn stable(version: i16) -> (i16, ListGroupsRequest) {
3443 (
3444 version,
3445 ListGroupsRequest {
3446 states_filter: stable_state_filter(),
3447 ..Default::default()
3448 },
3449 )
3450 }
3451
3452 fn classic_and_consumer(version: i16) -> (i16, ListGroupsRequest) {
3455 (
3456 version,
3457 ListGroupsRequest {
3458 types_filter: if version >= 5 {
3459 vec!["Consumer".into(), "Classic".into()]
3460 } else {
3461 Vec::new()
3462 },
3463 ..Default::default()
3464 },
3465 )
3466 }
3467
3468 fn consumer_type(version: i16) -> (i16, ListGroupsRequest) {
3469 (
3470 version,
3471 ListGroupsRequest {
3472 types_filter: vec!["Consumer".into()],
3473 ..Default::default()
3474 },
3475 )
3476 }
3477
3478 fn unsupported(broker_max: i16, client_min: i16) -> String {
3481 AdminError::Transport(ClientError::IncompatibleVersion {
3482 api_key: list_groups_request::API_KEY,
3483 broker_min: 0,
3484 broker_max,
3485 client_min,
3486 client_max: 5,
3487 })
3488 .to_string()
3489 }
3490
3491 fn states(states: &[GroupState]) -> ListGroupsOptions {
3492 ListGroupsOptions::builder()
3493 .group_states(states.iter().copied().collect())
3494 .build()
3495 }
3496
3497 fn stable_state_filter() -> Vec<String> {
3498 vec![GroupState::Stable.to_string()]
3499 }
3500
3501 fn single_broker_error() -> Vec<ExpectedError> {
3502 vec![(2, 31, None)]
3503 }
3504
3505 fn expected_kafka_error(code: i16, message: Option<String>) -> KafkaError {
3506 kafka_error_if(code, message).expect("error code is nonzero")
3507 }
3508
3509 fn types(types: &[GroupType]) -> ListGroupsOptions {
3510 ListGroupsOptions::builder()
3511 .types(types.iter().copied().collect())
3512 .build()
3513 }
3514
3515 fn broker_error(code: i16) -> ListGroupsBroker {
3516 ListGroupsBroker {
3517 max_version: 5,
3518 answers: vec![ListGroupsAnswer::Error(code)],
3519 }
3520 }
3521
3522 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3525 async fn list_groups_stops_a_broker_after_the_retry_limit() {
3526 let cluster = Arc::new(Mutex::new(Vec::new()));
3527 let sent = Arc::new(Mutex::new(Vec::new()));
3528 let broker =
3529 list_groups_broker(broker_error(14), Arc::clone(&cluster), Arc::clone(&sent)).await;
3530 *cluster.lock().expect("cluster lock") = vec![broker.addr];
3531 let admin = connect_admin(broker.addr, "admin connects").await;
3532
3533 let options = ListGroupsOptions::builder().build();
3534 let result = run_list_groups_with_retry(
3535 &admin,
3536 &options,
3537 list_groups_retry_policy(Duration::from_secs(10), Duration::from_millis(1), 2),
3538 )
3539 .await;
3540
3541 broker.stop();
3542 let codes = result
3543 .errors
3544 .iter()
3545 .map(|error| error.error.code)
3546 .collect::<Vec<_>>();
3547 assert!((codes, sent.lock().expect("sent lock").len()) == (vec![7], 3));
3548 }
3549
3550 fn below(max_version: i16, ids: &[&str]) -> ListGroupsBroker {
3551 ListGroupsBroker {
3552 max_version,
3553 ..groups(ids)
3554 }
3555 }
3556
3557 async fn run_list_groups_case(case: ListGroupsCase) {
3560 let cluster = Arc::new(Mutex::new(Vec::new()));
3561 let mut brokers = Vec::new();
3562 let mut requests = Vec::new();
3563 for script in case.brokers {
3564 let sent = Arc::new(Mutex::new(Vec::new()));
3565 brokers.push(list_groups_broker(script, Arc::clone(&cluster), Arc::clone(&sent)).await);
3566 requests.push(sent);
3567 }
3568 *cluster.lock().expect("cluster lock") = brokers.iter().map(|broker| broker.addr).collect();
3569 let admin = connect_admin(brokers[0].addr, "admin connects").await;
3570
3571 let result = run_list_groups_with_retry(
3572 &admin,
3573 &case.options,
3574 list_groups_retry_policy(case.timeout, BACKOFF, u32::MAX),
3575 )
3576 .await;
3577
3578 let expected = ListGroupsResult {
3579 valid: case.valid,
3580 errors: case
3581 .errors
3582 .into_iter()
3583 .map(|(node_id, code, message)| {
3584 let addr = brokers[usize::try_from(node_id - 1).expect("node id")].addr;
3585 ListGroupsError {
3586 node_id,
3587 host: addr.ip().to_string(),
3588 port: broker_port(addr),
3589 error: expected_kafka_error(code, message),
3590 }
3591 })
3592 .collect(),
3593 };
3594 for broker in brokers {
3595 broker.stop();
3596 }
3597 let requests = requests
3598 .iter()
3599 .map(|sent| sent.lock().expect("requests lock").clone())
3600 .collect::<Vec<_>>();
3601 let name = case.name;
3602 assert!(
3603 (result, requests) == (expected, case.requests.to_vec()),
3604 "case {name}"
3605 );
3606 }
3607
3608 async fn run_list_groups_with_retry(
3609 admin: &AdminClient,
3610 options: &ListGroupsOptions,
3611 policy: RetryPolicy,
3612 ) -> ListGroupsResult {
3613 admin
3614 .list_groups_with_retry(options, policy)
3615 .await
3616 .expect("list_groups succeeds")
3617 }
3618
3619 fn block_on_test(future: impl std::future::Future<Output = ()> + Send + 'static) {
3620 tokio::runtime::Builder::new_multi_thread()
3621 .worker_threads(2)
3622 .enable_all()
3623 .build()
3624 .expect("tokio runtime")
3625 .block_on(future);
3626 }
3627
3628 fn run_list_groups_case_sync(case: ListGroupsCase) {
3629 block_on_test(run_list_groups_case(case));
3630 }
3631
3632 #[yare::parameterized(
3638 one_group_each_broker = { 0 },
3639 empty_broker = { 1 },
3640 duplicate_group = { 2 },
3641 broker_timeout = { 3 },
3642 broker_retries = { 4 },
3643 authorization_error = { 5 },
3644 )]
3645 fn list_groups_asks_every_broker_and_merges(index: usize) {
3646 run_list_groups_case_sync(list_groups_merge_cases().remove(index));
3647 }
3648
3649 #[yare::parameterized(
3655 state_filter = { 0 },
3656 old_state_filter = { 1 },
3657 consumer_type_filter = { 2 },
3658 old_consumer_type_pair = { 3 },
3659 protocol_filter = { 4 },
3660 )]
3661 fn list_groups_sends_filters_as_kafka_does(index: usize) {
3662 run_list_groups_case_sync(list_groups_filter_cases().remove(index));
3663 }
3664
3665 #[derive(Clone, Copy, Debug)]
3668 enum SecuredBroker {
3669 Stalls,
3671 RejectsAuthentication,
3673 }
3674
3675 struct StartedSecuredBroker {
3678 addr: std::net::SocketAddr,
3679 connections: Arc<AtomicUsize>,
3680 mock: Option<MockBroker>,
3681 listener: Option<tokio::task::JoinHandle<()>>,
3682 }
3683
3684 impl SecuredBroker {
3685 async fn start(self) -> StartedSecuredBroker {
3686 let connections = Arc::new(AtomicUsize::new(0));
3687 let counter = Arc::clone(&connections);
3688 match self {
3689 Self::Stalls => {
3690 let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
3691 .await
3692 .expect("listener binds");
3693 let addr = listener.local_addr().expect("listener address");
3694 let task = tokio::spawn(async move {
3695 let mut open = Vec::new();
3696 while let Ok((stream, _)) = listener.accept().await {
3697 counter.fetch_add(1, Ordering::SeqCst);
3698 open.push(stream);
3699 }
3700 });
3701 StartedSecuredBroker {
3702 addr,
3703 connections,
3704 mock: None,
3705 listener: Some(task),
3706 }
3707 }
3708 Self::RejectsAuthentication => {
3709 let mock = MockBroker::start_with_replies(move |api_key, version, _, _| {
3710 if api_key == sasl_handshake_request::API_KEY {
3711 counter.fetch_add(1, Ordering::SeqCst);
3712 }
3713 MockSaslAnswer::AuthenticateError(SASL_AUTHENTICATION_FAILED)
3714 .reply(api_key, version)
3715 .unwrap_or(MockReply::Silent)
3716 })
3717 .await;
3718 StartedSecuredBroker {
3719 addr: mock.addr,
3720 connections,
3721 mock: Some(mock),
3722 listener: None,
3723 }
3724 }
3725 }
3726 }
3727
3728 fn expected_error(self, addr: std::net::SocketAddr) -> (i16, Option<String>) {
3731 match self {
3732 Self::Stalls => (
3733 7,
3734 Some("ListGroups timed out after 1 attempt(s)".to_owned()),
3735 ),
3736 Self::RejectsAuthentication => (
3737 SASL_AUTHENTICATION_FAILED,
3738 Some(
3739 AdminError::Transport(ClientError::Authentication {
3740 addr,
3741 source: AuthenticationError::Sasl(SaslAuthenticationError::Failed(
3742 "SaslAuthenticate(PLAIN) error_code=58 \
3743 error_message=Some(\"rejected by mock broker\")"
3744 .to_owned(),
3745 )),
3746 })
3747 .to_string(),
3748 ),
3749 ),
3750 }
3751 }
3752 }
3753
3754 impl StartedSecuredBroker {
3755 fn stop(self) {
3756 if let Some(mock) = self.mock {
3757 mock.stop();
3758 }
3759 if let Some(listener) = self.listener {
3760 listener.abort();
3761 }
3762 }
3763 }
3764
3765 const SASL_AUTHENTICATION_FAILED: i16 = 58;
3766
3767 #[yare::parameterized(
3783 stalls = { SecuredBroker::Stalls },
3784 rejects_authentication = { SecuredBroker::RejectsAuthentication },
3785 )]
3786 #[test_macro(tokio::test(flavor = "multi_thread", worker_threads = 2))]
3787 async fn list_groups_isolates_a_secured_broker_that_stalls_or_rejects(broker_2: SecuredBroker) {
3788 const RETRY: RetryPolicy = RetryPolicy {
3789 timeout: Duration::from_secs(2),
3790 initial_backoff: Duration::from_millis(100),
3791 max_backoff: Duration::from_millis(100),
3792 jitter: 0.0,
3793 max_retries: u32::MAX,
3794 };
3795 let cluster = Arc::new(Mutex::new(Vec::new()));
3796 let requests = Arc::new(Mutex::new(Vec::new()));
3797 let broker_1 =
3798 list_groups_broker(groups(&["a"]), Arc::clone(&cluster), Arc::clone(&requests)).await;
3799 let broker_2_started = broker_2.start().await;
3800 let broker_3 =
3801 list_groups_broker(groups(&["c"]), Arc::clone(&cluster), Arc::clone(&requests)).await;
3802 let broker_2_addr = broker_2_started.addr;
3803 *cluster.lock().expect("cluster lock") = vec![broker_1.addr, broker_2_addr, broker_3.addr];
3804 let admin = AdminClient::connect_secured(
3805 &[broker_1.addr.to_string()],
3806 Some(ClientSecurity {
3807 protocol: krabka_security::ListenerProtocol::SaslPlaintext,
3808 tls: None,
3809 sasl: Some(SaslCredentials::Plain {
3810 username: "alice".into(),
3811 password: "secret".into(),
3812 }),
3813 sasl_host: None,
3814 }),
3815 )
3816 .await
3817 .expect("admin connects");
3818
3819 let outcome = tokio::time::timeout(
3820 RETRY.timeout * 5,
3821 admin.list_groups_with_retry(&ListGroupsOptions::builder().build(), RETRY),
3822 )
3823 .await
3824 .map(|result| result.expect("list_groups succeeds"));
3825 let observed = (outcome, broker_2_started.connections.load(Ordering::SeqCst));
3826
3827 let (code, message) = broker_2.expected_error(broker_2_addr);
3828 let expected = (
3829 Ok(ListGroupsResult {
3830 valid: stable_listings(&["a", "c"]),
3831 errors: vec![ListGroupsError {
3832 node_id: 2,
3833 host: broker_2_addr.ip().to_string(),
3834 port: broker_port(broker_2_addr),
3835 error: expected_kafka_error(code, message),
3836 }],
3837 }),
3838 1,
3839 );
3840 broker_1.stop();
3841 broker_2_started.stop();
3842 broker_3.stop();
3843 assert2::assert!(observed == expected, "case {broker_2:?}");
3844 }
3845
3846 fn list_groups_merge_cases() -> Vec<ListGroupsCase> {
3847 vec![
3848 ListGroupsCase::builder()
3849 .name("one group on each broker")
3850 .brokers([groups(&["a"]), groups(&["b"]), groups(&["c"])])
3851 .valid(stable_listings(&["a", "b", "c"]))
3852 .build(),
3853 ListGroupsCase::builder()
3854 .name("broker 2 has no groups")
3855 .brokers([groups(&["a"]), groups(&[]), groups(&["c"])])
3856 .valid(stable_listings(&["a", "c"]))
3857 .build(),
3858 ListGroupsCase::builder()
3859 .name("a group on two brokers is listed once")
3860 .brokers([groups(&["a"]), groups(&["a", "b"]), groups(&["c"])])
3861 .valid(stable_listings(&["a", "b", "c"]))
3862 .build(),
3863 ListGroupsCase::builder()
3864 .name("broker 2 answers coordinator not available until the timeout")
3865 .brokers([groups(&["a"]), broker_error(15), groups(&["c"])])
3866 .timeout(SHORT)
3867 .valid(stable_listings(&["a", "c"]))
3868 .errors(vec![(
3869 2,
3870 7,
3871 Some(format!(
3872 "ListGroups timed out after 1 attempt(s); last error: {}",
3873 AdminError::Broker {
3874 api: "ListGroups",
3875 code: 15,
3876 name: "COORDINATOR_NOT_AVAILABLE",
3877 message: None,
3878 }
3879 )),
3880 )])
3881 .build(),
3882 ListGroupsCase::builder()
3883 .name("broker 2 answers coordinator load in progress, then its groups")
3884 .brokers([
3885 groups(&["a"]),
3886 ListGroupsBroker {
3887 max_version: 5,
3888 answers: vec![
3889 ListGroupsAnswer::Error(14),
3890 ListGroupsAnswer::Error(15),
3891 ListGroupsAnswer::Groups(vec![consumer_group("b")]),
3892 ],
3893 },
3894 groups(&["c"]),
3895 ])
3896 .valid(vec![
3897 stable_listing("a"),
3898 stable_listing("b"),
3899 stable_listing("c"),
3900 ])
3901 .requests([
3902 vec![plain(5)],
3903 vec![plain(5), plain(5), plain(5)],
3904 vec![plain(5)],
3905 ])
3906 .build(),
3907 ListGroupsCase::builder()
3908 .name("broker 2 answers cluster authorization failed")
3909 .brokers([groups(&["a"]), broker_error(31), groups(&["c"])])
3910 .valid(stable_listings(&["a", "c"]))
3911 .errors(single_broker_error())
3912 .build(),
3913 ]
3914 }
3915
3916 fn list_groups_filter_cases() -> Vec<ListGroupsCase> {
3917 vec![
3918 ListGroupsCase::builder()
3919 .name("state filter Stable is sent to every broker")
3920 .brokers([
3921 ListGroupsBroker {
3922 max_version: 5,
3923 answers: vec![ListGroupsAnswer::Groups(vec![
3924 listed("a", "consumer", "Empty", "Classic"),
3925 consumer_group("b"),
3926 ])],
3927 },
3928 groups(&["c"]),
3929 groups(&[]),
3930 ])
3931 .options(states(&[GroupState::Stable]))
3932 .valid(stable_listings(&["b", "c"]))
3933 .requests([vec![stable(5)], vec![stable(5)], vec![stable(5)]])
3934 .build(),
3935 ListGroupsCase::builder()
3936 .name("state filter on a broker below v4 is unsupported on that broker")
3937 .brokers([groups(&["a"]), below(3, &["b"]), groups(&["c"])])
3938 .options(states(&[GroupState::Stable]))
3939 .valid(stable_listings(&["a", "c"]))
3940 .errors(vec![(2, 35, Some(unsupported(3, 4)))])
3941 .requests([vec![stable(5)], Vec::new(), vec![stable(5)]])
3942 .build(),
3943 ListGroupsCase::builder()
3944 .name("type filter Consumer on a broker below v5 is unsupported on that broker")
3945 .brokers([groups(&["a"]), below(4, &["b"]), groups(&["c"])])
3946 .options(types(&[GroupType::Consumer]))
3947 .errors(vec![(2, 35, Some(unsupported(4, 5)))])
3948 .requests([vec![consumer_type(5)], Vec::new(), vec![consumer_type(5)]])
3949 .build(),
3950 ListGroupsCase::builder()
3951 .name("type filter Classic and Consumer on a broker below v5 is omitted")
3952 .brokers([groups(&["a"]), below(4, &["b"]), groups(&["c"])])
3953 .options(types(&[GroupType::Classic, GroupType::Consumer]))
3954 .valid(vec![
3955 stable_listing("a"),
3956 GroupListing {
3957 group_type: None,
3958 ..stable_listing("b")
3959 },
3960 stable_listing("c"),
3961 ])
3962 .requests([
3963 vec![classic_and_consumer(5)],
3964 vec![classic_and_consumer(4)],
3965 vec![classic_and_consumer(5)],
3966 ])
3967 .build(),
3968 ListGroupsCase::builder()
3969 .name("protocol type filter is applied by the client")
3970 .brokers([
3971 ListGroupsBroker {
3972 max_version: 5,
3973 answers: vec![ListGroupsAnswer::Groups(vec![
3974 consumer_group("a"),
3975 listed("connect-a", "connect", "Stable", "Classic"),
3976 listed("simple", "", "Empty", "Classic"),
3977 ])],
3978 },
3979 groups(&[]),
3980 ListGroupsBroker {
3981 max_version: 5,
3982 answers: vec![ListGroupsAnswer::Groups(vec![listed(
3983 "share", "share", "Stable", "Share",
3984 )])],
3985 },
3986 ])
3987 .options(ListGroupsOptions::for_consumer_groups())
3988 .valid(vec![
3989 stable_listing("a"),
3990 GroupListing {
3991 group_id: "simple".into(),
3992 group_type: Some(GroupType::Classic),
3993 protocol_type: String::new(),
3994 group_state: Some(GroupState::Empty),
3995 },
3996 ])
3997 .requests([
3998 vec![classic_and_consumer(5)],
3999 vec![classic_and_consumer(5)],
4000 vec![classic_and_consumer(5)],
4001 ])
4002 .build(),
4003 ]
4004 }
4005
4006 #[yare::parameterized(
4008 classic_without_protocol = {
4009 Some(GroupType::Classic),
4010 "",
4011 true,
4012 },
4013 classic_with_protocol = {
4014 Some(GroupType::Classic),
4015 "consumer",
4016 false,
4017 },
4018 consumer = { Some(GroupType::Consumer), "", false },
4019 unknown = { Some(GroupType::Unknown), "", false },
4020 no_type = { None, "", false },
4021 no_type_with_protocol = { None, "consumer", false },
4022 )]
4023 fn is_simple_consumer_group_matches_kafka(
4024 group_type: Option<GroupType>,
4025 protocol_type: &str,
4026 expected: bool,
4027 ) {
4028 let listing = GroupListing {
4029 group_id: "g".into(),
4030 group_type,
4031 protocol_type: protocol_type.into(),
4032 group_state: None,
4033 };
4034 assert2::assert!(listing.is_simple_consumer_group() == expected);
4035 }
4036
4037 #[yare::parameterized(
4041 consumer = {
4042 listed("g", "consumer", "Stable", "Consumer"),
4043 GroupListing {
4044 group_id: "g".into(),
4045 group_type: Some(GroupType::Consumer),
4046 protocol_type: "consumer".into(),
4047 group_state: Some(GroupState::Stable),
4048 },
4049 },
4050 empty = {
4051 listed("g", "", "", ""),
4052 GroupListing {
4053 group_id: "g".into(),
4054 group_type: None,
4055 protocol_type: String::new(),
4056 group_state: None,
4057 },
4058 },
4059 streams = {
4060 listed("g", "consumer", "preparingrebalance", "STREAMS"),
4061 GroupListing {
4062 group_id: "g".into(),
4063 group_type: Some(GroupType::Streams),
4064 protocol_type: "consumer".into(),
4065 group_state: Some(GroupState::PreparingRebalance),
4066 },
4067 },
4068 unknown = {
4069 listed("g", "consumer", "Rebalancing", "Future"),
4070 GroupListing {
4071 group_id: "g".into(),
4072 group_type: Some(GroupType::Unknown),
4073 protocol_type: "consumer".into(),
4074 group_state: Some(GroupState::Unknown),
4075 },
4076 },
4077 )]
4078 fn group_listing_reads_listed_group(group: ListedGroup, expected: GroupListing) {
4079 assert2::assert!(GroupListing::from(group) == expected);
4080 }
4081
4082 fn api_versions_all() -> Vec<u8> {
4087 let api = |api_key, max_version| ApiVersion {
4088 api_key,
4089 min_version: 0,
4090 max_version,
4091 ..Default::default()
4092 };
4093 encode_api_versions(vec![
4094 api(api_versions_request::API_KEY, 0),
4095 api(find_coordinator_request::API_KEY, 0),
4096 api(metadata_request::API_KEY, 13),
4097 api(offset_commit_request::API_KEY, 9),
4098 api(offset_fetch_request::API_KEY, 9),
4099 api(OffsetDeleteRequest::API_KEY, 0),
4100 api(LeaveGroupRequest::API_KEY, 5),
4101 api(AlterShareGroupOffsetsRequest::API_KEY, 1),
4102 api(DeleteShareGroupOffsetsRequest::API_KEY, 0),
4103 ])
4104 }
4105
4106 fn empty_metadata_response(version: i16) -> Vec<u8> {
4107 encode(&MetadataResponse::default(), version, true)
4108 }
4109
4110 fn offset_fetch_response_bytes(version: i16) -> Vec<u8> {
4111 encode(&offset_fetch_response(version), version, true)
4112 }
4113
4114 fn empty_metadata_reply(version: i16) -> Vec<u8> {
4115 empty_metadata_response(version)
4116 }
4117
4118 fn offset_fetch_reply(version: i16) -> Vec<u8> {
4119 offset_fetch_response_bytes(version)
4120 }
4121
4122 fn counted_reply(counter: &AtomicUsize, response: Option<Vec<u8>>) -> Option<Vec<u8>> {
4123 counter.fetch_add(1, Ordering::SeqCst);
4124 response
4125 }
4126
4127 async fn connect_self_coordinating(
4137 target_api_key: i16,
4138 codes: Vec<i16>,
4139 respond: impl Fn(i16, i16) -> Vec<u8> + Send + Sync + 'static,
4140 ) -> (AdminClient, MockBroker, Arc<AtomicUsize>) {
4141 let coordinator_addr = Arc::new(Mutex::new(None::<std::net::SocketAddr>));
4142 let addr_in_mock = Arc::clone(&coordinator_addr);
4143 let script = Arc::new(Mutex::new((codes, 0_usize)));
4144 let calls = Arc::new(AtomicUsize::new(0));
4145 let calls_in_mock = Arc::clone(&calls);
4146 let broker = MockBroker::start(move |api_key, version, _, _| {
4147 if api_key == target_api_key {
4148 calls_in_mock.fetch_add(1, Ordering::SeqCst);
4149 let mut guard = script.lock().expect("script lock");
4150 let (codes, requests) = &mut *guard;
4151 let code = RetryScript::next(codes, requests);
4152 Some(respond(code, version))
4153 } else {
4154 match api_key {
4155 api_versions_request::API_KEY => Some(api_versions_all()),
4156 find_coordinator_request::API_KEY => {
4157 let addr = addr_in_mock
4158 .lock()
4159 .expect("coordinator lock")
4160 .expect("coordinator address set before use");
4161 Some(encode(&mock_coordinator_response(addr, 0), version, false))
4162 }
4163 metadata_request::API_KEY => Some(empty_metadata_reply(version)),
4164 _ => None,
4165 }
4166 }
4167 })
4168 .await;
4169 *coordinator_addr.lock().expect("coordinator lock") = Some(broker.addr);
4170 let admin = connect_admin(broker.addr, "admin connects").await;
4171 (admin, broker, calls)
4172 }
4173
4174 fn successful_group_offset_outcome() -> Vec<ConsumerGroupOffsetOutcome> {
4175 vec![ConsumerGroupOffsetOutcome {
4176 topic: "orders".into(),
4177 partition: 2,
4178 error: None,
4179 }]
4180 }
4181
4182 macro_rules! successful_offset_topic {
4183 ($topic:ident, $partition:ident, $name_field:ident) => {
4184 $topic {
4185 $name_field: "orders".into(),
4186 partitions: vec![$partition {
4187 partition_index: 2,
4188 error_code: 0,
4189 ..Default::default()
4190 }],
4191 ..Default::default()
4192 }
4193 };
4194 }
4195
4196 fn successful_offset_delete_topic() -> OffsetDeleteResponseTopic {
4197 successful_offset_topic!(
4198 OffsetDeleteResponseTopic,
4199 OffsetDeleteResponsePartition,
4200 name
4201 )
4202 }
4203
4204 fn offset_delete_response(error_code: i16, version: i16) -> Vec<u8> {
4205 let topics =
4206 successful_response_rows(error_code, || vec![successful_offset_delete_topic()]);
4207 encode(
4208 &OffsetDeleteResponse {
4209 error_code,
4210 topics,
4211 ..Default::default()
4212 },
4213 version,
4214 false,
4215 )
4216 }
4217
4218 fn offset_commit_success_response(version: i16) -> Vec<u8> {
4219 encode(
4220 &OffsetCommitResponse {
4221 topics: vec![successful_offset_commit_topic()],
4222 ..Default::default()
4223 },
4224 version,
4225 true,
4226 )
4227 }
4228
4229 fn delete_share_offsets_response(error_code: i16, version: i16) -> Vec<u8> {
4230 let responses = successful_response_rows(error_code, || {
4231 vec![DeleteShareGroupOffsetsResponseTopic {
4232 topic_name: "orders".into(),
4233 error_code: 0,
4234 ..Default::default()
4235 }]
4236 });
4237 crate::encode_test_struct_fields!(
4238 encode,
4239 DeleteShareGroupOffsetsResponse,
4240 { error_code: error_code, responses: responses },
4241 version,
4242 true,
4243 )
4244 }
4245
4246 fn successful_offset_commit_topic() -> OffsetCommitResponseTopic {
4247 successful_offset_topic!(
4248 OffsetCommitResponseTopic,
4249 OffsetCommitResponsePartition,
4250 name
4251 )
4252 }
4253
4254 fn successful_share_offset_topic() -> AlterShareGroupOffsetsResponseTopic {
4255 successful_offset_topic!(
4256 AlterShareGroupOffsetsResponseTopic,
4257 AlterShareGroupOffsetsResponsePartition,
4258 topic_name
4259 )
4260 }
4261
4262 fn successful_response_rows<T>(error_code: i16, rows: impl FnOnce() -> Vec<T>) -> Vec<T> {
4263 if error_code == 0 { rows() } else { Vec::new() }
4264 }
4265
4266 fn assert_single_group_offset_success(
4267 result: Result<Vec<ConsumerGroupOffsetOutcome>, String>,
4268 calls: usize,
4269 ) {
4270 assert!((result, calls) == (Ok(successful_group_offset_outcome()), 1));
4271 }
4272
4273 struct CoordinatorRetryCase<T> {
4274 name: &'static str,
4275 codes: Vec<i16>,
4276 expected_calls: usize,
4277 expected: Result<T, i16>,
4278 }
4279
4280 #[bon::bon]
4281 impl<T> CoordinatorRetryCase<T> {
4282 #[builder]
4283 fn success(name: &'static str, codes: Vec<i16>, expected_calls: usize, outcome: T) -> Self {
4284 Self {
4285 name,
4286 codes,
4287 expected_calls,
4288 expected: Ok(outcome),
4289 }
4290 }
4291
4292 fn error(name: &'static str, code: i16) -> Self {
4293 Self {
4294 name,
4295 codes: vec![code],
4296 expected_calls: 1,
4297 expected: Err(code),
4298 }
4299 }
4300 }
4301
4302 impl<T: Clone> CoordinatorRetryCase<T> {
4303 fn standard(outcome: &T) -> Vec<Self> {
4304 vec![
4305 Self::success()
4306 .name("no error")
4307 .codes(vec![0])
4308 .expected_calls(1)
4309 .outcome(outcome.clone())
4310 .call(),
4311 Self::success()
4312 .name("coordinator load in progress retries on the same coordinator")
4313 .codes(vec![14, 0])
4314 .expected_calls(2)
4315 .outcome(outcome.clone())
4316 .call(),
4317 Self::success()
4318 .name("not coordinator finds the coordinator again")
4319 .codes(vec![16, 0])
4320 .expected_calls(2)
4321 .outcome(outcome.clone())
4322 .call(),
4323 ]
4324 }
4325
4326 fn with_rebalance_in_progress(outcome: &T) -> Vec<Self> {
4327 let mut cases = Self::standard(outcome);
4328 cases.push(
4329 Self::success()
4330 .name("rebalance in progress retries on the same coordinator")
4331 .codes(vec![27, 0])
4332 .expected_calls(2)
4333 .outcome(outcome.clone())
4334 .call(),
4335 );
4336 cases
4337 }
4338
4339 fn with_authorization_failure(mut cases: Vec<Self>) -> Vec<Self> {
4340 cases.push(Self::error("group authorization failed is final", 30));
4341 cases
4342 }
4343 }
4344
4345 #[bon::builder]
4346 async fn run_coordinator_retry_cases<T, F, Fut, Resp>(
4347 api_key: i16,
4348 respond: Resp,
4349 cases: Vec<CoordinatorRetryCase<T>>,
4350 action: F,
4351 ) where
4352 T: PartialEq + std::fmt::Debug,
4353 Resp: Fn(i16, i16) -> Vec<u8> + Copy + Send + Sync + 'static,
4354 F: Fn(AdminClient) -> Fut,
4355 Fut: std::future::Future<Output = Result<T, AdminError>>,
4356 {
4357 for case in cases {
4358 let (admin, broker, calls) =
4359 connect_self_coordinating(api_key, case.codes, respond).await;
4360 let result = action(admin).await.map_err(|error| match error {
4361 AdminError::Broker { code, .. } => code,
4362 other => panic!("case {}: unexpected error {other:?}", case.name),
4363 });
4364 broker.stop();
4365 assert!(
4366 (result, calls.load(Ordering::SeqCst)) == (case.expected, case.expected_calls),
4367 "case {}",
4368 case.name
4369 );
4370 }
4371 }
4372
4373 macro_rules! run_retry_cases {
4374 ($api_key:expr, $respond:expr, $cases:expr, $action:expr) => {
4375 run_coordinator_retry_cases()
4376 .api_key($api_key)
4377 .respond($respond)
4378 .cases($cases)
4379 .action($action)
4380 .call()
4381 .await
4382 };
4383 }
4384
4385 async fn delete_consumer_offsets(
4386 mut admin: AdminClient,
4387 ) -> Result<Vec<ConsumerGroupOffsetOutcome>, AdminError> {
4388 admin
4389 .delete_consumer_group_offsets("workers", &[("orders".into(), 2)])
4390 .await
4391 }
4392
4393 async fn alter_share_offsets(
4394 mut admin: AdminClient,
4395 ) -> Result<Vec<ConsumerGroupOffsetOutcome>, AdminError> {
4396 admin
4397 .alter_share_group_offsets("workers", &test_group_offsets())
4398 .await
4399 }
4400
4401 async fn delete_share_offsets(
4402 mut admin: AdminClient,
4403 ) -> Result<Vec<(String, Option<KafkaError>)>, AdminError> {
4404 admin
4405 .delete_share_group_offsets("workers", &["orders".to_string()])
4406 .await
4407 }
4408
4409 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
4414 async fn delete_consumer_group_offsets_retries_coordinator_errors_as_kafka_does() {
4415 let respond = offset_delete_response;
4416 let outcome = successful_group_offset_outcome();
4417 let mut cases = CoordinatorRetryCase::standard(&outcome);
4418 cases.push(CoordinatorRetryCase::error("non-empty group is final", 68));
4419 run_retry_cases!(
4420 OffsetDeleteRequest::API_KEY,
4421 respond,
4422 cases,
4423 delete_consumer_offsets
4424 );
4425 }
4426
4427 #[derive(Clone, Copy)]
4428 enum StreamsGroupOffsetCall {
4429 Delete,
4430 Alter,
4431 }
4432
4433 impl StreamsGroupOffsetCall {
4434 fn api_key(self) -> i16 {
4435 match self {
4436 Self::Delete => OffsetDeleteRequest::API_KEY,
4437 Self::Alter => offset_commit_request::API_KEY,
4438 }
4439 }
4440
4441 fn response(self, version: i16) -> Vec<u8> {
4442 match self {
4443 Self::Delete => offset_delete_response(0, version),
4444 Self::Alter => offset_commit_success_response(version),
4445 }
4446 }
4447
4448 async fn call(
4449 self,
4450 admin: &mut AdminClient,
4451 ) -> Result<Vec<ConsumerGroupOffsetOutcome>, AdminError> {
4452 match self {
4453 Self::Delete => {
4454 admin
4455 .delete_streams_group_offsets("workers", &[("orders".into(), 2)])
4456 .await
4457 }
4458 Self::Alter => {
4459 admin
4460 .alter_streams_group_offsets("workers", &test_group_offsets())
4461 .await
4462 }
4463 }
4464 }
4465 }
4466
4467 #[yare::parameterized(
4470 delete = { StreamsGroupOffsetCall::Delete },
4471 alter = { StreamsGroupOffsetCall::Alter },
4472 )]
4473 #[test_macro(tokio::test(flavor = "multi_thread", worker_threads = 2))]
4474 async fn streams_group_offset_calls_delegate_to_consumer_calls(call: StreamsGroupOffsetCall) {
4475 let respond = move |_code: i16, version: i16| call.response(version);
4476 let (mut admin, broker, calls) =
4477 connect_self_coordinating(call.api_key(), vec![0], respond).await;
4478 let result = call
4479 .call(&mut admin)
4480 .await
4481 .map_err(|error| error.to_string());
4482 broker.stop();
4483 assert_single_group_offset_success(result, calls.load(Ordering::SeqCst));
4484 }
4485
4486 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
4492 async fn remove_members_from_consumer_group_retries_coordinator_errors_as_kafka_does() {
4493 let answered = |instance_id: &str, error_code| MemberResponse {
4494 member_id: String::new(),
4495 group_instance_id: Some(instance_id.into()),
4496 error_code,
4497 ..Default::default()
4498 };
4499 let respond = move |code: i16, version: i16| {
4500 let members = successful_response_rows(code, || {
4501 vec![answered("b", 25), answered("a", 0)]
4503 });
4504 let response = LeaveGroupResponse {
4505 error_code: code,
4506 members,
4507 ..Default::default()
4508 };
4509 encode_flexible(&response, version)
4510 };
4511 let removed = |instance_id: &str| RemovedMember {
4512 member_id: String::new(),
4513 group_instance_id: Some(instance_id.into()),
4514 };
4515 let outcome = RemovedMembers {
4516 members: maplit::btreemap! {
4517 removed("a") => None,
4518 removed("b") => Some(KafkaError {
4519 code: 25,
4520 name: "UNKNOWN_MEMBER_ID",
4521 message: None,
4522 }),
4523 },
4524 };
4525 let cases = CoordinatorRetryCase::with_authorization_failure(
4526 CoordinatorRetryCase::standard(&outcome),
4527 );
4528 run_retry_cases!(
4529 LeaveGroupRequest::API_KEY,
4530 respond,
4531 cases,
4532 |mut admin| async move {
4533 admin
4534 .remove_members_from_consumer_group(
4535 "workers",
4536 &[
4537 MemberToRemove {
4538 group_instance_id: "a".into(),
4539 },
4540 MemberToRemove {
4541 group_instance_id: "b".into(),
4542 },
4543 ],
4544 &RemoveMembersOptions::builder().build(),
4545 )
4546 .await
4547 }
4548 );
4549 }
4550
4551 #[yare::parameterized(
4555 none = { None, DEFAULT_LEAVE_GROUP_REASON.to_owned() },
4556 empty = { Some(String::new()), DEFAULT_LEAVE_GROUP_REASON.to_owned() },
4557 short = { Some("rolling restart".to_owned()), "rolling restart".to_owned() },
4558 long = { Some("r".repeat(300)), "r".repeat(255) },
4559 )]
4560 fn leave_group_reason_matches_kafka(reason: Option<String>, expected: String) {
4561 assert2::assert!(leave_group_reason(reason.as_deref()) == expected);
4562 }
4563
4564 #[test]
4568 fn removed_members_member_result_matches_kafka() {
4569 let member = |instance_id: &str| MemberToRemove {
4570 group_instance_id: instance_id.into(),
4571 };
4572 let fenced = KafkaError {
4573 code: 82,
4574 name: kafka_error_name(82),
4575 message: None,
4576 };
4577 let result = RemovedMembers {
4578 members: maplit::btreemap! {
4579 RemovedMember {
4580 member_id: String::new(),
4581 group_instance_id: Some("a".into()),
4582 } => None,
4583 RemovedMember {
4584 member_id: String::new(),
4585 group_instance_id: Some("b".into()),
4586 } => Some(fenced.clone()),
4587 },
4588 };
4589 assert!(
4590 [member("a"), member("b"), member("c")].map(|member| result.member_result(&member))
4591 == [
4592 Ok(()),
4593 Err(fenced),
4594 Err(KafkaError {
4595 code: -1,
4596 name: "UNKNOWN_SERVER_ERROR",
4597 message: Some(
4598 "Member \"c\" was not included in the removal response".into()
4599 ),
4600 }),
4601 ]
4602 );
4603 }
4604
4605 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
4610 async fn alter_share_group_offsets_retries_coordinator_errors_as_kafka_does() {
4611 let respond = |code: i16, version: i16| {
4612 let responses =
4613 successful_response_rows(code, || vec![successful_share_offset_topic()]);
4614 let response = AlterShareGroupOffsetsResponse {
4615 error_code: code,
4616 responses,
4617 ..Default::default()
4618 };
4619 encode_flexible(&response, version)
4620 };
4621 let outcome = successful_group_offset_outcome();
4622 let mut cases = CoordinatorRetryCase::with_rebalance_in_progress(&outcome);
4623 cases.push(
4624 CoordinatorRetryCase::success()
4625 .name("a final group error is the error of each partition")
4626 .codes(vec![68])
4627 .expected_calls(1)
4628 .outcome(vec![ConsumerGroupOffsetOutcome {
4629 topic: "orders".into(),
4630 partition: 2,
4631 error: Some(KafkaError {
4632 code: 68,
4633 name: "NON_EMPTY_GROUP",
4634 message: None,
4635 }),
4636 }])
4637 .call(),
4638 );
4639 run_retry_cases!(
4640 AlterShareGroupOffsetsRequest::API_KEY,
4641 respond,
4642 cases,
4643 alter_share_offsets
4644 );
4645 }
4646
4647 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
4650 async fn list_streams_group_offsets_delegates_to_offset_fetch() {
4651 let respond =
4652 |_code: i16, version: i16| encode(&offset_fetch_response(version), version, true);
4653 let (mut admin, broker, calls) =
4654 connect_self_coordinating(offset_fetch_request::API_KEY, vec![0], respond).await;
4655 let result = admin
4656 .list_streams_group_offsets("workers")
4657 .await
4658 .map_err(|error| error.to_string());
4659 broker.stop();
4660 assert!((result, calls.load(Ordering::SeqCst)) == (Ok(test_group_offsets()), 1));
4661 }
4662
4663 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
4668 async fn delete_share_group_offsets_retries_coordinator_errors_as_kafka_does() {
4669 let respond = delete_share_offsets_response;
4670 let outcome = vec![("orders".to_string(), None)];
4671 let cases = CoordinatorRetryCase::with_authorization_failure(
4672 CoordinatorRetryCase::with_rebalance_in_progress(&outcome),
4673 );
4674 run_retry_cases!(
4675 DeleteShareGroupOffsetsRequest::API_KEY,
4676 respond,
4677 cases,
4678 delete_share_offsets
4679 );
4680 }
4681}