1use std::{
13 collections::{BTreeMap, BTreeSet},
14 sync::Mutex,
15};
16
17use bytes::{Buf, BufMut};
18use krabka_client_core::{ClientError, CoordinatorEndpoint};
19use krabka_protocol::{
20 Decode, Encode, ProtocolError, ProtocolRequest,
21 owned::{
22 consumer_group_describe_request::{self, ConsumerGroupDescribeRequest},
23 consumer_group_describe_response::{
24 DescribedGroup as ConsumerDescribedGroup, Member as ConsumerMember,
25 },
26 consumer_protocol_assignment::{self, ConsumerProtocolAssignment},
27 describe_groups_request::DescribeGroupsRequest,
28 describe_groups_response::{DescribedGroup as ClassicDescribedGroup, DescribedGroupMember},
29 share_group_describe_request::ShareGroupDescribeRequest,
30 share_group_describe_response::DescribedGroup as ShareDescribedGroup,
31 streams_group_describe_request::{self, StreamsGroupDescribeRequest},
32 streams_group_describe_response::{
33 DescribedGroup as StreamsDescribedGroup, StreamsGroupDescribeResponse,
34 },
35 },
36};
37
38use crate::{
39 AdminClient, AdminError, ClusterNode, KafkaError,
40 cluster::{authorized_operations, node_with_host_port},
41 group_coordinators::{GroupOutcome, group_error_outcome, kafka_error},
42 groups::{GroupState, GroupType, list_groups_kafka_error},
43 kafka_error_name,
44 retry::is_connection_failure,
45 users::AclOperation,
46};
47
48const UNKNOWN_SERVER_ERROR: i16 = -1;
51const UNSUPPORTED_VERSION: i16 = 35;
53const GROUP_ID_NOT_FOUND: i16 = 69;
55const CONSUMER_PROTOCOL_TYPE: &str = "consumer";
58
59pub type ClassicGroupDescription = ClassicDescribedGroup;
63
64pub type ShareGroupDescription = ShareDescribedGroup;
68
69pub type StreamsGroupDescription = StreamsDescribedGroup;
73
74#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, bon::Builder)]
77pub struct DescribeGroupsOptions {
78 #[builder(default)]
81 pub include_authorized_operations: bool,
82}
83
84#[derive(Debug, Clone, PartialEq, Eq)]
87pub struct MemberDescription {
88 pub member_id: String,
89 pub group_instance_id: Option<String>,
90 pub rack_id: Option<String>,
92 pub client_id: String,
93 pub host: String,
94 pub assignment: BTreeSet<(String, i32)>,
96 pub target_assignment: Option<BTreeSet<(String, i32)>>,
99 pub member_epoch: Option<i32>,
101 pub upgraded: Option<bool>,
105}
106
107#[derive(Debug, Clone, PartialEq, Eq)]
110pub struct ConsumerGroupDescription {
111 pub group_id: String,
112 pub is_simple_consumer_group: bool,
114 pub members: Vec<MemberDescription>,
115 pub partition_assignor: String,
118 pub group_type: GroupType,
120 pub group_state: GroupState,
121 pub coordinator: ClusterNode,
123 pub authorized_operations: Option<BTreeSet<AclOperation>>,
125 pub group_epoch: Option<i32>,
127 pub target_assignment_epoch: Option<i32>,
129}
130
131pub type GroupDescriptions<T> = BTreeMap<String, Result<T, KafkaError>>;
134
135#[derive(Clone, Debug, PartialEq, Eq)]
141struct ReleasedStreamsGroupDescribe(StreamsGroupDescribeRequest);
142
143impl Encode for ReleasedStreamsGroupDescribe {
144 fn encode<B: BufMut>(&self, buf: &mut B, version: i16) -> Result<(), ProtocolError> {
145 self.0.encode(buf, version)
146 }
147
148 fn encoded_len(&self, version: i16) -> usize {
149 self.0.encoded_len(version)
150 }
151}
152
153impl ProtocolRequest for ReleasedStreamsGroupDescribe {
154 const API_KEY: i16 = streams_group_describe_request::API_KEY;
155 const MIN_VERSION: i16 = streams_group_describe_request::MIN_VERSION;
156 const MAX_VERSION: i16 = 0;
158 const LATEST_STABLE_VERSION: i16 = Self::MAX_VERSION;
159 const FLEXIBLE_MIN: i16 = streams_group_describe_request::FLEXIBLE_MIN;
160 type Response = StreamsGroupDescribeResponse;
161}
162
163trait GroupDescribeRequest: ProtocolRequest {
164 fn for_groups(groups: Vec<String>, options: DescribeGroupsOptions) -> Self;
165}
166
167macro_rules! impl_group_describe_request {
168 ($request:ty, $groups_field:ident) => {
169 impl GroupDescribeRequest for $request {
170 fn for_groups(groups: Vec<String>, options: DescribeGroupsOptions) -> Self {
171 Self {
172 $groups_field: groups,
173 include_authorized_operations: options.include_authorized_operations,
174 ..Default::default()
175 }
176 }
177 }
178 };
179}
180
181impl_group_describe_request!(ConsumerGroupDescribeRequest, group_ids);
182impl_group_describe_request!(DescribeGroupsRequest, groups);
183impl_group_describe_request!(ShareGroupDescribeRequest, group_ids);
184
185impl GroupDescribeRequest for ReleasedStreamsGroupDescribe {
186 fn for_groups(groups: Vec<String>, options: DescribeGroupsOptions) -> Self {
187 Self(StreamsGroupDescribeRequest {
188 group_ids: groups,
189 include_authorized_operations: options.include_authorized_operations,
190 ..Default::default()
191 })
192 }
193}
194
195trait GroupDescriptionEntry: Sized {
196 fn group_id(&self) -> &str;
197 fn error_code(&self) -> i16;
198 fn error_message(self) -> Option<String>;
199 fn successful_outcome(self) -> GroupOutcome<Self>;
200
201 fn into_outcome(self) -> (String, GroupOutcome<Self>) {
202 let group_id = self.group_id().to_owned();
203 let error_code = self.error_code();
204 let outcome = if error_code == 0 {
205 self.successful_outcome()
206 } else {
207 group_error_outcome(error_code, self.error_message(), &[])
208 };
209 (group_id, outcome)
210 }
211}
212
213macro_rules! impl_group_description_entry {
214 ($entry:ty, $error_message:expr, $successful_outcome:expr) => {
215 impl GroupDescriptionEntry for $entry {
216 fn group_id(&self) -> &str {
217 &self.group_id
218 }
219
220 fn error_message(self) -> Option<String> {
221 ($error_message)(self)
222 }
223
224 fn successful_outcome(self) -> GroupOutcome<Self> {
225 ($successful_outcome)(self)
226 }
227
228 fn error_code(&self) -> i16 {
229 self.error_code
230 }
231 }
232 };
233}
234
235impl_group_description_entry!(
236 ClassicDescribedGroup,
237 |_: ClassicDescribedGroup| None,
238 |group: ClassicDescribedGroup| GroupOutcome::Done(Ok(group))
239);
240impl_group_description_entry!(
241 ShareDescribedGroup,
242 |group: ShareDescribedGroup| group.error_message,
243 |group: ShareDescribedGroup| GroupOutcome::Done(Ok(group))
244);
245impl_group_description_entry!(
246 StreamsDescribedGroup,
247 |group: StreamsDescribedGroup| group.error_message,
248 |group: StreamsDescribedGroup| {
249 if group.topology.is_none() {
250 GroupOutcome::Done(Err(KafkaError {
251 code: UNKNOWN_SERVER_ERROR,
252 name: kafka_error_name(UNKNOWN_SERVER_ERROR),
253 message: Some("Topology information is missing".to_owned()),
254 }))
255 } else {
256 GroupOutcome::Done(Ok(group))
257 }
258 }
259);
260
261trait GroupDescribeApi: GroupDescribeRequest {
262 type Description;
263
264 const METHOD_NAME: &'static str;
265
266 fn outcomes(response: Self::Response) -> BTreeMap<String, GroupOutcome<Self::Description>>;
267}
268
269macro_rules! impl_group_describe_api {
270 ($request:ty, $description:ty, $method_name:literal) => {
271 impl GroupDescribeApi for $request {
272 type Description = $description;
273
274 const METHOD_NAME: &'static str = $method_name;
275
276 fn outcomes(
277 response: Self::Response,
278 ) -> BTreeMap<String, GroupOutcome<Self::Description>> {
279 group_description_outcomes(response.groups)
280 }
281 }
282 };
283}
284
285impl_group_describe_api!(
286 DescribeGroupsRequest,
287 ClassicGroupDescription,
288 "describeClassicGroups"
289);
290impl_group_describe_api!(
291 ShareGroupDescribeRequest,
292 ShareGroupDescription,
293 "describeShareGroups"
294);
295impl_group_describe_api!(
296 ReleasedStreamsGroupDescribe,
297 StreamsGroupDescription,
298 "describeStreamsGroups"
299);
300
301fn group_description_outcomes<Group: GroupDescriptionEntry>(
302 groups: impl IntoIterator<Item = Group>,
303) -> BTreeMap<String, GroupOutcome<Group>> {
304 groups
305 .into_iter()
306 .map(GroupDescriptionEntry::into_outcome)
307 .collect()
308}
309
310fn insert_description_outcomes<Group, Description>(
311 outcomes: &mut BTreeMap<String, GroupOutcome<Description>>,
312 groups: impl IntoIterator<Item = Group>,
313 mut classify: impl FnMut(Group) -> (String, Option<GroupOutcome<Description>>),
314) {
315 for group in groups {
316 let (group_id, outcome) = classify(group);
317 if let Some(outcome) = outcome {
318 outcomes.insert(group_id, outcome);
319 }
320 }
321}
322
323fn consumer_description_outcome(
324 group: ConsumerDescribedGroup,
325 node: &ClusterNode,
326 classic_keys: &mut Vec<String>,
327 classic: &Mutex<BTreeMap<String, Option<String>>>,
328) -> (String, Option<GroupOutcome<ConsumerGroupDescription>>) {
329 let group_id = group.group_id.clone();
330 let outcome = match consumer_answer(group, node) {
331 ConsumerAnswer::Outcome(outcome) => Some(outcome),
332 ConsumerAnswer::Classic { not_found_message } => {
333 classic
334 .lock()
335 .unwrap_or_else(std::sync::PoisonError::into_inner)
336 .insert(group_id.clone(), not_found_message);
337 classic_keys.push(group_id.clone());
338 None
339 }
340 };
341 (group_id, outcome)
342}
343
344fn classic_consumer_description_outcome(
345 group: ClassicDescribedGroup,
346 node: &ClusterNode,
347 classic: &Mutex<BTreeMap<String, Option<String>>>,
348) -> (String, Option<GroupOutcome<ConsumerGroupDescription>>) {
349 let group_id = group.group_id.clone();
350 let not_found_message = classic
351 .lock()
352 .unwrap_or_else(std::sync::PoisonError::into_inner)
353 .get(&group_id)
354 .cloned()
355 .flatten();
356 (
357 group_id,
358 Some(classic_consumer_outcome(group, node, not_found_message)),
359 )
360}
361
362fn coordinator_node(coordinator: &CoordinatorEndpoint) -> ClusterNode {
364 node_with_host_port(
365 coordinator.node_id,
366 coordinator.host.clone(),
367 coordinator.port,
368 )
369}
370
371pub(crate) fn request_error_outcomes<V>(
376 keys: &[String],
377 error: &AdminError,
378) -> BTreeMap<String, GroupOutcome<V>> {
379 keys.iter()
380 .map(|key| {
381 let outcome = if is_connection_failure(error) {
382 GroupOutcome::Unmap(error.to_string())
383 } else {
384 GroupOutcome::Done(Err(list_groups_kafka_error(error)))
385 };
386 (key.clone(), outcome)
387 })
388 .collect()
389}
390
391fn consumer_assignment(
393 assignment: krabka_protocol::owned::common::consumer_group_describe_response::assignment::Assignment,
394) -> BTreeSet<(String, i32)> {
395 assignment_partitions(
396 assignment
397 .topic_partitions
398 .into_iter()
399 .map(|topic| (topic.topic_name, topic.partitions)),
400 )
401}
402
403fn assignment_partitions(
404 topics: impl IntoIterator<Item = (String, Vec<i32>)>,
405) -> BTreeSet<(String, i32)> {
406 topics
407 .into_iter()
408 .fold(BTreeSet::new(), |mut result, (name, partitions)| {
409 result.extend(
410 partitions
411 .into_iter()
412 .map(|partition| (name.clone(), partition)),
413 );
414 result
415 })
416}
417
418fn consumer_member(member: ConsumerMember) -> MemberDescription {
421 MemberDescription {
422 member_id: member.member_id,
423 group_instance_id: member.instance_id,
424 rack_id: member.rack_id,
425 client_id: member.client_id,
426 host: member.client_host,
427 assignment: consumer_assignment(member.assignment),
428 target_assignment: Some(consumer_assignment(member.target_assignment)),
429 member_epoch: Some(member.member_epoch),
430 upgraded: (member.member_type != -1).then_some(member.member_type == 1),
431 }
432}
433
434fn classic_assignment(mut bytes: &[u8]) -> Result<BTreeSet<(String, i32)>, String> {
439 if bytes.is_empty() {
440 return Ok(BTreeSet::new());
441 }
442 if bytes.len() < 2 {
443 return Err("Buffer underflow while parsing consumer protocol's header".to_owned());
444 }
445 let version = bytes.get_i16();
446 if version < consumer_protocol_assignment::MIN_VERSION {
447 return Err(format!("Unsupported assignment version: {version}"));
448 }
449 let version = version.min(consumer_protocol_assignment::MAX_VERSION);
450 let assignment = ConsumerProtocolAssignment::decode(&mut bytes, version)
451 .map_err(|error| format!("Error reading consumer assignment: {error}"))?;
452 Ok(assignment_partitions(
453 assignment
454 .assigned_partitions
455 .into_iter()
456 .map(|topic| (topic.topic, topic.partitions)),
457 ))
458}
459
460fn classic_member(member: DescribedGroupMember) -> Result<MemberDescription, String> {
463 Ok(MemberDescription {
464 assignment: classic_assignment(&member.member_assignment)?,
465 member_id: member.member_id,
466 group_instance_id: member.group_instance_id,
467 rack_id: None,
468 client_id: member.client_id,
469 host: member.client_host,
470 target_assignment: None,
471 member_epoch: None,
472 upgraded: None,
473 })
474}
475
476#[derive(Debug, Clone, PartialEq, Eq)]
478enum ConsumerAnswer {
479 Outcome(GroupOutcome<ConsumerGroupDescription>),
480 Classic {
484 not_found_message: Option<String>,
485 },
486}
487
488fn consumer_answer(group: ConsumerDescribedGroup, coordinator: &ClusterNode) -> ConsumerAnswer {
493 match group.error_code {
494 0 => ConsumerAnswer::Outcome(GroupOutcome::Done(Ok(ConsumerGroupDescription {
495 group_id: group.group_id,
496 is_simple_consumer_group: false,
497 members: group.members.into_iter().map(consumer_member).collect(),
498 partition_assignor: group.assignor_name,
499 group_type: GroupType::Consumer,
500 group_state: GroupState::parse(&group.group_state),
501 coordinator: coordinator.clone(),
502 authorized_operations: authorized_operations(group.authorized_operations),
503 group_epoch: Some(group.group_epoch),
504 target_assignment_epoch: Some(group.assignment_epoch),
505 }))),
506 UNSUPPORTED_VERSION => ConsumerAnswer::Classic {
507 not_found_message: None,
508 },
509 GROUP_ID_NOT_FOUND => ConsumerAnswer::Classic {
510 not_found_message: group.error_message,
511 },
512 code => ConsumerAnswer::Outcome(group_error_outcome(code, group.error_message, &[])),
513 }
514}
515
516fn classic_consumer_outcome(
522 group: ClassicDescribedGroup,
523 coordinator: &ClusterNode,
524 not_found_message: Option<String>,
525) -> GroupOutcome<ConsumerGroupDescription> {
526 match group.error_code {
527 0 => {}
528 GROUP_ID_NOT_FOUND => {
529 return GroupOutcome::Done(Err(kafka_error(
530 GROUP_ID_NOT_FOUND,
531 not_found_message.or(group.error_message),
532 )));
533 }
534 code => return group_error_outcome(code, group.error_message, &[]),
535 }
536 if group.protocol_type != CONSUMER_PROTOCOL_TYPE && !group.protocol_type.is_empty() {
537 return GroupOutcome::Done(Err(kafka_error(
538 UNKNOWN_SERVER_ERROR,
539 Some(format!(
540 "GroupId {} is not a consumer group ({}).",
541 group.group_id, group.protocol_type
542 )),
543 )));
544 }
545 let members = match group
546 .members
547 .into_iter()
548 .map(classic_member)
549 .collect::<Result<Vec<_>, _>>()
550 {
551 Ok(members) => members,
552 Err(message) => {
553 return GroupOutcome::Done(Err(kafka_error(UNKNOWN_SERVER_ERROR, Some(message))));
554 }
555 };
556 GroupOutcome::Done(Ok(ConsumerGroupDescription {
557 is_simple_consumer_group: group.protocol_type.is_empty(),
558 group_id: group.group_id,
559 members,
560 partition_assignor: group.protocol_data,
561 group_type: GroupType::Classic,
562 group_state: GroupState::parse(&group.group_state),
563 coordinator: coordinator.clone(),
564 authorized_operations: authorized_operations(group.authorized_operations),
565 group_epoch: None,
566 target_assignment_epoch: None,
567 }))
568}
569
570impl AdminClient {
571 pub async fn describe_consumer_groups(
598 &self,
599 groups: &[&str],
600 options: DescribeGroupsOptions,
601 ) -> GroupDescriptions<ConsumerGroupDescription> {
602 let classic = Mutex::new(BTreeMap::<String, Option<String>>::new());
603 self.call_group_coordinators("describeConsumerGroups", groups, |coordinator, keys| {
604 self.describe_consumer_groups_on(coordinator, keys, options, &classic)
605 })
606 .await
607 }
608
609 async fn describe_consumer_groups_on(
613 &self,
614 coordinator: CoordinatorEndpoint,
615 keys: Vec<String>,
616 options: DescribeGroupsOptions,
617 classic: &Mutex<BTreeMap<String, Option<String>>>,
618 ) -> Result<BTreeMap<String, GroupOutcome<ConsumerGroupDescription>>, AdminError> {
619 let connection = self.connect_coordinator(&coordinator).await?;
620 let node = coordinator_node(&coordinator);
621 let (classic_keys, consumer_keys): (Vec<_>, Vec<_>) = {
622 let classic = classic
623 .lock()
624 .unwrap_or_else(std::sync::PoisonError::into_inner);
625 keys.into_iter().partition(|key| classic.contains_key(key))
626 };
627 let mut outcomes = BTreeMap::new();
628 let mut classic_keys = classic_keys;
629 let consumer_describe_supported = connection
634 .advertised_api_range(consumer_group_describe_request::API_KEY)
635 .is_some();
636 if !consumer_describe_supported {
637 let mut classic = classic
638 .lock()
639 .unwrap_or_else(std::sync::PoisonError::into_inner);
640 for key in &consumer_keys {
641 classic.entry(key.clone()).or_insert(None);
642 }
643 classic_keys.extend(consumer_keys.iter().cloned());
644 } else if !consumer_keys.is_empty() {
645 let request = ConsumerGroupDescribeRequest::for_groups(consumer_keys.clone(), options);
646 match connection.send(request).await {
647 Ok(response) => {
648 insert_description_outcomes(&mut outcomes, response.groups, |group| {
649 consumer_description_outcome(group, &node, &mut classic_keys, classic)
650 });
651 }
652 Err(ClientError::IncompatibleVersion { .. }) => {
655 let mut classic = classic
656 .lock()
657 .unwrap_or_else(std::sync::PoisonError::into_inner);
658 for key in consumer_keys {
659 classic.entry(key.clone()).or_insert(None);
660 classic_keys.push(key);
661 }
662 }
663 Err(error) => {
664 let error = AdminError::from(error);
665 if is_connection_failure(&error) {
666 return Err(error);
667 }
668 outcomes.extend(request_error_outcomes(&consumer_keys, &error));
669 }
670 }
671 }
672 if classic_keys.is_empty() {
673 return Ok(outcomes);
674 }
675 let request = DescribeGroupsRequest::for_groups(classic_keys.clone(), options);
676 match connection.send(request).await {
677 Ok(response) => {
678 insert_description_outcomes(&mut outcomes, response.groups, |group| {
679 classic_consumer_description_outcome(group, &node, classic)
680 });
681 }
682 Err(error) => {
683 outcomes.extend(request_error_outcomes(&classic_keys, &error.into()));
684 }
685 }
686 Ok(outcomes)
687 }
688
689 pub async fn describe_classic_groups(
702 &self,
703 groups: &[&str],
704 options: DescribeGroupsOptions,
705 ) -> GroupDescriptions<ClassicGroupDescription> {
706 self.describe_group_api::<DescribeGroupsRequest>(groups, options)
707 .await
708 }
709
710 pub async fn describe_share_groups(
723 &self,
724 groups: &[&str],
725 options: DescribeGroupsOptions,
726 ) -> GroupDescriptions<ShareGroupDescription> {
727 self.describe_group_api::<ShareGroupDescribeRequest>(groups, options)
728 .await
729 }
730
731 pub async fn describe_streams_groups(
746 &self,
747 groups: &[&str],
748 options: DescribeGroupsOptions,
749 ) -> GroupDescriptions<StreamsGroupDescription> {
750 self.describe_group_api::<ReleasedStreamsGroupDescribe>(groups, options)
751 .await
752 }
753
754 async fn describe_group_api<Request: GroupDescribeApi>(
755 &self,
756 groups: &[&str],
757 options: DescribeGroupsOptions,
758 ) -> GroupDescriptions<Request::Description> {
759 self.call_group_coordinators(
760 Request::METHOD_NAME,
761 groups,
762 |coordinator, keys| async move {
763 let connection = self.connect_coordinator(&coordinator).await?;
764 let request = Request::for_groups(keys.clone(), options);
765 Ok(match connection.send(request).await {
766 Ok(response) => Request::outcomes(response),
767 Err(error) => request_error_outcomes(&keys, &error.into()),
768 })
769 },
770 )
771 .await
772 }
773}
774
775#[cfg(test)]
778mod tests {
779 use std::sync::{Arc, Mutex};
780
781 use assert2::assert;
782 use bytes::BytesMut;
783 use krabka_client_core::MockReply;
784 use krabka_protocol::owned::{
785 common::consumer_group_describe_response::{
786 assignment::Assignment, topic_partitions::TopicPartitions,
787 },
788 consumer_group_describe_response::ConsumerGroupDescribeResponse,
789 consumer_protocol_assignment::TopicPartition as AssignedTopic,
790 describe_groups_request,
791 describe_groups_response::DescribeGroupsResponse,
792 share_group_describe_request,
793 share_group_describe_response::ShareGroupDescribeResponse,
794 streams_group_describe_response::Topology,
795 };
796
797 use super::*;
798 use crate::{
799 group_coordinators::test_support::{GroupCluster, Lookup, Seen, group_cluster},
800 partition_leaders::test_support::{ApiVersionRange, decode_request, encode_response},
801 };
802
803 fn api_range(api_key: i16, max_version: i16) -> ApiVersionRange {
804 ApiVersionRange::builder()
805 .api_key(api_key)
806 .max_version(max_version)
807 .build()
808 }
809
810 fn mock_response(bytes: Vec<u8>) -> MockReply {
811 MockReply::Respond(bytes)
812 }
813
814 fn classic_response(groups: Vec<ClassicDescribedGroup>) -> DescribeGroupsResponse {
815 DescribeGroupsResponse {
816 groups,
817 ..DescribeGroupsResponse::default()
818 }
819 }
820
821 const FALLBACK_GROUPS: [&str; 5] = ["kip848", "classic", "other", "missing", "connect"];
822 const STREAMS_GROUPS: [&str; 2] = ["with", "without"];
823
824 struct AdminGroupCluster {
825 cluster: GroupCluster,
826 admin: AdminClient,
827 }
828
829 macro_rules! described_group {
830 ($entry:ident, $error_code:expr, $error_message:expr $(, $field:ident = $value:expr)* $(,)?) => {
831 $entry {
832 error_code: $error_code,
833 error_message: $error_message,
834 group_id: "workers".to_owned(),
835 $($field: $value,)*
836 ..Default::default()
837 }
838 };
839 }
840
841 macro_rules! encode_described_group {
842 ($response:ident, $entry:ident, $code:expr, $message:expr, $version:expr, $flexible:expr $(, $field:ident = $value:expr)* $(,)?) => {
843 crate::encode_test_struct_field!(
844 encode_response,
845 $response,
846 groups,
847 vec![described_group!($entry, $code, $message $(, $field = $value)*)],
848 $version,
849 $flexible,
850 )
851 };
852 }
853
854 #[bon::builder]
855 async fn admin_group_cluster(
856 apis: Vec<ApiVersionRange>,
857 answer: impl FnMut(usize, i16, i16, &[u8], std::net::SocketAddr) -> MockReply + Send + 'static,
858 ) -> AdminGroupCluster {
859 let cluster = group_cluster()
860 .count(1)
861 .apis(apis)
862 .lookup(|_, _| Lookup::At(0))
863 .answer(answer)
864 .call()
865 .await;
866 let admin = cluster.admin(krabka_units::secs(5)).await;
867 AdminGroupCluster { cluster, admin }
868 }
869
870 const OMITTED: i32 = i32::MIN;
871
872 fn assignment_bytes(version: i16, partitions: &[(&str, &[i32])]) -> bytes::Bytes {
873 let mut bytes = BytesMut::new();
874 bytes.put_i16(version);
875 ConsumerProtocolAssignment {
876 assigned_partitions: partitions
877 .iter()
878 .map(|(topic, partitions)| AssignedTopic {
879 topic: (*topic).to_owned(),
880 partitions: partitions.to_vec(),
881 ..Default::default()
882 })
883 .collect(),
884 ..Default::default()
885 }
886 .encode(&mut bytes, version.min(3))
887 .expect("assignment encodes");
888 bytes.freeze()
889 }
890
891 #[yare::parameterized(
892 empty = { bytes::Bytes::new(), Ok(BTreeSet::new()) },
893 v0 = { assignment_bytes(0, &[("orders", &[0, 2])]), Ok(maplit::btreeset! {
894 ("orders".to_owned(), 0),
895 ("orders".to_owned(), 2),
896 }) },
897 newer_version = { assignment_bytes(9, &[("orders", &[1])]), Ok(maplit::btreeset! {
898 ("orders".to_owned(), 1),
899 }) },
900 negative_version = { bytes::Bytes::from_static(&[0xff, 0xff]), Err("Unsupported assignment version: -1".to_owned()) },
901 truncated_header = { bytes::Bytes::from_static(&[0]), Err("Buffer underflow while parsing consumer protocol's header".to_owned()) },
902 )]
903 fn classic_assignment_reads_as_kafka_does(
904 bytes: bytes::Bytes,
905 expected: Result<BTreeSet<(String, i32)>, String>,
906 ) {
907 assert2::assert!(classic_assignment(&bytes) == expected);
908 }
909
910 fn node(addr: std::net::SocketAddr, id: i32) -> ClusterNode {
911 node_with_host_port(id, addr.ip().to_string(), i32::from(addr.port()))
912 }
913
914 fn nodes(addrs: [std::net::SocketAddr; 2]) -> [ClusterNode; 2] {
915 [node(addrs[0], 0), node(addrs[1], 1)]
916 }
917
918 fn consumer_entry(
919 group_id: &str,
920 error_code: i16,
921 message: Option<&str>,
922 ) -> ConsumerDescribedGroup {
923 ConsumerDescribedGroup {
924 error_code,
925 error_message: message.map(str::to_owned),
926 group_id: group_id.to_owned(),
927 group_state: "Stable".to_owned(),
928 group_epoch: 4,
929 assignment_epoch: 3,
930 assignor_name: "uniform".to_owned(),
931 members: vec![ConsumerMember {
932 member_id: "m1".to_owned(),
933 client_id: "c1".to_owned(),
934 client_host: "/10.0.0.1".to_owned(),
935 member_epoch: 4,
936 assignment: Assignment {
937 topic_partitions: vec![TopicPartitions {
938 topic_name: "orders".to_owned(),
939 partitions: vec![0],
940 ..Default::default()
941 }],
942 ..Default::default()
943 },
944 member_type: 1,
945 ..Default::default()
946 }],
947 authorized_operations: OMITTED,
948 ..Default::default()
949 }
950 }
951
952 fn classic_entry(
953 group_id: &str,
954 error_code: i16,
955 protocol_type: &str,
956 ) -> ClassicDescribedGroup {
957 ClassicDescribedGroup {
958 error_code,
959 group_id: group_id.to_owned(),
960 group_state: "Stable".to_owned(),
961 protocol_type: protocol_type.to_owned(),
962 protocol_data: "range".to_owned(),
963 members: vec![DescribedGroupMember {
964 member_id: "m2".to_owned(),
965 group_instance_id: Some("static-1".to_owned()),
966 client_id: "c2".to_owned(),
967 client_host: "/10.0.0.2".to_owned(),
968 member_assignment: assignment_bytes(1, &[("orders", &[1])]),
969 ..Default::default()
970 }],
971 authorized_operations: 1 << 3,
972 ..Default::default()
973 }
974 }
975
976 struct GroupDescriptionFields {
977 member: MemberDescription,
978 partition_assignor: String,
979 group_type: GroupType,
980 authorized_operations: Option<BTreeSet<AclOperation>>,
981 group_epoch: Option<i32>,
982 target_assignment_epoch: Option<i32>,
983 }
984
985 #[bon::builder]
986 fn group_description(
987 group_id: String,
988 coordinator: ClusterNode,
989 fields: GroupDescriptionFields,
990 ) -> ConsumerGroupDescription {
991 ConsumerGroupDescription {
992 group_id,
993 is_simple_consumer_group: false,
994 members: vec![fields.member],
995 partition_assignor: fields.partition_assignor,
996 group_type: fields.group_type,
997 group_state: GroupState::Stable,
998 coordinator,
999 authorized_operations: fields.authorized_operations,
1000 group_epoch: fields.group_epoch,
1001 target_assignment_epoch: fields.target_assignment_epoch,
1002 }
1003 }
1004
1005 #[derive(Clone, Copy)]
1006 enum GroupDescriptionKind {
1007 Consumer,
1008 Classic,
1009 }
1010
1011 #[bon::builder]
1012 fn member_description(
1013 member_id: String,
1014 group_instance_id: Option<String>,
1015 client_id: String,
1016 host: String,
1017 assignment: BTreeSet<(String, i32)>,
1018 target_assignment: Option<BTreeSet<(String, i32)>>,
1019 member_epoch: Option<i32>,
1020 upgraded: Option<bool>,
1021 ) -> MemberDescription {
1022 MemberDescription {
1023 member_id,
1024 group_instance_id,
1025 rack_id: None,
1026 client_id,
1027 host,
1028 assignment,
1029 target_assignment,
1030 member_epoch,
1031 upgraded,
1032 }
1033 }
1034
1035 fn group_description_fields(kind: GroupDescriptionKind) -> GroupDescriptionFields {
1036 match kind {
1037 GroupDescriptionKind::Consumer => GroupDescriptionFields {
1038 member: member_description()
1039 .member_id("m1".to_owned())
1040 .maybe_group_instance_id(None)
1041 .client_id("c1".to_owned())
1042 .host("/10.0.0.1".to_owned())
1043 .assignment(maplit::btreeset! {
1044 ("orders".to_owned(), 0),
1045 })
1046 .maybe_target_assignment(Some(BTreeSet::new()))
1047 .maybe_member_epoch(Some(4))
1048 .maybe_upgraded(Some(true))
1049 .call(),
1050 partition_assignor: "uniform".to_owned(),
1051 group_type: GroupType::Consumer,
1052 authorized_operations: None,
1053 group_epoch: Some(4),
1054 target_assignment_epoch: Some(3),
1055 },
1056 GroupDescriptionKind::Classic => GroupDescriptionFields {
1057 member: member_description()
1058 .member_id("m2".to_owned())
1059 .maybe_group_instance_id(Some("static-1".to_owned()))
1060 .client_id("c2".to_owned())
1061 .host("/10.0.0.2".to_owned())
1062 .assignment(maplit::btreeset! {
1063 ("orders".to_owned(), 1),
1064 })
1065 .maybe_target_assignment(None)
1066 .maybe_member_epoch(None)
1067 .maybe_upgraded(None)
1068 .call(),
1069 partition_assignor: "range".to_owned(),
1070 group_type: GroupType::Classic,
1071 authorized_operations: Some(maplit::btreeset! {
1072 AclOperation::Read,
1073 }),
1074 group_epoch: None,
1075 target_assignment_epoch: None,
1076 },
1077 }
1078 }
1079
1080 fn test_group_description(
1081 group_id: &str,
1082 coordinator: ClusterNode,
1083 kind: GroupDescriptionKind,
1084 ) -> ConsumerGroupDescription {
1085 let fields = group_description_fields(kind);
1086 group_description()
1087 .group_id(group_id.to_owned())
1088 .coordinator(coordinator)
1089 .fields(fields)
1090 .call()
1091 }
1092
1093 fn classic_group_description(coordinator: ClusterNode) -> ConsumerGroupDescription {
1094 test_group_description("classic", coordinator, GroupDescriptionKind::Classic)
1095 }
1096
1097 #[derive(Clone, Copy, Debug)]
1098 enum DescribeApiCall {
1099 Classic,
1100 Share,
1101 Streams,
1102 }
1103
1104 async fn describe_one_group(
1105 call: DescribeApiCall,
1106 admin: &mut AdminClient,
1107 options: DescribeGroupsOptions,
1108 ) -> Option<Result<(), KafkaError>> {
1109 match call {
1110 DescribeApiCall::Classic => admin
1111 .describe_classic_groups(&["workers"], options)
1112 .await
1113 .remove("workers")
1114 .map(|result| result.map(drop)),
1115 DescribeApiCall::Share => admin
1116 .describe_share_groups(&["workers"], options)
1117 .await
1118 .remove("workers")
1119 .map(|result| result.map(drop)),
1120 DescribeApiCall::Streams => admin
1121 .describe_streams_groups(&["workers"], options)
1122 .await
1123 .remove("workers")
1124 .map(|result| result.map(drop)),
1125 }
1126 }
1127
1128 fn default_describe_options() -> DescribeGroupsOptions {
1129 DescribeGroupsOptions::builder().build()
1130 }
1131
1132 async fn default_streams_descriptions(
1133 admin: &AdminClient,
1134 groups: &[&str],
1135 ) -> GroupDescriptions<StreamsGroupDescription> {
1136 admin
1137 .describe_streams_groups(groups, default_describe_options())
1138 .await
1139 }
1140
1141 fn requests_of<T>(
1144 seen: &Seen,
1145 api_key: i16,
1146 read: impl Fn(&[u8], i16) -> T,
1147 ) -> Vec<(usize, i16, T)> {
1148 let mut requests = seen
1149 .requests
1150 .iter()
1151 .filter(|request| request.api_key == api_key)
1152 .map(|request| {
1153 (
1154 request.broker,
1155 request.version,
1156 read(&request.body, request.version),
1157 )
1158 })
1159 .collect::<Vec<_>>();
1160 requests.sort_by_key(|(broker, _, _)| *broker);
1162 requests
1163 }
1164
1165 fn consumer_describe_ids(body: &[u8], version: i16) -> Vec<String> {
1166 decode_request::<ConsumerGroupDescribeRequest>(body, version, true).group_ids
1167 }
1168
1169 fn describe_groups_ids(body: &[u8], version: i16) -> Vec<String> {
1170 decode_request::<DescribeGroupsRequest>(body, version, version >= 5).groups
1171 }
1172
1173 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1180 async fn describe_consumer_groups_batches_and_falls_back_to_describe_groups() {
1181 let cluster: GroupCluster = group_cluster()
1184 .count(2)
1185 .apis(vec![
1186 api_range(consumer_group_describe_request::API_KEY, 1),
1187 api_range(describe_groups_request::API_KEY, 5),
1188 ])
1189 .lookup(|group, _| Lookup::At(usize::from(!matches!(group, "kip848" | "classic"))))
1190 .answer(|_, api_key, version, body, _| {
1191 let reply = match api_key {
1192 consumer_group_describe_request::API_KEY => {
1193 let ids = consumer_describe_ids(body, version);
1194 let groups = ids
1195 .iter()
1196 .map(|id| match id.as_str() {
1197 "kip848" | "other" => consumer_entry(id, 0, None),
1198 "classic" | "connect" => {
1199 consumer_entry(id, 69, Some("Group is not a consumer group."))
1200 }
1201 "missing" => {
1202 consumer_entry(id, 69, Some("Group missing does not exist."))
1203 }
1204 _ => consumer_entry(id, 35, None),
1205 })
1206 .collect();
1207 crate::encode_test_struct_field!(
1208 encode_response,
1209 ConsumerGroupDescribeResponse,
1210 groups,
1211 groups,
1212 version,
1213 true
1214 )
1215 }
1216 describe_groups_request::API_KEY => {
1217 let ids = describe_groups_ids(body, version);
1218 let groups = ids
1219 .iter()
1220 .map(|id| match id.as_str() {
1221 "classic" => classic_entry(id, 0, "consumer"),
1222 "connect" => classic_entry(id, 0, "connect"),
1223 _ => classic_entry(id, 69, ""),
1224 })
1225 .collect();
1226 encode_response(&classic_response(groups), version, version >= 5)
1227 }
1228 _ => return MockReply::Silent,
1229 };
1230 mock_response(reply)
1231 })
1232 .call()
1233 .await;
1234 let admin = cluster.admin(krabka_units::secs(5)).await;
1235
1236 let result = admin
1237 .describe_consumer_groups(&FALLBACK_GROUPS, default_describe_options())
1238 .await;
1239
1240 let [b0, b1] = nodes([cluster.addrs[0], cluster.addrs[1]]);
1241 let seen = cluster.stop();
1242 assert2::assert!(
1243 result
1244 == maplit::btreemap! {
1245 "kip848".to_owned() => Ok(test_group_description(
1246 "kip848",
1247 b0.clone(),
1248 GroupDescriptionKind::Consumer,
1249 )),
1250 "classic".to_owned() => Ok(classic_group_description(b0)),
1251 "other".to_owned() => Ok(test_group_description(
1252 "other",
1253 b1,
1254 GroupDescriptionKind::Consumer,
1255 )),
1256 "missing".to_owned() => Err(kafka_error(
1257 69,
1258 Some("Group missing does not exist.".to_owned())
1259 )),
1260 "connect".to_owned() => Err(kafka_error(
1261 -1,
1262 Some("GroupId connect is not a consumer group (connect).".to_owned())
1263 )),
1264 }
1265 );
1266 let ids = |ids: &[&str]| ids.iter().map(|id| (*id).to_owned()).collect::<Vec<_>>();
1267 assert!(
1268 (
1269 seen.find_coordinator.clone(),
1270 requests_of(
1271 &seen,
1272 consumer_group_describe_request::API_KEY,
1273 consumer_describe_ids
1274 ),
1275 requests_of(&seen, describe_groups_request::API_KEY, describe_groups_ids),
1276 ) == (
1277 vec![(
1278 6,
1279 ids(&["classic", "connect", "kip848", "missing", "other"])
1280 )],
1281 vec![
1282 (0, 1, ids(&["classic", "kip848"])),
1283 (1, 1, ids(&["connect", "missing", "other"])),
1284 ],
1285 vec![
1286 (0, 5, ids(&["classic"])),
1287 (1, 5, ids(&["connect", "missing"]))
1288 ],
1289 )
1290 );
1291 }
1292
1293 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1297 async fn describe_consumer_groups_uses_describe_groups_on_an_old_coordinator() {
1298 let AdminGroupCluster { cluster, admin } = admin_group_cluster()
1299 .apis(vec![api_range(describe_groups_request::API_KEY, 5)])
1300 .answer(|_, api_key, version, body, _| {
1301 if api_key != describe_groups_request::API_KEY {
1302 return MockReply::Silent;
1303 }
1304 let groups = describe_groups_ids(body, version)
1305 .iter()
1306 .map(|id| classic_entry(id, 0, "consumer"))
1307 .collect();
1308 mock_response(encode_response(&classic_response(groups), version, true))
1309 })
1310 .call()
1311 .await;
1312
1313 let result = admin
1314 .describe_consumer_groups(&["classic"], default_describe_options())
1315 .await;
1316
1317 let first_broker = cluster.addrs[0];
1318 let _seen = cluster.stop();
1319 let b0 = node(first_broker, 0);
1320 assert!(
1321 result
1322 == maplit::btreemap! {
1323 "classic".to_owned() => Ok(classic_group_description(b0)),
1324 }
1325 );
1326 }
1327
1328 #[yare::parameterized(
1334 classic_no_error = { DescribeApiCall::Classic, "no error", vec![0], 1, 1, Ok(()) },
1335 classic_load = { DescribeApiCall::Classic, "load in progress", vec![14, 0], 1, 2, Ok(()) },
1336 classic_not_coordinator = { DescribeApiCall::Classic, "not coordinator", vec![16, 0], 2, 2, Ok(()) },
1337 classic_not_available = { DescribeApiCall::Classic, "not available", vec![15, 0], 2, 2, Ok(()) },
1338 classic_authorization = { DescribeApiCall::Classic, "group authorization failed", vec![30], 1, 1, Err(30) },
1339 classic_not_found = { DescribeApiCall::Classic, "group id not found", vec![69], 1, 1, Err(69) },
1340 share_no_error = { DescribeApiCall::Share, "no error", vec![0], 1, 1, Ok(()) },
1341 share_load = { DescribeApiCall::Share, "load in progress", vec![14, 0], 1, 2, Ok(()) },
1342 share_not_coordinator = { DescribeApiCall::Share, "not coordinator", vec![16, 0], 2, 2, Ok(()) },
1343 share_not_available = { DescribeApiCall::Share, "not available", vec![15, 0], 2, 2, Ok(()) },
1344 share_authorization = { DescribeApiCall::Share, "group authorization failed", vec![30], 1, 1, Err(30) },
1345 share_not_found = { DescribeApiCall::Share, "group id not found", vec![69], 1, 1, Err(69) },
1346 streams_no_error = { DescribeApiCall::Streams, "no error", vec![0], 1, 1, Ok(()) },
1347 streams_load = { DescribeApiCall::Streams, "load in progress", vec![14, 0], 1, 2, Ok(()) },
1348 streams_not_coordinator = { DescribeApiCall::Streams, "not coordinator", vec![16, 0], 2, 2, Ok(()) },
1349 streams_not_available = { DescribeApiCall::Streams, "not available", vec![15, 0], 2, 2, Ok(()) },
1350 streams_authorization = { DescribeApiCall::Streams, "group authorization failed", vec![30], 1, 1, Err(30) },
1351 streams_not_found = { DescribeApiCall::Streams, "group id not found", vec![69], 1, 1, Err(69) },
1352 )]
1353 #[test_macro(tokio::test(flavor = "multi_thread", worker_threads = 2))]
1354 async fn describe_calls_retry_coordinator_errors_as_kafka_does(
1355 call: DescribeApiCall,
1356 name: &'static str,
1357 codes: Vec<i16>,
1358 expected_lookups: usize,
1359 expected_requests: usize,
1360 expected: Result<(), i16>,
1361 ) {
1362 let message = Some("from the broker".to_owned());
1363 let api_key = match call {
1364 DescribeApiCall::Classic => describe_groups_request::API_KEY,
1365 DescribeApiCall::Share => share_group_describe_request::API_KEY,
1366 DescribeApiCall::Streams => streams_group_describe_request::API_KEY,
1367 };
1368 let script = Arc::new(Mutex::new(codes));
1369 let reply_message = message.clone();
1370 let cluster = group_cluster()
1371 .count(1)
1372 .apis(vec![api_range(api_key, 6)])
1373 .lookup(|_, _| Lookup::At(0))
1374 .answer(move |_, key, version, _, _| {
1375 if key != api_key {
1376 return MockReply::Silent;
1377 }
1378 let code = {
1379 let mut script = script.lock().expect("script lock");
1380 if script.len() > 1 {
1381 script.remove(0)
1382 } else {
1383 script[0]
1384 }
1385 };
1386 let body = match call {
1387 DescribeApiCall::Classic => encode_described_group!(
1388 DescribeGroupsResponse,
1389 ClassicDescribedGroup,
1390 code,
1391 reply_message.clone(),
1392 version,
1393 version >= 5,
1394 ),
1395 DescribeApiCall::Share => encode_described_group!(
1396 ShareGroupDescribeResponse,
1397 ShareDescribedGroup,
1398 code,
1399 reply_message.clone(),
1400 version,
1401 true,
1402 ),
1403 DescribeApiCall::Streams => encode_described_group!(
1404 StreamsGroupDescribeResponse,
1405 StreamsDescribedGroup,
1406 code,
1407 reply_message.clone(),
1408 version,
1409 true,
1410 topology = Some(Topology::default()),
1411 ),
1412 };
1413 mock_response(body)
1414 })
1415 .call()
1416 .await;
1417 let mut admin = cluster.admin(krabka_units::secs(5)).await;
1418 let options = DescribeGroupsOptions::builder()
1419 .include_authorized_operations(true)
1420 .build();
1421 let result = describe_one_group(call, &mut admin, options).await;
1422 let seen = cluster.stop();
1423 let expected_message = match call {
1424 DescribeApiCall::Classic => None,
1425 DescribeApiCall::Share | DescribeApiCall::Streams => message.clone(),
1426 };
1427 let flags = seen
1428 .requests
1429 .iter()
1430 .map(|request| {
1431 let (version, body) = (request.version, &request.body[..]);
1432 match call {
1433 DescribeApiCall::Classic => {
1434 decode_request::<DescribeGroupsRequest>(body, version, version >= 5)
1435 .include_authorized_operations
1436 }
1437 DescribeApiCall::Share => {
1438 decode_request::<ShareGroupDescribeRequest>(body, version, true)
1439 .include_authorized_operations
1440 }
1441 DescribeApiCall::Streams => {
1442 decode_request::<StreamsGroupDescribeRequest>(body, version, true)
1443 .include_authorized_operations
1444 }
1445 }
1446 })
1447 .collect::<Vec<_>>();
1448 assert2::assert!(
1449 (result, seen.find_coordinator.len(), flags)
1450 == (
1451 Some(expected.map_err(|code| kafka_error(code, expected_message))),
1452 expected_lookups,
1453 vec![true; expected_requests],
1454 ),
1455 "case {call:?} {name}"
1456 );
1457 }
1458
1459 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1462 async fn describe_streams_groups_sends_v0_and_needs_a_topology() {
1463 let AdminGroupCluster { cluster, admin } = admin_group_cluster()
1464 .apis(vec![api_range(streams_group_describe_request::API_KEY, 1)])
1465 .answer(|_, api_key, version, _, _| {
1466 if api_key != streams_group_describe_request::API_KEY {
1467 return MockReply::Silent;
1468 }
1469 mock_response(crate::encode_test_struct_field!(
1470 encode_response,
1471 StreamsGroupDescribeResponse,
1472 groups,
1473 vec![
1474 StreamsDescribedGroup {
1475 group_id: "with".to_owned(),
1476 topology: Some(Topology::default()),
1477 ..Default::default()
1478 },
1479 StreamsDescribedGroup {
1480 group_id: "without".to_owned(),
1481 ..Default::default()
1482 },
1483 ],
1484 version,
1485 true,
1486 ))
1487 })
1488 .call()
1489 .await;
1490
1491 let result = default_streams_descriptions(&admin, &STREAMS_GROUPS).await;
1492
1493 let seen = cluster.stop();
1494 let versions = seen
1495 .requests
1496 .iter()
1497 .map(|request| request.version)
1498 .collect::<Vec<_>>();
1499 assert!(
1500 (result, versions)
1501 == (
1502 maplit::btreemap! {
1503 "with".to_owned() => Ok(StreamsDescribedGroup {
1504 group_id: "with".to_owned(),
1505 topology: Some(Topology::default()),
1506 ..Default::default()
1507 }),
1508 "without".to_owned() => Err(kafka_error(
1509 -1,
1510 Some("Topology information is missing".to_owned())
1511 )),
1512 },
1513 vec![0],
1514 )
1515 );
1516 }
1517}