1use std::collections::{BTreeMap, BTreeSet};
11
12use krabka_client_core::ClientError;
13use krabka_protocol::{
14 ProtocolRequest as _,
15 owned::{
16 describe_topic_partitions_request::{
17 Cursor as RequestCursor, DescribeTopicPartitionsRequest, TopicRequest,
18 },
19 describe_topic_partitions_response::{
20 DescribeTopicPartitionsResponse, DescribeTopicPartitionsResponsePartition,
21 DescribeTopicPartitionsResponseTopic,
22 },
23 metadata_request::{MetadataRequest, MetadataRequestTopic},
24 metadata_response::{MetadataResponse, MetadataResponsePartition, MetadataResponseTopic},
25 },
26 primitives::uuid::Uuid as ProtoUuid,
27};
28use uuid::Uuid;
29
30use crate::{
31 AdminClient, AdminError, ClusterNode, DescribeClusterOptions, KafkaError,
32 cluster::authorized_operations, groups::list_groups_kafka_error, kafka_error_name,
33 retry::ControllerRetry, users::AclOperation,
34};
35
36const UNKNOWN_TOPIC_OR_PARTITION: i16 = 3;
38const INVALID_TOPIC_EXCEPTION: i16 = 17;
40const UNKNOWN_TOPIC_ID: i16 = 100;
42
43#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, bon::Builder)]
45pub struct ListTopicsOptions {
46 #[builder(default)]
48 pub list_internal: bool,
49}
50
51#[derive(Debug, Clone, PartialEq, Eq)]
53pub struct TopicListing {
54 pub name: String,
56 pub topic_id: Option<Uuid>,
58 pub is_internal: bool,
60}
61
62#[derive(Debug, Clone, Copy, PartialEq, Eq, bon::Builder)]
65pub struct DescribeTopicsOptions {
66 #[builder(default)]
68 pub include_authorized_operations: bool,
69 #[builder(default = 2000)]
72 pub partition_size_limit_per_response: i32,
73}
74
75impl Default for DescribeTopicsOptions {
76 fn default() -> Self {
77 Self {
78 include_authorized_operations: false,
79 partition_size_limit_per_response: 2000,
80 }
81 }
82}
83
84#[derive(Debug, Clone, PartialEq, Eq)]
90pub struct TopicPartitionInfo {
91 pub partition: i32,
93 pub leader: Option<ClusterNode>,
96 pub replicas: Vec<ClusterNode>,
98 pub isr: Vec<ClusterNode>,
100 pub elr: Vec<ClusterNode>,
103 pub last_known_elr: Vec<ClusterNode>,
106}
107
108#[derive(Debug, Clone, PartialEq, Eq)]
111pub struct TopicDescription {
112 pub name: String,
114 pub topic_id: Option<Uuid>,
116 pub is_internal: bool,
118 pub partitions: Vec<TopicPartitionInfo>,
120 pub authorized_operations: Option<BTreeSet<AclOperation>>,
123}
124
125pub type TopicDescriptions<K> = BTreeMap<K, Result<TopicDescription, KafkaError>>;
127
128fn topic_id(id: ProtoUuid) -> Option<Uuid> {
129 (id != ProtoUuid::ZERO).then(|| Uuid::from_bytes(id.0))
130}
131
132fn topic_error(code: i16, message: String) -> KafkaError {
133 KafkaError {
134 code,
135 name: kafka_error_name(code),
136 message: Some(message),
137 }
138}
139
140fn node(nodes: &BTreeMap<i32, ClusterNode>, id: i32) -> ClusterNode {
142 nodes.get(&id).cloned().unwrap_or_else(|| ClusterNode {
143 id,
144 host: String::new(),
145 port: -1,
146 rack: None,
147 is_fenced: false,
148 })
149}
150
151fn node_list(nodes: &BTreeMap<i32, ClusterNode>, ids: &[i32]) -> Vec<ClusterNode> {
152 ids.iter().map(|id| node(nodes, *id)).collect()
153}
154
155fn all_topics_request() -> MetadataRequest {
157 MetadataRequest {
158 topics: None,
159 allow_auto_topic_creation: true,
160 ..Default::default()
161 }
162}
163
164fn topic_listings(
167 response: MetadataResponse,
168 options: ListTopicsOptions,
169) -> BTreeMap<String, TopicListing> {
170 response
171 .topics
172 .into_iter()
173 .filter(|topic| !topic.is_internal || options.list_internal)
174 .filter_map(|topic| {
175 let name = topic.name?;
176 Some((
177 name.clone(),
178 TopicListing {
179 name,
180 topic_id: topic_id(topic.topic_id),
181 is_internal: topic.is_internal,
182 },
183 ))
184 })
185 .collect()
186}
187
188fn describe_partitions_request(
193 pending: &BTreeSet<String>,
194 partial: Option<&TopicDescription>,
195 options: DescribeTopicsOptions,
196) -> DescribeTopicPartitionsRequest {
197 DescribeTopicPartitionsRequest {
198 topics: pending
199 .iter()
200 .map(|name| TopicRequest {
201 name: name.clone(),
202 ..Default::default()
203 })
204 .collect(),
205 response_partition_limit: options.partition_size_limit_per_response,
206 cursor: partial.map(|partial| RequestCursor {
207 topic_name: partial.name.clone(),
208 partition_index: i32::try_from(partial.partitions.len()).unwrap_or(i32::MAX),
209 ..Default::default()
210 }),
211 ..Default::default()
212 }
213}
214
215fn partition_from_describe(
216 partition: &DescribeTopicPartitionsResponsePartition,
217 nodes: &BTreeMap<i32, ClusterNode>,
218) -> TopicPartitionInfo {
219 TopicPartitionInfo {
220 partition: partition.partition_index,
221 leader: nodes.get(&partition.leader_id).cloned(),
222 replicas: node_list(nodes, &partition.replica_nodes),
223 isr: node_list(nodes, &partition.isr_nodes),
224 elr: node_list(
225 nodes,
226 partition
227 .eligible_leader_replicas
228 .as_deref()
229 .unwrap_or_default(),
230 ),
231 last_known_elr: node_list(
232 nodes,
233 partition.last_known_elr.as_deref().unwrap_or_default(),
234 ),
235 }
236}
237
238fn description_from_describe(
241 topic: &DescribeTopicPartitionsResponseTopic,
242 name: String,
243 nodes: &BTreeMap<i32, ClusterNode>,
244 include_authorized_operations: bool,
245) -> TopicDescription {
246 TopicDescription {
247 name,
248 topic_id: topic_id(topic.topic_id),
249 is_internal: topic.is_internal,
250 partitions: topic
251 .partitions
252 .iter()
253 .map(|partition| partition_from_describe(partition, nodes))
254 .collect(),
255 authorized_operations: if include_authorized_operations {
256 authorized_operations(topic.topic_authorized_operations)
257 } else {
258 None
259 },
260 }
261}
262
263#[derive(Debug, Default)]
265struct DescribePages {
266 pending: BTreeSet<String>,
268 partial: Option<TopicDescription>,
270 out: TopicDescriptions<String>,
272}
273
274impl DescribePages {
275 fn read(
279 &mut self,
280 response: DescribeTopicPartitionsResponse,
281 nodes: &BTreeMap<i32, ClusterNode>,
282 include_authorized_operations: bool,
283 ) {
284 let mut cursor = response.next_cursor.map(|cursor| cursor.topic_name);
285 let mut next = None;
286 for topic in response.topics {
287 let Some(name) = topic.name.clone() else {
288 tracing::warn!("the DescribeTopicPartitions response names a topic without a name");
289 continue;
290 };
291 let asked = self.pending.contains(&name)
292 || self.partial.as_ref().is_some_and(|p| p.name == name);
293 if !asked {
294 tracing::warn!(
295 topic = %name,
296 "the DescribeTopicPartitions response names a topic that is not in the request"
297 );
298 continue;
299 }
300 if topic.error_code != 0 {
301 self.out.entry(name.clone()).or_insert_with(|| {
302 Err(KafkaError {
303 code: topic.error_code,
304 name: kafka_error_name(topic.error_code),
305 message: None,
306 })
307 });
308 self.pending.remove(&name);
309 if cursor.as_deref() == Some(name.as_str()) {
310 cursor = None;
311 }
312 continue;
313 }
314 let current = description_from_describe(
315 &topic,
316 name.clone(),
317 nodes,
318 include_authorized_operations,
319 );
320 if let Some(partial) = self.partial.as_mut()
321 && partial.name == name
322 {
323 partial.partitions.extend(current.partitions);
324 continue;
325 }
326 if cursor.as_deref() == Some(name.as_str()) {
327 next = Some(current);
328 continue;
329 }
330 self.pending.remove(&name);
331 self.out.entry(name).or_insert(Ok(current));
332 }
333 if let Some(partial) = self
334 .partial
335 .take_if(|partial| cursor.as_deref() != Some(partial.name.as_str()))
336 {
337 self.pending.remove(&partial.name);
338 self.out.entry(partial.name.clone()).or_insert(Ok(partial));
339 }
340 if next.is_some() {
341 self.partial = next;
342 }
343 }
344
345 fn unfinished(&self) -> bool {
347 !self.pending.is_empty() || self.partial.is_some()
348 }
349
350 fn fail(mut self, error: &KafkaError) -> TopicDescriptions<String> {
352 let unfinished = std::mem::take(&mut self.pending)
353 .into_iter()
354 .chain(self.partial.take().map(|partial| partial.name));
355 for name in unfinished {
356 self.out.entry(name).or_insert_with(|| Err(error.clone()));
357 }
358 self.out
359 }
360}
361
362fn metadata_by_name_request(
365 names: &BTreeSet<String>,
366 options: DescribeTopicsOptions,
367) -> MetadataRequest {
368 MetadataRequest {
369 topics: Some(
370 names
371 .iter()
372 .map(|name| MetadataRequestTopic {
373 topic_id: ProtoUuid::ZERO,
374 name: Some(name.clone()),
375 ..Default::default()
376 })
377 .collect(),
378 ),
379 allow_auto_topic_creation: false,
380 include_topic_authorized_operations: options.include_authorized_operations,
381 ..Default::default()
382 }
383}
384
385fn metadata_by_id_request(ids: &BTreeSet<Uuid>, options: DescribeTopicsOptions) -> MetadataRequest {
388 MetadataRequest {
389 topics: Some(
390 ids.iter()
391 .map(|id| MetadataRequestTopic {
392 topic_id: ProtoUuid(*id.as_bytes()),
393 name: None,
394 ..Default::default()
395 })
396 .collect(),
397 ),
398 allow_auto_topic_creation: false,
399 include_topic_authorized_operations: options.include_authorized_operations,
400 ..Default::default()
401 }
402}
403
404fn metadata_nodes(response: &MetadataResponse) -> BTreeMap<i32, ClusterNode> {
405 response
406 .brokers
407 .iter()
408 .map(|broker| {
409 (
410 broker.node_id,
411 ClusterNode {
412 id: broker.node_id,
413 host: broker.host.clone(),
414 port: broker.port,
415 rack: broker.rack.clone(),
416 is_fenced: false,
417 },
418 )
419 })
420 .collect()
421}
422
423fn partition_from_metadata(
424 partition: &MetadataResponsePartition,
425 nodes: &BTreeMap<i32, ClusterNode>,
426) -> TopicPartitionInfo {
427 TopicPartitionInfo {
428 partition: partition.partition_index,
429 leader: (partition.leader_id >= 0)
430 .then(|| nodes.get(&partition.leader_id).cloned())
431 .flatten(),
432 replicas: node_list(nodes, &partition.replica_nodes),
433 isr: node_list(nodes, &partition.isr_nodes),
434 elr: Vec::new(),
435 last_known_elr: Vec::new(),
436 }
437}
438
439fn description_from_metadata(
443 topic: &MetadataResponseTopic,
444 name: String,
445 nodes: &BTreeMap<i32, ClusterNode>,
446) -> TopicDescription {
447 let mut partitions = topic
448 .partitions
449 .iter()
450 .map(|partition| partition_from_metadata(partition, nodes))
451 .collect::<Vec<_>>();
452 partitions.sort_by_key(|partition| partition.partition);
453 TopicDescription {
454 name,
455 topic_id: topic_id(topic.topic_id),
456 is_internal: topic.is_internal,
457 partitions,
458 authorized_operations: authorized_operations(topic.topic_authorized_operations),
459 }
460}
461
462fn descriptions_by_name(
467 names: &BTreeSet<String>,
468 response: MetadataResponse,
469) -> TopicDescriptions<String> {
470 let nodes = metadata_nodes(&response);
471 let mut topics = response
472 .topics
473 .into_iter()
474 .filter_map(|topic| Some((topic.name.clone()?, topic)))
475 .collect::<BTreeMap<_, _>>();
476 names
477 .iter()
478 .map(|name| {
479 let result = match topics.remove(name) {
480 Some(topic) if topic.error_code != 0 => Err(KafkaError {
481 code: topic.error_code,
482 name: kafka_error_name(topic.error_code),
483 message: None,
484 }),
485 Some(topic) => Ok(description_from_metadata(&topic, name.clone(), &nodes)),
486 None => Err(topic_error(
487 UNKNOWN_TOPIC_OR_PARTITION,
488 format!("Topic {name} not found."),
489 )),
490 };
491 (name.clone(), result)
492 })
493 .collect()
494}
495
496fn descriptions_by_id(ids: &BTreeSet<Uuid>, response: MetadataResponse) -> TopicDescriptions<Uuid> {
501 let nodes = metadata_nodes(&response);
502 let mut topics = response
503 .topics
504 .into_iter()
505 .filter(|topic| topic.error_code == 0)
506 .filter_map(|topic| Some((topic_id(topic.topic_id)?, topic)))
507 .collect::<BTreeMap<_, _>>();
508 ids.iter()
509 .map(|id| {
510 let result = match topics.remove(id) {
511 Some(mut topic) => match topic.name.take() {
512 Some(name) => Ok(description_from_metadata(&topic, name, &nodes)),
513 None => Err(topic_error(
514 UNKNOWN_TOPIC_ID,
515 format!("TopicId {id} not found."),
516 )),
517 },
518 None => Err(topic_error(
519 UNKNOWN_TOPIC_ID,
520 format!("TopicId {id} not found."),
521 )),
522 };
523 (*id, result)
524 })
525 .collect()
526}
527
528const fn is_unsupported_version(error: &AdminError) -> bool {
531 matches!(
532 error,
533 AdminError::Transport(ClientError::IncompatibleVersion { .. })
534 )
535}
536
537impl AdminClient {
538 pub async fn list_topics(
546 &self,
547 options: ListTopicsOptions,
548 ) -> Result<BTreeMap<String, TopicListing>, AdminError> {
549 let retry = ControllerRetry::new("Metadata", self.retry);
550 let response = retry
551 .bounded(self.conn.send_at_least(all_topics_request(), 1))
552 .await?;
553 Ok(topic_listings(response, options))
554 }
555
556 pub async fn describe_topics(
578 &self,
579 names: &[&str],
580 options: DescribeTopicsOptions,
581 ) -> TopicDescriptions<String> {
582 let mut pages = DescribePages::default();
583 for name in names {
584 if name.is_empty() {
585 pages.out.insert(
586 String::new(),
587 Err(topic_error(
588 INVALID_TOPIC_EXCEPTION,
589 "The given topic name '' cannot be represented in a request.".to_owned(),
590 )),
591 );
592 } else {
593 pages.pending.insert((*name).to_owned());
594 }
595 }
596 if pages.pending.is_empty() {
597 return pages.out;
598 }
599 let nodes = match self
600 .describe_cluster(DescribeClusterOptions::builder().build())
601 .await
602 {
603 Ok(cluster) => cluster
604 .nodes
605 .into_iter()
606 .map(|node| (node.id, node))
607 .collect::<BTreeMap<_, _>>(),
608 Err(error) => return pages.fail(&list_groups_kafka_error(&error)),
609 };
610 if self
611 .conn
612 .advertised_api_range(DescribeTopicPartitionsRequest::API_KEY)
613 .await
614 .is_none()
615 {
616 return self.describe_topics_with_metadata(pages, options).await;
617 }
618 let retry = ControllerRetry::new("DescribeTopicPartitions", self.retry);
619 while pages.unfinished() {
620 let request =
621 describe_partitions_request(&pages.pending, pages.partial.as_ref(), options);
622 match retry.bounded(self.conn.send(request)).await {
623 Ok(response) => pages.read(response, &nodes, options.include_authorized_operations),
624 Err(error) if is_unsupported_version(&error) => {
625 return self.describe_topics_with_metadata(pages, options).await;
626 }
627 Err(error) => return pages.fail(&list_groups_kafka_error(&error)),
628 }
629 }
630 pages.out
631 }
632
633 async fn describe_topics_with_metadata(
638 &self,
639 mut pages: DescribePages,
640 options: DescribeTopicsOptions,
641 ) -> TopicDescriptions<String> {
642 if let Some(partial) = pages.partial.take() {
643 pages.pending.insert(partial.name);
644 }
645 let retry = ControllerRetry::new("Metadata", self.retry);
646 let request = metadata_by_name_request(&pages.pending, options);
647 let response = match retry.bounded(self.conn.send_at_least(request, 4)).await {
648 Err(error) if is_unsupported_version(&error) => {
649 retry
650 .bounded(self.conn.send_at_least(all_topics_request(), 1))
651 .await
652 }
653 other => other,
654 };
655 match response {
656 Ok(response) => {
657 let pending = std::mem::take(&mut pages.pending);
658 for (name, result) in descriptions_by_name(&pending, response) {
659 pages.out.entry(name).or_insert(result);
660 }
661 pages.out
662 }
663 Err(error) => pages.fail(&list_groups_kafka_error(&error)),
664 }
665 }
666
667 pub async fn describe_topics_by_id(
677 &self,
678 ids: &[Uuid],
679 options: DescribeTopicsOptions,
680 ) -> TopicDescriptions<Uuid> {
681 let mut out = TopicDescriptions::new();
682 let mut pending = BTreeSet::new();
683 for id in ids {
684 if id.is_nil() {
685 out.insert(
686 *id,
687 Err(topic_error(
688 INVALID_TOPIC_EXCEPTION,
689 format!("The given topic id '{id}' cannot be represented in a request."),
690 )),
691 );
692 } else {
693 pending.insert(*id);
694 }
695 }
696 if pending.is_empty() {
697 return out;
698 }
699 let retry = ControllerRetry::new("Metadata", self.retry);
700 let request = metadata_by_id_request(&pending, options);
701 match retry.bounded(self.conn.send_at_least(request, 12)).await {
702 Ok(response) => out.extend(descriptions_by_id(&pending, response)),
703 Err(error) => {
704 let error = list_groups_kafka_error(&error);
705 out.extend(pending.into_iter().map(|id| (id, Err(error.clone()))));
706 }
707 }
708 out
709 }
710}
711
712#[cfg(test)]
713mod tests {
714 use std::{
715 net::SocketAddr,
716 sync::{Arc, Mutex},
717 };
718
719 use krabka_client_core::MockReply;
720 use krabka_protocol::owned::{
721 describe_cluster_request,
722 describe_cluster_response::{DescribeClusterBroker, DescribeClusterResponse},
723 describe_topic_partitions_request,
724 describe_topic_partitions_response::Cursor as ResponseCursor,
725 metadata_request,
726 metadata_response::MetadataResponseBroker,
727 };
728
729 use super::*;
730 use crate::partition_leaders::test_support::{
731 ApiVersionRange, decode_request, encode_response, fast_admin, scripted_broker,
732 };
733
734 const ALPHA: Uuid = Uuid::from_u128(0xa);
735 const BETA: Uuid = Uuid::from_u128(0xb);
736
737 fn id_of(name: &str) -> Uuid {
738 if name == "alpha" { ALPHA } else { BETA }
739 }
740
741 fn broker_one(own: SocketAddr) -> ClusterNode {
743 ClusterNode {
744 id: 1,
745 host: own.ip().to_string(),
746 port: i32::from(own.port()),
747 rack: None,
748 is_fenced: false,
749 }
750 }
751
752 fn unknown_two() -> ClusterNode {
754 ClusterNode {
755 id: 2,
756 host: String::new(),
757 port: -1,
758 rack: None,
759 is_fenced: false,
760 }
761 }
762
763 fn page_partition(index: i32) -> DescribeTopicPartitionsResponsePartition {
766 DescribeTopicPartitionsResponsePartition {
767 partition_index: index,
768 leader_id: 1,
769 replica_nodes: vec![1, 2],
770 isr_nodes: vec![1],
771 eligible_leader_replicas: Some(vec![2]),
772 last_known_elr: Some(Vec::new()),
773 ..Default::default()
774 }
775 }
776
777 fn page_topic(
780 name: &str,
781 error_code: i16,
782 partitions: &[i32],
783 ) -> DescribeTopicPartitionsResponseTopic {
784 DescribeTopicPartitionsResponseTopic {
785 error_code,
786 name: Some(name.to_owned()),
787 topic_id: ProtoUuid(*id_of(name).as_bytes()),
788 partitions: partitions.iter().copied().map(page_partition).collect(),
789 topic_authorized_operations: 1 << 3,
790 ..Default::default()
791 }
792 }
793
794 fn page(
795 topics: Vec<DescribeTopicPartitionsResponseTopic>,
796 cursor: Option<(&str, i32)>,
797 ) -> DescribeTopicPartitionsResponse {
798 DescribeTopicPartitionsResponse {
799 topics,
800 next_cursor: cursor.map(|(topic_name, partition_index)| ResponseCursor {
801 topic_name: topic_name.to_owned(),
802 partition_index,
803 ..Default::default()
804 }),
805 ..Default::default()
806 }
807 }
808
809 fn paged(name: &str, partitions: &[i32], own: SocketAddr) -> TopicDescription {
811 TopicDescription {
812 name: name.to_owned(),
813 topic_id: Some(id_of(name)),
814 is_internal: false,
815 partitions: partitions
816 .iter()
817 .map(|partition| TopicPartitionInfo {
818 partition: *partition,
819 leader: Some(broker_one(own)),
820 replicas: vec![broker_one(own), unknown_two()],
821 isr: vec![broker_one(own)],
822 elr: vec![unknown_two()],
823 last_known_elr: Vec::new(),
824 })
825 .collect(),
826 authorized_operations: None,
827 }
828 }
829
830 fn dtp_request(topics: &[&str], cursor: Option<(&str, i32)>) -> DescribeTopicPartitionsRequest {
831 DescribeTopicPartitionsRequest {
832 topics: topics
833 .iter()
834 .map(|name| TopicRequest {
835 name: (*name).to_owned(),
836 ..Default::default()
837 })
838 .collect(),
839 response_partition_limit: 2000,
840 cursor: cursor.map(|(topic_name, partition_index)| RequestCursor {
841 topic_name: topic_name.to_owned(),
842 partition_index,
843 ..Default::default()
844 }),
845 ..Default::default()
846 }
847 }
848
849 fn metadata_topic(
852 name: &str,
853 error_code: i16,
854 indexes: &[i32],
855 leader: i32,
856 ) -> MetadataResponseTopic {
857 MetadataResponseTopic {
858 error_code,
859 name: Some(name.to_owned()),
860 topic_id: ProtoUuid(*id_of(name).as_bytes()),
861 is_internal: name.starts_with("__"),
862 partitions: indexes
863 .iter()
864 .map(|index| MetadataResponsePartition {
865 partition_index: *index,
866 leader_id: leader,
867 replica_nodes: vec![1, 2],
868 isr_nodes: vec![1],
869 ..Default::default()
870 })
871 .collect(),
872 ..Default::default()
873 }
874 }
875
876 fn metadata(own: SocketAddr, topics: Vec<MetadataResponseTopic>) -> MetadataResponse {
877 MetadataResponse {
878 brokers: vec![MetadataResponseBroker {
879 node_id: 1,
880 host: own.ip().to_string(),
881 port: i32::from(own.port()),
882 ..Default::default()
883 }],
884 controller_id: 1,
885 topics,
886 ..Default::default()
887 }
888 }
889
890 fn from_metadata(
892 name: &str,
893 partitions: &[i32],
894 own: SocketAddr,
895 leader: bool,
896 ) -> TopicDescription {
897 TopicDescription {
898 name: name.to_owned(),
899 topic_id: Some(id_of(name)),
900 is_internal: false,
901 partitions: partitions
902 .iter()
903 .map(|partition| TopicPartitionInfo {
904 partition: *partition,
905 leader: leader.then(|| broker_one(own)),
906 replicas: vec![broker_one(own), unknown_two()],
907 isr: vec![broker_one(own)],
908 elr: Vec::new(),
909 last_known_elr: Vec::new(),
910 })
911 .collect(),
912 authorized_operations: None,
913 }
914 }
915
916 fn topic_err(code: i16, message: Option<String>) -> Result<TopicDescription, KafkaError> {
917 Err(KafkaError {
918 code,
919 name: kafka_error_name(code),
920 message,
921 })
922 }
923
924 #[derive(Clone)]
926 enum Answer {
927 Page(DescribeTopicPartitionsResponse),
928 Close,
929 }
930
931 #[derive(Debug, Clone, PartialEq)]
933 enum Received {
934 Pages(DescribeTopicPartitionsRequest),
935 Metadata(i16, MetadataRequest),
936 }
937
938 struct DescribeCase {
940 name: &'static str,
941 names: Vec<&'static str>,
942 cluster_error: i16,
943 dtp: bool,
944 answers: Vec<Answer>,
945 metadata_topics: Vec<MetadataResponseTopic>,
946 expected: fn(SocketAddr) -> (TopicDescriptions<String>, Vec<Received>),
947 }
948
949 async fn describe_case(
954 case: &DescribeCase,
955 ) -> (TopicDescriptions<String>, Vec<Received>, SocketAddr) {
956 let received = Arc::new(Mutex::new(Vec::new()));
957 let handler_received = Arc::clone(&received);
958 let mut apis = vec![
959 ApiVersionRange::builder()
960 .api_key(describe_cluster_request::API_KEY)
961 .max_version(2)
962 .build(),
963 ];
964 if case.dtp {
965 apis.push(
966 ApiVersionRange::builder()
967 .api_key(describe_topic_partitions_request::API_KEY)
968 .build(),
969 );
970 }
971 let cluster_error = case.cluster_error;
972 let metadata_topics = case.metadata_topics.clone();
973 let pages = Arc::new(Mutex::new(case.answers.clone().into_iter()));
974 let broker = scripted_broker(apis, move |api_key, version, body, own| {
975 let mut received = handler_received.lock().expect("received lock");
976 let reply = match api_key {
977 describe_cluster_request::API_KEY => encode_response(
978 &DescribeClusterResponse {
979 error_code: cluster_error,
980 controller_id: 1,
981 brokers: vec![DescribeClusterBroker {
982 broker_id: 1,
983 host: own.ip().to_string(),
984 port: i32::from(own.port()),
985 ..Default::default()
986 }],
987 ..Default::default()
988 },
989 version,
990 true,
991 ),
992 describe_topic_partitions_request::API_KEY => {
993 received.push(Received::Pages(decode_request(body, version, true)));
994 match pages.lock().expect("pages lock").next() {
995 Some(Answer::Page(page)) => encode_response(&page, version, true),
996 Some(Answer::Close) => return MockReply::Close,
997 None => return MockReply::Silent,
998 }
999 }
1000 metadata_request::API_KEY => {
1001 let flexible = version >= metadata_request::FLEXIBLE_MIN;
1002 received.push(Received::Metadata(
1003 version,
1004 decode_request(body, version, flexible),
1005 ));
1006 encode_response(&metadata(own, metadata_topics.clone()), version, flexible)
1007 }
1008 _ => return MockReply::Silent,
1009 };
1010 MockReply::Respond(reply)
1011 })
1012 .await;
1013 let admin = fast_admin(broker.addr, krabka_units::secs(5)).await;
1014 let result = admin
1015 .describe_topics(&case.names, DescribeTopicsOptions::builder().build())
1016 .await;
1017 let own = broker.addr;
1018 broker.stop();
1019 let received = received.lock().expect("received lock").clone();
1020 (result, received, own)
1021 }
1022
1023 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1029 async fn describe_topics_pages_and_falls_back_to_metadata() {
1030 let cases = [
1031 DescribeCase {
1032 name: "one page",
1033 names: vec!["beta", "alpha"],
1034 cluster_error: 0,
1035 dtp: true,
1036 answers: vec![Answer::Page(page(
1037 vec![page_topic("alpha", 0, &[0]), page_topic("beta", 0, &[0, 1])],
1038 None,
1039 ))],
1040 metadata_topics: Vec::new(),
1041 expected: |own| {
1042 (
1043 maplit::btreemap! {
1044 "alpha".to_owned() => Ok(paged("alpha", &[0], own)),
1045 "beta".to_owned() => Ok(paged("beta", &[0, 1], own)),
1046 },
1047 vec![Received::Pages(dtp_request(&["alpha", "beta"], None))],
1048 )
1049 },
1050 },
1051 DescribeCase {
1052 name: "the cursor splits a topic across pages",
1053 names: vec!["alpha", "beta"],
1054 cluster_error: 0,
1055 dtp: true,
1056 answers: vec![
1057 Answer::Page(page(
1058 vec![page_topic("alpha", 0, &[0]), page_topic("beta", 0, &[0])],
1059 Some(("beta", 1)),
1060 )),
1061 Answer::Page(page(vec![page_topic("beta", 0, &[1, 2])], None)),
1062 ],
1063 metadata_topics: Vec::new(),
1064 expected: |own| {
1065 (
1066 maplit::btreemap! {
1067 "alpha".to_owned() => Ok(paged("alpha", &[0], own)),
1068 "beta".to_owned() => Ok(paged("beta", &[0, 1, 2], own)),
1069 },
1070 vec![
1071 Received::Pages(dtp_request(&["alpha", "beta"], None)),
1072 Received::Pages(dtp_request(&["beta"], Some(("beta", 1)))),
1073 ],
1074 )
1075 },
1076 },
1077 DescribeCase {
1078 name: "a topic error fails that topic only",
1079 names: vec!["alpha", "beta"],
1080 cluster_error: 0,
1081 dtp: true,
1082 answers: vec![Answer::Page(page(
1083 vec![page_topic("alpha", 29, &[]), page_topic("beta", 0, &[0])],
1084 None,
1085 ))],
1086 metadata_topics: Vec::new(),
1087 expected: |own| {
1088 (
1089 maplit::btreemap! {
1090 "alpha".to_owned() => topic_err(29, None),
1091 "beta".to_owned() => Ok(paged("beta", &[0], own)),
1092 },
1093 vec![Received::Pages(dtp_request(&["alpha", "beta"], None))],
1094 )
1095 },
1096 },
1097 DescribeCase {
1098 name: "a lost connection is retried",
1099 names: vec!["alpha"],
1100 cluster_error: 0,
1101 dtp: true,
1102 answers: vec![
1103 Answer::Close,
1104 Answer::Page(page(vec![page_topic("alpha", 0, &[0])], None)),
1105 ],
1106 metadata_topics: Vec::new(),
1107 expected: |own| {
1108 (
1109 maplit::btreemap! {
1110 "alpha".to_owned() => Ok(paged("alpha", &[0], own)),
1111 },
1112 vec![
1113 Received::Pages(dtp_request(&["alpha"], None)),
1114 Received::Pages(dtp_request(&["alpha"], None)),
1115 ],
1116 )
1117 },
1118 },
1119 DescribeCase {
1120 name: "a peer without DescribeTopicPartitions is asked with Metadata",
1121 names: vec!["alpha", "beta", ""],
1122 cluster_error: 0,
1123 dtp: false,
1124 answers: Vec::new(),
1125 metadata_topics: vec![metadata_topic("alpha", 0, &[1, 0], -1)],
1126 expected: |own| {
1127 (
1128 maplit::btreemap! {
1129 String::new() => topic_err(
1130 17,
1131 Some(
1132 "The given topic name '' cannot be represented in a \
1133 request."
1134 .to_owned(),
1135 ),
1136 ),
1137 "alpha".to_owned() => Ok(from_metadata("alpha", &[0, 1], own, false)),
1138 "beta".to_owned() => topic_err(3, Some("Topic beta not found.".to_owned())),
1139 },
1140 vec![Received::Metadata(
1141 12,
1142 metadata_by_name_request(
1143 &maplit::btreeset! {
1144 "alpha".to_owned(),
1145 "beta".to_owned(),
1146 },
1147 DescribeTopicsOptions::builder().build(),
1148 ),
1149 )],
1150 )
1151 },
1152 },
1153 DescribeCase {
1154 name: "a failed DescribeCluster fails every topic",
1155 names: vec!["alpha"],
1156 cluster_error: 31,
1157 dtp: true,
1158 answers: Vec::new(),
1159 metadata_topics: Vec::new(),
1160 expected: |_| {
1161 (
1162 maplit::btreemap! {
1163 "alpha".to_owned() => topic_err(31, None),
1164 },
1165 Vec::new(),
1166 )
1167 },
1168 },
1169 ];
1170 for case in &cases {
1171 let (result, received, own) = describe_case(case).await;
1172 assert2::assert!(
1173 (result, received) == (case.expected)(own),
1174 "case {}",
1175 case.name
1176 );
1177 }
1178 }
1179
1180 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1183 async fn list_topics_keeps_internal_topics_on_request() {
1184 let listing = |name: &str, is_internal| TopicListing {
1185 name: name.to_owned(),
1186 topic_id: Some(id_of(name)),
1187 is_internal,
1188 };
1189 for (name, list_internal, close_first, expected) in [
1190 (
1191 "without internal topics",
1192 false,
1193 false,
1194 (
1195 maplit::btreemap! {
1196 "alpha".to_owned() => listing("alpha", false),
1197 },
1198 1,
1199 ),
1200 ),
1201 (
1202 "with internal topics",
1203 true,
1204 false,
1205 (
1206 maplit::btreemap! {
1207 "__consumer_offsets".to_owned() => listing("__consumer_offsets", true),
1208 "alpha".to_owned() => listing("alpha", false),
1209 },
1210 1,
1211 ),
1212 ),
1213 (
1214 "a lost connection is retried",
1215 false,
1216 true,
1217 (
1218 maplit::btreemap! {
1219 "alpha".to_owned() => listing("alpha", false),
1220 },
1221 2,
1222 ),
1223 ),
1224 ] {
1225 let requests = Arc::new(Mutex::new(Vec::new()));
1226 let handler_requests = Arc::clone(&requests);
1227 let broker = scripted_broker(Vec::new(), move |api_key, version, body, own| {
1228 if api_key != metadata_request::API_KEY {
1229 return MockReply::Silent;
1230 }
1231 let mut requests = handler_requests.lock().expect("requests lock");
1232 let request: MetadataRequest = decode_request(body, version, true);
1233 requests.push(request);
1234 if close_first && requests.len() == 1 {
1235 return MockReply::Close;
1236 }
1237 MockReply::Respond(encode_response(
1238 &metadata(
1239 own,
1240 vec![
1241 metadata_topic("alpha", 0, &[0], 1),
1242 metadata_topic("__consumer_offsets", 0, &[0], 1),
1243 ],
1244 ),
1245 version,
1246 true,
1247 ))
1248 })
1249 .await;
1250 let admin = fast_admin(broker.addr, krabka_units::secs(5)).await;
1251 let result = admin
1252 .list_topics(
1253 ListTopicsOptions::builder()
1254 .list_internal(list_internal)
1255 .build(),
1256 )
1257 .await
1258 .expect("list topics");
1259 broker.stop();
1260 let requests = requests.lock().expect("requests lock").clone();
1261 assert2::assert!(
1262 (result, requests) == (expected.0, vec![all_topics_request(); expected.1]),
1263 "case {name}"
1264 );
1265 }
1266 }
1267
1268 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1273 async fn describe_topics_by_id_uses_metadata_v12() {
1274 let nil = topic_err(
1275 17,
1276 Some(format!(
1277 "The given topic id '{}' cannot be represented in a request.",
1278 Uuid::nil()
1279 )),
1280 );
1281 let unknown_beta = topic_err(100, Some(format!("TopicId {BETA} not found.")));
1282 let unsupported = topic_err(35, None);
1283 let options = DescribeTopicsOptions::builder()
1284 .include_authorized_operations(true)
1285 .build();
1286 let asked = maplit::btreeset! {
1287 ALPHA,
1288 BETA,
1289 };
1290 for (name, max_metadata, topics, expected_requests) in [
1291 (
1292 "found and missing",
1293 12,
1294 vec![metadata_topic("alpha", 0, &[0], 1)],
1295 vec![(12, metadata_by_id_request(&asked, options))],
1296 ),
1297 (
1298 "a topic with an error is unknown",
1299 13,
1300 vec![
1301 metadata_topic("alpha", 0, &[0], 1),
1302 metadata_topic("beta", 29, &[], -1),
1303 ],
1304 vec![(13, metadata_by_id_request(&asked, options))],
1305 ),
1306 ("a peer below v12", 11, Vec::new(), Vec::new()),
1307 ] {
1308 let requests = Arc::new(Mutex::new(Vec::new()));
1309 let handler_requests = Arc::clone(&requests);
1310 let broker = scripted_broker(
1311 vec![
1312 ApiVersionRange::builder()
1313 .api_key(metadata_request::API_KEY)
1314 .min_version(1)
1315 .max_version(max_metadata)
1316 .build(),
1317 ],
1318 move |api_key, version, body, own| {
1319 if api_key != metadata_request::API_KEY {
1320 return MockReply::Silent;
1321 }
1322 let request: MetadataRequest = decode_request(body, version, true);
1323 handler_requests
1324 .lock()
1325 .expect("requests lock")
1326 .push((version, request));
1327 MockReply::Respond(encode_response(
1328 &metadata(own, topics.clone()),
1329 version,
1330 true,
1331 ))
1332 },
1333 )
1334 .await;
1335 let admin = fast_admin(broker.addr, krabka_units::secs(5)).await;
1336 let result = admin
1337 .describe_topics_by_id(&[BETA, Uuid::nil(), ALPHA], options)
1338 .await
1339 .into_iter()
1340 .map(|(id, result)| {
1341 let result = result.map_err(|error| KafkaError {
1344 message: error.message.filter(|_| error.code != 35),
1345 ..error
1346 });
1347 (id, result)
1348 })
1349 .collect::<BTreeMap<_, _>>();
1350 let own = broker.addr;
1351 broker.stop();
1352 let expected = if max_metadata < 12 {
1353 maplit::btreemap! {
1354 Uuid::nil() => nil.clone(),
1355 ALPHA => unsupported.clone(),
1356 BETA => unsupported.clone(),
1357 }
1358 } else {
1359 maplit::btreemap! {
1360 Uuid::nil() => nil.clone(),
1361 ALPHA => Ok(from_metadata("alpha", &[0], own, true)),
1362 BETA => unknown_beta.clone(),
1363 }
1364 };
1365 assert2::assert!(
1366 (result, requests.lock().expect("requests lock").clone())
1367 == (expected, expected_requests),
1368 "case {name}"
1369 );
1370 }
1371 }
1372
1373 #[test]
1376 fn describe_page_reads_authorized_operations_on_request() {
1377 for (include, expected) in [
1378 (false, None),
1379 (
1380 true,
1381 Some(maplit::btreeset! {
1382 AclOperation::Read,
1383 }),
1384 ),
1385 ] {
1386 let description = description_from_describe(
1387 &page_topic("alpha", 0, &[]),
1388 "alpha".to_owned(),
1389 &BTreeMap::new(),
1390 include,
1391 );
1392 assert2::assert!(
1393 description.authorized_operations == expected,
1394 "include {include}"
1395 );
1396 }
1397 }
1398}