Skip to main content

krabka_client_admin/
topic_descriptions.rs

1//! Topic listing and description, as Kafka's `Admin.listTopics` and
2//! `Admin.describeTopics`.
3//!
4//! `listTopics` and `describeTopics` by topic ID read `Metadata`.
5//! `describeTopics` by name first reads the cluster's nodes with
6//! `DescribeCluster`, then pages through `DescribeTopicPartitions` (KIP-966),
7//! and falls back to `Metadata` when the peer does not support it, as Kafka's
8//! `KafkaAdminClient` does.
9
10use std::collections::{BTreeMap, BTreeSet};
11
12use krabka_client_core::ClientError;
13use krabka_protocol::{
14    ProtocolRequest as _,
15    owned::{
16        describe_topic_partitions_request::{
17            Cursor as RequestCursor, DescribeTopicPartitionsRequest, TopicRequest,
18        },
19        describe_topic_partitions_response::{
20            DescribeTopicPartitionsResponse, DescribeTopicPartitionsResponsePartition,
21            DescribeTopicPartitionsResponseTopic,
22        },
23        metadata_request::{MetadataRequest, MetadataRequestTopic},
24        metadata_response::{MetadataResponse, MetadataResponsePartition, MetadataResponseTopic},
25    },
26    primitives::uuid::Uuid as ProtoUuid,
27};
28use uuid::Uuid;
29
30use crate::{
31    AdminClient, AdminError, ClusterNode, DescribeClusterOptions, KafkaError,
32    cluster::authorized_operations, groups::list_groups_kafka_error, kafka_error_name,
33    retry::ControllerRetry, users::AclOperation,
34};
35
36/// `UNKNOWN_TOPIC_OR_PARTITION`.
37const UNKNOWN_TOPIC_OR_PARTITION: i16 = 3;
38/// `INVALID_TOPIC_EXCEPTION`.
39const INVALID_TOPIC_EXCEPTION: i16 = 17;
40/// `UNKNOWN_TOPIC_ID`.
41const UNKNOWN_TOPIC_ID: i16 = 100;
42
43/// What [`AdminClient::list_topics`] lists, as Kafka's `ListTopicsOptions`.
44#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, bon::Builder)]
45pub struct ListTopicsOptions {
46    /// List internal topics, such as `__consumer_offsets`, too.
47    #[builder(default)]
48    pub list_internal: bool,
49}
50
51/// One topic of [`AdminClient::list_topics`], as Kafka's `TopicListing`.
52#[derive(Debug, Clone, PartialEq, Eq)]
53pub struct TopicListing {
54    /// The topic name.
55    pub name: String,
56    /// The topic ID, or `None` when the response carries the zero ID.
57    pub topic_id: Option<Uuid>,
58    /// Whether the topic is internal.
59    pub is_internal: bool,
60}
61
62/// What [`AdminClient::describe_topics`] asks for, as Kafka's
63/// `DescribeTopicsOptions`.
64#[derive(Debug, Clone, Copy, PartialEq, Eq, bon::Builder)]
65pub struct DescribeTopicsOptions {
66    /// Ask for the operations that the caller may perform on each topic.
67    #[builder(default)]
68    pub include_authorized_operations: bool,
69    /// The most partitions that one `DescribeTopicPartitions` answer
70    /// carries. Kafka's default is 2000.
71    #[builder(default = 2000)]
72    pub partition_size_limit_per_response: i32,
73}
74
75impl Default for DescribeTopicsOptions {
76    fn default() -> Self {
77        Self {
78            include_authorized_operations: false,
79            partition_size_limit_per_response: 2000,
80        }
81    }
82}
83
84/// One partition of a [`TopicDescription`], as Kafka's
85/// `TopicPartitionInfo`.
86///
87/// A replica that the cluster's node list does not name appears with an
88/// empty host and port -1, as Kafka's `new Node(id, "", -1)`.
89#[derive(Debug, Clone, PartialEq, Eq)]
90pub struct TopicPartitionInfo {
91    /// The partition index.
92    pub partition: i32,
93    /// The leader, or `None` when the partition has no leader or the leader
94    /// is not a known node.
95    pub leader: Option<ClusterNode>,
96    /// The replicas, preferred replica first.
97    pub replicas: Vec<ClusterNode>,
98    /// The in-sync replicas.
99    pub isr: Vec<ClusterNode>,
100    /// The eligible leader replicas (KIP-966). Always empty through
101    /// `Metadata`.
102    pub elr: Vec<ClusterNode>,
103    /// The last known eligible leader replicas (KIP-966). Always empty
104    /// through `Metadata`.
105    pub last_known_elr: Vec<ClusterNode>,
106}
107
108/// One topic of [`AdminClient::describe_topics`], as Kafka's
109/// `TopicDescription`.
110#[derive(Debug, Clone, PartialEq, Eq)]
111pub struct TopicDescription {
112    /// The topic name.
113    pub name: String,
114    /// The topic ID, or `None` when the response carries the zero ID.
115    pub topic_id: Option<Uuid>,
116    /// Whether the topic is internal.
117    pub is_internal: bool,
118    /// The partitions, in partition order.
119    pub partitions: Vec<TopicPartitionInfo>,
120    /// The operations that the caller may perform on the topic, or `None`
121    /// when the caller did not ask for them.
122    pub authorized_operations: Option<BTreeSet<AclOperation>>,
123}
124
125/// The description of each topic of a describe call, or its error.
126pub type TopicDescriptions<K> = BTreeMap<K, Result<TopicDescription, KafkaError>>;
127
128fn topic_id(id: ProtoUuid) -> Option<Uuid> {
129    (id != ProtoUuid::ZERO).then(|| Uuid::from_bytes(id.0))
130}
131
132fn topic_error(code: i16, message: String) -> KafkaError {
133    KafkaError {
134        code,
135        name: kafka_error_name(code),
136        message: Some(message),
137    }
138}
139
140/// The node of `id`, or Kafka's placeholder `Node(id, "", -1)`.
141fn node(nodes: &BTreeMap<i32, ClusterNode>, id: i32) -> ClusterNode {
142    nodes.get(&id).cloned().unwrap_or_else(|| ClusterNode {
143        id,
144        host: String::new(),
145        port: -1,
146        rack: None,
147        is_fenced: false,
148    })
149}
150
151fn node_list(nodes: &BTreeMap<i32, ClusterNode>, ids: &[i32]) -> Vec<ClusterNode> {
152    ids.iter().map(|id| node(nodes, *id)).collect()
153}
154
155/// The request of Kafka's `MetadataRequest.Builder.allTopics()`.
156fn all_topics_request() -> MetadataRequest {
157    MetadataRequest {
158        topics: None,
159        allow_auto_topic_creation: true,
160        ..Default::default()
161    }
162}
163
164/// The listings of a `Metadata` answer, as Kafka's `listTopics`
165/// `handleResponse` reads it.
166fn topic_listings(
167    response: MetadataResponse,
168    options: ListTopicsOptions,
169) -> BTreeMap<String, TopicListing> {
170    response
171        .topics
172        .into_iter()
173        .filter(|topic| !topic.is_internal || options.list_internal)
174        .filter_map(|topic| {
175            let name = topic.name?;
176            Some((
177                name.clone(),
178                TopicListing {
179                    name,
180                    topic_id: topic_id(topic.topic_id),
181                    is_internal: topic.is_internal,
182                },
183            ))
184        })
185        .collect()
186}
187
188/// One `DescribeTopicPartitions` request, as Kafka's
189/// `generateDescribeTopicsCallWithDescribeTopicPartitionsApi`
190/// `createRequest` builds it. The cursor points past the partitions already
191/// read of the topic that the previous answer cut.
192fn describe_partitions_request(
193    pending: &BTreeSet<String>,
194    partial: Option<&TopicDescription>,
195    options: DescribeTopicsOptions,
196) -> DescribeTopicPartitionsRequest {
197    DescribeTopicPartitionsRequest {
198        topics: pending
199            .iter()
200            .map(|name| TopicRequest {
201                name: name.clone(),
202                ..Default::default()
203            })
204            .collect(),
205        response_partition_limit: options.partition_size_limit_per_response,
206        cursor: partial.map(|partial| RequestCursor {
207            topic_name: partial.name.clone(),
208            partition_index: i32::try_from(partial.partitions.len()).unwrap_or(i32::MAX),
209            ..Default::default()
210        }),
211        ..Default::default()
212    }
213}
214
215fn partition_from_describe(
216    partition: &DescribeTopicPartitionsResponsePartition,
217    nodes: &BTreeMap<i32, ClusterNode>,
218) -> TopicPartitionInfo {
219    TopicPartitionInfo {
220        partition: partition.partition_index,
221        leader: nodes.get(&partition.leader_id).cloned(),
222        replicas: node_list(nodes, &partition.replica_nodes),
223        isr: node_list(nodes, &partition.isr_nodes),
224        elr: node_list(
225            nodes,
226            partition
227                .eligible_leader_replicas
228                .as_deref()
229                .unwrap_or_default(),
230        ),
231        last_known_elr: node_list(
232            nodes,
233            partition.last_known_elr.as_deref().unwrap_or_default(),
234        ),
235    }
236}
237
238/// The description of one `DescribeTopicPartitions` topic, as Kafka's
239/// `getTopicDescriptionFromDescribeTopicsResponseTopic`.
240fn description_from_describe(
241    topic: &DescribeTopicPartitionsResponseTopic,
242    name: String,
243    nodes: &BTreeMap<i32, ClusterNode>,
244    include_authorized_operations: bool,
245) -> TopicDescription {
246    TopicDescription {
247        name,
248        topic_id: topic_id(topic.topic_id),
249        is_internal: topic.is_internal,
250        partitions: topic
251            .partitions
252            .iter()
253            .map(|partition| partition_from_describe(partition, nodes))
254            .collect(),
255        authorized_operations: if include_authorized_operations {
256            authorized_operations(topic.topic_authorized_operations)
257        } else {
258            None
259        },
260    }
261}
262
263/// The paging state of one `describeTopics` by name.
264#[derive(Debug, Default)]
265struct DescribePages {
266    /// The topics that no answer has finished.
267    pending: BTreeSet<String>,
268    /// The topic that the last answer cut, with the partitions read so far.
269    partial: Option<TopicDescription>,
270    /// The finished topics.
271    out: TopicDescriptions<String>,
272}
273
274impl DescribePages {
275    /// Reads one answer, as Kafka's `handleResponse` does: a topic error
276    /// finishes the topic, the cursor topic waits for the next page, and the
277    /// partial topic of the previous page gets the new partitions.
278    fn read(
279        &mut self,
280        response: DescribeTopicPartitionsResponse,
281        nodes: &BTreeMap<i32, ClusterNode>,
282        include_authorized_operations: bool,
283    ) {
284        let mut cursor = response.next_cursor.map(|cursor| cursor.topic_name);
285        let mut next = None;
286        for topic in response.topics {
287            let Some(name) = topic.name.clone() else {
288                tracing::warn!("the DescribeTopicPartitions response names a topic without a name");
289                continue;
290            };
291            let asked = self.pending.contains(&name)
292                || self.partial.as_ref().is_some_and(|p| p.name == name);
293            if !asked {
294                tracing::warn!(
295                    topic = %name,
296                    "the DescribeTopicPartitions response names a topic that is not in the request"
297                );
298                continue;
299            }
300            if topic.error_code != 0 {
301                self.out.entry(name.clone()).or_insert_with(|| {
302                    Err(KafkaError {
303                        code: topic.error_code,
304                        name: kafka_error_name(topic.error_code),
305                        message: None,
306                    })
307                });
308                self.pending.remove(&name);
309                if cursor.as_deref() == Some(name.as_str()) {
310                    cursor = None;
311                }
312                continue;
313            }
314            let current = description_from_describe(
315                &topic,
316                name.clone(),
317                nodes,
318                include_authorized_operations,
319            );
320            if let Some(partial) = self.partial.as_mut()
321                && partial.name == name
322            {
323                partial.partitions.extend(current.partitions);
324                continue;
325            }
326            if cursor.as_deref() == Some(name.as_str()) {
327                next = Some(current);
328                continue;
329            }
330            self.pending.remove(&name);
331            self.out.entry(name).or_insert(Ok(current));
332        }
333        if let Some(partial) = self
334            .partial
335            .take_if(|partial| cursor.as_deref() != Some(partial.name.as_str()))
336        {
337            self.pending.remove(&partial.name);
338            self.out.entry(partial.name.clone()).or_insert(Ok(partial));
339        }
340        if next.is_some() {
341            self.partial = next;
342        }
343    }
344
345    /// Whether another page is needed.
346    fn unfinished(&self) -> bool {
347        !self.pending.is_empty() || self.partial.is_some()
348    }
349
350    /// Fails every unfinished topic with `error`.
351    fn fail(mut self, error: &KafkaError) -> TopicDescriptions<String> {
352        let unfinished = std::mem::take(&mut self.pending)
353            .into_iter()
354            .chain(self.partial.take().map(|partial| partial.name));
355        for name in unfinished {
356            self.out.entry(name).or_insert_with(|| Err(error.clone()));
357        }
358        self.out
359    }
360}
361
362/// The `Metadata` request of `describeTopics` by name, as Kafka's
363/// `generateDescribeTopicsCallWithMetadataApi` builds it.
364fn metadata_by_name_request(
365    names: &BTreeSet<String>,
366    options: DescribeTopicsOptions,
367) -> MetadataRequest {
368    MetadataRequest {
369        topics: Some(
370            names
371                .iter()
372                .map(|name| MetadataRequestTopic {
373                    topic_id: ProtoUuid::ZERO,
374                    name: Some(name.clone()),
375                    ..Default::default()
376                })
377                .collect(),
378        ),
379        allow_auto_topic_creation: false,
380        include_topic_authorized_operations: options.include_authorized_operations,
381        ..Default::default()
382    }
383}
384
385/// The `Metadata` request of `describeTopics` by ID, as Kafka's
386/// `handleDescribeTopicsByIds` builds it.
387fn metadata_by_id_request(ids: &BTreeSet<Uuid>, options: DescribeTopicsOptions) -> MetadataRequest {
388    MetadataRequest {
389        topics: Some(
390            ids.iter()
391                .map(|id| MetadataRequestTopic {
392                    topic_id: ProtoUuid(*id.as_bytes()),
393                    name: None,
394                    ..Default::default()
395                })
396                .collect(),
397        ),
398        allow_auto_topic_creation: false,
399        include_topic_authorized_operations: options.include_authorized_operations,
400        ..Default::default()
401    }
402}
403
404fn metadata_nodes(response: &MetadataResponse) -> BTreeMap<i32, ClusterNode> {
405    response
406        .brokers
407        .iter()
408        .map(|broker| {
409            (
410                broker.node_id,
411                ClusterNode {
412                    id: broker.node_id,
413                    host: broker.host.clone(),
414                    port: broker.port,
415                    rack: broker.rack.clone(),
416                    is_fenced: false,
417                },
418            )
419        })
420        .collect()
421}
422
423fn partition_from_metadata(
424    partition: &MetadataResponsePartition,
425    nodes: &BTreeMap<i32, ClusterNode>,
426) -> TopicPartitionInfo {
427    TopicPartitionInfo {
428        partition: partition.partition_index,
429        leader: (partition.leader_id >= 0)
430            .then(|| nodes.get(&partition.leader_id).cloned())
431            .flatten(),
432        replicas: node_list(nodes, &partition.replica_nodes),
433        isr: node_list(nodes, &partition.isr_nodes),
434        elr: Vec::new(),
435        last_known_elr: Vec::new(),
436    }
437}
438
439/// The description of one `Metadata` topic, as Kafka's
440/// `getTopicDescriptionFromCluster`: partitions in partition order, and the
441/// authorized operations whenever the answer carries them.
442fn description_from_metadata(
443    topic: &MetadataResponseTopic,
444    name: String,
445    nodes: &BTreeMap<i32, ClusterNode>,
446) -> TopicDescription {
447    let mut partitions = topic
448        .partitions
449        .iter()
450        .map(|partition| partition_from_metadata(partition, nodes))
451        .collect::<Vec<_>>();
452    partitions.sort_by_key(|partition| partition.partition);
453    TopicDescription {
454        name,
455        topic_id: topic_id(topic.topic_id),
456        is_internal: topic.is_internal,
457        partitions,
458        authorized_operations: authorized_operations(topic.topic_authorized_operations),
459    }
460}
461
462/// The description of each topic of `names` in a `Metadata` answer, as
463/// Kafka's `generateDescribeTopicsCallWithMetadataApi` `handleResponse`
464/// reads it: a topic error first, then `UNKNOWN_TOPIC_OR_PARTITION` for a
465/// topic the answer does not name.
466fn descriptions_by_name(
467    names: &BTreeSet<String>,
468    response: MetadataResponse,
469) -> TopicDescriptions<String> {
470    let nodes = metadata_nodes(&response);
471    let mut topics = response
472        .topics
473        .into_iter()
474        .filter_map(|topic| Some((topic.name.clone()?, topic)))
475        .collect::<BTreeMap<_, _>>();
476    names
477        .iter()
478        .map(|name| {
479            let result = match topics.remove(name) {
480                Some(topic) if topic.error_code != 0 => Err(KafkaError {
481                    code: topic.error_code,
482                    name: kafka_error_name(topic.error_code),
483                    message: None,
484                }),
485                Some(topic) => Ok(description_from_metadata(&topic, name.clone(), &nodes)),
486                None => Err(topic_error(
487                    UNKNOWN_TOPIC_OR_PARTITION,
488                    format!("Topic {name} not found."),
489                )),
490            };
491            (name.clone(), result)
492        })
493        .collect()
494}
495
496/// The description of each topic of `ids` in a `Metadata` answer, as
497/// Kafka's `handleDescribeTopicsByIds` `handleResponse` reads it. Kafka's
498/// `Cluster` holds only topics without an error, so an ID that the answer
499/// does not name without an error gives `UNKNOWN_TOPIC_ID`.
500fn descriptions_by_id(ids: &BTreeSet<Uuid>, response: MetadataResponse) -> TopicDescriptions<Uuid> {
501    let nodes = metadata_nodes(&response);
502    let mut topics = response
503        .topics
504        .into_iter()
505        .filter(|topic| topic.error_code == 0)
506        .filter_map(|topic| Some((topic_id(topic.topic_id)?, topic)))
507        .collect::<BTreeMap<_, _>>();
508    ids.iter()
509        .map(|id| {
510            let result = match topics.remove(id) {
511                Some(mut topic) => match topic.name.take() {
512                    Some(name) => Ok(description_from_metadata(&topic, name, &nodes)),
513                    None => Err(topic_error(
514                        UNKNOWN_TOPIC_ID,
515                        format!("TopicId {id} not found."),
516                    )),
517                },
518                None => Err(topic_error(
519                    UNKNOWN_TOPIC_ID,
520                    format!("TopicId {id} not found."),
521                )),
522            };
523            (*id, result)
524        })
525        .collect()
526}
527
528/// Whether `error` means that the peer does not support the request's
529/// version, as Kafka's `UnsupportedVersionException`.
530const fn is_unsupported_version(error: &AdminError) -> bool {
531    matches!(
532        error,
533        AdminError::Transport(ClientError::IncompatibleVersion { .. })
534    )
535}
536
537impl AdminClient {
538    /// Lists the topics of the cluster, as Kafka's `listTopics` operation
539    /// does: one `Metadata` request for all topics, keeping internal topics
540    /// only with [`ListTopicsOptions::list_internal`].
541    ///
542    /// # Errors
543    /// Returns a transport error, and [`AdminError::Broker`] with
544    /// `REQUEST_TIMED_OUT` (7) at `default.api.timeout.ms` (60 s).
545    pub async fn list_topics(
546        &self,
547        options: ListTopicsOptions,
548    ) -> Result<BTreeMap<String, TopicListing>, AdminError> {
549        let retry = ControllerRetry::new("Metadata", self.retry);
550        let response = retry
551            .bounded(self.conn.send_at_least(all_topics_request(), 1))
552            .await?;
553        Ok(topic_listings(response, options))
554    }
555
556    /// Describes each topic of `names`, as Kafka's `describeTopics`
557    /// operation does for a `TopicNameCollection`.
558    ///
559    /// The client reads the nodes of the cluster with `DescribeCluster`,
560    /// then sends `DescribeTopicPartitions` (KIP-966) and follows its cursor
561    /// until every topic is complete, at most
562    /// [`DescribeTopicsOptions::partition_size_limit_per_response`]
563    /// partitions per answer. When the peer does not support
564    /// `DescribeTopicPartitions`, such as a controller reached through
565    /// controller bootstrap, it sends `Metadata` instead.
566    ///
567    /// The result has one entry for each name:
568    ///
569    /// - The description of the topic.
570    /// - `INVALID_TOPIC_EXCEPTION` (17) for an empty name, which Kafka
571    ///   cannot send.
572    /// - The topic error of the answer, such as `UNKNOWN_TOPIC_OR_PARTITION`
573    ///   (3) or `TOPIC_AUTHORIZATION_FAILED` (29).
574    /// - The error of the call for every topic it did not finish, such as a
575    ///   failed `DescribeCluster`, a transport error, or `REQUEST_TIMED_OUT`
576    ///   (7) at `default.api.timeout.ms` (60 s).
577    pub async fn describe_topics(
578        &self,
579        names: &[&str],
580        options: DescribeTopicsOptions,
581    ) -> TopicDescriptions<String> {
582        let mut pages = DescribePages::default();
583        for name in names {
584            if name.is_empty() {
585                pages.out.insert(
586                    String::new(),
587                    Err(topic_error(
588                        INVALID_TOPIC_EXCEPTION,
589                        "The given topic name '' cannot be represented in a request.".to_owned(),
590                    )),
591                );
592            } else {
593                pages.pending.insert((*name).to_owned());
594            }
595        }
596        if pages.pending.is_empty() {
597            return pages.out;
598        }
599        let nodes = match self
600            .describe_cluster(DescribeClusterOptions::builder().build())
601            .await
602        {
603            Ok(cluster) => cluster
604                .nodes
605                .into_iter()
606                .map(|node| (node.id, node))
607                .collect::<BTreeMap<_, _>>(),
608            Err(error) => return pages.fail(&list_groups_kafka_error(&error)),
609        };
610        if self
611            .conn
612            .advertised_api_range(DescribeTopicPartitionsRequest::API_KEY)
613            .await
614            .is_none()
615        {
616            return self.describe_topics_with_metadata(pages, options).await;
617        }
618        let retry = ControllerRetry::new("DescribeTopicPartitions", self.retry);
619        while pages.unfinished() {
620            let request =
621                describe_partitions_request(&pages.pending, pages.partial.as_ref(), options);
622            match retry.bounded(self.conn.send(request)).await {
623                Ok(response) => pages.read(response, &nodes, options.include_authorized_operations),
624                Err(error) if is_unsupported_version(&error) => {
625                    return self.describe_topics_with_metadata(pages, options).await;
626                }
627                Err(error) => return pages.fail(&list_groups_kafka_error(&error)),
628            }
629        }
630        pages.out
631    }
632
633    /// The `Metadata` fallback of [`Self::describe_topics`] for the topics
634    /// that `pages` has not finished. A peer below `Metadata` v4 cannot
635    /// refuse auto-creation, so it gets a request for all topics, as
636    /// Kafka's `supportsDisablingTopicCreation` fallback sends.
637    async fn describe_topics_with_metadata(
638        &self,
639        mut pages: DescribePages,
640        options: DescribeTopicsOptions,
641    ) -> TopicDescriptions<String> {
642        if let Some(partial) = pages.partial.take() {
643            pages.pending.insert(partial.name);
644        }
645        let retry = ControllerRetry::new("Metadata", self.retry);
646        let request = metadata_by_name_request(&pages.pending, options);
647        let response = match retry.bounded(self.conn.send_at_least(request, 4)).await {
648            Err(error) if is_unsupported_version(&error) => {
649                retry
650                    .bounded(self.conn.send_at_least(all_topics_request(), 1))
651                    .await
652            }
653            other => other,
654        };
655        match response {
656            Ok(response) => {
657                let pending = std::mem::take(&mut pages.pending);
658                for (name, result) in descriptions_by_name(&pending, response) {
659                    pages.out.entry(name).or_insert(result);
660                }
661                pages.out
662            }
663            Err(error) => pages.fail(&list_groups_kafka_error(&error)),
664        }
665    }
666
667    /// Describes each topic of `ids`, as Kafka's `describeTopics` operation
668    /// does for a `TopicIdCollection`: one `Metadata` request (v12 or
669    /// higher, which Kafka needs for topic IDs).
670    ///
671    /// The result has one entry for each ID: the description of the topic,
672    /// `INVALID_TOPIC_EXCEPTION` (17) for the nil ID, `UNKNOWN_TOPIC_ID`
673    /// (100) for an ID that the answer does not describe, or the error of
674    /// the call, such as `UNSUPPORTED_VERSION` (35) for a peer below
675    /// `Metadata` v12.
676    pub async fn describe_topics_by_id(
677        &self,
678        ids: &[Uuid],
679        options: DescribeTopicsOptions,
680    ) -> TopicDescriptions<Uuid> {
681        let mut out = TopicDescriptions::new();
682        let mut pending = BTreeSet::new();
683        for id in ids {
684            if id.is_nil() {
685                out.insert(
686                    *id,
687                    Err(topic_error(
688                        INVALID_TOPIC_EXCEPTION,
689                        format!("The given topic id '{id}' cannot be represented in a request."),
690                    )),
691                );
692            } else {
693                pending.insert(*id);
694            }
695        }
696        if pending.is_empty() {
697            return out;
698        }
699        let retry = ControllerRetry::new("Metadata", self.retry);
700        let request = metadata_by_id_request(&pending, options);
701        match retry.bounded(self.conn.send_at_least(request, 12)).await {
702            Ok(response) => out.extend(descriptions_by_id(&pending, response)),
703            Err(error) => {
704                let error = list_groups_kafka_error(&error);
705                out.extend(pending.into_iter().map(|id| (id, Err(error.clone()))));
706            }
707        }
708        out
709    }
710}
711
712#[cfg(test)]
713mod tests {
714    use std::{
715        net::SocketAddr,
716        sync::{Arc, Mutex},
717    };
718
719    use krabka_client_core::MockReply;
720    use krabka_protocol::owned::{
721        describe_cluster_request,
722        describe_cluster_response::{DescribeClusterBroker, DescribeClusterResponse},
723        describe_topic_partitions_request,
724        describe_topic_partitions_response::Cursor as ResponseCursor,
725        metadata_request,
726        metadata_response::MetadataResponseBroker,
727    };
728
729    use super::*;
730    use crate::partition_leaders::test_support::{
731        ApiVersionRange, decode_request, encode_response, fast_admin, scripted_broker,
732    };
733
734    const ALPHA: Uuid = Uuid::from_u128(0xa);
735    const BETA: Uuid = Uuid::from_u128(0xb);
736
737    fn id_of(name: &str) -> Uuid {
738        if name == "alpha" { ALPHA } else { BETA }
739    }
740
741    /// Broker 1 at `own`, the only node that the cluster names.
742    fn broker_one(own: SocketAddr) -> ClusterNode {
743        ClusterNode {
744            id: 1,
745            host: own.ip().to_string(),
746            port: i32::from(own.port()),
747            rack: None,
748            is_fenced: false,
749        }
750    }
751
752    /// Broker 2, which the cluster does not name: Kafka's placeholder node.
753    fn unknown_two() -> ClusterNode {
754        ClusterNode {
755            id: 2,
756            host: String::new(),
757            port: -1,
758            rack: None,
759            is_fenced: false,
760        }
761    }
762
763    /// Partition `index` as `DescribeTopicPartitions` gives it: leader 1,
764    /// replicas 1 and 2, ISR 1, ELR 2.
765    fn page_partition(index: i32) -> DescribeTopicPartitionsResponsePartition {
766        DescribeTopicPartitionsResponsePartition {
767            partition_index: index,
768            leader_id: 1,
769            replica_nodes: vec![1, 2],
770            isr_nodes: vec![1],
771            eligible_leader_replicas: Some(vec![2]),
772            last_known_elr: Some(Vec::new()),
773            ..Default::default()
774        }
775    }
776
777    /// A `DescribeTopicPartitions` topic whose authorized operations are
778    /// `READ` (bit 3).
779    fn page_topic(
780        name: &str,
781        error_code: i16,
782        partitions: &[i32],
783    ) -> DescribeTopicPartitionsResponseTopic {
784        DescribeTopicPartitionsResponseTopic {
785            error_code,
786            name: Some(name.to_owned()),
787            topic_id: ProtoUuid(*id_of(name).as_bytes()),
788            partitions: partitions.iter().copied().map(page_partition).collect(),
789            topic_authorized_operations: 1 << 3,
790            ..Default::default()
791        }
792    }
793
794    fn page(
795        topics: Vec<DescribeTopicPartitionsResponseTopic>,
796        cursor: Option<(&str, i32)>,
797    ) -> DescribeTopicPartitionsResponse {
798        DescribeTopicPartitionsResponse {
799            topics,
800            next_cursor: cursor.map(|(topic_name, partition_index)| ResponseCursor {
801                topic_name: topic_name.to_owned(),
802                partition_index,
803                ..Default::default()
804            }),
805            ..Default::default()
806        }
807    }
808
809    /// The description that `DescribeTopicPartitions` pages give `name`.
810    fn paged(name: &str, partitions: &[i32], own: SocketAddr) -> TopicDescription {
811        TopicDescription {
812            name: name.to_owned(),
813            topic_id: Some(id_of(name)),
814            is_internal: false,
815            partitions: partitions
816                .iter()
817                .map(|partition| TopicPartitionInfo {
818                    partition: *partition,
819                    leader: Some(broker_one(own)),
820                    replicas: vec![broker_one(own), unknown_two()],
821                    isr: vec![broker_one(own)],
822                    elr: vec![unknown_two()],
823                    last_known_elr: Vec::new(),
824                })
825                .collect(),
826            authorized_operations: None,
827        }
828    }
829
830    fn dtp_request(topics: &[&str], cursor: Option<(&str, i32)>) -> DescribeTopicPartitionsRequest {
831        DescribeTopicPartitionsRequest {
832            topics: topics
833                .iter()
834                .map(|name| TopicRequest {
835                    name: (*name).to_owned(),
836                    ..Default::default()
837                })
838                .collect(),
839            response_partition_limit: 2000,
840            cursor: cursor.map(|(topic_name, partition_index)| RequestCursor {
841                topic_name: topic_name.to_owned(),
842                partition_index,
843                ..Default::default()
844            }),
845            ..Default::default()
846        }
847    }
848
849    /// A `Metadata` topic with partitions `indexes` in that order: leader
850    /// `leader`, replicas 1 and 2, ISR 1.
851    fn metadata_topic(
852        name: &str,
853        error_code: i16,
854        indexes: &[i32],
855        leader: i32,
856    ) -> MetadataResponseTopic {
857        MetadataResponseTopic {
858            error_code,
859            name: Some(name.to_owned()),
860            topic_id: ProtoUuid(*id_of(name).as_bytes()),
861            is_internal: name.starts_with("__"),
862            partitions: indexes
863                .iter()
864                .map(|index| MetadataResponsePartition {
865                    partition_index: *index,
866                    leader_id: leader,
867                    replica_nodes: vec![1, 2],
868                    isr_nodes: vec![1],
869                    ..Default::default()
870                })
871                .collect(),
872            ..Default::default()
873        }
874    }
875
876    fn metadata(own: SocketAddr, topics: Vec<MetadataResponseTopic>) -> MetadataResponse {
877        MetadataResponse {
878            brokers: vec![MetadataResponseBroker {
879                node_id: 1,
880                host: own.ip().to_string(),
881                port: i32::from(own.port()),
882                ..Default::default()
883            }],
884            controller_id: 1,
885            topics,
886            ..Default::default()
887        }
888    }
889
890    /// The description that `Metadata` gives `name`, partitions sorted.
891    fn from_metadata(
892        name: &str,
893        partitions: &[i32],
894        own: SocketAddr,
895        leader: bool,
896    ) -> TopicDescription {
897        TopicDescription {
898            name: name.to_owned(),
899            topic_id: Some(id_of(name)),
900            is_internal: false,
901            partitions: partitions
902                .iter()
903                .map(|partition| TopicPartitionInfo {
904                    partition: *partition,
905                    leader: leader.then(|| broker_one(own)),
906                    replicas: vec![broker_one(own), unknown_two()],
907                    isr: vec![broker_one(own)],
908                    elr: Vec::new(),
909                    last_known_elr: Vec::new(),
910                })
911                .collect(),
912            authorized_operations: None,
913        }
914    }
915
916    fn topic_err(code: i16, message: Option<String>) -> Result<TopicDescription, KafkaError> {
917        Err(KafkaError {
918            code,
919            name: kafka_error_name(code),
920            message,
921        })
922    }
923
924    /// What the mock broker of one case answers to `DescribeTopicPartitions`.
925    #[derive(Clone)]
926    enum Answer {
927        Page(DescribeTopicPartitionsResponse),
928        Close,
929    }
930
931    /// What one mock broker received.
932    #[derive(Debug, Clone, PartialEq)]
933    enum Received {
934        Pages(DescribeTopicPartitionsRequest),
935        Metadata(i16, MetadataRequest),
936    }
937
938    /// One case of [`describe_topics_pages_and_falls_back_to_metadata`].
939    struct DescribeCase {
940        name: &'static str,
941        names: Vec<&'static str>,
942        cluster_error: i16,
943        dtp: bool,
944        answers: Vec<Answer>,
945        metadata_topics: Vec<MetadataResponseTopic>,
946        expected: fn(SocketAddr) -> (TopicDescriptions<String>, Vec<Received>),
947    }
948
949    /// Runs `describe_topics` against a broker that answers
950    /// `DescribeCluster` with the case's error and broker 1, each
951    /// `DescribeTopicPartitions` request with the next answer (when the case
952    /// advertises the API), and `Metadata` with the case's topics.
953    async fn describe_case(
954        case: &DescribeCase,
955    ) -> (TopicDescriptions<String>, Vec<Received>, SocketAddr) {
956        let received = Arc::new(Mutex::new(Vec::new()));
957        let handler_received = Arc::clone(&received);
958        let mut apis = vec![
959            ApiVersionRange::builder()
960                .api_key(describe_cluster_request::API_KEY)
961                .max_version(2)
962                .build(),
963        ];
964        if case.dtp {
965            apis.push(
966                ApiVersionRange::builder()
967                    .api_key(describe_topic_partitions_request::API_KEY)
968                    .build(),
969            );
970        }
971        let cluster_error = case.cluster_error;
972        let metadata_topics = case.metadata_topics.clone();
973        let pages = Arc::new(Mutex::new(case.answers.clone().into_iter()));
974        let broker = scripted_broker(apis, move |api_key, version, body, own| {
975            let mut received = handler_received.lock().expect("received lock");
976            let reply = match api_key {
977                describe_cluster_request::API_KEY => encode_response(
978                    &DescribeClusterResponse {
979                        error_code: cluster_error,
980                        controller_id: 1,
981                        brokers: vec![DescribeClusterBroker {
982                            broker_id: 1,
983                            host: own.ip().to_string(),
984                            port: i32::from(own.port()),
985                            ..Default::default()
986                        }],
987                        ..Default::default()
988                    },
989                    version,
990                    true,
991                ),
992                describe_topic_partitions_request::API_KEY => {
993                    received.push(Received::Pages(decode_request(body, version, true)));
994                    match pages.lock().expect("pages lock").next() {
995                        Some(Answer::Page(page)) => encode_response(&page, version, true),
996                        Some(Answer::Close) => return MockReply::Close,
997                        None => return MockReply::Silent,
998                    }
999                }
1000                metadata_request::API_KEY => {
1001                    let flexible = version >= metadata_request::FLEXIBLE_MIN;
1002                    received.push(Received::Metadata(
1003                        version,
1004                        decode_request(body, version, flexible),
1005                    ));
1006                    encode_response(&metadata(own, metadata_topics.clone()), version, flexible)
1007                }
1008                _ => return MockReply::Silent,
1009            };
1010            MockReply::Respond(reply)
1011        })
1012        .await;
1013        let admin = fast_admin(broker.addr, krabka_units::secs(5)).await;
1014        let result = admin
1015            .describe_topics(&case.names, DescribeTopicsOptions::builder().build())
1016            .await;
1017        let own = broker.addr;
1018        broker.stop();
1019        let received = received.lock().expect("received lock").clone();
1020        (result, received, own)
1021    }
1022
1023    /// Kafka's `describeTopics` by name pages through
1024    /// `DescribeTopicPartitions` with its cursor, maps each replica to its
1025    /// node, fails a topic with its error, retries a lost connection, falls
1026    /// back to `Metadata` when the peer lacks the API, and fails every topic
1027    /// when `DescribeCluster` fails.
1028    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1029    async fn describe_topics_pages_and_falls_back_to_metadata() {
1030        let cases = [
1031            DescribeCase {
1032                name: "one page",
1033                names: vec!["beta", "alpha"],
1034                cluster_error: 0,
1035                dtp: true,
1036                answers: vec![Answer::Page(page(
1037                    vec![page_topic("alpha", 0, &[0]), page_topic("beta", 0, &[0, 1])],
1038                    None,
1039                ))],
1040                metadata_topics: Vec::new(),
1041                expected: |own| {
1042                    (
1043                        maplit::btreemap! {
1044                            "alpha".to_owned() => Ok(paged("alpha", &[0], own)),
1045                            "beta".to_owned() => Ok(paged("beta", &[0, 1], own)),
1046                        },
1047                        vec![Received::Pages(dtp_request(&["alpha", "beta"], None))],
1048                    )
1049                },
1050            },
1051            DescribeCase {
1052                name: "the cursor splits a topic across pages",
1053                names: vec!["alpha", "beta"],
1054                cluster_error: 0,
1055                dtp: true,
1056                answers: vec![
1057                    Answer::Page(page(
1058                        vec![page_topic("alpha", 0, &[0]), page_topic("beta", 0, &[0])],
1059                        Some(("beta", 1)),
1060                    )),
1061                    Answer::Page(page(vec![page_topic("beta", 0, &[1, 2])], None)),
1062                ],
1063                metadata_topics: Vec::new(),
1064                expected: |own| {
1065                    (
1066                        maplit::btreemap! {
1067                            "alpha".to_owned() => Ok(paged("alpha", &[0], own)),
1068                            "beta".to_owned() => Ok(paged("beta", &[0, 1, 2], own)),
1069                        },
1070                        vec![
1071                            Received::Pages(dtp_request(&["alpha", "beta"], None)),
1072                            Received::Pages(dtp_request(&["beta"], Some(("beta", 1)))),
1073                        ],
1074                    )
1075                },
1076            },
1077            DescribeCase {
1078                name: "a topic error fails that topic only",
1079                names: vec!["alpha", "beta"],
1080                cluster_error: 0,
1081                dtp: true,
1082                answers: vec![Answer::Page(page(
1083                    vec![page_topic("alpha", 29, &[]), page_topic("beta", 0, &[0])],
1084                    None,
1085                ))],
1086                metadata_topics: Vec::new(),
1087                expected: |own| {
1088                    (
1089                        maplit::btreemap! {
1090                            "alpha".to_owned() => topic_err(29, None),
1091                            "beta".to_owned() => Ok(paged("beta", &[0], own)),
1092                        },
1093                        vec![Received::Pages(dtp_request(&["alpha", "beta"], None))],
1094                    )
1095                },
1096            },
1097            DescribeCase {
1098                name: "a lost connection is retried",
1099                names: vec!["alpha"],
1100                cluster_error: 0,
1101                dtp: true,
1102                answers: vec![
1103                    Answer::Close,
1104                    Answer::Page(page(vec![page_topic("alpha", 0, &[0])], None)),
1105                ],
1106                metadata_topics: Vec::new(),
1107                expected: |own| {
1108                    (
1109                        maplit::btreemap! {
1110                            "alpha".to_owned() => Ok(paged("alpha", &[0], own)),
1111                        },
1112                        vec![
1113                            Received::Pages(dtp_request(&["alpha"], None)),
1114                            Received::Pages(dtp_request(&["alpha"], None)),
1115                        ],
1116                    )
1117                },
1118            },
1119            DescribeCase {
1120                name: "a peer without DescribeTopicPartitions is asked with Metadata",
1121                names: vec!["alpha", "beta", ""],
1122                cluster_error: 0,
1123                dtp: false,
1124                answers: Vec::new(),
1125                metadata_topics: vec![metadata_topic("alpha", 0, &[1, 0], -1)],
1126                expected: |own| {
1127                    (
1128                        maplit::btreemap! {
1129                            String::new() => topic_err(
1130                            17,
1131                            Some(
1132                                "The given topic name '' cannot be represented in a \
1133                                 request."
1134                                    .to_owned(),
1135                            ),
1136                        ),
1137                            "alpha".to_owned() => Ok(from_metadata("alpha", &[0, 1], own, false)),
1138                            "beta".to_owned() => topic_err(3, Some("Topic beta not found.".to_owned())),
1139                        },
1140                        vec![Received::Metadata(
1141                            12,
1142                            metadata_by_name_request(
1143                                &maplit::btreeset! {
1144                                    "alpha".to_owned(),
1145                                    "beta".to_owned(),
1146                                },
1147                                DescribeTopicsOptions::builder().build(),
1148                            ),
1149                        )],
1150                    )
1151                },
1152            },
1153            DescribeCase {
1154                name: "a failed DescribeCluster fails every topic",
1155                names: vec!["alpha"],
1156                cluster_error: 31,
1157                dtp: true,
1158                answers: Vec::new(),
1159                metadata_topics: Vec::new(),
1160                expected: |_| {
1161                    (
1162                        maplit::btreemap! {
1163                            "alpha".to_owned() => topic_err(31, None),
1164                        },
1165                        Vec::new(),
1166                    )
1167                },
1168            },
1169        ];
1170        for case in &cases {
1171            let (result, received, own) = describe_case(case).await;
1172            assert2::assert!(
1173                (result, received) == (case.expected)(own),
1174                "case {}",
1175                case.name
1176            );
1177        }
1178    }
1179
1180    /// Kafka's `listTopics` asks `Metadata` for all topics, keeps internal
1181    /// topics only with `listInternal`, and retries a lost connection.
1182    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1183    async fn list_topics_keeps_internal_topics_on_request() {
1184        let listing = |name: &str, is_internal| TopicListing {
1185            name: name.to_owned(),
1186            topic_id: Some(id_of(name)),
1187            is_internal,
1188        };
1189        for (name, list_internal, close_first, expected) in [
1190            (
1191                "without internal topics",
1192                false,
1193                false,
1194                (
1195                    maplit::btreemap! {
1196                        "alpha".to_owned() => listing("alpha", false),
1197                    },
1198                    1,
1199                ),
1200            ),
1201            (
1202                "with internal topics",
1203                true,
1204                false,
1205                (
1206                    maplit::btreemap! {
1207                        "__consumer_offsets".to_owned() => listing("__consumer_offsets", true),
1208                        "alpha".to_owned() => listing("alpha", false),
1209                    },
1210                    1,
1211                ),
1212            ),
1213            (
1214                "a lost connection is retried",
1215                false,
1216                true,
1217                (
1218                    maplit::btreemap! {
1219                        "alpha".to_owned() => listing("alpha", false),
1220                    },
1221                    2,
1222                ),
1223            ),
1224        ] {
1225            let requests = Arc::new(Mutex::new(Vec::new()));
1226            let handler_requests = Arc::clone(&requests);
1227            let broker = scripted_broker(Vec::new(), move |api_key, version, body, own| {
1228                if api_key != metadata_request::API_KEY {
1229                    return MockReply::Silent;
1230                }
1231                let mut requests = handler_requests.lock().expect("requests lock");
1232                let request: MetadataRequest = decode_request(body, version, true);
1233                requests.push(request);
1234                if close_first && requests.len() == 1 {
1235                    return MockReply::Close;
1236                }
1237                MockReply::Respond(encode_response(
1238                    &metadata(
1239                        own,
1240                        vec![
1241                            metadata_topic("alpha", 0, &[0], 1),
1242                            metadata_topic("__consumer_offsets", 0, &[0], 1),
1243                        ],
1244                    ),
1245                    version,
1246                    true,
1247                ))
1248            })
1249            .await;
1250            let admin = fast_admin(broker.addr, krabka_units::secs(5)).await;
1251            let result = admin
1252                .list_topics(
1253                    ListTopicsOptions::builder()
1254                        .list_internal(list_internal)
1255                        .build(),
1256                )
1257                .await
1258                .expect("list topics");
1259            broker.stop();
1260            let requests = requests.lock().expect("requests lock").clone();
1261            assert2::assert!(
1262                (result, requests) == (expected.0, vec![all_topics_request(); expected.1]),
1263                "case {name}"
1264            );
1265        }
1266    }
1267
1268    /// Kafka's `describeTopics` by ID sends `Metadata` v12 or higher with
1269    /// the IDs, gives `UNKNOWN_TOPIC_ID` to an ID it does not find or that
1270    /// has an error, refuses the nil ID, and fails every ID on a peer below
1271    /// v12.
1272    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1273    async fn describe_topics_by_id_uses_metadata_v12() {
1274        let nil = topic_err(
1275            17,
1276            Some(format!(
1277                "The given topic id '{}' cannot be represented in a request.",
1278                Uuid::nil()
1279            )),
1280        );
1281        let unknown_beta = topic_err(100, Some(format!("TopicId {BETA} not found.")));
1282        let unsupported = topic_err(35, None);
1283        let options = DescribeTopicsOptions::builder()
1284            .include_authorized_operations(true)
1285            .build();
1286        let asked = maplit::btreeset! {
1287            ALPHA,
1288            BETA,
1289        };
1290        for (name, max_metadata, topics, expected_requests) in [
1291            (
1292                "found and missing",
1293                12,
1294                vec![metadata_topic("alpha", 0, &[0], 1)],
1295                vec![(12, metadata_by_id_request(&asked, options))],
1296            ),
1297            (
1298                "a topic with an error is unknown",
1299                13,
1300                vec![
1301                    metadata_topic("alpha", 0, &[0], 1),
1302                    metadata_topic("beta", 29, &[], -1),
1303                ],
1304                vec![(13, metadata_by_id_request(&asked, options))],
1305            ),
1306            ("a peer below v12", 11, Vec::new(), Vec::new()),
1307        ] {
1308            let requests = Arc::new(Mutex::new(Vec::new()));
1309            let handler_requests = Arc::clone(&requests);
1310            let broker = scripted_broker(
1311                vec![
1312                    ApiVersionRange::builder()
1313                        .api_key(metadata_request::API_KEY)
1314                        .min_version(1)
1315                        .max_version(max_metadata)
1316                        .build(),
1317                ],
1318                move |api_key, version, body, own| {
1319                    if api_key != metadata_request::API_KEY {
1320                        return MockReply::Silent;
1321                    }
1322                    let request: MetadataRequest = decode_request(body, version, true);
1323                    handler_requests
1324                        .lock()
1325                        .expect("requests lock")
1326                        .push((version, request));
1327                    MockReply::Respond(encode_response(
1328                        &metadata(own, topics.clone()),
1329                        version,
1330                        true,
1331                    ))
1332                },
1333            )
1334            .await;
1335            let admin = fast_admin(broker.addr, krabka_units::secs(5)).await;
1336            let result = admin
1337                .describe_topics_by_id(&[BETA, Uuid::nil(), ALPHA], options)
1338                .await
1339                .into_iter()
1340                .map(|(id, result)| {
1341                    // A version error names the version ranges in its
1342                    // message; the code is what the case checks.
1343                    let result = result.map_err(|error| KafkaError {
1344                        message: error.message.filter(|_| error.code != 35),
1345                        ..error
1346                    });
1347                    (id, result)
1348                })
1349                .collect::<BTreeMap<_, _>>();
1350            let own = broker.addr;
1351            broker.stop();
1352            let expected = if max_metadata < 12 {
1353                maplit::btreemap! {
1354                    Uuid::nil() => nil.clone(),
1355                    ALPHA => unsupported.clone(),
1356                    BETA => unsupported.clone(),
1357                }
1358            } else {
1359                maplit::btreemap! {
1360                    Uuid::nil() => nil.clone(),
1361                    ALPHA => Ok(from_metadata("alpha", &[0], own, true)),
1362                    BETA => unknown_beta.clone(),
1363                }
1364            };
1365            assert2::assert!(
1366                (result, requests.lock().expect("requests lock").clone())
1367                    == (expected, expected_requests),
1368                "case {name}"
1369            );
1370        }
1371    }
1372
1373    /// Kafka's `getTopicDescriptionFromDescribeTopicsResponseTopic` reads
1374    /// the authorized operations only when the caller asked for them.
1375    #[test]
1376    fn describe_page_reads_authorized_operations_on_request() {
1377        for (include, expected) in [
1378            (false, None),
1379            (
1380                true,
1381                Some(maplit::btreeset! {
1382                    AclOperation::Read,
1383                }),
1384            ),
1385        ] {
1386            let description = description_from_describe(
1387                &page_topic("alpha", 0, &[]),
1388                "alpha".to_owned(),
1389                &BTreeMap::new(),
1390                include,
1391            );
1392            assert2::assert!(
1393                description.authorized_operations == expected,
1394                "include {include}"
1395            );
1396        }
1397    }
1398}