Skip to main content

krabka_client_admin/
log_dirs.rs

1//! KIP-113 admin RPCs: `AlterReplicaLogDirs` (`api_key` 34) and
2//! `DescribeLogDirs` (`api_key` 35).
3//!
4//! Both act only on the broker that receives the request. As Kafka's
5//! `KafkaAdminClient.describeLogDirs` and `alterReplicaLogDirs` do, the admin
6//! client finds each broker by id in the metadata and sends one request to
7//! each broker (`ConstantNodeIdProvider`). A broker that fails gives an error
8//! for that broker only.
9
10use std::collections::{BTreeMap, BTreeSet};
11
12use futures_util::{StreamExt as _, stream::FuturesUnordered};
13use krabka_client_core::{Connection, ConnectionOptions};
14use krabka_protocol::{
15    ProtocolRequest,
16    owned::{
17        alter_replica_log_dirs_request::{
18            AlterReplicaLogDir, AlterReplicaLogDirTopic, AlterReplicaLogDirsRequest,
19        },
20        alter_replica_log_dirs_response::AlterReplicaLogDirsResponse,
21        describe_cluster_request::DescribeClusterRequest,
22        describe_log_dirs_request::{DescribableLogDirTopic, DescribeLogDirsRequest},
23        describe_log_dirs_response::DescribeLogDirsResponse,
24        metadata_request::MetadataRequest,
25    },
26};
27use krabka_units::{ByteSize, convert::ByteSizeExt as _};
28
29use crate::{
30    AdminClient, AdminError, KafkaError, RecoveringConnection, format_host_port,
31    groups::list_groups_kafka_error,
32    kafka_error_if, kafka_error_name,
33    retry::{CoordinatorRetry, RetryAction, RetryPolicy, is_connection_failure},
34    send_connection_at_least,
35};
36
37/// `UNKNOWN_SERVER_ERROR`.
38const UNKNOWN_SERVER_ERROR: i16 = -1;
39/// `REQUEST_TIMED_OUT`: Kafka's `TimeoutException`.
40const REQUEST_TIMED_OUT: i16 = 7;
41/// `CLUSTER_AUTHORIZATION_FAILED`.
42const CLUSTER_AUTHORIZATION_FAILED: i16 = 31;
43/// `KAFKA_STORAGE_ERROR`: the log dir is offline, or the broker does not
44/// host the replica.
45const KAFKA_STORAGE_ERROR: i16 = 56;
46/// `DescribeCluster` `EndpointType.CONTROLLER` (KIP-919).
47const ENDPOINT_TYPE_CONTROLLERS: i8 = 2;
48
49/// One replica of a partition on one broker, as Kafka's
50/// `TopicPartitionReplica`.
51#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
52pub struct TopicPartitionReplica {
53    pub topic: String,
54    pub partition: i32,
55    pub broker_id: i32,
56}
57
58/// One log dir from a `DescribeLogDirs` response, as Kafka's
59/// `LogDirDescription`.
60#[derive(Debug, Clone, PartialEq)]
61pub struct LogDirInfo {
62    pub log_dir: String,
63    pub error: Option<KafkaError>,
64    pub topics: Vec<LogDirTopicInfo>,
65    /// The size of the volume. `None` when the broker does not report it
66    /// (`DescribeLogDirs` below v4, or `-1`).
67    pub total: Option<ByteSize>,
68    /// The free space of the volume that the log dir can use. `None` when the
69    /// broker does not report it.
70    pub usable: Option<ByteSize>,
71    /// Whether the log dir takes no new partitions (KIP-1066).
72    pub is_cordoned: bool,
73}
74
75#[derive(Debug, Clone, PartialEq, Eq)]
76pub struct LogDirTopicInfo {
77    pub name: String,
78    pub partitions: Vec<LogDirPartitionInfo>,
79}
80
81#[derive(Debug, Clone, PartialEq, Eq)]
82pub struct LogDirPartitionInfo {
83    pub partition_index: i32,
84    pub partition_size: i64,
85    pub offset_lag: i64,
86    pub is_future_key: bool,
87}
88
89/// The result of one broker: its value, or the Kafka error of the broker.
90pub type BrokerResult<T> = Result<T, KafkaError>;
91
92/// Where one replica of a partition lives on its broker.
93#[derive(Debug, Clone, PartialEq, Eq)]
94pub struct ReplicaLogDir {
95    /// The absolute path of the log dir.
96    pub path: String,
97    /// How far the replica lags behind the log end offset of the partition
98    /// on this broker (for the current replica) or behind the current
99    /// replica (for a future replica).
100    pub offset_lag: i64,
101}
102
103/// The log dirs of one replica, as Kafka's
104/// `DescribeReplicaLogDirsResult.ReplicaLogDirInfo`. A replica that the
105/// broker does not hold, or holds only in an offline log dir, has neither.
106#[derive(Debug, Clone, Default, PartialEq, Eq)]
107pub struct ReplicaLogDirInfo {
108    /// The log dir of the current replica.
109    pub current: Option<ReplicaLogDir>,
110    /// The log dir that an `AlterReplicaLogDirs` move is copying the replica
111    /// to (KIP-113).
112    pub future: Option<ReplicaLogDir>,
113}
114
115impl AdminClient {
116    /// `AlterReplicaLogDirs` (KIP-113): moves replicas between the
117    /// `log.dirs` of their brokers, as Kafka's
118    /// `KafkaAdminClient.alterReplicaLogDirs` does.
119    ///
120    /// `assignments` maps each replica to the absolute path of its target log
121    /// dir. The call groups the replicas by broker id and sends one request to
122    /// each broker, all at the same time. The result has one entry for each
123    /// replica:
124    ///
125    /// - The error code of the broker for that partition, `Ok(())` for 0.
126    /// - `UNKNOWN_SERVER_ERROR` (-1) when the response of the broker has no
127    ///   result for the replica (`completeUnrealizedFutures`).
128    /// - The error of the broker call for every replica of a broker that
129    ///   fails. The call finds each broker by id in fresh metadata for each
130    ///   connection attempt, so a broker that is missing from the metadata or
131    ///   that moved to a new address is found again. A missing broker and a
132    ///   failed or lost connection are tried again with Kafka's backoff until
133    ///   `default.api.timeout.ms` (60 s), and then give `REQUEST_TIMED_OUT`
134    ///   (7). A slow or missing broker does not delay the other brokers.
135    pub async fn alter_replica_log_dirs(
136        &mut self,
137        assignments: &BTreeMap<TopicPartitionReplica, String>,
138    ) -> BTreeMap<TopicPartitionReplica, BrokerResult<()>> {
139        self.alter_replica_log_dirs_with_retry(assignments, self.retry)
140            .await
141    }
142
143    async fn alter_replica_log_dirs_with_retry(
144        &mut self,
145        assignments: &BTreeMap<TopicPartitionReplica, String>,
146        retry: RetryPolicy,
147    ) -> BTreeMap<TopicPartitionReplica, BrokerResult<()>> {
148        let start = retry.start();
149        let mut by_broker = BTreeMap::<i32, BTreeMap<TopicPartitionReplica, String>>::new();
150        for (replica, path) in assignments {
151            by_broker
152                .entry(replica.broker_id)
153                .or_default()
154                .insert(replica.clone(), path.clone());
155        }
156        let (conn, options) = (&self.conn, &self.options);
157        let answers = futures_util::future::join_all(by_broker.iter().map(|(broker_id, moves)| {
158            call_broker(
159                conn,
160                *broker_id,
161                options,
162                alter_request(moves),
163                CoordinatorRetry::from_deadline(start),
164            )
165        }))
166        .await;
167
168        let mut out = BTreeMap::new();
169        for ((broker_id, moves), answer) in by_broker.into_iter().zip(answers) {
170            match answer {
171                Ok(response) => out.extend(alter_results(broker_id, &moves, response)),
172                Err(error) => {
173                    out.extend(
174                        moves
175                            .into_keys()
176                            .map(|replica| (replica, Err(error.clone()))),
177                    );
178                }
179            }
180        }
181        out
182    }
183
184    /// `DescribeLogDirs` (KIP-113): lists every configured `log.dir` of each
185    /// broker in `brokers`, with the partitions each one holds, as Kafka's
186    /// `KafkaAdminClient.describeLogDirs` does.
187    ///
188    /// Pass `None` to fetch all partitions, as Kafka does. Pass `Some` with a
189    /// topic to partitions filter to narrow the result. An empty inner vec
190    /// means all partitions of that topic.
191    ///
192    /// The result has one entry for each broker id, and the brokers are asked
193    /// at the same time. A broker that answers with no log dir gives its
194    /// top-level error code, or `CLUSTER_AUTHORIZATION_FAILED` (31) when it
195    /// has none, as Kafka does. The call finds each broker by id in fresh
196    /// metadata for each connection attempt. A missing broker and a failed or
197    /// lost connection are tried again with Kafka's backoff until
198    /// `default.api.timeout.ms` (60 s), and then give `REQUEST_TIMED_OUT` (7).
199    pub async fn describe_log_dirs(
200        &mut self,
201        brokers: &[i32],
202        filter: Option<&BTreeMap<String, Vec<i32>>>,
203    ) -> BTreeMap<i32, BrokerResult<Vec<LogDirInfo>>> {
204        self.describe_log_dirs_with_retry(brokers, filter, self.retry)
205            .await
206    }
207
208    async fn describe_log_dirs_with_retry(
209        &mut self,
210        brokers: &[i32],
211        filter: Option<&BTreeMap<String, Vec<i32>>>,
212        retry: RetryPolicy,
213    ) -> BTreeMap<i32, BrokerResult<Vec<LogDirInfo>>> {
214        let start = retry.start();
215        let broker_ids = brokers.iter().copied().collect::<BTreeSet<_>>();
216        let request = describe_request(filter);
217        let (conn, options) = (&self.conn, &self.options);
218        let answers = futures_util::future::join_all(broker_ids.iter().map(|broker_id| {
219            let request = request.clone();
220            async move {
221                let response = call_broker(
222                    conn,
223                    *broker_id,
224                    options,
225                    request,
226                    CoordinatorRetry::from_deadline(start),
227                )
228                .await?;
229                log_dir_infos(response)
230            }
231        }))
232        .await;
233        broker_ids.into_iter().zip(answers).collect()
234    }
235
236    /// `DescribeLogDirs` for single replicas: finds the log dir of each
237    /// replica of `replicas`, and of its future replica when one is being
238    /// moved, as Kafka's `KafkaAdminClient.describeReplicaLogDirs` does.
239    ///
240    /// The call groups the replicas by broker id and sends each broker one
241    /// `DescribeLogDirs` request that names only its partitions, all at the
242    /// same time. Each broker is found and retried as in
243    /// [`Self::describe_log_dirs`]. The result has one entry for each
244    /// replica, and the brokers complete their replicas in the order they
245    /// answer, as Kafka completes the replica futures:
246    ///
247    /// - The log dirs that the broker reports for the partition. A log dir
248    ///   with `KAFKA_STORAGE_ERROR` (56) is offline and skipped, as Kafka
249    ///   does, so a replica that lives only there has neither log dir. A
250    ///   replica that the answer does not name has neither log dir.
251    /// - The top-level error code of the answer is not read, as Kafka's
252    ///   `describeReplicaLogDirs` does not read it: a `DescribeLogDirs` v3+
253    ///   answer with `CLUSTER_AUTHORIZATION_FAILED` (31) and no log dir gives
254    ///   each replica of that broker neither log dir.
255    /// - A log dir with any other error fails the call, as Kafka's
256    ///   `handleFailure` with an `IllegalStateException` completes every
257    ///   replica future of the call that is still pending: the replicas of
258    ///   that broker and of every broker that has not answered yet get that
259    ///   error. The replicas of brokers that answered before keep their log
260    ///   dirs.
261    /// - A broker call that fails, with `REQUEST_TIMED_OUT` (7) at
262    ///   `default.api.timeout.ms` or another error, fails every pending
263    ///   replica of the call in the same way.
264    ///
265    /// The call returns once no replica is pending; it does not wait for the
266    /// brokers that have not answered after a failure.
267    pub async fn describe_replica_log_dirs(
268        &mut self,
269        replicas: &[TopicPartitionReplica],
270    ) -> BTreeMap<TopicPartitionReplica, BrokerResult<ReplicaLogDirInfo>> {
271        self.describe_replica_log_dirs_with_retry(replicas, self.retry)
272            .await
273    }
274
275    async fn describe_replica_log_dirs_with_retry(
276        &mut self,
277        replicas: &[TopicPartitionReplica],
278        retry: RetryPolicy,
279    ) -> BTreeMap<TopicPartitionReplica, BrokerResult<ReplicaLogDirInfo>> {
280        let start = retry.start();
281        let mut by_broker = BTreeMap::<i32, BTreeMap<String, BTreeSet<i32>>>::new();
282        for replica in replicas {
283            by_broker
284                .entry(replica.broker_id)
285                .or_default()
286                .entry(replica.topic.clone())
287                .or_default()
288                .insert(replica.partition);
289        }
290        let (conn, options) = (&self.conn, &self.options);
291        let mut results = ReplicaResults::new(&by_broker);
292        let mut answers = by_broker
293            .iter()
294            .map(|(broker_id, topics)| async move {
295                let answer = call_broker(
296                    conn,
297                    *broker_id,
298                    options,
299                    replica_request(topics),
300                    CoordinatorRetry::from_deadline(start),
301                )
302                .await;
303                (
304                    *broker_id,
305                    topics,
306                    answer.and_then(|response| replica_log_dirs(*broker_id, topics, response)),
307                )
308            })
309            .collect::<FuturesUnordered<_>>();
310        while !results.is_complete()
311            && let Some((broker_id, topics, answer)) = answers.next().await
312        {
313            results.complete(broker_id, topics, answer);
314        }
315        results.finish()
316    }
317}
318
319/// The result of each replica of a `describe_replica_log_dirs` call, as
320/// Kafka's replica futures: a broker's answer completes the replicas of that
321/// broker that are still pending, and a failure completes every replica of
322/// the call that is still pending (`completeAllExceptionally`).
323#[derive(Debug)]
324struct ReplicaResults {
325    pending: BTreeSet<TopicPartitionReplica>,
326    done: BTreeMap<TopicPartitionReplica, BrokerResult<ReplicaLogDirInfo>>,
327}
328
329impl ReplicaResults {
330    fn new(by_broker: &BTreeMap<i32, BTreeMap<String, BTreeSet<i32>>>) -> Self {
331        let pending = by_broker
332            .iter()
333            .flat_map(|(broker_id, topics)| {
334                topics.iter().flat_map(move |(topic, partitions)| {
335                    partitions
336                        .iter()
337                        .map(move |partition| TopicPartitionReplica {
338                            topic: topic.clone(),
339                            partition: *partition,
340                            broker_id: *broker_id,
341                        })
342                })
343            })
344            .collect();
345        Self {
346            pending,
347            done: BTreeMap::new(),
348        }
349    }
350
351    /// Complete the replicas with the answer of broker `broker_id`, which the
352    /// request asked about `topics`.
353    fn complete(
354        &mut self,
355        broker_id: i32,
356        topics: &BTreeMap<String, BTreeSet<i32>>,
357        answer: BrokerResult<BTreeMap<(String, i32), ReplicaLogDirInfo>>,
358    ) {
359        match answer {
360            Ok(mut infos) => {
361                for (topic, partitions) in topics {
362                    for partition in partitions {
363                        let replica = TopicPartitionReplica {
364                            topic: topic.clone(),
365                            partition: *partition,
366                            broker_id,
367                        };
368                        if self.pending.remove(&replica) {
369                            let info = infos
370                                .remove(&(topic.clone(), *partition))
371                                .unwrap_or_default();
372                            self.done.insert(replica, Ok(info));
373                        }
374                    }
375                }
376            }
377            Err(error) => {
378                for replica in std::mem::take(&mut self.pending) {
379                    self.done.insert(replica, Err(error.clone()));
380                }
381            }
382        }
383    }
384
385    fn is_complete(&self) -> bool {
386        self.pending.is_empty()
387    }
388
389    fn finish(self) -> BTreeMap<TopicPartitionReplica, BrokerResult<ReplicaLogDirInfo>> {
390        self.done
391    }
392}
393
394/// The `DescribeLogDirs` request of one broker for the partitions of
395/// `topics`, as Kafka's `describeReplicaLogDirs` builds it.
396fn replica_request(topics: &BTreeMap<String, BTreeSet<i32>>) -> DescribeLogDirsRequest {
397    DescribeLogDirsRequest {
398        topics: Some(
399            topics
400                .iter()
401                .map(|(topic, partitions)| DescribableLogDirTopic {
402                    topic: topic.clone(),
403                    partitions: partitions.iter().copied().collect(),
404                    ..Default::default()
405                })
406                .collect(),
407        ),
408        ..Default::default()
409    }
410}
411
412/// The log dirs of each asked-for partition in one broker's answer, as
413/// Kafka's `describeReplicaLogDirs` `handleResponse` reads them. The
414/// top-level error code is not read, as Kafka does not read it. A partition
415/// that the request did not name is skipped with a warning.
416fn replica_log_dirs(
417    broker_id: i32,
418    topics: &BTreeMap<String, BTreeSet<i32>>,
419    response: DescribeLogDirsResponse,
420) -> BrokerResult<BTreeMap<(String, i32), ReplicaLogDirInfo>> {
421    let mut out = BTreeMap::<(String, i32), ReplicaLogDirInfo>::new();
422    for dir in response.results {
423        if dir.error_code == KAFKA_STORAGE_ERROR {
424            continue;
425        }
426        if dir.error_code != 0 {
427            return Err(KafkaError {
428                code: dir.error_code,
429                name: kafka_error_name(dir.error_code),
430                message: Some(format!(
431                    "the error {} for log directory {} in the response from broker {broker_id} \
432                     is illegal",
433                    kafka_error_name(dir.error_code),
434                    dir.log_dir
435                )),
436            });
437        }
438        for topic in dir.topics {
439            for partition in topic.partitions {
440                let asked = topics
441                    .get(&topic.name)
442                    .is_some_and(|partitions| partitions.contains(&partition.partition_index));
443                if !asked {
444                    tracing::warn!(
445                        broker_id,
446                        topic = %topic.name,
447                        partition = partition.partition_index,
448                        "the DescribeLogDirs response names a partition that is not in the request"
449                    );
450                    continue;
451                }
452                let info = out
453                    .entry((topic.name.clone(), partition.partition_index))
454                    .or_default();
455                let location = Some(ReplicaLogDir {
456                    path: dir.log_dir.clone(),
457                    offset_lag: partition.offset_lag,
458                });
459                if partition.is_future_key {
460                    info.future = location;
461                } else {
462                    info.current = location;
463                }
464            }
465        }
466    }
467    Ok(out)
468}
469
470/// Send `request` to broker `broker_id` until it answers, the error is
471/// final, or the call deadline passes, as one Kafka `Call` with a
472/// `ConstantNodeIdProvider` does.
473///
474/// Each connection attempt looks the broker up in fresh metadata. A broker
475/// that the metadata does not name, and a failed or lost connection
476/// (including a TLS or SASL handshake with no verdict), are tried again after
477/// the backoff, as Kafka's `Call.fail` retries a `RetriableException`.
478async fn call_broker<R>(
479    conn: &RecoveringConnection,
480    broker_id: i32,
481    options: &ConnectionOptions,
482    request: R,
483    retry: CoordinatorRetry,
484) -> BrokerResult<R::Response>
485where
486    R: ProtocolRequest + Clone,
487{
488    call_node(conn, NodeTarget::broker(broker_id), options, request, retry).await
489}
490
491/// The node of a [`call_node`] call, as Kafka's `ConstantNodeIdProvider`.
492#[derive(Clone, Copy, Debug, PartialEq, Eq)]
493pub(crate) struct NodeTarget {
494    /// The id of the node.
495    pub(crate) node_id: i32,
496    /// Whether the call finds the node among the controllers when the client
497    /// bootstraps from controllers (KIP-919), as a provider with
498    /// `supportsUseControllers` does. A call without it looks the node up in
499    /// `Metadata`.
500    pub(crate) supports_controllers: bool,
501    /// The lowest version of the request that the call may send.
502    pub(crate) min_version: Option<i16>,
503}
504
505impl NodeTarget {
506    /// A broker, found in `Metadata`, at any version.
507    const fn broker(node_id: i32) -> Self {
508        Self {
509            node_id,
510            supports_controllers: false,
511            min_version: None,
512        }
513    }
514}
515
516/// Send `request` to the node of `target`, as [`call_broker`] does for a
517/// broker.
518pub(crate) async fn call_node<R>(
519    conn: &RecoveringConnection,
520    target: NodeTarget,
521    options: &ConnectionOptions,
522    request: R,
523    mut retry: CoordinatorRetry,
524) -> BrokerResult<R::Response>
525where
526    R: ProtocolRequest + Clone,
527{
528    let mut connection: Option<Connection> = None;
529    loop {
530        let action = retry
531            .run(broker_attempt(
532                &mut connection,
533                conn,
534                target,
535                options,
536                request.clone(),
537            ))
538            .await;
539        if let Some(result) = retry.next(action).await {
540            return result.map_err(|error| broker_error(target.node_id, &error));
541        }
542    }
543}
544
545/// The Kafka error of a failed broker call. A timeout is the error of a call
546/// past its deadline, so it gives `REQUEST_TIMED_OUT` (7), as Kafka's
547/// `Call.handleTimeoutFailure` gives a `TimeoutException`.
548fn broker_error(broker_id: i32, error: &AdminError) -> KafkaError {
549    if is_connection_failure(error) {
550        return KafkaError {
551            code: REQUEST_TIMED_OUT,
552            name: kafka_error_name(REQUEST_TIMED_OUT),
553            message: Some(format!("the call to broker {broker_id} timed out: {error}")),
554        };
555    }
556    list_groups_kafka_error(error)
557}
558
559/// One attempt of [`call_broker`]. Without a connection, the attempt finds
560/// the broker in the metadata and connects to it. A missing broker or a
561/// connection failure empties `connection` and asks for another attempt.
562async fn broker_attempt<R>(
563    connection: &mut Option<Connection>,
564    conn: &RecoveringConnection,
565    target: NodeTarget,
566    options: &ConnectionOptions,
567    request: R,
568) -> RetryAction<R::Response>
569where
570    R: ProtocolRequest,
571{
572    let broker_id = target.node_id;
573    if connection.is_none() {
574        let endpoint = match node_endpoint(conn, target).await {
575            Ok(Some(endpoint)) => endpoint,
576            Ok(None) => {
577                tracing::debug!(broker_id, "the broker is not in the metadata; retrying");
578                return RetryAction::SameCoordinator(Err(AdminError::Broker {
579                    api: "Metadata",
580                    code: REQUEST_TIMED_OUT,
581                    name: kafka_error_name(REQUEST_TIMED_OUT),
582                    message: Some(format!("broker {broker_id} is not in the metadata")),
583                }));
584            }
585            Err(error) if is_connection_failure(&error) => {
586                return RetryAction::SameCoordinator(Err(error));
587            }
588            Err(error) => return RetryAction::Done(Err(error)),
589        };
590        match AdminClient::connect_one(&endpoint, options.clone()).await {
591            Ok(new) => *connection = Some(new),
592            Err(error) if is_connection_failure(&error) => {
593                return RetryAction::SameCoordinator(Err(error));
594            }
595            Err(error) => return RetryAction::Done(Err(error)),
596        }
597    }
598    let Some(current) = connection.as_ref() else {
599        return RetryAction::SameCoordinator(Err(AdminError::Transport(
600            krabka_client_core::ClientError::Disconnected,
601        )));
602    };
603    let sent = match target.min_version {
604        Some(min_version) => send_connection_at_least(current, request, min_version).await,
605        None => current.send(request).await,
606    };
607    match sent {
608        Ok(response) => RetryAction::Done(Ok(response)),
609        Err(error) => {
610            let error = AdminError::from(error);
611            if is_connection_failure(&error) {
612                *connection = None;
613                RetryAction::SameCoordinator(Err(error))
614            } else {
615                RetryAction::Done(Err(error))
616            }
617        }
618    }
619}
620
621/// The `host:port` of the node of `target`, or `None` when no node has its
622/// id. With a controller bootstrap and `supports_controllers`, the node is
623/// one of the controllers that `DescribeCluster` names, as Kafka's admin
624/// metadata holds the controllers then; otherwise a broker of fresh metadata.
625async fn node_endpoint(
626    conn: &RecoveringConnection,
627    target: NodeTarget,
628) -> Result<Option<String>, AdminError> {
629    if !(target.supports_controllers && conn.uses_controller_bootstrap()) {
630        return broker_endpoint(conn, target.node_id).await;
631    }
632    let response = conn
633        .send_at_least(
634            DescribeClusterRequest {
635                endpoint_type: ENDPOINT_TYPE_CONTROLLERS,
636                ..Default::default()
637            },
638            1,
639        )
640        .await?;
641    if response.error_code != 0 {
642        return Err(AdminError::Broker {
643            api: "DescribeCluster",
644            code: response.error_code,
645            name: kafka_error_name(response.error_code),
646            message: response.error_message,
647        });
648    }
649    Ok(response
650        .brokers
651        .into_iter()
652        .find(|controller| controller.broker_id == target.node_id)
653        .map(|controller| format_host_port(&controller.host, controller.port)))
654}
655
656/// The `host:port` of `broker_id` in fresh metadata, or `None` when the
657/// metadata does not name it.
658async fn broker_endpoint(
659    conn: &RecoveringConnection,
660    broker_id: i32,
661) -> Result<Option<String>, AdminError> {
662    let response = conn
663        .send(MetadataRequest {
664            topics: Some(Vec::new()),
665            allow_auto_topic_creation: true,
666            ..Default::default()
667        })
668        .await?;
669    Ok(response
670        .brokers
671        .into_iter()
672        .find(|broker| broker.node_id == broker_id)
673        .map(|broker| format_host_port(&broker.host, broker.port)))
674}
675
676/// The `AlterReplicaLogDirs` request of one broker. It lists each log dir
677/// once, each topic once per log dir, and the partitions in order.
678fn alter_request(moves: &BTreeMap<TopicPartitionReplica, String>) -> AlterReplicaLogDirsRequest {
679    let mut dirs = BTreeMap::<&str, BTreeMap<&str, Vec<i32>>>::new();
680    for (replica, path) in moves {
681        dirs.entry(path.as_str())
682            .or_default()
683            .entry(replica.topic.as_str())
684            .or_default()
685            .push(replica.partition);
686    }
687    AlterReplicaLogDirsRequest {
688        dirs: dirs
689            .into_iter()
690            .map(|(path, topics)| AlterReplicaLogDir {
691                path: path.to_owned(),
692                topics: topics
693                    .into_iter()
694                    .map(|(name, partitions)| AlterReplicaLogDirTopic {
695                        name: name.to_owned(),
696                        partitions,
697                        ..Default::default()
698                    })
699                    .collect(),
700                ..Default::default()
701            })
702            .collect(),
703        ..Default::default()
704    }
705}
706
707/// The result of each replica that `broker_id` moves. A result for a
708/// partition that is not in the request is skipped, and a replica with no
709/// result gets `UNKNOWN_SERVER_ERROR`, as Kafka does.
710fn alter_results(
711    broker_id: i32,
712    moves: &BTreeMap<TopicPartitionReplica, String>,
713    response: AlterReplicaLogDirsResponse,
714) -> BTreeMap<TopicPartitionReplica, BrokerResult<()>> {
715    let mut out = BTreeMap::new();
716    for topic in response.results {
717        for partition in topic.partitions {
718            let replica = TopicPartitionReplica {
719                topic: topic.topic_name.clone(),
720                partition: partition.partition_index,
721                broker_id,
722            };
723            if !moves.contains_key(&replica) {
724                tracing::warn!(
725                    broker_id,
726                    topic = %replica.topic,
727                    partition = replica.partition,
728                    "the AlterReplicaLogDirs response names a partition that is not in the request"
729                );
730                continue;
731            }
732            let result = kafka_error_if(partition.error_code, None).map_or(Ok(()), Err);
733            out.insert(replica, result);
734        }
735    }
736    for replica in moves.keys() {
737        out.entry(replica.clone()).or_insert_with(|| {
738            Err(KafkaError {
739                code: UNKNOWN_SERVER_ERROR,
740                name: kafka_error_name(UNKNOWN_SERVER_ERROR),
741                message: Some(format!(
742                    "the response from broker {broker_id} did not contain a result for replica \
743                     {}-{}",
744                    replica.topic, replica.partition
745                )),
746            })
747        });
748    }
749    out
750}
751
752fn describe_request(filter: Option<&BTreeMap<String, Vec<i32>>>) -> DescribeLogDirsRequest {
753    DescribeLogDirsRequest {
754        topics: filter.map(|filter| {
755            filter
756                .iter()
757                .map(|(name, partitions)| DescribableLogDirTopic {
758                    topic: name.clone(),
759                    partitions: partitions.clone(),
760                    ..Default::default()
761                })
762                .collect()
763        }),
764        ..Default::default()
765    }
766}
767
768/// A volume size of the response, `None` for `-1`.
769fn volume_bytes(bytes: i64) -> Option<ByteSize> {
770    u64::try_from(bytes).ok().map(ByteSize::from_bytes)
771}
772
773/// The log dirs of one response. No log dir is an error of the broker, as
774/// Kafka's `describeLogDirs` `handleResponse` makes it.
775fn log_dir_infos(response: DescribeLogDirsResponse) -> BrokerResult<Vec<LogDirInfo>> {
776    if response.results.is_empty() {
777        let code = if response.error_code == 0 {
778            CLUSTER_AUTHORIZATION_FAILED
779        } else {
780            response.error_code
781        };
782        return Err(KafkaError {
783            code,
784            name: kafka_error_name(code),
785            message: None,
786        });
787    }
788    Ok(response
789        .results
790        .into_iter()
791        .map(|result| LogDirInfo {
792            log_dir: result.log_dir,
793            error: kafka_error_if(result.error_code, None),
794            topics: result
795                .topics
796                .into_iter()
797                .map(|topic| LogDirTopicInfo {
798                    name: topic.name,
799                    partitions: topic
800                        .partitions
801                        .into_iter()
802                        .map(|partition| LogDirPartitionInfo {
803                            partition_index: partition.partition_index,
804                            partition_size: partition.partition_size,
805                            offset_lag: partition.offset_lag,
806                            is_future_key: partition.is_future_key,
807                        })
808                        .collect(),
809                })
810                .collect(),
811            total: volume_bytes(result.total_bytes),
812            usable: volume_bytes(result.usable_bytes),
813            is_cordoned: result.is_cordoned,
814        })
815        .collect())
816}
817
818#[cfg(test)]
819mod tests {
820    use std::{
821        net::SocketAddr,
822        sync::{Arc, Mutex},
823        time::Duration,
824    };
825
826    use assert2::assert;
827    use bytes::{Buf, BytesMut};
828    use krabka_client_core::MockBroker;
829    use krabka_protocol::{
830        Decode, Encode,
831        owned::{
832            alter_replica_log_dirs_request,
833            alter_replica_log_dirs_response::{
834                AlterReplicaLogDirPartitionResult, AlterReplicaLogDirTopicResult,
835            },
836            api_versions_request,
837            api_versions_response::{ApiVersion, ApiVersionsResponse},
838            describe_log_dirs_request,
839            describe_log_dirs_response::{
840                DescribeLogDirsPartition, DescribeLogDirsResult, DescribeLogDirsTopic,
841            },
842            metadata_request,
843            metadata_response::{MetadataResponse, MetadataResponseBroker},
844        },
845    };
846
847    use super::*;
848
849    const LONG: Duration = Duration::from_secs(5);
850    const SHORT: Duration = Duration::from_millis(300);
851
852    /// What each mock broker received: the broker id, and the request.
853    #[derive(Clone, Debug, PartialEq, Eq)]
854    enum Received {
855        Describe(i32, DescribeLogDirsRequest),
856        Alter(i32, AlterReplicaLogDirsRequest),
857    }
858
859    /// How one mock broker of the cluster behaves.
860    #[derive(Clone, Copy, Debug, PartialEq, Eq)]
861    enum Behavior {
862        /// It answers for its own log dir and replicas.
863        Normal,
864        /// It answers `DescribeLogDirs` with no log dir and error code 0.
865        NoLogDirs,
866        /// The metadata names it at an address that refuses connections.
867        Down,
868        /// The first metadata response names it at an address that refuses
869        /// connections, and later responses name its real address, as after
870        /// a restart with a new advertised address.
871        Moved,
872    }
873
874    fn encode(response: &impl Encode, version: i16, flexible: bool) -> Vec<u8> {
875        let mut buf = BytesMut::new();
876        if flexible {
877            buf.extend_from_slice(&[0]);
878        }
879        response.encode(&mut buf, version).unwrap();
880        buf.to_vec()
881    }
882
883    fn decode<R: for<'de> Decode<'de>>(mut body: &[u8], version: i16, flexible: bool) -> R {
884        let client_id_len = body.get_i16();
885        body.advance(usize::try_from(client_id_len).expect("client id length"));
886        if flexible {
887            body.advance(1);
888        }
889        R::decode(&mut body, version).expect("request decodes")
890    }
891
892    fn api_versions() -> Vec<u8> {
893        let api = |api_key, max_version| ApiVersion {
894            api_key,
895            min_version: 0,
896            max_version,
897            ..Default::default()
898        };
899        encode(
900            &ApiVersionsResponse {
901                api_keys: vec![
902                    api(api_versions_request::API_KEY, 0),
903                    api(metadata_request::API_KEY, 12),
904                    api(describe_log_dirs_request::API_KEY, 4),
905                    api(alter_replica_log_dirs_request::API_KEY, 2),
906                ],
907                ..Default::default()
908            },
909            0,
910            false,
911        )
912    }
913
914    fn log_dir(broker_id: i32) -> String {
915        format!("/data/broker-{broker_id}")
916    }
917
918    /// The log dir that broker `broker_id` describes: partition
919    /// `broker_id - 1` of `orders`, and a 1 GiB volume.
920    fn described(broker_id: i32) -> Vec<LogDirInfo> {
921        vec![LogDirInfo {
922            log_dir: log_dir(broker_id),
923            error: None,
924            topics: vec![LogDirTopicInfo {
925                name: "orders".into(),
926                partitions: vec![LogDirPartitionInfo {
927                    partition_index: broker_id - 1,
928                    partition_size: 100,
929                    offset_lag: 0,
930                    is_future_key: false,
931                }],
932            }],
933            total: Some(krabka_units::gibibytes(1)),
934            usable: None,
935            is_cordoned: false,
936        }]
937    }
938
939    /// Start one broker of a two-broker cluster. Broker `broker_id` hosts
940    /// partition `broker_id - 1` of `orders`, and `addresses` holds the
941    /// address of each broker id that the metadata names.
942    async fn cluster_broker(
943        broker_id: i32,
944        behavior: Behavior,
945        addresses: Arc<Mutex<Addresses>>,
946        received: Arc<Mutex<Vec<Received>>>,
947    ) -> MockBroker {
948        MockBroker::start(move |api_key, version, _, body| match api_key {
949            api_versions_request::API_KEY => Some(api_versions()),
950            metadata_request::API_KEY => Some(encode(
951                &MetadataResponse {
952                    brokers: addresses
953                        .lock()
954                        .expect("addresses lock")
955                        .next()
956                        .iter()
957                        .map(|(node_id, addr)| MetadataResponseBroker {
958                            node_id: *node_id,
959                            host: addr.ip().to_string(),
960                            port: i32::from(addr.port()),
961                            ..Default::default()
962                        })
963                        .collect(),
964                    ..Default::default()
965                },
966                version,
967                version >= metadata_request::FLEXIBLE_MIN,
968            )),
969            describe_log_dirs_request::API_KEY => {
970                let flexible = version >= describe_log_dirs_request::FLEXIBLE_MIN;
971                let request: DescribeLogDirsRequest = decode(body, version, flexible);
972                received
973                    .lock()
974                    .expect("received lock")
975                    .push(Received::Describe(broker_id, request));
976                let results = if behavior == Behavior::NoLogDirs {
977                    Vec::new()
978                } else {
979                    vec![DescribeLogDirsResult {
980                        log_dir: log_dir(broker_id),
981                        topics: vec![DescribeLogDirsTopic {
982                            name: "orders".into(),
983                            partitions: vec![DescribeLogDirsPartition {
984                                partition_index: broker_id - 1,
985                                partition_size: 100,
986                                ..Default::default()
987                            }],
988                            ..Default::default()
989                        }],
990                        total_bytes: 1 << 30,
991                        usable_bytes: -1,
992                        ..Default::default()
993                    }]
994                };
995                Some(encode(
996                    &DescribeLogDirsResponse {
997                        results,
998                        ..Default::default()
999                    },
1000                    version,
1001                    flexible,
1002                ))
1003            }
1004            alter_replica_log_dirs_request::API_KEY => {
1005                let flexible = version >= alter_replica_log_dirs_request::FLEXIBLE_MIN;
1006                let request: AlterReplicaLogDirsRequest = decode(body, version, flexible);
1007                let results = request
1008                    .dirs
1009                    .iter()
1010                    .flat_map(|dir| dir.topics.iter())
1011                    .map(|topic| AlterReplicaLogDirTopicResult {
1012                        topic_name: topic.name.clone(),
1013                        partitions: topic
1014                            .partitions
1015                            .iter()
1016                            .map(|partition| AlterReplicaLogDirPartitionResult {
1017                                partition_index: *partition,
1018                                error_code: if *partition == broker_id - 1 {
1019                                    0
1020                                } else {
1021                                    KAFKA_STORAGE_ERROR
1022                                },
1023                                ..Default::default()
1024                            })
1025                            .collect(),
1026                        ..Default::default()
1027                    })
1028                    .collect();
1029                received
1030                    .lock()
1031                    .expect("received lock")
1032                    .push(Received::Alter(broker_id, request));
1033                Some(encode(
1034                    &AlterReplicaLogDirsResponse {
1035                        results,
1036                        ..Default::default()
1037                    },
1038                    version,
1039                    flexible,
1040                ))
1041            }
1042            _ => None,
1043        })
1044        .await
1045    }
1046
1047    /// The broker addresses that the metadata names. `first` is used for the
1048    /// first metadata response only.
1049    #[derive(Default)]
1050    struct Addresses {
1051        first: Option<Vec<(i32, SocketAddr)>>,
1052        later: Vec<(i32, SocketAddr)>,
1053    }
1054
1055    impl Addresses {
1056        fn next(&mut self) -> Vec<(i32, SocketAddr)> {
1057            self.first.take().unwrap_or_else(|| self.later.clone())
1058        }
1059    }
1060
1061    /// A running two-broker cluster.
1062    struct Cluster {
1063        brokers: Vec<MockBroker>,
1064        received: Arc<Mutex<Vec<Received>>>,
1065        admin: AdminClient,
1066    }
1067
1068    impl Cluster {
1069        /// Start brokers 1 and 2. Broker 1 is the bootstrap broker.
1070        async fn start(broker_2: Behavior) -> Self {
1071            let addresses = Arc::new(Mutex::new(Addresses::default()));
1072            let received = Arc::new(Mutex::new(Vec::new()));
1073            let one = cluster_broker(
1074                1,
1075                Behavior::Normal,
1076                Arc::clone(&addresses),
1077                Arc::clone(&received),
1078            )
1079            .await;
1080            let two =
1081                cluster_broker(2, broker_2, Arc::clone(&addresses), Arc::clone(&received)).await;
1082            let refused = {
1083                let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
1084                    .await
1085                    .expect("bind a port");
1086                listener.local_addr().expect("local address")
1087            };
1088            *addresses.lock().expect("addresses lock") = match broker_2 {
1089                Behavior::Down => Addresses {
1090                    first: None,
1091                    later: vec![(1, one.addr), (2, refused)],
1092                },
1093                Behavior::Moved => Addresses {
1094                    first: Some(vec![(1, one.addr), (2, refused)]),
1095                    later: vec![(1, one.addr), (2, two.addr)],
1096                },
1097                Behavior::Normal | Behavior::NoLogDirs => Addresses {
1098                    first: None,
1099                    later: vec![(1, one.addr), (2, two.addr)],
1100                },
1101            };
1102            let admin = AdminClient::connect(&[one.addr.to_string()])
1103                .await
1104                .expect("admin connects");
1105            Self {
1106                brokers: vec![one, two],
1107                received,
1108                admin,
1109            }
1110        }
1111
1112        fn stop(self) -> Vec<Received> {
1113            for broker in self.brokers {
1114                broker.stop();
1115            }
1116            let mut received = self.received.lock().expect("received lock").clone();
1117            received.sort_by_key(|entry| match entry {
1118                Received::Describe(id, _) | Received::Alter(id, _) => *id,
1119            });
1120            received
1121        }
1122    }
1123
1124    fn policy(timeout: Duration) -> RetryPolicy {
1125        let backoff = if timeout == LONG {
1126            Duration::from_millis(1)
1127        } else {
1128            timeout
1129        };
1130        RetryPolicy {
1131            timeout,
1132            initial_backoff: backoff,
1133            max_backoff: backoff,
1134            jitter: 0.0,
1135            max_retries: u32::MAX,
1136        }
1137    }
1138
1139    /// The Kafka error code of each broker result.
1140    fn codes<T>(results: BTreeMap<i32, BrokerResult<T>>) -> BTreeMap<i32, Result<T, i16>> {
1141        results
1142            .into_iter()
1143            .map(|(broker_id, result)| (broker_id, result.map_err(|error| error.code)))
1144            .collect()
1145    }
1146
1147    /// Kafka's `describeLogDirs` sends one `DescribeLogDirs` to each named
1148    /// broker (`ConstantNodeIdProvider`), waits for an unknown broker until
1149    /// the deadline, fails a broker that answers with no log dir with its
1150    /// error code or `CLUSTER_AUTHORIZATION_FAILED`, and times out a broker
1151    /// that it cannot reach.
1152    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1153    async fn describe_log_dirs_asks_each_broker_by_id() {
1154        let all = describe_request(None);
1155        for (name, broker_2, brokers, timeout, expected_requests, expected) in [
1156            (
1157                "both brokers",
1158                Behavior::Normal,
1159                vec![1, 2],
1160                LONG,
1161                vec![
1162                    Received::Describe(1, all.clone()),
1163                    Received::Describe(2, all.clone()),
1164                ],
1165                maplit::btreemap! {
1166                    1 => Ok(described(1)),
1167                    2 => Ok(described(2)),
1168                },
1169            ),
1170            (
1171                "one broker",
1172                Behavior::Normal,
1173                vec![2],
1174                LONG,
1175                vec![Received::Describe(2, all.clone())],
1176                maplit::btreemap! {
1177                    2 => Ok(described(2)),
1178                },
1179            ),
1180            (
1181                "an unknown broker times out",
1182                Behavior::Normal,
1183                vec![3],
1184                SHORT,
1185                vec![],
1186                maplit::btreemap! {
1187                    3 => Err(REQUEST_TIMED_OUT),
1188                },
1189            ),
1190            (
1191                "a known broker is asked while an unknown broker waits",
1192                Behavior::Normal,
1193                vec![1, 3],
1194                SHORT,
1195                vec![Received::Describe(1, all.clone())],
1196                maplit::btreemap! {
1197                    1 => Ok(described(1)),
1198                    3 => Err(REQUEST_TIMED_OUT),
1199                },
1200            ),
1201            (
1202                "a broker at a new address is found again",
1203                Behavior::Moved,
1204                vec![2],
1205                LONG,
1206                vec![Received::Describe(2, all.clone())],
1207                maplit::btreemap! {
1208                    2 => Ok(described(2)),
1209                },
1210            ),
1211            (
1212                "a broker with no log dir",
1213                Behavior::NoLogDirs,
1214                vec![1, 2],
1215                LONG,
1216                vec![
1217                    Received::Describe(1, all.clone()),
1218                    Received::Describe(2, all.clone()),
1219                ],
1220                maplit::btreemap! {
1221                    1 => Ok(described(1)),
1222                    2 => Err(CLUSTER_AUTHORIZATION_FAILED),
1223                },
1224            ),
1225            (
1226                "an unreachable broker times out",
1227                Behavior::Down,
1228                vec![1, 2],
1229                SHORT,
1230                vec![Received::Describe(1, all.clone())],
1231                maplit::btreemap! {
1232                    1 => Ok(described(1)),
1233                    2 => Err(REQUEST_TIMED_OUT),
1234                },
1235            ),
1236        ] {
1237            let mut cluster = Cluster::start(broker_2).await;
1238            let result = cluster
1239                .admin
1240                .describe_log_dirs_with_retry(&brokers, None, policy(timeout))
1241                .await;
1242            let received = cluster.stop();
1243            assert!(
1244                (codes(result), received) == (expected, expected_requests),
1245                "case {name}"
1246            );
1247        }
1248    }
1249
1250    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1251    async fn describe_log_dirs_sends_the_topic_filter() {
1252        let mut cluster = Cluster::start(Behavior::Normal).await;
1253        let filter = maplit::btreemap! {
1254            "orders".to_string() => vec![0],
1255        };
1256        cluster.admin.describe_log_dirs(&[1], Some(&filter)).await;
1257        let received = cluster.stop();
1258        assert!(
1259            received
1260                == vec![Received::Describe(
1261                    1,
1262                    DescribeLogDirsRequest {
1263                        topics: Some(vec![DescribableLogDirTopic {
1264                            topic: "orders".into(),
1265                            partitions: vec![0],
1266                            ..Default::default()
1267                        }]),
1268                        ..Default::default()
1269                    }
1270                )]
1271        );
1272    }
1273
1274    fn replica(partition: i32, broker_id: i32) -> TopicPartitionReplica {
1275        TopicPartitionReplica {
1276            topic: "orders".into(),
1277            partition,
1278            broker_id,
1279        }
1280    }
1281
1282    fn alter(dirs: &[(&str, &[i32])]) -> AlterReplicaLogDirsRequest {
1283        AlterReplicaLogDirsRequest {
1284            dirs: dirs
1285                .iter()
1286                .map(|(path, partitions)| AlterReplicaLogDir {
1287                    path: (*path).to_owned(),
1288                    topics: vec![AlterReplicaLogDirTopic {
1289                        name: "orders".into(),
1290                        partitions: partitions.to_vec(),
1291                        ..Default::default()
1292                    }],
1293                    ..Default::default()
1294                })
1295                .collect(),
1296            ..Default::default()
1297        }
1298    }
1299
1300    /// Kafka's `alterReplicaLogDirs` groups the moves by
1301    /// `TopicPartitionReplica.brokerId` and sends each group to its broker.
1302    /// A replica with no result in the response fails with
1303    /// `UNKNOWN_SERVER_ERROR`, and a broker that fails fails only its own
1304    /// replicas.
1305    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1306    async fn alter_replica_log_dirs_sends_each_move_to_its_broker() {
1307        for (name, broker_2, moves, timeout, expected_requests, expected) in [
1308            (
1309                "each move goes to the broker of its replica",
1310                Behavior::Normal,
1311                vec![(replica(0, 1), "/a"), (replica(1, 2), "/b")],
1312                LONG,
1313                vec![
1314                    Received::Alter(1, alter(&[("/a", &[0])])),
1315                    Received::Alter(2, alter(&[("/b", &[1])])),
1316                ],
1317                maplit::btreemap! {
1318                    replica(0, 1) => Ok(()),
1319                    replica(1, 2) => Ok(()),
1320                },
1321            ),
1322            (
1323                "a broker groups its moves by log dir",
1324                Behavior::Normal,
1325                vec![
1326                    (replica(0, 1), "/a"),
1327                    (replica(5, 1), "/a"),
1328                    (replica(6, 1), "/b"),
1329                ],
1330                LONG,
1331                vec![Received::Alter(1, alter(&[("/a", &[0, 5]), ("/b", &[6])]))],
1332                maplit::btreemap! {
1333                    replica(0, 1) => Ok(()),
1334                    replica(5, 1) => Err(KAFKA_STORAGE_ERROR),
1335                    replica(6, 1) => Err(KAFKA_STORAGE_ERROR),
1336                },
1337            ),
1338            (
1339                "an unreachable broker fails only its replicas",
1340                Behavior::Down,
1341                vec![(replica(0, 1), "/a"), (replica(1, 2), "/b")],
1342                SHORT,
1343                vec![Received::Alter(1, alter(&[("/a", &[0])]))],
1344                maplit::btreemap! {
1345                    replica(0, 1) => Ok(()),
1346                    replica(1, 2) => Err(REQUEST_TIMED_OUT),
1347                },
1348            ),
1349            (
1350                "an unknown broker times out",
1351                Behavior::Normal,
1352                vec![(replica(0, 9), "/a")],
1353                SHORT,
1354                vec![],
1355                maplit::btreemap! {
1356                    replica(0, 9) => Err(REQUEST_TIMED_OUT),
1357                },
1358            ),
1359        ] {
1360            let mut cluster = Cluster::start(broker_2).await;
1361            let assignments = moves
1362                .into_iter()
1363                .map(|(replica, path)| (replica, path.to_owned()))
1364                .collect();
1365            let result = cluster
1366                .admin
1367                .alter_replica_log_dirs_with_retry(&assignments, policy(timeout))
1368                .await
1369                .into_iter()
1370                .map(|(replica, result)| (replica, result.map_err(|error| error.code)))
1371                .collect::<BTreeMap<_, _>>();
1372            let received = cluster.stop();
1373            assert!(
1374                (result, received) == (expected, expected_requests),
1375                "case {name}"
1376            );
1377        }
1378    }
1379
1380    fn replica_request_of(broker: &[(&str, &[i32])]) -> DescribeLogDirsRequest {
1381        DescribeLogDirsRequest {
1382            topics: Some(
1383                broker
1384                    .iter()
1385                    .map(|(topic, partitions)| DescribableLogDirTopic {
1386                        topic: (*topic).to_owned(),
1387                        partitions: partitions.to_vec(),
1388                        ..Default::default()
1389                    })
1390                    .collect(),
1391            ),
1392            ..Default::default()
1393        }
1394    }
1395
1396    /// The current log dir of the replica that broker `broker_id` hosts in
1397    /// the mock cluster.
1398    fn hosted(broker_id: i32) -> ReplicaLogDirInfo {
1399        ReplicaLogDirInfo {
1400            current: Some(ReplicaLogDir {
1401                path: log_dir(broker_id),
1402                offset_lag: 0,
1403            }),
1404            future: None,
1405        }
1406    }
1407
1408    /// Kafka's `describeReplicaLogDirs` sends each broker one
1409    /// `DescribeLogDirs` that names only its replicas, finds a moved broker
1410    /// again, gives an empty `ReplicaLogDirInfo` to a replica the broker does
1411    /// not report, and times out a broker it cannot reach.
1412    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1413    async fn describe_replica_log_dirs_asks_the_broker_of_each_replica() {
1414        for (name, broker_2, replicas, timeout, expected_requests, expected) in [
1415            (
1416                "each replica is asked of its broker",
1417                Behavior::Normal,
1418                vec![replica(0, 1), replica(5, 1), replica(1, 2)],
1419                LONG,
1420                vec![
1421                    Received::Describe(1, replica_request_of(&[("orders", &[0, 5])])),
1422                    Received::Describe(2, replica_request_of(&[("orders", &[1])])),
1423                ],
1424                maplit::btreemap! {
1425                    replica(0, 1) => Ok(hosted(1)),
1426                    replica(5, 1) => Ok(ReplicaLogDirInfo::default()),
1427                    replica(1, 2) => Ok(hosted(2)),
1428                },
1429            ),
1430            (
1431                "a broker at a new address is found again",
1432                Behavior::Moved,
1433                vec![replica(1, 2)],
1434                LONG,
1435                vec![Received::Describe(
1436                    2,
1437                    replica_request_of(&[("orders", &[1])]),
1438                )],
1439                maplit::btreemap! {
1440                    replica(1, 2) => Ok(hosted(2)),
1441                },
1442            ),
1443            (
1444                "a broker with no log dir reports no replica",
1445                Behavior::NoLogDirs,
1446                vec![replica(1, 2)],
1447                LONG,
1448                vec![Received::Describe(
1449                    2,
1450                    replica_request_of(&[("orders", &[1])]),
1451                )],
1452                maplit::btreemap! {
1453                    replica(1, 2) => Ok(ReplicaLogDirInfo::default()),
1454                },
1455            ),
1456            (
1457                "an unreachable broker fails the replicas still pending",
1458                Behavior::Down,
1459                vec![replica(0, 1), replica(1, 2)],
1460                SHORT,
1461                vec![Received::Describe(
1462                    1,
1463                    replica_request_of(&[("orders", &[0])]),
1464                )],
1465                maplit::btreemap! {
1466                    replica(0, 1) => Ok(hosted(1)),
1467                    replica(1, 2) => Err(REQUEST_TIMED_OUT),
1468                },
1469            ),
1470        ] {
1471            let mut cluster = Cluster::start(broker_2).await;
1472            let result = cluster
1473                .admin
1474                .describe_replica_log_dirs_with_retry(&replicas, policy(timeout))
1475                .await
1476                .into_iter()
1477                .map(|(replica, result)| (replica, result.map_err(|error| error.code)))
1478                .collect::<BTreeMap<_, _>>();
1479            let received = cluster.stop();
1480            assert!(
1481                (result, received) == (expected, expected_requests),
1482                "case {name}"
1483            );
1484        }
1485    }
1486
1487    /// How Kafka's `describeReplicaLogDirs` `handleResponse` reads one
1488    /// answer: a future replica fills the future slot, an offline log dir is
1489    /// skipped, any other log-dir error fails the answer, and the top-level
1490    /// error code is not read.
1491    #[test]
1492    fn replica_log_dirs_reads_current_and_future_replicas() {
1493        let dir =
1494            |path: &str, error_code, partitions: Vec<(i32, i64, bool)>| DescribeLogDirsResult {
1495                error_code,
1496                log_dir: path.to_owned(),
1497                topics: vec![DescribeLogDirsTopic {
1498                    name: "orders".into(),
1499                    partitions: partitions
1500                        .into_iter()
1501                        .map(|(partition_index, offset_lag, is_future_key)| {
1502                            DescribeLogDirsPartition {
1503                                partition_index,
1504                                offset_lag,
1505                                is_future_key,
1506                                ..Default::default()
1507                            }
1508                        })
1509                        .collect(),
1510                    ..Default::default()
1511                }],
1512                ..Default::default()
1513            };
1514        let at = |path: &str, offset_lag| {
1515            Some(ReplicaLogDir {
1516                path: path.to_owned(),
1517                offset_lag,
1518            })
1519        };
1520        let asked = maplit::btreemap! {
1521            "orders".to_owned() => maplit::btreeset! {
1522                0,
1523                1,
1524            },
1525        };
1526        for (name, error_code, results, expected) in [
1527            (
1528                "a replica being moved has a current and a future log dir",
1529                0,
1530                vec![
1531                    dir("/a", 0, vec![(0, 0, false), (1, 3, false)]),
1532                    dir("/b", 0, vec![(0, 42, true), (9, 0, false)]),
1533                ],
1534                Ok(maplit::btreemap! {
1535                    ("orders".to_owned(), 0) => ReplicaLogDirInfo {
1536                    current: at("/a", 0),
1537                    future: at("/b", 42),
1538                },
1539                    ("orders".to_owned(), 1) => ReplicaLogDirInfo {
1540                    current: at("/a", 3),
1541                    future: None,
1542                },
1543                }),
1544            ),
1545            (
1546                "an offline log dir is skipped",
1547                0,
1548                vec![
1549                    dir("/offline", KAFKA_STORAGE_ERROR, vec![(0, 0, false)]),
1550                    dir("/a", 0, vec![(1, 0, false)]),
1551                ],
1552                Ok(maplit::btreemap! {
1553                    ("orders".to_owned(), 1) => ReplicaLogDirInfo {
1554                    current: at("/a", 0),
1555                    future: None,
1556                },
1557                }),
1558            ),
1559            (
1560                "another log dir error fails the broker",
1561                0,
1562                vec![dir("/a", 57, vec![])],
1563                Err(KafkaError {
1564                    code: 57,
1565                    name: "LOG_DIR_NOT_FOUND",
1566                    message: Some(
1567                        "the error LOG_DIR_NOT_FOUND for log directory /a in the response from \
1568                         broker 1 is illegal"
1569                            .to_owned(),
1570                    ),
1571                }),
1572            ),
1573            (
1574                "the top-level error is not read",
1575                CLUSTER_AUTHORIZATION_FAILED,
1576                vec![dir("/a", 0, vec![(1, 0, false)])],
1577                Ok(maplit::btreemap! {
1578                    ("orders".to_owned(), 1) => ReplicaLogDirInfo {
1579                    current: at("/a", 0),
1580                    future: None,
1581                },
1582                }),
1583            ),
1584        ] {
1585            let response = DescribeLogDirsResponse {
1586                error_code,
1587                results,
1588                ..Default::default()
1589            };
1590            assert!(
1591                replica_log_dirs(1, &asked, response) == expected,
1592                "case {name}"
1593            );
1594        }
1595    }
1596
1597    /// Kafka's replica futures: an answer completes the pending replicas of
1598    /// its broker, and a failure (an illegal log-dir error or a failed call)
1599    /// completes every replica of the call that is still pending with that
1600    /// error. Replicas that are complete keep their result.
1601    #[test]
1602    fn a_failure_fails_every_replica_still_pending() {
1603        let asked = |partitions: &[i32]| {
1604            maplit::btreemap! {
1605                "orders".to_owned() => partitions.iter().copied().collect::<BTreeSet<_>>(),
1606            }
1607        };
1608        let by_broker = maplit::btreemap! {
1609            1 => asked(&[0, 5]),
1610            2 => asked(&[1]),
1611        };
1612        let found = |partition: i32, broker_id: i32| {
1613            (
1614                ("orders".to_owned(), partition),
1615                ReplicaLogDirInfo {
1616                    current: Some(ReplicaLogDir {
1617                        path: log_dir(broker_id),
1618                        offset_lag: 0,
1619                    }),
1620                    future: None,
1621                },
1622            )
1623        };
1624        let answer = |broker_id: i32, partition: i32| {
1625            let (key, value) = found(partition, broker_id);
1626            (
1627                broker_id,
1628                Ok(maplit::btreemap! {
1629                    key => value,
1630                }),
1631            )
1632        };
1633        let illegal = KafkaError {
1634            code: 57,
1635            name: "LOG_DIR_NOT_FOUND",
1636            message: Some("illegal".to_owned()),
1637        };
1638        let timed_out = KafkaError {
1639            code: REQUEST_TIMED_OUT,
1640            name: "REQUEST_TIMED_OUT",
1641            message: None,
1642        };
1643        let hosted = |partition, broker_id| Ok(found(partition, broker_id).1);
1644        for (name, answers, expected) in [
1645            (
1646                "every broker answers",
1647                vec![answer(2, 1), answer(1, 0)],
1648                maplit::btreemap! {
1649                    replica(0, 1) => hosted(0, 1),
1650                    replica(5, 1) => Ok(ReplicaLogDirInfo::default()),
1651                    replica(1, 2) => hosted(1, 2),
1652                },
1653            ),
1654            (
1655                "an illegal log dir after an answer",
1656                vec![answer(1, 0), (2, Err(illegal.clone()))],
1657                maplit::btreemap! {
1658                    replica(0, 1) => hosted(0, 1),
1659                    replica(5, 1) => Ok(ReplicaLogDirInfo::default()),
1660                    replica(1, 2) => Err(illegal.clone()),
1661                },
1662            ),
1663            (
1664                "an illegal log dir before an answer",
1665                vec![(2, Err(illegal.clone())), answer(1, 0)],
1666                maplit::btreemap! {
1667                    replica(0, 1) => Err(illegal.clone()),
1668                    replica(5, 1) => Err(illegal.clone()),
1669                    replica(1, 2) => Err(illegal.clone()),
1670                },
1671            ),
1672            (
1673                "a failed call before an answer",
1674                vec![(1, Err(timed_out.clone())), answer(2, 1)],
1675                maplit::btreemap! {
1676                    replica(0, 1) => Err(timed_out.clone()),
1677                    replica(5, 1) => Err(timed_out.clone()),
1678                    replica(1, 2) => Err(timed_out.clone()),
1679                },
1680            ),
1681        ] {
1682            let mut results = ReplicaResults::new(&by_broker);
1683            let mut completed_after = Vec::new();
1684            for (broker_id, answer) in answers {
1685                results.complete(broker_id, &by_broker[&broker_id], answer);
1686                completed_after.push(results.is_complete());
1687            }
1688            assert!(
1689                (completed_after.last().copied(), results.finish()) == (Some(true), expected),
1690                "case {name}"
1691            );
1692        }
1693    }
1694
1695    #[test]
1696    fn a_replica_without_a_result_fails_with_unknown_server_error() {
1697        let moves = maplit::btreemap! {
1698            replica(0, 1) => "/a".to_owned(),
1699            replica(1, 1) => "/a".to_owned(),
1700        };
1701        let response = AlterReplicaLogDirsResponse {
1702            results: vec![AlterReplicaLogDirTopicResult {
1703                topic_name: "orders".into(),
1704                partitions: vec![
1705                    AlterReplicaLogDirPartitionResult {
1706                        partition_index: 0,
1707                        ..Default::default()
1708                    },
1709                    AlterReplicaLogDirPartitionResult {
1710                        partition_index: 7,
1711                        ..Default::default()
1712                    },
1713                ],
1714                ..Default::default()
1715            }],
1716            ..Default::default()
1717        };
1718        assert!(
1719            alter_results(1, &moves, response)
1720                == maplit::btreemap! {
1721                    replica(0, 1) => Ok(()),
1722                    replica(1, 1) => Err(KafkaError {
1723                    code: UNKNOWN_SERVER_ERROR,
1724                    name: "UNKNOWN_SERVER_ERROR",
1725                    message: Some(
1726                        "the response from broker 1 did not contain a result for replica \
1727                         orders-1"
1728                            .into()
1729                    ),
1730                }),
1731                }
1732        );
1733    }
1734
1735    #[test]
1736    fn a_log_dir_reports_its_volume_sizes() {
1737        let response = DescribeLogDirsResponse {
1738            results: vec![DescribeLogDirsResult {
1739                log_dir: "/d".into(),
1740                error_code: 57,
1741                total_bytes: 4096,
1742                usable_bytes: 1024,
1743                is_cordoned: true,
1744                ..Default::default()
1745            }],
1746            ..Default::default()
1747        };
1748        assert!(
1749            log_dir_infos(response)
1750                == Ok(vec![LogDirInfo {
1751                    log_dir: "/d".into(),
1752                    error: kafka_error_if(57, None),
1753                    topics: vec![],
1754                    total: Some(krabka_units::kibibytes(4)),
1755                    usable: Some(krabka_units::kibibytes(1)),
1756                    is_cordoned: true,
1757                }])
1758        );
1759    }
1760}