Skip to main content

krabka_client_admin/
quorum.rs

1//! KIP-853 metadata-quorum administration: `DescribeQuorum`, `AddRaftVoter`
2//! and `RemoveRaftVoter`, as Kafka's `Admin.describeMetadataQuorum`,
3//! `addRaftVoter` and `removeRaftVoter`.
4
5use std::collections::BTreeSet;
6
7use krabka_protocol::{
8    owned::{
9        add_raft_voter_request::{AddRaftVoterRequest, Listener},
10        describe_quorum_request::{
11            DescribeQuorumRequest, PartitionData as RequestPartition, TopicData as RequestTopic,
12        },
13        remove_raft_voter_request::RemoveRaftVoterRequest,
14    },
15    primitives::uuid::Uuid as ProtoUuid,
16};
17
18use crate::{AdminClient, AdminError, NOT_CONTROLLER, kafka_error_name, retry::ControllerRetry};
19
20const CLUSTER_METADATA_TOPIC: &str = "__cluster_metadata";
21
22/// One listener endpoint of a voter, as Kafka's `RaftVoterEndpoint`.
23#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
24pub struct RaftVoterEndpoint {
25    listener: String,
26    host: String,
27    port: u16,
28}
29
30impl RaftVoterEndpoint {
31    /// An endpoint of `listener` at `host:port`.
32    ///
33    /// # Errors
34    /// Returns [`AdminError::InvalidArgument`] when `listener` is empty, has
35    /// leading or trailing whitespace, or is not in upper case, as Kafka's
36    /// `RaftVoterEndpoint` constructor refuses it.
37    pub fn new(
38        listener: impl Into<String>,
39        host: impl Into<String>,
40        port: u16,
41    ) -> Result<Self, AdminError> {
42        let listener = listener.into();
43        if listener.trim() != listener {
44            return Err(AdminError::InvalidArgument(format!(
45                "listener {listener:?}: leading or trailing whitespace is not allowed"
46            )));
47        }
48        if listener.is_empty() {
49            return Err(AdminError::InvalidArgument(
50                "listener: empty string is not allowed".to_owned(),
51            ));
52        }
53        if listener.to_uppercase() != listener {
54            return Err(AdminError::InvalidArgument(format!(
55                "listener {listener:?}: string must be UPPERCASE"
56            )));
57        }
58        Ok(Self {
59            listener,
60            host: host.into(),
61            port,
62        })
63    }
64
65    /// The listener name.
66    #[must_use]
67    pub fn listener(&self) -> &str {
68        &self.listener
69    }
70
71    /// The advertised host.
72    #[must_use]
73    pub fn host(&self) -> &str {
74        &self.host
75    }
76
77    /// The advertised port.
78    #[must_use]
79    pub const fn port(&self) -> u16 {
80        self.port
81    }
82}
83
84/// The `AddRaftVoter` request of one voter, as Kafka's `addRaftVoter`
85/// `createRequest` builds it. Each endpoint appears once, as Kafka takes a
86/// `Set`. `ack_when_committed` keeps its schema default, `true`, which
87/// Kafka's client does not change.
88fn add_raft_voter_request(
89    cluster_id: Option<&str>,
90    voter_id: i32,
91    voter_directory_id: uuid::Uuid,
92    endpoints: &[RaftVoterEndpoint],
93    timeout_ms: i32,
94) -> AddRaftVoterRequest {
95    let mut seen = BTreeSet::new();
96    AddRaftVoterRequest {
97        cluster_id: cluster_id.map(str::to_owned),
98        timeout_ms,
99        voter_id,
100        voter_directory_id: ProtoUuid(*voter_directory_id.as_bytes()),
101        listeners: endpoints
102            .iter()
103            .filter(|endpoint| seen.insert(*endpoint))
104            .map(|endpoint| Listener {
105                name: endpoint.listener.clone(),
106                host: endpoint.host.clone(),
107                port: endpoint.port,
108                ..Default::default()
109            })
110            .collect(),
111        ..Default::default()
112    }
113}
114
115/// The `RemoveRaftVoter` request of one voter, as Kafka's `removeRaftVoter`
116/// `createRequest` builds it: the cluster ID as the caller gave it, or null.
117fn remove_raft_voter_request(
118    cluster_id: Option<&str>,
119    voter_id: i32,
120    voter_directory_id: uuid::Uuid,
121) -> RemoveRaftVoterRequest {
122    RemoveRaftVoterRequest {
123        cluster_id: cluster_id.map(str::to_owned),
124        voter_id,
125        voter_directory_id: ProtoUuid(*voter_directory_id.as_bytes()),
126        ..Default::default()
127    }
128}
129
130/// `Ok` for code 0, and the broker error of `api` for any other code.
131fn voter_result(api: &'static str, code: i16, message: Option<String>) -> Result<(), AdminError> {
132    if code == 0 {
133        Ok(())
134    } else {
135        Err(broker_error(api, code, message))
136    }
137}
138
139#[derive(Debug, Clone, PartialEq, Eq)]
140pub struct QuorumReplica {
141    pub node_id: i32,
142    pub directory_id: uuid::Uuid,
143    pub log_end_offset: i64,
144    pub last_fetch_timestamp: i64,
145    pub last_caught_up_timestamp: i64,
146}
147
148#[derive(Debug, Clone, Default, PartialEq, Eq)]
149pub struct MetadataQuorum {
150    pub leader_id: i32,
151    pub leader_epoch: i32,
152    pub high_watermark: i64,
153    pub voters: Vec<QuorumReplica>,
154    pub observers: Vec<QuorumReplica>,
155    /// The listener endpoints of each node, as Kafka's `QuorumInfo.nodes()`
156    /// reads them from the response's `Nodes` (`DescribeQuorum` v2 and
157    /// later). Empty when the controller answered at v0 or v1.
158    pub nodes: Vec<QuorumNode>,
159}
160
161/// One node of `QuorumInfo.nodes()`: its id and listener endpoints.
162#[derive(Debug, Clone, PartialEq, Eq)]
163pub struct QuorumNode {
164    pub node_id: i32,
165    pub endpoints: Vec<RaftVoterEndpoint>,
166}
167
168fn broker_error(api: &'static str, code: i16, message: Option<String>) -> AdminError {
169    AdminError::Broker {
170        api,
171        code,
172        name: kafka_error_name(code),
173        message,
174    }
175}
176
177fn replica(
178    value: &krabka_protocol::owned::common::describe_quorum_response::replica_state::ReplicaState,
179) -> QuorumReplica {
180    QuorumReplica {
181        node_id: value.replica_id,
182        directory_id: uuid::Uuid::from_bytes(value.replica_directory_id.0),
183        log_end_offset: value.log_end_offset,
184        last_fetch_timestamp: value.last_fetch_timestamp,
185        last_caught_up_timestamp: value.last_caught_up_timestamp,
186    }
187}
188
189/// A response node as Kafka's `DescribeQuorumResponse.getNodes` maps it. The
190/// listener fields come straight from the wire, so they are not re-validated
191/// the way a caller-built `RaftVoterEndpoint` is.
192fn quorum_node(node: &krabka_protocol::owned::describe_quorum_response::Node) -> QuorumNode {
193    QuorumNode {
194        node_id: node.node_id,
195        endpoints: node
196            .listeners
197            .iter()
198            .map(|listener| RaftVoterEndpoint {
199                listener: listener.name.clone(),
200                host: listener.host.clone(),
201                port: listener.port,
202            })
203            .collect(),
204    }
205}
206
207impl AdminClient {
208    /// Return the live `__cluster_metadata` partition-zero quorum view.
209    ///
210    /// # Errors
211    /// Returns a transport, protocol, or Kafka error from `DescribeQuorum`.
212    pub async fn describe_metadata_quorum(&mut self) -> Result<MetadataQuorum, AdminError> {
213        let response = self
214            .conn
215            .send(DescribeQuorumRequest {
216                topics: vec![RequestTopic {
217                    topic_name: CLUSTER_METADATA_TOPIC.into(),
218                    partitions: vec![RequestPartition {
219                        partition_index: 0,
220                        ..Default::default()
221                    }],
222                    ..Default::default()
223                }],
224                ..Default::default()
225            })
226            .await?;
227        if response.error_code != 0 {
228            return Err(broker_error(
229                "DescribeQuorum",
230                response.error_code,
231                response.error_message,
232            ));
233        }
234        let nodes = response.nodes.iter().map(quorum_node).collect();
235        let partition = response
236            .topics
237            .into_iter()
238            .find(|topic| topic.topic_name == CLUSTER_METADATA_TOPIC)
239            .and_then(|topic| {
240                topic
241                    .partitions
242                    .into_iter()
243                    .find(|partition| partition.partition_index == 0)
244            })
245            .ok_or_else(|| {
246                AdminError::Protocol("DescribeQuorum omitted __cluster_metadata partition 0".into())
247            })?;
248        if partition.error_code != 0 {
249            return Err(broker_error(
250                "DescribeQuorum",
251                partition.error_code,
252                partition.error_message,
253            ));
254        }
255        Ok(MetadataQuorum {
256            leader_id: partition.leader_id,
257            leader_epoch: partition.leader_epoch,
258            high_watermark: partition.high_watermark,
259            voters: partition.current_voters.iter().map(replica).collect(),
260            observers: partition.observers.iter().map(replica).collect(),
261            nodes,
262        })
263    }
264
265    /// Adds a voter to the metadata quorum (KIP-853), as Kafka's
266    /// `addRaftVoter` operation does.
267    ///
268    /// The client sends `AddRaftVoter` on its connection: the active
269    /// controller through controller bootstrap (KIP-919), or a broker, which
270    /// forwards it, as Kafka's `LeastLoadedBrokerOrActiveKController` picks.
271    /// `cluster_id` is Kafka's `AddRaftVoterOptions.clusterId`; `None` sends
272    /// a null cluster ID, which the controller does not check. The request
273    /// carries the time left before the call deadline as its `timeout_ms`.
274    ///
275    /// `NOT_CONTROLLER` (41) makes the client find the controller again and
276    /// resend the request with the backoff until `default.api.timeout.ms`
277    /// (60 s), as Kafka's `handleNotControllerError` does.
278    ///
279    /// # Errors
280    /// Returns [`AdminError::Broker`] with the controller's error, such as
281    /// `DUPLICATE_VOTER` (126), and with `REQUEST_TIMED_OUT` (7) at the
282    /// deadline, or a transport error.
283    pub async fn add_raft_voter(
284        &mut self,
285        cluster_id: Option<&str>,
286        voter_id: i32,
287        voter_directory_id: uuid::Uuid,
288        endpoints: &[RaftVoterEndpoint],
289    ) -> Result<(), AdminError> {
290        let mut retry = ControllerRetry::new("AddRaftVoter", self.retry);
291        loop {
292            let request = add_raft_voter_request(
293                cluster_id,
294                voter_id,
295                voter_directory_id,
296                endpoints,
297                retry.remaining_millis(),
298            );
299            let response = retry.bounded(self.conn.send(request)).await?;
300            if response.error_code != NOT_CONTROLLER {
301                return voter_result("AddRaftVoter", response.error_code, response.error_message);
302            }
303            retry.after_not_controller(self).await?;
304        }
305    }
306
307    /// Remove one exact node and directory identity from the metadata quorum,
308    /// as Kafka's `removeRaftVoter` operation does.
309    ///
310    /// `cluster_id` is Kafka's `RemoveRaftVoterOptions.clusterId`, the
311    /// cluster ID string as `meta.properties` holds it; `None` sends a null
312    /// cluster ID, which the controller does not check.
313    ///
314    /// `NOT_CONTROLLER` (41) makes the client find the controller again and
315    /// resend the request with the backoff until `default.api.timeout.ms`
316    /// (60 s), as Kafka's `handleNotControllerError` does.
317    ///
318    /// # Errors
319    /// Returns a transport, protocol, or Kafka error from `RemoveRaftVoter`,
320    /// and `REQUEST_TIMED_OUT` (7) at the deadline.
321    pub async fn remove_raft_voter(
322        &mut self,
323        cluster_id: Option<&str>,
324        node_id: i32,
325        directory_id: uuid::Uuid,
326    ) -> Result<(), AdminError> {
327        let request = remove_raft_voter_request(cluster_id, node_id, directory_id);
328        let mut retry = ControllerRetry::new("RemoveRaftVoter", self.retry);
329        loop {
330            let response = retry.bounded(self.conn.send(request.clone())).await?;
331            if response.error_code != NOT_CONTROLLER {
332                return voter_result(
333                    "RemoveRaftVoter",
334                    response.error_code,
335                    response.error_message,
336                );
337            }
338            retry.after_not_controller(self).await?;
339        }
340    }
341}
342
343#[cfg(test)]
344mod tests {
345    use std::sync::{Arc, Mutex};
346
347    use krabka_client_core::MockReply;
348    use krabka_protocol::owned::{
349        add_raft_voter_request, add_raft_voter_response::AddRaftVoterResponse,
350        remove_raft_voter_request, remove_raft_voter_response::RemoveRaftVoterResponse,
351    };
352
353    use super::*;
354    use crate::partition_leaders::test_support::{
355        ApiVersionRange, decode_request, encode_response, fast_admin, scripted_broker,
356    };
357
358    const DIRECTORY: uuid::Uuid = uuid::Uuid::from_u128(0x0123_4567_89ab_cdef_0123_4567_89ab_cdef);
359
360    fn endpoint(listener: &str, host: &str, port: u16) -> RaftVoterEndpoint {
361        RaftVoterEndpoint::new(listener, host, port).expect("valid endpoint")
362    }
363
364    #[test]
365    fn endpoint_listener_must_be_trimmed_non_empty_upper_case() {
366        for (name, listener, valid) in [
367            ("upper case", "CONTROLLER", true),
368            ("upper case with digits and symbols", "CONTROLLER_2-X", true),
369            ("empty", "", false),
370            ("lower case", "controller", false),
371            ("leading whitespace", " CONTROLLER", false),
372            ("trailing whitespace", "CONTROLLER ", false),
373        ] {
374            let result = RaftVoterEndpoint::new(listener, "localhost", 9093);
375            assert2::assert!(
376                matches!(result, Err(AdminError::InvalidArgument(_))) != valid,
377                "case {name}"
378            );
379        }
380    }
381
382    #[test]
383    fn add_request_names_each_endpoint_once() {
384        let endpoints = [
385            endpoint("CONTROLLER", "c1", 9093),
386            endpoint("CONTROLLER", "c1", 9093),
387            endpoint("SSL", "c1", 9094),
388        ];
389        for (name, cluster_id, expected_cluster) in [
390            (
391                "with cluster id",
392                Some("MkU3OEVBNTcwNTJENDM2Qk"),
393                Some("MkU3OEVBNTcwNTJENDM2Qk"),
394            ),
395            ("without cluster id", None, None),
396        ] {
397            assert2::assert!(
398                add_raft_voter_request(cluster_id, 3, DIRECTORY, &endpoints, 1_000)
399                    == AddRaftVoterRequest {
400                        cluster_id: expected_cluster.map(str::to_owned),
401                        timeout_ms: 1_000,
402                        voter_id: 3,
403                        voter_directory_id: ProtoUuid(*DIRECTORY.as_bytes()),
404                        listeners: vec![
405                            Listener {
406                                name: "CONTROLLER".to_owned(),
407                                host: "c1".to_owned(),
408                                port: 9093,
409                                ..Default::default()
410                            },
411                            Listener {
412                                name: "SSL".to_owned(),
413                                host: "c1".to_owned(),
414                                port: 9094,
415                                ..Default::default()
416                            },
417                        ],
418                        ack_when_committed: true,
419                        ..Default::default()
420                    },
421                "case {name}"
422            );
423        }
424    }
425
426    #[test]
427    fn remove_request_sends_the_cluster_id_as_given() {
428        for (name, cluster_id) in [
429            ("with cluster id", Some("MkU3OEVBNTcwNTJENDM2Qk")),
430            ("without cluster id", None),
431        ] {
432            assert2::assert!(
433                remove_raft_voter_request(cluster_id, 3, DIRECTORY)
434                    == RemoveRaftVoterRequest {
435                        cluster_id: cluster_id.map(str::to_owned),
436                        voter_id: 3,
437                        voter_directory_id: ProtoUuid(*DIRECTORY.as_bytes()),
438                        ..Default::default()
439                    },
440                "case {name}"
441            );
442        }
443    }
444
445    /// `describe_metadata_quorum` reads the leader, voters, observers and,
446    /// from v2, every node's listener endpoints, as Kafka's
447    /// `DescribeQuorumResponse.getQuorumInfo` does. A v1 answer carries no
448    /// nodes.
449    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
450    async fn describe_metadata_quorum_reads_nodes_from_v2() {
451        use krabka_protocol::owned::{
452            common::describe_quorum_response::replica_state::ReplicaState,
453            describe_quorum_request,
454            describe_quorum_response::{
455                DescribeQuorumResponse, Listener, Node, PartitionData, TopicData,
456            },
457        };
458
459        let voter = |replica_id| ReplicaState {
460            replica_id,
461            log_end_offset: 40,
462            ..Default::default()
463        };
464        let response = DescribeQuorumResponse {
465            topics: vec![TopicData {
466                topic_name: CLUSTER_METADATA_TOPIC.into(),
467                partitions: vec![PartitionData {
468                    leader_id: 1,
469                    leader_epoch: 3,
470                    high_watermark: 41,
471                    current_voters: vec![voter(1), voter(2)],
472                    observers: vec![voter(7)],
473                    ..Default::default()
474                }],
475                ..Default::default()
476            }],
477            nodes: vec![Node {
478                node_id: 1,
479                listeners: vec![Listener {
480                    name: "CONTROLLER".into(),
481                    host: "c1.example".into(),
482                    port: 9093,
483                    ..Default::default()
484                }],
485                ..Default::default()
486            }],
487            ..Default::default()
488        };
489        let quorum = |nodes| MetadataQuorum {
490            leader_id: 1,
491            leader_epoch: 3,
492            high_watermark: 41,
493            voters: vec![
494                QuorumReplica {
495                    node_id: 1,
496                    directory_id: uuid::Uuid::nil(),
497                    log_end_offset: 40,
498                    last_fetch_timestamp: -1,
499                    last_caught_up_timestamp: -1,
500                },
501                QuorumReplica {
502                    node_id: 2,
503                    directory_id: uuid::Uuid::nil(),
504                    log_end_offset: 40,
505                    last_fetch_timestamp: -1,
506                    last_caught_up_timestamp: -1,
507                },
508            ],
509            observers: vec![QuorumReplica {
510                node_id: 7,
511                directory_id: uuid::Uuid::nil(),
512                log_end_offset: 40,
513                last_fetch_timestamp: -1,
514                last_caught_up_timestamp: -1,
515            }],
516            nodes,
517        };
518        for (name, max_version, expected) in [
519            (
520                "v2 carries nodes",
521                2,
522                quorum(vec![QuorumNode {
523                    node_id: 1,
524                    endpoints: vec![endpoint("CONTROLLER", "c1.example", 9093)],
525                }]),
526            ),
527            ("v1 carries none", 1, quorum(vec![])),
528        ] {
529            let answer = response.clone();
530            let broker = scripted_broker(
531                vec![
532                    ApiVersionRange::builder()
533                        .api_key(describe_quorum_request::API_KEY)
534                        .max_version(max_version)
535                        .build(),
536                ],
537                move |key, version, _, _| {
538                    if key == describe_quorum_request::API_KEY {
539                        MockReply::Respond(encode_response(&answer, version, true))
540                    } else {
541                        MockReply::Silent
542                    }
543                },
544            )
545            .await;
546            let mut admin = fast_admin(broker.addr, krabka_units::secs(5)).await;
547
548            let described = admin.describe_metadata_quorum().await.expect("describes");
549
550            broker.stop();
551            assert2::assert!(described == expected, "case {name}");
552        }
553    }
554
555    /// Kafka's `addRaftVoter` and `removeRaftVoter` resend after
556    /// `NOT_CONTROLLER` and fail at once on every other error code.
557    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
558    async fn voter_calls_retry_not_controller_and_stop_on_other_codes() {
559        for (name, add, codes, expected) in [
560            ("add succeeds", true, vec![0], (Ok(()), 1)),
561            (
562                "add after not controller",
563                true,
564                vec![NOT_CONTROLLER, 0],
565                (Ok(()), 2),
566            ),
567            (
568                "add duplicate voter is final",
569                true,
570                vec![126],
571                (Err(("AddRaftVoter", 126, Some("dup".to_owned()))), 1),
572            ),
573            ("remove succeeds", false, vec![0], (Ok(()), 1)),
574            (
575                "remove after not controller",
576                false,
577                vec![NOT_CONTROLLER, NOT_CONTROLLER, 0],
578                (Ok(()), 3),
579            ),
580            (
581                "remove voter not found is final",
582                false,
583                vec![127],
584                (Err(("RemoveRaftVoter", 127, Some("dup".to_owned()))), 1),
585            ),
586        ] {
587            let requests = Arc::new(Mutex::new(Vec::<(i16, i32)>::new()));
588            let handler_requests = Arc::clone(&requests);
589            let broker = scripted_broker(
590                vec![
591                    ApiVersionRange::builder()
592                        .api_key(add_raft_voter_request::API_KEY)
593                        .max_version(1)
594                        .build(),
595                    ApiVersionRange::builder()
596                        .api_key(remove_raft_voter_request::API_KEY)
597                        .build(),
598                ],
599                move |api_key, version, body, _| {
600                    let mut requests = handler_requests.lock().expect("requests lock");
601                    let code = codes[requests.len().min(codes.len() - 1)];
602                    let message = (code != 0).then(|| "dup".to_owned());
603                    let reply = match api_key {
604                        add_raft_voter_request::API_KEY => {
605                            let request: AddRaftVoterRequest = decode_request(body, version, true);
606                            requests.push((version, request.voter_id));
607                            encode_response(
608                                &AddRaftVoterResponse {
609                                    error_code: code,
610                                    error_message: message,
611                                    ..Default::default()
612                                },
613                                version,
614                                true,
615                            )
616                        }
617                        remove_raft_voter_request::API_KEY => {
618                            let request: RemoveRaftVoterRequest =
619                                decode_request(body, version, true);
620                            requests.push((version, request.voter_id));
621                            encode_response(
622                                &RemoveRaftVoterResponse {
623                                    error_code: code,
624                                    error_message: message,
625                                    ..Default::default()
626                                },
627                                version,
628                                true,
629                            )
630                        }
631                        _ => return MockReply::Silent,
632                    };
633                    MockReply::Respond(reply)
634                },
635            )
636            .await;
637            let mut admin = fast_admin(broker.addr, krabka_units::secs(5)).await;
638
639            let result = if add {
640                admin
641                    .add_raft_voter(None, 3, DIRECTORY, &[endpoint("CONTROLLER", "c3", 9093)])
642                    .await
643            } else {
644                admin.remove_raft_voter(None, 3, DIRECTORY).await
645            }
646            .map_err(|error| match error {
647                AdminError::Broker {
648                    api, code, message, ..
649                } => (api, code, message),
650                other => panic!("case {name}: unexpected error {other:?}"),
651            });
652
653            broker.stop();
654            let version = i16::from(add);
655            assert2::assert!(
656                (result, requests.lock().expect("requests lock").clone())
657                    == (expected.0, vec![(version, 3); expected.1]),
658                "case {name}"
659            );
660        }
661    }
662}