1use std::collections::BTreeSet;
6
7use krabka_protocol::{
8 owned::{
9 add_raft_voter_request::{AddRaftVoterRequest, Listener},
10 describe_quorum_request::{
11 DescribeQuorumRequest, PartitionData as RequestPartition, TopicData as RequestTopic,
12 },
13 remove_raft_voter_request::RemoveRaftVoterRequest,
14 },
15 primitives::uuid::Uuid as ProtoUuid,
16};
17
18use crate::{AdminClient, AdminError, NOT_CONTROLLER, kafka_error_name, retry::ControllerRetry};
19
20const CLUSTER_METADATA_TOPIC: &str = "__cluster_metadata";
21
22#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
24pub struct RaftVoterEndpoint {
25 listener: String,
26 host: String,
27 port: u16,
28}
29
30impl RaftVoterEndpoint {
31 pub fn new(
38 listener: impl Into<String>,
39 host: impl Into<String>,
40 port: u16,
41 ) -> Result<Self, AdminError> {
42 let listener = listener.into();
43 if listener.trim() != listener {
44 return Err(AdminError::InvalidArgument(format!(
45 "listener {listener:?}: leading or trailing whitespace is not allowed"
46 )));
47 }
48 if listener.is_empty() {
49 return Err(AdminError::InvalidArgument(
50 "listener: empty string is not allowed".to_owned(),
51 ));
52 }
53 if listener.to_uppercase() != listener {
54 return Err(AdminError::InvalidArgument(format!(
55 "listener {listener:?}: string must be UPPERCASE"
56 )));
57 }
58 Ok(Self {
59 listener,
60 host: host.into(),
61 port,
62 })
63 }
64
65 #[must_use]
67 pub fn listener(&self) -> &str {
68 &self.listener
69 }
70
71 #[must_use]
73 pub fn host(&self) -> &str {
74 &self.host
75 }
76
77 #[must_use]
79 pub const fn port(&self) -> u16 {
80 self.port
81 }
82}
83
84fn add_raft_voter_request(
89 cluster_id: Option<&str>,
90 voter_id: i32,
91 voter_directory_id: uuid::Uuid,
92 endpoints: &[RaftVoterEndpoint],
93 timeout_ms: i32,
94) -> AddRaftVoterRequest {
95 let mut seen = BTreeSet::new();
96 AddRaftVoterRequest {
97 cluster_id: cluster_id.map(str::to_owned),
98 timeout_ms,
99 voter_id,
100 voter_directory_id: ProtoUuid(*voter_directory_id.as_bytes()),
101 listeners: endpoints
102 .iter()
103 .filter(|endpoint| seen.insert(*endpoint))
104 .map(|endpoint| Listener {
105 name: endpoint.listener.clone(),
106 host: endpoint.host.clone(),
107 port: endpoint.port,
108 ..Default::default()
109 })
110 .collect(),
111 ..Default::default()
112 }
113}
114
115fn remove_raft_voter_request(
118 cluster_id: Option<&str>,
119 voter_id: i32,
120 voter_directory_id: uuid::Uuid,
121) -> RemoveRaftVoterRequest {
122 RemoveRaftVoterRequest {
123 cluster_id: cluster_id.map(str::to_owned),
124 voter_id,
125 voter_directory_id: ProtoUuid(*voter_directory_id.as_bytes()),
126 ..Default::default()
127 }
128}
129
130fn voter_result(api: &'static str, code: i16, message: Option<String>) -> Result<(), AdminError> {
132 if code == 0 {
133 Ok(())
134 } else {
135 Err(broker_error(api, code, message))
136 }
137}
138
139#[derive(Debug, Clone, PartialEq, Eq)]
140pub struct QuorumReplica {
141 pub node_id: i32,
142 pub directory_id: uuid::Uuid,
143 pub log_end_offset: i64,
144 pub last_fetch_timestamp: i64,
145 pub last_caught_up_timestamp: i64,
146}
147
148#[derive(Debug, Clone, Default, PartialEq, Eq)]
149pub struct MetadataQuorum {
150 pub leader_id: i32,
151 pub leader_epoch: i32,
152 pub high_watermark: i64,
153 pub voters: Vec<QuorumReplica>,
154 pub observers: Vec<QuorumReplica>,
155 pub nodes: Vec<QuorumNode>,
159}
160
161#[derive(Debug, Clone, PartialEq, Eq)]
163pub struct QuorumNode {
164 pub node_id: i32,
165 pub endpoints: Vec<RaftVoterEndpoint>,
166}
167
168fn broker_error(api: &'static str, code: i16, message: Option<String>) -> AdminError {
169 AdminError::Broker {
170 api,
171 code,
172 name: kafka_error_name(code),
173 message,
174 }
175}
176
177fn replica(
178 value: &krabka_protocol::owned::common::describe_quorum_response::replica_state::ReplicaState,
179) -> QuorumReplica {
180 QuorumReplica {
181 node_id: value.replica_id,
182 directory_id: uuid::Uuid::from_bytes(value.replica_directory_id.0),
183 log_end_offset: value.log_end_offset,
184 last_fetch_timestamp: value.last_fetch_timestamp,
185 last_caught_up_timestamp: value.last_caught_up_timestamp,
186 }
187}
188
189fn quorum_node(node: &krabka_protocol::owned::describe_quorum_response::Node) -> QuorumNode {
193 QuorumNode {
194 node_id: node.node_id,
195 endpoints: node
196 .listeners
197 .iter()
198 .map(|listener| RaftVoterEndpoint {
199 listener: listener.name.clone(),
200 host: listener.host.clone(),
201 port: listener.port,
202 })
203 .collect(),
204 }
205}
206
207impl AdminClient {
208 pub async fn describe_metadata_quorum(&mut self) -> Result<MetadataQuorum, AdminError> {
213 let response = self
214 .conn
215 .send(DescribeQuorumRequest {
216 topics: vec![RequestTopic {
217 topic_name: CLUSTER_METADATA_TOPIC.into(),
218 partitions: vec![RequestPartition {
219 partition_index: 0,
220 ..Default::default()
221 }],
222 ..Default::default()
223 }],
224 ..Default::default()
225 })
226 .await?;
227 if response.error_code != 0 {
228 return Err(broker_error(
229 "DescribeQuorum",
230 response.error_code,
231 response.error_message,
232 ));
233 }
234 let nodes = response.nodes.iter().map(quorum_node).collect();
235 let partition = response
236 .topics
237 .into_iter()
238 .find(|topic| topic.topic_name == CLUSTER_METADATA_TOPIC)
239 .and_then(|topic| {
240 topic
241 .partitions
242 .into_iter()
243 .find(|partition| partition.partition_index == 0)
244 })
245 .ok_or_else(|| {
246 AdminError::Protocol("DescribeQuorum omitted __cluster_metadata partition 0".into())
247 })?;
248 if partition.error_code != 0 {
249 return Err(broker_error(
250 "DescribeQuorum",
251 partition.error_code,
252 partition.error_message,
253 ));
254 }
255 Ok(MetadataQuorum {
256 leader_id: partition.leader_id,
257 leader_epoch: partition.leader_epoch,
258 high_watermark: partition.high_watermark,
259 voters: partition.current_voters.iter().map(replica).collect(),
260 observers: partition.observers.iter().map(replica).collect(),
261 nodes,
262 })
263 }
264
265 pub async fn add_raft_voter(
284 &mut self,
285 cluster_id: Option<&str>,
286 voter_id: i32,
287 voter_directory_id: uuid::Uuid,
288 endpoints: &[RaftVoterEndpoint],
289 ) -> Result<(), AdminError> {
290 let mut retry = ControllerRetry::new("AddRaftVoter", self.retry);
291 loop {
292 let request = add_raft_voter_request(
293 cluster_id,
294 voter_id,
295 voter_directory_id,
296 endpoints,
297 retry.remaining_millis(),
298 );
299 let response = retry.bounded(self.conn.send(request)).await?;
300 if response.error_code != NOT_CONTROLLER {
301 return voter_result("AddRaftVoter", response.error_code, response.error_message);
302 }
303 retry.after_not_controller(self).await?;
304 }
305 }
306
307 pub async fn remove_raft_voter(
322 &mut self,
323 cluster_id: Option<&str>,
324 node_id: i32,
325 directory_id: uuid::Uuid,
326 ) -> Result<(), AdminError> {
327 let request = remove_raft_voter_request(cluster_id, node_id, directory_id);
328 let mut retry = ControllerRetry::new("RemoveRaftVoter", self.retry);
329 loop {
330 let response = retry.bounded(self.conn.send(request.clone())).await?;
331 if response.error_code != NOT_CONTROLLER {
332 return voter_result(
333 "RemoveRaftVoter",
334 response.error_code,
335 response.error_message,
336 );
337 }
338 retry.after_not_controller(self).await?;
339 }
340 }
341}
342
343#[cfg(test)]
344mod tests {
345 use std::sync::{Arc, Mutex};
346
347 use krabka_client_core::MockReply;
348 use krabka_protocol::owned::{
349 add_raft_voter_request, add_raft_voter_response::AddRaftVoterResponse,
350 remove_raft_voter_request, remove_raft_voter_response::RemoveRaftVoterResponse,
351 };
352
353 use super::*;
354 use crate::partition_leaders::test_support::{
355 ApiVersionRange, decode_request, encode_response, fast_admin, scripted_broker,
356 };
357
358 const DIRECTORY: uuid::Uuid = uuid::Uuid::from_u128(0x0123_4567_89ab_cdef_0123_4567_89ab_cdef);
359
360 fn endpoint(listener: &str, host: &str, port: u16) -> RaftVoterEndpoint {
361 RaftVoterEndpoint::new(listener, host, port).expect("valid endpoint")
362 }
363
364 #[test]
365 fn endpoint_listener_must_be_trimmed_non_empty_upper_case() {
366 for (name, listener, valid) in [
367 ("upper case", "CONTROLLER", true),
368 ("upper case with digits and symbols", "CONTROLLER_2-X", true),
369 ("empty", "", false),
370 ("lower case", "controller", false),
371 ("leading whitespace", " CONTROLLER", false),
372 ("trailing whitespace", "CONTROLLER ", false),
373 ] {
374 let result = RaftVoterEndpoint::new(listener, "localhost", 9093);
375 assert2::assert!(
376 matches!(result, Err(AdminError::InvalidArgument(_))) != valid,
377 "case {name}"
378 );
379 }
380 }
381
382 #[test]
383 fn add_request_names_each_endpoint_once() {
384 let endpoints = [
385 endpoint("CONTROLLER", "c1", 9093),
386 endpoint("CONTROLLER", "c1", 9093),
387 endpoint("SSL", "c1", 9094),
388 ];
389 for (name, cluster_id, expected_cluster) in [
390 (
391 "with cluster id",
392 Some("MkU3OEVBNTcwNTJENDM2Qk"),
393 Some("MkU3OEVBNTcwNTJENDM2Qk"),
394 ),
395 ("without cluster id", None, None),
396 ] {
397 assert2::assert!(
398 add_raft_voter_request(cluster_id, 3, DIRECTORY, &endpoints, 1_000)
399 == AddRaftVoterRequest {
400 cluster_id: expected_cluster.map(str::to_owned),
401 timeout_ms: 1_000,
402 voter_id: 3,
403 voter_directory_id: ProtoUuid(*DIRECTORY.as_bytes()),
404 listeners: vec![
405 Listener {
406 name: "CONTROLLER".to_owned(),
407 host: "c1".to_owned(),
408 port: 9093,
409 ..Default::default()
410 },
411 Listener {
412 name: "SSL".to_owned(),
413 host: "c1".to_owned(),
414 port: 9094,
415 ..Default::default()
416 },
417 ],
418 ack_when_committed: true,
419 ..Default::default()
420 },
421 "case {name}"
422 );
423 }
424 }
425
426 #[test]
427 fn remove_request_sends_the_cluster_id_as_given() {
428 for (name, cluster_id) in [
429 ("with cluster id", Some("MkU3OEVBNTcwNTJENDM2Qk")),
430 ("without cluster id", None),
431 ] {
432 assert2::assert!(
433 remove_raft_voter_request(cluster_id, 3, DIRECTORY)
434 == RemoveRaftVoterRequest {
435 cluster_id: cluster_id.map(str::to_owned),
436 voter_id: 3,
437 voter_directory_id: ProtoUuid(*DIRECTORY.as_bytes()),
438 ..Default::default()
439 },
440 "case {name}"
441 );
442 }
443 }
444
445 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
450 async fn describe_metadata_quorum_reads_nodes_from_v2() {
451 use krabka_protocol::owned::{
452 common::describe_quorum_response::replica_state::ReplicaState,
453 describe_quorum_request,
454 describe_quorum_response::{
455 DescribeQuorumResponse, Listener, Node, PartitionData, TopicData,
456 },
457 };
458
459 let voter = |replica_id| ReplicaState {
460 replica_id,
461 log_end_offset: 40,
462 ..Default::default()
463 };
464 let response = DescribeQuorumResponse {
465 topics: vec![TopicData {
466 topic_name: CLUSTER_METADATA_TOPIC.into(),
467 partitions: vec![PartitionData {
468 leader_id: 1,
469 leader_epoch: 3,
470 high_watermark: 41,
471 current_voters: vec![voter(1), voter(2)],
472 observers: vec![voter(7)],
473 ..Default::default()
474 }],
475 ..Default::default()
476 }],
477 nodes: vec![Node {
478 node_id: 1,
479 listeners: vec![Listener {
480 name: "CONTROLLER".into(),
481 host: "c1.example".into(),
482 port: 9093,
483 ..Default::default()
484 }],
485 ..Default::default()
486 }],
487 ..Default::default()
488 };
489 let quorum = |nodes| MetadataQuorum {
490 leader_id: 1,
491 leader_epoch: 3,
492 high_watermark: 41,
493 voters: vec![
494 QuorumReplica {
495 node_id: 1,
496 directory_id: uuid::Uuid::nil(),
497 log_end_offset: 40,
498 last_fetch_timestamp: -1,
499 last_caught_up_timestamp: -1,
500 },
501 QuorumReplica {
502 node_id: 2,
503 directory_id: uuid::Uuid::nil(),
504 log_end_offset: 40,
505 last_fetch_timestamp: -1,
506 last_caught_up_timestamp: -1,
507 },
508 ],
509 observers: vec![QuorumReplica {
510 node_id: 7,
511 directory_id: uuid::Uuid::nil(),
512 log_end_offset: 40,
513 last_fetch_timestamp: -1,
514 last_caught_up_timestamp: -1,
515 }],
516 nodes,
517 };
518 for (name, max_version, expected) in [
519 (
520 "v2 carries nodes",
521 2,
522 quorum(vec![QuorumNode {
523 node_id: 1,
524 endpoints: vec![endpoint("CONTROLLER", "c1.example", 9093)],
525 }]),
526 ),
527 ("v1 carries none", 1, quorum(vec![])),
528 ] {
529 let answer = response.clone();
530 let broker = scripted_broker(
531 vec![
532 ApiVersionRange::builder()
533 .api_key(describe_quorum_request::API_KEY)
534 .max_version(max_version)
535 .build(),
536 ],
537 move |key, version, _, _| {
538 if key == describe_quorum_request::API_KEY {
539 MockReply::Respond(encode_response(&answer, version, true))
540 } else {
541 MockReply::Silent
542 }
543 },
544 )
545 .await;
546 let mut admin = fast_admin(broker.addr, krabka_units::secs(5)).await;
547
548 let described = admin.describe_metadata_quorum().await.expect("describes");
549
550 broker.stop();
551 assert2::assert!(described == expected, "case {name}");
552 }
553 }
554
555 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
558 async fn voter_calls_retry_not_controller_and_stop_on_other_codes() {
559 for (name, add, codes, expected) in [
560 ("add succeeds", true, vec![0], (Ok(()), 1)),
561 (
562 "add after not controller",
563 true,
564 vec![NOT_CONTROLLER, 0],
565 (Ok(()), 2),
566 ),
567 (
568 "add duplicate voter is final",
569 true,
570 vec![126],
571 (Err(("AddRaftVoter", 126, Some("dup".to_owned()))), 1),
572 ),
573 ("remove succeeds", false, vec![0], (Ok(()), 1)),
574 (
575 "remove after not controller",
576 false,
577 vec![NOT_CONTROLLER, NOT_CONTROLLER, 0],
578 (Ok(()), 3),
579 ),
580 (
581 "remove voter not found is final",
582 false,
583 vec![127],
584 (Err(("RemoveRaftVoter", 127, Some("dup".to_owned()))), 1),
585 ),
586 ] {
587 let requests = Arc::new(Mutex::new(Vec::<(i16, i32)>::new()));
588 let handler_requests = Arc::clone(&requests);
589 let broker = scripted_broker(
590 vec![
591 ApiVersionRange::builder()
592 .api_key(add_raft_voter_request::API_KEY)
593 .max_version(1)
594 .build(),
595 ApiVersionRange::builder()
596 .api_key(remove_raft_voter_request::API_KEY)
597 .build(),
598 ],
599 move |api_key, version, body, _| {
600 let mut requests = handler_requests.lock().expect("requests lock");
601 let code = codes[requests.len().min(codes.len() - 1)];
602 let message = (code != 0).then(|| "dup".to_owned());
603 let reply = match api_key {
604 add_raft_voter_request::API_KEY => {
605 let request: AddRaftVoterRequest = decode_request(body, version, true);
606 requests.push((version, request.voter_id));
607 encode_response(
608 &AddRaftVoterResponse {
609 error_code: code,
610 error_message: message,
611 ..Default::default()
612 },
613 version,
614 true,
615 )
616 }
617 remove_raft_voter_request::API_KEY => {
618 let request: RemoveRaftVoterRequest =
619 decode_request(body, version, true);
620 requests.push((version, request.voter_id));
621 encode_response(
622 &RemoveRaftVoterResponse {
623 error_code: code,
624 error_message: message,
625 ..Default::default()
626 },
627 version,
628 true,
629 )
630 }
631 _ => return MockReply::Silent,
632 };
633 MockReply::Respond(reply)
634 },
635 )
636 .await;
637 let mut admin = fast_admin(broker.addr, krabka_units::secs(5)).await;
638
639 let result = if add {
640 admin
641 .add_raft_voter(None, 3, DIRECTORY, &[endpoint("CONTROLLER", "c3", 9093)])
642 .await
643 } else {
644 admin.remove_raft_voter(None, 3, DIRECTORY).await
645 }
646 .map_err(|error| match error {
647 AdminError::Broker {
648 api, code, message, ..
649 } => (api, code, message),
650 other => panic!("case {name}: unexpected error {other:?}"),
651 });
652
653 broker.stop();
654 let version = i16::from(add);
655 assert2::assert!(
656 (result, requests.lock().expect("requests lock").clone())
657 == (expected.0, vec![(version, 3); expected.1]),
658 "case {name}"
659 );
660 }
661 }
662}