1use std::collections::{BTreeMap, BTreeSet};
4
5use bytes::BufMut;
6use krabka_client_core::{
7 Connection, ConnectionOptions, CoordinatorKeyType, build_find_coordinator,
8};
9use krabka_protocol::{
10 Encode, ProtocolError, ProtocolRequest,
11 owned::{
12 describe_producers_request::{
13 DescribeProducersRequest, TopicRequest as DescribeProducersTopicRequest,
14 },
15 describe_producers_response::{
16 DescribeProducersResponse, ProducerState as WireProducerState,
17 },
18 describe_transactions_request::DescribeTransactionsRequest,
19 describe_transactions_response::{DescribeTransactionsResponse, TransactionState},
20 find_coordinator_response::FindCoordinatorResponse,
21 init_producer_id_request::{self, InitProducerIdRequest},
22 init_producer_id_response::InitProducerIdResponse,
23 list_transactions_request::ListTransactionsRequest,
24 list_transactions_response::{
25 ListTransactionsResponse, TransactionState as ListedTransactionState,
26 },
27 metadata_request::{MetadataRequest, MetadataRequestTopic},
28 metadata_response::MetadataResponse,
29 write_txn_markers_request::{
30 WritableTxnMarker, WritableTxnMarkerTopic, WriteTxnMarkersRequest,
31 },
32 write_txn_markers_response::WriteTxnMarkersResponse,
33 },
34 primitives::uuid::Uuid as ProtoUuid,
35};
36use krabka_units::{Time, convert::TimeExt as _};
37
38#[cfg(test)]
39use crate::AdminClientConfig;
40use crate::{
41 AdminClient, AdminError, KafkaError, format_host_port, kafka_error_name,
42 partition_leaders::{PartitionKey, PartitionResults, complete_results},
43 retry::{CoordinatorRetry, RetryAction, RetryDeadline, RetryPolicy, connection_failure_action},
44};
45
46const COORDINATOR_LOAD_IN_PROGRESS: i16 = 14;
48const COORDINATOR_NOT_AVAILABLE: i16 = 15;
50const NOT_COORDINATOR: i16 = 16;
52const CONCURRENT_TRANSACTIONS: i16 = 51;
54const LEADER_NOT_AVAILABLE: i16 = 5;
56const NOT_LEADER_OR_FOLLOWER: i16 = 6;
59
60fn describe_retry_action(
65 result: Result<TransactionDescription, AdminError>,
66) -> RetryAction<TransactionDescription> {
67 match result {
68 Err(AdminError::Broker {
69 code: COORDINATOR_LOAD_IN_PROGRESS,
70 ..
71 }) => RetryAction::SameCoordinator(result),
72 Err(AdminError::Broker {
73 code: COORDINATOR_NOT_AVAILABLE | NOT_COORDINATOR,
74 ..
75 }) => RetryAction::FindCoordinator(result),
76 result => RetryAction::Done(result),
77 }
78}
79
80fn fence_retry_action(error_code: i16) -> RetryAction<()> {
85 if error_code == 0 {
86 return RetryAction::Done(Ok(()));
87 }
88 let result = Err(AdminError::Broker {
89 api: "InitProducerId",
90 code: error_code,
91 name: kafka_error_name(error_code),
92 message: None,
93 });
94 match error_code {
95 COORDINATOR_LOAD_IN_PROGRESS | CONCURRENT_TRANSACTIONS => {
96 RetryAction::SameCoordinator(result)
97 }
98 COORDINATOR_NOT_AVAILABLE | NOT_COORDINATOR => RetryAction::FindCoordinator(result),
99 _ => RetryAction::Done(result),
100 }
101}
102
103#[derive(Debug, Clone, PartialEq)]
112pub struct TransactionDescription {
113 pub transactional_id: String,
115 pub coordinator_id: i32,
117 pub state: String,
119 pub producer_id: i64,
121 pub producer_epoch: i16,
124 pub timeout: Time,
126 pub start_time_ms: Option<i64>,
131 pub topic_partitions: BTreeSet<(String, i32)>,
135}
136
137struct TransactionCoordinator {
140 node_id: i32,
142 connection: Connection,
143}
144
145#[derive(Debug, Clone, PartialEq, Eq)]
148pub struct TransactionListing {
149 pub transactional_id: String,
151 pub producer_id: i64,
153 pub state: String,
155}
156
157#[derive(Debug, Clone, Default, PartialEq, bon::Builder)]
161pub struct ListTransactionsFilter {
162 #[builder(default)]
165 pub state_filters: Vec<String>,
166 #[builder(default)]
169 pub producer_id_filters: Vec<i64>,
170 #[builder(with = |millis: i64| Time::from_millis(millis))]
173 pub min_duration: Option<Time>,
174}
175
176#[derive(Debug, Clone, PartialEq, Eq)]
179pub struct ProducerStateInfo {
180 pub producer_id: i64,
182 pub producer_epoch: i32,
184 pub last_sequence: i32,
187 pub last_timestamp_ms: i64,
190 pub coordinator_epoch: i32,
192 pub current_txn_start_offset: Option<i64>,
195}
196
197fn coordinator_address(
200 transactional_id: &str,
201 response: FindCoordinatorResponse,
202) -> Result<(i32, String), AdminError> {
203 if let Some(coordinator) = response
204 .coordinators
205 .into_iter()
206 .find(|coordinator| coordinator.key == transactional_id)
207 {
208 if coordinator.error_code != 0 {
209 return Err(AdminError::Broker {
210 api: "FindCoordinator",
211 code: coordinator.error_code,
212 name: kafka_error_name(coordinator.error_code),
213 message: coordinator.error_message,
214 });
215 }
216 return Ok((
217 coordinator.node_id,
218 format!("{}:{}", coordinator.host, coordinator.port),
219 ));
220 }
221
222 if response.error_code != 0 {
223 return Err(AdminError::Broker {
224 api: "FindCoordinator",
225 code: response.error_code,
226 name: kafka_error_name(response.error_code),
227 message: response.error_message,
228 });
229 }
230 if response.host.is_empty() {
231 return Err(AdminError::Protocol(format!(
232 "FindCoordinator returned no entry for transactional id {transactional_id:?}"
233 )));
234 }
235 Ok((
236 response.node_id,
237 format!("{}:{}", response.host, response.port),
238 ))
239}
240
241#[derive(Clone, Debug, PartialEq, Eq)]
252struct ReleasedInitProducerId(InitProducerIdRequest);
253
254impl Encode for ReleasedInitProducerId {
255 fn encode<B: BufMut>(&self, buf: &mut B, version: i16) -> Result<(), ProtocolError> {
256 self.0.encode(buf, version)
257 }
258
259 fn encoded_len(&self, version: i16) -> usize {
260 self.0.encoded_len(version)
261 }
262}
263
264impl ProtocolRequest for ReleasedInitProducerId {
265 const API_KEY: i16 = init_producer_id_request::API_KEY;
266 const MIN_VERSION: i16 = init_producer_id_request::MIN_VERSION;
267 const MAX_VERSION: i16 = 5;
269 const LATEST_STABLE_VERSION: i16 = Self::MAX_VERSION;
271 const FLEXIBLE_MIN: i16 = init_producer_id_request::FLEXIBLE_MIN;
272 type Response = InitProducerIdResponse;
273}
274
275fn force_terminate_request(
276 transactional_id: &str,
277 transaction_timeout_ms: i32,
278) -> ReleasedInitProducerId {
279 ReleasedInitProducerId(InitProducerIdRequest {
280 transactional_id: Some(transactional_id.to_owned()),
281 transaction_timeout_ms,
282 producer_id: -1,
283 producer_epoch: -1,
284 enable2_pc: false,
285 keep_prepared_txn: false,
286 ..Default::default()
287 })
288}
289
290fn describe_transactions_request(transactional_id: &str) -> DescribeTransactionsRequest {
291 DescribeTransactionsRequest {
292 transactional_ids: vec![transactional_id.to_owned()],
293 ..Default::default()
294 }
295}
296
297fn described_transaction(
301 coordinator_id: i32,
302 state: TransactionState,
303) -> Result<TransactionDescription, AdminError> {
304 if state.error_code != 0 {
305 return Err(AdminError::Broker {
306 api: "DescribeTransactions",
307 code: state.error_code,
308 name: kafka_error_name(state.error_code),
309 message: None,
310 });
311 }
312 Ok(TransactionDescription {
313 transactional_id: state.transactional_id,
314 coordinator_id,
315 state: state.transaction_state,
316 producer_id: state.producer_id,
317 producer_epoch: state.producer_epoch,
318 timeout: Time::from_millis(i64::from(state.transaction_timeout_ms)),
319 start_time_ms: (state.transaction_start_time_ms >= 0)
320 .then_some(state.transaction_start_time_ms),
321 topic_partitions: state
322 .topics
323 .into_iter()
324 .flat_map(|topic| {
325 topic
326 .partitions
327 .into_iter()
328 .map(move |partition| (topic.topic.clone(), partition))
329 })
330 .collect(),
331 })
332}
333
334fn transaction_description(
337 transactional_id: &str,
338 coordinator_id: i32,
339 response: DescribeTransactionsResponse,
340) -> Result<TransactionDescription, AdminError> {
341 response
342 .transaction_states
343 .into_iter()
344 .find(|state| state.transactional_id == transactional_id)
345 .ok_or_else(|| {
346 AdminError::Protocol(format!(
347 "DescribeTransactions returned no state for transactional id {transactional_id:?}"
348 ))
349 })
350 .and_then(|state| described_transaction(coordinator_id, state))
351}
352
353fn build_topic_metadata(topics: &[&str]) -> MetadataRequest {
357 MetadataRequest {
358 topics: if topics.is_empty() {
359 Some(Vec::new())
360 } else {
361 Some(
362 topics
363 .iter()
364 .map(|name| MetadataRequestTopic {
365 topic_id: ProtoUuid::ZERO,
366 name: Some((*name).to_owned()),
367 ..Default::default()
368 })
369 .collect(),
370 )
371 },
372 allow_auto_topic_creation: false,
373 ..Default::default()
374 }
375}
376
377fn partition_leader_address(
380 topic: &str,
381 partition: i32,
382 metadata: &MetadataResponse,
383) -> Result<String, AdminError> {
384 let topic_metadata = metadata
385 .topics
386 .iter()
387 .find(|entry| entry.name.as_deref() == Some(topic))
388 .ok_or_else(|| {
389 AdminError::Protocol(format!("Metadata returned no entry for topic {topic:?}"))
390 })?;
391 if topic_metadata.error_code != 0 {
392 return Err(AdminError::Broker {
393 api: "Metadata",
394 code: topic_metadata.error_code,
395 name: kafka_error_name(topic_metadata.error_code),
396 message: None,
397 });
398 }
399 let partition_metadata = topic_metadata
400 .partitions
401 .iter()
402 .find(|entry| entry.partition_index == partition)
403 .ok_or_else(|| {
404 AdminError::Protocol(format!(
405 "Metadata returned no entry for partition {topic}-{partition}"
406 ))
407 })?;
408 if partition_metadata.error_code != 0 {
409 return Err(AdminError::Broker {
410 api: "Metadata",
411 code: partition_metadata.error_code,
412 name: kafka_error_name(partition_metadata.error_code),
413 message: None,
414 });
415 }
416 metadata
417 .brokers
418 .iter()
419 .find(|broker| broker.node_id == partition_metadata.leader_id)
420 .map(|broker| format_host_port(&broker.host, broker.port))
421 .ok_or_else(|| {
422 AdminError::Protocol(format!(
423 "Metadata named leader {} for {topic}-{partition} but listed no such broker",
424 partition_metadata.leader_id
425 ))
426 })
427}
428
429#[derive(Debug, Clone, PartialEq, Eq)]
433pub struct AbortTransactionSpec {
434 pub topic: String,
436 pub partition: i32,
438 pub producer_id: i64,
440 pub producer_epoch: i16,
442 pub coordinator_epoch: i32,
444}
445
446fn abort_transaction_request(spec: &AbortTransactionSpec) -> WriteTxnMarkersRequest {
448 WriteTxnMarkersRequest {
449 markers: vec![WritableTxnMarker {
450 producer_id: spec.producer_id,
451 producer_epoch: spec.producer_epoch,
452 transaction_result: false,
453 topics: vec![WritableTxnMarkerTopic {
454 name: spec.topic.clone(),
455 partition_indexes: vec![spec.partition],
456 ..Default::default()
457 }],
458 coordinator_epoch: spec.coordinator_epoch,
459 ..Default::default()
460 }],
461 ..Default::default()
462 }
463}
464
465fn abort_transaction_result(
468 spec: &AbortTransactionSpec,
469 response: WriteTxnMarkersResponse,
470) -> Result<(), AdminError> {
471 let marker = response
472 .markers
473 .into_iter()
474 .find(|marker| marker.producer_id == spec.producer_id)
475 .ok_or_else(|| {
476 AdminError::Protocol(format!(
477 "WriteTxnMarkers returned no result for producer id {}",
478 spec.producer_id
479 ))
480 })?;
481 let topic_result = marker
482 .topics
483 .into_iter()
484 .find(|entry| entry.name == spec.topic)
485 .ok_or_else(|| {
486 AdminError::Protocol(format!(
487 "WriteTxnMarkers returned no result for topic {:?}",
488 spec.topic
489 ))
490 })?;
491 let partition_result = topic_result
492 .partitions
493 .into_iter()
494 .find(|entry| entry.partition_index == spec.partition)
495 .ok_or_else(|| {
496 AdminError::Protocol(format!(
497 "WriteTxnMarkers returned no result for partition {}-{}",
498 spec.topic, spec.partition
499 ))
500 })?;
501 if partition_result.error_code != 0 {
502 return Err(AdminError::Broker {
503 api: "WriteTxnMarkers",
504 code: partition_result.error_code,
505 name: kafka_error_name(partition_result.error_code),
506 message: None,
507 });
508 }
509 Ok(())
510}
511
512fn abort_transaction_retry_action(result: Result<(), AdminError>) -> RetryAction<()> {
517 match result {
518 Err(AdminError::Broker {
519 code: NOT_LEADER_OR_FOLLOWER | LEADER_NOT_AVAILABLE,
520 ..
521 }) => RetryAction::FindCoordinator(result),
522 result => RetryAction::Done(result),
523 }
524}
525
526fn describe_producers_request(keys: &[(String, i32)]) -> DescribeProducersRequest {
530 let mut by_topic = BTreeMap::<&str, Vec<i32>>::new();
531 for (topic, partition) in keys {
532 by_topic.entry(topic.as_str()).or_default().push(*partition);
533 }
534 DescribeProducersRequest {
535 topics: by_topic
536 .into_iter()
537 .map(|(name, partition_indexes)| DescribeProducersTopicRequest {
538 name: name.to_owned(),
539 partition_indexes,
540 ..Default::default()
541 })
542 .collect(),
543 ..Default::default()
544 }
545}
546
547fn producer_state_info(state: &WireProducerState) -> ProducerStateInfo {
550 ProducerStateInfo {
551 producer_id: state.producer_id,
552 producer_epoch: state.producer_epoch,
553 last_sequence: state.last_sequence,
554 last_timestamp_ms: state.last_timestamp,
555 coordinator_epoch: state.coordinator_epoch,
556 current_txn_start_offset: (state.current_txn_start_offset != -1)
557 .then_some(state.current_txn_start_offset),
558 }
559}
560
561fn describe_producers_results(
565 keys: &[PartitionKey],
566 response: DescribeProducersResponse,
567) -> PartitionResults<Vec<ProducerStateInfo>> {
568 let mut out = BTreeMap::new();
569 for topic in response.topics {
570 for partition in topic.partitions {
571 let result = if partition.error_code == 0 {
572 Ok(partition
573 .active_producers
574 .iter()
575 .map(producer_state_info)
576 .collect())
577 } else {
578 Err(KafkaError {
579 code: partition.error_code,
580 name: kafka_error_name(partition.error_code),
581 message: partition.error_message,
582 })
583 };
584 out.insert((topic.name.clone(), partition.partition_index), result);
585 }
586 }
587 complete_results("DescribeProducers", keys, out)
588}
589
590const fn describe_producers_retry_code(code: i16) -> bool {
594 code == NOT_LEADER_OR_FOLLOWER
595}
596
597async fn describe_producers_on_leader(
600 address: String,
601 options: ConnectionOptions,
602 keys: Vec<PartitionKey>,
603) -> Result<PartitionResults<Vec<ProducerStateInfo>>, AdminError> {
604 let connection = AdminClient::connect_one(&address, options).await?;
605 let response = connection.send(describe_producers_request(&keys)).await?;
606 Ok(describe_producers_results(&keys, response))
607}
608
609fn list_transactions_request(filter: &ListTransactionsFilter) -> ListTransactionsRequest {
612 ListTransactionsRequest {
613 state_filters: filter.state_filters.clone(),
614 producer_id_filters: filter.producer_id_filters.clone(),
615 duration_filter: filter
616 .min_duration
617 .map_or(-1, krabka_units::convert::TimeExt::millis_i64),
618 ..Default::default()
619 }
620}
621
622fn transaction_listing(state: ListedTransactionState) -> TransactionListing {
624 TransactionListing {
625 transactional_id: state.transactional_id,
626 producer_id: state.producer_id,
627 state: state.transaction_state,
628 }
629}
630
631async fn list_transactions_on_broker(
634 address: String,
635 options: ConnectionOptions,
636 request: ListTransactionsRequest,
637) -> Result<Vec<TransactionListing>, AdminError> {
638 let connection = AdminClient::connect_one(&address, options).await?;
639 let response: ListTransactionsResponse = connection.send(request).await?;
640 if response.error_code != 0 {
641 return Err(AdminError::Broker {
642 api: "ListTransactions",
643 code: response.error_code,
644 name: kafka_error_name(response.error_code),
645 message: None,
646 });
647 }
648 Ok(response
649 .transaction_states
650 .into_iter()
651 .map(transaction_listing)
652 .collect())
653}
654
655impl AdminClient {
656 pub async fn force_terminate_transaction(
692 &self,
693 transactional_id: &str,
694 ) -> Result<(), AdminError> {
695 if transactional_id.is_empty() {
696 return Err(AdminError::Protocol(
697 "transactional id must not be empty".to_owned(),
698 ));
699 }
700
701 self.force_terminate_transaction_with_retry(transactional_id, self.retry)
702 .await
703 }
704
705 async fn force_terminate_transaction_with_retry(
706 &self,
707 transactional_id: &str,
708 retry: RetryPolicy,
709 ) -> Result<(), AdminError> {
710 self.force_terminate_transaction_until(transactional_id, retry.start())
711 .await
712 }
713
714 async fn force_terminate_transaction_until(
717 &self,
718 transactional_id: &str,
719 deadline: RetryDeadline,
720 ) -> Result<(), AdminError> {
721 let mut coordinator = None;
722 let mut retry = CoordinatorRetry::from_deadline(deadline);
723 loop {
724 let action = retry
725 .run(self.force_terminate_attempt(
726 transactional_id,
727 &mut coordinator,
728 retry.find_coordinator(),
729 ))
730 .await;
731 if let Some(result) = retry.next(action).await {
732 return result;
733 }
734 }
735 }
736
737 pub async fn fence_producers(
757 &self,
758 transactional_ids: &[&str],
759 ) -> BTreeMap<String, Result<(), AdminError>> {
760 let deadline = self.retry.start();
761 futures_util::future::join_all(transactional_ids.iter().map(|transactional_id| {
762 let transactional_id = (*transactional_id).to_owned();
763 async move {
764 let result = if transactional_id.is_empty() {
765 Err(AdminError::Protocol(
766 "transactional id must not be empty".to_owned(),
767 ))
768 } else {
769 self.force_terminate_transaction_until(&transactional_id, deadline)
770 .await
771 };
772 (transactional_id, result)
773 }
774 }))
775 .await
776 .into_iter()
777 .collect()
778 }
779
780 pub async fn abort_transaction(&self, spec: &AbortTransactionSpec) -> Result<(), AdminError> {
808 if spec.topic.is_empty() {
809 return Err(AdminError::Protocol("topic must not be empty".to_owned()));
810 }
811
812 let mut leader = None;
813 let mut retry = CoordinatorRetry::new(self.retry);
814 loop {
815 let action = retry
816 .run(self.abort_transaction_attempt(spec, &mut leader, retry.find_coordinator()))
817 .await;
818 if let Some(result) = retry.next(action).await {
819 return result;
820 }
821 }
822 }
823
824 async fn abort_transaction_attempt(
826 &self,
827 spec: &AbortTransactionSpec,
828 leader: &mut Option<Connection>,
829 find_leader: bool,
830 ) -> RetryAction<()> {
831 let connection = match self
832 .partition_leader_connection(&spec.topic, spec.partition, leader, find_leader)
833 .await
834 {
835 Ok(connection) => connection,
836 Err(action) => return action,
837 };
838 match connection.send(abort_transaction_request(spec)).await {
839 Ok(response) => {
840 abort_transaction_retry_action(abort_transaction_result(spec, response))
841 }
842 Err(error) => connection_failure_action(error.into()),
843 }
844 }
845
846 async fn partition_leader_connection<'c, T>(
850 &self,
851 topic: &str,
852 partition: i32,
853 leader: &'c mut Option<Connection>,
854 find_leader: bool,
855 ) -> Result<&'c Connection, RetryAction<T>> {
856 if find_leader {
857 *leader = None;
858 }
859 if let Some(connection) = leader {
860 return Ok(connection);
861 }
862 let metadata = self
863 .conn
864 .send(build_topic_metadata(&[topic]))
865 .await
866 .map_err(connection_failure_action)?;
867 let address = match partition_leader_address(topic, partition, &metadata) {
868 Ok(address) => address,
869 Err(
870 error @ AdminError::Broker {
871 code: NOT_LEADER_OR_FOLLOWER | LEADER_NOT_AVAILABLE,
872 ..
873 },
874 ) => return Err(RetryAction::FindCoordinator(Err(error))),
875 Err(error) => return Err(RetryAction::Done(Err(error))),
876 };
877 match Self::connect_one(&address, self.options.clone()).await {
878 Ok(connection) => Ok(leader.insert(connection)),
879 Err(error) => Err(connection_failure_action(error)),
880 }
881 }
882
883 pub async fn describe_producers(
910 &self,
911 partitions: &[(String, i32)],
912 ) -> BTreeMap<(String, i32), Result<Vec<ProducerStateInfo>, KafkaError>> {
913 self.call_partition_leaders(
914 "DescribeProducers",
915 partitions,
916 describe_producers_retry_code,
917 |address, keys| describe_producers_on_leader(address, self.options.clone(), keys),
918 )
919 .await
920 }
921
922 pub async fn list_transactions(
940 &self,
941 filter: &ListTransactionsFilter,
942 ) -> Result<Vec<TransactionListing>, AdminError> {
943 let metadata = self.conn.send(build_topic_metadata(&[])).await?;
944 let request = list_transactions_request(filter);
945 let options = self.options.clone();
946 let answers = futures_util::future::join_all(metadata.brokers.into_iter().map(|broker| {
947 list_transactions_on_broker(
948 format_host_port(&broker.host, broker.port),
949 options.clone(),
950 request.clone(),
951 )
952 }))
953 .await;
954
955 let mut listings = Vec::new();
956 for answer in answers {
957 listings.extend(answer?);
958 }
959 Ok(listings)
960 }
961
962 async fn force_terminate_attempt(
964 &self,
965 transactional_id: &str,
966 coordinator: &mut Option<TransactionCoordinator>,
967 find_coordinator: bool,
968 ) -> RetryAction<()> {
969 let coordinator = match self
970 .transaction_coordinator(transactional_id, coordinator, find_coordinator)
971 .await
972 {
973 Ok(coordinator) => coordinator,
974 Err(action) => return action,
975 };
976 let request =
977 force_terminate_request(transactional_id, self.options.request_timeout.millis_i32());
978 match coordinator.connection.send(request).await {
979 Ok(response) => fence_retry_action(response.error_code),
980 Err(error) => connection_failure_action(error.into()),
981 }
982 }
983
984 pub async fn describe_transaction(
1019 &self,
1020 transactional_id: &str,
1021 ) -> Result<TransactionDescription, AdminError> {
1022 if transactional_id.is_empty() {
1023 return Err(AdminError::Protocol(
1024 "transactional id must not be empty".to_owned(),
1025 ));
1026 }
1027
1028 self.describe_transaction_with_retry(transactional_id, self.retry)
1029 .await
1030 }
1031
1032 async fn describe_transaction_with_retry(
1033 &self,
1034 transactional_id: &str,
1035 retry: RetryPolicy,
1036 ) -> Result<TransactionDescription, AdminError> {
1037 self.describe_transaction_until(transactional_id, retry.start())
1038 .await
1039 }
1040
1041 async fn describe_transaction_until(
1044 &self,
1045 transactional_id: &str,
1046 deadline: RetryDeadline,
1047 ) -> Result<TransactionDescription, AdminError> {
1048 let mut coordinator = None;
1049 let mut retry = CoordinatorRetry::from_deadline(deadline);
1050 loop {
1051 let action = retry
1052 .run(self.describe_transaction_attempt(
1053 transactional_id,
1054 &mut coordinator,
1055 retry.find_coordinator(),
1056 ))
1057 .await;
1058 if let Some(result) = retry.next(action).await {
1059 return result;
1060 }
1061 }
1062 }
1063
1064 async fn describe_transaction_attempt(
1066 &self,
1067 transactional_id: &str,
1068 coordinator: &mut Option<TransactionCoordinator>,
1069 find_coordinator: bool,
1070 ) -> RetryAction<TransactionDescription> {
1071 let coordinator = match self
1072 .transaction_coordinator(transactional_id, coordinator, find_coordinator)
1073 .await
1074 {
1075 Ok(coordinator) => coordinator,
1076 Err(action) => return action,
1077 };
1078 match coordinator
1079 .connection
1080 .send(describe_transactions_request(transactional_id))
1081 .await
1082 {
1083 Ok(response) => describe_retry_action(transaction_description(
1084 transactional_id,
1085 coordinator.node_id,
1086 response,
1087 )),
1088 Err(error) => connection_failure_action(error.into()),
1089 }
1090 }
1091
1092 async fn transaction_coordinator<'c, T>(
1102 &self,
1103 transactional_id: &str,
1104 coordinator: &'c mut Option<TransactionCoordinator>,
1105 find_coordinator: bool,
1106 ) -> Result<&'c TransactionCoordinator, RetryAction<T>> {
1107 if find_coordinator {
1108 *coordinator = None;
1109 }
1110 if let Some(coordinator) = coordinator {
1111 return Ok(coordinator);
1112 }
1113 let response = self
1114 .conn
1115 .send(build_find_coordinator(
1116 transactional_id,
1117 CoordinatorKeyType::Transaction,
1118 ))
1119 .await
1120 .map_err(connection_failure_action)?;
1121 let (node_id, address) = match coordinator_address(transactional_id, response) {
1122 Ok(found) => found,
1123 Err(
1124 error @ AdminError::Broker {
1125 code: COORDINATOR_LOAD_IN_PROGRESS | COORDINATOR_NOT_AVAILABLE,
1126 ..
1127 },
1128 ) => return Err(RetryAction::FindCoordinator(Err(error))),
1129 Err(error) => return Err(RetryAction::Done(Err(error))),
1130 };
1131 match Self::connect_one(&address, self.options.clone()).await {
1132 Ok(connection) => Ok(coordinator.insert(TransactionCoordinator {
1133 node_id,
1134 connection,
1135 })),
1136 Err(error) => Err(connection_failure_action(error)),
1137 }
1138 }
1139
1140 pub async fn describe_transactions(
1160 &self,
1161 transactional_ids: &[&str],
1162 ) -> Result<Vec<TransactionDescription>, AdminError> {
1163 let deadline = self.retry.start();
1166 let mut descriptions = Vec::with_capacity(transactional_ids.len());
1167 for transactional_id in transactional_ids {
1168 if transactional_id.is_empty() {
1169 return Err(AdminError::Protocol(
1170 "transactional id must not be empty".to_owned(),
1171 ));
1172 }
1173 descriptions.push(
1174 self.describe_transaction_until(transactional_id, deadline)
1175 .await?,
1176 );
1177 }
1178 Ok(descriptions)
1179 }
1180}
1181
1182#[cfg(test)]
1183mod tests {
1184 use std::{
1185 sync::{Arc, Mutex},
1186 time::Duration,
1187 };
1188
1189 use assert2::check;
1190 use bytes::{Buf, BytesMut};
1191 use krabka_client_core::{ClientError, MockBroker, MockReply};
1192 use krabka_protocol::{
1193 Decode,
1194 owned::{
1195 api_versions_request,
1196 api_versions_response::{ApiVersion, ApiVersionsResponse},
1197 describe_producers_request,
1198 describe_producers_response::{PartitionResponse, TopicResponse},
1199 describe_transactions_request,
1200 describe_transactions_response::TopicData,
1201 find_coordinator_request,
1202 find_coordinator_response::Coordinator,
1203 list_transactions_request, metadata_request,
1204 metadata_response::{
1205 MetadataResponse, MetadataResponseBroker, MetadataResponsePartition,
1206 MetadataResponseTopic,
1207 },
1208 write_txn_markers_request,
1209 write_txn_markers_response::{
1210 WritableTxnMarkerPartitionResult, WritableTxnMarkerResult,
1211 WritableTxnMarkerTopicResult,
1212 },
1213 },
1214 };
1215
1216 use super::*;
1217 use crate::partition_leaders::test_support::{
1218 ApiVersionRange, Seen, fast_admin as leader_fast_admin,
1219 leader_broker as partition_leader_broker,
1220 };
1221
1222 fn state_row(transactional_id: &str, error_code: i16) -> TransactionState {
1223 TransactionState {
1224 error_code,
1225 transactional_id: transactional_id.to_owned(),
1226 ..Default::default()
1227 }
1228 }
1229
1230 fn fence_payments() -> InitProducerIdRequest {
1233 InitProducerIdRequest {
1234 transactional_id: Some("payments".to_owned()),
1235 transaction_timeout_ms: 30_000,
1236 producer_id: -1,
1237 producer_epoch: -1,
1238 enable2_pc: false,
1239 keep_prepared_txn: false,
1240 ..Default::default()
1241 }
1242 }
1243
1244 #[test]
1245 fn force_termination_request_fences_without_preserving_transaction() {
1246 let request = force_terminate_request("payments", 30_000);
1247
1248 assert2::assert!(request == ReleasedInitProducerId(fence_payments()));
1249 }
1250
1251 fn encode_response(response: &impl Encode, version: i16, flexible: bool) -> Vec<u8> {
1254 let mut bytes = BytesMut::new();
1255 if flexible {
1256 bytes.extend_from_slice(&[0]);
1257 }
1258 response
1259 .encode(&mut bytes, version)
1260 .expect("response encodes");
1261 bytes.to_vec()
1262 }
1263
1264 fn api_versions(init_producer_id_range: (i16, i16)) -> Vec<u8> {
1267 let api_version = |api_key, min_version, max_version| ApiVersion {
1268 api_key,
1269 min_version,
1270 max_version,
1271 ..Default::default()
1272 };
1273 encode_response(
1274 &ApiVersionsResponse {
1275 api_keys: vec![
1276 api_version(api_versions_request::API_KEY, 0, 0),
1277 api_version(find_coordinator_request::API_KEY, 0, 0),
1278 api_version(metadata_request::API_KEY, 13, 13),
1279 api_version(
1280 init_producer_id_request::API_KEY,
1281 init_producer_id_range.0,
1282 init_producer_id_range.1,
1283 ),
1284 ],
1285 ..Default::default()
1286 },
1287 0,
1288 false,
1289 )
1290 }
1291
1292 type FenceResult = Result<(), (i16, i16, i16, i16, i16)>;
1295
1296 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1300 async fn force_terminate_transaction_sends_a_released_init_producer_id_version() {
1301 let sent_request = |version| vec![(version, fence_payments())];
1302 for (name, init_producer_id_range, expected_requests, expected_result) in [
1303 ("coordinator stops at v1", (0, 1), sent_request(1), Ok(())),
1304 ("coordinator stops at v4", (0, 4), sent_request(4), Ok(())),
1305 ("coordinator stops at v5", (0, 5), sent_request(5), Ok(())),
1306 (
1307 "coordinator supports unstable v6",
1308 (0, 6),
1309 sent_request(5),
1310 Ok(()),
1311 ),
1312 (
1313 "coordinator supports only unstable v6",
1314 (6, 6),
1315 Vec::new(),
1316 Err((init_producer_id_request::API_KEY, 6, 6, 0, 5)),
1317 ),
1318 ] {
1319 let requests = Arc::new(Mutex::new(Vec::new()));
1320 let requests_in_mock = Arc::clone(&requests);
1321 let coordinator =
1322 MockBroker::start(move |api_key, version, _, mut body| match api_key {
1323 api_versions_request::API_KEY => Some(api_versions(init_producer_id_range)),
1324 init_producer_id_request::API_KEY => {
1325 let client_id_len = body.get_i16();
1326 body.advance(usize::try_from(client_id_len).expect("client id length"));
1327 let flexible = version >= init_producer_id_request::FLEXIBLE_MIN;
1328 if flexible {
1329 body.advance(1);
1330 }
1331 let request = InitProducerIdRequest::decode(&mut body, version)
1332 .expect("init producer id request decodes");
1333 requests_in_mock
1334 .lock()
1335 .expect("requests lock")
1336 .push((version, request));
1337 Some(encode_response(
1338 &InitProducerIdResponse {
1339 producer_id: 4242,
1340 producer_epoch: 1,
1341 ..Default::default()
1342 },
1343 version,
1344 flexible,
1345 ))
1346 }
1347 _ => None,
1348 })
1349 .await;
1350 let coordinator_addr = coordinator.addr;
1351 let bootstrap = MockBroker::start(move |api_key, version, _, _| match api_key {
1352 api_versions_request::API_KEY => Some(api_versions(init_producer_id_range)),
1353 find_coordinator_request::API_KEY => Some(encode_response(
1354 &FindCoordinatorResponse {
1355 node_id: 2,
1356 host: coordinator_addr.ip().to_string(),
1357 port: i32::from(coordinator_addr.port()),
1358 ..Default::default()
1359 },
1360 version,
1361 false,
1362 )),
1363 metadata_request::API_KEY => {
1364 Some(encode_response(&MetadataResponse::default(), version, true))
1365 }
1366 _ => None,
1367 })
1368 .await;
1369 let admin = AdminClient::connect(&[bootstrap.addr.to_string()])
1370 .await
1371 .expect("admin connects");
1372
1373 let result: FenceResult =
1374 admin
1375 .force_terminate_transaction("payments")
1376 .await
1377 .map_err(|error| match error {
1378 AdminError::Transport(ClientError::IncompatibleVersion {
1379 api_key,
1380 broker_min,
1381 broker_max,
1382 client_min,
1383 client_max,
1384 }) => (api_key, broker_min, broker_max, client_min, client_max),
1385 other => panic!("case {name}: unexpected error {other:?}"),
1386 });
1387
1388 bootstrap.stop();
1389 coordinator.stop();
1390 let requests = requests.lock().expect("requests lock").clone();
1391 assert2::assert!(
1392 (requests, result) == (expected_requests, expected_result),
1393 "case {name}"
1394 );
1395 }
1396 }
1397
1398 #[test]
1401 fn coordinator_lookup_names_the_node_and_address_of_the_key() {
1402 for (name, response) in [
1403 (
1404 "batched entry",
1405 FindCoordinatorResponse {
1406 coordinators: vec![
1407 Coordinator {
1408 key: "other".to_owned(),
1409 node_id: 1,
1410 host: "wrong".to_owned(),
1411 port: 1,
1412 ..Default::default()
1413 },
1414 Coordinator {
1415 key: "payments".to_owned(),
1416 node_id: 3,
1417 host: "coordinator".to_owned(),
1418 port: 9092,
1419 ..Default::default()
1420 },
1421 ],
1422 ..Default::default()
1423 },
1424 ),
1425 (
1426 "top-level fields",
1427 FindCoordinatorResponse {
1428 node_id: 3,
1429 host: "coordinator".to_owned(),
1430 port: 9092,
1431 ..Default::default()
1432 },
1433 ),
1434 ] {
1435 let found = coordinator_address("payments", response).expect("matching coordinator");
1436 assert2::assert!(found == (3, "coordinator:9092".to_owned()), "case {name}");
1437 }
1438 }
1439
1440 #[test]
1441 fn describe_request_asks_for_exactly_one_transactional_id() {
1442 let request = describe_transactions_request("payments");
1443
1444 assert2::assert!(
1445 request
1446 == DescribeTransactionsRequest {
1447 transactional_ids: vec!["payments".to_owned()],
1448 ..Default::default()
1449 }
1450 );
1451 }
1452
1453 #[test]
1457 fn description_maps_the_matching_row_to_the_domain_struct() {
1458 let topic = |topic: &str, partitions: Vec<i32>| TopicData {
1459 topic: topic.to_owned(),
1460 partitions,
1461 ..Default::default()
1462 };
1463 let partitions = |entries: &[(&str, i32)]| {
1464 entries
1465 .iter()
1466 .map(|(topic, partition)| ((*topic).to_owned(), *partition))
1467 .collect::<BTreeSet<_>>()
1468 };
1469 for (name, state, start, topics, expected_start, expected_partitions) in [
1470 (
1471 "ongoing with partitions of two topics",
1472 "Ongoing",
1473 1_700_000_000_123,
1474 vec![topic("orders", vec![1, 0]), topic("audit", vec![2])],
1475 Some(1_700_000_000_123),
1476 partitions(&[("audit", 2), ("orders", 0), ("orders", 1)]),
1477 ),
1478 (
1479 "empty with no start time",
1480 "Empty",
1481 -1,
1482 Vec::new(),
1483 None,
1484 BTreeSet::new(),
1485 ),
1486 (
1487 "start time at the epoch",
1488 "Ongoing",
1489 0,
1490 Vec::new(),
1491 Some(0),
1492 BTreeSet::new(),
1493 ),
1494 ] {
1495 let response = DescribeTransactionsResponse {
1496 transaction_states: vec![
1497 TransactionState {
1498 transactional_id: "other".to_owned(),
1499 producer_id: 11,
1500 producer_epoch: 1,
1501 ..Default::default()
1502 },
1503 TransactionState {
1504 transactional_id: "payments".to_owned(),
1505 transaction_state: state.to_owned(),
1506 transaction_timeout_ms: 60_000,
1507 transaction_start_time_ms: start,
1508 producer_id: 4242,
1509 producer_epoch: 7,
1510 topics,
1511 ..Default::default()
1512 },
1513 ],
1514 ..Default::default()
1515 };
1516
1517 let described = transaction_description("payments", 3, response).expect("matching row");
1518
1519 assert2::assert!(
1520 described
1521 == TransactionDescription {
1522 transactional_id: "payments".to_owned(),
1523 coordinator_id: 3,
1524 state: state.to_owned(),
1525 producer_id: 4242,
1526 producer_epoch: 7,
1527 timeout: Time::from_millis(60_000),
1528 start_time_ms: expected_start,
1529 topic_partitions: expected_partitions,
1530 },
1531 "case {name}"
1532 );
1533 }
1534 }
1535
1536 #[test]
1537 fn description_surfaces_the_error_code_of_the_matching_row() {
1538 for (_case, code, want_name) in [
1539 ("unknown id", 105, "TRANSACTIONAL_ID_NOT_FOUND"),
1540 (
1541 "not authorized",
1542 53,
1543 "TRANSACTIONAL_ID_AUTHORIZATION_FAILED",
1544 ),
1545 ("moved coordinator", 16, "NOT_COORDINATOR"),
1546 ("loading", 14, "COORDINATOR_LOAD_IN_PROGRESS"),
1547 ] {
1548 let response = DescribeTransactionsResponse {
1549 transaction_states: vec![state_row("other", 0), state_row("payments", code)],
1550 ..Default::default()
1551 };
1552
1553 let error = transaction_description("payments", 3, response).expect_err("broker error");
1554 match error {
1555 AdminError::Broker {
1556 api,
1557 code: got,
1558 name,
1559 message,
1560 } => {
1561 check!(
1562 (api, got, name, message)
1563 == ("DescribeTransactions", code, want_name, None)
1564 );
1565 }
1566 other => panic!("expected AdminError::Broker, got {other:?}"),
1567 }
1568 }
1569 }
1570
1571 #[test]
1572 fn description_rejects_a_response_that_omits_the_requested_id() {
1573 let response = DescribeTransactionsResponse {
1574 transaction_states: vec![state_row("other", 0)],
1575 ..Default::default()
1576 };
1577
1578 let error = transaction_description("payments", 3, response).expect_err("missing row");
1579 assert2::assert!(matches!(error, AdminError::Protocol(_)));
1580 }
1581
1582 #[derive(Clone, Copy, Debug)]
1584 enum TransactionCall {
1585 Describe,
1586 ForceTerminate,
1587 }
1588
1589 #[derive(Default)]
1592 struct CoordinatorScript {
1593 find_coordinator: Vec<i16>,
1594 coordinator: Vec<i16>,
1595 find_coordinator_requests: usize,
1596 coordinator_requests: usize,
1597 unreachable_answers: usize,
1600 unreachable: Option<std::net::SocketAddr>,
1601 silent: bool,
1603 closed_find_coordinators: usize,
1606 }
1607
1608 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
1610 enum CoordinatorBehavior {
1611 Normal,
1613 UnreachableOnce,
1616 Silent,
1618 FindCoordinatorClosedOnce,
1621 }
1622
1623 async fn refused_address() -> std::net::SocketAddr {
1625 let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
1626 .await
1627 .expect("bind a port");
1628 listener.local_addr().expect("local address")
1629 }
1630
1631 fn next_code(codes: &[i16], requests: &mut usize) -> i16 {
1632 let code = codes[(*requests).min(codes.len() - 1)];
1633 *requests += 1;
1634 code
1635 }
1636
1637 fn retry_api_versions() -> Vec<u8> {
1639 let api_version = |api_key, min_version, max_version| ApiVersion {
1640 api_key,
1641 min_version,
1642 max_version,
1643 ..Default::default()
1644 };
1645 encode_response(
1646 &ApiVersionsResponse {
1647 api_keys: vec![
1648 api_version(api_versions_request::API_KEY, 0, 0),
1649 api_version(find_coordinator_request::API_KEY, 0, 0),
1650 api_version(metadata_request::API_KEY, 13, 13),
1651 api_version(init_producer_id_request::API_KEY, 0, 0),
1652 api_version(describe_transactions_request::API_KEY, 0, 0),
1653 ],
1654 ..Default::default()
1655 },
1656 0,
1657 false,
1658 )
1659 }
1660
1661 async fn scripted_transaction_broker(
1664 script: Arc<Mutex<CoordinatorScript>>,
1665 coordinator: Arc<Mutex<Option<std::net::SocketAddr>>>,
1666 ) -> MockBroker {
1667 MockBroker::start_with_replies(move |api_key, version, _, _| {
1668 let mut script = script.lock().expect("script lock");
1669 let script = &mut *script;
1670 if api_key == find_coordinator_request::API_KEY && script.closed_find_coordinators > 0 {
1671 script.closed_find_coordinators -= 1;
1672 script.find_coordinator_requests += 1;
1673 return MockReply::Close;
1674 }
1675 let reply = match api_key {
1676 api_versions_request::API_KEY => Some(retry_api_versions()),
1677 find_coordinator_request::API_KEY => {
1678 let error_code = next_code(
1679 &script.find_coordinator,
1680 &mut script.find_coordinator_requests,
1681 );
1682 let addr = match script.unreachable {
1683 Some(unreachable)
1684 if script.find_coordinator_requests <= script.unreachable_answers =>
1685 {
1686 unreachable
1687 }
1688 _ => coordinator
1689 .lock()
1690 .expect("coordinator lock")
1691 .expect("coordinator address"),
1692 };
1693 Some(encode_response(
1694 &FindCoordinatorResponse {
1695 error_code,
1696 node_id: 2,
1697 host: addr.ip().to_string(),
1698 port: i32::from(addr.port()),
1699 ..Default::default()
1700 },
1701 version,
1702 false,
1703 ))
1704 }
1705 describe_transactions_request::API_KEY => {
1706 let error_code =
1707 next_code(&script.coordinator, &mut script.coordinator_requests);
1708 if script.silent {
1709 return MockReply::Silent;
1710 }
1711 Some(encode_response(
1712 &DescribeTransactionsResponse {
1713 transaction_states: vec![TransactionState {
1714 error_code,
1715 transactional_id: "payments".to_owned(),
1716 transaction_state: "Ongoing".to_owned(),
1717 producer_id: 4242,
1718 producer_epoch: 7,
1719 ..Default::default()
1720 }],
1721 ..Default::default()
1722 },
1723 version,
1724 true,
1725 ))
1726 }
1727 init_producer_id_request::API_KEY => {
1728 let error_code =
1729 next_code(&script.coordinator, &mut script.coordinator_requests);
1730 if script.silent {
1731 return MockReply::Silent;
1732 }
1733 Some(encode_response(
1734 &InitProducerIdResponse {
1735 error_code,
1736 producer_id: 4242,
1737 producer_epoch: 8,
1738 ..Default::default()
1739 },
1740 version,
1741 false,
1742 ))
1743 }
1744 _ => None,
1745 };
1746 reply.map_or(MockReply::Silent, MockReply::Respond)
1747 })
1748 .await
1749 }
1750
1751 const TIMED_OUT: i16 = 7;
1754 const LONG: Duration = Duration::from_secs(5);
1756 const NOW: Duration = Duration::from_millis(500);
1759 const SHORT: Duration = Duration::from_millis(300);
1761
1762 type TransactionOutcome = Result<(), (&'static str, i16)>;
1763
1764 type TransactionCase = (
1765 &'static str,
1766 TransactionCall,
1767 Vec<i16>,
1768 Vec<i16>,
1769 (Duration, CoordinatorBehavior),
1770 (TransactionOutcome, usize, usize),
1771 );
1772
1773 const OK: TransactionOutcome = Ok(());
1774
1775 fn failed(api: &'static str, code: i16) -> TransactionOutcome {
1776 Err((api, code))
1777 }
1778
1779 async fn run_transaction_case(case: TransactionCase) {
1782 use TransactionCall::{Describe, ForceTerminate};
1783
1784 let (name, call, find_coordinator, coordinator, (timeout, behavior), expected) = case;
1785 let backoff = if timeout == LONG {
1786 Duration::from_millis(1)
1787 } else {
1788 timeout
1789 };
1790 let script = Arc::new(Mutex::new(CoordinatorScript {
1791 find_coordinator,
1792 coordinator,
1793 unreachable_answers: usize::from(behavior == CoordinatorBehavior::UnreachableOnce),
1794 unreachable: Some(refused_address().await),
1795 silent: behavior == CoordinatorBehavior::Silent,
1796 closed_find_coordinators: usize::from(
1797 behavior == CoordinatorBehavior::FindCoordinatorClosedOnce,
1798 ),
1799 ..CoordinatorScript::default()
1800 }));
1801 let coordinator_addr = Arc::new(Mutex::new(None));
1802 let coordinator =
1803 scripted_transaction_broker(Arc::clone(&script), Arc::clone(&coordinator_addr)).await;
1804 *coordinator_addr.lock().expect("coordinator lock") = Some(coordinator.addr);
1805 let bootstrap =
1806 scripted_transaction_broker(Arc::clone(&script), Arc::clone(&coordinator_addr)).await;
1807 let admin = AdminClient::connect(&[bootstrap.addr.to_string()])
1808 .await
1809 .expect("admin connects");
1810 let retry = RetryPolicy {
1811 timeout,
1812 initial_backoff: backoff,
1813 max_backoff: backoff,
1814 jitter: 0.0,
1815 max_retries: u32::MAX,
1816 };
1817
1818 let started = tokio::time::Instant::now();
1819 let result = match call {
1820 Describe => admin
1821 .describe_transaction_with_retry("payments", retry)
1822 .await
1823 .map(|_| ()),
1824 ForceTerminate => {
1825 admin
1826 .force_terminate_transaction_with_retry("payments", retry)
1827 .await
1828 }
1829 }
1830 .map_err(|error| match error {
1831 AdminError::Broker { api, code, .. } => (api, code),
1832 AdminError::Transport(ClientError::Timeout(_)) => ("timeout", TIMED_OUT),
1833 other => panic!("case {name}: unexpected error {other:?}"),
1834 });
1835
1836 bootstrap.stop();
1837 coordinator.stop();
1838 let script = script.lock().expect("script lock");
1839 let within_deadline = started.elapsed() < timeout + Duration::from_secs(2);
1841 assert2::assert!(
1842 (
1843 result,
1844 script.find_coordinator_requests,
1845 script.coordinator_requests,
1846 within_deadline
1847 ) == (expected.0, expected.1, expected.2, true),
1848 "case {name}"
1849 );
1850 }
1851
1852 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1862 async fn transaction_calls_retry_coordinator_errors_as_kafka_does() {
1863 use TransactionCall::{Describe, ForceTerminate};
1864
1865 for case in [
1866 (
1867 "describe: no error",
1868 Describe,
1869 vec![0],
1870 vec![0],
1871 (LONG, CoordinatorBehavior::Normal),
1872 (OK, 1, 1),
1873 ),
1874 (
1875 "describe: coordinator load in progress retries on the same coordinator",
1876 Describe,
1877 vec![0],
1878 vec![14, 14, 0],
1879 (LONG, CoordinatorBehavior::Normal),
1880 (OK, 1, 3),
1881 ),
1882 (
1883 "describe: not coordinator finds the coordinator again",
1884 Describe,
1885 vec![0],
1886 vec![16, 0],
1887 (LONG, CoordinatorBehavior::Normal),
1888 (OK, 2, 2),
1889 ),
1890 (
1891 "describe: coordinator not available finds the coordinator again",
1892 Describe,
1893 vec![0],
1894 vec![15, 0],
1895 (LONG, CoordinatorBehavior::Normal),
1896 (OK, 2, 2),
1897 ),
1898 (
1899 "describe: concurrent transactions is final",
1900 Describe,
1901 vec![0],
1902 vec![51],
1903 (LONG, CoordinatorBehavior::Normal),
1904 (failed("DescribeTransactions", 51), 1, 1),
1905 ),
1906 (
1907 "describe: transactional id not found is final",
1908 Describe,
1909 vec![0],
1910 vec![105],
1911 (LONG, CoordinatorBehavior::Normal),
1912 (failed("DescribeTransactions", 105), 1, 1),
1913 ),
1914 (
1915 "describe: coordinator load in progress past the timeout times out",
1916 Describe,
1917 vec![0],
1918 vec![14],
1919 (NOW, CoordinatorBehavior::Normal),
1920 (failed("timeout", TIMED_OUT), 1, 1),
1921 ),
1922 (
1923 "describe: find coordinator not available, then the coordinator",
1924 Describe,
1925 vec![15, 0],
1926 vec![0],
1927 (LONG, CoordinatorBehavior::Normal),
1928 (OK, 2, 1),
1929 ),
1930 (
1931 "describe: find coordinator authorization failed is final",
1932 Describe,
1933 vec![53],
1934 vec![0],
1935 (LONG, CoordinatorBehavior::Normal),
1936 (failed("FindCoordinator", 53), 1, 0),
1937 ),
1938 (
1939 "fence: no error",
1940 ForceTerminate,
1941 vec![0],
1942 vec![0],
1943 (LONG, CoordinatorBehavior::Normal),
1944 (OK, 1, 1),
1945 ),
1946 (
1947 "fence: coordinator load in progress retries on the same coordinator",
1948 ForceTerminate,
1949 vec![0],
1950 vec![14, 0],
1951 (LONG, CoordinatorBehavior::Normal),
1952 (OK, 1, 2),
1953 ),
1954 (
1955 "fence: concurrent transactions retries on the same coordinator",
1956 ForceTerminate,
1957 vec![0],
1958 vec![51, 51, 0],
1959 (LONG, CoordinatorBehavior::Normal),
1960 (OK, 1, 3),
1961 ),
1962 (
1963 "fence: not coordinator finds the coordinator again",
1964 ForceTerminate,
1965 vec![0],
1966 vec![16, 0],
1967 (LONG, CoordinatorBehavior::Normal),
1968 (OK, 2, 2),
1969 ),
1970 (
1971 "fence: cluster authorization failed is final",
1972 ForceTerminate,
1973 vec![0],
1974 vec![31],
1975 (LONG, CoordinatorBehavior::Normal),
1976 (failed("InitProducerId", 31), 1, 1),
1977 ),
1978 (
1979 "fence: concurrent transactions past the timeout times out",
1980 ForceTerminate,
1981 vec![0],
1982 vec![51],
1983 (NOW, CoordinatorBehavior::Normal),
1984 (failed("timeout", TIMED_OUT), 1, 1),
1985 ),
1986 (
1987 "fence: find coordinator load in progress, then the coordinator",
1988 ForceTerminate,
1989 vec![14, 0],
1990 vec![0],
1991 (LONG, CoordinatorBehavior::Normal),
1992 (OK, 2, 1),
1993 ),
1994 ] {
1995 run_transaction_case(case).await;
1996 }
1997 }
1998
1999 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2003 async fn transaction_calls_find_the_coordinator_again_after_connection_failures() {
2004 use TransactionCall::{Describe, ForceTerminate};
2005
2006 for case in [
2007 (
2008 "describe: a closed connection during FindCoordinator recovers",
2009 Describe,
2010 vec![0],
2011 vec![0],
2012 (LONG, CoordinatorBehavior::FindCoordinatorClosedOnce),
2013 (OK, 2, 1),
2014 ),
2015 (
2016 "fence: a closed connection during FindCoordinator recovers",
2017 ForceTerminate,
2018 vec![0],
2019 vec![0],
2020 (LONG, CoordinatorBehavior::FindCoordinatorClosedOnce),
2021 (OK, 2, 1),
2022 ),
2023 (
2024 "describe: an unreachable coordinator finds the coordinator again",
2025 Describe,
2026 vec![0],
2027 vec![0],
2028 (LONG, CoordinatorBehavior::UnreachableOnce),
2029 (OK, 2, 1),
2030 ),
2031 (
2032 "describe: a silent coordinator stops the attempt at the deadline",
2033 Describe,
2034 vec![0],
2035 vec![0],
2036 (SHORT, CoordinatorBehavior::Silent),
2037 (failed("timeout", TIMED_OUT), 1, 1),
2038 ),
2039 (
2040 "fence: an unreachable coordinator finds the coordinator again",
2041 ForceTerminate,
2042 vec![0],
2043 vec![0],
2044 (LONG, CoordinatorBehavior::UnreachableOnce),
2045 (OK, 2, 1),
2046 ),
2047 (
2048 "fence: a silent coordinator stops the attempt at the deadline",
2049 ForceTerminate,
2050 vec![0],
2051 vec![0],
2052 (SHORT, CoordinatorBehavior::Silent),
2053 (failed("timeout", TIMED_OUT), 1, 1),
2054 ),
2055 ] {
2056 run_transaction_case(case).await;
2057 }
2058 }
2059
2060 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2063 async fn describe_transaction_names_the_coordinator_that_find_coordinator_gave() {
2064 let script = Arc::new(Mutex::new(CoordinatorScript {
2065 find_coordinator: vec![0],
2066 coordinator: vec![0],
2067 ..CoordinatorScript::default()
2068 }));
2069 let coordinator_addr = Arc::new(Mutex::new(None));
2070 let coordinator =
2071 scripted_transaction_broker(Arc::clone(&script), Arc::clone(&coordinator_addr)).await;
2072 *coordinator_addr.lock().expect("coordinator lock") = Some(coordinator.addr);
2073 let bootstrap =
2074 scripted_transaction_broker(Arc::clone(&script), Arc::clone(&coordinator_addr)).await;
2075 let admin = AdminClient::connect(&[bootstrap.addr.to_string()])
2076 .await
2077 .expect("admin connects");
2078
2079 let described = admin.describe_transaction("payments").await;
2080
2081 bootstrap.stop();
2082 coordinator.stop();
2083 assert2::assert!(let Ok(described) = described);
2084 assert2::assert!(
2085 described
2086 == TransactionDescription {
2087 transactional_id: "payments".to_owned(),
2088 coordinator_id: 2,
2089 state: "Ongoing".to_owned(),
2090 producer_id: 4242,
2091 producer_epoch: 7,
2092 timeout: Time::from_millis(0),
2093 start_time_ms: Some(0),
2094 topic_partitions: BTreeSet::new(),
2095 }
2096 );
2097 }
2098
2099 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2102 async fn describe_transactions_shares_one_deadline_across_ids() {
2103 let mut coordinator_codes = vec![14; 8];
2104 coordinator_codes.extend([0, 14]);
2105 let script = Arc::new(Mutex::new(CoordinatorScript {
2106 find_coordinator: vec![0],
2107 coordinator: coordinator_codes,
2108 ..CoordinatorScript::default()
2109 }));
2110 let coordinator_addr = Arc::new(Mutex::new(None));
2111 let coordinator =
2112 scripted_transaction_broker(Arc::clone(&script), Arc::clone(&coordinator_addr)).await;
2113 *coordinator_addr.lock().expect("coordinator lock") = Some(coordinator.addr);
2114 let bootstrap =
2115 scripted_transaction_broker(Arc::clone(&script), Arc::clone(&coordinator_addr)).await;
2116 let admin = AdminClient::connect_with_config(
2117 &[bootstrap.addr.to_string()],
2118 crate::AdminClientConfig {
2119 request_timeout: krabka_units::millis(200),
2120 default_api_timeout: Some(krabka_units::millis(1500)),
2121 retry_backoff: krabka_units::millis(100),
2122 retry_backoff_max: krabka_units::millis(100),
2123 ..AdminClientConfig::default()
2124 },
2125 )
2126 .await
2127 .expect("admin connects");
2128
2129 let started = tokio::time::Instant::now();
2130 let result = admin.describe_transactions(&["payments", "payments"]).await;
2131 let elapsed = started.elapsed();
2132
2133 bootstrap.stop();
2134 coordinator.stop();
2135 assert2::assert!(matches!(
2138 result,
2139 Err(AdminError::Transport(ClientError::Timeout(_)))
2140 ));
2141 assert2::assert!(elapsed < Duration::from_millis(2200), "{elapsed:?}");
2142 }
2143
2144 fn orders_spec() -> AbortTransactionSpec {
2146 AbortTransactionSpec {
2147 topic: "orders".to_owned(),
2148 partition: 1,
2149 producer_id: 4242,
2150 producer_epoch: 7,
2151 coordinator_epoch: 3,
2152 }
2153 }
2154
2155 fn orders_metadata(addr: std::net::SocketAddr, partition_error: i16) -> MetadataResponse {
2159 let partition = |partition_index| MetadataResponsePartition {
2160 error_code: partition_error,
2161 partition_index,
2162 leader_id: 1,
2163 ..Default::default()
2164 };
2165 MetadataResponse {
2166 brokers: vec![MetadataResponseBroker {
2167 node_id: 1,
2168 host: addr.ip().to_string(),
2169 port: i32::from(addr.port()),
2170 ..Default::default()
2171 }],
2172 topics: vec![
2173 MetadataResponseTopic {
2174 name: Some("orders".to_owned()),
2175 partitions: vec![partition(0), partition(1)],
2176 ..Default::default()
2177 },
2178 MetadataResponseTopic {
2179 error_code: 3,
2180 name: Some("missing".to_owned()),
2181 ..Default::default()
2182 },
2183 ],
2184 ..Default::default()
2185 }
2186 }
2187
2188 fn outcome_of(result: Result<(), AdminError>) -> TransactionOutcome {
2191 result.map_err(|error| match error {
2192 AdminError::Broker { api, code, .. } => (api, code),
2193 AdminError::Protocol(_) => ("protocol", 0),
2194 other => panic!("unexpected error {other:?}"),
2195 })
2196 }
2197
2198 #[test]
2199 fn partition_leader_address_resolves_the_leader_or_reports_the_metadata_error() {
2200 let addr: std::net::SocketAddr = "127.0.0.1:9092".parse().expect("address");
2201 let mut leaderless = orders_metadata(addr, 0);
2202 leaderless.topics[0].partitions[1].leader_id = 9;
2203 for (name, metadata, topic, partition, expected) in [
2204 (
2205 "leader",
2206 orders_metadata(addr, 0),
2207 "orders",
2208 1,
2209 Ok("127.0.0.1:9092".to_owned()),
2210 ),
2211 (
2212 "partition error",
2213 orders_metadata(addr, NOT_LEADER_OR_FOLLOWER),
2214 "orders",
2215 1,
2216 Err(("Metadata", NOT_LEADER_OR_FOLLOWER)),
2217 ),
2218 (
2219 "topic error",
2220 orders_metadata(addr, 0),
2221 "missing",
2222 0,
2223 Err(("Metadata", 3)),
2224 ),
2225 (
2226 "unlisted topic",
2227 orders_metadata(addr, 0),
2228 "absent",
2229 0,
2230 Err(("protocol", 0)),
2231 ),
2232 (
2233 "unlisted partition",
2234 orders_metadata(addr, 0),
2235 "orders",
2236 5,
2237 Err(("protocol", 0)),
2238 ),
2239 (
2240 "unlisted leader",
2241 leaderless,
2242 "orders",
2243 1,
2244 Err(("protocol", 0)),
2245 ),
2246 ] {
2247 let address = partition_leader_address(topic, partition, &metadata);
2248 let actual = match address {
2249 Ok(address) => Ok(address),
2250 Err(error) => Err(outcome_of(Err(error)).expect_err("an error")),
2251 };
2252 assert2::assert!(actual == expected, "case {name}");
2253 }
2254 }
2255
2256 #[test]
2257 fn abort_request_writes_one_abort_marker_for_the_partition() {
2258 assert2::assert!(
2259 abort_transaction_request(&orders_spec())
2260 == WriteTxnMarkersRequest {
2261 markers: vec![WritableTxnMarker {
2262 producer_id: 4242,
2263 producer_epoch: 7,
2264 transaction_result: false,
2265 topics: vec![WritableTxnMarkerTopic {
2266 name: "orders".to_owned(),
2267 partition_indexes: vec![1],
2268 ..Default::default()
2269 }],
2270 coordinator_epoch: 3,
2271 ..Default::default()
2272 }],
2273 ..Default::default()
2274 }
2275 );
2276 }
2277
2278 fn markers_response(
2281 producer_id: i64,
2282 topic: &str,
2283 partition: i32,
2284 code: i16,
2285 ) -> WriteTxnMarkersResponse {
2286 WriteTxnMarkersResponse {
2287 markers: vec![WritableTxnMarkerResult {
2288 producer_id,
2289 topics: vec![WritableTxnMarkerTopicResult {
2290 name: topic.to_owned(),
2291 partitions: vec![WritableTxnMarkerPartitionResult {
2292 partition_index: partition,
2293 error_code: code,
2294 ..Default::default()
2295 }],
2296 ..Default::default()
2297 }],
2298 ..Default::default()
2299 }],
2300 ..Default::default()
2301 }
2302 }
2303
2304 #[test]
2305 fn abort_result_maps_the_matching_partition_result() {
2306 for (name, response, expected) in [
2307 ("no error", markers_response(4242, "orders", 1, 0), OK),
2308 (
2309 "partition error",
2310 markers_response(4242, "orders", 1, 53),
2311 failed("WriteTxnMarkers", 53),
2312 ),
2313 (
2314 "other producer",
2315 markers_response(1, "orders", 1, 0),
2316 failed("protocol", 0),
2317 ),
2318 (
2319 "other topic",
2320 markers_response(4242, "payments", 1, 0),
2321 failed("protocol", 0),
2322 ),
2323 (
2324 "other partition",
2325 markers_response(4242, "orders", 0, 0),
2326 failed("protocol", 0),
2327 ),
2328 ] {
2329 let result = outcome_of(abort_transaction_result(&orders_spec(), response));
2330 assert2::assert!(result == expected, "case {name}");
2331 }
2332 }
2333
2334 #[test]
2338 fn abort_retries_only_leadership_errors() {
2339 #[derive(Debug, PartialEq, Eq)]
2340 enum Action {
2341 Done,
2342 FindLeader,
2343 }
2344 for (code, expected) in [
2345 (0, Action::Done),
2346 (LEADER_NOT_AVAILABLE, Action::FindLeader),
2347 (NOT_LEADER_OR_FOLLOWER, Action::FindLeader),
2348 (53, Action::Done),
2349 (47, Action::Done),
2350 ] {
2351 let result =
2352 abort_transaction_result(&orders_spec(), markers_response(4242, "orders", 1, code));
2353 let action = match abort_transaction_retry_action(result) {
2354 RetryAction::Done(_) => Action::Done,
2355 RetryAction::FindCoordinator(_) => Action::FindLeader,
2356 RetryAction::SameCoordinator(_) => panic!("code {code}: same leader"),
2357 };
2358 assert2::assert!(action == expected, "code {code}");
2359 }
2360 }
2361
2362 #[test]
2363 fn describe_producers_request_groups_partitions_by_topic() {
2364 let keys = [
2365 ("orders".to_owned(), 1),
2366 ("audit".to_owned(), 0),
2367 ("orders".to_owned(), 0),
2368 ];
2369
2370 assert2::assert!(
2371 describe_producers_request(&keys)
2372 == DescribeProducersRequest {
2373 topics: vec![
2374 DescribeProducersTopicRequest {
2375 name: "audit".to_owned(),
2376 partition_indexes: vec![0],
2377 ..Default::default()
2378 },
2379 DescribeProducersTopicRequest {
2380 name: "orders".to_owned(),
2381 partition_indexes: vec![1, 0],
2382 ..Default::default()
2383 },
2384 ],
2385 ..Default::default()
2386 }
2387 );
2388 }
2389
2390 fn orders_producers() -> DescribeProducersResponse {
2394 DescribeProducersResponse {
2395 topics: vec![TopicResponse {
2396 name: "orders".to_owned(),
2397 partitions: vec![
2398 PartitionResponse {
2399 partition_index: 0,
2400 active_producers: vec![
2401 WireProducerState {
2402 producer_id: 4242,
2403 producer_epoch: 7,
2404 last_sequence: 12,
2405 last_timestamp: 1_700_000_000_123,
2406 coordinator_epoch: 3,
2407 current_txn_start_offset: 88,
2408 ..Default::default()
2409 },
2410 WireProducerState {
2411 producer_id: 11,
2412 ..Default::default()
2413 },
2414 ],
2415 ..Default::default()
2416 },
2417 PartitionResponse {
2418 partition_index: 1,
2419 error_code: NOT_LEADER_OR_FOLLOWER,
2420 error_message: Some("moved".to_owned()),
2421 ..Default::default()
2422 },
2423 PartitionResponse {
2424 partition_index: 9,
2425 ..Default::default()
2426 },
2427 ],
2428 ..Default::default()
2429 }],
2430 ..Default::default()
2431 }
2432 }
2433
2434 fn orders_producer_results()
2437 -> BTreeMap<(String, i32), Result<Vec<ProducerStateInfo>, KafkaError>> {
2438 maplit::btreemap! {
2439 ("orders".to_owned(), 0) => Ok(vec![
2440 ProducerStateInfo {
2441 producer_id: 4242,
2442 producer_epoch: 7,
2443 last_sequence: 12,
2444 last_timestamp_ms: 1_700_000_000_123,
2445 coordinator_epoch: 3,
2446 current_txn_start_offset: Some(88),
2447 },
2448 ProducerStateInfo {
2449 producer_id: 11,
2450 producer_epoch: 0,
2451 last_sequence: -1,
2452 last_timestamp_ms: -1,
2453 coordinator_epoch: 0,
2454 current_txn_start_offset: None,
2455 },
2456 ]),
2457 ("orders".to_owned(), 1) => Err(KafkaError {
2458 code: NOT_LEADER_OR_FOLLOWER,
2459 name: "NOT_LEADER_OR_FOLLOWER",
2460 message: Some("moved".to_owned()),
2461 }),
2462 }
2463 }
2464
2465 #[test]
2466 fn describe_producers_results_map_each_requested_partition() {
2467 let keys = [
2468 ("orders".to_owned(), 0),
2469 ("orders".to_owned(), 1),
2470 ("orders".to_owned(), 2),
2471 ];
2472
2473 let mut expected = orders_producer_results();
2474 expected.insert(
2475 ("orders".to_owned(), 2),
2476 Err(KafkaError {
2477 code: -1,
2478 name: "UNKNOWN_SERVER_ERROR",
2479 message: Some(
2480 "the DescribeProducers response did not contain a result for partition \
2481 orders-2"
2482 .to_owned(),
2483 ),
2484 }),
2485 );
2486 assert2::assert!(describe_producers_results(&keys, orders_producers()) == expected);
2487 }
2488
2489 #[test]
2490 fn list_transactions_request_carries_the_filter() {
2491 for (name, filter, expected) in [
2492 (
2493 "default matches everything",
2494 ListTransactionsFilter::builder().build(),
2495 ListTransactionsRequest::default(),
2496 ),
2497 (
2498 "every filter",
2499 ListTransactionsFilter::builder()
2500 .state_filters(vec!["Ongoing".to_owned()])
2501 .producer_id_filters(vec![4242])
2502 .min_duration(90_000)
2503 .build(),
2504 ListTransactionsRequest {
2505 state_filters: vec!["Ongoing".to_owned()],
2506 producer_id_filters: vec![4242],
2507 duration_filter: 90_000,
2508 ..Default::default()
2509 },
2510 ),
2511 ] {
2512 assert2::assert!(
2513 list_transactions_request(&filter) == expected,
2514 "case {name}"
2515 );
2516 }
2517 }
2518
2519 fn decode_request<T: for<'a> Decode<'a>>(mut body: &[u8], version: i16, flexible: bool) -> T {
2522 let client_id_len = body.get_i16();
2523 body.advance(usize::try_from(client_id_len).expect("client id length"));
2524 if flexible {
2525 body.advance(1);
2526 }
2527 T::decode(&mut body, version).expect("request decodes")
2528 }
2529
2530 fn leader_api_versions() -> Vec<u8> {
2532 let api_version = |api_key, min_version, max_version| ApiVersion {
2533 api_key,
2534 min_version,
2535 max_version,
2536 ..Default::default()
2537 };
2538 encode_response(
2539 &ApiVersionsResponse {
2540 api_keys: vec![
2541 api_version(api_versions_request::API_KEY, 0, 0),
2542 api_version(metadata_request::API_KEY, 13, 13),
2543 api_version(write_txn_markers_request::API_KEY, 1, 2),
2544 api_version(list_transactions_request::API_KEY, 0, 2),
2545 ],
2546 ..Default::default()
2547 },
2548 0,
2549 false,
2550 )
2551 }
2552
2553 #[derive(Default)]
2555 struct LeaderScript {
2556 write_codes: Vec<i16>,
2559 write_requests: Vec<WriteTxnMarkersRequest>,
2561 metadata_requests: usize,
2563 list_requests: Vec<ListTransactionsRequest>,
2565 listing: ListTransactionsResponse,
2567 }
2568
2569 async fn leader_broker(
2572 script: Arc<Mutex<LeaderScript>>,
2573 cluster: Arc<Mutex<Vec<std::net::SocketAddr>>>,
2574 ) -> MockBroker {
2575 MockBroker::start(move |api_key, version, _, body| {
2576 let mut script = script.lock().expect("script lock");
2577 match api_key {
2578 api_versions_request::API_KEY => Some(leader_api_versions()),
2579 metadata_request::API_KEY => {
2580 script.metadata_requests += 1;
2581 let cluster = cluster.lock().expect("cluster lock").clone();
2582 let mut metadata = orders_metadata(cluster[0], 0);
2583 metadata.brokers = cluster
2584 .iter()
2585 .zip(1..)
2586 .map(|(addr, node_id)| MetadataResponseBroker {
2587 node_id,
2588 host: addr.ip().to_string(),
2589 port: i32::from(addr.port()),
2590 ..Default::default()
2591 })
2592 .collect();
2593 Some(encode_response(&metadata, version, true))
2594 }
2595 write_txn_markers_request::API_KEY => {
2596 let request: WriteTxnMarkersRequest = decode_request(body, version, true);
2597 let code = script.write_codes[script
2598 .write_requests
2599 .len()
2600 .min(script.write_codes.len() - 1)];
2601 script.write_requests.push(request);
2602 Some(encode_response(
2603 &markers_response(4242, "orders", 1, code),
2604 version,
2605 true,
2606 ))
2607 }
2608 list_transactions_request::API_KEY => {
2609 script
2610 .list_requests
2611 .push(decode_request(body, version, true));
2612 Some(encode_response(&script.listing, version, true))
2613 }
2614 _ => None,
2615 }
2616 })
2617 .await
2618 }
2619
2620 async fn fast_admin(bootstrap: std::net::SocketAddr) -> AdminClient {
2622 AdminClient::connect_with_config(
2623 &[bootstrap.to_string()],
2624 crate::AdminClientConfig {
2625 request_timeout: krabka_units::secs(2),
2626 retry_backoff: krabka_units::millis(1),
2627 retry_backoff_max: krabka_units::millis(1),
2628 default_api_timeout: Some(krabka_units::secs(5)),
2629 ..AdminClientConfig::default()
2630 },
2631 )
2632 .await
2633 .expect("admin connects")
2634 }
2635
2636 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2640 async fn abort_transaction_writes_an_abort_marker_to_the_leader() {
2641 for (name, write_codes, expected) in [
2642 ("no error", vec![0], (OK, 1, 1)),
2643 (
2644 "not leader or follower finds the leader again",
2645 vec![NOT_LEADER_OR_FOLLOWER, 0],
2646 (OK, 2, 2),
2647 ),
2648 (
2649 "transactional id authorization failed is final",
2650 vec![53],
2651 (failed("WriteTxnMarkers", 53), 1, 1),
2652 ),
2653 ] {
2654 let script = Arc::new(Mutex::new(LeaderScript {
2655 write_codes,
2656 ..LeaderScript::default()
2657 }));
2658 let cluster = Arc::new(Mutex::new(Vec::new()));
2659 let broker = leader_broker(Arc::clone(&script), Arc::clone(&cluster)).await;
2660 cluster.lock().expect("cluster lock").push(broker.addr);
2661 let admin = fast_admin(broker.addr).await;
2662
2663 let result = outcome_of(admin.abort_transaction(&orders_spec()).await);
2664
2665 broker.stop();
2666 let script = script.lock().expect("script lock");
2667 let expected_requests = vec![abort_transaction_request(&orders_spec()); expected.2];
2668 assert2::assert!(
2669 (
2670 result,
2671 script.metadata_requests,
2672 script.write_requests.clone()
2673 ) == (expected.0, expected.1, expected_requests),
2674 "case {name}"
2675 );
2676 }
2677 }
2678
2679 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2680 async fn abort_transaction_rejects_an_empty_topic() {
2681 let script = Arc::new(Mutex::new(LeaderScript::default()));
2682 let cluster = Arc::new(Mutex::new(Vec::new()));
2683 let broker = leader_broker(Arc::clone(&script), Arc::clone(&cluster)).await;
2684 cluster.lock().expect("cluster lock").push(broker.addr);
2685 let admin = fast_admin(broker.addr).await;
2686
2687 let result = admin
2688 .abort_transaction(&AbortTransactionSpec {
2689 topic: String::new(),
2690 ..orders_spec()
2691 })
2692 .await;
2693
2694 broker.stop();
2695 assert2::assert!(outcome_of(result) == failed("protocol", 0));
2696 assert2::assert!(script.lock().expect("script lock").metadata_requests == 0);
2697 }
2698
2699 fn producers_answer(
2702 request: &DescribeProducersRequest,
2703 orders_1_code: i16,
2704 ) -> DescribeProducersResponse {
2705 let orders_0 = orders_producers().topics[0].partitions[0].clone();
2706 DescribeProducersResponse {
2707 topics: request
2708 .topics
2709 .iter()
2710 .map(|topic| TopicResponse {
2711 name: topic.name.clone(),
2712 partitions: topic
2713 .partition_indexes
2714 .iter()
2715 .map(|&partition_index| {
2716 if partition_index == 0 {
2717 orders_0.clone()
2718 } else {
2719 PartitionResponse {
2720 partition_index,
2721 error_code: orders_1_code,
2722 error_message: (orders_1_code != 0)
2723 .then(|| "refused".to_owned()),
2724 ..Default::default()
2725 }
2726 }
2727 })
2728 .collect(),
2729 ..Default::default()
2730 })
2731 .collect(),
2732 ..Default::default()
2733 }
2734 }
2735
2736 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2741 async fn describe_producers_finds_the_leader_again_as_kafka_does() {
2742 let both = vec![("orders".to_owned(), 0), ("orders".to_owned(), 1)];
2743 let only_1 = vec![("orders".to_owned(), 1)];
2744 let refused = refused_address().await;
2745 let producers_0 = orders_producer_results()[&both[0]].clone();
2746 for (name, leaders, orders_1_codes, expected) in [
2747 (
2748 "success",
2749 vec![None],
2750 vec![0],
2751 (Ok(Vec::new()), 1, vec![both.clone()]),
2752 ),
2753 (
2754 "not leader or follower finds the leader again",
2755 vec![None],
2756 vec![NOT_LEADER_OR_FOLLOWER, 0],
2757 (Ok(Vec::new()), 2, vec![both.clone(), only_1.clone()]),
2758 ),
2759 (
2760 "a refused connection finds the leader again",
2761 vec![Some(refused), None],
2762 vec![0],
2763 (Ok(Vec::new()), 2, vec![both.clone()]),
2764 ),
2765 (
2766 "topic authorization failed is final",
2767 vec![None],
2768 vec![29],
2769 (
2770 Err(KafkaError {
2771 code: 29,
2772 name: "TOPIC_AUTHORIZATION_FAILED",
2773 message: Some("refused".to_owned()),
2774 }),
2775 1,
2776 vec![both.clone()],
2777 ),
2778 ),
2779 ] {
2780 let seen = Arc::new(Mutex::new(Seen::default()));
2781 let broker = partition_leader_broker(
2782 vec![
2783 ApiVersionRange::builder()
2784 .api_key(describe_producers_request::API_KEY)
2785 .build(),
2786 ],
2787 describe_producers_request::API_KEY,
2788 leaders,
2789 Arc::clone(&seen),
2790 move |n, version, body| {
2791 let request: DescribeProducersRequest = decode_request(body, version, true);
2792 let code = orders_1_codes[n.min(orders_1_codes.len() - 1)];
2793 encode_response(&producers_answer(&request, code), version, true)
2794 },
2795 )
2796 .await;
2797 let admin = leader_fast_admin(broker.addr, krabka_units::secs(5)).await;
2798
2799 let result = admin.describe_producers(&both).await;
2800
2801 broker.stop();
2802 let seen = seen.lock().expect("seen lock");
2803 let requests = seen
2804 .requests
2805 .iter()
2806 .map(|(version, body)| {
2807 decode_request::<DescribeProducersRequest>(body, *version, true)
2808 })
2809 .collect::<Vec<_>>();
2810 assert2::assert!(
2811 (result, seen.metadata_requests, requests)
2812 == (
2813 maplit::btreemap! {
2814 both[0].clone() => producers_0.clone(),
2815 both[1].clone() => expected.0,
2816 },
2817 expected.1,
2818 expected
2819 .2
2820 .iter()
2821 .map(|keys| describe_producers_request(keys))
2822 .collect::<Vec<_>>(),
2823 ),
2824 "case {name}"
2825 );
2826 }
2827 }
2828
2829 fn listing(error_code: i16, transactional_ids: &[&str]) -> ListTransactionsResponse {
2832 ListTransactionsResponse {
2833 error_code,
2834 transaction_states: transactional_ids
2835 .iter()
2836 .zip(1..)
2837 .map(|(transactional_id, producer_id)| ListedTransactionState {
2838 transactional_id: (*transactional_id).to_owned(),
2839 producer_id,
2840 transaction_state: "Ongoing".to_owned(),
2841 ..Default::default()
2842 })
2843 .collect(),
2844 ..Default::default()
2845 }
2846 }
2847
2848 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2852 async fn list_transactions_merges_the_listing_of_every_broker() {
2853 let listed = |transactional_id: &str, producer_id| TransactionListing {
2854 transactional_id: transactional_id.to_owned(),
2855 producer_id,
2856 state: "Ongoing".to_owned(),
2857 };
2858 for (name, second_listing, expected) in [
2859 (
2860 "every broker answers",
2861 listing(0, &["audit"]),
2862 Ok(vec![
2863 listed("payments", 1),
2864 listed("orders", 2),
2865 listed("audit", 1),
2866 ]),
2867 ),
2868 (
2869 "a broker is loading its coordinator",
2870 listing(COORDINATOR_LOAD_IN_PROGRESS, &[]),
2871 Err(("ListTransactions", COORDINATOR_LOAD_IN_PROGRESS)),
2872 ),
2873 ] {
2874 let first = Arc::new(Mutex::new(LeaderScript {
2875 listing: listing(0, &["payments", "orders"]),
2876 ..LeaderScript::default()
2877 }));
2878 let second = Arc::new(Mutex::new(LeaderScript {
2879 listing: second_listing,
2880 ..LeaderScript::default()
2881 }));
2882 let cluster = Arc::new(Mutex::new(Vec::new()));
2883 let first_broker = leader_broker(Arc::clone(&first), Arc::clone(&cluster)).await;
2884 let second_broker = leader_broker(Arc::clone(&second), Arc::clone(&cluster)).await;
2885 cluster
2886 .lock()
2887 .expect("cluster lock")
2888 .extend([first_broker.addr, second_broker.addr]);
2889 let admin = fast_admin(first_broker.addr).await;
2890 let filter = ListTransactionsFilter::builder()
2891 .state_filters(vec!["Ongoing".to_owned()])
2892 .build();
2893
2894 let result = admin
2895 .list_transactions(&filter)
2896 .await
2897 .map_err(|error| match error {
2898 AdminError::Broker { api, code, .. } => (api, code),
2899 other => panic!("case {name}: unexpected error {other:?}"),
2900 });
2901
2902 first_broker.stop();
2903 second_broker.stop();
2904 let requests = [&first, &second]
2905 .map(|script| script.lock().expect("script lock").list_requests.clone());
2906 let expected_request = list_transactions_request(&filter);
2907 assert2::assert!(
2908 (result, requests)
2909 == (
2910 expected,
2911 [vec![expected_request.clone()], vec![expected_request]]
2912 ),
2913 "case {name}"
2914 );
2915 }
2916 }
2917
2918 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2921 async fn fence_producers_gives_each_transactional_id_its_own_result() {
2922 let script = Arc::new(Mutex::new(CoordinatorScript {
2923 find_coordinator: vec![0],
2924 coordinator: vec![0],
2925 ..CoordinatorScript::default()
2926 }));
2927 let coordinator_addr = Arc::new(Mutex::new(None));
2928 let coordinator =
2929 scripted_transaction_broker(Arc::clone(&script), Arc::clone(&coordinator_addr)).await;
2930 *coordinator_addr.lock().expect("coordinator lock") = Some(coordinator.addr);
2931 let bootstrap =
2932 scripted_transaction_broker(Arc::clone(&script), Arc::clone(&coordinator_addr)).await;
2933 let admin = AdminClient::connect(&[bootstrap.addr.to_string()])
2934 .await
2935 .expect("admin connects");
2936
2937 let results = admin.fence_producers(&["payments", "", "orders"]).await;
2938
2939 bootstrap.stop();
2940 coordinator.stop();
2941 let results = results
2942 .into_iter()
2943 .map(|(transactional_id, result)| (transactional_id, outcome_of(result)))
2944 .collect::<BTreeMap<_, _>>();
2945 let script = script.lock().expect("script lock");
2946 assert2::assert!(
2947 (
2948 results,
2949 script.find_coordinator_requests,
2950 script.coordinator_requests
2951 ) == (
2952 maplit::btreemap! {
2953 String::new() => failed("protocol", 0),
2954 "orders".to_owned() => OK,
2955 "payments".to_owned() => OK,
2956 },
2957 2,
2958 2
2959 )
2960 );
2961 }
2962
2963 fn assert_send<T: Send>(_: T) {}
2964
2965 #[test]
2968 fn retrying_admin_call_futures_are_send() {
2969 let _ = |admin: &mut AdminClient,
2970 offsets: &std::collections::BTreeMap<(String, i32), i64>| {
2971 assert_send(admin.alter_consumer_group_offsets("workers", offsets));
2972 assert_send(admin.list_consumer_group_offsets("workers"));
2973 assert_send(admin.describe_transaction("payments"));
2974 assert_send(admin.force_terminate_transaction("payments"));
2975 assert_send(admin.fence_producers(&["payments"]));
2976 assert_send(admin.abort_transaction(&orders_spec()));
2977 assert_send(admin.describe_producers(&[("orders".to_owned(), 0)]));
2978 assert_send(admin.list_transactions(&ListTransactionsFilter::builder().build()));
2979 };
2980 }
2981}