Skip to main content

krabka_client_admin/
transactions.rs

1//! Transaction administration.
2
3use std::collections::{BTreeMap, BTreeSet};
4
5use bytes::BufMut;
6use krabka_client_core::{
7    Connection, ConnectionOptions, CoordinatorKeyType, build_find_coordinator,
8};
9use krabka_protocol::{
10    Encode, ProtocolError, ProtocolRequest,
11    owned::{
12        describe_producers_request::{
13            DescribeProducersRequest, TopicRequest as DescribeProducersTopicRequest,
14        },
15        describe_producers_response::{
16            DescribeProducersResponse, ProducerState as WireProducerState,
17        },
18        describe_transactions_request::DescribeTransactionsRequest,
19        describe_transactions_response::{DescribeTransactionsResponse, TransactionState},
20        find_coordinator_response::FindCoordinatorResponse,
21        init_producer_id_request::{self, InitProducerIdRequest},
22        init_producer_id_response::InitProducerIdResponse,
23        list_transactions_request::ListTransactionsRequest,
24        list_transactions_response::{
25            ListTransactionsResponse, TransactionState as ListedTransactionState,
26        },
27        metadata_request::{MetadataRequest, MetadataRequestTopic},
28        metadata_response::MetadataResponse,
29        write_txn_markers_request::{
30            WritableTxnMarker, WritableTxnMarkerTopic, WriteTxnMarkersRequest,
31        },
32        write_txn_markers_response::WriteTxnMarkersResponse,
33    },
34    primitives::uuid::Uuid as ProtoUuid,
35};
36use krabka_units::{Time, convert::TimeExt as _};
37
38#[cfg(test)]
39use crate::AdminClientConfig;
40use crate::{
41    AdminClient, AdminError, KafkaError, format_host_port, kafka_error_name,
42    partition_leaders::{PartitionKey, PartitionResults, complete_results},
43    retry::{CoordinatorRetry, RetryAction, RetryDeadline, RetryPolicy, connection_failure_action},
44};
45
46/// `COORDINATOR_LOAD_IN_PROGRESS`: the coordinator is loading its state.
47const COORDINATOR_LOAD_IN_PROGRESS: i16 = 14;
48/// `COORDINATOR_NOT_AVAILABLE`: no broker coordinates the transactional ID now.
49const COORDINATOR_NOT_AVAILABLE: i16 = 15;
50/// `NOT_COORDINATOR`: the broker does not coordinate the transactional ID.
51const NOT_COORDINATOR: i16 = 16;
52/// `CONCURRENT_TRANSACTIONS`: the coordinator is completing a transaction.
53const CONCURRENT_TRANSACTIONS: i16 = 51;
54/// `LEADER_NOT_AVAILABLE`: the metadata has no leader for a partition yet.
55const LEADER_NOT_AVAILABLE: i16 = 5;
56/// `NOT_LEADER_OR_FOLLOWER`: the broker is no longer (or not yet) the leader
57/// of a partition.
58const NOT_LEADER_OR_FOLLOWER: i16 = 6;
59
60/// Map the result of one `DescribeTransactions` attempt to a retry action, as
61/// Kafka's `DescribeTransactionsHandler.handleError` does. 14 retries on the
62/// same coordinator. 15 and 16 unmap the transactional ID, so the next attempt
63/// finds the coordinator again. Every other result is final.
64fn describe_retry_action(
65    result: Result<TransactionDescription, AdminError>,
66) -> RetryAction<TransactionDescription> {
67    match result {
68        Err(AdminError::Broker {
69            code: COORDINATOR_LOAD_IN_PROGRESS,
70            ..
71        }) => RetryAction::SameCoordinator(result),
72        Err(AdminError::Broker {
73            code: COORDINATOR_NOT_AVAILABLE | NOT_COORDINATOR,
74            ..
75        }) => RetryAction::FindCoordinator(result),
76        result => RetryAction::Done(result),
77    }
78}
79
80/// Map the error code of one `InitProducerId` answer to a retry action, as
81/// Kafka's `FenceProducersHandler.handleError` does. 14 and 51 retry on the
82/// same coordinator. 15 and 16 unmap the transactional ID, so the next attempt
83/// finds the coordinator again. Every other code is final.
84fn fence_retry_action(error_code: i16) -> RetryAction<()> {
85    if error_code == 0 {
86        return RetryAction::Done(Ok(()));
87    }
88    let result = Err(AdminError::Broker {
89        api: "InitProducerId",
90        code: error_code,
91        name: kafka_error_name(error_code),
92        message: None,
93    });
94    match error_code {
95        COORDINATOR_LOAD_IN_PROGRESS | CONCURRENT_TRANSACTIONS => {
96            RetryAction::SameCoordinator(result)
97        }
98        COORDINATOR_NOT_AVAILABLE | NOT_COORDINATOR => RetryAction::FindCoordinator(result),
99        _ => RetryAction::Done(result),
100    }
101}
102
103/// The transaction coordinator's view of one transactional ID, as Kafka's
104/// `TransactionDescription`.
105///
106/// A third party reads [`producer_id`](Self::producer_id) and
107/// [`producer_epoch`](Self::producer_epoch) to verify the authority of a
108/// writer. The coordinator raises the epoch each time it fences the previous
109/// producer generation, so a writer that presents an older epoch no longer
110/// holds authority over the transactional ID.
111#[derive(Debug, Clone, PartialEq)]
112pub struct TransactionDescription {
113    /// The transactional ID that the caller asked about.
114    pub transactional_id: String,
115    /// The broker ID of the transaction coordinator that answered.
116    pub coordinator_id: i32,
117    /// The coordinator's state name, such as `Ongoing` or `CompleteCommit`.
118    pub state: String,
119    /// The producer ID that the coordinator holds for this transactional ID.
120    pub producer_id: i64,
121    /// The producer epoch that the coordinator holds for this transactional
122    /// ID.
123    pub producer_epoch: i16,
124    /// The transaction timeout that the producer registered.
125    pub timeout: Time,
126    /// The start of the current transaction, in Kafka epoch milliseconds, or
127    /// `None` when no transaction is open. The coordinator reports `-1` for
128    /// none, and Kafka maps any negative value to an empty `OptionalLong`. An
129    /// instant is a coordinate, so it stays a raw integer.
130    pub start_time_ms: Option<i64>,
131    /// The partitions of the current transaction. While the transaction
132    /// prepares to commit or abort, only the partitions that have no marker
133    /// yet.
134    pub topic_partitions: BTreeSet<(String, i32)>,
135}
136
137/// An open connection to the transaction coordinator of one transactional
138/// ID.
139struct TransactionCoordinator {
140    /// The broker ID that `FindCoordinator` named.
141    node_id: i32,
142    connection: Connection,
143}
144
145/// One transaction that `list_transactions` found on a broker, as Kafka's
146/// `TransactionListing`.
147#[derive(Debug, Clone, PartialEq, Eq)]
148pub struct TransactionListing {
149    /// The transactional ID.
150    pub transactional_id: String,
151    /// The producer ID currently bound to the transactional ID.
152    pub producer_id: i64,
153    /// The coordinator's state name, such as `Ongoing` or `CompleteCommit`.
154    pub state: String,
155}
156
157/// Narrows a [`AdminClient::list_transactions`] call, as Kafka's
158/// `ListTransactionsOptions` does. The default matches every transaction, as
159/// Kafka's default options do.
160#[derive(Debug, Clone, Default, PartialEq, bon::Builder)]
161pub struct ListTransactionsFilter {
162    /// Only transactions whose state name is in this list. Empty matches
163    /// every state.
164    #[builder(default)]
165    pub state_filters: Vec<String>,
166    /// Only transactions whose producer ID is in this list. Empty matches
167    /// every producer ID.
168    #[builder(default)]
169    pub producer_id_filters: Vec<i64>,
170    /// Only transactions that have run for at least this long. `None`
171    /// matches every duration, as Kafka's `-1` filter does.
172    #[builder(with = |millis: i64| Time::from_millis(millis))]
173    pub min_duration: Option<Time>,
174}
175
176/// One producer's state on one partition, as Kafka's `ProducerState`
177/// (`DescribeProducers`, KIP-664).
178#[derive(Debug, Clone, PartialEq, Eq)]
179pub struct ProducerStateInfo {
180    /// The producer ID.
181    pub producer_id: i64,
182    /// The producer epoch.
183    pub producer_epoch: i32,
184    /// The last sequence number the producer wrote, or `-1` when it has
185    /// written nothing yet.
186    pub last_sequence: i32,
187    /// The timestamp of the last write, in Kafka epoch milliseconds. An
188    /// instant is a coordinate, so it stays a raw integer.
189    pub last_timestamp_ms: i64,
190    /// The epoch of the coordinator that last fenced this producer.
191    pub coordinator_epoch: i32,
192    /// The start offset of the producer's open transaction on this
193    /// partition, or `None` when it has no open transaction.
194    pub current_txn_start_offset: Option<i64>,
195}
196
197/// The broker ID and the address of the coordinator that a
198/// `FindCoordinator` answer names for `transactional_id`.
199fn coordinator_address(
200    transactional_id: &str,
201    response: FindCoordinatorResponse,
202) -> Result<(i32, String), AdminError> {
203    if let Some(coordinator) = response
204        .coordinators
205        .into_iter()
206        .find(|coordinator| coordinator.key == transactional_id)
207    {
208        if coordinator.error_code != 0 {
209            return Err(AdminError::Broker {
210                api: "FindCoordinator",
211                code: coordinator.error_code,
212                name: kafka_error_name(coordinator.error_code),
213                message: coordinator.error_message,
214            });
215        }
216        return Ok((
217            coordinator.node_id,
218            format!("{}:{}", coordinator.host, coordinator.port),
219        ));
220    }
221
222    if response.error_code != 0 {
223        return Err(AdminError::Broker {
224            api: "FindCoordinator",
225            code: response.error_code,
226            name: kafka_error_name(response.error_code),
227            message: response.error_message,
228        });
229    }
230    if response.host.is_empty() {
231        return Err(AdminError::Protocol(format!(
232            "FindCoordinator returned no entry for transactional id {transactional_id:?}"
233        )));
234    }
235    Ok((
236        response.node_id,
237        format!("{}:{}", response.host, response.port),
238    ))
239}
240
241/// An `InitProducerId` request limited to the released versions, v0 to v5.
242///
243/// `InitProducerIdRequest.json` marks v6 as its unstable latest version
244/// (`"latestVersionUnstable": true`, KIP-939), and krabka-protocol's
245/// generated `MAX_VERSION` includes v6. Apache Kafka's
246/// `FenceProducersHandler.buildSingleRequest` uses
247/// `new InitProducerIdRequest.Builder(data)`. That builder calls
248/// `AbstractRequest.Builder(ApiKeys.INIT_PRODUCER_ID)`, which allows
249/// `apiKey.latestVersion(false)`, the latest released version. This type
250/// gives the same cap to version negotiation.
251#[derive(Clone, Debug, PartialEq, Eq)]
252struct ReleasedInitProducerId(InitProducerIdRequest);
253
254impl Encode for ReleasedInitProducerId {
255    fn encode<B: BufMut>(&self, buf: &mut B, version: i16) -> Result<(), ProtocolError> {
256        self.0.encode(buf, version)
257    }
258
259    fn encoded_len(&self, version: i16) -> usize {
260        self.0.encoded_len(version)
261    }
262}
263
264impl ProtocolRequest for ReleasedInitProducerId {
265    const API_KEY: i16 = init_producer_id_request::API_KEY;
266    const MIN_VERSION: i16 = init_producer_id_request::MIN_VERSION;
267    /// The latest released `InitProducerId` version.
268    const MAX_VERSION: i16 = 5;
269    /// The cap is a released version, so it is also the stable maximum.
270    const LATEST_STABLE_VERSION: i16 = Self::MAX_VERSION;
271    const FLEXIBLE_MIN: i16 = init_producer_id_request::FLEXIBLE_MIN;
272    type Response = InitProducerIdResponse;
273}
274
275fn force_terminate_request(
276    transactional_id: &str,
277    transaction_timeout_ms: i32,
278) -> ReleasedInitProducerId {
279    ReleasedInitProducerId(InitProducerIdRequest {
280        transactional_id: Some(transactional_id.to_owned()),
281        transaction_timeout_ms,
282        producer_id: -1,
283        producer_epoch: -1,
284        enable2_pc: false,
285        keep_prepared_txn: false,
286        ..Default::default()
287    })
288}
289
290fn describe_transactions_request(transactional_id: &str) -> DescribeTransactionsRequest {
291    DescribeTransactionsRequest {
292        transactional_ids: vec![transactional_id.to_owned()],
293        ..Default::default()
294    }
295}
296
297/// Converts one row of coordinator `coordinator_id` into the domain struct,
298/// as Kafka's `DescribeTransactionsHandler.handleResponse` does, or surfaces
299/// the error code that the coordinator attached to that row.
300fn described_transaction(
301    coordinator_id: i32,
302    state: TransactionState,
303) -> Result<TransactionDescription, AdminError> {
304    if state.error_code != 0 {
305        return Err(AdminError::Broker {
306            api: "DescribeTransactions",
307            code: state.error_code,
308            name: kafka_error_name(state.error_code),
309            message: None,
310        });
311    }
312    Ok(TransactionDescription {
313        transactional_id: state.transactional_id,
314        coordinator_id,
315        state: state.transaction_state,
316        producer_id: state.producer_id,
317        producer_epoch: state.producer_epoch,
318        timeout: Time::from_millis(i64::from(state.transaction_timeout_ms)),
319        start_time_ms: (state.transaction_start_time_ms >= 0)
320            .then_some(state.transaction_start_time_ms),
321        topic_partitions: state
322            .topics
323            .into_iter()
324            .flat_map(|topic| {
325                topic
326                    .partitions
327                    .into_iter()
328                    .map(move |partition| (topic.topic.clone(), partition))
329            })
330            .collect(),
331    })
332}
333
334/// Picks the row for `transactional_id` out of the `DescribeTransactions`
335/// response of coordinator `coordinator_id` and maps it to the domain struct.
336fn transaction_description(
337    transactional_id: &str,
338    coordinator_id: i32,
339    response: DescribeTransactionsResponse,
340) -> Result<TransactionDescription, AdminError> {
341    response
342        .transaction_states
343        .into_iter()
344        .find(|state| state.transactional_id == transactional_id)
345        .ok_or_else(|| {
346            AdminError::Protocol(format!(
347                "DescribeTransactions returned no state for transactional id {transactional_id:?}"
348            ))
349        })
350        .and_then(|state| described_transaction(coordinator_id, state))
351}
352
353/// A `Metadata` request for `topics`, as Kafka's leader-routed handlers send
354/// it. An empty slice asks for the broker list only, with no topics, as
355/// `list_transactions` uses it.
356fn build_topic_metadata(topics: &[&str]) -> MetadataRequest {
357    MetadataRequest {
358        topics: if topics.is_empty() {
359            Some(Vec::new())
360        } else {
361            Some(
362                topics
363                    .iter()
364                    .map(|name| MetadataRequestTopic {
365                        topic_id: ProtoUuid::ZERO,
366                        name: Some((*name).to_owned()),
367                        ..Default::default()
368                    })
369                    .collect(),
370            )
371        },
372        allow_auto_topic_creation: false,
373        ..Default::default()
374    }
375}
376
377/// The `host:port` of the leader of `topic`-`partition` in `metadata`, as
378/// Kafka's `PartitionLeaderStrategy` resolves it.
379fn partition_leader_address(
380    topic: &str,
381    partition: i32,
382    metadata: &MetadataResponse,
383) -> Result<String, AdminError> {
384    let topic_metadata = metadata
385        .topics
386        .iter()
387        .find(|entry| entry.name.as_deref() == Some(topic))
388        .ok_or_else(|| {
389            AdminError::Protocol(format!("Metadata returned no entry for topic {topic:?}"))
390        })?;
391    if topic_metadata.error_code != 0 {
392        return Err(AdminError::Broker {
393            api: "Metadata",
394            code: topic_metadata.error_code,
395            name: kafka_error_name(topic_metadata.error_code),
396            message: None,
397        });
398    }
399    let partition_metadata = topic_metadata
400        .partitions
401        .iter()
402        .find(|entry| entry.partition_index == partition)
403        .ok_or_else(|| {
404            AdminError::Protocol(format!(
405                "Metadata returned no entry for partition {topic}-{partition}"
406            ))
407        })?;
408    if partition_metadata.error_code != 0 {
409        return Err(AdminError::Broker {
410            api: "Metadata",
411            code: partition_metadata.error_code,
412            name: kafka_error_name(partition_metadata.error_code),
413            message: None,
414        });
415    }
416    metadata
417        .brokers
418        .iter()
419        .find(|broker| broker.node_id == partition_metadata.leader_id)
420        .map(|broker| format_host_port(&broker.host, broker.port))
421        .ok_or_else(|| {
422            AdminError::Protocol(format!(
423                "Metadata named leader {} for {topic}-{partition} but listed no such broker",
424                partition_metadata.leader_id
425            ))
426        })
427}
428
429/// The identity of a transaction to abort, as Kafka's `AbortTransactionSpec`
430/// bundles it: the partition it last wrote to, and the producer generation
431/// that owns it.
432#[derive(Debug, Clone, PartialEq, Eq)]
433pub struct AbortTransactionSpec {
434    /// The topic that the transaction wrote to.
435    pub topic: String,
436    /// The partition that the transaction wrote to.
437    pub partition: i32,
438    /// The producer ID of the transaction to fence.
439    pub producer_id: i64,
440    /// The producer epoch of the transaction to fence.
441    pub producer_epoch: i16,
442    /// The epoch of the coordinator that assigned `producer_id`.
443    pub coordinator_epoch: i32,
444}
445
446/// The `WriteTxnMarkers` request that aborts the transaction of `spec`.
447fn abort_transaction_request(spec: &AbortTransactionSpec) -> WriteTxnMarkersRequest {
448    WriteTxnMarkersRequest {
449        markers: vec![WritableTxnMarker {
450            producer_id: spec.producer_id,
451            producer_epoch: spec.producer_epoch,
452            transaction_result: false,
453            topics: vec![WritableTxnMarkerTopic {
454                name: spec.topic.clone(),
455                partition_indexes: vec![spec.partition],
456                ..Default::default()
457            }],
458            coordinator_epoch: spec.coordinator_epoch,
459            ..Default::default()
460        }],
461        ..Default::default()
462    }
463}
464
465/// Picks the result of `spec`'s topic and partition out of a `WriteTxnMarkers`
466/// response and maps it to an [`AdminError`].
467fn abort_transaction_result(
468    spec: &AbortTransactionSpec,
469    response: WriteTxnMarkersResponse,
470) -> Result<(), AdminError> {
471    let marker = response
472        .markers
473        .into_iter()
474        .find(|marker| marker.producer_id == spec.producer_id)
475        .ok_or_else(|| {
476            AdminError::Protocol(format!(
477                "WriteTxnMarkers returned no result for producer id {}",
478                spec.producer_id
479            ))
480        })?;
481    let topic_result = marker
482        .topics
483        .into_iter()
484        .find(|entry| entry.name == spec.topic)
485        .ok_or_else(|| {
486            AdminError::Protocol(format!(
487                "WriteTxnMarkers returned no result for topic {:?}",
488                spec.topic
489            ))
490        })?;
491    let partition_result = topic_result
492        .partitions
493        .into_iter()
494        .find(|entry| entry.partition_index == spec.partition)
495        .ok_or_else(|| {
496            AdminError::Protocol(format!(
497                "WriteTxnMarkers returned no result for partition {}-{}",
498                spec.topic, spec.partition
499            ))
500        })?;
501    if partition_result.error_code != 0 {
502        return Err(AdminError::Broker {
503            api: "WriteTxnMarkers",
504            code: partition_result.error_code,
505            name: kafka_error_name(partition_result.error_code),
506            message: None,
507        });
508    }
509    Ok(())
510}
511
512/// Maps the outcome of one `WriteTxnMarkers` attempt to a retry action.
513/// `NOT_LEADER_OR_FOLLOWER` and `LEADER_NOT_AVAILABLE` look the leader up
514/// again, as Kafka's `AbortTransactionHandler.handleResponse` does. Every
515/// other outcome is final.
516fn abort_transaction_retry_action(result: Result<(), AdminError>) -> RetryAction<()> {
517    match result {
518        Err(AdminError::Broker {
519            code: NOT_LEADER_OR_FOLLOWER | LEADER_NOT_AVAILABLE,
520            ..
521        }) => RetryAction::FindCoordinator(result),
522        result => RetryAction::Done(result),
523    }
524}
525
526/// The `DescribeProducers` request that asks for every partition in `keys`,
527/// grouped by topic, as Kafka's `DescribeProducersHandler.buildBatchedRequest`
528/// does.
529fn describe_producers_request(keys: &[(String, i32)]) -> DescribeProducersRequest {
530    let mut by_topic = BTreeMap::<&str, Vec<i32>>::new();
531    for (topic, partition) in keys {
532        by_topic.entry(topic.as_str()).or_default().push(*partition);
533    }
534    DescribeProducersRequest {
535        topics: by_topic
536            .into_iter()
537            .map(|(name, partition_indexes)| DescribeProducersTopicRequest {
538                name: name.to_owned(),
539                partition_indexes,
540                ..Default::default()
541            })
542            .collect(),
543        ..Default::default()
544    }
545}
546
547/// Converts one wire `ProducerState` row into the domain struct. `-1` marks
548/// no open transaction, as Kafka's `ProducerState` does.
549fn producer_state_info(state: &WireProducerState) -> ProducerStateInfo {
550    ProducerStateInfo {
551        producer_id: state.producer_id,
552        producer_epoch: state.producer_epoch,
553        last_sequence: state.last_sequence,
554        last_timestamp_ms: state.last_timestamp,
555        coordinator_epoch: state.coordinator_epoch,
556        current_txn_start_offset: (state.current_txn_start_offset != -1)
557            .then_some(state.current_txn_start_offset),
558    }
559}
560
561/// Maps a `DescribeProducers` response to a result for each of `keys`. A
562/// partition that the response omits gets `UNKNOWN_SERVER_ERROR`, as Kafka's
563/// `DescribeProducersHandler.handleResponse` does for a missing result.
564fn describe_producers_results(
565    keys: &[PartitionKey],
566    response: DescribeProducersResponse,
567) -> PartitionResults<Vec<ProducerStateInfo>> {
568    let mut out = BTreeMap::new();
569    for topic in response.topics {
570        for partition in topic.partitions {
571            let result = if partition.error_code == 0 {
572                Ok(partition
573                    .active_producers
574                    .iter()
575                    .map(producer_state_info)
576                    .collect())
577            } else {
578                Err(KafkaError {
579                    code: partition.error_code,
580                    name: kafka_error_name(partition.error_code),
581                    message: partition.error_message,
582                })
583            };
584            out.insert((topic.name.clone(), partition.partition_index), result);
585        }
586    }
587    complete_results("DescribeProducers", keys, out)
588}
589
590/// Whether a `DescribeProducers` partition code sends the partition back to
591/// the leader lookup. Kafka's `DescribeProducersHandler.handlePartitionError`
592/// unmaps the partition on `NOT_LEADER_OR_FOLLOWER` only.
593const fn describe_producers_retry_code(code: i16) -> bool {
594    code == NOT_LEADER_OR_FOLLOWER
595}
596
597/// Sends one `DescribeProducers` request for `keys` to the leader at
598/// `address`.
599async fn describe_producers_on_leader(
600    address: String,
601    options: ConnectionOptions,
602    keys: Vec<PartitionKey>,
603) -> Result<PartitionResults<Vec<ProducerStateInfo>>, AdminError> {
604    let connection = AdminClient::connect_one(&address, options).await?;
605    let response = connection.send(describe_producers_request(&keys)).await?;
606    Ok(describe_producers_results(&keys, response))
607}
608
609/// The `ListTransactions` request of `filter`, as Kafka's
610/// `ListTransactionsHandler.buildRequest` builds it for every broker.
611fn list_transactions_request(filter: &ListTransactionsFilter) -> ListTransactionsRequest {
612    ListTransactionsRequest {
613        state_filters: filter.state_filters.clone(),
614        producer_id_filters: filter.producer_id_filters.clone(),
615        duration_filter: filter
616            .min_duration
617            .map_or(-1, krabka_units::convert::TimeExt::millis_i64),
618        ..Default::default()
619    }
620}
621
622/// Converts one wire `TransactionState` row into the domain struct.
623fn transaction_listing(state: ListedTransactionState) -> TransactionListing {
624    TransactionListing {
625        transactional_id: state.transactional_id,
626        producer_id: state.producer_id,
627        state: state.transaction_state,
628    }
629}
630
631/// Sends `request` to one broker and returns its transaction listings, or the
632/// broker's top-level error.
633async fn list_transactions_on_broker(
634    address: String,
635    options: ConnectionOptions,
636    request: ListTransactionsRequest,
637) -> Result<Vec<TransactionListing>, AdminError> {
638    let connection = AdminClient::connect_one(&address, options).await?;
639    let response: ListTransactionsResponse = connection.send(request).await?;
640    if response.error_code != 0 {
641        return Err(AdminError::Broker {
642            api: "ListTransactions",
643            code: response.error_code,
644            name: kafka_error_name(response.error_code),
645            message: None,
646        });
647    }
648    Ok(response
649        .transaction_states
650        .into_iter()
651        .map(transaction_listing)
652        .collect())
653}
654
655impl AdminClient {
656    /// Fences the current producer generation and aborts any ongoing
657    /// transaction for `transactional_id`.
658    ///
659    /// This is Kafka's `forceTerminateTransaction` operation: it discovers the
660    /// transaction coordinator and sends `InitProducerId` with no producer
661    /// identity and `keepPreparedTxn=false`. It is safe to call when no
662    /// transaction is open; the coordinator still advances the producer
663    /// generation so stale writers are fenced.
664    ///
665    /// The client negotiates `InitProducerId` v5 or lower, as Kafka's
666    /// `FenceProducersHandler` does. v6 is an unstable version.
667    ///
668    /// The call retries as Apache Kafka's `FenceProducersHandler.handleError` does:
669    ///
670    /// - `COORDINATOR_LOAD_IN_PROGRESS` (14) and `CONCURRENT_TRANSACTIONS` (51): send the request again to the same coordinator.
671    /// - `COORDINATOR_NOT_AVAILABLE` (15) and `NOT_COORDINATOR` (16): find the
672    ///   coordinator again, then send the request again.
673    ///
674    /// A `FindCoordinator` answer of 14 or 15, and a failed or lost connection
675    /// to the coordinator (including a TLS or SASL handshake that stops
676    /// before the broker gives a verdict), also make the call find the
677    /// coordinator again. The call waits between attempts with Kafka's
678    /// backoff, and it stops when Kafka's default `default.api.timeout.ms`
679    /// (60 s) elapses. A call whose retries are unresolved at the deadline, or
680    /// whose attempt is still running, fails with a timeout error, as Kafka
681    /// gives a `TimeoutException`.
682    ///
683    /// # Errors
684    ///
685    /// Returns [`AdminError::Protocol`] for an empty transactional ID, or the
686    /// coordinator lookup, connection, transport, and broker errors returned
687    /// by Kafka. Returns [`ClientError::IncompatibleVersion`] when the
688    /// coordinator does not support `InitProducerId` v5 or lower.
689    ///
690    /// [`ClientError::IncompatibleVersion`]: krabka_client_core::ClientError::IncompatibleVersion
691    pub async fn force_terminate_transaction(
692        &self,
693        transactional_id: &str,
694    ) -> Result<(), AdminError> {
695        if transactional_id.is_empty() {
696            return Err(AdminError::Protocol(
697                "transactional id must not be empty".to_owned(),
698            ));
699        }
700
701        self.force_terminate_transaction_with_retry(transactional_id, self.retry)
702            .await
703    }
704
705    async fn force_terminate_transaction_with_retry(
706        &self,
707        transactional_id: &str,
708        retry: RetryPolicy,
709    ) -> Result<(), AdminError> {
710        self.force_terminate_transaction_until(transactional_id, retry.start())
711            .await
712    }
713
714    /// `force_terminate_transaction` with a call deadline that already runs,
715    /// so that every ID of one `fence_producers` call shares one deadline.
716    async fn force_terminate_transaction_until(
717        &self,
718        transactional_id: &str,
719        deadline: RetryDeadline,
720    ) -> Result<(), AdminError> {
721        let mut coordinator = None;
722        let mut retry = CoordinatorRetry::from_deadline(deadline);
723        loop {
724            let action = retry
725                .run(self.force_terminate_attempt(
726                    transactional_id,
727                    &mut coordinator,
728                    retry.find_coordinator(),
729                ))
730                .await;
731            if let Some(result) = retry.next(action).await {
732                return result;
733            }
734        }
735    }
736
737    /// Fences the current producer generation and aborts any ongoing
738    /// transaction for each ID in `transactional_ids`, as Kafka's
739    /// `fenceProducers` operation does.
740    ///
741    /// Two transactional IDs can live on two different coordinators, so the
742    /// client fences them at the same time rather than one at a time. All IDs
743    /// share one call deadline, as Kafka's `FenceProducersHandler` and
744    /// `AdminApiDriver` run one `invokeDriver` for the whole call. The result
745    /// has one entry for each ID in `transactional_ids`, in no particular
746    /// order.
747    ///
748    /// See [`Self::force_terminate_transaction`] for the retry behavior and
749    /// the `InitProducerId` version that the client negotiates.
750    ///
751    /// # Errors
752    ///
753    /// Each entry gets [`AdminError::Protocol`] for an empty transactional
754    /// ID, or the coordinator lookup, connection, transport, and broker
755    /// errors returned by Kafka for that ID.
756    pub async fn fence_producers(
757        &self,
758        transactional_ids: &[&str],
759    ) -> BTreeMap<String, Result<(), AdminError>> {
760        let deadline = self.retry.start();
761        futures_util::future::join_all(transactional_ids.iter().map(|transactional_id| {
762            let transactional_id = (*transactional_id).to_owned();
763            async move {
764                let result = if transactional_id.is_empty() {
765                    Err(AdminError::Protocol(
766                        "transactional id must not be empty".to_owned(),
767                    ))
768                } else {
769                    self.force_terminate_transaction_until(&transactional_id, deadline)
770                        .await
771                };
772                (transactional_id, result)
773            }
774        }))
775        .await
776        .into_iter()
777        .collect()
778    }
779
780    /// Fences the current producer generation and aborts the ongoing
781    /// transaction that wrote to `topic`-`partition`, without contacting the
782    /// transaction coordinator.
783    ///
784    /// This is Kafka's `abortTransaction` operation (KIP-664): it sends
785    /// `WriteTxnMarkers` straight to the partition leader with the identity of
786    /// the transaction to fence, as `AbortTransactionHandler` does. A
787    /// consumer that reads `describe_producers` or the partition's records
788    /// supplies `spec`.
789    ///
790    /// The call retries as Apache Kafka's
791    /// `AbortTransactionHandler.handleResponse` does: `NOT_LEADER_OR_FOLLOWER`
792    /// (6) and `LEADER_NOT_AVAILABLE` (5), from either the leader lookup or
793    /// the `WriteTxnMarkers` answer, look the leader up again. A failed or
794    /// lost connection to the leader does the same. Every other broker error
795    /// is final. The call waits between attempts with Kafka's backoff, and it
796    /// stops when Kafka's default `default.api.timeout.ms` (60 s) elapses.
797    ///
798    /// # Errors
799    ///
800    /// Returns [`AdminError::Protocol`] for an empty topic name, or when the
801    /// broker's answer names no result for the requested producer, topic, or
802    /// partition. Returns [`AdminError::Broker`] when the leader lookup or the
803    /// `WriteTxnMarkers` call fails, such as with
804    /// `TRANSACTIONAL_ID_AUTHORIZATION_FAILED` or `FENCED_INSTANCE_ID`.
805    /// Returns [`AdminError::Transport`] when the connection to the leader
806    /// fails.
807    pub async fn abort_transaction(&self, spec: &AbortTransactionSpec) -> Result<(), AdminError> {
808        if spec.topic.is_empty() {
809            return Err(AdminError::Protocol("topic must not be empty".to_owned()));
810        }
811
812        let mut leader = None;
813        let mut retry = CoordinatorRetry::new(self.retry);
814        loop {
815            let action = retry
816                .run(self.abort_transaction_attempt(spec, &mut leader, retry.find_coordinator()))
817                .await;
818            if let Some(result) = retry.next(action).await {
819                return result;
820            }
821        }
822    }
823
824    /// One attempt of `abort_transaction`.
825    async fn abort_transaction_attempt(
826        &self,
827        spec: &AbortTransactionSpec,
828        leader: &mut Option<Connection>,
829        find_leader: bool,
830    ) -> RetryAction<()> {
831        let connection = match self
832            .partition_leader_connection(&spec.topic, spec.partition, leader, find_leader)
833            .await
834        {
835            Ok(connection) => connection,
836            Err(action) => return action,
837        };
838        match connection.send(abort_transaction_request(spec)).await {
839            Ok(response) => {
840                abort_transaction_retry_action(abort_transaction_result(spec, response))
841            }
842            Err(error) => connection_failure_action(error.into()),
843        }
844    }
845
846    /// Returns the connection to the leader of `topic`-`partition`. When
847    /// `find_leader` is set, or when no connection is open, the attempt first
848    /// fetches fresh metadata and connects to the leader that it names.
849    async fn partition_leader_connection<'c, T>(
850        &self,
851        topic: &str,
852        partition: i32,
853        leader: &'c mut Option<Connection>,
854        find_leader: bool,
855    ) -> Result<&'c Connection, RetryAction<T>> {
856        if find_leader {
857            *leader = None;
858        }
859        if let Some(connection) = leader {
860            return Ok(connection);
861        }
862        let metadata = self
863            .conn
864            .send(build_topic_metadata(&[topic]))
865            .await
866            .map_err(connection_failure_action)?;
867        let address = match partition_leader_address(topic, partition, &metadata) {
868            Ok(address) => address,
869            Err(
870                error @ AdminError::Broker {
871                    code: NOT_LEADER_OR_FOLLOWER | LEADER_NOT_AVAILABLE,
872                    ..
873                },
874            ) => return Err(RetryAction::FindCoordinator(Err(error))),
875            Err(error) => return Err(RetryAction::Done(Err(error))),
876        };
877        match Self::connect_one(&address, self.options.clone()).await {
878            Ok(connection) => Ok(leader.insert(connection)),
879            Err(error) => Err(connection_failure_action(error)),
880        }
881    }
882
883    /// Reads the producer state of each requested partition, as Kafka's
884    /// `describeProducers` operation does (KIP-664).
885    ///
886    /// The client finds the leader of each partition with `Metadata` and
887    /// sends one `DescribeProducers` request to each leader, all at the same
888    /// time, batching every partition of that leader into it, as Kafka's
889    /// `DescribeProducersHandler` with its `PartitionLeaderStrategy` does. The
890    /// result has one entry for each requested `(topic, partition)` pair.
891    ///
892    /// A partition goes back to the leader lookup, and the call finds its
893    /// leader again after the backoff, when:
894    ///
895    /// - the lookup names no leader yet, such as for an unknown topic or a
896    ///   partition with `LEADER_NOT_AVAILABLE`;
897    /// - the connection to its leader fails or is lost;
898    /// - the leader answers `NOT_LEADER_OR_FOLLOWER` (6) for it.
899    ///
900    /// A slow or failed leader does not delay the other leaders. The call
901    /// stops at Kafka's default `default.api.timeout.ms` (60 s), where each
902    /// unresolved partition gets `REQUEST_TIMED_OUT` (7).
903    ///
904    /// # Errors
905    ///
906    /// The call itself does not fail. Each partition gets its own
907    /// [`KafkaError`] in the result, such as the final lookup or
908    /// `DescribeProducers` error of that partition.
909    pub async fn describe_producers(
910        &self,
911        partitions: &[(String, i32)],
912    ) -> BTreeMap<(String, i32), Result<Vec<ProducerStateInfo>, KafkaError>> {
913        self.call_partition_leaders(
914            "DescribeProducers",
915            partitions,
916            describe_producers_retry_code,
917            |address, keys| describe_producers_on_leader(address, self.options.clone(), keys),
918        )
919        .await
920    }
921
922    /// Lists every transaction that the cluster's coordinators currently
923    /// hold, as Kafka's `listTransactions` operation does (KIP-664).
924    ///
925    /// The client fetches the broker list from `Metadata` and sends
926    /// `ListTransactions` to every broker at the same time, since each
927    /// transaction coordinator answers only for the transactions it
928    /// coordinates. `filter` narrows the state, producer ID, and minimum
929    /// duration of the listed transactions, as Kafka's
930    /// `ListTransactionsOptions` does; the default matches every transaction.
931    ///
932    /// # Errors
933    ///
934    /// Returns [`AdminError::Broker`] when any broker gives a top-level
935    /// error, and [`AdminError::Transport`] when the connection to any broker
936    /// fails. A partial listing from the healthy brokers is not returned;
937    /// callers that need per-broker results should read
938    /// [`AdminError`] for which broker failed and retry.
939    pub async fn list_transactions(
940        &self,
941        filter: &ListTransactionsFilter,
942    ) -> Result<Vec<TransactionListing>, AdminError> {
943        let metadata = self.conn.send(build_topic_metadata(&[])).await?;
944        let request = list_transactions_request(filter);
945        let options = self.options.clone();
946        let answers = futures_util::future::join_all(metadata.brokers.into_iter().map(|broker| {
947            list_transactions_on_broker(
948                format_host_port(&broker.host, broker.port),
949                options.clone(),
950                request.clone(),
951            )
952        }))
953        .await;
954
955        let mut listings = Vec::new();
956        for answer in answers {
957            listings.extend(answer?);
958        }
959        Ok(listings)
960    }
961
962    /// One attempt of `force_terminate_transaction`.
963    async fn force_terminate_attempt(
964        &self,
965        transactional_id: &str,
966        coordinator: &mut Option<TransactionCoordinator>,
967        find_coordinator: bool,
968    ) -> RetryAction<()> {
969        let coordinator = match self
970            .transaction_coordinator(transactional_id, coordinator, find_coordinator)
971            .await
972        {
973            Ok(coordinator) => coordinator,
974            Err(action) => return action,
975        };
976        let request =
977            force_terminate_request(transactional_id, self.options.request_timeout.millis_i32());
978        match coordinator.connection.send(request).await {
979            Ok(response) => fence_retry_action(response.error_code),
980            Err(error) => connection_failure_action(error.into()),
981        }
982    }
983
984    /// Reads the transaction coordinator's current state for one transactional
985    /// ID.
986    ///
987    /// This is Kafka's `describeTransactions` operation (KIP-664). The client
988    /// discovers the transaction coordinator for `transactional_id` and sends
989    /// `DescribeTransactions` to that coordinator. An external system calls
990    /// this to verify the authority of a writer. It joins no group and holds
991    /// no producer state. The returned producer ID and producer epoch are the
992    /// only generation from which the coordinator accepts writes.
993    ///
994    /// The call retries as Apache Kafka's `DescribeTransactionsHandler.handleError` does:
995    ///
996    /// - `COORDINATOR_LOAD_IN_PROGRESS` (14): send the request again to the same coordinator.
997    /// - `COORDINATOR_NOT_AVAILABLE` (15) and `NOT_COORDINATOR` (16): find the
998    ///   coordinator again, then send the request again.
999    ///
1000    /// A `FindCoordinator` answer of 14 or 15, and a failed or lost connection
1001    /// to the coordinator (including a TLS or SASL handshake that stops
1002    /// before the broker gives a verdict), also make the call find the
1003    /// coordinator again. The call waits between attempts with Kafka's
1004    /// backoff, and it stops when Kafka's default `default.api.timeout.ms`
1005    /// (60 s) elapses. A call whose retries are unresolved at the deadline, or
1006    /// whose attempt is still running, fails with a timeout error, as Kafka
1007    /// gives a `TimeoutException`.
1008    ///
1009    /// # Errors
1010    ///
1011    /// Returns [`AdminError::Protocol`] for an empty transactional ID, and for
1012    /// a response that carries no row for `transactional_id`. Returns
1013    /// [`AdminError::Broker`] when the coordinator lookup fails, or when the
1014    /// coordinator attaches an error code to the row, such as
1015    /// `TRANSACTIONAL_ID_NOT_FOUND`,
1016    /// `TRANSACTIONAL_ID_AUTHORIZATION_FAILED`, or `NOT_COORDINATOR`. Returns
1017    /// [`AdminError::Transport`] when the connection to the coordinator fails.
1018    pub async fn describe_transaction(
1019        &self,
1020        transactional_id: &str,
1021    ) -> Result<TransactionDescription, AdminError> {
1022        if transactional_id.is_empty() {
1023            return Err(AdminError::Protocol(
1024                "transactional id must not be empty".to_owned(),
1025            ));
1026        }
1027
1028        self.describe_transaction_with_retry(transactional_id, self.retry)
1029            .await
1030    }
1031
1032    async fn describe_transaction_with_retry(
1033        &self,
1034        transactional_id: &str,
1035        retry: RetryPolicy,
1036    ) -> Result<TransactionDescription, AdminError> {
1037        self.describe_transaction_until(transactional_id, retry.start())
1038            .await
1039    }
1040
1041    /// `describe_transaction` with a call deadline that already runs, so that
1042    /// the IDs of one `describe_transactions` call share one deadline.
1043    async fn describe_transaction_until(
1044        &self,
1045        transactional_id: &str,
1046        deadline: RetryDeadline,
1047    ) -> Result<TransactionDescription, AdminError> {
1048        let mut coordinator = None;
1049        let mut retry = CoordinatorRetry::from_deadline(deadline);
1050        loop {
1051            let action = retry
1052                .run(self.describe_transaction_attempt(
1053                    transactional_id,
1054                    &mut coordinator,
1055                    retry.find_coordinator(),
1056                ))
1057                .await;
1058            if let Some(result) = retry.next(action).await {
1059                return result;
1060            }
1061        }
1062    }
1063
1064    /// One attempt of `describe_transaction`.
1065    async fn describe_transaction_attempt(
1066        &self,
1067        transactional_id: &str,
1068        coordinator: &mut Option<TransactionCoordinator>,
1069        find_coordinator: bool,
1070    ) -> RetryAction<TransactionDescription> {
1071        let coordinator = match self
1072            .transaction_coordinator(transactional_id, coordinator, find_coordinator)
1073            .await
1074        {
1075            Ok(coordinator) => coordinator,
1076            Err(action) => return action,
1077        };
1078        match coordinator
1079            .connection
1080            .send(describe_transactions_request(transactional_id))
1081            .await
1082        {
1083            Ok(response) => describe_retry_action(transaction_description(
1084                transactional_id,
1085                coordinator.node_id,
1086                response,
1087            )),
1088            Err(error) => connection_failure_action(error.into()),
1089        }
1090    }
1091
1092    /// Returns the transaction coordinator of `transactional_id`. When
1093    /// `find_coordinator` is set, or when no connection is open, the attempt
1094    /// first sends `FindCoordinator` and connects to the coordinator that it
1095    /// names.
1096    ///
1097    /// A `FindCoordinator` answer of `COORDINATOR_LOAD_IN_PROGRESS` (14) or
1098    /// `COORDINATOR_NOT_AVAILABLE` (15) and a failed connection to the
1099    /// coordinator give [`RetryAction::FindCoordinator`], as Kafka's
1100    /// `CoordinatorStrategy.handleError` and `AdminApiDriver.onFailure` do.
1101    async fn transaction_coordinator<'c, T>(
1102        &self,
1103        transactional_id: &str,
1104        coordinator: &'c mut Option<TransactionCoordinator>,
1105        find_coordinator: bool,
1106    ) -> Result<&'c TransactionCoordinator, RetryAction<T>> {
1107        if find_coordinator {
1108            *coordinator = None;
1109        }
1110        if let Some(coordinator) = coordinator {
1111            return Ok(coordinator);
1112        }
1113        let response = self
1114            .conn
1115            .send(build_find_coordinator(
1116                transactional_id,
1117                CoordinatorKeyType::Transaction,
1118            ))
1119            .await
1120            .map_err(connection_failure_action)?;
1121        let (node_id, address) = match coordinator_address(transactional_id, response) {
1122            Ok(found) => found,
1123            Err(
1124                error @ AdminError::Broker {
1125                    code: COORDINATOR_LOAD_IN_PROGRESS | COORDINATOR_NOT_AVAILABLE,
1126                    ..
1127                },
1128            ) => return Err(RetryAction::FindCoordinator(Err(error))),
1129            Err(error) => return Err(RetryAction::Done(Err(error))),
1130        };
1131        match Self::connect_one(&address, self.options.clone()).await {
1132            Ok(connection) => Ok(coordinator.insert(TransactionCoordinator {
1133                node_id,
1134                connection,
1135            })),
1136            Err(error) => Err(connection_failure_action(error)),
1137        }
1138    }
1139
1140    /// Reads the transaction coordinator's current state for each ID in
1141    /// `transactional_ids`.
1142    ///
1143    /// Two transactional IDs can live on two different coordinators, so the
1144    /// client describes them one at a time. Each ID costs one
1145    /// `FindCoordinator` round trip, one new connection to the coordinator
1146    /// that the lookup names, and one `DescribeTransactions` round trip. The
1147    /// cost grows linearly with the number of IDs. Callers that describe a
1148    /// large set should expect that cost. The results keep the order of
1149    /// `transactional_ids`.
1150    ///
1151    /// # Errors
1152    ///
1153    /// Stops at the first ID that fails and returns its error. Returns
1154    /// [`AdminError::Protocol`] for an empty transactional ID, and for a
1155    /// response that carries no row for the ID. Returns
1156    /// [`AdminError::Broker`] when the coordinator lookup fails, or when the
1157    /// coordinator attaches an error code to the row. Returns
1158    /// [`AdminError::Transport`] when the connection to a coordinator fails.
1159    pub async fn describe_transactions(
1160        &self,
1161        transactional_ids: &[&str],
1162    ) -> Result<Vec<TransactionDescription>, AdminError> {
1163        // Kafka's `describeTransactions` runs all IDs under one deadline
1164        // (`invokeDriver` with one `deadlineMs`).
1165        let deadline = self.retry.start();
1166        let mut descriptions = Vec::with_capacity(transactional_ids.len());
1167        for transactional_id in transactional_ids {
1168            if transactional_id.is_empty() {
1169                return Err(AdminError::Protocol(
1170                    "transactional id must not be empty".to_owned(),
1171                ));
1172            }
1173            descriptions.push(
1174                self.describe_transaction_until(transactional_id, deadline)
1175                    .await?,
1176            );
1177        }
1178        Ok(descriptions)
1179    }
1180}
1181
1182#[cfg(test)]
1183mod tests {
1184    use std::{
1185        sync::{Arc, Mutex},
1186        time::Duration,
1187    };
1188
1189    use assert2::check;
1190    use bytes::{Buf, BytesMut};
1191    use krabka_client_core::{ClientError, MockBroker, MockReply};
1192    use krabka_protocol::{
1193        Decode,
1194        owned::{
1195            api_versions_request,
1196            api_versions_response::{ApiVersion, ApiVersionsResponse},
1197            describe_producers_request,
1198            describe_producers_response::{PartitionResponse, TopicResponse},
1199            describe_transactions_request,
1200            describe_transactions_response::TopicData,
1201            find_coordinator_request,
1202            find_coordinator_response::Coordinator,
1203            list_transactions_request, metadata_request,
1204            metadata_response::{
1205                MetadataResponse, MetadataResponseBroker, MetadataResponsePartition,
1206                MetadataResponseTopic,
1207            },
1208            write_txn_markers_request,
1209            write_txn_markers_response::{
1210                WritableTxnMarkerPartitionResult, WritableTxnMarkerResult,
1211                WritableTxnMarkerTopicResult,
1212            },
1213        },
1214    };
1215
1216    use super::*;
1217    use crate::partition_leaders::test_support::{
1218        ApiVersionRange, Seen, fast_admin as leader_fast_admin,
1219        leader_broker as partition_leader_broker,
1220    };
1221
1222    fn state_row(transactional_id: &str, error_code: i16) -> TransactionState {
1223        TransactionState {
1224            error_code,
1225            transactional_id: transactional_id.to_owned(),
1226            ..Default::default()
1227        }
1228    }
1229
1230    /// The `InitProducerId` request that fences `payments` with no producer
1231    /// identity and no kept prepared transaction.
1232    fn fence_payments() -> InitProducerIdRequest {
1233        InitProducerIdRequest {
1234            transactional_id: Some("payments".to_owned()),
1235            transaction_timeout_ms: 30_000,
1236            producer_id: -1,
1237            producer_epoch: -1,
1238            enable2_pc: false,
1239            keep_prepared_txn: false,
1240            ..Default::default()
1241        }
1242    }
1243
1244    #[test]
1245    fn force_termination_request_fences_without_preserving_transaction() {
1246        let request = force_terminate_request("payments", 30_000);
1247
1248        assert2::assert!(request == ReleasedInitProducerId(fence_payments()));
1249    }
1250
1251    /// The body of `response` at `version`, behind an empty tagged-field
1252    /// byte when `flexible` is set.
1253    fn encode_response(response: &impl Encode, version: i16, flexible: bool) -> Vec<u8> {
1254        let mut bytes = BytesMut::new();
1255        if flexible {
1256            bytes.extend_from_slice(&[0]);
1257        }
1258        response
1259            .encode(&mut bytes, version)
1260            .expect("response encodes");
1261        bytes.to_vec()
1262    }
1263
1264    /// An `ApiVersions` response that advertises `InitProducerId` in
1265    /// `init_producer_id_range`.
1266    fn api_versions(init_producer_id_range: (i16, i16)) -> Vec<u8> {
1267        let api_version = |api_key, min_version, max_version| ApiVersion {
1268            api_key,
1269            min_version,
1270            max_version,
1271            ..Default::default()
1272        };
1273        encode_response(
1274            &ApiVersionsResponse {
1275                api_keys: vec![
1276                    api_version(api_versions_request::API_KEY, 0, 0),
1277                    api_version(find_coordinator_request::API_KEY, 0, 0),
1278                    api_version(metadata_request::API_KEY, 13, 13),
1279                    api_version(
1280                        init_producer_id_request::API_KEY,
1281                        init_producer_id_range.0,
1282                        init_producer_id_range.1,
1283                    ),
1284                ],
1285                ..Default::default()
1286            },
1287            0,
1288            false,
1289        )
1290    }
1291
1292    /// The negotiated `InitProducerId` version and the result of
1293    /// `force_terminate_transaction`, with a version error as its ranges.
1294    type FenceResult = Result<(), (i16, i16, i16, i16, i16)>;
1295
1296    /// Apache Kafka's `FenceProducersHandler` builds `InitProducerId` with
1297    /// `new InitProducerIdRequest.Builder(data)`, which allows only released
1298    /// versions. v6 is unstable, so the request stops at v5.
1299    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1300    async fn force_terminate_transaction_sends_a_released_init_producer_id_version() {
1301        let sent_request = |version| vec![(version, fence_payments())];
1302        for (name, init_producer_id_range, expected_requests, expected_result) in [
1303            ("coordinator stops at v1", (0, 1), sent_request(1), Ok(())),
1304            ("coordinator stops at v4", (0, 4), sent_request(4), Ok(())),
1305            ("coordinator stops at v5", (0, 5), sent_request(5), Ok(())),
1306            (
1307                "coordinator supports unstable v6",
1308                (0, 6),
1309                sent_request(5),
1310                Ok(()),
1311            ),
1312            (
1313                "coordinator supports only unstable v6",
1314                (6, 6),
1315                Vec::new(),
1316                Err((init_producer_id_request::API_KEY, 6, 6, 0, 5)),
1317            ),
1318        ] {
1319            let requests = Arc::new(Mutex::new(Vec::new()));
1320            let requests_in_mock = Arc::clone(&requests);
1321            let coordinator =
1322                MockBroker::start(move |api_key, version, _, mut body| match api_key {
1323                    api_versions_request::API_KEY => Some(api_versions(init_producer_id_range)),
1324                    init_producer_id_request::API_KEY => {
1325                        let client_id_len = body.get_i16();
1326                        body.advance(usize::try_from(client_id_len).expect("client id length"));
1327                        let flexible = version >= init_producer_id_request::FLEXIBLE_MIN;
1328                        if flexible {
1329                            body.advance(1);
1330                        }
1331                        let request = InitProducerIdRequest::decode(&mut body, version)
1332                            .expect("init producer id request decodes");
1333                        requests_in_mock
1334                            .lock()
1335                            .expect("requests lock")
1336                            .push((version, request));
1337                        Some(encode_response(
1338                            &InitProducerIdResponse {
1339                                producer_id: 4242,
1340                                producer_epoch: 1,
1341                                ..Default::default()
1342                            },
1343                            version,
1344                            flexible,
1345                        ))
1346                    }
1347                    _ => None,
1348                })
1349                .await;
1350            let coordinator_addr = coordinator.addr;
1351            let bootstrap = MockBroker::start(move |api_key, version, _, _| match api_key {
1352                api_versions_request::API_KEY => Some(api_versions(init_producer_id_range)),
1353                find_coordinator_request::API_KEY => Some(encode_response(
1354                    &FindCoordinatorResponse {
1355                        node_id: 2,
1356                        host: coordinator_addr.ip().to_string(),
1357                        port: i32::from(coordinator_addr.port()),
1358                        ..Default::default()
1359                    },
1360                    version,
1361                    false,
1362                )),
1363                metadata_request::API_KEY => {
1364                    Some(encode_response(&MetadataResponse::default(), version, true))
1365                }
1366                _ => None,
1367            })
1368            .await;
1369            let admin = AdminClient::connect(&[bootstrap.addr.to_string()])
1370                .await
1371                .expect("admin connects");
1372
1373            let result: FenceResult =
1374                admin
1375                    .force_terminate_transaction("payments")
1376                    .await
1377                    .map_err(|error| match error {
1378                        AdminError::Transport(ClientError::IncompatibleVersion {
1379                            api_key,
1380                            broker_min,
1381                            broker_max,
1382                            client_min,
1383                            client_max,
1384                        }) => (api_key, broker_min, broker_max, client_min, client_max),
1385                        other => panic!("case {name}: unexpected error {other:?}"),
1386                    });
1387
1388            bootstrap.stop();
1389            coordinator.stop();
1390            let requests = requests.lock().expect("requests lock").clone();
1391            assert2::assert!(
1392                (requests, result) == (expected_requests, expected_result),
1393                "case {name}"
1394            );
1395        }
1396    }
1397
1398    /// Kafka's `CoordinatorStrategy` reads the batched entry of the key from
1399    /// v4 on, and the top-level fields of an older answer.
1400    #[test]
1401    fn coordinator_lookup_names_the_node_and_address_of_the_key() {
1402        for (name, response) in [
1403            (
1404                "batched entry",
1405                FindCoordinatorResponse {
1406                    coordinators: vec![
1407                        Coordinator {
1408                            key: "other".to_owned(),
1409                            node_id: 1,
1410                            host: "wrong".to_owned(),
1411                            port: 1,
1412                            ..Default::default()
1413                        },
1414                        Coordinator {
1415                            key: "payments".to_owned(),
1416                            node_id: 3,
1417                            host: "coordinator".to_owned(),
1418                            port: 9092,
1419                            ..Default::default()
1420                        },
1421                    ],
1422                    ..Default::default()
1423                },
1424            ),
1425            (
1426                "top-level fields",
1427                FindCoordinatorResponse {
1428                    node_id: 3,
1429                    host: "coordinator".to_owned(),
1430                    port: 9092,
1431                    ..Default::default()
1432                },
1433            ),
1434        ] {
1435            let found = coordinator_address("payments", response).expect("matching coordinator");
1436            assert2::assert!(found == (3, "coordinator:9092".to_owned()), "case {name}");
1437        }
1438    }
1439
1440    #[test]
1441    fn describe_request_asks_for_exactly_one_transactional_id() {
1442        let request = describe_transactions_request("payments");
1443
1444        assert2::assert!(
1445            request
1446                == DescribeTransactionsRequest {
1447                    transactional_ids: vec!["payments".to_owned()],
1448                    ..Default::default()
1449                }
1450        );
1451    }
1452
1453    /// Kafka's `DescribeTransactionsHandler.handleResponse` names the
1454    /// coordinator that answered, maps a negative start time to an empty
1455    /// `OptionalLong`, and collects the partitions of every topic.
1456    #[test]
1457    fn description_maps_the_matching_row_to_the_domain_struct() {
1458        let topic = |topic: &str, partitions: Vec<i32>| TopicData {
1459            topic: topic.to_owned(),
1460            partitions,
1461            ..Default::default()
1462        };
1463        let partitions = |entries: &[(&str, i32)]| {
1464            entries
1465                .iter()
1466                .map(|(topic, partition)| ((*topic).to_owned(), *partition))
1467                .collect::<BTreeSet<_>>()
1468        };
1469        for (name, state, start, topics, expected_start, expected_partitions) in [
1470            (
1471                "ongoing with partitions of two topics",
1472                "Ongoing",
1473                1_700_000_000_123,
1474                vec![topic("orders", vec![1, 0]), topic("audit", vec![2])],
1475                Some(1_700_000_000_123),
1476                partitions(&[("audit", 2), ("orders", 0), ("orders", 1)]),
1477            ),
1478            (
1479                "empty with no start time",
1480                "Empty",
1481                -1,
1482                Vec::new(),
1483                None,
1484                BTreeSet::new(),
1485            ),
1486            (
1487                "start time at the epoch",
1488                "Ongoing",
1489                0,
1490                Vec::new(),
1491                Some(0),
1492                BTreeSet::new(),
1493            ),
1494        ] {
1495            let response = DescribeTransactionsResponse {
1496                transaction_states: vec![
1497                    TransactionState {
1498                        transactional_id: "other".to_owned(),
1499                        producer_id: 11,
1500                        producer_epoch: 1,
1501                        ..Default::default()
1502                    },
1503                    TransactionState {
1504                        transactional_id: "payments".to_owned(),
1505                        transaction_state: state.to_owned(),
1506                        transaction_timeout_ms: 60_000,
1507                        transaction_start_time_ms: start,
1508                        producer_id: 4242,
1509                        producer_epoch: 7,
1510                        topics,
1511                        ..Default::default()
1512                    },
1513                ],
1514                ..Default::default()
1515            };
1516
1517            let described = transaction_description("payments", 3, response).expect("matching row");
1518
1519            assert2::assert!(
1520                described
1521                    == TransactionDescription {
1522                        transactional_id: "payments".to_owned(),
1523                        coordinator_id: 3,
1524                        state: state.to_owned(),
1525                        producer_id: 4242,
1526                        producer_epoch: 7,
1527                        timeout: Time::from_millis(60_000),
1528                        start_time_ms: expected_start,
1529                        topic_partitions: expected_partitions,
1530                    },
1531                "case {name}"
1532            );
1533        }
1534    }
1535
1536    #[test]
1537    fn description_surfaces_the_error_code_of_the_matching_row() {
1538        for (_case, code, want_name) in [
1539            ("unknown id", 105, "TRANSACTIONAL_ID_NOT_FOUND"),
1540            (
1541                "not authorized",
1542                53,
1543                "TRANSACTIONAL_ID_AUTHORIZATION_FAILED",
1544            ),
1545            ("moved coordinator", 16, "NOT_COORDINATOR"),
1546            ("loading", 14, "COORDINATOR_LOAD_IN_PROGRESS"),
1547        ] {
1548            let response = DescribeTransactionsResponse {
1549                transaction_states: vec![state_row("other", 0), state_row("payments", code)],
1550                ..Default::default()
1551            };
1552
1553            let error = transaction_description("payments", 3, response).expect_err("broker error");
1554            match error {
1555                AdminError::Broker {
1556                    api,
1557                    code: got,
1558                    name,
1559                    message,
1560                } => {
1561                    check!(
1562                        (api, got, name, message)
1563                            == ("DescribeTransactions", code, want_name, None)
1564                    );
1565                }
1566                other => panic!("expected AdminError::Broker, got {other:?}"),
1567            }
1568        }
1569    }
1570
1571    #[test]
1572    fn description_rejects_a_response_that_omits_the_requested_id() {
1573        let response = DescribeTransactionsResponse {
1574            transaction_states: vec![state_row("other", 0)],
1575            ..Default::default()
1576        };
1577
1578        let error = transaction_description("payments", 3, response).expect_err("missing row");
1579        assert2::assert!(matches!(error, AdminError::Protocol(_)));
1580    }
1581
1582    /// The transaction admin call of a retry case.
1583    #[derive(Clone, Copy, Debug)]
1584    enum TransactionCall {
1585        Describe,
1586        ForceTerminate,
1587    }
1588
1589    /// The error codes that the mock brokers answer, one per request. The last
1590    /// code repeats.
1591    #[derive(Default)]
1592    struct CoordinatorScript {
1593        find_coordinator: Vec<i16>,
1594        coordinator: Vec<i16>,
1595        find_coordinator_requests: usize,
1596        coordinator_requests: usize,
1597        /// The first `unreachable_answers` `FindCoordinator` answers name
1598        /// `unreachable`, an address that refuses connections.
1599        unreachable_answers: usize,
1600        unreachable: Option<std::net::SocketAddr>,
1601        /// The coordinator does not answer the transaction request.
1602        silent: bool,
1603        /// The first `closed_find_coordinators` `FindCoordinator` requests
1604        /// close the connection with no answer.
1605        closed_find_coordinators: usize,
1606    }
1607
1608    /// How the coordinator of a retry case behaves.
1609    #[derive(Clone, Copy, Debug, PartialEq, Eq)]
1610    enum CoordinatorBehavior {
1611        /// It answers from the script.
1612        Normal,
1613        /// The first `FindCoordinator` answer names an address that refuses
1614        /// connections.
1615        UnreachableOnce,
1616        /// It does not answer the transaction request.
1617        Silent,
1618        /// The bootstrap broker closes the connection on the first
1619        /// `FindCoordinator`.
1620        FindCoordinatorClosedOnce,
1621    }
1622
1623    /// An address on which no listener accepts connections.
1624    async fn refused_address() -> std::net::SocketAddr {
1625        let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
1626            .await
1627            .expect("bind a port");
1628        listener.local_addr().expect("local address")
1629    }
1630
1631    fn next_code(codes: &[i16], requests: &mut usize) -> i16 {
1632        let code = codes[(*requests).min(codes.len() - 1)];
1633        *requests += 1;
1634        code
1635    }
1636
1637    /// An `ApiVersions` response for the retry mocks.
1638    fn retry_api_versions() -> Vec<u8> {
1639        let api_version = |api_key, min_version, max_version| ApiVersion {
1640            api_key,
1641            min_version,
1642            max_version,
1643            ..Default::default()
1644        };
1645        encode_response(
1646            &ApiVersionsResponse {
1647                api_keys: vec![
1648                    api_version(api_versions_request::API_KEY, 0, 0),
1649                    api_version(find_coordinator_request::API_KEY, 0, 0),
1650                    api_version(metadata_request::API_KEY, 13, 13),
1651                    api_version(init_producer_id_request::API_KEY, 0, 0),
1652                    api_version(describe_transactions_request::API_KEY, 0, 0),
1653                ],
1654                ..Default::default()
1655            },
1656            0,
1657            false,
1658        )
1659    }
1660
1661    /// A mock broker that answers `FindCoordinator` with `coordinator`, and
1662    /// `DescribeTransactions` and `InitProducerId` with the codes of `script`.
1663    async fn scripted_transaction_broker(
1664        script: Arc<Mutex<CoordinatorScript>>,
1665        coordinator: Arc<Mutex<Option<std::net::SocketAddr>>>,
1666    ) -> MockBroker {
1667        MockBroker::start_with_replies(move |api_key, version, _, _| {
1668            let mut script = script.lock().expect("script lock");
1669            let script = &mut *script;
1670            if api_key == find_coordinator_request::API_KEY && script.closed_find_coordinators > 0 {
1671                script.closed_find_coordinators -= 1;
1672                script.find_coordinator_requests += 1;
1673                return MockReply::Close;
1674            }
1675            let reply = match api_key {
1676                api_versions_request::API_KEY => Some(retry_api_versions()),
1677                find_coordinator_request::API_KEY => {
1678                    let error_code = next_code(
1679                        &script.find_coordinator,
1680                        &mut script.find_coordinator_requests,
1681                    );
1682                    let addr = match script.unreachable {
1683                        Some(unreachable)
1684                            if script.find_coordinator_requests <= script.unreachable_answers =>
1685                        {
1686                            unreachable
1687                        }
1688                        _ => coordinator
1689                            .lock()
1690                            .expect("coordinator lock")
1691                            .expect("coordinator address"),
1692                    };
1693                    Some(encode_response(
1694                        &FindCoordinatorResponse {
1695                            error_code,
1696                            node_id: 2,
1697                            host: addr.ip().to_string(),
1698                            port: i32::from(addr.port()),
1699                            ..Default::default()
1700                        },
1701                        version,
1702                        false,
1703                    ))
1704                }
1705                describe_transactions_request::API_KEY => {
1706                    let error_code =
1707                        next_code(&script.coordinator, &mut script.coordinator_requests);
1708                    if script.silent {
1709                        return MockReply::Silent;
1710                    }
1711                    Some(encode_response(
1712                        &DescribeTransactionsResponse {
1713                            transaction_states: vec![TransactionState {
1714                                error_code,
1715                                transactional_id: "payments".to_owned(),
1716                                transaction_state: "Ongoing".to_owned(),
1717                                producer_id: 4242,
1718                                producer_epoch: 7,
1719                                ..Default::default()
1720                            }],
1721                            ..Default::default()
1722                        },
1723                        version,
1724                        true,
1725                    ))
1726                }
1727                init_producer_id_request::API_KEY => {
1728                    let error_code =
1729                        next_code(&script.coordinator, &mut script.coordinator_requests);
1730                    if script.silent {
1731                        return MockReply::Silent;
1732                    }
1733                    Some(encode_response(
1734                        &InitProducerIdResponse {
1735                            error_code,
1736                            producer_id: 4242,
1737                            producer_epoch: 8,
1738                            ..Default::default()
1739                        },
1740                        version,
1741                        false,
1742                    ))
1743                }
1744                _ => None,
1745            };
1746            reply.map_or(MockReply::Silent, MockReply::Respond)
1747        })
1748        .await
1749    }
1750
1751    /// A call that stops at its deadline gives a timeout. The retry tests give
1752    /// it Kafka's `REQUEST_TIMED_OUT` code.
1753    const TIMED_OUT: i16 = 7;
1754    /// A deadline that several attempts fit in.
1755    const LONG: Duration = Duration::from_secs(5);
1756    /// One attempt fits in `NOW`, and the backoff after it reaches the
1757    /// deadline, so the call makes no second attempt.
1758    const NOW: Duration = Duration::from_millis(500);
1759    /// A deadline shorter than the request timeout.
1760    const SHORT: Duration = Duration::from_millis(300);
1761
1762    type TransactionOutcome = Result<(), (&'static str, i16)>;
1763
1764    type TransactionCase = (
1765        &'static str,
1766        TransactionCall,
1767        Vec<i16>,
1768        Vec<i16>,
1769        (Duration, CoordinatorBehavior),
1770        (TransactionOutcome, usize, usize),
1771    );
1772
1773    const OK: TransactionOutcome = Ok(());
1774
1775    fn failed(api: &'static str, code: i16) -> TransactionOutcome {
1776        Err((api, code))
1777    }
1778
1779    /// Run one retry case against a scripted bootstrap broker and
1780    /// coordinator.
1781    async fn run_transaction_case(case: TransactionCase) {
1782        use TransactionCall::{Describe, ForceTerminate};
1783
1784        let (name, call, find_coordinator, coordinator, (timeout, behavior), expected) = case;
1785        let backoff = if timeout == LONG {
1786            Duration::from_millis(1)
1787        } else {
1788            timeout
1789        };
1790        let script = Arc::new(Mutex::new(CoordinatorScript {
1791            find_coordinator,
1792            coordinator,
1793            unreachable_answers: usize::from(behavior == CoordinatorBehavior::UnreachableOnce),
1794            unreachable: Some(refused_address().await),
1795            silent: behavior == CoordinatorBehavior::Silent,
1796            closed_find_coordinators: usize::from(
1797                behavior == CoordinatorBehavior::FindCoordinatorClosedOnce,
1798            ),
1799            ..CoordinatorScript::default()
1800        }));
1801        let coordinator_addr = Arc::new(Mutex::new(None));
1802        let coordinator =
1803            scripted_transaction_broker(Arc::clone(&script), Arc::clone(&coordinator_addr)).await;
1804        *coordinator_addr.lock().expect("coordinator lock") = Some(coordinator.addr);
1805        let bootstrap =
1806            scripted_transaction_broker(Arc::clone(&script), Arc::clone(&coordinator_addr)).await;
1807        let admin = AdminClient::connect(&[bootstrap.addr.to_string()])
1808            .await
1809            .expect("admin connects");
1810        let retry = RetryPolicy {
1811            timeout,
1812            initial_backoff: backoff,
1813            max_backoff: backoff,
1814            jitter: 0.0,
1815            max_retries: u32::MAX,
1816        };
1817
1818        let started = tokio::time::Instant::now();
1819        let result = match call {
1820            Describe => admin
1821                .describe_transaction_with_retry("payments", retry)
1822                .await
1823                .map(|_| ()),
1824            ForceTerminate => {
1825                admin
1826                    .force_terminate_transaction_with_retry("payments", retry)
1827                    .await
1828            }
1829        }
1830        .map_err(|error| match error {
1831            AdminError::Broker { api, code, .. } => (api, code),
1832            AdminError::Transport(ClientError::Timeout(_)) => ("timeout", TIMED_OUT),
1833            other => panic!("case {name}: unexpected error {other:?}"),
1834        });
1835
1836        bootstrap.stop();
1837        coordinator.stop();
1838        let script = script.lock().expect("script lock");
1839        // A call never runs far past its deadline.
1840        let within_deadline = started.elapsed() < timeout + Duration::from_secs(2);
1841        assert2::assert!(
1842            (
1843                result,
1844                script.find_coordinator_requests,
1845                script.coordinator_requests,
1846                within_deadline
1847            ) == (expected.0, expected.1, expected.2, true),
1848            "case {name}"
1849        );
1850    }
1851
1852    /// Apache Kafka's `DescribeTransactionsHandler.handleError` retries
1853    /// `COORDINATOR_LOAD_IN_PROGRESS` (14) on the same coordinator.
1854    /// `FenceProducersHandler.handleError` retries 14 and
1855    /// `CONCURRENT_TRANSACTIONS` (51) on the same coordinator. Both unmap the
1856    /// transactional ID on `COORDINATOR_NOT_AVAILABLE` (15) and
1857    /// `NOT_COORDINATOR` (16), so the driver finds the coordinator again, and
1858    /// fail on every other code. `CoordinatorStrategy.handleError` retries a
1859    /// `FindCoordinator` answer of 14 or 15. The driver stops at the call
1860    /// timeout.
1861    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1862    async fn transaction_calls_retry_coordinator_errors_as_kafka_does() {
1863        use TransactionCall::{Describe, ForceTerminate};
1864
1865        for case in [
1866            (
1867                "describe: no error",
1868                Describe,
1869                vec![0],
1870                vec![0],
1871                (LONG, CoordinatorBehavior::Normal),
1872                (OK, 1, 1),
1873            ),
1874            (
1875                "describe: coordinator load in progress retries on the same coordinator",
1876                Describe,
1877                vec![0],
1878                vec![14, 14, 0],
1879                (LONG, CoordinatorBehavior::Normal),
1880                (OK, 1, 3),
1881            ),
1882            (
1883                "describe: not coordinator finds the coordinator again",
1884                Describe,
1885                vec![0],
1886                vec![16, 0],
1887                (LONG, CoordinatorBehavior::Normal),
1888                (OK, 2, 2),
1889            ),
1890            (
1891                "describe: coordinator not available finds the coordinator again",
1892                Describe,
1893                vec![0],
1894                vec![15, 0],
1895                (LONG, CoordinatorBehavior::Normal),
1896                (OK, 2, 2),
1897            ),
1898            (
1899                "describe: concurrent transactions is final",
1900                Describe,
1901                vec![0],
1902                vec![51],
1903                (LONG, CoordinatorBehavior::Normal),
1904                (failed("DescribeTransactions", 51), 1, 1),
1905            ),
1906            (
1907                "describe: transactional id not found is final",
1908                Describe,
1909                vec![0],
1910                vec![105],
1911                (LONG, CoordinatorBehavior::Normal),
1912                (failed("DescribeTransactions", 105), 1, 1),
1913            ),
1914            (
1915                "describe: coordinator load in progress past the timeout times out",
1916                Describe,
1917                vec![0],
1918                vec![14],
1919                (NOW, CoordinatorBehavior::Normal),
1920                (failed("timeout", TIMED_OUT), 1, 1),
1921            ),
1922            (
1923                "describe: find coordinator not available, then the coordinator",
1924                Describe,
1925                vec![15, 0],
1926                vec![0],
1927                (LONG, CoordinatorBehavior::Normal),
1928                (OK, 2, 1),
1929            ),
1930            (
1931                "describe: find coordinator authorization failed is final",
1932                Describe,
1933                vec![53],
1934                vec![0],
1935                (LONG, CoordinatorBehavior::Normal),
1936                (failed("FindCoordinator", 53), 1, 0),
1937            ),
1938            (
1939                "fence: no error",
1940                ForceTerminate,
1941                vec![0],
1942                vec![0],
1943                (LONG, CoordinatorBehavior::Normal),
1944                (OK, 1, 1),
1945            ),
1946            (
1947                "fence: coordinator load in progress retries on the same coordinator",
1948                ForceTerminate,
1949                vec![0],
1950                vec![14, 0],
1951                (LONG, CoordinatorBehavior::Normal),
1952                (OK, 1, 2),
1953            ),
1954            (
1955                "fence: concurrent transactions retries on the same coordinator",
1956                ForceTerminate,
1957                vec![0],
1958                vec![51, 51, 0],
1959                (LONG, CoordinatorBehavior::Normal),
1960                (OK, 1, 3),
1961            ),
1962            (
1963                "fence: not coordinator finds the coordinator again",
1964                ForceTerminate,
1965                vec![0],
1966                vec![16, 0],
1967                (LONG, CoordinatorBehavior::Normal),
1968                (OK, 2, 2),
1969            ),
1970            (
1971                "fence: cluster authorization failed is final",
1972                ForceTerminate,
1973                vec![0],
1974                vec![31],
1975                (LONG, CoordinatorBehavior::Normal),
1976                (failed("InitProducerId", 31), 1, 1),
1977            ),
1978            (
1979                "fence: concurrent transactions past the timeout times out",
1980                ForceTerminate,
1981                vec![0],
1982                vec![51],
1983                (NOW, CoordinatorBehavior::Normal),
1984                (failed("timeout", TIMED_OUT), 1, 1),
1985            ),
1986            (
1987                "fence: find coordinator load in progress, then the coordinator",
1988                ForceTerminate,
1989                vec![14, 0],
1990                vec![0],
1991                (LONG, CoordinatorBehavior::Normal),
1992                (OK, 2, 1),
1993            ),
1994        ] {
1995            run_transaction_case(case).await;
1996        }
1997    }
1998
1999    /// Kafka's `AdminApiDriver.onFailure` looks the coordinator up again
2000    /// after a disconnect, and `KafkaAdminClient` times out a call in flight
2001    /// at its deadline.
2002    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2003    async fn transaction_calls_find_the_coordinator_again_after_connection_failures() {
2004        use TransactionCall::{Describe, ForceTerminate};
2005
2006        for case in [
2007            (
2008                "describe: a closed connection during FindCoordinator recovers",
2009                Describe,
2010                vec![0],
2011                vec![0],
2012                (LONG, CoordinatorBehavior::FindCoordinatorClosedOnce),
2013                (OK, 2, 1),
2014            ),
2015            (
2016                "fence: a closed connection during FindCoordinator recovers",
2017                ForceTerminate,
2018                vec![0],
2019                vec![0],
2020                (LONG, CoordinatorBehavior::FindCoordinatorClosedOnce),
2021                (OK, 2, 1),
2022            ),
2023            (
2024                "describe: an unreachable coordinator finds the coordinator again",
2025                Describe,
2026                vec![0],
2027                vec![0],
2028                (LONG, CoordinatorBehavior::UnreachableOnce),
2029                (OK, 2, 1),
2030            ),
2031            (
2032                "describe: a silent coordinator stops the attempt at the deadline",
2033                Describe,
2034                vec![0],
2035                vec![0],
2036                (SHORT, CoordinatorBehavior::Silent),
2037                (failed("timeout", TIMED_OUT), 1, 1),
2038            ),
2039            (
2040                "fence: an unreachable coordinator finds the coordinator again",
2041                ForceTerminate,
2042                vec![0],
2043                vec![0],
2044                (LONG, CoordinatorBehavior::UnreachableOnce),
2045                (OK, 2, 1),
2046            ),
2047            (
2048                "fence: a silent coordinator stops the attempt at the deadline",
2049                ForceTerminate,
2050                vec![0],
2051                vec![0],
2052                (SHORT, CoordinatorBehavior::Silent),
2053                (failed("timeout", TIMED_OUT), 1, 1),
2054            ),
2055        ] {
2056            run_transaction_case(case).await;
2057        }
2058    }
2059
2060    /// Kafka's `DescribeTransactionsHandler` names the broker that answered
2061    /// as the coordinator, which is the node that `FindCoordinator` gave.
2062    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2063    async fn describe_transaction_names_the_coordinator_that_find_coordinator_gave() {
2064        let script = Arc::new(Mutex::new(CoordinatorScript {
2065            find_coordinator: vec![0],
2066            coordinator: vec![0],
2067            ..CoordinatorScript::default()
2068        }));
2069        let coordinator_addr = Arc::new(Mutex::new(None));
2070        let coordinator =
2071            scripted_transaction_broker(Arc::clone(&script), Arc::clone(&coordinator_addr)).await;
2072        *coordinator_addr.lock().expect("coordinator lock") = Some(coordinator.addr);
2073        let bootstrap =
2074            scripted_transaction_broker(Arc::clone(&script), Arc::clone(&coordinator_addr)).await;
2075        let admin = AdminClient::connect(&[bootstrap.addr.to_string()])
2076            .await
2077            .expect("admin connects");
2078
2079        let described = admin.describe_transaction("payments").await;
2080
2081        bootstrap.stop();
2082        coordinator.stop();
2083        assert2::assert!(let Ok(described) = described);
2084        assert2::assert!(
2085            described
2086                == TransactionDescription {
2087                    transactional_id: "payments".to_owned(),
2088                    coordinator_id: 2,
2089                    state: "Ongoing".to_owned(),
2090                    producer_id: 4242,
2091                    producer_epoch: 7,
2092                    timeout: Time::from_millis(0),
2093                    start_time_ms: Some(0),
2094                    topic_partitions: BTreeSet::new(),
2095                }
2096        );
2097    }
2098
2099    /// Kafka's `describeTransactions` runs every ID under one call deadline
2100    /// (`invokeDriver`), so a slow first ID leaves less time for the next.
2101    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2102    async fn describe_transactions_shares_one_deadline_across_ids() {
2103        let mut coordinator_codes = vec![14; 8];
2104        coordinator_codes.extend([0, 14]);
2105        let script = Arc::new(Mutex::new(CoordinatorScript {
2106            find_coordinator: vec![0],
2107            coordinator: coordinator_codes,
2108            ..CoordinatorScript::default()
2109        }));
2110        let coordinator_addr = Arc::new(Mutex::new(None));
2111        let coordinator =
2112            scripted_transaction_broker(Arc::clone(&script), Arc::clone(&coordinator_addr)).await;
2113        *coordinator_addr.lock().expect("coordinator lock") = Some(coordinator.addr);
2114        let bootstrap =
2115            scripted_transaction_broker(Arc::clone(&script), Arc::clone(&coordinator_addr)).await;
2116        let admin = AdminClient::connect_with_config(
2117            &[bootstrap.addr.to_string()],
2118            crate::AdminClientConfig {
2119                request_timeout: krabka_units::millis(200),
2120                default_api_timeout: Some(krabka_units::millis(1500)),
2121                retry_backoff: krabka_units::millis(100),
2122                retry_backoff_max: krabka_units::millis(100),
2123                ..AdminClientConfig::default()
2124            },
2125        )
2126        .await
2127        .expect("admin connects");
2128
2129        let started = tokio::time::Instant::now();
2130        let result = admin.describe_transactions(&["payments", "payments"]).await;
2131        let elapsed = started.elapsed();
2132
2133        bootstrap.stop();
2134        coordinator.stop();
2135        // The first ID needs about 800 ms. The second ID gets the rest of the
2136        // 1.5 s deadline, not a new 1.5 s.
2137        assert2::assert!(matches!(
2138            result,
2139            Err(AdminError::Transport(ClientError::Timeout(_)))
2140        ));
2141        assert2::assert!(elapsed < Duration::from_millis(2200), "{elapsed:?}");
2142    }
2143
2144    /// The transaction that the abort tests fence.
2145    fn orders_spec() -> AbortTransactionSpec {
2146        AbortTransactionSpec {
2147            topic: "orders".to_owned(),
2148            partition: 1,
2149            producer_id: 4242,
2150            producer_epoch: 7,
2151            coordinator_epoch: 3,
2152        }
2153    }
2154
2155    /// A `Metadata` answer in which broker 1 at `addr` leads `orders`-0 and
2156    /// `orders`-1, with `partition_error` on each partition, and `missing`
2157    /// is an unknown topic.
2158    fn orders_metadata(addr: std::net::SocketAddr, partition_error: i16) -> MetadataResponse {
2159        let partition = |partition_index| MetadataResponsePartition {
2160            error_code: partition_error,
2161            partition_index,
2162            leader_id: 1,
2163            ..Default::default()
2164        };
2165        MetadataResponse {
2166            brokers: vec![MetadataResponseBroker {
2167                node_id: 1,
2168                host: addr.ip().to_string(),
2169                port: i32::from(addr.port()),
2170                ..Default::default()
2171            }],
2172            topics: vec![
2173                MetadataResponseTopic {
2174                    name: Some("orders".to_owned()),
2175                    partitions: vec![partition(0), partition(1)],
2176                    ..Default::default()
2177                },
2178                MetadataResponseTopic {
2179                    error_code: 3,
2180                    name: Some("missing".to_owned()),
2181                    ..Default::default()
2182                },
2183            ],
2184            ..Default::default()
2185        }
2186    }
2187
2188    /// The outcome of an admin call reduced to what the tests compare: the
2189    /// failing API and error code, or `("protocol", 0)` for a protocol error.
2190    fn outcome_of(result: Result<(), AdminError>) -> TransactionOutcome {
2191        result.map_err(|error| match error {
2192            AdminError::Broker { api, code, .. } => (api, code),
2193            AdminError::Protocol(_) => ("protocol", 0),
2194            other => panic!("unexpected error {other:?}"),
2195        })
2196    }
2197
2198    #[test]
2199    fn partition_leader_address_resolves_the_leader_or_reports_the_metadata_error() {
2200        let addr: std::net::SocketAddr = "127.0.0.1:9092".parse().expect("address");
2201        let mut leaderless = orders_metadata(addr, 0);
2202        leaderless.topics[0].partitions[1].leader_id = 9;
2203        for (name, metadata, topic, partition, expected) in [
2204            (
2205                "leader",
2206                orders_metadata(addr, 0),
2207                "orders",
2208                1,
2209                Ok("127.0.0.1:9092".to_owned()),
2210            ),
2211            (
2212                "partition error",
2213                orders_metadata(addr, NOT_LEADER_OR_FOLLOWER),
2214                "orders",
2215                1,
2216                Err(("Metadata", NOT_LEADER_OR_FOLLOWER)),
2217            ),
2218            (
2219                "topic error",
2220                orders_metadata(addr, 0),
2221                "missing",
2222                0,
2223                Err(("Metadata", 3)),
2224            ),
2225            (
2226                "unlisted topic",
2227                orders_metadata(addr, 0),
2228                "absent",
2229                0,
2230                Err(("protocol", 0)),
2231            ),
2232            (
2233                "unlisted partition",
2234                orders_metadata(addr, 0),
2235                "orders",
2236                5,
2237                Err(("protocol", 0)),
2238            ),
2239            (
2240                "unlisted leader",
2241                leaderless,
2242                "orders",
2243                1,
2244                Err(("protocol", 0)),
2245            ),
2246        ] {
2247            let address = partition_leader_address(topic, partition, &metadata);
2248            let actual = match address {
2249                Ok(address) => Ok(address),
2250                Err(error) => Err(outcome_of(Err(error)).expect_err("an error")),
2251            };
2252            assert2::assert!(actual == expected, "case {name}");
2253        }
2254    }
2255
2256    #[test]
2257    fn abort_request_writes_one_abort_marker_for_the_partition() {
2258        assert2::assert!(
2259            abort_transaction_request(&orders_spec())
2260                == WriteTxnMarkersRequest {
2261                    markers: vec![WritableTxnMarker {
2262                        producer_id: 4242,
2263                        producer_epoch: 7,
2264                        transaction_result: false,
2265                        topics: vec![WritableTxnMarkerTopic {
2266                            name: "orders".to_owned(),
2267                            partition_indexes: vec![1],
2268                            ..Default::default()
2269                        }],
2270                        coordinator_epoch: 3,
2271                        ..Default::default()
2272                    }],
2273                    ..Default::default()
2274                }
2275        );
2276    }
2277
2278    /// A `WriteTxnMarkers` answer with `code` for `producer_id` on
2279    /// `topic`-`partition`.
2280    fn markers_response(
2281        producer_id: i64,
2282        topic: &str,
2283        partition: i32,
2284        code: i16,
2285    ) -> WriteTxnMarkersResponse {
2286        WriteTxnMarkersResponse {
2287            markers: vec![WritableTxnMarkerResult {
2288                producer_id,
2289                topics: vec![WritableTxnMarkerTopicResult {
2290                    name: topic.to_owned(),
2291                    partitions: vec![WritableTxnMarkerPartitionResult {
2292                        partition_index: partition,
2293                        error_code: code,
2294                        ..Default::default()
2295                    }],
2296                    ..Default::default()
2297                }],
2298                ..Default::default()
2299            }],
2300            ..Default::default()
2301        }
2302    }
2303
2304    #[test]
2305    fn abort_result_maps_the_matching_partition_result() {
2306        for (name, response, expected) in [
2307            ("no error", markers_response(4242, "orders", 1, 0), OK),
2308            (
2309                "partition error",
2310                markers_response(4242, "orders", 1, 53),
2311                failed("WriteTxnMarkers", 53),
2312            ),
2313            (
2314                "other producer",
2315                markers_response(1, "orders", 1, 0),
2316                failed("protocol", 0),
2317            ),
2318            (
2319                "other topic",
2320                markers_response(4242, "payments", 1, 0),
2321                failed("protocol", 0),
2322            ),
2323            (
2324                "other partition",
2325                markers_response(4242, "orders", 0, 0),
2326                failed("protocol", 0),
2327            ),
2328        ] {
2329            let result = outcome_of(abort_transaction_result(&orders_spec(), response));
2330            assert2::assert!(result == expected, "case {name}");
2331        }
2332    }
2333
2334    /// Apache Kafka's `AbortTransactionHandler.handleResponse` looks the
2335    /// leader up again after `NOT_LEADER_OR_FOLLOWER` and
2336    /// `LEADER_NOT_AVAILABLE`, and fails on every other code.
2337    #[test]
2338    fn abort_retries_only_leadership_errors() {
2339        #[derive(Debug, PartialEq, Eq)]
2340        enum Action {
2341            Done,
2342            FindLeader,
2343        }
2344        for (code, expected) in [
2345            (0, Action::Done),
2346            (LEADER_NOT_AVAILABLE, Action::FindLeader),
2347            (NOT_LEADER_OR_FOLLOWER, Action::FindLeader),
2348            (53, Action::Done),
2349            (47, Action::Done),
2350        ] {
2351            let result =
2352                abort_transaction_result(&orders_spec(), markers_response(4242, "orders", 1, code));
2353            let action = match abort_transaction_retry_action(result) {
2354                RetryAction::Done(_) => Action::Done,
2355                RetryAction::FindCoordinator(_) => Action::FindLeader,
2356                RetryAction::SameCoordinator(_) => panic!("code {code}: same leader"),
2357            };
2358            assert2::assert!(action == expected, "code {code}");
2359        }
2360    }
2361
2362    #[test]
2363    fn describe_producers_request_groups_partitions_by_topic() {
2364        let keys = [
2365            ("orders".to_owned(), 1),
2366            ("audit".to_owned(), 0),
2367            ("orders".to_owned(), 0),
2368        ];
2369
2370        assert2::assert!(
2371            describe_producers_request(&keys)
2372                == DescribeProducersRequest {
2373                    topics: vec![
2374                        DescribeProducersTopicRequest {
2375                            name: "audit".to_owned(),
2376                            partition_indexes: vec![0],
2377                            ..Default::default()
2378                        },
2379                        DescribeProducersTopicRequest {
2380                            name: "orders".to_owned(),
2381                            partition_indexes: vec![1, 0],
2382                            ..Default::default()
2383                        },
2384                    ],
2385                    ..Default::default()
2386                }
2387        );
2388    }
2389
2390    /// The `DescribeProducers` answer of the tests: one producer with an open
2391    /// transaction and one without on `orders`-0, `NOT_LEADER_OR_FOLLOWER` on
2392    /// `orders`-1, and a partition that nobody asked about.
2393    fn orders_producers() -> DescribeProducersResponse {
2394        DescribeProducersResponse {
2395            topics: vec![TopicResponse {
2396                name: "orders".to_owned(),
2397                partitions: vec![
2398                    PartitionResponse {
2399                        partition_index: 0,
2400                        active_producers: vec![
2401                            WireProducerState {
2402                                producer_id: 4242,
2403                                producer_epoch: 7,
2404                                last_sequence: 12,
2405                                last_timestamp: 1_700_000_000_123,
2406                                coordinator_epoch: 3,
2407                                current_txn_start_offset: 88,
2408                                ..Default::default()
2409                            },
2410                            WireProducerState {
2411                                producer_id: 11,
2412                                ..Default::default()
2413                            },
2414                        ],
2415                        ..Default::default()
2416                    },
2417                    PartitionResponse {
2418                        partition_index: 1,
2419                        error_code: NOT_LEADER_OR_FOLLOWER,
2420                        error_message: Some("moved".to_owned()),
2421                        ..Default::default()
2422                    },
2423                    PartitionResponse {
2424                        partition_index: 9,
2425                        ..Default::default()
2426                    },
2427                ],
2428                ..Default::default()
2429            }],
2430            ..Default::default()
2431        }
2432    }
2433
2434    /// The per-partition results of [`orders_producers`] for `orders`-0 and
2435    /// `orders`-1.
2436    fn orders_producer_results()
2437    -> BTreeMap<(String, i32), Result<Vec<ProducerStateInfo>, KafkaError>> {
2438        maplit::btreemap! {
2439            ("orders".to_owned(), 0) => Ok(vec![
2440            ProducerStateInfo {
2441                producer_id: 4242,
2442                producer_epoch: 7,
2443                last_sequence: 12,
2444                last_timestamp_ms: 1_700_000_000_123,
2445                coordinator_epoch: 3,
2446                current_txn_start_offset: Some(88),
2447            },
2448            ProducerStateInfo {
2449                producer_id: 11,
2450                producer_epoch: 0,
2451                last_sequence: -1,
2452                last_timestamp_ms: -1,
2453                coordinator_epoch: 0,
2454                current_txn_start_offset: None,
2455            },
2456        ]),
2457            ("orders".to_owned(), 1) => Err(KafkaError {
2458            code: NOT_LEADER_OR_FOLLOWER,
2459            name: "NOT_LEADER_OR_FOLLOWER",
2460            message: Some("moved".to_owned()),
2461        }),
2462        }
2463    }
2464
2465    #[test]
2466    fn describe_producers_results_map_each_requested_partition() {
2467        let keys = [
2468            ("orders".to_owned(), 0),
2469            ("orders".to_owned(), 1),
2470            ("orders".to_owned(), 2),
2471        ];
2472
2473        let mut expected = orders_producer_results();
2474        expected.insert(
2475            ("orders".to_owned(), 2),
2476            Err(KafkaError {
2477                code: -1,
2478                name: "UNKNOWN_SERVER_ERROR",
2479                message: Some(
2480                    "the DescribeProducers response did not contain a result for partition \
2481                     orders-2"
2482                        .to_owned(),
2483                ),
2484            }),
2485        );
2486        assert2::assert!(describe_producers_results(&keys, orders_producers()) == expected);
2487    }
2488
2489    #[test]
2490    fn list_transactions_request_carries_the_filter() {
2491        for (name, filter, expected) in [
2492            (
2493                "default matches everything",
2494                ListTransactionsFilter::builder().build(),
2495                ListTransactionsRequest::default(),
2496            ),
2497            (
2498                "every filter",
2499                ListTransactionsFilter::builder()
2500                    .state_filters(vec!["Ongoing".to_owned()])
2501                    .producer_id_filters(vec![4242])
2502                    .min_duration(90_000)
2503                    .build(),
2504                ListTransactionsRequest {
2505                    state_filters: vec!["Ongoing".to_owned()],
2506                    producer_id_filters: vec![4242],
2507                    duration_filter: 90_000,
2508                    ..Default::default()
2509                },
2510            ),
2511        ] {
2512            assert2::assert!(
2513                list_transactions_request(&filter) == expected,
2514                "case {name}"
2515            );
2516        }
2517    }
2518
2519    /// Decodes the request body of a mock broker: the client id, the tagged
2520    /// field byte of a flexible header, and the request itself.
2521    fn decode_request<T: for<'a> Decode<'a>>(mut body: &[u8], version: i16, flexible: bool) -> T {
2522        let client_id_len = body.get_i16();
2523        body.advance(usize::try_from(client_id_len).expect("client id length"));
2524        if flexible {
2525            body.advance(1);
2526        }
2527        T::decode(&mut body, version).expect("request decodes")
2528    }
2529
2530    /// An `ApiVersions` response for the leader-routed mocks.
2531    fn leader_api_versions() -> Vec<u8> {
2532        let api_version = |api_key, min_version, max_version| ApiVersion {
2533            api_key,
2534            min_version,
2535            max_version,
2536            ..Default::default()
2537        };
2538        encode_response(
2539            &ApiVersionsResponse {
2540                api_keys: vec![
2541                    api_version(api_versions_request::API_KEY, 0, 0),
2542                    api_version(metadata_request::API_KEY, 13, 13),
2543                    api_version(write_txn_markers_request::API_KEY, 1, 2),
2544                    api_version(list_transactions_request::API_KEY, 0, 2),
2545                ],
2546                ..Default::default()
2547            },
2548            0,
2549            false,
2550        )
2551    }
2552
2553    /// What a leader-routed mock broker saw and answers.
2554    #[derive(Default)]
2555    struct LeaderScript {
2556        /// The `WriteTxnMarkers` error codes, one per request. The last
2557        /// repeats.
2558        write_codes: Vec<i16>,
2559        /// The `WriteTxnMarkers` requests that the broker received.
2560        write_requests: Vec<WriteTxnMarkersRequest>,
2561        /// The `Metadata` requests that the broker received.
2562        metadata_requests: usize,
2563        /// The `ListTransactions` requests that the broker received.
2564        list_requests: Vec<ListTransactionsRequest>,
2565        /// The `ListTransactions` answer of the broker.
2566        listing: ListTransactionsResponse,
2567    }
2568
2569    /// A mock broker that leads every `orders` partition. `cluster` names the
2570    /// brokers that its `Metadata` answer lists, itself first.
2571    async fn leader_broker(
2572        script: Arc<Mutex<LeaderScript>>,
2573        cluster: Arc<Mutex<Vec<std::net::SocketAddr>>>,
2574    ) -> MockBroker {
2575        MockBroker::start(move |api_key, version, _, body| {
2576            let mut script = script.lock().expect("script lock");
2577            match api_key {
2578                api_versions_request::API_KEY => Some(leader_api_versions()),
2579                metadata_request::API_KEY => {
2580                    script.metadata_requests += 1;
2581                    let cluster = cluster.lock().expect("cluster lock").clone();
2582                    let mut metadata = orders_metadata(cluster[0], 0);
2583                    metadata.brokers = cluster
2584                        .iter()
2585                        .zip(1..)
2586                        .map(|(addr, node_id)| MetadataResponseBroker {
2587                            node_id,
2588                            host: addr.ip().to_string(),
2589                            port: i32::from(addr.port()),
2590                            ..Default::default()
2591                        })
2592                        .collect();
2593                    Some(encode_response(&metadata, version, true))
2594                }
2595                write_txn_markers_request::API_KEY => {
2596                    let request: WriteTxnMarkersRequest = decode_request(body, version, true);
2597                    let code = script.write_codes[script
2598                        .write_requests
2599                        .len()
2600                        .min(script.write_codes.len() - 1)];
2601                    script.write_requests.push(request);
2602                    Some(encode_response(
2603                        &markers_response(4242, "orders", 1, code),
2604                        version,
2605                        true,
2606                    ))
2607                }
2608                list_transactions_request::API_KEY => {
2609                    script
2610                        .list_requests
2611                        .push(decode_request(body, version, true));
2612                    Some(encode_response(&script.listing, version, true))
2613                }
2614                _ => None,
2615            }
2616        })
2617        .await
2618    }
2619
2620    /// An admin client on `bootstrap` that retries after 1 ms.
2621    async fn fast_admin(bootstrap: std::net::SocketAddr) -> AdminClient {
2622        AdminClient::connect_with_config(
2623            &[bootstrap.to_string()],
2624            crate::AdminClientConfig {
2625                request_timeout: krabka_units::secs(2),
2626                retry_backoff: krabka_units::millis(1),
2627                retry_backoff_max: krabka_units::millis(1),
2628                default_api_timeout: Some(krabka_units::secs(5)),
2629                ..AdminClientConfig::default()
2630            },
2631        )
2632        .await
2633        .expect("admin connects")
2634    }
2635
2636    /// Apache Kafka's `AbortTransactionHandler` sends one `WriteTxnMarkers`
2637    /// to the partition leader, looks the leader up again after
2638    /// `NOT_LEADER_OR_FOLLOWER`, and fails on every other code.
2639    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2640    async fn abort_transaction_writes_an_abort_marker_to_the_leader() {
2641        for (name, write_codes, expected) in [
2642            ("no error", vec![0], (OK, 1, 1)),
2643            (
2644                "not leader or follower finds the leader again",
2645                vec![NOT_LEADER_OR_FOLLOWER, 0],
2646                (OK, 2, 2),
2647            ),
2648            (
2649                "transactional id authorization failed is final",
2650                vec![53],
2651                (failed("WriteTxnMarkers", 53), 1, 1),
2652            ),
2653        ] {
2654            let script = Arc::new(Mutex::new(LeaderScript {
2655                write_codes,
2656                ..LeaderScript::default()
2657            }));
2658            let cluster = Arc::new(Mutex::new(Vec::new()));
2659            let broker = leader_broker(Arc::clone(&script), Arc::clone(&cluster)).await;
2660            cluster.lock().expect("cluster lock").push(broker.addr);
2661            let admin = fast_admin(broker.addr).await;
2662
2663            let result = outcome_of(admin.abort_transaction(&orders_spec()).await);
2664
2665            broker.stop();
2666            let script = script.lock().expect("script lock");
2667            let expected_requests = vec![abort_transaction_request(&orders_spec()); expected.2];
2668            assert2::assert!(
2669                (
2670                    result,
2671                    script.metadata_requests,
2672                    script.write_requests.clone()
2673                ) == (expected.0, expected.1, expected_requests),
2674                "case {name}"
2675            );
2676        }
2677    }
2678
2679    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2680    async fn abort_transaction_rejects_an_empty_topic() {
2681        let script = Arc::new(Mutex::new(LeaderScript::default()));
2682        let cluster = Arc::new(Mutex::new(Vec::new()));
2683        let broker = leader_broker(Arc::clone(&script), Arc::clone(&cluster)).await;
2684        cluster.lock().expect("cluster lock").push(broker.addr);
2685        let admin = fast_admin(broker.addr).await;
2686
2687        let result = admin
2688            .abort_transaction(&AbortTransactionSpec {
2689                topic: String::new(),
2690                ..orders_spec()
2691            })
2692            .await;
2693
2694        broker.stop();
2695        assert2::assert!(outcome_of(result) == failed("protocol", 0));
2696        assert2::assert!(script.lock().expect("script lock").metadata_requests == 0);
2697    }
2698
2699    /// The `DescribeProducers` answer to `request`: the producers of
2700    /// [`orders_producers`] on `orders`-0, and `orders_1_code` on `orders`-1.
2701    fn producers_answer(
2702        request: &DescribeProducersRequest,
2703        orders_1_code: i16,
2704    ) -> DescribeProducersResponse {
2705        let orders_0 = orders_producers().topics[0].partitions[0].clone();
2706        DescribeProducersResponse {
2707            topics: request
2708                .topics
2709                .iter()
2710                .map(|topic| TopicResponse {
2711                    name: topic.name.clone(),
2712                    partitions: topic
2713                        .partition_indexes
2714                        .iter()
2715                        .map(|&partition_index| {
2716                            if partition_index == 0 {
2717                                orders_0.clone()
2718                            } else {
2719                                PartitionResponse {
2720                                    partition_index,
2721                                    error_code: orders_1_code,
2722                                    error_message: (orders_1_code != 0)
2723                                        .then(|| "refused".to_owned()),
2724                                    ..Default::default()
2725                                }
2726                            }
2727                        })
2728                        .collect(),
2729                    ..Default::default()
2730                })
2731                .collect(),
2732            ..Default::default()
2733        }
2734    }
2735
2736    /// Apache Kafka's `DescribeProducersHandler` batches the partitions of
2737    /// one leader into one request, gives each partition its own result, and
2738    /// sends a partition back to the `PartitionLeaderStrategy` lookup after
2739    /// `NOT_LEADER_OR_FOLLOWER` or a lost connection to its leader.
2740    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2741    async fn describe_producers_finds_the_leader_again_as_kafka_does() {
2742        let both = vec![("orders".to_owned(), 0), ("orders".to_owned(), 1)];
2743        let only_1 = vec![("orders".to_owned(), 1)];
2744        let refused = refused_address().await;
2745        let producers_0 = orders_producer_results()[&both[0]].clone();
2746        for (name, leaders, orders_1_codes, expected) in [
2747            (
2748                "success",
2749                vec![None],
2750                vec![0],
2751                (Ok(Vec::new()), 1, vec![both.clone()]),
2752            ),
2753            (
2754                "not leader or follower finds the leader again",
2755                vec![None],
2756                vec![NOT_LEADER_OR_FOLLOWER, 0],
2757                (Ok(Vec::new()), 2, vec![both.clone(), only_1.clone()]),
2758            ),
2759            (
2760                "a refused connection finds the leader again",
2761                vec![Some(refused), None],
2762                vec![0],
2763                (Ok(Vec::new()), 2, vec![both.clone()]),
2764            ),
2765            (
2766                "topic authorization failed is final",
2767                vec![None],
2768                vec![29],
2769                (
2770                    Err(KafkaError {
2771                        code: 29,
2772                        name: "TOPIC_AUTHORIZATION_FAILED",
2773                        message: Some("refused".to_owned()),
2774                    }),
2775                    1,
2776                    vec![both.clone()],
2777                ),
2778            ),
2779        ] {
2780            let seen = Arc::new(Mutex::new(Seen::default()));
2781            let broker = partition_leader_broker(
2782                vec![
2783                    ApiVersionRange::builder()
2784                        .api_key(describe_producers_request::API_KEY)
2785                        .build(),
2786                ],
2787                describe_producers_request::API_KEY,
2788                leaders,
2789                Arc::clone(&seen),
2790                move |n, version, body| {
2791                    let request: DescribeProducersRequest = decode_request(body, version, true);
2792                    let code = orders_1_codes[n.min(orders_1_codes.len() - 1)];
2793                    encode_response(&producers_answer(&request, code), version, true)
2794                },
2795            )
2796            .await;
2797            let admin = leader_fast_admin(broker.addr, krabka_units::secs(5)).await;
2798
2799            let result = admin.describe_producers(&both).await;
2800
2801            broker.stop();
2802            let seen = seen.lock().expect("seen lock");
2803            let requests = seen
2804                .requests
2805                .iter()
2806                .map(|(version, body)| {
2807                    decode_request::<DescribeProducersRequest>(body, *version, true)
2808                })
2809                .collect::<Vec<_>>();
2810            assert2::assert!(
2811                (result, seen.metadata_requests, requests)
2812                    == (
2813                        maplit::btreemap! {
2814                            both[0].clone() => producers_0.clone(),
2815                            both[1].clone() => expected.0,
2816                        },
2817                        expected.1,
2818                        expected
2819                            .2
2820                            .iter()
2821                            .map(|keys| describe_producers_request(keys))
2822                            .collect::<Vec<_>>(),
2823                    ),
2824                "case {name}"
2825            );
2826        }
2827    }
2828
2829    /// A `ListTransactions` answer with `error_code` and one transaction per
2830    /// ID in `transactional_ids`.
2831    fn listing(error_code: i16, transactional_ids: &[&str]) -> ListTransactionsResponse {
2832        ListTransactionsResponse {
2833            error_code,
2834            transaction_states: transactional_ids
2835                .iter()
2836                .zip(1..)
2837                .map(|(transactional_id, producer_id)| ListedTransactionState {
2838                    transactional_id: (*transactional_id).to_owned(),
2839                    producer_id,
2840                    transaction_state: "Ongoing".to_owned(),
2841                    ..Default::default()
2842                })
2843                .collect(),
2844            ..Default::default()
2845        }
2846    }
2847
2848    /// Apache Kafka's `ListTransactionsHandler` asks every broker
2849    /// (`AllBrokersStrategy`) and merges the listings. A broker error fails
2850    /// the call.
2851    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2852    async fn list_transactions_merges_the_listing_of_every_broker() {
2853        let listed = |transactional_id: &str, producer_id| TransactionListing {
2854            transactional_id: transactional_id.to_owned(),
2855            producer_id,
2856            state: "Ongoing".to_owned(),
2857        };
2858        for (name, second_listing, expected) in [
2859            (
2860                "every broker answers",
2861                listing(0, &["audit"]),
2862                Ok(vec![
2863                    listed("payments", 1),
2864                    listed("orders", 2),
2865                    listed("audit", 1),
2866                ]),
2867            ),
2868            (
2869                "a broker is loading its coordinator",
2870                listing(COORDINATOR_LOAD_IN_PROGRESS, &[]),
2871                Err(("ListTransactions", COORDINATOR_LOAD_IN_PROGRESS)),
2872            ),
2873        ] {
2874            let first = Arc::new(Mutex::new(LeaderScript {
2875                listing: listing(0, &["payments", "orders"]),
2876                ..LeaderScript::default()
2877            }));
2878            let second = Arc::new(Mutex::new(LeaderScript {
2879                listing: second_listing,
2880                ..LeaderScript::default()
2881            }));
2882            let cluster = Arc::new(Mutex::new(Vec::new()));
2883            let first_broker = leader_broker(Arc::clone(&first), Arc::clone(&cluster)).await;
2884            let second_broker = leader_broker(Arc::clone(&second), Arc::clone(&cluster)).await;
2885            cluster
2886                .lock()
2887                .expect("cluster lock")
2888                .extend([first_broker.addr, second_broker.addr]);
2889            let admin = fast_admin(first_broker.addr).await;
2890            let filter = ListTransactionsFilter::builder()
2891                .state_filters(vec!["Ongoing".to_owned()])
2892                .build();
2893
2894            let result = admin
2895                .list_transactions(&filter)
2896                .await
2897                .map_err(|error| match error {
2898                    AdminError::Broker { api, code, .. } => (api, code),
2899                    other => panic!("case {name}: unexpected error {other:?}"),
2900                });
2901
2902            first_broker.stop();
2903            second_broker.stop();
2904            let requests = [&first, &second]
2905                .map(|script| script.lock().expect("script lock").list_requests.clone());
2906            let expected_request = list_transactions_request(&filter);
2907            assert2::assert!(
2908                (result, requests)
2909                    == (
2910                        expected,
2911                        [vec![expected_request.clone()], vec![expected_request]]
2912                    ),
2913                "case {name}"
2914            );
2915        }
2916    }
2917
2918    /// Apache Kafka's `fenceProducers` gives each transactional ID its own
2919    /// result, and an invalid ID does not stop the others.
2920    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2921    async fn fence_producers_gives_each_transactional_id_its_own_result() {
2922        let script = Arc::new(Mutex::new(CoordinatorScript {
2923            find_coordinator: vec![0],
2924            coordinator: vec![0],
2925            ..CoordinatorScript::default()
2926        }));
2927        let coordinator_addr = Arc::new(Mutex::new(None));
2928        let coordinator =
2929            scripted_transaction_broker(Arc::clone(&script), Arc::clone(&coordinator_addr)).await;
2930        *coordinator_addr.lock().expect("coordinator lock") = Some(coordinator.addr);
2931        let bootstrap =
2932            scripted_transaction_broker(Arc::clone(&script), Arc::clone(&coordinator_addr)).await;
2933        let admin = AdminClient::connect(&[bootstrap.addr.to_string()])
2934            .await
2935            .expect("admin connects");
2936
2937        let results = admin.fence_producers(&["payments", "", "orders"]).await;
2938
2939        bootstrap.stop();
2940        coordinator.stop();
2941        let results = results
2942            .into_iter()
2943            .map(|(transactional_id, result)| (transactional_id, outcome_of(result)))
2944            .collect::<BTreeMap<_, _>>();
2945        let script = script.lock().expect("script lock");
2946        assert2::assert!(
2947            (
2948                results,
2949                script.find_coordinator_requests,
2950                script.coordinator_requests
2951            ) == (
2952                maplit::btreemap! {
2953                    String::new() => failed("protocol", 0),
2954                    "orders".to_owned() => OK,
2955                    "payments".to_owned() => OK,
2956                },
2957                2,
2958                2
2959            )
2960        );
2961    }
2962
2963    fn assert_send<T: Send>(_: T) {}
2964
2965    /// A caller can spawn every retrying admin call on a multi-thread
2966    /// runtime, so each future must be `Send`.
2967    #[test]
2968    fn retrying_admin_call_futures_are_send() {
2969        let _ = |admin: &mut AdminClient,
2970                 offsets: &std::collections::BTreeMap<(String, i32), i64>| {
2971            assert_send(admin.alter_consumer_group_offsets("workers", offsets));
2972            assert_send(admin.list_consumer_group_offsets("workers"));
2973            assert_send(admin.describe_transaction("payments"));
2974            assert_send(admin.force_terminate_transaction("payments"));
2975            assert_send(admin.fence_producers(&["payments"]));
2976            assert_send(admin.abort_transaction(&orders_spec()));
2977            assert_send(admin.describe_producers(&[("orders".to_owned(), 0)]));
2978            assert_send(admin.list_transactions(&ListTransactionsFilter::builder().build()));
2979        };
2980    }
2981}