Skip to main content

krabka_client_admin/
group_deletions.rs

1//! Batched group deletion: [`AdminClient::delete_consumer_groups`],
2//! [`AdminClient::delete_share_groups`] and
3//! [`AdminClient::delete_streams_groups`].
4//!
5//! Each call finds the coordinator of every group with one batched
6//! `FindCoordinator` request and sends one `DeleteGroups` request to each
7//! coordinator, as Apache Kafka's `DeleteGroupsHandler` does with its
8//! `CoordinatorStrategy`. A request to a coordinator that
9//! fails or loses its connection finds the coordinator again.
10
11use std::collections::BTreeMap;
12
13use krabka_protocol::owned::delete_groups_request::DeleteGroupsRequest;
14
15use crate::{
16    AdminClient, KafkaError,
17    group_coordinators::{GroupOutcome, group_error_outcome},
18    group_descriptions::request_error_outcomes,
19};
20
21/// The per-group results of a delete call: one entry for each requested
22/// group, `Ok(())` for a deleted group.
23pub type GroupDeletions = BTreeMap<String, Result<(), KafkaError>>;
24
25impl AdminClient {
26    /// Deletes empty consumer groups with `DeleteGroups`, as Apache Kafka's
27    /// `deleteConsumerGroups` does with its `DeleteConsumerGroupsHandler`.
28    ///
29    /// The call finds the coordinator of every group with one batched
30    /// `FindCoordinator` request and sends one `DeleteGroups` request to each
31    /// coordinator with the groups that it coordinates.
32    /// `COORDINATOR_LOAD_IN_PROGRESS` (14) sends a group to the same
33    /// coordinator again, and `COORDINATOR_NOT_AVAILABLE` (15) and
34    /// `NOT_COORDINATOR` (16) find its coordinator again. A lost
35    /// connection finds the coordinator again. The call waits with Kafka's
36    /// backoff between rounds and stops at `default.api.timeout.ms` (60 s),
37    /// where each pending group gets `REQUEST_TIMED_OUT` (7).
38    ///
39    /// # Errors
40    /// The call itself does not fail. Each group gets its own [`KafkaError`],
41    /// such as `NON_EMPTY_GROUP` (68), `GROUP_ID_NOT_FOUND` (69) or
42    /// `REQUEST_TIMED_OUT` (7) at the deadline.
43    pub async fn delete_consumer_groups(&self, groups: &[&str]) -> GroupDeletions {
44        self.delete_groups("deleteConsumerGroups", groups).await
45    }
46
47    /// Deletes empty share groups. Apache Kafka's `deleteShareGroups` uses a
48    /// `DeleteShareGroupsHandler` that builds the same `DeleteGroups` request
49    /// as `deleteConsumerGroups`.
50    ///
51    /// # Errors
52    /// See [`AdminClient::delete_consumer_groups`].
53    pub async fn delete_share_groups(&self, groups: &[&str]) -> GroupDeletions {
54        self.delete_groups("deleteShareGroups", groups).await
55    }
56
57    /// Deletes empty streams groups. Apache Kafka's `deleteStreamsGroups`
58    /// delegates to `deleteConsumerGroups`.
59    ///
60    /// # Errors
61    /// See [`AdminClient::delete_consumer_groups`].
62    pub async fn delete_streams_groups(&self, groups: &[&str]) -> GroupDeletions {
63        self.delete_consumer_groups(groups).await
64    }
65
66    /// The `DeleteGroups` call of Kafka's `DeleteGroupsHandler`.
67    async fn delete_groups(&self, api: &'static str, groups: &[&str]) -> GroupDeletions {
68        self.call_group_coordinators(api, groups, |coordinator, keys| async move {
69            let connection = self.connect_coordinator(&coordinator).await?;
70            let request = DeleteGroupsRequest {
71                groups_names: keys.clone(),
72                ..Default::default()
73            };
74            Ok(match connection.send(request).await {
75                Ok(response) => response
76                    .results
77                    .into_iter()
78                    .map(|result| {
79                        let outcome = match result.error_code {
80                            0 => GroupOutcome::Done(Ok(())),
81                            code => group_error_outcome(code, None, &[]),
82                        };
83                        (result.group_id, outcome)
84                    })
85                    .collect(),
86                Err(error) => request_error_outcomes(&keys, &error.into()),
87            })
88        })
89        .await
90    }
91}
92
93#[cfg(test)]
94mod tests {
95    use std::sync::{Arc, Mutex};
96
97    use assert2::assert;
98    use krabka_client_core::MockReply;
99    use krabka_protocol::owned::{
100        delete_groups_request,
101        delete_groups_response::{DeletableGroupResult, DeleteGroupsResponse},
102        find_coordinator_request,
103    };
104
105    use super::*;
106    use crate::{
107        group_coordinators::{
108            kafka_error,
109            test_support::{GroupCluster, Lookup, group_cluster},
110        },
111        partition_leaders::test_support::{ApiVersionRange, decode_request, encode_response},
112    };
113
114    fn names(names: &[&str]) -> Vec<String> {
115        names.iter().map(|name| (*name).to_owned()).collect()
116    }
117
118    async fn delete_groups_cluster(find_coordinator_max: i16) -> GroupCluster {
119        group_cluster()
120            .count(1)
121            .apis(vec![
122                ApiVersionRange::builder()
123                    .api_key(delete_groups_request::API_KEY)
124                    .max_version(2)
125                    .build(),
126                ApiVersionRange::builder()
127                    .api_key(find_coordinator_request::API_KEY)
128                    .max_version(find_coordinator_max)
129                    .build(),
130            ])
131            .lookup(|_, _| Lookup::At(0))
132            .answer(mock_delete_groups)
133            .call()
134            .await
135    }
136
137    /// Kafka's `DeleteGroupsHandler` batches the groups of one coordinator,
138    /// retries 14 on the same coordinator, finds the coordinator again on 15
139    /// and 16, and fails every other code. A `FindCoordinator` error that
140    /// Kafka does not retry fails the group without a `DeleteGroups` request.
141    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
142    async fn delete_groups_batches_and_retries_as_kafka_does() {
143        #[derive(Clone, Copy, Debug)]
144        enum Call {
145            Consumer,
146            Share,
147            Streams,
148        }
149        for call in [Call::Consumer, Call::Share, Call::Streams] {
150            // "moved" is on broker 1 first, and then on broker 0.
151            let codes = Arc::new(Mutex::new(maplit::btreemap! {
152                "done" => vec![0],
153                "loading" => vec![14, 0],
154                "moved" => vec![16, 0],
155                "busy" => vec![68],
156            }));
157            let cluster = group_cluster()
158                .count(2)
159                .apis(vec![
160                    ApiVersionRange::builder()
161                        .api_key(delete_groups_request::API_KEY)
162                        .max_version(2)
163                        .build(),
164                ])
165                .lookup(|group, lookups| match (group, lookups) {
166                    ("denied", _) => Lookup::Error(30),
167                    ("moved", 0) => Lookup::At(1),
168                    _ => Lookup::At(0),
169                })
170                .answer(move |_, api_key, version, body, _| {
171                    if api_key != delete_groups_request::API_KEY {
172                        return MockReply::Silent;
173                    }
174                    let request: DeleteGroupsRequest = decode_request(body, version, version >= 2);
175                    let mut codes = codes.lock().expect("codes lock");
176                    let results = request
177                        .groups_names
178                        .iter()
179                        .map(|group| {
180                            let script = codes.get_mut(group.as_str()).expect("scripted group");
181                            let error_code = if script.len() > 1 {
182                                script.remove(0)
183                            } else {
184                                script[0]
185                            };
186                            DeletableGroupResult {
187                                group_id: group.clone(),
188                                error_code,
189                                ..Default::default()
190                            }
191                        })
192                        .collect();
193                    MockReply::Respond(encode_response(
194                        &DeleteGroupsResponse {
195                            results,
196                            ..Default::default()
197                        },
198                        version,
199                        version >= 2,
200                    ))
201                })
202                .call()
203                .await;
204            let admin = cluster.admin(krabka_units::secs(5)).await;
205            let groups = ["done", "loading", "moved", "busy", "denied"];
206
207            let result = match call {
208                Call::Consumer => admin.delete_consumer_groups(&groups).await,
209                Call::Share => admin.delete_share_groups(&groups).await,
210                Call::Streams => admin.delete_streams_groups(&groups).await,
211            };
212
213            let seen = cluster.stop();
214            let requests = seen
215                .requests
216                .iter()
217                .map(|request| {
218                    let body: DeleteGroupsRequest =
219                        decode_request(&request.body, request.version, request.version >= 2);
220                    (request.broker, request.version, body.groups_names)
221                })
222                .collect::<Vec<_>>();
223            let lookups = seen
224                .find_coordinator
225                .into_iter()
226                .map(|(_, keys)| keys)
227                .collect::<Vec<_>>();
228            assert!(
229                (result, lookups)
230                    == (
231                        maplit::btreemap! {
232                            "busy".to_owned() => Err(kafka_error(68, None)),
233                            "denied".to_owned() => Err(kafka_error(30, None)),
234                            "done".to_owned() => Ok(()),
235                            "loading".to_owned() => Ok(()),
236                            "moved".to_owned() => Ok(()),
237                        },
238                        vec![
239                            names(&["busy", "denied", "done", "loading", "moved"]),
240                            names(&["moved"]),
241                        ],
242                    ),
243                "case {call:?}"
244            );
245            // The first round sends one request to each coordinator; the
246            // second retries "loading" and the moved "moved" together.
247            let mut first_round = requests[..2].to_vec();
248            first_round.sort();
249            assert!(
250                (first_round, requests[2..].to_vec())
251                    == (
252                        vec![
253                            (0, 2, names(&["busy", "done", "loading"])),
254                            (1, 2, names(&["moved"])),
255                        ],
256                        vec![(0, 2, names(&["loading", "moved"]))],
257                    ),
258                "case {call:?}"
259            );
260        }
261    }
262
263    fn mock_delete_groups(
264        _broker: usize,
265        api_key: i16,
266        version: i16,
267        body: &[u8],
268        _addr: std::net::SocketAddr,
269    ) -> MockReply {
270        if api_key != delete_groups_request::API_KEY {
271            return MockReply::Silent;
272        }
273        let request: DeleteGroupsRequest = decode_request(body, version, version >= 2);
274        MockReply::Respond(encode_response(
275            &DeleteGroupsResponse {
276                results: request
277                    .groups_names
278                    .into_iter()
279                    .map(|group_id| DeletableGroupResult {
280                        group_id,
281                        ..Default::default()
282                    })
283                    .collect(),
284                ..Default::default()
285            },
286            version,
287            version >= 2,
288        ))
289    }
290
291    /// A broker at `FindCoordinator` v4 or later gets one lookup for every
292    /// group, and a broker below v4 gets one lookup for each group, sent all at
293    /// once. Each row compares the version and keys of every lookup, sorted
294    /// because concurrent lookups arrive in any order.
295    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
296    async fn group_lookups_batch_from_find_coordinator_v4_and_fan_out_below() {
297        for (name, find_coordinator_max, expected) in [
298            ("v6 broker batches", 6, vec![(4, names(&["a", "b", "c"]))]),
299            ("v4 broker batches", 4, vec![(4, names(&["a", "b", "c"]))]),
300            (
301                "v3 broker looks up each group",
302                3,
303                vec![(3, names(&["a"])), (3, names(&["b"])), (3, names(&["c"]))],
304            ),
305        ] {
306            let cluster = delete_groups_cluster(find_coordinator_max).await;
307            let admin = cluster.admin(krabka_units::secs(5)).await;
308
309            let result = admin.delete_consumer_groups(&["a", "b", "c"]).await;
310
311            let mut lookups = cluster
312                .stop()
313                .find_coordinator
314                .into_iter()
315                .map(|(version, keys)| (version.min(4), keys))
316                .collect::<Vec<_>>();
317            lookups.sort();
318            assert!(
319                (result, lookups)
320                    == (
321                        maplit::btreemap! {
322                            "a".to_owned() => Ok(()),
323                            "b".to_owned() => Ok(()),
324                            "c".to_owned() => Ok(()),
325                        },
326                        expected,
327                    ),
328                "case {name}"
329            );
330        }
331    }
332
333    /// A broker below `FindCoordinator` v4 cannot look up many keys at once,
334    /// so each group gets its own lookup, as Kafka's `CoordinatorStrategy`
335    /// does after `disableBatch`. The groups of one coordinator still share
336    /// one `DeleteGroups` request.
337    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
338    async fn delete_groups_looks_up_one_group_at_a_time_below_find_coordinator_v4() {
339        let cluster = delete_groups_cluster(3).await;
340        let admin = cluster.admin(krabka_units::secs(5)).await;
341
342        let result = admin.delete_consumer_groups(&["a", "b"]).await;
343
344        let seen = cluster.stop();
345        assert!(
346            (result, seen.find_coordinator, seen.requests.len())
347                == (
348                    maplit::btreemap! {
349                        "a".to_owned() => Ok(()),
350                        "b".to_owned() => Ok(()),
351                    },
352                    vec![(3, names(&["a"])), (3, names(&["b"]))],
353                    1,
354                )
355        );
356    }
357}