crabka-client-consumer 0.3.5

Subscribe-style consumer client for Apache Kafka in Rust
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116

//! Range assignor (Kafka's classic default).
//!
//! Given a set of members and a per-topic partition count, hands each
//! member a contiguous range of partitions per topic. The trailing
//! members get one less partition when the partition count doesn't
//! divide evenly.

use std::collections::HashMap;

/// Returns `member_id → Vec<(topic, partition)>` assignments.
#[must_use]
pub fn assign(
    mut members: Vec<(String, Vec<String>)>,
    topic_partitions: &HashMap<String, i32>,
) -> HashMap<String, Vec<(String, i32)>> {
    members.sort_by(|(a, _), (b, _)| a.cmp(b));
    let mut out: HashMap<String, Vec<(String, i32)>> = members
        .iter()
        .map(|(m, _)| (m.clone(), Vec::new()))
        .collect();

    // For determinism, iterate topics in sorted order.
    let mut topics: Vec<(&String, &i32)> = topic_partitions.iter().collect();
    topics.sort_by_key(|(a, _)| *a);

    for (topic, &partition_count) in topics {
        let subscribed: Vec<&String> = members
            .iter()
            .filter(|(_, subs)| subs.iter().any(|t| t == topic))
            .map(|(m, _)| m)
            .collect();
        if subscribed.is_empty() || partition_count <= 0 {
            continue;
        }
        let n = i32::try_from(subscribed.len()).unwrap_or(i32::MAX);
        let per = partition_count / n;
        let extras = partition_count % n;
        let mut next: i32 = 0;
        for (i, m) in subscribed.iter().enumerate() {
            let extra = i32::from(i32::try_from(i).unwrap_or(i32::MAX) < extras);
            let take = per + extra;
            for p in next..(next + take) {
                if let Some(slot) = out.get_mut(*m) {
                    slot.push((topic.clone(), p));
                }
            }
            next += take;
        }
    }
    out
}

#[cfg(test)]
mod tests {
    use super::*;
    use assert2::assert;

    #[test]
    fn one_member_takes_everything() {
        let mut tp = HashMap::new();
        tp.insert("t".into(), 4);
        let a = assign(vec![("m1".into(), vec!["t".into()])], &tp);
        assert!(a["m1"].len() == 4);
        assert!(
            a["m1"]
                == vec![
                    ("t".into(), 0),
                    ("t".into(), 1),
                    ("t".into(), 2),
                    ("t".into(), 3),
                ]
        );
    }

    #[test]
    fn two_members_split_evenly() {
        let mut tp = HashMap::new();
        tp.insert("t".into(), 4);
        let a = assign(
            vec![
                ("m1".into(), vec!["t".into()]),
                ("m2".into(), vec!["t".into()]),
            ],
            &tp,
        );
        assert!(a["m1"] == vec![("t".into(), 0), ("t".into(), 1)]);
        assert!(a["m2"] == vec![("t".into(), 2), ("t".into(), 3)]);
    }

    #[test]
    fn extras_go_to_lower_member_ids() {
        let mut tp = HashMap::new();
        tp.insert("t".into(), 5);
        let a = assign(
            vec![
                ("m1".into(), vec!["t".into()]),
                ("m2".into(), vec!["t".into()]),
            ],
            &tp,
        );
        assert!(a["m1"].len() == 3);
        assert!(a["m2"].len() == 2);
    }

    #[test]
    fn member_with_no_subscriptions_gets_empty() {
        let mut tp = HashMap::new();
        tp.insert("t".into(), 2);
        let a = assign(
            vec![("m1".into(), vec!["t".into()]), ("m2".into(), vec![])],
            &tp,
        );
        assert!(a["m1"].len() == 2);
        assert!(a["m2"].len() == 0);
    }
}