dbsp 0.287.0

Continuous streaming analytics engine
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271

#![cfg_attr(miri, ignore)]

use crate::{
    dynamic::LeanVec,
    utils::{
        Tup2, VecExt,
        consolidation::{
            consolidate, consolidate_from, consolidate_paired_slices, consolidate_payload_from,
            consolidate_slice,
            quicksort::quicksort,
            utils::{dedup_payload_starting_at, retain_starting_at},
        },
    },
};
use proptest::{collection::vec, prelude::*};
use std::collections::BTreeMap;

prop_compose! {
    /// Create a batch data tuple
    fn tuple()(key in 0..10_000usize, value in 0..10_000usize, diff in -10_000..=10_000isize) -> ((usize, usize), isize) {
        ((key, value), diff)
    }
}

prop_compose! {
    /// Generate a random batch of data
    fn batch()(batch in vec(tuple().prop_map(|((k,v), w)| Tup2(Tup2(k, v), w)), 0..5_000)) -> LeanVec<Tup2<Tup2<usize, usize>, isize>> {
        LeanVec::from(batch)
    }
}

prop_compose! {
    fn random_vec()(batch in vec(any::<u16>(), 0..5000)) -> Vec<u16> {
        batch
    }
}

prop_compose! {
    fn random_paired_vecs()
        (len in 0..=5000usize)
        (left in vec(any::<u16>(), len), right in vec(any::<i16>(), len))
    -> (Vec<u16>, Vec<i16>) {
        debug_assert_eq!(left.len(), right.len());
        (left, right)
    }
}

prop_compose! {
    fn multiple_payloads()
        (len in 0..=5000usize)
        (
            left in vec(any::<u8>(), len),
            values1 in vec(any::<i16>(), len),
            values2 in vec(any::<u8>(), len),
            values3 in vec(any::<u32>(), len),
        )
    -> (Vec<u8>, Vec<i16>, Vec<u8>, Vec<u32>) {
        debug_assert_eq!(left.len(), values1.len());
        debug_assert_eq!(left.len(), values2.len());
        debug_assert_eq!(left.len(), values3.len());
        (left, values1, values2, values3)
    }
}

fn batch_data(batch: &[Tup2<Tup2<usize, usize>, isize>]) -> BTreeMap<Tup2<usize, usize>, i64> {
    let mut values = BTreeMap::new();
    for &Tup2(tuple, diff) in batch {
        values
            .entry(tuple)
            .and_modify(|acc| *acc += diff as i64)
            .or_insert(diff as i64);
    }

    // Elements with a value of zero are removed in consolidation
    values.retain(|_, &mut diff| diff != 0);
    values
}

fn paired_batch_data(
    keys: &[Tup2<usize, usize>],
    diffs: &[isize],
) -> BTreeMap<Tup2<usize, usize>, i64> {
    let mut values = BTreeMap::new();
    for (&tuple, &diff) in keys.iter().zip(diffs) {
        values
            .entry(tuple)
            .and_modify(|acc| *acc += diff as i64)
            .or_insert(diff as i64);
    }

    // Elements with a value of zero are removed in consolidation
    values.retain(|_, &mut diff| diff != 0);
    values
}

proptest! {
    #[test]
    fn consolidate_batch(mut batch in batch()) {
        let expected = batch_data(batch.as_slice());
        consolidate(&mut batch);
        let output = batch_data(batch.as_slice());

        // Ensure the batch is sorted
        prop_assert!(batch.is_sorted_by(|Tup2(a, _), Tup2(b, _)| a.cmp(b)));
        // Ensure no diff values are zero
        prop_assert!(batch.as_slice().iter().all(|&Tup2(_, diff)| diff != 0));
        // Ensure the aggregated data is the same
        prop_assert_eq!(expected, output);
    }

    #[test]
    fn consolidate_impls_are_equivalent(batch in batch()) {
        let expected = batch_data(batch.as_slice());

        let mut vec = batch.clone();
        consolidate(&mut vec);
        prop_assert!(vec.as_slice().iter().all(|&Tup2(_, diff)| diff != 0));
        prop_assert!(vec.is_sorted_by(|Tup2(a, _), Tup2(b, _)| a.cmp(b)));
        prop_assert_eq!(&expected, &batch_data(vec.as_slice()));

        let mut vec_offset = batch.clone();
        consolidate_from(&mut vec_offset, 0);
        prop_assert!(vec_offset.as_slice().iter().all(|&Tup2(_, diff)| diff != 0));
        prop_assert!(vec_offset.is_sorted_by(|Tup2(a, _), Tup2(b, _)| a.cmp(b)));
        prop_assert_eq!(&expected, &batch_data(vec.as_slice()));
        prop_assert_eq!(&vec, &vec_offset);

        let mut slice = batch;
        let len = consolidate_slice(slice.as_mut_slice());
        slice.truncate(len);
        prop_assert!(slice.as_slice().iter().all(|&Tup2(_, diff)| diff != 0));
        prop_assert!(slice.is_sorted_by(|Tup2(a, _), Tup2(b, _)| a.cmp(b)));
        prop_assert_eq!(&expected, &batch_data(slice.as_slice()));
        prop_assert_eq!(&vec, &slice);
    }

    #[test]
    fn consolidate_pair_is_equivalent(batch in batch()) {
        let expected = batch_data(batch.as_slice());

        let mut consolidated = batch.clone();
        consolidate(&mut consolidated);

        let (mut keys, mut diffs): (Vec<_>, Vec<_>) = batch.as_slice().iter().cloned().map(|Tup2(x,y)| (x,y)).unzip();
        let len = consolidate_paired_slices(&mut keys, &mut diffs);
        keys.truncate(len);
        diffs.truncate(len);

        prop_assert!(diffs.iter().all(|&diff| diff != 0));
        prop_assert!(keys.is_sorted_by(|a, b| a.partial_cmp(b)));
        prop_assert_eq!(expected, paired_batch_data(&keys, &diffs));

        let (consolidated_keys, consolidated_diffs): (Vec<_>, Vec<_>) = consolidated.as_slice().iter().cloned().map(|Tup2(x,y)| (x,y)).unzip();
        prop_assert_eq!(consolidated_keys, keys);
        prop_assert_eq!(consolidated_diffs, diffs);
    }

    #[test]
    fn consolidate_payload_from_is_equivalent(batch in batch()) {
        let expected = batch_data(batch.as_slice());

        let mut consolidated = batch.clone();
        consolidate(&mut consolidated);

        let (mut keys, mut diffs): (Vec<_>, Vec<_>) = batch.as_slice().iter().cloned().map(|Tup2(x,y)| (x,y)).unzip();
        consolidate_payload_from(&mut keys, &mut diffs, 0);

        prop_assert!(diffs.iter().all(|&diff| diff != 0));
        prop_assert!(keys.is_sorted_by(|a, b| a.partial_cmp(b)));
        prop_assert_eq!(expected, paired_batch_data(&keys, &diffs));

        let (consolidated_keys, consolidated_diffs): (Vec<_>, Vec<_>) = consolidated.as_slice().iter().cloned().map(|Tup2(x,y)| (x,y)).unzip();
        prop_assert_eq!(consolidated_keys, keys);
        prop_assert_eq!(consolidated_diffs, diffs);
    }

    #[test]
    fn dual_quicksort_smoke(mut data in vec(any::<(u32, u32)>(), 0..=5000)) {
        let (mut keys, mut values): (Vec<_>, Vec<_>) = data.clone().into_iter().unzip();
        quicksort(&mut keys, &mut values);

        data.sort_unstable_by_key(|&(key, _)| key);

        super::assert_sorted_eq(& data, &keys, &values);
    }

    #[test]
    fn retain_equivalence(mut expected in random_vec()) {
        let mut output = expected.clone();
        retain_starting_at(&mut output, 0, |a| *a % 5 == 0);
        expected.retain(|a| *a % 5 == 0);
        prop_assert_eq!(output, expected);
    }

    #[test]
    fn dedup_equivalence(mut expected in random_vec()) {
        let mut output = expected.clone();
        dedup_payload_starting_at(&mut output, (), 0, |a, (), b, ()| *a == *b);
        expected.dedup_by(|a, b| *a == *b);
        prop_assert_eq!(output, expected);
    }

    #[test]
    fn dedup_payload_equivalence((mut keys, mut values) in random_paired_vecs()) {
        keys.sort_unstable();

        let mut expected: Vec<(_, _)> = keys.iter().copied().zip(values.iter().copied()).collect();
        expected.dedup_by(|(a, _), (b, _)| a == b);
        let (expected_keys, expected_values): (Vec<_>, Vec<_>) = expected.into_iter().unzip();

        dedup_payload_starting_at(&mut keys, &mut values, 0, |a, _, b, _| *a == *b);
        prop_assert_eq!(keys, expected_keys);
        prop_assert_eq!(values, expected_values);
    }

    #[test]
    fn dedup_multiple_payloads((mut keys, mut values1, mut values2, mut values3) in multiple_payloads()) {
        let mut expected = Vec::with_capacity(keys.len());
        for idx in 0..keys.len() {
            expected.push((keys[idx], values1[idx], values2[idx], values3[idx]));
        }
        expected.dedup_by(|(a, ..), (b, ..)| a == b);

        let (mut expected_keys, mut expected_values1, mut expected_values2, mut expected_values3) = (
            Vec::with_capacity(expected.len()),
            Vec::with_capacity(expected.len()),
            Vec::with_capacity(expected.len()),
            Vec::with_capacity(expected.len()),
        );
        for (key, value1, value2, value3) in expected{
            expected_keys.push(key);
            expected_values1.push(value1);
            expected_values2.push(value2);
            expected_values3.push(value3);
        }

        dedup_payload_starting_at(&mut keys, (&mut values1, (&mut values2, &mut values3)), 0, |a, _, b, _| *a == *b);
        prop_assert_eq!(keys, expected_keys);
        prop_assert_eq!(values1, expected_values1);
        prop_assert_eq!(values2, expected_values2);
        prop_assert_eq!(values3, expected_values3);
    }

    #[test]
    fn quicksort_correctness(mut batch in vec(any::<(u16, u16)>(), 0..50_000)) {
        // Split the data into keys and values
        let (mut keys, mut values): (Vec<_>, Vec<_>) = batch.clone().into_iter().unzip();

        // Sort the given data
        quicksort(&mut keys, &mut values);

        // Make sure we didn't lose any elements
        prop_assert_eq!(keys.len(), values.len());
        prop_assert_eq!(keys.len(), batch.len());

        // Recombine the keys and values into a single vector
        let mut results: Vec<_> = keys.into_iter().zip(values).collect();

        // Ensure that the results vec is properly sorted
        prop_assert!(results.is_sorted_by_key(|&(key, _)| key));

        // Sort both the input batch and the results the same way
        // so that if their contents are equal (that is, if `quicksort()`
        // didn't drop any elements) then both vectors will be equal
        batch.sort();
        results.sort();

        // Ensure that no elements were lost during sorting
        prop_assert_eq!(batch, results);
    }
}