timely 0.29.0

A low-latency data-parallel dataflow system in Rust
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95

use std::cmp::Ordering;

use timely::dataflow::*;
use timely::progress::Timestamp;
use timely::dataflow::operators::{Input, Exchange, Probe};
use timely::dataflow::operators::generic::operator::Operator;
use timely::dataflow::channels::pact::Pipeline;

fn main() {

    // command-line args: numbers of nodes and edges in the random graph.
    let nodes: usize = std::env::args().nth(1).unwrap().parse().unwrap();
    let edges: usize = std::env::args().nth(2).unwrap().parse().unwrap();
    let batch: usize = std::env::args().nth(3).unwrap().parse().unwrap();

    timely::execute_from_args(std::env::args().skip(4), move |worker| {

        let index = worker.index();
        let peers = worker.peers();

        let mut input = InputHandle::new();
        let probe = ProbeHandle::new();

        worker.dataflow(|scope| {
            scope.input_from(&mut input)
                //  .exchange(move |x: &(usize, usize)| (x.0 % (peers - 1)) as u64 + 1)
                 .union_find()
                 .exchange(|_| 0)
                 .union_find()
                 .probe_with(&probe);
        });

        // Generate roughly random data.
        use std::hash::{BuildHasher, BuildHasherDefault, DefaultHasher};
        let hasher = BuildHasherDefault::<DefaultHasher>::new();
        let insert = (0..).map(move |i| (hasher.hash_one(&(i,index,0)) as usize % nodes,
                                         hasher.hash_one(&(i,index,1)) as usize % nodes));

        for (edge, arc) in insert.take(edges / peers).enumerate() {
            input.send(arc);
            if edge % batch == (batch - 1) {
                let next = input.time() + 1;
                input.advance_to(next);
                while probe.less_than(input.time()) {
                    worker.step();
                }
            }
        }

    }).unwrap(); // asserts error-free execution;
}

trait UnionFind {
    fn union_find(self) -> Self;
}

impl<'scope, T: Timestamp> UnionFind for StreamVec<'scope, T, (usize, usize)> {
    fn union_find(self) -> Self {

        self.unary(Pipeline, "UnionFind", |_,_| {

            let mut roots = vec![];  // u32 works, and is smaller than uint/u64
            let mut ranks = vec![];  // u8 should be large enough (n < 2^256)

            move |input, output| {

                input.for_each_time(|time, data| {
                    let mut session = output.session(&time);
                    for &mut (mut x, mut y) in data.flatten() {

                        // grow arrays if required.
                        let m = ::std::cmp::max(x, y);
                        for i in roots.len() .. (m + 1) {
                            roots.push(i);
                            ranks.push(0);
                        }

                        // look up roots for `x` and `y`.
                        while x != roots[x] { x = roots[x]; }
                        while y != roots[y] { y = roots[y]; }

                        if x != y {
                            session.give((x, y));
                            match ranks[x].cmp(&ranks[y]) {
                                Ordering::Less    => { roots[x] = y },
                                Ordering::Greater => { roots[y] = x },
                                Ordering::Equal   => { roots[y] = x; ranks[x] += 1 },
                            }
                        }
                    }
                });
            }
        })
    }
}