timely 0.2.0

A low-latency data-parallel dataflow system in Rust
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74

//! A child dataflow scope, used to build nested dataflow scopes.

use std::cell::RefCell;

use progress::{Timestamp, Operate, Subgraph};
use progress::nested::{Source, Target};
use progress::nested::product::Product;
use timely_communication::{Allocate, Data};
use {Push, Pull};

use super::{ScopeParent, Scope};

/// A `Child` wraps a `Subgraph` and a parent `G: Scope`. It manages the addition
/// of `Operate`s to a subgraph, and the connection of edges between them.
pub struct Child<'a, G: ScopeParent, T: Timestamp> {
    /// The subgraph under assembly.
    pub subgraph: &'a RefCell<Subgraph<G::Timestamp, T>>,
    /// A copy of the child's parent scope.
    pub parent:   G,
}

impl<'a, G: ScopeParent, T: Timestamp> Child<'a, G, T> {
    /// This worker's unique identifier.
    ///
    /// Ranges from `0` to `self.peers() - 1`.
    pub fn index(&self) -> usize { self.parent.index() }
    /// The total number of workers in the computation.
    pub fn peers(&self) -> usize { self.parent.peers() }
}

impl<'a, G: ScopeParent, T: Timestamp> ScopeParent for Child<'a, G, T> {
    type Timestamp = Product<G::Timestamp, T>;

    fn new_identifier(&mut self) -> usize {
        self.parent.new_identifier()
    }
}

impl<'a, G: ScopeParent, T: Timestamp> Scope for Child<'a, G, T> {
    fn name(&self) -> String { self.subgraph.borrow().name().to_owned() }
    fn addr(&self) -> Vec<usize> { self.subgraph.borrow().path.clone() }
    fn add_edge(&self, source: Source, target: Target) {
        self.subgraph.borrow_mut().connect(source, target);
    }

    fn add_operator_with_index<SC: Operate<Self::Timestamp>+'static>(&mut self, scope: SC, index: usize) {
        let identifier = self.new_identifier();
        self.subgraph.borrow_mut().add_child(Box::new(scope), index, identifier);
    }

    fn add_operator<SC: Operate<Self::Timestamp>+'static>(&mut self, scope: SC) -> usize {
        let index = self.subgraph.borrow_mut().allocate_child_id();
        self.add_operator_with_index(scope, index);
        index
    }

    fn new_subscope<T2: Timestamp>(&mut self) -> Subgraph<Product<G::Timestamp, T>, T2> {
        let index = self.subgraph.borrow_mut().allocate_child_id();
        let path = self.subgraph.borrow().path.clone();
        Subgraph::new_from(self, index, path)
    }
}

impl<'a, G: ScopeParent, T: Timestamp> Allocate for Child<'a, G, T> {
    fn index(&self) -> usize { self.parent.index() }
    fn peers(&self) -> usize { self.parent.peers() }
    fn allocate<D: Data>(&mut self) -> (Vec<Box<Push<D>>>, Box<Pull<D>>) {
        self.parent.allocate()
    }
}

impl<'a, G: ScopeParent, T: Timestamp> Clone for Child<'a, G, T> {
    fn clone(&self) -> Self { Child { subgraph: self.subgraph, parent: self.parent.clone() }}
}