rlink 0.6.16

High performance Stream Processing Framework
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79

use std::fmt::Debug;

use crate::core::runtime::{CheckpointId, OperatorId, TaskId};

/// This struct provides a context in which user functions that use managed state metadata
#[derive(Clone, Debug)]
pub struct FunctionSnapshotContext {
    pub operator_id: OperatorId,
    pub task_id: TaskId,
    pub checkpoint_id: CheckpointId,
    pub completed_checkpoint_id: Option<CheckpointId>,
}

impl FunctionSnapshotContext {
    pub fn new(
        operator_id: OperatorId,
        task_id: TaskId,
        checkpoint_id: CheckpointId,
        completed_checkpoint_id: Option<CheckpointId>,
    ) -> Self {
        FunctionSnapshotContext {
            operator_id,
            task_id,
            checkpoint_id,
            completed_checkpoint_id,
        }
    }
}

/// checkpoint handle
/// identify a checkpoint resource
/// eg: 1. checkpoint state and save to distribution file system, mark the file path as handle.
///     2. as mq system, mark the offset as handle
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct CheckpointHandle {
    pub handle: String,
}

impl Default for CheckpointHandle {
    fn default() -> Self {
        Self {
            handle: "".to_string(),
        }
    }
}

/// descriptor a `Checkpoint`
/// use for network communication between `Coordinator` and `Worker`
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Checkpoint {
    pub operator_id: OperatorId,
    pub task_id: TaskId,
    pub checkpoint_id: CheckpointId,
    pub completed_checkpoint_id: Option<CheckpointId>,
    pub handle: CheckpointHandle,
}

pub trait CheckpointFunction {
    fn consult_version(
        &mut self,
        context: &FunctionSnapshotContext,
        _handle: &Option<CheckpointHandle>,
    ) -> CheckpointId {
        context.checkpoint_id
    }

    /// trigger the method when a `operator` initialization
    fn initialize_state(
        &mut self,
        _context: &FunctionSnapshotContext,
        _handle: &Option<CheckpointHandle>,
    ) {
    }

    /// trigger the method when the `operator` operate a `Barrier` event
    fn snapshot_state(&mut self, _context: &FunctionSnapshotContext) -> Option<CheckpointHandle> {
        None
    }
}