tinyflow-framework 0.1.1

Streaming runtime engine — checkpoint, chained task execution, and state store
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85

use std::any::Any;
use std::marker::PhantomData;
use std::sync::{Arc, Mutex};

use async_trait::async_trait;

use tinyflow_api::error::StreamResult;
use tinyflow_api::functions::SourceFunction;

use super::{
    ChainSegmentFactory, ChainSourceSegment, ChainStepKindInner, NamedStep,
    impl_chain_segment_lifecycle,
};

struct SourceChainSegment<T, S>
where
    T: Send,
    S: SourceFunction<T>,
{
    function: S,
    task_id: u32,
    input_cnt: u64,
    output_cnt: u64,
    last_input_cnt: u64,
    last_output_cnt: u64,
    _marker: PhantomData<T>,
}

impl<T, S> SourceChainSegment<T, S>
where
    T: Send + 'static,
    S: SourceFunction<T> + Send,
{
    fn new(function: S) -> Self {
        Self {
            function,
            task_id: 0,
            input_cnt: 0,
            output_cnt: 0,
            last_input_cnt: 0,
            last_output_cnt: 0,
            _marker: PhantomData,
        }
    }
}

impl_chain_segment_lifecycle! {
    impl<T, S>
    ChainSegment for SourceChainSegment<T, S>,
    with_metrics
    where
        T: Send + 'static,
        S: SourceFunction<T> + Send,
}

#[async_trait]
impl<T, S> ChainSourceSegment for SourceChainSegment<T, S>
where
    T: Send + 'static,
    S: SourceFunction<T> + Send,
{
    async fn poll_next(&mut self) -> StreamResult<Option<Box<dyn Any + Send>>> {
        let result = self.function.poll_next().await?;
        if result.is_some() {
            self.input_cnt += 1;
            self.output_cnt += 1;
        }
        Ok(result.map(|v| Box::new(v) as Box<dyn Any + Send>))
    }
}

pub fn source_chain_factory<T, S, F>(factory: Arc<Mutex<F>>) -> ChainSegmentFactory
where
    T: Send + 'static,
    S: SourceFunction<T> + Send + 'static,
    F: Fn() -> S + Send + 'static,
{
    Box::new(move || {
        let source = factory.lock().expect("source factory lock poisoned")();
        NamedStep {
            name: "Source".into(),
            kind: ChainStepKindInner::Source(Box::new(SourceChainSegment::new(source))),
        }
    })
}