tinyflow-framework 0.1.1

Streaming runtime engine — checkpoint, chained task execution, and state store
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

use std::any::Any;

use crate::runtime::chain_segment::ChainTailSegment;
use tinyflow_api::context::RuntimeContext;
use tinyflow_api::error::StreamResult;

pub(super) async fn run_through_chain(
    tail_steps: &mut [Box<dyn ChainTailSegment>],
    input: Box<dyn Any + Send>,
    runtime: &mut RuntimeContext,
) -> StreamResult<()> {
    let mut pending = vec![input];
    let last = tail_steps.len().saturating_sub(1);
    for (i, step) in tail_steps.iter_mut().enumerate() {
        let mut next = Vec::new();
        for record in pending {
            next.extend(step.process_record(record, runtime).await?);
        }
        if next.is_empty() {
            return Ok(());
        }
        if i == last {
            // Tail outputs are consumed by the step (e.g. Iceberg writer registers to job_state).
            return Ok(());
        }
        pending = next;
    }
    Ok(())
}