fv-streams 0.8.0

The FusionVault Streams standalone runner: run a stream pipeline defined in one TOML file (steps or SQL) with no control plane — every connector built in, the Nexmark event generator, checkpoints to a local directory or an object store, scale-out across workers.
Documentation
//! The runner's tests across its parts: a pipeline file parsed and answered through its control
//! plane.

use super::*;
use fv_streams_engine::BuildSignal;
use fv_streams_types::ControlPlane;
use std::sync::atomic::AtomicBool;
use std::sync::atomic::Ordering;
use std::sync::Arc;

const FILE: &str = r#"
        name = "demo"
        [settings]
        STREAM_BATCH_MAX = "10"
        [brokers]
        default = "broker:9092"
        [datasets.events]
        topic = "in"
        [datasets.out]
        topic = "out"
        brokers = "other:9092"
        [[transforms]]
        inputs = ["events"]
        output = "out"
          [[transforms.steps]]
          op = "filter"
          expression = "x > 1"
    "#;

#[test]
fn parses_name_brokers_and_topology() {
    let p = Pipeline::parse(FILE, "demo.toml").unwrap();
    assert_eq!(p.name, "demo");
    assert_eq!(p.default_brokers(), "broker:9092");
    let cp = FileControlPlane::new(&p, Arc::new(AtomicBool::new(false)));
    let topo = tokio::runtime::Runtime::new()
        .unwrap()
        .block_on(cp.topology("demo"))
        .unwrap();
    assert_eq!(topo.stages.len(), 1);
    assert_eq!(topo.stages[0].inputs, vec!["events".to_string()]);
    assert_eq!(topo.stages[0].steps[0]["op"], "filter");
}

/// A dataset the file says where to write (a `sink` or a `topic`) is published, so an
/// intermediate that declares one is written as well as read. One that says nothing is not.
#[test]
fn a_dataset_that_declares_a_destination_is_published() {
    let file = r#"
            name = "demo"
            [datasets.a]
            sink = "stdout"
            [datasets.b]
            topic = "b"
            [datasets.c]
            [[transforms]]
            inputs = ["a"]
            output = "b"
        "#;
    let p = Pipeline::parse(file, "demo.toml").unwrap();
    let cp = FileControlPlane::new(&p, Arc::new(AtomicBool::new(false)));
    let rt = tokio::runtime::Runtime::new().unwrap();
    let published = |d: &str| rt.block_on(cp.publishes(d));
    assert!(published("a") && published("b"), "a sink or a topic is a destination");
    assert!(
        !published("c") && !published("nowhere"),
        "a bare or unknown dataset is not"
    );
}

#[test]
fn resolves_datasets_through_the_registry_with_per_dataset_broker_override() {
    let p = Pipeline::parse(FILE, "demo.toml").unwrap();
    let cp = FileControlPlane::new(&p, Arc::new(AtomicBool::new(false)));
    let rt = tokio::runtime::Runtime::new().unwrap();
    let events = rt.block_on(cp.resolve_dataset("events")).unwrap();
    assert_eq!(events.source.name(), "in");
    assert!(
        events.source.describe().contains("broker:9092"),
        "{}",
        events.source.describe()
    );
    let out = rt.block_on(cp.resolve_dataset("out")).unwrap();
    assert!(out.sink.describe().contains("other:9092"), "{}", out.sink.describe());
    let err = rt.block_on(cp.resolve_dataset("nope")).unwrap_err();
    assert!(err.contains("not defined under [datasets]"), "{err}");
}

#[test]
fn the_stop_flag_drives_the_heartbeat() {
    let p = Pipeline::parse(FILE, "demo.toml").unwrap();
    let stop = Arc::new(AtomicBool::new(false));
    let cp = FileControlPlane::new(&p, stop.clone());
    let rt = tokio::runtime::Runtime::new().unwrap();
    assert_eq!(rt.block_on(cp.heartbeat("demo")), BuildSignal::Continue);
    stop.store(true, Ordering::Relaxed);
    assert_eq!(rt.block_on(cp.heartbeat("demo")), BuildSignal::Stop);
}