fv-streams 0.1.0

The FusionVault Streams standalone runner: run a stream pipeline defined in one TOML file against Kafka or Redpanda with no control plane, including the built-in Nexmark event generator.
Documentation
//! Run a stream pipeline from one TOML file, with no control plane anywhere.
//!
//! The engine (`fv-streams-engine`) asks its host four questions through the
//! [`ControlPlane`] trait: what is the pipeline's topology, which topic backs a dataset, should I
//! keep running, and here is my build record. This crate answers them from a file. The same engine
//! that a platform hosts over HTTP therefore runs from a laptop, a container, or a test with the
//! same semantics: checkpoints, exactly-once output, stateful windows and joins.
//!
//! ```no_run
//! use std::sync::atomic::AtomicBool;
//! use std::sync::Arc;
//! use fv_streams::Pipeline;
//!
//! # tokio::runtime::Runtime::new().unwrap().block_on(async {
//! let stop = Arc::new(AtomicBool::new(false));      // set it to stop cleanly (Ctrl-C does)
//! Pipeline::load("pipeline.toml")?.run(stop).await;
//! # Ok::<(), String>(())
//! # });
//! ```
//!
//! A pipeline file names its datasets (Kafka topics), its transforms (the engine's own step
//! vocabulary: inline steps, `wasm`/`container` compute, `windowedAggregate`, `sessionAggregate`,
//! `streamJoin`, `topN`, `lastN`) and optional engine settings. The pipeline's `name` is its stable
//! identity: the consumer group `fv-stream-<name>`, the state topic `fv-state-<name>` and the
//! exactly-once transactional namespace all derive from it, so a restarted run restores its
//! checkpoints and fences its own zombies.

/// Every Rust code block in the README is compiled and run as a doctest.
#[cfg(doctest)]
#[doc = include_str!("../README.md")]
pub struct ReadmeDoctests;

use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;

use async_trait::async_trait;
use fv_streams_engine::{BuildSignal, ControlPlane, StageDef, TopicBinding, Topology};
use serde_json::{json, Value as J};

pub mod nexmark;

/// Convert a parsed TOML value into the engine's JSON vocabulary (steps are defined by the engine's
/// spec parser, not by this crate — a 1:1 mapping keeps one source of truth).
fn toml_to_json(v: &toml::Value) -> J {
    match v {
        toml::Value::String(s) => J::String(s.clone()),
        toml::Value::Integer(i) => json!(i),
        toml::Value::Float(f) => json!(f),
        toml::Value::Boolean(b) => json!(b),
        toml::Value::Datetime(d) => J::String(d.to_string()),
        toml::Value::Array(a) => J::Array(a.iter().map(toml_to_json).collect()),
        toml::Value::Table(t) => J::Object(t.iter().map(|(k, v)| (k.clone(), toml_to_json(v))).collect()),
    }
}

/// A pipeline definition loaded from a TOML file.
#[derive(Clone, Debug)]
pub struct Pipeline {
    /// The pipeline's stable identity (`name` in the file; `"pipeline"` when absent).
    pub name: String,
    /// Where it was loaded from (for messages only).
    pub path: String,
    /// The definition in the engine's JSON vocabulary.
    pub def: J,
}

impl Pipeline {
    /// Read and parse a pipeline file. Fails with a readable message for a missing file or invalid
    /// TOML; the engine validates the steps themselves when the run starts.
    pub fn load(path: &str) -> Result<Self, String> {
        let raw = std::fs::read_to_string(path).map_err(|e| format!("cannot read {path}: {e}"))?;
        Self::parse(&raw, path)
    }

    /// Parse a pipeline from TOML text (`path` is only used in messages).
    pub fn parse(toml_text: &str, path: &str) -> Result<Self, String> {
        let parsed: toml::Value = toml_text
            .parse()
            .map_err(|e| format!("{path} is not valid TOML: {e}"))?;
        let def = toml_to_json(&parsed);
        let name = def["name"].as_str().unwrap_or("pipeline").to_string();
        Ok(Self {
            name,
            path: path.to_string(),
            def,
        })
    }

    /// The default broker list (`[brokers] default`, else `localhost:9092`).
    pub fn default_brokers(&self) -> String {
        self.def["brokers"]["default"]
            .as_str()
            .unwrap_or("localhost:9092")
            .to_string()
    }

    /// Apply the file's `[settings]` table as `STREAM_*` environment variables for the engine. An
    /// explicit environment variable always wins, so a deployment can override a file.
    pub fn apply_settings(&self) {
        if let Some(settings) = self.def["settings"].as_object() {
            for (k, v) in settings {
                if std::env::var(k).is_err() {
                    let val = match v {
                        J::String(s) => s.clone(),
                        other => other.to_string(),
                    };
                    std::env::set_var(k, val);
                }
            }
        }
    }

    /// Start a generator for every dataset that declares `connector = "nexmark"`: the topic is
    /// created (like any dataset's), then a detached task streams the deterministic event mix
    /// while the pipeline runs. Returns the datasets that got a generator.
    pub async fn start_generators(&self) -> Result<Vec<String>, String> {
        let default_brokers = self.default_brokers();
        let mut started = Vec::new();
        if let Some(datasets) = self.def["datasets"].as_object() {
            for (ds_name, ds) in datasets {
                if let Some(parsed) = nexmark::parse(ds_name, ds, &default_brokers) {
                    let cfg = parsed?;
                    fv_streams_engine::ensure_topic(&cfg.brokers, &cfg.topic, cfg.partitions, &[])
                        .await
                        .map_err(|e| format!("nexmark[{ds_name}]: create topic {}: {e}", cfg.topic))?;
                    println!(
                        "fv-streams: nexmark[{ds_name}] → {} ({} events, seed {}, step {}ms{})",
                        cfg.topic,
                        cfg.events,
                        cfg.seed,
                        cfg.step_ms,
                        if cfg.rate > 0 {
                            format!(", {} ev/s", cfg.rate)
                        } else {
                            String::new()
                        }
                    );
                    nexmark::spawn(cfg); // detached: bounded, prints its own completion line
                    started.push(ds_name.clone());
                }
            }
        }
        Ok(started)
    }

    /// Run the pipeline until `stop` is set, then unwind in order: final checkpoint, flush, commit
    /// offsets. Applies the file's settings and starts any Nexmark generators first. The live
    /// counters print on the terminal as the run progresses.
    pub async fn run(self, stop: Arc<AtomicBool>) -> Result<(), String> {
        self.apply_settings();
        self.start_generators().await?;
        let record = json!({
            "id": self.name,
            "pipeline": self.name,
            "kind": "stream",
            "status": "RUNNING",
            "startedAt": chrono::Utc::now().to_rfc3339(),
        });
        println!(
            "fv-streams: running pipeline '{}' from {} (Ctrl-C to stop)",
            self.name, self.path
        );
        let name = self.name.clone();
        let cp = Arc::new(FileControlPlane { def: self.def, stop });
        fv_streams_engine::run_stream(cp, name.clone(), name, record).await;
        Ok(())
    }
}

/// The file-defined host: the engine's [`ControlPlane`] answered from a [`Pipeline`] definition.
/// Heartbeat = run until the stop flag; the build record prints as a status line (there is no
/// server to send it to).
pub struct FileControlPlane {
    def: J,
    stop: Arc<AtomicBool>,
}

impl FileControlPlane {
    pub fn new(pipeline: &Pipeline, stop: Arc<AtomicBool>) -> Self {
        Self {
            def: pipeline.def.clone(),
            stop,
        }
    }
}

#[async_trait]
impl ControlPlane for FileControlPlane {
    async fn topology(&self, _pipeline: &str) -> Result<Topology, String> {
        let transforms = self.def["transforms"]
            .as_array()
            .ok_or("pipeline.toml: needs at least one [[transforms]] table")?;
        let stages = transforms
            .iter()
            .map(|t| StageDef {
                steps: t["steps"].as_array().cloned().unwrap_or_default(),
                inputs: t["inputs"]
                    .as_array()
                    .map(|a| a.iter().filter_map(|v| v.as_str().map(String::from)).collect())
                    .unwrap_or_default(),
                output: t["output"].as_str().unwrap_or("").to_string(),
            })
            .collect();
        Ok(Topology { stages })
    }

    async fn resolve_topic(&self, dataset: &str) -> Result<TopicBinding, String> {
        let ds = &self.def["datasets"][dataset];
        if ds.is_null() {
            return Err(format!(
                "pipeline.toml: dataset '{dataset}' is not defined under [datasets] — every transform input/output must be"
            ));
        }
        let topic = ds["topic"]
            .as_str()
            .ok_or_else(|| format!("pipeline.toml: [datasets.{dataset}] needs `topic = \"…\"`"))?
            .to_string();
        let brokers = ds["brokers"]
            .as_str()
            .or_else(|| self.def["brokers"]["default"].as_str())
            .unwrap_or("localhost:9092")
            .to_string();
        Ok(TopicBinding {
            api_name: dataset.to_string(),
            topic,
            brokers,
        })
    }

    async fn heartbeat(&self, _build_id: &str) -> BuildSignal {
        if self.stop.load(Ordering::Relaxed) {
            BuildSignal::Stop
        } else {
            BuildSignal::Continue
        }
    }

    async fn put_record(&self, _build_id: &str, record: &J) {
        // No server to persist to — surface the live counters on the terminal instead.
        let m = &record["metrics"];
        let status = record["status"].as_str().unwrap_or("RUNNING");
        println!(
            "▸ {status}  consumed={} emitted={} dropped={}",
            m["consumed"].as_u64().unwrap_or(0),
            m["emitted"].as_u64().unwrap_or(0),
            m["dropped"].as_u64().unwrap_or(0)
        );
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    const FILE: &str = r#"
        name = "demo"
        [settings]
        STREAM_BATCH_MAX = "10"
        [brokers]
        default = "broker:9092"
        [datasets.events]
        topic = "in"
        [datasets.out]
        topic = "out"
        brokers = "other:9092"
        [[transforms]]
        inputs = ["events"]
        output = "out"
          [[transforms.steps]]
          op = "filter"
          expression = "x > 1"
    "#;

    #[test]
    fn parses_name_brokers_and_topology() {
        let p = Pipeline::parse(FILE, "demo.toml").unwrap();
        assert_eq!(p.name, "demo");
        assert_eq!(p.default_brokers(), "broker:9092");
        let cp = FileControlPlane::new(&p, Arc::new(AtomicBool::new(false)));
        let topo = tokio::runtime::Runtime::new()
            .unwrap()
            .block_on(cp.topology("demo"))
            .unwrap();
        assert_eq!(topo.stages.len(), 1);
        assert_eq!(topo.stages[0].inputs, vec!["events".to_string()]);
        assert_eq!(topo.stages[0].steps[0]["op"], "filter");
    }

    #[test]
    fn resolves_topics_with_per_dataset_broker_override() {
        let p = Pipeline::parse(FILE, "demo.toml").unwrap();
        let cp = FileControlPlane::new(&p, Arc::new(AtomicBool::new(false)));
        let rt = tokio::runtime::Runtime::new().unwrap();
        let events = rt.block_on(cp.resolve_topic("events")).unwrap();
        assert_eq!((events.topic.as_str(), events.brokers.as_str()), ("in", "broker:9092"));
        let out = rt.block_on(cp.resolve_topic("out")).unwrap();
        assert_eq!(out.brokers, "other:9092");
        let err = rt.block_on(cp.resolve_topic("nope")).unwrap_err();
        assert!(err.contains("not defined under [datasets]"), "{err}");
    }

    #[test]
    fn the_stop_flag_drives_the_heartbeat() {
        let p = Pipeline::parse(FILE, "demo.toml").unwrap();
        let stop = Arc::new(AtomicBool::new(false));
        let cp = FileControlPlane::new(&p, stop.clone());
        let rt = tokio::runtime::Runtime::new().unwrap();
        assert_eq!(rt.block_on(cp.heartbeat("demo")), BuildSignal::Continue);
        stop.store(true, Ordering::Relaxed);
        assert_eq!(rt.block_on(cp.heartbeat("demo")), BuildSignal::Stop);
    }

    #[test]
    fn invalid_toml_is_a_readable_error() {
        let err = Pipeline::parse("name = [", "bad.toml").unwrap_err();
        assert!(err.starts_with("bad.toml is not valid TOML"), "{err}");
    }
}