fv-streams 0.9.0

The FusionVault Streams standalone runner: run a stream pipeline defined in one TOML file (steps or SQL) with no control plane — every connector built in, the Nexmark event generator, checkpoints to a local directory or an object store, scale-out across workers.
Documentation
//! A pipeline file: parsed from TOML into the engine's vocabulary, and run, alone or serving its
//! status.

use super::control_plane::FileControlPlane;
use super::{serve, settings};
use serde_json::{json, Value as J};
use std::sync::atomic::AtomicBool;
use std::sync::Arc;

/// Convert a parsed TOML value into the engine's JSON vocabulary (steps are defined by the engine's
/// spec parser, not by this crate — a 1:1 mapping keeps one source of truth).
fn toml_to_json(v: &toml::Value) -> J {
    match v {
        toml::Value::String(s) => J::String(s.clone()),
        toml::Value::Integer(i) => json!(i),
        toml::Value::Float(f) => json!(f),
        toml::Value::Boolean(b) => json!(b),
        toml::Value::Datetime(d) => J::String(d.to_string()),
        toml::Value::Array(a) => J::Array(a.iter().map(toml_to_json).collect()),
        toml::Value::Table(t) => J::Object(t.iter().map(|(k, v)| (k.clone(), toml_to_json(v))).collect()),
    }
}

/// A pipeline definition loaded from a TOML file.
#[derive(Clone, Debug)]
pub struct Pipeline {
    /// The pipeline's stable identity (`name` in the file; `"pipeline"` when absent).
    pub name: String,
    /// Where it was loaded from (for messages only).
    pub path: String,
    /// The definition in the engine's JSON vocabulary.
    pub def: J,
}

impl Pipeline {
    /// Read and parse a pipeline file. Fails with a readable message for a missing file or invalid
    /// TOML; the engine validates the steps themselves when the run starts.
    pub fn load(path: &str) -> Result<Self, String> {
        let raw = std::fs::read_to_string(path).map_err(|e| format!("cannot read {path}: {e}"))?;
        Self::parse(&raw, path)
    }

    /// Parse a pipeline from TOML text (`path` is only used in messages).
    pub fn parse(toml_text: &str, path: &str) -> Result<Self, String> {
        let parsed: toml::Value = toml_text
            .parse()
            .map_err(|e| format!("{path} is not valid TOML: {e}"))?;
        let def = toml_to_json(&parsed);
        let name = def["name"].as_str().unwrap_or("pipeline").to_string();
        Ok(Self {
            name,
            path: path.to_string(),
            def,
        })
    }

    /// Apply the file's `[settings]` table as `STREAM_*` environment variables for the engine; an
    /// explicit environment variable always wins, so a deployment can override a file. Refused
    /// before anything is set: a key that is no setting, or a value its setting does not take
    /// (either would be silently ignored). Then the whole environment is checked: a value no
    /// setting takes is an error, a `STREAM_…` variable that is no setting a printed warning.
    pub fn apply_settings(&self) -> Result<(), String> {
        let known = settings();
        fv_streams_types::settings::check_table(&self.def["settings"], &format!("{}: [settings]", self.path), &known)?;
        if let Some(table) = self.def["settings"].as_object() {
            for (k, v) in table {
                if std::env::var_os(k).is_none() {
                    std::env::set_var(k, fv_streams_types::settings::as_env(v));
                }
            }
        }
        for warning in fv_streams_types::settings::check_environment(&known)? {
            eprintln!("fv-streams: {warning}");
        }
        Ok(())
    }

    /// Start a generator for every dataset that declares `connector = "nexmark"`: the topic is
    /// created (like any dataset's), then a detached task streams the deterministic event mix
    /// while the pipeline runs. Returns the datasets that got a generator.
    pub async fn start_generators(&self) -> Result<Vec<String>, String> {
        // In a cluster the input is seeded once, by the leader: a joined worker (`--join`) consumes
        // the topic the leader fills, so it must not run its own generator and double the events.
        if !fv_streams_engine::settings::JOIN.raw().is_empty() {
            return Ok(Vec::new());
        }
        let brokers = &self.def["brokers"];
        let mut started = Vec::new();
        if let Some(datasets) = self.def["datasets"].as_object() {
            for (ds_name, ds) in datasets {
                if let Some(parsed) = fv_streams_connectors::nexmark::parse(ds_name, ds) {
                    // a misspelt option fails here, before the generator writes a single event
                    // (its warnings print once, when the dataset resolves)
                    fv_streams_connectors::check_dataset(&fv_streams_connectors::Ctx {
                        dataset: ds_name,
                        ds,
                        brokers,
                    })?;
                    let cfg = parsed?;
                    if cfg.inline {
                        continue; // the runtime opens it as source tasks (`resolve_dataset`)
                    }
                    // the generator connects as the dataset's source does: one rule for every client
                    let connection = fv_streams_connectors::kafka::Connection::resolve(ds_name, ds, brokers)?;
                    fv_streams_connectors::kafka::ensure_topic(&connection, &cfg.topic, cfg.partitions, 1, &[]) // a benchmark topic: one copy
                        .await
                        .map_err(|e| format!("nexmark[{ds_name}]: create topic {}: {e}", cfg.topic))?;
                    let (probe, on) = (cfg.clone(), connection.clone());
                    let written =
                        tokio::task::spawn_blocking(move || fv_streams_connectors::nexmark::written(&probe, &on))
                            .await
                            .map_err(|e| format!("nexmark[{ds_name}]: {e}"))??;
                    if let fv_streams_connectors::nexmark::Written::All(n) = written {
                        // a restart: the events are there, and the pipeline's position is in them
                        println!(
                            "fv-streams: nexmark[{ds_name}] → {}: the topic already holds all {n} messages; not writing them again (delete the topic to generate it afresh)",
                            cfg.topic
                        );
                        continue;
                    }
                    println!(
                        "fv-streams: nexmark[{ds_name}] → {} ({} events, seed {}, step {}ms{})",
                        cfg.topic,
                        cfg.events,
                        cfg.seed,
                        cfg.step_ms,
                        if cfg.rate > 0 {
                            format!(", {} ev/s", cfg.rate)
                        } else {
                            String::new()
                        }
                    );
                    fv_streams_connectors::nexmark::spawn(cfg, connection); // detached: bounded, prints its own completion line
                    started.push(ds_name.clone());
                }
            }
        }
        Ok(started)
    }

    /// Run the pipeline until `stop` is set, then stop in order: a final checkpoint, then its
    /// commit. Applies the file's settings and starts any Nexmark generators first. The live
    /// counters print on the terminal as the run progresses.
    pub async fn run(self, stop: Arc<AtomicBool>) -> Result<(), String> {
        self.run_serving(stop, None).await
    }

    /// [`Pipeline::run`] with the observability surface on `serve` (`--serve <addr>` /
    /// `STREAM_SERVE`): `/health`, `/ready`, `/status`, `/metrics`.
    pub async fn run_serving(self, stop: Arc<AtomicBool>, serve: Option<&str>) -> Result<(), String> {
        self.apply_settings()?;
        self.start_generators().await?;
        let record = json!({
            "id": self.name,
            "pipeline": self.name,
            "kind": "stream",
            "status": "RUNNING",
            "startedAt": chrono::Utc::now().to_rfc3339(),
        });
        println!(
            "fv-streams: running pipeline '{}' from {} (Ctrl-C to stop)",
            self.name, self.path
        );
        let name = self.name.clone();
        let cp = Arc::new(FileControlPlane {
            def: self.def,
            stop,
            live: tokio::sync::watch::channel(json!({ "status": "STARTING", "metrics": {} })).0,
        });
        if let Some(addr) = serve {
            let bound = serve::start(addr, cp.live()).await?;
            println!("fv-streams: serving /health /ready /status /metrics on http://{bound}");
        }
        fv_streams_engine::run_stream(cp, name.clone(), name, record).await
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn invalid_toml_is_a_readable_error() {
        let err = Pipeline::parse("name = [", "bad.toml").unwrap_err();
        assert!(err.starts_with("bad.toml is not valid TOML"), "{err}");
    }
}