fv-streams 0.9.0

The FusionVault Streams standalone runner: run a stream pipeline defined in one TOML file (steps or SQL) with no control plane — every connector built in, the Nexmark event generator, checkpoints to a local directory or an object store, scale-out across workers.
Documentation
//! The file-defined host: the engine's control plane answered from a pipeline file.

use super::pipeline::Pipeline;
use super::serve;
use async_trait::async_trait;
use fv_streams_engine::{Binding, BuildSignal, ControlPlane, StageDef, Topology};
use serde_json::{json, Value as J};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;

/// The file-defined host: the engine's [`ControlPlane`] answered from a [`Pipeline`] definition.
/// Heartbeat = run until the stop flag; the build record prints as a status line (there is no
/// server to send it to).
pub struct FileControlPlane {
    pub(crate) def: J,
    pub(crate) stop: Arc<AtomicBool>,
    /// The live build record, for the observability surface (`--serve`).
    pub(crate) live: tokio::sync::watch::Sender<J>,
}

impl FileControlPlane {
    pub fn new(pipeline: &Pipeline, stop: Arc<AtomicBool>) -> Self {
        Self {
            def: pipeline.def.clone(),
            stop,
            live: tokio::sync::watch::channel(json!({ "status": "STARTING", "metrics": {} })).0,
        }
    }

    /// A receiver of the live build record (every heartbeat's record).
    pub fn live(&self) -> serve::Live {
        self.live.subscribe()
    }
}

#[async_trait]
impl ControlPlane for FileControlPlane {
    async fn topology(&self, _pipeline: &str) -> Result<Topology, String> {
        // A `sql = "…"` field is the SQL front door: it compiles to the SAME stages a
        // `[[transforms]]` block would (one runtime, two front doors). Datasets/brokers/settings
        // stay as config; the SQL only replaces the transforms.
        if let Some(sql) = self.def["sql"].as_str() {
            return fv_streams_sql::compile(sql).map_err(|e| format!("pipeline SQL: {e}"));
        }
        let transforms = self.def["transforms"]
            .as_array()
            .ok_or("pipeline.toml: needs at least one [[transforms]] table (or a `sql = \"…\"` field)")?;
        let stages = transforms
            .iter()
            .map(|t| StageDef {
                steps: t["steps"].as_array().cloned().unwrap_or_default(),
                inputs: t["inputs"]
                    .as_array()
                    .map(|a| a.iter().filter_map(|v| v.as_str().map(String::from)).collect())
                    .unwrap_or_default(),
                output: t["output"].as_str().unwrap_or("").to_string(),
            })
            .collect();
        Ok(Topology { stages })
    }

    async fn resolve_dataset(&self, dataset: &str) -> Result<Binding, String> {
        let ds = &self.def["datasets"][dataset];
        if ds.is_null() {
            return Err(format!(
                "pipeline.toml: dataset '{dataset}' is not defined under [datasets] — every transform input/output must be"
            ));
        }
        // The connectors crate's registry resolves both halves: `connector = "…"` names the
        // source (Kafka when absent), `sink = "…"` the sink (Kafka when absent). An input's sink
        // and an output's source are resolved and never opened.
        let cx = fv_streams_connectors::Ctx {
            dataset,
            ds,
            brokers: &self.def["brokers"],
        };
        for warning in fv_streams_connectors::check_dataset(&cx)? {
            eprintln!("fv-streams: {warning}");
        }
        Ok(Binding {
            api_name: dataset.to_string(),
            source: fv_streams_connectors::resolve_source(&cx)?,
            sink: fv_streams_connectors::resolve_sink(&cx)?,
        })
    }

    async fn publishes(&self, dataset: &str) -> bool {
        let ds = &self.def["datasets"][dataset];
        ds.get("sink").is_some() || ds.get("topic").is_some()
    }

    async fn heartbeat(&self, _build_id: &str) -> BuildSignal {
        if self.stop.load(Ordering::Relaxed) {
            BuildSignal::Stop
        } else {
            BuildSignal::Continue
        }
    }

    async fn put_record(&self, _build_id: &str, record: &J) {
        // No server to persist to — publish to the observability surface and print the counters.
        let _ = self.live.send(record.clone());
        println!("{}", progress_line(record));
    }
}

/// The line a heartbeat prints: the status and the counters, and how many messages the sinks are
/// waiting on when there are any, so a run that is not delivering shows it.
pub(crate) fn progress_line(record: &J) -> String {
    let m = &record["metrics"];
    let n = |k: &str| m[k].as_u64().unwrap_or(0);
    let status = record["status"].as_str().unwrap_or("RUNNING");
    let mut line = format!(
        "▸ {status}  consumed={} emitted={} dropped={}",
        n("consumed"),
        n("emitted"),
        n("dropped")
    );
    if n("pending") > 0 {
        line.push_str(&format!(" pending={}", n("pending")));
    }
    line
}