#[cfg(doctest)]
#[doc = include_str!("../README.md")]
pub struct ReadmeDoctests;
use fv_streams_types::settings::{Section, Setting};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use async_trait::async_trait;
use fv_streams_engine::{Binding, BuildSignal, ControlPlane, StageDef, Topology};
use serde_json::{json, Value as J};
pub mod flight_sql;
#[cfg(feature = "sandbox")]
pub mod sandbox;
pub mod serve;
fn toml_to_json(v: &toml::Value) -> J {
match v {
toml::Value::String(s) => J::String(s.clone()),
toml::Value::Integer(i) => json!(i),
toml::Value::Float(f) => json!(f),
toml::Value::Boolean(b) => json!(b),
toml::Value::Datetime(d) => J::String(d.to_string()),
toml::Value::Array(a) => J::Array(a.iter().map(toml_to_json).collect()),
toml::Value::Table(t) => J::Object(t.iter().map(|(k, v)| (k.clone(), toml_to_json(v))).collect()),
}
}
#[derive(Clone, Debug)]
pub struct Pipeline {
pub name: String,
pub path: String,
pub def: J,
}
impl Pipeline {
pub fn load(path: &str) -> Result<Self, String> {
let raw = std::fs::read_to_string(path).map_err(|e| format!("cannot read {path}: {e}"))?;
Self::parse(&raw, path)
}
pub fn parse(toml_text: &str, path: &str) -> Result<Self, String> {
let parsed: toml::Value = toml_text
.parse()
.map_err(|e| format!("{path} is not valid TOML: {e}"))?;
let def = toml_to_json(&parsed);
let name = def["name"].as_str().unwrap_or("pipeline").to_string();
Ok(Self {
name,
path: path.to_string(),
def,
})
}
pub fn default_brokers(&self) -> String {
self.def["brokers"]["default"]
.as_str()
.unwrap_or("localhost:9092")
.to_string()
}
pub fn apply_settings(&self) -> Result<(), String> {
let known = settings();
fv_streams_types::settings::check_table(&self.def["settings"], &format!("{}: [settings]", self.path), &known)?;
if let Some(table) = self.def["settings"].as_object() {
for (k, v) in table {
if std::env::var_os(k).is_none() {
std::env::set_var(k, fv_streams_types::settings::as_env(v));
}
}
}
for warning in fv_streams_types::settings::check_environment(&known)? {
eprintln!("fv-streams: {warning}");
}
Ok(())
}
pub async fn start_generators(&self) -> Result<Vec<String>, String> {
if !fv_streams_engine::settings::JOIN.raw().is_empty() {
return Ok(Vec::new());
}
let default_brokers = self.default_brokers();
let mut started = Vec::new();
if let Some(datasets) = self.def["datasets"].as_object() {
for (ds_name, ds) in datasets {
if let Some(parsed) = fv_streams_connectors::nexmark::parse(ds_name, ds, &default_brokers) {
fv_streams_connectors::check_dataset(&fv_streams_connectors::Ctx {
dataset: ds_name,
ds,
default_brokers: &default_brokers,
})?;
let cfg = parsed?;
if cfg.inline {
continue; }
fv_streams_connectors::kafka::ensure_topic(&cfg.brokers, &cfg.topic, cfg.partitions, 1, &[]) .await
.map_err(|e| format!("nexmark[{ds_name}]: create topic {}: {e}", cfg.topic))?;
let probe = cfg.clone();
let written = tokio::task::spawn_blocking(move || fv_streams_connectors::nexmark::written(&probe))
.await
.map_err(|e| format!("nexmark[{ds_name}]: {e}"))??;
if let fv_streams_connectors::nexmark::Written::All(n) = written {
println!(
"fv-streams: nexmark[{ds_name}] → {}: the topic already holds all {n} messages; not writing them again (delete the topic to generate it afresh)",
cfg.topic
);
continue;
}
println!(
"fv-streams: nexmark[{ds_name}] → {} ({} events, seed {}, step {}ms{})",
cfg.topic,
cfg.events,
cfg.seed,
cfg.step_ms,
if cfg.rate > 0 {
format!(", {} ev/s", cfg.rate)
} else {
String::new()
}
);
fv_streams_connectors::nexmark::spawn(cfg); started.push(ds_name.clone());
}
}
}
Ok(started)
}
pub async fn run(self, stop: Arc<AtomicBool>) -> Result<(), String> {
self.run_serving(stop, None).await
}
pub async fn run_serving(self, stop: Arc<AtomicBool>, serve: Option<&str>) -> Result<(), String> {
self.apply_settings()?;
self.start_generators().await?;
let record = json!({
"id": self.name,
"pipeline": self.name,
"kind": "stream",
"status": "RUNNING",
"startedAt": chrono::Utc::now().to_rfc3339(),
});
println!(
"fv-streams: running pipeline '{}' from {} (Ctrl-C to stop)",
self.name, self.path
);
let name = self.name.clone();
let cp = Arc::new(FileControlPlane {
def: self.def,
stop,
live: tokio::sync::watch::channel(json!({ "status": "STARTING", "metrics": {} })).0,
});
if let Some(addr) = serve {
let bound = serve::start(addr, cp.live()).await?;
println!("fv-streams: serving /health /ready /status /metrics on http://{bound}");
}
fv_streams_engine::run_stream(cp, name.clone(), name, record).await
}
}
pub const SERVE: Setting = Setting {
key: "STREAM_SERVE",
kind: fv_streams_types::options::Kind::Text,
default: "",
doc: "Serve `/health`, `/ready`, `/status` and `/metrics` on this address, `host:port`. Set by `--serve`. Empty: not served.",
};
static RUNNER: Section = Section {
title: "The runner",
intro: "The `fv-streams` command's own settings.",
settings: &[SERVE],
};
pub fn settings() -> Vec<&'static Section> {
fv_streams_engine::settings::SECTIONS
.iter()
.chain(fv_streams_connectors::settings())
.chain([&RUNNER])
.collect()
}
pub struct FileControlPlane {
def: J,
stop: Arc<AtomicBool>,
live: tokio::sync::watch::Sender<J>,
}
impl FileControlPlane {
pub fn new(pipeline: &Pipeline, stop: Arc<AtomicBool>) -> Self {
Self {
def: pipeline.def.clone(),
stop,
live: tokio::sync::watch::channel(json!({ "status": "STARTING", "metrics": {} })).0,
}
}
pub fn live(&self) -> serve::Live {
self.live.subscribe()
}
}
#[async_trait]
impl ControlPlane for FileControlPlane {
async fn topology(&self, _pipeline: &str) -> Result<Topology, String> {
if let Some(sql) = self.def["sql"].as_str() {
return fv_streams_sql::compile(sql).map_err(|e| format!("pipeline SQL: {e}"));
}
let transforms = self.def["transforms"]
.as_array()
.ok_or("pipeline.toml: needs at least one [[transforms]] table (or a `sql = \"…\"` field)")?;
let stages = transforms
.iter()
.map(|t| StageDef {
steps: t["steps"].as_array().cloned().unwrap_or_default(),
inputs: t["inputs"]
.as_array()
.map(|a| a.iter().filter_map(|v| v.as_str().map(String::from)).collect())
.unwrap_or_default(),
output: t["output"].as_str().unwrap_or("").to_string(),
})
.collect();
Ok(Topology { stages })
}
async fn resolve_dataset(&self, dataset: &str) -> Result<Binding, String> {
let ds = &self.def["datasets"][dataset];
if ds.is_null() {
return Err(format!(
"pipeline.toml: dataset '{dataset}' is not defined under [datasets] — every transform input/output must be"
));
}
let default_brokers = self.def["brokers"]["default"]
.as_str()
.unwrap_or("localhost:9092")
.to_string();
let cx = fv_streams_connectors::Ctx {
dataset,
ds,
default_brokers: &default_brokers,
};
for warning in fv_streams_connectors::check_dataset(&cx)? {
eprintln!("fv-streams: {warning}");
}
Ok(Binding {
api_name: dataset.to_string(),
source: fv_streams_connectors::resolve_source(&cx)?,
sink: fv_streams_connectors::resolve_sink(&cx)?,
})
}
async fn heartbeat(&self, _build_id: &str) -> BuildSignal {
if self.stop.load(Ordering::Relaxed) {
BuildSignal::Stop
} else {
BuildSignal::Continue
}
}
async fn put_record(&self, _build_id: &str, record: &J) {
let _ = self.live.send(record.clone());
let m = &record["metrics"];
let status = record["status"].as_str().unwrap_or("RUNNING");
println!(
"▸ {status} consumed={} emitted={} dropped={}",
m["consumed"].as_u64().unwrap_or(0),
m["emitted"].as_u64().unwrap_or(0),
m["dropped"].as_u64().unwrap_or(0)
);
}
}
#[cfg(test)]
mod tests {
use super::*;
const FILE: &str = r#"
name = "demo"
[settings]
STREAM_BATCH_MAX = "10"
[brokers]
default = "broker:9092"
[datasets.events]
topic = "in"
[datasets.out]
topic = "out"
brokers = "other:9092"
[[transforms]]
inputs = ["events"]
output = "out"
[[transforms.steps]]
op = "filter"
expression = "x > 1"
"#;
#[test]
fn parses_name_brokers_and_topology() {
let p = Pipeline::parse(FILE, "demo.toml").unwrap();
assert_eq!(p.name, "demo");
assert_eq!(p.default_brokers(), "broker:9092");
let cp = FileControlPlane::new(&p, Arc::new(AtomicBool::new(false)));
let topo = tokio::runtime::Runtime::new()
.unwrap()
.block_on(cp.topology("demo"))
.unwrap();
assert_eq!(topo.stages.len(), 1);
assert_eq!(topo.stages[0].inputs, vec!["events".to_string()]);
assert_eq!(topo.stages[0].steps[0]["op"], "filter");
}
#[test]
fn resolves_datasets_through_the_registry_with_per_dataset_broker_override() {
let p = Pipeline::parse(FILE, "demo.toml").unwrap();
let cp = FileControlPlane::new(&p, Arc::new(AtomicBool::new(false)));
let rt = tokio::runtime::Runtime::new().unwrap();
let events = rt.block_on(cp.resolve_dataset("events")).unwrap();
assert_eq!(events.source.name(), "in");
assert!(
events.source.describe().contains("broker:9092"),
"{}",
events.source.describe()
);
let out = rt.block_on(cp.resolve_dataset("out")).unwrap();
assert!(out.sink.describe().contains("other:9092"), "{}", out.sink.describe());
let err = rt.block_on(cp.resolve_dataset("nope")).unwrap_err();
assert!(err.contains("not defined under [datasets]"), "{err}");
}
#[test]
fn the_stop_flag_drives_the_heartbeat() {
let p = Pipeline::parse(FILE, "demo.toml").unwrap();
let stop = Arc::new(AtomicBool::new(false));
let cp = FileControlPlane::new(&p, stop.clone());
let rt = tokio::runtime::Runtime::new().unwrap();
assert_eq!(rt.block_on(cp.heartbeat("demo")), BuildSignal::Continue);
stop.store(true, Ordering::Relaxed);
assert_eq!(rt.block_on(cp.heartbeat("demo")), BuildSignal::Stop);
}
#[test]
fn invalid_toml_is_a_readable_error() {
let err = Pipeline::parse("name = [", "bad.toml").unwrap_err();
assert!(err.starts_with("bad.toml is not valid TOML"), "{err}");
}
}