#[cfg(doctest)]
#[doc = include_str!("../README.md")]
pub struct ReadmeDoctests;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use async_trait::async_trait;
use fv_streams_engine::{BuildSignal, ControlPlane, StageDef, TopicBinding, Topology};
use serde_json::{json, Value as J};
pub mod arrow_source;
pub mod flight_sql;
pub mod nexmark;
#[cfg(feature = "sandbox")]
pub mod sandbox;
fn toml_to_json(v: &toml::Value) -> J {
match v {
toml::Value::String(s) => J::String(s.clone()),
toml::Value::Integer(i) => json!(i),
toml::Value::Float(f) => json!(f),
toml::Value::Boolean(b) => json!(b),
toml::Value::Datetime(d) => J::String(d.to_string()),
toml::Value::Array(a) => J::Array(a.iter().map(toml_to_json).collect()),
toml::Value::Table(t) => J::Object(t.iter().map(|(k, v)| (k.clone(), toml_to_json(v))).collect()),
}
}
#[derive(Clone, Debug)]
pub struct Pipeline {
pub name: String,
pub path: String,
pub def: J,
}
impl Pipeline {
pub fn load(path: &str) -> Result<Self, String> {
let raw = std::fs::read_to_string(path).map_err(|e| format!("cannot read {path}: {e}"))?;
Self::parse(&raw, path)
}
pub fn parse(toml_text: &str, path: &str) -> Result<Self, String> {
let parsed: toml::Value = toml_text
.parse()
.map_err(|e| format!("{path} is not valid TOML: {e}"))?;
let def = toml_to_json(&parsed);
let name = def["name"].as_str().unwrap_or("pipeline").to_string();
Ok(Self {
name,
path: path.to_string(),
def,
})
}
pub fn default_brokers(&self) -> String {
self.def["brokers"]["default"]
.as_str()
.unwrap_or("localhost:9092")
.to_string()
}
pub fn apply_settings(&self) {
if let Some(settings) = self.def["settings"].as_object() {
for (k, v) in settings {
if std::env::var(k).is_err() {
let val = match v {
J::String(s) => s.clone(),
other => other.to_string(),
};
std::env::set_var(k, val);
}
}
}
}
pub async fn start_generators(&self) -> Result<Vec<String>, String> {
if std::env::var("STREAM_JOIN").map(|v| !v.is_empty()).unwrap_or(false) {
return Ok(Vec::new());
}
let default_brokers = self.default_brokers();
let mut started = Vec::new();
if let Some(datasets) = self.def["datasets"].as_object() {
for (ds_name, ds) in datasets {
if let Some(parsed) = nexmark::parse(ds_name, ds, &default_brokers) {
let cfg = parsed?;
if cfg.inline {
continue; }
fv_streams_engine::ensure_topic(&cfg.brokers, &cfg.topic, cfg.partitions, &[])
.await
.map_err(|e| format!("nexmark[{ds_name}]: create topic {}: {e}", cfg.topic))?;
println!(
"fv-streams: nexmark[{ds_name}] → {} ({} events, seed {}, step {}ms{})",
cfg.topic,
cfg.events,
cfg.seed,
cfg.step_ms,
if cfg.rate > 0 {
format!(", {} ev/s", cfg.rate)
} else {
String::new()
}
);
nexmark::spawn(cfg); started.push(ds_name.clone());
}
}
}
Ok(started)
}
pub async fn run(self, stop: Arc<AtomicBool>) -> Result<(), String> {
self.apply_settings();
self.start_generators().await?;
let record = json!({
"id": self.name,
"pipeline": self.name,
"kind": "stream",
"status": "RUNNING",
"startedAt": chrono::Utc::now().to_rfc3339(),
});
println!(
"fv-streams: running pipeline '{}' from {} (Ctrl-C to stop)",
self.name, self.path
);
let name = self.name.clone();
let cp = Arc::new(FileControlPlane { def: self.def, stop });
fv_streams_engine::run_stream(cp, name.clone(), name, record).await
}
}
pub struct FileControlPlane {
def: J,
stop: Arc<AtomicBool>,
}
impl FileControlPlane {
pub fn new(pipeline: &Pipeline, stop: Arc<AtomicBool>) -> Self {
Self {
def: pipeline.def.clone(),
stop,
}
}
}
#[async_trait]
impl ControlPlane for FileControlPlane {
async fn topology(&self, _pipeline: &str) -> Result<Topology, String> {
if let Some(sql) = self.def["sql"].as_str() {
return fv_streams_sql::compile(sql).map_err(|e| format!("pipeline SQL: {e}"));
}
let transforms = self.def["transforms"]
.as_array()
.ok_or("pipeline.toml: needs at least one [[transforms]] table (or a `sql = \"…\"` field)")?;
let stages = transforms
.iter()
.map(|t| StageDef {
steps: t["steps"].as_array().cloned().unwrap_or_default(),
inputs: t["inputs"]
.as_array()
.map(|a| a.iter().filter_map(|v| v.as_str().map(String::from)).collect())
.unwrap_or_default(),
output: t["output"].as_str().unwrap_or("").to_string(),
})
.collect();
Ok(Topology { stages })
}
async fn resolve_topic(&self, dataset: &str) -> Result<TopicBinding, String> {
let ds = &self.def["datasets"][dataset];
if ds.is_null() {
return Err(format!(
"pipeline.toml: dataset '{dataset}' is not defined under [datasets] — every transform input/output must be"
));
}
let default_brokers = self.def["brokers"]["default"]
.as_str()
.unwrap_or("localhost:9092")
.to_string();
let source = if let Some(parsed) = arrow_source::parse(dataset, ds) {
fv_streams_engine::SourceKind::Inline(std::sync::Arc::new(arrow_source::ArrowInline {
cfg: parsed?,
}))
} else {
match nexmark::parse(dataset, ds, &default_brokers) {
Some(parsed) => {
let cfg = parsed?;
if cfg.inline {
fv_streams_engine::SourceKind::Inline(std::sync::Arc::new(nexmark::NexmarkInline { cfg }))
} else {
fv_streams_engine::SourceKind::Kafka
}
}
None => fv_streams_engine::SourceKind::Kafka,
}
};
let sink = match ds["sink"].as_str() {
Some("blackhole") => fv_streams_engine::SinkKind::Blackhole,
Some("arrow") => fv_streams_engine::SinkKind::Arrow,
Some("flight") => fv_streams_engine::SinkKind::Flight,
Some(other) => {
return Err(format!(
"pipeline.toml: [datasets.{dataset}] sink = \"{other}\" is not known (blackhole, arrow, flight)"
))
}
None => fv_streams_engine::SinkKind::Kafka,
};
let topicless = matches!(source, fv_streams_engine::SourceKind::Inline(_))
|| matches!(
sink,
fv_streams_engine::SinkKind::Blackhole
| fv_streams_engine::SinkKind::Arrow
| fv_streams_engine::SinkKind::Flight
);
let topic = if matches!(sink, fv_streams_engine::SinkKind::Arrow) {
ds["path"]
.as_str()
.ok_or_else(|| format!("pipeline.toml: [datasets.{dataset}] sink = \"arrow\" needs `path = \"…\"`"))?
.to_string()
} else if matches!(sink, fv_streams_engine::SinkKind::Flight) {
let port = ds["port"]
.as_i64()
.filter(|p| (1..=65535).contains(p))
.ok_or_else(|| format!("pipeline.toml: [datasets.{dataset}] sink = \"flight\" needs `port = <1..=65535>`"))?;
let host = ds["host"].as_str().unwrap_or("127.0.0.1");
format!("{host}:{port}")
} else {
match ds["topic"].as_str() {
Some(t) => t.to_string(),
None if topicless => String::new(),
None => return Err(format!("pipeline.toml: [datasets.{dataset}] needs `topic = \"…\"`")),
}
};
let brokers = ds["brokers"]
.as_str()
.or_else(|| self.def["brokers"]["default"].as_str())
.unwrap_or("localhost:9092")
.to_string();
let schema = match ds["schema"].as_array() {
Some(cols) => {
let fields = cols
.iter()
.map(|c| {
let name = c["name"]
.as_str()
.ok_or_else(|| format!("pipeline.toml: [datasets.{dataset}] schema column needs `name`"))?;
let ty = c["type"].as_str().unwrap_or("string");
fv_streams_engine::decode::declared_field(name, ty)
.map_err(|e| format!("pipeline.toml: [datasets.{dataset}] {e}"))
})
.collect::<Result<Vec<_>, String>>()?;
fv_streams_engine::decode::SchemaMode::Declared(std::sync::Arc::new(
fv_streams_engine::decode::arrow_schema(fields),
))
}
None => fv_streams_engine::decode::SchemaMode::Inferred {
sample: ds["infer_sample"].as_u64().unwrap_or(5000) as usize,
},
};
let on_bad_data = match ds["on_bad_data"].as_str() {
Some(s) => fv_streams_engine::decode::OnBadData::parse(s)
.map_err(|e| format!("pipeline.toml: [datasets.{dataset}] {e}"))?,
None => Default::default(),
};
Ok(TopicBinding {
api_name: dataset.to_string(),
topic,
brokers,
policy: fv_streams_engine::decode::SourcePolicy { schema, on_bad_data },
source,
sink,
})
}
async fn heartbeat(&self, _build_id: &str) -> BuildSignal {
if self.stop.load(Ordering::Relaxed) {
BuildSignal::Stop
} else {
BuildSignal::Continue
}
}
async fn put_record(&self, _build_id: &str, record: &J) {
let m = &record["metrics"];
let status = record["status"].as_str().unwrap_or("RUNNING");
println!(
"▸ {status} consumed={} emitted={} dropped={}",
m["consumed"].as_u64().unwrap_or(0),
m["emitted"].as_u64().unwrap_or(0),
m["dropped"].as_u64().unwrap_or(0)
);
}
}
#[cfg(test)]
mod tests {
use super::*;
const FILE: &str = r#"
name = "demo"
[settings]
STREAM_BATCH_MAX = "10"
[brokers]
default = "broker:9092"
[datasets.events]
topic = "in"
[datasets.out]
topic = "out"
brokers = "other:9092"
[[transforms]]
inputs = ["events"]
output = "out"
[[transforms.steps]]
op = "filter"
expression = "x > 1"
"#;
#[test]
fn parses_name_brokers_and_topology() {
let p = Pipeline::parse(FILE, "demo.toml").unwrap();
assert_eq!(p.name, "demo");
assert_eq!(p.default_brokers(), "broker:9092");
let cp = FileControlPlane::new(&p, Arc::new(AtomicBool::new(false)));
let topo = tokio::runtime::Runtime::new()
.unwrap()
.block_on(cp.topology("demo"))
.unwrap();
assert_eq!(topo.stages.len(), 1);
assert_eq!(topo.stages[0].inputs, vec!["events".to_string()]);
assert_eq!(topo.stages[0].steps[0]["op"], "filter");
}
#[test]
fn resolves_topics_with_per_dataset_broker_override() {
let p = Pipeline::parse(FILE, "demo.toml").unwrap();
let cp = FileControlPlane::new(&p, Arc::new(AtomicBool::new(false)));
let rt = tokio::runtime::Runtime::new().unwrap();
let events = rt.block_on(cp.resolve_topic("events")).unwrap();
assert_eq!((events.topic.as_str(), events.brokers.as_str()), ("in", "broker:9092"));
let out = rt.block_on(cp.resolve_topic("out")).unwrap();
assert_eq!(out.brokers, "other:9092");
let err = rt.block_on(cp.resolve_topic("nope")).unwrap_err();
assert!(err.contains("not defined under [datasets]"), "{err}");
}
#[test]
fn the_stop_flag_drives_the_heartbeat() {
let p = Pipeline::parse(FILE, "demo.toml").unwrap();
let stop = Arc::new(AtomicBool::new(false));
let cp = FileControlPlane::new(&p, stop.clone());
let rt = tokio::runtime::Runtime::new().unwrap();
assert_eq!(rt.block_on(cp.heartbeat("demo")), BuildSignal::Continue);
stop.store(true, Ordering::Relaxed);
assert_eq!(rt.block_on(cp.heartbeat("demo")), BuildSignal::Stop);
}
#[test]
fn invalid_toml_is_a_readable_error() {
let err = Pipeline::parse("name = [", "bad.toml").unwrap_err();
assert!(err.starts_with("bad.toml is not valid TOML"), "{err}");
}
}