Skip to main content

fv_streams/
lib.rs

1//! Run a stream pipeline from one TOML file, with no control plane anywhere.
2//!
3//! The engine (`fv-streams-engine`) asks its host four questions through the
4//! [`ControlPlane`](fv_streams_engine::ControlPlane) trait: what is the pipeline's topology, which topic backs a dataset, should I
5//! keep running, and here is my build record. This crate answers them from a file. The same engine
6//! that a platform hosts over HTTP therefore runs from a laptop, a container, or a test with the
7//! same semantics: checkpoints, exactly-once output, stateful windows and joins.
8//!
9//! ```no_run
10//! use std::sync::atomic::AtomicBool;
11//! use std::sync::Arc;
12//! use fv_streams::Pipeline;
13//!
14//! # tokio::runtime::Runtime::new().unwrap().block_on(async {
15//! let stop = Arc::new(AtomicBool::new(false));      // set it to stop cleanly (Ctrl-C does)
16//! Pipeline::load("pipeline.toml")?.run(stop).await;
17//! # Ok::<(), String>(())
18//! # });
19//! ```
20//!
21//! A pipeline file names its datasets (Kafka topics), its transforms (the engine's own step
22//! vocabulary: inline steps, `wasm`/`container` compute, `windowedAggregate`, `sessionAggregate`,
23//! `streamJoin`, `topN`, `lastN`) and optional engine settings. The pipeline's `name` is its stable
24//! identity: the consumer group `fv-stream-<name>`, the state topic `fv-state-<name>` and the
25//! exactly-once transactional namespace all derive from it, so a restarted run restores its
26//! checkpoints and fences its own zombies.
27
28/// Every Rust code block in the README is compiled and run as a doctest.
29#[cfg(doctest)]
30#[doc = include_str!("../README.md")]
31pub struct ReadmeDoctests;
32
33use fv_streams_types::settings::{Section, Setting};
34
35pub mod flight_sql;
36pub mod serve;
37
38mod control_plane;
39mod pipeline;
40
41pub use control_plane::FileControlPlane;
42pub use pipeline::Pipeline;
43
44/// `STREAM_SERVE`: the runner's observability listener (`--serve`).
45pub const SERVE: Setting = Setting {
46    key: "STREAM_SERVE",
47    kind: fv_streams_types::options::Kind::Text,
48    default: "",
49    doc: "Serve `/health`, `/ready`, `/status` and `/metrics` on this address, `host:port`. Set by `--serve`. Empty: not served.",
50};
51
52static RUNNER: Section = Section {
53    title: "The runner",
54    intro: "The `fv-streams` command's own settings.",
55    settings: &[SERVE],
56};
57
58/// Every setting this runner reads, by area — the engine's, the connectors', its own — what a
59/// pipeline's `[settings]` and the environment are checked against, and what `docs/settings.md`
60/// lists.
61pub fn settings() -> Vec<&'static Section> {
62    fv_streams_engine::settings::SECTIONS
63        .iter()
64        .chain(fv_streams_connectors::settings())
65        .chain([&RUNNER])
66        .collect()
67}
68
69#[cfg(test)]
70mod tests;