Skip to main content

fv_streams_connectors/
lib.rs

1//! THE CONNECTORS of FusionVault Streams — one directory per connector, each behind its own
2//! feature, all through the contract crate's seams: a source is an
3//! [`fv_streams_types::InlineSource`], a sink an
4//! [`fv_streams_types::OutputSink`]; the engine runs them as tasks and never knows
5//! which is which. Adding a connector is a directory (`mod.rs`: the config parse, the [`Connector`]
6//! registration and its description, [`Connector::about`]; `source.rs` / `sink.rs`: the
7//! implementations), a feature line, and one entry in [`connectors`]. The description is what the
8//! dataset check ([`check_dataset`]) accepts and what `docs/reference/connectors.md` shows.
9//!
10//! | `connector = …` / `sink = …` | directory | feature |
11//! |---|---|---|
12//! | `blackhole` (sink) | `blackhole` | — |
13//! | `file`, `arrow` (source + sink) | `file` | — (`s3` / `gcs` / `azure` for an object store) |
14//! | `table` (source) | `table` | `sqlite` / `postgres` / `mysql` |
15//! | `nexmark` (source) | `nexmark` | — |
16//! | `iceberg` (sink) | `iceberg` | `iceberg` |
17//! | `flight` (sink) | `flight` | `flight` |
18//! | `kafka` (source + sink) — **the default** when a dataset names neither | `kafka` | `kafka` |
19//! | `nats` (source + sink; JetStream) | `nats` | `nats` |
20//! | `stdout` (sink) — the minimum connector, the worked example | `stdout` | — |
21//! | `http` (polling source), `webhook` (sink) | `http` | `http` |
22//! | `websocket` (source: a live feed) | `websocket` | `websocket` |
23
24/// Every Rust code block in the README is compiled and run as a doctest.
25#[cfg(doctest)]
26#[doc = include_str!("../README.md")]
27pub struct ReadmeDoctests;
28
29use std::sync::Arc;
30
31use fv_streams_types::{InlineSource, OutputSink};
32use serde_json::Value as J;
33
34mod block;
35pub mod exactly_once;
36pub mod fanout;
37pub mod messaging;
38pub mod reference;
39
40pub mod blackhole;
41pub mod file;
42#[cfg(feature = "flight")]
43pub mod flight;
44#[cfg(feature = "http")]
45pub mod http;
46#[cfg(feature = "iceberg")]
47pub mod iceberg;
48#[cfg(feature = "kafka")]
49pub mod kafka;
50#[cfg(feature = "nats")]
51pub mod nats;
52pub mod nexmark;
53pub mod stdout;
54pub mod table;
55#[cfg(feature = "websocket")]
56pub mod websocket;
57
58pub use fv_streams_formats as formats;
59
60/// What a connector parses: the dataset's name and its table from the pipeline definition, and the
61/// pipeline's `[brokers]` table (`Null` when it has none): the default brokers, and the connection
62/// options, for a connector that reaches Kafka.
63pub struct Ctx<'a> {
64    pub dataset: &'a str,
65    pub ds: &'a J,
66    pub brokers: &'a J,
67}
68
69/// What a connector is and every option it takes, per role — the one description the dataset check
70/// ([`check_dataset`]) and the generated reference (`docs/reference/connectors.md`, [`reference`](mod@reference))
71/// both read.
72#[derive(Debug)]
73pub struct ConnectorDoc {
74    /// What it connects to, in a sentence or two.
75    pub summary: &'static str,
76    /// The cargo feature that builds it; `""` when every build has it.
77    pub feature: &'static str,
78    pub source: Option<RoleDoc>,
79    pub sink: Option<RoleDoc>,
80    /// What it does not do yet, one plain sentence each.
81    pub limits: &'static [&'static str],
82    /// Complete `[datasets.<name>]` tables using it (one per role when it has both).
83    pub example: &'static str,
84}
85
86/// One role of a connector (source or sink).
87#[derive(Debug)]
88pub struct RoleDoc {
89    /// How it reads or writes, what decides the parallelism, and what the row key is.
90    pub summary: &'static str,
91    /// Source: whether it ends (`bounded`), never ends (`unbounded`), or when each. Sink: what it
92    /// writes, and in what format. A few words: the reference's overview table shows it.
93    pub shape: &'static str,
94    /// Source: what a restart resumes from. Sink: the delivery guarantee, per mode. A few words:
95    /// the overview table shows it.
96    pub guarantee: &'static str,
97    pub options: &'static [fv_streams_types::options::Opt],
98}
99
100/// One connector: the `connector = "…"` spellings it reads as a source, the `sink = "…"` spellings
101/// it writes as a sink, and the parse of each from the dataset table into the seam's kinds.
102pub trait Connector: Send + Sync {
103    /// What the connector is and every option it takes. `None`: not described — its datasets'
104    /// keys are not checked (a connector written outside this crate may leave it out).
105    fn about(&self) -> Option<ConnectorDoc> {
106        None
107    }
108    /// The `STREAM_*` settings it reads (its client's tuning), described once; they are checked
109    /// at start and listed in `docs/settings.md`.
110    fn settings(&self) -> Option<&'static fv_streams_types::settings::Section> {
111        None
112    }
113    fn name(&self) -> &'static str;
114    fn sources(&self) -> &'static [&'static str] {
115        &[]
116    }
117    fn sinks(&self) -> &'static [&'static str] {
118        &[]
119    }
120    /// True when its `format` is a message format ([`fv_streams_types::format`]), read through
121    /// [`messaging::format_for`]: a dataset naming it in one role then shares its `format` with a
122    /// message connector in the other. False (the default) when `format` means something of its own,
123    /// such as a file's.
124    fn message_format(&self) -> bool {
125        false
126    }
127    fn source(&self, cx: &Ctx<'_>) -> Result<Arc<dyn InlineSource>, String> {
128        Err(format!(
129            "connector {} has no source (dataset {})",
130            self.name(),
131            cx.dataset
132        ))
133    }
134    fn sink(&self, cx: &Ctx<'_>) -> Result<Arc<dyn OutputSink>, String> {
135        Err(format!(
136            "connector {} has no sink (dataset {})",
137            self.name(),
138            cx.dataset
139        ))
140    }
141}
142
143/// Every connector this build carries.
144pub fn connectors() -> Vec<Box<dyn Connector>> {
145    // pushes for every connector (a build with no optional feature still uses `mut`)
146    let mut all: Vec<Box<dyn Connector>> = Vec::with_capacity(12);
147    all.push(Box::new(blackhole::Blackhole));
148    all.push(Box::new(file::File));
149    all.push(Box::new(table::Table));
150    all.push(Box::new(nexmark::Nexmark));
151    all.push(Box::new(stdout::Stdout));
152    #[cfg(feature = "iceberg")]
153    all.push(Box::new(iceberg::Iceberg));
154    #[cfg(feature = "flight")]
155    all.push(Box::new(flight::Flight));
156    #[cfg(feature = "http")]
157    all.push(Box::new(http::Http));
158    #[cfg(feature = "kafka")]
159    all.push(Box::new(kafka::Kafka));
160    #[cfg(feature = "nats")]
161    all.push(Box::new(nats::Nats));
162    #[cfg(feature = "websocket")]
163    all.push(Box::new(websocket::Websocket));
164    all
165}
166
167/// Every setting the connectors of this build read, by area: the shared source settings, then each
168/// connector's, by title.
169pub fn settings() -> Vec<&'static fv_streams_types::settings::Section> {
170    let mut each: Vec<_> = connectors().iter().filter_map(|c| c.settings()).collect();
171    each.sort_by_key(|s| s.title);
172    std::iter::once(&messaging::SETTINGS).chain(each).collect()
173}
174
175/// The connector a dataset that names none gets: Kafka, when this build carries it.
176pub(crate) const DEFAULT: &str = "kafka";
177
178/// The source a dataset's `connector = "…"` names — or the default connector's when it names none
179/// (a build without the default names the gap). `Err` names the dataset and what is wrong.
180pub fn resolve_source(cx: &Ctx<'_>) -> Result<Arc<dyn InlineSource>, String> {
181    let all = connectors();
182    match cx.ds["connector"].as_str() {
183        Some(name) => match all.iter().find(|c| c.sources().contains(&name)) {
184            Some(c) => c.source(cx),
185            None => Err(format!(
186                "pipeline.toml: [datasets.{}] connector = \"{name}\" is not known ({})",
187                cx.dataset,
188                all.iter().flat_map(|c| c.sources().iter().copied()).collect::<Vec<_>>().join(", ")
189            )),
190        },
191        None => match all.iter().find(|c| c.name() == DEFAULT) {
192            Some(c) => c.source(cx),
193            None => Err(format!(
194                "pipeline.toml: [datasets.{}] names no `connector`, and this build has no default ({DEFAULT}) connector",
195                cx.dataset
196            )),
197        },
198    }
199}
200
201/// Dataset options that were renamed, `(old, new)`: an old name is still accepted wherever its new
202/// name is, with a warning that names the new one; setting both is refused.
203pub const RENAMED: &[(&str, &str)] = &[("infer_sample", "inferSample"), ("on_bad_data", "onBadData")];
204
205/// Check a dataset's keys against what its connectors take: every key must be `connector`, `sink`,
206/// or an option of the source the dataset names (or the default's) or of its sink — or the old name
207/// of one ([`RENAMED`]). An unknown key is refused with the closest right one; the error lists what
208/// the dataset takes, the roles it names first. `Ok` holds a warning for each old name used. A
209/// dataset whose source or sink is not described is not checked.
210pub fn check_dataset(cx: &Ctx<'_>) -> Result<Vec<String>, String> {
211    let all = connectors();
212    let named = |key: &str, of: fn(&dyn Connector) -> &'static [&'static str]| match cx.ds[key].as_str() {
213        Some(name) => all.iter().find(|c| of(c.as_ref()).contains(&name)),
214        None => all.iter().find(|c| c.name() == DEFAULT),
215    };
216    let (Some(source), Some(sink)) = (named("connector", |c| c.sources()), named("sink", |c| c.sinks())) else {
217        return Ok(Vec::new()); // an unknown spelling is the resolver's error, with its list
218    };
219    let (Some(source), Some(sink)) = (source.about(), sink.about()) else {
220        return Ok(Vec::new());
221    };
222    let mut roles = [
223        (cx.ds["connector"].is_null(), source.source.as_ref()),
224        (cx.ds["sink"].is_null(), sink.sink.as_ref()),
225    ];
226    roles.sort_by_key(|(defaulted, _)| *defaulted);
227    let mut accepted: Vec<&str> = vec!["connector", "sink"];
228    for o in roles.iter().filter_map(|(_, r)| *r).flat_map(|r| r.options) {
229        if !accepted.contains(&o.key) {
230            accepted.push(o.key);
231        }
232    }
233    let at = format!("pipeline.toml: [datasets.{}]", cx.dataset);
234    let renamed: Vec<(&str, &str)> = RENAMED
235        .iter()
236        .copied()
237        .filter(|(_, new)| accepted.contains(new))
238        .collect();
239    fv_streams_types::options::check_keys(
240        cx.ds,
241        &at,
242        &renamed.iter().map(|(old, _)| *old).collect::<Vec<_>>(),
243        accepted.iter().copied(),
244    )?;
245    let mut warnings = Vec::new();
246    for (old, new) in renamed.into_iter().filter(|(old, _)| !cx.ds[*old].is_null()) {
247        if !cx.ds[new].is_null() {
248            return Err(format!(
249                "{at}: sets both `{new}` and its old name `{old}`; keep `{new}`"
250            ));
251        }
252        warnings.push(format!(
253            "{at}: `{old}` is the old name of `{new}`; it still works — rename it"
254        ));
255    }
256    Ok(warnings)
257}
258
259/// The sink a dataset's `sink = "…"` names — or the default connector's when it names none.
260pub fn resolve_sink(cx: &Ctx<'_>) -> Result<Arc<dyn OutputSink>, String> {
261    let all = connectors();
262    match cx.ds["sink"].as_str() {
263        Some(name) => match all.iter().find(|c| c.sinks().contains(&name)) {
264            Some(c) => c.sink(cx),
265            None => Err(format!(
266                "pipeline.toml: [datasets.{}] sink = \"{name}\" is not known ({})",
267                cx.dataset,
268                all.iter()
269                    .flat_map(|c| c.sinks().iter().copied())
270                    .collect::<Vec<_>>()
271                    .join(", ")
272            )),
273        },
274        None => match all.iter().find(|c| c.name() == DEFAULT) {
275            Some(c) => c.sink(cx),
276            None => Err(format!(
277                "pipeline.toml: [datasets.{}] names no `sink`, and this build has no default ({DEFAULT}) connector",
278                cx.dataset
279            )),
280        },
281    }
282}
283
284#[cfg(test)]
285mod tests;