fv_streams_connectors/
lib.rs1#[cfg(doctest)]
26#[doc = include_str!("../README.md")]
27pub struct ReadmeDoctests;
28
29use std::sync::Arc;
30
31use fv_streams_types::{InlineSource, OutputSink};
32use serde_json::Value as J;
33
34mod block;
35pub mod exactly_once;
36pub mod fanout;
37pub mod messaging;
38pub mod reference;
39
40pub mod blackhole;
41pub mod file;
42#[cfg(feature = "flight")]
43pub mod flight;
44#[cfg(feature = "http")]
45pub mod http;
46#[cfg(feature = "iceberg")]
47pub mod iceberg;
48#[cfg(feature = "kafka")]
49pub mod kafka;
50#[cfg(feature = "nats")]
51pub mod nats;
52pub mod nexmark;
53pub mod stdout;
54pub mod table;
55#[cfg(feature = "websocket")]
56pub mod websocket;
57
58pub use fv_streams_formats as formats;
59
60pub struct Ctx<'a> {
64 pub dataset: &'a str,
65 pub ds: &'a J,
66 pub brokers: &'a J,
67}
68
69#[derive(Debug)]
73pub struct ConnectorDoc {
74 pub summary: &'static str,
76 pub feature: &'static str,
78 pub source: Option<RoleDoc>,
79 pub sink: Option<RoleDoc>,
80 pub limits: &'static [&'static str],
82 pub example: &'static str,
84}
85
86#[derive(Debug)]
88pub struct RoleDoc {
89 pub summary: &'static str,
91 pub shape: &'static str,
94 pub guarantee: &'static str,
97 pub options: &'static [fv_streams_types::options::Opt],
98}
99
100pub trait Connector: Send + Sync {
103 fn about(&self) -> Option<ConnectorDoc> {
106 None
107 }
108 fn settings(&self) -> Option<&'static fv_streams_types::settings::Section> {
111 None
112 }
113 fn name(&self) -> &'static str;
114 fn sources(&self) -> &'static [&'static str] {
115 &[]
116 }
117 fn sinks(&self) -> &'static [&'static str] {
118 &[]
119 }
120 fn message_format(&self) -> bool {
125 false
126 }
127 fn source(&self, cx: &Ctx<'_>) -> Result<Arc<dyn InlineSource>, String> {
128 Err(format!(
129 "connector {} has no source (dataset {})",
130 self.name(),
131 cx.dataset
132 ))
133 }
134 fn sink(&self, cx: &Ctx<'_>) -> Result<Arc<dyn OutputSink>, String> {
135 Err(format!(
136 "connector {} has no sink (dataset {})",
137 self.name(),
138 cx.dataset
139 ))
140 }
141}
142
143pub fn connectors() -> Vec<Box<dyn Connector>> {
145 let mut all: Vec<Box<dyn Connector>> = Vec::with_capacity(12);
147 all.push(Box::new(blackhole::Blackhole));
148 all.push(Box::new(file::File));
149 all.push(Box::new(table::Table));
150 all.push(Box::new(nexmark::Nexmark));
151 all.push(Box::new(stdout::Stdout));
152 #[cfg(feature = "iceberg")]
153 all.push(Box::new(iceberg::Iceberg));
154 #[cfg(feature = "flight")]
155 all.push(Box::new(flight::Flight));
156 #[cfg(feature = "http")]
157 all.push(Box::new(http::Http));
158 #[cfg(feature = "kafka")]
159 all.push(Box::new(kafka::Kafka));
160 #[cfg(feature = "nats")]
161 all.push(Box::new(nats::Nats));
162 #[cfg(feature = "websocket")]
163 all.push(Box::new(websocket::Websocket));
164 all
165}
166
167pub fn settings() -> Vec<&'static fv_streams_types::settings::Section> {
170 let mut each: Vec<_> = connectors().iter().filter_map(|c| c.settings()).collect();
171 each.sort_by_key(|s| s.title);
172 std::iter::once(&messaging::SETTINGS).chain(each).collect()
173}
174
175pub(crate) const DEFAULT: &str = "kafka";
177
178pub fn resolve_source(cx: &Ctx<'_>) -> Result<Arc<dyn InlineSource>, String> {
181 let all = connectors();
182 match cx.ds["connector"].as_str() {
183 Some(name) => match all.iter().find(|c| c.sources().contains(&name)) {
184 Some(c) => c.source(cx),
185 None => Err(format!(
186 "pipeline.toml: [datasets.{}] connector = \"{name}\" is not known ({})",
187 cx.dataset,
188 all.iter().flat_map(|c| c.sources().iter().copied()).collect::<Vec<_>>().join(", ")
189 )),
190 },
191 None => match all.iter().find(|c| c.name() == DEFAULT) {
192 Some(c) => c.source(cx),
193 None => Err(format!(
194 "pipeline.toml: [datasets.{}] names no `connector`, and this build has no default ({DEFAULT}) connector",
195 cx.dataset
196 )),
197 },
198 }
199}
200
201pub const RENAMED: &[(&str, &str)] = &[("infer_sample", "inferSample"), ("on_bad_data", "onBadData")];
204
205pub fn check_dataset(cx: &Ctx<'_>) -> Result<Vec<String>, String> {
211 let all = connectors();
212 let named = |key: &str, of: fn(&dyn Connector) -> &'static [&'static str]| match cx.ds[key].as_str() {
213 Some(name) => all.iter().find(|c| of(c.as_ref()).contains(&name)),
214 None => all.iter().find(|c| c.name() == DEFAULT),
215 };
216 let (Some(source), Some(sink)) = (named("connector", |c| c.sources()), named("sink", |c| c.sinks())) else {
217 return Ok(Vec::new()); };
219 let (Some(source), Some(sink)) = (source.about(), sink.about()) else {
220 return Ok(Vec::new());
221 };
222 let mut roles = [
223 (cx.ds["connector"].is_null(), source.source.as_ref()),
224 (cx.ds["sink"].is_null(), sink.sink.as_ref()),
225 ];
226 roles.sort_by_key(|(defaulted, _)| *defaulted);
227 let mut accepted: Vec<&str> = vec!["connector", "sink"];
228 for o in roles.iter().filter_map(|(_, r)| *r).flat_map(|r| r.options) {
229 if !accepted.contains(&o.key) {
230 accepted.push(o.key);
231 }
232 }
233 let at = format!("pipeline.toml: [datasets.{}]", cx.dataset);
234 let renamed: Vec<(&str, &str)> = RENAMED
235 .iter()
236 .copied()
237 .filter(|(_, new)| accepted.contains(new))
238 .collect();
239 fv_streams_types::options::check_keys(
240 cx.ds,
241 &at,
242 &renamed.iter().map(|(old, _)| *old).collect::<Vec<_>>(),
243 accepted.iter().copied(),
244 )?;
245 let mut warnings = Vec::new();
246 for (old, new) in renamed.into_iter().filter(|(old, _)| !cx.ds[*old].is_null()) {
247 if !cx.ds[new].is_null() {
248 return Err(format!(
249 "{at}: sets both `{new}` and its old name `{old}`; keep `{new}`"
250 ));
251 }
252 warnings.push(format!(
253 "{at}: `{old}` is the old name of `{new}`; it still works — rename it"
254 ));
255 }
256 Ok(warnings)
257}
258
259pub fn resolve_sink(cx: &Ctx<'_>) -> Result<Arc<dyn OutputSink>, String> {
261 let all = connectors();
262 match cx.ds["sink"].as_str() {
263 Some(name) => match all.iter().find(|c| c.sinks().contains(&name)) {
264 Some(c) => c.sink(cx),
265 None => Err(format!(
266 "pipeline.toml: [datasets.{}] sink = \"{name}\" is not known ({})",
267 cx.dataset,
268 all.iter()
269 .flat_map(|c| c.sinks().iter().copied())
270 .collect::<Vec<_>>()
271 .join(", ")
272 )),
273 },
274 None => match all.iter().find(|c| c.name() == DEFAULT) {
275 Some(c) => c.sink(cx),
276 None => Err(format!(
277 "pipeline.toml: [datasets.{}] names no `sink`, and this build has no default ({DEFAULT}) connector",
278 cx.dataset
279 )),
280 },
281 }
282}
283
284#[cfg(test)]
285mod tests;