1#[cfg(doctest)]
12#[doc = include_str!("../README.md")]
13pub struct ReadmeDoctests;
14
15use fv_mac::{where_sql, Bindings, Taxonomy};
16use fv_value::Value;
17use tokio_postgres::types::Type;
18use tokio_postgres::NoTls;
19
20#[cfg(test)]
24pub(crate) static ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
25
26pub mod binding;
27pub mod changelog;
28mod config;
29mod ident;
30pub mod labels;
31pub mod lake;
32pub mod log;
33mod mysql;
34pub mod rest;
35pub mod scan;
36pub mod source;
37pub mod sqlite;
38mod write;
39
40pub use binding::{bindings_from_definitions, primary_binding, Binding};
41pub use changelog::{diff_snapshots, Change, Snapshot};
42pub use config::{secret_env_var, ConnectorConfig};
43pub use labels::{normalize_row, LabelBinding};
44pub use lake::{Checkpoint, SnapshotStore};
45pub use log::{fold_compacted_log, LogMessage};
46pub use mysql::read_mysql_with_mac;
47pub use scan::{scan_source, CollectSink, LandingSource, MemorySource, Sink, Source};
48pub use source::{register_source, registered_kinds, source_for, Caps, Source as ConnectorSource};
49pub use write::{write_and_promote_mysql, write_and_promote_postgres};
50
51pub type Row = Vec<(String, Value)>;
53
54pub async fn read_source(
57 config: &ConnectorConfig,
58 password: &str,
59 table: &str,
60 taxonomy: &Taxonomy,
61 grants: &Bindings,
62 label_column: impl Fn(&str) -> String + Copy + Sync,
63 limit: Option<usize>,
64) -> Result<Vec<Row>, ConnectError> {
65 match source_for(&config.kind) {
66 Some(s) if s.capabilities().read => {
67 s.read(config, password, table, taxonomy, grants, &label_column, limit)
68 .await
69 }
70 Some(_) => err(format!("connector type '{}' has no read capability", config.kind)),
71 None => err(format!(
72 "connector type '{}' has no registered source (have {})",
73 config.kind,
74 registered_kinds().join(", ")
75 )),
76 }
77}
78
79pub async fn read_source_preferring_landing(
86 config: &ConnectorConfig,
87 password: &str,
88 table: &str,
89 taxonomy: &Taxonomy,
90 grants: &Bindings,
91 label_column: impl Fn(&str) -> String + Copy + Sync,
92 limit: Option<usize>,
93) -> Result<Vec<Row>, ConnectError> {
94 if !table.is_empty() {
95 let root = std::env::var("PG_LANDING_DIR")
96 .or_else(|_| std::env::var("LAKE_DIR"))
97 .unwrap_or_else(|_| "/tmp/fv-funnel/landing".to_string());
98 let store = lake::SnapshotStore::new(&root, table);
99 if !store.list().is_empty() {
100 let mut rows = store.last_raw().map_err(ConnectError)?;
101 if let Some(n) = limit {
102 rows.truncate(n);
103 }
104 return Ok(rows);
105 }
106 }
107 read_source(config, password, table, taxonomy, grants, label_column, limit).await
108}
109
110pub async fn write_and_promote(
113 config: &ConnectorConfig,
114 password: &str,
115 table: &str,
116 columns: &[String],
117 rows: &[Row],
118) -> Result<(), ConnectError> {
119 match source_for(&config.kind) {
120 Some(s) if s.capabilities().write => s.write_and_promote(config, password, table, columns, rows).await,
121 Some(_) => err(format!("connector type '{}' has no write capability", config.kind)),
122 None => err(format!(
123 "connector type '{}' has no registered source (have {})",
124 config.kind,
125 registered_kinds().join(", ")
126 )),
127 }
128}
129
130#[derive(Debug)]
131pub struct ConnectError(pub String);
132
133impl std::fmt::Display for ConnectError {
134 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
135 write!(f, "{}", self.0)
136 }
137}
138
139impl std::error::Error for ConnectError {}
140
141fn err<T>(e: impl std::fmt::Display) -> Result<T, ConnectError> {
142 Err(ConnectError(e.to_string()))
143}
144
145pub async fn read_with_mac(
151 dsn: &str,
152 table: &str,
153 taxonomy: &Taxonomy,
154 grants: &Bindings,
155 label_column: impl Fn(&str) -> String,
156 limit: Option<usize>,
157) -> Result<Vec<Row>, ConnectError> {
158 let (client, connection) = match tokio_postgres::connect(dsn, NoTls).await {
159 Ok(pair) => pair,
160 Err(e) => return err(format!("connect: {e}")),
161 };
162 let handle = tokio::spawn(async move {
164 let _ = connection.await;
165 });
166
167 let predicate = where_sql(taxonomy, grants, &label_column);
168 let mut sql = format!("SELECT * FROM {} WHERE {predicate}", ident::pg_ident(table));
171 if let Some(n) = limit {
172 sql.push_str(&format!(" LIMIT {n}"));
173 }
174
175 let result = client.query(&sql, &[]).await;
176 drop(client);
177 let _ = handle.await;
178
179 let pg_rows = match result {
180 Ok(r) => r,
181 Err(e) => {
182 let cause = e
184 .as_db_error()
185 .map(|d| d.message().to_string())
186 .or_else(|| std::error::Error::source(&e).map(|s| s.to_string()))
187 .unwrap_or_default();
188 return err(format!("query `{sql}`: {e} {cause}"));
189 }
190 };
191
192 let mut out = Vec::with_capacity(pg_rows.len());
193 for r in &pg_rows {
194 let mut row: Row = Vec::with_capacity(r.columns().len());
195 for (i, col) in r.columns().iter().enumerate() {
196 row.push((col.name().to_string(), cell(r, i, col.type_())));
197 }
198 out.push(row);
199 }
200 Ok(out)
201}
202
203#[allow(clippy::too_many_arguments)] pub async fn read_series(
210 dsn: &str,
211 series_id: &str,
212 from: Option<i64>,
213 to: Option<i64>,
214 size: usize,
215 taxonomy: &Taxonomy,
216 grants: &Bindings,
217 column_for: impl Fn(&str) -> String,
218) -> Result<Vec<Row>, ConnectError> {
219 let (client, connection) = match tokio_postgres::connect(dsn, NoTls).await {
220 Ok(pair) => pair,
221 Err(e) => return err(format!("connect: {e}")),
222 };
223 let handle = tokio::spawn(async move {
224 let _ = connection.await;
225 });
226
227 let mut clauses = vec![format!("series_id = {}", sql_str(series_id))];
228 if let Some(f) = from {
229 clauses.push(format!("ts >= {f}"));
230 }
231 if let Some(t) = to {
232 clauses.push(format!("ts <= {t}"));
233 }
234 clauses.push(fv_mac::pg_where(taxonomy, grants, column_for)); let sql = format!(
236 "SELECT ts, value_num, value_str, value_geo FROM series.observations WHERE {} ORDER BY ts ASC LIMIT {size}",
237 clauses.join(" AND ")
238 );
239
240 let result = client.query(&sql, &[]).await;
241 drop(client);
242 let _ = handle.await;
243
244 let pg_rows = match result {
245 Ok(r) => r,
246 Err(e) => return err(format!("series query `{sql}`: {e}")),
247 };
248 let mut out = Vec::with_capacity(pg_rows.len());
249 for r in &pg_rows {
250 let mut row: Row = Vec::with_capacity(r.columns().len());
251 for (i, col) in r.columns().iter().enumerate() {
252 row.push((col.name().to_string(), cell(r, i, col.type_())));
253 }
254 out.push(row);
255 }
256 Ok(out)
257}
258
259fn sql_str(s: &str) -> String {
261 format!("'{}'", s.replace('\'', "''"))
262}
263
264fn cell(row: &tokio_postgres::Row, i: usize, ty: &Type) -> Value {
267 match *ty {
268 Type::TEXT | Type::VARCHAR | Type::BPCHAR | Type::NAME => row
269 .try_get::<_, Option<String>>(i)
270 .ok()
271 .flatten()
272 .map(Value::Str)
273 .unwrap_or(Value::Null),
274 Type::INT2 => row
275 .try_get::<_, Option<i16>>(i)
276 .ok()
277 .flatten()
278 .map(|v| Value::Num(v as f64))
279 .unwrap_or(Value::Null),
280 Type::INT4 => row
281 .try_get::<_, Option<i32>>(i)
282 .ok()
283 .flatten()
284 .map(|v| Value::Num(v as f64))
285 .unwrap_or(Value::Null),
286 Type::INT8 => row
287 .try_get::<_, Option<i64>>(i)
288 .ok()
289 .flatten()
290 .map(|v| Value::Num(v as f64))
291 .unwrap_or(Value::Null),
292 Type::FLOAT4 => row
293 .try_get::<_, Option<f32>>(i)
294 .ok()
295 .flatten()
296 .map(|v| Value::Num(v as f64))
297 .unwrap_or(Value::Null),
298 Type::FLOAT8 => row
299 .try_get::<_, Option<f64>>(i)
300 .ok()
301 .flatten()
302 .map(Value::Num)
303 .unwrap_or(Value::Null),
304 Type::BOOL => row
305 .try_get::<_, Option<bool>>(i)
306 .ok()
307 .flatten()
308 .map(Value::Bool)
309 .unwrap_or(Value::Null),
310 _ => row
311 .try_get::<_, Option<String>>(i)
312 .ok()
313 .flatten()
314 .map(Value::Str)
315 .unwrap_or(Value::Null),
316 }
317}
318
319pub fn get<'a>(row: &'a Row, key: &str) -> &'a Value {
321 row.iter()
322 .find(|(k, _)| k == key)
323 .map(|(_, v)| v)
324 .unwrap_or(&Value::Null)
325}
326
327#[cfg(test)]
328mod tee_tests {
329 use super::*;
330 use fv_value::Value;
331
332 #[test]
333 fn runner_read_prefers_the_latest_landing_and_falls_back_without_one() {
334 let dir = std::env::temp_dir().join(format!("fv-landing-tests-{}", std::process::id()));
335 let store = lake::SnapshotStore::at(dir.join("curated.tee-vessel"));
336 let row = |rid: &str, speed: f64| -> Row {
337 vec![
338 ("rid".into(), Value::Str(rid.into())),
339 ("speedMps".into(), Value::Num(speed)),
340 ]
341 };
342 store.land(&[row("v1", 1.0)]).unwrap();
343 store.land(&[row("v1", 2.0), row("v2", 3.0)]).unwrap();
344 let _env = crate::ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); std::env::set_var("PG_LANDING_DIR", dir.to_str().unwrap());
346 let config = ConnectorConfig {
347 kind: "REST_POLL".into(),
348 host: String::new(),
349 port: 0,
350 database: String::new(),
351 user: String::new(),
352 options: serde_json::json!({}),
353 };
354 let rt = tokio::runtime::Builder::new_current_thread()
355 .enable_all()
356 .build()
357 .unwrap();
358 let read = |table: &'static str, limit| {
359 rt.block_on(read_source_preferring_landing(
360 &config,
361 "",
362 table,
363 &Taxonomy::default(),
364 &Bindings::new(),
365 |c| c.to_string(),
366 limit,
367 ))
368 };
369 let rows = read("curated.tee-vessel", None).unwrap();
371 assert_eq!(rows.len(), 2);
372 assert!(rows.iter().any(|r| r
373 .iter()
374 .any(|(k, v)| k == "speedMps" && matches!(v, Value::Num(n) if *n == 3.0))));
375 assert_eq!(read("curated.tee-vessel", Some(1)).unwrap().len(), 1);
376 assert!(read("curated.never-landed", None)
378 .unwrap_err()
379 .0
380 .contains("config.options.url is required"));
381 std::env::remove_var("PG_LANDING_DIR"); }
383}