Skip to main content

fv_connect/
lib.rs

1//! Source connectivity — read a relational source into rows with the
2//! MAC filter pushed DOWN into the source query, so hidden rows never leave the database.
3//!
4//! Local dev connects plaintext (`sslmode=disable`) via `tokio-postgres` (NoTls) — no native TLS /
5//! openssl. The MAC WHERE comes from `fv_mac::where_sql`; for a DOMINANCE axis the fragment
6//! (`col <= n`) is valid in Postgres as-is. (Array axes ALL_OF/ANY_OF need the Postgres operator
7//! dialect `<@` / `&&` — a Postgres-flavoured builder, or a DataFusion custom TableProvider that lets
8//! datafusion-federation translate — is the follow-up; DOMINANCE covers the common curated-table case.)
9
10/// Every Rust code block in the README is compiled as a doctest.
11#[cfg(doctest)]
12#[doc = include_str!("../README.md")]
13pub struct ReadmeDoctests;
14
15use fv_mac::{where_sql, Bindings, Taxonomy};
16use fv_value::Value;
17use tokio_postgres::types::Type;
18use tokio_postgres::NoTls;
19
20/// Tests that mutate the PROCESS environment (`std::env::set_var`) share this lock: cargo runs a crate's
21/// tests on parallel threads inside ONE process, so two tests writing the same variable race unless
22/// they serialise. Hold the guard for the whole test; restore the variable before releasing it.
23#[cfg(test)]
24pub(crate) static ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
25
26pub mod binding;
27pub mod changelog;
28mod config;
29mod ident;
30pub mod labels;
31pub mod lake;
32pub mod log;
33mod mysql;
34pub mod rest;
35pub mod scan;
36pub mod source;
37pub mod sqlite;
38mod write;
39
40pub use binding::{bindings_from_definitions, primary_binding, Binding};
41pub use changelog::{diff_snapshots, Change, Snapshot};
42pub use config::{secret_env_var, ConnectorConfig};
43pub use labels::{normalize_row, LabelBinding};
44pub use lake::{Checkpoint, SnapshotStore};
45pub use log::{fold_compacted_log, LogMessage};
46pub use mysql::read_mysql_with_mac;
47pub use scan::{scan_source, CollectSink, LandingSource, MemorySource, Sink, Source};
48pub use source::{register_source, registered_kinds, source_for, Caps, Source as ConnectorSource};
49pub use write::{write_and_promote_mysql, write_and_promote_postgres};
50
51/// One source row as ordered (column, value) pairs (matches the engine's Row shape).
52pub type Row = Vec<(String, Value)>;
53
54/// Read a registered source with MAC pushed into the query — a registry lookup on the connector
55/// type (Source SPI); a new driver is one `register_source` call away, no new match arms.
56pub async fn read_source(
57    config: &ConnectorConfig,
58    password: &str,
59    table: &str,
60    taxonomy: &Taxonomy,
61    grants: &Bindings,
62    label_column: impl Fn(&str) -> String + Copy + Sync,
63    limit: Option<usize>,
64) -> Result<Vec<Row>, ConnectError> {
65    match source_for(&config.kind) {
66        Some(s) if s.capabilities().read => {
67            s.read(config, password, table, taxonomy, grants, &label_column, limit)
68                .await
69        }
70        Some(_) => err(format!("connector type '{}' has no read capability", config.kind)),
71        None => err(format!(
72            "connector type '{}' has no registered source (have {})",
73            config.kind,
74            registered_kinds().join(", ")
75        )),
76    }
77}
78
79/// The S12 landing tee — the PIPELINE RUNNER'S dataset read: prefer the funnel's LANDED snapshot of
80/// `table` (the exact curated rows the changelog diffed → deterministic inputs, and a rate-limited
81/// public API is never double-fetched), falling back to the live `read_source` when the funnel has
82/// never landed this table (e.g. a raw relational table outside any object-type binding).
83/// NEVER use this from the funnel's own scan legs — they fetch-then-land, and a landing-preferring
84/// read would freeze ingestion on its own previous state (pinned by rest.rs's regression guard).
85pub async fn read_source_preferring_landing(
86    config: &ConnectorConfig,
87    password: &str,
88    table: &str,
89    taxonomy: &Taxonomy,
90    grants: &Bindings,
91    label_column: impl Fn(&str) -> String + Copy + Sync,
92    limit: Option<usize>,
93) -> Result<Vec<Row>, ConnectError> {
94    if !table.is_empty() {
95        let root = std::env::var("PG_LANDING_DIR")
96            .or_else(|_| std::env::var("LAKE_DIR"))
97            .unwrap_or_else(|_| "/tmp/fv-funnel/landing".to_string());
98        let store = lake::SnapshotStore::new(&root, table);
99        if !store.list().is_empty() {
100            let mut rows = store.last_raw().map_err(ConnectError)?;
101            if let Some(n) = limit {
102                rows.truncate(n);
103            }
104            return Ok(rows);
105        }
106    }
107    read_source(config, password, table, taxonomy, grants, label_column, limit).await
108}
109
110/// Stage + atomically promote `rows` as `table` on the connector — the write half of the SPI.
111/// Replaces the per-capability `match config.kind` the worker carried.
112pub async fn write_and_promote(
113    config: &ConnectorConfig,
114    password: &str,
115    table: &str,
116    columns: &[String],
117    rows: &[Row],
118) -> Result<(), ConnectError> {
119    match source_for(&config.kind) {
120        Some(s) if s.capabilities().write => s.write_and_promote(config, password, table, columns, rows).await,
121        Some(_) => err(format!("connector type '{}' has no write capability", config.kind)),
122        None => err(format!(
123            "connector type '{}' has no registered source (have {})",
124            config.kind,
125            registered_kinds().join(", ")
126        )),
127    }
128}
129
130#[derive(Debug)]
131pub struct ConnectError(pub String);
132
133impl std::fmt::Display for ConnectError {
134    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
135        write!(f, "{}", self.0)
136    }
137}
138
139impl std::error::Error for ConnectError {}
140
141fn err<T>(e: impl std::fmt::Display) -> Result<T, ConnectError> {
142    Err(ConnectError(e.to_string()))
143}
144
145/// Read `table` (schema-qualified, e.g. `curated.customer`) with MAC pushed into the query.
146///
147/// `label_column(axis)` maps a taxonomy axis to the source's label column (e.g. classification →
148/// `classification_level`). The predicate is enforced by the database — it only ever returns rows the
149/// subject may see.
150pub async fn read_with_mac(
151    dsn: &str,
152    table: &str,
153    taxonomy: &Taxonomy,
154    grants: &Bindings,
155    label_column: impl Fn(&str) -> String,
156    limit: Option<usize>,
157) -> Result<Vec<Row>, ConnectError> {
158    let (client, connection) = match tokio_postgres::connect(dsn, NoTls).await {
159        Ok(pair) => pair,
160        Err(e) => return err(format!("connect: {e}")),
161    };
162    // Drive the connection in the background; it ends when the client drops.
163    let handle = tokio::spawn(async move {
164        let _ = connection.await;
165    });
166
167    let predicate = where_sql(taxonomy, grants, &label_column);
168    // Quote the (possibly schema-qualified) table so hyphenated/kebab-case identifiers read correctly
169    // — `curated.delivery-site` must be `"curated"."delivery-site"`, else `-` parses as subtraction.
170    let mut sql = format!("SELECT * FROM {} WHERE {predicate}", ident::pg_ident(table));
171    if let Some(n) = limit {
172        sql.push_str(&format!(" LIMIT {n}"));
173    }
174
175    let result = client.query(&sql, &[]).await;
176    drop(client);
177    let _ = handle.await;
178
179    let pg_rows = match result {
180        Ok(r) => r,
181        Err(e) => {
182            // tokio-postgres Display is just "db error" — the actual server message rides source().
183            let cause = e
184                .as_db_error()
185                .map(|d| d.message().to_string())
186                .or_else(|| std::error::Error::source(&e).map(|s| s.to_string()))
187                .unwrap_or_default();
188            return err(format!("query `{sql}`: {e} {cause}"));
189        }
190    };
191
192    let mut out = Vec::with_capacity(pg_rows.len());
193    for r in &pg_rows {
194        let mut row: Row = Vec::with_capacity(r.columns().len());
195        for (i, col) in r.columns().iter().enumerate() {
196            row.push((col.name().to_string(), cell(r, i, col.type_())));
197        }
198        out.push(row);
199    }
200    Ok(out)
201}
202
203/// Read a time-series from a PG-wire sidecar plane — `series.observations` (equally a
204/// Timescale hypertable / PostGIS store; all PG-wire). MAC is pushed into the WHERE via the
205/// Postgres-native [`fv_mac::pg_where`] (enforcement point): a plane read can never return an
206/// observation the caller couldn't see. Returns time-ascending rows (ts + value_num/str/geo columns);
207/// the caller resolves the primary value. Reuses the same connection plumbing as [`read_with_mac`].
208#[allow(clippy::too_many_arguments)] // the series-read contract: dsn/series/window/paging in one call
209pub async fn read_series(
210    dsn: &str,
211    series_id: &str,
212    from: Option<i64>,
213    to: Option<i64>,
214    size: usize,
215    taxonomy: &Taxonomy,
216    grants: &Bindings,
217    column_for: impl Fn(&str) -> String,
218) -> Result<Vec<Row>, ConnectError> {
219    let (client, connection) = match tokio_postgres::connect(dsn, NoTls).await {
220        Ok(pair) => pair,
221        Err(e) => return err(format!("connect: {e}")),
222    };
223    let handle = tokio::spawn(async move {
224        let _ = connection.await;
225    });
226
227    let mut clauses = vec![format!("series_id = {}", sql_str(series_id))];
228    if let Some(f) = from {
229        clauses.push(format!("ts >= {f}"));
230    }
231    if let Some(t) = to {
232        clauses.push(format!("ts <= {t}"));
233    }
234    clauses.push(fv_mac::pg_where(taxonomy, grants, column_for)); // PG-native MAC pushdown
235    let sql = format!(
236        "SELECT ts, value_num, value_str, value_geo FROM series.observations WHERE {} ORDER BY ts ASC LIMIT {size}",
237        clauses.join(" AND ")
238    );
239
240    let result = client.query(&sql, &[]).await;
241    drop(client);
242    let _ = handle.await;
243
244    let pg_rows = match result {
245        Ok(r) => r,
246        Err(e) => return err(format!("series query `{sql}`: {e}")),
247    };
248    let mut out = Vec::with_capacity(pg_rows.len());
249    for r in &pg_rows {
250        let mut row: Row = Vec::with_capacity(r.columns().len());
251        for (i, col) in r.columns().iter().enumerate() {
252            row.push((col.name().to_string(), cell(r, i, col.type_())));
253        }
254        out.push(row);
255    }
256    Ok(out)
257}
258
259/// Escape a Postgres string literal (single quotes doubled). Shared by the series read.
260fn sql_str(s: &str) -> String {
261    format!("'{}'", s.replace('\'', "''"))
262}
263
264/// Map a Postgres cell to a dialect Value (the types curated tables commonly use). Unknown types
265/// fall back to their text form, else Null.
266fn cell(row: &tokio_postgres::Row, i: usize, ty: &Type) -> Value {
267    match *ty {
268        Type::TEXT | Type::VARCHAR | Type::BPCHAR | Type::NAME => row
269            .try_get::<_, Option<String>>(i)
270            .ok()
271            .flatten()
272            .map(Value::Str)
273            .unwrap_or(Value::Null),
274        Type::INT2 => row
275            .try_get::<_, Option<i16>>(i)
276            .ok()
277            .flatten()
278            .map(|v| Value::Num(v as f64))
279            .unwrap_or(Value::Null),
280        Type::INT4 => row
281            .try_get::<_, Option<i32>>(i)
282            .ok()
283            .flatten()
284            .map(|v| Value::Num(v as f64))
285            .unwrap_or(Value::Null),
286        Type::INT8 => row
287            .try_get::<_, Option<i64>>(i)
288            .ok()
289            .flatten()
290            .map(|v| Value::Num(v as f64))
291            .unwrap_or(Value::Null),
292        Type::FLOAT4 => row
293            .try_get::<_, Option<f32>>(i)
294            .ok()
295            .flatten()
296            .map(|v| Value::Num(v as f64))
297            .unwrap_or(Value::Null),
298        Type::FLOAT8 => row
299            .try_get::<_, Option<f64>>(i)
300            .ok()
301            .flatten()
302            .map(Value::Num)
303            .unwrap_or(Value::Null),
304        Type::BOOL => row
305            .try_get::<_, Option<bool>>(i)
306            .ok()
307            .flatten()
308            .map(Value::Bool)
309            .unwrap_or(Value::Null),
310        _ => row
311            .try_get::<_, Option<String>>(i)
312            .ok()
313            .flatten()
314            .map(Value::Str)
315            .unwrap_or(Value::Null),
316    }
317}
318
319/// Convenience: pull the value of one column from a row (Null if absent).
320pub fn get<'a>(row: &'a Row, key: &str) -> &'a Value {
321    row.iter()
322        .find(|(k, _)| k == key)
323        .map(|(_, v)| v)
324        .unwrap_or(&Value::Null)
325}
326
327#[cfg(test)]
328mod tee_tests {
329    use super::*;
330    use fv_value::Value;
331
332    #[test]
333    fn runner_read_prefers_the_latest_landing_and_falls_back_without_one() {
334        let dir = std::env::temp_dir().join(format!("fv-landing-tests-{}", std::process::id()));
335        let store = lake::SnapshotStore::at(dir.join("curated.tee-vessel"));
336        let row = |rid: &str, speed: f64| -> Row {
337            vec![
338                ("rid".into(), Value::Str(rid.into())),
339                ("speedMps".into(), Value::Num(speed)),
340            ]
341        };
342        store.land(&[row("v1", 1.0)]).unwrap();
343        store.land(&[row("v1", 2.0), row("v2", 3.0)]).unwrap();
344        let _env = crate::ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); // serialise env-mutating tests
345        std::env::set_var("PG_LANDING_DIR", dir.to_str().unwrap());
346        let config = ConnectorConfig {
347            kind: "REST_POLL".into(),
348            host: String::new(),
349            port: 0,
350            database: String::new(),
351            user: String::new(),
352            options: serde_json::json!({}),
353        };
354        let rt = tokio::runtime::Builder::new_current_thread()
355            .enable_all()
356            .build()
357            .unwrap();
358        let read = |table: &'static str, limit| {
359            rt.block_on(read_source_preferring_landing(
360                &config,
361                "",
362                table,
363                &Taxonomy::default(),
364                &Bindings::new(),
365                |c| c.to_string(),
366                limit,
367            ))
368        };
369        // Landed → the LATEST snapshot, no network; limit honoured.
370        let rows = read("curated.tee-vessel", None).unwrap();
371        assert_eq!(rows.len(), 2);
372        assert!(rows.iter().any(|r| r
373            .iter()
374            .any(|(k, v)| k == "speedMps" && matches!(v, Value::Num(n) if *n == 3.0))));
375        assert_eq!(read("curated.tee-vessel", Some(1)).unwrap().len(), 1);
376        // Never landed → live fallback, which demands a URL (the path is exercised, no network).
377        assert!(read("curated.never-landed", None)
378            .unwrap_err()
379            .0
380            .contains("config.options.url is required"));
381        std::env::remove_var("PG_LANDING_DIR"); // leave the process env as we found it
382    }
383}