fv-connect 0.1.0

FusionVault Conduit: read relational, file and REST sources into rows with mandatory access control pushed into the read, write results back with atomic promotion, and drive incremental base-ingest scans, changelogs and label normalisation checked against the published conformance vectors.
Documentation
//! Source connectivity — read a relational source into rows with the
//! MAC filter pushed DOWN into the source query, so hidden rows never leave the database.
//!
//! Local dev connects plaintext (`sslmode=disable`) via `tokio-postgres` (NoTls) — no native TLS /
//! openssl. The MAC WHERE comes from `fv_mac::where_sql`; for a DOMINANCE axis the fragment
//! (`col <= n`) is valid in Postgres as-is. (Array axes ALL_OF/ANY_OF need the Postgres operator
//! dialect `<@` / `&&` — a Postgres-flavoured builder, or a DataFusion custom TableProvider that lets
//! datafusion-federation translate — is the follow-up; DOMINANCE covers the common curated-table case.)

/// Every Rust code block in the README is compiled as a doctest.
#[cfg(doctest)]
#[doc = include_str!("../README.md")]
pub struct ReadmeDoctests;

use fv_mac::{where_sql, Bindings, Taxonomy};
use fv_value::Value;
use tokio_postgres::types::Type;
use tokio_postgres::NoTls;

/// Tests that mutate the PROCESS environment (`std::env::set_var`) share this lock: cargo runs a crate's
/// tests on parallel threads inside ONE process, so two tests writing the same variable race unless
/// they serialise. Hold the guard for the whole test; restore the variable before releasing it.
#[cfg(test)]
pub(crate) static ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());

pub mod binding;
pub mod changelog;
mod config;
mod ident;
pub mod labels;
pub mod lake;
pub mod log;
mod mysql;
pub mod rest;
pub mod scan;
pub mod source;
pub mod sqlite;
mod write;

pub use binding::{bindings_from_definitions, primary_binding, Binding};
pub use changelog::{diff_snapshots, Change, Snapshot};
pub use config::{secret_env_var, ConnectorConfig};
pub use labels::{normalize_row, LabelBinding};
pub use lake::{Checkpoint, SnapshotStore};
pub use log::{fold_compacted_log, LogMessage};
pub use mysql::read_mysql_with_mac;
pub use scan::{scan_source, CollectSink, LandingSource, MemorySource, Sink, Source};
pub use source::{register_source, registered_kinds, source_for, Caps, Source as ConnectorSource};
pub use write::{write_and_promote_mysql, write_and_promote_postgres};

/// One source row as ordered (column, value) pairs (matches the engine's Row shape).
pub type Row = Vec<(String, Value)>;

/// Read a registered source with MAC pushed into the query — a registry lookup on the connector
/// type (Source SPI); a new driver is one `register_source` call away, no new match arms.
pub async fn read_source(
    config: &ConnectorConfig,
    password: &str,
    table: &str,
    taxonomy: &Taxonomy,
    grants: &Bindings,
    label_column: impl Fn(&str) -> String + Copy + Sync,
    limit: Option<usize>,
) -> Result<Vec<Row>, ConnectError> {
    match source_for(&config.kind) {
        Some(s) if s.capabilities().read => {
            s.read(config, password, table, taxonomy, grants, &label_column, limit)
                .await
        }
        Some(_) => err(format!("connector type '{}' has no read capability", config.kind)),
        None => err(format!(
            "connector type '{}' has no registered source (have {})",
            config.kind,
            registered_kinds().join(", ")
        )),
    }
}

/// The S12 landing tee — the PIPELINE RUNNER'S dataset read: prefer the funnel's LANDED snapshot of
/// `table` (the exact curated rows the changelog diffed → deterministic inputs, and a rate-limited
/// public API is never double-fetched), falling back to the live `read_source` when the funnel has
/// never landed this table (e.g. a raw relational table outside any object-type binding).
/// NEVER use this from the funnel's own scan legs — they fetch-then-land, and a landing-preferring
/// read would freeze ingestion on its own previous state (pinned by rest.rs's regression guard).
pub async fn read_source_preferring_landing(
    config: &ConnectorConfig,
    password: &str,
    table: &str,
    taxonomy: &Taxonomy,
    grants: &Bindings,
    label_column: impl Fn(&str) -> String + Copy + Sync,
    limit: Option<usize>,
) -> Result<Vec<Row>, ConnectError> {
    if !table.is_empty() {
        let root = std::env::var("PG_LANDING_DIR")
            .or_else(|_| std::env::var("LAKE_DIR"))
            .unwrap_or_else(|_| "/tmp/fv-funnel/landing".to_string());
        let store = lake::SnapshotStore::new(&root, table);
        if !store.list().is_empty() {
            let mut rows = store.last_raw().map_err(ConnectError)?;
            if let Some(n) = limit {
                rows.truncate(n);
            }
            return Ok(rows);
        }
    }
    read_source(config, password, table, taxonomy, grants, label_column, limit).await
}

/// Stage + atomically promote `rows` as `table` on the connector — the write half of the SPI.
/// Replaces the per-capability `match config.kind` the worker carried.
pub async fn write_and_promote(
    config: &ConnectorConfig,
    password: &str,
    table: &str,
    columns: &[String],
    rows: &[Row],
) -> Result<(), ConnectError> {
    match source_for(&config.kind) {
        Some(s) if s.capabilities().write => s.write_and_promote(config, password, table, columns, rows).await,
        Some(_) => err(format!("connector type '{}' has no write capability", config.kind)),
        None => err(format!(
            "connector type '{}' has no registered source (have {})",
            config.kind,
            registered_kinds().join(", ")
        )),
    }
}

#[derive(Debug)]
pub struct ConnectError(pub String);

impl std::fmt::Display for ConnectError {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        write!(f, "{}", self.0)
    }
}

impl std::error::Error for ConnectError {}

fn err<T>(e: impl std::fmt::Display) -> Result<T, ConnectError> {
    Err(ConnectError(e.to_string()))
}

/// Read `table` (schema-qualified, e.g. `curated.customer`) with MAC pushed into the query.
///
/// `label_column(axis)` maps a taxonomy axis to the source's label column (e.g. classification →
/// `classification_level`). The predicate is enforced by the database — it only ever returns rows the
/// subject may see.
pub async fn read_with_mac(
    dsn: &str,
    table: &str,
    taxonomy: &Taxonomy,
    grants: &Bindings,
    label_column: impl Fn(&str) -> String,
    limit: Option<usize>,
) -> Result<Vec<Row>, ConnectError> {
    let (client, connection) = match tokio_postgres::connect(dsn, NoTls).await {
        Ok(pair) => pair,
        Err(e) => return err(format!("connect: {e}")),
    };
    // Drive the connection in the background; it ends when the client drops.
    let handle = tokio::spawn(async move {
        let _ = connection.await;
    });

    let predicate = where_sql(taxonomy, grants, &label_column);
    // Quote the (possibly schema-qualified) table so hyphenated/kebab-case identifiers read correctly
    // — `curated.delivery-site` must be `"curated"."delivery-site"`, else `-` parses as subtraction.
    let mut sql = format!("SELECT * FROM {} WHERE {predicate}", ident::pg_ident(table));
    if let Some(n) = limit {
        sql.push_str(&format!(" LIMIT {n}"));
    }

    let result = client.query(&sql, &[]).await;
    drop(client);
    let _ = handle.await;

    let pg_rows = match result {
        Ok(r) => r,
        Err(e) => {
            // tokio-postgres Display is just "db error" — the actual server message rides source().
            let cause = e
                .as_db_error()
                .map(|d| d.message().to_string())
                .or_else(|| std::error::Error::source(&e).map(|s| s.to_string()))
                .unwrap_or_default();
            return err(format!("query `{sql}`: {e} {cause}"));
        }
    };

    let mut out = Vec::with_capacity(pg_rows.len());
    for r in &pg_rows {
        let mut row: Row = Vec::with_capacity(r.columns().len());
        for (i, col) in r.columns().iter().enumerate() {
            row.push((col.name().to_string(), cell(r, i, col.type_())));
        }
        out.push(row);
    }
    Ok(out)
}

/// Read a time-series from a PG-wire sidecar plane — `series.observations` (equally a
/// Timescale hypertable / PostGIS store; all PG-wire). MAC is pushed into the WHERE via the
/// Postgres-native [`fv_mac::pg_where`] (enforcement point): a plane read can never return an
/// observation the caller couldn't see. Returns time-ascending rows (ts + value_num/str/geo columns);
/// the caller resolves the primary value. Reuses the same connection plumbing as [`read_with_mac`].
#[allow(clippy::too_many_arguments)] // the series-read contract: dsn/series/window/paging in one call
pub async fn read_series(
    dsn: &str,
    series_id: &str,
    from: Option<i64>,
    to: Option<i64>,
    size: usize,
    taxonomy: &Taxonomy,
    grants: &Bindings,
    column_for: impl Fn(&str) -> String,
) -> Result<Vec<Row>, ConnectError> {
    let (client, connection) = match tokio_postgres::connect(dsn, NoTls).await {
        Ok(pair) => pair,
        Err(e) => return err(format!("connect: {e}")),
    };
    let handle = tokio::spawn(async move {
        let _ = connection.await;
    });

    let mut clauses = vec![format!("series_id = {}", sql_str(series_id))];
    if let Some(f) = from {
        clauses.push(format!("ts >= {f}"));
    }
    if let Some(t) = to {
        clauses.push(format!("ts <= {t}"));
    }
    clauses.push(fv_mac::pg_where(taxonomy, grants, column_for)); // PG-native MAC pushdown
    let sql = format!(
        "SELECT ts, value_num, value_str, value_geo FROM series.observations WHERE {} ORDER BY ts ASC LIMIT {size}",
        clauses.join(" AND ")
    );

    let result = client.query(&sql, &[]).await;
    drop(client);
    let _ = handle.await;

    let pg_rows = match result {
        Ok(r) => r,
        Err(e) => return err(format!("series query `{sql}`: {e}")),
    };
    let mut out = Vec::with_capacity(pg_rows.len());
    for r in &pg_rows {
        let mut row: Row = Vec::with_capacity(r.columns().len());
        for (i, col) in r.columns().iter().enumerate() {
            row.push((col.name().to_string(), cell(r, i, col.type_())));
        }
        out.push(row);
    }
    Ok(out)
}

/// Escape a Postgres string literal (single quotes doubled). Shared by the series read.
fn sql_str(s: &str) -> String {
    format!("'{}'", s.replace('\'', "''"))
}

/// Map a Postgres cell to a dialect Value (the types curated tables commonly use). Unknown types
/// fall back to their text form, else Null.
fn cell(row: &tokio_postgres::Row, i: usize, ty: &Type) -> Value {
    match *ty {
        Type::TEXT | Type::VARCHAR | Type::BPCHAR | Type::NAME => row
            .try_get::<_, Option<String>>(i)
            .ok()
            .flatten()
            .map(Value::Str)
            .unwrap_or(Value::Null),
        Type::INT2 => row
            .try_get::<_, Option<i16>>(i)
            .ok()
            .flatten()
            .map(|v| Value::Num(v as f64))
            .unwrap_or(Value::Null),
        Type::INT4 => row
            .try_get::<_, Option<i32>>(i)
            .ok()
            .flatten()
            .map(|v| Value::Num(v as f64))
            .unwrap_or(Value::Null),
        Type::INT8 => row
            .try_get::<_, Option<i64>>(i)
            .ok()
            .flatten()
            .map(|v| Value::Num(v as f64))
            .unwrap_or(Value::Null),
        Type::FLOAT4 => row
            .try_get::<_, Option<f32>>(i)
            .ok()
            .flatten()
            .map(|v| Value::Num(v as f64))
            .unwrap_or(Value::Null),
        Type::FLOAT8 => row
            .try_get::<_, Option<f64>>(i)
            .ok()
            .flatten()
            .map(Value::Num)
            .unwrap_or(Value::Null),
        Type::BOOL => row
            .try_get::<_, Option<bool>>(i)
            .ok()
            .flatten()
            .map(Value::Bool)
            .unwrap_or(Value::Null),
        _ => row
            .try_get::<_, Option<String>>(i)
            .ok()
            .flatten()
            .map(Value::Str)
            .unwrap_or(Value::Null),
    }
}

/// Convenience: pull the value of one column from a row (Null if absent).
pub fn get<'a>(row: &'a Row, key: &str) -> &'a Value {
    row.iter()
        .find(|(k, _)| k == key)
        .map(|(_, v)| v)
        .unwrap_or(&Value::Null)
}

#[cfg(test)]
mod tee_tests {
    use super::*;
    use fv_value::Value;

    #[test]
    fn runner_read_prefers_the_latest_landing_and_falls_back_without_one() {
        let dir = std::env::temp_dir().join(format!("fv-landing-tests-{}", std::process::id()));
        let store = lake::SnapshotStore::at(dir.join("curated.tee-vessel"));
        let row = |rid: &str, speed: f64| -> Row {
            vec![
                ("rid".into(), Value::Str(rid.into())),
                ("speedMps".into(), Value::Num(speed)),
            ]
        };
        store.land(&[row("v1", 1.0)]).unwrap();
        store.land(&[row("v1", 2.0), row("v2", 3.0)]).unwrap();
        let _env = crate::ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); // serialise env-mutating tests
        std::env::set_var("PG_LANDING_DIR", dir.to_str().unwrap());
        let config = ConnectorConfig {
            kind: "REST_POLL".into(),
            host: String::new(),
            port: 0,
            database: String::new(),
            user: String::new(),
            options: serde_json::json!({}),
        };
        let rt = tokio::runtime::Builder::new_current_thread()
            .enable_all()
            .build()
            .unwrap();
        let read = |table: &'static str, limit| {
            rt.block_on(read_source_preferring_landing(
                &config,
                "",
                table,
                &Taxonomy::default(),
                &Bindings::new(),
                |c| c.to_string(),
                limit,
            ))
        };
        // Landed → the LATEST snapshot, no network; limit honoured.
        let rows = read("curated.tee-vessel", None).unwrap();
        assert_eq!(rows.len(), 2);
        assert!(rows.iter().any(|r| r
            .iter()
            .any(|(k, v)| k == "speedMps" && matches!(v, Value::Num(n) if *n == 3.0))));
        assert_eq!(read("curated.tee-vessel", Some(1)).unwrap().len(), 1);
        // Never landed → live fallback, which demands a URL (the path is exercised, no network).
        assert!(read("curated.never-landed", None)
            .unwrap_err()
            .0
            .contains("config.options.url is required"));
        std::env::remove_var("PG_LANDING_DIR"); // leave the process env as we found it
    }
}