fv-connect 0.1.0

FusionVault Conduit: read relational, file and REST sources into rows with mandatory access control pushed into the read, write results back with atomic promotion, and drive incremental base-ingest scans, changelogs and label normalisation checked against the published conformance vectors.
Documentation
//! The connector `Source` SPI — one trait + a registry, so adding a driver is
//! "implement the trait + one `register_source` call" instead of growing a `match config.kind`
//! in every capability (the read/write dispatches used to be copied across fv-connect and the
//! fv-datafusion worker; they all collapse to registry lookups here).
//!
//! Governance is part of the CONTRACT, not the driver's discretion: `read` takes the taxonomy +
//! grants and must push MAC into the scan — a driver that cannot translate the
//! predicate must refuse, not return unfiltered rows. `write_and_promote` must stage + atomically
//! swap (never partially-visible outputs). Scan-leg classification (polled / log / snapshots —
//! Kafka / Iceberg style sources) is declared per source; the shipped drivers cover the relational
//! read/write pair, SQLite and REST.

use std::collections::HashMap;
use std::sync::{Arc, LazyLock, RwLock};

use fv_mac::{Bindings, Taxonomy};

use crate::config::ConnectorConfig;
use crate::write::{write_and_promote_mysql, write_and_promote_postgres};
use crate::{read_mysql_with_mac, read_with_mac, ConnectError, Row};

/// What a driver can do. Checked at dispatch so a capability gap is a clear error
/// ("MYSQL cannot X") rather than a wrong-arm `match` panic or a silent no-op.
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct Caps {
    pub read: bool,
    pub write: bool,
}

/// How the funnel drives this source's BASE-ingest scan (replaces the type-string
/// matches in scan_binding and the funnel-follow classification). The leg MACHINERY (consumer
/// drains, snapshot prefetch) lives with the funnel; the driver only declares which leg it needs,
/// so a new log-shaped source (MQTT, webhook, CDC) inherits the event-driven leg by registration.
#[derive(Debug, Clone, Copy, PartialEq)]
pub enum ScanLeg {
    /// Fresh full read per tick, diffed against the landing (relational shape). Uses `Source::read`.
    Polled,
    /// Keyed compacted log: consume messages, fold onto the landed state (Kafka shape).
    Log,
    /// Native snapshot cursor: the source versions itself (Iceberg shape).
    Snapshots,
}

/// A label-column mapper: taxonomy axis → the source's column (e.g. classification →
/// `classification_level`). Object-safe form of the closures call sites already pass.
pub type LabelColumn<'a> = &'a (dyn Fn(&str) -> String + Sync);

// The SPI read takes the full gated-read contract; every driver shares the shape.
#[allow(clippy::too_many_arguments)]
#[async_trait::async_trait]
pub trait Source: Send + Sync {
    fn capabilities(&self) -> Caps;

    /// Which funnel scan leg drives this source. Polled by default (the relational shape).
    fn scan_leg(&self) -> ScanLeg {
        ScanLeg::Polled
    }

    /// MAC-pushed-down read of one (schema-qualified) table.
    async fn read(
        &self,
        config: &ConnectorConfig,
        password: &str,
        table: &str,
        taxonomy: &Taxonomy,
        grants: &Bindings,
        label_column: LabelColumn<'_>,
        limit: Option<usize>,
    ) -> Result<Vec<Row>, ConnectError>;

    /// Stage `rows` and atomically promote them as `table` (never partially visible).
    async fn write_and_promote(
        &self,
        config: &ConnectorConfig,
        password: &str,
        table: &str,
        columns: &[String],
        rows: &[Row],
    ) -> Result<(), ConnectError>;
}

struct PostgresSource;

#[async_trait::async_trait]
impl Source for PostgresSource {
    fn capabilities(&self) -> Caps {
        Caps {
            read: true,
            write: true,
        }
    }

    async fn read(
        &self,
        config: &ConnectorConfig,
        password: &str,
        table: &str,
        taxonomy: &Taxonomy,
        grants: &Bindings,
        label_column: LabelColumn<'_>,
        limit: Option<usize>,
    ) -> Result<Vec<Row>, ConnectError> {
        read_with_mac(
            &config.postgres_dsn(password),
            table,
            taxonomy,
            grants,
            label_column,
            limit,
        )
        .await
    }

    async fn write_and_promote(
        &self,
        config: &ConnectorConfig,
        password: &str,
        table: &str,
        columns: &[String],
        rows: &[Row],
    ) -> Result<(), ConnectError> {
        write_and_promote_postgres(&config.postgres_dsn(password), table, columns, rows).await
    }
}

struct MysqlSource;

#[async_trait::async_trait]
impl Source for MysqlSource {
    fn capabilities(&self) -> Caps {
        Caps {
            read: true,
            write: true,
        }
    }

    async fn read(
        &self,
        config: &ConnectorConfig,
        password: &str,
        table: &str,
        taxonomy: &Taxonomy,
        grants: &Bindings,
        label_column: LabelColumn<'_>,
        limit: Option<usize>,
    ) -> Result<Vec<Row>, ConnectError> {
        read_mysql_with_mac(
            &config.mysql_url(password),
            table,
            taxonomy,
            grants,
            label_column,
            limit,
        )
        .await
    }

    async fn write_and_promote(
        &self,
        config: &ConnectorConfig,
        password: &str,
        table: &str,
        columns: &[String],
        rows: &[Row],
    ) -> Result<(), ConnectError> {
        write_and_promote_mysql(&config.mysql_url(password), table, columns, rows).await
    }
}

/// SQLite — the DX proof: a full read+write driver is this impl + one registry line.
/// `config.database` is the db file path (SQLite has no host/port); the sync rusqlite core runs on
/// the blocking pool.
struct SqliteSource;

#[async_trait::async_trait]
impl Source for SqliteSource {
    fn capabilities(&self) -> Caps {
        Caps {
            read: true,
            write: true,
        }
    }

    async fn read(
        &self,
        config: &ConnectorConfig,
        _password: &str,
        table: &str,
        taxonomy: &Taxonomy,
        grants: &Bindings,
        label_column: LabelColumn<'_>,
        limit: Option<usize>,
    ) -> Result<Vec<Row>, ConnectError> {
        let (path, table, taxonomy, grants) = (
            config.database.clone(),
            table.to_string(),
            taxonomy.clone(),
            grants.clone(),
        );
        let label: Vec<(String, String)> = taxonomy
            .axes
            .iter()
            .map(|a| (a.name.clone(), label_column(&a.name)))
            .collect();
        tokio::task::spawn_blocking(move || {
            let map: std::collections::HashMap<String, String> = label.into_iter().collect();
            crate::sqlite::read_sqlite_with_mac_sync(
                &path,
                &table,
                &taxonomy,
                &grants,
                |a| map.get(a).cloned().unwrap_or_else(|| a.to_string()),
                limit,
            )
        })
        .await
        .map_err(|e| ConnectError(format!("sqlite read task: {e}")))?
    }

    async fn write_and_promote(
        &self,
        config: &ConnectorConfig,
        _password: &str,
        table: &str,
        columns: &[String],
        rows: &[Row],
    ) -> Result<(), ConnectError> {
        let (path, table, columns, rows) = (
            config.database.clone(),
            table.to_string(),
            columns.to_vec(),
            rows.to_vec(),
        );
        tokio::task::spawn_blocking(move || {
            crate::sqlite::write_and_promote_sqlite_sync(&path, &table, &columns, &rows)
        })
        .await
        .map_err(|e| ConnectError(format!("sqlite write task: {e}")))?
    }
}

/// A scan-leg-only registration: the source is driven entirely by a funnel leg (its consume /
/// snapshot machinery lives there), so relational read/write are capability errors, not matches.
struct LegOnly(ScanLeg);

#[async_trait::async_trait]
impl Source for LegOnly {
    fn capabilities(&self) -> Caps {
        Caps {
            read: false,
            write: false,
        }
    }
    fn scan_leg(&self) -> ScanLeg {
        self.0
    }
    async fn read(
        &self,
        config: &ConnectorConfig,
        _: &str,
        _: &str,
        _: &Taxonomy,
        _: &Bindings,
        _: LabelColumn<'_>,
        _: Option<usize>,
    ) -> Result<Vec<Row>, ConnectError> {
        Err(ConnectError(format!(
            "connector type '{}' is scan-leg driven ({:?}); it has no relational read",
            config.kind, self.0
        )))
    }
    async fn write_and_promote(
        &self,
        config: &ConnectorConfig,
        _: &str,
        _: &str,
        _: &[String],
        _: &[Row],
    ) -> Result<(), ConnectError> {
        Err(ConnectError(format!(
            "connector type '{}' has no write capability",
            config.kind
        )))
    }
}

/// Connector type (upper-cased) → driver. Seeded with the built-ins; `register_source` adds more
/// (a custom driver registers at process start — the "define a connector once" seam).
static REGISTRY: LazyLock<RwLock<HashMap<String, Arc<dyn Source>>>> = LazyLock::new(|| {
    let mut m: HashMap<String, Arc<dyn Source>> = HashMap::new();
    m.insert("POSTGRES".into(), Arc::new(PostgresSource));
    m.insert("MYSQL".into(), Arc::new(MysqlSource));
    m.insert("SQLITE".into(), Arc::new(SqliteSource));
    m.insert("KAFKA".into(), Arc::new(LegOnly(ScanLeg::Log)));
    m.insert("ICEBERG".into(), Arc::new(LegOnly(ScanLeg::Snapshots)));
    m.insert("ICEBERG_REST".into(), Arc::new(LegOnly(ScanLeg::Snapshots)));
    // The generic REST connector (read-only, Polled): one driver, two kinds — REST = batch pull,
    // REST_POLL = the same driver the funnel-follow polls on a cadence (a live feed).
    m.insert("REST".into(), Arc::new(crate::rest::RestSource));
    m.insert("REST_POLL".into(), Arc::new(crate::rest::RestSource));
    RwLock::new(m)
});

/// Register (or replace) a driver for `kind`. Case-insensitive on lookup.
pub fn register_source(kind: &str, source: Arc<dyn Source>) {
    REGISTRY.write().unwrap().insert(kind.to_ascii_uppercase(), source);
}

/// Look up the driver for a connector type, if one is registered.
pub fn source_for(kind: &str) -> Option<Arc<dyn Source>> {
    REGISTRY.read().unwrap().get(&kind.to_ascii_uppercase()).cloned()
}

/// The registered connector types, sorted — for "have X, Y" error messages that stay truthful
/// as drivers are added.
pub fn registered_kinds() -> Vec<String> {
    let mut kinds: Vec<String> = REGISTRY.read().unwrap().keys().cloned().collect();
    kinds.sort();
    kinds
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn builtins_are_registered_with_full_caps() {
        for kind in ["POSTGRES", "postgres", "MYSQL", "mysql"] {
            let s = source_for(kind).expect(kind);
            assert_eq!(
                s.capabilities(),
                Caps {
                    read: true,
                    write: true
                }
            );
        }
        assert!(source_for("ORACLE").is_none());
        // Contains, not equals: the registry is process-global and other tests register drivers.
        let kinds = registered_kinds();
        assert!(kinds.contains(&"MYSQL".to_string()) && kinds.contains(&"POSTGRES".to_string()));
    }

    #[test]
    fn a_registered_custom_driver_is_dispatchable() {
        struct ReadOnly;
        #[async_trait::async_trait]
        impl Source for ReadOnly {
            fn capabilities(&self) -> Caps {
                Caps {
                    read: true,
                    write: false,
                }
            }
            async fn read(
                &self,
                _: &ConnectorConfig,
                _: &str,
                _: &str,
                _: &Taxonomy,
                _: &Bindings,
                _: LabelColumn<'_>,
                _: Option<usize>,
            ) -> Result<Vec<Row>, ConnectError> {
                Ok(vec![vec![("rid".into(), fv_value::Value::Str("r-1".into()))]])
            }
            async fn write_and_promote(
                &self,
                _: &ConnectorConfig,
                _: &str,
                _: &str,
                _: &[String],
                _: &[Row],
            ) -> Result<(), ConnectError> {
                Err(ConnectError("read-only".into()))
            }
        }
        register_source("TESTKIND", Arc::new(ReadOnly));
        let s = source_for("testkind").expect("registered");
        assert!(s.capabilities().read && !s.capabilities().write);
    }
}