fv-connect 0.1.0

FusionVault Conduit: read relational, file and REST sources into rows with mandatory access control pushed into the read, write results back with atomic promotion, and drive incremental base-ingest scans, changelogs and label normalisation checked against the published conformance vectors.
Documentation
//! Incremental base-ingest scan. Drives the changelog: for a source, emit BASE changes for every snapshot newer
//! than the checkpoint, in order, and return the new checkpoint (highest snapshot processed).
//!
//! A [`Source`] is where base rows come from, abstracted so the scan is identical regardless of
//! connector. [`LandingSource`] adapts a poll — any `fetch()` that returns the current table state — to
//! the snapshot cursor by landing each poll (`SnapshotStore`) and normalizing on read
//! (`labels::normalize_row`). The relational poll itself is `read_source` (MAC pushed into the query);
//! wiring that fetch into a `LandingSource` is the worker's job (Slice 4).

use crate::changelog::{diff_snapshots, Change, Snapshot};
use crate::labels::{normalize_row, LabelBinding};
use crate::lake::SnapshotStore;
use crate::Row;

/// Where the Funnel reads base rows from — the snapshot cursor every connector exposes.
pub trait Source {
    /// The snapshot ids available now (a relational source polls + lands here, so this may do IO).
    fn snapshots(&mut self) -> Result<Vec<i64>, String>;
    /// A snapshot as normalized `(rid, row)` pairs (labels folded, claims projected).
    fn read_snapshot(&mut self, snapshot_id: i64) -> Result<Snapshot, String>;
}

/// Where BASE changes go — the co-partitioned merged topic (Kafka), a file, or a test collector.
pub trait Sink {
    fn emit(&mut self, rid: &str, change: &Change);
    fn flush(&mut self) -> Result<(), String>;
}

/// Emit BASE changes for every snapshot newer than `last_snapshot` (and `<= up_to`, if the batch plane
/// has only produced up to S_n this tick), in order. Returns the new checkpoint (highest snapshot
/// processed), or `last_snapshot` unchanged if nothing was new — mirroring the Funnel resuming from its
/// persisted snapshot id. `datasource`/`primary` tag emitted changes and select
/// primary-vs-secondary DELETE semantics.
// One argument per leg of the scan contract (source/type/cursor/sink/clock/window/tagging) —
// a struct would just relocate the eight names.
#[allow(clippy::too_many_arguments)]
pub fn scan_source<S: Source + ?Sized>(
    source: &mut S,
    object_type: &str,
    last_snapshot: Option<i64>,
    sink: &mut dyn Sink,
    now_ms: i64,
    up_to: Option<i64>,
    datasource: Option<&str>,
    primary: bool,
    // The datasource's `sourceUpdatedAtColumn` — event-time stamping (see diff_snapshots).
    event_time_column: Option<&str>,
) -> Result<Option<i64>, String> {
    let snaps = source.snapshots()?;
    let new: Vec<i64> = snaps
        .into_iter()
        .filter(|s| last_snapshot.is_none_or(|l| *s > l) && up_to.is_none_or(|u| *s <= u))
        .collect();
    if new.is_empty() {
        return Ok(last_snapshot);
    }

    let mut prev: Snapshot = match last_snapshot {
        Some(l) => source.read_snapshot(l)?,
        None => Vec::new(),
    };
    let mut processed = last_snapshot;
    for sid in new {
        let curr = source.read_snapshot(sid)?;
        for change in diff_snapshots(
            &prev,
            &curr,
            object_type,
            sid,
            now_ms,
            datasource,
            primary,
            event_time_column,
        ) {
            sink.emit(&change.rid, &change);
        }
        prev = curr;
        processed = Some(sid);
    }
    Ok(processed)
}

/// A poll-and-land source: land the current table state (`fetch`) as a snapshot, expose the snapshot
/// cursor, and normalize each snapshot on read via the binding. `fetch` is given the PRIOR landed state
/// (the last snapshot's raw rows) so a compacted-log (Kafka) source can fold newly-consumed messages
/// onto it; a relational (Postgres/MySQL) source ignores it and returns a fresh `SELECT *`.
pub struct LandingSource<F> {
    store: SnapshotStore,
    binding: LabelBinding,
    fetch: F,
}

impl<F> LandingSource<F>
where
    F: FnMut(&[Row]) -> Result<Vec<Row>, String>,
{
    pub fn new(store: SnapshotStore, binding: LabelBinding, fetch: F) -> Self {
        LandingSource { store, binding, fetch }
    }
}

impl<F> Source for LandingSource<F>
where
    F: FnMut(&[Row]) -> Result<Vec<Row>, String>,
{
    fn snapshots(&mut self) -> Result<Vec<i64>, String> {
        let prior = self.store.last_raw()?;
        let rows = (self.fetch)(&prior)?;
        self.store.land(&rows)?;
        Ok(self.store.list())
    }

    fn read_snapshot(&mut self, snapshot_id: i64) -> Result<Snapshot, String> {
        let raw = self.store.read_raw(snapshot_id)?;
        let mut out: Snapshot = Vec::with_capacity(raw.len());
        for row in raw {
            let rid = row
                .iter()
                .find(|(k, _)| k == "rid")
                .and_then(|(_, v)| match v {
                    fv_value::Value::Str(s) => Some(s.clone()),
                    _ => None,
                })
                .ok_or("curated row has no string rid")?;
            out.push((rid, normalize_row(&row, &self.binding)));
        }
        Ok(out)
    }
}

/// A Source backed by pre-materialized snapshots (`{id → normalized rows}`) plus the full list of
/// available snapshot ids. For sources whose read is async and done up-front — e.g. an Iceberg table
/// whose snapshots are scanned to Arrow in the worker before the (sync) scan loop runs. An id not in
/// the map reads as an empty snapshot (a lenient baseline, e.g. an expired prior snapshot).
pub struct MemorySource {
    all_ids: Vec<i64>,
    snapshots: std::collections::BTreeMap<i64, Snapshot>,
}

impl MemorySource {
    pub fn new(all_ids: Vec<i64>, snapshots: std::collections::BTreeMap<i64, Snapshot>) -> Self {
        MemorySource { all_ids, snapshots }
    }
}

impl Source for MemorySource {
    fn snapshots(&mut self) -> Result<Vec<i64>, String> {
        Ok(self.all_ids.clone())
    }
    fn read_snapshot(&mut self, snapshot_id: i64) -> Result<Snapshot, String> {
        Ok(self.snapshots.get(&snapshot_id).cloned().unwrap_or_default())
    }
}

/// An in-memory sink that collects emitted changes — for tests and the file replay path.
#[derive(Default)]
pub struct CollectSink {
    pub changes: Vec<Change>,
}

impl Sink for CollectSink {
    fn emit(&mut self, _rid: &str, change: &Change) {
        self.changes.push(change.clone());
    }
    fn flush(&mut self) -> Result<(), String> {
        Ok(())
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use fv_value::Value;

    /// An in-memory Source: `{snapshot_id: [(rid, row)]}` — mirrors the Python `DictSource` test double.
    struct DictSource {
        snaps: std::collections::BTreeMap<i64, Snapshot>,
    }

    impl Source for DictSource {
        fn snapshots(&mut self) -> Result<Vec<i64>, String> {
            Ok(self.snaps.keys().copied().collect())
        }
        fn read_snapshot(&mut self, id: i64) -> Result<Snapshot, String> {
            Ok(self.snaps.get(&id).cloned().unwrap_or_default())
        }
    }

    fn r(rid: &str, status: &str) -> (String, Row) {
        (
            rid.to_string(),
            vec![
                ("rid".into(), Value::Str(rid.into())),
                ("status".into(), Value::Str(status.into())),
            ],
        )
    }

    #[test]
    fn first_scan_inserts_all_then_incremental_resumes() {
        let mut src = DictSource {
            snaps: [
                (1, vec![r("A-1", "IN_SERVICE"), r("A-2", "GROUNDED")]),
                (2, vec![r("A-1", "GROUNDED"), r("A-2", "GROUNDED")]),
            ]
            .into_iter()
            .collect(),
        };
        let mut sink = CollectSink::default();
        let cp = scan_source(&mut src, "Aircraft", None, &mut sink, 1, None, None, true, None).unwrap();
        assert_eq!(cp, Some(2));
        let seen: Vec<(&str, &Value)> = sink
            .changes
            .iter()
            .map(|c| {
                (
                    c.rid.as_str(),
                    c.properties
                        .iter()
                        .find(|(k, _)| k == "status")
                        .map(|(_, v)| v)
                        .unwrap(),
                )
            })
            .collect();
        assert_eq!(
            seen,
            vec![
                ("A-1", &Value::Str("IN_SERVICE".into())),
                ("A-2", &Value::Str("GROUNDED".into())),
                ("A-1", &Value::Str("GROUNDED".into())),
            ]
        );

        // Resume from the checkpoint: nothing new → nothing emitted, checkpoint unchanged.
        let mut sink2 = CollectSink::default();
        assert_eq!(
            scan_source(&mut src, "Aircraft", cp, &mut sink2, 2, None, None, true, None).unwrap(),
            Some(2)
        );
        assert!(sink2.changes.is_empty());
    }

    #[test]
    fn memory_source_serves_prefetched_snapshots_and_diffs_them() {
        // Pre-materialized snapshots (as an Iceberg prefetch would build): snapshot 5 then 7.
        let snaps: std::collections::BTreeMap<i64, Snapshot> =
            [(5, vec![r("A-1", "IN_SERVICE")]), (7, vec![r("A-1", "GROUNDED")])]
                .into_iter()
                .collect();
        let mut src = MemorySource::new(vec![5, 7], snaps);
        let mut sink = CollectSink::default();
        // From checkpoint 5: only snapshot 7 is new → one UPDATE change; checkpoint advances to 7.
        let cp = scan_source(&mut src, "Aircraft", Some(5), &mut sink, 1, None, None, true, None).unwrap();
        assert_eq!(cp, Some(7));
        assert_eq!(sink.changes.len(), 1);
        assert_eq!(
            sink.changes[0]
                .properties
                .iter()
                .find(|(k, _)| k == "status")
                .map(|(_, v)| v),
            Some(&Value::Str("GROUNDED".into()))
        );
    }

    #[test]
    fn up_to_caps_the_tick() {
        let mut src = DictSource {
            snaps: [(1, vec![r("A-1", "a")]), (2, vec![r("A-1", "b")])]
                .into_iter()
                .collect(),
        };
        let mut sink = CollectSink::default();
        assert_eq!(
            scan_source(&mut src, "T", None, &mut sink, 1, Some(1), None, true, None).unwrap(),
            Some(1)
        );
        assert_eq!(sink.changes.len(), 1);
        assert_eq!(
            sink.changes[0]
                .properties
                .iter()
                .find(|(k, _)| k == "status")
                .map(|(_, v)| v),
            Some(&Value::Str("a".into()))
        );
    }

    #[test]
    fn landing_source_lands_polls_and_normalizes_on_read() {
        let tmp = std::env::temp_dir().join(format!("fv-scan-{}", std::process::id()));
        let _ = std::fs::remove_dir_all(&tmp);
        let store = SnapshotStore::at(&tmp);
        let binding = LabelBinding {
            label_columns: [("classification".to_string(), "classification_level".to_string())]
                .into_iter()
                .collect(),
            on_missing_label: "DENY".into(),
            primary: true,
            ..Default::default()
        };

        // A poll returns a raw row carrying the label COLUMN; the scan must emit it folded into _labels.
        let fetch = |_prior: &[Row]| {
            Ok(vec![vec![
                ("rid".to_string(), Value::Str("A-1".into())),
                ("status".to_string(), Value::Str("IN_SERVICE".into())),
                ("classification_level".to_string(), Value::Num(10.0)),
            ]])
        };
        let mut source = LandingSource::new(store, binding, fetch);
        let mut sink = CollectSink::default();
        let cp = scan_source(&mut source, "Aircraft", None, &mut sink, 100, None, None, true, None).unwrap();
        assert_eq!(cp, Some(1));
        assert_eq!(sink.changes.len(), 1);
        let change = &sink.changes[0];
        // raw label column is gone from properties; folded into the change's labels.
        assert!(change.properties.iter().all(|(k, _)| k != "classification_level"));
        assert_eq!(
            change.labels,
            Some(Value::Obj(
                [("classification".to_string(), Value::Num(10.0))].into_iter().collect()
            ))
        );
        let _ = std::fs::remove_dir_all(&tmp);
    }
}