fv-connect 0.1.0

FusionVault Conduit: read relational, file and REST sources into rows with mandatory access control pushed into the read, write results back with atomic promotion, and drive incremental base-ingest scans, changelogs and label normalisation checked against the published conformance vectors.
Documentation
//! The landing "lake" — snapshot staging + checkpoint for polled sources.
//!
//! A relational source has no native snapshot/time-travel, so each poll of a source is LANDED as a
//! full-state snapshot and the changelog diff + checkpoint machinery does the rest (property-level
//! change detection, DELETE detection, resumable incremental scans). A table is a directory
//! `<dir>/<table>/snapshot-<id>.json`; each file is the full raw table state at that snapshot (a JSON
//! array of curated rows). Snapshots are landed only when the state actually changed, so an unchanged
//! poll produces no new snapshot (idempotent). Rows are stored RAW (label columns intact) — the
//! normalization into `_labels` + claims happens on read, at the source boundary (`labels::normalize_row`).
//!
//! This is deliberately a faithful JSON staging area, not a columnar table: the changelog reads whole
//! rows and nested `_labels` maps must round-trip exactly. Swapping this for a columnar/Iceberg store
//! (/) is behind the changelog vectors — a storage change, not a behaviour change.

use std::collections::BTreeMap;
use std::path::{Path, PathBuf};

use fv_value::{json_to_value, value_to_json, Value};

use crate::Row;

fn rid_of(row: &Row) -> Option<String> {
    row.iter().find(|(k, _)| k == "rid").and_then(|(_, v)| match v {
        Value::Str(s) => Some(s.clone()),
        _ => None,
    })
}

/// A raw row → its JSON object form (label columns intact).
fn row_to_json(row: &Row) -> serde_json::Value {
    serde_json::Value::Object(row.iter().map(|(k, v)| (k.clone(), value_to_json(v))).collect())
}

/// A JSON object → a raw row (ordered by the object's key order).
fn json_to_row(j: &serde_json::Value) -> Row {
    j.as_object()
        .map(|o| o.iter().map(|(k, v)| (k.clone(), json_to_value(v))).collect())
        .unwrap_or_default()
}

/// A per-table snapshot directory: `<root>/<table>/snapshot-<id>.json`.
pub struct SnapshotStore {
    dir: PathBuf,
}

impl SnapshotStore {
    /// The store for one table under a landing root (`<root>/<table>`), matching the Python layout.
    pub fn new(root: &str, table: &str) -> Self {
        SnapshotStore {
            dir: PathBuf::from(root.trim_end_matches('/')).join(table),
        }
    }

    /// The store rooted directly at `dir` (one table per directory).
    pub fn at(dir: impl Into<PathBuf>) -> Self {
        SnapshotStore { dir: dir.into() }
    }

    /// Sorted snapshot ids present for this table.
    pub fn list(&self) -> Vec<i64> {
        let mut ids: Vec<i64> = match std::fs::read_dir(&self.dir) {
            Ok(entries) => entries
                .filter_map(|e| e.ok())
                .filter_map(|e| {
                    let name = e.file_name();
                    let name = name.to_str()?;
                    name.strip_prefix("snapshot-")?
                        .strip_suffix(".json")?
                        .parse::<i64>()
                        .ok()
                })
                .collect(),
            Err(_) => Vec::new(),
        };
        ids.sort_unstable();
        ids
    }

    /// The last landed snapshot as raw rows (empty if none) — the prior state a compacted-log source
    /// folds new messages onto.
    pub fn last_raw(&self) -> Result<Vec<Row>, String> {
        match self.list().last() {
            Some(&id) => self.read_raw(id),
            None => Ok(Vec::new()),
        }
    }

    /// Load a snapshot as raw rows (label columns intact), in file order.
    pub fn read_raw(&self, snapshot_id: i64) -> Result<Vec<Row>, String> {
        let path = self.dir.join(format!("snapshot-{snapshot_id}.json"));
        let bytes = std::fs::read(&path).map_err(|e| format!("read {}: {e}", path.display()))?;
        let doc: serde_json::Value =
            serde_json::from_slice(&bytes).map_err(|e| format!("parse {}: {e}", path.display()))?;
        Ok(doc
            .as_array()
            .map(|a| a.iter().map(json_to_row).collect())
            .unwrap_or_default())
    }

    /// Write the current table state as the next snapshot — but only if it differs from the latest
    /// landed snapshot (keyed by rid), so an unchanged poll produces no new snapshot. Returns the new
    /// snapshot id if one was written, else `None`.
    pub fn land(&self, rows: &[Row]) -> Result<Option<i64>, String> {
        let existing = self.list();
        // Compare state keyed by rid, each row as a MAP — order-insensitive, since the JSON round-trip
        // reorders columns (a reshuffled poll with no logical change must not land a new snapshot).
        let as_state = |rows: Vec<Row>| -> BTreeMap<String, BTreeMap<String, Value>> {
            rows.into_iter()
                .filter_map(|r| rid_of(&r).map(|rid| (rid, r.into_iter().collect())))
                .collect()
        };
        let current = as_state(rows.to_vec());
        if let Some(&last) = existing.last() {
            if as_state(self.read_raw(last)?) == current {
                return Ok(None);
            }
        }
        let next_id = existing.last().map_or(1, |&l| l + 1);
        std::fs::create_dir_all(&self.dir).map_err(|e| format!("mkdir {}: {e}", self.dir.display()))?;
        let body = serde_json::Value::Array(rows.iter().map(row_to_json).collect());
        write_atomic(
            &self.dir.join(format!("snapshot-{next_id}.json")),
            &serde_json::to_vec(&body).unwrap(),
        )?;
        Ok(Some(next_id))
    }
}

/// Per-(type, source) checkpoint — `{key: last_consumed_snapshot_id}`. Persisted so re-runs resume
/// from the last snapshot rather than rescanning.
#[derive(Debug, Default, Clone, PartialEq)]
pub struct Checkpoint {
    map: BTreeMap<String, i64>,
}

impl Checkpoint {
    pub fn get(&self, key: &str) -> Option<i64> {
        self.map.get(key).copied()
    }

    pub fn set(&mut self, key: &str, snapshot_id: i64) {
        self.map.insert(key.to_string(), snapshot_id);
    }

    pub fn is_empty(&self) -> bool {
        self.map.is_empty()
    }

    /// Load a checkpoint file (empty if it does not exist yet).
    pub fn read(path: &str) -> Result<Checkpoint, String> {
        match std::fs::read(path) {
            Ok(bytes) => {
                let map: BTreeMap<String, i64> =
                    serde_json::from_slice(&bytes).map_err(|e| format!("parse checkpoint {path}: {e}"))?;
                Ok(Checkpoint { map })
            }
            Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(Checkpoint::default()),
            Err(e) => Err(format!("read checkpoint {path}: {e}")),
        }
    }

    /// Atomically persist the checkpoint (a crash mid-write never corrupts it).
    pub fn write(&self, path: &str) -> Result<(), String> {
        let bytes = serde_json::to_vec_pretty(&self.map).unwrap();
        write_atomic(Path::new(path), &bytes)
    }
}

/// Write `bytes` to `path` atomically via a temp file + rename (creating parent dirs).
fn write_atomic(path: &Path, bytes: &[u8]) -> Result<(), String> {
    if let Some(parent) = path.parent() {
        std::fs::create_dir_all(parent).map_err(|e| format!("mkdir {}: {e}", parent.display()))?;
    }
    let tmp = path.with_extension("tmp");
    std::fs::write(&tmp, bytes).map_err(|e| format!("write {}: {e}", tmp.display()))?;
    std::fs::rename(&tmp, path).map_err(|e| format!("rename {}: {e}", path.display()))
}

#[cfg(test)]
mod tests {
    use super::*;

    fn row(pairs: &[(&str, Value)]) -> Row {
        pairs.iter().map(|(k, v)| (k.to_string(), v.clone())).collect()
    }

    #[test]
    fn land_is_idempotent_and_faithful_to_nested_labels() {
        let tmp = std::env::temp_dir().join(format!("fv-lake-{}", std::process::id()));
        let _ = std::fs::remove_dir_all(&tmp);
        let store = SnapshotStore::at(&tmp);

        let labels = Value::Obj([("classification".to_string(), Value::Num(10.0))].into_iter().collect());
        let rows = vec![row(&[
            ("rid", Value::Str("A-1".into())),
            ("status", Value::Str("IN_SERVICE".into())),
            ("_labels", labels.clone()),
        ])];

        assert_eq!(store.land(&rows).unwrap(), Some(1));
        assert_eq!(store.land(&rows).unwrap(), None, "unchanged poll lands nothing");
        assert_eq!(store.list(), vec![1]);

        // Nested `_labels` object round-trips exactly (the reason this stays JSON, not the lossy Arrow path).
        let read = store.read_raw(1).unwrap();
        assert_eq!(read.len(), 1);
        assert_eq!(
            read[0].iter().find(|(k, _)| k == "_labels").map(|(_, v)| v),
            Some(&labels)
        );

        let changed = vec![row(&[
            ("rid", Value::Str("A-1".into())),
            ("status", Value::Str("GROUNDED".into())),
        ])];
        assert_eq!(store.land(&changed).unwrap(), Some(2));
        assert_eq!(store.list(), vec![1, 2]);

        let _ = std::fs::remove_dir_all(&tmp);
    }

    #[test]
    fn checkpoint_round_trips_and_defaults_empty() {
        let path = std::env::temp_dir().join(format!("fv-cp-{}.json", std::process::id()));
        let p = path.to_str().unwrap();
        let _ = std::fs::remove_file(p);

        assert!(
            Checkpoint::read(p).unwrap().is_empty(),
            "missing checkpoint reads empty"
        );
        let mut cp = Checkpoint::default();
        cp.set("Customer:crm", 4);
        cp.set("Aircraft", 9);
        cp.write(p).unwrap();

        let back = Checkpoint::read(p).unwrap();
        assert_eq!(back.get("Customer:crm"), Some(4));
        assert_eq!(back.get("Aircraft"), Some(9));
        assert_eq!(back.get("missing"), None);
        let _ = std::fs::remove_file(p);
    }
}