fv-connect 0.1.0

FusionVault Conduit: read relational, file and REST sources into rows with mandatory access control pushed into the read, write results back with atomic promotion, and drive incremental base-ingest scans, changelogs and label normalisation checked against the published conformance vectors.
Documentation
//! BASE-CDC changelog diff — the pure core of the base-ingest scan, pinned to the contract's
//! `changelog` conformance vectors.
//!
//! Diffing two curated snapshots into property-level BASE changes — and, within a changed row, only
//! the *changed* properties — is what lets the merge apply a batch update to one property without
//! clobbering a concurrent user edit to another (property-level provenance). A snapshot that
//! rewrote files without logical changes (compaction) yields nothing.
//!
//! A BASE change deliberately carries NO correlation token and NO source-updated-at: a batch snapshot
//! is anonymous base data, so it cannot absorb a pinned edit on a ROUND_TRIP property — which is
//! exactly what makes the lost-update race structurally safe. That guarantee is structural here: the
//! [`Change`] struct simply has no field for either.

use std::collections::HashMap;

use fv_value::{value_to_json, Value};
use serde_json::json;

use crate::Row;

/// One snapshot as ordered `(rid, row)` pairs. Order is preserved so the emitted change stream keeps a
/// stable order (the merge is co-partitioned by rid, but a deterministic order keeps tests + replay
/// reproducible). A `row` carries `rid`, the business properties, and optionally a `_labels` map.
pub type Snapshot = Vec<(String, Row)>;

/// A BASE change envelope: `{src:"BASE", objectType, rid, properties, snapshot, ts, labels?, deleted?,
/// datasource?}`. `src` is always `"BASE"`.
#[derive(Debug, Clone, PartialEq)]
pub struct Change {
    pub object_type: String,
    pub rid: String,
    /// Only the changed properties (INSERT = all; UPDATE = the diff; secondary DELETE = its claims → null).
    pub properties: Vec<(String, Value)>,
    pub snapshot: i64,
    pub ts: i64,
    /// The row's security labels — present only on an INSERT that carried a `_labels` map.
    pub labels: Option<Value>,
    /// A primary DELETE tombstones the object (`deleted: true`).
    pub deleted: bool,
    /// The binding name — tags which source each value came from (`_prov.ds`). `None` for
    /// the singular-sugar binding, keeping legacy envelopes byte-identical.
    pub datasource: Option<String>,
}

impl Change {
    /// The `merged.v1` envelope this change produces on the sink:
    /// `{src:"BASE", objectType, rid, properties, snapshot, ts, labels?, deleted?, datasource?}` —
    /// byte-for-byte the shape merge-core consumes (`labels`/`deleted`/`datasource` present only when
    /// they apply; no `correlationToken`/`sourceUpdatedAt`, which is why a BASE change is anonymous).
    pub fn to_json(&self) -> serde_json::Value {
        let mut m = serde_json::Map::new();
        m.insert("src".into(), json!("BASE"));
        m.insert("objectType".into(), json!(self.object_type));
        m.insert("rid".into(), json!(self.rid));
        m.insert(
            "properties".into(),
            serde_json::Value::Object(
                self.properties
                    .iter()
                    .map(|(k, v)| (k.clone(), value_to_json(v)))
                    .collect(),
            ),
        );
        m.insert("snapshot".into(), json!(self.snapshot));
        m.insert("ts".into(), json!(self.ts));
        if let Some(labels) = &self.labels {
            m.insert("labels".into(), value_to_json(labels));
        }
        if self.deleted {
            m.insert("deleted".into(), json!(true));
        }
        if let Some(ds) = &self.datasource {
            m.insert("datasource".into(), json!(ds));
        }
        serde_json::Value::Object(m)
    }
}

/// The object properties of a curated row — everything except `rid` and the `_labels` column.
fn business_props(row: &Row) -> Vec<(String, Value)> {
    row.iter()
        .filter(|(k, _)| k != "rid" && k != "_labels")
        .map(|(k, v)| (k.clone(), v.clone()))
        .collect()
}

/// The row's `_labels` map, if present and non-null (mirrors Python's `row.get("_labels") is not None`).
fn row_labels(row: &Row) -> Option<Value> {
    row.iter()
        .find(|(k, _)| k == "_labels")
        .map(|(_, v)| v.clone())
        .filter(|v| !matches!(v, Value::Null))
}

/// Diff two snapshots (each `{rid: curated_row}`) into an ordered list of BASE change envelopes.
///
/// - rid in `curr` only            → INSERT: all properties (+ labels if present).
/// - rid in both, props differ     → UPDATE: only the changed properties.
/// - rid in both, no prop change   → nothing (compaction / no-op).
/// - rid in `prev` only → DELETE. Lifecycle belongs to the PRIMARY source: a
///   primary delete tombstones (`deleted: true`); a SECONDARY delete only nulls the properties that
///   source last supplied ("I no longer know these") — the object itself survives.
///
/// `datasource` (the binding name) tags every emitted change so the materializer records which source
/// each value came from.
#[allow(clippy::too_many_arguments)] // the changelog contract, one name per leg
pub fn diff_snapshots(
    prev: &Snapshot,
    curr: &Snapshot,
    object_type: &str,
    snapshot_id: i64,
    ts: i64,
    datasource: Option<&str>,
    primary: bool,
    // EVENT-TIME stamping (owner directive 2026-09-02): the datasource's `sourceUpdatedAtColumn` —
    // when a row carries it, the change is stamped with the DATA's own time, not the scan clock.
    // Late imports, replays and backfills then land in history at their TRUE time (and a looped
    // replay collapses onto itself instead of sawtoothing forward). None/absent/non-numeric rows
    // keep the scan-time fallback — the exact pre-existing behaviour.
    event_time_column: Option<&str>,
) -> Vec<Change> {
    let prev_idx: HashMap<&str, &Row> = prev.iter().map(|(rid, row)| (rid.as_str(), row)).collect();
    let curr_idx: HashMap<&str, &Row> = curr.iter().map(|(rid, row)| (rid.as_str(), row)).collect();

    // The row's event time, when declared + present + sane; else the scan clock.
    let event_ts = |row: Option<&&Row>| -> i64 {
        let Some(col) = event_time_column else { return ts };
        row.and_then(|r| r.iter().find(|(k, _)| k == col))
            .and_then(|(_, v)| match v {
                Value::Num(n) if n.is_finite() && *n > 0.0 => Some(*n as i64),
                // Strings: epoch-ms first, then RFC3339/ISO-8601 WITH its own zone offset (a backlog
                // import can say "2024-03-01T09:00:00+10:00" and land at the right instant). Anything
                // messier is the pipeline builder's job to normalize BEFORE ingest (owner's model).
                Value::Str(s) => {
                    let t = s.trim();
                    t.parse::<f64>()
                        .ok()
                        .filter(|n| n.is_finite() && *n > 0.0)
                        .map(|n| n as i64)
                        .or_else(|| {
                            chrono::DateTime::parse_from_rfc3339(t)
                                .ok()
                                .map(|d| d.timestamp_millis())
                        })
                }
                _ => None,
            })
            .unwrap_or(ts)
    };
    let envelope = |rid: &str, props: Vec<(String, Value)>, change_ts: i64| Change {
        object_type: object_type.to_string(),
        rid: rid.to_string(),
        properties: props,
        snapshot: snapshot_id,
        ts: change_ts,
        labels: None,
        deleted: false,
        datasource: datasource.map(str::to_string),
    };

    let mut out = Vec::new();

    for (rid, row) in curr {
        let props = business_props(row);
        let row_ts = event_ts(Some(&row));
        match prev_idx.get(rid.as_str()) {
            None => {
                let mut change = envelope(rid, props, row_ts);
                change.labels = row_labels(row);
                out.push(change);
            }
            Some(prev_row) => {
                let prev_props = business_props(prev_row);
                let prev_map: HashMap<&str, &Value> = prev_props.iter().map(|(k, v)| (k.as_str(), v)).collect();
                let changed: Vec<(String, Value)> = props
                    .iter()
                    .filter(|(k, v)| prev_map.get(k.as_str()).copied() != Some(v))
                    .map(|(k, v)| (k.clone(), v.clone()))
                    .collect();
                if !changed.is_empty() {
                    out.push(envelope(rid, changed, row_ts));
                }
            }
        }
    }

    for (rid, row) in prev {
        if curr_idx.contains_key(rid.as_str()) {
            continue;
        }
        // Disappearance is DETECTED at scan time — that IS its event time (the row carries no
        // future timestamp for its own absence).
        if primary {
            let mut change = envelope(rid, Vec::new(), ts);
            change.deleted = true;
            out.push(change);
        } else {
            let nulled: Vec<(String, Value)> = business_props(row).into_iter().map(|(k, _)| (k, Value::Null)).collect();
            if !nulled.is_empty() {
                out.push(envelope(rid, nulled, ts));
            }
        }
    }

    out
}

#[cfg(test)]
mod event_time_tests {
    use super::*;
    use fv_value::Value;

    fn row(rid: &str, speed: f64, observed: Option<Value>) -> (String, Row) {
        let mut r: Row = vec![
            ("rid".into(), Value::Str(rid.into())),
            ("speedMps".into(), Value::Num(speed)),
        ];
        if let Some(o) = observed {
            r.push(("observedAt".into(), o));
        }
        (rid.into(), r)
    }

    #[test]
    fn changes_are_stamped_with_the_rows_own_event_time_not_the_scan_clock() {
        let curr = vec![
            row("v1", 5.0, Some(Value::Num(1_000_000.0))),
            row("v2", 6.0, Some(Value::Str("2000000".into()))), // stringified on the wire still counts
            row("v3", 7.0, None),                               // no event column → scan clock
        ];
        let out = diff_snapshots(
            &Vec::new(),
            &curr,
            "Vessel",
            1,
            9_999_999,
            None,
            true,
            Some("observedAt"),
        );
        let ts_of = |rid: &str| out.iter().find(|c| c.rid == rid).unwrap().ts;
        assert_eq!(ts_of("v1"), 1_000_000);
        assert_eq!(ts_of("v2"), 2_000_000);
        assert_eq!(ts_of("v3"), 9_999_999);
        // A backlog import with a ZONED ISO timestamp lands at the true instant (owner: sources
        // may use any timezone — the offset is honoured, messier formats are the pipeline's job).
        let iso = vec![row("v4", 8.0, Some(Value::Str("2024-03-01T09:00:00+10:00".into())))];
        let out = diff_snapshots(
            &Vec::new(),
            &iso,
            "Vessel",
            1,
            9_999_999,
            None,
            true,
            Some("observedAt"),
        );
        assert_eq!(out[0].ts, 1_709_247_600_000); // 2024-02-29T23:00:00Z in epoch-ms
                                                  // None column = the exact pre-existing behaviour (the frozen vectors pin this).
        let legacy = diff_snapshots(&Vec::new(), &curr, "Vessel", 1, 9_999_999, None, true, None);
        assert!(legacy.iter().all(|c| c.ts == 9_999_999));
    }

    #[test]
    fn a_looped_replay_collapses_instead_of_sawtoothing() {
        // The same observation re-scanned later: identical values → NO change at all (dedup by
        // diff); a changed value with the SAME event ts re-lands at that ts (history overwrites
        // rid#prop#ts) — either way, no forward-marching sawtooth.
        let a = vec![row("v1", 5.0, Some(Value::Num(1_000.0)))];
        let again = diff_snapshots(&a, &a, "Vessel", 2, 8_888, None, true, Some("observedAt"));
        assert!(again.is_empty());
        let b = vec![row("v1", 9.0, Some(Value::Num(1_000.0)))];
        let changed = diff_snapshots(&a, &b, "Vessel", 3, 8_888, None, true, Some("observedAt"));
        assert_eq!(changed[0].ts, 1_000);
    }

    #[test]
    fn deletes_are_stamped_at_scan_time_and_junk_event_values_fall_back() {
        let prev = vec![row("v1", 5.0, Some(Value::Num(1_000.0)))];
        let out = diff_snapshots(&prev, &Vec::new(), "Vessel", 2, 7_777, None, true, Some("observedAt"));
        assert!(out[0].deleted);
        assert_eq!(out[0].ts, 7_777); // absence is detected at scan time — that IS its event time
        let junk = vec![row("v2", 5.0, Some(Value::Str("not-a-time".into())))];
        let out = diff_snapshots(&Vec::new(), &junk, "Vessel", 1, 7_777, None, true, Some("observedAt"));
        assert_eq!(out[0].ts, 7_777);
    }
}