fv-connect 0.1.0

FusionVault Conduit: read relational, file and REST sources into rows with mandatory access control pushed into the read, write results back with atomic promotion, and drive incremental base-ingest scans, changelogs and label normalisation checked against the published conformance vectors.
Documentation
//! Compacted-log fold — the pure core of a streaming (Kafka) source.
//!
//! A Kafka topic is a stream, not a table, so the funnel reads it as a CDC / compacted log: each keyed
//! message is the latest FULL state of one object, and a null value (a tombstone) is a delete. Folding
//! newly-consumed messages onto the previously-landed state yields the current full-table state, which
//! the existing `LandingSource` then lands + diffs — so the changelog emits exactly the property-level
//! INSERT/UPDATE/DELETEs the stream implies, with no new diff machinery. The broker consume
//! itself is the caller's IO.

use fv_value::Value;

use crate::Row;

/// One consumed message: a key and a value. A `None` value is a tombstone (delete). The value, when
/// present, is the full latest row state for the object.
#[derive(Debug, Clone)]
pub struct LogMessage {
    pub key: Option<String>,
    pub value: Option<Row>,
}

fn rid_of(row: &Row) -> Option<String> {
    row.iter().find(|(k, _)| k == "rid").and_then(|(_, v)| match v {
        Value::Str(s) => Some(s.clone()),
        _ => None,
    })
}

/// Fold consumed messages onto the prior full-table state → the new full-table state, in first-seen
/// order (an upsert keeps the object's position; a new object appends; a tombstone removes it). The
/// object rid is the message value's `rid` field, falling back to the message key; a message with
/// neither has nowhere to land and is skipped. The upserted row always carries its rid.
pub fn fold_compacted_log(prior: &[Row], messages: &[LogMessage]) -> Vec<Row> {
    // Ordered rids + their rows (mirrors the Python dict: update keeps position, insert appends).
    let mut order: Vec<String> = Vec::new();
    let mut rows: std::collections::HashMap<String, Row> = std::collections::HashMap::new();
    for row in prior {
        if let Some(rid) = rid_of(row) {
            if !rows.contains_key(&rid) {
                order.push(rid.clone());
            }
            rows.insert(rid, row.clone());
        }
    }

    for msg in messages {
        match &msg.value {
            // Tombstone: the rid is the key (no value to read a rid from).
            None => {
                if let Some(rid) = &msg.key {
                    if rows.remove(rid).is_some() {
                        order.retain(|k| k != rid);
                    }
                }
            }
            Some(value) => {
                let rid = match rid_of(value).or_else(|| msg.key.clone()) {
                    Some(r) => r,
                    None => continue, // neither a rid nor a key → nowhere to land
                };
                // Upsert the full latest state, rid first (== Python `{"rid": rid, **value}`).
                let mut row: Row = vec![("rid".to_string(), Value::Str(rid.clone()))];
                row.extend(value.iter().filter(|(k, _)| k != "rid").cloned());
                if !rows.contains_key(&rid) {
                    order.push(rid.clone());
                }
                rows.insert(rid, row);
            }
        }
    }

    order.into_iter().filter_map(|rid| rows.remove(&rid)).collect()
}

#[cfg(test)]
mod tests {
    use super::*;

    fn row(rid: &str, fields: &[(&str, &str)]) -> Row {
        let mut r: Row = vec![("rid".into(), Value::Str(rid.into()))];
        r.extend(
            fields
                .iter()
                .map(|(k, v)| ((*k).to_string(), Value::Str((*v).to_string()))),
        );
        r
    }

    fn msg(key: Option<&str>, value: Option<Row>) -> LogMessage {
        LogMessage {
            key: key.map(String::from),
            value,
        }
    }

    #[test]
    fn empty_prior_reconstitutes_full_rows() {
        // The property the reseed baseline-reset (/) depends on: re-landing a topic from
        // scratch (cleared landing + group replay from earliest) yields COMPLETE rows — every field
        // of the latest message state, not a diff shape.
        let m = row(
            "fv.o.m.object.speed-alert.TEL-1",
            &[
                ("deviceRid", "fv.o.m.object.telematics-device.TEL-1"),
                ("alert", "OVERSPEED"),
                ("speedKph", "97"),
            ],
        );
        let folded = fold_compacted_log(&[], &[msg(None, Some(m.clone()))]);
        assert_eq!(folded, vec![m]);
    }

    #[test]
    fn a_message_is_full_state_not_a_patch() {
        // A later message REPLACES the landed row wholesale: a field absent from the new message is
        // gone, not inherited. (The funnel's diff-vs-landing then emits its removal — the drift bug
        // was never here, it was landing state surviving an index wipe.)
        let prior = row("r-1", &[("a", "1"), ("b", "2")]);
        let update = row("r-1", &[("a", "9")]);
        let folded = fold_compacted_log(&[prior], &[msg(None, Some(update.clone()))]);
        assert_eq!(folded, vec![update]);
    }

    #[test]
    fn upsert_keeps_position_insert_appends_tombstone_removes() {
        let prior = vec![
            row("r-1", &[("a", "1")]),
            row("r-2", &[("a", "2")]),
            row("r-3", &[("a", "3")]),
        ];
        let folded = fold_compacted_log(
            &prior,
            &[
                msg(None, Some(row("r-2", &[("a", "2b")]))), // upsert: keeps slot 2
                msg(Some("r-1"), None),                      // tombstone: removes r-1
                msg(None, Some(row("r-4", &[("a", "4")]))),  // insert: appends
            ],
        );
        let rids: Vec<&str> = folded
            .iter()
            .map(|r| match &r[0].1 {
                Value::Str(s) => s.as_str(),
                _ => "?",
            })
            .collect();
        assert_eq!(rids, vec!["r-2", "r-3", "r-4"]);
        assert_eq!(folded[0], row("r-2", &[("a", "2b")]));
    }

    #[test]
    fn rid_falls_back_to_the_message_key_and_ridless_messages_are_skipped() {
        // A value with no rid lands under its kafka key (and the folded row gains that rid).
        let keyed = fold_compacted_log(
            &[],
            &[msg(Some("k-1"), Some(vec![("a".into(), Value::Str("1".into()))]))],
        );
        assert_eq!(keyed, vec![row("k-1", &[("a", "1")])]);
        // No rid and no key → nowhere to land; a keyless tombstone is equally inert.
        assert!(fold_compacted_log(
            &[],
            &[
                msg(None, Some(vec![("a".into(), Value::Str("1".into()))])),
                msg(None, None)
            ]
        )
        .is_empty());
    }
}