Skip to main content

fv_connect/
log.rs

1//! Compacted-log fold — the pure core of a streaming (Kafka) source.
2//!
3//! A Kafka topic is a stream, not a table, so the funnel reads it as a CDC / compacted log: each keyed
4//! message is the latest FULL state of one object, and a null value (a tombstone) is a delete. Folding
5//! newly-consumed messages onto the previously-landed state yields the current full-table state, which
6//! the existing `LandingSource` then lands + diffs — so the changelog emits exactly the property-level
7//! INSERT/UPDATE/DELETEs the stream implies, with no new diff machinery. The broker consume
8//! itself is the caller's IO.
9
10use fv_value::Value;
11
12use crate::Row;
13
14/// One consumed message: a key and a value. A `None` value is a tombstone (delete). The value, when
15/// present, is the full latest row state for the object.
16#[derive(Debug, Clone)]
17pub struct LogMessage {
18    pub key: Option<String>,
19    pub value: Option<Row>,
20}
21
22fn rid_of(row: &Row) -> Option<String> {
23    row.iter().find(|(k, _)| k == "rid").and_then(|(_, v)| match v {
24        Value::Str(s) => Some(s.clone()),
25        _ => None,
26    })
27}
28
29/// Fold consumed messages onto the prior full-table state → the new full-table state, in first-seen
30/// order (an upsert keeps the object's position; a new object appends; a tombstone removes it). The
31/// object rid is the message value's `rid` field, falling back to the message key; a message with
32/// neither has nowhere to land and is skipped. The upserted row always carries its rid.
33pub fn fold_compacted_log(prior: &[Row], messages: &[LogMessage]) -> Vec<Row> {
34    // Ordered rids + their rows (mirrors the Python dict: update keeps position, insert appends).
35    let mut order: Vec<String> = Vec::new();
36    let mut rows: std::collections::HashMap<String, Row> = std::collections::HashMap::new();
37    for row in prior {
38        if let Some(rid) = rid_of(row) {
39            if !rows.contains_key(&rid) {
40                order.push(rid.clone());
41            }
42            rows.insert(rid, row.clone());
43        }
44    }
45
46    for msg in messages {
47        match &msg.value {
48            // Tombstone: the rid is the key (no value to read a rid from).
49            None => {
50                if let Some(rid) = &msg.key {
51                    if rows.remove(rid).is_some() {
52                        order.retain(|k| k != rid);
53                    }
54                }
55            }
56            Some(value) => {
57                let rid = match rid_of(value).or_else(|| msg.key.clone()) {
58                    Some(r) => r,
59                    None => continue, // neither a rid nor a key → nowhere to land
60                };
61                // Upsert the full latest state, rid first (== Python `{"rid": rid, **value}`).
62                let mut row: Row = vec![("rid".to_string(), Value::Str(rid.clone()))];
63                row.extend(value.iter().filter(|(k, _)| k != "rid").cloned());
64                if !rows.contains_key(&rid) {
65                    order.push(rid.clone());
66                }
67                rows.insert(rid, row);
68            }
69        }
70    }
71
72    order.into_iter().filter_map(|rid| rows.remove(&rid)).collect()
73}
74
75#[cfg(test)]
76mod tests {
77    use super::*;
78
79    fn row(rid: &str, fields: &[(&str, &str)]) -> Row {
80        let mut r: Row = vec![("rid".into(), Value::Str(rid.into()))];
81        r.extend(
82            fields
83                .iter()
84                .map(|(k, v)| ((*k).to_string(), Value::Str((*v).to_string()))),
85        );
86        r
87    }
88
89    fn msg(key: Option<&str>, value: Option<Row>) -> LogMessage {
90        LogMessage {
91            key: key.map(String::from),
92            value,
93        }
94    }
95
96    #[test]
97    fn empty_prior_reconstitutes_full_rows() {
98        // The property the reseed baseline-reset (/) depends on: re-landing a topic from
99        // scratch (cleared landing + group replay from earliest) yields COMPLETE rows — every field
100        // of the latest message state, not a diff shape.
101        let m = row(
102            "fv.o.m.object.speed-alert.TEL-1",
103            &[
104                ("deviceRid", "fv.o.m.object.telematics-device.TEL-1"),
105                ("alert", "OVERSPEED"),
106                ("speedKph", "97"),
107            ],
108        );
109        let folded = fold_compacted_log(&[], &[msg(None, Some(m.clone()))]);
110        assert_eq!(folded, vec![m]);
111    }
112
113    #[test]
114    fn a_message_is_full_state_not_a_patch() {
115        // A later message REPLACES the landed row wholesale: a field absent from the new message is
116        // gone, not inherited. (The funnel's diff-vs-landing then emits its removal — the drift bug
117        // was never here, it was landing state surviving an index wipe.)
118        let prior = row("r-1", &[("a", "1"), ("b", "2")]);
119        let update = row("r-1", &[("a", "9")]);
120        let folded = fold_compacted_log(&[prior], &[msg(None, Some(update.clone()))]);
121        assert_eq!(folded, vec![update]);
122    }
123
124    #[test]
125    fn upsert_keeps_position_insert_appends_tombstone_removes() {
126        let prior = vec![
127            row("r-1", &[("a", "1")]),
128            row("r-2", &[("a", "2")]),
129            row("r-3", &[("a", "3")]),
130        ];
131        let folded = fold_compacted_log(
132            &prior,
133            &[
134                msg(None, Some(row("r-2", &[("a", "2b")]))), // upsert: keeps slot 2
135                msg(Some("r-1"), None),                      // tombstone: removes r-1
136                msg(None, Some(row("r-4", &[("a", "4")]))),  // insert: appends
137            ],
138        );
139        let rids: Vec<&str> = folded
140            .iter()
141            .map(|r| match &r[0].1 {
142                Value::Str(s) => s.as_str(),
143                _ => "?",
144            })
145            .collect();
146        assert_eq!(rids, vec!["r-2", "r-3", "r-4"]);
147        assert_eq!(folded[0], row("r-2", &[("a", "2b")]));
148    }
149
150    #[test]
151    fn rid_falls_back_to_the_message_key_and_ridless_messages_are_skipped() {
152        // A value with no rid lands under its kafka key (and the folded row gains that rid).
153        let keyed = fold_compacted_log(
154            &[],
155            &[msg(Some("k-1"), Some(vec![("a".into(), Value::Str("1".into()))]))],
156        );
157        assert_eq!(keyed, vec![row("k-1", &[("a", "1")])]);
158        // No rid and no key → nowhere to land; a keyless tombstone is equally inert.
159        assert!(fold_compacted_log(
160            &[],
161            &[
162                msg(None, Some(vec![("a".into(), Value::Str("1".into()))])),
163                msg(None, None)
164            ]
165        )
166        .is_empty());
167    }
168}