Skip to main content

fv_connect/
scan.rs

1//! Incremental base-ingest scan. Drives the changelog: for a source, emit BASE changes for every snapshot newer
2//! than the checkpoint, in order, and return the new checkpoint (highest snapshot processed).
3//!
4//! A [`Source`] is where base rows come from, abstracted so the scan is identical regardless of
5//! connector. [`LandingSource`] adapts a poll — any `fetch()` that returns the current table state — to
6//! the snapshot cursor by landing each poll (`SnapshotStore`) and normalizing on read
7//! (`labels::normalize_row`). The relational poll itself is `read_source` (MAC pushed into the query);
8//! wiring that fetch into a `LandingSource` is the worker's job (Slice 4).
9
10use crate::changelog::{diff_snapshots, Change, Snapshot};
11use crate::labels::{normalize_row, LabelBinding};
12use crate::lake::SnapshotStore;
13use crate::Row;
14
15/// Where the Funnel reads base rows from — the snapshot cursor every connector exposes.
16pub trait Source {
17    /// The snapshot ids available now (a relational source polls + lands here, so this may do IO).
18    fn snapshots(&mut self) -> Result<Vec<i64>, String>;
19    /// A snapshot as normalized `(rid, row)` pairs (labels folded, claims projected).
20    fn read_snapshot(&mut self, snapshot_id: i64) -> Result<Snapshot, String>;
21}
22
23/// Where BASE changes go — the co-partitioned merged topic (Kafka), a file, or a test collector.
24pub trait Sink {
25    fn emit(&mut self, rid: &str, change: &Change);
26    fn flush(&mut self) -> Result<(), String>;
27}
28
29/// Emit BASE changes for every snapshot newer than `last_snapshot` (and `<= up_to`, if the batch plane
30/// has only produced up to S_n this tick), in order. Returns the new checkpoint (highest snapshot
31/// processed), or `last_snapshot` unchanged if nothing was new — mirroring the Funnel resuming from its
32/// persisted snapshot id. `datasource`/`primary` tag emitted changes and select
33/// primary-vs-secondary DELETE semantics.
34// One argument per leg of the scan contract (source/type/cursor/sink/clock/window/tagging) —
35// a struct would just relocate the eight names.
36#[allow(clippy::too_many_arguments)]
37pub fn scan_source<S: Source + ?Sized>(
38    source: &mut S,
39    object_type: &str,
40    last_snapshot: Option<i64>,
41    sink: &mut dyn Sink,
42    now_ms: i64,
43    up_to: Option<i64>,
44    datasource: Option<&str>,
45    primary: bool,
46    // The datasource's `sourceUpdatedAtColumn` — event-time stamping (see diff_snapshots).
47    event_time_column: Option<&str>,
48) -> Result<Option<i64>, String> {
49    let snaps = source.snapshots()?;
50    let new: Vec<i64> = snaps
51        .into_iter()
52        .filter(|s| last_snapshot.is_none_or(|l| *s > l) && up_to.is_none_or(|u| *s <= u))
53        .collect();
54    if new.is_empty() {
55        return Ok(last_snapshot);
56    }
57
58    let mut prev: Snapshot = match last_snapshot {
59        Some(l) => source.read_snapshot(l)?,
60        None => Vec::new(),
61    };
62    let mut processed = last_snapshot;
63    for sid in new {
64        let curr = source.read_snapshot(sid)?;
65        for change in diff_snapshots(
66            &prev,
67            &curr,
68            object_type,
69            sid,
70            now_ms,
71            datasource,
72            primary,
73            event_time_column,
74        ) {
75            sink.emit(&change.rid, &change);
76        }
77        prev = curr;
78        processed = Some(sid);
79    }
80    Ok(processed)
81}
82
83/// A poll-and-land source: land the current table state (`fetch`) as a snapshot, expose the snapshot
84/// cursor, and normalize each snapshot on read via the binding. `fetch` is given the PRIOR landed state
85/// (the last snapshot's raw rows) so a compacted-log (Kafka) source can fold newly-consumed messages
86/// onto it; a relational (Postgres/MySQL) source ignores it and returns a fresh `SELECT *`.
87pub struct LandingSource<F> {
88    store: SnapshotStore,
89    binding: LabelBinding,
90    fetch: F,
91}
92
93impl<F> LandingSource<F>
94where
95    F: FnMut(&[Row]) -> Result<Vec<Row>, String>,
96{
97    pub fn new(store: SnapshotStore, binding: LabelBinding, fetch: F) -> Self {
98        LandingSource { store, binding, fetch }
99    }
100}
101
102impl<F> Source for LandingSource<F>
103where
104    F: FnMut(&[Row]) -> Result<Vec<Row>, String>,
105{
106    fn snapshots(&mut self) -> Result<Vec<i64>, String> {
107        let prior = self.store.last_raw()?;
108        let rows = (self.fetch)(&prior)?;
109        self.store.land(&rows)?;
110        Ok(self.store.list())
111    }
112
113    fn read_snapshot(&mut self, snapshot_id: i64) -> Result<Snapshot, String> {
114        let raw = self.store.read_raw(snapshot_id)?;
115        let mut out: Snapshot = Vec::with_capacity(raw.len());
116        for row in raw {
117            let rid = row
118                .iter()
119                .find(|(k, _)| k == "rid")
120                .and_then(|(_, v)| match v {
121                    fv_value::Value::Str(s) => Some(s.clone()),
122                    _ => None,
123                })
124                .ok_or("curated row has no string rid")?;
125            out.push((rid, normalize_row(&row, &self.binding)));
126        }
127        Ok(out)
128    }
129}
130
131/// A Source backed by pre-materialized snapshots (`{id → normalized rows}`) plus the full list of
132/// available snapshot ids. For sources whose read is async and done up-front — e.g. an Iceberg table
133/// whose snapshots are scanned to Arrow in the worker before the (sync) scan loop runs. An id not in
134/// the map reads as an empty snapshot (a lenient baseline, e.g. an expired prior snapshot).
135pub struct MemorySource {
136    all_ids: Vec<i64>,
137    snapshots: std::collections::BTreeMap<i64, Snapshot>,
138}
139
140impl MemorySource {
141    pub fn new(all_ids: Vec<i64>, snapshots: std::collections::BTreeMap<i64, Snapshot>) -> Self {
142        MemorySource { all_ids, snapshots }
143    }
144}
145
146impl Source for MemorySource {
147    fn snapshots(&mut self) -> Result<Vec<i64>, String> {
148        Ok(self.all_ids.clone())
149    }
150    fn read_snapshot(&mut self, snapshot_id: i64) -> Result<Snapshot, String> {
151        Ok(self.snapshots.get(&snapshot_id).cloned().unwrap_or_default())
152    }
153}
154
155/// An in-memory sink that collects emitted changes — for tests and the file replay path.
156#[derive(Default)]
157pub struct CollectSink {
158    pub changes: Vec<Change>,
159}
160
161impl Sink for CollectSink {
162    fn emit(&mut self, _rid: &str, change: &Change) {
163        self.changes.push(change.clone());
164    }
165    fn flush(&mut self) -> Result<(), String> {
166        Ok(())
167    }
168}
169
170#[cfg(test)]
171mod tests {
172    use super::*;
173    use fv_value::Value;
174
175    /// An in-memory Source: `{snapshot_id: [(rid, row)]}` — mirrors the Python `DictSource` test double.
176    struct DictSource {
177        snaps: std::collections::BTreeMap<i64, Snapshot>,
178    }
179
180    impl Source for DictSource {
181        fn snapshots(&mut self) -> Result<Vec<i64>, String> {
182            Ok(self.snaps.keys().copied().collect())
183        }
184        fn read_snapshot(&mut self, id: i64) -> Result<Snapshot, String> {
185            Ok(self.snaps.get(&id).cloned().unwrap_or_default())
186        }
187    }
188
189    fn r(rid: &str, status: &str) -> (String, Row) {
190        (
191            rid.to_string(),
192            vec![
193                ("rid".into(), Value::Str(rid.into())),
194                ("status".into(), Value::Str(status.into())),
195            ],
196        )
197    }
198
199    #[test]
200    fn first_scan_inserts_all_then_incremental_resumes() {
201        let mut src = DictSource {
202            snaps: [
203                (1, vec![r("A-1", "IN_SERVICE"), r("A-2", "GROUNDED")]),
204                (2, vec![r("A-1", "GROUNDED"), r("A-2", "GROUNDED")]),
205            ]
206            .into_iter()
207            .collect(),
208        };
209        let mut sink = CollectSink::default();
210        let cp = scan_source(&mut src, "Aircraft", None, &mut sink, 1, None, None, true, None).unwrap();
211        assert_eq!(cp, Some(2));
212        let seen: Vec<(&str, &Value)> = sink
213            .changes
214            .iter()
215            .map(|c| {
216                (
217                    c.rid.as_str(),
218                    c.properties
219                        .iter()
220                        .find(|(k, _)| k == "status")
221                        .map(|(_, v)| v)
222                        .unwrap(),
223                )
224            })
225            .collect();
226        assert_eq!(
227            seen,
228            vec![
229                ("A-1", &Value::Str("IN_SERVICE".into())),
230                ("A-2", &Value::Str("GROUNDED".into())),
231                ("A-1", &Value::Str("GROUNDED".into())),
232            ]
233        );
234
235        // Resume from the checkpoint: nothing new → nothing emitted, checkpoint unchanged.
236        let mut sink2 = CollectSink::default();
237        assert_eq!(
238            scan_source(&mut src, "Aircraft", cp, &mut sink2, 2, None, None, true, None).unwrap(),
239            Some(2)
240        );
241        assert!(sink2.changes.is_empty());
242    }
243
244    #[test]
245    fn memory_source_serves_prefetched_snapshots_and_diffs_them() {
246        // Pre-materialized snapshots (as an Iceberg prefetch would build): snapshot 5 then 7.
247        let snaps: std::collections::BTreeMap<i64, Snapshot> =
248            [(5, vec![r("A-1", "IN_SERVICE")]), (7, vec![r("A-1", "GROUNDED")])]
249                .into_iter()
250                .collect();
251        let mut src = MemorySource::new(vec![5, 7], snaps);
252        let mut sink = CollectSink::default();
253        // From checkpoint 5: only snapshot 7 is new → one UPDATE change; checkpoint advances to 7.
254        let cp = scan_source(&mut src, "Aircraft", Some(5), &mut sink, 1, None, None, true, None).unwrap();
255        assert_eq!(cp, Some(7));
256        assert_eq!(sink.changes.len(), 1);
257        assert_eq!(
258            sink.changes[0]
259                .properties
260                .iter()
261                .find(|(k, _)| k == "status")
262                .map(|(_, v)| v),
263            Some(&Value::Str("GROUNDED".into()))
264        );
265    }
266
267    #[test]
268    fn up_to_caps_the_tick() {
269        let mut src = DictSource {
270            snaps: [(1, vec![r("A-1", "a")]), (2, vec![r("A-1", "b")])]
271                .into_iter()
272                .collect(),
273        };
274        let mut sink = CollectSink::default();
275        assert_eq!(
276            scan_source(&mut src, "T", None, &mut sink, 1, Some(1), None, true, None).unwrap(),
277            Some(1)
278        );
279        assert_eq!(sink.changes.len(), 1);
280        assert_eq!(
281            sink.changes[0]
282                .properties
283                .iter()
284                .find(|(k, _)| k == "status")
285                .map(|(_, v)| v),
286            Some(&Value::Str("a".into()))
287        );
288    }
289
290    #[test]
291    fn landing_source_lands_polls_and_normalizes_on_read() {
292        let tmp = std::env::temp_dir().join(format!("fv-scan-{}", std::process::id()));
293        let _ = std::fs::remove_dir_all(&tmp);
294        let store = SnapshotStore::at(&tmp);
295        let binding = LabelBinding {
296            label_columns: [("classification".to_string(), "classification_level".to_string())]
297                .into_iter()
298                .collect(),
299            on_missing_label: "DENY".into(),
300            primary: true,
301            ..Default::default()
302        };
303
304        // A poll returns a raw row carrying the label COLUMN; the scan must emit it folded into _labels.
305        let fetch = |_prior: &[Row]| {
306            Ok(vec![vec![
307                ("rid".to_string(), Value::Str("A-1".into())),
308                ("status".to_string(), Value::Str("IN_SERVICE".into())),
309                ("classification_level".to_string(), Value::Num(10.0)),
310            ]])
311        };
312        let mut source = LandingSource::new(store, binding, fetch);
313        let mut sink = CollectSink::default();
314        let cp = scan_source(&mut source, "Aircraft", None, &mut sink, 100, None, None, true, None).unwrap();
315        assert_eq!(cp, Some(1));
316        assert_eq!(sink.changes.len(), 1);
317        let change = &sink.changes[0];
318        // raw label column is gone from properties; folded into the change's labels.
319        assert!(change.properties.iter().all(|(k, _)| k != "classification_level"));
320        assert_eq!(
321            change.labels,
322            Some(Value::Obj(
323                [("classification".to_string(), Value::Num(10.0))].into_iter().collect()
324            ))
325        );
326        let _ = std::fs::remove_dir_all(&tmp);
327    }
328}