1use crate::changelog::{diff_snapshots, Change, Snapshot};
11use crate::labels::{normalize_row, LabelBinding};
12use crate::lake::SnapshotStore;
13use crate::Row;
14
15pub trait Source {
17 fn snapshots(&mut self) -> Result<Vec<i64>, String>;
19 fn read_snapshot(&mut self, snapshot_id: i64) -> Result<Snapshot, String>;
21}
22
23pub trait Sink {
25 fn emit(&mut self, rid: &str, change: &Change);
26 fn flush(&mut self) -> Result<(), String>;
27}
28
29#[allow(clippy::too_many_arguments)]
37pub fn scan_source<S: Source + ?Sized>(
38 source: &mut S,
39 object_type: &str,
40 last_snapshot: Option<i64>,
41 sink: &mut dyn Sink,
42 now_ms: i64,
43 up_to: Option<i64>,
44 datasource: Option<&str>,
45 primary: bool,
46 event_time_column: Option<&str>,
48) -> Result<Option<i64>, String> {
49 let snaps = source.snapshots()?;
50 let new: Vec<i64> = snaps
51 .into_iter()
52 .filter(|s| last_snapshot.is_none_or(|l| *s > l) && up_to.is_none_or(|u| *s <= u))
53 .collect();
54 if new.is_empty() {
55 return Ok(last_snapshot);
56 }
57
58 let mut prev: Snapshot = match last_snapshot {
59 Some(l) => source.read_snapshot(l)?,
60 None => Vec::new(),
61 };
62 let mut processed = last_snapshot;
63 for sid in new {
64 let curr = source.read_snapshot(sid)?;
65 for change in diff_snapshots(
66 &prev,
67 &curr,
68 object_type,
69 sid,
70 now_ms,
71 datasource,
72 primary,
73 event_time_column,
74 ) {
75 sink.emit(&change.rid, &change);
76 }
77 prev = curr;
78 processed = Some(sid);
79 }
80 Ok(processed)
81}
82
83pub struct LandingSource<F> {
88 store: SnapshotStore,
89 binding: LabelBinding,
90 fetch: F,
91}
92
93impl<F> LandingSource<F>
94where
95 F: FnMut(&[Row]) -> Result<Vec<Row>, String>,
96{
97 pub fn new(store: SnapshotStore, binding: LabelBinding, fetch: F) -> Self {
98 LandingSource { store, binding, fetch }
99 }
100}
101
102impl<F> Source for LandingSource<F>
103where
104 F: FnMut(&[Row]) -> Result<Vec<Row>, String>,
105{
106 fn snapshots(&mut self) -> Result<Vec<i64>, String> {
107 let prior = self.store.last_raw()?;
108 let rows = (self.fetch)(&prior)?;
109 self.store.land(&rows)?;
110 Ok(self.store.list())
111 }
112
113 fn read_snapshot(&mut self, snapshot_id: i64) -> Result<Snapshot, String> {
114 let raw = self.store.read_raw(snapshot_id)?;
115 let mut out: Snapshot = Vec::with_capacity(raw.len());
116 for row in raw {
117 let rid = row
118 .iter()
119 .find(|(k, _)| k == "rid")
120 .and_then(|(_, v)| match v {
121 fv_value::Value::Str(s) => Some(s.clone()),
122 _ => None,
123 })
124 .ok_or("curated row has no string rid")?;
125 out.push((rid, normalize_row(&row, &self.binding)));
126 }
127 Ok(out)
128 }
129}
130
131pub struct MemorySource {
136 all_ids: Vec<i64>,
137 snapshots: std::collections::BTreeMap<i64, Snapshot>,
138}
139
140impl MemorySource {
141 pub fn new(all_ids: Vec<i64>, snapshots: std::collections::BTreeMap<i64, Snapshot>) -> Self {
142 MemorySource { all_ids, snapshots }
143 }
144}
145
146impl Source for MemorySource {
147 fn snapshots(&mut self) -> Result<Vec<i64>, String> {
148 Ok(self.all_ids.clone())
149 }
150 fn read_snapshot(&mut self, snapshot_id: i64) -> Result<Snapshot, String> {
151 Ok(self.snapshots.get(&snapshot_id).cloned().unwrap_or_default())
152 }
153}
154
155#[derive(Default)]
157pub struct CollectSink {
158 pub changes: Vec<Change>,
159}
160
161impl Sink for CollectSink {
162 fn emit(&mut self, _rid: &str, change: &Change) {
163 self.changes.push(change.clone());
164 }
165 fn flush(&mut self) -> Result<(), String> {
166 Ok(())
167 }
168}
169
170#[cfg(test)]
171mod tests {
172 use super::*;
173 use fv_value::Value;
174
175 struct DictSource {
177 snaps: std::collections::BTreeMap<i64, Snapshot>,
178 }
179
180 impl Source for DictSource {
181 fn snapshots(&mut self) -> Result<Vec<i64>, String> {
182 Ok(self.snaps.keys().copied().collect())
183 }
184 fn read_snapshot(&mut self, id: i64) -> Result<Snapshot, String> {
185 Ok(self.snaps.get(&id).cloned().unwrap_or_default())
186 }
187 }
188
189 fn r(rid: &str, status: &str) -> (String, Row) {
190 (
191 rid.to_string(),
192 vec![
193 ("rid".into(), Value::Str(rid.into())),
194 ("status".into(), Value::Str(status.into())),
195 ],
196 )
197 }
198
199 #[test]
200 fn first_scan_inserts_all_then_incremental_resumes() {
201 let mut src = DictSource {
202 snaps: [
203 (1, vec![r("A-1", "IN_SERVICE"), r("A-2", "GROUNDED")]),
204 (2, vec![r("A-1", "GROUNDED"), r("A-2", "GROUNDED")]),
205 ]
206 .into_iter()
207 .collect(),
208 };
209 let mut sink = CollectSink::default();
210 let cp = scan_source(&mut src, "Aircraft", None, &mut sink, 1, None, None, true, None).unwrap();
211 assert_eq!(cp, Some(2));
212 let seen: Vec<(&str, &Value)> = sink
213 .changes
214 .iter()
215 .map(|c| {
216 (
217 c.rid.as_str(),
218 c.properties
219 .iter()
220 .find(|(k, _)| k == "status")
221 .map(|(_, v)| v)
222 .unwrap(),
223 )
224 })
225 .collect();
226 assert_eq!(
227 seen,
228 vec![
229 ("A-1", &Value::Str("IN_SERVICE".into())),
230 ("A-2", &Value::Str("GROUNDED".into())),
231 ("A-1", &Value::Str("GROUNDED".into())),
232 ]
233 );
234
235 let mut sink2 = CollectSink::default();
237 assert_eq!(
238 scan_source(&mut src, "Aircraft", cp, &mut sink2, 2, None, None, true, None).unwrap(),
239 Some(2)
240 );
241 assert!(sink2.changes.is_empty());
242 }
243
244 #[test]
245 fn memory_source_serves_prefetched_snapshots_and_diffs_them() {
246 let snaps: std::collections::BTreeMap<i64, Snapshot> =
248 [(5, vec![r("A-1", "IN_SERVICE")]), (7, vec![r("A-1", "GROUNDED")])]
249 .into_iter()
250 .collect();
251 let mut src = MemorySource::new(vec![5, 7], snaps);
252 let mut sink = CollectSink::default();
253 let cp = scan_source(&mut src, "Aircraft", Some(5), &mut sink, 1, None, None, true, None).unwrap();
255 assert_eq!(cp, Some(7));
256 assert_eq!(sink.changes.len(), 1);
257 assert_eq!(
258 sink.changes[0]
259 .properties
260 .iter()
261 .find(|(k, _)| k == "status")
262 .map(|(_, v)| v),
263 Some(&Value::Str("GROUNDED".into()))
264 );
265 }
266
267 #[test]
268 fn up_to_caps_the_tick() {
269 let mut src = DictSource {
270 snaps: [(1, vec![r("A-1", "a")]), (2, vec![r("A-1", "b")])]
271 .into_iter()
272 .collect(),
273 };
274 let mut sink = CollectSink::default();
275 assert_eq!(
276 scan_source(&mut src, "T", None, &mut sink, 1, Some(1), None, true, None).unwrap(),
277 Some(1)
278 );
279 assert_eq!(sink.changes.len(), 1);
280 assert_eq!(
281 sink.changes[0]
282 .properties
283 .iter()
284 .find(|(k, _)| k == "status")
285 .map(|(_, v)| v),
286 Some(&Value::Str("a".into()))
287 );
288 }
289
290 #[test]
291 fn landing_source_lands_polls_and_normalizes_on_read() {
292 let tmp = std::env::temp_dir().join(format!("fv-scan-{}", std::process::id()));
293 let _ = std::fs::remove_dir_all(&tmp);
294 let store = SnapshotStore::at(&tmp);
295 let binding = LabelBinding {
296 label_columns: [("classification".to_string(), "classification_level".to_string())]
297 .into_iter()
298 .collect(),
299 on_missing_label: "DENY".into(),
300 primary: true,
301 ..Default::default()
302 };
303
304 let fetch = |_prior: &[Row]| {
306 Ok(vec![vec![
307 ("rid".to_string(), Value::Str("A-1".into())),
308 ("status".to_string(), Value::Str("IN_SERVICE".into())),
309 ("classification_level".to_string(), Value::Num(10.0)),
310 ]])
311 };
312 let mut source = LandingSource::new(store, binding, fetch);
313 let mut sink = CollectSink::default();
314 let cp = scan_source(&mut source, "Aircraft", None, &mut sink, 100, None, None, true, None).unwrap();
315 assert_eq!(cp, Some(1));
316 assert_eq!(sink.changes.len(), 1);
317 let change = &sink.changes[0];
318 assert!(change.properties.iter().all(|(k, _)| k != "classification_level"));
320 assert_eq!(
321 change.labels,
322 Some(Value::Obj(
323 [("classification".to_string(), Value::Num(10.0))].into_iter().collect()
324 ))
325 );
326 let _ = std::fs::remove_dir_all(&tmp);
327 }
328}