use crate::changelog::{diff_snapshots, Change, Snapshot};
use crate::labels::{normalize_row, LabelBinding};
use crate::lake::SnapshotStore;
use crate::Row;
pub trait Source {
fn snapshots(&mut self) -> Result<Vec<i64>, String>;
fn read_snapshot(&mut self, snapshot_id: i64) -> Result<Snapshot, String>;
}
pub trait Sink {
fn emit(&mut self, rid: &str, change: &Change);
fn flush(&mut self) -> Result<(), String>;
}
#[allow(clippy::too_many_arguments)]
pub fn scan_source<S: Source + ?Sized>(
source: &mut S,
object_type: &str,
last_snapshot: Option<i64>,
sink: &mut dyn Sink,
now_ms: i64,
up_to: Option<i64>,
datasource: Option<&str>,
primary: bool,
event_time_column: Option<&str>,
) -> Result<Option<i64>, String> {
let snaps = source.snapshots()?;
let new: Vec<i64> = snaps
.into_iter()
.filter(|s| last_snapshot.is_none_or(|l| *s > l) && up_to.is_none_or(|u| *s <= u))
.collect();
if new.is_empty() {
return Ok(last_snapshot);
}
let mut prev: Snapshot = match last_snapshot {
Some(l) => source.read_snapshot(l)?,
None => Vec::new(),
};
let mut processed = last_snapshot;
for sid in new {
let curr = source.read_snapshot(sid)?;
for change in diff_snapshots(
&prev,
&curr,
object_type,
sid,
now_ms,
datasource,
primary,
event_time_column,
) {
sink.emit(&change.rid, &change);
}
prev = curr;
processed = Some(sid);
}
Ok(processed)
}
pub struct LandingSource<F> {
store: SnapshotStore,
binding: LabelBinding,
fetch: F,
}
impl<F> LandingSource<F>
where
F: FnMut(&[Row]) -> Result<Vec<Row>, String>,
{
pub fn new(store: SnapshotStore, binding: LabelBinding, fetch: F) -> Self {
LandingSource { store, binding, fetch }
}
}
impl<F> Source for LandingSource<F>
where
F: FnMut(&[Row]) -> Result<Vec<Row>, String>,
{
fn snapshots(&mut self) -> Result<Vec<i64>, String> {
let prior = self.store.last_raw()?;
let rows = (self.fetch)(&prior)?;
self.store.land(&rows)?;
Ok(self.store.list())
}
fn read_snapshot(&mut self, snapshot_id: i64) -> Result<Snapshot, String> {
let raw = self.store.read_raw(snapshot_id)?;
let mut out: Snapshot = Vec::with_capacity(raw.len());
for row in raw {
let rid = row
.iter()
.find(|(k, _)| k == "rid")
.and_then(|(_, v)| match v {
fv_value::Value::Str(s) => Some(s.clone()),
_ => None,
})
.ok_or("curated row has no string rid")?;
out.push((rid, normalize_row(&row, &self.binding)));
}
Ok(out)
}
}
pub struct MemorySource {
all_ids: Vec<i64>,
snapshots: std::collections::BTreeMap<i64, Snapshot>,
}
impl MemorySource {
pub fn new(all_ids: Vec<i64>, snapshots: std::collections::BTreeMap<i64, Snapshot>) -> Self {
MemorySource { all_ids, snapshots }
}
}
impl Source for MemorySource {
fn snapshots(&mut self) -> Result<Vec<i64>, String> {
Ok(self.all_ids.clone())
}
fn read_snapshot(&mut self, snapshot_id: i64) -> Result<Snapshot, String> {
Ok(self.snapshots.get(&snapshot_id).cloned().unwrap_or_default())
}
}
#[derive(Default)]
pub struct CollectSink {
pub changes: Vec<Change>,
}
impl Sink for CollectSink {
fn emit(&mut self, _rid: &str, change: &Change) {
self.changes.push(change.clone());
}
fn flush(&mut self) -> Result<(), String> {
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
use fv_value::Value;
struct DictSource {
snaps: std::collections::BTreeMap<i64, Snapshot>,
}
impl Source for DictSource {
fn snapshots(&mut self) -> Result<Vec<i64>, String> {
Ok(self.snaps.keys().copied().collect())
}
fn read_snapshot(&mut self, id: i64) -> Result<Snapshot, String> {
Ok(self.snaps.get(&id).cloned().unwrap_or_default())
}
}
fn r(rid: &str, status: &str) -> (String, Row) {
(
rid.to_string(),
vec![
("rid".into(), Value::Str(rid.into())),
("status".into(), Value::Str(status.into())),
],
)
}
#[test]
fn first_scan_inserts_all_then_incremental_resumes() {
let mut src = DictSource {
snaps: [
(1, vec![r("A-1", "IN_SERVICE"), r("A-2", "GROUNDED")]),
(2, vec![r("A-1", "GROUNDED"), r("A-2", "GROUNDED")]),
]
.into_iter()
.collect(),
};
let mut sink = CollectSink::default();
let cp = scan_source(&mut src, "Aircraft", None, &mut sink, 1, None, None, true, None).unwrap();
assert_eq!(cp, Some(2));
let seen: Vec<(&str, &Value)> = sink
.changes
.iter()
.map(|c| {
(
c.rid.as_str(),
c.properties
.iter()
.find(|(k, _)| k == "status")
.map(|(_, v)| v)
.unwrap(),
)
})
.collect();
assert_eq!(
seen,
vec![
("A-1", &Value::Str("IN_SERVICE".into())),
("A-2", &Value::Str("GROUNDED".into())),
("A-1", &Value::Str("GROUNDED".into())),
]
);
let mut sink2 = CollectSink::default();
assert_eq!(
scan_source(&mut src, "Aircraft", cp, &mut sink2, 2, None, None, true, None).unwrap(),
Some(2)
);
assert!(sink2.changes.is_empty());
}
#[test]
fn memory_source_serves_prefetched_snapshots_and_diffs_them() {
let snaps: std::collections::BTreeMap<i64, Snapshot> =
[(5, vec![r("A-1", "IN_SERVICE")]), (7, vec![r("A-1", "GROUNDED")])]
.into_iter()
.collect();
let mut src = MemorySource::new(vec![5, 7], snaps);
let mut sink = CollectSink::default();
let cp = scan_source(&mut src, "Aircraft", Some(5), &mut sink, 1, None, None, true, None).unwrap();
assert_eq!(cp, Some(7));
assert_eq!(sink.changes.len(), 1);
assert_eq!(
sink.changes[0]
.properties
.iter()
.find(|(k, _)| k == "status")
.map(|(_, v)| v),
Some(&Value::Str("GROUNDED".into()))
);
}
#[test]
fn up_to_caps_the_tick() {
let mut src = DictSource {
snaps: [(1, vec![r("A-1", "a")]), (2, vec![r("A-1", "b")])]
.into_iter()
.collect(),
};
let mut sink = CollectSink::default();
assert_eq!(
scan_source(&mut src, "T", None, &mut sink, 1, Some(1), None, true, None).unwrap(),
Some(1)
);
assert_eq!(sink.changes.len(), 1);
assert_eq!(
sink.changes[0]
.properties
.iter()
.find(|(k, _)| k == "status")
.map(|(_, v)| v),
Some(&Value::Str("a".into()))
);
}
#[test]
fn landing_source_lands_polls_and_normalizes_on_read() {
let tmp = std::env::temp_dir().join(format!("fv-scan-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&tmp);
let store = SnapshotStore::at(&tmp);
let binding = LabelBinding {
label_columns: [("classification".to_string(), "classification_level".to_string())]
.into_iter()
.collect(),
on_missing_label: "DENY".into(),
primary: true,
..Default::default()
};
let fetch = |_prior: &[Row]| {
Ok(vec![vec![
("rid".to_string(), Value::Str("A-1".into())),
("status".to_string(), Value::Str("IN_SERVICE".into())),
("classification_level".to_string(), Value::Num(10.0)),
]])
};
let mut source = LandingSource::new(store, binding, fetch);
let mut sink = CollectSink::default();
let cp = scan_source(&mut source, "Aircraft", None, &mut sink, 100, None, None, true, None).unwrap();
assert_eq!(cp, Some(1));
assert_eq!(sink.changes.len(), 1);
let change = &sink.changes[0];
assert!(change.properties.iter().all(|(k, _)| k != "classification_level"));
assert_eq!(
change.labels,
Some(Value::Obj(
[("classification".to_string(), Value::Num(10.0))].into_iter().collect()
))
);
let _ = std::fs::remove_dir_all(&tmp);
}
}