use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::Arc;
use tokio::sync::Notify;
use crate::pipeline::{PublishHook, Snapshot, SnapshotHandle};
use super::FleetStore;
pub type SnapshotSource = Arc<dyn Fn() -> Arc<Snapshot> + Send + Sync>;
pub struct FleetSink {
store: FleetStore,
snapshots: SnapshotSource,
doorbell: Arc<Notify>,
skipped: Arc<AtomicU64>,
stored: Arc<AtomicU64>,
last_stored: AtomicU64,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct SinkStats {
pub stored: u64,
pub skipped: u64,
}
impl FleetSink {
pub fn from_handle(store: FleetStore, handle: SnapshotHandle) -> Self {
Self::new(store, Arc::new(move || handle.latest()))
}
pub fn new(store: FleetStore, snapshots: SnapshotSource) -> Self {
Self {
store,
snapshots,
doorbell: Arc::new(Notify::new()),
skipped: Arc::new(AtomicU64::new(0)),
stored: Arc::new(AtomicU64::new(0)),
last_stored: AtomicU64::new(0),
}
}
pub fn publish_hook(&self) -> PublishHook {
let doorbell = Arc::clone(&self.doorbell);
Arc::new(move || doorbell.notify_one())
}
pub fn stats(&self) -> SinkStats {
SinkStats {
stored: self.stored.load(Ordering::Relaxed),
skipped: self.skipped.load(Ordering::Relaxed),
}
}
pub async fn append_latest(&self) -> Result<usize, iceberg::Error> {
let snapshot = (self.snapshots)();
let generation = snapshot.generation;
if generation != 0 && generation <= self.last_stored.load(Ordering::Relaxed) {
return Ok(0);
}
let rows = self.store.append(&snapshot).await?;
if rows > 0 {
let previous = self.last_stored.swap(generation, Ordering::Relaxed);
if previous != 0 && generation > previous + 1 {
self.skipped
.fetch_add(generation - previous - 1, Ordering::Relaxed);
}
self.stored.fetch_add(1, Ordering::Relaxed);
}
Ok(rows)
}
pub async fn run(
&self,
mut stop: tokio::sync::oneshot::Receiver<()>,
) -> Result<SinkStats, iceberg::Error> {
loop {
tokio::select! {
_ = self.doorbell.notified() => {
self.append_latest().await?;
}
_ = &mut stop => return Ok(self.stats()),
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::fleet_store::store::test_support::in_memory_store;
use arc_swap::ArcSwap;
fn source(initial: Snapshot) -> (SnapshotSource, Arc<ArcSwap<Snapshot>>) {
let slot = Arc::new(ArcSwap::from(Arc::new(initial)));
let reader = Arc::clone(&slot);
(Arc::new(move || reader.load_full()), slot)
}
#[tokio::test]
async fn the_publish_hook_only_signals() {
let (_warehouse, store) = in_memory_store("hook").await;
let (src, _slot) = source(Snapshot::default());
let sink = FleetSink::new(store, src);
let hook = sink.publish_hook();
hook();
hook();
assert_eq!(
sink.stats(),
SinkStats {
stored: 0,
skipped: 0
},
"ringing the bell must not itself write anything"
);
}
#[tokio::test]
async fn generations_missed_while_appending_are_counted() {
let (_warehouse, store) = in_memory_store("gap").await;
let (src, slot) = source(Snapshot {
generation: 1,
collected_at: Some(1_700_000_000),
..Default::default()
});
let sink = FleetSink::new(store, src);
sink.append_latest().await.expect("first");
assert_eq!(sink.stats().stored, 1);
assert_eq!(sink.stats().skipped, 0);
slot.store(Arc::new(Snapshot {
generation: 4,
collected_at: Some(1_700_000_003),
..Default::default()
}));
sink.append_latest().await.expect("second");
assert_eq!(sink.stats().stored, 2);
assert_eq!(
sink.stats().skipped,
2,
"generations 2 and 3 never reached the table, and the table cannot say so itself"
);
}
#[tokio::test]
async fn the_same_generation_is_not_stored_twice() {
let (_warehouse, store) = in_memory_store("dup").await;
let (src, _slot) = source(Snapshot {
generation: 9,
collected_at: Some(1_700_000_000),
..Default::default()
});
let sink = FleetSink::new(store, src);
assert_eq!(sink.append_latest().await.expect("first"), 1);
assert_eq!(
sink.append_latest().await.expect("second"),
0,
"a re-notified tick must not duplicate a row"
);
assert_eq!(sink.stats().stored, 1);
assert_eq!(
sink.stats().skipped,
0,
"nothing was lost, so nothing may be reported as lost"
);
}
}