differential_dataflow/columnar/
spill.rs

1//! Traits for paging chunks of merge-batcher state to and from backing storage.
2//!
3//! Modeled on timely's pager traits in
4//! `timely-dataflow/communication/src/allocator/zero_copy/spill.rs`
5//! (`SpillPolicy`, `BytesSpill`, `BytesFetch`), but parameterized over a chunk
6//! type `C` rather than fixed to `timely::bytes::arc::Bytes`. For the columnar
7//! batcher we expect `C = Updates<U>`; that wiring lives elsewhere — this file
8//! only defines the trait shapes.
9
10use std::collections::VecDeque;
11
12/// A queue entry: either an in-memory chunk or a handle that can fetch one
13/// (or several) from backing storage.
14pub enum Entry<C> {
15    /// In-memory chunk.
16    Typed(C),
17    /// Paged-out chunk(s); fetch via the handle.
18    Paged(Box<dyn Fetch<C>>),
19}
20
21/// Decides which queue entries to spill out and which to keep resident.
22///
23/// Invoked at well-defined moments by the holder of the queue (e.g., after
24/// pushing a new chunk). The implementation may rewrite entries in either
25/// direction: convert `Typed` to `Paged` (spill out) or `Paged` to `Typed`
26/// (fetch back).
27pub trait SpillPolicy<C> {
28    /// Optionally transform the queue.
29    fn apply(&mut self, queue: &mut VecDeque<Entry<C>>);
30}
31
32/// Move in-memory chunks to backing storage, returning fetch handles.
33///
34/// The implementation should drain from `chunks` and push to `handles` as it
35/// goes; on failure it may stop partway, leaving the lists in a consistent
36/// state that will be retried in the future. If it cannot leave the lists in
37/// a consistent state it should panic.
38pub trait Spill<C> {
39    /// Spill `chunks` to storage, producing one fetch handle per spilled group.
40    fn spill(&mut self, chunks: &mut Vec<C>, handles: &mut Vec<Box<dyn Fetch<C>>>);
41}
42
43/// Handle to spilled chunk(s). Consume to retrieve them from storage.
44pub trait Fetch<C> {
45    /// Consume the handle and return the spilled chunks.
46    ///
47    /// On failure, the handle is returned so the caller can retry later.
48    fn fetch(self: Box<Self>) -> Result<Vec<C>, Box<dyn Fetch<C>>>;
49}
differential_dataflow/columnar/spill.rs

differential_dataflow/columnar/
spill.rs
