timelake 0.7.0

The open time-series lakehouse: fast queries on your own object storage, with no database and no maintenance jobs
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51

//! Writes that finish after the call that started them returns. A commit is done once its
//! metadata is written; updating the version hint and the head after that are only shortcuts for
//! readers, so a commit doesn't wait for them (SPEC ยง19.1, P4).

use std::future::Future;
use std::sync::atomic::{AtomicUsize, Ordering};

use tokio::sync::Notify;

/// Background writes started and not yet finished or dropped.
static IN_FLIGHT: AtomicUsize = AtomicUsize::new(0);
static FINISHED: Notify = Notify::const_new();

/// Counts a task as in flight until it completes or is dropped with its runtime.
struct InFlight;

impl InFlight {
    fn start() -> Self {
        IN_FLIGHT.fetch_add(1, Ordering::SeqCst);
        Self
    }
}

impl Drop for InFlight {
    fn drop(&mut self) {
        IN_FLIGHT.fetch_sub(1, Ordering::SeqCst);
        FINISHED.notify_waiters();
    }
}

/// Runs `task` on the current Tokio runtime without waiting for it.
pub(crate) fn spawn(task: impl Future<Output = ()> + Send + 'static) {
    let in_flight = InFlight::start();
    tokio::spawn(async move {
        task.await;
        drop(in_flight);
    });
}

/// Waits for the background writes this process started. Call it before the process exits, or
/// the last commit's version hint and head may not be updated: readers still find the newest
/// version, but need extra requests to do so. The CLI and the Python package call it for you.
pub async fn flush() {
    loop {
        let finished = FINISHED.notified();
        if IN_FLIGHT.load(Ordering::SeqCst) == 0 {
            return;
        }
        finished.await;
    }
}