slatedb 0.14.0

A cloud native embedded storage engine built on object storage.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115

use bytes::Bytes;
use parking_lot::RwLockWriteGuard;

use crate::db::DbInner;
use crate::db_state::DbState;
use crate::error::SlateDBError;
use crate::oracle::Oracle;
use crate::prefix_extractor::{PrefixExtractor, PrefixTarget};
use crate::wal_replay::ReplayedMemtable;

/// Extract the segment prefix (RFC-0024) from `key` under `extractor`.
///
/// Returns the prefix bytes — a cheap refcount slice of `key`. An
/// absent or zero-length prefix is a hard error
/// ([`SlateDBError::EmptySegmentPrefix`]): every key in a segmented DB
/// must map to a non-empty segment. This is the single point where a
/// key's segment prefix is derived; the write path, WAL replay, and
/// checkpoint filtering all route through it so they agree on both the
/// extraction and the empty-prefix error.
pub(crate) fn extract_segment_prefix(
    extractor: &dyn PrefixExtractor,
    key: &Bytes,
) -> Result<Bytes, SlateDBError> {
    match extractor.prefix_len(&PrefixTarget::Point(key.clone())) {
        Some(0) | None => Err(SlateDBError::EmptySegmentPrefix { key: key.clone() }),
        Some(n) => Ok(key.slice(0..n)),
    }
}

impl DbInner {
    pub(crate) fn maybe_freeze_current_memtable(&self) -> Result<(), SlateDBError> {
        let wal_id = self.wal_buffer.recent_flushed_wal_id();
        let mut guard = self.state.write();
        let meta = guard.memtable().metadata();

        let last_freeze_wal_id = guard
            .state()
            .imm_memtable
            .front()
            .map(|imm| imm.recent_flushed_wal_id())
            .unwrap_or(guard.state().core().replay_after_wal_id);

        let l0_sst_size_est = self
            .table_store
            .estimate_encoded_size_compacted(meta.entry_num, meta.entries_size_in_bytes);

        let wal_id_gap = wal_id
            .checked_sub(last_freeze_wal_id)
            .ok_or_else(|| SlateDBError::InvalidDBState)?;

        if wal_id_gap < self.settings.max_wal_flushes_before_l0_flush
            && l0_sst_size_est < self.settings.l0_sst_size_bytes
        {
            Ok(())
        } else {
            self.freeze_memtable(&mut guard, wal_id)
        }
    }

    pub(crate) fn freeze_current_memtable(&self) -> Result<(), SlateDBError> {
        let wal_id = self.wal_buffer.recent_flushed_wal_id();
        let mut guard = self.state.write();
        self.freeze_memtable(&mut guard, wal_id)
    }

    pub(crate) fn freeze_memtable(
        &self,
        guard: &mut RwLockWriteGuard<'_, DbState>,
        wal_id: u64,
    ) -> Result<(), SlateDBError> {
        if guard.memtable().is_empty() {
            return Ok(());
        }

        guard.freeze_memtable(wal_id);
        let _ = self.memtable_flusher().notify_memtable_frozen();
        Ok(())
    }

    pub(crate) fn replay_memtable(
        &self,
        replayed_memtable: ReplayedMemtable,
    ) -> Result<(), SlateDBError> {
        let current_memtable_wal_id = self.wal_buffer.recent_flushed_wal_id();
        let mut guard = self.state.write();

        // The active memtable was installed by the previous replay step, so its
        // durable WAL boundary is the WAL buffer's current boundary. Stamp the
        // frozen table with that boundary before advancing the buffer for the new
        // replayed memtable.
        self.freeze_memtable(&mut guard, current_memtable_wal_id)?;

        let last_wal = replayed_memtable.last_wal_id;
        guard.modify(|modifier| modifier.state.manifest.value.core.next_wal_sst_id = last_wal + 1);

        // update seqs and clock
        // we know these won't move backwards (even though the replayed wal files might contain some
        // older rows) because the wal replay iterator ignores any entries with seq num lower than
        // l0_last_seq from the manifest
        assert!(self.oracle.last_seq() <= replayed_memtable.last_seq);
        self.oracle.advance_last_seq(replayed_memtable.last_seq);
        assert!(self.oracle.last_committed_seq() <= replayed_memtable.last_seq);
        self.oracle
            .advance_committed_seq(replayed_memtable.last_seq);
        self.mono_clock.set_last_tick(replayed_memtable.last_tick)?;

        // replace the memtable
        guard.replace_memtable(replayed_memtable.table);
        self.wal_buffer.advance_recent_flushed_wal_id(last_wal);
        let dirty_manifest = guard.state().manifest.clone();
        drop(guard);
        self.status_manager.report_manifest(dirty_manifest.into());
        Ok(())
    }
}