use crate::types::SequenceNumber;
pub(crate) struct RangeTombstone {
pub begin: Vec<u8>,
pub end: Vec<u8>,
pub seq: SequenceNumber,
}
pub(crate) struct RangeTombstoneTracker {
tombstones: Vec<RangeTombstone>,
sorted: bool,
next_idx: usize,
active: Vec<usize>,
}
impl RangeTombstoneTracker {
pub fn new() -> Self {
Self {
tombstones: Vec::new(),
sorted: false,
next_idx: 0,
active: Vec::new(),
}
}
pub fn add(&mut self, begin: Vec<u8>, end: Vec<u8>, seq: SequenceNumber) {
self.tombstones.push(RangeTombstone { begin, end, seq });
self.sorted = false;
}
pub fn reset(&mut self) {
if !self.sorted {
if self.tombstones.len() > 1 {
self.tombstones.sort_by(|a, b| a.begin.cmp(&b.begin));
}
self.sorted = true;
}
self.next_idx = 0;
self.active.clear();
}
pub fn is_deleted(
&mut self,
user_key: &[u8],
seq: SequenceNumber,
snapshot: SequenceNumber,
) -> bool {
if self.tombstones.len() <= 4 {
return self.linear_check(user_key, seq, snapshot);
}
if !self.sorted {
self.reset();
}
while self.next_idx < self.tombstones.len() {
if self.tombstones[self.next_idx].begin.as_slice() <= user_key {
self.active.push(self.next_idx);
self.next_idx += 1;
} else {
break;
}
}
let tombstones = &self.tombstones;
self.active
.retain(|&idx| tombstones[idx].end.as_slice() > user_key);
for &idx in &self.active {
let rt = &self.tombstones[idx];
if rt.seq <= snapshot && rt.seq > seq {
return true;
}
}
false
}
fn linear_check(&self, user_key: &[u8], seq: SequenceNumber, snapshot: SequenceNumber) -> bool {
for rt in &self.tombstones {
if rt.seq <= snapshot
&& user_key >= rt.begin.as_slice()
&& user_key < rt.end.as_slice()
&& rt.seq > seq
{
return true;
}
}
false
}
pub fn is_empty(&self) -> bool {
self.tombstones.is_empty()
}
}
type Bounds = Vec<Vec<u8>>;
type Tree = Vec<Vec<(SequenceNumber, usize)>>;
fn tree_node_capacity(num_intervals: usize) -> usize {
if num_intervals == 0 {
0
} else {
4 * num_intervals
}
}
fn tree_insert(
tree: &mut Tree,
num_intervals: usize,
lo: usize,
hi: usize,
val: (SequenceNumber, usize),
) {
if lo >= hi {
return; }
tree_insert_rec(tree, 1, 0, num_intervals, lo, hi, val);
}
fn tree_insert_rec(
tree: &mut Tree,
node: usize,
node_lo: usize,
node_hi: usize,
lo: usize,
hi: usize,
val: (SequenceNumber, usize),
) {
if hi <= node_lo || node_hi <= lo {
return; }
if lo <= node_lo && node_hi <= hi {
tree[node].push(val); return;
}
let mid = node_lo + (node_hi - node_lo) / 2;
tree_insert_rec(tree, 2 * node, node_lo, mid, lo, hi, val);
tree_insert_rec(tree, 2 * node + 1, mid, node_hi, lo, hi, val);
}
fn tree_query(
tree: &Tree,
num_intervals: usize,
leaf: usize,
snapshot: SequenceNumber,
source_level: Option<usize>,
) -> SequenceNumber {
let mut node = 1usize;
let mut node_lo = 0usize;
let mut node_hi = num_intervals;
let mut best: SequenceNumber = 0;
loop {
for &(seq, level) in &tree[node] {
if seq > snapshot {
continue;
}
if let Some(src_lvl) = source_level
&& level > src_lvl
{
continue;
}
if seq > best {
best = seq;
}
break;
}
if node_hi - node_lo <= 1 {
break;
}
let mid = node_lo + (node_hi - node_lo) / 2;
if leaf < mid {
node_hi = mid;
node *= 2;
} else {
node_lo = mid;
node = 2 * node + 1;
}
}
best
}
pub(crate) struct FragmentedRangeTombstoneList {
raw: Vec<(Vec<u8>, Vec<u8>, SequenceNumber, usize)>,
bounds: Bounds,
tree: Tree,
}
impl FragmentedRangeTombstoneList {
pub fn empty() -> Self {
Self {
raw: Vec::new(),
bounds: Vec::new(),
tree: Vec::new(),
}
}
pub fn new(raw: Vec<(Vec<u8>, Vec<u8>, SequenceNumber)>) -> Self {
let with_levels: Vec<_> = raw.into_iter().map(|(b, e, s)| (b, e, s, 0usize)).collect();
Self::new_with_levels(with_levels)
}
pub fn new_with_levels(raw: Vec<(Vec<u8>, Vec<u8>, SequenceNumber, usize)>) -> Self {
if raw.is_empty() {
return Self::empty();
}
let mut bounds: Bounds = Vec::with_capacity(raw.len() * 2);
for (begin, end, _, _) in &raw {
bounds.push(begin.clone());
bounds.push(end.clone());
}
bounds.sort();
bounds.dedup();
let num_intervals = bounds.len() - 1;
let mut tree: Tree = vec![Vec::new(); tree_node_capacity(num_intervals)];
for (begin, end, seq, level) in &raw {
if begin >= end {
continue; }
let lo = bounds
.binary_search(begin)
.expect("begin was pushed into bounds above");
let hi = bounds
.binary_search(end)
.expect("end was pushed into bounds above");
tree_insert(&mut tree, num_intervals, lo, hi, (*seq, *level));
}
for node in &mut tree {
node.sort_unstable_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(&b.1)));
}
Self { raw, bounds, tree }
}
pub fn max_covering_tombstone_seq(
&self,
user_key: &[u8],
snapshot: SequenceNumber,
) -> SequenceNumber {
self.max_covering_tombstone_seq_for_level(user_key, snapshot, None)
}
pub fn max_covering_tombstone_seq_for_level(
&self,
user_key: &[u8],
snapshot: SequenceNumber,
source_level: Option<usize>,
) -> SequenceNumber {
let num_intervals = match self.bounds.len().checked_sub(1) {
Some(0) | None => return 0,
Some(n) => n,
};
let idx = self.bounds[..num_intervals].partition_point(|b| b.as_slice() <= user_key);
if idx == 0 {
return 0;
}
let leaf = idx - 1;
if user_key >= self.bounds[leaf + 1].as_slice() {
return 0;
}
tree_query(&self.tree, num_intervals, leaf, snapshot, source_level)
}
pub fn is_empty(&self) -> bool {
self.raw.is_empty()
}
pub fn tombstones(&self) -> Vec<(Vec<u8>, Vec<u8>, SequenceNumber)> {
self.raw
.iter()
.map(|(b, e, s, _)| (b.clone(), e.clone(), *s))
.collect()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_empty_tracker() {
let mut tracker = RangeTombstoneTracker::new();
assert!(!tracker.is_deleted(b"key", 1, 10));
}
#[test]
fn test_single_tombstone() {
let mut tracker = RangeTombstoneTracker::new();
tracker.add(b"aaa".to_vec(), b"zzz".to_vec(), 5);
tracker.reset();
assert!(tracker.is_deleted(b"bbb", 3, 10));
assert!(!tracker.is_deleted(b"bbb", 6, 10)); assert!(!tracker.is_deleted(b"000", 3, 10)); }
#[test]
fn test_same_seq_not_deleted() {
let mut tracker = RangeTombstoneTracker::new();
tracker.add(b"a".to_vec(), b"z".to_vec(), 5);
tracker.reset();
assert!(!tracker.is_deleted(b"m", 5, 10));
assert!(tracker.is_deleted(b"m", 4, 10));
}
#[test]
fn test_forward_sweep() {
let mut tracker = RangeTombstoneTracker::new();
tracker.add(b"b".to_vec(), b"d".to_vec(), 5);
tracker.add(b"f".to_vec(), b"h".to_vec(), 5);
tracker.reset();
assert!(!tracker.is_deleted(b"a", 1, 10));
assert!(tracker.is_deleted(b"b", 1, 10));
assert!(tracker.is_deleted(b"c", 1, 10));
assert!(!tracker.is_deleted(b"d", 1, 10));
assert!(!tracker.is_deleted(b"e", 1, 10));
assert!(tracker.is_deleted(b"f", 1, 10));
assert!(tracker.is_deleted(b"g", 1, 10));
assert!(!tracker.is_deleted(b"h", 1, 10));
}
#[test]
fn test_many_tombstones() {
let mut tracker = RangeTombstoneTracker::new();
for i in 0..100u32 {
let begin = format!("key_{:04}", i * 2);
let end = format!("key_{:04}", i * 2 + 1);
tracker.add(begin.into_bytes(), end.into_bytes(), 5);
}
tracker.reset();
for i in 0..200u32 {
let key = format!("key_{:04}", i);
let deleted = tracker.is_deleted(key.as_bytes(), 1, 10);
if i % 2 == 0 && i < 200 {
assert!(deleted, "key_{:04} should be deleted", i);
}
}
}
#[test]
fn test_fragmented_empty() {
let list = FragmentedRangeTombstoneList::empty();
assert!(list.is_empty());
assert_eq!(list.max_covering_tombstone_seq(b"any", 100), 0);
let list2 = FragmentedRangeTombstoneList::new(vec![]);
assert!(list2.is_empty());
}
#[test]
fn test_fragmented_single_tombstone() {
let list = FragmentedRangeTombstoneList::new(vec![(b"a".to_vec(), b"z".to_vec(), 5)]);
assert!(!list.is_empty());
assert_eq!(list.max_covering_tombstone_seq(b"a", 10), 5);
assert_eq!(list.max_covering_tombstone_seq(b"m", 10), 5);
assert_eq!(list.max_covering_tombstone_seq(b"y", 10), 5);
assert_eq!(list.max_covering_tombstone_seq(b"z", 10), 0); assert_eq!(list.max_covering_tombstone_seq(b"\0", 10), 0);
assert_eq!(list.max_covering_tombstone_seq(b"m", 3), 0);
assert_eq!(list.max_covering_tombstone_seq(b"m", 5), 5);
}
#[test]
fn test_fragmented_same_seq_not_deleted() {
let list = FragmentedRangeTombstoneList::new(vec![(b"a".to_vec(), b"z".to_vec(), 5)]);
let max_seq = list.max_covering_tombstone_seq(b"m", 10);
assert_eq!(max_seq, 5);
assert!(max_seq <= 5); assert!(max_seq > 4); }
#[test]
fn test_fragmented_overlapping() {
let list = FragmentedRangeTombstoneList::new(vec![
(b"a".to_vec(), b"m".to_vec(), 5),
(b"f".to_vec(), b"z".to_vec(), 8),
]);
assert_eq!(list.max_covering_tombstone_seq(b"a", 10), 5);
assert_eq!(list.max_covering_tombstone_seq(b"c", 10), 5);
assert_eq!(list.max_covering_tombstone_seq(b"e", 10), 5);
assert_eq!(list.max_covering_tombstone_seq(b"f", 10), 8);
assert_eq!(list.max_covering_tombstone_seq(b"h", 10), 8);
assert_eq!(list.max_covering_tombstone_seq(b"l", 10), 8);
assert_eq!(list.max_covering_tombstone_seq(b"m", 10), 8);
assert_eq!(list.max_covering_tombstone_seq(b"p", 10), 8);
assert_eq!(list.max_covering_tombstone_seq(b"y", 10), 8);
assert_eq!(list.max_covering_tombstone_seq(b"z", 10), 0);
assert_eq!(list.max_covering_tombstone_seq(b"h", 6), 5); assert_eq!(list.max_covering_tombstone_seq(b"p", 6), 0); }
#[test]
fn test_fragmented_nested() {
let list = FragmentedRangeTombstoneList::new(vec![
(b"a".to_vec(), b"z".to_vec(), 5),
(b"d".to_vec(), b"f".to_vec(), 8),
]);
assert_eq!(list.max_covering_tombstone_seq(b"b", 10), 5);
assert_eq!(list.max_covering_tombstone_seq(b"d", 10), 8); assert_eq!(list.max_covering_tombstone_seq(b"e", 10), 8);
assert_eq!(list.max_covering_tombstone_seq(b"f", 10), 5); assert_eq!(list.max_covering_tombstone_seq(b"x", 10), 5);
}
#[test]
fn test_fragmented_adjacent() {
let list = FragmentedRangeTombstoneList::new(vec![
(b"a".to_vec(), b"c".to_vec(), 5),
(b"c".to_vec(), b"f".to_vec(), 8),
]);
assert_eq!(list.max_covering_tombstone_seq(b"a", 10), 5);
assert_eq!(list.max_covering_tombstone_seq(b"b", 10), 5);
assert_eq!(list.max_covering_tombstone_seq(b"c", 10), 8); assert_eq!(list.max_covering_tombstone_seq(b"d", 10), 8);
assert_eq!(list.max_covering_tombstone_seq(b"f", 10), 0); }
#[test]
fn test_fragmented_many_tombstones() {
let raw: Vec<_> = (0..100u32)
.map(|i| {
let begin = format!("key_{:04}", i * 2);
let end = format!("key_{:04}", i * 2 + 1);
(begin.into_bytes(), end.into_bytes(), 5u64)
})
.collect();
let list = FragmentedRangeTombstoneList::new(raw);
for i in 0..200u32 {
let key = format!("key_{:04}", i);
let max_seq = list.max_covering_tombstone_seq(key.as_bytes(), 10);
if i % 2 == 0 {
assert_eq!(max_seq, 5, "key_{:04} should be covered", i);
} else {
assert_eq!(max_seq, 0, "key_{:04} should NOT be covered", i);
}
}
}
#[test]
fn test_fragmented_duplicate_seqs() {
let list = FragmentedRangeTombstoneList::new(vec![
(b"a".to_vec(), b"d".to_vec(), 5),
(b"c".to_vec(), b"f".to_vec(), 5),
]);
assert_eq!(list.max_covering_tombstone_seq(b"b", 10), 5);
assert_eq!(list.max_covering_tombstone_seq(b"c", 10), 5);
assert_eq!(list.max_covering_tombstone_seq(b"e", 10), 5);
}
#[test]
fn test_fragmented_nested_bounded_storage() {
let n: usize = 2000;
let raw: Vec<(Vec<u8>, Vec<u8>, SequenceNumber)> = (0..n)
.map(|i| {
let begin = format!("{i:06}").into_bytes();
let end = format!("{:06}", 2 * n - i).into_bytes();
(begin, end, (i + 1) as SequenceNumber)
})
.collect();
let list = FragmentedRangeTombstoneList::new(raw);
let total_entries: usize = list.tree.iter().map(|node| node.len()).sum();
assert!(
total_entries < n * 40,
"expected roughly O(n log n) (n*40 = {}) but stored {total_entries} \
entries for n={n} fully-nested tombstones",
n * 40
);
assert!(
total_entries < n * n / 4,
"storage did not avoid the O(n^2) blowup: {total_entries} entries for n={n}"
);
let center = format!("{n:06}").into_bytes();
assert_eq!(
list.max_covering_tombstone_seq(¢er, n as SequenceNumber),
n as SequenceNumber,
"innermost key must see the highest (innermost) seq"
);
assert_eq!(
list.max_covering_tombstone_seq(b"000000", n as SequenceNumber),
1,
"outermost key is covered only by the outermost (seq=1) tombstone"
);
assert_eq!(list.max_covering_tombstone_seq(¢er, 0), 0);
}
#[test]
fn test_fragmented_matches_brute_force_oracle() {
fn brute_force(
raw: &[(Vec<u8>, Vec<u8>, SequenceNumber, usize)],
key: &[u8],
snapshot: SequenceNumber,
source_level: Option<usize>,
) -> SequenceNumber {
let mut best = 0;
for (b, e, s, l) in raw {
let level_ok = source_level.is_none_or(|sl| *l <= sl);
if *s <= snapshot
&& key >= b.as_slice()
&& key < e.as_slice()
&& level_ok
&& *s > best
{
best = *s;
}
}
best
}
let raw: Vec<(Vec<u8>, Vec<u8>, SequenceNumber, usize)> = vec![
(vec![0], vec![100], 1, 0),
(vec![10], vec![90], 5, 1),
(vec![20], vec![80], 3, 0),
(vec![20], vec![80], 9, 2),
(vec![30], vec![40], 20, 3),
(vec![40], vec![50], 21, 0),
(vec![60], vec![70], 2, 2),
(vec![0], vec![5], 0, 0),
(vec![0], vec![5], 0, 1),
(vec![95], vec![100], 30, 5),
(vec![100], vec![110], 31, 0),
];
let list = FragmentedRangeTombstoneList::new_with_levels(raw.clone());
for key_byte in 0u8..=120 {
let key = [key_byte];
for snapshot in [0u64, 1, 2, 5, 9, 20, 21, 30, 31, u64::MAX] {
for source_level in [None, Some(0usize), Some(1), Some(2), Some(3), Some(5)] {
let expected = brute_force(&raw, &key, snapshot, source_level);
let actual =
list.max_covering_tombstone_seq_for_level(&key, snapshot, source_level);
assert_eq!(
actual, expected,
"key={key_byte} snapshot={snapshot} source_level={source_level:?}"
);
}
}
}
}
}