use std::collections::HashMap;
use std::sync::Arc;
use test_log::test;
use crate::batch::Batch;
use crate::memtable::MemTable;
use crate::{InternalKeyKind, LSMIterator, Value};
fn assert_value(encoded_value: &Value, expected_value: &[u8]) {
let value_content = &encoded_value[..];
assert_eq!(value_content, expected_value);
}
#[test]
fn memtable_get() {
let memtable = MemTable::default();
let key = b"foo".to_vec();
let value = b"value".to_vec();
let mut batch = Batch::new(1);
batch.set(key, value.clone(), 0).unwrap();
memtable.add(&batch).unwrap();
let res = memtable.get(b"foo", None).unwrap();
assert_value(&res.1, &value);
}
#[test]
fn memtable_size() {
let memtable = MemTable::default();
let key = b"foo".to_vec();
let value = b"value".to_vec();
let mut batch = Batch::new(1);
batch.set(key, value, 0).unwrap();
memtable.add(&batch).unwrap();
assert!(memtable.size() > 0);
}
#[test]
fn memtable_lsn() {
let memtable = MemTable::default();
let key = b"foo".to_vec();
let value = b"value".to_vec();
let seq_num = 100;
let mut batch = Batch::new(seq_num);
batch.set(key, value, 0).unwrap();
memtable.add(&batch).unwrap();
assert_eq!(seq_num, memtable.lsn());
}
#[test]
fn memtable_add_and_get() {
let memtable = MemTable::default();
let key1 = b"key1".to_vec();
let value1 = b"value1".to_vec();
let mut batch1 = Batch::new(1);
batch1.set(key1, value1.clone(), 0).unwrap();
memtable.add(&batch1).unwrap();
let key2 = b"key2".to_vec();
let value2 = b"value2".to_vec();
let mut batch2 = Batch::new(2);
batch2.set(key2, value2.clone(), 0).unwrap();
memtable.add(&batch2).unwrap();
let res = memtable.get(b"key1", None).unwrap();
assert_value(&res.1, &value1);
let res = memtable.get(b"key2", None).unwrap();
assert_value(&res.1, &value2);
}
#[test]
fn memtable_get_latest_seq_no() {
let memtable = MemTable::default();
let key1 = b"key1".to_vec();
let value1 = b"value1".to_vec();
let value2 = b"value2".to_vec();
let value3 = b"value3".to_vec();
let mut batch1 = Batch::new(1);
batch1.set(key1.clone(), value1, 0).unwrap();
memtable.add(&batch1).unwrap();
let mut batch2 = Batch::new(2);
batch2.set(key1.clone(), value2, 0).unwrap();
memtable.add(&batch2).unwrap();
let mut batch3 = Batch::new(3);
batch3.set(key1, value3.clone(), 0).unwrap();
memtable.add(&batch3).unwrap();
let res = memtable.get(b"key1", None).unwrap();
assert_value(&res.1, &value3);
}
#[test]
fn memtable_prefix() {
let memtable = MemTable::default();
let key1 = b"foo".to_vec();
let value1 = b"value1".to_vec();
let key2 = b"foo1".to_vec();
let value2 = b"value2".to_vec();
let mut batch1 = Batch::new(0);
batch1.set(key1, value1.clone(), 0).unwrap();
memtable.add(&batch1).unwrap();
let mut batch2 = Batch::new(1);
batch2.set(key2, value2.clone(), 0).unwrap();
memtable.add(&batch2).unwrap();
let res = memtable.get(b"foo", None).unwrap();
assert_value(&res.1, &value1);
let res = memtable.get(b"foo1", None).unwrap();
assert_value(&res.1, &value2);
}
type TestEntry = (Vec<u8>, Vec<u8>, InternalKeyKind, Option<u64>);
fn create_test_memtable(entries: Vec<TestEntry>) -> (Arc<MemTable>, u64) {
let memtable = Arc::new(MemTable::default());
let mut last_seq = 0;
for (key, value, kind, custom_seq) in entries {
let seq_num = custom_seq.unwrap_or_else(|| {
last_seq += 1;
last_seq
});
let mut batch = Batch::new(seq_num);
match kind {
InternalKeyKind::Set => {
batch.set(key.clone(), value.clone(), 0).unwrap();
}
InternalKeyKind::Delete => {
batch.delete(key.clone(), 0).unwrap();
}
_ => {
batch.add_record(kind, key.clone(), Some(value.clone()), 0).unwrap();
}
}
memtable.add(&batch).unwrap();
if custom_seq.is_some() {
last_seq = std::cmp::max(last_seq, seq_num);
}
}
(memtable, last_seq)
}
#[test]
fn test_empty_memtable() {
let memtable = Arc::new(MemTable::default());
let mut iter = memtable.iter();
iter.seek_first().unwrap();
let mut count = 0;
while iter.valid() {
count += 1;
if !iter.next().unwrap_or(false) {
break;
}
}
assert_eq!(count, 0);
assert!(memtable.is_empty());
}
#[test]
fn test_single_key() {
let (memtable, _) = create_test_memtable(vec![(
b"key1".to_vec(),
b"value1".to_vec(),
InternalKeyKind::Set,
None,
)]);
let mut iter = memtable.iter();
iter.seek_first().unwrap();
let mut entries = Vec::new();
while iter.valid() {
let key = iter.key().to_owned();
let value_bytes = iter.value_encoded();
let value = value_bytes.unwrap().to_vec();
entries.push((key, value));
if !iter.next().unwrap_or(false) {
break;
}
}
assert_eq!(entries.len(), 1);
let (key, encoded_value) = &entries[0];
let user_key = &key.user_key;
assert_eq!(user_key, b"key1");
assert_value(encoded_value, &b"value1"[..]);
let result = memtable.get(b"key1", None);
assert!(result.is_some());
let (ikey, encoded_val) = result.unwrap();
assert_eq!(&ikey.user_key, b"key1");
assert_value(&encoded_val, b"value1");
}
#[test]
fn test_multiple_keys() {
let (memtable, _) = create_test_memtable(vec![
(b"key1".to_vec(), b"value1".to_vec(), InternalKeyKind::Set, None),
(b"key3".to_vec(), b"value3".to_vec(), InternalKeyKind::Set, None),
(b"key5".to_vec(), b"value5".to_vec(), InternalKeyKind::Set, None),
]);
let mut iter = memtable.iter();
iter.seek_first().unwrap();
let mut entries = Vec::new();
while iter.valid() {
let key = iter.key().to_owned();
let value_bytes = iter.value_encoded();
let value = value_bytes.unwrap().to_vec();
entries.push((key, value));
if !iter.next().unwrap_or(false) {
break;
}
}
assert_eq!(entries.len(), 3);
let user_keys: Vec<_> = entries.iter().map(|(key, _)| key.user_key.clone()).collect();
assert_eq!(&user_keys[0], b"key1");
assert_eq!(&user_keys[1], b"key3");
assert_eq!(&user_keys[2], b"key5");
assert!(memtable.get(b"key1", None).is_some());
assert!(memtable.get(b"key3", None).is_some());
assert!(memtable.get(b"key5", None).is_some());
assert!(memtable.get(b"key2", None).is_none());
assert!(memtable.get(b"key4", None).is_none());
}
#[test]
fn test_sequence_number_ordering() {
let (memtable, _) = create_test_memtable(vec![
(b"key1".to_vec(), b"value1".to_vec(), InternalKeyKind::Set, Some(10)),
(b"key1".to_vec(), b"value2".to_vec(), InternalKeyKind::Set, Some(20)),
(b"key1".to_vec(), b"value3".to_vec(), InternalKeyKind::Set, Some(5)),
]);
let mut iter = memtable.iter();
iter.seek_first().unwrap();
let mut entries = Vec::new();
while iter.valid() {
let key = iter.key().to_owned();
let value_bytes = iter.value_encoded();
let value = value_bytes.unwrap().to_vec();
entries.push((key, value));
if !iter.next().unwrap_or(false) {
break;
}
}
assert_eq!(entries.len(), 3);
let mut key1_entries = Vec::new();
for (key, encoded_value) in &entries {
let (user_key, seq_num, _) = (key.user_key.clone(), key.seq_num(), key.kind());
if &user_key == b"key1" {
key1_entries.push((seq_num, encoded_value));
}
}
assert_eq!(key1_entries.len(), 3);
assert_eq!(key1_entries[0].0, 20);
assert_eq!(key1_entries[0].1, b"value2");
assert_eq!(key1_entries[1].0, 10);
assert_eq!(key1_entries[1].1, b"value1");
assert_eq!(key1_entries[2].0, 5);
assert_eq!(key1_entries[2].1, b"value3");
let result = memtable.get(b"key1", None);
assert!(result.is_some());
let (ikey, encoded_val) = result.unwrap();
assert_eq!(ikey.seq_num(), 20);
assert_eq!(&encoded_val, b"value2");
}
#[test]
fn test_key_updates_with_sequence_numbers() {
let (memtable, _) = create_test_memtable(vec![
(b"key1".to_vec(), b"old_value".to_vec(), InternalKeyKind::Set, Some(5)),
(b"key1".to_vec(), b"new_value".to_vec(), InternalKeyKind::Set, Some(10)),
(b"key2".to_vec(), b"value2".to_vec(), InternalKeyKind::Set, Some(7)),
]);
let result = memtable.get(b"key1", None);
assert!(result.is_some());
let (_, encoded_val) = result.unwrap();
assert_value(&encoded_val, b"new_value");
let result = memtable.get(b"key1", Some(8));
assert!(result.is_some());
let (_, encoded_val) = result.unwrap();
assert_value(&encoded_val, b"old_value"); }
#[test]
fn test_tombstones() {
let (memtable, _) = create_test_memtable(vec![
(b"key1".to_vec(), b"value1".to_vec(), InternalKeyKind::Set, Some(1)),
(b"key2".to_vec(), b"value2".to_vec(), InternalKeyKind::Set, Some(2)),
(b"key3".to_vec(), b"value3".to_vec(), InternalKeyKind::Set, Some(3)),
(b"key2".to_vec(), vec![], InternalKeyKind::Delete, Some(4)), ]);
let mut iter = memtable.iter();
iter.seek_first().unwrap();
let mut entries = Vec::new();
while iter.valid() {
let key = iter.key().to_owned();
let value_bytes = iter.value_encoded();
let value = value_bytes.unwrap().to_vec();
entries.push((key, value));
if !iter.next().unwrap_or(false) {
break;
}
}
let mut key_counts = HashMap::new();
for (key, _) in &entries {
let user_key = &key.user_key;
*key_counts.entry(user_key).or_insert(0) += 1;
}
assert_eq!(key_counts[&b"key1".to_vec()], 1);
assert_eq!(key_counts[&b"key2".to_vec()], 2); assert_eq!(key_counts[&b"key3".to_vec()], 1);
}
#[test]
fn test_key_kinds() {
let (memtable, _) = create_test_memtable(vec![
(b"key1".to_vec(), b"value1".to_vec(), InternalKeyKind::Set, Some(10)),
(b"key2".to_vec(), vec![], InternalKeyKind::Delete, Some(20)),
(b"key3".to_vec(), b"value3".to_vec(), InternalKeyKind::Set, Some(30)),
(b"key4".to_vec(), vec![], InternalKeyKind::Delete, Some(40)),
]);
let mut iter = memtable.iter();
iter.seek_first().unwrap();
let mut entries = Vec::new();
while iter.valid() {
let key = iter.key().to_owned();
let value_bytes = iter.value_encoded();
let value = value_bytes.unwrap().to_vec();
entries.push((key, value));
if !iter.next().unwrap_or(false) {
break;
}
}
assert_eq!(entries.len(), 4);
let mut key_info = Vec::new();
for (key, encoded_value) in &entries {
let (user_key, seq_num, kind) = (key.user_key.clone(), key.seq_num(), key.kind());
key_info.push((user_key, seq_num, kind, encoded_value.len()));
}
assert_eq!(&key_info[0].0, b"key1");
assert_eq!(key_info[0].2, InternalKeyKind::Set);
assert!(key_info[0].3 > 0);
assert_eq!(&key_info[1].0, b"key2");
assert_eq!(key_info[1].2, InternalKeyKind::Delete);
assert_eq!(key_info[1].3, 0);
assert_eq!(&key_info[2].0, b"key3");
assert_eq!(key_info[2].2, InternalKeyKind::Set);
assert!(key_info[2].3 > 0);
assert_eq!(&key_info[3].0, b"key4");
assert_eq!(key_info[3].2, InternalKeyKind::Delete);
assert_eq!(key_info[3].3, 0);
let result = memtable.get(b"key1", None);
assert!(result.is_some());
let (ikey, _) = result.unwrap();
assert_eq!(ikey.kind(), InternalKeyKind::Set);
let result = memtable.get(b"key2", None);
assert!(result.is_some());
let (ikey, encoded_val) = result.unwrap();
assert_eq!(ikey.kind(), InternalKeyKind::Delete);
assert_eq!(encoded_val.len(), 0);
}
#[test]
fn test_range_query() {
let (memtable, _) = create_test_memtable(vec![
(b"a".to_vec(), b"value-a".to_vec(), InternalKeyKind::Set, None),
(b"c".to_vec(), b"value-c".to_vec(), InternalKeyKind::Set, None),
(b"e".to_vec(), b"value-e".to_vec(), InternalKeyKind::Set, None),
(b"g".to_vec(), b"value-g".to_vec(), InternalKeyKind::Set, None),
(b"i".to_vec(), b"value-i".to_vec(), InternalKeyKind::Set, None),
(b"k".to_vec(), b"value-k".to_vec(), InternalKeyKind::Set, None),
(b"m".to_vec(), b"value-m".to_vec(), InternalKeyKind::Set, None),
]);
let mut range_iter = memtable.range(
Some("c".as_bytes()), Some("l".as_bytes()), );
range_iter.seek_first().unwrap();
let mut range_entries = Vec::new();
while range_iter.valid() {
let key = range_iter.key().to_owned();
let value_bytes = range_iter.value_encoded();
let value = value_bytes.unwrap().to_vec();
range_entries.push((key, value));
if !range_iter.next().unwrap_or(false) {
break;
}
}
let user_keys: Vec<_> = range_entries.iter().map(|(key, _)| key.user_key.clone()).collect();
assert_eq!(user_keys.len(), 5);
assert_eq!(&user_keys[0], b"c");
assert_eq!(&user_keys[1], b"e");
assert_eq!(&user_keys[2], b"g");
assert_eq!(&user_keys[3], b"i");
assert_eq!(&user_keys[4], b"k");
let mut range_iter = memtable.range(
Some("c".as_bytes()), Some("k".as_bytes()), );
range_iter.seek_first().unwrap();
let mut range_entries = Vec::new();
while range_iter.valid() {
let key = range_iter.key().to_owned();
let value_bytes = range_iter.value_encoded();
let value = value_bytes.unwrap().to_vec();
range_entries.push((key, value));
if !range_iter.next().unwrap_or(false) {
break;
}
}
let user_keys: Vec<_> = range_entries.iter().map(|(key, _)| key.user_key.clone()).collect();
assert_eq!(user_keys.len(), 4); assert_eq!(&user_keys[0], b"c");
assert_eq!(&user_keys[1], b"e");
assert_eq!(&user_keys[2], b"g");
assert_eq!(&user_keys[3], b"i");
}
#[test]
fn test_range_query_with_sequence_numbers() {
let (memtable, _) = create_test_memtable(vec![
(b"a".to_vec(), b"value-a1".to_vec(), InternalKeyKind::Set, Some(10)),
(b"a".to_vec(), b"value-a2".to_vec(), InternalKeyKind::Set, Some(20)), (b"c".to_vec(), b"value-c1".to_vec(), InternalKeyKind::Set, Some(15)),
(b"e".to_vec(), b"value-e1".to_vec(), InternalKeyKind::Set, Some(25)),
(b"e".to_vec(), b"value-e2".to_vec(), InternalKeyKind::Set, Some(15)), ]);
let mut range_iter = memtable.range(
Some("a".as_bytes()), Some("f".as_bytes()), );
range_iter.seek_first().unwrap();
let mut range_entries = Vec::new();
while range_iter.valid() {
let key = range_iter.key().to_owned();
let value_bytes = range_iter.value_encoded();
let value = value_bytes.unwrap().to_vec();
range_entries.push((key, value));
if !range_iter.next().unwrap_or(false) {
break;
}
}
let mut entries_info = Vec::new();
for (key, encoded_value) in &range_entries {
let (user_key, seq_num, _) = (key.user_key.clone(), key.seq_num(), key.kind());
entries_info.push((user_key, seq_num, encoded_value));
}
assert_eq!(entries_info.len(), 5);
assert_eq!(&entries_info[0].0, b"a");
assert_eq!(entries_info[0].1, 20);
assert_eq!(entries_info[0].2, b"value-a2");
assert_eq!(entries_info[1].0, b"a");
assert_eq!(entries_info[1].1, 10);
assert_eq!(entries_info[1].2, b"value-a1");
assert_eq!(entries_info[2].0, b"c");
assert_eq!(entries_info[2].1, 15);
assert_eq!(entries_info[2].2, b"value-c1");
assert_eq!(entries_info[3].0, b"e");
assert_eq!(entries_info[3].1, 25);
assert_eq!(entries_info[3].2, b"value-e1");
assert_eq!(entries_info[4].0, b"e");
assert_eq!(entries_info[4].1, 15);
assert_eq!(entries_info[4].2, b"value-e2");
}
#[test]
fn test_binary_keys() {
let (memtable, _) = create_test_memtable(vec![
(vec![0, 0, 1], b"value1".to_vec(), InternalKeyKind::Set, None),
(vec![0, 1, 0], b"value2".to_vec(), InternalKeyKind::Set, None),
(vec![1, 0, 0], b"value3".to_vec(), InternalKeyKind::Set, None),
(vec![0xFF, 0xFE, 0xFD], b"value4".to_vec(), InternalKeyKind::Set, None),
]);
let mut iter = memtable.iter();
iter.seek_first().unwrap();
let mut entries = Vec::new();
while iter.valid() {
let key = iter.key().to_owned();
let value_bytes = iter.value_encoded();
let value = value_bytes.unwrap().to_vec();
entries.push((key, value));
if !iter.next().unwrap_or(false) {
break;
}
}
assert_eq!(entries.len(), 4);
let user_keys: Vec<_> = entries.iter().map(|(key, _)| key.user_key.clone()).collect();
assert_eq!(user_keys[0].as_ref(), vec![0, 0, 1]);
assert_eq!(user_keys[1].as_ref(), vec![0, 1, 0]);
assert_eq!(user_keys[2].as_ref(), vec![1, 0, 0]);
assert_eq!(user_keys[3].as_ref(), vec![0xFF, 0xFE, 0xFD]);
}
#[test]
fn test_large_dataset() {
let mut entries = Vec::new();
for i in 0..1000 {
let key = format!("key{i:04}").as_bytes().to_vec();
let value = format!("value{i:04}").as_bytes().to_vec();
entries.push((key, value, InternalKeyKind::Set, None));
}
let (memtable, _) = create_test_memtable(entries);
let mut iter = memtable.iter();
iter.seek_first().unwrap();
let mut count = 0;
while iter.valid() {
count += 1;
if !iter.next().unwrap_or(false) {
break;
}
}
assert_eq!(count, 1000);
let result = memtable.get(b"key0000", None);
assert!(result.is_some());
let (_, encoded_val) = result.unwrap();
assert_value(&encoded_val, b"value0000");
let result = memtable.get(b"key0500", None);
assert!(result.is_some());
let (_, encoded_val) = result.unwrap();
assert_value(&encoded_val, b"value0500");
let result = memtable.get(b"key0999", None);
assert!(result.is_some());
let (_, encoded_val) = result.unwrap();
assert_value(&encoded_val, b"value0999");
let result = memtable.get(b"key1000", None);
assert!(result.is_none());
}
#[test]
fn test_memtable_size_tracking() {
let memtable = Arc::new(MemTable::default());
let mut batch = Batch::new(1);
batch.set(b"key1".to_vec(), b"value1".to_vec(), 0).unwrap();
batch.set(b"key2".to_vec(), b"value2".to_vec(), 0).unwrap();
memtable.add(&batch).unwrap();
let size1 = memtable.size();
assert!(size1 > 0);
let mut batch2 = Batch::new(2);
batch2.set(b"key3".to_vec(), b"value3".to_vec(), 0).unwrap();
memtable.add(&batch2).unwrap();
let size2 = memtable.size();
assert!(size2 > size1);
}
#[test]
fn test_latest_sequence_number() {
let memtable = Arc::new(MemTable::default());
assert_eq!(memtable.lsn(), 0);
let mut batch1 = Batch::new(10);
batch1.set(b"key1".to_vec(), b"value1".to_vec(), 0).unwrap();
memtable.add(&batch1).unwrap();
assert_eq!(memtable.lsn(), 10);
let mut batch2 = Batch::new(5);
batch2.set(b"key2".to_vec(), b"value2".to_vec(), 0).unwrap();
memtable.add(&batch2).unwrap();
assert_eq!(memtable.lsn(), 10);
let mut batch3 = Batch::new(20);
batch3.set(b"key3".to_vec(), b"value3".to_vec(), 0).unwrap();
memtable.add(&batch3).unwrap();
assert_eq!(memtable.lsn(), 20);
}
#[test]
fn test_get_highest_seq_num() {
let mut batch = Batch::new(10);
batch.set(b"key1".to_vec(), b"value1".to_vec(), 0).unwrap();
batch.set(b"key2".to_vec(), b"value2".to_vec(), 0).unwrap();
batch.set(b"key3".to_vec(), b"value3".to_vec(), 0).unwrap();
batch.set(b"key4".to_vec(), b"value4".to_vec(), 0).unwrap();
batch.set(b"key5".to_vec(), b"value5".to_vec(), 0).unwrap();
assert_eq!(batch.get_highest_seq_num(), 14);
}
#[test]
fn test_excluded_bound_skips_all_versions_of_key() {
let (memtable, _) = create_test_memtable(vec![
(b"a".to_vec(), b"value-a".to_vec(), InternalKeyKind::Set, Some(10)),
(b"b".to_vec(), b"value-b1".to_vec(), InternalKeyKind::Set, Some(30)), (b"b".to_vec(), b"value-b2".to_vec(), InternalKeyKind::Set, Some(20)), (b"b".to_vec(), b"value-b3".to_vec(), InternalKeyKind::Set, Some(10)), (b"c".to_vec(), b"value-c".to_vec(), InternalKeyKind::Set, Some(15)),
]);
let mut iter = memtable.range(
Some("b".as_bytes()), None, );
iter.seek_first().unwrap();
let mut range_entries = Vec::new();
while iter.valid() {
let key = iter.key().to_owned();
let value_bytes = iter.value_encoded();
let value = value_bytes.unwrap().to_vec();
if key.user_key != b"b" {
range_entries.push((key, value));
while iter.next().unwrap_or(false) && iter.valid() {
let key = iter.key().to_owned();
let value_bytes = iter.value_encoded();
let value = value_bytes.unwrap().to_vec();
range_entries.push((key, value));
}
break;
}
if !iter.next().unwrap_or(false) {
break;
}
}
let user_keys: Vec<_> = range_entries.iter().map(|(key, _)| key.user_key.clone()).collect();
assert_eq!(user_keys.len(), 1, "Expected only 'c', but got {:?}", user_keys);
assert_eq!(&user_keys[0], b"c");
}
#[test]
fn test_excluded_bound_first_skips_all_versions() {
let (memtable, _) = create_test_memtable(vec![
(b"a".to_vec(), b"value-a".to_vec(), InternalKeyKind::Set, Some(10)),
(b"b".to_vec(), b"value-b1".to_vec(), InternalKeyKind::Set, Some(30)),
(b"b".to_vec(), b"value-b2".to_vec(), InternalKeyKind::Set, Some(20)),
(b"b".to_vec(), b"value-b3".to_vec(), InternalKeyKind::Set, Some(10)),
(b"c".to_vec(), b"value-c".to_vec(), InternalKeyKind::Set, Some(15)),
]);
let mut iter = memtable.range(
Some("b".as_bytes()), None, );
iter.seek_first().unwrap();
while iter.valid() {
let key = iter.key().to_owned();
if key.user_key != b"b" {
break;
}
if !iter.next().unwrap_or(false) {
break;
}
}
let first = iter.seek_first().unwrap();
assert!(first);
let key = iter.key().to_owned();
assert_eq!(&key.user_key, b"b", "First key should be 'b'");
let mut iter2 = memtable.range(Some("b".as_bytes()), None);
iter2.seek_first().unwrap();
while iter2.valid() {
let key = iter2.key().to_owned();
if key.user_key != b"b" {
break;
}
if !iter2.next().unwrap_or(false) {
break;
}
}
let first = iter2.seek_first().unwrap();
assert!(first);
let key = iter2.key().to_owned();
assert_eq!(&key.user_key, b"b", "After reset, first key should still be 'b'");
}