cuendillar 0.1.0

Embedded persistent key–value storage engine (LSM-tree based)
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112

use std::{fs::File, path::Path};

use crate::database::wal::{WAL, errors::WALError};
mod default_wal;

pub fn test_wal_append(wal: &mut impl WAL) {
    let mut prev = 0;
    for i in 0..100 as i32 {
        let offset = wal.append_log(&i.to_be_bytes()).unwrap();
        assert!(prev == 0 || prev < offset);
        prev = offset;
    }
}

pub fn test_wal_read(wal: &mut impl WAL) {
    for i in 0..100 as i32 {
        wal.append_log(&i.to_be_bytes()).unwrap();
    }
    // now we will read the wal
    let wal_iterator = wal.read(0).unwrap();
    let mut curr: i32 = 0;
    for i in wal_iterator {
        let (_, val) = i.unwrap();
        assert_eq!(val, curr.to_be_bytes());
        curr += 1;
    }
    assert_eq!(curr, 100);
}

pub fn test_wal_rotation(wal: &mut impl WAL, wal_file_size: u64) {
    let wal_record_size = 40;
    let iterations = ((wal_file_size * 5) / wal_record_size) - 1; // there will be multiple files
    for i in 0..iterations {
        wal.append_log(&i.to_be_bytes()).unwrap();
    }
    // now we will read the wal
    let wal_iterator = wal.read(0).unwrap();
    let mut curr: u64 = 0;
    for i in wal_iterator {
        let (_, val) = i.unwrap();
        assert_eq!(val, curr.to_be_bytes());
        curr += 1;
    }
    assert_eq!(curr, iterations);
}

pub fn test_wal_flush(wal: &mut impl WAL, wal_file_size: u64) {
    let wal_record_size = 40;
    let iterations = ((wal_file_size * 4) / wal_record_size) - 1; // there will be multiple files
    // we will flush almost half the logs
    let mut wals_to_flush = 0;
    for i in 0..iterations {
        let offset = wal.append_log(&i.to_be_bytes()).unwrap();
        if i == iterations / 2 {
            wals_to_flush = offset;
        }
    }
    // as it is clear that logs may be readable even after flush
    // but current implementation delete file which have all logs < offset this way
    // we can assume that the file 0.wal will be deleted
    wal.flush_wal(wals_to_flush).unwrap();
    assert_eq!(Some(WALError::OffsetUnderflow), wal.read(0).err());
    // now we will read the wal
    let wal_iterator = wal.read(wals_to_flush).unwrap();
    let mut curr: u64 = iterations / 2 + 1; // as wals to flushe will be starting offset of next log 
    for i in wal_iterator {
        let (_, val) = i.unwrap();
        assert_eq!(val, curr.to_be_bytes());
        curr += 1;
    }
}

pub fn test_wal_corruption(wal: &mut impl WAL, wal_file_size: u64) {
    let wal_record_size = 40;
    let iterations = ((wal_file_size * 3) / wal_record_size) - 1; // there will be multiple files
    // we will flush almost half the logs
    let mut wals_to_flush = 0;
    for i in 0..iterations {
        let offset = wal.append_log(&i.to_be_bytes()).unwrap();
        if i == iterations / 2 {
            wals_to_flush = offset;
        }
    }
    // now we will read the wal by missaliging the offset
    let mut wal_iterator = wal.read(wals_to_flush + 3).unwrap();
    assert!(wal_iterator.next().unwrap().is_err()); // there will be value but it should be error
}

pub fn test_wal_invalid_file_name(wal: &mut impl WAL, root_dir: &Path) {
    let invalid_file = root_dir.join("invalid_name.wal");
    File::create(invalid_file).unwrap();
    assert!(matches!(wal.read(0), Err(WALError::InvalidFileName(_))));
}

pub fn test_wal_max_palyload(wal: &mut impl WAL, wal_max_payload_len: u64) {
    let payload = vec![0u8; (wal_max_payload_len + 3) as usize];
    assert!(matches!(
        wal.append_log(&payload),
        Err(WALError::PayloadLengthOutOfBound(_))
    ));
}

/// This function assumes that proved wal new without any existing entries
pub fn test_wal_empty(wal: &mut impl WAL) {
    assert!(matches!(wal.read(0).unwrap().next(), None));
}

/// This function assumes that proved wal is wal which is using the same root in which there are some logs (recovered)
pub fn test_wal_recovery_on_new(wal: &mut impl WAL) {
    let offset = wal.append_log(b"test").unwrap();
    assert!(offset > 0);
}