logfs 0.1.1

Simple append-only log based filesystem with encryption and compression
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126

use std::{collections::BTreeMap, sync::RwLock};

use crate::LogFsError;

use super::Path;

pub type DataOffset = u64;

/// Stores the offset to a key.
///
/// Only used at runtime.
#[derive(Clone, Debug)]
pub struct KeyPointer {
    pub sequence_id: u64,
    pub file_offset: DataOffset,
    pub size: u64,
    pub chunk_size: Option<u32>,
}

/// Runtime state of the db.
pub struct State {
    /// A tree mapping keys to key metadata.
    /// This allows quickly finding keys and their file system location.
    ///
    /// NOTE: all paths are kept in memory, which increases memory usage but
    /// allows for keeping the on-disk log structure very simple and enables
    /// fast key lookups.
    /// The memory usage overhead is mitigated a bit by using a [`BTreeMap`],
    /// which means that keys with the same prefix only consume extra memory
    /// for the unique segments.
    pub(crate) tree: BTreeMap<Path, KeyPointer>,

    /// Amount of bytes of redundant (deleted) file data that could be removed
    /// by re-writing the log.
    /// Does not include the space taken up by journal log messages, only the
    /// aggregated file size.
    redundant_data_bytes_estimate: Option<u128>,

    pub(crate) write_counter: u64,
}

pub type SharedTree = std::sync::Arc<RwLock<State>>;

impl State {
    pub fn new() -> Self {
        Self {
            tree: BTreeMap::new(),
            redundant_data_bytes_estimate: Some(0),
            write_counter: 0,
        }
    }

    pub(crate) fn set_tree(&mut self, tree: BTreeMap<Path, KeyPointer>) {
        self.tree = tree;
        self.redundant_data_bytes_estimate = None;
    }

    pub fn get_key(&self, path: &str) -> Option<&KeyPointer> {
        self.tree.get(path)
    }

    pub fn paths_range<R>(&self, range: R) -> Vec<Path>
    where
        R: std::ops::RangeBounds<String>,
    {
        self.tree.range(range).map(|x| x.0).cloned().collect()
    }

    pub fn paths_offset(&self, offset: usize, max: usize) -> Vec<Path> {
        self.tree
            .iter()
            .skip(offset)
            .take(max)
            .map(|x| x.0)
            .cloned()
            .collect()
    }

    pub fn paths_prefix(&self, prefix: &str) -> Vec<Path> {
        self.tree
            .range(prefix.to_string()..)
            .take_while(|(path, _v)| path.starts_with(prefix))
            .map(|x| x.0)
            .cloned()
            .collect()
    }

    pub fn add_key(&mut self, path: Path, pointer: KeyPointer) {
        self.tree.insert(path, pointer);
        self.write_counter += 1;
    }

    pub fn remove_key(&mut self, path: &str) -> Option<KeyPointer> {
        if let Some(pointer) = self.tree.remove(path) {
            let old = self.redundant_data_bytes_estimate.unwrap_or_default();
            self.redundant_data_bytes_estimate = Some(old + pointer.size as u128);
            self.write_counter += 1;
            Some(pointer)
        } else {
            None
        }
    }

    pub fn rename_key(&mut self, old_path: &Path, new_path: Path) -> Result<(), LogFsError> {
        if let Some(old) = self.tree.remove(old_path) {
            self.tree.insert(new_path, old);
            self.write_counter += 1;
            Ok(())
        } else {
            Err(LogFsError::NotFound {
                path: old_path.clone(),
            })
        }
    }

    /// Get a reference to the state's redundant data bytes estimate.
    pub fn redundant_data_bytes_estimate(&self) -> Option<u128> {
        self.redundant_data_bytes_estimate
    }
}

impl Default for State {
    fn default() -> Self {
        Self::new()
    }
}