landfill 0.5.0

Various types for dealing with on-disk data
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126

use std::hash::Hash;
use std::io;
use std::marker::PhantomData;

use bytemuck_derive::*;
use digest::Digest;

use crate::{AppendOnly, GuardedLandfill, SmashMap, Substructure};

#[repr(C)]
#[derive(Clone, Copy, Pod, Zeroable)]
struct Entry {
    ofs: u64,
    len: u32,
    tag: u32,
}

#[repr(C)]
#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Pod, Zeroable)]
pub struct ContentId([u8; 32]);

impl ContentId {
    fn from_bytes<D: Digest>(bytes: &[u8]) -> Self {
        let mut hash_bytes = [0u8; 32];
        let digest = D::digest(bytes);
        hash_bytes.copy_from_slice(digest.as_ref());
        ContentId(hash_bytes)
    }
}

/// A storage for content-adressable byte-slices
pub struct Content<D> {
    data: AppendOnly,
    index: SmashMap<ContentId, Entry>,
    _marker: PhantomData<D>,
}

impl<D> Substructure for Content<D> {
    fn init(lf: GuardedLandfill) -> io::Result<Self> {
        Ok(Content {
            data: lf.substructure("data")?,
            index: lf.substructure("index")?,
            _marker: PhantomData,
        })
    }

    fn flush(&self) -> io::Result<()> {
        self.data.flush()?;
        self.index.flush()
    }
}

impl<D> Content<D>
where
    D: Digest,
{
    /// Insert bytes into the Content store, returning the content id
    pub fn insert(&self, bytes: &[u8]) -> io::Result<ContentId> {
        self.insert_aligned(bytes, 1)
    }

    /// Insert bytes aligned to `alignment` into the Content store,
    /// returning the content id
    pub fn insert_aligned(
        &self,
        bytes: &[u8],
        alignment: usize,
    ) -> io::Result<ContentId> {
        let id = ContentId::from_bytes::<D>(bytes);

        self.index.insert(
            &id,
            |search, entry| {
                let search_tag = search.tag_u32();

                if search_tag == entry.tag {
                    let stored = self.data.get(entry.ofs, entry.len);
                    let stored_id = ContentId::from_bytes::<D>(stored);

                    if id == stored_id {
                        search.halt()
                    } else {
                        search.proceed()
                    }
                } else {
                    search.proceed()
                }
            },
            |search| {
                let ofs = self.data.write_aligned(bytes, alignment)?;

                Ok(Entry {
                    ofs,
                    len: bytes.len() as u32,
                    tag: search.tag_u32(),
                })
            },
        )?;
        Ok(id)
    }

    /// Gets the value corresponding to the key, if any
    pub fn get(&self, id: ContentId) -> Option<&[u8]> {
        let mut result = None;
        self.index.get(&id, |search, entry| {
            let search_tag = search.tag_u32();

            if search_tag == entry.tag {
                let stored = self.data.get(entry.ofs, entry.len);

                let stored_id = ContentId::from_bytes::<D>(stored);

                if stored_id == id {
                    // found it!
                    result = Some(stored);
                    search.halt()
                } else {
                    search.proceed()
                }
            } else {
                search.proceed()
            }
        });
        result
    }
}