firecloud-storage 0.2.0

Chunking, compression, and local storage for FireCloud distributed storage
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126

//! Local chunk storage using Sled (lightweight embedded DB)

use crate::{StorageError, StorageResult};
use firecloud_core::{Chunk, ChunkHash, ChunkMetadata};
use sled::Db;
use std::path::Path;
use tracing::{debug, info};

/// Tree names for different data types
const TREE_CHUNKS: &str = "chunks";
const TREE_METADATA: &str = "metadata";

/// Local chunk store backed by Sled
pub struct ChunkStore {
    db: Db,
}

impl ChunkStore {
    /// Open or create a chunk store at the given path
    pub fn open<P: AsRef<Path>>(path: P) -> StorageResult<Self> {
        let db = sled::open(path)
            .map_err(|e| StorageError::Database(e.to_string()))?;
        info!("Opened chunk store with Sled");
        Ok(Self { db })
    }

    /// Store a chunk
    pub fn put(&self, chunk: &Chunk) -> StorageResult<()> {
        let chunks_tree = self.db.open_tree(TREE_CHUNKS)
            .map_err(|e| StorageError::Database(e.to_string()))?;
        let metadata_tree = self.db.open_tree(TREE_METADATA)
            .map_err(|e| StorageError::Database(e.to_string()))?;

        let key = chunk.metadata.hash.0;

        // Store chunk data
        chunks_tree.insert(key, chunk.data.as_ref())
            .map_err(|e| StorageError::Database(e.to_string()))?;

        // Store metadata
        let metadata_bytes = bincode::serialize(&chunk.metadata)
            .map_err(|e| StorageError::Serialization(e.to_string()))?;
        metadata_tree.insert(key, metadata_bytes)
            .map_err(|e| StorageError::Database(e.to_string()))?;

        debug!("Stored chunk: {}", chunk.metadata.hash);
        Ok(())
    }

    /// Retrieve a chunk by hash
    pub fn get(&self, hash: &ChunkHash) -> StorageResult<Option<Chunk>> {
        let chunks_tree = self.db.open_tree(TREE_CHUNKS)
            .map_err(|e| StorageError::Database(e.to_string()))?;
        let metadata_tree = self.db.open_tree(TREE_METADATA)
            .map_err(|e| StorageError::Database(e.to_string()))?;

        let key = hash.0;

        let data = match chunks_tree.get(key)
            .map_err(|e| StorageError::Database(e.to_string()))? {
            Some(d) => d.to_vec(),
            None => return Ok(None),
        };

        let metadata_bytes = metadata_tree.get(key)
            .map_err(|e| StorageError::Database(e.to_string()))?
            .ok_or_else(|| StorageError::ChunkNotFound(hash.to_hex()))?;

        let metadata: ChunkMetadata = bincode::deserialize(&metadata_bytes)
            .map_err(|e| StorageError::Serialization(e.to_string()))?;

        Ok(Some(Chunk { data: data.into(), metadata }))
    }

    /// Check if a chunk exists
    pub fn contains(&self, hash: &ChunkHash) -> StorageResult<bool> {
        let chunks_tree = self.db.open_tree(TREE_CHUNKS)
            .map_err(|e| StorageError::Database(e.to_string()))?;
        Ok(chunks_tree.contains_key(hash.0)
            .map_err(|e| StorageError::Database(e.to_string()))?)
    }

    /// Delete a chunk
    pub fn delete(&self, hash: &ChunkHash) -> StorageResult<bool> {
        let chunks_tree = self.db.open_tree(TREE_CHUNKS)
            .map_err(|e| StorageError::Database(e.to_string()))?;
        let metadata_tree = self.db.open_tree(TREE_METADATA)
            .map_err(|e| StorageError::Database(e.to_string()))?;

        let key = hash.0;
        let existed = chunks_tree.remove(key)
            .map_err(|e| StorageError::Database(e.to_string()))?
            .is_some();
        metadata_tree.remove(key)
            .map_err(|e| StorageError::Database(e.to_string()))?;

        if existed { debug!("Deleted chunk: {}", hash); }
        Ok(existed)
    }

    /// Get total number of chunks stored
    pub fn count(&self) -> StorageResult<usize> {
        let chunks_tree = self.db.open_tree(TREE_CHUNKS)
            .map_err(|e| StorageError::Database(e.to_string()))?;
        Ok(chunks_tree.len())
    }

    /// Get total size of all stored chunks in bytes
    pub fn total_size(&self) -> StorageResult<u64> {
        let chunks_tree = self.db.open_tree(TREE_CHUNKS)
            .map_err(|e| StorageError::Database(e.to_string()))?;
        
        let mut total = 0u64;
        for result in chunks_tree.iter() {
            let (_, value) = result.map_err(|e| StorageError::Database(e.to_string()))?;
            total += value.len() as u64;
        }
        Ok(total)
    }

    /// Flush all pending writes to disk
    pub fn flush(&self) -> StorageResult<()> {
        self.db.flush().map_err(|e| StorageError::Database(e.to_string()))?;
        Ok(())
    }
}