rustic_core/
index.rs

1use std::{sync::Arc, thread::sleep, time::Duration};
2
3use bytes::Bytes;
4use derive_more::Constructor;
5
6use crate::{
7    backend::{FileType, decrypt::DecryptReadBackend},
8    blob::{BlobId, BlobLocation, BlobType, DataId, tree::TreeId},
9    error::{ErrorKind, RusticError, RusticResult},
10    index::binarysorted::{Index, IndexCollector, IndexType},
11    progress::Progress,
12    repofile::{
13        indexfile::{IndexBlob, IndexFile},
14        packfile::PackId,
15    },
16};
17
18pub(crate) mod binarysorted;
19pub(crate) mod indexer;
20
21/// An entry in the index
22#[derive(Debug, Clone, Copy, PartialEq, Eq, Constructor)]
23pub struct IndexEntry {
24    /// The type of the blob
25    blob_type: BlobType,
26    /// The pack the blob is in
27    pub pack: PackId,
28    /// The location of the blob in the pack
29    pub location: BlobLocation,
30}
31
32impl IndexEntry {
33    /// Create an [`IndexEntry`] from an [`IndexBlob`]
34    ///
35    /// # Arguments
36    ///
37    /// * `blob` - The [`IndexBlob`] to create the [`IndexEntry`] from
38    /// * `pack` - The pack the blob is in
39    #[must_use]
40    pub const fn from_index_blob(blob: &IndexBlob, pack: PackId) -> Self {
41        Self {
42            blob_type: blob.tpe,
43            pack,
44            location: blob.location,
45        }
46    }
47
48    /// Get a blob described by [`IndexEntry`] from the backend
49    ///
50    /// # Arguments
51    ///
52    /// * `be` - The backend to read from
53    ///
54    /// # Errors
55    ///
56    // TODO:  add error! This function will return an error if the blob is not found in the backend.
57    pub fn read_data<B: DecryptReadBackend>(&self, be: &B) -> RusticResult<Bytes> {
58        let data = be.read_encrypted_partial(
59            FileType::Pack,
60            &self.pack,
61            self.blob_type.is_cacheable(),
62            self.location,
63        )?;
64
65        Ok(data)
66    }
67
68    /// Get the length of the data described by the [`IndexEntry`]
69    #[must_use]
70    pub const fn data_length(&self) -> u32 {
71        self.location.data_length()
72    }
73}
74
75/// The index of the repository
76///
77/// The index is a list of [`IndexEntry`]s
78pub trait ReadIndex {
79    /// Get an [`IndexEntry`] from the index
80    ///
81    /// # Arguments
82    ///
83    /// * `tpe` - The type of the blob
84    /// * `id` - The id of the blob
85    ///
86    /// # Returns
87    ///
88    /// The [`IndexEntry`] - If it exists otherwise `None`
89    fn get_id(&self, tpe: BlobType, id: &BlobId) -> Option<IndexEntry>;
90
91    /// Get the total size of all blobs of the given type
92    ///
93    /// # Arguments
94    ///
95    /// * `tpe` - The type of the blobs
96    fn total_size(&self, tpe: BlobType) -> u64;
97
98    /// Check if the index contains the given blob
99    ///
100    /// # Arguments
101    ///
102    /// * `tpe` - The type of the blob
103    /// * `id` - The id of the blob
104    fn has(&self, tpe: BlobType, id: &BlobId) -> bool;
105
106    /// Get a tree from the index
107    ///
108    /// # Arguments
109    ///
110    /// * `id` - The id of the tree
111    ///
112    /// # Returns
113    ///
114    /// The [`IndexEntry`] of the tree if it exists otherwise `None`
115    fn get_tree(&self, id: &TreeId) -> Option<IndexEntry> {
116        self.get_id(BlobType::Tree, &BlobId::from(**id))
117    }
118
119    /// Get a data blob from the index
120    ///
121    /// # Arguments
122    ///
123    /// * `id` - The id of the data blob
124    ///
125    /// # Returns
126    ///
127    /// The [`IndexEntry`] of the data blob if it exists otherwise `None`
128    fn get_data(&self, id: &DataId) -> Option<IndexEntry> {
129        self.get_id(BlobType::Data, &BlobId::from(**id))
130    }
131
132    /// Check if the index contains the given tree
133    ///
134    /// # Arguments
135    ///
136    /// * `id` - The id of the tree
137    ///
138    /// # Returns
139    ///
140    /// `true` if the index contains the tree otherwise `false`
141    fn has_tree(&self, id: &TreeId) -> bool {
142        self.has(BlobType::Tree, &BlobId::from(**id))
143    }
144
145    /// Check if the index contains the given data blob
146    ///
147    /// # Arguments
148    ///
149    /// * `id` - The id of the data blob
150    ///
151    /// # Returns
152    ///
153    /// `true` if the index contains the data blob otherwise `false`
154    fn has_data(&self, id: &DataId) -> bool {
155        self.has(BlobType::Data, &BlobId::from(**id))
156    }
157
158    /// Get a blob from the backend
159    ///
160    /// # Arguments
161    ///
162    /// * `tpe` - The type of the blob
163    /// * `id` - The id of the blob
164    ///
165    /// # Errors
166    ///
167    /// * If the blob could not be found in the index
168    fn blob_from_backend(
169        &self,
170        be: &impl DecryptReadBackend,
171        tpe: BlobType,
172        id: &BlobId,
173    ) -> RusticResult<Bytes> {
174        self.get_id(tpe, id).map_or_else(
175            || {
176                Err(RusticError::new(
177                    ErrorKind::Internal,
178                    "Blob `{id}` with type `{type}` not found in index",
179                )
180                .attach_context("id", id.to_string())
181                .attach_context("type", tpe.to_string()))
182            },
183            |ie| ie.read_data(be),
184        )
185    }
186}
187
188/// A trait for a global index
189pub trait ReadGlobalIndex: ReadIndex + Clone + Sync + Send + 'static {}
190
191/// A global index
192#[derive(Clone, Debug)]
193pub struct GlobalIndex {
194    /// The atomic reference counted, sharable index.
195    index: Arc<Index>,
196}
197
198impl ReadIndex for GlobalIndex {
199    /// Get an [`IndexEntry`] from the index
200    ///
201    /// # Arguments
202    ///
203    /// * `tpe` - The type of the blob
204    /// * `id` - The id of the blob
205    ///
206    /// # Returns
207    ///
208    /// The [`IndexEntry`] - If it exists otherwise `None`
209    fn get_id(&self, tpe: BlobType, id: &BlobId) -> Option<IndexEntry> {
210        self.index.get_id(tpe, id)
211    }
212
213    /// Get the total size of all blobs of the given type
214    ///
215    /// # Arguments
216    ///
217    /// * `tpe` - The type of the blobs
218    fn total_size(&self, tpe: BlobType) -> u64 {
219        self.index.total_size(tpe)
220    }
221
222    /// Check if the index contains the given blob
223    ///
224    /// # Arguments
225    ///
226    /// * `tpe` - The type of the blob
227    /// * `id` - The id of the blob
228    ///
229    /// # Returns
230    ///
231    /// `true` if the index contains the blob otherwise `false`
232    fn has(&self, tpe: BlobType, id: &BlobId) -> bool {
233        self.index.has(tpe, id)
234    }
235}
236
237impl GlobalIndex {
238    /// Create a new [`GlobalIndex`] from an [`Index`]
239    ///
240    /// # Type Parameters
241    ///
242    /// * `BE` - The backend type
243    ///
244    /// # Arguments
245    ///
246    /// * `be` - The backend to read from
247    /// * `index` - The index to use
248    pub fn new_from_index(index: Index) -> Self {
249        Self {
250            index: Arc::new(index),
251        }
252    }
253
254    /// Create a new [`GlobalIndex`] from an [`IndexCollector`]
255    ///
256    /// # Arguments
257    ///
258    /// * `be` - The backend to read from
259    /// * `p` - The progress tracker
260    /// * `collector` - The [`IndexCollector`] to use
261    ///
262    /// # Errors
263    ///
264    /// * If the index could not be read
265    fn new_from_collector(
266        be: &impl DecryptReadBackend,
267        p: &Progress,
268        mut collector: IndexCollector,
269    ) -> RusticResult<Self> {
270        p.set_title("reading index...");
271        for index in be.stream_all::<IndexFile>(p)? {
272            collector.extend(index?.1.packs);
273        }
274
275        p.finish();
276
277        Ok(Self::new_from_index(collector.into_index()))
278    }
279
280    /// Create a new [`GlobalIndex`]
281    ///
282    /// # Arguments
283    ///
284    /// * `be` - The backend to read from
285    /// * `p` - The progress tracker
286    pub fn new(be: &impl DecryptReadBackend, p: &Progress) -> RusticResult<Self> {
287        Self::new_from_collector(be, p, IndexCollector::new(IndexType::Full))
288    }
289
290    /// Create a new [`GlobalIndex`] with only full trees
291    ///
292    /// # Arguments
293    ///
294    /// * `be` - The backend to read from
295    /// * `p` - The progress tracker
296    ///
297    /// # Errors
298    ///
299    /// * If the index could not be read
300    pub fn only_full_trees(be: &impl DecryptReadBackend, p: &Progress) -> RusticResult<Self> {
301        Self::new_from_collector(be, p, IndexCollector::new(IndexType::DataIds))
302    }
303
304    /// Convert the `Arc<Index>` to an Index
305    pub fn into_index(self) -> Index {
306        match Arc::try_unwrap(self.index) {
307            Ok(index) => index,
308            Err(arc) => {
309                // Seems index is still in use; this could be due to some threads using it which didn't yet completely shut down.
310                // sleep a bit to let threads using the index shut down, after this index should be available to unwrap
311                sleep(Duration::from_millis(100));
312                Arc::try_unwrap(arc).expect("index still in use")
313            }
314        }
315    }
316
317    pub(crate) fn drop_data(self) -> Self {
318        Self {
319            index: Arc::new(self.into_index().drop_data()),
320        }
321    }
322}
323
324impl ReadGlobalIndex for GlobalIndex {}
rustic_core/index.rs

rustic_core/
index.rs
