iroh_sync/
store.rs

1//! Storage trait and implementation for iroh-sync documents
2use std::num::NonZeroUsize;
3
4use anyhow::Result;
5use bytes::Bytes;
6use serde::{Deserialize, Serialize};
7
8use crate::{AuthorId, Entry, NamespaceId};
9
10pub mod fs;
11mod pubkeys;
12mod util;
13pub use pubkeys::*;
14
15pub use fs::Store;
16
17/// Number of peers to cache per document.
18pub(crate) const PEERS_PER_DOC_CACHE_SIZE: NonZeroUsize = match NonZeroUsize::new(5) {
19    Some(val) => val,
20    None => panic!("this is clearly non zero"),
21};
22
23/// Error return from [`Store::open_replica`]
24#[derive(Debug, thiserror::Error)]
25pub enum OpenError {
26    /// The replica does not exist.
27    #[error("Replica not found")]
28    NotFound,
29    /// Other error while opening the replica.
30    #[error("{0}")]
31    Other(#[from] anyhow::Error),
32}
33
34/// Store that gives read access to download policies for a document.
35pub trait DownloadPolicyStore {
36    /// Get the download policy for a document.
37    fn get_download_policy(&mut self, namespace: &NamespaceId) -> Result<DownloadPolicy>;
38}
39
40impl<T: DownloadPolicyStore> DownloadPolicyStore for &mut T {
41    fn get_download_policy(&mut self, namespace: &NamespaceId) -> Result<DownloadPolicy> {
42        DownloadPolicyStore::get_download_policy(*self, namespace)
43    }
44}
45
46impl DownloadPolicyStore for crate::store::Store {
47    fn get_download_policy(&mut self, namespace: &NamespaceId) -> Result<DownloadPolicy> {
48        self.get_download_policy(namespace)
49    }
50}
51
52/// Outcome of [`Store::import_namespace`]
53#[derive(Debug, Clone, Copy)]
54pub enum ImportNamespaceOutcome {
55    /// The namespace did not exist before and is now inserted.
56    Inserted,
57    /// The namespace existed and now has an upgraded capability.
58    Upgraded,
59    /// The namespace existed and its capability remains unchanged.
60    NoChange,
61}
62
63/// Download policy to decide which content blobs shall be downloaded.
64#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
65pub enum DownloadPolicy {
66    /// Do not download any key unless it matches one of the filters.
67    NothingExcept(Vec<FilterKind>),
68    /// Download every key unless it matches one of the filters.
69    EverythingExcept(Vec<FilterKind>),
70}
71
72impl Default for DownloadPolicy {
73    fn default() -> Self {
74        DownloadPolicy::EverythingExcept(Vec::default())
75    }
76}
77
78/// Filter strategy used in download policies.
79#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
80pub enum FilterKind {
81    /// Matches if the contained bytes are a prefix of the key.
82    Prefix(Bytes),
83    /// Matches if the contained bytes and the key are the same.
84    Exact(Bytes),
85}
86
87impl std::fmt::Display for FilterKind {
88    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
89        // hardly usable but good enough as a poc
90        let (kind, bytes) = match self {
91            FilterKind::Prefix(bytes) => ("prefix", bytes),
92            FilterKind::Exact(bytes) => ("exact", bytes),
93        };
94        let (encoding, repr) = match String::from_utf8(bytes.to_vec()) {
95            Ok(repr) => ("utf8", repr),
96            Err(_) => ("hex", hex::encode(bytes)),
97        };
98        write!(f, "{kind}:{encoding}:{repr}")
99    }
100}
101
102impl std::str::FromStr for FilterKind {
103    type Err = anyhow::Error;
104
105    fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
106        let Some((kind, rest)) = s.split_once(':') else {
107            anyhow::bail!("missing filter kind, either \"prefix:\" or \"exact:\"")
108        };
109        let Some((encoding, rest)) = rest.split_once(':') else {
110            anyhow::bail!("missing encoding: either \"hex:\" or \"utf8:\"")
111        };
112
113        let is_exact = match kind {
114            "exact" => true,
115            "prefix" => false,
116            other => {
117                anyhow::bail!("expected filter kind \"prefix:\" or \"exact:\", found {other}")
118            }
119        };
120
121        let decoded = match encoding {
122            "utf8" => Bytes::from(rest.to_owned()),
123            "hex" => match hex::decode(rest) {
124                Ok(bytes) => Bytes::from(bytes),
125                Err(_) => anyhow::bail!("failed to decode hex"),
126            },
127            other => {
128                anyhow::bail!("expected encoding: either \"hex:\" or \"utf8:\", found {other}")
129            }
130        };
131
132        if is_exact {
133            Ok(FilterKind::Exact(decoded))
134        } else {
135            Ok(FilterKind::Prefix(decoded))
136        }
137    }
138}
139
140impl FilterKind {
141    /// Verifies whether this filter matches a given key
142    pub fn matches(&self, key: impl AsRef<[u8]>) -> bool {
143        match self {
144            FilterKind::Prefix(prefix) => key.as_ref().starts_with(prefix),
145            FilterKind::Exact(expected) => expected == key.as_ref(),
146        }
147    }
148}
149
150impl DownloadPolicy {
151    /// Check if an entry should be downloaded according to this policy.
152    pub fn matches(&self, entry: &Entry) -> bool {
153        let key = entry.key();
154        match self {
155            DownloadPolicy::NothingExcept(patterns) => {
156                patterns.iter().any(|pattern| pattern.matches(key))
157            }
158            DownloadPolicy::EverythingExcept(patterns) => {
159                patterns.iter().all(|pattern| !pattern.matches(key))
160            }
161        }
162    }
163}
164
165/// A query builder for document queries.
166#[derive(Debug, Default)]
167pub struct QueryBuilder<K> {
168    kind: K,
169    filter_author: AuthorFilter,
170    filter_key: KeyFilter,
171    limit: Option<u64>,
172    offset: u64,
173    include_empty: bool,
174    sort_direction: SortDirection,
175}
176
177impl<K> QueryBuilder<K> {
178    /// Call to include empty entries (deletion markers).
179    pub fn include_empty(mut self) -> Self {
180        self.include_empty = true;
181        self
182    }
183    /// Filter by exact key match.
184    pub fn key_exact(mut self, key: impl AsRef<[u8]>) -> Self {
185        self.filter_key = KeyFilter::Exact(key.as_ref().to_vec().into());
186        self
187    }
188    /// Filter by key prefix.
189    pub fn key_prefix(mut self, key: impl AsRef<[u8]>) -> Self {
190        self.filter_key = KeyFilter::Prefix(key.as_ref().to_vec().into());
191        self
192    }
193    /// Filter by author.
194    pub fn author(mut self, author: AuthorId) -> Self {
195        self.filter_author = AuthorFilter::Exact(author);
196        self
197    }
198    /// Set the maximum number of entries to be returned.
199    pub fn limit(mut self, limit: u64) -> Self {
200        self.limit = Some(limit);
201        self
202    }
203    /// Set the offset within the result set from where to start returning results.
204    pub fn offset(mut self, offset: u64) -> Self {
205        self.offset = offset;
206        self
207    }
208}
209
210/// Query on all entries without aggregation.
211#[derive(Debug, Clone, Default, Serialize, Deserialize)]
212pub struct FlatQuery {
213    sort_by: SortBy,
214}
215
216/// Query that only returns the latest entry for a key which has entries from multiple authors.
217#[derive(Debug, Clone, Default, Serialize, Deserialize)]
218pub struct SingleLatestPerKeyQuery {}
219
220impl QueryBuilder<FlatQuery> {
221    /// Set the sort for the query.
222    ///
223    /// The default is to sort by author, then by key, in ascending order.
224    pub fn sort_by(mut self, sort_by: SortBy, direction: SortDirection) -> Self {
225        self.kind.sort_by = sort_by;
226        self.sort_direction = direction;
227        self
228    }
229
230    /// Build the query.
231    pub fn build(self) -> Query {
232        Query::from(self)
233    }
234}
235
236impl QueryBuilder<SingleLatestPerKeyQuery> {
237    /// Set the order direction for the query.
238    ///
239    /// Ordering is always by key for this query type.
240    /// Default direction is ascending.
241    pub fn sort_direction(mut self, direction: SortDirection) -> Self {
242        self.sort_direction = direction;
243        self
244    }
245
246    /// Build the query.
247    pub fn build(self) -> Query {
248        Query::from(self)
249    }
250}
251
252impl From<QueryBuilder<SingleLatestPerKeyQuery>> for Query {
253    fn from(builder: QueryBuilder<SingleLatestPerKeyQuery>) -> Query {
254        Query {
255            kind: QueryKind::SingleLatestPerKey(builder.kind),
256            filter_author: builder.filter_author,
257            filter_key: builder.filter_key,
258            limit: builder.limit,
259            offset: builder.offset,
260            include_empty: builder.include_empty,
261            sort_direction: builder.sort_direction,
262        }
263    }
264}
265
266impl From<QueryBuilder<FlatQuery>> for Query {
267    fn from(builder: QueryBuilder<FlatQuery>) -> Query {
268        Query {
269            kind: QueryKind::Flat(builder.kind),
270            filter_author: builder.filter_author,
271            filter_key: builder.filter_key,
272            limit: builder.limit,
273            offset: builder.offset,
274            include_empty: builder.include_empty,
275            sort_direction: builder.sort_direction,
276        }
277    }
278}
279
280/// Note: When using the `SingleLatestPerKey` query kind, the key filter is applied *before* the
281/// grouping, the author filter is applied *after* the grouping.
282#[derive(Debug, Clone, Serialize, Deserialize)]
283pub struct Query {
284    kind: QueryKind,
285    filter_author: AuthorFilter,
286    filter_key: KeyFilter,
287    limit: Option<u64>,
288    offset: u64,
289    include_empty: bool,
290    sort_direction: SortDirection,
291}
292
293impl Query {
294    /// Query all records.
295    pub fn all() -> QueryBuilder<FlatQuery> {
296        Default::default()
297    }
298    /// Query only the latest entry for each key, omitting older entries if the entry was written
299    /// to by multiple authors.
300    pub fn single_latest_per_key() -> QueryBuilder<SingleLatestPerKeyQuery> {
301        Default::default()
302    }
303
304    /// Create a [`Query::all`] query filtered by a single author.
305    pub fn author(author: AuthorId) -> QueryBuilder<FlatQuery> {
306        Self::all().author(author)
307    }
308
309    /// Create a [`Query::all`] query filtered by a single key.
310    pub fn key_exact(key: impl AsRef<[u8]>) -> QueryBuilder<FlatQuery> {
311        Self::all().key_exact(key)
312    }
313
314    /// Create a [`Query::all`] query filtered by a key prefix.
315    pub fn key_prefix(prefix: impl AsRef<[u8]>) -> QueryBuilder<FlatQuery> {
316        Self::all().key_prefix(prefix)
317    }
318
319    /// Get the limit for this query (max. number of entries to emit).
320    pub fn limit(&self) -> Option<u64> {
321        self.limit
322    }
323
324    /// Get the offset for this query (number of entries to skip at the beginning).
325    pub fn offset(&self) -> u64 {
326        self.offset
327    }
328}
329
330/// Sort direction
331#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)]
332pub enum SortDirection {
333    /// Sort ascending
334    #[default]
335    Asc,
336    /// Sort descending
337    Desc,
338}
339
340#[derive(derive_more::Debug, Clone, Serialize, Deserialize)]
341enum QueryKind {
342    #[debug("Flat {{ sort_by: {:?}}}", _0)]
343    Flat(FlatQuery),
344    #[debug("SingleLatestPerKey")]
345    SingleLatestPerKey(SingleLatestPerKeyQuery),
346}
347
348/// Fields by which the query can be sorted
349#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)]
350pub enum SortBy {
351    /// Sort by key, then author.
352    KeyAuthor,
353    /// Sort by author, then key.
354    #[default]
355    AuthorKey,
356}
357
358/// Key matching.
359#[derive(Debug, Serialize, Deserialize, Clone, Default, Eq, PartialEq)]
360pub enum KeyFilter {
361    /// Matches any key.
362    #[default]
363    Any,
364    /// Only keys that are exactly the provided value.
365    Exact(Bytes),
366    /// All keys that start with the provided value.
367    Prefix(Bytes),
368}
369
370impl<T: AsRef<[u8]>> From<T> for KeyFilter {
371    fn from(value: T) -> Self {
372        KeyFilter::Exact(Bytes::copy_from_slice(value.as_ref()))
373    }
374}
375
376impl KeyFilter {
377    /// Test if a key is matched by this [`KeyFilter`].
378    pub fn matches(&self, key: &[u8]) -> bool {
379        match self {
380            Self::Any => true,
381            Self::Exact(k) => &k[..] == key,
382            Self::Prefix(p) => key.starts_with(p),
383        }
384    }
385}
386
387/// Author matching.
388#[derive(Debug, Serialize, Deserialize, Clone, Default, Eq, PartialEq)]
389pub enum AuthorFilter {
390    /// Matches any author.
391    #[default]
392    Any,
393    /// Matches exactly the provided author.
394    Exact(AuthorId),
395}
396
397impl AuthorFilter {
398    /// Test if an author is matched by this [`AuthorFilter`].
399    pub fn matches(&self, author: &AuthorId) -> bool {
400        match self {
401            Self::Any => true,
402            Self::Exact(a) => a == author,
403        }
404    }
405}
406
407impl From<AuthorId> for AuthorFilter {
408    fn from(value: AuthorId) -> Self {
409        AuthorFilter::Exact(value)
410    }
411}
412
413#[cfg(test)]
414mod tests {
415    use super::*;
416
417    #[test]
418    fn test_filter_kind_encode_decode() {
419        const REPR: &str = "prefix:utf8:memes/futurama";
420        let filter: FilterKind = REPR.parse().expect("should decode");
421        assert_eq!(
422            filter,
423            FilterKind::Prefix(Bytes::from(String::from("memes/futurama")))
424        );
425        assert_eq!(filter.to_string(), REPR)
426    }
427}
iroh_sync/store.rs

iroh_sync/
store.rs
