1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113

#![deny(missing_docs)]
#![deny(missing_debug_implementations)]
#![warn(clippy::all)]
// #![warn(clippy::nursery)]
// #![warn(clippy::pedantic)]
// #![warn(clippy::cargo)]

//! # pearl
//!
//! The `pearl` library is an asyncronous Append only key-value blob storage on disk.
//! Crate `pearl` provides [`Futures 0.3`] interface. Tokio runtime required.
//! Storage follows no harm policy, which means that it won't delete or change any of the stored data.
//!
//! [`Futures 0.3`]: https://rust-lang-nursery.github.io/futures-api-docs#latest
//!
//! # Examples
//! The following example shows a storage building and initialization.
//! For more advanced usage see the benchmark tool as the example
//!
//! ```no_run
//! use pearl::{Storage, Builder, ArrayKey, BlobRecordTimestamp};
//!
//! #[tokio::main]
//! async fn main() {
//!     let mut storage: Storage<ArrayKey<8>> = Builder::new()
//!         .work_dir("/tmp/pearl/")
//!         .max_blob_size(1_000_000)
//!         .max_data_in_blob(1_000_000_000)
//!         .blob_file_name_prefix("pearl-test")
//!         .allow_duplicates()
//!         .build()
//!         .unwrap();
//!     storage.init().await.unwrap();
//!     let key = ArrayKey::<8>::default();
//!     let data = b"Hello World!".to_vec();
//!     let timestamp = BlobRecordTimestamp::now();
//!     storage.write(key, data.into(), timestamp).await.unwrap();
//! }
//! ```

#[macro_use]
extern crate log;
#[macro_use]
extern crate serde_derive;
#[macro_use]
extern crate anyhow;

extern crate bytes;

/// Basic info about current build.
pub mod build_info;

mod blob;
/// Types representing various errors that can occur in pearl.
pub mod error;
mod io;
pub use io::IoDriver;
mod record;
mod storage;

/// bloom filter for faster check record contains in blob
pub mod filter;
pub use filter::{Bloom, BloomDataProvider, BloomProvider, Config as BloomConfig, FilterResult};

/// tools to interact with pearl structures
pub mod tools;

pub use blob::Entry;
pub use error::{Error, Kind as ErrorKind};
pub use record::Meta;
pub use storage::{ArrayKey, BlobRecordTimestamp, Builder, Key, ReadResult, RefKey, Storage};

mod prelude {
    use crc::{Crc, CRC_32_ISCSI};
    pub const CRC32C: Crc<u32> = Crc::<u32>::new(&CRC_32_ISCSI);

    pub(crate) use super::*;
    pub(crate) use std::collections::BTreeMap;

    pub(crate) use anyhow::{Context as ErrorContexts, Result};
    pub(crate) use bincode::{deserialize, serialize, serialize_into, serialized_size};
    pub(crate) use blob::{self, Blob, BlobConfig, IndexConfig};
    pub(crate) use filter::{Bloom, BloomProvider, Config as BloomConfig, HierarchicalFilters};
    pub(crate) use futures::{stream::futures_unordered::FuturesUnordered};
    pub(crate) use record::{Header as RecordHeader, Record, RECORD_MAGIC_BYTE};

    pub(crate) use io::{File, WritableData, WritableDataCreator};

    pub(crate) use std::{
        cmp::Ordering as CmpOrdering,
        collections::HashMap,
        convert::TryInto,
        fmt::{Debug, Display, Formatter, Result as FmtResult},
        fs::File as StdFile,
        io::Error as IOError,
        io::ErrorKind as IOErrorKind,
        io::Result as IOResult,
        marker::PhantomData,
        path::{Path, PathBuf},
        sync::{
            atomic::{AtomicUsize, Ordering},
            Arc,
        },
    };
    pub(crate) use thiserror::Error as ThisError;
    pub(crate) use tokio::{
        fs::{read_dir, DirEntry},
        sync::{RwLock, Semaphore},
        time::{Instant, Duration},
    };
    pub(crate) use tokio_stream::StreamExt;
    pub(crate) use error::IntoBincodeIfUnexpectedEofTrait;
}