1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
use std::path::PathBuf;
use super::{IpfsStoreConf, Location};
use datafusion::datasource::object_store::ObjectStoreUrl;
use object_store::path::Path as ObjStorePath;
use once_cell::sync::Lazy;
use serde::{Deserialize, Serialize};
pub static DEFAULT_DATADIR: Lazy<std::path::PathBuf> = Lazy::new(|| {
std::path::PathBuf::new()
.join(dirs::home_dir().expect("failed to detect home directory!"))
.join(".chaindexer")
});
/// just for serde defaults
fn defaultdir() -> PathBuf {
DEFAULT_DATADIR.to_path_buf()
}
/// configures a storage layer. each variant represents a different storage type
#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Eq, Hash)]
#[serde(tag = "type")]
pub enum StorageConf {
/// store on local file system. dirpath will default to `~/.chaindexer` if not set
#[serde(rename = "file")]
File {
/// Location on disk where data files will be placed (typically would just be
/// the users application data dir.)
#[serde(default = "defaultdir")]
dirpath: PathBuf,
/// Location, relative to `dirpath` where the index file is stored.
filename: String,
},
#[serde(rename = "ipfs")]
Ipfs(IpfsStoreConf),
/// Memory store is basically only for testing.
/// Just holds a hash map from [`String`] => [`bytes::Bytes`].
///
/// Has a bucket for testing actual stores with buckets (e.g. s3)
///
/// Not sure when one would want to use it forreal
///
#[serde(rename = "memory")]
Memory {
/// bucket is basically just a prefix in the in memory hashmap key.
bucket: String,
},
/// Configuration for persisting a mapping in S3. Will use `AWS_*` variables from
/// the environment to authenticate, e.g. `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, etc.
#[serde(rename = "s3")]
S3 {
bucket: String,
/// prefix to append to all keys generated by the store. includes the
/// main database file (at `filename`).
prefix: Option<String>,
filename: String,
},
}
impl StorageConf {
/// For the given data location, does this storage conf support it?
///
/// For example if the location is `file:///var/data`, then
/// `File {dirpath: "/var" }` would match it, but `Ipfs{..}` would not.
pub fn location_is_valid(&self, loc: &Location) -> bool {
// schemes must always match regardless of anything else
if loc.scheme() != self.scheme() {
return false;
}
// buckets must also always match (including nulls)
if loc.bucket() != self.bucket() {
return false;
}
match self {
StorageConf::Ipfs(_) | StorageConf::Memory { .. } => true,
StorageConf::File { dirpath, .. } => {
let parsed_result = if dirpath.is_absolute() {
ObjStorePath::from_absolute_path(dirpath)
} else {
ObjStorePath::from_filesystem_path(dirpath)
};
if let Ok(dirpath) = parsed_result {
loc.path().prefix_matches(&dirpath)
} else {
false
}
}
StorageConf::S3 { prefix, .. } => {
if let Some(prefix) = prefix {
let prefix = ObjStorePath::parse(prefix)
.expect("failed to convert s3 path prefix into object store path");
loc.path().prefix_matches(&prefix)
} else {
// no prefix matches everything
true
}
}
}
}
/// get the url scheme that this conf is for. mostly used for loggin' and debuggin'
pub fn scheme(&self) -> &str {
match self {
StorageConf::File { .. } => "file",
StorageConf::Ipfs(_) => "ipfs",
StorageConf::Memory { .. } => "memory",
StorageConf::S3 { .. } => "s3",
}
}
/// get the bucket (if any) that this storage conf corresponds to
pub fn bucket(&self) -> Option<&str> {
match self {
StorageConf::S3 { bucket, .. } => Some(bucket),
StorageConf::Memory { bucket } => Some(bucket),
StorageConf::File { .. } | StorageConf::Ipfs(_) => None,
}
}
pub fn object_store_url(&self) -> ObjectStoreUrl {
let scheme = self.scheme();
let host = self.bucket().unwrap_or("");
ObjectStoreUrl::parse(format!("{scheme}://{host}")).unwrap()
}
}