1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334
//! Traits for in-memory or persistent maps of blob with bao encoded outboards.
use std::{io, path::PathBuf};
use crate::{
util::{
progress::{IdGenerator, ProgressSender},
RpcError,
},
Hash,
};
use bao_tree::{blake3, ChunkNum};
use bytes::Bytes;
use futures::future::BoxFuture;
use iroh_io::AsyncSliceReader;
use range_collections::RangeSet2;
use serde::{Deserialize, Serialize};
use tokio::sync::mpsc;
pub use bao_tree;
pub use range_collections;
/// The availability status of an entry in a store.
#[derive(Debug, Clone, Eq, PartialEq)]
pub enum EntryStatus {
/// The entry is completely available.
Complete,
/// The entry is partially available.
Partial,
/// The entry is not in the store.
NotFound,
}
/// An entry for one hash in a bao collection
///
/// The entry has the ability to provide you with an (outboard, data)
/// reader pair. Creating the reader is async and may fail. The futures that
/// create the readers must be `Send`, but the readers themselves don't have to
/// be.
pub trait MapEntry<D: Map>: Clone + Send + Sync + 'static {
/// The hash of the entry.
fn hash(&self) -> blake3::Hash;
/// The size of the entry.
fn size(&self) -> u64;
/// Returns `true` if the entry is complete.
///
/// Note that this does not actually verify if the bytes on disk are complete, it only checks
/// if the entry is among the partial or complete section of the [`Map`]. To verify if all
/// bytes are actually available on disk, use [`MapEntry::available_ranges`].
fn is_complete(&self) -> bool;
/// Compute the available ranges.
///
/// Depending on the implementation, this may be an expensive operation.
///
/// It can also only ever be a best effort, since the underlying data may
/// change at any time. E.g. somebody could flip a bit in the file, or download
/// more chunks.
fn available_ranges(&self) -> BoxFuture<'_, io::Result<RangeSet2<ChunkNum>>>;
/// A future that resolves to a reader that can be used to read the outboard
fn outboard(&self) -> BoxFuture<'_, io::Result<D::Outboard>>;
/// A future that resolves to a reader that can be used to read the data
fn data_reader(&self) -> BoxFuture<'_, io::Result<D::DataReader>>;
}
/// A generic collection of blobs with precomputed outboards
pub trait Map: Clone + Send + Sync + 'static {
/// The outboard type. This can be an in memory outboard or an outboard that
/// retrieves the data asynchronously from a remote database.
type Outboard: bao_tree::io::fsm::Outboard;
/// The reader type.
type DataReader: AsyncSliceReader;
/// The entry type. An entry is a cheaply cloneable handle that can be used
/// to open readers for both the data and the outboard
type Entry: MapEntry<Self>;
/// Get an entry for a hash.
///
/// This can also be used for a membership test by just checking if there
/// is an entry. Creating an entry should be cheap, any expensive ops should
/// be deferred to the creation of the actual readers.
///
/// It is not guaranteed that the entry is complete. A [PartialMap] would return
/// here both complete and partial entries, so that you can share partial entries.
///
/// This function should not block to perform io. The knowledge about
/// existing entries must be present in memory.
fn get(&self, hash: &Hash) -> Option<Self::Entry>;
/// Find out if the data behind a `hash` is complete, partial, or not present.
///
/// Note that this does not actually verify the on-disc data, but only checks in which section
/// of the store the entry is present.
fn contains(&self, hash: &Hash) -> EntryStatus;
}
/// A partial entry
pub trait PartialMapEntry<D: PartialMap>: MapEntry<D> {
/// A future that resolves to an writeable outboard
fn outboard_mut(&self) -> BoxFuture<'_, io::Result<D::OutboardMut>>;
/// A future that resolves to a writer that can be used to write the data
fn data_writer(&self) -> BoxFuture<'_, io::Result<D::DataWriter>>;
}
/// A mutable bao map
pub trait PartialMap: Map {
/// The outboard type to write data to the partial entry.
type OutboardMut: bao_tree::io::fsm::OutboardMut;
/// The writer type to write data to the partial entry.
type DataWriter: iroh_io::AsyncSliceWriter;
/// A partial entry. This is an entry that is writeable and possibly incomplete.
///
/// It must also be readable.
type PartialEntry: PartialMapEntry<Self>;
/// Get an existing partial entry, or create a new one.
///
/// We need to know the size of the partial entry. This might produce an
/// error e.g. if there is not enough space on disk.
fn get_or_create_partial(&self, hash: Hash, size: u64) -> io::Result<Self::PartialEntry>;
/// Get an existing partial entry.
///
/// This will return `None` if there is no partial entry for this hash.
///
/// This function should not block to perform io. The knowledge about
/// partial entries must be present in memory.
fn get_partial(&self, hash: &Hash) -> Option<Self::PartialEntry>;
/// Upgrade a partial entry to a complete entry.
fn insert_complete(&self, entry: Self::PartialEntry) -> BoxFuture<'_, io::Result<()>>;
}
/// Extension of BaoMap to add misc methods used by the rpc calls.
pub trait ReadableStore: Map {
/// list all blobs in the database. This should include collections, since
/// collections are blobs and can be requested as blobs.
///
/// This function should not block to perform io. The knowledge about
/// existing blobs must be present in memory.
fn blobs(&self) -> Box<dyn Iterator<Item = Hash> + Send + Sync + 'static>;
/// list all roots (collections or other explicitly added things) in the database
///
/// This function should not block to perform io. The knowledge about
/// existing roots must be present in memory.
fn roots(&self) -> Box<dyn Iterator<Item = Hash> + Send + Sync + 'static>;
/// Validate the database
fn validate(&self, tx: mpsc::Sender<ValidateProgress>) -> BoxFuture<'_, anyhow::Result<()>>;
/// list partial blobs in the database
fn partial_blobs(&self) -> Box<dyn Iterator<Item = Hash> + Send + Sync + 'static>;
/// This trait method extracts a file to a local path.
///
/// `hash` is the hash of the file
/// `target` is the path to the target file
/// `mode` is a hint how the file should be exported.
/// `progress` is a callback that is called with the total number of bytes that have been written
fn export(
&self,
hash: Hash,
target: PathBuf,
mode: ExportMode,
progress: impl Fn(u64) -> io::Result<()> + Send + Sync + 'static,
) -> BoxFuture<'_, io::Result<()>>;
}
/// The mutable part of a BaoDb
pub trait Store: ReadableStore + PartialMap {
/// This trait method imports a file from a local path.
///
/// `data` is the path to the file.
/// `mode` is a hint how the file should be imported.
/// `progress` is a sender that provides a way for the importer to send progress messages
/// when importing large files. This also serves as a way to cancel the import. If the
/// consumer of the progress messages is dropped, subsequent attempts to send progress
/// will fail.
///
/// Returns the hash of the imported file. The reason to have this method is that some database
/// implementations might be able to import a file without copying it.
fn import(
&self,
data: PathBuf,
mode: ImportMode,
progress: impl ProgressSender<Msg = ImportProgress> + IdGenerator,
) -> BoxFuture<'_, io::Result<(Hash, u64)>>;
/// This trait method imports data from memory.
///
/// It is a special case of `import` that does not use the file system.
fn import_bytes(&self, bytes: Bytes) -> BoxFuture<'_, io::Result<Hash>>;
}
/// Progress messages for an import operation
///
/// An import operation involves computing the outboard of a file, and then
/// either copying or moving the file into the database.
#[allow(missing_docs)]
#[derive(Debug)]
pub enum ImportProgress {
/// Found a path
///
/// This will be the first message for an id
Found { id: u64, path: PathBuf },
/// Progress when copying the file to the store
///
/// This will be omitted if the store can use the file in place
///
/// There will be multiple of these messages for an id
CopyProgress { id: u64, offset: u64 },
/// Determined the size
///
/// This will come after `Found` and zero or more `CopyProgress` messages.
/// For unstable files, determining the size will only be done once the file
/// is fully copied.
Size { id: u64, size: u64 },
/// Progress when computing the outboard
///
/// There will be multiple of these messages for an id
OutboardProgress { id: u64, offset: u64 },
/// Done computing the outboard
///
/// This comes after `Size` and zero or more `OutboardProgress` messages
OutboardDone { id: u64, hash: Hash },
}
/// The import mode describes how files will be imported.
///
/// This is a hint to the import trait method. For some implementations, this
/// does not make any sense. E.g. an in memory implementation will always have
/// to copy the file into memory. Also, a disk based implementation might choose
/// to copy small files even if the mode is `Reference`.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum ImportMode {
/// This mode will copy the file into the database before hashing.
///
/// This is the safe default because the file can not be accidentally modified
/// after it has been imported.
#[default]
Copy,
/// This mode will try to reference the file in place and assume it is unchanged after import.
///
/// This has a large performance and storage benefit, but it is less safe since
/// the file might be modified after it has been imported.
///
/// Stores are allowed to ignore this mode and always copy the file, e.g.
/// if the file is very small or if the store does not support referencing files.
TryReference,
}
/// The import mode describes how files will be imported.
///
/// This is a hint to the import trait method. For some implementations, this
/// does not make any sense. E.g. an in memory implementation will always have
/// to copy the file into memory. Also, a disk based implementation might choose
/// to copy small files even if the mode is `Reference`.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum ExportMode {
/// This mode will copy the file to the target directory.
///
/// This is the safe default because the file can not be accidentally modified
/// after it has been exported.
#[default]
Copy,
/// This mode will try to move the file to the target directory and then reference it from
/// the database.
///
/// This has a large performance and storage benefit, but it is less safe since
/// the file might be modified in the target directory after it has been exported.
///
/// Stores are allowed to ignore this mode and always copy the file, e.g.
/// if the file is very small or if the store does not support referencing files.
TryReference,
}
#[allow(missing_docs)]
#[derive(Debug)]
pub enum ExportProgress {
/// Starting to export to a file
///
/// This will be the first message for an id
Start {
id: u64,
hash: Hash,
path: PathBuf,
stable: bool,
},
/// Progress when copying the file to the target
///
/// This will be omitted if the store can move the file or use copy on write
///
/// There will be multiple of these messages for an id
Progress { id: u64, offset: u64 },
/// Done exporting
Done { id: u64 },
}
/// Progress updates for the provide operation
#[derive(Debug, Serialize, Deserialize)]
pub enum ValidateProgress {
/// started validating
Starting {
/// The total number of entries to validate
total: u64,
},
/// We started validating an entry
Entry {
/// a new unique id for this entry
id: u64,
/// the hash of the entry
hash: Hash,
/// location of the entry.
///
/// In case of a file, this is the path to the file.
/// Otherwise it might be an url or something else to uniquely identify the entry.
path: Option<String>,
/// the size of the entry
size: u64,
},
/// We got progress ingesting item `id`.
Progress {
/// The unique id of the entry.
id: u64,
/// The offset of the progress, in bytes.
offset: u64,
},
/// We are done with `id`
Done {
/// The unique id of the entry.
id: u64,
/// An error if we failed to validate the entry.
error: Option<String>,
},
/// We are done with the whole operation.
AllDone,
/// We got an error and need to abort.
Abort(RpcError),
}