nydus_storage/
lib.rs

1// Copyright 2020 Ant Group. All rights reserved.
2//
3// SPDX-License-Identifier: Apache-2.0
4
5//! Chunked blob storage service to support Rafs filesystem.
6//!
7//! The Rafs filesystem is blob based filesystem with chunk deduplication. A Rafs filesystem is
8//! composed up of a metadata blob and zero or more data blobs. A blob is just a plain object
9//! storage containing data chunks. Data chunks may be compressed, encrypted and deduplicated by
10//! content digest value. When Rafs file is used for container images, Rafs metadata blob contains
11//! all filesystem metadatas, such as directory, file name, permission etc. Actually file contents
12//! are split into chunks and stored into data blobs. Rafs may build one data blob for each
13//! container image layer or build a  single data blob for the whole image, according to building
14//! options.
15//!
16//! The nydus-storage crate is used to manage and access chunked blobs for Rafs filesystem, which
17//! contains three layers:
18//! - [Backend](backend/index.html): access raw blob objects on remote storage backends.
19//! - [Cache](cache/index.html): cache remote blob contents onto local storage in forms
20//!   optimized for performance.
21//! - [Device](device/index.html): public APIs for chunked blobs
22//!
23//! There are several core abstractions provided by the public APIs:
24//! - [BlobInfo](device/struct.BlobInfo.html): provides information about blobs, which is typically
25//!   constructed from the `blob array` in Rafs filesystem metadata.
26//! - [BlobDevice](device/struct.BlobDevice.html): provides access to all blobs of a Rafs filesystem,
27//!   which is constructed from an array of [BlobInfo](device/struct.BlobInfo.html) objects.
28//! - [BlobChunkInfo](device/trait.BlobChunkInfo.html): provides information about a data chunk, which
29//!   is loaded from Rafs metadata.
30//! - [BlobIoDesc](device/struct.BlobIoDesc.html): a blob IO descriptor, containing information for a
31//!   continuous IO range within a chunk.
32//! - [BlobIoVec](device/struct.BlobIoVec.html): a scatter/gather list for blob IO operation, containing
33//!   one or more blob IO descriptors
34//!
35//! To read data from the Rafs filesystem, the Rafs filesystem driver will prepare a
36//! [BlobIoVec](device/struct.BlobIoVec.html)
37//! object and submit it to the corresponding [BlobDevice](device/struct.BlobDevice.html)
38//!  object to actually execute the IO
39//! operations.
40#[macro_use]
41extern crate log;
42#[macro_use]
43extern crate bitflags;
44#[macro_use]
45extern crate nydus_api;
46
47use std::fmt::{Display, Formatter};
48
49pub mod backend;
50pub mod cache;
51pub mod device;
52pub mod factory;
53pub mod meta;
54//pub mod remote;
55#[cfg(test)]
56pub(crate) mod test;
57pub mod utils;
58
59// A helper to impl RafsChunkInfo for upper layers like Rafs different metadata mode.
60#[doc(hidden)]
61#[macro_export]
62macro_rules! impl_getter {
63    ($G: ident, $F: ident, $U: ty) => {
64        fn $G(&self) -> $U {
65            self.$F
66        }
67    };
68}
69
70/// Default blob chunk size.
71pub const RAFS_DEFAULT_CHUNK_SIZE: u64 = 1024 * 1024;
72/// Maximum blob chunk size, 16MB.
73pub const RAFS_MAX_CHUNK_SIZE: u64 = 1024 * 1024 * 16;
74/// Maximum numbers of chunk per data blob
75pub const RAFS_MAX_CHUNKS_PER_BLOB: u32 = 1u32 << 24;
76/// Generate maximum gap between chunks from merging size.
77pub const RAFS_BATCH_SIZE_TO_GAP_SHIFT: u64 = 7;
78
79/// Error codes related to storage subsystem.
80#[derive(Debug)]
81pub enum StorageError {
82    Unsupported,
83    Timeout,
84    VolatileSlice(vm_memory::VolatileMemoryError),
85    MemOverflow,
86    NotContinuous,
87    CacheIndex(std::io::Error),
88}
89
90impl Display for StorageError {
91    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
92        match self {
93            StorageError::Unsupported => write!(f, "unsupported storage operation"),
94            StorageError::Timeout => write!(f, "timeout when reading data from storage backend"),
95            StorageError::MemOverflow => write!(f, "memory overflow when doing storage backend IO"),
96            StorageError::NotContinuous => write!(f, "address ranges are not continuous"),
97            StorageError::VolatileSlice(e) => write!(f, "{}", e),
98            StorageError::CacheIndex(e) => write!(f, "Wrong cache index {}", e),
99        }
100    }
101}
102
103/// Specialized std::result::Result for storage subsystem.
104pub type StorageResult<T> = std::result::Result<T, StorageError>;
nydus_storage/lib.rs

nydus_storage/
lib.rs
