nydus_rafs/
lib.rs

1// Copyright 2020 Ant Group. All rights reserved.
2//
3// SPDX-License-Identifier: Apache-2.0
4
5//! RAFS: a chunk dedup, on-demand loading, readonly fuse filesystem.
6//!
7//! The Rafs filesystem is blob based readonly filesystem with chunk deduplication. A Rafs
8//! filesystem is composed up of a metadata blob and zero or more data blobs. A blob is just a
9//! plain object containing data chunks. Data chunks may be compressed, encrypted and deduplicated
10//! by chunk content digest value. When Rafs file is used for container images, Rafs metadata blob
11//! contains all filesystem metadatas, such as directory, file name, permission etc. Actually file
12//! contents are divided into chunks and stored into data blobs. Rafs may built one data blob for
13//! each container image layer or build a single data blob for the whole image, according to
14//! building options.
15//!
16//! There are several versions of Rafs filesystem defined:
17//! - V4: the original Rafs filesystem format
18//! - V5: an optimized version based on V4 with metadata direct mapping, data prefetching etc.
19//! - V6: a redesigned version to reduce metadata blob size and inter-operable with in kernel erofs,
20//!   better support of virtio-fs.
21//!
22//! The nydus-rafs crate depends on the nydus-storage crate to access metadata and data blobs and
23//! improve performance by caching data on local storage. The nydus-rafs itself includes two main
24//! sub modules:
25//! - [fs](fs/index.html): the Rafs core to glue fuse, storage backend and filesystem metadata.
26//! - [metadata](metadata/index.html): defines and accesses Rafs filesystem metadata.
27//!
28//! For more information, please refer to
29//! [Dragonfly Image Service](https://github.com/dragonflyoss/nydus)
30
31#[macro_use]
32extern crate log;
33#[macro_use]
34extern crate bitflags;
35#[macro_use]
36extern crate nydus_api;
37#[macro_use]
38extern crate nydus_storage as storage;
39
40use std::any::Any;
41use std::borrow::Cow;
42use std::fmt::Debug;
43use std::fs::File;
44use std::io::{BufWriter, Error, Read, Result, Seek, SeekFrom, Write};
45use std::os::unix::io::AsRawFd;
46use std::path::{Path, PathBuf};
47use std::sync::Arc;
48
49use crate::metadata::{RafsInodeExt, RafsSuper};
50
51#[cfg(feature = "virtio-fs")]
52pub mod blobfs;
53pub mod fs;
54pub mod metadata;
55#[cfg(test)]
56pub mod mock;
57
58/// Error codes for rafs related operations.
59#[derive(thiserror::Error, Debug)]
60pub enum RafsError {
61    #[error("Operation is not supported.")]
62    Unsupported,
63    #[error("Rafs is not initialized.")]
64    Uninitialized,
65    #[error("Rafs is already mounted.")]
66    AlreadyMounted,
67    #[error("Failed to read metadata: {0}`")]
68    ReadMetadata(Error, String),
69    #[error("Failed to load config: {0}`")]
70    LoadConfig(Error),
71    #[error("Failed to parse config: {0}`")]
72    ParseConfig(#[source] serde_json::Error),
73    #[error("Failed to create swap backend: {0}`")]
74    SwapBackend(Error),
75    #[error("Failed to fill superBlock: {0}`")]
76    FillSuperBlock(Error),
77    #[error("Failed to create device: {0}`")]
78    CreateDevice(Error),
79    #[error("Failed to prefetch data: {0}`")]
80    Prefetch(String),
81    #[error("Failed to configure device: {0}`")]
82    Configure(String),
83    #[error("Incompatible RAFS version: `{0}`")]
84    Incompatible(u16),
85    #[error("Illegal meta struct, type is `{0:?}` and content is `{1}`")]
86    IllegalMetaStruct(MetaType, String),
87    #[error("Invalid image data")]
88    InvalidImageData,
89}
90
91#[derive(Debug)]
92pub enum MetaType {
93    Regular,
94    Dir,
95    Symlink,
96}
97
98/// Specialized version of std::result::Result<> for Rafs.
99pub type RafsResult<T> = std::result::Result<T, RafsError>;
100
101/// Handler to read file system bootstrap.
102pub type RafsIoReader = Box<dyn RafsIoRead>;
103
104/// A helper trait for RafsIoReader.
105pub trait RafsIoRead: Read + AsRawFd + Seek + Send {}
106
107impl RafsIoRead for File {}
108
109/// Handler to write file system bootstrap.
110pub type RafsIoWriter = Box<dyn RafsIoWrite>;
111
112/// A helper trait for RafsIoWriter.
113pub trait RafsIoWrite: Write + Seek + 'static {
114    fn as_any(&self) -> &dyn Any;
115
116    fn validate_alignment(&mut self, size: usize, alignment: usize) -> Result<usize> {
117        if alignment != 0 {
118            let cur = self.stream_position()?;
119
120            if (size & (alignment - 1) != 0) || (cur & (alignment as u64 - 1) != 0) {
121                return Err(einval!("unaligned data"));
122            }
123        }
124
125        Ok(size)
126    }
127
128    /// write padding to align to RAFS_ALIGNMENT.
129    fn write_padding(&mut self, size: usize) -> Result<()> {
130        if size > WRITE_PADDING_DATA.len() {
131            return Err(einval!("invalid padding size"));
132        }
133        self.write_all(&WRITE_PADDING_DATA[0..size])
134    }
135
136    /// Seek the writer to the end.
137    fn seek_to_end(&mut self) -> Result<u64> {
138        self.seek(SeekFrom::End(0)).map_err(|e| {
139            error!("Seeking to end fails, {}", e);
140            e
141        })
142    }
143
144    /// Seek the writer to the `offset`.
145    fn seek_offset(&mut self, offset: u64) -> Result<u64> {
146        self.seek(SeekFrom::Start(offset)).map_err(|e| {
147            error!("Seeking to offset {} from start fails, {}", offset, e);
148            e
149        })
150    }
151
152    /// Seek the writer to current position plus the specified offset.
153    fn seek_current(&mut self, offset: i64) -> Result<u64> {
154        self.seek(SeekFrom::Current(offset))
155    }
156
157    /// Do some finalization works.
158    fn finalize(&mut self, _name: Option<String>) -> anyhow::Result<()> {
159        Ok(())
160    }
161
162    /// Return a slice to get all data written.
163    ///
164    /// No more data should be written after calling as_bytes().
165    fn as_bytes(&mut self) -> std::io::Result<Cow<'_, [u8]>> {
166        unimplemented!()
167    }
168}
169
170impl RafsIoWrite for File {
171    fn as_any(&self) -> &dyn Any {
172        self
173    }
174}
175
176// Rust file I/O is un-buffered by default. If we have many small write calls
177// to a file, should use BufWriter. BufWriter maintains an in-memory buffer
178// for writing, minimizing the number of system calls required.
179impl RafsIoWrite for BufWriter<File> {
180    fn as_any(&self) -> &dyn Any {
181        self
182    }
183}
184
185const WRITE_PADDING_DATA: [u8; 64] = [0u8; 64];
186
187impl dyn RafsIoRead {
188    /// Seek the reader to next aligned position.
189    pub fn seek_to_next_aligned(&mut self, last_read_len: usize, alignment: usize) -> Result<u64> {
190        let suffix = last_read_len & (alignment - 1);
191        let offset = if suffix == 0 { 0 } else { alignment - suffix };
192
193        self.seek(SeekFrom::Current(offset as i64)).map_err(|e| {
194            error!("Seeking to offset {} from current fails, {}", offset, e);
195            e
196        })
197    }
198
199    /// Move the reader current position forward with `plus_offset` bytes.
200    pub fn seek_plus_offset(&mut self, plus_offset: i64) -> Result<u64> {
201        // Seek should not fail otherwise rafs goes insane.
202        self.seek(SeekFrom::Current(plus_offset)).map_err(|e| {
203            error!(
204                "Seeking to offset {} from current fails, {}",
205                plus_offset, e
206            );
207            e
208        })
209    }
210
211    /// Seek the reader to the `offset`.
212    pub fn seek_to_offset(&mut self, offset: u64) -> Result<u64> {
213        self.seek(SeekFrom::Start(offset)).map_err(|e| {
214            error!("Seeking to offset {} from start fails, {}", offset, e);
215            e
216        })
217    }
218
219    /// Seek the reader to the end.
220    pub fn seek_to_end(&mut self, offset: i64) -> Result<u64> {
221        self.seek(SeekFrom::End(offset)).map_err(|e| {
222            error!("Seeking to end fails, {}", e);
223            e
224        })
225    }
226
227    /// Create a reader from a file path.
228    pub fn from_file(path: impl AsRef<Path>) -> RafsResult<RafsIoReader> {
229        let f = File::open(&path).map_err(|e| {
230            RafsError::ReadMetadata(e, path.as_ref().to_string_lossy().into_owned())
231        })?;
232
233        Ok(Box::new(f))
234    }
235}
236
237///  Iterator to walk all inodes of a Rafs filesystem.
238pub struct RafsIterator<'a> {
239    _rs: &'a RafsSuper,
240    cursor_stack: Vec<(Arc<dyn RafsInodeExt>, PathBuf)>,
241}
242
243impl<'a> RafsIterator<'a> {
244    /// Create a new iterator to visit a Rafs filesystem.
245    pub fn new(rs: &'a RafsSuper) -> Self {
246        let cursor_stack = match rs.get_extended_inode(rs.superblock.root_ino(), false) {
247            Ok(node) => {
248                let path = PathBuf::from("/");
249                vec![(node, path)]
250            }
251            Err(e) => {
252                error!(
253                    "failed to get root inode from the bootstrap {}, damaged or malicious file?",
254                    e
255                );
256                vec![]
257            }
258        };
259
260        RafsIterator {
261            _rs: rs,
262            cursor_stack,
263        }
264    }
265}
266
267impl Iterator for RafsIterator<'_> {
268    type Item = (Arc<dyn RafsInodeExt>, PathBuf);
269
270    fn next(&mut self) -> Option<Self::Item> {
271        let (node, path) = self.cursor_stack.pop()?;
272        if node.is_dir() {
273            let children = 0..node.get_child_count();
274            for idx in children.rev() {
275                if let Ok(child) = node.get_child_by_index(idx) {
276                    let child_path = path.join(child.name());
277                    self.cursor_stack.push((child, child_path));
278                } else {
279                    error!(
280                        "failed to get child inode from the bootstrap, damaged or malicious file?"
281                    );
282                }
283            }
284        }
285        Some((node, path))
286    }
287}
288
289#[cfg(test)]
290mod tests {
291    use super::*;
292    use crate::metadata::RafsMode;
293    use std::fs::OpenOptions;
294    use vmm_sys_util::tempfile::TempFile;
295
296    #[test]
297    fn test_rafs_io_writer() {
298        let mut file = TempFile::new().unwrap().into_file();
299
300        assert!(file.validate_alignment(2, 8).is_err());
301        assert!(file.validate_alignment(7, 8).is_err());
302        assert!(file.validate_alignment(9, 8).is_err());
303        assert!(file.validate_alignment(8, 8).is_ok());
304
305        file.write_all(&[0x0u8; 7]).unwrap();
306        assert!(file.validate_alignment(8, 8).is_err());
307        {
308            let obj: &mut dyn RafsIoWrite = &mut file;
309            obj.write_padding(1).unwrap();
310        }
311        assert!(file.validate_alignment(8, 8).is_ok());
312        file.write_all(&[0x0u8; 1]).unwrap();
313        assert!(file.validate_alignment(8, 8).is_err());
314
315        let obj: &mut dyn RafsIoRead = &mut file;
316        assert_eq!(obj.seek_to_offset(0).unwrap(), 0);
317        assert_eq!(obj.seek_plus_offset(7).unwrap(), 7);
318        assert_eq!(obj.seek_to_next_aligned(7, 8).unwrap(), 8);
319        assert_eq!(obj.seek_plus_offset(7).unwrap(), 15);
320    }
321
322    #[test]
323    fn test_rafs_iterator() {
324        let root_dir = &std::env::var("CARGO_MANIFEST_DIR").expect("$CARGO_MANIFEST_DIR");
325        let path = PathBuf::from(root_dir).join("../tests/texture/bootstrap/rafs-v5.boot");
326        let bootstrap = OpenOptions::new()
327            .read(true)
328            .write(false)
329            .open(path)
330            .unwrap();
331        let mut rs = RafsSuper {
332            mode: RafsMode::Direct,
333            validate_digest: false,
334            ..Default::default()
335        };
336        rs.load(&mut (Box::new(bootstrap) as RafsIoReader)).unwrap();
337        let iter = RafsIterator::new(&rs);
338
339        let mut last = false;
340        for (idx, (_node, path)) in iter.enumerate() {
341            assert!(!last);
342            if idx == 1 {
343                assert_eq!(path, PathBuf::from("/bin"));
344            } else if idx == 2 {
345                assert_eq!(path, PathBuf::from("/boot"));
346            } else if idx == 3 {
347                assert_eq!(path, PathBuf::from("/dev"));
348            } else if idx == 10 {
349                assert_eq!(path, PathBuf::from("/etc/DIR_COLORS.256color"));
350            } else if idx == 11 {
351                assert_eq!(path, PathBuf::from("/etc/DIR_COLORS.lightbgcolor"));
352            } else if path == PathBuf::from("/var/yp") {
353                last = true;
354            }
355        }
356        assert!(last);
357    }
358}
nydus_rafs/lib.rs

nydus_rafs/
lib.rs
