bias_vfs/impls/
overlay.rs

1//! An overlay file system combining two filesystems, an upper layer with read/write access and a lower layer with only read access
2
3use crate::error::VfsErrorKind;
4use crate::{FileSystem, SeekAndRead, SeekAndWrite, VfsMetadata, VfsPath, VfsResult};
5use std::collections::HashSet;
6use std::path::PathBuf;
7use std::time::SystemTime;
8
9/// An overlay file system combining several filesystems into one, an upper layer with read/write access and lower layers with only read access
10///
11/// Files in upper layers shadow those in lower layers. Directories are the merged view of all layers.
12///
13/// NOTE: To allow removing files and directories (e.g. via remove_file()) from the lower layer filesystems, this mechanism creates a `.whiteout` folder in the root of the upper level filesystem to mark removed files
14///
15#[derive(Debug, Clone)]
16pub struct OverlayFS {
17    layers: Vec<VfsPath>,
18}
19
20impl OverlayFS {
21    /// Create a new overlay FileSystem from the given layers, only the first layer is written to
22    pub fn new(layers: &[VfsPath]) -> Self {
23        if layers.is_empty() {
24            panic!("OverlayFS needs at least one layer")
25        }
26        OverlayFS {
27            layers: layers.to_vec(),
28        }
29    }
30
31    fn write_layer(&self) -> &VfsPath {
32        &self.layers[0]
33    }
34
35    fn read_path(&self, path: &str) -> VfsResult<VfsPath> {
36        if path.is_empty() {
37            return Ok(self.layers[0].clone());
38        }
39        if self.whiteout_path(path)?.exists()? {
40            return Err(VfsErrorKind::FileNotFound.into());
41        }
42        for layer in &self.layers {
43            let layer_path = layer.join(&path[1..])?;
44            if layer_path.exists()? {
45                return Ok(layer_path);
46            }
47        }
48        let read_path = self.write_layer().join(&path[1..])?;
49        if !read_path.exists()? {
50            return Err(VfsErrorKind::FileNotFound.into());
51        }
52        Ok(read_path)
53    }
54
55    fn write_path(&self, path: &str) -> VfsResult<VfsPath> {
56        if path.is_empty() {
57            return Ok(self.layers[0].clone());
58        }
59        self.write_layer().join(&path[1..])
60    }
61
62    fn whiteout_path(&self, path: &str) -> VfsResult<VfsPath> {
63        if path.is_empty() {
64            return self.write_layer().join(".whiteout/_wo");
65        }
66        self.write_layer()
67            .join(format!(".whiteout/{}_wo", &path[1..]))
68    }
69
70    fn ensure_has_parent(&self, path: &str) -> VfsResult<()> {
71        let separator = path.rfind('/');
72        if let Some(index) = separator {
73            let parent_path = &path[..index];
74            if self.exists(parent_path)? {
75                self.write_path(parent_path)?.create_dir_all()?;
76                return Ok(());
77            }
78        }
79        Err(VfsErrorKind::Other("Parent path does not exist".into()).into())
80    }
81}
82
83impl FileSystem for OverlayFS {
84    fn read_dir(&self, path: &str) -> VfsResult<Box<dyn Iterator<Item = String> + Send>> {
85        let actual_path = if !path.is_empty() { &path[1..] } else { path };
86        if !self.read_path(path)?.exists()? {
87            return Err(VfsErrorKind::FileNotFound.into());
88        }
89        let mut entries = HashSet::<String>::new();
90        for layer in &self.layers {
91            let layer_path = layer.join(actual_path)?;
92            if layer_path.exists()? {
93                for path in layer_path.read_dir()? {
94                    entries.insert(path.filename());
95                }
96            }
97        }
98        // remove whiteout entries that have been removed
99        let whiteout_path = self.write_layer().join(format!(".whiteout{}", path))?;
100        if whiteout_path.exists()? {
101            for path in whiteout_path.read_dir()? {
102                let filename = path.filename();
103                if filename.ends_with("_wo") {
104                    entries.remove(&filename[..filename.len() - 3]);
105                }
106            }
107        }
108        Ok(Box::new(entries.into_iter()))
109    }
110
111    fn create_dir(&self, path: &str) -> VfsResult<()> {
112        self.ensure_has_parent(path)?;
113        self.write_path(path)?.create_dir()?;
114        let whiteout_path = self.whiteout_path(path)?;
115        if whiteout_path.exists()? {
116            whiteout_path.remove_file()?;
117        }
118        Ok(())
119    }
120
121    fn open_file(&self, path: &str) -> VfsResult<Box<dyn SeekAndRead + Send>> {
122        self.read_path(path)?.open_file()
123    }
124
125    fn create_file(&self, path: &str) -> VfsResult<Box<dyn SeekAndWrite + Send>> {
126        self.ensure_has_parent(path)?;
127        let result = self.write_path(path)?.create_file()?;
128        let whiteout_path = self.whiteout_path(path)?;
129        if whiteout_path.exists()? {
130            whiteout_path.remove_file()?;
131        }
132        Ok(result)
133    }
134
135    fn append_file(&self, path: &str) -> VfsResult<Box<dyn SeekAndWrite + Send>> {
136        let write_path = self.write_path(path)?;
137        if !write_path.exists()? {
138            self.ensure_has_parent(path)?;
139            self.read_path(path)?.copy_file(&write_path)?;
140        }
141        write_path.append_file()
142    }
143
144    fn metadata(&self, path: &str) -> VfsResult<VfsMetadata> {
145        self.read_path(path)?.metadata()
146    }
147
148    fn set_creation_time(&self, path: &str, time: SystemTime) -> VfsResult<()> {
149        self.write_path(path)?.set_creation_time(time)
150    }
151
152    fn set_modification_time(&self, path: &str, time: SystemTime) -> VfsResult<()> {
153        self.write_path(path)?.set_modification_time(time)
154    }
155
156    fn set_access_time(&self, path: &str, time: SystemTime) -> VfsResult<()> {
157        self.write_path(path)?.set_access_time(time)
158    }
159
160    fn exists(&self, path: &str) -> VfsResult<bool> {
161        if self
162            .whiteout_path(path)
163            .map_err(|err| err.with_context(|| "whiteout_path"))?
164            .exists()?
165        {
166            return Ok(false);
167        }
168        self.read_path(path)
169            .map(|path| path.exists())
170            .unwrap_or(Ok(false))
171    }
172
173    fn remove_file(&self, path: &str) -> VfsResult<()> {
174        // Ensure path exists
175        self.read_path(path)?;
176        let write_path = self.write_path(path)?;
177        if write_path.exists()? {
178            write_path.remove_file()?;
179        }
180        let whiteout_path = self.whiteout_path(path)?;
181        whiteout_path.parent().create_dir_all()?;
182        whiteout_path.create_file()?;
183        Ok(())
184    }
185
186    fn remove_dir(&self, path: &str) -> VfsResult<()> {
187        // Ensure path exists
188        self.read_path(path)?;
189        let write_path = self.write_path(path)?;
190        if write_path.exists()? {
191            write_path.remove_dir()?;
192        }
193        let whiteout_path = self.whiteout_path(path)?;
194        whiteout_path.parent().create_dir_all()?;
195        whiteout_path.create_file()?;
196        Ok(())
197    }
198
199    fn real_path(&self, path: &str) -> VfsResult<PathBuf> {
200        let vspath = self.read_path(path)?;
201
202        vspath.filesystem().real_path(path)
203    }
204}
205
206#[cfg(test)]
207mod tests {
208    use super::*;
209    use crate::MemoryFS;
210    test_vfs!({
211        let upper_root: VfsPath = MemoryFS::new().into();
212        let lower_root: VfsPath = MemoryFS::new().into();
213        OverlayFS::new(&[upper_root, lower_root])
214    });
215
216    fn create_roots() -> (VfsPath, VfsPath, VfsPath) {
217        let lower_root: VfsPath = MemoryFS::new().into();
218        let upper_root: VfsPath = MemoryFS::new().into();
219        let overlay_root: VfsPath =
220            OverlayFS::new(&[upper_root.clone(), lower_root.clone()]).into();
221        (lower_root, upper_root, overlay_root)
222    }
223
224    #[test]
225    fn read() -> VfsResult<()> {
226        let (lower_root, upper_root, overlay_root) = create_roots();
227        let lower_path = lower_root.join("foo.txt")?;
228        let upper_path = upper_root.join("foo.txt")?;
229        let overlay_path = overlay_root.join("foo.txt")?;
230        lower_path.create_file()?.write_all(b"Hello Lower")?;
231        assert_eq!(&overlay_path.read_to_string()?, "Hello Lower");
232        upper_path.create_file()?.write_all(b"Hello Upper")?;
233        assert_eq!(&overlay_path.read_to_string()?, "Hello Upper");
234        lower_path.remove_file()?;
235        assert_eq!(&overlay_path.read_to_string()?, "Hello Upper");
236        upper_path.remove_file()?;
237        assert!(!overlay_path.exists()?, "File should not exist anymore");
238        Ok(())
239    }
240
241    #[test]
242    fn read_dir() -> VfsResult<()> {
243        let (lower_root, upper_root, overlay_root) = create_roots();
244        upper_root.join("foo/upper")?.create_dir_all()?;
245        upper_root.join("foo/common")?.create_dir_all()?;
246        lower_root.join("foo/common")?.create_dir_all()?;
247        lower_root.join("foo/lower")?.create_dir_all()?;
248        let entries: Vec<_> = overlay_root.join("foo")?.read_dir()?.collect();
249        let mut paths: Vec<_> = entries.iter().map(|path| path.as_str()).collect();
250        paths.sort();
251        assert_eq!(paths, vec!["/foo/common", "/foo/lower", "/foo/upper"]);
252        Ok(())
253    }
254
255    #[test]
256    fn read_dir_root() -> VfsResult<()> {
257        let (lower_root, upper_root, overlay_root) = create_roots();
258        upper_root.join("upper")?.create_dir_all()?;
259        upper_root.join("common")?.create_dir_all()?;
260        lower_root.join("common")?.create_dir_all()?;
261        lower_root.join("lower")?.create_dir_all()?;
262        let entries: Vec<_> = overlay_root.read_dir()?.collect();
263        let mut paths: Vec<_> = entries.iter().map(|path| path.as_str()).collect();
264        paths.sort();
265        assert_eq!(paths, vec!["/common", "/lower", "/upper"]);
266        Ok(())
267    }
268
269    #[test]
270    fn create_dir() -> VfsResult<()> {
271        let (lower_root, _upper_root, overlay_root) = create_roots();
272        lower_root.join("foo")?.create_dir_all()?;
273        assert!(overlay_root.join("foo")?.exists()?, "dir should exist");
274        overlay_root.join("foo/bar")?.create_dir()?;
275        assert!(overlay_root.join("foo/bar")?.exists()?, "dir should exist");
276        Ok(())
277    }
278
279    #[test]
280    fn create_file() -> VfsResult<()> {
281        let (lower_root, _upper_root, overlay_root) = create_roots();
282        lower_root.join("foo")?.create_dir_all()?;
283        assert!(overlay_root.join("foo")?.exists()?, "dir should exist");
284        overlay_root.join("foo/bar")?.create_file()?;
285        assert!(overlay_root.join("foo/bar")?.exists()?, "file should exist");
286        Ok(())
287    }
288
289    #[test]
290    fn append_file() -> VfsResult<()> {
291        let (lower_root, _upper_root, overlay_root) = create_roots();
292        lower_root.join("foo")?.create_dir_all()?;
293        lower_root
294            .join("foo/bar.txt")?
295            .create_file()?
296            .write_all(b"Hello Lower\n")?;
297        overlay_root
298            .join("foo/bar.txt")?
299            .append_file()?
300            .write_all(b"Hello Overlay\n")?;
301        assert_eq!(
302            &overlay_root.join("foo/bar.txt")?.read_to_string()?,
303            "Hello Lower\nHello Overlay\n"
304        );
305        Ok(())
306    }
307
308    #[test]
309    fn remove_file() -> VfsResult<()> {
310        let (lower_root, _upper_root, overlay_root) = create_roots();
311        lower_root.join("foo")?.create_dir_all()?;
312        lower_root
313            .join("foo/bar.txt")?
314            .create_file()?
315            .write_all(b"Hello Lower\n")?;
316        assert!(
317            overlay_root.join("foo/bar.txt")?.exists()?,
318            "file should exist"
319        );
320
321        overlay_root.join("foo/bar.txt")?.remove_file()?;
322        assert!(
323            !overlay_root.join("foo/bar.txt")?.exists()?,
324            "file should not exist anymore"
325        );
326
327        overlay_root
328            .join("foo/bar.txt")?
329            .create_file()?
330            .write_all(b"Hello Overlay\n")?;
331        assert!(
332            overlay_root.join("foo/bar.txt")?.exists()?,
333            "file should exist"
334        );
335        assert_eq!(
336            &overlay_root.join("foo/bar.txt")?.read_to_string()?,
337            "Hello Overlay\n"
338        );
339        Ok(())
340    }
341
342    #[test]
343    fn remove_dir() -> VfsResult<()> {
344        let (lower_root, _upper_root, overlay_root) = create_roots();
345        lower_root.join("foo")?.create_dir_all()?;
346        lower_root.join("foo/bar")?.create_dir_all()?;
347        assert!(overlay_root.join("foo/bar")?.exists()?, "dir should exist");
348
349        overlay_root.join("foo/bar")?.remove_dir()?;
350        assert!(
351            !overlay_root.join("foo/bar")?.exists()?,
352            "dir should not exist anymore"
353        );
354
355        overlay_root.join("foo/bar")?.create_dir()?;
356        assert!(overlay_root.join("foo/bar")?.exists()?, "dir should exist");
357        Ok(())
358    }
359
360    #[test]
361    fn read_dir_removed_entries() -> VfsResult<()> {
362        let (lower_root, _upper_root, overlay_root) = create_roots();
363        lower_root.join("foo")?.create_dir_all()?;
364        lower_root.join("foo/bar")?.create_dir_all()?;
365        lower_root.join("foo/bar.txt")?.create_dir_all()?;
366
367        let entries: Vec<_> = overlay_root.join("foo")?.read_dir()?.collect();
368        let mut paths: Vec<_> = entries.iter().map(|path| path.as_str()).collect();
369        paths.sort();
370        assert_eq!(paths, vec!["/foo/bar", "/foo/bar.txt"]);
371        overlay_root.join("foo/bar")?.remove_dir()?;
372        overlay_root.join("foo/bar.txt")?.remove_file()?;
373
374        let entries: Vec<_> = overlay_root.join("foo")?.read_dir()?.collect();
375        let mut paths: Vec<_> = entries.iter().map(|path| path.as_str()).collect();
376        paths.sort();
377        assert_eq!(paths, vec![] as Vec<&str>);
378
379        Ok(())
380    }
381}
382
383#[cfg(test)]
384mod tests_physical {
385    use super::*;
386    use crate::PhysicalFS;
387    test_vfs!({
388        let temp_dir = std::env::temp_dir();
389        let dir = temp_dir.join(uuid::Uuid::new_v4().to_string());
390        let lower_path = dir.join("lower");
391        std::fs::create_dir_all(&lower_path).unwrap();
392        let upper_path = dir.join("upper");
393        std::fs::create_dir_all(&upper_path).unwrap();
394
395        let upper_root: VfsPath = PhysicalFS::new(upper_path).into();
396        let lower_root: VfsPath = PhysicalFS::new(lower_path).into();
397        OverlayFS::new(&[upper_root, lower_root])
398    });
399}
400
401#[cfg(test)]
402mod tests_overlay {
403    use super::*;
404    use crate::{AltrootFS, PhysicalFS};
405
406    #[test]
407    fn test_real_paths() {
408        let temp_dir = std::env::temp_dir();
409        let dir = temp_dir.join(uuid::Uuid::new_v4().to_string());
410        let lower_path = dir.join("overlay1").join("lower");
411        std::fs::create_dir_all(&lower_path).unwrap();
412        let upper_path = dir.join("overlay2").join("upper");
413        std::fs::create_dir_all(&upper_path).unwrap();
414
415        let root1 = dir.join("overlay1");
416        let root2 = dir.join("overlay2");
417
418        let upper_root: VfsPath = AltrootFS::new(PhysicalFS::new(root1).into()).into();
419        let lower_root: VfsPath = AltrootFS::new(PhysicalFS::new(root2).into()).into();
420
421        let fs = OverlayFS::new(&[upper_root, lower_root]);
422
423        assert_eq!(fs.real_path("/lower").unwrap(), lower_path);
424        assert_eq!(fs.real_path("/upper").unwrap(), upper_path);
425    }
426}
bias_vfs/impls/overlay.rs

bias_vfs/impls/
overlay.rs
