remotefs_memory/
lib.rs

1#![crate_name = "remotefs_memory"]
2#![crate_type = "lib"]
3
4//! # remotefs-memory
5//!
6//! remotefs-memory is a [remotefs](https://github.com/remotefs-rs/remotefs-rs)
7//! client implementation backed entirely by an in-memory tree, useful for
8//! tests and simulations that need a [`RemoteFs`] without a real server.
9//!
10//! It exposes a single type, [`MemoryFs`], which implements the [`RemoteFs`]
11//! trait and can therefore be used interchangeably with any other remotefs
12//! client.
13//!
14//! ## Getting Started
15//!
16//! Add `remotefs-memory` to your `Cargo.toml`:
17//!
18//! ```toml
19//! remotefs = "1"
20//! remotefs-memory = "1"
21//! ```
22//!
23//! ## Example
24//!
25//! ```rust
26//! use std::path::{Path, PathBuf};
27//!
28//! use remotefs_memory::{Inode, MemoryFs, node, Node, Tree};
29//! use remotefs::RemoteFs;
30//! use remotefs::fs::UnixPex;
31//!
32//! let tempdir = PathBuf::from("/tmp");
33//! let tree = Tree::new(node!(
34//!     PathBuf::from("/"),
35//!     Inode::dir(0, 0, UnixPex::from(0o755)),
36//!     node!(tempdir.clone(), Inode::dir(0, 0, UnixPex::from(0o755)))
37//! ));
38//!
39//! let mut client = MemoryFs::new(tree);
40//! client.connect().unwrap();
41//! // Every path is absolute; there is no working directory.
42//! assert!(client.stat(Path::new("/tmp")).unwrap().is_dir());
43//! ```
44//!
45//! ## Transfers
46//!
47//! `open`, `create`, and `append` return owned streams. A write stream stages
48//! bytes and commits them into the tree only when `finish` is called;
49//! dropping it discards the staged bytes and logs at `debug` level.
50//!
51//! ```rust
52//! use std::io::Write;
53//! use std::path::{Path, PathBuf};
54//!
55//! use remotefs::RemoteFs;
56//! use remotefs::fs::{ReadOptions, UnixPex, WriteOptions};
57//! use remotefs_memory::{Inode, MemoryFs, node, Node, Tree};
58//!
59//! let tree = Tree::new(node!(
60//!     PathBuf::from("/"),
61//!     Inode::dir(0, 0, UnixPex::from(0o755))
62//! ));
63//! let mut client = MemoryFs::new(tree);
64//! client.connect().unwrap();
65//!
66//! let mut stream = client
67//!     .create(Path::new("/hello.txt"), &WriteOptions::default())
68//!     .unwrap();
69//! stream.write_all(b"hello").unwrap();
70//! stream.finish().unwrap();
71//!
72//! let mut output = Vec::new();
73//! client
74//!     .read_file(Path::new("/hello.txt"), &ReadOptions::default(), &mut output)
75//!     .unwrap();
76//! assert_eq!(output, b"hello");
77//! ```
78//!
79//! ## Async consumers
80//!
81//! `MemoryFs` is a native blocking client. Wrap it in
82//! `remotefs::adapters::r#async::Unblock` (remotefs `tokio` feature) to use it
83//! from an `AsyncRemoteFs` consumer.
84//!
85
86#![doc(html_playground_url = "https://play.rust-lang.org")]
87#![doc(
88    html_favicon_url = "https://raw.githubusercontent.com/remotefs-rs/remotefs-rs/main/assets/logo-128.png"
89)]
90#![doc(
91    html_logo_url = "https://raw.githubusercontent.com/remotefs-rs/remotefs-rs/main/assets/logo.png"
92)]
93
94#[macro_use]
95extern crate log;
96
97mod inode;
98mod stream;
99#[cfg(test)]
100mod test;
101
102use std::path::{Path, PathBuf};
103use std::sync::atomic::{AtomicBool, Ordering};
104use std::sync::{Arc, Mutex, MutexGuard};
105
106pub use orange_trees::{Node, Tree, node};
107use remotefs::fs::{
108    Capabilities, ExecOutput, ReadOptions, ReadStream, SetMetadata, UnixPex, WriteOptions,
109    WriteStream,
110};
111use remotefs::path::ensure_absolute;
112use remotefs::{File, RemoteError, RemoteErrorType, RemoteFs, RemoteResult};
113
114pub use self::inode::Inode;
115use self::stream::{MemoryReader, MemoryWriter};
116
117/// Alias for the filesystem tree. It is a [`Tree`] of [`PathBuf`] and [`Inode`].
118pub type FsTree = Tree<PathBuf, Inode>;
119
120/// Operations `MemoryFs` performs natively.
121const CAPABILITIES: Capabilities = Capabilities::STREAM_READ
122    .union(Capabilities::STREAM_WRITE)
123    .union(Capabilities::APPEND)
124    .union(Capabilities::RANGE_READ)
125    .union(Capabilities::SEEK_READ)
126    .union(Capabilities::SEEK_WRITE)
127    .union(Capabilities::COPY)
128    .union(Capabilities::SYMLINK)
129    .union(Capabilities::SET_METADATA)
130    .union(Capabilities::POSIX_MODE);
131
132/// Default POSIX mode for files and directories created without one.
133const DEFAULT_MODE: u32 = 0o755;
134
135/// Locks the shared tree, mapping a poisoned lock to a protocol error.
136pub(crate) fn lock_tree(tree: &Mutex<FsTree>) -> RemoteResult<MutexGuard<'_, FsTree>> {
137    tree.lock().map_err(|_| {
138        RemoteError::with_message(
139            RemoteErrorType::ProtocolError,
140            "in-memory filesystem lock poisoned",
141        )
142    })
143}
144
145/// MemoryFs is a simple in-memory filesystem that can be used for testing purposes.
146///
147/// It implements the [`RemoteFs`] trait.
148///
149/// The [`MemoryFs`] is instantiated providing a [`orange_trees::Tree`] which contains the filesystem data.
150///
151/// When reading or writing files, the [`MemoryFs`] will use the [`orange_trees::Tree`] to store the data.
152///
153/// You can easily create the [`MemoryFs`] using the [`MemoryFs::new`] method, providing the tree.
154/// Use the [`node!`] macro to create the tree or use the [`orange_trees`] crate to create it programmatically.
155///
156/// The tree contains nodes identified by a [`PathBuf`] and a value of type [`Inode`].
157/// Every path passed to the client must be absolute; the tree root is `/`.
158pub struct MemoryFs {
159    tree: Arc<Mutex<FsTree>>,
160    connected: AtomicBool,
161    // Fn to get uid
162    get_uid: Box<dyn Fn() -> u32 + Send + Sync>,
163    // Fn to get gid
164    get_gid: Box<dyn Fn() -> u32 + Send + Sync>,
165}
166
167impl MemoryFs {
168    /// Create a client backed by `tree`.
169    ///
170    /// `uid`/`gid` default to `0` for every file until
171    /// [`MemoryFs::with_get_uid`] or [`MemoryFs::with_get_gid`] is used to
172    /// override them.
173    ///
174    /// # Examples
175    ///
176    /// ```rust
177    /// use std::path::PathBuf;
178    ///
179    /// use remotefs::fs::UnixPex;
180    /// use remotefs_memory::{Inode, MemoryFs, Node, Tree, node};
181    ///
182    /// let tree = Tree::new(node!(
183    ///     PathBuf::from("/"),
184    ///     Inode::dir(0, 0, UnixPex::from(0o755))
185    /// ));
186    /// let client = MemoryFs::new(tree);
187    /// ```
188    pub fn new(tree: FsTree) -> Self {
189        Self {
190            tree: Arc::new(Mutex::new(tree)),
191            connected: AtomicBool::new(false),
192            get_uid: Box::new(|| 0),
193            get_gid: Box::new(|| 0),
194        }
195    }
196
197    /// Override the closure used to fill the uid of newly created files.
198    ///
199    /// # Examples
200    ///
201    /// ```rust
202    /// use std::path::PathBuf;
203    ///
204    /// use remotefs::fs::UnixPex;
205    /// use remotefs_memory::{Inode, MemoryFs, Node, Tree, node};
206    ///
207    /// let tree = Tree::new(node!(
208    ///     PathBuf::from("/"),
209    ///     Inode::dir(0, 0, UnixPex::from(0o755))
210    /// ));
211    /// let client = MemoryFs::new(tree).with_get_uid(|| 1000);
212    /// ```
213    pub fn with_get_uid<F>(mut self, get_uid: F) -> Self
214    where
215        F: Fn() -> u32 + Send + Sync + 'static,
216    {
217        self.get_uid = Box::new(get_uid);
218        self
219    }
220
221    /// Override the closure used to fill the gid of newly created files.
222    ///
223    /// # Examples
224    ///
225    /// ```rust
226    /// use std::path::PathBuf;
227    ///
228    /// use remotefs::fs::UnixPex;
229    /// use remotefs_memory::{Inode, MemoryFs, Node, Tree, node};
230    ///
231    /// let tree = Tree::new(node!(
232    ///     PathBuf::from("/"),
233    ///     Inode::dir(0, 0, UnixPex::from(0o755))
234    /// ));
235    /// let client = MemoryFs::new(tree).with_get_gid(|| 1000);
236    /// ```
237    pub fn with_get_gid<F>(mut self, get_gid: F) -> Self
238    where
239        F: Fn() -> u32 + Send + Sync + 'static,
240    {
241        self.get_gid = Box::new(get_gid);
242        self
243    }
244
245    /// Validates that `path` is absolute and that the client is connected.
246    fn validate(&self, path: &Path) -> RemoteResult<()> {
247        ensure_absolute(path)?;
248        if self.is_connected() {
249            Ok(())
250        } else {
251            Err(RemoteError::new(RemoteErrorType::NotConnected))
252        }
253    }
254
255    /// Locks the shared tree, mapping a poisoned lock to a protocol error.
256    fn lock_tree(&self) -> RemoteResult<MutexGuard<'_, FsTree>> {
257        lock_tree(&self.tree)
258    }
259
260    fn missing() -> RemoteError {
261        RemoteError::new(RemoteErrorType::NoSuchFileOrDirectory)
262    }
263
264    /// Returns the parent of `path`, or `InvalidPath` for a root path.
265    fn parent_of(path: &Path) -> RemoteResult<PathBuf> {
266        path.parent().map(Path::to_path_buf).ok_or_else(|| {
267            RemoteError::with_message(RemoteErrorType::InvalidPath, "path has no parent")
268        })
269    }
270
271    /// Returns a mutable directory node, or `NoSuchFileOrDirectory`.
272    fn directory_mut<'a>(
273        tree: &'a mut FsTree,
274        path: &PathBuf,
275    ) -> RemoteResult<&'a mut Node<PathBuf, Inode>> {
276        tree.root_mut()
277            .query_mut(path)
278            .filter(|node| node.value().metadata().is_dir())
279            .ok_or_else(Self::missing)
280    }
281
282    /// Clones the subtree rooted at `node`, rewriting ids from `src` to `dest`.
283    fn rekey(node: &Node<PathBuf, Inode>, src: &Path, dest: &Path) -> Node<PathBuf, Inode> {
284        let id = match node.id().strip_prefix(src) {
285            Ok(suffix) if !suffix.as_os_str().is_empty() => dest.join(suffix),
286            _ => dest.to_path_buf(),
287        };
288        let children = node
289            .children()
290            .iter()
291            .map(|child| Self::rekey(child, src, dest))
292            .collect();
293        Node::new(id, node.value().clone_with_new_identity()).with_children(children)
294    }
295
296    /// Copies the subtree at `src` under `dest`, replacing an existing `dest`.
297    fn clone_subtree(tree: &mut FsTree, src: &PathBuf, dest: &PathBuf) -> RemoteResult<()> {
298        if dest.starts_with(src) || src.starts_with(dest) {
299            return Err(RemoteError::with_message(
300                RemoteErrorType::BadFile,
301                "source and destination overlap",
302            ));
303        }
304        let dest_parent = Self::parent_of(dest)?;
305        let subtree = Self::rekey(tree.root().query(src).ok_or_else(Self::missing)?, src, dest);
306        let parent = Self::directory_mut(tree, &dest_parent)?;
307        parent.remove_child(dest);
308        parent.add_child(subtree);
309        Ok(())
310    }
311
312    /// Validates every path before checking whether the client is connected.
313    fn validate_paths(&self, paths: &[&Path]) -> RemoteResult<()> {
314        for path in paths {
315            ensure_absolute(path)?;
316        }
317        if self.is_connected() {
318            Ok(())
319        } else {
320            Err(RemoteError::new(RemoteErrorType::NotConnected))
321        }
322    }
323
324    /// Slices `bytes` by the read options, clamping to the file length.
325    fn ranged(bytes: &[u8], opts: &ReadOptions) -> Vec<u8> {
326        let length = bytes.len() as u64;
327        let offset = opts.offset.unwrap_or(0).min(length);
328        let end = opts.length.map_or(length, |requested| {
329            offset.saturating_add(requested).min(length)
330        });
331        bytes[offset as usize..end as usize].to_vec()
332    }
333
334    /// Inserts or reuses the inode at `path` and returns a staged writer.
335    fn open_writer(
336        &self,
337        path: &Path,
338        opts: &WriteOptions,
339        append: bool,
340    ) -> RemoteResult<WriteStream> {
341        let path = path.to_path_buf();
342        let parent_path = Self::parent_of(&path)?;
343        let mut tree = self.lock_tree()?;
344        let parent = Self::directory_mut(&mut tree, &parent_path)?;
345        let existing = parent
346            .children()
347            .iter()
348            .find(|child| *child.id() == path)
349            .map(|child| child.value().clone());
350        if existing
351            .as_ref()
352            .is_some_and(|inode| inode.metadata().is_dir())
353        {
354            return Err(RemoteError::with_message(
355                RemoteErrorType::BadFile,
356                "is a directory",
357            ));
358        }
359        let (inode, initial) = match existing {
360            Some(inode) if append => {
361                if inode.metadata().is_symlink() {
362                    return Err(RemoteError::with_message(
363                        RemoteErrorType::BadFile,
364                        "cannot append to a symlink",
365                    ));
366                }
367                let initial = inode.content().map(<[u8]>::to_vec).unwrap_or_default();
368                (inode.clone_with_new_identity(), initial)
369            }
370            _ => (
371                Inode::file(
372                    (self.get_uid)(),
373                    (self.get_gid)(),
374                    opts.mode.unwrap_or_else(|| UnixPex::from(DEFAULT_MODE)),
375                    Vec::new(),
376                ),
377                Vec::new(),
378            ),
379        };
380        let identity = inode.identity();
381        parent.add_child(Node::new(path.clone(), inode));
382        drop(tree);
383        let mut writer = MemoryWriter::new(
384            Arc::clone(&self.tree),
385            path,
386            initial,
387            opts.modified,
388            identity,
389        );
390        if append {
391            writer.seek_to_end();
392        }
393        Ok(WriteStream::new(writer))
394    }
395}
396
397impl RemoteFs for MemoryFs {
398    fn connect(&mut self) -> RemoteResult<()> {
399        debug!("connect()");
400        if self.connected.swap(true, Ordering::AcqRel) {
401            return Err(RemoteError::new(RemoteErrorType::AlreadyConnected));
402        }
403        Ok(())
404    }
405
406    fn disconnect(&mut self) -> RemoteResult<()> {
407        debug!("disconnect()");
408        if self.connected.swap(false, Ordering::AcqRel) {
409            Ok(())
410        } else {
411            Err(RemoteError::new(RemoteErrorType::NotConnected))
412        }
413    }
414
415    fn is_connected(&self) -> bool {
416        self.connected.load(Ordering::Acquire)
417    }
418
419    fn capabilities(&self) -> Capabilities {
420        CAPABILITIES
421    }
422
423    fn list_dir(&self, path: &Path) -> RemoteResult<Vec<File>> {
424        self.validate(path)?;
425        debug!("list_dir({path:?})");
426        let tree = self.lock_tree()?;
427        let node = tree
428            .root()
429            .query(&path.to_path_buf())
430            .ok_or_else(Self::missing)?;
431        if !node.value().metadata().is_dir() {
432            return Err(RemoteError::with_message(
433                RemoteErrorType::BadFile,
434                "not a directory",
435            ));
436        }
437        Ok(node
438            .children()
439            .iter()
440            .map(|child| File::new(child.id().clone(), child.value().metadata().clone()))
441            .collect())
442    }
443
444    fn stat(&self, path: &Path) -> RemoteResult<File> {
445        self.validate(path)?;
446        debug!("stat({path:?})");
447        let tree = self.lock_tree()?;
448        let node = tree
449            .root()
450            .query(&path.to_path_buf())
451            .ok_or_else(Self::missing)?;
452        Ok(File::new(
453            node.id().clone(),
454            node.value().metadata().clone(),
455        ))
456    }
457
458    fn exists(&self, path: &Path) -> RemoteResult<bool> {
459        self.validate(path)?;
460        debug!("exists({path:?})");
461        let tree = self.lock_tree()?;
462        Ok(tree.root().query(&path.to_path_buf()).is_some())
463    }
464
465    fn set_metadata(&self, path: &Path, metadata: &SetMetadata) -> RemoteResult<()> {
466        self.validate(path)?;
467        debug!("set_metadata({path:?}, {metadata:?})");
468        let mut tree = self.lock_tree()?;
469        let node = tree
470            .root_mut()
471            .query_mut(&path.to_path_buf())
472            .ok_or_else(Self::missing)?;
473        let mut inode = node.value().clone();
474        if let Some(mode) = metadata.mode {
475            inode.metadata.mode = Some(mode);
476        }
477        if let Some(uid) = metadata.uid {
478            inode.metadata.uid = Some(uid);
479        }
480        if let Some(gid) = metadata.gid {
481            inode.metadata.gid = Some(gid);
482        }
483        if let Some(accessed) = metadata.accessed {
484            inode.metadata.accessed = Some(accessed);
485        }
486        if let Some(modified) = metadata.modified {
487            inode.metadata.modified = Some(modified);
488        }
489        node.set_value(inode);
490        Ok(())
491    }
492
493    fn create_dir(&self, path: &Path, mode: Option<UnixPex>) -> RemoteResult<()> {
494        self.validate(path)?;
495        debug!("create_dir({path:?}, {mode:?})");
496        let path = path.to_path_buf();
497        let parent_path = Self::parent_of(&path)?;
498        let dir = Inode::dir(
499            (self.get_uid)(),
500            (self.get_gid)(),
501            mode.unwrap_or_else(|| UnixPex::from(DEFAULT_MODE)),
502        );
503        let mut tree = self.lock_tree()?;
504        let parent = Self::directory_mut(&mut tree, &parent_path)?;
505        if parent.children().iter().any(|child| *child.id() == path) {
506            return Err(RemoteError::new(RemoteErrorType::AlreadyExists));
507        }
508        parent.add_child(Node::new(path, dir));
509        Ok(())
510    }
511
512    fn remove_file(&self, path: &Path) -> RemoteResult<()> {
513        self.validate(path)?;
514        debug!("remove_file({path:?})");
515        let path = path.to_path_buf();
516        let mut tree = self.lock_tree()?;
517        let node = tree.root().query(&path).ok_or_else(Self::missing)?;
518        if node.value().metadata().is_dir() {
519            return Err(RemoteError::with_message(
520                RemoteErrorType::CouldNotRemoveFile,
521                "is a directory",
522            ));
523        }
524        tree.root_mut()
525            .parent_mut(&path)
526            .ok_or_else(Self::missing)?
527            .remove_child(&path);
528        Ok(())
529    }
530
531    fn remove_dir(&self, path: &Path) -> RemoteResult<()> {
532        self.validate(path)?;
533        debug!("remove_dir({path:?})");
534        let path = path.to_path_buf();
535        let mut tree = self.lock_tree()?;
536        let node = tree.root().query(&path).ok_or_else(Self::missing)?;
537        if !node.value().metadata().is_dir() {
538            return Err(RemoteError::with_message(
539                RemoteErrorType::CouldNotRemoveFile,
540                "not a directory",
541            ));
542        }
543        if !node.is_leaf() {
544            return Err(RemoteError::new(RemoteErrorType::DirectoryNotEmpty));
545        }
546        tree.root_mut()
547            .parent_mut(&path)
548            .ok_or_else(Self::missing)?
549            .remove_child(&path);
550        Ok(())
551    }
552
553    fn remove_dir_all(&self, path: &Path) -> RemoteResult<()> {
554        self.validate(path)?;
555        debug!("remove_dir_all({path:?})");
556        let path = path.to_path_buf();
557        let mut tree = self.lock_tree()?;
558        let parent = tree
559            .root_mut()
560            .parent_mut(&path)
561            .ok_or_else(Self::missing)?;
562        if !parent.children().iter().any(|child| *child.id() == path) {
563            return Err(Self::missing());
564        }
565        parent.remove_child(&path);
566        Ok(())
567    }
568
569    fn rename(&self, src: &Path, dest: &Path) -> RemoteResult<()> {
570        self.validate_paths(&[src, dest])?;
571        debug!("rename({src:?}, {dest:?})");
572        let (src, dest) = (src.to_path_buf(), dest.to_path_buf());
573        let mut tree = self.lock_tree()?;
574        Self::clone_subtree(&mut tree, &src, &dest)?;
575        tree.root_mut()
576            .parent_mut(&src)
577            .ok_or_else(Self::missing)?
578            .remove_child(&src);
579        Ok(())
580    }
581
582    fn copy(&self, src: &Path, dest: &Path) -> RemoteResult<()> {
583        self.validate_paths(&[src, dest])?;
584        debug!("copy({src:?}, {dest:?})");
585        let mut tree = self.lock_tree()?;
586        Self::clone_subtree(&mut tree, &src.to_path_buf(), &dest.to_path_buf())
587    }
588
589    fn symlink(&self, path: &Path, target: &Path) -> RemoteResult<()> {
590        self.validate_paths(&[path, target])?;
591        debug!("symlink({path:?}, {target:?})");
592        let path = path.to_path_buf();
593        let parent_path = Self::parent_of(&path)?;
594        let link = Inode::symlink((self.get_uid)(), (self.get_gid)(), target.to_path_buf());
595        let mut tree = self.lock_tree()?;
596        if tree.root().query(&target.to_path_buf()).is_none() {
597            return Err(Self::missing());
598        }
599        let parent = Self::directory_mut(&mut tree, &parent_path)?;
600        if parent.children().iter().any(|child| *child.id() == path) {
601            return Err(RemoteError::new(RemoteErrorType::AlreadyExists));
602        }
603        parent.add_child(Node::new(path, link));
604        Ok(())
605    }
606
607    fn open(&self, path: &Path, opts: &ReadOptions) -> RemoteResult<ReadStream> {
608        self.validate(path)?;
609        debug!("open({path:?}, {opts:?})");
610        let tree = self.lock_tree()?;
611        let node = tree
612            .root()
613            .query(&path.to_path_buf())
614            .ok_or_else(Self::missing)?;
615        let content = node.value().content().ok_or_else(|| {
616            RemoteError::with_message(RemoteErrorType::BadFile, "cannot open a directory")
617        })?;
618        Ok(ReadStream::new(MemoryReader::new(Self::ranged(
619            content, opts,
620        ))))
621    }
622
623    fn create(&self, path: &Path, opts: &WriteOptions) -> RemoteResult<WriteStream> {
624        self.validate(path)?;
625        debug!("create({path:?}, {opts:?})");
626        self.open_writer(path, opts, false)
627    }
628
629    fn append(&self, path: &Path, opts: &WriteOptions) -> RemoteResult<WriteStream> {
630        self.validate(path)?;
631        debug!("append({path:?}, {opts:?})");
632        self.open_writer(path, opts, true)
633    }
634
635    fn exec(&self, _cmd: &str) -> RemoteResult<ExecOutput> {
636        if !self.is_connected() {
637            return Err(RemoteError::new(RemoteErrorType::NotConnected));
638        }
639        Err(RemoteError::new(RemoteErrorType::UnsupportedFeature))
640    }
641}
remotefs_memory/lib.rs

remotefs_memory/
lib.rs
