rspack_storage/fs/
mod.rs

1use std::sync::Arc;
2
3mod error;
4use error::FsResultToStorageFsResult;
5pub use error::{BatchFSError, BatchFSResult, FSError, FSOperation, FSResult};
6use rspack_fs::{FileMetadata, IntermediateFileSystem, ReadStream, WriteStream};
7use rspack_paths::{Utf8Path, Utf8PathBuf};
8use rustc_hash::FxHashSet as HashSet;
9
10#[async_trait::async_trait]
11pub trait FileSystem: std::fmt::Debug + Sync + Send {
12  async fn exists(&self, path: &Utf8Path) -> FSResult<bool>;
13  async fn remove_dir(&self, path: &Utf8Path) -> FSResult<()>;
14  async fn ensure_dir(&self, path: &Utf8Path) -> FSResult<()>;
15  async fn write_file(&self, path: &Utf8Path) -> FSResult<Writer>;
16  async fn read_file(&self, path: &Utf8Path) -> FSResult<Reader>;
17  async fn read_dir(&self, path: &Utf8Path) -> FSResult<HashSet<String>>;
18  async fn metadata(&self, path: &Utf8Path) -> FSResult<FileMetadata>;
19  async fn remove_file(&self, path: &Utf8Path) -> FSResult<()>;
20  async fn move_file(&self, from: &Utf8Path, to: &Utf8Path) -> FSResult<()>;
21}
22
23#[derive(Debug)]
24pub struct Writer {
25  path: Utf8PathBuf,
26  stream: Box<dyn WriteStream>,
27}
28
29impl Writer {
30  pub async fn write_line(&mut self, line: &str) -> FSResult<()> {
31    self
32      .stream
33      .write_line(line)
34      .await
35      .to_storage_fs_result(&self.path, FSOperation::Write)
36  }
37  pub async fn write(&mut self, buf: &[u8]) -> FSResult<usize> {
38    self
39      .stream
40      .write(buf)
41      .await
42      .to_storage_fs_result(&self.path, FSOperation::Write)
43  }
44  pub async fn write_all(&mut self, buf: &[u8]) -> FSResult<()> {
45    self
46      .stream
47      .write_all(buf)
48      .await
49      .to_storage_fs_result(&self.path, FSOperation::Write)
50  }
51  pub async fn flush(&mut self) -> FSResult<()> {
52    self
53      .stream
54      .flush()
55      .await
56      .to_storage_fs_result(&self.path, FSOperation::Write)
57  }
58  pub async fn close(&mut self) -> FSResult<()> {
59    self
60      .stream
61      .close()
62      .await
63      .to_storage_fs_result(&self.path, FSOperation::Write)
64  }
65}
66
67#[derive(Debug)]
68pub struct Reader {
69  path: Utf8PathBuf,
70  stream: Box<dyn ReadStream>,
71}
72
73impl Reader {
74  pub async fn read_line(&mut self) -> FSResult<String> {
75    self
76      .stream
77      .read_line()
78      .await
79      .to_storage_fs_result(&self.path, FSOperation::Read)
80  }
81  pub async fn read(&mut self, length: usize) -> FSResult<Vec<u8>> {
82    self
83      .stream
84      .read(length)
85      .await
86      .to_storage_fs_result(&self.path, FSOperation::Read)
87  }
88  pub async fn read_until(&mut self, byte: u8) -> FSResult<Vec<u8>> {
89    self
90      .stream
91      .read_until(byte)
92      .await
93      .to_storage_fs_result(&self.path, FSOperation::Read)
94  }
95  pub async fn read_to_end(&mut self) -> FSResult<Vec<u8>> {
96    self
97      .stream
98      .read_to_end()
99      .await
100      .to_storage_fs_result(&self.path, FSOperation::Read)
101  }
102  pub async fn skip(&mut self, offset: usize) -> FSResult<()> {
103    self
104      .stream
105      .skip(offset)
106      .await
107      .to_storage_fs_result(&self.path, FSOperation::Read)
108  }
109  pub async fn close(&mut self) -> FSResult<()> {
110    self
111      .stream
112      .close()
113      .await
114      .to_storage_fs_result(&self.path, FSOperation::Read)
115  }
116}
117
118#[derive(Debug)]
119pub struct BridgeFileSystem(pub Arc<dyn IntermediateFileSystem>);
120
121#[async_trait::async_trait]
122impl FileSystem for BridgeFileSystem {
123  async fn exists(&self, path: &Utf8Path) -> FSResult<bool> {
124    match self.metadata(path).await {
125      Ok(_) => Ok(true),
126      Err(e) => {
127        if e.is_not_found() {
128          Ok(false)
129        } else {
130          Err(e)
131        }
132      }
133    }
134  }
135
136  async fn remove_dir(&self, path: &Utf8Path) -> FSResult<()> {
137    if self.exists(path).await? {
138      self
139        .0
140        .remove_dir_all(path)
141        .await
142        .to_storage_fs_result(path, FSOperation::Remove)?;
143    }
144    Ok(())
145  }
146
147  async fn ensure_dir(&self, path: &Utf8Path) -> FSResult<()> {
148    self
149      .0
150      .create_dir_all(path)
151      .await
152      .to_storage_fs_result(path, FSOperation::Dir)?;
153    Ok(())
154  }
155
156  async fn write_file(&self, path: &Utf8Path) -> FSResult<Writer> {
157    if self.exists(path).await? {
158      self.remove_file(path).await?;
159    }
160    self
161      .ensure_dir(path.parent().expect("should have parent"))
162      .await?;
163
164    let stream = self
165      .0
166      .create_write_stream(path)
167      .await
168      .to_storage_fs_result(path, FSOperation::Write)?;
169
170    Ok(Writer {
171      path: path.to_path_buf(),
172      stream,
173    })
174  }
175
176  async fn read_file(&self, path: &Utf8Path) -> FSResult<Reader> {
177    let stream = self
178      .0
179      .create_read_stream(path)
180      .await
181      .to_storage_fs_result(path, FSOperation::Read)?;
182    Ok(Reader {
183      path: path.to_path_buf(),
184      stream,
185    })
186  }
187
188  async fn read_dir(&self, path: &Utf8Path) -> FSResult<HashSet<String>> {
189    let files = self
190      .0
191      .read_dir(path)
192      .await
193      .to_storage_fs_result(path, FSOperation::Read)?;
194    Ok(files.into_iter().collect::<HashSet<_>>())
195  }
196
197  async fn metadata(&self, path: &Utf8Path) -> FSResult<FileMetadata> {
198    let res = self
199      .0
200      .stat(path)
201      .await
202      .to_storage_fs_result(path, FSOperation::Stat)?;
203    Ok(res)
204  }
205
206  async fn remove_file(&self, path: &Utf8Path) -> FSResult<()> {
207    if self.exists(path).await? {
208      self
209        .0
210        .remove_file(path)
211        .await
212        .to_storage_fs_result(path, FSOperation::Remove)?;
213    }
214    Ok(())
215  }
216
217  async fn move_file(&self, from: &Utf8Path, to: &Utf8Path) -> FSResult<()> {
218    if self.exists(from).await? {
219      self
220        .ensure_dir(to.parent().expect("should have parent"))
221        .await?;
222      self
223        .0
224        .rename(from, to)
225        .await
226        .to_storage_fs_result(from, FSOperation::Move)?;
227    }
228    Ok(())
229  }
230}
231
232#[cfg(test)]
233mod tests {
234  use std::sync::Arc;
235
236  use rspack_fs::MemoryFileSystem;
237  use rspack_paths::Utf8PathBuf;
238
239  use super::{BridgeFileSystem, FSResult};
240  use crate::FileSystem;
241
242  fn get_path(p: &str) -> Utf8PathBuf {
243    Utf8PathBuf::from(p)
244  }
245
246  async fn test_create_dir(fs: &BridgeFileSystem) -> FSResult<()> {
247    fs.ensure_dir(&get_path("/parent/from")).await?;
248    fs.ensure_dir(&get_path("/parent/to")).await?;
249
250    assert!(fs.exists(&get_path("/parent/from")).await?);
251    assert!(fs.exists(&get_path("/parent/to")).await?);
252
253    assert!(fs.metadata(&get_path("/parent/from")).await?.is_directory);
254    assert!(fs.metadata(&get_path("/parent/to")).await?.is_directory);
255
256    Ok(())
257  }
258
259  async fn test_write_file(fs: &BridgeFileSystem) -> FSResult<()> {
260    let mut writer = fs.write_file(&get_path("/parent/from/file.txt")).await?;
261
262    writer.write_line("hello").await?;
263    writer.write(b" world").await?;
264    writer.flush().await?;
265
266    assert!(fs.exists(&get_path("/parent/from/file.txt")).await?);
267    assert!(
268      fs.metadata(&get_path("/parent/from/file.txt"))
269        .await?
270        .is_file
271    );
272
273    Ok(())
274  }
275
276  async fn test_read_file(fs: &BridgeFileSystem) -> FSResult<()> {
277    let mut reader = fs.read_file(&get_path("/parent/from/file.txt")).await?;
278
279    assert_eq!(reader.read_line().await?, "hello");
280    assert_eq!(reader.read(b" world".len()).await?, b" world");
281
282    Ok(())
283  }
284
285  async fn test_move_file(fs: &BridgeFileSystem) -> FSResult<()> {
286    fs.move_file(
287      &get_path("/parent/from/file.txt"),
288      &get_path("/parent/to/file.txt"),
289    )
290    .await?;
291    assert!(!fs.exists(&get_path("/parent/from/file.txt")).await?);
292    assert!(fs.exists(&get_path("/parent/to/file.txt")).await?);
293    assert!(fs.metadata(&get_path("/parent/to/file.txt")).await?.is_file);
294
295    Ok(())
296  }
297
298  async fn test_remove_file(fs: &BridgeFileSystem) -> FSResult<()> {
299    fs.remove_file(&get_path("/parent/to/file.txt")).await?;
300    assert!(!fs.exists(&get_path("/parent/to/file.txt")).await?);
301    Ok(())
302  }
303
304  async fn test_remove_dir(fs: &BridgeFileSystem) -> FSResult<()> {
305    fs.remove_dir(&get_path("/parent/from")).await?;
306    fs.remove_dir(&get_path("/parent/to")).await?;
307    assert!(!fs.exists(&get_path("/parent/from")).await?);
308    assert!(!fs.exists(&get_path("/parent/to")).await?);
309    Ok(())
310  }
311
312  async fn test_error(fs: &BridgeFileSystem) -> FSResult<()> {
313    match fs.metadata(&get_path("/parent/from/not_exist.txt")).await {
314      Ok(_) => panic!("should error"),
315      Err(e) => assert_eq!(
316        e.to_string(),
317        r#"stat `/parent/from/not_exist.txt` failed due to `file not exist`"#
318      ),
319    };
320
321    Ok(())
322  }
323
324  async fn test_memory_fs(fs: &BridgeFileSystem) -> FSResult<()> {
325    test_create_dir(fs).await?;
326    test_write_file(fs).await?;
327    test_read_file(fs).await?;
328    test_move_file(fs).await?;
329    test_remove_file(fs).await?;
330    test_remove_dir(fs).await?;
331    test_error(fs).await?;
332
333    Ok(())
334  }
335
336  #[tokio::test]
337  #[cfg_attr(miri, ignore)]
338  async fn should_storage_bridge_fs_work() -> FSResult<()> {
339    let fs = BridgeFileSystem(Arc::new(MemoryFileSystem::default()));
340
341    test_memory_fs(&fs).await?;
342    Ok(())
343  }
344}
rspack_storage/fs/mod.rs

rspack_storage/fs/
mod.rs
