tycho_storage/fs/
mod.rs

1#![allow(clippy::disallowed_methods)]
2#![allow(clippy::disallowed_types)] // it's wrapper around Files so
3
4use std::borrow::Cow;
5use std::ffi::OsStr;
6use std::fs::{File, Metadata, OpenOptions};
7use std::os::fd::AsRawFd;
8use std::path::{Path, PathBuf};
9use std::sync::Arc;
10use std::time::Duration;
11
12use anyhow::{Context, Result};
13
14pub use self::mapped_file::{MappedFile, MappedFileMut};
15
16mod mapped_file;
17
18const BASE_DIR: &str = "temp";
19
20#[derive(Clone)]
21pub struct TempFileStorage {
22    storage_dir: Dir,
23}
24
25impl TempFileStorage {
26    const MAX_FILE_TTL: Duration = Duration::from_secs(86400); // 1 day
27
28    pub fn new(files_dir: &Dir) -> Result<Self> {
29        Ok(Self {
30            storage_dir: files_dir.create_subdir(BASE_DIR)?,
31        })
32    }
33
34    pub async fn remove_outdated_files(&self) -> Result<()> {
35        let now = std::time::SystemTime::now();
36
37        let this = self.clone();
38        tokio::task::spawn_blocking(move || {
39            this.retain_files(|path, metadata| match metadata.modified() {
40                Ok(modified) => {
41                    let since_modified = now.duration_since(modified).unwrap_or(Duration::ZERO);
42                    since_modified <= Self::MAX_FILE_TTL
43                }
44                Err(e) => {
45                    tracing::warn!(
46                        path = %path.display(),
47                        "failed to check file metadata: {e:?}"
48                    );
49                    false
50                }
51            })
52        })
53        .await?
54    }
55
56    pub fn retain_files<F>(&self, mut f: F) -> Result<()>
57    where
58        F: FnMut(&Path, &Metadata) -> bool,
59    {
60        let entries = self.storage_dir.entries()?;
61        for e in entries {
62            let e = e?;
63
64            let path = e.path();
65            let Ok(metadata) = std::fs::metadata(&path) else {
66                tracing::warn!(
67                    path = %path.display(),
68                    "failed to check downloaded file metadata: {e:?}"
69                );
70                continue;
71            };
72
73            let is_file = metadata.is_file();
74            let keep = is_file && f(&path, &metadata);
75            tracing::debug!(keep, path = %path.display(), "found downloaded file");
76
77            if keep {
78                continue;
79            }
80
81            let e = if is_file {
82                std::fs::remove_file(&path)
83            } else {
84                std::fs::remove_dir_all(&path)
85            };
86            if let Err(e) = e {
87                tracing::warn!(path = %path.display(), "failed to remove downloads entry: {e:?}");
88            }
89        }
90
91        Ok(())
92    }
93
94    pub fn file<P: AsRef<Path>>(&self, rel_path: P) -> FileBuilder {
95        self.storage_dir.file(&rel_path)
96    }
97
98    pub fn unnamed_file(&self) -> UnnamedFileBuilder {
99        self.storage_dir.unnamed_file()
100    }
101}
102
103#[derive(Clone)]
104pub struct Dir(Arc<DirInner>);
105
106impl Dir {
107    /// Creates a new `Dir` instance.
108    /// If the `root` directory does not exist, it will be created.
109    pub fn new<P>(root: P) -> Result<Self>
110    where
111        P: AsRef<Path>,
112    {
113        std::fs::create_dir_all(root.as_ref())
114            .with_context(|| format!("failed to create {}", root.as_ref().display()))?;
115        Ok(Self(Arc::new(DirInner {
116            base_dir: root.as_ref().to_path_buf(),
117        })))
118    }
119
120    /// Creates a new `Dir` without creating the root directory tree
121    pub fn new_readonly<P>(root: P) -> Self
122    where
123        P: AsRef<Path>,
124    {
125        Self(Arc::new(DirInner {
126            base_dir: root.as_ref().to_path_buf(),
127        }))
128    }
129
130    pub fn path(&self) -> &Path {
131        &self.0.base_dir
132    }
133
134    pub fn create_if_not_exists(&self) -> std::io::Result<()> {
135        std::fs::create_dir_all(&self.0.base_dir)
136    }
137
138    pub fn create_dir_all<P: AsRef<Path>>(&self, rel_path: P) -> std::io::Result<()> {
139        std::fs::create_dir_all(self.0.base_dir.join(rel_path))
140    }
141
142    pub fn remove_file<P: AsRef<Path>>(&self, rel_path: P) -> std::io::Result<()> {
143        std::fs::remove_file(self.0.base_dir.join(rel_path))
144    }
145
146    pub fn file<P: AsRef<Path>>(&self, rel_path: P) -> FileBuilder {
147        FileBuilder {
148            path: self.0.base_dir.join(rel_path.as_ref()),
149            options: std::fs::OpenOptions::new(),
150            prealloc: None,
151        }
152    }
153
154    pub fn unnamed_file(&self) -> UnnamedFileBuilder {
155        UnnamedFileBuilder {
156            base_dir: self.0.base_dir.clone(),
157            prealloc: None,
158        }
159    }
160
161    /// Creates `Dir` instance for a subdirectory of the current one.
162    /// **Note**: The subdirectory will not be created if it does not exist.
163    /// Use `create_subdir` to create it.
164    pub fn subdir_readonly<P: AsRef<Path>>(&self, rel_path: P) -> Self {
165        Self(Arc::new(DirInner {
166            base_dir: self.0.base_dir.join(rel_path),
167        }))
168    }
169
170    /// Creates `Dir` instance for a subdirectory of the current one.
171    /// The subdirectory will be created if it does not exist.
172    pub fn create_subdir<P: AsRef<Path>>(&self, rel_path: P) -> Result<Self> {
173        Self::new(self.0.base_dir.join(rel_path))
174    }
175
176    pub fn file_exists<P: AsRef<Path>>(&self, rel_path: P) -> bool {
177        self.path().join(rel_path).is_file()
178    }
179
180    pub fn entries(&self) -> std::io::Result<std::fs::ReadDir> {
181        std::fs::read_dir(&self.0.base_dir)
182    }
183}
184
185struct DirInner {
186    base_dir: PathBuf,
187}
188
189#[derive(Clone)]
190pub struct FileBuilder {
191    path: PathBuf,
192    options: OpenOptions,
193    prealloc: Option<usize>,
194}
195
196impl FileBuilder {
197    pub fn with_extension<S: AsRef<OsStr>>(&self, extension: S) -> Self {
198        Self {
199            path: self.path.with_extension(extension),
200            options: self.options.clone(),
201            prealloc: self.prealloc,
202        }
203    }
204
205    pub fn open(&self) -> Result<File> {
206        let file = self
207            .options
208            .open(&self.path)
209            .with_context(|| format!("failed to open {}", self.path.display()))?;
210        if let Some(prealloc) = self.prealloc {
211            alloc_file(&file, prealloc)?;
212        }
213        Ok(file)
214    }
215
216    pub fn rename<P: AsRef<Path>>(&self, new_path: P) -> std::io::Result<()> {
217        let new_path = match self.path.parent() {
218            Some(parent) => Cow::Owned(parent.join(new_path)),
219            None => Cow::Borrowed(new_path.as_ref()),
220        };
221        std::fs::rename(&self.path, new_path)
222    }
223
224    pub fn exists(&self) -> bool {
225        std::fs::metadata(&self.path)
226            .ok()
227            .map(|m| m.is_file())
228            .unwrap_or_default()
229    }
230
231    pub fn append(&mut self, append: bool) -> &mut Self {
232        self.options.append(append);
233        self
234    }
235
236    pub fn create(&mut self, create: bool) -> &mut Self {
237        self.options.create(create);
238        self
239    }
240
241    pub fn create_new(&mut self, create_new: bool) -> &mut Self {
242        self.options.create_new(create_new);
243        self
244    }
245
246    pub fn read(&mut self, read: bool) -> &mut Self {
247        self.options.read(read);
248        self
249    }
250
251    pub fn truncate(&mut self, truncate: bool) -> &mut Self {
252        self.options.truncate(truncate);
253        self
254    }
255
256    pub fn write(&mut self, write: bool) -> &mut Self {
257        self.options.write(write);
258        self
259    }
260
261    pub fn prealloc(&mut self, prealloc: usize) -> &mut Self {
262        self.prealloc = Some(prealloc);
263        self
264    }
265
266    pub fn path(&self) -> &Path {
267        &self.path
268    }
269}
270
271pub struct UnnamedFileBuilder {
272    base_dir: PathBuf,
273    prealloc: Option<usize>,
274}
275
276impl UnnamedFileBuilder {
277    pub fn open(self) -> Result<File> {
278        let file = tempfile::tempfile_in(&self.base_dir)?;
279        if let Some(prealloc) = self.prealloc {
280            file.set_len(prealloc as u64)?;
281        }
282
283        Ok(file)
284    }
285
286    pub fn open_as_mapped_mut(&self) -> Result<MappedFileMut> {
287        let file = tempfile::tempfile_in(&self.base_dir).with_context(|| {
288            format!("failed to create a tempfile in {}", self.base_dir.display())
289        })?;
290
291        if let Some(prealloc) = self.prealloc {
292            #[cfg(target_os = "linux")]
293            alloc_file(&file, prealloc)?;
294
295            file.set_len(prealloc as u64)?;
296        } else {
297            anyhow::bail!("prealloc is required for mapping unnamed files");
298        }
299
300        MappedFileMut::from_existing_file(file).map_err(Into::into)
301    }
302
303    pub fn prealloc(&mut self, prealloc: usize) -> &mut Self {
304        self.prealloc = Some(prealloc);
305        self
306    }
307}
308
309#[cfg(not(target_os = "macos"))]
310fn alloc_file(file: &File, len: usize) -> std::io::Result<()> {
311    let res = unsafe { libc::posix_fallocate(file.as_raw_fd(), 0, len as i64) };
312    if res == 0 {
313        Ok(())
314    } else {
315        Err(std::io::Error::last_os_error())
316    }
317}
318
319#[cfg(target_os = "macos")]
320pub fn alloc_file(file: &File, len: usize) -> std::io::Result<()> {
321    let res = unsafe { libc::ftruncate(file.as_raw_fd(), len as i64) };
322    if res < 0 {
323        Err(std::io::Error::last_os_error())
324    } else {
325        Ok(())
326    }
327}
tycho_storage/fs/mod.rs

tycho_storage/fs/
mod.rs
