gix_features/
fs.rs

1//! Filesystem utilities
2//!
3//! These are will be parallel if the `parallel` feature is enabled, at the expense of compiling additional dependencies
4//! along with runtime costs for maintaining a global [`rayon`](https://docs.rs/rayon) thread pool.
5//!
6//! For information on how to use the [`WalkDir`] type, have a look at
7//! * [`jwalk::WalkDir`](https://docs.rs/jwalk/0.5.1/jwalk/type.WalkDir.html) if `parallel` feature is enabled
8//! * [walkdir::WalkDir](https://docs.rs/walkdir/2.3.1/walkdir/struct.WalkDir.html) otherwise
9
10#[cfg(any(feature = "walkdir", feature = "fs-walkdir-parallel"))]
11mod shared {
12    /// The desired level of parallelism.
13    pub enum Parallelism {
14        /// Do not parallelize at all by making a serial traversal on the current thread.
15        Serial,
16        /// Create a new thread pool for each traversal with up to 16 threads or the amount of logical cores of the machine.
17        ThreadPoolPerTraversal {
18            /// The base name of the threads we create as part of the thread-pool.
19            thread_name: &'static str,
20        },
21    }
22}
23
24#[cfg(any(feature = "walkdir", feature = "fs-walkdir-parallel", feature = "fs-read-dir"))]
25mod walkdir_precompose {
26    use std::borrow::Cow;
27    use std::ffi::OsStr;
28    use std::path::Path;
29
30    #[derive(Debug)]
31    pub struct DirEntry<T: std::fmt::Debug> {
32        inner: T,
33        precompose_unicode: bool,
34    }
35
36    impl<T: std::fmt::Debug> DirEntry<T> {
37        /// Create a new instance.
38        pub fn new(inner: T, precompose_unicode: bool) -> Self {
39            Self {
40                inner,
41                precompose_unicode,
42            }
43        }
44    }
45
46    pub trait DirEntryApi {
47        fn path(&self) -> Cow<'_, Path>;
48        fn file_name(&self) -> Cow<'_, OsStr>;
49        fn file_type(&self) -> std::io::Result<std::fs::FileType>;
50    }
51
52    impl<T: DirEntryApi + std::fmt::Debug> DirEntry<T> {
53        /// Obtain the full path of this entry, possibly with precomposed unicode if enabled.
54        ///
55        /// Note that decomposing filesystem like those made by Apple accept both precomposed and
56        /// decomposed names, and consider them equal.
57        pub fn path(&self) -> Cow<'_, Path> {
58            let path = self.inner.path();
59            if self.precompose_unicode {
60                gix_utils::str::precompose_path(path)
61            } else {
62                path
63            }
64        }
65
66        /// Obtain filen name of this entry, possibly with precomposed unicode if enabled.
67        pub fn file_name(&self) -> Cow<'_, OsStr> {
68            let name = self.inner.file_name();
69            if self.precompose_unicode {
70                gix_utils::str::precompose_os_string(name)
71            } else {
72                name
73            }
74        }
75
76        /// Return the file type for the file that this entry points to.
77        ///
78        /// If `follow_links` was `true`, this is the file type of the item the link points to.
79        pub fn file_type(&self) -> std::io::Result<std::fs::FileType> {
80            self.inner.file_type()
81        }
82    }
83
84    /// A platform over entries in a directory, which may or may not precompose unicode after retrieving
85    /// paths from the file system.
86    #[cfg(any(feature = "walkdir", feature = "fs-walkdir-parallel"))]
87    pub struct WalkDir<T> {
88        pub(crate) inner: Option<T>,
89        pub(crate) precompose_unicode: bool,
90    }
91
92    #[cfg(any(feature = "walkdir", feature = "fs-walkdir-parallel"))]
93    pub struct WalkDirIter<T, I, E>
94    where
95        T: Iterator<Item = Result<I, E>>,
96        I: DirEntryApi,
97    {
98        pub(crate) inner: T,
99        pub(crate) precompose_unicode: bool,
100    }
101
102    #[cfg(any(feature = "walkdir", feature = "fs-walkdir-parallel"))]
103    impl<T, I, E> Iterator for WalkDirIter<T, I, E>
104    where
105        T: Iterator<Item = Result<I, E>>,
106        I: DirEntryApi + std::fmt::Debug,
107    {
108        type Item = Result<DirEntry<I>, E>;
109
110        fn next(&mut self) -> Option<Self::Item> {
111            self.inner
112                .next()
113                .map(|res| res.map(|entry| DirEntry::new(entry, self.precompose_unicode)))
114        }
115    }
116}
117
118///
119#[cfg(feature = "fs-read-dir")]
120pub mod read_dir {
121    use std::borrow::Cow;
122    use std::ffi::OsStr;
123    use std::fs::FileType;
124    use std::path::Path;
125
126    /// A directory entry adding precompose-unicode support to [`std::fs::DirEntry`].
127    pub type DirEntry = super::walkdir_precompose::DirEntry<std::fs::DirEntry>;
128
129    impl super::walkdir_precompose::DirEntryApi for std::fs::DirEntry {
130        fn path(&self) -> Cow<'_, Path> {
131            self.path().into()
132        }
133
134        fn file_name(&self) -> Cow<'_, OsStr> {
135            self.file_name().into()
136        }
137
138        fn file_type(&self) -> std::io::Result<FileType> {
139            self.file_type()
140        }
141    }
142}
143
144///
145#[cfg(feature = "fs-walkdir-parallel")]
146pub mod walkdir {
147    use std::borrow::Cow;
148    use std::ffi::OsStr;
149    use std::fs::FileType;
150    use std::path::Path;
151
152    use jwalk::WalkDir as WalkDirImpl;
153    pub use jwalk::{DirEntry as DirEntryGeneric, DirEntryIter as DirEntryIterGeneric, Error};
154
155    pub use super::shared::Parallelism;
156
157    type DirEntryImpl = DirEntryGeneric<((), ())>;
158
159    /// A directory entry returned by [DirEntryIter].
160    pub type DirEntry = super::walkdir_precompose::DirEntry<DirEntryImpl>;
161    /// A platform to create a [DirEntryIter] from.
162    pub type WalkDir = super::walkdir_precompose::WalkDir<WalkDirImpl>;
163
164    impl super::walkdir_precompose::DirEntryApi for DirEntryImpl {
165        fn path(&self) -> Cow<'_, Path> {
166            self.path().into()
167        }
168
169        fn file_name(&self) -> Cow<'_, OsStr> {
170            self.file_name().into()
171        }
172
173        fn file_type(&self) -> std::io::Result<FileType> {
174            Ok(self.file_type())
175        }
176    }
177
178    impl IntoIterator for WalkDir {
179        type Item = Result<DirEntry, jwalk::Error>;
180        type IntoIter = DirEntryIter;
181
182        fn into_iter(self) -> Self::IntoIter {
183            DirEntryIter {
184                inner: self.inner.expect("always set (builder fix)").into_iter(),
185                precompose_unicode: self.precompose_unicode,
186            }
187        }
188    }
189
190    impl WalkDir {
191        /// Set the minimum component depth of paths of entries.
192        pub fn min_depth(mut self, min: usize) -> Self {
193            self.inner = Some(self.inner.take().expect("always set").min_depth(min));
194            self
195        }
196        /// Set the maximum component depth of paths of entries.
197        pub fn max_depth(mut self, max: usize) -> Self {
198            self.inner = Some(self.inner.take().expect("always set").max_depth(max));
199            self
200        }
201        /// Follow symbolic links.
202        pub fn follow_links(mut self, toggle: bool) -> Self {
203            self.inner = Some(self.inner.take().expect("always set").follow_links(toggle));
204            self
205        }
206    }
207
208    impl From<Parallelism> for jwalk::Parallelism {
209        fn from(v: Parallelism) -> Self {
210            match v {
211                Parallelism::Serial => jwalk::Parallelism::Serial,
212                Parallelism::ThreadPoolPerTraversal { thread_name } => std::thread::available_parallelism()
213                    .map_or_else(
214                        |_| Parallelism::Serial.into(),
215                        |threads| {
216                            let pool = jwalk::rayon::ThreadPoolBuilder::new()
217                                .num_threads(threads.get().min(16))
218                                .stack_size(128 * 1024)
219                                .thread_name(move |idx| format!("{thread_name} {idx}"))
220                                .build()
221                                .expect("we only set options that can't cause a build failure");
222                            jwalk::Parallelism::RayonExistingPool {
223                                pool: pool.into(),
224                                busy_timeout: None,
225                            }
226                        },
227                    ),
228            }
229        }
230    }
231
232    /// Instantiate a new directory iterator which will not skip hidden files, with the given level of `parallelism`.
233    ///
234    /// Use `precompose_unicode` to represent the `core.precomposeUnicode` configuration option.
235    pub fn walkdir_new(root: &Path, parallelism: Parallelism, precompose_unicode: bool) -> WalkDir {
236        WalkDir {
237            inner: WalkDirImpl::new(root)
238                .skip_hidden(false)
239                .parallelism(parallelism.into())
240                .into(),
241            precompose_unicode,
242        }
243    }
244
245    /// Instantiate a new directory iterator which will not skip hidden files and is sorted
246    ///
247    /// Use `precompose_unicode` to represent the `core.precomposeUnicode` configuration option.
248    pub fn walkdir_sorted_new(root: &Path, parallelism: Parallelism, precompose_unicode: bool) -> WalkDir {
249        WalkDir {
250            inner: WalkDirImpl::new(root)
251                .skip_hidden(false)
252                .sort(true)
253                .parallelism(parallelism.into())
254                .into(),
255            precompose_unicode,
256        }
257    }
258
259    type DirEntryIterImpl = DirEntryIterGeneric<((), ())>;
260
261    /// The Iterator yielding directory items
262    pub type DirEntryIter = super::walkdir_precompose::WalkDirIter<DirEntryIterImpl, DirEntryImpl, jwalk::Error>;
263}
264
265///
266#[cfg(all(feature = "walkdir", not(feature = "fs-walkdir-parallel")))]
267pub mod walkdir {
268    use std::borrow::Cow;
269    use std::ffi::OsStr;
270    use std::fs::FileType;
271    use std::path::Path;
272
273    pub use walkdir::Error;
274    use walkdir::{DirEntry as DirEntryImpl, WalkDir as WalkDirImpl};
275
276    /// A directory entry returned by [DirEntryIter].
277    pub type DirEntry = super::walkdir_precompose::DirEntry<DirEntryImpl>;
278    /// A platform to create a [DirEntryIter] from.
279    pub type WalkDir = super::walkdir_precompose::WalkDir<WalkDirImpl>;
280
281    pub use super::shared::Parallelism;
282
283    impl super::walkdir_precompose::DirEntryApi for DirEntryImpl {
284        fn path(&self) -> Cow<'_, Path> {
285            self.path().into()
286        }
287
288        fn file_name(&self) -> Cow<'_, OsStr> {
289            self.file_name().into()
290        }
291
292        fn file_type(&self) -> std::io::Result<FileType> {
293            Ok(self.file_type())
294        }
295    }
296
297    impl IntoIterator for WalkDir {
298        type Item = Result<DirEntry, walkdir::Error>;
299        type IntoIter = DirEntryIter;
300
301        fn into_iter(self) -> Self::IntoIter {
302            DirEntryIter {
303                inner: self.inner.expect("always set (builder fix)").into_iter(),
304                precompose_unicode: self.precompose_unicode,
305            }
306        }
307    }
308
309    impl WalkDir {
310        /// Set the minimum component depth of paths of entries.
311        pub fn min_depth(mut self, min: usize) -> Self {
312            self.inner = Some(self.inner.take().expect("always set").min_depth(min));
313            self
314        }
315        /// Set the maximum component depth of paths of entries.
316        pub fn max_depth(mut self, max: usize) -> Self {
317            self.inner = Some(self.inner.take().expect("always set").max_depth(max));
318            self
319        }
320        /// Follow symbolic links.
321        pub fn follow_links(mut self, toggle: bool) -> Self {
322            self.inner = Some(self.inner.take().expect("always set").follow_links(toggle));
323            self
324        }
325    }
326
327    /// Instantiate a new directory iterator which will not skip hidden files, with the given level of `parallelism`.
328    ///
329    /// Use `precompose_unicode` to represent the `core.precomposeUnicode` configuration option.
330    pub fn walkdir_new(root: &Path, _: Parallelism, precompose_unicode: bool) -> WalkDir {
331        WalkDir {
332            inner: WalkDirImpl::new(root).into(),
333            precompose_unicode,
334        }
335    }
336
337    /// Instantiate a new directory iterator which will not skip hidden files and is sorted, with the given level of `parallelism`.
338    ///
339    /// Use `precompose_unicode` to represent the `core.precomposeUnicode` configuration option.
340    pub fn walkdir_sorted_new(root: &Path, _: Parallelism, precompose_unicode: bool) -> WalkDir {
341        WalkDir {
342            inner: WalkDirImpl::new(root).sort_by_file_name().into(),
343            precompose_unicode,
344        }
345    }
346
347    /// The Iterator yielding directory items
348    pub type DirEntryIter = super::walkdir_precompose::WalkDirIter<walkdir::IntoIter, DirEntryImpl, walkdir::Error>;
349}
350
351#[cfg(any(feature = "walkdir", feature = "fs-walkdir-parallel"))]
352pub use self::walkdir::{walkdir_new, walkdir_sorted_new, WalkDir};
353
354/// Prepare open options which won't follow symlinks when the file is opened.
355///
356/// Note: only effective on unix currently.
357pub fn open_options_no_follow() -> std::fs::OpenOptions {
358    #[cfg_attr(not(unix), allow(unused_mut))]
359    let mut options = std::fs::OpenOptions::new();
360    #[cfg(unix)]
361    {
362        /// Make sure that it's impossible to follow through to the target of symlinks.
363        /// Note that this will still follow symlinks in the path, which is what we assume
364        /// has been checked separately.
365        use std::os::unix::fs::OpenOptionsExt;
366        options.custom_flags(libc::O_NOFOLLOW);
367    }
368    options
369}
gix_features/fs.rs

gix_features/
fs.rs
