plugx_config/loader/
fs.rs

1//! File system configuration loader (`fs` feature).
2//!
3//! * Supported schema: `fs` and `file`  
4//! *
5//!
6//! ### Example
7//! ```rust
8//! use std::{fs, collections::HashMap};
9//! use tempdir::TempDir;
10//! use plugx_config::loader::{Loader, fs::{Fs, SoftErrorsFs}};
11//! use url::Url;
12//!
13//! // Create a temporary directory containing `foo.json`, `bar.yaml`, and `baz.toml`:
14//! let tmp_dir = TempDir::new("fs-example").unwrap();
15//! let foo = tmp_dir.path().join("foo.json");
16//! fs::write(&foo, "{\"hello\": \"world\"}").unwrap();
17//! let bar = tmp_dir.path().join("bar.yaml");
18//! fs::write(&bar, "hello: world").unwrap();
19//! let baz = tmp_dir.path().join("baz.toml");
20//! fs::write(&baz, "hello = \"world\"").unwrap();
21//! let url = Url::try_from(format!("file://{}", tmp_dir.path().to_str().unwrap()).as_str()).unwrap();
22//!
23//! let mut loader = Fs::new();
24//! // You could set some skippable errors here.
25//! // For example if you're loading contents of one file that may potentially not exists:
26//! // loader.add_skippable_error(SkippbaleErrorKind::NotFound)
27//!
28//! // Load all configurations inside directory:
29//! let loaded = loader.load(&url, None, false).unwrap();
30//! assert_eq!(loaded.len(), 3);
31//! let (_, foo) = loaded.iter().find(|(plugin_name, _)| plugin_name == "foo").expect("`foo` plugin config");
32//! assert_eq!(foo.maybe_format(), Some(&"json".to_string()));
33//! let (_, bar) = loaded.iter().find(|(plugin_name, _)| plugin_name == "bar").expect("`bar` plugin config");
34//! assert_eq!(bar.maybe_contents(), Some(&"hello: world".to_string()));
35//!
36//! // Only load `foo` and `bar`:
37//! let whitelist = ["foo".into(), "bar".into()].to_vec();
38//! let loaded = loader.load(&url, Some(&whitelist), false).unwrap();
39//! assert_eq!(loaded.len(), 2);
40//!
41//! // Load just one file:
42//! let qux = tmp_dir.path().join("qux.env");
43//! fs::write(&qux, "hello=\"world\"").unwrap();
44//! let url = Url::try_from(format!("file://{}", qux.to_str().unwrap()).as_str()).unwrap();
45//! let loaded = loader.load(&url, None, false).unwrap();
46//! assert_eq!(loaded.len(), 1);
47//! ```
48//!
49//! See [loader] documentation to known how loaders work.
50
51use crate::{
52    entity::ConfigurationEntity,
53    loader::{self, Error, Loader, SoftErrors},
54};
55use anyhow::anyhow;
56use cfg_if::cfg_if;
57use serde::Deserialize;
58use std::fmt::{Display, Formatter};
59use std::{
60    collections::HashMap,
61    env::current_dir,
62    fmt::Debug,
63    fs, io,
64    path::{Path, PathBuf},
65};
66use url::Url;
67
68pub const NAME: &str = "File";
69pub const SCHEME_LIST: &[&str] = &["fs", "file"];
70
71/// Loads configurations from filesystem.
72#[derive(Default, Clone, Debug)]
73pub struct Fs {
74    options: FsOptions,
75}
76
77#[derive(Debug, Clone, Default, Deserialize)]
78#[serde(default, rename_all = "kebab-case")]
79pub struct FsOptions {
80    strip_slash: Option<bool>,
81    soft_errors: SoftErrors<SoftErrorsFs>,
82}
83
84impl FsOptions {
85    pub fn contains(&self, error: io::ErrorKind) -> bool {
86        SoftErrorsFs::try_from(error)
87            .map(|error| self.soft_errors.contains(&error))
88            .unwrap_or_default()
89    }
90}
91
92/// Supported soft errors when loading filesystem contents.
93#[derive(Debug, Clone, Copy, Deserialize, PartialEq)]
94#[serde(rename_all = "kebab-case")]
95pub enum SoftErrorsFs {
96    NotFound,
97    PermissionDenied,
98}
99
100impl TryFrom<io::ErrorKind> for SoftErrorsFs {
101    type Error = String;
102
103    fn try_from(value: io::ErrorKind) -> Result<Self, Self::Error> {
104        match value {
105            io::ErrorKind::NotFound => Ok(Self::NotFound),
106            io::ErrorKind::PermissionDenied => Ok(Self::PermissionDenied),
107            _ => Err("Unhandled IO error".into()),
108        }
109    }
110}
111
112#[doc(hidden)]
113impl Fs {
114    #[inline]
115    pub fn get_plugin_name_and_format<P: AsRef<Path>>(path: P) -> Option<(String, String)> {
116        Self::get_plugin_name(&path)
117            .and_then(|name| Self::get_format(&path).map(|format| (name, format)))
118    }
119
120    #[inline]
121    pub fn get_plugin_name<P: AsRef<Path>>(path: P) -> Option<String> {
122        path.as_ref()
123            .file_stem()
124            .and_then(|name| name.to_str())
125            .map(|name| name.to_lowercase())
126            .and_then(|name| if name.is_empty() { None } else { Some(name) })
127    }
128
129    #[inline]
130    pub fn get_format<P: AsRef<Path>>(path: P) -> Option<String> {
131        path.as_ref()
132            .extension()
133            .and_then(|format| format.to_str())
134            .map(|format| format.to_lowercase())
135            .and_then(|format| {
136                if format.is_empty() {
137                    None
138                } else {
139                    Some(format)
140                }
141            })
142    }
143
144    #[inline]
145    pub(super) fn get_entity_list(
146        url: &Url,
147        options: &FsOptions,
148        maybe_whitelist: Option<&[String]>,
149        skip_soft_errors: bool,
150    ) -> Result<Vec<ConfigurationEntity>, Error> {
151        let path = Self::url_to_path(url, options)
152            .map_err(|_| Error::Other(anyhow!("Could not detect current working directory")))?;
153        if path.is_dir() {
154            let list = match Self::get_directory_file_list(&path, maybe_whitelist) {
155                Ok(list) => list,
156                Err(error) => {
157                    return if skip_soft_errors
158                        && (options.soft_errors.skip_all() || options.contains(error.kind()))
159                    {
160                        cfg_if! {
161                            if #[cfg(feature = "tracing")] {
162                                tracing::info!(path=?path, skip_error=true, "Could not load directory contents");
163                            } else if #[cfg(feature = "logging")] {
164                                log::info!("msg=\"Could not load directory contents\" path={path:?} skip_error=true");
165                            }
166                        }
167                        Ok(Vec::new())
168                    } else {
169                        Err(Error::Load {
170                            loader: NAME.to_string(),
171                            url: url.clone(),
172                            description: "load directory file list".to_string().into(),
173                            source: error.into(),
174                        })
175                    }
176                }
177            };
178            let mut plugins: HashMap<&String, &String> = HashMap::with_capacity(list.len());
179            for (plugin_name, format, _) in list.iter() {
180                if let Some(other_format) = plugins.get(plugin_name) {
181                    let mut url = url.clone();
182                    url.set_query(None);
183                    return Err(Error::Duplicate {
184                        loader: NAME.to_string().into(),
185                        url,
186                        plugin: plugin_name.to_string().into(),
187                        format_1: other_format.to_string().into(),
188                        format_2: format.to_string().into(),
189                    });
190                } else {
191                    plugins.insert(plugin_name, format);
192                }
193            }
194            Ok(list
195                .into_iter()
196                .map(|(plugin_name, format, path)| {
197                    ConfigurationEntity::new(path.to_str().unwrap(), url.clone(), plugin_name, NAME)
198                        .with_format(format)
199                })
200                .collect())
201        } else if path.is_file() {
202            if let Some((plugin_name, format)) = Self::get_plugin_name_and_format(&path) {
203                if maybe_whitelist
204                    .map(|whitelist| whitelist.contains(&plugin_name))
205                    .unwrap_or(true)
206                {
207                    let entity = ConfigurationEntity::new(
208                        path.to_str().unwrap(),
209                        url.clone(),
210                        plugin_name,
211                        NAME,
212                    )
213                    .with_format(format);
214                    Ok([entity].into())
215                } else {
216                    Ok(Vec::new())
217                }
218            } else if skip_soft_errors && options.soft_errors.skip_all() {
219                cfg_if! {
220                    if #[cfg(feature = "tracing")] {
221                        tracing::info!(url=%url, skip_error=true, "Could not parse plugin name/format");
222                    } else if #[cfg(feature = "logging")] {
223                        log::info!(
224                            "msg=\"Could not parse plugin name/format\" url={:?} skip_error=true",
225                            url.to_string()
226                        );
227                    }
228                }
229                Ok(Vec::new())
230            } else {
231                Err(Error::InvalidUrl {
232                    loader: NAME.to_string(),
233                    url: url.to_string(),
234                    source: anyhow!("Could not parse plugin name/format"),
235                })
236            }
237        } else if path.exists() {
238            if skip_soft_errors && options.soft_errors.skip_all() {
239                cfg_if! {
240                    if #[cfg(feature = "tracing")] {
241                        tracing::info!(url=%url, skip_error=true, "URL is not pointing to a directory or regular file");
242                    } else if #[cfg(feature = "logging")] {
243                        log::info!(
244                            "msg=\"URL is not pointing to a directory or regular file\" url={:?} skip_error=true",
245                            url.to_string()
246                        );
247                    }
248                }
249                Ok(Vec::new())
250            } else {
251                Err(Error::InvalidUrl {
252                    loader: NAME.to_string(),
253                    url: url.to_string(),
254                    source: anyhow!("URL is not pointing to a directory or regular file"),
255                })
256            }
257        } else if skip_soft_errors && options.contains(io::ErrorKind::NotFound) {
258            cfg_if! {
259                if #[cfg(feature = "tracing")] {
260                    tracing::info!(url=%url, skip_error=true, "Could not find path");
261                } else if #[cfg(feature = "logging")] {
262                    log::info!(
263                        "msg=\"Could not find path\" url={:?} skip_error=true",
264                        url.to_string()
265                    );
266                }
267            }
268            Ok(Vec::new())
269        } else {
270            Err(Error::NotFound {
271                loader: NAME.to_string(),
272                url: url.clone(),
273                item: format!("path `{path:?}`").into(),
274            })
275        }
276    }
277
278    #[inline]
279    pub fn get_directory_file_list<P: AsRef<Path>>(
280        path: P,
281        maybe_whitelist: Option<&[String]>,
282    ) -> Result<Vec<(String, String, PathBuf)>, io::Error> {
283        Ok(fs::read_dir(path)?
284            .filter_map(|maybe_entry| maybe_entry.ok())
285            .map(|entry| entry.path())
286            .filter_map(|path| {
287                if let Some((plugin_name, format)) = Self::get_plugin_name_and_format(&path) {
288                    cfg_if! {
289                        if #[cfg(feature = "tracing")] {
290                            tracing::trace!(plugin=plugin_name, path=?path, "Detected configuration file");
291                        } else if #[cfg(feature = "logging")] {
292                            log::trace!("msg=\"Detected configuration file\" plugin={plugin_name:?} path={path:?}");
293                        }
294                    }
295                    Some((plugin_name, format, path))
296                } else {
297                    cfg_if! {
298                        if #[cfg(feature = "tracing")] {
299                            tracing::warn!(path=?path, "Could not parse plugin name/format");
300                        } else if #[cfg(feature = "logging")] {
301                            log::warn!("msg=\"Could not parse plugin name/format\" path={path:?}");
302                        }
303                    }
304                    None
305                }
306            })
307            .filter(|(plugin_name, _, _)| {
308                maybe_whitelist
309                    .map(|whitelist| whitelist.contains(plugin_name))
310                    .unwrap_or(true)
311            })
312            .filter_map(|(plugin_name, format, path)| {
313                if path.is_file() {
314                    Some((plugin_name, format, path))
315                } else {
316                    cfg_if! {
317                        if #[cfg(feature = "tracing")] {
318                            tracing::warn!(path=?path, "Path is not pointing to a regular file");
319                        } else if #[cfg(feature = "logging")] {
320                            log::warn!("msg=\"Path is not pointing to a regular file\" path={path:?}");
321                        }
322                    }
323                    None
324                }
325            })
326            .collect())
327    }
328
329    #[inline]
330    pub fn read_entity_contents(entity: &mut ConfigurationEntity) -> Result<(), io::Error> {
331        fs::read_to_string(entity.item()).map(|contents| {
332            entity.set_contents(contents);
333        })
334    }
335
336    #[inline]
337    pub fn url_to_path(url: &Url, options: &FsOptions) -> Result<PathBuf, io::Error> {
338        cfg_if! {
339            if #[cfg(target_os="windows")] {
340                let is_windows = true;
341            } else {
342                let is_windows = false;
343            }
344        }
345        let url_path = if url.path() == "/" || url.path().is_empty() {
346            let cwd = current_dir()?;
347            cfg_if! {
348                if #[cfg(feature = "tracing")] {
349                    tracing::debug!(url=%url, cwd=?cwd,"set current working directory");
350                } else if #[cfg(feature = "logging")] {
351                    log::debug!("msg=\"set current working directory\" url=\"{url}\", cwd={cwd:?}");
352                }
353            }
354            cwd
355        } else if (options.strip_slash.unwrap_or(false) || is_windows)
356            && url.path().starts_with('/')
357        {
358            PathBuf::from(
359                url.path()
360                    .strip_prefix('/')
361                    .expect("URL path with length > 1"),
362            )
363        } else {
364            PathBuf::from(url.path())
365        };
366        let mut path = PathBuf::new();
367        url_path.components().for_each(|component| {
368            path = path.join(component);
369        });
370        cfg_if! {
371            if #[cfg(feature = "tracing")] {
372                tracing::trace!(url_path=?url_path, os_path=?path,"Changed URL path to OS path");
373            } else if #[cfg(feature = "logging")] {
374                log::trace!("msg=\"Changed URL path to OS path\" url_path={url_path:?}, os_path={path:?}");
375            }
376        }
377        Ok(path)
378    }
379}
380
381impl Fs {
382    pub fn new() -> Self {
383        Default::default()
384    }
385
386    pub fn add_soft_error(&mut self, error: SoftErrorsFs) {
387        self.options.soft_errors.add_soft_error(error)
388    }
389
390    pub fn with_soft_error(mut self, error: SoftErrorsFs) -> Self {
391        self.add_soft_error(error);
392        self
393    }
394
395    fn get_options(&self, url: &Url) -> Result<FsOptions, Error> {
396        loader::deserialize_query_string::<FsOptions>(NAME, url).map(|mut options| {
397            if let Some(soft_errors) = self.options.soft_errors.maybe_soft_error_list() {
398                soft_errors
399                    .iter()
400                    .for_each(|soft_error| options.soft_errors.add_soft_error(*soft_error))
401            }
402            options
403        })
404    }
405}
406
407impl Display for Fs {
408    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
409        f.write_str(NAME)
410    }
411}
412
413impl Loader for Fs {
414    /// In this case "fs" and "file".
415    fn scheme_list(&self) -> Vec<String> {
416        SCHEME_LIST.iter().cloned().map(String::from).collect()
417    }
418
419    fn load(
420        &self,
421        url: &Url,
422        maybe_whitelist: Option<&[String]>,
423        skip_soft_errors: bool,
424    ) -> Result<Vec<(String, ConfigurationEntity)>, Error> {
425        let options = self.get_options(url)?;
426        let mut entity_list =
427            Self::get_entity_list(url, &options, maybe_whitelist, skip_soft_errors)?;
428        entity_list.iter_mut().try_for_each(|entity| {
429            match Self::read_entity_contents(entity) {
430                Ok(_) => {
431                    cfg_if! {
432                        if #[cfg(feature = "tracing")] {
433                            tracing::trace!(
434                                url=%entity.url(),
435                                contents=entity
436                                    .maybe_contents()
437                                    .expect("Contents is set inside `utils::read_entity_contents`"),
438                                "Read configuration file"
439                            );
440                        } else if #[cfg(feature = "logging")] {
441                            log::trace!(
442                                "msg=\"Read configuration file\" url={:?} contents={:?}",
443                                entity.url().to_string(),
444                                entity.maybe_contents().expect("Contents is set inside `utils::read_entity_contents`")
445                            );
446                        }
447                    }
448                    Ok(())
449                },
450                Err(error) => {
451                    if skip_soft_errors && (options.soft_errors.skip_all() || options.contains(error.kind())) {
452                        cfg_if! {
453                            if #[cfg(feature = "tracing")] {
454                                tracing::info!(
455                                    path=entity.url().path(),
456                                    skip_error=true,
457                                    "Could not read contents of file"
458                                );
459                            } else if #[cfg(feature = "logging")] {
460                                log::info!(
461                                    "msg=\"Could not read contents of file\" path={:?} skip_error=true",
462                                    entity.url().path()
463                                );
464                            }
465                        }
466                        Ok(())
467                    } else {
468                        Err(Error::Load {
469                            loader: NAME.to_string(),
470                            url: entity.url().clone(),
471                            description: "read contents of file".to_string().into(),
472                            source: error.into(),
473                        })
474                    }
475                }
476            }
477        })?;
478        let result = entity_list
479            .into_iter()
480            // Maybe we have skipped soft errors in above:
481            .filter(|entity| entity.maybe_contents().is_some())
482            .map(|entity| (entity.plugin_name().clone(), entity))
483            .collect();
484        Ok(result)
485    }
486}
plugx_config/loader/fs.rs

plugx_config/loader/
fs.rs
