Skip to main content

c4/
loader.rs

1//! The loader: scan sources, pick parsers by extension, deep-merge. One
2//! code path for `trace()` and `load()`, generic over the merge target:
3//! the traced tree, or a plain value that skips the per-leaf provenance.
4
5use std::collections::BTreeMap;
6use std::path::{Path, PathBuf};
7
8use serde::de::DeserializeOwned;
9
10use crate::trace::{SourceRef, Trace, TracedValue, Warning};
11use crate::{Error, Format, FormatKind, Options, Order, Result, Source, Value, error, format};
12
13/// Loads and merges the config sources named by its [`Options`].
14#[derive(Debug, Clone, Default)]
15pub struct Loader {
16    pub options: Options,
17}
18
19impl Loader {
20    /// Everything — sources included — comes in through [`Options`];
21    /// there is no separate builder step.
22    pub fn new(options: Options) -> Self {
23        Self { options }
24    }
25
26    /// Load and merge all sources, deserializing into `T`.
27    ///
28    /// [`Value`] implements `Deserialize`, so `T = Value` gives dynamic
29    /// access and any other serde type gives typed config. A `Value` —
30    /// the whole result, or a field of your struct — is handed over as
31    /// merged, typed variants (`DateTime`, `Ipv4`, …) included, exactly as
32    /// [`Loader::trace`] shows them.
33    ///
34    /// A **string** reaching a `bool`, integer or float field parses with
35    /// the rules of that table type id (`u16`, `bool`, … — see the crate
36    /// docs' table section), so ini, env and cnf values, which are all
37    /// strings, load into typed fields. The explicit `bool` tokens apply
38    /// (`yes`/`no`, `on`/`off`, `1`/`0`, …), and out-of-range or malformed
39    /// text is [`Error::Deserialize`] (`'70000' is not a valid u16`). Untyped
40    /// targets (`Value`, `String`) keep the string.
41    ///
42    /// The reverse holds too: a number or bool reaching a `String` field
43    /// becomes its text (`password: 123456` → `"123456"`; a float in its
44    /// shortest round-trip form, so `1.0` stays `"1.0"`).
45    ///
46    /// Every [`Error::Deserialize`] names the key path it failed at —
47    /// `db.password: invalid type: …`, `items[2].id: …`. An empty value
48    /// (`PORT=`) cannot fill a number or bool field, not even an `Option`
49    /// one (serde picks `None` or `Some` before it sees the inner type);
50    /// the error says to remove the line instead.
51    ///
52    /// To type one part of a loaded [`Value`], use [`Value::into_typed`].
53    ///
54    /// ```
55    /// #[derive(serde::Deserialize)]
56    /// struct Server {
57    ///     port: u16,
58    ///     debug: bool,
59    /// }
60    ///
61    /// # fn main() -> Result<(), c4::Error> {
62    /// // the 1-tuple value source stands in for an ini/env/cnf file here
63    /// let server: Server = c4::Loader::new(c4::Options {
64    ///     sources: vec![(serde_json::json!({ "port": "8080", "debug": "yes" }),).into()],
65    ///     ..c4::Options::default()
66    /// })
67    /// .load()?;
68    /// assert_eq!(server.port, 8080);
69    /// assert!(server.debug);
70    /// # Ok(())
71    /// # }
72    /// ```
73    ///
74    /// [`Warning`]s are dropped — call [`Loader::trace`] to see why some
75    /// input was not used (a skipped cnf `!include`, for instance).
76    pub fn load<T: DeserializeOwned>(&self) -> Result<T> {
77        let (value, _warnings) = self.run::<Value>()?;
78        value.into_typed()
79    }
80
81    /// Like [`Loader::load`], but keeps per-leaf provenance and the
82    /// [`Warning`]s — see [`Trace`]. This powers the CLI `--trace` mode;
83    /// source labels are a debug/testing aid, not config data.
84    ///
85    /// ```
86    /// use c4::{CustomFormat, Loader, TracedValue};
87    ///
88    /// # fn main() -> Result<(), c4::Error> {
89    /// // a custom-format string source keeps this example runnable
90    /// // under every feature combination (`Loader::default()` would
91    /// // read ./config instead)
92    /// let kv = CustomFormat::new("kv", ["kv"], |text, path, options| {
93    ///     let rows = text
94    ///         .lines()
95    ///         .map(|line| line.split('=').map(str::to_owned).collect())
96    ///         .collect();
97    ///     c4::parse_table(rows, &c4::TableLayout::Kv, path, options)
98    /// });
99    ///
100    /// let trace = Loader::new(c4::Options {
101    ///     sources: vec![(kv, "port=8080").into()],
102    ///     ..c4::Options::default()
103    /// })
104    /// .trace()?;
105    /// assert!(trace.warnings.is_empty());
106    /// let TracedValue::Object(root) = &trace.value else { unreachable!() };
107    /// let TracedValue::Leaf { value, source } = &root["port"] else { unreachable!() };
108    /// assert_eq!(value.as_u64(), Some(8080));
109    /// assert_eq!(source, &c4::SourceRef::String(0)); // "string:0" in traces
110    /// # Ok(())
111    /// # }
112    /// ```
113    pub fn trace(&self) -> Result<Trace> {
114        let (value, warnings) = self.run::<TracedValue>()?;
115        Ok(Trace { value, warnings })
116    }
117
118    /// Scan, parse and merge every source into `R` — the one code path
119    /// behind both [`Loader::trace`] (a traced tree) and [`Loader::load`]
120    /// (a plain value, skipping the per-leaf provenance it would drop).
121    fn run<R: Tree>(&self) -> Result<(R, Vec<Warning>)> {
122        let extensions = extension_map(&self.options);
123        let mut root = R::root();
124        let mut warnings = Vec::new();
125
126        for (index, source) in self.options.sources.iter().enumerate() {
127            match source {
128                // one path source: a folder whose files merge, or a single
129                // file — detected here (same rule as `c4::load`)
130                Source::Path(path) if path.is_dir() => {
131                    self.load_folder(path, &extensions, &mut root, &mut warnings)?;
132                }
133                Source::Path(path) if path.is_file() => {
134                    let claim = claimed_format(path, &extensions).ok_or_else(|| Error::Parse {
135                        path: path.clone(),
136                        message: unclaimed_message(path),
137                    })?;
138                    let source = SourceRef::File(path.clone());
139                    let mut messages = Vec::new();
140                    let value = self.parse_file(claim, path, &mut messages)?;
141                    note(&mut warnings, &source, messages);
142                    if !matches!(value, Value::Null) {
143                        self.merge(&mut root, value, &source);
144                    }
145                }
146                Source::Path(path) => return Err(Error::NotFound(path.clone())),
147                Source::Table {
148                    path,
149                    format,
150                    sheet,
151                    layout,
152                } => {
153                    // one file of a table format, read under an explicit
154                    // layout (and, for spreadsheets, an explicit sheet)
155                    if !path.is_file() {
156                        return Err(Error::Parse {
157                            path: path.clone(),
158                            message: "table source is not a file (table sources read \
159                                      exactly one file; for in-code text use a \
160                                      (format, text) string source)"
161                                .into(),
162                        });
163                    }
164                    let value = format::parse_table_file(
165                        *format,
166                        path,
167                        sheet.as_deref(),
168                        layout,
169                        &self.options,
170                    )?;
171                    if !matches!(value, Value::Null) {
172                        self.merge(&mut root, value, &SourceRef::File(path.clone()));
173                    }
174                }
175                Source::Value(result) => {
176                    let mut value = result.clone().map_err(|message| Error::Parse {
177                        path: PathBuf::from(format!("value:{index}")),
178                        message,
179                    })?;
180                    format::strip_commented_data_keys(&mut value, &self.options);
181                    if !matches!(value, Value::Null) {
182                        self.merge(&mut root, value, &SourceRef::Value(index));
183                    }
184                }
185                Source::String { format, content } => {
186                    let label = PathBuf::from(format!("string:{index}"));
187                    let mut messages = Vec::new();
188                    let value = match format {
189                        // `format::parse` filters commented data keys
190                        // itself; a custom parser's result is filtered here
191                        FormatKind::Builtin(format) => format::parse(
192                            *format,
193                            format::strip_bom(content),
194                            &crate::TableLayout::Kv,
195                            &label,
196                            &self.options,
197                            &mut messages,
198                        )?,
199                        FormatKind::Custom(custom) => {
200                            let mut value =
201                                (custom.parser)(format::strip_bom(content), &label, &self.options)?;
202                            format::strip_commented_data_keys(&mut value, &self.options);
203                            value
204                        }
205                    };
206                    let source = SourceRef::String(index);
207                    note(&mut warnings, &source, messages);
208                    if !matches!(value, Value::Null) {
209                        self.merge(&mut root, value, &source);
210                    }
211                }
212            }
213        }
214        Ok((root, warnings))
215    }
216
217    fn parse_file(&self, claim: Claim, path: &Path, warnings: &mut Vec<String>) -> Result<Value> {
218        match claim {
219            Claim::Builtin { format, spec } => {
220                // the claiming formats entry decides the table layout
221                let layout = &self.options.formats[spec].layout;
222                // binary formats (spreadsheets) parse from the path
223                // directly — their bytes must never go through the text
224                // pipeline
225                if format::is_binary(format) {
226                    return format::parse_binary(format, path, layout, &self.options);
227                }
228                let text = format::read_text(path)?;
229                format::parse(format, &text, layout, path, &self.options, warnings)
230            }
231            Claim::Custom(index) => {
232                let FormatKind::Custom(custom) = &self.options.formats[index].format else {
233                    unreachable!("custom claims always index custom entries");
234                };
235                let text = format::read_text(path)?;
236                // built-in formats filter commented data keys inside
237                // `format::parse`/`parse_binary`; do it for custom ones here
238                let mut value = (custom.parser)(&text, path, &self.options)?;
239                format::strip_commented_data_keys(&mut value, &self.options);
240                Ok(value)
241            }
242        }
243    }
244
245    fn merge<R: Tree>(&self, target: &mut R, value: Value, source: &SourceRef) {
246        target.merge(value, source, !self.options.case_sensitive);
247    }
248
249    /// Load one folder source: scan it (honoring [`Options::dir_depth`]),
250    /// parse each file, and merge it in. The keying options decide the
251    /// shape — [`filename_as_key`](Options::filename_as_key) wraps each
252    /// file's value under a key named after the file, and
253    /// [`dirname_as_key`](Options::dirname_as_key) wraps under the
254    /// subfolder names leading to it. With both off this is a flat merge;
255    /// with both on (the `tree` feature's default) it is a folder tree.
256    /// Entries load in [`Options::order`], so key collisions deep-merge
257    /// with the later entry winning.
258    fn load_folder<R: Tree>(
259        &self,
260        folder: &Path,
261        extensions: &[(String, Claim)],
262        root: &mut R,
263        warnings: &mut Vec<Warning>,
264    ) -> Result<()> {
265        let mut files = Vec::new();
266        walk(
267            folder,
268            Vec::new(),
269            &self.options,
270            self.options.dir_depth,
271            &mut files,
272        )?;
273        for (file, dirs) in files {
274            let name = file
275                .file_name()
276                .and_then(|n| n.to_str())
277                .unwrap_or_default()
278                .to_owned();
279            let mut messages = Vec::new();
280            let value = match claimed_format(&file, extensions) {
281                Some(claim) => self.parse_file(claim, &file, &mut messages)?,
282                // extension-less / unknown-extension handling only applies
283                // when a file becomes a key; otherwise the file has no key
284                // to contribute under and is skipped
285                None if self.options.filename_as_key
286                    && file_extension(&name).is_none()
287                    && self.options.auto_no_ext_files =>
288                {
289                    // the trimmed content goes through the table auto
290                    // detection (same feature-gated guesses)
291                    let text = format::read_text(&file)?;
292                    format::table_auto(text.trim())
293                }
294                None if self.options.filename_as_key && !self.options.ignore_unknown_ext => {
295                    return Err(Error::Parse {
296                        message: unclaimed_message(&file),
297                        path: file,
298                    });
299                }
300                None => continue, // no active format claims this extension
301            };
302            let source = SourceRef::File(file);
303            note(warnings, &source, messages);
304            if matches!(value, Value::Null) {
305                continue; // empty file contributes nothing
306            }
307            let mut value = value;
308            if self.options.filename_as_key {
309                value = Value::Object(BTreeMap::from([(file_stem_key(&name), value)]));
310            }
311            if self.options.dirname_as_key {
312                for key in dirs.into_iter().rev() {
313                    value = Value::Object(BTreeMap::from([(key, value)]));
314                }
315            }
316            self.merge(root, value, &source);
317        }
318        Ok(())
319    }
320}
321
322/// The [`filename_as_key`](Options::filename_as_key) key of a file: its
323/// name without the extension. A hidden file whose whole name is its
324/// extension (`.env`) keeps the full name.
325fn file_stem_key(name: &str) -> String {
326    let hidden = name.starts_with('.');
327    let body = if hidden { &name[1..] } else { name };
328    match body.rfind('.') {
329        Some(i) => format!("{}{}", if hidden { "." } else { "" }, &body[..i]),
330        None => name.to_owned(),
331    }
332}
333
334/// What an extension resolves to: a built-in format (with the index of
335/// its [`Options::formats`] entry, whose layout applies), or a custom
336/// entry (by index).
337#[derive(Clone, Copy)]
338enum Claim {
339    Builtin { format: Format, spec: usize },
340    Custom(usize),
341}
342
343/// The extension→parser mapping of [`Options::formats`]: lowercased
344/// extension, last claimer wins — built-in and custom entries follow the
345/// same rule.
346fn extension_map(options: &Options) -> Vec<(String, Claim)> {
347    let mut map: Vec<(String, Claim)> = Vec::new();
348    for (index, spec) in options.formats.iter().enumerate() {
349        let claim = match &spec.format {
350            FormatKind::Builtin(format) => Claim::Builtin {
351                format: *format,
352                spec: index,
353            },
354            FormatKind::Custom(_) => Claim::Custom(index),
355        };
356        for ext in &spec.extensions {
357            let ext = ext.to_lowercase();
358            map.retain(|(e, _)| *e != ext);
359            map.push((ext, claim));
360        }
361    }
362    map
363}
364
365/// The extension of a file name, lowercased. A hidden file `.X` counts as
366/// having extension `X` (that is how `.env` matches `env`).
367fn file_extension(name: &str) -> Option<String> {
368    let hidden = name.starts_with('.');
369    let stem = if hidden { &name[1..] } else { name };
370    match stem.rfind('.') {
371        Some(i) => Some(stem[i + 1..].to_lowercase()),
372        None if hidden && !stem.is_empty() => Some(stem.to_lowercase()),
373        None => None,
374    }
375}
376
377/// Why no format claims `path`, and how to fix it: the built-in format
378/// whose default extensions include it is either compiled out (name the
379/// feature) or missing from `Options.formats`.
380fn unclaimed_message(path: &Path) -> String {
381    let base = "no active format claims this file's extension";
382    let Some(ext) = path
383        .file_name()
384        .and_then(|n| n.to_str())
385        .and_then(file_extension)
386    else {
387        return base.into();
388    };
389    // jsonc before json: it is the default reader of `.json`
390    let reader = Format::ALL
391        .into_iter()
392        .find(|f| f.default_extensions().contains(&ext.as_str()));
393    match reader {
394        Some(f) if !format::is_compiled(f) => format!(
395            "{base}: '.{ext}' is read by the `{id}` format — enable the `{id}` feature",
396            id = f.id()
397        ),
398        Some(f) => format!(
399            "{base}: the `{}` format reads it but is not in `formats`",
400            f.id()
401        ),
402        None => base.into(),
403    }
404}
405
406fn claimed_format(path: &Path, extensions: &[(String, Claim)]) -> Option<Claim> {
407    let name = path.file_name()?.to_str()?;
408    let ext = file_extension(name)?;
409    extensions
410        .iter()
411        .find(|(e, _)| *e == ext)
412        .map(|(_, claim)| *claim)
413}
414
415/// A directory's entries as `(name, path, is_dir)`.
416fn read_entries(dir: &Path) -> Result<Vec<(String, PathBuf, bool)>> {
417    let mut entries = Vec::new();
418    for entry in std::fs::read_dir(dir).map_err(error::io(dir))? {
419        let entry = entry.map_err(error::io(dir))?;
420        let path = entry.path();
421        let Some(name) = path.file_name().and_then(|n| n.to_str()) else {
422            continue;
423        };
424        entries.push((name.to_owned(), path.clone(), path.is_dir()));
425    }
426    Ok(entries)
427}
428
429fn sort_entries(entries: &mut [(String, PathBuf, bool)], order: Order) {
430    match order {
431        Order::FoldersFirstAlphabetic => entries.sort_by(|a, b| (!a.2, &a.0).cmp(&(!b.2, &b.0))),
432        Order::Alphabetic => entries.sort_by(|a, b| a.0.cmp(&b.0)),
433        Order::ReverseAlphabetic => entries.sort_by(|a, b| b.0.cmp(&a.0)),
434    }
435}
436
437/// All files of a folder in load order, each with the subfolder names
438/// leading to it. `depth` is the remaining [`Options::dir_depth`] budget:
439/// `0` stops descent, `-1` recurses without limit, a positive value
440/// decrements per level. Commented-out names (`#…`, `_…`) are skipped
441/// here — by name, so keyed or not, a `_draft.yml` file and everything
442/// under a `_wip/` folder stay out of the scan.
443fn walk(
444    dir: &Path,
445    prefix: Vec<String>,
446    options: &Options,
447    depth: isize,
448    out: &mut Vec<(PathBuf, Vec<String>)>,
449) -> Result<()> {
450    let mut entries = read_entries(dir)?;
451    sort_entries(&mut entries, options.order);
452    for (name, path, is_dir) in entries {
453        let ignore_commented = if is_dir {
454            options.ignore_commented_dirnames
455        } else {
456            options.ignore_commented_filenames
457        };
458        if ignore_commented && crate::options::is_commented(&name) {
459            continue;
460        }
461        if is_dir {
462            if depth != 0 {
463                let mut prefix = prefix.clone();
464                prefix.push(name);
465                walk(
466                    &path,
467                    prefix,
468                    options,
469                    if depth < 0 { -1 } else { depth - 1 },
470                    out,
471                )?;
472            }
473        } else {
474            out.push((path, prefix.clone()));
475        }
476    }
477    Ok(())
478}
479
480fn note(warnings: &mut Vec<Warning>, source: &SourceRef, messages: Vec<String>) {
481    warnings.extend(messages.into_iter().map(|message| Warning {
482        source: source.clone(),
483        message,
484    }));
485}
486
487/// What the sources merge into. Both trees deep-merge the same way —
488/// objects recurse, everything else replaces, keys lowercased when merging
489/// is case-insensitive — and differ only in whether leaves keep `source`.
490trait Tree {
491    fn root() -> Self;
492    fn merge(&mut self, value: Value, source: &SourceRef, lowercase: bool);
493}
494
495impl Tree for TracedValue {
496    fn root() -> Self {
497        TracedValue::Object(BTreeMap::new())
498    }
499
500    fn merge(&mut self, value: Value, source: &SourceRef, lowercase: bool) {
501        merge_traced(self, value, source, lowercase);
502    }
503}
504
505impl Tree for Value {
506    fn root() -> Self {
507        Value::Object(BTreeMap::new())
508    }
509
510    fn merge(&mut self, value: Value, _source: &SourceRef, lowercase: bool) {
511        merge_plain(self, value, lowercase);
512    }
513}
514
515/// [`merge_traced`] without the provenance.
516fn merge_plain(target: &mut Value, value: Value, lowercase: bool) {
517    match (target, value) {
518        (Value::Object(entries), Value::Object(incoming)) => {
519            for (key, value) in incoming {
520                let key = if lowercase { key.to_lowercase() } else { key };
521                match entries.get_mut(&key) {
522                    Some(slot) => merge_plain(slot, value, lowercase),
523                    None => {
524                        entries.insert(key, lowercased(value, lowercase));
525                    }
526                }
527            }
528        }
529        (slot, value) => *slot = lowercased(value, lowercase),
530    }
531}
532
533/// [`to_traced`] without the provenance: object keys lowercased at every
534/// level (arrays are leaves, so their objects keep their keys).
535fn lowercased(value: Value, lowercase: bool) -> Value {
536    match value {
537        Value::Object(map) if lowercase => Value::Object(
538            map.into_iter()
539                .map(|(key, value)| (key.to_lowercase(), lowercased(value, lowercase)))
540                .collect(),
541        ),
542        other => other,
543    }
544}
545
546/// Deep-merge `value` (labeled with `source`) into the traced tree:
547/// objects merge recursively, everything else replaces.
548fn merge_traced(target: &mut TracedValue, value: Value, source: &SourceRef, lowercase: bool) {
549    match (target, value) {
550        (TracedValue::Object(entries), Value::Object(incoming)) => {
551            for (key, value) in incoming {
552                let key = if lowercase { key.to_lowercase() } else { key };
553                match entries.get_mut(&key) {
554                    Some(slot) => merge_traced(slot, value, source, lowercase),
555                    None => {
556                        entries.insert(key, to_traced(value, source, lowercase));
557                    }
558                }
559            }
560        }
561        (slot, value) => *slot = to_traced(value, source, lowercase),
562    }
563}
564
565fn to_traced(value: Value, source: &SourceRef, lowercase: bool) -> TracedValue {
566    match value {
567        Value::Object(map) => TracedValue::Object(
568            map.into_iter()
569                .map(|(key, value)| {
570                    let key = if lowercase { key.to_lowercase() } else { key };
571                    (key, to_traced(value, source, lowercase))
572                })
573                .collect(),
574        ),
575        other => TracedValue::Leaf {
576            value: other,
577            source: source.clone(),
578        },
579    }
580}