c4/loader.rs
1//! The loader: scan sources, pick parsers by extension, deep-merge. One
2//! code path for `trace()` and `load()`, generic over the merge target:
3//! the traced tree, or a plain value that skips the per-leaf provenance.
4
5use std::collections::BTreeMap;
6use std::path::{Path, PathBuf};
7
8use serde::de::DeserializeOwned;
9
10use crate::trace::{SourceRef, Trace, TracedValue, Warning};
11use crate::{Error, Format, FormatKind, Options, Order, Result, Source, Value, error, format};
12
13/// Loads and merges the config sources named by its [`Options`].
14#[derive(Debug, Clone, Default)]
15pub struct Loader {
16 pub options: Options,
17}
18
19impl Loader {
20 /// Everything — sources included — comes in through [`Options`];
21 /// there is no separate builder step.
22 pub fn new(options: Options) -> Self {
23 Self { options }
24 }
25
26 /// Load and merge all sources, deserializing into `T`.
27 ///
28 /// [`Value`] implements `Deserialize`, so `T = Value` gives dynamic
29 /// access and any other serde type gives typed config. A `Value` —
30 /// the whole result, or a field of your struct — is handed over as
31 /// merged, typed variants (`DateTime`, `Ipv4`, …) included, exactly as
32 /// [`Loader::trace`] shows them.
33 ///
34 /// A **string** reaching a `bool`, integer or float field parses with
35 /// the rules of that table type id (`u16`, `bool`, … — see the crate
36 /// docs' table section), so ini, env and cnf values, which are all
37 /// strings, load into typed fields. The explicit `bool` tokens apply
38 /// (`yes`/`no`, `on`/`off`, `1`/`0`, …), and out-of-range or malformed
39 /// text is [`Error::Deserialize`] (`'70000' is not a valid u16`). Untyped
40 /// targets (`Value`, `String`) keep the string.
41 ///
42 /// The reverse holds too: a number or bool reaching a `String` field
43 /// becomes its text (`password: 123456` → `"123456"`; a float in its
44 /// shortest round-trip form, so `1.0` stays `"1.0"`).
45 ///
46 /// Every [`Error::Deserialize`] names the key path it failed at —
47 /// `db.password: invalid type: …`, `items[2].id: …`. An empty value
48 /// (`PORT=`) cannot fill a number or bool field, not even an `Option`
49 /// one (serde picks `None` or `Some` before it sees the inner type);
50 /// the error says to remove the line instead.
51 ///
52 /// To type one part of a loaded [`Value`], use [`Value::into_typed`].
53 ///
54 /// ```
55 /// #[derive(serde::Deserialize)]
56 /// struct Server {
57 /// port: u16,
58 /// debug: bool,
59 /// }
60 ///
61 /// # fn main() -> Result<(), c4::Error> {
62 /// // the 1-tuple value source stands in for an ini/env/cnf file here
63 /// let server: Server = c4::Loader::new(c4::Options {
64 /// sources: vec![(serde_json::json!({ "port": "8080", "debug": "yes" }),).into()],
65 /// ..c4::Options::default()
66 /// })
67 /// .load()?;
68 /// assert_eq!(server.port, 8080);
69 /// assert!(server.debug);
70 /// # Ok(())
71 /// # }
72 /// ```
73 ///
74 /// [`Warning`]s are dropped — call [`Loader::trace`] to see why some
75 /// input was not used (a skipped cnf `!include`, for instance).
76 pub fn load<T: DeserializeOwned>(&self) -> Result<T> {
77 let (value, _warnings) = self.run::<Value>()?;
78 value.into_typed()
79 }
80
81 /// Like [`Loader::load`], but keeps per-leaf provenance and the
82 /// [`Warning`]s — see [`Trace`]. This powers the CLI `--trace` mode;
83 /// source labels are a debug/testing aid, not config data.
84 ///
85 /// ```
86 /// use c4::{CustomFormat, Loader, TracedValue};
87 ///
88 /// # fn main() -> Result<(), c4::Error> {
89 /// // a custom-format string source keeps this example runnable
90 /// // under every feature combination (`Loader::default()` would
91 /// // read ./config instead)
92 /// let kv = CustomFormat::new("kv", ["kv"], |text, path, options| {
93 /// let rows = text
94 /// .lines()
95 /// .map(|line| line.split('=').map(str::to_owned).collect())
96 /// .collect();
97 /// c4::parse_table(rows, &c4::TableLayout::Kv, path, options)
98 /// });
99 ///
100 /// let trace = Loader::new(c4::Options {
101 /// sources: vec![(kv, "port=8080").into()],
102 /// ..c4::Options::default()
103 /// })
104 /// .trace()?;
105 /// assert!(trace.warnings.is_empty());
106 /// let TracedValue::Object(root) = &trace.value else { unreachable!() };
107 /// let TracedValue::Leaf { value, source } = &root["port"] else { unreachable!() };
108 /// assert_eq!(value.as_u64(), Some(8080));
109 /// assert_eq!(source, &c4::SourceRef::String(0)); // "string:0" in traces
110 /// # Ok(())
111 /// # }
112 /// ```
113 pub fn trace(&self) -> Result<Trace> {
114 let (value, warnings) = self.run::<TracedValue>()?;
115 Ok(Trace { value, warnings })
116 }
117
118 /// Scan, parse and merge every source into `R` — the one code path
119 /// behind both [`Loader::trace`] (a traced tree) and [`Loader::load`]
120 /// (a plain value, skipping the per-leaf provenance it would drop).
121 fn run<R: Tree>(&self) -> Result<(R, Vec<Warning>)> {
122 let extensions = extension_map(&self.options);
123 let mut root = R::root();
124 let mut warnings = Vec::new();
125
126 for (index, source) in self.options.sources.iter().enumerate() {
127 match source {
128 // one path source: a folder whose files merge, or a single
129 // file — detected here (same rule as `c4::load`)
130 Source::Path(path) if path.is_dir() => {
131 self.load_folder(path, &extensions, &mut root, &mut warnings)?;
132 }
133 Source::Path(path) if path.is_file() => {
134 let claim = claimed_format(path, &extensions).ok_or_else(|| Error::Parse {
135 path: path.clone(),
136 message: unclaimed_message(path),
137 })?;
138 let source = SourceRef::File(path.clone());
139 let mut messages = Vec::new();
140 let value = self.parse_file(claim, path, &mut messages)?;
141 note(&mut warnings, &source, messages);
142 if !matches!(value, Value::Null) {
143 self.merge(&mut root, value, &source);
144 }
145 }
146 Source::Path(path) => return Err(Error::NotFound(path.clone())),
147 Source::Table {
148 path,
149 format,
150 sheet,
151 layout,
152 } => {
153 // one file of a table format, read under an explicit
154 // layout (and, for spreadsheets, an explicit sheet)
155 if !path.is_file() {
156 return Err(Error::Parse {
157 path: path.clone(),
158 message: "table source is not a file (table sources read \
159 exactly one file; for in-code text use a \
160 (format, text) string source)"
161 .into(),
162 });
163 }
164 let value = format::parse_table_file(
165 *format,
166 path,
167 sheet.as_deref(),
168 layout,
169 &self.options,
170 )?;
171 if !matches!(value, Value::Null) {
172 self.merge(&mut root, value, &SourceRef::File(path.clone()));
173 }
174 }
175 Source::Value(result) => {
176 let mut value = result.clone().map_err(|message| Error::Parse {
177 path: PathBuf::from(format!("value:{index}")),
178 message,
179 })?;
180 format::strip_commented_data_keys(&mut value, &self.options);
181 if !matches!(value, Value::Null) {
182 self.merge(&mut root, value, &SourceRef::Value(index));
183 }
184 }
185 Source::String { format, content } => {
186 let label = PathBuf::from(format!("string:{index}"));
187 let mut messages = Vec::new();
188 let value = match format {
189 // `format::parse` filters commented data keys
190 // itself; a custom parser's result is filtered here
191 FormatKind::Builtin(format) => format::parse(
192 *format,
193 format::strip_bom(content),
194 &crate::TableLayout::Kv,
195 &label,
196 &self.options,
197 &mut messages,
198 )?,
199 FormatKind::Custom(custom) => {
200 let mut value =
201 (custom.parser)(format::strip_bom(content), &label, &self.options)?;
202 format::strip_commented_data_keys(&mut value, &self.options);
203 value
204 }
205 };
206 let source = SourceRef::String(index);
207 note(&mut warnings, &source, messages);
208 if !matches!(value, Value::Null) {
209 self.merge(&mut root, value, &source);
210 }
211 }
212 }
213 }
214 Ok((root, warnings))
215 }
216
217 fn parse_file(&self, claim: Claim, path: &Path, warnings: &mut Vec<String>) -> Result<Value> {
218 match claim {
219 Claim::Builtin { format, spec } => {
220 // the claiming formats entry decides the table layout
221 let layout = &self.options.formats[spec].layout;
222 // binary formats (spreadsheets) parse from the path
223 // directly — their bytes must never go through the text
224 // pipeline
225 if format::is_binary(format) {
226 return format::parse_binary(format, path, layout, &self.options);
227 }
228 let text = format::read_text(path)?;
229 format::parse(format, &text, layout, path, &self.options, warnings)
230 }
231 Claim::Custom(index) => {
232 let FormatKind::Custom(custom) = &self.options.formats[index].format else {
233 unreachable!("custom claims always index custom entries");
234 };
235 let text = format::read_text(path)?;
236 // built-in formats filter commented data keys inside
237 // `format::parse`/`parse_binary`; do it for custom ones here
238 let mut value = (custom.parser)(&text, path, &self.options)?;
239 format::strip_commented_data_keys(&mut value, &self.options);
240 Ok(value)
241 }
242 }
243 }
244
245 fn merge<R: Tree>(&self, target: &mut R, value: Value, source: &SourceRef) {
246 target.merge(value, source, !self.options.case_sensitive);
247 }
248
249 /// Load one folder source: scan it (honoring [`Options::dir_depth`]),
250 /// parse each file, and merge it in. The keying options decide the
251 /// shape — [`filename_as_key`](Options::filename_as_key) wraps each
252 /// file's value under a key named after the file, and
253 /// [`dirname_as_key`](Options::dirname_as_key) wraps under the
254 /// subfolder names leading to it. With both off this is a flat merge;
255 /// with both on (the `tree` feature's default) it is a folder tree.
256 /// Entries load in [`Options::order`], so key collisions deep-merge
257 /// with the later entry winning.
258 fn load_folder<R: Tree>(
259 &self,
260 folder: &Path,
261 extensions: &[(String, Claim)],
262 root: &mut R,
263 warnings: &mut Vec<Warning>,
264 ) -> Result<()> {
265 let mut files = Vec::new();
266 walk(
267 folder,
268 Vec::new(),
269 &self.options,
270 self.options.dir_depth,
271 &mut files,
272 )?;
273 for (file, dirs) in files {
274 let name = file
275 .file_name()
276 .and_then(|n| n.to_str())
277 .unwrap_or_default()
278 .to_owned();
279 let mut messages = Vec::new();
280 let value = match claimed_format(&file, extensions) {
281 Some(claim) => self.parse_file(claim, &file, &mut messages)?,
282 // extension-less / unknown-extension handling only applies
283 // when a file becomes a key; otherwise the file has no key
284 // to contribute under and is skipped
285 None if self.options.filename_as_key
286 && file_extension(&name).is_none()
287 && self.options.auto_no_ext_files =>
288 {
289 // the trimmed content goes through the table auto
290 // detection (same feature-gated guesses)
291 let text = format::read_text(&file)?;
292 format::table_auto(text.trim())
293 }
294 None if self.options.filename_as_key && !self.options.ignore_unknown_ext => {
295 return Err(Error::Parse {
296 message: unclaimed_message(&file),
297 path: file,
298 });
299 }
300 None => continue, // no active format claims this extension
301 };
302 let source = SourceRef::File(file);
303 note(warnings, &source, messages);
304 if matches!(value, Value::Null) {
305 continue; // empty file contributes nothing
306 }
307 let mut value = value;
308 if self.options.filename_as_key {
309 value = Value::Object(BTreeMap::from([(file_stem_key(&name), value)]));
310 }
311 if self.options.dirname_as_key {
312 for key in dirs.into_iter().rev() {
313 value = Value::Object(BTreeMap::from([(key, value)]));
314 }
315 }
316 self.merge(root, value, &source);
317 }
318 Ok(())
319 }
320}
321
322/// The [`filename_as_key`](Options::filename_as_key) key of a file: its
323/// name without the extension. A hidden file whose whole name is its
324/// extension (`.env`) keeps the full name.
325fn file_stem_key(name: &str) -> String {
326 let hidden = name.starts_with('.');
327 let body = if hidden { &name[1..] } else { name };
328 match body.rfind('.') {
329 Some(i) => format!("{}{}", if hidden { "." } else { "" }, &body[..i]),
330 None => name.to_owned(),
331 }
332}
333
334/// What an extension resolves to: a built-in format (with the index of
335/// its [`Options::formats`] entry, whose layout applies), or a custom
336/// entry (by index).
337#[derive(Clone, Copy)]
338enum Claim {
339 Builtin { format: Format, spec: usize },
340 Custom(usize),
341}
342
343/// The extension→parser mapping of [`Options::formats`]: lowercased
344/// extension, last claimer wins — built-in and custom entries follow the
345/// same rule.
346fn extension_map(options: &Options) -> Vec<(String, Claim)> {
347 let mut map: Vec<(String, Claim)> = Vec::new();
348 for (index, spec) in options.formats.iter().enumerate() {
349 let claim = match &spec.format {
350 FormatKind::Builtin(format) => Claim::Builtin {
351 format: *format,
352 spec: index,
353 },
354 FormatKind::Custom(_) => Claim::Custom(index),
355 };
356 for ext in &spec.extensions {
357 let ext = ext.to_lowercase();
358 map.retain(|(e, _)| *e != ext);
359 map.push((ext, claim));
360 }
361 }
362 map
363}
364
365/// The extension of a file name, lowercased. A hidden file `.X` counts as
366/// having extension `X` (that is how `.env` matches `env`).
367fn file_extension(name: &str) -> Option<String> {
368 let hidden = name.starts_with('.');
369 let stem = if hidden { &name[1..] } else { name };
370 match stem.rfind('.') {
371 Some(i) => Some(stem[i + 1..].to_lowercase()),
372 None if hidden && !stem.is_empty() => Some(stem.to_lowercase()),
373 None => None,
374 }
375}
376
377/// Why no format claims `path`, and how to fix it: the built-in format
378/// whose default extensions include it is either compiled out (name the
379/// feature) or missing from `Options.formats`.
380fn unclaimed_message(path: &Path) -> String {
381 let base = "no active format claims this file's extension";
382 let Some(ext) = path
383 .file_name()
384 .and_then(|n| n.to_str())
385 .and_then(file_extension)
386 else {
387 return base.into();
388 };
389 // jsonc before json: it is the default reader of `.json`
390 let reader = Format::ALL
391 .into_iter()
392 .find(|f| f.default_extensions().contains(&ext.as_str()));
393 match reader {
394 Some(f) if !format::is_compiled(f) => format!(
395 "{base}: '.{ext}' is read by the `{id}` format — enable the `{id}` feature",
396 id = f.id()
397 ),
398 Some(f) => format!(
399 "{base}: the `{}` format reads it but is not in `formats`",
400 f.id()
401 ),
402 None => base.into(),
403 }
404}
405
406fn claimed_format(path: &Path, extensions: &[(String, Claim)]) -> Option<Claim> {
407 let name = path.file_name()?.to_str()?;
408 let ext = file_extension(name)?;
409 extensions
410 .iter()
411 .find(|(e, _)| *e == ext)
412 .map(|(_, claim)| *claim)
413}
414
415/// A directory's entries as `(name, path, is_dir)`.
416fn read_entries(dir: &Path) -> Result<Vec<(String, PathBuf, bool)>> {
417 let mut entries = Vec::new();
418 for entry in std::fs::read_dir(dir).map_err(error::io(dir))? {
419 let entry = entry.map_err(error::io(dir))?;
420 let path = entry.path();
421 let Some(name) = path.file_name().and_then(|n| n.to_str()) else {
422 continue;
423 };
424 entries.push((name.to_owned(), path.clone(), path.is_dir()));
425 }
426 Ok(entries)
427}
428
429fn sort_entries(entries: &mut [(String, PathBuf, bool)], order: Order) {
430 match order {
431 Order::FoldersFirstAlphabetic => entries.sort_by(|a, b| (!a.2, &a.0).cmp(&(!b.2, &b.0))),
432 Order::Alphabetic => entries.sort_by(|a, b| a.0.cmp(&b.0)),
433 Order::ReverseAlphabetic => entries.sort_by(|a, b| b.0.cmp(&a.0)),
434 }
435}
436
437/// All files of a folder in load order, each with the subfolder names
438/// leading to it. `depth` is the remaining [`Options::dir_depth`] budget:
439/// `0` stops descent, `-1` recurses without limit, a positive value
440/// decrements per level. Commented-out names (`#…`, `_…`) are skipped
441/// here — by name, so keyed or not, a `_draft.yml` file and everything
442/// under a `_wip/` folder stay out of the scan.
443fn walk(
444 dir: &Path,
445 prefix: Vec<String>,
446 options: &Options,
447 depth: isize,
448 out: &mut Vec<(PathBuf, Vec<String>)>,
449) -> Result<()> {
450 let mut entries = read_entries(dir)?;
451 sort_entries(&mut entries, options.order);
452 for (name, path, is_dir) in entries {
453 let ignore_commented = if is_dir {
454 options.ignore_commented_dirnames
455 } else {
456 options.ignore_commented_filenames
457 };
458 if ignore_commented && crate::options::is_commented(&name) {
459 continue;
460 }
461 if is_dir {
462 if depth != 0 {
463 let mut prefix = prefix.clone();
464 prefix.push(name);
465 walk(
466 &path,
467 prefix,
468 options,
469 if depth < 0 { -1 } else { depth - 1 },
470 out,
471 )?;
472 }
473 } else {
474 out.push((path, prefix.clone()));
475 }
476 }
477 Ok(())
478}
479
480fn note(warnings: &mut Vec<Warning>, source: &SourceRef, messages: Vec<String>) {
481 warnings.extend(messages.into_iter().map(|message| Warning {
482 source: source.clone(),
483 message,
484 }));
485}
486
487/// What the sources merge into. Both trees deep-merge the same way —
488/// objects recurse, everything else replaces, keys lowercased when merging
489/// is case-insensitive — and differ only in whether leaves keep `source`.
490trait Tree {
491 fn root() -> Self;
492 fn merge(&mut self, value: Value, source: &SourceRef, lowercase: bool);
493}
494
495impl Tree for TracedValue {
496 fn root() -> Self {
497 TracedValue::Object(BTreeMap::new())
498 }
499
500 fn merge(&mut self, value: Value, source: &SourceRef, lowercase: bool) {
501 merge_traced(self, value, source, lowercase);
502 }
503}
504
505impl Tree for Value {
506 fn root() -> Self {
507 Value::Object(BTreeMap::new())
508 }
509
510 fn merge(&mut self, value: Value, _source: &SourceRef, lowercase: bool) {
511 merge_plain(self, value, lowercase);
512 }
513}
514
515/// [`merge_traced`] without the provenance.
516fn merge_plain(target: &mut Value, value: Value, lowercase: bool) {
517 match (target, value) {
518 (Value::Object(entries), Value::Object(incoming)) => {
519 for (key, value) in incoming {
520 let key = if lowercase { key.to_lowercase() } else { key };
521 match entries.get_mut(&key) {
522 Some(slot) => merge_plain(slot, value, lowercase),
523 None => {
524 entries.insert(key, lowercased(value, lowercase));
525 }
526 }
527 }
528 }
529 (slot, value) => *slot = lowercased(value, lowercase),
530 }
531}
532
533/// [`to_traced`] without the provenance: object keys lowercased at every
534/// level (arrays are leaves, so their objects keep their keys).
535fn lowercased(value: Value, lowercase: bool) -> Value {
536 match value {
537 Value::Object(map) if lowercase => Value::Object(
538 map.into_iter()
539 .map(|(key, value)| (key.to_lowercase(), lowercased(value, lowercase)))
540 .collect(),
541 ),
542 other => other,
543 }
544}
545
546/// Deep-merge `value` (labeled with `source`) into the traced tree:
547/// objects merge recursively, everything else replaces.
548fn merge_traced(target: &mut TracedValue, value: Value, source: &SourceRef, lowercase: bool) {
549 match (target, value) {
550 (TracedValue::Object(entries), Value::Object(incoming)) => {
551 for (key, value) in incoming {
552 let key = if lowercase { key.to_lowercase() } else { key };
553 match entries.get_mut(&key) {
554 Some(slot) => merge_traced(slot, value, source, lowercase),
555 None => {
556 entries.insert(key, to_traced(value, source, lowercase));
557 }
558 }
559 }
560 }
561 (slot, value) => *slot = to_traced(value, source, lowercase),
562 }
563}
564
565fn to_traced(value: Value, source: &SourceRef, lowercase: bool) -> TracedValue {
566 match value {
567 Value::Object(map) => TracedValue::Object(
568 map.into_iter()
569 .map(|(key, value)| {
570 let key = if lowercase { key.to_lowercase() } else { key };
571 (key, to_traced(value, source, lowercase))
572 })
573 .collect(),
574 ),
575 other => TracedValue::Leaf {
576 value: other,
577 source: source.clone(),
578 },
579 }
580}