dre_core/config/
node.rs

1//! A parsed YAML document as a tree of nodes, each with the line and column it was written at.
2//!
3//! YAML is parsed by `serde-saphyr`, YAML 1.2 with strict booleans: only `true` and `false` are
4//! booleans, so `yes`, `no`, `on` and `off` stay strings. Duplicate keys are errors; merge keys
5//! (`<<: *base`) are expanded. Typed config is then read from the tree (see [`super::de`]), so every
6//! diagnostic can point at the exact line of the key or value it's about.
7
8use std::fmt;
9
10use serde::de::{Deserialize, Deserializer, MapAccess, SeqAccess, Visitor};
11use serde_saphyr::Spanned;
12
13/// A YAML value and where it was written (1-based; 0 when unknown).
14#[derive(Debug, Clone, PartialEq)]
15pub struct Node {
16    pub kind: Kind,
17    pub line: usize,
18    pub column: usize,
19}
20
21#[derive(Debug, Clone, PartialEq)]
22pub enum Kind {
23    Null,
24    Bool(bool),
25    Int(i64),
26    UInt(u64),
27    Float(f64),
28    Str(String),
29    Seq(Vec<Node>),
30    /// In the file's order. Keys are strings: a scalar key (`2024:`) keeps its spelling.
31    Map(Vec<(Key, Node)>),
32}
33
34/// A map key and its line.
35#[derive(Debug, Clone, PartialEq, Eq)]
36pub struct Key {
37    pub name: String,
38    pub line: usize,
39}
40
41/// A YAML syntax error, with its line when known.
42#[derive(Debug, Clone, PartialEq, Eq)]
43pub struct SyntaxError {
44    pub line: Option<usize>,
45    pub message: String,
46}
47
48impl Kind {
49    /// How a value of this kind is named in diagnostics.
50    pub fn name(&self) -> &'static str {
51        match self {
52            Kind::Null => "nothing",
53            Kind::Bool(_) => "a boolean",
54            Kind::Int(_) | Kind::UInt(_) | Kind::Float(_) => "a number",
55            Kind::Str(_) => "a string",
56            Kind::Seq(_) => "a list",
57            Kind::Map(_) => "a map",
58        }
59    }
60}
61
62impl Node {
63    pub fn null() -> Node {
64        Node {
65            kind: Kind::Null,
66            line: 0,
67            column: 0,
68        }
69    }
70
71    pub fn as_str(&self) -> Option<&str> {
72        match &self.kind {
73            Kind::Str(s) => Some(s),
74            _ => None,
75        }
76    }
77
78    pub fn as_map(&self) -> Option<&[(Key, Node)]> {
79        match &self.kind {
80            Kind::Map(m) => Some(m),
81            _ => None,
82        }
83    }
84
85    /// A key's value in a map node.
86    pub fn get(&self, key: &str) -> Option<&Node> {
87        self.as_map()?.iter().find(|(k, _)| k.name == key).map(|(_, v)| v)
88    }
89
90    /// A key's entry (key and value) in a map node.
91    pub fn entry(&self, key: &str) -> Option<&(Key, Node)> {
92        self.as_map()?.iter().find(|(k, _)| k.name == key)
93    }
94
95    /// The node as JSON, for values handed on unvalidated (a plugin's fields). Keys are strings;
96    /// numbers that don't fit JSON's become strings.
97    pub fn to_json(&self) -> serde_json::Value {
98        use serde_json::Value as J;
99        match &self.kind {
100            Kind::Null => J::Null,
101            Kind::Bool(b) => J::Bool(*b),
102            Kind::Int(i) => J::from(*i),
103            Kind::UInt(u) => J::from(*u),
104            Kind::Float(f) => {
105                serde_json::Number::from_f64(*f).map_or_else(|| J::String(f.to_string()), J::Number)
106            }
107            Kind::Str(s) => J::String(s.clone()),
108            Kind::Seq(s) => J::Array(s.iter().map(Node::to_json).collect()),
109            Kind::Map(m) => J::Object(m.iter().map(|(k, v)| (k.name.clone(), v.to_json())).collect()),
110        }
111    }
112}
113
114/// Parse a YAML document. An empty document is a null node.
115pub fn parse(text: &str) -> Result<Node, SyntaxError> {
116    let mut options = serde_saphyr::options::Options::default();
117    options.strict_booleans = true;
118    options.duplicate_keys = serde_saphyr::options::DuplicateKeyPolicy::Error;
119    options.with_snippet = false;
120    // `.inf`, `.nan` and out-of-range floats like `1e999` reach DRE as the strings they are; a
121    // plugin decides what they mean.
122    options.non_finite_float_policy = serde_saphyr::options::NonFiniteFloatPolicy::AsString;
123    if text.trim().is_empty() {
124        return Ok(Node::null());
125    }
126    match serde_saphyr::from_str_with_options::<Spanned<Raw>>(text, options) {
127        Ok(raw) => {
128            let mut node: Node = raw.into();
129            // Top-level `x-*` keys (the docker-compose convention) hold YAML anchors to reuse
130            // with aliases and `<<:` merges; once those are resolved, DRE ignores them.
131            if let Kind::Map(m) = &mut node.kind {
132                m.retain(|(k, _)| !k.name.starts_with("x-"));
133            }
134            Ok(node)
135        }
136        Err(e) => Err(syntax_error(&e)),
137    }
138}
139
140fn syntax_error(e: &serde_saphyr::Error) -> SyntaxError {
141    static LOCATION: std::sync::LazyLock<regex::Regex> = std::sync::LazyLock::new(|| {
142        regex::Regex::new(r"^(?:error: )?(?:line \d+ column \d+: )?(.*?)(?: at line \d+, column \d+)?$")
143            .unwrap()
144    });
145    let line = e.location().map(|l| l.line() as usize).filter(|l| *l > 0);
146    let text = e.to_string();
147    let first = text.lines().next().unwrap_or_default();
148    // DRE prints the line itself, and serde-saphyr's hints name its own options and functions.
149    let message = LOCATION
150        .captures(first)
151        .map_or(first, |c| c.get(1).map_or(first, |m| m.as_str()));
152    let message = message.replace(", set DuplicateKeyPolicy in Options if acceptable", "");
153    let message = message
154        .split("; use from_")
155        .next()
156        .unwrap_or_default()
157        .to_string();
158    let message = if message.contains("merge value must be") {
159        "invalid YAML merge key (`<<`): `<<` must be a map or a list of maps".to_string()
160    } else {
161        message
162    };
163    SyntaxError { line, message }
164}
165
166/// The tree as serde-saphyr hands it over: every value with its location.
167enum Raw {
168    Null,
169    Bool(bool),
170    Int(i64),
171    UInt(u64),
172    Float(f64),
173    Str(String),
174    Seq(Vec<Spanned<Raw>>),
175    Map(Vec<(Spanned<KeyRaw>, Spanned<Raw>)>),
176}
177
178/// A map key as written: a scalar, kept as its spelling (`null`, `true`, `2024`).
179struct KeyRaw(String);
180
181impl From<Spanned<Raw>> for Node {
182    fn from(s: Spanned<Raw>) -> Node {
183        let kind = match s.value {
184            Raw::Null => Kind::Null,
185            Raw::Bool(b) => Kind::Bool(b),
186            Raw::Int(i) => Kind::Int(i),
187            Raw::UInt(u) => Kind::UInt(u),
188            Raw::Float(f) => Kind::Float(f),
189            Raw::Str(t) => Kind::Str(t),
190            Raw::Seq(items) => Kind::Seq(items.into_iter().map(Node::from).collect()),
191            Raw::Map(entries) => Kind::Map(
192                entries
193                    .into_iter()
194                    .map(|(k, v)| {
195                        let key = Key {
196                            name: k.value.0,
197                            line: k.referenced.line() as usize,
198                        };
199                        (key, Node::from(v))
200                    })
201                    .collect(),
202            ),
203        };
204        Node {
205            kind,
206            line: s.referenced.line() as usize,
207            column: s.referenced.column() as usize,
208        }
209    }
210}
211
212impl<'de> Deserialize<'de> for Raw {
213    fn deserialize<D: Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
214        struct V;
215        impl<'de> Visitor<'de> for V {
216            type Value = Raw;
217            fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result {
218                f.write_str("a YAML value")
219            }
220            fn visit_unit<E>(self) -> Result<Raw, E> {
221                Ok(Raw::Null)
222            }
223            fn visit_none<E>(self) -> Result<Raw, E> {
224                Ok(Raw::Null)
225            }
226            fn visit_some<D: Deserializer<'de>>(self, d: D) -> Result<Raw, D::Error> {
227                Raw::deserialize(d)
228            }
229            fn visit_bool<E>(self, v: bool) -> Result<Raw, E> {
230                Ok(Raw::Bool(v))
231            }
232            fn visit_i64<E>(self, v: i64) -> Result<Raw, E> {
233                Ok(Raw::Int(v))
234            }
235            fn visit_u64<E>(self, v: u64) -> Result<Raw, E> {
236                Ok(i64::try_from(v).map_or(Raw::UInt(v), Raw::Int))
237            }
238            fn visit_f64<E>(self, v: f64) -> Result<Raw, E> {
239                Ok(Raw::Float(v))
240            }
241            fn visit_str<E>(self, v: &str) -> Result<Raw, E> {
242                Ok(Raw::Str(v.to_string()))
243            }
244            fn visit_string<E>(self, v: String) -> Result<Raw, E> {
245                Ok(Raw::Str(v))
246            }
247            fn visit_seq<A: SeqAccess<'de>>(self, mut a: A) -> Result<Raw, A::Error> {
248                let mut items = Vec::new();
249                while let Some(x) = a.next_element()? {
250                    items.push(x);
251                }
252                Ok(Raw::Seq(items))
253            }
254            fn visit_map<A: MapAccess<'de>>(self, mut a: A) -> Result<Raw, A::Error> {
255                let mut entries = Vec::new();
256                while let Some(e) = a.next_entry()? {
257                    entries.push(e);
258                }
259                Ok(Raw::Map(entries))
260            }
261        }
262        d.deserialize_any(V)
263    }
264}
265
266/// The message for a key that's a list or a map.
267pub const COMPLEX_KEY: &str = "a map key is a list or a map; keys must be names";
268
269impl<'de> Deserialize<'de> for KeyRaw {
270    fn deserialize<D: Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
271        struct V;
272        impl<'de> Visitor<'de> for V {
273            type Value = KeyRaw;
274            fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result {
275                f.write_str("a key")
276            }
277            fn visit_unit<E>(self) -> Result<KeyRaw, E> {
278                Ok(KeyRaw("null".into()))
279            }
280            fn visit_none<E>(self) -> Result<KeyRaw, E> {
281                Ok(KeyRaw("null".into()))
282            }
283            fn visit_bool<E>(self, v: bool) -> Result<KeyRaw, E> {
284                Ok(KeyRaw(v.to_string()))
285            }
286            fn visit_i64<E>(self, v: i64) -> Result<KeyRaw, E> {
287                Ok(KeyRaw(v.to_string()))
288            }
289            fn visit_u64<E>(self, v: u64) -> Result<KeyRaw, E> {
290                Ok(KeyRaw(v.to_string()))
291            }
292            fn visit_f64<E>(self, v: f64) -> Result<KeyRaw, E> {
293                Ok(KeyRaw(v.to_string()))
294            }
295            fn visit_str<E>(self, v: &str) -> Result<KeyRaw, E> {
296                Ok(KeyRaw(v.to_string()))
297            }
298            fn visit_string<E>(self, v: String) -> Result<KeyRaw, E> {
299                Ok(KeyRaw(v))
300            }
301            fn visit_seq<A: SeqAccess<'de>>(self, _: A) -> Result<KeyRaw, A::Error> {
302                Err(serde::de::Error::custom(COMPLEX_KEY))
303            }
304            fn visit_map<A: MapAccess<'de>>(self, _: A) -> Result<KeyRaw, A::Error> {
305                Err(serde::de::Error::custom(COMPLEX_KEY))
306            }
307        }
308        d.deserialize_any(V)
309    }
310}
311
312#[cfg(test)]
313mod tests {
314    use super::*;
315
316    #[test]
317    fn values_keep_their_lines() {
318        let n = parse("a:\n  b: 1\n  c: [x, y]\n").unwrap();
319        let a = n.get("a").unwrap();
320        assert_eq!(n.entry("a").unwrap().0.line, 1);
321        assert_eq!(a.entry("c").unwrap().0.line, 3);
322        assert_eq!(a.get("b").unwrap().kind, Kind::Int(1));
323        assert_eq!(a.get("b").unwrap().line, 2);
324    }
325
326    #[test]
327    fn only_true_and_false_are_booleans() {
328        let n = parse("a: yes\nb: no\nc: on\nd: true\n").unwrap();
329        assert_eq!(n.get("a").unwrap().as_str(), Some("yes"));
330        assert_eq!(n.get("c").unwrap().as_str(), Some("on"));
331        assert_eq!(n.get("d").unwrap().kind, Kind::Bool(true));
332    }
333
334    #[test]
335    fn merge_keys_are_expanded_and_written_keys_win() {
336        let n = parse("base: &b {type: duckdb, path: x}\ndev: {<<: *b, path: y}\n").unwrap();
337        let dev = n.get("dev").unwrap();
338        assert_eq!(dev.get("type").unwrap().as_str(), Some("duckdb"));
339        assert_eq!(dev.get("path").unwrap().as_str(), Some("y"));
340    }
341
342    #[test]
343    fn duplicate_keys_are_errors_with_their_line() {
344        let e = parse("a: 1\na: 2\n").unwrap_err();
345        assert_eq!(e.line, Some(2));
346        assert_eq!(e.message, "duplicate mapping key: a");
347    }
348
349    #[test]
350    fn syntax_errors_have_a_line() {
351        let e = parse("a:\n  b: [1\n").unwrap_err();
352        assert_eq!(e.line, Some(2));
353        assert!(!e.message.starts_with("line"), "{}", e.message);
354    }
355
356    #[test]
357    fn scalar_keys_are_strings_and_empty_files_are_null() {
358        assert_eq!(
359            parse("2024: x\n").unwrap().get("2024").unwrap().as_str(),
360            Some("x")
361        );
362        assert_eq!(parse("\n# nothing\n").unwrap().kind, Kind::Null);
363    }
364
365    #[test]
366    fn merged_keys_report_the_merge_line() {
367        let n = parse("base: &b\n  x: 1\ndev:\n  <<: *b\n  z: 3\n").unwrap();
368        let dev = n.get("dev").unwrap();
369        assert_eq!(dev.entry("x").unwrap().0.line, 4);
370        assert_eq!(dev.entry("z").unwrap().0.line, 5);
371    }
372
373    #[test]
374    fn multiple_documents_name_no_library_functions() {
375        let e = parse("a: 1\n---\nb: 2\n").unwrap_err();
376        assert_eq!(e.line, Some(3));
377        assert_eq!(e.message, "multiple YAML documents detected");
378    }
379
380    #[test]
381    fn non_finite_floats_are_strings() {
382        let j = parse("a: .inf\nb: -.inf\nc: .nan\nd: 1e999\n").unwrap().to_json();
383        assert_eq!(
384            j,
385            serde_json::json!({"a": ".inf", "b": "-.inf", "c": ".nan", "d": ".inf"})
386        );
387    }
388
389    #[test]
390    fn relaxed_yaml_forms_parse() {
391        let quoted = parse("a: \"one\n two\"\n").unwrap();
392        assert_eq!(quoted.get("a").unwrap().as_str(), Some("one two"));
393        assert_eq!(parse("?\tk\n").unwrap().to_json(), serde_json::json!({"k": null}));
394        assert!(parse("?").is_ok());
395        let block = parse("s: |\n  a\n  ").unwrap();
396        assert_eq!(block.get("s").unwrap().as_str(), Some("a\n"));
397    }
398
399    #[test]
400    fn jinja_in_quoted_values_is_a_string() {
401        let n = parse("pw: \"{{ env_var('X') }}\"\n").unwrap();
402        assert_eq!(n.get("pw").unwrap().as_str(), Some("{{ env_var('X') }}"));
403    }
404}
405
406#[cfg(test)]
407mod x_key_tests {
408    use super::*;
409
410    #[test]
411    fn top_level_x_keys_hold_anchors_and_are_dropped() {
412        let n = parse("x-dest: &dest {profile: s3, path: a.csv}\noutput:\n  destination:\n    <<: *dest\n    path: b.csv\n").unwrap();
413        let j = n.to_json();
414        assert!(j.get("x-dest").is_none(), "{j}");
415        assert_eq!(
416            j["output"]["destination"],
417            serde_json::json!({"profile": "s3", "path": "b.csv"})
418        );
419    }
420}
dre_core/config/node.rs

dre_core/config/
node.rs
